From 85a7f475392b60b678ebed2377b51a24dabf2fee Mon Sep 17 00:00:00 2001 From: tamaina Date: Sun, 5 Feb 2023 12:34:04 +0000 Subject: [PATCH] =?UTF-8?q?=E3=83=91=E3=83=83=E3=82=B1=E3=83=BC=E3=82=B8?= =?UTF-8?q?=E5=85=AC=E9=96=8B=E3=81=AB=E6=9C=80=E9=81=A9=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 23 ++++- built/const.d.ts | 1 + built/const.js | 41 +++++++- built/create-temp.d.ts | 2 + built/create-temp.js | 22 ++++- built/download.d.ts | 22 +++++ built/download.js | 88 ++++++++++++++++- built/file-info.d.ts | 11 +++ built/file-info.js | 59 +++++++++++- built/http.d.ts | 8 ++ built/http.js | 46 ++++++++- built/image-processor.d.ts | 18 ++++ built/image-processor.js | 27 +++++- built/index.d.ts | 17 ++++ built/index.js | 193 ++++++++++++++++++++++++++++++++++++- built/status-error.d.ts | 6 ++ built/status-error.js | 10 +- package.json | 8 +- pnpm-lock.yaml | 34 +++++-- server.js | 6 ++ src/download.ts | 47 ++++++--- src/http.ts | 48 +++++---- src/index.ts | 58 ++++++++++- tsconfig.json | 2 +- 24 files changed, 737 insertions(+), 60 deletions(-) create mode 100644 built/const.d.ts create mode 100644 built/create-temp.d.ts create mode 100644 built/download.d.ts create mode 100644 built/file-info.d.ts create mode 100644 built/http.d.ts create mode 100644 built/image-processor.d.ts create mode 100644 built/index.d.ts create mode 100644 built/status-error.d.ts create mode 100644 server.js diff --git a/README.md b/README.md index 89abebf..70da8ab 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,25 @@ Misskeyの/proxyが単体で動作します（Misskeyのコードがほぼその **Fastifyプラグインとして動作する気がします。** `pnpm start`は[fastify-cli](https://github.com/fastify/fastify-cli)が動作します。 -## セットアップ方法 +## Fastifyプラグインとして動作させる +### npm install + +``` +npm install git+https://github.com/misskey-dev/media-proxy.git +``` + +### Fastifyプラグインを書く +``` +import MediaProxy from 'misskey-media-proxy'; + +// ...... + +fastify.register(MediaProxy); +``` + +オプションを指定できます。オプションの内容はindex.tsのMediaProxyOptionsに指定してあります。 + +## サーバーのセットアップ方法 まずはgit cloneしてcdしてください。 ``` @@ -25,7 +43,7 @@ NODE_ENV=production pnpm install ```js import { readFileSync } from 'node:fs'; -const repo = JSON.stringify(readFileSync('./package.json', 'utf8')); +const repo = JSON.parse(readFileSync('./package.json', 'utf8')); export default { // UA @@ -91,3 +109,4 @@ mediaProxyの指定をdefault.ymlに追記し、Misskeyを再起動してくだ ```yml mediaProxy: https://mediaproxy.example.com ``` + diff --git a/built/const.d.ts b/built/const.d.ts new file mode 100644 index 0000000..68ff82d --- /dev/null +++ b/built/const.d.ts @@ -0,0 +1 @@ +export declare const FILE_TYPE_BROWSERSAFE: string[]; diff --git a/built/const.js b/built/const.js index 952d3f0..9cdbd5e 100644 --- a/built/const.js +++ b/built/const.js @@ -1 +1,40 @@ -export const FILE_TYPE_BROWSERSAFE=["image/png","image/gif","image/jpeg","image/webp","image/avif","image/apng","image/bmp","image/tiff","image/x-icon","audio/opus","video/ogg","audio/ogg","application/ogg","video/quicktime","video/mp4","audio/mp4","video/x-m4v","audio/x-m4a","video/3gpp","video/3gpp2","video/mpeg","audio/mpeg","video/webm","audio/webm","audio/aac","audio/x-flac","audio/vnd.wave"]; \ No newline at end of file +// ブラウザで直接表示することを許可するファイルの種類のリスト +// ここに含まれないものは application/octet-stream としてレスポンスされる +// SVGはXSSを生むので許可しない +export const FILE_TYPE_BROWSERSAFE = [ + // Images + 'image/png', + 'image/gif', + 'image/jpeg', + 'image/webp', + 'image/avif', + 'image/apng', + 'image/bmp', + 'image/tiff', + 'image/x-icon', + // OggS + 'audio/opus', + 'video/ogg', + 'audio/ogg', + 'application/ogg', + // ISO/IEC base media file format + 'video/quicktime', + 'video/mp4', + 'audio/mp4', + 'video/x-m4v', + 'audio/x-m4a', + 'video/3gpp', + 'video/3gpp2', + 'video/mpeg', + 'audio/mpeg', + 'video/webm', + 'audio/webm', + 'audio/aac', + 'audio/x-flac', + 'audio/vnd.wave', +]; +/* +https://github.com/sindresorhus/file-type/blob/main/supported.js +https://github.com/sindresorhus/file-type/blob/main/core.js +https://developer.mozilla.org/en-US/docs/Web/Media/Formats/Containers +*/ diff --git a/built/create-temp.d.ts b/built/create-temp.d.ts new file mode 100644 index 0000000..55102c5 --- /dev/null +++ b/built/create-temp.d.ts @@ -0,0 +1,2 @@ +export declare function createTemp(): Promise<[string, () => void]>; +export declare function createTempDir(): Promise<[string, () => void]>; diff --git a/built/create-temp.js b/built/create-temp.js index a517156..f65205f 100644 --- a/built/create-temp.js +++ b/built/create-temp.js @@ -1 +1,21 @@ -import*as tmp from"tmp";export function createTemp(){return new Promise((res,rej)=>{tmp.file((e,path,fd,cleanup)=>{if(e)return rej(e);res([path,process.env.NODE_ENV==="production"?cleanup:()=>{}])})})}export function createTempDir(){return new Promise((res,rej)=>{tmp.dir({unsafeCleanup:true},(e,path,cleanup)=>{if(e)return rej(e);res([path,process.env.NODE_ENV==="production"?cleanup:()=>{}])})})} \ No newline at end of file +import * as tmp from 'tmp'; +export function createTemp() { + return new Promise((res, rej) => { + tmp.file((e, path, fd, cleanup) => { + if (e) + return rej(e); + res([path, process.env.NODE_ENV === 'production' ? cleanup : () => { }]); + }); + }); +} +export function createTempDir() { + return new Promise((res, rej) => { + tmp.dir({ + unsafeCleanup: true, + }, (e, path, cleanup) => { + if (e) + return rej(e); + res([path, process.env.NODE_ENV === 'production' ? cleanup : () => { }]); + }); + }); +} diff --git a/built/download.d.ts b/built/download.d.ts new file mode 100644 index 0000000..7bcd2ca --- /dev/null +++ b/built/download.d.ts @@ -0,0 +1,22 @@ +/// +/// +import * as http from 'node:http'; +import * as https from 'node:https'; +export type DownloadConfig = { + [x: string]: any; + userAgent: string; + allowedPrivateNetworks: string[]; + maxSize: number; + httpAgent: http.Agent; + httpsAgent: https.Agent; + proxy?: boolean; +}; +export declare const defaultDownloadConfig: { + httpAgent: http.Agent; + httpsAgent: https.Agent; + userAgent: string; + allowedPrivateNetworks: never[]; + maxSize: number; + proxy: boolean; +}; +export declare function downloadUrl(url: string, path: string, settings?: DownloadConfig): Promise; diff --git a/built/download.js b/built/download.js index 4a982c5..9b32599 100644 --- a/built/download.js +++ b/built/download.js @@ -1 +1,87 @@ -import*as fs from"node:fs";import*as stream from"node:stream";import*as util from"node:util";import got,*as Got from"got";import IPCIDR from"ip-cidr";import PrivateIp from"private-ip";import{StatusError}from"./status-error.js";import config from"../config.js";import{httpAgent,httpsAgent}from"./http.js";const pipeline=util.promisify(stream.pipeline);export async function downloadUrl(url,path){if(process.env.NODE_ENV!=="production")console.log(`Downloading ${url} to ${path} ...`);const timeout=30*1e3;const operationTimeout=60*1e3;const req=got.stream(url,{headers:{"User-Agent":config.userAgent},timeout:{lookup:timeout,connect:timeout,secureConnect:timeout,socket:timeout,response:timeout,send:timeout,request:operationTimeout},agent:{http:httpAgent,https:httpsAgent},http2:true,retry:{limit:0},enableUnixSockets:false}).on("response",res=>{if((process.env.NODE_ENV==="production"||process.env.NODE_ENV==="test")&&!config.proxy&&res.ip){if(isPrivateIp(res.ip)){console.log(`Blocked address: ${res.ip}`);req.destroy()}}const contentLength=res.headers["content-length"];if(contentLength!=null){const size=Number(contentLength);if(size>config.maxSize){console.log(`maxSize exceeded (${size} > ${config.maxSize}) on response`);req.destroy()}}}).on("downloadProgress",progress=>{if(progress.transferred>config.maxSize){console.log(`maxSize exceeded (${progress.transferred} > ${config.maxSize}) on downloadProgress`);req.destroy()}});try{await pipeline(req,fs.createWriteStream(path))}catch(e){if(e instanceof Got.HTTPError){throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`,e.response.statusCode,e.response.statusMessage)}else{throw e}}if(process.env.NODE_ENV!=="production")console.log(`Download finished: ${url}`)}function isPrivateIp(ip){for(const net of config.allowedPrivateNetworks??[]){const cidr=new IPCIDR(net);if(cidr.contains(ip)){return false}}return PrivateIp(ip)??false} \ No newline at end of file +import * as fs from 'node:fs'; +import * as stream from 'node:stream'; +import * as util from 'node:util'; +import got, * as Got from 'got'; +import IPCIDR from 'ip-cidr'; +import PrivateIp from 'private-ip'; +import { StatusError } from './status-error.js'; +import { getAgents } from './http.js'; +const pipeline = util.promisify(stream.pipeline); +export const defaultDownloadConfig = { + userAgent: `MisskeyMediaProxy/0.0.0`, + allowedPrivateNetworks: [], + maxSize: 262144000, + proxy: false, + ...getAgents() +}; +export async function downloadUrl(url, path, settings = defaultDownloadConfig) { + if (process.env.NODE_ENV !== 'production') + console.log(`Downloading ${url} to ${path} ...`); + const timeout = 30 * 1000; + const operationTimeout = 60 * 1000; + const req = got.stream(url, { + headers: { + 'User-Agent': settings.userAgent, + }, + timeout: { + lookup: timeout, + connect: timeout, + secureConnect: timeout, + socket: timeout, + response: timeout, + send: timeout, + request: operationTimeout, // whole operation timeout + }, + agent: { + http: settings.httpAgent, + https: settings.httpsAgent, + }, + http2: true, + retry: { + limit: 0, + }, + enableUnixSockets: false, + }).on('response', (res) => { + if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !settings.proxy && res.ip) { + if (isPrivateIp(res.ip, settings.allowedPrivateNetworks)) { + console.log(`Blocked address: ${res.ip}`); + req.destroy(); + } + } + const contentLength = res.headers['content-length']; + if (contentLength != null) { + const size = Number(contentLength); + if (size > settings.maxSize) { + console.log(`maxSize exceeded (${size} > ${settings.maxSize}) on response`); + req.destroy(); + } + } + }).on('downloadProgress', (progress) => { + if (progress.transferred > settings.maxSize) { + console.log(`maxSize exceeded (${progress.transferred} > ${settings.maxSize}) on downloadProgress`); + req.destroy(); + } + }); + try { + await pipeline(req, fs.createWriteStream(path)); + } + catch (e) { + if (e instanceof Got.HTTPError) { + throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage); + } + else { + throw e; + } + } + if (process.env.NODE_ENV !== 'production') + console.log(`Download finished: ${url}`); +} +function isPrivateIp(ip, allowedPrivateNetworks) { + for (const net of allowedPrivateNetworks ?? []) { + const cidr = new IPCIDR(net); + if (cidr.contains(ip)) { + return false; + } + } + return PrivateIp(ip) ?? false; +} diff --git a/built/file-info.d.ts b/built/file-info.d.ts new file mode 100644 index 0000000..76fc22b --- /dev/null +++ b/built/file-info.d.ts @@ -0,0 +1,11 @@ +export declare function detectType(path: string): Promise<{ + mime: string; + ext: string | null; +}>; +declare const dictionary: { + 'safe-file': string[]; + 'sharp-convertible-image': string[]; + 'sharp-animation-convertible-image': string[]; +}; +export declare const isMimeImage: (mime: string, type: keyof typeof dictionary) => boolean; +export {}; diff --git a/built/file-info.js b/built/file-info.js index a9dd393..a04c39b 100644 --- a/built/file-info.js +++ b/built/file-info.js @@ -1 +1,58 @@ -import fs from"node:fs";import{fileTypeFromFile}from"file-type";import isSvg from"is-svg";import{promisify}from"node:util";const TYPE_OCTET_STREAM={mime:"application/octet-stream",ext:null};const TYPE_SVG={mime:"image/svg+xml",ext:"svg"};async function getFileSize(path){const getStat=promisify(fs.stat);return(await getStat(path)).size}export async function detectType(path){const fileSize=await getFileSize(path);if(fileSize===0){return TYPE_OCTET_STREAM}const type=await fileTypeFromFile(path);if(type){if(type.mime==="application/xml"&&await checkSvg(path)){return TYPE_SVG}return{mime:type.mime,ext:type.ext}}if(await checkSvg(path)){return TYPE_SVG}return TYPE_OCTET_STREAM}async function checkSvg(path){try{const size=await getFileSize(path);if(size>1*1024*1024)return false;return isSvg(fs.readFileSync(path))}catch{return false}}import{FILE_TYPE_BROWSERSAFE}from"./const.js";const dictionary={"safe-file":FILE_TYPE_BROWSERSAFE,"sharp-convertible-image":["image/jpeg","image/png","image/gif","image/apng","image/vnd.mozilla.apng","image/webp","image/avif","image/svg+xml"],"sharp-animation-convertible-image":["image/jpeg","image/png","image/gif","image/webp","image/avif","image/svg+xml"]};export const isMimeImage=(mime,type)=>dictionary[type].includes(mime); \ No newline at end of file +import fs from 'node:fs'; +import { fileTypeFromFile } from 'file-type'; +import isSvg from 'is-svg'; +import { promisify } from 'node:util'; +const TYPE_OCTET_STREAM = { + mime: 'application/octet-stream', + ext: null, +}; +const TYPE_SVG = { + mime: 'image/svg+xml', + ext: 'svg', +}; +async function getFileSize(path) { + const getStat = promisify(fs.stat); + return (await getStat(path)).size; +} +export async function detectType(path) { + // Check 0 byte + const fileSize = await getFileSize(path); + if (fileSize === 0) { + return TYPE_OCTET_STREAM; + } + const type = await fileTypeFromFile(path); + if (type) { + // XMLはSVGかもしれない + if (type.mime === 'application/xml' && await checkSvg(path)) { + return TYPE_SVG; + } + return { + mime: type.mime, + ext: type.ext, + }; + } + // 種類が不明でもSVGかもしれない + if (await checkSvg(path)) { + return TYPE_SVG; + } + // それでも種類が不明なら application/octet-stream にする + return TYPE_OCTET_STREAM; +} +async function checkSvg(path) { + try { + const size = await getFileSize(path); + if (size > 1 * 1024 * 1024) + return false; + return isSvg(fs.readFileSync(path)); + } + catch { + return false; + } +} +import { FILE_TYPE_BROWSERSAFE } from './const.js'; +const dictionary = { + 'safe-file': FILE_TYPE_BROWSERSAFE, + 'sharp-convertible-image': ['image/jpeg', 'image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng', 'image/webp', 'image/avif', 'image/svg+xml'], + 'sharp-animation-convertible-image': ['image/jpeg', 'image/png', 'image/gif', 'image/webp', 'image/avif', 'image/svg+xml'], +}; +export const isMimeImage = (mime, type) => dictionary[type].includes(mime); diff --git a/built/http.d.ts b/built/http.d.ts new file mode 100644 index 0000000..fa2402f --- /dev/null +++ b/built/http.d.ts @@ -0,0 +1,8 @@ +/// +/// +import * as http from 'node:http'; +import * as https from 'node:https'; +export declare function getAgents(proxy?: string): { + httpAgent: http.Agent; + httpsAgent: https.Agent; +}; diff --git a/built/http.js b/built/http.js index 1c4189f..e9ff3ae 100644 --- a/built/http.js +++ b/built/http.js @@ -1 +1,45 @@ -import*as http from"node:http";import*as https from"node:https";import CacheableLookup from"cacheable-lookup";import{HttpProxyAgent,HttpsProxyAgent}from"hpagent";import config from"../config.js";const cache=new CacheableLookup({maxTtl:3600,errorTtl:30,lookup:false});const _http=new http.Agent({keepAlive:true,keepAliveMsecs:30*1e3,lookup:cache.lookup});const _https=new https.Agent({keepAlive:true,keepAliveMsecs:30*1e3,lookup:cache.lookup});export const httpAgent=config.proxy?new HttpProxyAgent({keepAlive:true,keepAliveMsecs:30*1e3,maxSockets:256,maxFreeSockets:256,scheduling:"lifo",proxy:config.proxy}):_http;export const httpsAgent=config.proxy?new HttpsProxyAgent({keepAlive:true,keepAliveMsecs:30*1e3,maxSockets:256,maxFreeSockets:256,scheduling:"lifo",proxy:config.proxy}):_https; \ No newline at end of file +import * as http from 'node:http'; +import * as https from 'node:https'; +import CacheableLookup from 'cacheable-lookup'; +import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'; +const cache = new CacheableLookup({ + maxTtl: 3600, + errorTtl: 30, + lookup: false, // nativeのdns.lookupにfallbackしない +}); +const _http = new http.Agent({ + keepAlive: true, + keepAliveMsecs: 30 * 1000, + lookup: cache.lookup, +}); +const _https = new https.Agent({ + keepAlive: true, + keepAliveMsecs: 30 * 1000, + lookup: cache.lookup, +}); +export function getAgents(proxy) { + const httpAgent = proxy + ? new HttpProxyAgent({ + keepAlive: true, + keepAliveMsecs: 30 * 1000, + maxSockets: 256, + maxFreeSockets: 256, + scheduling: 'lifo', + proxy: proxy, + }) + : _http; + const httpsAgent = proxy + ? new HttpsProxyAgent({ + keepAlive: true, + keepAliveMsecs: 30 * 1000, + maxSockets: 256, + maxFreeSockets: 256, + scheduling: 'lifo', + proxy: proxy, + }) + : _https; + return { + httpAgent, + httpsAgent, + }; +} diff --git a/built/image-processor.d.ts b/built/image-processor.d.ts new file mode 100644 index 0000000..717e6b8 --- /dev/null +++ b/built/image-processor.d.ts @@ -0,0 +1,18 @@ +/// +/// +import sharp from 'sharp'; +import { Readable } from 'node:stream'; +export type IImage = { + data: Buffer; + ext: string | null; + type: string; +}; +export type IImageStream = { + data: Readable; + ext: string | null; + type: string; +}; +export type IImageStreamable = IImage | IImageStream; +export declare const webpDefault: sharp.WebpOptions; +export declare function convertToWebpStream(path: string, width: number, height: number, options?: sharp.WebpOptions): IImageStream; +export declare function convertSharpToWebpStream(sharp: sharp.Sharp, width: number, height: number, options?: sharp.WebpOptions): IImageStream; diff --git a/built/image-processor.js b/built/image-processor.js index a62f340..be63182 100644 --- a/built/image-processor.js +++ b/built/image-processor.js @@ -1 +1,26 @@ -import sharp from"sharp";export const webpDefault={quality:85,alphaQuality:95,lossless:false,nearLossless:false,smartSubsample:true,mixed:true};export function convertToWebpStream(path,width,height,options=webpDefault){return convertSharpToWebpStream(sharp(path),width,height,options)}export function convertSharpToWebpStream(sharp,width,height,options=webpDefault){const data=sharp.resize(width,height,{fit:"inside",withoutEnlargement:true}).rotate().webp(options);return{data,ext:"webp",type:"image/webp"}} \ No newline at end of file +import sharp from 'sharp'; +export const webpDefault = { + quality: 85, + alphaQuality: 95, + lossless: false, + nearLossless: false, + smartSubsample: true, + mixed: true, +}; +export function convertToWebpStream(path, width, height, options = webpDefault) { + return convertSharpToWebpStream(sharp(path), width, height, options); +} +export function convertSharpToWebpStream(sharp, width, height, options = webpDefault) { + const data = sharp + .resize(width, height, { + fit: 'inside', + withoutEnlargement: true, + }) + .rotate() + .webp(options); + return { + data, + ext: 'webp', + type: 'image/webp', + }; +} diff --git a/built/index.d.ts b/built/index.d.ts new file mode 100644 index 0000000..9032e39 --- /dev/null +++ b/built/index.d.ts @@ -0,0 +1,17 @@ +/// +/// +import * as http from 'node:http'; +import * as https from 'node:https'; +import type { FastifyInstance } from 'fastify'; +export type MediaProxyOptions = { + userAgent?: string; + allowedPrivateNetworks?: string[]; + maxSize?: number; +} & ({ + proxy?: string; +} | { + httpAgent: http.Agent; + httpsAgent: https.Agent; +}); +export declare function setMediaProxyConfig(setting?: MediaProxyOptions | null): void; +export default function (fastify: FastifyInstance, options: MediaProxyOptions | null | undefined, done: (err?: Error) => void): void; diff --git a/built/index.js b/built/index.js index 95fdfa2..54744f9 100644 --- a/built/index.js +++ b/built/index.js @@ -1 +1,192 @@ -import*as fs from"node:fs";import{fileURLToPath}from"node:url";import{dirname}from"node:path";import fastifyStatic from"@fastify/static";import{createTemp}from"./create-temp.js";import{FILE_TYPE_BROWSERSAFE}from"./const.js";import{convertToWebpStream,webpDefault}from"./image-processor.js";import{detectType,isMimeImage}from"./file-info.js";import sharp from"sharp";import{StatusError}from"./status-error.js";import{downloadUrl}from"./download.js";const _filename=fileURLToPath(import.meta.url);const _dirname=dirname(_filename);const assets=`${_dirname}/../../server/file/assets/`;export default function(fastify,options,done){fastify.addHook("onRequest",(request,reply,done)=>{reply.header("Content-Security-Policy",`default-src 'none'; img-src 'self'; media-src 'self'; style-src 'unsafe-inline'`);done()});fastify.register(fastifyStatic,{root:_dirname,serve:false});fastify.get("/:url*",async(request,reply)=>{return await proxyHandler(request,reply).catch(err=>errorHandler(request,reply,err))});done()}function errorHandler(request,reply,err){console.log(`${err}`);reply.header("Cache-Control","max-age=300");if(request.query&&"fallback"in request.query){return reply.sendFile("/dummy.png",assets)}if(err instanceof StatusError&&(err.statusCode===302||err.isClientError)){reply.code(err.statusCode);return}reply.code(500);return}async function proxyHandler(request,reply){const url="url"in request.query?request.query.url:"https://"+request.params.url;if(typeof url!=="string"){reply.code(400);return}const file=await downloadAndDetectTypeFromUrl(url);try{const isConvertibleImage=isMimeImage(file.mime,"sharp-convertible-image");const isAnimationConvertibleImage=isMimeImage(file.mime,"sharp-animation-convertible-image");let image=null;if(("emoji"in request.query||"avatar"in request.query)&&isConvertibleImage){if(!isAnimationConvertibleImage&&!("static"in request.query)){image={data:fs.createReadStream(file.path),ext:file.ext,type:file.mime}}else{const data=sharp(file.path,{animated:!("static"in request.query)}).resize({height:"emoji"in request.query?128:320,withoutEnlargement:true}).webp(webpDefault);image={data,ext:"webp",type:"image/webp"}}}else if("static"in request.query&&isConvertibleImage){image=convertToWebpStream(file.path,498,280)}else if("preview"in request.query&&isConvertibleImage){image=convertToWebpStream(file.path,200,200)}else if("badge"in request.query){if(!isConvertibleImage){throw new StatusError("Unexpected mime",404)}const mask=sharp(file.path).resize(96,96,{fit:"inside",withoutEnlargement:false}).greyscale().normalise().linear(1.75,-128*1.75+128).flatten({background:"#000"}).toColorspace("b-w");const stats=await mask.clone().stats();if(stats.entropy<.1){throw new StatusError("Skip to provide badge",404)}const data=sharp({create:{width:96,height:96,channels:4,background:{r:0,g:0,b:0,alpha:0}}}).pipelineColorspace("b-w").boolean(await mask.png().toBuffer(),"eor");image={data:await data.png().toBuffer(),ext:"png",type:"image/png"}}else if(file.mime==="image/svg+xml"){image=convertToWebpStream(file.path,2048,2048)}else if(!file.mime.startsWith("image/")||!FILE_TYPE_BROWSERSAFE.includes(file.mime)){throw new StatusError("Rejected type",403,"Rejected type")}if(!image){image={data:fs.createReadStream(file.path),ext:file.ext,type:file.mime}}if("cleanup"in file){if("pipe"in image.data&&typeof image.data.pipe==="function"){image.data.on("end",file.cleanup);image.data.on("close",file.cleanup)}else{file.cleanup()}}reply.header("Content-Type",image.type);reply.header("Cache-Control","max-age=31536000, immutable");return image.data}catch(e){if("cleanup"in file)file.cleanup();throw e}}async function downloadAndDetectTypeFromUrl(url){const[path,cleanup]=await createTemp();try{await downloadUrl(url,path);const{mime,ext}=await detectType(path);return{state:"remote",mime,ext,path,cleanup}}catch(e){cleanup();throw e}} \ No newline at end of file +import * as fs from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname } from 'node:path'; +import fastifyStatic from '@fastify/static'; +import { createTemp } from './create-temp.js'; +import { FILE_TYPE_BROWSERSAFE } from './const.js'; +import { convertToWebpStream, webpDefault } from './image-processor.js'; +import { detectType, isMimeImage } from './file-info.js'; +import sharp from 'sharp'; +import { StatusError } from './status-error.js'; +import { defaultDownloadConfig, downloadUrl } from './download.js'; +import { getAgents } from './http.js'; +const _filename = fileURLToPath(import.meta.url); +const _dirname = dirname(_filename); +const assets = `${_dirname}/../../server/file/assets/`; +let config = defaultDownloadConfig; +export function setMediaProxyConfig(setting) { + const proxy = process.env.HTTP_PROXY ?? process.env.http_proxy; + if (!setting) { + config = { + ...defaultDownloadConfig, + ...(proxy ? getAgents(proxy) : {}), + proxy: !!proxy, + }; + console.log(config); + return; + } + config = { + userAgent: setting.userAgent ?? defaultDownloadConfig.userAgent, + allowedPrivateNetworks: setting.allowedPrivateNetworks ?? defaultDownloadConfig.allowedPrivateNetworks, + maxSize: setting.maxSize ?? defaultDownloadConfig.maxSize, + ...('proxy' in setting ? + { ...getAgents(setting.proxy), proxy: !!setting.proxy } : + 'httpAgent' in setting ? { + httpAgent: setting.httpAgent, + httpsAgent: setting.httpsAgent, + proxy: true, + } : + { ...getAgents(proxy), proxy: !!proxy }), + }; + console.log(config); +} +export default function (fastify, options, done) { + setMediaProxyConfig(options); + fastify.addHook('onRequest', (request, reply, done) => { + reply.header('Content-Security-Policy', `default-src 'none'; img-src 'self'; media-src 'self'; style-src 'unsafe-inline'`); + done(); + }); + fastify.register(fastifyStatic, { + root: _dirname, + serve: false, + }); + fastify.get('/:url*', async (request, reply) => { + return await proxyHandler(request, reply) + .catch(err => errorHandler(request, reply, err)); + }); + done(); +} +function errorHandler(request, reply, err) { + console.log(`${err}`); + reply.header('Cache-Control', 'max-age=300'); + if (request.query && 'fallback' in request.query) { + return reply.sendFile('/dummy.png', assets); + } + if (err instanceof StatusError && (err.statusCode === 302 || err.isClientError)) { + reply.code(err.statusCode); + return; + } + reply.code(500); + return; +} +async function proxyHandler(request, reply) { + const url = 'url' in request.query ? request.query.url : (request.params.url && 'https://' + request.params.url); + if (!url || typeof url !== 'string') { + reply.code(400); + return; + } + // Create temp file + const file = await downloadAndDetectTypeFromUrl(url); + try { + const isConvertibleImage = isMimeImage(file.mime, 'sharp-convertible-image'); + const isAnimationConvertibleImage = isMimeImage(file.mime, 'sharp-animation-convertible-image'); + let image = null; + if (('emoji' in request.query || 'avatar' in request.query) && isConvertibleImage) { + if (!isAnimationConvertibleImage && !('static' in request.query)) { + image = { + data: fs.createReadStream(file.path), + ext: file.ext, + type: file.mime, + }; + } + else { + const data = sharp(file.path, { animated: !('static' in request.query) }) + .resize({ + height: 'emoji' in request.query ? 128 : 320, + withoutEnlargement: true, + }) + .webp(webpDefault); + image = { + data, + ext: 'webp', + type: 'image/webp', + }; + } + } + else if ('static' in request.query && isConvertibleImage) { + image = convertToWebpStream(file.path, 498, 280); + } + else if ('preview' in request.query && isConvertibleImage) { + image = convertToWebpStream(file.path, 200, 200); + } + else if ('badge' in request.query) { + if (!isConvertibleImage) { + // 画像でないなら404でお茶を濁す + throw new StatusError('Unexpected mime', 404); + } + const mask = sharp(file.path) + .resize(96, 96, { + fit: 'inside', + withoutEnlargement: false, + }) + .greyscale() + .normalise() + .linear(1.75, -(128 * 1.75) + 128) // 1.75x contrast + .flatten({ background: '#000' }) + .toColorspace('b-w'); + const stats = await mask.clone().stats(); + if (stats.entropy < 0.1) { + // エントロピーがあまりない場合は404にする + throw new StatusError('Skip to provide badge', 404); + } + const data = sharp({ + create: { width: 96, height: 96, channels: 4, background: { r: 0, g: 0, b: 0, alpha: 0 } }, + }) + .pipelineColorspace('b-w') + .boolean(await mask.png().toBuffer(), 'eor'); + image = { + data: await data.png().toBuffer(), + ext: 'png', + type: 'image/png', + }; + } + else if (file.mime === 'image/svg+xml') { + image = convertToWebpStream(file.path, 2048, 2048); + } + else if (!file.mime.startsWith('image/') || !FILE_TYPE_BROWSERSAFE.includes(file.mime)) { + throw new StatusError('Rejected type', 403, 'Rejected type'); + } + if (!image) { + image = { + data: fs.createReadStream(file.path), + ext: file.ext, + type: file.mime, + }; + } + if ('cleanup' in file) { + if ('pipe' in image.data && typeof image.data.pipe === 'function') { + // image.dataがstreamなら、stream終了後にcleanup + image.data.on('end', file.cleanup); + image.data.on('close', file.cleanup); + } + else { + // image.dataがstreamでないなら直ちにcleanup + file.cleanup(); + } + } + reply.header('Content-Type', image.type); + reply.header('Cache-Control', 'max-age=31536000, immutable'); + return image.data; + } + catch (e) { + if ('cleanup' in file) + file.cleanup(); + throw e; + } +} +async function downloadAndDetectTypeFromUrl(url) { + const [path, cleanup] = await createTemp(); + try { + await downloadUrl(url, path, config); + const { mime, ext } = await detectType(path); + return { + state: 'remote', + mime, ext, + path, cleanup, + }; + } + catch (e) { + cleanup(); + throw e; + } +} diff --git a/built/status-error.d.ts b/built/status-error.d.ts new file mode 100644 index 0000000..0f3f195 --- /dev/null +++ b/built/status-error.d.ts @@ -0,0 +1,6 @@ +export declare class StatusError extends Error { + statusCode: number; + statusMessage?: string; + isClientError: boolean; + constructor(message: string, statusCode: number, statusMessage?: string); +} diff --git a/built/status-error.js b/built/status-error.js index 7a0e647..348e1a4 100644 --- a/built/status-error.js +++ b/built/status-error.js @@ -1 +1,9 @@ -export class StatusError extends Error{constructor(message,statusCode,statusMessage){super(message);this.name="StatusError";this.statusCode=statusCode;this.statusMessage=statusMessage;this.isClientError=typeof this.statusCode==="number"&&this.statusCode>=400&&this.statusCode<500}} \ No newline at end of file +export class StatusError extends Error { + constructor(message, statusCode, statusMessage) { + super(message); + this.name = 'StatusError'; + this.statusCode = statusCode; + this.statusMessage = statusMessage; + this.isClientError = typeof this.statusCode === 'number' && this.statusCode >= 400 && this.statusCode < 500; + } +} diff --git a/package.json b/package.json index 33a8086..ac8492b 100644 --- a/package.json +++ b/package.json @@ -6,9 +6,9 @@ "packageManager": "pnpm@7.26.0", "type": "module", "scripts": { - "build": "swc src -d built -D", + "build": "tsc -p tsconfig.json", "watch": "swc src -d built -D -w & fastify start -w -l info -P ./built/index.js", - "start": "fastify start ./built/index.js" + "start": "fastify start ./server.js" }, "repository": { "type": "git", @@ -23,8 +23,10 @@ "devDependencies": { "@swc/cli": "^0.1.61", "@swc/core": "^1.3.32", + "@types/node": "^18.11.19", "@types/sharp": "^0.31.1", - "@types/tmp": "^0.2.3" + "@types/tmp": "^0.2.3", + "typescript": "^4.9.5" }, "dependencies": { "@fastify/static": "^6.8.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8e104f3..1f38951 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4,6 +4,7 @@ specifiers: '@fastify/static': ^6.8.0 '@swc/cli': ^0.1.61 '@swc/core': ^1.3.32 + '@types/node': ^18.11.19 '@types/sharp': ^0.31.1 '@types/tmp': ^0.2.3 cacheable-lookup: ^7.0.0 @@ -17,6 +18,7 @@ specifiers: private-ip: ^3.0.0 sharp: ^0.31.3 tmp: ^0.2.1 + typescript: ^4.9.5 dependencies: '@fastify/static': 6.8.0 @@ -35,8 +37,10 @@ dependencies: devDependencies: '@swc/cli': 0.1.61_@swc+core@1.3.32 '@swc/core': 1.3.32 + '@types/node': 18.11.19 '@types/sharp': 0.31.1 '@types/tmp': 0.2.3 + typescript: 4.9.5 packages: @@ -47,6 +51,7 @@ packages: /@fastify/accept-negotiator/1.1.0: resolution: {integrity: sha512-OIHZrb2ImZ7XG85HXOONLcJWGosv7sIvM2ifAPQVhg9Lv7qdmMBNVaai4QTdyuaqbKM5eO6sLSQOYI7wEQeCJQ==} engines: {node: '>=14'} + dev: false /@fastify/ajv-compiler/3.5.0: resolution: {integrity: sha512-ebbEtlI7dxXF5ziNdr05mOY8NnDiPB1XvAlLHctRt/Rc+C3LCOVW5imUVX+mhvUhnNzmPBHewUkOFgGlCxgdAA==} @@ -78,6 +83,7 @@ packages: fast-decode-uri-component: 1.0.1 http-errors: 2.0.0 mime: 3.0.0 + dev: false /@fastify/static/6.8.0: resolution: {integrity: sha512-MNQp7KM0NIC+722OPN3MholnfvM+Vg2ao4OwbWWNJhAJEWOKGe4fJsEjIh3OkN0z5ymhklc7EXGCG0zDaIU5ZQ==} @@ -94,6 +100,7 @@ packages: /@lukeed/ms/2.0.1: resolution: {integrity: sha512-Xs/4RZltsAL7pkvaNStUQt7netTkyxrS0K+RILcVr3TRMS/ToOg4I6uNfhB9SlGsnWBym4U+EaXq0f0cEMNkHA==} engines: {node: '>=8'} + dev: false /@mole-inc/bin-wrapper/8.0.1: resolution: {integrity: sha512-sTGoeZnjI8N4KS+sW2AN95gDBErhAguvkw/tWdCjeM8bvxpz5lqrnd0vOJABA1A+Ic3zED7PYoLP/RANLgVotA==} @@ -289,7 +296,7 @@ packages: dependencies: '@types/http-cache-semantics': 4.0.1 '@types/keyv': 3.1.4 - '@types/node': 18.11.18 + '@types/node': 18.11.19 '@types/responselike': 1.0.0 dev: true @@ -299,23 +306,23 @@ packages: /@types/keyv/3.1.4: resolution: {integrity: sha512-BQ5aZNSCpj7D6K2ksrRCTmKRLEpnPvWDiLPfoGyhZ++8YtiK9d/3DBKPJgry359X/P1PfruyYwvnvwFjuEiEIg==} dependencies: - '@types/node': 18.11.18 + '@types/node': 18.11.19 dev: true - /@types/node/18.11.18: - resolution: {integrity: sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==} + /@types/node/18.11.19: + resolution: {integrity: sha512-YUgMWAQBWLObABqrvx8qKO1enAvBUdjZOAWQ5grBAkp5LQv45jBvYKZ3oFS9iKRCQyFjqw6iuEa1vmFqtxYLZw==} dev: true /@types/responselike/1.0.0: resolution: {integrity: sha512-85Y2BjiufFzaMIlvJDvTTB8Fxl2xfLo4HgmHzVBz08w4wDePCTjYw66PdrolO0kzli3yam/YCgRufyo1DdQVTA==} dependencies: - '@types/node': 18.11.18 + '@types/node': 18.11.19 dev: true /@types/sharp/0.31.1: resolution: {integrity: sha512-5nWwamN9ZFHXaYEincMSuza8nNfOof8nmO+mcI+Agx1uMUk4/pQnNIcix+9rLPXzKrm1pS34+6WRDbDV0Jn7ag==} dependencies: - '@types/node': 18.11.18 + '@types/node': 18.11.19 dev: true /@types/tmp/0.2.3: @@ -646,6 +653,7 @@ packages: /depd/2.0.0: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} engines: {node: '>= 0.8'} + dev: false /detect-libc/2.0.1: resolution: {integrity: sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==} @@ -664,6 +672,7 @@ packages: /escape-html/1.0.3: resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==} + dev: false /escape-string-regexp/5.0.0: resolution: {integrity: sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==} @@ -1063,6 +1072,7 @@ packages: setprototypeof: 1.2.0 statuses: 2.0.1 toidentifier: 1.0.1 + dev: false /http2-wrapper/1.0.3: resolution: {integrity: sha512-V+23sDMr12Wnz7iTcDeJr3O6AIxlnvT/bmaAAAP/Xda35C90p9599p0F1eHR/N1KILWSoWVAiOMFjBBXaXSMxg==} @@ -1291,6 +1301,7 @@ packages: resolution: {integrity: sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==} engines: {node: '>=10.0.0'} hasBin: true + dev: false /mimic-fn/2.1.0: resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==} @@ -1435,6 +1446,7 @@ packages: engines: {node: '>=10'} dependencies: yocto-queue: 0.1.0 + dev: false /p-locate/3.0.0: resolution: {integrity: sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==} @@ -1755,6 +1767,7 @@ packages: /setprototypeof/1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} + dev: false /sharp/0.31.3: resolution: {integrity: sha512-XcR4+FCLBFKw1bdB+GEhnUNXNXvnt0tDo4WsBsraKymuo/IAuPuCBVAL2wIkUw2r/dwFW5Q5+g66Kwl2dgDFVg==} @@ -1865,6 +1878,7 @@ packages: /statuses/2.0.1: resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==} engines: {node: '>= 0.8'} + dev: false /string_decoder/1.3.0: resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} @@ -1965,6 +1979,7 @@ packages: /toidentifier/1.0.1: resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} engines: {node: '>=0.6'} + dev: false /token-types/5.0.1: resolution: {integrity: sha512-Y2fmSnZjQdDb9W4w4r1tswlMHylzWIeOKpx0aZH9BgGtACHhrk3OkT52AzwcuqTRBZtvvnTjDBh8eynMulu8Vg==} @@ -1986,6 +2001,12 @@ packages: safe-buffer: 5.2.1 dev: false + /typescript/4.9.5: + resolution: {integrity: sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==} + engines: {node: '>=4.2.0'} + hasBin: true + dev: true + /uri-js/4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} dependencies: @@ -2034,3 +2055,4 @@ packages: /yocto-queue/0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} + dev: false diff --git a/server.js b/server.js new file mode 100644 index 0000000..813feea --- /dev/null +++ b/server.js @@ -0,0 +1,6 @@ +import config from './config.js'; +import app from './built/index.js'; + +export default function (fastify, opts, next) { + return app(fastify, { ...config, ...opts }, next); +} diff --git a/src/download.ts b/src/download.ts index 04d6995..4401aa6 100644 --- a/src/download.ts +++ b/src/download.ts @@ -1,16 +1,35 @@ import * as fs from 'node:fs'; import * as stream from 'node:stream'; import * as util from 'node:util'; +import * as http from 'node:http'; +import * as https from 'node:https'; import got, * as Got from 'got'; import IPCIDR from 'ip-cidr'; import PrivateIp from 'private-ip'; import { StatusError } from './status-error.js'; -import config from '../config.js'; -import { httpAgent, httpsAgent } from './http.js'; +import { getAgents } from './http.js'; const pipeline = util.promisify(stream.pipeline); -export async function downloadUrl(url: string, path: string): Promise { +export type DownloadConfig = { + [x: string]: any; + userAgent: string; + allowedPrivateNetworks: string[]; + maxSize: number; + httpAgent: http.Agent, + httpsAgent: https.Agent, + proxy?: boolean; +} + +export const defaultDownloadConfig = { + userAgent: `MisskeyMediaProxy/0.0.0`, + allowedPrivateNetworks: [], + maxSize: 262144000, + proxy: false, + ...getAgents() +} + +export async function downloadUrl(url: string, path: string, settings:DownloadConfig = defaultDownloadConfig): Promise { if (process.env.NODE_ENV !== 'production') console.log(`Downloading ${url} to ${path} ...`); const timeout = 30 * 1000; @@ -18,7 +37,7 @@ export async function downloadUrl(url: string, path: string): Promise { const req = got.stream(url, { headers: { - 'User-Agent': config.userAgent, + 'User-Agent': settings.userAgent, }, timeout: { lookup: timeout, @@ -30,8 +49,8 @@ export async function downloadUrl(url: string, path: string): Promise { request: operationTimeout, // whole operation timeout }, agent: { - http: httpAgent, - https: httpsAgent, + http: settings.httpAgent, + https: settings.httpsAgent, }, http2: true, retry: { @@ -39,8 +58,8 @@ export async function downloadUrl(url: string, path: string): Promise { }, enableUnixSockets: false, }).on('response', (res: Got.Response) => { - if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !config.proxy && res.ip) { - if (isPrivateIp(res.ip)) { + if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !settings.proxy && res.ip) { + if (isPrivateIp(res.ip, settings.allowedPrivateNetworks)) { console.log(`Blocked address: ${res.ip}`); req.destroy(); } @@ -49,14 +68,14 @@ export async function downloadUrl(url: string, path: string): Promise { const contentLength = res.headers['content-length']; if (contentLength != null) { const size = Number(contentLength); - if (size > config.maxSize) { - console.log(`maxSize exceeded (${size} > ${config.maxSize}) on response`); + if (size > settings.maxSize) { + console.log(`maxSize exceeded (${size} > ${settings.maxSize}) on response`); req.destroy(); } } }).on('downloadProgress', (progress: Got.Progress) => { - if (progress.transferred > config.maxSize) { - console.log(`maxSize exceeded (${progress.transferred} > ${config.maxSize}) on downloadProgress`); + if (progress.transferred > settings.maxSize) { + console.log(`maxSize exceeded (${progress.transferred} > ${settings.maxSize}) on downloadProgress`); req.destroy(); } }); @@ -75,8 +94,8 @@ export async function downloadUrl(url: string, path: string): Promise { } -function isPrivateIp(ip: string): boolean { - for (const net of config.allowedPrivateNetworks ?? []) { +function isPrivateIp(ip: string, allowedPrivateNetworks: string[]): boolean { + for (const net of allowedPrivateNetworks ?? []) { const cidr = new IPCIDR(net); if (cidr.contains(ip)) { return false; diff --git a/src/http.ts b/src/http.ts index 40f7e7b..c8c45cd 100644 --- a/src/http.ts +++ b/src/http.ts @@ -2,7 +2,6 @@ import * as http from 'node:http'; import * as https from 'node:https'; import CacheableLookup from 'cacheable-lookup'; import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'; -import config from "../config.js"; const cache = new CacheableLookup({ maxTtl: 3600, // 1hours @@ -22,24 +21,31 @@ const _https = new https.Agent({ lookup: cache.lookup, } as https.AgentOptions); -export const httpAgent = config.proxy - ? new HttpProxyAgent({ - keepAlive: true, - keepAliveMsecs: 30 * 1000, - maxSockets: 256, - maxFreeSockets: 256, - scheduling: 'lifo', - proxy: config.proxy, - }) - : _http; +export function getAgents(proxy?: string) { + const httpAgent = proxy + ? new HttpProxyAgent({ + keepAlive: true, + keepAliveMsecs: 30 * 1000, + maxSockets: 256, + maxFreeSockets: 256, + scheduling: 'lifo', + proxy: proxy, + }) + : _http; -export const httpsAgent = config.proxy - ? new HttpsProxyAgent({ - keepAlive: true, - keepAliveMsecs: 30 * 1000, - maxSockets: 256, - maxFreeSockets: 256, - scheduling: 'lifo', - proxy: config.proxy, - }) - : _https; \ No newline at end of file + const httpsAgent = proxy + ? new HttpsProxyAgent({ + keepAlive: true, + keepAliveMsecs: 30 * 1000, + maxSockets: 256, + maxFreeSockets: 256, + scheduling: 'lifo', + proxy: proxy, + }) + : _https; + + return { + httpAgent, + httpsAgent, + }; +} diff --git a/src/index.ts b/src/index.ts index 1c116f7..96d8c49 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,6 @@ import * as fs from 'node:fs'; +import * as http from 'node:http'; +import * as https from 'node:https'; import { fileURLToPath } from 'node:url'; import { dirname } from 'node:path'; import fastifyStatic from '@fastify/static'; @@ -9,14 +11,60 @@ import type { FastifyInstance, FastifyRequest, FastifyReply, FastifyPluginOption import { detectType, isMimeImage } from './file-info.js'; import sharp from 'sharp'; import { StatusError } from './status-error.js'; -import { downloadUrl } from './download.js'; +import { DownloadConfig, defaultDownloadConfig, downloadUrl } from './download.js'; +import { getAgents } from './http.js'; const _filename = fileURLToPath(import.meta.url); const _dirname = dirname(_filename); const assets = `${_dirname}/../../server/file/assets/`; -export default function (fastify: FastifyInstance, options: FastifyPluginOptions, done: (err?: Error) => void) { +export type MediaProxyOptions = { + userAgent?: string; + allowedPrivateNetworks?: string[]; + maxSize?: number; +} & ({ + proxy?: string; +} | { + httpAgent: http.Agent; + httpsAgent: https.Agent; +}); + +let config: DownloadConfig = defaultDownloadConfig; + +export function setMediaProxyConfig(setting?: MediaProxyOptions | null) { + const proxy = process.env.HTTP_PROXY ?? process.env.http_proxy; + + if (!setting) { + config = { + ...defaultDownloadConfig, + ...(proxy ? getAgents(proxy) : {}), + proxy: !!proxy, + }; + console.log(config); + return; + } + + config = { + userAgent: setting.userAgent ?? defaultDownloadConfig.userAgent, + allowedPrivateNetworks: setting.allowedPrivateNetworks ?? defaultDownloadConfig.allowedPrivateNetworks, + maxSize: setting.maxSize ?? defaultDownloadConfig.maxSize, + ...('proxy' in setting ? + { ...getAgents(setting.proxy), proxy: !!setting.proxy } : + 'httpAgent' in setting ? { + httpAgent: setting.httpAgent, + httpsAgent: setting.httpsAgent, + proxy: true, + } : + { ...getAgents(proxy), proxy: !!proxy }), + }; + + console.log(config); +} + +export default function (fastify: FastifyInstance, options: MediaProxyOptions | null | undefined, done: (err?: Error) => void) { + setMediaProxyConfig(options); + fastify.addHook('onRequest', (request, reply, done) => { reply.header('Content-Security-Policy', `default-src 'none'; img-src 'self'; media-src 'self'; style-src 'unsafe-inline'`); done(); @@ -57,9 +105,9 @@ function errorHandler(request: FastifyRequest<{ Params?: { [x: string]: any }; Q } async function proxyHandler(request: FastifyRequest<{ Params: { url: string; }; Querystring: { url?: string; }; }>, reply: FastifyReply) { - const url = 'url' in request.query ? request.query.url : 'https://' + request.params.url; + const url = 'url' in request.query ? request.query.url : (request.params.url && 'https://' + request.params.url); - if (typeof url !== 'string') { + if (!url || typeof url !== 'string') { reply.code(400); return; } @@ -171,7 +219,7 @@ async function downloadAndDetectTypeFromUrl(url: string): Promise< > { const [path, cleanup] = await createTemp(); try { - await downloadUrl(url, path); + await downloadUrl(url, path, config); const { mime, ext } = await detectType(path); diff --git a/tsconfig.json b/tsconfig.json index 544b529..b9a7abf 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -7,7 +7,7 @@ "noUnusedParameters": false, "noUnusedLocals": false, "noFallthroughCasesInSwitch": true, - "declaration": false, + "declaration": true, "sourceMap": false, "target": "es2021", "module": "esnext",