HTTPリクエストのKeep-AliveとPrxoy対応など (#5226)

* DriveのKeep-Alive, Proxy と APのProxy対応

* request系でKeep-Aliveするように

* fix lookup-dns-cache.d.ts

* remove debug output

src/@types/lookup-dns-cache.d.ts

@@ -1,17 +1,9 @@
 declare module 'lookup-dns-cache' {
-	type IPv4 = 4;
+	import { LookupOneOptions, LookupAllOptions, LookupOptions, LookupAddress } from 'dns'
 
-	type IPv6 = 6;
-
-	type Family = IPv4 | IPv6 | undefined;
-
-	interface IRunOptions {
-		family?: Family;
-		all?: boolean;
-	}
-
-	type RunCallback = (error: Error | null, address?: string | string[], family?: Family) => void;
-
-	export function lookup(hostname: string, options: IRunOptions | Family, callback: RunCallback): {} | undefined;
-	export function lookup(hostname: string, callback: RunCallback): {} | undefined;
+	function lookup(hostname: string, family: number, callback: (err: NodeJS.ErrnoException | null, address: string, family: number) => void): void;
+	function lookup(hostname: string, options: LookupOneOptions, callback: (err: NodeJS.ErrnoException | null, address: string, family: number) => void): void;
+	function lookup(hostname: string, options: LookupAllOptions, callback: (err: NodeJS.ErrnoException | null, addresses: LookupAddress[]) => void): void;
+	function lookup(hostname: string, options: LookupOptions, callback: (err: NodeJS.ErrnoException | null, address: string | LookupAddress[], family: number) => void): void;
+	function lookup(hostname: string, callback: (err: NodeJS.ErrnoException | null, address: string, family: number) => void): void;
 }

src/@types/promise-any.d.ts

@@ -1,7 +0,0 @@
-declare module 'promise-any' {
-	function promiseAny<T>(iterable: Iterable<T | PromiseLike<T>>): Promise<T>;
-
-	namespace promiseAny {} // Hack
-
-	export = promiseAny;
-}

src/misc/donwload-url.ts

@@ -29,6 +29,7 @@ export async function downloadUrl(url: string, path: string) {
 			url: new URL(url).href, // https://github.com/syuilo/misskey/issues/2637
 			proxy: config.proxy,
 			timeout: 10 * 1000,
+			forever: true,
 			headers: {
 				'User-Agent': config.userAgent
 			}

src/remote/activitypub/request.ts

@@ -1,8 +1,7 @@
-import { request } from 'https';
+import * as https from 'https';
 import { sign } from 'http-signature';
 import * as crypto from 'crypto';
-import { lookup, IRunOptions } from 'lookup-dns-cache';
-import * as promiseAny from 'promise-any';
+import * as cache from 'lookup-dns-cache';
 
 import config from '../../config';
 import { ILocalUser } from '../../models/entities/user';
@@ -12,9 +11,16 @@ import { UserKeypairs, Instances } from '../../models';
 import { fetchMeta } from '../../misc/fetch-meta';
 import { toPuny } from '../../misc/convert-host';
 import { ensure } from '../../prelude/ensure';
+import * as httpsProxyAgent from 'https-proxy-agent';
 
 export const logger = apLogger.createSubLogger('deliver');
 
+const agent = config.proxy
+	? new httpsProxyAgent(config.proxy)
+	: new https.Agent({
+			lookup: cache.lookup,
+		});
+
 export default async (user: ILocalUser, url: string, object: any) => {
 	const timeout = 10 * 1000;
 
@@ -47,24 +53,20 @@ export default async (user: ILocalUser, url: string, object: any) => {
 	sha256.update(data);
 	const hash = sha256.digest('base64');
 
-	const addr = await resolveAddr(hostname);
-	if (!addr) return;
-
 	const keypair = await UserKeypairs.findOne({
 		userId: user.id
 	}).then(ensure);
 
 	await new Promise((resolve, reject) => {
-		const req = request({
+		const req = https.request({
+			agent,
 			protocol,
-			hostname: addr,
-			setHost: false,
+			hostname,
 			port,
 			method: 'POST',
 			path: pathname + search,
 			timeout,
 			headers: {
-				'Host': host,
 				'User-Agent': config.userAgent,
 				'Content-Type': 'application/activity+json',
 				'Digest': `SHA-256=${hash}`
@@ -110,30 +112,3 @@ export default async (user: ILocalUser, url: string, object: any) => {
 	});
 	//#endregion
 };
-
-/**
- * Resolve host (with cached, asynchrony)
- */
-async function resolveAddr(domain: string) {
-	const af = config.outgoingAddressFamily || 'ipv4';
-	const useV4 = af == 'ipv4' || af == 'dual';
-	const useV6 = af == 'ipv6' || af == 'dual';
-
-	const promises = [];
-
-	if (!useV4 && !useV6) throw 'No usable address family available';
-	if (useV4) promises.push(resolveAddrInner(domain, { family: 4 }));
-	if (useV6) promises.push(resolveAddrInner(domain, { family: 6 }));
-
-	// v4/v6で先に取得できた方を採用する
-	return await promiseAny(promises);
-}
-
-function resolveAddrInner(domain: string, options: IRunOptions = {}): Promise<string> {
-	return new Promise((res, rej) => {
-		lookup(domain, options, (error, address) => {
-			if (error) return rej(error);
-			return res(Array.isArray(address) ? address[0] : address);
-		});
-	});
-}

src/remote/activitypub/resolver.ts

@@ -57,6 +57,7 @@ export default class Resolver {
 			url: value,
 			proxy: config.proxy,
 			timeout: this.timeout,
+			forever: true,
 			headers: {
 				'User-Agent': config.userAgent,
 				Accept: 'application/activity+json, application/ld+json'

src/server/web/url-preview.ts

@@ -22,6 +22,7 @@ module.exports = async (ctx: Koa.BaseContext) => {
 				url: ctx.query.url,
 				lang: ctx.query.lang || 'ja-JP'
 			},
+			forever: true,
 			json: true
 		}) : await summaly(ctx.query.url, {
 			followRedirects: false,

src/services/drive/add-file.ts

@@ -2,7 +2,6 @@ import { Buffer } from 'buffer';
 import * as fs from 'fs';
 
 import * as crypto from 'crypto';
-import * as Minio from 'minio';
 import * as uuid from 'uuid';
 import * as sharp from 'sharp';
 
@@ -21,6 +20,8 @@ import { IRemoteUser, User } from '../../models/entities/user';
 import { driveChart, perUserDriveChart, instanceChart } from '../chart';
 import { genId } from '../../misc/gen-id';
 import { isDuplicateKeyValueError } from '../../misc/is-duplicate-key-value-error';
+import * as S3 from 'aws-sdk/clients/s3';
+import { getS3 } from './s3';
 
 const logger = driveLogger.createSubLogger('register', 'yellow');
 
@@ -211,23 +212,21 @@ async function upload(key: string, stream: fs.ReadStream | Buffer, type: string,
 
 	const meta = await fetchMeta();
 
-	const minio = new Minio.Client({
-		endPoint: meta.objectStorageEndpoint!,
-		region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined,
-		port: meta.objectStoragePort ? meta.objectStoragePort : undefined,
-		useSSL: meta.objectStorageUseSSL,
-		accessKey: meta.objectStorageAccessKey!,
-		secretKey: meta.objectStorageSecretKey!,
-	});
+	const params = {
+		Bucket: meta.objectStorageBucket,
+		Key: key,
+		Body: stream,
+		ContentType: type,
+		CacheControl: 'max-age=31536000, immutable',
+	} as S3.PutObjectRequest;
 
-	const metadata = {
-		'Content-Type': type,
-		'Cache-Control': 'max-age=31536000, immutable'
-	} as Minio.ItemBucketMetadata;
+	if (filename) params.ContentDisposition = contentDisposition('inline', filename);
 
-	if (filename) metadata['Content-Disposition'] = contentDisposition('inline', filename);
+	const s3 = getS3(meta);
 
-	await minio.putObject(meta.objectStorageBucket!, key, stream, undefined, metadata);
+	const upload = s3.upload(params);
+
+	await upload.promise();
 }
 
 async function deleteOldFile(user: IRemoteUser) {

src/services/drive/delete-file.ts

@@ -1,10 +1,10 @@
-import * as Minio from 'minio';
 import { DriveFile } from '../../models/entities/drive-file';
 import { InternalStorage } from './internal-storage';
 import { DriveFiles, Instances, Notes } from '../../models';
 import { driveChart, perUserDriveChart, instanceChart } from '../chart';
 import { createDeleteObjectStorageFileJob } from '../../queue';
 import { fetchMeta } from '../../misc/fetch-meta';
+import { getS3 } from './s3';
 
 export async function deleteFile(file: DriveFile, isExpired = false) {
 	if (file.storedInternal) {
@@ -93,14 +93,10 @@ function postProcess(file: DriveFile, isExpired = false) {
 export async function deleteObjectStorageFile(key: string) {
 	const meta = await fetchMeta();
 
-	const minio = new Minio.Client({
-		endPoint: meta.objectStorageEndpoint!,
-		region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined,
-		port: meta.objectStoragePort ? meta.objectStoragePort : undefined,
-		useSSL: meta.objectStorageUseSSL,
-		accessKey: meta.objectStorageAccessKey!,
-		secretKey: meta.objectStorageSecretKey!,
-	});
+	const s3 = getS3(meta);
 
-	await minio.removeObject(meta.objectStorageBucket!, key);
+	await s3.deleteObject({
+		Bucket: meta.objectStorageBucket!,
+		Key: key
+	}).promise();
 }

src/services/drive/s3.ts

@@ -0,0 +1,31 @@
+import * as S3 from 'aws-sdk/clients/s3';
+import config from '../../config';
+import { Meta } from '../../models/entities/meta';
+import * as httpsProxyAgent from 'https-proxy-agent';
+import * as agentkeepalive from 'agentkeepalive';
+
+const httpsAgent = config.proxy
+	? new httpsProxyAgent(config.proxy)
+	: new agentkeepalive.HttpsAgent({
+			freeSocketTimeout: 30 * 1000
+		});
+
+export function getS3(meta: Meta) {
+	const conf = {
+		endpoint: meta.objectStorageEndpoint,
+		accessKeyId: meta.objectStorageAccessKey,
+		secretAccessKey: meta.objectStorageSecretKey,
+		region: meta.objectStorageRegion,
+		sslEnabled: meta.objectStorageUseSSL,
+		httpOptions: {
+		}
+	} as S3.ClientConfiguration;
+
+	if (meta.objectStorageUseSSL) {
+		conf.httpOptions!.agent = httpsAgent;
+	}
+
+	const s3 = new S3(conf);
+
+	return s3;
+}