refactoring

Resolve #7779

packages/backend/src/services/drive/add-file.ts

@@ -0,0 +1,466 @@
+import * as fs from 'fs';
+
+import { v4 as uuid } from 'uuid';
+
+import { publishMainStream, publishDriveStream } from '@/services/stream';
+import { deleteFile } from './delete-file';
+import { fetchMeta } from '@/misc/fetch-meta';
+import { GenerateVideoThumbnail } from './generate-video-thumbnail';
+import { driveLogger } from './logger';
+import { IImage, convertSharpToJpeg, convertSharpToWebp, convertSharpToPng, convertSharpToPngOrJpeg } from './image-processor';
+import { contentDisposition } from '@/misc/content-disposition';
+import { getFileInfo } from '@/misc/get-file-info';
+import { DriveFiles, DriveFolders, Users, Instances, UserProfiles } from '@/models/index';
+import { InternalStorage } from './internal-storage';
+import { DriveFile } from '@/models/entities/drive-file';
+import { IRemoteUser, User } from '@/models/entities/user';
+import { driveChart, perUserDriveChart, instanceChart } from '@/services/chart/index';
+import { genId } from '@/misc/gen-id';
+import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error';
+import * as S3 from 'aws-sdk/clients/s3';
+import { getS3 } from './s3';
+import * as sharp from 'sharp';
+
+const logger = driveLogger.createSubLogger('register', 'yellow');
+
+/***
+ * Save file
+ * @param path Path for original
+ * @param name Name for original
+ * @param type Content-Type for original
+ * @param hash Hash for original
+ * @param size Size for original
+ */
+async function save(file: DriveFile, path: string, name: string, type: string, hash: string, size: number): Promise<DriveFile> {
+	// thunbnail, webpublic を必要なら生成
+	const alts = await generateAlts(path, type, !file.uri);
+
+	const meta = await fetchMeta();
+
+	if (meta.useObjectStorage) {
+		//#region ObjectStorage params
+		let [ext] = (name.match(/\.([a-zA-Z0-9_-]+)$/) || ['']);
+
+		if (ext === '') {
+			if (type === 'image/jpeg') ext = '.jpg';
+			if (type === 'image/png') ext = '.png';
+			if (type === 'image/webp') ext = '.webp';
+			if (type === 'image/apng') ext = '.apng';
+			if (type === 'image/vnd.mozilla.apng') ext = '.apng';
+		}
+
+		const baseUrl = meta.objectStorageBaseUrl
+			|| `${ meta.objectStorageUseSSL ? 'https' : 'http' }://${ meta.objectStorageEndpoint }${ meta.objectStoragePort ? `:${meta.objectStoragePort}` : '' }/${ meta.objectStorageBucket }`;
+
+		// for original
+		const key = `${meta.objectStoragePrefix}/${uuid()}${ext}`;
+		const url = `${ baseUrl }/${ key }`;
+
+		// for alts
+		let webpublicKey: string | null = null;
+		let webpublicUrl: string | null = null;
+		let thumbnailKey: string | null = null;
+		let thumbnailUrl: string | null = null;
+		//#endregion
+
+		//#region Uploads
+		logger.info(`uploading original: ${key}`);
+		const uploads = [
+			upload(key, fs.createReadStream(path), type, name)
+		];
+
+		if (alts.webpublic) {
+			webpublicKey = `${meta.objectStoragePrefix}/webpublic-${uuid()}.${alts.webpublic.ext}`;
+			webpublicUrl = `${ baseUrl }/${ webpublicKey }`;
+
+			logger.info(`uploading webpublic: ${webpublicKey}`);
+			uploads.push(upload(webpublicKey, alts.webpublic.data, alts.webpublic.type, name));
+		}
+
+		if (alts.thumbnail) {
+			thumbnailKey = `${meta.objectStoragePrefix}/thumbnail-${uuid()}.${alts.thumbnail.ext}`;
+			thumbnailUrl = `${ baseUrl }/${ thumbnailKey }`;
+
+			logger.info(`uploading thumbnail: ${thumbnailKey}`);
+			uploads.push(upload(thumbnailKey, alts.thumbnail.data, alts.thumbnail.type));
+		}
+
+		await Promise.all(uploads);
+		//#endregion
+
+		file.url = url;
+		file.thumbnailUrl = thumbnailUrl;
+		file.webpublicUrl = webpublicUrl;
+		file.accessKey = key;
+		file.thumbnailAccessKey = thumbnailKey;
+		file.webpublicAccessKey = webpublicKey;
+		file.name = name;
+		file.type = type;
+		file.md5 = hash;
+		file.size = size;
+		file.storedInternal = false;
+
+		return await DriveFiles.save(file);
+	} else { // use internal storage
+		const accessKey = uuid();
+		const thumbnailAccessKey = 'thumbnail-' + uuid();
+		const webpublicAccessKey = 'webpublic-' + uuid();
+
+		const url = InternalStorage.saveFromPath(accessKey, path);
+
+		let thumbnailUrl: string | null = null;
+		let webpublicUrl: string | null = null;
+
+		if (alts.thumbnail) {
+			thumbnailUrl = InternalStorage.saveFromBuffer(thumbnailAccessKey, alts.thumbnail.data);
+			logger.info(`thumbnail stored: ${thumbnailAccessKey}`);
+		}
+
+		if (alts.webpublic) {
+			webpublicUrl = InternalStorage.saveFromBuffer(webpublicAccessKey, alts.webpublic.data);
+			logger.info(`web stored: ${webpublicAccessKey}`);
+		}
+
+		file.storedInternal = true;
+		file.url = url;
+		file.thumbnailUrl = thumbnailUrl;
+		file.webpublicUrl = webpublicUrl;
+		file.accessKey = accessKey;
+		file.thumbnailAccessKey = thumbnailAccessKey;
+		file.webpublicAccessKey = webpublicAccessKey;
+		file.name = name;
+		file.type = type;
+		file.md5 = hash;
+		file.size = size;
+
+		return await DriveFiles.save(file);
+	}
+}
+
+/**
+ * Generate webpublic, thumbnail, etc
+ * @param path Path for original
+ * @param type Content-Type for original
+ * @param generateWeb Generate webpublic or not
+ */
+export async function generateAlts(path: string, type: string, generateWeb: boolean) {
+	if (type.startsWith('video/')) {
+		try {
+			const thumbnail = await GenerateVideoThumbnail(path);
+			return {
+				webpublic: null,
+				thumbnail
+			};
+		} catch (e) {
+			logger.warn(`GenerateVideoThumbnail failed: ${e}`);
+			return {
+				webpublic: null,
+				thumbnail: null
+			};
+		}
+	}
+
+	if (!['image/jpeg', 'image/png', 'image/webp'].includes(type)) {
+		logger.debug(`web image and thumbnail not created (not an required file)`);
+		return {
+			webpublic: null,
+			thumbnail: null
+		};
+	}
+
+	let img: sharp.Sharp | null = null;
+
+	try {
+		img = sharp(path);
+		const metadata = await img.metadata();
+		const isAnimated = metadata.pages && metadata.pages > 1;
+
+		// skip animated
+		if (isAnimated) {
+			return {
+				webpublic: null,
+				thumbnail: null
+			};
+		}
+	} catch (e) {
+		logger.warn(`sharp failed: ${e}`);
+		return {
+			webpublic: null,
+			thumbnail: null
+		};
+	}
+
+	// #region webpublic
+	let webpublic: IImage | null = null;
+
+	if (generateWeb) {
+		logger.info(`creating web image`);
+
+		try {
+			if (['image/jpeg'].includes(type)) {
+				webpublic = await convertSharpToJpeg(img, 2048, 2048);
+			} else if (['image/webp'].includes(type)) {
+				webpublic = await convertSharpToWebp(img, 2048, 2048);
+			} else if (['image/png'].includes(type)) {
+				webpublic = await convertSharpToPng(img, 2048, 2048);
+			} else {
+				logger.debug(`web image not created (not an required image)`);
+			}
+		} catch (e) {
+			logger.warn(`web image not created (an error occured)`, e);
+		}
+	} else {
+		logger.info(`web image not created (from remote)`);
+	}
+	// #endregion webpublic
+
+	// #region thumbnail
+	let thumbnail: IImage | null = null;
+
+	try {
+		if (['image/jpeg', 'image/webp'].includes(type)) {
+			thumbnail = await convertSharpToJpeg(img, 498, 280);
+		} else if (['image/png'].includes(type)) {
+			thumbnail = await convertSharpToPngOrJpeg(img, 498, 280);
+		} else {
+			logger.debug(`thumbnail not created (not an required file)`);
+		}
+	} catch (e) {
+		logger.warn(`thumbnail not created (an error occured)`, e);
+	}
+	// #endregion thumbnail
+
+	return {
+		webpublic,
+		thumbnail,
+	};
+}
+
+/**
+ * Upload to ObjectStorage
+ */
+async function upload(key: string, stream: fs.ReadStream | Buffer, type: string, filename?: string) {
+	if (type === 'image/apng') type = 'image/png';
+
+	const meta = await fetchMeta();
+
+	const params = {
+		Bucket: meta.objectStorageBucket,
+		Key: key,
+		Body: stream,
+		ContentType: type,
+		CacheControl: 'max-age=31536000, immutable',
+	} as S3.PutObjectRequest;
+
+	if (filename) params.ContentDisposition = contentDisposition('inline', filename);
+	if (meta.objectStorageSetPublicRead) params.ACL = 'public-read';
+
+	const s3 = getS3(meta);
+
+	const upload = s3.upload(params, {
+		partSize: s3.endpoint?.hostname === 'storage.googleapis.com' ? 500 * 1024 * 1024 : 8 * 1024 * 1024
+	});
+
+	const result = await upload.promise();
+	if (result) logger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`);
+}
+
+async function deleteOldFile(user: IRemoteUser) {
+	const q = DriveFiles.createQueryBuilder('file')
+		.where('file.userId = :userId', { userId: user.id })
+		.andWhere('file.isLink = FALSE');
+
+	if (user.avatarId) {
+		q.andWhere('file.id != :avatarId', { avatarId: user.avatarId });
+	}
+
+	if (user.bannerId) {
+		q.andWhere('file.id != :bannerId', { bannerId: user.bannerId });
+	}
+
+	q.orderBy('file.id', 'ASC');
+
+	const oldFile = await q.getOne();
+
+	if (oldFile) {
+		deleteFile(oldFile, true);
+	}
+}
+
+/**
+ * Add file to drive
+ *
+ * @param user User who wish to add file
+ * @param path File path
+ * @param name Name
+ * @param comment Comment
+ * @param folderId Folder ID
+ * @param force If set to true, forcibly upload the file even if there is a file with the same hash.
+ * @param isLink Do not save file to local
+ * @param url URL of source (URLからアップロードされた場合(ローカル/リモート)の元URL)
+ * @param uri URL of source (リモートインスタンスのURLからアップロードされた場合の元URL)
+ * @param sensitive Mark file as sensitive
+ * @return Created drive file
+ */
+export default async function(
+	user: { id: User['id']; host: User['host'] } | null,
+	path: string,
+	name: string | null = null,
+	comment: string | null = null,
+	folderId: any = null,
+	force: boolean = false,
+	isLink: boolean = false,
+	url: string | null = null,
+	uri: string | null = null,
+	sensitive: boolean | null = null
+): Promise<DriveFile> {
+	const info = await getFileInfo(path);
+	logger.info(`${JSON.stringify(info)}`);
+
+	// detect name
+	const detectedName = name || (info.type.ext ? `untitled.${info.type.ext}` : 'untitled');
+
+	if (user && !force) {
+		// Check if there is a file with the same hash
+		const much = await DriveFiles.findOne({
+			md5: info.md5,
+			userId: user.id,
+		});
+
+		if (much) {
+			logger.info(`file with same hash is found: ${much.id}`);
+			return much;
+		}
+	}
+
+	//#region Check drive usage
+	if (user && !isLink) {
+		const usage = await DriveFiles.calcDriveUsageOf(user);
+
+		const instance = await fetchMeta();
+		const driveCapacity = 1024 * 1024 * (Users.isLocalUser(user) ? instance.localDriveCapacityMb : instance.remoteDriveCapacityMb);
+
+		logger.debug(`drive usage is ${usage} (max: ${driveCapacity})`);
+
+		// If usage limit exceeded
+		if (usage + info.size > driveCapacity) {
+			if (Users.isLocalUser(user)) {
+				throw new Error('no-free-space');
+			} else {
+				// (アバターまたはバナーを含まず)最も古いファイルを削除する
+				deleteOldFile(await Users.findOneOrFail(user.id) as IRemoteUser);
+			}
+		}
+	}
+	//#endregion
+
+	const fetchFolder = async () => {
+		if (!folderId) {
+			return null;
+		}
+
+		const driveFolder = await DriveFolders.findOne({
+			id: folderId,
+			userId: user ? user.id : null
+		});
+
+		if (driveFolder == null) throw new Error('folder-not-found');
+
+		return driveFolder;
+	};
+
+	const properties: {
+		width?: number;
+		height?: number;
+	} = {};
+
+	if (info.width) {
+		properties['width'] = info.width;
+		properties['height'] = info.height;
+	}
+
+	const profile = user ? await UserProfiles.findOne(user.id) : null;
+
+	const folder = await fetchFolder();
+
+	let file = new DriveFile();
+	file.id = genId();
+	file.createdAt = new Date();
+	file.userId = user ? user.id : null;
+	file.userHost = user ? user.host : null;
+	file.folderId = folder !== null ? folder.id : null;
+	file.comment = comment;
+	file.properties = properties;
+	file.blurhash = info.blurhash || null;
+	file.isLink = isLink;
+	file.isSensitive = user
+		? Users.isLocalUser(user) && profile!.alwaysMarkNsfw ? true :
+			(sensitive !== null && sensitive !== undefined)
+				? sensitive
+				: false
+		: false;
+
+	if (url !== null) {
+		file.src = url;
+
+		if (isLink) {
+			file.url = url;
+			// ローカルプロキシ用
+			file.accessKey = uuid();
+			file.thumbnailAccessKey = 'thumbnail-' + uuid();
+			file.webpublicAccessKey = 'webpublic-' + uuid();
+		}
+	}
+
+	if (uri !== null) {
+		file.uri = uri;
+	}
+
+	if (isLink) {
+		try {
+			file.size = 0;
+			file.md5 = info.md5;
+			file.name = detectedName;
+			file.type = info.type.mime;
+			file.storedInternal = false;
+
+			file = await DriveFiles.save(file);
+		} catch (e) {
+			// duplicate key error (when already registered)
+			if (isDuplicateKeyValueError(e)) {
+				logger.info(`already registered ${file.uri}`);
+
+				file = await DriveFiles.findOne({
+					uri: file.uri,
+					userId: user ? user.id : null
+				}) as DriveFile;
+			} else {
+				logger.error(e);
+				throw e;
+			}
+		}
+	} else {
+		file = await (save(file, path, detectedName, info.type.mime, info.md5, info.size));
+	}
+
+	logger.succ(`drive file has been created ${file.id}`);
+
+	if (user) {
+		DriveFiles.pack(file, { self: true }).then(packedFile => {
+			// Publish driveFileCreated event
+			publishMainStream(user.id, 'driveFileCreated', packedFile);
+			publishDriveStream(user.id, 'fileCreated', packedFile);
+		});
+	}
+
+	// 統計を更新
+	driveChart.update(file, true);
+	perUserDriveChart.update(file, true);
+	if (file.userHost !== null) {
+		instanceChart.updateDrive(file, true);
+		Instances.increment({ host: file.userHost }, 'driveUsage', file.size);
+		Instances.increment({ host: file.userHost }, 'driveFiles', 1);
+	}
+
+	return file;
+}

packages/backend/src/services/drive/delete-file.ts

@@ -0,0 +1,103 @@
+import { DriveFile } from '@/models/entities/drive-file';
+import { InternalStorage } from './internal-storage';
+import { DriveFiles, Instances } from '@/models/index';
+import { driveChart, perUserDriveChart, instanceChart } from '@/services/chart/index';
+import { createDeleteObjectStorageFileJob } from '@/queue/index';
+import { fetchMeta } from '@/misc/fetch-meta';
+import { getS3 } from './s3';
+import { v4 as uuid } from 'uuid';
+
+export async function deleteFile(file: DriveFile, isExpired = false) {
+	if (file.storedInternal) {
+		InternalStorage.del(file.accessKey!);
+
+		if (file.thumbnailUrl) {
+			InternalStorage.del(file.thumbnailAccessKey!);
+		}
+
+		if (file.webpublicUrl) {
+			InternalStorage.del(file.webpublicAccessKey!);
+		}
+	} else if (!file.isLink) {
+		createDeleteObjectStorageFileJob(file.accessKey!);
+
+		if (file.thumbnailUrl) {
+			createDeleteObjectStorageFileJob(file.thumbnailAccessKey!);
+		}
+
+		if (file.webpublicUrl) {
+			createDeleteObjectStorageFileJob(file.webpublicAccessKey!);
+		}
+	}
+
+	postProcess(file, isExpired);
+}
+
+export async function deleteFileSync(file: DriveFile, isExpired = false) {
+	if (file.storedInternal) {
+		InternalStorage.del(file.accessKey!);
+
+		if (file.thumbnailUrl) {
+			InternalStorage.del(file.thumbnailAccessKey!);
+		}
+
+		if (file.webpublicUrl) {
+			InternalStorage.del(file.webpublicAccessKey!);
+		}
+	} else if (!file.isLink) {
+		const promises = [];
+
+		promises.push(deleteObjectStorageFile(file.accessKey!));
+
+		if (file.thumbnailUrl) {
+			promises.push(deleteObjectStorageFile(file.thumbnailAccessKey!));
+		}
+
+		if (file.webpublicUrl) {
+			promises.push(deleteObjectStorageFile(file.webpublicAccessKey!));
+		}
+
+		await Promise.all(promises);
+	}
+
+	postProcess(file, isExpired);
+}
+
+async function postProcess(file: DriveFile, isExpired = false) {
+	// リモートファイル期限切れ削除後は直リンクにする
+	if (isExpired && file.userHost !== null && file.uri != null) {
+		DriveFiles.update(file.id, {
+			isLink: true,
+			url: file.uri,
+			thumbnailUrl: null,
+			webpublicUrl: null,
+			storedInternal: false,
+			// ローカルプロキシ用
+			accessKey: uuid(),
+			thumbnailAccessKey: 'thumbnail-' + uuid(),
+			webpublicAccessKey: 'webpublic-' + uuid(),
+		});
+	} else {
+		DriveFiles.delete(file.id);
+	}
+
+	// 統計を更新
+	driveChart.update(file, false);
+	perUserDriveChart.update(file, false);
+	if (file.userHost !== null) {
+		instanceChart.updateDrive(file, false);
+		Instances.decrement({ host: file.userHost }, 'driveUsage', file.size);
+		Instances.decrement({ host: file.userHost }, 'driveFiles', 1);
+	}
+}
+
+export async function deleteObjectStorageFile(key: string) {
+	const meta = await fetchMeta();
+
+	const s3 = getS3(meta);
+
+	await s3.deleteObject({
+		Bucket: meta.objectStorageBucket!,
+		Key: key
+	}).promise();
+}

packages/backend/src/services/drive/generate-video-thumbnail.ts

@@ -0,0 +1,37 @@
+import * as fs from 'fs';
+import * as tmp from 'tmp';
+import { IImage, convertToJpeg } from './image-processor';
+import * as FFmpeg from 'fluent-ffmpeg';
+
+export async function GenerateVideoThumbnail(path: string): Promise<IImage> {
+	const [outDir, cleanup] = await new Promise<[string, any]>((res, rej) => {
+		tmp.dir((e, path, cleanup) => {
+			if (e) return rej(e);
+			res([path, cleanup]);
+		});
+	});
+
+	await new Promise((res, rej) => {
+		FFmpeg({
+			source: path
+		})
+		.on('end', res)
+		.on('error', rej)
+		.screenshot({
+			folder: outDir,
+			filename: 'output.png',
+			count: 1,
+			timestamps: ['5%']
+		});
+	});
+
+	const outPath = `${outDir}/output.png`;
+
+	const thumbnail = await convertToJpeg(outPath, 498, 280);
+
+	// cleanup
+	await fs.promises.unlink(outPath);
+	cleanup();
+
+	return thumbnail;
+}

packages/backend/src/services/drive/image-processor.ts

@@ -0,0 +1,107 @@
+import * as sharp from 'sharp';
+
+export type IImage = {
+	data: Buffer;
+	ext: string | null;
+	type: string;
+};
+
+/**
+ * Convert to JPEG
+ *   with resize, remove metadata, resolve orientation, stop animation
+ */
+export async function convertToJpeg(path: string, width: number, height: number): Promise<IImage> {
+	return convertSharpToJpeg(await sharp(path), width, height);
+}
+
+export async function convertSharpToJpeg(sharp: sharp.Sharp, width: number, height: number): Promise<IImage> {
+	const data = await sharp
+		.resize(width, height, {
+			fit: 'inside',
+			withoutEnlargement: true
+		})
+		.rotate()
+		.jpeg({
+			quality: 85,
+			progressive: true
+		})
+		.toBuffer();
+
+	return {
+		data,
+		ext: 'jpg',
+		type: 'image/jpeg'
+	};
+}
+
+/**
+ * Convert to WebP
+ *   with resize, remove metadata, resolve orientation, stop animation
+ */
+export async function convertToWebp(path: string, width: number, height: number): Promise<IImage> {
+	return convertSharpToWebp(await sharp(path), width, height);
+}
+
+export async function convertSharpToWebp(sharp: sharp.Sharp, width: number, height: number): Promise<IImage> {
+	const data = await sharp
+		.resize(width, height, {
+			fit: 'inside',
+			withoutEnlargement: true
+		})
+		.rotate()
+		.webp({
+			quality: 85
+		})
+		.toBuffer();
+
+	return {
+		data,
+		ext: 'webp',
+		type: 'image/webp'
+	};
+}
+
+/**
+ * Convert to PNG
+ *   with resize, remove metadata, resolve orientation, stop animation
+ */
+export async function convertToPng(path: string, width: number, height: number): Promise<IImage> {
+	return convertSharpToPng(await sharp(path), width, height);
+}
+
+export async function convertSharpToPng(sharp: sharp.Sharp, width: number, height: number): Promise<IImage> {
+	const data = await sharp
+		.resize(width, height, {
+			fit: 'inside',
+			withoutEnlargement: true
+		})
+		.rotate()
+		.png()
+		.toBuffer();
+
+	return {
+		data,
+		ext: 'png',
+		type: 'image/png'
+	};
+}
+
+/**
+ * Convert to PNG or JPEG
+ *   with resize, remove metadata, resolve orientation, stop animation
+ */
+export async function convertToPngOrJpeg(path: string, width: number, height: number): Promise<IImage> {
+	return convertSharpToPngOrJpeg(await sharp(path), width, height);
+}
+
+export async function convertSharpToPngOrJpeg(sharp: sharp.Sharp, width: number, height: number): Promise<IImage> {
+	const stats = await sharp.stats();
+	const metadata = await sharp.metadata();
+
+	// 不透明で300x300pxの範囲を超えていればJPEG
+	if (stats.isOpaque && ((metadata.width && metadata.width >= 300) || (metadata.height && metadata!.height >= 300))) {
+		return await convertSharpToJpeg(sharp, width, height);
+	} else {
+		return await convertSharpToPng(sharp, width, height);
+	}
+}

packages/backend/src/services/drive/internal-storage.ts

@@ -0,0 +1,35 @@
+import * as fs from 'fs';
+import * as Path from 'path';
+import { fileURLToPath } from 'url';
+import { dirname } from 'path';
+import config from '@/config/index';
+
+//const _filename = fileURLToPath(import.meta.url);
+const _filename = __filename;
+const _dirname = dirname(_filename);
+
+export class InternalStorage {
+	private static readonly path = Path.resolve(_dirname, '../../../../../files');
+
+	public static resolvePath = (key: string) => Path.resolve(InternalStorage.path, key);
+
+	public static read(key: string) {
+		return fs.createReadStream(InternalStorage.resolvePath(key));
+	}
+
+	public static saveFromPath(key: string, srcPath: string) {
+		fs.mkdirSync(InternalStorage.path, { recursive: true });
+		fs.copyFileSync(srcPath, InternalStorage.resolvePath(key));
+		return `${config.url}/files/${key}`;
+	}
+
+	public static saveFromBuffer(key: string, data: Buffer) {
+		fs.mkdirSync(InternalStorage.path, { recursive: true });
+		fs.writeFileSync(InternalStorage.resolvePath(key), data);
+		return `${config.url}/files/${key}`;
+	}
+
+	public static del(key: string) {
+		fs.unlink(InternalStorage.resolvePath(key), () => {});
+	}
+}

packages/backend/src/services/drive/logger.ts

@@ -0,0 +1,3 @@
+import Logger from '../logger';
+
+export const driveLogger = new Logger('drive', 'blue');

packages/backend/src/services/drive/s3.ts

@@ -0,0 +1,24 @@
+import { URL } from 'url';
+import * as S3 from 'aws-sdk/clients/s3';
+import { Meta } from '@/models/entities/meta';
+import { getAgentByUrl } from '@/misc/fetch';
+
+export function getS3(meta: Meta) {
+	const u = meta.objectStorageEndpoint != null
+		? `${meta.objectStorageUseSSL ? 'https://' : 'http://'}${meta.objectStorageEndpoint}`
+		: `${meta.objectStorageUseSSL ? 'https://' : 'http://'}example.net`;
+
+	return new S3({
+		endpoint: meta.objectStorageEndpoint || undefined,
+		accessKeyId: meta.objectStorageAccessKey!,
+		secretAccessKey: meta.objectStorageSecretKey!,
+		region: meta.objectStorageRegion || undefined,
+		sslEnabled: meta.objectStorageUseSSL,
+		s3ForcePathStyle: !meta.objectStorageEndpoint	// AWS with endPoint omitted
+			? false
+			: meta.objectStorageS3ForcePathStyle,
+		httpOptions: {
+			agent: getAgentByUrl(new URL(u), !meta.objectStorageUseProxy)
+		}
+	});
+}

packages/backend/src/services/drive/upload-from-url.ts

@@ -0,0 +1,62 @@
+import { URL } from 'url';
+import create from './add-file';
+import { User } from '@/models/entities/user';
+import { driveLogger } from './logger';
+import { createTemp } from '@/misc/create-temp';
+import { downloadUrl } from '@/misc/download-url';
+import { DriveFolder } from '@/models/entities/drive-folder';
+import { DriveFile } from '@/models/entities/drive-file';
+import { DriveFiles } from '@/models/index';
+
+const logger = driveLogger.createSubLogger('downloader');
+
+export default async (
+	url: string,
+	user: { id: User['id']; host: User['host'] } | null,
+	folderId: DriveFolder['id'] | null = null,
+	uri: string | null = null,
+	sensitive = false,
+	force = false,
+	link = false,
+	comment = null
+): Promise<DriveFile> => {
+	let name = new URL(url).pathname.split('/').pop() || null;
+	if (name == null || !DriveFiles.validateFileName(name)) {
+		name = null;
+	}
+
+	// If the comment is same as the name, skip comment
+	// (image.name is passed in when receiving attachment)
+	if (comment !== null && name == comment) {
+		comment = null;
+	}
+
+	// Create temp file
+	const [path, cleanup] = await createTemp();
+
+	// write content at URL to temp file
+	await downloadUrl(url, path);
+
+	let driveFile: DriveFile;
+	let error;
+
+	try {
+		driveFile = await create(user, path, name, comment, folderId, force, link, url, uri, sensitive);
+		logger.succ(`Got: ${driveFile.id}`);
+	} catch (e) {
+		error = e;
+		logger.error(`Failed to create drive file: ${e}`, {
+			url: url,
+			e: e
+		});
+	}
+
+	// clean-up
+	cleanup();
+
+	if (error) {
+		throw error;
+	} else {
+		return driveFile!;
+	}
+};