enhance: Proxy custom emojis to reduce image size and accelerate the frontend (#9431)

* fix(server): /emoji to accept `@.` host expression

* fix(client): use MkEmoji for custom emoji in MkEmojiPicker

* change convertToWebp

* nanka iroiro

* remove

* fix

* nearLosslessは労多くして益少なしなのでやめる

* do not cleanup tmp for development

* update sharp.js to 0.31.3

* mixed: true

* fix MkAutocomplete of 912791b3ab

* clean up

* https://github.com/misskey-dev/misskey/pull/9431#discussion_r1059215943

packages/backend/src/core/CustomEmojiService.ts

@@ -2,12 +2,10 @@ import { Inject, Injectable } from '@nestjs/common';
 import { DataSource, In, IsNull } from 'typeorm';
 import { GlobalEventService } from '@/core/GlobalEventService.js';
 import { DI } from '@/di-symbols.js';
-import type { Config } from '@/config.js';
 import { IdService } from '@/core/IdService.js';
 import type { DriveFile } from '@/models/entities/DriveFile.js';
 import type { Emoji } from '@/models/entities/Emoji.js';
 import { Cache } from '@/misc/cache.js';
-import { query } from '@/misc/prelude/url.js';
 import type { Note } from '@/models/entities/Note.js';
 import type { EmojisRepository } from '@/models/index.js';
 import { UtilityService } from '@/core/UtilityService.js';
@@ -27,9 +25,6 @@ export class CustomEmojiService {
 	private cache: Cache<Emoji | null>;
 
 	constructor(
-		@Inject(DI.config)
-		private config: Config,
-
 		@Inject(DI.db)
 		private db: DataSource,
 
@@ -117,7 +112,7 @@ export class CustomEmojiService {
 
 		const isLocal = emoji.host == null;
 		const emojiUrl = emoji.publicUrl || emoji.originalUrl; // || emoji.originalUrl してるのは後方互換性のため
-		const url = isLocal ? emojiUrl : `${this.config.url}/proxy/${encodeURIComponent((new URL(emojiUrl)).pathname)}?${query({ url: emojiUrl })}`;
+		const url = emojiUrl;
 
 		return {
 			name: emojiName,

packages/backend/src/core/DownloadService.ts

@@ -33,7 +33,7 @@ export class DownloadService {
 
 	@bindThis
 	public async downloadUrl(url: string, path: string): Promise<void> {
-		this.logger.info(`Downloading ${chalk.cyan(url)} ...`);
+		this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`);
 	
 		const timeout = 30 * 1000;
 		const operationTimeout = 60 * 1000;

packages/backend/src/core/ImageProcessingService.ts

@@ -8,6 +8,16 @@ export type IImage = {
 	ext: string | null;
 	type: string;
 };
+
+export const webpDefault: sharp.WebpOptions = {
+	quality: 85,
+	alphaQuality: 95,
+	lossless: false,
+	nearLossless: false,
+	smartSubsample: true,
+	mixed: true,
+};
+
 import { bindThis } from '@/decorators.js';
 
 @Injectable()
@@ -53,21 +63,19 @@ export class ImageProcessingService {
 	 *   with resize, remove metadata, resolve orientation, stop animation
 	 */
 	@bindThis
-	public async convertToWebp(path: string, width: number, height: number, quality = 85): Promise<IImage> {
-		return this.convertSharpToWebp(await sharp(path), width, height, quality);
+	public async convertToWebp(path: string, width: number, height: number, options: sharp.WebpOptions = webpDefault): Promise<IImage> {
+		return this.convertSharpToWebp(await sharp(path), width, height, options);
 	}
 
 	@bindThis
-	public async convertSharpToWebp(sharp: sharp.Sharp, width: number, height: number, quality = 85): Promise<IImage> {
+	public async convertSharpToWebp(sharp: sharp.Sharp, width: number, height: number, options: sharp.WebpOptions = webpDefault): Promise<IImage> {
 		const data = await sharp
 			.resize(width, height, {
 				fit: 'inside',
 				withoutEnlargement: true,
 			})
 			.rotate()
-			.webp({
-				quality,
-			})
+			.webp(options)
 			.toBuffer();
 
 		return {

packages/backend/src/misc/create-temp.ts

@@ -4,7 +4,7 @@ export function createTemp(): Promise<[string, () => void]> {
 	return new Promise<[string, () => void]>((res, rej) => {
 		tmp.file((e, path, fd, cleanup) => {
 			if (e) return rej(e);
-			res([path, cleanup]);
+			res([path, process.env.NODE_ENV === 'production' ? cleanup : () => {}]);
 		});
 	});
 }
@@ -17,7 +17,7 @@ export function createTempDir(): Promise<[string, () => void]> {
 			},
 			(e, path, cleanup) => {
 				if (e) return rej(e);
-				res([path, cleanup]);
+				res([path, process.env.NODE_ENV === 'production' ? cleanup : () => {}]);
 			},
 		);
 	});

packages/backend/src/misc/prelude/url.ts

@@ -1,3 +1,8 @@
+/* objを検査して
+ * 1. 配列に何も入っていない時はクエリを付けない
+ * 2. プロパティがundefinedの時はクエリを付けない
+ * （new URLSearchParams(obj)ではそこまで丁寧なことをしてくれない）
+ */ 
 export function query(obj: Record<string, unknown>): string {
 	const params = Object.entries(obj)
 		.filter(([, v]) => Array.isArray(v) ? v.length : v !== undefined)

packages/backend/src/server/MediaProxyServerService.ts

@@ -9,7 +9,7 @@ import type { Config } from '@/config.js';
 import { isMimeImage } from '@/misc/is-mime-image.js';
 import { createTemp } from '@/misc/create-temp.js';
 import { DownloadService } from '@/core/DownloadService.js';
-import { ImageProcessingService } from '@/core/ImageProcessingService.js';
+import { ImageProcessingService, webpDefault } from '@/core/ImageProcessingService.js';
 import type { IImage } from '@/core/ImageProcessingService.js';
 import { FILE_TYPE_BROWSERSAFE } from '@/const.js';
 import { StatusError } from '@/misc/status-error.js';
@@ -81,8 +81,21 @@ export class MediaProxyServerService {
 			const isConvertibleImage = isMimeImage(mime, 'sharp-convertible-image');
 	
 			let image: IImage;
-	
-			if ('static' in request.query && isConvertibleImage) {
+			if ('emoji' in request.query && isConvertibleImage) {
+				const data = await sharp(path, { animated: !('static' in request.query) })
+					.resize({
+						height: 128,
+						withoutEnlargement: true,
+					})
+					.webp(webpDefault)
+					.toBuffer();
+
+				image = {
+					data,
+					ext: 'webp',
+					type: 'image/webp',
+				};
+			} else if ('static' in request.query && isConvertibleImage) {
 				image = await this.imageProcessingService.convertToWebp(path, 498, 280);
 			} else if ('preview' in request.query && isConvertibleImage) {
 				image = await this.imageProcessingService.convertToWebp(path, 200, 200);
@@ -91,7 +104,7 @@ export class MediaProxyServerService {
 					// 画像でないなら404でお茶を濁す
 					throw new StatusError('Unexpected mime', 404);
 				}
-	
+
 				const mask = sharp(path)
 					.resize(96, 96, {
 						fit: 'inside',
@@ -121,8 +134,8 @@ export class MediaProxyServerService {
 					ext: 'png',
 					type: 'image/png',
 				};
-			}	else if (mime === 'image/svg+xml') {
-				image = await this.imageProcessingService.convertToWebp(path, 2048, 2048, 1);
+			} else if (mime === 'image/svg+xml') {
+				image = await this.imageProcessingService.convertToWebp(path, 2048, 2048, webpDefault);
 			} else if (!mime.startsWith('image/') || !FILE_TYPE_BROWSERSAFE.includes(mime)) {
 				throw new StatusError('Rejected type', 403, 'Rejected type');
 			} else {

packages/backend/src/server/web/ClientServerService.ts

@@ -220,7 +220,7 @@ export class ClientServerService {
 			return reply.sendFile('/apple-touch-icon.png', staticAssets);
 		});
 
-		fastify.get<{ Params: { path: string } }>('/emoji/:path(.*)', async (request, reply) => {
+		fastify.get<{ Params: { path: string }; Querystring: { static?: any; }; }>('/emoji/:path(.*)', async (request, reply) => {
 			const path = request.params.path;
 
 			if (!path.match(/^[a-zA-Z0-9\-_@\.]+?\.webp$/)) {
@@ -244,8 +244,15 @@ export class ClientServerService {
 
 			reply.header('Content-Security-Policy', 'default-src \'none\'; style-src \'unsafe-inline\'');
 
-			// ?? emoji.originalUrl してるのは後方互換性のため
-			return await reply.redirect(301, emoji.publicUrl ?? emoji.originalUrl);
+			const url = new URL("/proxy/emoji.webp", this.config.url);
+			url.searchParams.set('url', emoji.publicUrl ?? emoji.originalUrl); // ?? emoji.originalUrl してるのは後方互換性のため
+			url.searchParams.set('emoji', '1');
+			if ('static' in request.query) url.searchParams.set('static', '1');
+
+			return await reply.redirect(
+				301,
+				url.toString(),
+			);
 		});
 
 		fastify.get<{ Params: { path: string } }>('/fluent-emoji/:path(.*)', async (request, reply) => {