refactoring

Resolve #7779

packages/backend/src/misc/antenna-cache.ts

@@ -0,0 +1,36 @@
+import { Antennas } from '@/models/index';
+import { Antenna } from '@/models/entities/antenna';
+import { subsdcriber } from '../db/redis';
+
+let antennasFetched = false;
+let antennas: Antenna[] = [];
+
+export async function getAntennas() {
+	if (!antennasFetched) {
+		antennas = await Antennas.find();
+		antennasFetched = true;
+	}
+
+	return antennas;
+}
+
+subsdcriber.on('message', async (_, data) => {
+	const obj = JSON.parse(data);
+
+	if (obj.channel === 'internal') {
+		const { type, body } = obj.message;
+		switch (type) {
+			case 'antennaCreated':
+				antennas.push(body);
+				break;
+			case 'antennaUpdated':
+				antennas[antennas.findIndex(a => a.id === body.id)] = body;
+				break;
+			case 'antennaDeleted':
+				antennas = antennas.filter(a => a.id !== body.id);
+				break;
+			default:
+				break;
+		}
+	}
+});

packages/backend/src/misc/api-permissions.ts

@@ -0,0 +1,35 @@
+export const kinds = [
+	'read:account',
+	'write:account',
+	'read:blocks',
+	'write:blocks',
+	'read:drive',
+	'write:drive',
+	'read:favorites',
+	'write:favorites',
+	'read:following',
+	'write:following',
+	'read:messaging',
+	'write:messaging',
+	'read:mutes',
+	'write:mutes',
+	'write:notes',
+	'read:notifications',
+	'write:notifications',
+	'read:reactions',
+	'write:reactions',
+	'write:votes',
+	'read:pages',
+	'write:pages',
+	'write:page-likes',
+	'read:page-likes',
+	'read:user-groups',
+	'write:user-groups',
+	'read:channels',
+	'write:channels',
+	'read:gallery',
+	'write:gallery',
+	'read:gallery-likes',
+	'write:gallery-likes',
+];
+// IF YOU ADD KINDS(PERMISSIONS), YOU MUST ADD TRANSLATIONS (under _permissions).

packages/backend/src/misc/app-lock.ts

@@ -0,0 +1,31 @@
+import { redisClient } from '../db/redis';
+import { promisify } from 'util';
+import * as redisLock from 'redis-lock';
+
+/**
+ * Retry delay (ms) for lock acquisition
+ */
+const retryDelay = 100;
+
+const lock: (key: string, timeout?: number) => Promise<() => void>
+	= redisClient
+	? promisify(redisLock(redisClient, retryDelay))
+	: async () => () => { };
+
+/**
+ * Get AP Object lock
+ * @param uri AP object ID
+ * @param timeout Lock timeout (ms), The timeout releases previous lock.
+ * @returns Unlock function
+ */
+export function getApLock(uri: string, timeout = 30 * 1000) {
+	return lock(`ap-object:${uri}`, timeout);
+}
+
+export function getFetchInstanceMetadataLock(host: string, timeout = 30 * 1000) {
+	return lock(`instance:${host}`, timeout);
+}
+
+export function getChartInsertLock(lockKey: string, timeout = 30 * 1000) {
+	return lock(`chart-insert:${lockKey}`, timeout);
+}

packages/backend/src/misc/before-shutdown.ts

@@ -0,0 +1,92 @@
+// https://gist.github.com/nfantone/1eaa803772025df69d07f4dbf5df7e58
+
+'use strict';
+
+/**
+ * @callback BeforeShutdownListener
+ * @param {string} [signalOrEvent] The exit signal or event name received on the process.
+ */
+
+/**
+ * System signals the app will listen to initiate shutdown.
+ * @const {string[]}
+ */
+const SHUTDOWN_SIGNALS = ['SIGINT', 'SIGTERM'];
+
+/**
+ * Time in milliseconds to wait before forcing shutdown.
+ * @const {number}
+ */
+const SHUTDOWN_TIMEOUT = 15000;
+
+/**
+ * A queue of listener callbacks to execute before shutting
+ * down the process.
+ * @type {BeforeShutdownListener[]}
+ */
+const shutdownListeners = [];
+
+/**
+ * Listen for signals and execute given `fn` function once.
+ * @param  {string[]} signals System signals to listen to.
+ * @param  {function(string)} fn Function to execute on shutdown.
+ */
+const processOnce = (signals, fn) => {
+	for (const sig of signals) {
+		process.once(sig, fn);
+	}
+};
+
+/**
+ * Sets a forced shutdown mechanism that will exit the process after `timeout` milliseconds.
+ * @param {number} timeout Time to wait before forcing shutdown (milliseconds)
+ */
+const forceExitAfter = timeout => () => {
+	setTimeout(() => {
+		// Force shutdown after timeout
+		console.warn(`Could not close resources gracefully after ${timeout}ms: forcing shutdown`);
+		return process.exit(1);
+	}, timeout).unref();
+};
+
+/**
+ * Main process shutdown handler. Will invoke every previously registered async shutdown listener
+ * in the queue and exit with a code of `0`. Any `Promise` rejections from any listener will
+ * be logged out as a warning, but won't prevent other callbacks from executing.
+ * @param {string} signalOrEvent The exit signal or event name received on the process.
+ */
+async function shutdownHandler(signalOrEvent) {
+	if (process.env.NODE_ENV === 'test') return process.exit(0);
+
+	console.warn(`Shutting down: received [${signalOrEvent}] signal`);
+
+	for (const listener of shutdownListeners) {
+		try {
+			await listener(signalOrEvent);
+		} catch (err) {
+			console.warn(`A shutdown handler failed before completing with: ${err.message || err}`);
+		}
+	}
+
+	return process.exit(0);
+}
+
+/**
+ * Registers a new shutdown listener to be invoked before exiting
+ * the main process. Listener handlers are guaranteed to be called in the order
+ * they were registered.
+ * @param {BeforeShutdownListener} listener The shutdown listener to register.
+ * @returns {BeforeShutdownListener} Echoes back the supplied `listener`.
+ */
+export function beforeShutdown(listener) {
+	shutdownListeners.push(listener);
+	return listener;
+}
+
+// Register shutdown callback that kills the process after `SHUTDOWN_TIMEOUT` milliseconds
+// This prevents custom shutdown handlers from hanging the process indefinitely
+processOnce(SHUTDOWN_SIGNALS, forceExitAfter(SHUTDOWN_TIMEOUT));
+
+// Register process shutdown callback
+// Will listen to incoming signal events and execute all registered handlers in the stack
+processOnce(SHUTDOWN_SIGNALS, shutdownHandler);

packages/backend/src/misc/cache.ts

@@ -0,0 +1,43 @@
+export class Cache<T> {
+	private cache: Map<string | null, { date: number; value: T; }>;
+	private lifetime: number;
+
+	constructor(lifetime: Cache<never>['lifetime']) {
+		this.cache = new Map();
+		this.lifetime = lifetime;
+	}
+
+	public set(key: string | null, value: T): void {
+		this.cache.set(key, {
+			date: Date.now(),
+			value
+		});
+	}
+
+	public get(key: string | null): T | undefined {
+		const cached = this.cache.get(key);
+		if (cached == null) return undefined;
+		if ((Date.now() - cached.date) > this.lifetime) {
+			this.cache.delete(key);
+			return undefined;
+		}
+		return cached.value;
+	}
+
+	public delete(key: string | null) {
+		this.cache.delete(key);
+	}
+
+	public async fetch(key: string | null, fetcher: () => Promise<T>): Promise<T> {
+		const cachedValue = this.get(key);
+		if (cachedValue !== undefined) {
+			// Cache HIT
+			return cachedValue;
+		}
+
+		// Cache MISS
+		const value = await fetcher();
+		this.set(key, value);
+		return value;
+	}
+}

packages/backend/src/misc/cafy-id.ts

@@ -0,0 +1,32 @@
+import { Context } from 'cafy';
+
+export class ID<Maybe = string> extends Context<string | (Maybe extends {} ? string : Maybe)> {
+	public readonly name = 'ID';
+
+	constructor(optional = false, nullable = false) {
+		super(optional, nullable);
+
+		this.push((v: any) => {
+			if (typeof v !== 'string') {
+				return new Error('must-be-an-id');
+			}
+			return true;
+		});
+	}
+
+	public getType() {
+		return super.getType('String');
+	}
+
+	public makeOptional(): ID<undefined> {
+		return new ID(true, false);
+	}
+
+	public makeNullable(): ID<null> {
+		return new ID(false, true);
+	}
+
+	public makeOptionalNullable(): ID<undefined | null> {
+		return new ID(true, true);
+	}
+}

packages/backend/src/misc/captcha.ts

@@ -0,0 +1,56 @@
+import fetch from 'node-fetch';
+import { URLSearchParams } from 'url';
+import { getAgentByUrl } from './fetch';
+import config from '@/config/index';
+
+export async function verifyRecaptcha(secret: string, response: string) {
+	const result = await getCaptchaResponse('https://www.recaptcha.net/recaptcha/api/siteverify', secret, response).catch(e => {
+		throw `recaptcha-request-failed: ${e}`;
+	});
+
+	if (result.success !== true) {
+		const errorCodes = result['error-codes'] ? result['error-codes']?.join(', ') : '';
+		throw `recaptcha-failed: ${errorCodes}`;
+	}
+}
+
+export async function verifyHcaptcha(secret: string, response: string) {
+	const result = await getCaptchaResponse('https://hcaptcha.com/siteverify', secret, response).catch(e => {
+		throw `hcaptcha-request-failed: ${e}`;
+	});
+
+	if (result.success !== true) {
+		const errorCodes = result['error-codes'] ? result['error-codes']?.join(', ') : '';
+		throw `hcaptcha-failed: ${errorCodes}`;
+	}
+}
+
+type CaptchaResponse = {
+	success: boolean;
+	'error-codes'?: string[];
+};
+
+async function getCaptchaResponse(url: string, secret: string, response: string): Promise<CaptchaResponse> {
+	const params = new URLSearchParams({
+		secret,
+		response
+	});
+
+	const res = await fetch(url, {
+		method: 'POST',
+		body: params,
+		headers: {
+			'User-Agent': config.userAgent
+		},
+		timeout: 10 * 1000,
+		agent: getAgentByUrl
+	}).catch(e => {
+		throw `${e.message || e}`;
+	});
+
+	if (!res.ok) {
+		throw `${res.status}`;
+	}
+
+	return await res.json() as CaptchaResponse;
+}

packages/backend/src/misc/check-hit-antenna.ts

@@ -0,0 +1,90 @@
+import { Antenna } from '@/models/entities/antenna';
+import { Note } from '@/models/entities/note';
+import { User } from '@/models/entities/user';
+import { UserListJoinings, UserGroupJoinings } from '@/models/index';
+import { getFullApAccount } from './convert-host';
+import * as Acct from 'misskey-js/built/acct';
+import { Packed } from './schema';
+
+/**
+ * noteUserFollowers / antennaUserFollowing はどちらか一方が指定されていればよい
+ */
+export async function checkHitAntenna(antenna: Antenna, note: (Note | Packed<'Note'>), noteUser: { username: string; host: string | null; }, noteUserFollowers?: User['id'][], antennaUserFollowing?: User['id'][]): Promise<boolean> {
+	if (note.visibility === 'specified') return false;
+
+	if (note.visibility === 'followers') {
+		if (noteUserFollowers && !noteUserFollowers.includes(antenna.userId)) return false;
+		if (antennaUserFollowing && !antennaUserFollowing.includes(note.userId)) return false;
+	}
+
+	if (!antenna.withReplies && note.replyId != null) return false;
+
+	if (antenna.src === 'home') {
+		if (noteUserFollowers && !noteUserFollowers.includes(antenna.userId)) return false;
+		if (antennaUserFollowing && !antennaUserFollowing.includes(note.userId)) return false;
+	} else if (antenna.src === 'list') {
+		const listUsers = (await UserListJoinings.find({
+			userListId: antenna.userListId!
+		})).map(x => x.userId);
+
+		if (!listUsers.includes(note.userId)) return false;
+	} else if (antenna.src === 'group') {
+		const joining = await UserGroupJoinings.findOneOrFail(antenna.userGroupJoiningId!);
+
+		const groupUsers = (await UserGroupJoinings.find({
+			userGroupId: joining.userGroupId
+		})).map(x => x.userId);
+
+		if (!groupUsers.includes(note.userId)) return false;
+	} else if (antenna.src === 'users') {
+		const accts = antenna.users.map(x => {
+			const { username, host } = Acct.parse(x);
+			return getFullApAccount(username, host).toLowerCase();
+		});
+		if (!accts.includes(getFullApAccount(noteUser.username, noteUser.host).toLowerCase())) return false;
+	}
+
+	const keywords = antenna.keywords
+		// Clean up
+		.map(xs => xs.filter(x => x !== ''))
+		.filter(xs => xs.length > 0);
+
+	if (keywords.length > 0) {
+		if (note.text == null) return false;
+
+		const matched = keywords.some(and =>
+			and.every(keyword =>
+				antenna.caseSensitive
+					? note.text!.includes(keyword)
+					: note.text!.toLowerCase().includes(keyword.toLowerCase())
+			));
+
+		if (!matched) return false;
+	}
+
+	const excludeKeywords = antenna.excludeKeywords
+		// Clean up
+		.map(xs => xs.filter(x => x !== ''))
+		.filter(xs => xs.length > 0);
+
+	if (excludeKeywords.length > 0) {
+		if (note.text == null) return false;
+
+		const matched = excludeKeywords.some(and =>
+			and.every(keyword =>
+				antenna.caseSensitive
+					? note.text!.includes(keyword)
+					: note.text!.toLowerCase().includes(keyword.toLowerCase())
+			));
+
+		if (matched) return false;
+	}
+
+	if (antenna.withFile) {
+		if (note.fileIds && note.fileIds.length === 0) return false;
+	}
+
+	// TODO: eval expression
+
+	return true;
+}

packages/backend/src/misc/check-word-mute.ts

@@ -0,0 +1,39 @@
+const RE2 = require('re2');
+import { Note } from '@/models/entities/note';
+import { User } from '@/models/entities/user';
+
+type NoteLike = {
+	userId: Note['userId'];
+	text: Note['text'];
+};
+
+type UserLike = {
+	id: User['id'];
+};
+
+export async function checkWordMute(note: NoteLike, me: UserLike | null | undefined, mutedWords: string[][]): Promise<boolean> {
+	// 自分自身
+	if (me && (note.userId === me.id)) return false;
+
+	const words = mutedWords
+		// Clean up
+		.map(xs => xs.filter(x => x !== ''))
+		.filter(xs => xs.length > 0);
+
+	if (words.length > 0) {
+		if (note.text == null) return false;
+
+		const matched = words.some(and =>
+			and.every(keyword => {
+				const regexp = keyword.match(/^\/(.+)\/(.*)$/);
+				if (regexp) {
+					return new RE2(regexp[1], regexp[2]).test(note.text!);
+				}
+				return note.text!.includes(keyword);
+			}));
+
+		if (matched) return true;
+	}
+
+	return false;
+}

packages/backend/src/misc/content-disposition.ts

@@ -0,0 +1,6 @@
+const cd = require('content-disposition');
+
+export function contentDisposition(type: 'inline' | 'attachment', filename: string): string {
+	const fallback = filename.replace(/[^\w.-]/g, '_');
+	return cd(filename, { type, fallback });
+}

packages/backend/src/misc/convert-host.ts

@@ -0,0 +1,26 @@
+import { URL } from 'url';
+import config from '@/config/index';
+import { toASCII } from 'punycode/';
+
+export function getFullApAccount(username: string, host: string | null) {
+	return host ? `${username}@${toPuny(host)}` : `${username}@${toPuny(config.host)}`;
+}
+
+export function isSelfHost(host: string) {
+	if (host == null) return true;
+	return toPuny(config.host) === toPuny(host);
+}
+
+export function extractDbHost(uri: string) {
+	const url = new URL(uri);
+	return toPuny(url.hostname);
+}
+
+export function toPuny(host: string) {
+	return toASCII(host.toLowerCase());
+}
+
+export function toPunyNullable(host: string | null | undefined): string | null {
+	if (host == null) return null;
+	return toASCII(host.toLowerCase());
+}

packages/backend/src/misc/count-same-renotes.ts

@@ -0,0 +1,15 @@
+import { Notes } from '@/models/index';
+
+export async function countSameRenotes(userId: string, renoteId: string, excludeNoteId: string | undefined): Promise<number> {
+	// 指定したユーザーの指定したノートのリノートがいくつあるか数える
+	const query = Notes.createQueryBuilder('note')
+		.where('note.userId = :userId', { userId })
+		.andWhere('note.renoteId = :renoteId', { renoteId });
+
+	// 指定した投稿を除く
+	if (excludeNoteId) {
+		query.andWhere('note.id != :excludeNoteId', { excludeNoteId });
+	}
+
+	return await query.getCount();
+}

packages/backend/src/misc/create-temp.ts

@@ -0,0 +1,10 @@
+import * as tmp from 'tmp';
+
+export function createTemp(): Promise<[string, any]> {
+	return new Promise<[string, any]>((res, rej) => {
+		tmp.file((e, path, fd, cleanup) => {
+			if (e) return rej(e);
+			res([path, cleanup]);
+		});
+	});
+}

packages/backend/src/misc/detect-url-mime.ts

@@ -0,0 +1,15 @@
+import { createTemp } from './create-temp';
+import { downloadUrl } from './download-url';
+import { detectType } from './get-file-info';
+
+export async function detectUrlMime(url: string) {
+	const [path, cleanup] = await createTemp();
+
+	try {
+		await downloadUrl(url, path);
+		const { mime } = await detectType(path);
+		return mime;
+	} finally {
+		cleanup();
+	}
+}

packages/backend/src/misc/download-text-file.ts

@@ -0,0 +1,25 @@
+import * as fs from 'fs';
+import * as util from 'util';
+import Logger from '@/services/logger';
+import { createTemp } from './create-temp';
+import { downloadUrl } from './download-url';
+
+const logger = new Logger('download-text-file');
+
+export async function downloadTextFile(url: string): Promise<string> {
+	// Create temp file
+	const [path, cleanup] = await createTemp();
+
+	logger.info(`Temp file is ${path}`);
+
+	try {
+		// write content at URL to temp file
+		await downloadUrl(url, path);
+
+		const text = await util.promisify(fs.readFile)(path, 'utf8');
+
+		return text;
+	} finally {
+		cleanup();
+	}
+}

packages/backend/src/misc/download-url.ts

@@ -0,0 +1,87 @@
+import * as fs from 'fs';
+import * as stream from 'stream';
+import * as util from 'util';
+import got, * as Got from 'got';
+import { httpAgent, httpsAgent, StatusError } from './fetch';
+import config from '@/config/index';
+import * as chalk from 'chalk';
+import Logger from '@/services/logger';
+import * as IPCIDR from 'ip-cidr';
+const PrivateIp = require('private-ip');
+
+const pipeline = util.promisify(stream.pipeline);
+
+export async function downloadUrl(url: string, path: string) {
+	const logger = new Logger('download');
+
+	logger.info(`Downloading ${chalk.cyan(url)} ...`);
+
+	const timeout = 30 * 1000;
+	const operationTimeout = 60 * 1000;
+	const maxSize = config.maxFileSize || 262144000;
+
+	const req = got.stream(url, {
+		headers: {
+			'User-Agent': config.userAgent
+		},
+		timeout: {
+			lookup: timeout,
+			connect: timeout,
+			secureConnect: timeout,
+			socket: timeout,	// read timeout
+			response: timeout,
+			send: timeout,
+			request: operationTimeout,	// whole operation timeout
+		},
+		agent: {
+			http: httpAgent,
+			https: httpsAgent,
+		},
+		http2: false,	// default
+		retry: 0,
+	}).on('response', (res: Got.Response) => {
+		if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !config.proxy && res.ip) {
+			if (isPrivateIp(res.ip)) {
+				logger.warn(`Blocked address: ${res.ip}`);
+				req.destroy();
+			}
+		}
+
+		const contentLength = res.headers['content-length'];
+		if (contentLength != null) {
+			const size = Number(contentLength);
+			if (size > maxSize) {
+				logger.warn(`maxSize exceeded (${size} > ${maxSize}) on response`);
+				req.destroy();
+			}
+		}
+	}).on('downloadProgress', (progress: Got.Progress) => {
+		if (progress.transferred > maxSize) {
+			logger.warn(`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`);
+			req.destroy();
+		}
+	});
+
+	try {
+		await pipeline(req, fs.createWriteStream(path));
+	} catch (e) {
+		if (e instanceof Got.HTTPError) {
+			throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage);
+		} else {
+			throw e;
+		}
+	}
+
+	logger.succ(`Download finished: ${chalk.cyan(url)}`);
+}
+
+function isPrivateIp(ip: string) {
+	for (const net of config.allowedPrivateNetworks || []) {
+		const cidr = new IPCIDR(net);
+		if (cidr.contains(ip)) {
+			return false;
+		}
+	}
+
+	return PrivateIp(ip);
+}

packages/backend/src/misc/emoji-regex.ts

@@ -0,0 +1,3 @@
+const twemojiRegex = require('twemoji-parser/dist/lib/regex').default;
+
+export const emojiRegex = new RegExp(`(${twemojiRegex.source})`);

packages/backend/src/misc/extract-custom-emojis-from-mfm.ts

@@ -0,0 +1,10 @@
+import * as mfm from 'mfm-js';
+import { unique } from '@/prelude/array';
+
+export function extractCustomEmojisFromMfm(nodes: mfm.MfmNode[]): string[] {
+	const emojiNodes = mfm.extract(nodes, (node) => {
+		return (node.type === 'emojiCode' && node.props.name.length <= 100);
+	});
+
+	return unique(emojiNodes.map(x => x.props.name));
+}

packages/backend/src/misc/extract-hashtags.ts

@@ -0,0 +1,9 @@
+import * as mfm from 'mfm-js';
+import { unique } from '@/prelude/array';
+
+export function extractHashtags(nodes: mfm.MfmNode[]): string[] {
+	const hashtagNodes = mfm.extract(nodes, (node) => node.type === 'hashtag');
+	const hashtags = unique(hashtagNodes.map(x => x.props.hashtag));
+
+	return hashtags;
+}

packages/backend/src/misc/extract-mentions.ts

@@ -0,0 +1,11 @@
+// test is located in test/extract-mentions
+
+import * as mfm from 'mfm-js';
+
+export function extractMentions(nodes: mfm.MfmNode[]): mfm.MfmMention['props'][] {
+	// TODO: 重複を削除
+	const mentionNodes = mfm.extract(nodes, (node) => node.type === 'mention');
+	const mentions = mentionNodes.map(x => x.props);
+
+	return mentions;
+}

packages/backend/src/misc/fetch-meta.ts

@@ -0,0 +1,35 @@
+import { Meta } from '@/models/entities/meta';
+import { getConnection } from 'typeorm';
+
+let cache: Meta;
+
+export async function fetchMeta(noCache = false): Promise<Meta> {
+	if (!noCache && cache) return cache;
+
+	return await getConnection().transaction(async transactionalEntityManager => {
+		// 過去のバグでレコードが複数出来てしまっている可能性があるので新しいIDを優先する
+		const meta = await transactionalEntityManager.findOne(Meta, {
+			order: {
+				id: 'DESC'
+			}
+		});
+
+		if (meta) {
+			cache = meta;
+			return meta;
+		} else {
+			const saved = await transactionalEntityManager.save(Meta, {
+				id: 'x'
+			}) as Meta;
+
+			cache = saved;
+			return saved;
+		}
+	});
+}
+
+setInterval(() => {
+	fetchMeta(true).then(meta => {
+		cache = meta;
+	});
+}, 1000 * 10);

packages/backend/src/misc/fetch-proxy-account.ts

@@ -0,0 +1,9 @@
+import { fetchMeta } from './fetch-meta';
+import { ILocalUser } from '@/models/entities/user';
+import { Users } from '@/models/index';
+
+export async function fetchProxyAccount(): Promise<ILocalUser | null> {
+	const meta = await fetchMeta();
+	if (meta.proxyAccountId == null) return null;
+	return await Users.findOneOrFail(meta.proxyAccountId) as ILocalUser;
+}

packages/backend/src/misc/fetch.ts

@@ -0,0 +1,141 @@
+import * as http from 'http';
+import * as https from 'https';
+import CacheableLookup from 'cacheable-lookup';
+import fetch from 'node-fetch';
+import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
+import config from '@/config/index';
+import { URL } from 'url';
+
+export async function getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: Record<string, string>) {
+	const res = await getResponse({
+		url,
+		method: 'GET',
+		headers: Object.assign({
+			'User-Agent': config.userAgent,
+			Accept: accept
+		}, headers || {}),
+		timeout
+	});
+
+	return await res.json();
+}
+
+export async function getHtml(url: string, accept = 'text/html, */*', timeout = 10000, headers?: Record<string, string>) {
+	const res = await getResponse({
+		url,
+		method: 'GET',
+		headers: Object.assign({
+			'User-Agent': config.userAgent,
+			Accept: accept
+		}, headers || {}),
+		timeout
+	});
+
+	return await res.text();
+}
+
+export async function getResponse(args: { url: string, method: string, body?: string, headers: Record<string, string>, timeout?: number, size?: number }) {
+	const timeout = args?.timeout || 10 * 1000;
+
+	const controller = new AbortController();
+	setTimeout(() => {
+		controller.abort();
+	}, timeout * 6);
+
+	const res = await fetch(args.url, {
+		method: args.method,
+		headers: args.headers,
+		body: args.body,
+		timeout,
+		size: args?.size || 10 * 1024 * 1024,
+		agent: getAgentByUrl,
+		signal: controller.signal,
+	});
+
+	if (!res.ok) {
+		throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText);
+	}
+
+	return res;
+}
+
+const cache = new CacheableLookup({
+	maxTtl: 3600,	// 1hours
+	errorTtl: 30,	// 30secs
+	lookup: false,	// nativeのdns.lookupにfallbackしない
+});
+
+/**
+ * Get http non-proxy agent
+ */
+const _http = new http.Agent({
+	keepAlive: true,
+	keepAliveMsecs: 30 * 1000,
+	lookup: cache.lookup,
+} as http.AgentOptions);
+
+/**
+ * Get https non-proxy agent
+ */
+const _https = new https.Agent({
+	keepAlive: true,
+	keepAliveMsecs: 30 * 1000,
+	lookup: cache.lookup,
+} as https.AgentOptions);
+
+const maxSockets = Math.max(256, config.deliverJobConcurrency || 128);
+
+/**
+ * Get http proxy or non-proxy agent
+ */
+export const httpAgent = config.proxy
+	? new HttpProxyAgent({
+		keepAlive: true,
+		keepAliveMsecs: 30 * 1000,
+		maxSockets,
+		maxFreeSockets: 256,
+		scheduling: 'lifo',
+		proxy: config.proxy
+	})
+	: _http;
+
+/**
+ * Get https proxy or non-proxy agent
+ */
+export const httpsAgent = config.proxy
+	? new HttpsProxyAgent({
+		keepAlive: true,
+		keepAliveMsecs: 30 * 1000,
+		maxSockets,
+		maxFreeSockets: 256,
+		scheduling: 'lifo',
+		proxy: config.proxy
+	})
+	: _https;
+
+/**
+ * Get agent by URL
+ * @param url URL
+ * @param bypassProxy Allways bypass proxy
+ */
+export function getAgentByUrl(url: URL, bypassProxy = false) {
+	if (bypassProxy || (config.proxyBypassHosts || []).includes(url.hostname)) {
+		return url.protocol == 'http:' ? _http : _https;
+	} else {
+		return url.protocol == 'http:' ? httpAgent : httpsAgent;
+	}
+}
+
+export class StatusError extends Error {
+	public statusCode: number;
+	public statusMessage?: string;
+	public isClientError: boolean;
+
+	constructor(message: string, statusCode: number, statusMessage?: string) {
+		super(message);
+		this.name = 'StatusError';
+		this.statusCode = statusCode;
+		this.statusMessage = statusMessage;
+		this.isClientError = typeof this.statusCode === 'number' && this.statusCode >= 400 && this.statusCode < 500;
+	}
+}

packages/backend/src/misc/gen-avatar.ts

@@ -0,0 +1,90 @@
+/**
+ * Random avatar generator
+ */
+
+import * as p from 'pureimage';
+import * as gen from 'random-seed';
+import { WriteStream } from 'fs';
+
+const size = 256; // px
+const n = 5; // resolution
+const margin = (size / n);
+const colors = [
+	'#e57373',
+	'#F06292',
+	'#BA68C8',
+	'#9575CD',
+	'#7986CB',
+	'#64B5F6',
+	'#4FC3F7',
+	'#4DD0E1',
+	'#4DB6AC',
+	'#81C784',
+	'#8BC34A',
+	'#AFB42B',
+	'#F57F17',
+	'#FF5722',
+	'#795548',
+	'#455A64',
+];
+const bg = '#e9e9e9';
+
+const actualSize = size - (margin * 2);
+const cellSize = actualSize / n;
+const sideN = Math.floor(n / 2);
+
+/**
+ * Generate buffer of random avatar by seed
+ */
+export function genAvatar(seed: string, stream: WriteStream): Promise<void> {
+	const rand = gen.create(seed);
+	const canvas = p.make(size, size);
+	const ctx = canvas.getContext('2d');
+
+	ctx.fillStyle = bg;
+	ctx.beginPath();
+	ctx.fillRect(0, 0, size, size);
+
+	ctx.fillStyle = colors[rand(colors.length)];
+
+	// side bitmap (filled by false)
+	const side: boolean[][] = new Array(sideN);
+	for (let i = 0; i < side.length; i++) {
+		side[i] = new Array(n).fill(false);
+	}
+
+	// 1*n (filled by false)
+	const center: boolean[] = new Array(n).fill(false);
+
+	// tslint:disable-next-line:prefer-for-of
+	for (let x = 0; x < side.length; x++) {
+		for (let y = 0; y < side[x].length; y++) {
+			side[x][y] = rand(3) === 0;
+		}
+	}
+
+	for (let i = 0; i < center.length; i++) {
+		center[i] = rand(3) === 0;
+	}
+
+	// Draw
+	for (let x = 0; x < n; x++) {
+		for (let y = 0; y < n; y++) {
+			const isXCenter = x === ((n - 1) / 2);
+			if (isXCenter && !center[y]) continue;
+
+			const isLeftSide = x < ((n - 1) / 2);
+			if (isLeftSide && !side[x][y]) continue;
+
+			const isRightSide = x > ((n - 1) / 2);
+			if (isRightSide && !side[sideN - (x - sideN)][y]) continue;
+
+			const actualX = margin + (cellSize * x);
+			const actualY = margin + (cellSize * y);
+			ctx.beginPath();
+			ctx.fillRect(actualX, actualY, cellSize, cellSize);
+		}
+	}
+
+	return p.encodePNGToStream(canvas, stream);
+}

packages/backend/src/misc/gen-id.ts

@@ -0,0 +1,21 @@
+import { ulid } from 'ulid';
+import { genAid } from './id/aid';
+import { genMeid } from './id/meid';
+import { genMeidg } from './id/meidg';
+import { genObjectId } from './id/object-id';
+import config from '@/config/index';
+
+const metohd = config.id.toLowerCase();
+
+export function genId(date?: Date): string {
+	if (!date || (date > new Date())) date = new Date();
+
+	switch (metohd) {
+		case 'aid': return genAid(date);
+		case 'meid': return genMeid(date);
+		case 'meidg': return genMeidg(date);
+		case 'ulid': return ulid(date.getTime());
+		case 'objectid': return genObjectId(date);
+		default: throw new Error('unrecognized id generation method');
+	}
+}

packages/backend/src/misc/gen-key-pair.ts

@@ -0,0 +1,36 @@
+import * as crypto from 'crypto';
+import * as util from 'util';
+
+const generateKeyPair = util.promisify(crypto.generateKeyPair);
+
+export async function genRsaKeyPair(modulusLength = 2048) {
+	return await generateKeyPair('rsa', {
+		modulusLength,
+		publicKeyEncoding: {
+			type: 'spki',
+			format: 'pem'
+		},
+		privateKeyEncoding: {
+			type: 'pkcs8',
+			format: 'pem',
+			cipher: undefined,
+			passphrase: undefined
+		}
+	});
+}
+
+export async function genEcKeyPair(namedCurve: 'prime256v1' | 'secp384r1' | 'secp521r1' | 'curve25519' = 'prime256v1') {
+	return await generateKeyPair('ec', {
+		namedCurve,
+		publicKeyEncoding: {
+			type: 'spki',
+			format: 'pem'
+		},
+		privateKeyEncoding: {
+			type: 'pkcs8',
+			format: 'pem',
+			cipher: undefined,
+			passphrase: undefined
+		}
+	});
+}

packages/backend/src/misc/get-file-info.ts

@@ -0,0 +1,196 @@
+import * as fs from 'fs';
+import * as crypto from 'crypto';
+import * as stream from 'stream';
+import * as util from 'util';
+import * as fileType from 'file-type';
+import isSvg from 'is-svg';
+import * as probeImageSize from 'probe-image-size';
+import * as sharp from 'sharp';
+import { encode } from 'blurhash';
+
+const pipeline = util.promisify(stream.pipeline);
+
+export type FileInfo = {
+	size: number;
+	md5: string;
+	type: {
+		mime: string;
+		ext: string | null;
+	};
+	width?: number;
+	height?: number;
+	blurhash?: string;
+	warnings: string[];
+};
+
+const TYPE_OCTET_STREAM = {
+	mime: 'application/octet-stream',
+	ext: null
+};
+
+const TYPE_SVG = {
+	mime: 'image/svg+xml',
+	ext: 'svg'
+};
+
+/**
+ * Get file information
+ */
+export async function getFileInfo(path: string): Promise<FileInfo> {
+	const warnings = [] as string[];
+
+	const size = await getFileSize(path);
+	const md5 = await calcHash(path);
+
+	let type = await detectType(path);
+
+	// image dimensions
+	let width: number | undefined;
+	let height: number | undefined;
+
+	if (['image/jpeg', 'image/gif', 'image/png', 'image/apng', 'image/webp', 'image/bmp', 'image/tiff', 'image/svg+xml', 'image/vnd.adobe.photoshop'].includes(type.mime)) {
+		const imageSize = await detectImageSize(path).catch(e => {
+			warnings.push(`detectImageSize failed: ${e}`);
+			return undefined;
+		});
+
+		// うまく判定できない画像は octet-stream にする
+		if (!imageSize) {
+			warnings.push(`cannot detect image dimensions`);
+			type = TYPE_OCTET_STREAM;
+		} else if (imageSize.wUnits === 'px') {
+			width = imageSize.width;
+			height = imageSize.height;
+
+			// 制限を超えている画像は octet-stream にする
+			if (imageSize.width > 16383 || imageSize.height > 16383) {
+				warnings.push(`image dimensions exceeds limits`);
+				type = TYPE_OCTET_STREAM;
+			}
+		} else {
+			warnings.push(`unsupported unit type: ${imageSize.wUnits}`);
+		}
+	}
+
+	let blurhash: string | undefined;
+
+	if (['image/jpeg', 'image/gif', 'image/png', 'image/apng', 'image/webp', 'image/svg+xml'].includes(type.mime)) {
+		blurhash = await getBlurhash(path).catch(e => {
+			warnings.push(`getBlurhash failed: ${e}`);
+			return undefined;
+		});
+	}
+
+	return {
+		size,
+		md5,
+		type,
+		width,
+		height,
+		blurhash,
+		warnings,
+	};
+}
+
+/**
+ * Detect MIME Type and extension
+ */
+export async function detectType(path: string) {
+	// Check 0 byte
+	const fileSize = await getFileSize(path);
+	if (fileSize === 0) {
+		return TYPE_OCTET_STREAM;
+	}
+
+	const type = await fileType.fromFile(path);
+
+	if (type) {
+		// XMLはSVGかもしれない
+		if (type.mime === 'application/xml' && await checkSvg(path)) {
+			return TYPE_SVG;
+		}
+
+		return {
+			mime: type.mime,
+			ext: type.ext
+		};
+	}
+
+	// 種類が不明でもSVGかもしれない
+	if (await checkSvg(path)) {
+		return TYPE_SVG;
+	}
+
+	// それでも種類が不明なら application/octet-stream にする
+	return TYPE_OCTET_STREAM;
+}
+
+/**
+ * Check the file is SVG or not
+ */
+export async function checkSvg(path: string) {
+	try {
+		const size = await getFileSize(path);
+		if (size > 1 * 1024 * 1024) return false;
+		return isSvg(fs.readFileSync(path));
+	} catch {
+		return false;
+	}
+}
+
+/**
+ * Get file size
+ */
+export async function getFileSize(path: string): Promise<number> {
+	const getStat = util.promisify(fs.stat);
+	return (await getStat(path)).size;
+}
+
+/**
+ * Calculate MD5 hash
+ */
+async function calcHash(path: string): Promise<string> {
+	const hash = crypto.createHash('md5').setEncoding('hex');
+	await pipeline(fs.createReadStream(path), hash);
+	return hash.read();
+}
+
+/**
+ * Detect dimensions of image
+ */
+async function detectImageSize(path: string): Promise<{
+	width: number;
+	height: number;
+	wUnits: string;
+	hUnits: string;
+}> {
+	const readable = fs.createReadStream(path);
+	const imageSize = await probeImageSize(readable);
+	readable.destroy();
+	return imageSize;
+}
+
+/**
+ * Calculate average color of image
+ */
+function getBlurhash(path: string): Promise<string> {
+	return new Promise((resolve, reject) => {
+		sharp(path)
+			.raw()
+			.ensureAlpha()
+			.resize(64, 64, { fit: 'inside' })
+			.toBuffer((err, buffer, { width, height }) => {
+				if (err) return reject(err);
+
+				let hash;
+
+				try {
+					hash = encode(new Uint8ClampedArray(buffer), width, height, 7, 7);
+				} catch (e) {
+					return reject(e);
+				}
+
+				resolve(hash);
+			});
+	});
+}

packages/backend/src/misc/get-note-summary.ts

@@ -0,0 +1,54 @@
+import { Packed } from './schema';
+
+/**
+ * 投稿を表す文字列を取得します。
+ * @param {*} note (packされた)投稿
+ */
+export const getNoteSummary = (note: Packed<'Note'>): string => {
+	if (note.deletedAt) {
+		return `(❌⛔)`;
+	}
+
+	if (note.isHidden) {
+		return `(⛔)`;
+	}
+
+	let summary = '';
+
+	// 本文
+	if (note.cw != null) {
+		summary += note.cw;
+	} else {
+		summary += note.text ? note.text : '';
+	}
+
+	// ファイルが添付されているとき
+	if ((note.files || []).length != 0) {
+		summary += ` (📎${note.files!.length})`;
+	}
+
+	// 投票が添付されているとき
+	if (note.poll) {
+		summary += ` (📊)`;
+	}
+
+	// 返信のとき
+	if (note.replyId) {
+		if (note.reply) {
+			summary += `\n\nRE: ${getNoteSummary(note.reply)}`;
+		} else {
+			summary += '\n\nRE: ...';
+		}
+	}
+
+	// Renoteのとき
+	if (note.renoteId) {
+		if (note.renote) {
+			summary += `\n\nRN: ${getNoteSummary(note.renote)}`;
+		} else {
+			summary += '\n\nRN: ...';
+		}
+	}
+
+	return summary.trim();
+};

packages/backend/src/misc/get-reaction-emoji.ts

@@ -0,0 +1,16 @@
+export default function(reaction: string): string {
+	switch (reaction) {
+		case 'like': return '👍';
+		case 'love': return '❤️';
+		case 'laugh': return '😆';
+		case 'hmm': return '🤔';
+		case 'surprise': return '😮';
+		case 'congrats': return '🎉';
+		case 'angry': return '💢';
+		case 'confused': return '😥';
+		case 'rip': return '😇';
+		case 'pudding': return '🍮';
+		case 'star': return '⭐';
+		default: return reaction;
+	}
+}

packages/backend/src/misc/hard-limits.ts

@@ -0,0 +1,14 @@
+
+// If you change DB_* values, you must also change the DB schema.
+
+/**
+ * Maximum note text length that can be stored in DB.
+ * Surrogate pairs count as one
+ */
+export const DB_MAX_NOTE_TEXT_LENGTH = 8192;
+
+/**
+ * Maximum image description length that can be stored in DB.
+ * Surrogate pairs count as one
+ */
+export const DB_MAX_IMAGE_COMMENT_LENGTH = 512;

packages/backend/src/misc/i18n.ts

@@ -0,0 +1,29 @@
+export class I18n<T extends Record<string, any>> {
+	public locale: T;
+
+	constructor(locale: T) {
+		this.locale = locale;
+
+		//#region BIND
+		this.t = this.t.bind(this);
+		//#endregion
+	}
+
+	// string にしているのは、ドット区切りでのパス指定を許可するため
+	// なるべくこのメソッド使うよりもlocale直接参照の方がvueのキャッシュ効いてパフォーマンスが良いかも
+	public t(key: string, args?: Record<string, any>): string {
+		try {
+			let str = key.split('.').reduce((o, i) => o[i], this.locale) as string;
+
+			if (args) {
+				for (const [k, v] of Object.entries(args)) {
+					str = str.replace(`{${k}}`, v);
+				}
+			}
+			return str;
+		} catch (e) {
+			console.warn(`missing localization '${key}'`);
+			return key;
+		}
+	}
+}

packages/backend/src/misc/id/aid.ts

@@ -0,0 +1,25 @@
+// AID
+// 長さ8の[2000年1月1日からの経過ミリ秒をbase36でエンコードしたもの] + 長さ2の[ノイズ文字列]
+
+import * as crypto from 'crypto';
+
+const TIME2000 = 946684800000;
+let counter = crypto.randomBytes(2).readUInt16LE(0);
+
+function getTime(time: number) {
+	time = time - TIME2000;
+	if (time < 0) time = 0;
+
+	return time.toString(36).padStart(8, '0');
+}
+
+function getNoise() {
+	return counter.toString(36).padStart(2, '0').slice(-2);
+}
+
+export function genAid(date: Date): string {
+	const t = date.getTime();
+	if (isNaN(t)) throw 'Failed to create AID: Invalid Date';
+	counter++;
+	return getTime(t) + getNoise();
+}

packages/backend/src/misc/id/meid.ts

@@ -0,0 +1,26 @@
+const CHARS = '0123456789abcdef';
+
+function getTime(time: number) {
+	if (time < 0) time = 0;
+	if (time === 0) {
+		return CHARS[0];
+	}
+
+	time += 0x800000000000;
+
+	return time.toString(16).padStart(12, CHARS[0]);
+}
+
+function getRandom() {
+	let str = '';
+
+	for (let i = 0; i < 12; i++) {
+		str += CHARS[Math.floor(Math.random() * CHARS.length)];
+	}
+
+	return str;
+}
+
+export function genMeid(date: Date): string {
+	return getTime(date.getTime()) + getRandom();
+}

packages/backend/src/misc/id/meidg.ts

@@ -0,0 +1,28 @@
+const CHARS = '0123456789abcdef';
+
+//  4bit Fixed hex value 'g'
+// 44bit UNIX Time ms in Hex
+// 48bit Random value in Hex
+
+function getTime(time: number) {
+	if (time < 0) time = 0;
+	if (time === 0) {
+		return CHARS[0];
+	}
+
+	return time.toString(16).padStart(11, CHARS[0]);
+}
+
+function getRandom() {
+	let str = '';
+
+	for (let i = 0; i < 12; i++) {
+		str += CHARS[Math.floor(Math.random() * CHARS.length)];
+	}
+
+	return str;
+}
+
+export function genMeidg(date: Date): string {
+	return 'g' + getTime(date.getTime()) + getRandom();
+}

packages/backend/src/misc/id/object-id.ts

@@ -0,0 +1,26 @@
+const CHARS = '0123456789abcdef';
+
+function getTime(time: number) {
+	if (time < 0) time = 0;
+	if (time === 0) {
+		return CHARS[0];
+	}
+
+	time = Math.floor(time / 1000);
+
+	return time.toString(16).padStart(8, CHARS[0]);
+}
+
+function getRandom() {
+	let str = '';
+
+	for (let i = 0; i < 16; i++) {
+		str += CHARS[Math.floor(Math.random() * CHARS.length)];
+	}
+
+	return str;
+}
+
+export function genObjectId(date: Date): string {
+	return getTime(date.getTime()) + getRandom();
+}

packages/backend/src/misc/identifiable-error.ts

@@ -0,0 +1,13 @@
+/**
+ * ID付きエラー
+ */
+export class IdentifiableError extends Error {
+	public message: string;
+	public id: string;
+
+	constructor(id: string, message?: string) {
+		super(message);
+		this.message = message || '';
+		this.id = id;
+	}
+}

packages/backend/src/misc/is-blocker-user-related.ts

@@ -0,0 +1,15 @@
+export function isBlockerUserRelated(note: any, blockerUserIds: Set<string>): boolean {
+	if (blockerUserIds.has(note.userId)) {
+		return true;
+	}
+
+	if (note.reply != null && blockerUserIds.has(note.reply.userId)) {
+		return true;
+	}
+
+	if (note.renote != null && blockerUserIds.has(note.renote.userId)) {
+		return true;
+	}
+
+	return false;
+}

packages/backend/src/misc/is-duplicate-key-value-error.ts

@@ -0,0 +1,3 @@
+export function isDuplicateKeyValueError(e: Error): boolean {
+	return e.message.startsWith('duplicate key value');
+}

packages/backend/src/misc/is-muted-user-related.ts

@@ -0,0 +1,15 @@
+export function isMutedUserRelated(note: any, mutedUserIds: Set<string>): boolean {
+	if (mutedUserIds.has(note.userId)) {
+		return true;
+	}
+
+	if (note.reply != null && mutedUserIds.has(note.reply.userId)) {
+		return true;
+	}
+
+	if (note.renote != null && mutedUserIds.has(note.renote.userId)) {
+		return true;
+	}
+
+	return false;
+}

packages/backend/src/misc/is-quote.ts

@@ -0,0 +1,5 @@
+import { Note } from '@/models/entities/note';
+
+export default function(note: Note): boolean {
+	return note.renoteId != null && (note.text != null || note.hasPoll || (note.fileIds != null && note.fileIds.length > 0));
+}

packages/backend/src/misc/keypair-store.ts

@@ -0,0 +1,10 @@
+import { UserKeypairs } from '@/models/index';
+import { User } from '@/models/entities/user';
+import { UserKeypair } from '@/models/entities/user-keypair';
+import { Cache } from './cache';
+
+const cache = new Cache<UserKeypair>(Infinity);
+
+export async function getUserKeypair(userId: User['id']): Promise<UserKeypair> {
+	return await cache.fetch(userId, () => UserKeypairs.findOneOrFail(userId));
+}

packages/backend/src/misc/normalize-for-search.ts

@@ -0,0 +1,6 @@
+export function normalizeForSearch(tag: string): string {
+	// ref.
+	// - https://analytics-note.xyz/programming/unicode-normalization-forms/
+	// - https://maku77.github.io/js/string/normalize.html
+	return tag.normalize('NFKC').toLowerCase();
+}

packages/backend/src/misc/nyaize.ts

@@ -0,0 +1,15 @@
+export function nyaize(text: string): string {
+	return text
+		// ja-JP
+		.replace(/な/g, 'にゃ').replace(/ナ/g, 'ニャ').replace(/ﾅ/g, 'ﾆｬ')
+		// en-US
+		.replace(/(?<=n)a/gi, x => x === 'A' ? 'YA' : 'ya')
+		.replace(/(?<=morn)ing/gi, x => x === 'ING' ? 'YAN' : 'yan')
+		.replace(/(?<=every)one/gi, x => x === 'ONE' ? 'NYAN' : 'nyan')
+		// ko-KR
+		.replace(/[나-낳]/g, match => String.fromCharCode(
+			match.charCodeAt(0)! + '냐'.charCodeAt(0) - '나'.charCodeAt(0)
+		))
+		.replace(/(다$)|(다(?=\.))|(다(?= ))|(다(?=!))|(다(?=\?))/gm, '다냥')
+		.replace(/(야(?=\?))|(야$)|(야(?= ))/gm, '냥');
+}

packages/backend/src/misc/populate-emojis.ts

@@ -0,0 +1,124 @@
+import { In } from 'typeorm';
+import { Emojis } from '@/models/index';
+import { Emoji } from '@/models/entities/emoji';
+import { Note } from '@/models/entities/note';
+import { Cache } from './cache';
+import { isSelfHost, toPunyNullable } from './convert-host';
+import { decodeReaction } from './reaction-lib';
+import config from '@/config/index';
+import { query } from '@/prelude/url';
+
+const cache = new Cache<Emoji | null>(1000 * 60 * 60 * 12);
+
+/**
+ * 添付用絵文字情報
+ */
+type PopulatedEmoji = {
+	name: string;
+	url: string;
+};
+
+function normalizeHost(src: string | undefined, noteUserHost: string | null): string | null {
+	// クエリに使うホスト
+	let host = src === '.' ? null	// .はローカルホスト (ここがマッチするのはリアクションのみ)
+		: src === undefined ? noteUserHost	// ノートなどでホスト省略表記の場合はローカルホスト (ここがリアクションにマッチすることはない)
+		: isSelfHost(src) ? null	// 自ホスト指定
+		: (src || noteUserHost);	// 指定されたホスト || ノートなどの所有者のホスト (こっちがリアクションにマッチすることはない)
+
+	host = toPunyNullable(host);
+
+	return host;
+}
+
+function parseEmojiStr(emojiName: string, noteUserHost: string | null) {
+	const match = emojiName.match(/^(\w+)(?:@([\w.-]+))?$/);
+	if (!match) return { name: null, host: null };
+
+	const name = match[1];
+
+	// ホスト正規化
+	const host = toPunyNullable(normalizeHost(match[2], noteUserHost));
+
+	return { name, host };
+}
+
+/**
+ * 添付用絵文字情報を解決する
+ * @param emojiName ノートやユーザープロフィールに添付された、またはリアクションのカスタム絵文字名 (:は含めない, リアクションでローカルホストの場合は@.を付ける (これはdecodeReactionで可能))
+ * @param noteUserHost ノートやユーザープロフィールの所有者のホスト
+ * @returns 絵文字情報, nullは未マッチを意味する
+ */
+export async function populateEmoji(emojiName: string, noteUserHost: string | null): Promise<PopulatedEmoji | null> {
+	const { name, host } = parseEmojiStr(emojiName, noteUserHost);
+	if (name == null) return null;
+
+	const queryOrNull = async () => (await Emojis.findOne({
+		name,
+		host
+	})) || null;
+
+	const emoji = await cache.fetch(`${name} ${host}`, queryOrNull);
+
+	if (emoji == null) return null;
+
+	const isLocal = emoji.host == null;
+	const url = isLocal ? emoji.url : `${config.url}/proxy/image.png?${query({url: emoji.url})}`;
+
+	return {
+		name: emojiName,
+		url,
+	};
+}
+
+/**
+ * 複数の添付用絵文字情報を解決する (キャシュ付き, 存在しないものは結果から除外される)
+ */
+export async function populateEmojis(emojiNames: string[], noteUserHost: string | null): Promise<PopulatedEmoji[]> {
+	const emojis = await Promise.all(emojiNames.map(x => populateEmoji(x, noteUserHost)));
+	return emojis.filter((x): x is PopulatedEmoji => x != null);
+}
+
+export function aggregateNoteEmojis(notes: Note[]) {
+	let emojis: { name: string | null; host: string | null; }[] = [];
+	for (const note of notes) {
+		emojis = emojis.concat(note.emojis
+			.map(e => parseEmojiStr(e, note.userHost)));
+		if (note.renote) {
+			emojis = emojis.concat(note.renote.emojis
+				.map(e => parseEmojiStr(e, note.renote!.userHost)));
+			if (note.renote.user) {
+				emojis = emojis.concat(note.renote.user.emojis
+					.map(e => parseEmojiStr(e, note.renote!.userHost)));
+			}
+		}
+		const customReactions = Object.keys(note.reactions).map(x => decodeReaction(x)).filter(x => x.name != null) as typeof emojis;
+		emojis = emojis.concat(customReactions);
+		if (note.user) {
+			emojis = emojis.concat(note.user.emojis
+				.map(e => parseEmojiStr(e, note.userHost)));
+		}
+	}
+	return emojis.filter(x => x.name != null) as { name: string; host: string | null; }[];
+}
+
+/**
+ * 与えられた絵文字のリストをデータベースから取得し、キャッシュに追加します
+ */
+export async function prefetchEmojis(emojis: { name: string; host: string | null; }[]): Promise<void> {
+	const notCachedEmojis = emojis.filter(emoji => cache.get(`${emoji.name} ${emoji.host}`) == null);
+	const emojisQuery: any[] = [];
+	const hosts = new Set(notCachedEmojis.map(e => e.host));
+	for (const host of hosts) {
+		emojisQuery.push({
+			name: In(notCachedEmojis.filter(e => e.host === host).map(e => e.name)),
+			host: host
+		});
+	}
+	const _emojis = emojisQuery.length > 0 ? await Emojis.find({
+		where: emojisQuery,
+		select: ['name', 'host', 'url']
+	}) : [];
+	for (const emoji of _emojis) {
+		cache.set(`${emoji.name} ${emoji.host}`, emoji);
+	}
+}

packages/backend/src/misc/reaction-lib.ts

@@ -0,0 +1,129 @@
+import { emojiRegex } from './emoji-regex';
+import { fetchMeta } from './fetch-meta';
+import { Emojis } from '@/models/index';
+import { toPunyNullable } from './convert-host';
+
+const legacies: Record<string, string> = {
+	'like':     '👍',
+	'love':     '❤', // ここに記述する場合は異体字セレクタを入れない
+	'laugh':    '😆',
+	'hmm':      '🤔',
+	'surprise': '😮',
+	'congrats': '🎉',
+	'angry':    '💢',
+	'confused': '😥',
+	'rip':      '😇',
+	'pudding':  '🍮',
+	'star':     '⭐',
+};
+
+export async function getFallbackReaction(): Promise<string> {
+	const meta = await fetchMeta();
+	return meta.useStarForReactionFallback ? '⭐' : '👍';
+}
+
+export function convertLegacyReactions(reactions: Record<string, number>) {
+	const _reactions = {} as Record<string, number>;
+
+	for (const reaction of Object.keys(reactions)) {
+		if (reactions[reaction] <= 0) continue;
+
+		if (Object.keys(legacies).includes(reaction)) {
+			if (_reactions[legacies[reaction]]) {
+				_reactions[legacies[reaction]] += reactions[reaction];
+			} else {
+				_reactions[legacies[reaction]] = reactions[reaction];
+			}
+		} else {
+			if (_reactions[reaction]) {
+				_reactions[reaction] += reactions[reaction];
+			} else {
+				_reactions[reaction] = reactions[reaction];
+			}
+		}
+	}
+
+	const _reactions2 = {} as Record<string, number>;
+
+	for (const reaction of Object.keys(_reactions)) {
+		_reactions2[decodeReaction(reaction).reaction] = _reactions[reaction];
+	}
+
+	return _reactions2;
+}
+
+export async function toDbReaction(reaction?: string | null, reacterHost?: string | null): Promise<string> {
+	if (reaction == null) return await getFallbackReaction();
+
+	reacterHost = toPunyNullable(reacterHost);
+
+	// 文字列タイプのリアクションを絵文字に変換
+	if (Object.keys(legacies).includes(reaction)) return legacies[reaction];
+
+	// Unicode絵文字
+	const match = emojiRegex.exec(reaction);
+	if (match) {
+		// 合字を含む1つの絵文字
+		const unicode = match[0];
+
+		// 異体字セレクタ除去
+		return unicode.match('\u200d') ? unicode : unicode.replace(/\ufe0f/g, '');
+	}
+
+	const custom = reaction.match(/^:([\w+-]+)(?:@\.)?:$/);
+	if (custom) {
+		const name = custom[1];
+		const emoji = await Emojis.findOne({
+			host: reacterHost || null,
+			name,
+		});
+
+		if (emoji) return reacterHost ? `:${name}@${reacterHost}:` : `:${name}:`;
+	}
+
+	return await getFallbackReaction();
+}
+
+type DecodedReaction = {
+	/**
+	 * リアクション名 (Unicode Emoji or ':name@hostname' or ':name@.')
+	 */
+	reaction: string;
+
+	/**
+	 * name (カスタム絵文字の場合name, Emojiクエリに使う)
+	 */
+	name?: string;
+
+	/**
+	 * host (カスタム絵文字の場合host, Emojiクエリに使う)
+	 */
+	host?: string | null;
+};
+
+export function decodeReaction(str: string): DecodedReaction {
+	const custom = str.match(/^:([\w+-]+)(?:@([\w.-]+))?:$/);
+
+	if (custom) {
+		const name = custom[1];
+		const host = custom[2] || null;
+
+		return {
+			reaction: `:${name}@${host || '.'}:`,	// ローカル分は@以降を省略するのではなく.にする
+			name,
+			host
+		};
+	}
+
+	return {
+		reaction: str,
+		name: undefined,
+		host: undefined
+	};
+}
+
+export function convertLegacyReaction(reaction: string): string {
+	reaction = decodeReaction(reaction).reaction;
+	if (Object.keys(legacies).includes(reaction)) return legacies[reaction];
+	return reaction;
+}

packages/backend/src/misc/safe-for-sql.ts

@@ -0,0 +1,3 @@
+export function safeForSql(text: string): boolean {
+	return !/[\0\x08\x09\x1a\n\r"'\\\%]/g.test(text);
+}

packages/backend/src/misc/schema.ts

@@ -0,0 +1,107 @@
+import { SimpleObj, SimpleSchema } from './simple-schema';
+import { packedUserSchema } from '@/models/repositories/user';
+import { packedNoteSchema } from '@/models/repositories/note';
+import { packedUserListSchema } from '@/models/repositories/user-list';
+import { packedAppSchema } from '@/models/repositories/app';
+import { packedMessagingMessageSchema } from '@/models/repositories/messaging-message';
+import { packedNotificationSchema } from '@/models/repositories/notification';
+import { packedDriveFileSchema } from '@/models/repositories/drive-file';
+import { packedDriveFolderSchema } from '@/models/repositories/drive-folder';
+import { packedFollowingSchema } from '@/models/repositories/following';
+import { packedMutingSchema } from '@/models/repositories/muting';
+import { packedBlockingSchema } from '@/models/repositories/blocking';
+import { packedNoteReactionSchema } from '@/models/repositories/note-reaction';
+import { packedHashtagSchema } from '@/models/repositories/hashtag';
+import { packedPageSchema } from '@/models/repositories/page';
+import { packedUserGroupSchema } from '@/models/repositories/user-group';
+import { packedNoteFavoriteSchema } from '@/models/repositories/note-favorite';
+import { packedChannelSchema } from '@/models/repositories/channel';
+import { packedAntennaSchema } from '@/models/repositories/antenna';
+import { packedClipSchema } from '@/models/repositories/clip';
+import { packedFederationInstanceSchema } from '@/models/repositories/federation-instance';
+import { packedQueueCountSchema } from '@/models/repositories/queue';
+import { packedGalleryPostSchema } from '@/models/repositories/gallery-post';
+import { packedEmojiSchema } from '@/models/repositories/emoji';
+import { packedReversiGameSchema } from '@/models/repositories/games/reversi/game';
+import { packedReversiMatchingSchema } from '@/models/repositories/games/reversi/matching';
+
+export const refs = {
+	User: packedUserSchema,
+	UserList: packedUserListSchema,
+	UserGroup: packedUserGroupSchema,
+	App: packedAppSchema,
+	MessagingMessage: packedMessagingMessageSchema,
+	Note: packedNoteSchema,
+	NoteReaction: packedNoteReactionSchema,
+	NoteFavorite: packedNoteFavoriteSchema,
+	Notification: packedNotificationSchema,
+	DriveFile: packedDriveFileSchema,
+	DriveFolder: packedDriveFolderSchema,
+	Following: packedFollowingSchema,
+	Muting: packedMutingSchema,
+	Blocking: packedBlockingSchema,
+	Hashtag: packedHashtagSchema,
+	Page: packedPageSchema,
+	Channel: packedChannelSchema,
+	QueueCount: packedQueueCountSchema,
+	Antenna: packedAntennaSchema,
+	Clip: packedClipSchema,
+	FederationInstance: packedFederationInstanceSchema,
+	GalleryPost: packedGalleryPostSchema,
+	Emoji: packedEmojiSchema,
+	ReversiGame: packedReversiGameSchema,
+	ReversiMatching: packedReversiMatchingSchema,
+};
+
+export type Packed<x extends keyof typeof refs> = ObjType<(typeof refs[x])['properties']>;
+
+export interface Schema extends SimpleSchema {
+	items?: Schema;
+	properties?: Obj;
+	ref?: keyof typeof refs;
+}
+
+type NonUndefinedPropertyNames<T extends Obj> = {
+	[K in keyof T]: T[K]['optional'] extends true ? never : K
+}[keyof T];
+
+type UndefinedPropertyNames<T extends Obj> = {
+	[K in keyof T]: T[K]['optional'] extends true ? K : never
+}[keyof T];
+
+type OnlyRequired<T extends Obj> = Pick<T, NonUndefinedPropertyNames<T>>;
+type OnlyOptional<T extends Obj> = Pick<T, UndefinedPropertyNames<T>>;
+
+export interface Obj extends SimpleObj { [key: string]: Schema; }
+
+export type ObjType<s extends Obj> =
+	{ [P in keyof OnlyOptional<s>]?: SchemaType<s[P]> } &
+	{ [P in keyof OnlyRequired<s>]: SchemaType<s[P]> };
+
+// https://qiita.com/hrsh7th@github/items/84e8968c3601009cdcf2
+type MyType<T extends Schema> = {
+	0: any;
+	1: SchemaType<T>;
+}[T extends Schema ? 1 : 0];
+
+type NullOrUndefined<p extends Schema, T> =
+	p['nullable'] extends true
+		?	p['optional'] extends true
+			? (T | null | undefined)
+			: (T | null)
+		: p['optional'] extends true
+			? (T | undefined)
+			: T;
+
+export type SchemaType<p extends Schema> =
+	p['type'] extends 'number' ? NullOrUndefined<p, number> :
+	p['type'] extends 'string' ? NullOrUndefined<p, string> :
+	p['type'] extends 'boolean' ? NullOrUndefined<p, boolean> :
+	p['type'] extends 'array' ? NullOrUndefined<p, MyType<NonNullable<p['items']>>[]> :
+	p['type'] extends 'object' ? (
+		p['ref'] extends keyof typeof refs
+			? NullOrUndefined<p, Packed<p['ref']>>
+			: NullOrUndefined<p, ObjType<NonNullable<p['properties']>>>
+	) :
+	p['type'] extends 'any' ? NullOrUndefined<p, any> :
+	any;

packages/backend/src/misc/secure-rndstr.ts

@@ -0,0 +1,21 @@
+import * as crypto from 'crypto';
+
+const L_CHARS = '0123456789abcdefghijklmnopqrstuvwxyz';
+const LU_CHARS = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
+
+export function secureRndstr(length = 32, useLU = true): string {
+	const chars = useLU ? LU_CHARS : L_CHARS;
+	const chars_len = chars.length;
+
+	let str = '';
+
+	for (let i = 0; i < length; i++) {
+		let rand = Math.floor((crypto.randomBytes(1).readUInt8(0) / 0xFF) * chars_len);
+		if (rand === chars_len) {
+			rand = chars_len - 1;
+		}
+		str += chars.charAt(rand);
+	}
+
+	return str;
+}

packages/backend/src/misc/show-machine-info.ts

@@ -0,0 +1,13 @@
+import * as os from 'os';
+import * as sysUtils from 'systeminformation';
+import Logger from '@/services/logger';
+
+export async function showMachineInfo(parentLogger: Logger) {
+	const logger = parentLogger.createSubLogger('machine');
+	logger.debug(`Hostname: ${os.hostname()}`);
+	logger.debug(`Platform: ${process.platform} Arch: ${process.arch}`);
+	const mem = await sysUtils.mem();
+	const totalmem = (mem.total / 1024 / 1024 / 1024).toFixed(1);
+	const availmem = (mem.available / 1024 / 1024 / 1024).toFixed(1);
+	logger.debug(`CPU: ${os.cpus().length} core MEM: ${totalmem}GB (available: ${availmem}GB)`);
+}

packages/backend/src/misc/simple-schema.ts

@@ -0,0 +1,15 @@
+export interface SimpleSchema {
+	type: 'boolean' | 'number' | 'string' | 'array' | 'object' | 'any';
+	nullable: boolean;
+	optional: boolean;
+	items?: SimpleSchema;
+	properties?: SimpleObj;
+	description?: string;
+	example?: any;
+	format?: string;
+	ref?: string;
+	enum?: string[];
+	default?: boolean | null;
+}
+
+export interface SimpleObj { [key: string]: SimpleSchema; }

packages/backend/src/misc/truncate.ts

@@ -0,0 +1,11 @@
+import { substring } from 'stringz';
+
+export function truncate(input: string, size: number): string;
+export function truncate(input: string | undefined, size: number): string | undefined;
+export function truncate(input: string | undefined, size: number): string | undefined {
+	if (!input) {
+		return input;
+	} else {
+		return substring(input, 0, size);
+	}
+}