12.106.0あっぷだて

packages/backend/src/misc/check-word-mute.ts

@@ -11,26 +11,31 @@ type UserLike = {
 	id: User['id'];
 };
 
-export async function checkWordMute(note: NoteLike, me: UserLike | null | undefined, mutedWords: string[][]): Promise<boolean> {
+export async function checkWordMute(note: NoteLike, me: UserLike | null | undefined, mutedWords: Array<string | string[]>): Promise<boolean> {
 	// 自分自身
 	if (me && (note.userId === me.id)) return false;
 
-	const words = mutedWords
-		// Clean up
-		.map(xs => xs.filter(x => x !== ''))
-		.filter(xs => xs.length > 0);
-
-	if (words.length > 0) {
+	if (mutedWords.length > 0) {
 		if (note.text == null) return false;
 
-		const matched = words.some(and =>
-			and.every(keyword => {
-				const regexp = keyword.match(/^\/(.+)\/(.*)$/);
-				if (regexp) {
+		const matched = mutedWords.some(filter => {
+			if (Array.isArray(filter)) {
+				return filter.every(keyword => note.text!.includes(keyword));
+			} else {
+				// represents RegExp
+				const regexp = filter.match(/^\/(.+)\/(.*)$/);
+
+				// This should never happen due to input sanitisation.
+				if (!regexp) return false;
+
+				try {
 					return new RE2(regexp[1], regexp[2]).test(note.text!);
+				} catch (err) {
+					// This should never happen due to input sanitisation.
+					return false;
 				}
-				return note.text!.includes(keyword);
-			}));
+			}
+		});
 
 		if (matched) return true;
 	}

packages/backend/src/models/entities/following.ts

@@ -41,6 +41,7 @@ export class Following {
 	public follower: User | null;
 
 	//#region Denormalized fields
+	@Index()
 	@Column('varchar', {
 		length: 128, nullable: true,
 		comment: '[Denormalized]',
@@ -59,6 +60,7 @@ export class Following {
 	})
 	public followerSharedInbox: string | null;
 
+	@Index()
 	@Column('varchar', {
 		length: 128, nullable: true,
 		comment: '[Denormalized]',

packages/backend/src/queue/index.ts

@@ -258,6 +258,11 @@ export default function() {
 	processDb(dbQueue);
 	processObjectStorage(objectStorageQueue);
 
+	systemQueue.add('tickCharts', {
+	}, {
+		repeat: { cron: '55 * * * *' },
+	});
+
 	systemQueue.add('resyncCharts', {
 	}, {
 		repeat: { cron: '0 0 * * *' },

packages/backend/src/queue/processors/system/index.ts

@@ -1,8 +1,10 @@
 import * as Bull from 'bull';
+import { tickCharts } from './tick-charts';
 import { resyncCharts } from './resync-charts';
 import { cleanCharts } from './clean-charts';
 
 const jobs = {
+	tickCharts,
 	resyncCharts,
 	cleanCharts,
 } as Record<string, Bull.ProcessCallbackFunction<Record<string, unknown>> | Bull.ProcessPromiseFunction<Record<string, unknown>>>;

packages/backend/src/queue/processors/system/tick-charts.ts

@@ -0,0 +1,28 @@
+import * as Bull from 'bull';
+
+import { queueLogger } from '../../logger';
+import { activeUsersChart, driveChart, federationChart, hashtagChart, instanceChart, notesChart, perUserDriveChart, perUserFollowingChart, perUserNotesChart, perUserReactionsChart, usersChart, apRequestChart } from '@/services/chart/index';
+
+const logger = queueLogger.createSubLogger('tick-charts');
+
+export async function tickCharts(job: Bull.Job<Record<string, unknown>>, done: any): Promise<void> {
+	logger.info(`Tick charts...`);
+
+	await Promise.all([
+		federationChart.tick(false),
+		notesChart.tick(false),
+		usersChart.tick(false),
+		activeUsersChart.tick(false),
+		instanceChart.tick(false),
+		perUserNotesChart.tick(false),
+		driveChart.tick(false),
+		perUserReactionsChart.tick(false),
+		hashtagChart.tick(false),
+		perUserFollowingChart.tick(false),
+		perUserDriveChart.tick(false),
+		apRequestChart.tick(false),
+	]);
+
+	logger.succ(`All charts successfully ticked.`);
+	done();
+}

packages/backend/src/server/api/endpoints/admin/resync-chart.ts

@@ -1,22 +0,0 @@
-import define from '../../define';
-import { driveChart, notesChart, usersChart } from '@/services/chart/index';
-import { insertModerationLog } from '@/services/insert-moderation-log';
-
-export const meta = {
-	tags: ['admin'],
-
-	requireCredential: true,
-	requireModerator: true,
-} as const;
-
-// eslint-disable-next-line import/no-default-export
-export default define(meta, async (ps, me) => {
-	insertModerationLog(me, 'chartResync');
-
-	driveChart.resync();
-	notesChart.resync();
-	usersChart.resync();
-
-	// TODO: ユーザーごとのチャートもキューに入れて更新する
-	// TODO: インスタンスごとのチャートもキューに入れて更新する
-});

packages/backend/src/server/api/endpoints/i/update.ts

@@ -1,3 +1,4 @@
+const RE2 = require('re2');
 import $ from 'cafy';
 import * as mfm from 'mfm-js';
 import { ID } from '@/misc/cafy-id';
@@ -117,7 +118,7 @@ export const meta = {
 		},
 
 		mutedWords: {
-			validator: $.optional.arr($.arr($.str)),
+			validator: $.optional.arr($.either($.arr($.str.min(1)).min(1), $.str)),
 		},
 
 		mutedInstances: {
@@ -163,6 +164,12 @@ export const meta = {
 			code: 'NO_SUCH_PAGE',
 			id: '8e01b590-7eb9-431b-a239-860e086c408e',
 		},
+
+		invalidRegexp: {
+			message: 'Invalid Regular Expression.',
+			code: 'INVALID_REGEXP',
+			id: '0d786918-10df-41cd-8f33-8dec7d9a89a5',
+		}
 	},
 
 	res: {
@@ -191,6 +198,18 @@ export default define(meta, async (ps, _user, token) => {
 	if (ps.avatarId !== undefined) updates.avatarId = ps.avatarId;
 	if (ps.bannerId !== undefined) updates.bannerId = ps.bannerId;
 	if (ps.mutedWords !== undefined) {
+		// validate regular expression syntax
+		ps.mutedWords.filter(x => !Array.isArray(x)).forEach(x => {
+			const regexp = x.match(/^\/(.+)\/(.*)$/);
+			if (!regexp) throw new ApiError(meta.errors.invalidRegexp);
+
+			try {
+				new RE2(regexp[1], regexp[2]);
+			} catch (err) {
+				throw new ApiError(meta.errors.invalidRegexp);
+			}
+		});
+
 		profileUpdates.mutedWords = ps.mutedWords;
 		profileUpdates.enableWordMute = ps.mutedWords.length > 0;
 	}

packages/backend/src/services/chart/charts/active-users.ts

@@ -18,7 +18,12 @@ export default class ActiveUsersChart extends Chart<typeof schema> {
 	}
 
 	@autobind
-	protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
+	protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
+		return {};
+	}
+
+	@autobind
+	protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
 		return {};
 	}
 

packages/backend/src/services/chart/charts/ap-request.ts

@@ -12,7 +12,12 @@ export default class ApRequestChart extends Chart<typeof schema> {
 	}
 
 	@autobind
-	protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
+	protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
+		return {};
+	}
+
+	@autobind
+	protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
 		return {};
 	}
 

packages/backend/src/services/chart/charts/drive.ts

@@ -15,7 +15,12 @@ export default class DriveChart extends Chart<typeof schema> {
 	}
 
 	@autobind
-	protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
+	protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
+		return {};
+	}
+
+	@autobind
+	protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
 		return {};
 	}
 

packages/backend/src/services/chart/charts/entities/federation.ts

@@ -3,12 +3,11 @@ import Chart from '../../core';
 export const name = 'federation';
 
 export const schema = {
-	'instance.total': { accumulate: true },
-	'instance.inc': { range: 'small' },
-	'instance.dec': { range: 'small' },
 	'deliveredInstances': { uniqueIncrement: true, range: 'small' },
 	'inboxInstances': { uniqueIncrement: true, range: 'small' },
 	'stalled': { uniqueIncrement: true, range: 'small' },
+	'sub': { accumulate: true, range: 'small' },
+	'pub': { accumulate: true, range: 'small' },
 } as const;
 
 export const entity = Chart.schemaToEntity(name, schema);

packages/backend/src/services/chart/charts/federation.ts

@@ -1,6 +1,6 @@
 import autobind from 'autobind-decorator';
 import Chart, { KVs } from '../core';
-import { Instances } from '@/models/index';
+import { Followings } from '@/models/index';
 import { name, schema } from './entities/federation';
 
 /**
@@ -13,23 +13,30 @@ export default class FederationChart extends Chart<typeof schema> {
 	}
 
 	@autobind
-	protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
-		const [total] = await Promise.all([
-			Instances.count({}),
-		]);
-
+	protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
 		return {
-			'instance.total': total,
 		};
 	}
 
 	@autobind
-	public async update(isAdditional: boolean): Promise<void> {
-		await this.commit({
-			'instance.total': isAdditional ? 1 : -1,
-			'instance.inc': isAdditional ? 1 : 0,
-			'instance.dec': isAdditional ? 0 : 1,
-		});
+	protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
+		const [sub, pub] = await Promise.all([
+			Followings.createQueryBuilder('following')
+				.select('COUNT(DISTINCT following.followeeHost)')
+				.where('following.followeeHost IS NOT NULL')
+				.getRawOne()
+				.then(x => parseInt(x.count, 10)),
+			Followings.createQueryBuilder('following')
+				.select('COUNT(DISTINCT following.followerHost)')
+				.where('following.followerHost IS NOT NULL')
+				.getRawOne()
+				.then(x => parseInt(x.count, 10)),
+		]);
+
+		return {
+			'sub': sub,
+			'pub': pub,
+		};
 	}
 
 	@autobind

packages/backend/src/services/chart/charts/hashtag.ts

@@ -14,7 +14,12 @@ export default class HashtagChart extends Chart<typeof schema> {
 	}
 
 	@autobind
-	protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
+	protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
+		return {};
+	}
+
+	@autobind
+	protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
 		return {};
 	}
 

packages/backend/src/services/chart/charts/instance.ts

@@ -16,7 +16,7 @@ export default class InstanceChart extends Chart<typeof schema> {
 	}
 
 	@autobind
-	protected async queryCurrentState(group: string): Promise<Partial<KVs<typeof schema>>> {
+	protected async tickMajor(group: string): Promise<Partial<KVs<typeof schema>>> {
 		const [
 			notesCount,
 			usersCount,
@@ -42,6 +42,11 @@ export default class InstanceChart extends Chart<typeof schema> {
 		};
 	}
 
+	@autobind
+	protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
+		return {};
+	}
+
 	@autobind
 	public async requestReceived(host: string): Promise<void> {
 		await this.commit({

packages/backend/src/services/chart/charts/notes.ts

@@ -15,7 +15,7 @@ export default class NotesChart extends Chart<typeof schema> {
 	}
 
 	@autobind
-	protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
+	protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
 		const [localCount, remoteCount] = await Promise.all([
 			Notes.count({ userHost: null }),
 			Notes.count({ userHost: Not(IsNull()) }),
@@ -27,6 +27,11 @@ export default class NotesChart extends Chart<typeof schema> {
 		};
 	}
 
+	@autobind
+	protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
+		return {};
+	}
+
 	@autobind
 	public async update(note: Note, isAdditional: boolean): Promise<void> {
 		const prefix = note.userHost === null ? 'local' : 'remote';

packages/backend/src/services/chart/charts/per-user-drive.ts

@@ -14,7 +14,7 @@ export default class PerUserDriveChart extends Chart<typeof schema> {
 	}
 
 	@autobind
-	protected async queryCurrentState(group: string): Promise<Partial<KVs<typeof schema>>> {
+	protected async tickMajor(group: string): Promise<Partial<KVs<typeof schema>>> {
 		const [count, size] = await Promise.all([
 			DriveFiles.count({ userId: group }),
 			DriveFiles.calcDriveUsageOf(group),
@@ -26,6 +26,11 @@ export default class PerUserDriveChart extends Chart<typeof schema> {
 		};
 	}
 
+	@autobind
+	protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
+		return {};
+	}
+
 	@autobind
 	public async update(file: DriveFile, isAdditional: boolean): Promise<void> {
 		const fileSizeKb = file.size / 1000;

packages/backend/src/services/chart/charts/per-user-following.ts

@@ -15,7 +15,7 @@ export default class PerUserFollowingChart extends Chart<typeof schema> {
 	}
 
 	@autobind
-	protected async queryCurrentState(group: string): Promise<Partial<KVs<typeof schema>>> {
+	protected async tickMajor(group: string): Promise<Partial<KVs<typeof schema>>> {
 		const [
 			localFollowingsCount,
 			localFollowersCount,
@@ -36,6 +36,11 @@ export default class PerUserFollowingChart extends Chart<typeof schema> {
 		};
 	}
 
+	@autobind
+	protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
+		return {};
+	}
+
 	@autobind
 	public async update(follower: { id: User['id']; host: User['host']; }, followee: { id: User['id']; host: User['host']; }, isFollow: boolean): Promise<void> {
 		const prefixFollower = Users.isLocalUser(follower) ? 'local' : 'remote';

packages/backend/src/services/chart/charts/per-user-notes.ts

@@ -15,7 +15,7 @@ export default class PerUserNotesChart extends Chart<typeof schema> {
 	}
 
 	@autobind
-	protected async queryCurrentState(group: string): Promise<Partial<KVs<typeof schema>>> {
+	protected async tickMajor(group: string): Promise<Partial<KVs<typeof schema>>> {
 		const [count] = await Promise.all([
 			Notes.count({ userId: group }),
 		]);
@@ -25,6 +25,11 @@ export default class PerUserNotesChart extends Chart<typeof schema> {
 		};
 	}
 
+	@autobind
+	protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
+		return {};
+	}
+
 	@autobind
 	public async update(user: { id: User['id'] }, note: Note, isAdditional: boolean): Promise<void> {
 		await this.commit({

packages/backend/src/services/chart/charts/per-user-reactions.ts

@@ -15,7 +15,12 @@ export default class PerUserReactionsChart extends Chart<typeof schema> {
 	}
 
 	@autobind
-	protected async queryCurrentState(group: string): Promise<Partial<KVs<typeof schema>>> {
+	protected async tickMajor(group: string): Promise<Partial<KVs<typeof schema>>> {
+		return {};
+	}
+
+	@autobind
+	protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
 		return {};
 	}
 

packages/backend/src/services/chart/charts/test-grouped.ts

@@ -14,12 +14,17 @@ export default class TestGroupedChart extends Chart<typeof schema> {
 	}
 
 	@autobind
-	protected async queryCurrentState(group: string): Promise<Partial<KVs<typeof schema>>> {
+	protected async tickMajor(group: string): Promise<Partial<KVs<typeof schema>>> {
 		return {
 			'foo.total': this.total[group],
 		};
 	}
 
+	@autobind
+	protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
+		return {};
+	}
+
 	@autobind
 	public async increment(group: string): Promise<void> {
 		if (this.total[group] == null) this.total[group] = 0;

packages/backend/src/services/chart/charts/test-intersection.ts

@@ -12,7 +12,12 @@ export default class TestIntersectionChart extends Chart<typeof schema> {
 	}
 
 	@autobind
-	protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
+	protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
+		return {};
+	}
+
+	@autobind
+	protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
 		return {};
 	}
 

packages/backend/src/services/chart/charts/test-unique.ts

@@ -12,7 +12,12 @@ export default class TestUniqueChart extends Chart<typeof schema> {
 	}
 
 	@autobind
-	protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
+	protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
+		return {};
+	}
+
+	@autobind
+	protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
 		return {};
 	}
 

packages/backend/src/services/chart/charts/test.ts

@@ -14,12 +14,17 @@ export default class TestChart extends Chart<typeof schema> {
 	}
 
 	@autobind
-	protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
+	protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
 		return {
 			'foo.total': this.total,
 		};
 	}
 
+	@autobind
+	protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
+		return {};
+	}
+
 	@autobind
 	public async increment(): Promise<void> {
 		this.total++;

packages/backend/src/services/chart/charts/users.ts

@@ -15,7 +15,7 @@ export default class UsersChart extends Chart<typeof schema> {
 	}
 
 	@autobind
-	protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
+	protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
 		const [localCount, remoteCount] = await Promise.all([
 			Users.count({ host: null }),
 			Users.count({ host: Not(IsNull()) }),
@@ -27,6 +27,11 @@ export default class UsersChart extends Chart<typeof schema> {
 		};
 	}
 
+	@autobind
+	protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
+		return {};
+	}
+
 	@autobind
 	public async update(user: { id: User['id'], host: User['host'] }, isAdditional: boolean): Promise<void> {
 		const prefix = Users.isLocalUser(user) ? 'local' : 'remote';

packages/backend/src/services/chart/core.ts

@@ -81,7 +81,15 @@ export default abstract class Chart<T extends Schema> {
 	protected repositoryForHour: Repository<RawRecord<T>>;
 	protected repositoryForDay: Repository<RawRecord<T>>;
 
-	protected abstract queryCurrentState(group: string | null): Promise<Partial<KVs<T>>>;
+	/**
+	 * 1日に一回程度実行されれば良いような計算処理を入れる(主にCASCADE削除などアプリケーション側で感知できない変動によるズレの修正用)
+	 */
+	protected abstract tickMajor(group: string | null): Promise<Partial<KVs<T>>>;
+
+	/**
+	 * 少なくとも最小スパン内に1回は実行されて欲しい計算処理を入れる
+	 */
+	protected abstract tickMinor(group: string | null): Promise<Partial<KVs<T>>>;
 
 	@autobind
 	private static convertSchemaToColumnDefinitions(schema: Schema): Record<string, { type: string; array?: boolean; default?: any; }> {
@@ -445,8 +453,8 @@ export default abstract class Chart<T extends Schema> {
 	}
 
 	@autobind
-	public async resync(group: string | null = null): Promise<void> {
-		const data = await this.queryCurrentState(group);
+	public async tick(major: boolean, group: string | null = null): Promise<void> {
+		const data = major ? await this.tickMajor(group) : await this.tickMinor(group);
 
 		const columns = {} as Record<string, number>;
 		for (const [k, v] of Object.entries(data)) {
@@ -480,6 +488,11 @@ export default abstract class Chart<T extends Schema> {
 			update(logHour, logDay));
 	}
 
+	@autobind
+	public resync(group: string | null = null): Promise<void> {
+		return this.tick(true, group);
+	}
+
 	@autobind
 	public async clean(): Promise<void> {
 		const current = dateUTC(Chart.getCurrentDate());

packages/backend/src/services/register-or-fetch-instance-doc.ts

@@ -1,6 +1,5 @@
 import { Instance } from '@/models/entities/instance';
 import { Instances } from '@/models/index';
-import { federationChart } from '@/services/chart/index';
 import { genId } from '@/misc/gen-id';
 import { toPuny } from '@/misc/convert-host';
 import { Cache } from '@/misc/cache';
@@ -23,8 +22,6 @@ export async function registerOrFetchInstanceDoc(host: string): Promise<Instance
 			lastCommunicatedAt: new Date(),
 		}).then(x => Instances.findOneOrFail(x.identifiers[0]));
 
-		federationChart.update(true);
-
 		cache.set(host, i);
 		return i;
 	} else {