なんかもうめっちゃ変えた

This commit is contained in:
syuilo
2022-09-18 03:27:08 +09:00
committed by GitHub
parent d9ab03f086
commit b75184ec8e
946 changed files with 41219 additions and 28839 deletions

View File

@ -0,0 +1,59 @@
import { Injectable, Inject } from '@nestjs/common';
import { beforeShutdown } from '@/misc/before-shutdown.js';
import FederationChart from './charts/federation.js';
import NotesChart from './charts/notes.js';
import UsersChart from './charts/users.js';
import ActiveUsersChart from './charts/active-users.js';
import InstanceChart from './charts/instance.js';
import PerUserNotesChart from './charts/per-user-notes.js';
import DriveChart from './charts/drive.js';
import PerUserReactionsChart from './charts/per-user-reactions.js';
import HashtagChart from './charts/hashtag.js';
import PerUserFollowingChart from './charts/per-user-following.js';
import PerUserDriveChart from './charts/per-user-drive.js';
import ApRequestChart from './charts/ap-request.js';
@Injectable()
export class ChartManagementService {
constructor(
private federationChart: FederationChart,
private notesChart: NotesChart,
private usersChart: UsersChart,
private activeUsersChart: ActiveUsersChart,
private instanceChart: InstanceChart,
private perUserNotesChart: PerUserNotesChart,
private driveChart: DriveChart,
private perUserReactionsChart: PerUserReactionsChart,
private hashtagChart: HashtagChart,
private perUserFollowingChart: PerUserFollowingChart,
private perUserDriveChart: PerUserDriveChart,
private apRequestChart: ApRequestChart,
) {}
public async run() {
const charts = [
this.federationChart,
this.notesChart,
this.usersChart,
this.activeUsersChart,
this.instanceChart,
this.perUserNotesChart,
this.driveChart,
this.perUserReactionsChart,
this.hashtagChart,
this.perUserFollowingChart,
this.perUserDriveChart,
this.apRequestChart,
];
// 20分おきにメモリ情報をDBに書き込み
setInterval(() => {
for (const chart of charts) {
chart.save();
}
}, 1000 * 60 * 20);
beforeShutdown(() => Promise.all(charts.map(chart => chart.save())));
}
}

View File

@ -0,0 +1,54 @@
import { Injectable, Inject } from '@nestjs/common';
import { DataSource } from 'typeorm';
import { AppLockService } from '@/core/AppLockService.js';
import type { User } from '@/models/entities/User.js';
import { DI } from '@/di-symbols.js';
import Chart from '../core.js';
import { name, schema } from './entities/active-users.js';
import type { KVs } from '../core.js';
const week = 1000 * 60 * 60 * 24 * 7;
const month = 1000 * 60 * 60 * 24 * 30;
const year = 1000 * 60 * 60 * 24 * 365;
/**
* アクティブユーザーに関するチャート
*/
// eslint-disable-next-line import/no-default-export
@Injectable()
export default class ActiveUsersChart extends Chart<typeof schema> {
constructor(
@Inject(DI.db)
private db: DataSource,
private appLockService: AppLockService,
) {
super(db, (k) => appLockService.getChartInsertLock(k), name, schema);
}
protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async read(user: { id: User['id'], host: null, createdAt: User['createdAt'] }): Promise<void> {
await this.commit({
'read': [user.id],
'registeredWithinWeek': (Date.now() - user.createdAt.getTime() < week) ? [user.id] : [],
'registeredWithinMonth': (Date.now() - user.createdAt.getTime() < month) ? [user.id] : [],
'registeredWithinYear': (Date.now() - user.createdAt.getTime() < year) ? [user.id] : [],
'registeredOutsideWeek': (Date.now() - user.createdAt.getTime() > week) ? [user.id] : [],
'registeredOutsideMonth': (Date.now() - user.createdAt.getTime() > month) ? [user.id] : [],
'registeredOutsideYear': (Date.now() - user.createdAt.getTime() > year) ? [user.id] : [],
});
}
public async write(user: { id: User['id'], host: null, createdAt: User['createdAt'] }): Promise<void> {
await this.commit({
'write': [user.id],
});
}
}

View File

@ -0,0 +1,49 @@
import { Injectable, Inject } from '@nestjs/common';
import { DataSource } from 'typeorm';
import { AppLockService } from '@/core/AppLockService.js';
import { DI } from '@/di-symbols.js';
import Chart from '../core.js';
import { name, schema } from './entities/ap-request.js';
import type { KVs } from '../core.js';
/**
* Chart about ActivityPub requests
*/
// eslint-disable-next-line import/no-default-export
@Injectable()
export default class ApRequestChart extends Chart<typeof schema> {
constructor(
@Inject(DI.db)
private db: DataSource,
private appLockService: AppLockService,
) {
super(db, (k) => appLockService.getChartInsertLock(k), name, schema);
}
protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async deliverSucc(): Promise<void> {
await this.commit({
'deliverSucceeded': 1,
});
}
public async deliverFail(): Promise<void> {
await this.commit({
'deliverFailed': 1,
});
}
public async inbox(): Promise<void> {
await this.commit({
'inboxReceived': 1,
});
}
}

View File

@ -0,0 +1,47 @@
import { Injectable, Inject } from '@nestjs/common';
import { Not, IsNull, DataSource } from 'typeorm';
import type { DriveFile } from '@/models/entities/DriveFile.js';
import { AppLockService } from '@/core/AppLockService.js';
import { DI } from '@/di-symbols.js';
import Chart from '../core.js';
import { name, schema } from './entities/drive.js';
import type { KVs } from '../core.js';
/**
* ドライブに関するチャート
*/
// eslint-disable-next-line import/no-default-export
@Injectable()
export default class DriveChart extends Chart<typeof schema> {
constructor(
@Inject(DI.db)
private db: DataSource,
private appLockService: AppLockService,
) {
super(db, (k) => appLockService.getChartInsertLock(k), name, schema);
}
protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async update(file: DriveFile, isAdditional: boolean): Promise<void> {
const fileSizeKb = file.size / 1000;
await this.commit(file.userHost === null ? {
'local.incCount': isAdditional ? 1 : 0,
'local.incSize': isAdditional ? fileSizeKb : 0,
'local.decCount': isAdditional ? 0 : 1,
'local.decSize': isAdditional ? 0 : fileSizeKb,
} : {
'remote.incCount': isAdditional ? 1 : 0,
'remote.incSize': isAdditional ? fileSizeKb : 0,
'remote.decCount': isAdditional ? 0 : 1,
'remote.decSize': isAdditional ? 0 : fileSizeKb,
});
}
}

View File

@ -0,0 +1,17 @@
import Chart from '../../core.js';
export const name = 'activeUsers';
export const schema = {
'readWrite': { intersection: ['read', 'write'], range: 'small' },
'read': { uniqueIncrement: true, range: 'small' },
'write': { uniqueIncrement: true, range: 'small' },
'registeredWithinWeek': { uniqueIncrement: true, range: 'small' },
'registeredWithinMonth': { uniqueIncrement: true, range: 'small' },
'registeredWithinYear': { uniqueIncrement: true, range: 'small' },
'registeredOutsideWeek': { uniqueIncrement: true, range: 'small' },
'registeredOutsideMonth': { uniqueIncrement: true, range: 'small' },
'registeredOutsideYear': { uniqueIncrement: true, range: 'small' },
} as const;
export const entity = Chart.schemaToEntity(name, schema);

View File

@ -0,0 +1,11 @@
import Chart from '../../core.js';
export const name = 'apRequest';
export const schema = {
'deliverFailed': { },
'deliverSucceeded': { },
'inboxReceived': { },
} as const;
export const entity = Chart.schemaToEntity(name, schema);

View File

@ -0,0 +1,16 @@
import Chart from '../../core.js';
export const name = 'drive';
export const schema = {
'local.incCount': {},
'local.incSize': {}, // in kilobyte
'local.decCount': {},
'local.decSize': {}, // in kilobyte
'remote.incCount': {},
'remote.incSize': {}, // in kilobyte
'remote.decCount': {},
'remote.decSize': {}, // in kilobyte
} as const;
export const entity = Chart.schemaToEntity(name, schema);

View File

@ -0,0 +1,16 @@
import Chart from '../../core.js';
export const name = 'federation';
export const schema = {
'deliveredInstances': { uniqueIncrement: true, range: 'small' },
'inboxInstances': { uniqueIncrement: true, range: 'small' },
'stalled': { uniqueIncrement: true, range: 'small' },
'sub': { accumulate: true, range: 'small' },
'pub': { accumulate: true, range: 'small' },
'pubsub': { accumulate: true, range: 'small' },
'subActive': { accumulate: true, range: 'small' },
'pubActive': { accumulate: true, range: 'small' },
} as const;
export const entity = Chart.schemaToEntity(name, schema);

View File

@ -0,0 +1,10 @@
import Chart from '../../core.js';
export const name = 'hashtag';
export const schema = {
'local.users': { uniqueIncrement: true },
'remote.users': { uniqueIncrement: true },
} as const;
export const entity = Chart.schemaToEntity(name, schema, true);

View File

@ -0,0 +1,32 @@
import Chart from '../../core.js';
export const name = 'instance';
export const schema = {
'requests.failed': { range: 'small' },
'requests.succeeded': { range: 'small' },
'requests.received': { range: 'small' },
'notes.total': { accumulate: true },
'notes.inc': {},
'notes.dec': {},
'notes.diffs.normal': {},
'notes.diffs.reply': {},
'notes.diffs.renote': {},
'notes.diffs.withFile': {},
'users.total': { accumulate: true },
'users.inc': { range: 'small' },
'users.dec': { range: 'small' },
'following.total': { accumulate: true },
'following.inc': { range: 'small' },
'following.dec': { range: 'small' },
'followers.total': { accumulate: true },
'followers.inc': { range: 'small' },
'followers.dec': { range: 'small' },
'drive.totalFiles': { accumulate: true },
'drive.incFiles': {},
'drive.decFiles': {},
'drive.incUsage': {}, // in kilobyte
'drive.decUsage': {}, // in kilobyte
} as const;
export const entity = Chart.schemaToEntity(name, schema, true);

View File

@ -0,0 +1,22 @@
import Chart from '../../core.js';
export const name = 'notes';
export const schema = {
'local.total': { accumulate: true },
'local.inc': {},
'local.dec': {},
'local.diffs.normal': {},
'local.diffs.reply': {},
'local.diffs.renote': {},
'local.diffs.withFile': {},
'remote.total': { accumulate: true },
'remote.inc': {},
'remote.dec': {},
'remote.diffs.normal': {},
'remote.diffs.reply': {},
'remote.diffs.renote': {},
'remote.diffs.withFile': {},
} as const;
export const entity = Chart.schemaToEntity(name, schema);

View File

@ -0,0 +1,14 @@
import Chart from '../../core.js';
export const name = 'perUserDrive';
export const schema = {
'totalCount': { accumulate: true },
'totalSize': { accumulate: true }, // in kilobyte
'incCount': { range: 'small' },
'incSize': {}, // in kilobyte
'decCount': { range: 'small' },
'decSize': {}, // in kilobyte
} as const;
export const entity = Chart.schemaToEntity(name, schema, true);

View File

@ -0,0 +1,20 @@
import Chart from '../../core.js';
export const name = 'perUserFollowing';
export const schema = {
'local.followings.total': { accumulate: true },
'local.followings.inc': { range: 'small' },
'local.followings.dec': { range: 'small' },
'local.followers.total': { accumulate: true },
'local.followers.inc': { range: 'small' },
'local.followers.dec': { range: 'small' },
'remote.followings.total': { accumulate: true },
'remote.followings.inc': { range: 'small' },
'remote.followings.dec': { range: 'small' },
'remote.followers.total': { accumulate: true },
'remote.followers.inc': { range: 'small' },
'remote.followers.dec': { range: 'small' },
} as const;
export const entity = Chart.schemaToEntity(name, schema, true);

View File

@ -0,0 +1,15 @@
import Chart from '../../core.js';
export const name = 'perUserNotes';
export const schema = {
'total': { accumulate: true },
'inc': { range: 'small' },
'dec': { range: 'small' },
'diffs.normal': { range: 'small' },
'diffs.reply': { range: 'small' },
'diffs.renote': { range: 'small' },
'diffs.withFile': { range: 'small' },
} as const;
export const entity = Chart.schemaToEntity(name, schema, true);

View File

@ -0,0 +1,10 @@
import Chart from '../../core.js';
export const name = 'perUserReaction';
export const schema = {
'local.count': { range: 'small' },
'remote.count': { range: 'small' },
} as const;
export const entity = Chart.schemaToEntity(name, schema, true);

View File

@ -0,0 +1,11 @@
import Chart from '../../core.js';
export const name = 'testGrouped';
export const schema = {
'foo.total': { accumulate: true },
'foo.inc': {},
'foo.dec': {},
} as const;
export const entity = Chart.schemaToEntity(name, schema, true);

View File

@ -0,0 +1,11 @@
import Chart from '../../core.js';
export const name = 'testIntersection';
export const schema = {
'a': { uniqueIncrement: true },
'b': { uniqueIncrement: true },
'aAndB': { intersection: ['a', 'b'] },
} as const;
export const entity = Chart.schemaToEntity(name, schema);

View File

@ -0,0 +1,9 @@
import Chart from '../../core.js';
export const name = 'testUnique';
export const schema = {
'foo': { uniqueIncrement: true },
} as const;
export const entity = Chart.schemaToEntity(name, schema);

View File

@ -0,0 +1,11 @@
import Chart from '../../core.js';
export const name = 'test';
export const schema = {
'foo.total': { accumulate: true },
'foo.inc': {},
'foo.dec': {},
} as const;
export const entity = Chart.schemaToEntity(name, schema);

View File

@ -0,0 +1,14 @@
import Chart from '../../core.js';
export const name = 'users';
export const schema = {
'local.total': { accumulate: true },
'local.inc': { range: 'small' },
'local.dec': { range: 'small' },
'remote.total': { accumulate: true },
'remote.inc': { range: 'small' },
'remote.dec': { range: 'small' },
} as const;
export const entity = Chart.schemaToEntity(name, schema);

View File

@ -0,0 +1,121 @@
import { Injectable, Inject } from '@nestjs/common';
import { DataSource } from 'typeorm';
import { FollowingsRepository, InstancesRepository } from '@/models/index.js';
import { AppLockService } from '@/core/AppLockService.js';
import { DI } from '@/di-symbols.js';
import { MetaService } from '@/core/MetaService.js';
import Chart from '../core.js';
import { name, schema } from './entities/federation.js';
import type { KVs } from '../core.js';
/**
* フェデレーションに関するチャート
*/
// eslint-disable-next-line import/no-default-export
@Injectable()
export default class FederationChart extends Chart<typeof schema> {
constructor(
@Inject(DI.db)
private db: DataSource,
@Inject(DI.followingsRepository)
private followingsRepository: FollowingsRepository,
@Inject(DI.instancesRepository)
private instancesRepository: InstancesRepository,
private metaService: MetaService,
private appLockService: AppLockService,
) {
super(db, (k) => appLockService.getChartInsertLock(k), name, schema);
}
protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
return {
};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
const meta = await this.metaService.fetch();
const suspendedInstancesQuery = this.instancesRepository.createQueryBuilder('instance')
.select('instance.host')
.where('instance.isSuspended = true');
const pubsubSubQuery = this.followingsRepository.createQueryBuilder('f')
.select('f.followerHost')
.where('f.followerHost IS NOT NULL');
const subInstancesQuery = this.followingsRepository.createQueryBuilder('f')
.select('f.followeeHost')
.where('f.followeeHost IS NOT NULL');
const pubInstancesQuery = this.followingsRepository.createQueryBuilder('f')
.select('f.followerHost')
.where('f.followerHost IS NOT NULL');
const [sub, pub, pubsub, subActive, pubActive] = await Promise.all([
this.followingsRepository.createQueryBuilder('following')
.select('COUNT(DISTINCT following.followeeHost)')
.where('following.followeeHost IS NOT NULL')
.andWhere(meta.blockedHosts.length === 0 ? '1=1' : 'following.followeeHost NOT IN (:...blocked)', { blocked: meta.blockedHosts })
.andWhere(`following.followeeHost NOT IN (${ suspendedInstancesQuery.getQuery() })`)
.getRawOne()
.then(x => parseInt(x.count, 10)),
this.followingsRepository.createQueryBuilder('following')
.select('COUNT(DISTINCT following.followerHost)')
.where('following.followerHost IS NOT NULL')
.andWhere(meta.blockedHosts.length === 0 ? '1=1' : 'following.followerHost NOT IN (:...blocked)', { blocked: meta.blockedHosts })
.andWhere(`following.followerHost NOT IN (${ suspendedInstancesQuery.getQuery() })`)
.getRawOne()
.then(x => parseInt(x.count, 10)),
this.followingsRepository.createQueryBuilder('following')
.select('COUNT(DISTINCT following.followeeHost)')
.where('following.followeeHost IS NOT NULL')
.andWhere(meta.blockedHosts.length === 0 ? '1=1' : 'following.followeeHost NOT IN (:...blocked)', { blocked: meta.blockedHosts })
.andWhere(`following.followeeHost NOT IN (${ suspendedInstancesQuery.getQuery() })`)
.andWhere(`following.followeeHost IN (${ pubsubSubQuery.getQuery() })`)
.setParameters(pubsubSubQuery.getParameters())
.getRawOne()
.then(x => parseInt(x.count, 10)),
this.instancesRepository.createQueryBuilder('instance')
.select('COUNT(instance.id)')
.where(`instance.host IN (${ subInstancesQuery.getQuery() })`)
.andWhere(meta.blockedHosts.length === 0 ? '1=1' : 'instance.host NOT IN (:...blocked)', { blocked: meta.blockedHosts })
.andWhere('instance.isSuspended = false')
.andWhere('instance.lastCommunicatedAt > :gt', { gt: new Date(Date.now() - (1000 * 60 * 60 * 24 * 30)) })
.getRawOne()
.then(x => parseInt(x.count, 10)),
this.instancesRepository.createQueryBuilder('instance')
.select('COUNT(instance.id)')
.where(`instance.host IN (${ pubInstancesQuery.getQuery() })`)
.andWhere(meta.blockedHosts.length === 0 ? '1=1' : 'instance.host NOT IN (:...blocked)', { blocked: meta.blockedHosts })
.andWhere('instance.isSuspended = false')
.andWhere('instance.lastCommunicatedAt > :gt', { gt: new Date(Date.now() - (1000 * 60 * 60 * 24 * 30)) })
.getRawOne()
.then(x => parseInt(x.count, 10)),
]);
return {
'sub': sub,
'pub': pub,
'pubsub': pubsub,
'subActive': subActive,
'pubActive': pubActive,
};
}
public async deliverd(host: string, succeeded: boolean): Promise<void> {
await this.commit(succeeded ? {
'deliveredInstances': [host],
} : {
'stalled': [host],
});
}
public async inbox(host: string): Promise<void> {
await this.commit({
'inboxInstances': [host],
});
}
}

View File

@ -0,0 +1,41 @@
import { Injectable, Inject } from '@nestjs/common';
import { DataSource } from 'typeorm';
import type { User } from '@/models/entities/User.js';
import { AppLockService } from '@/core/AppLockService.js';
import { DI } from '@/di-symbols.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import Chart from '../core.js';
import { name, schema } from './entities/hashtag.js';
import type { KVs } from '../core.js';
/**
* ハッシュタグに関するチャート
*/
// eslint-disable-next-line import/no-default-export
@Injectable()
export default class HashtagChart extends Chart<typeof schema> {
constructor(
@Inject(DI.db)
private db: DataSource,
private appLockService: AppLockService,
private userEntityService: UserEntityService,
) {
super(db, (k) => appLockService.getChartInsertLock(k), name, schema, true);
}
protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async update(hashtag: string, user: { id: User['id'], host: User['host'] }): Promise<void> {
await this.commit({
'local.users': this.userEntityService.isLocalUser(user) ? [user.id] : [],
'remote.users': this.userEntityService.isLocalUser(user) ? [] : [user.id],
}, hashtag);
}
}

View File

@ -0,0 +1,127 @@
import { Injectable, Inject } from '@nestjs/common';
import { DataSource } from 'typeorm';
import { DriveFilesRepository, FollowingsRepository, UsersRepository, NotesRepository } from '@/models/index.js';
import type { DriveFile } from '@/models/entities/DriveFile.js';
import type { Note } from '@/models/entities/Note.js';
import { AppLockService } from '@/core/AppLockService.js';
import { DI } from '@/di-symbols.js';
import { UtilityService } from '@/core/UtilityService.js';
import Chart from '../core.js';
import { name, schema } from './entities/instance.js';
import type { KVs } from '../core.js';
/**
* インスタンスごとのチャート
*/
// eslint-disable-next-line import/no-default-export
@Injectable()
export default class InstanceChart extends Chart<typeof schema> {
constructor(
@Inject(DI.db)
private db: DataSource,
@Inject(DI.usersRepository)
private usersRepository: UsersRepository,
@Inject(DI.notesRepository)
private notesRepository: NotesRepository,
@Inject(DI.driveFilesRepository)
private driveFilesRepository: DriveFilesRepository,
@Inject(DI.followingsRepository)
private followingsRepository: FollowingsRepository,
private utilityService: UtilityService,
private appLockService: AppLockService,
) {
super(db, (k) => appLockService.getChartInsertLock(k), name, schema, true);
}
protected async tickMajor(group: string): Promise<Partial<KVs<typeof schema>>> {
const [
notesCount,
usersCount,
followingCount,
followersCount,
driveFiles,
] = await Promise.all([
this.notesRepository.countBy({ userHost: group }),
this.usersRepository.countBy({ host: group }),
this.followingsRepository.countBy({ followerHost: group }),
this.followingsRepository.countBy({ followeeHost: group }),
this.driveFilesRepository.countBy({ userHost: group }),
]);
return {
'notes.total': notesCount,
'users.total': usersCount,
'following.total': followingCount,
'followers.total': followersCount,
'drive.totalFiles': driveFiles,
};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async requestReceived(host: string): Promise<void> {
await this.commit({
'requests.received': 1,
}, this.utilityService.toPuny(host));
}
public async requestSent(host: string, isSucceeded: boolean): Promise<void> {
await this.commit({
'requests.succeeded': isSucceeded ? 1 : 0,
'requests.failed': isSucceeded ? 0 : 1,
}, this.utilityService.toPuny(host));
}
public async newUser(host: string): Promise<void> {
await this.commit({
'users.total': 1,
'users.inc': 1,
}, this.utilityService.toPuny(host));
}
public async updateNote(host: string, note: Note, isAdditional: boolean): Promise<void> {
await this.commit({
'notes.total': isAdditional ? 1 : -1,
'notes.inc': isAdditional ? 1 : 0,
'notes.dec': isAdditional ? 0 : 1,
'notes.diffs.normal': note.replyId == null && note.renoteId == null ? (isAdditional ? 1 : -1) : 0,
'notes.diffs.renote': note.renoteId != null ? (isAdditional ? 1 : -1) : 0,
'notes.diffs.reply': note.replyId != null ? (isAdditional ? 1 : -1) : 0,
'notes.diffs.withFile': note.fileIds.length > 0 ? (isAdditional ? 1 : -1) : 0,
}, this.utilityService.toPuny(host));
}
public async updateFollowing(host: string, isAdditional: boolean): Promise<void> {
await this.commit({
'following.total': isAdditional ? 1 : -1,
'following.inc': isAdditional ? 1 : 0,
'following.dec': isAdditional ? 0 : 1,
}, this.utilityService.toPuny(host));
}
public async updateFollowers(host: string, isAdditional: boolean): Promise<void> {
await this.commit({
'followers.total': isAdditional ? 1 : -1,
'followers.inc': isAdditional ? 1 : 0,
'followers.dec': isAdditional ? 0 : 1,
}, this.utilityService.toPuny(host));
}
public async updateDrive(file: DriveFile, isAdditional: boolean): Promise<void> {
const fileSizeKb = file.size / 1000;
await this.commit({
'drive.totalFiles': isAdditional ? 1 : -1,
'drive.incFiles': isAdditional ? 1 : 0,
'drive.incUsage': isAdditional ? fileSizeKb : 0,
'drive.decFiles': isAdditional ? 1 : 0,
'drive.decUsage': isAdditional ? fileSizeKb : 0,
}, file.userHost);
}
}

View File

@ -0,0 +1,58 @@
import { Injectable, Inject } from '@nestjs/common';
import { Not, IsNull, DataSource } from 'typeorm';
import { NotesRepository } from '@/models/index.js';
import type { Note } from '@/models/entities/Note.js';
import { AppLockService } from '@/core/AppLockService.js';
import { DI } from '@/di-symbols.js';
import Chart from '../core.js';
import { name, schema } from './entities/notes.js';
import type { KVs } from '../core.js';
/**
* ノートに関するチャート
*/
// eslint-disable-next-line import/no-default-export
@Injectable()
export default class NotesChart extends Chart<typeof schema> {
constructor(
@Inject(DI.db)
private db: DataSource,
@Inject(DI.notesRepository)
private notesRepository: NotesRepository,
private appLockService: AppLockService,
) {
super(db, (k) => appLockService.getChartInsertLock(k), name, schema);
}
protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
const [localCount, remoteCount] = await Promise.all([
this.notesRepository.countBy({ userHost: IsNull() }),
this.notesRepository.countBy({ userHost: Not(IsNull()) }),
]);
return {
'local.total': localCount,
'remote.total': remoteCount,
};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async update(note: Note, isAdditional: boolean): Promise<void> {
const prefix = note.userHost === null ? 'local' : 'remote';
await this.commit({
[`${prefix}.total`]: isAdditional ? 1 : -1,
[`${prefix}.inc`]: isAdditional ? 1 : 0,
[`${prefix}.dec`]: isAdditional ? 0 : 1,
[`${prefix}.diffs.normal`]: note.replyId == null && note.renoteId == null ? (isAdditional ? 1 : -1) : 0,
[`${prefix}.diffs.renote`]: note.renoteId != null ? (isAdditional ? 1 : -1) : 0,
[`${prefix}.diffs.reply`]: note.replyId != null ? (isAdditional ? 1 : -1) : 0,
[`${prefix}.diffs.withFile`]: note.fileIds.length > 0 ? (isAdditional ? 1 : -1) : 0,
});
}
}

View File

@ -0,0 +1,58 @@
import { Injectable, Inject } from '@nestjs/common';
import { DataSource } from 'typeorm';
import { DriveFilesRepository } from '@/models/index.js';
import type { DriveFile } from '@/models/entities/DriveFile.js';
import { AppLockService } from '@/core/AppLockService.js';
import { DI } from '@/di-symbols.js';
import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js';
import Chart from '../core.js';
import { name, schema } from './entities/per-user-drive.js';
import type { KVs } from '../core.js';
/**
* ユーザーごとのドライブに関するチャート
*/
// eslint-disable-next-line import/no-default-export
@Injectable()
export default class PerUserDriveChart extends Chart<typeof schema> {
constructor(
@Inject(DI.db)
private db: DataSource,
@Inject(DI.driveFilesRepository)
private driveFilesRepository: DriveFilesRepository,
private appLockService: AppLockService,
private driveFileEntityService: DriveFileEntityService,
) {
super(db, (k) => appLockService.getChartInsertLock(k), name, schema, true);
}
protected async tickMajor(group: string): Promise<Partial<KVs<typeof schema>>> {
const [count, size] = await Promise.all([
this.driveFilesRepository.countBy({ userId: group }),
this.driveFileEntityService.calcDriveUsageOf(group),
]);
return {
'totalCount': count,
'totalSize': size,
};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async update(file: DriveFile, isAdditional: boolean): Promise<void> {
const fileSizeKb = file.size / 1000;
await this.commit({
'totalCount': isAdditional ? 1 : -1,
'totalSize': isAdditional ? fileSizeKb : -fileSizeKb,
'incCount': isAdditional ? 1 : 0,
'incSize': isAdditional ? fileSizeKb : 0,
'decCount': isAdditional ? 0 : 1,
'decSize': isAdditional ? 0 : fileSizeKb,
}, file.userId);
}
}

View File

@ -0,0 +1,71 @@
import { Injectable, Inject } from '@nestjs/common';
import { Not, IsNull, DataSource } from 'typeorm';
import type { User } from '@/models/entities/User.js';
import { AppLockService } from '@/core/AppLockService.js';
import { DI } from '@/di-symbols.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { FollowingsRepository } from '@/models/index.js';
import Chart from '../core.js';
import { name, schema } from './entities/per-user-following.js';
import type { KVs } from '../core.js';
/**
* ユーザーごとのフォローに関するチャート
*/
// eslint-disable-next-line import/no-default-export
@Injectable()
export default class PerUserFollowingChart extends Chart<typeof schema> {
constructor(
@Inject(DI.db)
private db: DataSource,
@Inject(DI.followingsRepository)
private followingsRepository: FollowingsRepository,
private appLockService: AppLockService,
private userEntityService: UserEntityService,
) {
super(db, (k) => appLockService.getChartInsertLock(k), name, schema, true);
}
protected async tickMajor(group: string): Promise<Partial<KVs<typeof schema>>> {
const [
localFollowingsCount,
localFollowersCount,
remoteFollowingsCount,
remoteFollowersCount,
] = await Promise.all([
this.followingsRepository.countBy({ followerId: group, followeeHost: IsNull() }),
this.followingsRepository.countBy({ followeeId: group, followerHost: IsNull() }),
this.followingsRepository.countBy({ followerId: group, followeeHost: Not(IsNull()) }),
this.followingsRepository.countBy({ followeeId: group, followerHost: Not(IsNull()) }),
]);
return {
'local.followings.total': localFollowingsCount,
'local.followers.total': localFollowersCount,
'remote.followings.total': remoteFollowingsCount,
'remote.followers.total': remoteFollowersCount,
};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async update(follower: { id: User['id']; host: User['host']; }, followee: { id: User['id']; host: User['host']; }, isFollow: boolean): Promise<void> {
const prefixFollower = this.userEntityService.isLocalUser(follower) ? 'local' : 'remote';
const prefixFollowee = this.userEntityService.isLocalUser(followee) ? 'local' : 'remote';
this.commit({
[`${prefixFollower}.followings.total`]: isFollow ? 1 : -1,
[`${prefixFollower}.followings.inc`]: isFollow ? 1 : 0,
[`${prefixFollower}.followings.dec`]: isFollow ? 0 : 1,
}, follower.id);
this.commit({
[`${prefixFollowee}.followers.total`]: isFollow ? 1 : -1,
[`${prefixFollowee}.followers.inc`]: isFollow ? 1 : 0,
[`${prefixFollowee}.followers.dec`]: isFollow ? 0 : 1,
}, followee.id);
}
}

View File

@ -0,0 +1,55 @@
import { Injectable, Inject } from '@nestjs/common';
import { DataSource } from 'typeorm';
import type { User } from '@/models/entities/User.js';
import type { Note } from '@/models/entities/Note.js';
import { AppLockService } from '@/core/AppLockService.js';
import { DI } from '@/di-symbols.js';
import { NotesRepository } from '@/models/index.js';
import Chart from '../core.js';
import { name, schema } from './entities/per-user-notes.js';
import type { KVs } from '../core.js';
/**
* ユーザーごとのノートに関するチャート
*/
// eslint-disable-next-line import/no-default-export
@Injectable()
export default class PerUserNotesChart extends Chart<typeof schema> {
constructor(
@Inject(DI.db)
private db: DataSource,
@Inject(DI.notesRepository)
private notesRepository: NotesRepository,
private appLockService: AppLockService,
) {
super(db, (k) => appLockService.getChartInsertLock(k), name, schema, true);
}
protected async tickMajor(group: string): Promise<Partial<KVs<typeof schema>>> {
const [count] = await Promise.all([
this.notesRepository.countBy({ userId: group }),
]);
return {
total: count,
};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async update(user: { id: User['id'] }, note: Note, isAdditional: boolean): Promise<void> {
await this.commit({
'total': isAdditional ? 1 : -1,
'inc': isAdditional ? 1 : 0,
'dec': isAdditional ? 0 : 1,
'diffs.normal': note.replyId == null && note.renoteId == null ? (isAdditional ? 1 : -1) : 0,
'diffs.renote': note.renoteId != null ? (isAdditional ? 1 : -1) : 0,
'diffs.reply': note.replyId != null ? (isAdditional ? 1 : -1) : 0,
'diffs.withFile': note.fileIds.length > 0 ? (isAdditional ? 1 : -1) : 0,
}, user.id);
}
}

View File

@ -0,0 +1,42 @@
import { Injectable, Inject } from '@nestjs/common';
import { DataSource } from 'typeorm';
import type { User } from '@/models/entities/User.js';
import type { Note } from '@/models/entities/Note.js';
import { AppLockService } from '@/core/AppLockService.js';
import { DI } from '@/di-symbols.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import Chart from '../core.js';
import { name, schema } from './entities/per-user-reactions.js';
import type { KVs } from '../core.js';
/**
* ユーザーごとのリアクションに関するチャート
*/
// eslint-disable-next-line import/no-default-export
@Injectable()
export default class PerUserReactionsChart extends Chart<typeof schema> {
constructor(
@Inject(DI.db)
private db: DataSource,
private appLockService: AppLockService,
private userEntityService: UserEntityService,
) {
super(db, (k) => appLockService.getChartInsertLock(k), name, schema, true);
}
protected async tickMajor(group: string): Promise<Partial<KVs<typeof schema>>> {
return {};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async update(user: { id: User['id'], host: User['host'] }, note: Note): Promise<void> {
const prefix = this.userEntityService.isLocalUser(user) ? 'local' : 'remote';
this.commit({
[`${prefix}.count`]: 1,
}, note.userId);
}
}

View File

@ -0,0 +1,46 @@
import { Injectable, Inject } from '@nestjs/common';
import { DataSource } from 'typeorm';
import { AppLockService } from '@/core/AppLockService.js';
import { DI } from '@/di-symbols.js';
import Chart from '../core.js';
import { name, schema } from './entities/test-grouped.js';
import type { KVs } from '../core.js';
/**
* For testing
*/
// eslint-disable-next-line import/no-default-export
@Injectable()
export default class TestGroupedChart extends Chart<typeof schema> {
private total = {} as Record<string, number>;
constructor(
@Inject(DI.db)
private db: DataSource,
private appLockService: AppLockService,
) {
super(db, (k) => appLockService.getChartInsertLock(k), name, schema, true);
}
protected async tickMajor(group: string): Promise<Partial<KVs<typeof schema>>> {
return {
'foo.total': this.total[group],
};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async increment(group: string): Promise<void> {
if (this.total[group] == null) this.total[group] = 0;
this.total[group]++;
await this.commit({
'foo.total': 1,
'foo.inc': 1,
}, group);
}
}

View File

@ -0,0 +1,43 @@
import { Injectable, Inject } from '@nestjs/common';
import { DataSource } from 'typeorm';
import { AppLockService } from '@/core/AppLockService.js';
import { DI } from '@/di-symbols.js';
import Chart from '../core.js';
import { name, schema } from './entities/test-intersection.js';
import type { KVs } from '../core.js';
/**
* For testing
*/
// eslint-disable-next-line import/no-default-export
@Injectable()
export default class TestIntersectionChart extends Chart<typeof schema> {
constructor(
@Inject(DI.db)
private db: DataSource,
private appLockService: AppLockService,
) {
super(db, (k) => appLockService.getChartInsertLock(k), name, schema);
}
protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async addA(key: string): Promise<void> {
await this.commit({
a: [key],
});
}
public async addB(key: string): Promise<void> {
await this.commit({
b: [key],
});
}
}

View File

@ -0,0 +1,37 @@
import { Injectable, Inject } from '@nestjs/common';
import { DataSource } from 'typeorm';
import { AppLockService } from '@/core/AppLockService.js';
import { DI } from '@/di-symbols.js';
import Chart from '../core.js';
import { name, schema } from './entities/test-unique.js';
import type { KVs } from '../core.js';
/**
* For testing
*/
// eslint-disable-next-line import/no-default-export
@Injectable()
export default class TestUniqueChart extends Chart<typeof schema> {
constructor(
@Inject(DI.db)
private db: DataSource,
private appLockService: AppLockService,
) {
super(db, (k) => appLockService.getChartInsertLock(k), name, schema);
}
protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async uniqueIncrement(key: string): Promise<void> {
await this.commit({
foo: [key],
});
}
}

View File

@ -0,0 +1,53 @@
import { Injectable, Inject } from '@nestjs/common';
import { DataSource } from 'typeorm';
import { AppLockService } from '@/core/AppLockService.js';
import { DI } from '@/di-symbols.js';
import Chart from '../core.js';
import { name, schema } from './entities/test.js';
import type { KVs } from '../core.js';
/**
* For testing
*/
// eslint-disable-next-line import/no-default-export
@Injectable()
export default class TestChart extends Chart<typeof schema> {
public total = 0; // publicにするのはテストのため
constructor(
@Inject(DI.db)
private db: DataSource,
private appLockService: AppLockService,
) {
super(db, (k) => appLockService.getChartInsertLock(k), name, schema);
}
protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
return {
'foo.total': this.total,
};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async increment(): Promise<void> {
this.total++;
await this.commit({
'foo.total': 1,
'foo.inc': 1,
});
}
public async decrement(): Promise<void> {
this.total--;
await this.commit({
'foo.total': -1,
'foo.dec': 1,
});
}
}

View File

@ -0,0 +1,56 @@
import { Injectable, Inject } from '@nestjs/common';
import { Not, IsNull, DataSource } from 'typeorm';
import type { User } from '@/models/entities/User.js';
import { AppLockService } from '@/core/AppLockService.js';
import { DI } from '@/di-symbols.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { UsersRepository } from '@/models/index.js';
import Chart from '../core.js';
import { name, schema } from './entities/users.js';
import type { KVs } from '../core.js';
/**
* ユーザー数に関するチャート
*/
// eslint-disable-next-line import/no-default-export
@Injectable()
export default class UsersChart extends Chart<typeof schema> {
constructor(
@Inject(DI.db)
private db: DataSource,
@Inject(DI.usersRepository)
private usersRepository: UsersRepository,
private appLockService: AppLockService,
private userEntityService: UserEntityService,
) {
super(db, (k) => appLockService.getChartInsertLock(k), name, schema);
}
protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
const [localCount, remoteCount] = await Promise.all([
this.usersRepository.countBy({ host: IsNull() }),
this.usersRepository.countBy({ host: Not(IsNull()) }),
]);
return {
'local.total': localCount,
'remote.total': remoteCount,
};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async update(user: { id: User['id'], host: User['host'] }, isAdditional: boolean): Promise<void> {
const prefix = this.userEntityService.isLocalUser(user) ? 'local' : 'remote';
await this.commit({
[`${prefix}.total`]: isAdditional ? 1 : -1,
[`${prefix}.inc`]: isAdditional ? 1 : 0,
[`${prefix}.dec`]: isAdditional ? 0 : 1,
});
}
}

View File

@ -0,0 +1,679 @@
/**
* チャートエンジン
*
* Tests located in test/chart
*/
import * as nestedProperty from 'nested-property';
import { EntitySchema, LessThan, Between } from 'typeorm';
import { dateUTC, isTimeSame, isTimeBefore, subtractTime, addTime } from '@/misc/prelude/time.js';
import Logger from '@/logger.js';
import type { Repository, DataSource } from 'typeorm';
const logger = new Logger('chart', 'white', process.env.NODE_ENV !== 'test');
const columnPrefix = '___' as const;
const uniqueTempColumnPrefix = 'unique_temp___' as const;
const columnDot = '_' as const;
type Schema = Record<string, {
uniqueIncrement?: boolean;
intersection?: string[] | ReadonlyArray<string>;
range?: 'big' | 'small' | 'medium';
// previousな値を引き継ぐかどうか
accumulate?: boolean;
}>;
type KeyToColumnName<T extends string> = T extends `${infer R1}.${infer R2}` ? `${R1}${typeof columnDot}${KeyToColumnName<R2>}` : T;
type Columns<S extends Schema> = {
[K in keyof S as `${typeof columnPrefix}${KeyToColumnName<string & K>}`]: number;
};
type TempColumnsForUnique<S extends Schema> = {
[K in keyof S as `${typeof uniqueTempColumnPrefix}${KeyToColumnName<string & K>}`]: S[K]['uniqueIncrement'] extends true ? string[] : never;
};
type RawRecord<S extends Schema> = {
id: number;
/**
* 集計のグループ
*/
group?: string | null;
/**
* 集計日時のUnixタイムスタンプ(秒)
*/
date: number;
} & TempColumnsForUnique<S> & Columns<S>;
const camelToSnake = (str: string): string => {
return str.replace(/([A-Z])/g, s => '_' + s.charAt(0).toLowerCase());
};
const removeDuplicates = (array: any[]) => Array.from(new Set(array));
type Commit<S extends Schema> = {
[K in keyof S]?: S[K]['uniqueIncrement'] extends true ? string[] : number;
};
export type KVs<S extends Schema> = {
[K in keyof S]: number;
};
type ChartResult<T extends Schema> = {
[P in keyof T]: number[];
};
type UnionToIntersection<T> = (T extends any ? (x: T) => any : never) extends (x: infer R) => any ? R : never;
type UnflattenSingleton<K extends string, V> = K extends `${infer A}.${infer B}`
? { [_ in A]: UnflattenSingleton<B, V>; }
: { [_ in K]: V; };
type Unflatten<T extends Record<string, any>> = UnionToIntersection<
{
[K in Extract<keyof T, string>]: UnflattenSingleton<K, T[K]>;
}[Extract<keyof T, string>]
>;
type ToJsonSchema<S> = {
type: 'object';
properties: {
[K in keyof S]: S[K] extends number[] ? { type: 'array'; items: { type: 'number'; }; } : ToJsonSchema<S[K]>;
},
required: (keyof S)[];
};
export function getJsonSchema<S extends Schema>(schema: S): ToJsonSchema<Unflatten<ChartResult<S>>> {
const jsonSchema = {
type: 'object',
properties: {} as Record<string, unknown>,
required: [],
};
for (const k in schema) {
jsonSchema.properties[k] = {
type: 'array',
items: { type: 'number' },
};
}
return jsonSchema as ToJsonSchema<Unflatten<ChartResult<S>>>;
}
/**
* 様々なチャートの管理を司るクラス
*/
// eslint-disable-next-line import/no-default-export
export default abstract class Chart<T extends Schema> {
public schema: T;
private name: string;
private buffer: {
diff: Commit<T>;
group: string | null;
}[] = [];
// ↓にしたいけどfindOneとかで型エラーになる
//private repositoryForHour: Repository<RawRecord<T>>;
//private repositoryForDay: Repository<RawRecord<T>>;
private repositoryForHour: Repository<{ id: number; group?: string | null; date: number; }>;
private repositoryForDay: Repository<{ id: number; group?: string | null; date: number; }>;
/**
* 1日に一回程度実行されれば良いような計算処理を入れる(主にCASCADE削除などアプリケーション側で感知できない変動によるズレの修正用)
*/
protected abstract tickMajor(group: string | null): Promise<Partial<KVs<T>>>;
/**
* 少なくとも最小スパン内に1回は実行されて欲しい計算処理を入れる
*/
protected abstract tickMinor(group: string | null): Promise<Partial<KVs<T>>>;
private static convertSchemaToColumnDefinitions(schema: Schema): Record<string, { type: string; array?: boolean; default?: any; }> {
const columns = {} as Record<string, { type: string; array?: boolean; default?: any; }>;
for (const [k, v] of Object.entries(schema)) {
const name = k.replaceAll('.', columnDot);
const type = v.range === 'big' ? 'bigint' : v.range === 'small' ? 'smallint' : 'integer';
if (v.uniqueIncrement) {
columns[uniqueTempColumnPrefix + name] = {
type: 'varchar',
array: true,
default: '{}',
};
columns[columnPrefix + name] = {
type,
default: 0,
};
} else {
columns[columnPrefix + name] = {
type,
default: 0,
};
}
}
return columns;
}
private static dateToTimestamp(x: Date): number {
return Math.floor(x.getTime() / 1000);
}
private static parseDate(date: Date): [number, number, number, number, number, number, number] {
const y = date.getUTCFullYear();
const m = date.getUTCMonth();
const d = date.getUTCDate();
const h = date.getUTCHours();
const _m = date.getUTCMinutes();
const _s = date.getUTCSeconds();
const _ms = date.getUTCMilliseconds();
return [y, m, d, h, _m, _s, _ms];
}
private static getCurrentDate() {
return Chart.parseDate(new Date());
}
public static schemaToEntity(name: string, schema: Schema, grouped = false): {
hour: EntitySchema,
day: EntitySchema,
} {
const createEntity = (span: 'hour' | 'day'): EntitySchema => new EntitySchema({
name:
span === 'hour' ? `__chart__${camelToSnake(name)}` :
span === 'day' ? `__chart_day__${camelToSnake(name)}` :
new Error('not happen') as never,
columns: {
id: {
type: 'integer',
primary: true,
generated: true,
},
date: {
type: 'integer',
},
...(grouped ? {
group: {
type: 'varchar',
length: 128,
},
} : {}),
...Chart.convertSchemaToColumnDefinitions(schema),
},
indices: [{
columns: grouped ? ['date', 'group'] : ['date'],
unique: true,
}],
uniques: [{
columns: grouped ? ['date', 'group'] : ['date'],
}],
relations: {
/* TODO
group: {
target: () => Foo,
type: 'many-to-one',
onDelete: 'CASCADE',
},
*/
},
});
return {
hour: createEntity('hour'),
day: createEntity('day'),
};
}
private lock: (key: string) => Promise<() => void>;
constructor(db: DataSource, lock: (key: string) => Promise<() => void>, name: string, schema: T, grouped = false) {
this.name = name;
this.schema = schema;
this.lock = lock;
const { hour, day } = Chart.schemaToEntity(name, schema, grouped);
this.repositoryForHour = db.getRepository<{ id: number; group?: string | null; date: number; }>(hour);
this.repositoryForDay = db.getRepository<{ id: number; group?: string | null; date: number; }>(day);
}
private convertRawRecord(x: RawRecord<T>): KVs<T> {
const kvs = {} as Record<string, number>;
for (const k of Object.keys(x).filter((k) => k.startsWith(columnPrefix)) as (keyof Columns<T>)[]) {
kvs[(k as string).substr(columnPrefix.length).split(columnDot).join('.')] = x[k] as unknown as number;
}
return kvs as KVs<T>;
}
private getNewLog(latest: KVs<T> | null): KVs<T> {
const log = {} as Record<keyof T, number>;
for (const [k, v] of Object.entries(this.schema) as ([keyof typeof this['schema'], this['schema'][string]])[]) {
if (v.accumulate && latest) {
log[k] = latest[k];
} else {
log[k] = 0;
}
}
return log as KVs<T>;
}
private getLatestLog(group: string | null, span: 'hour' | 'day'): Promise<RawRecord<T> | null> {
const repository =
span === 'hour' ? this.repositoryForHour :
span === 'day' ? this.repositoryForDay :
new Error('not happen') as never;
return repository.findOne({
where: group ? {
group: group,
} : {},
order: {
date: -1,
},
}).then(x => x ?? null) as Promise<RawRecord<T> | null>;
}
/**
* 現在(=今のHour or Day)のログをデータベースから探して、あればそれを返し、なければ作成して返します。
*/
private async claimCurrentLog(group: string | null, span: 'hour' | 'day'): Promise<RawRecord<T>> {
const [y, m, d, h] = Chart.getCurrentDate();
const current = dateUTC(
span === 'hour' ? [y, m, d, h] :
span === 'day' ? [y, m, d] :
new Error('not happen') as never);
const repository =
span === 'hour' ? this.repositoryForHour :
span === 'day' ? this.repositoryForDay :
new Error('not happen') as never;
// 現在(=今のHour or Day)のログ
const currentLog = await repository.findOneBy({
date: Chart.dateToTimestamp(current),
...(group ? { group: group } : {}),
}) as RawRecord<T> | undefined;
// ログがあればそれを返して終了
if (currentLog != null) {
return currentLog;
}
let log: RawRecord<T>;
let data: KVs<T>;
// 集計期間が変わってから、初めてのチャート更新なら
// 最も最近のログを持ってくる
// * 例えば集計期間が「日」である場合で考えると、
// * 昨日何もチャートを更新するような出来事がなかった場合は、
// * ログがそもそも作られずドキュメントが存在しないということがあり得るため、
// * 「昨日の」と決め打ちせずに「もっとも最近の」とします
const latest = await this.getLatestLog(group, span);
if (latest != null) {
// 空ログデータを作成
data = this.getNewLog(this.convertRawRecord(latest));
} else {
// ログが存在しなかったら
// (Misskeyインスタンスを建てて初めてのチャート更新時など)
// 初期ログデータを作成
data = this.getNewLog(null);
logger.info(`${this.name + (group ? `:${group}` : '')}(${span}): Initial commit created`);
}
const date = Chart.dateToTimestamp(current);
const lockKey = group ? `${this.name}:${date}:${span}:${group}` : `${this.name}:${date}:${span}`;
const unlock = await this.lock(lockKey);
try {
// ロック内でもう1回チェックする
const currentLog = await repository.findOneBy({
date: date,
...(group ? { group: group } : {}),
}) as RawRecord<T> | undefined;
// ログがあればそれを返して終了
if (currentLog != null) return currentLog;
const columns = {} as Record<string, number | unknown[]>;
for (const [k, v] of Object.entries(data)) {
const name = k.replaceAll('.', columnDot);
columns[columnPrefix + name] = v;
}
// 新規ログ挿入
log = await repository.insert({
date: date,
...(group ? { group: group } : {}),
...columns,
}).then(x => repository.findOneByOrFail(x.identifiers[0])) as RawRecord<T>;
logger.info(`${this.name + (group ? `:${group}` : '')}(${span}): New commit created`);
return log;
} finally {
unlock();
}
}
protected commit(diff: Commit<T>, group: string | null = null): void {
for (const [k, v] of Object.entries(diff)) {
if (v == null || v === 0 || (Array.isArray(v) && v.length === 0)) delete diff[k];
}
this.buffer.push({
diff, group,
});
}
public async save(): Promise<void> {
if (this.buffer.length === 0) {
logger.info(`${this.name}: Write skipped`);
return;
}
// TODO: 前の時間のログがbufferにあった場合のハンドリング
// 例えば、save が20分ごとに行われるとして、前回行われたのは 01:50 だったとする。
// 次に save が行われるのは 02:10 ということになるが、もし 01:55 に新規ログが buffer に追加されたとすると、
// そのログは本来は 01:00~ のログとしてDBに保存されて欲しいのに、02:00~ のログ扱いになってしまう。
// これを回避するための実装は複雑になりそうなため、一旦保留。
const update = async (logHour: RawRecord<T>, logDay: RawRecord<T>): Promise<void> => {
const finalDiffs = {} as Record<string, number | string[]>;
for (const diff of this.buffer.filter(q => q.group == null || (q.group === logHour.group)).map(q => q.diff)) {
for (const [k, v] of Object.entries(diff)) {
if (finalDiffs[k] == null) {
finalDiffs[k] = v;
} else {
if (typeof finalDiffs[k] === 'number') {
(finalDiffs[k] as number) += v as number;
} else {
(finalDiffs[k] as string[]) = (finalDiffs[k] as string[]).concat(v);
}
}
}
}
const queryForHour: Record<keyof RawRecord<T>, number | (() => string)> = {} as any;
const queryForDay: Record<keyof RawRecord<T>, number | (() => string)> = {} as any;
for (const [k, v] of Object.entries(finalDiffs)) {
if (typeof v === 'number') {
const name = columnPrefix + k.replaceAll('.', columnDot) as string & keyof Columns<T>;
if (v > 0) queryForHour[name] = () => `"${name}" + ${v}`;
if (v < 0) queryForHour[name] = () => `"${name}" - ${Math.abs(v)}`;
if (v > 0) queryForDay[name] = () => `"${name}" + ${v}`;
if (v < 0) queryForDay[name] = () => `"${name}" - ${Math.abs(v)}`;
} else if (Array.isArray(v) && v.length > 0) { // ユニークインクリメント
const tempColumnName = uniqueTempColumnPrefix + k.replaceAll('.', columnDot) as string & keyof TempColumnsForUnique<T>;
// TODO: item をSQLエスケープ
const itemsForHour = v.filter(item => !(logHour[tempColumnName] as unknown as string[]).includes(item)).map(item => `"${item}"`);
const itemsForDay = v.filter(item => !(logDay[tempColumnName] as unknown as string[]).includes(item)).map(item => `"${item}"`);
if (itemsForHour.length > 0) queryForHour[tempColumnName] = () => `array_cat("${tempColumnName}", '{${itemsForHour.join(',')}}'::varchar[])`;
if (itemsForDay.length > 0) queryForDay[tempColumnName] = () => `array_cat("${tempColumnName}", '{${itemsForDay.join(',')}}'::varchar[])`;
}
}
// bake unique count
for (const [k, v] of Object.entries(finalDiffs)) {
if (this.schema[k].uniqueIncrement) {
const name = columnPrefix + k.replaceAll('.', columnDot) as keyof Columns<T>;
const tempColumnName = uniqueTempColumnPrefix + k.replaceAll('.', columnDot) as keyof TempColumnsForUnique<T>;
queryForHour[name] = new Set([...(v as string[]), ...(logHour[tempColumnName] as unknown as string[])]).size;
queryForDay[name] = new Set([...(v as string[]), ...(logDay[tempColumnName] as unknown as string[])]).size;
}
}
// compute intersection
// TODO: intersectionに指定されたカラムがintersectionだった場合の対応
for (const [k, v] of Object.entries(this.schema)) {
const intersection = v.intersection;
if (intersection) {
const name = columnPrefix + k.replaceAll('.', columnDot) as keyof Columns<T>;
const firstKey = intersection[0];
const firstTempColumnName = uniqueTempColumnPrefix + firstKey.replaceAll('.', columnDot) as keyof TempColumnsForUnique<T>;
const firstValues = finalDiffs[firstKey] as string[] | undefined;
const currentValuesForHour = new Set([...(firstValues ?? []), ...(logHour[firstTempColumnName] as unknown as string[])]);
const currentValuesForDay = new Set([...(firstValues ?? []), ...(logDay[firstTempColumnName] as unknown as string[])]);
for (let i = 1; i < intersection.length; i++) {
const targetKey = intersection[i];
const targetTempColumnName = uniqueTempColumnPrefix + targetKey.replaceAll('.', columnDot) as keyof TempColumnsForUnique<T>;
const targetValues = finalDiffs[targetKey] as string[] | undefined;
const targetValuesForHour = new Set([...(targetValues ?? []), ...(logHour[targetTempColumnName] as unknown as string[])]);
const targetValuesForDay = new Set([...(targetValues ?? []), ...(logDay[targetTempColumnName] as unknown as string[])]);
currentValuesForHour.forEach(v => {
if (!targetValuesForHour.has(v)) currentValuesForHour.delete(v);
});
currentValuesForDay.forEach(v => {
if (!targetValuesForDay.has(v)) currentValuesForDay.delete(v);
});
}
queryForHour[name] = currentValuesForHour.size;
queryForDay[name] = currentValuesForDay.size;
}
}
// ログ更新
await Promise.all([
this.repositoryForHour.createQueryBuilder()
.update()
.set(queryForHour as any)
.where('id = :id', { id: logHour.id })
.execute(),
this.repositoryForDay.createQueryBuilder()
.update()
.set(queryForDay as any)
.where('id = :id', { id: logDay.id })
.execute(),
]);
logger.info(`${this.name + (logHour.group ? `:${logHour.group}` : '')}: Updated`);
// TODO: この一連の処理が始まった後に新たにbufferに入ったものは消さないようにする
this.buffer = this.buffer.filter(q => q.group != null && (q.group !== logHour.group));
};
const groups = removeDuplicates(this.buffer.map(log => log.group));
await Promise.all(
groups.map(group =>
Promise.all([
this.claimCurrentLog(group, 'hour'),
this.claimCurrentLog(group, 'day'),
]).then(([logHour, logDay]) =>
update(logHour, logDay))));
}
public async tick(major: boolean, group: string | null = null): Promise<void> {
const data = major ? await this.tickMajor(group) : await this.tickMinor(group);
const columns = {} as Record<keyof Columns<T>, number>;
for (const [k, v] of Object.entries(data) as ([keyof typeof data, number])[]) {
const name = columnPrefix + (k as string).replaceAll('.', columnDot) as keyof Columns<T>;
columns[name] = v;
}
if (Object.keys(columns).length === 0) {
return;
}
const update = async (logHour: RawRecord<T>, logDay: RawRecord<T>): Promise<void> => {
await Promise.all([
this.repositoryForHour.createQueryBuilder()
.update()
.set(columns)
.where('id = :id', { id: logHour.id })
.execute(),
this.repositoryForDay.createQueryBuilder()
.update()
.set(columns)
.where('id = :id', { id: logDay.id })
.execute(),
]);
};
return Promise.all([
this.claimCurrentLog(group, 'hour'),
this.claimCurrentLog(group, 'day'),
]).then(([logHour, logDay]) =>
update(logHour, logDay));
}
public resync(group: string | null = null): Promise<void> {
return this.tick(true, group);
}
public async clean(): Promise<void> {
const current = dateUTC(Chart.getCurrentDate());
// 一日以上前かつ三日以内
const gt = Chart.dateToTimestamp(current) - (60 * 60 * 24 * 3);
const lt = Chart.dateToTimestamp(current) - (60 * 60 * 24);
const columns = {} as Record<keyof TempColumnsForUnique<T>, []>;
for (const [k, v] of Object.entries(this.schema)) {
if (v.uniqueIncrement) {
const name = uniqueTempColumnPrefix + k.replaceAll('.', columnDot) as keyof TempColumnsForUnique<T>;
columns[name] = [];
}
}
if (Object.keys(columns).length === 0) {
return;
}
await Promise.all([
this.repositoryForHour.createQueryBuilder()
.update()
.set(columns)
.where('date > :gt', { gt })
.andWhere('date < :lt', { lt })
.execute(),
this.repositoryForDay.createQueryBuilder()
.update()
.set(columns)
.where('date > :gt', { gt })
.andWhere('date < :lt', { lt })
.execute(),
]);
}
public async getChartRaw(span: 'hour' | 'day', amount: number, cursor: Date | null, group: string | null = null): Promise<ChartResult<T>> {
const [y, m, d, h, _m, _s, _ms] = cursor ? Chart.parseDate(subtractTime(addTime(cursor, 1, span), 1)) : Chart.getCurrentDate();
const [y2, m2, d2, h2] = cursor ? Chart.parseDate(addTime(cursor, 1, span)) : [] as never;
const lt = dateUTC([y, m, d, h, _m, _s, _ms]);
const gt =
span === 'day' ? subtractTime(cursor ? dateUTC([y2, m2, d2, 0]) : dateUTC([y, m, d, 0]), amount - 1, 'day') :
span === 'hour' ? subtractTime(cursor ? dateUTC([y2, m2, d2, h2]) : dateUTC([y, m, d, h]), amount - 1, 'hour') :
new Error('not happen') as never;
const repository =
span === 'hour' ? this.repositoryForHour :
span === 'day' ? this.repositoryForDay :
new Error('not happen') as never;
// ログ取得
let logs = await repository.find({
where: {
date: Between(Chart.dateToTimestamp(gt), Chart.dateToTimestamp(lt)),
...(group ? { group: group } : {}),
},
order: {
date: -1,
},
}) as RawRecord<T>[];
// 要求された範囲にログがひとつもなかったら
if (logs.length === 0) {
// もっとも新しいログを持ってくる
// (すくなくともひとつログが無いと隙間埋めできないため)
const recentLog = await repository.findOne({
where: group ? {
group: group,
} : {},
order: {
date: -1,
},
}) as RawRecord<T> | undefined;
if (recentLog) {
logs = [recentLog];
}
// 要求された範囲の最も古い箇所に位置するログが存在しなかったら
} else if (!isTimeSame(new Date(logs[logs.length - 1].date * 1000), gt)) {
// 要求された範囲の最も古い箇所時点での最も新しいログを持ってきて末尾に追加する
// (隙間埋めできないため)
const outdatedLog = await repository.findOne({
where: {
date: LessThan(Chart.dateToTimestamp(gt)),
...(group ? { group: group } : {}),
},
order: {
date: -1,
},
}) as RawRecord<T> | undefined;
if (outdatedLog) {
logs.push(outdatedLog);
}
}
const chart: KVs<T>[] = [];
for (let i = (amount - 1); i >= 0; i--) {
const current =
span === 'hour' ? subtractTime(dateUTC([y, m, d, h]), i, 'hour') :
span === 'day' ? subtractTime(dateUTC([y, m, d]), i, 'day') :
new Error('not happen') as never;
const log = logs.find(l => isTimeSame(new Date(l.date * 1000), current));
if (log) {
chart.unshift(this.convertRawRecord(log));
} else {
// 隙間埋め
const latest = logs.find(l => isTimeBefore(new Date(l.date * 1000), current));
const data = latest ? this.convertRawRecord(latest) : null;
chart.unshift(this.getNewLog(data));
}
}
const res = {} as ChartResult<T>;
/**
* [{ foo: 1, bar: 5 }, { foo: 2, bar: 6 }, { foo: 3, bar: 7 }]
* を
* { foo: [1, 2, 3], bar: [5, 6, 7] }
* にする
*/
for (const record of chart) {
for (const [k, v] of Object.entries(record) as ([keyof typeof record, number])[]) {
if (res[k]) {
res[k].push(v);
} else {
res[k] = [v];
}
}
}
return res;
}
public async getChart(span: 'hour' | 'day', amount: number, cursor: Date | null, group: string | null = null): Promise<Unflatten<ChartResult<T>>> {
const result = await this.getChartRaw(span, amount, cursor, group);
const object = {};
for (const [k, v] of Object.entries(result)) {
nestedProperty.set(object, k, v);
}
return object as Unflatten<ChartResult<T>>;
}
}

View File

@ -0,0 +1,39 @@
import { entity as FederationChart } from './charts/entities/federation.js';
import { entity as NotesChart } from './charts/entities/notes.js';
import { entity as UsersChart } from './charts/entities/users.js';
import { entity as ActiveUsersChart } from './charts/entities/active-users.js';
import { entity as InstanceChart } from './charts/entities/instance.js';
import { entity as PerUserNotesChart } from './charts/entities/per-user-notes.js';
import { entity as DriveChart } from './charts/entities/drive.js';
import { entity as PerUserReactionsChart } from './charts/entities/per-user-reactions.js';
import { entity as HashtagChart } from './charts/entities/hashtag.js';
import { entity as PerUserFollowingChart } from './charts/entities/per-user-following.js';
import { entity as PerUserDriveChart } from './charts/entities/per-user-drive.js';
import { entity as ApRequestChart } from './charts/entities/ap-request.js';
import { entity as TestChart } from './charts/entities/test.js';
import { entity as TestGroupedChart } from './charts/entities/test-grouped.js';
import { entity as TestUniqueChart } from './charts/entities/test-unique.js';
import { entity as TestIntersectionChart } from './charts/entities/test-intersection.js';
export const entities = [
FederationChart.hour, FederationChart.day,
NotesChart.hour, NotesChart.day,
UsersChart.hour, UsersChart.day,
ActiveUsersChart.hour, ActiveUsersChart.day,
InstanceChart.hour, InstanceChart.day,
PerUserNotesChart.hour, PerUserNotesChart.day,
DriveChart.hour, DriveChart.day,
PerUserReactionsChart.hour, PerUserReactionsChart.day,
HashtagChart.hour, HashtagChart.day,
PerUserFollowingChart.hour, PerUserFollowingChart.day,
PerUserDriveChart.hour, PerUserDriveChart.day,
ApRequestChart.hour, ApRequestChart.day,
...(process.env.NODE_ENV === 'test' ? [
TestChart.hour, TestChart.day,
TestGroupedChart.hour, TestGroupedChart.day,
TestUniqueChart.hour, TestUniqueChart.day,
TestIntersectionChart.hour, TestIntersectionChart.day,
] : []),
];