refactoring

Resolve #7779
This commit is contained in:
syuilo
2021-11-12 02:02:25 +09:00
parent 037837b551
commit 0e4a111f81
1714 changed files with 20803 additions and 11751 deletions

View File

@ -0,0 +1,47 @@
import autobind from 'autobind-decorator';
import Chart, { Obj, DeepPartial } from '../../core';
import { User } from '@/models/entities/user';
import { SchemaType } from '@/misc/schema';
import { Users } from '@/models/index';
import { name, schema } from '../schemas/active-users';
type ActiveUsersLog = SchemaType<typeof schema>;
export default class ActiveUsersChart extends Chart<ActiveUsersLog> {
constructor() {
super(name, schema);
}
@autobind
protected genNewLog(latest: ActiveUsersLog): DeepPartial<ActiveUsersLog> {
return {};
}
@autobind
protected aggregate(logs: ActiveUsersLog[]): ActiveUsersLog {
return {
local: {
users: logs.reduce((a, b) => a.concat(b.local.users), [] as ActiveUsersLog['local']['users']),
},
remote: {
users: logs.reduce((a, b) => a.concat(b.remote.users), [] as ActiveUsersLog['remote']['users']),
},
};
}
@autobind
protected async fetchActual(): Promise<DeepPartial<ActiveUsersLog>> {
return {};
}
@autobind
public async update(user: { id: User['id'], host: User['host'] }) {
const update: Obj = {
users: [user.id]
};
await this.inc({
[Users.isLocalUser(user) ? 'local' : 'remote']: update
});
}
}

View File

@ -0,0 +1,91 @@
import autobind from 'autobind-decorator';
import Chart, { Obj, DeepPartial } from '../../core';
import { SchemaType } from '@/misc/schema';
import { DriveFiles } from '@/models/index';
import { Not, IsNull } from 'typeorm';
import { DriveFile } from '@/models/entities/drive-file';
import { name, schema } from '../schemas/drive';
type DriveLog = SchemaType<typeof schema>;
export default class DriveChart extends Chart<DriveLog> {
constructor() {
super(name, schema);
}
@autobind
protected genNewLog(latest: DriveLog): DeepPartial<DriveLog> {
return {
local: {
totalCount: latest.local.totalCount,
totalSize: latest.local.totalSize,
},
remote: {
totalCount: latest.remote.totalCount,
totalSize: latest.remote.totalSize,
}
};
}
@autobind
protected aggregate(logs: DriveLog[]): DriveLog {
return {
local: {
totalCount: logs[0].local.totalCount,
totalSize: logs[0].local.totalSize,
incCount: logs.reduce((a, b) => a + b.local.incCount, 0),
incSize: logs.reduce((a, b) => a + b.local.incSize, 0),
decCount: logs.reduce((a, b) => a + b.local.decCount, 0),
decSize: logs.reduce((a, b) => a + b.local.decSize, 0),
},
remote: {
totalCount: logs[0].remote.totalCount,
totalSize: logs[0].remote.totalSize,
incCount: logs.reduce((a, b) => a + b.remote.incCount, 0),
incSize: logs.reduce((a, b) => a + b.remote.incSize, 0),
decCount: logs.reduce((a, b) => a + b.remote.decCount, 0),
decSize: logs.reduce((a, b) => a + b.remote.decSize, 0),
},
};
}
@autobind
protected async fetchActual(): Promise<DeepPartial<DriveLog>> {
const [localCount, remoteCount, localSize, remoteSize] = await Promise.all([
DriveFiles.count({ userHost: null }),
DriveFiles.count({ userHost: Not(IsNull()) }),
DriveFiles.calcDriveUsageOfLocal(),
DriveFiles.calcDriveUsageOfRemote()
]);
return {
local: {
totalCount: localCount,
totalSize: localSize,
},
remote: {
totalCount: remoteCount,
totalSize: remoteSize,
}
};
}
@autobind
public async update(file: DriveFile, isAdditional: boolean) {
const update: Obj = {};
update.totalCount = isAdditional ? 1 : -1;
update.totalSize = isAdditional ? file.size : -file.size;
if (isAdditional) {
update.incCount = 1;
update.incSize = file.size;
} else {
update.decCount = 1;
update.decSize = file.size;
}
await this.inc({
[file.userHost === null ? 'local' : 'remote']: update
});
}
}

View File

@ -0,0 +1,62 @@
import autobind from 'autobind-decorator';
import Chart, { Obj, DeepPartial } from '../../core';
import { SchemaType } from '@/misc/schema';
import { Instances } from '@/models/index';
import { name, schema } from '../schemas/federation';
type FederationLog = SchemaType<typeof schema>;
export default class FederationChart extends Chart<FederationLog> {
constructor() {
super(name, schema);
}
@autobind
protected genNewLog(latest: FederationLog): DeepPartial<FederationLog> {
return {
instance: {
total: latest.instance.total,
}
};
}
@autobind
protected aggregate(logs: FederationLog[]): FederationLog {
return {
instance: {
total: logs[0].instance.total,
inc: logs.reduce((a, b) => a + b.instance.inc, 0),
dec: logs.reduce((a, b) => a + b.instance.dec, 0),
},
};
}
@autobind
protected async fetchActual(): Promise<DeepPartial<FederationLog>> {
const [total] = await Promise.all([
Instances.count({})
]);
return {
instance: {
total: total,
}
};
}
@autobind
public async update(isAdditional: boolean) {
const update: Obj = {};
update.total = isAdditional ? 1 : -1;
if (isAdditional) {
update.inc = 1;
} else {
update.dec = 1;
}
await this.inc({
instance: update
});
}
}

View File

@ -0,0 +1,47 @@
import autobind from 'autobind-decorator';
import Chart, { Obj, DeepPartial } from '../../core';
import { User } from '@/models/entities/user';
import { SchemaType } from '@/misc/schema';
import { Users } from '@/models/index';
import { name, schema } from '../schemas/hashtag';
type HashtagLog = SchemaType<typeof schema>;
export default class HashtagChart extends Chart<HashtagLog> {
constructor() {
super(name, schema, true);
}
@autobind
protected genNewLog(latest: HashtagLog): DeepPartial<HashtagLog> {
return {};
}
@autobind
protected aggregate(logs: HashtagLog[]): HashtagLog {
return {
local: {
users: logs.reduce((a, b) => a.concat(b.local.users), [] as HashtagLog['local']['users']),
},
remote: {
users: logs.reduce((a, b) => a.concat(b.remote.users), [] as HashtagLog['remote']['users']),
},
};
}
@autobind
protected async fetchActual(): Promise<DeepPartial<HashtagLog>> {
return {};
}
@autobind
public async update(hashtag: string, user: { id: User['id'], host: User['host'] }) {
const update: Obj = {
users: [user.id]
};
await this.inc({
[Users.isLocalUser(user) ? 'local' : 'remote']: update
}, hashtag);
}
}

View File

@ -0,0 +1,217 @@
import autobind from 'autobind-decorator';
import Chart, { Obj, DeepPartial } from '../../core';
import { SchemaType } from '@/misc/schema';
import { DriveFiles, Followings, Users, Notes } from '@/models/index';
import { DriveFile } from '@/models/entities/drive-file';
import { name, schema } from '../schemas/instance';
import { Note } from '@/models/entities/note';
import { toPuny } from '@/misc/convert-host';
type InstanceLog = SchemaType<typeof schema>;
export default class InstanceChart extends Chart<InstanceLog> {
constructor() {
super(name, schema);
}
@autobind
protected genNewLog(latest: InstanceLog): DeepPartial<InstanceLog> {
return {
notes: {
total: latest.notes.total,
},
users: {
total: latest.users.total,
},
following: {
total: latest.following.total,
},
followers: {
total: latest.followers.total,
},
drive: {
totalFiles: latest.drive.totalFiles,
totalUsage: latest.drive.totalUsage,
}
};
}
@autobind
protected aggregate(logs: InstanceLog[]): InstanceLog {
return {
requests: {
failed: logs.reduce((a, b) => a + b.requests.failed, 0),
succeeded: logs.reduce((a, b) => a + b.requests.succeeded, 0),
received: logs.reduce((a, b) => a + b.requests.received, 0),
},
notes: {
total: logs[0].notes.total,
inc: logs.reduce((a, b) => a + b.notes.inc, 0),
dec: logs.reduce((a, b) => a + b.notes.dec, 0),
diffs: {
reply: logs.reduce((a, b) => a + b.notes.diffs.reply, 0),
renote: logs.reduce((a, b) => a + b.notes.diffs.renote, 0),
normal: logs.reduce((a, b) => a + b.notes.diffs.normal, 0),
},
},
users: {
total: logs[0].users.total,
inc: logs.reduce((a, b) => a + b.users.inc, 0),
dec: logs.reduce((a, b) => a + b.users.dec, 0),
},
following: {
total: logs[0].following.total,
inc: logs.reduce((a, b) => a + b.following.inc, 0),
dec: logs.reduce((a, b) => a + b.following.dec, 0),
},
followers: {
total: logs[0].followers.total,
inc: logs.reduce((a, b) => a + b.followers.inc, 0),
dec: logs.reduce((a, b) => a + b.followers.dec, 0),
},
drive: {
totalFiles: logs[0].drive.totalFiles,
totalUsage: logs[0].drive.totalUsage,
incFiles: logs.reduce((a, b) => a + b.drive.incFiles, 0),
incUsage: logs.reduce((a, b) => a + b.drive.incUsage, 0),
decFiles: logs.reduce((a, b) => a + b.drive.decFiles, 0),
decUsage: logs.reduce((a, b) => a + b.drive.decUsage, 0),
},
};
}
@autobind
protected async fetchActual(group: string): Promise<DeepPartial<InstanceLog>> {
const [
notesCount,
usersCount,
followingCount,
followersCount,
driveFiles,
driveUsage,
] = await Promise.all([
Notes.count({ userHost: group }),
Users.count({ host: group }),
Followings.count({ followerHost: group }),
Followings.count({ followeeHost: group }),
DriveFiles.count({ userHost: group }),
DriveFiles.calcDriveUsageOfHost(group),
]);
return {
notes: {
total: notesCount,
},
users: {
total: usersCount,
},
following: {
total: followingCount,
},
followers: {
total: followersCount,
},
drive: {
totalFiles: driveFiles,
totalUsage: driveUsage,
}
};
}
@autobind
public async requestReceived(host: string) {
await this.inc({
requests: {
received: 1
}
}, toPuny(host));
}
@autobind
public async requestSent(host: string, isSucceeded: boolean) {
const update: Obj = {};
if (isSucceeded) {
update.succeeded = 1;
} else {
update.failed = 1;
}
await this.inc({
requests: update
}, toPuny(host));
}
@autobind
public async newUser(host: string) {
await this.inc({
users: {
total: 1,
inc: 1
}
}, toPuny(host));
}
@autobind
public async updateNote(host: string, note: Note, isAdditional: boolean) {
const diffs = {} as any;
if (note.replyId != null) {
diffs.reply = isAdditional ? 1 : -1;
} else if (note.renoteId != null) {
diffs.renote = isAdditional ? 1 : -1;
} else {
diffs.normal = isAdditional ? 1 : -1;
}
await this.inc({
notes: {
total: isAdditional ? 1 : -1,
inc: isAdditional ? 1 : 0,
dec: isAdditional ? 0 : 1,
diffs: diffs
}
}, toPuny(host));
}
@autobind
public async updateFollowing(host: string, isAdditional: boolean) {
await this.inc({
following: {
total: isAdditional ? 1 : -1,
inc: isAdditional ? 1 : 0,
dec: isAdditional ? 0 : 1,
}
}, toPuny(host));
}
@autobind
public async updateFollowers(host: string, isAdditional: boolean) {
await this.inc({
followers: {
total: isAdditional ? 1 : -1,
inc: isAdditional ? 1 : 0,
dec: isAdditional ? 0 : 1,
}
}, toPuny(host));
}
@autobind
public async updateDrive(file: DriveFile, isAdditional: boolean) {
const update: Obj = {};
update.totalFiles = isAdditional ? 1 : -1;
update.totalUsage = isAdditional ? file.size : -file.size;
if (isAdditional) {
update.incFiles = 1;
update.incUsage = file.size;
} else {
update.decFiles = 1;
update.decUsage = file.size;
}
await this.inc({
drive: update
}, file.userHost);
}
}

View File

@ -0,0 +1,45 @@
import autobind from 'autobind-decorator';
import Chart, { DeepPartial } from '../../core';
import { SchemaType } from '@/misc/schema';
import { name, schema } from '../schemas/network';
type NetworkLog = SchemaType<typeof schema>;
export default class NetworkChart extends Chart<NetworkLog> {
constructor() {
super(name, schema);
}
@autobind
protected genNewLog(latest: NetworkLog): DeepPartial<NetworkLog> {
return {};
}
@autobind
protected aggregate(logs: NetworkLog[]): NetworkLog {
return {
incomingRequests: logs.reduce((a, b) => a + b.incomingRequests, 0),
outgoingRequests: logs.reduce((a, b) => a + b.outgoingRequests, 0),
totalTime: logs.reduce((a, b) => a + b.totalTime, 0),
incomingBytes: logs.reduce((a, b) => a + b.incomingBytes, 0),
outgoingBytes: logs.reduce((a, b) => a + b.outgoingBytes, 0),
};
}
@autobind
protected async fetchActual(): Promise<DeepPartial<NetworkLog>> {
return {};
}
@autobind
public async update(incomingRequests: number, time: number, incomingBytes: number, outgoingBytes: number) {
const inc: DeepPartial<NetworkLog> = {
incomingRequests: incomingRequests,
totalTime: time,
incomingBytes: incomingBytes,
outgoingBytes: outgoingBytes
};
await this.inc(inc);
}
}

View File

@ -0,0 +1,97 @@
import autobind from 'autobind-decorator';
import Chart, { Obj, DeepPartial } from '../../core';
import { SchemaType } from '@/misc/schema';
import { Notes } from '@/models/index';
import { Not, IsNull } from 'typeorm';
import { Note } from '@/models/entities/note';
import { name, schema } from '../schemas/notes';
type NotesLog = SchemaType<typeof schema>;
export default class NotesChart extends Chart<NotesLog> {
constructor() {
super(name, schema);
}
@autobind
protected genNewLog(latest: NotesLog): DeepPartial<NotesLog> {
return {
local: {
total: latest.local.total,
},
remote: {
total: latest.remote.total,
}
};
}
@autobind
protected aggregate(logs: NotesLog[]): NotesLog {
return {
local: {
total: logs[0].local.total,
inc: logs.reduce((a, b) => a + b.local.inc, 0),
dec: logs.reduce((a, b) => a + b.local.dec, 0),
diffs: {
reply: logs.reduce((a, b) => a + b.local.diffs.reply, 0),
renote: logs.reduce((a, b) => a + b.local.diffs.renote, 0),
normal: logs.reduce((a, b) => a + b.local.diffs.normal, 0),
},
},
remote: {
total: logs[0].remote.total,
inc: logs.reduce((a, b) => a + b.remote.inc, 0),
dec: logs.reduce((a, b) => a + b.remote.dec, 0),
diffs: {
reply: logs.reduce((a, b) => a + b.remote.diffs.reply, 0),
renote: logs.reduce((a, b) => a + b.remote.diffs.renote, 0),
normal: logs.reduce((a, b) => a + b.remote.diffs.normal, 0),
},
},
};
}
@autobind
protected async fetchActual(): Promise<DeepPartial<NotesLog>> {
const [localCount, remoteCount] = await Promise.all([
Notes.count({ userHost: null }),
Notes.count({ userHost: Not(IsNull()) })
]);
return {
local: {
total: localCount,
},
remote: {
total: remoteCount,
}
};
}
@autobind
public async update(note: Note, isAdditional: boolean) {
const update: Obj = {
diffs: {}
};
update.total = isAdditional ? 1 : -1;
if (isAdditional) {
update.inc = 1;
} else {
update.dec = 1;
}
if (note.replyId != null) {
update.diffs.reply = isAdditional ? 1 : -1;
} else if (note.renoteId != null) {
update.diffs.renote = isAdditional ? 1 : -1;
} else {
update.diffs.normal = isAdditional ? 1 : -1;
}
await this.inc({
[note.userHost === null ? 'local' : 'remote']: update
});
}
}

View File

@ -0,0 +1,64 @@
import autobind from 'autobind-decorator';
import Chart, { Obj, DeepPartial } from '../../core';
import { SchemaType } from '@/misc/schema';
import { DriveFiles } from '@/models/index';
import { DriveFile } from '@/models/entities/drive-file';
import { name, schema } from '../schemas/per-user-drive';
type PerUserDriveLog = SchemaType<typeof schema>;
export default class PerUserDriveChart extends Chart<PerUserDriveLog> {
constructor() {
super(name, schema, true);
}
@autobind
protected genNewLog(latest: PerUserDriveLog): DeepPartial<PerUserDriveLog> {
return {
totalCount: latest.totalCount,
totalSize: latest.totalSize,
};
}
@autobind
protected aggregate(logs: PerUserDriveLog[]): PerUserDriveLog {
return {
totalCount: logs[0].totalCount,
totalSize: logs[0].totalSize,
incCount: logs.reduce((a, b) => a + b.incCount, 0),
incSize: logs.reduce((a, b) => a + b.incSize, 0),
decCount: logs.reduce((a, b) => a + b.decCount, 0),
decSize: logs.reduce((a, b) => a + b.decSize, 0),
};
}
@autobind
protected async fetchActual(group: string): Promise<DeepPartial<PerUserDriveLog>> {
const [count, size] = await Promise.all([
DriveFiles.count({ userId: group }),
DriveFiles.calcDriveUsageOf(group)
]);
return {
totalCount: count,
totalSize: size,
};
}
@autobind
public async update(file: DriveFile, isAdditional: boolean) {
const update: Obj = {};
update.totalCount = isAdditional ? 1 : -1;
update.totalSize = isAdditional ? file.size : -file.size;
if (isAdditional) {
update.incCount = 1;
update.incSize = file.size;
} else {
update.decCount = 1;
update.decSize = file.size;
}
await this.inc(update, file.userId);
}
}

View File

@ -0,0 +1,121 @@
import autobind from 'autobind-decorator';
import Chart, { Obj, DeepPartial } from '../../core';
import { SchemaType } from '@/misc/schema';
import { Followings, Users } from '@/models/index';
import { Not, IsNull } from 'typeorm';
import { User } from '@/models/entities/user';
import { name, schema } from '../schemas/per-user-following';
type PerUserFollowingLog = SchemaType<typeof schema>;
export default class PerUserFollowingChart extends Chart<PerUserFollowingLog> {
constructor() {
super(name, schema, true);
}
@autobind
protected genNewLog(latest: PerUserFollowingLog): DeepPartial<PerUserFollowingLog> {
return {
local: {
followings: {
total: latest.local.followings.total,
},
followers: {
total: latest.local.followers.total,
}
},
remote: {
followings: {
total: latest.remote.followings.total,
},
followers: {
total: latest.remote.followers.total,
}
}
};
}
@autobind
protected aggregate(logs: PerUserFollowingLog[]): PerUserFollowingLog {
return {
local: {
followings: {
total: logs[0].local.followings.total,
inc: logs.reduce((a, b) => a + b.local.followings.inc, 0),
dec: logs.reduce((a, b) => a + b.local.followings.dec, 0),
},
followers: {
total: logs[0].local.followers.total,
inc: logs.reduce((a, b) => a + b.local.followers.inc, 0),
dec: logs.reduce((a, b) => a + b.local.followers.dec, 0),
},
},
remote: {
followings: {
total: logs[0].remote.followings.total,
inc: logs.reduce((a, b) => a + b.remote.followings.inc, 0),
dec: logs.reduce((a, b) => a + b.remote.followings.dec, 0),
},
followers: {
total: logs[0].remote.followers.total,
inc: logs.reduce((a, b) => a + b.remote.followers.inc, 0),
dec: logs.reduce((a, b) => a + b.remote.followers.dec, 0),
},
},
};
}
@autobind
protected async fetchActual(group: string): Promise<DeepPartial<PerUserFollowingLog>> {
const [
localFollowingsCount,
localFollowersCount,
remoteFollowingsCount,
remoteFollowersCount
] = await Promise.all([
Followings.count({ followerId: group, followeeHost: null }),
Followings.count({ followeeId: group, followerHost: null }),
Followings.count({ followerId: group, followeeHost: Not(IsNull()) }),
Followings.count({ followeeId: group, followerHost: Not(IsNull()) })
]);
return {
local: {
followings: {
total: localFollowingsCount,
},
followers: {
total: localFollowersCount,
}
},
remote: {
followings: {
total: remoteFollowingsCount,
},
followers: {
total: remoteFollowersCount,
}
}
};
}
@autobind
public async update(follower: { id: User['id']; host: User['host']; }, followee: { id: User['id']; host: User['host']; }, isFollow: boolean) {
const update: Obj = {};
update.total = isFollow ? 1 : -1;
if (isFollow) {
update.inc = 1;
} else {
update.dec = 1;
}
this.inc({
[Users.isLocalUser(follower) ? 'local' : 'remote']: { followings: update }
}, follower.id);
this.inc({
[Users.isLocalUser(followee) ? 'local' : 'remote']: { followers: update }
}, followee.id);
}
}

View File

@ -0,0 +1,72 @@
import autobind from 'autobind-decorator';
import Chart, { Obj, DeepPartial } from '../../core';
import { User } from '@/models/entities/user';
import { SchemaType } from '@/misc/schema';
import { Notes } from '@/models/index';
import { Note } from '@/models/entities/note';
import { name, schema } from '../schemas/per-user-notes';
type PerUserNotesLog = SchemaType<typeof schema>;
export default class PerUserNotesChart extends Chart<PerUserNotesLog> {
constructor() {
super(name, schema, true);
}
@autobind
protected genNewLog(latest: PerUserNotesLog): DeepPartial<PerUserNotesLog> {
return {
total: latest.total,
};
}
@autobind
protected aggregate(logs: PerUserNotesLog[]): PerUserNotesLog {
return {
total: logs[0].total,
inc: logs.reduce((a, b) => a + b.inc, 0),
dec: logs.reduce((a, b) => a + b.dec, 0),
diffs: {
reply: logs.reduce((a, b) => a + b.diffs.reply, 0),
renote: logs.reduce((a, b) => a + b.diffs.renote, 0),
normal: logs.reduce((a, b) => a + b.diffs.normal, 0),
},
};
}
@autobind
protected async fetchActual(group: string): Promise<DeepPartial<PerUserNotesLog>> {
const [count] = await Promise.all([
Notes.count({ userId: group }),
]);
return {
total: count,
};
}
@autobind
public async update(user: { id: User['id'] }, note: Note, isAdditional: boolean) {
const update: Obj = {
diffs: {}
};
update.total = isAdditional ? 1 : -1;
if (isAdditional) {
update.inc = 1;
} else {
update.dec = 1;
}
if (note.replyId != null) {
update.diffs.reply = isAdditional ? 1 : -1;
} else if (note.renoteId != null) {
update.diffs.renote = isAdditional ? 1 : -1;
} else {
update.diffs.normal = isAdditional ? 1 : -1;
}
await this.inc(update, user.id);
}
}

View File

@ -0,0 +1,44 @@
import autobind from 'autobind-decorator';
import Chart, { DeepPartial } from '../../core';
import { User } from '@/models/entities/user';
import { Note } from '@/models/entities/note';
import { SchemaType } from '@/misc/schema';
import { Users } from '@/models/index';
import { name, schema } from '../schemas/per-user-reactions';
type PerUserReactionsLog = SchemaType<typeof schema>;
export default class PerUserReactionsChart extends Chart<PerUserReactionsLog> {
constructor() {
super(name, schema, true);
}
@autobind
protected genNewLog(latest: PerUserReactionsLog): DeepPartial<PerUserReactionsLog> {
return {};
}
@autobind
protected aggregate(logs: PerUserReactionsLog[]): PerUserReactionsLog {
return {
local: {
count: logs.reduce((a, b) => a + b.local.count, 0),
},
remote: {
count: logs.reduce((a, b) => a + b.remote.count, 0),
},
};
}
@autobind
protected async fetchActual(group: string): Promise<DeepPartial<PerUserReactionsLog>> {
return {};
}
@autobind
public async update(user: { id: User['id'], host: User['host'] }, note: Note) {
this.inc({
[Users.isLocalUser(user) ? 'local' : 'remote']: { count: 1 }
}, note.userId);
}
}

View File

@ -0,0 +1,58 @@
import autobind from 'autobind-decorator';
import Chart, { Obj, DeepPartial } from '../../core';
import { SchemaType } from '@/misc/schema';
import { name, schema } from '../schemas/test-grouped';
type TestGroupedLog = SchemaType<typeof schema>;
export default class TestGroupedChart extends Chart<TestGroupedLog> {
private total = {} as Record<string, number>;
constructor() {
super(name, schema, true);
}
@autobind
protected genNewLog(latest: TestGroupedLog): DeepPartial<TestGroupedLog> {
return {
foo: {
total: latest.foo.total,
},
};
}
@autobind
protected aggregate(logs: TestGroupedLog[]): TestGroupedLog {
return {
foo: {
total: logs[0].foo.total,
inc: logs.reduce((a, b) => a + b.foo.inc, 0),
dec: logs.reduce((a, b) => a + b.foo.dec, 0),
},
};
}
@autobind
protected async fetchActual(group: string): Promise<DeepPartial<TestGroupedLog>> {
return {
foo: {
total: this.total[group],
},
};
}
@autobind
public async increment(group: string) {
if (this.total[group] == null) this.total[group] = 0;
const update: Obj = {};
update.total = 1;
update.inc = 1;
this.total[group]++;
await this.inc({
foo: update
}, group);
}
}

View File

@ -0,0 +1,36 @@
import autobind from 'autobind-decorator';
import Chart, { DeepPartial } from '../../core';
import { SchemaType } from '@/misc/schema';
import { name, schema } from '../schemas/test-unique';
type TestUniqueLog = SchemaType<typeof schema>;
export default class TestUniqueChart extends Chart<TestUniqueLog> {
constructor() {
super(name, schema);
}
@autobind
protected genNewLog(latest: TestUniqueLog): DeepPartial<TestUniqueLog> {
return {};
}
@autobind
protected aggregate(logs: TestUniqueLog[]): TestUniqueLog {
return {
foo: logs.reduce((a, b) => a.concat(b.foo), [] as TestUniqueLog['foo']),
};
}
@autobind
protected async fetchActual(): Promise<DeepPartial<TestUniqueLog>> {
return {};
}
@autobind
public async uniqueIncrement(key: string) {
await this.inc({
foo: [key]
});
}
}

View File

@ -0,0 +1,69 @@
import autobind from 'autobind-decorator';
import Chart, { Obj, DeepPartial } from '../../core';
import { SchemaType } from '@/misc/schema';
import { name, schema } from '../schemas/test';
type TestLog = SchemaType<typeof schema>;
export default class TestChart extends Chart<TestLog> {
public total = 0; // publicにするのはテストのため
constructor() {
super(name, schema);
}
@autobind
protected genNewLog(latest: TestLog): DeepPartial<TestLog> {
return {
foo: {
total: latest.foo.total,
},
};
}
@autobind
protected aggregate(logs: TestLog[]): TestLog {
return {
foo: {
total: logs[0].foo.total,
inc: logs.reduce((a, b) => a + b.foo.inc, 0),
dec: logs.reduce((a, b) => a + b.foo.dec, 0),
},
};
}
@autobind
protected async fetchActual(): Promise<DeepPartial<TestLog>> {
return {
foo: {
total: this.total,
},
};
}
@autobind
public async increment() {
const update: Obj = {};
update.total = 1;
update.inc = 1;
this.total++;
await this.inc({
foo: update
});
}
@autobind
public async decrement() {
const update: Obj = {};
update.total = -1;
update.dec = 1;
this.total--;
await this.inc({
foo: update
});
}
}

View File

@ -0,0 +1,76 @@
import autobind from 'autobind-decorator';
import Chart, { Obj, DeepPartial } from '../../core';
import { SchemaType } from '@/misc/schema';
import { Users } from '@/models/index';
import { Not, IsNull } from 'typeorm';
import { User } from '@/models/entities/user';
import { name, schema } from '../schemas/users';
type UsersLog = SchemaType<typeof schema>;
export default class UsersChart extends Chart<UsersLog> {
constructor() {
super(name, schema);
}
@autobind
protected genNewLog(latest: UsersLog): DeepPartial<UsersLog> {
return {
local: {
total: latest.local.total,
},
remote: {
total: latest.remote.total,
}
};
}
@autobind
protected aggregate(logs: UsersLog[]): UsersLog {
return {
local: {
total: logs[0].local.total,
inc: logs.reduce((a, b) => a + b.local.inc, 0),
dec: logs.reduce((a, b) => a + b.local.dec, 0),
},
remote: {
total: logs[0].remote.total,
inc: logs.reduce((a, b) => a + b.remote.inc, 0),
dec: logs.reduce((a, b) => a + b.remote.dec, 0),
},
};
}
@autobind
protected async fetchActual(): Promise<DeepPartial<UsersLog>> {
const [localCount, remoteCount] = await Promise.all([
Users.count({ host: null }),
Users.count({ host: Not(IsNull()) })
]);
return {
local: {
total: localCount,
},
remote: {
total: remoteCount,
}
};
}
@autobind
public async update(user: { id: User['id'], host: User['host'] }, isAdditional: boolean) {
const update: Obj = {};
update.total = isAdditional ? 1 : -1;
if (isAdditional) {
update.inc = 1;
} else {
update.dec = 1;
}
await this.inc({
[Users.isLocalUser(user) ? 'local' : 'remote']: update
});
}
}

View File

@ -0,0 +1,35 @@
export const logSchema = {
/**
* アクティブユーザー
*/
users: {
type: 'array' as const,
optional: false as const, nullable: false as const,
items: {
type: 'string' as const,
optional: false as const, nullable: false as const,
}
},
};
/**
* アクティブユーザーに関するチャート
*/
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
local: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema
},
remote: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema
},
}
};
export const name = 'activeUsers';

View File

@ -0,0 +1,68 @@
const logSchema = {
/**
* 集計期間時点での、全ドライブファイル数
*/
totalCount: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* 集計期間時点での、全ドライブファイルの合計サイズ
*/
totalSize: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* 増加したドライブファイル数
*/
incCount: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* 増加したドライブ使用量
*/
incSize: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* 減少したドライブファイル数
*/
decCount: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* 減少したドライブ使用量
*/
decSize: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
};
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
local: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema
},
remote: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema
},
}
};
export const name = 'drive';

View File

@ -0,0 +1,29 @@
/**
* フェデレーションに関するチャート
*/
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
instance: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
total: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
inc: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
dec: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
}
}
}
};
export const name = 'federation';

View File

@ -0,0 +1,35 @@
export const logSchema = {
/**
* 投稿したユーザー
*/
users: {
type: 'array' as const,
optional: false as const, nullable: false as const,
items: {
type: 'string' as const,
optional: false as const, nullable: false as const,
}
},
};
/**
* ハッシュタグに関するチャート
*/
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
local: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema
},
remote: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema
},
}
};
export const name = 'hashtag';

View File

@ -0,0 +1,157 @@
/**
* インスタンスごとのチャート
*/
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
requests: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
failed: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
succeeded: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
received: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
}
},
notes: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
total: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
inc: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
dec: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
diffs: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
normal: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
reply: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
renote: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
}
},
}
},
users: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
total: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
inc: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
dec: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
}
},
following: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
total: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
inc: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
dec: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
}
},
followers: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
total: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
inc: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
dec: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
}
},
drive: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
totalFiles: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
totalUsage: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
incFiles: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
incUsage: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
decFiles: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
decUsage: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
}
},
}
};
export const name = 'instance';

View File

@ -0,0 +1,31 @@
/**
* ネットワークに関するチャート
*/
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
incomingRequests: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
outgoingRequests: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
totalTime: { // TIP: (totalTime / incomingRequests) でひとつのリクエストに平均でどれくらいの時間がかかったか知れる
type: 'number' as const,
optional: false as const, nullable: false as const,
},
incomingBytes: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
outgoingBytes: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
}
};
export const name = 'network';

View File

@ -0,0 +1,56 @@
const logSchema = {
total: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
inc: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
dec: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
diffs: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
normal: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
reply: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
renote: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
}
},
};
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
local: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema
},
remote: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema
},
}
};
export const name = 'notes';

View File

@ -0,0 +1,55 @@
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
/**
* 集計期間時点での、全ドライブファイル数
*/
totalCount: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* 集計期間時点での、全ドライブファイルの合計サイズ
*/
totalSize: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* 増加したドライブファイル数
*/
incCount: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* 増加したドライブ使用量
*/
incSize: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* 減少したドライブファイル数
*/
decCount: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* 減少したドライブ使用量
*/
decSize: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
}
};
export const name = 'perUserDrive';

View File

@ -0,0 +1,86 @@
export const logSchema = {
/**
* フォローしている
*/
followings: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
/**
* フォローしている合計
*/
total: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* フォローした数
*/
inc: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* フォロー解除した数
*/
dec: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
}
},
/**
* フォローされている
*/
followers: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
/**
* フォローされている合計
*/
total: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* フォローされた数
*/
inc: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* フォロー解除された数
*/
dec: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
}
},
};
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
local: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema
},
remote: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema
},
}
};
export const name = 'perUserFollowing';

View File

@ -0,0 +1,43 @@
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
total: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
inc: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
dec: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
diffs: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
normal: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
reply: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
renote: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
}
},
}
};
export const name = 'perUserNotes';

View File

@ -0,0 +1,31 @@
export const logSchema = {
/**
* フォローしている合計
*/
count: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
};
/**
* ユーザーごとのリアクションに関するチャート
*/
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
local: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema
},
remote: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema
},
}
};
export const name = 'perUserReaction';

View File

@ -0,0 +1,28 @@
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
foo: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
total: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
inc: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
dec: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
}
}
}
};
export const name = 'testGrouped';

View File

@ -0,0 +1,16 @@
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
foo: {
type: 'array' as const,
optional: false as const, nullable: false as const,
items: {
type: 'string' as const,
optional: false as const, nullable: false as const,
}
},
}
};
export const name = 'testUnique';

View File

@ -0,0 +1,28 @@
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
foo: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
total: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
inc: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
dec: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
}
}
}
};
export const name = 'test';

View File

@ -0,0 +1,44 @@
const logSchema = {
/**
* 集計期間時点での、全ユーザー数
*/
total: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* 増加したユーザー数
*/
inc: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* 減少したユーザー数
*/
dec: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
};
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
local: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema
},
remote: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema
},
}
};
export const name = 'users';

View File

@ -0,0 +1,563 @@
/**
* チャートエンジン
*
* Tests located in test/chart
*/
import * as nestedProperty from 'nested-property';
import autobind from 'autobind-decorator';
import Logger from '../logger';
import { SimpleSchema } from '@/misc/simple-schema';
import { EntitySchema, getRepository, Repository, LessThan, Between } from 'typeorm';
import { dateUTC, isTimeSame, isTimeBefore, subtractTime, addTime } from '@/prelude/time';
import { getChartInsertLock } from '@/misc/app-lock';
const logger = new Logger('chart', 'white', process.env.NODE_ENV !== 'test');
export type Obj = { [key: string]: any };
export type DeepPartial<T> = {
[P in keyof T]?: DeepPartial<T[P]>;
};
type ArrayValue<T> = {
[P in keyof T]: T[P] extends number ? T[P][] : ArrayValue<T[P]>;
};
type Log = {
id: number;
/**
* 集計のグループ
*/
group: string | null;
/**
* 集計日時のUnixタイムスタンプ(秒)
*/
date: number;
};
const camelToSnake = (str: string) => {
return str.replace(/([A-Z])/g, s => '_' + s.charAt(0).toLowerCase());
};
const removeDuplicates = (array: any[]) => Array.from(new Set(array));
/**
* 様々なチャートの管理を司るクラス
*/
export default abstract class Chart<T extends Record<string, any>> {
private static readonly columnPrefix = '___';
private static readonly columnDot = '_';
private name: string;
private buffer: {
diff: DeepPartial<T>;
group: string | null;
}[] = [];
public schema: SimpleSchema;
protected repository: Repository<Log>;
protected abstract genNewLog(latest: T): DeepPartial<T>;
/**
* @param logs 日時が新しい方が先頭
*/
protected abstract aggregate(logs: T[]): T;
protected abstract fetchActual(group: string | null): Promise<DeepPartial<T>>;
@autobind
private static convertSchemaToFlatColumnDefinitions(schema: SimpleSchema) {
const columns = {} as any;
const flatColumns = (x: Obj, path?: string) => {
for (const [k, v] of Object.entries(x)) {
const p = path ? `${path}${this.columnDot}${k}` : k;
if (v.type === 'object') {
flatColumns(v.properties, p);
} else if (v.type === 'number') {
columns[this.columnPrefix + p] = {
type: 'bigint',
};
} else if (v.type === 'array' && v.items.type === 'string') {
columns[this.columnPrefix + p] = {
type: 'varchar',
array: true,
};
}
}
};
flatColumns(schema.properties!);
return columns;
}
@autobind
private static convertFlattenColumnsToObject(x: Record<string, any>): Record<string, any> {
const obj = {} as any;
for (const k of Object.keys(x).filter(k => k.startsWith(Chart.columnPrefix))) {
// now k is ___x_y_z
const path = k.substr(Chart.columnPrefix.length).split(Chart.columnDot).join('.');
nestedProperty.set(obj, path, x[k]);
}
return obj;
}
@autobind
private static convertObjectToFlattenColumns(x: Record<string, any>) {
const columns = {} as Record<string, number | unknown[]>;
const flatten = (x: Obj, path?: string) => {
for (const [k, v] of Object.entries(x)) {
const p = path ? `${path}${this.columnDot}${k}` : k;
if (typeof v === 'object' && !Array.isArray(v)) {
flatten(v, p);
} else {
columns[this.columnPrefix + p] = v;
}
}
};
flatten(x);
return columns;
}
@autobind
private static countUniqueFields(x: Record<string, any>) {
const exec = (x: Obj) => {
const res = {} as Record<string, any>;
for (const [k, v] of Object.entries(x)) {
if (typeof v === 'object' && !Array.isArray(v)) {
res[k] = exec(v);
} else if (Array.isArray(v)) {
res[k] = Array.from(new Set(v)).length;
} else {
res[k] = v;
}
}
return res;
};
return exec(x);
}
@autobind
private static convertQuery(diff: Record<string, number | unknown[]>) {
const query: Record<string, Function> = {};
for (const [k, v] of Object.entries(diff)) {
if (typeof v === 'number') {
if (v > 0) query[k] = () => `"${k}" + ${v}`;
if (v < 0) query[k] = () => `"${k}" - ${Math.abs(v)}`;
} else if (Array.isArray(v)) {
// TODO: item が文字列以外の場合も対応
// TODO: item をSQLエスケープ
const items = v.map(item => `"${item}"`).join(',');
query[k] = () => `array_cat("${k}", '{${items}}'::varchar[])`;
}
}
return query;
}
@autobind
private static dateToTimestamp(x: Date): Log['date'] {
return Math.floor(x.getTime() / 1000);
}
@autobind
private static parseDate(date: Date): [number, number, number, number, number, number, number] {
const y = date.getUTCFullYear();
const m = date.getUTCMonth();
const d = date.getUTCDate();
const h = date.getUTCHours();
const _m = date.getUTCMinutes();
const _s = date.getUTCSeconds();
const _ms = date.getUTCMilliseconds();
return [y, m, d, h, _m, _s, _ms];
}
@autobind
private static getCurrentDate() {
return Chart.parseDate(new Date());
}
@autobind
public static schemaToEntity(name: string, schema: SimpleSchema): EntitySchema {
return new EntitySchema({
name: `__chart__${camelToSnake(name)}`,
columns: {
id: {
type: 'integer',
primary: true,
generated: true
},
date: {
type: 'integer',
},
group: {
type: 'varchar',
length: 128,
nullable: true
},
...Chart.convertSchemaToFlatColumnDefinitions(schema)
},
indices: [{
columns: ['date', 'group'],
unique: true,
}, { // groupにnullが含まれると↑のuniqueは機能しないので↓の部分インデックスでカバー
columns: ['date'],
unique: true,
where: '"group" IS NULL'
}]
});
}
constructor(name: string, schema: SimpleSchema, grouped = false) {
this.name = name;
this.schema = schema;
const entity = Chart.schemaToEntity(name, schema);
const keys = ['date'];
if (grouped) keys.push('group');
entity.options.uniques = [{
columns: keys
}];
this.repository = getRepository<Log>(entity);
}
@autobind
private getNewLog(latest: T | null): T {
const log = latest ? this.genNewLog(latest) : {};
const flatColumns = (x: Obj, path?: string) => {
for (const [k, v] of Object.entries(x)) {
const p = path ? `${path}.${k}` : k;
if (v.type === 'object') {
flatColumns(v.properties, p);
} else {
if (nestedProperty.get(log, p) == null) {
const emptyValue = v.type === 'number' ? 0 : [];
nestedProperty.set(log, p, emptyValue);
}
}
}
};
flatColumns(this.schema.properties!);
return log as T;
}
@autobind
private getLatestLog(group: string | null = null): Promise<Log | null> {
return this.repository.findOne({
group: group,
}, {
order: {
date: -1
}
}).then(x => x || null);
}
@autobind
private async getCurrentLog(group: string | null = null): Promise<Log> {
const [y, m, d, h] = Chart.getCurrentDate();
const current = dateUTC([y, m, d, h]);
// 現在(=今のHour)のログ
const currentLog = await this.repository.findOne({
date: Chart.dateToTimestamp(current),
...(group ? { group: group } : {})
});
// ログがあればそれを返して終了
if (currentLog != null) {
return currentLog;
}
let log: Log;
let data: T;
// 集計期間が変わってから、初めてのチャート更新なら
// 最も最近のログを持ってくる
// * 例えば集計期間が「日」である場合で考えると、
// * 昨日何もチャートを更新するような出来事がなかった場合は、
// * ログがそもそも作られずドキュメントが存在しないということがあり得るため、
// * 「昨日の」と決め打ちせずに「もっとも最近の」とします
const latest = await this.getLatestLog(group);
if (latest != null) {
const obj = Chart.convertFlattenColumnsToObject(latest) as T;
// 空ログデータを作成
data = this.getNewLog(obj);
} else {
// ログが存在しなかったら
// (Misskeyインスタンスを建てて初めてのチャート更新時など)
// 初期ログデータを作成
data = this.getNewLog(null);
logger.info(`${this.name + (group ? `:${group}` : '')}: Initial commit created`);
}
const date = Chart.dateToTimestamp(current);
const lockKey = `${this.name}:${date}:${group}`;
const unlock = await getChartInsertLock(lockKey);
try {
// ロック内でもう1回チェックする
const currentLog = await this.repository.findOne({
date: date,
...(group ? { group: group } : {})
});
// ログがあればそれを返して終了
if (currentLog != null) return currentLog;
// 新規ログ挿入
log = await this.repository.insert({
group: group,
date: date,
...Chart.convertObjectToFlattenColumns(data)
}).then(x => this.repository.findOneOrFail(x.identifiers[0]));
logger.info(`${this.name + (group ? `:${group}` : '')}: New commit created`);
return log;
} finally {
unlock();
}
}
@autobind
protected commit(diff: DeepPartial<T>, group: string | null = null): void {
this.buffer.push({
diff, group,
});
}
@autobind
public async save() {
if (this.buffer.length === 0) {
logger.info(`${this.name}: Write skipped`);
return;
}
// TODO: 前の時間のログがbufferにあった場合のハンドリング
// 例えば、save が20分ごとに行われるとして、前回行われたのは 01:50 だったとする。
// 次に save が行われるのは 02:10 ということになるが、もし 01:55 に新規ログが buffer に追加されたとすると、
// そのログは本来は 01:00~ のログとしてDBに保存されて欲しいのに、02:00~ のログ扱いになってしまう。
// これを回避するための実装は複雑になりそうなため、一旦保留。
const update = async (log: Log) => {
const finalDiffs = {} as Record<string, number | unknown[]>;
for (const diff of this.buffer.filter(q => q.group === log.group).map(q => q.diff)) {
const columns = Chart.convertObjectToFlattenColumns(diff);
for (const [k, v] of Object.entries(columns)) {
if (finalDiffs[k] == null) {
finalDiffs[k] = v;
} else {
if (typeof finalDiffs[k] === 'number') {
(finalDiffs[k] as number) += v as number;
} else {
(finalDiffs[k] as unknown[]) = (finalDiffs[k] as unknown[]).concat(v);
}
}
}
}
const query = Chart.convertQuery(finalDiffs);
// ログ更新
await this.repository.createQueryBuilder()
.update()
.set(query)
.where('id = :id', { id: log.id })
.execute();
logger.info(`${this.name + (log.group ? `:${log.group}` : '')}: Updated`);
// TODO: この一連の処理が始まった後に新たにbufferに入ったものは消さないようにする
this.buffer = this.buffer.filter(q => q.group !== log.group);
};
const groups = removeDuplicates(this.buffer.map(log => log.group));
await Promise.all(groups.map(group => this.getCurrentLog(group).then(log => update(log))));
}
@autobind
public async resync(group: string | null = null): Promise<any> {
const data = await this.fetchActual(group);
const update = async (log: Log) => {
await this.repository.createQueryBuilder()
.update()
.set(Chart.convertObjectToFlattenColumns(data))
.where('id = :id', { id: log.id })
.execute();
};
return this.getCurrentLog(group).then(log => update(log));
}
@autobind
protected async inc(inc: DeepPartial<T>, group: string | null = null): Promise<void> {
await this.commit(inc, group);
}
@autobind
public async getChart(span: 'hour' | 'day', amount: number, cursor: Date | null, group: string | null = null): Promise<ArrayValue<T>> {
const [y, m, d, h, _m, _s, _ms] = cursor ? Chart.parseDate(subtractTime(addTime(cursor, 1, span), 1)) : Chart.getCurrentDate();
const [y2, m2, d2, h2] = cursor ? Chart.parseDate(addTime(cursor, 1, span)) : [] as never;
const lt = dateUTC([y, m, d, h, _m, _s, _ms]);
const gt =
span === 'day' ? subtractTime(cursor ? dateUTC([y2, m2, d2, 0]) : dateUTC([y, m, d, 0]), amount - 1, 'day') :
span === 'hour' ? subtractTime(cursor ? dateUTC([y2, m2, d2, h2]) : dateUTC([y, m, d, h]), amount - 1, 'hour') :
null as never;
// ログ取得
let logs = await this.repository.find({
where: {
group: group,
date: Between(Chart.dateToTimestamp(gt), Chart.dateToTimestamp(lt))
},
order: {
date: -1
},
});
// 要求された範囲にログがひとつもなかったら
if (logs.length === 0) {
// もっとも新しいログを持ってくる
// (すくなくともひとつログが無いと隙間埋めできないため)
const recentLog = await this.repository.findOne({
group: group,
}, {
order: {
date: -1
},
});
if (recentLog) {
logs = [recentLog];
}
// 要求された範囲の最も古い箇所に位置するログが存在しなかったら
} else if (!isTimeSame(new Date(logs[logs.length - 1].date * 1000), gt)) {
// 要求された範囲の最も古い箇所時点での最も新しいログを持ってきて末尾に追加する
// (隙間埋めできないため)
const outdatedLog = await this.repository.findOne({
group: group,
date: LessThan(Chart.dateToTimestamp(gt))
}, {
order: {
date: -1
},
});
if (outdatedLog) {
logs.push(outdatedLog);
}
}
const chart: T[] = [];
if (span === 'hour') {
for (let i = (amount - 1); i >= 0; i--) {
const current = subtractTime(dateUTC([y, m, d, h]), i, 'hour');
const log = logs.find(l => isTimeSame(new Date(l.date * 1000), current));
if (log) {
const data = Chart.convertFlattenColumnsToObject(log);
chart.unshift(Chart.countUniqueFields(data) as T);
} else {
// 隙間埋め
const latest = logs.find(l => isTimeBefore(new Date(l.date * 1000), current));
const data = latest ? Chart.convertFlattenColumnsToObject(latest) as T : null;
chart.unshift(Chart.countUniqueFields(this.getNewLog(data)) as T);
}
}
} else if (span === 'day') {
const logsForEachDays: T[][] = [];
let currentDay = -1;
let currentDayIndex = -1;
for (let i = ((amount - 1) * 24) + h; i >= 0; i--) {
const current = subtractTime(dateUTC([y, m, d, h]), i, 'hour');
const _currentDay = Chart.parseDate(current)[2];
if (currentDay != _currentDay) currentDayIndex++;
currentDay = _currentDay;
const log = logs.find(l => isTimeSame(new Date(l.date * 1000), current));
if (log) {
if (logsForEachDays[currentDayIndex]) {
logsForEachDays[currentDayIndex].unshift(Chart.convertFlattenColumnsToObject(log) as T);
} else {
logsForEachDays[currentDayIndex] = [Chart.convertFlattenColumnsToObject(log) as T];
}
} else {
// 隙間埋め
const latest = logs.find(l => isTimeBefore(new Date(l.date * 1000), current));
const data = latest ? Chart.convertFlattenColumnsToObject(latest) as T : null;
const newLog = this.getNewLog(data);
if (logsForEachDays[currentDayIndex]) {
logsForEachDays[currentDayIndex].unshift(newLog);
} else {
logsForEachDays[currentDayIndex] = [newLog];
}
}
}
for (const logs of logsForEachDays) {
const log = this.aggregate(logs);
chart.unshift(Chart.countUniqueFields(log) as T);
}
}
const res: ArrayValue<T> = {} as any;
/**
* [{ foo: 1, bar: 5 }, { foo: 2, bar: 6 }, { foo: 3, bar: 7 }]
* を
* { foo: [1, 2, 3], bar: [5, 6, 7] }
* にする
*/
const compact = (x: Obj, path?: string) => {
for (const [k, v] of Object.entries(x)) {
const p = path ? `${path}.${k}` : k;
if (typeof v === 'object' && !Array.isArray(v)) {
compact(v, p);
} else {
const values = chart.map(s => nestedProperty.get(s, p));
nestedProperty.set(res, p, values);
}
}
};
compact(chart[0]);
return res;
}
}
export function convertLog(logSchema: SimpleSchema): SimpleSchema {
const v: SimpleSchema = JSON.parse(JSON.stringify(logSchema)); // copy
if (v.type === 'number') {
v.type = 'array';
v.items = {
type: 'number' as const,
optional: false as const, nullable: false as const,
};
} else if (v.type === 'object') {
for (const k of Object.keys(v.properties!)) {
v.properties![k] = convertLog(v.properties![k]);
}
}
return v;
}

View File

@ -0,0 +1,15 @@
import { fileURLToPath } from 'url';
import { dirname } from 'path';
import Chart from './core';
//const _filename = fileURLToPath(import.meta.url);
const _filename = __filename;
const _dirname = dirname(_filename);
export const entities = Object.values(require('require-all')({
dirname: _dirname + '/charts/schemas',
filter: /^.+\.[jt]s$/,
resolve: (x: any) => {
return Chart.schemaToEntity(x.name, x.schema);
}
}));

View File

@ -0,0 +1,50 @@
import FederationChart from './charts/classes/federation';
import NotesChart from './charts/classes/notes';
import UsersChart from './charts/classes/users';
import NetworkChart from './charts/classes/network';
import ActiveUsersChart from './charts/classes/active-users';
import InstanceChart from './charts/classes/instance';
import PerUserNotesChart from './charts/classes/per-user-notes';
import DriveChart from './charts/classes/drive';
import PerUserReactionsChart from './charts/classes/per-user-reactions';
import HashtagChart from './charts/classes/hashtag';
import PerUserFollowingChart from './charts/classes/per-user-following';
import PerUserDriveChart from './charts/classes/per-user-drive';
import { beforeShutdown } from '@/misc/before-shutdown';
export const federationChart = new FederationChart();
export const notesChart = new NotesChart();
export const usersChart = new UsersChart();
export const networkChart = new NetworkChart();
export const activeUsersChart = new ActiveUsersChart();
export const instanceChart = new InstanceChart();
export const perUserNotesChart = new PerUserNotesChart();
export const driveChart = new DriveChart();
export const perUserReactionsChart = new PerUserReactionsChart();
export const hashtagChart = new HashtagChart();
export const perUserFollowingChart = new PerUserFollowingChart();
export const perUserDriveChart = new PerUserDriveChart();
const charts = [
federationChart,
notesChart,
usersChart,
networkChart,
activeUsersChart,
instanceChart,
perUserNotesChart,
driveChart,
perUserReactionsChart,
hashtagChart,
perUserFollowingChart,
perUserDriveChart,
];
// 20分おきにメモリ情報をDBに書き込み
setInterval(() => {
for (const chart of charts) {
chart.save();
}
}, 1000 * 60 * 20);
beforeShutdown(() => Promise.all(charts.map(chart => chart.save())));