sim1222/test1
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

refactor: temporary files (#8713)

Browse Source 
* simplify temporary files for thumbnails

Because only a single file will be written to the directory, creating a
separate directory seems unnecessary. If only a temporary file is created,
the code from `createTemp` can be reused here as well.

* refactor: deduplicate code for temporary files/directories

To follow the DRY principle, the same code should not be duplicated
across different files. Instead an already existing function is used.

Because temporary directories are also create in multiple locations,
a function for this is also newly added to reduce duplication.

* fix: clean up identicon temp files

The temporary files for identicons are not reused and can be deleted
after they are fully read. This condition is met when the stream is closed
and so the file can be cleaned up using the events API of the stream.

* fix: ensure cleanup is called when download fails

* fix: ensure cleanup is called in error conditions

This covers import/export queue jobs and is mostly just wrapping all
code in a try...finally statement where the finally runs the cleanup.

* fix: use correct type instead of `any`
This commit is contained in:
Johann150
2022-05-25 09:50:22 +02:00
committed by GitHub
parent b049633db7
commit e27c6abaea
12 changed files with 307 additions and 341 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
packages/backend/src/queue/processors/db/export-following.ts
View File

@ -1,11 +1,11 @@
import Bull from 'bull';
import * as tmp from 'tmp';
import * as fs from 'node:fs';
import { queueLogger } from '../../logger.js';
import { addFile } from '@/services/drive/add-file.js';
import { format as dateFormat } from 'date-fns';
import { getFullApAccount } from '@/misc/convert-host.js';
import { createTemp } from '@/misc/create-temp.js';
import { Users, Followings, Mutings } from '@/models/index.js';
import { In, MoreThan, Not } from 'typeorm';
import { DbUserJobData } from '@/queue/types.js';
@ -23,73 +23,72 @@ export async function exportFollowing(job: Bull.Job<DbUserJobData>, done: () =>
}
// Create temp file
const [path, cleanup] = await new Promise<[string, () => void]>((res, rej) => {
tmp.file((e, path, fd, cleanup) => {
if (e) return rej(e);
res([path, cleanup]);
});
});
const [path, cleanup] = await createTemp();
logger.info(`Temp file is ${path}`);
const stream = fs.createWriteStream(path, { flags: 'a' });
try {
const stream = fs.createWriteStream(path, { flags: 'a' });
let cursor: Following['id'] | null = null;
let cursor: Following['id'] | null = null;
const mutings = job.data.excludeMuting ? await Mutings.findBy({
muterId: user.id,
}) : [];
const mutings = job.data.excludeMuting ? await Mutings.findBy({
muterId: user.id,
}) : [];
while (true) {
const followings = await Followings.find({
where: {
followerId: user.id,
...(mutings.length > 0 ? { followeeId: Not(In(mutings.map(x => x.muteeId))) } : {}),
...(cursor ? { id: MoreThan(cursor) } : {}),
},
take: 100,
order: {
id: 1,
},
}) as Following[];
while (true) {
const followings = await Followings.find({
where: {
followerId: user.id,
...(mutings.length > 0 ? { followeeId: Not(In(mutings.map(x => x.muteeId))) } : {}),
...(cursor ? { id: MoreThan(cursor) } : {}),
},
take: 100,
order: {
id: 1,
},
}) as Following[];
if (followings.length === 0) {
break;
}
cursor = followings[followings.length - 1].id;
for (const following of followings) {
const u = await Users.findOneBy({ id: following.followeeId });
if (u == null) {
continue;
if (followings.length === 0) {
break;
}
if (job.data.excludeInactive && u.updatedAt && (Date.now() - u.updatedAt.getTime() > 1000 * 60 * 60 * 24 * 90)) {
continue;
}
cursor = followings[followings.length - 1].id;
const content = getFullApAccount(u.username, u.host);
await new Promise<void>((res, rej) => {
stream.write(content + '\n', err => {
if (err) {
logger.error(err);
rej(err);
} else {
res();
}
for (const following of followings) {
const u = await Users.findOneBy({ id: following.followeeId });
if (u == null) {
continue;
}
if (job.data.excludeInactive && u.updatedAt && (Date.now() - u.updatedAt.getTime() > 1000 * 60 * 60 * 24 * 90)) {
continue;
}
const content = getFullApAccount(u.username, u.host);
await new Promise<void>((res, rej) => {
stream.write(content + '\n', err => {
if (err) {
logger.error(err);
rej(err);
} else {
res();
}
});
});
});
}
}
stream.end();
logger.succ(`Exported to: ${path}`);
const fileName = 'following-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv';
const driveFile = await addFile({ user, path, name: fileName, force: true });
logger.succ(`Exported to: ${driveFile.id}`);
} finally {
cleanup();
}
stream.end();
logger.succ(`Exported to: ${path}`);
const fileName = 'following-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv';
const driveFile = await addFile({ user, path, name: fileName, force: true });
logger.succ(`Exported to: ${driveFile.id}`);
cleanup();
done();
}