refactoring

Resolve #7779

packages/backend/src/queue/processors/object-storage/clean-remote-files.ts

@@ -0,0 +1,50 @@
+import * as Bull from 'bull';
+
+import { queueLogger } from '../../logger';
+import { deleteFileSync } from '@/services/drive/delete-file';
+import { DriveFiles } from '@/models/index';
+import { MoreThan, Not, IsNull } from 'typeorm';
+
+const logger = queueLogger.createSubLogger('clean-remote-files');
+
+export default async function cleanRemoteFiles(job: Bull.Job<{}>, done: any): Promise<void> {
+	logger.info(`Deleting cached remote files...`);
+
+	let deletedCount = 0;
+	let cursor: any = null;
+
+	while (true) {
+		const files = await DriveFiles.find({
+			where: {
+				userHost: Not(IsNull()),
+				isLink: false,
+				...(cursor ? { id: MoreThan(cursor) } : {})
+			},
+			take: 8,
+			order: {
+				id: 1
+			}
+		});
+
+		if (files.length === 0) {
+			job.progress(100);
+			break;
+		}
+
+		cursor = files[files.length - 1].id;
+
+		await Promise.all(files.map(file => deleteFileSync(file, true)));
+
+		deletedCount += 8;
+
+		const total = await DriveFiles.count({
+			userHost: Not(IsNull()),
+			isLink: false,
+		});
+
+		job.progress(deletedCount / total);
+	}
+
+	logger.succ(`All cahced remote files has been deleted.`);
+	done();
+}

packages/backend/src/queue/processors/object-storage/delete-file.ts

@@ -0,0 +1,11 @@
+import { ObjectStorageFileJobData } from '@/queue/types';
+import * as Bull from 'bull';
+import { deleteObjectStorageFile } from '@/services/drive/delete-file';
+
+export default async (job: Bull.Job<ObjectStorageFileJobData>) => {
+	const key: string = job.data.key;
+
+	await deleteObjectStorageFile(key);
+
+	return 'Success';
+};

packages/backend/src/queue/processors/object-storage/index.ts

@@ -0,0 +1,15 @@
+import * as Bull from 'bull';
+import { ObjectStorageJobData } from '@/queue/types';
+import deleteFile from './delete-file';
+import cleanRemoteFiles from './clean-remote-files';
+
+const jobs = {
+	deleteFile,
+	cleanRemoteFiles,
+} as Record<string, Bull.ProcessCallbackFunction<ObjectStorageJobData> | Bull.ProcessPromiseFunction<ObjectStorageJobData>>;
+
+export default function(q: Bull.Queue) {
+	for (const [k, v] of Object.entries(jobs)) {
+		q.process(k, 16, v);
+	}
+}