feat: auto nsfw detection (#8840)

* feat: auto nsfw detection

* ✌️

* Update ja-JP.yml

* Update ja-JP.yml

* ポルノ判定のしきい値を高めに

* エラーハンドリングちゃんとした

* Update ja-JP.yml

* 感度設定を強化

* refactor

* feat: add video support for auto nsfw detection

* rename: image -> media

* .js

* fix: add missing error handling

* fix: use valid pathname instead of using filename due to invalid usage

* perf(nsfw-detection): decode frames

* disable detection of video for some reasons

* perf(nsfw-detection): streamify detection process for video

* disable disallowUploadWhenPredictedAsPorn option

* fix(nsfw-detection): improve reliability

* fix(nsfw-detection): use Math.ceil instead of Math.round

* perf(nsfw-detection): delete tmp frames after used

* fix(nsfw-detection): FSWatcher does not emit ready event

* perf(nsfw-detection): skip black frames

* refactor: strip exists check

* Update package.json

* めっちゃ変えた

* lint

* Update COPYING

* オプションで動画解析できるように

* Update yarn.lock

* Update CHANGELOG.md

Co-authored-by: Acid Chicken (硫酸鶏) <root@acid-chicken.com>

packages/client/src/pages/admin/security.vue

@@ -14,6 +14,49 @@
 					<XBotProtection/>
 				</FormFolder>
 
+				<FormFolder class="_formBlock">
+					<template #icon><i class="fas fa-eye-slash"></i></template>
+					<template #label>{{ i18n.ts.sensitiveMediaDetection }}</template>
+					<template v-if="sensitiveMediaDetection === 'all'" #suffix>{{ i18n.ts.all }}</template>
+					<template v-else-if="sensitiveMediaDetection === 'local'" #suffix>{{ i18n.ts.localOnly }}</template>
+					<template v-else-if="sensitiveMediaDetection === 'remote'" #suffix>{{ i18n.ts.remoteOnly }}</template>
+					<template v-else #suffix>{{ i18n.ts.none }}</template>
+
+					<div class="_formRoot">
+						<span class="_formBlock">{{ i18n.ts._sensitiveMediaDetection.description }}</span>
+
+						<FormRadios v-model="sensitiveMediaDetection" class="_formBlock">
+							<option value="none">{{ i18n.ts.none }}</option>
+							<option value="all">{{ i18n.ts.all }}</option>
+							<option value="local">{{ i18n.ts.localOnly }}</option>
+							<option value="remote">{{ i18n.ts.remoteOnly }}</option>
+						</FormRadios>
+
+						<FormRange v-model="sensitiveMediaDetectionSensitivity" :min="0" :max="4" :step="1" :text-converter="(v) => `${v + 1}`" class="_formBlock">
+							<template #label>{{ i18n.ts._sensitiveMediaDetection.sensitivity }}</template>
+							<template #caption>{{ i18n.ts._sensitiveMediaDetection.sensitivityDescription }}</template>
+						</FormRange>
+
+						<FormSwitch v-model="enableSensitiveMediaDetectionForVideos" class="_formBlock">
+							<template #label>{{ i18n.ts._sensitiveMediaDetection.analyzeVideos }}<span class="_beta">{{ i18n.ts.beta }}</span></template>
+							<template #caption>{{ i18n.ts._sensitiveMediaDetection.analyzeVideosDescription }}</template>
+						</FormSwitch>
+
+						<FormSwitch v-model="setSensitiveFlagAutomatically" class="_formBlock">
+							<template #label>{{ i18n.ts._sensitiveMediaDetection.setSensitiveFlagAutomatically }} ({{ i18n.ts.notRecommended }})</template>
+							<template #caption>{{ i18n.ts._sensitiveMediaDetection.setSensitiveFlagAutomaticallyDescription }}</template>
+						</FormSwitch>
+
+						<!-- 現状 false positive が多すぎて実用に耐えない
+						<FormSwitch v-model="disallowUploadWhenPredictedAsPorn" class="_formBlock">
+							<template #label>{{ i18n.ts._sensitiveMediaDetection.disallowUploadWhenPredictedAsPorn }}</template>
+						</FormSwitch>
+						-->
+
+						<FormButton primary class="_formBlock" @click="save"><i class="fas fa-save"></i> {{ i18n.ts.save }}</FormButton>
+					</div>
+				</FormFolder>
+
 				<FormFolder class="_formBlock">
 					<template #label>Log IP address</template>
 					<template v-if="enableIpLogging" #suffix>Enabled</template>
@@ -49,10 +92,11 @@ import { } from 'vue';
 import XBotProtection from './bot-protection.vue';
 import XHeader from './_header_.vue';
 import FormFolder from '@/components/form/folder.vue';
+import FormRadios from '@/components/form/radios.vue';
 import FormSwitch from '@/components/form/switch.vue';
 import FormInfo from '@/components/ui/info.vue';
 import FormSuspense from '@/components/form/suspense.vue';
-import FormSection from '@/components/form/section.vue';
+import FormRange from '@/components/form/range.vue';
 import FormInput from '@/components/form/input.vue';
 import FormButton from '@/components/ui/button.vue';
 import * as os from '@/os';
@@ -63,6 +107,10 @@ import { definePageMetadata } from '@/scripts/page-metadata';
 let summalyProxy: string = $ref('');
 let enableHcaptcha: boolean = $ref(false);
 let enableRecaptcha: boolean = $ref(false);
+let sensitiveMediaDetection: string = $ref('none');
+let sensitiveMediaDetectionSensitivity: number = $ref(0);
+let setSensitiveFlagAutomatically: boolean = $ref(false);
+let enableSensitiveMediaDetectionForVideos: boolean = $ref(false);
 let enableIpLogging: boolean = $ref(false);
 
 async function init() {
@@ -70,12 +118,31 @@ async function init() {
 	summalyProxy = meta.summalyProxy;
 	enableHcaptcha = meta.enableHcaptcha;
 	enableRecaptcha = meta.enableRecaptcha;
+	sensitiveMediaDetection = meta.sensitiveMediaDetection;
+	sensitiveMediaDetectionSensitivity =
+		meta.sensitiveMediaDetectionSensitivity === 'veryLow' ? 0 :
+		meta.sensitiveMediaDetectionSensitivity === 'low' ? 1 :
+		meta.sensitiveMediaDetectionSensitivity === 'medium' ? 2 :
+		meta.sensitiveMediaDetectionSensitivity === 'high' ? 3 :
+		meta.sensitiveMediaDetectionSensitivity === 'veryHigh' ? 4 : 0;
+	setSensitiveFlagAutomatically = meta.setSensitiveFlagAutomatically;
+	enableSensitiveMediaDetectionForVideos = meta.enableSensitiveMediaDetectionForVideos;
 	enableIpLogging = meta.enableIpLogging;
 }
 
 function save() {
 	os.apiWithDialog('admin/update-meta', {
 		summalyProxy,
+		sensitiveMediaDetection,
+		sensitiveMediaDetectionSensitivity:
+			sensitiveMediaDetectionSensitivity === 0 ? 'veryLow' :
+			sensitiveMediaDetectionSensitivity === 1 ? 'low' :
+			sensitiveMediaDetectionSensitivity === 2 ? 'medium' :
+			sensitiveMediaDetectionSensitivity === 3 ? 'high' :
+			sensitiveMediaDetectionSensitivity === 4 ? 'veryHigh' :
+			0,
+		setSensitiveFlagAutomatically,
+		enableSensitiveMediaDetectionForVideos,
 		enableIpLogging,
 	}).then(() => {
 		fetchInstance();

packages/client/src/pages/settings/drive.vue

@@ -28,7 +28,17 @@
 			<template #suffix>{{ uploadFolder ? uploadFolder.name : '-' }}</template>
 			<template #suffixIcon><i class="fas fa-folder-open"></i></template>
 		</FormLink>
-		<FormSwitch v-model="keepOriginalUploading" class="_formBlock">{{ i18n.ts.keepOriginalUploading }}<template #caption>{{ i18n.ts.keepOriginalUploadingDescription }}</template></FormSwitch>
+		<FormSwitch v-model="keepOriginalUploading" class="_formBlock">
+			<template #label>{{ i18n.ts.keepOriginalUploading }}</template>
+			<template #caption>{{ i18n.ts.keepOriginalUploadingDescription }}</template>
+		</FormSwitch>
+		<FormSwitch v-model="alwaysMarkNsfw" class="_formBlock" @update:modelValue="saveProfile()">
+			<template #label>{{ i18n.ts.alwaysMarkSensitive }}</template>
+		</FormSwitch>
+		<FormSwitch v-model="autoSensitive" class="_formBlock" @update:modelValue="saveProfile()">
+			<template #label>{{ i18n.ts.enableAutoSensitive }}<span class="_beta">{{ i18n.ts.beta }}</span></template>
+			<template #caption>{{ i18n.ts.enableAutoSensitiveDescription }}</template>
+		</FormSwitch>
 	</FormSection>
 </div>
 </template>
@@ -47,11 +57,14 @@ import { defaultStore } from '@/store';
 import MkChart from '@/components/chart.vue';
 import { i18n } from '@/i18n';
 import { definePageMetadata } from '@/scripts/page-metadata';
+import { $i } from '@/account';
 
 const fetching = ref(true);
 const usage = ref<any>(null);
 const capacity = ref<any>(null);
 const uploadFolder = ref<any>(null);
+let alwaysMarkNsfw = $ref($i.alwaysMarkNsfw);
+let autoSensitive = $ref($i.autoSensitive);
 
 const meterStyle = computed(() => {
 	return {
@@ -94,6 +107,13 @@ function chooseUploadFolder() {
 	});
 }
 
+function saveProfile() {
+	os.api('i/update', {
+		alwaysMarkNsfw: !!alwaysMarkNsfw,
+		autoSensitive: !!autoSensitive,
+	});
+}
+
 const headerActions = $computed(() => []);
 
 const headerTabs = $computed(() => []);

packages/client/src/pages/settings/profile.vue

@@ -56,8 +56,6 @@
 	<FormSwitch v-model="profile.isCat" class="_formBlock">{{ i18n.ts.flagAsCat }}<template #caption>{{ i18n.ts.flagAsCatDescription }}</template></FormSwitch>
 	<FormSwitch v-model="profile.showTimelineReplies" class="_formBlock">{{ i18n.ts.flagShowTimelineReplies }}<template #caption>{{ i18n.ts.flagShowTimelineRepliesDescription }} {{ i18n.ts.reflectMayTakeTime }}</template></FormSwitch>
 	<FormSwitch v-model="profile.isBot" class="_formBlock">{{ i18n.ts.flagAsBot }}<template #caption>{{ i18n.ts.flagAsBotDescription }}</template></FormSwitch>
-
-	<FormSwitch v-model="profile.alwaysMarkNsfw" class="_formBlock">{{ i18n.ts.alwaysMarkSensitive }}</FormSwitch>
 </div>
 </template>
 
@@ -88,7 +86,6 @@ const profile = reactive({
 	isBot: $i.isBot,
 	isCat: $i.isCat,
 	showTimelineReplies: $i.showTimelineReplies,
-	alwaysMarkNsfw: $i.alwaysMarkNsfw,
 });
 
 watch(() => profile, () => {
@@ -126,7 +123,6 @@ function save() {
 		isBot: !!profile.isBot,
 		isCat: !!profile.isCat,
 		showTimelineReplies: !!profile.showTimelineReplies,
-		alwaysMarkNsfw: !!profile.alwaysMarkNsfw,
 	});
 }
 

packages/client/src/scripts/select-file.ts

@@ -1,9 +1,9 @@
 import { ref } from 'vue';
+import { DriveFile } from 'misskey-js/built/entities';
 import * as os from '@/os';
 import { stream } from '@/stream';
 import { i18n } from '@/i18n';
 import { defaultStore } from '@/store';
-import { DriveFile } from 'misskey-js/built/entities';
 import { uploadFile } from '@/scripts/upload';
 
 function select(src: any, label: string | null, multiple: boolean): Promise<DriveFile | DriveFile[]> {
@@ -20,10 +20,7 @@ function select(src: any, label: string | null, multiple: boolean): Promise<Driv
 				Promise.all(promises).then(driveFiles => {
 					res(multiple ? driveFiles : driveFiles[0]);
 				}).catch(err => {
-					os.alert({
-						type: 'error',
-						text: err
-					});
+					// アップロードのエラーは uploadFile 内でハンドリングされているためアラートダイアログを出したりはしてはいけない
 				});
 
 				// 一応廃棄
@@ -47,7 +44,7 @@ function select(src: any, label: string | null, multiple: boolean): Promise<Driv
 			os.inputText({
 				title: i18n.ts.uploadFromUrl,
 				type: 'url',
-				placeholder: i18n.ts.uploadFromUrlDescription
+				placeholder: i18n.ts.uploadFromUrlDescription,
 			}).then(({ canceled, result: url }) => {
 				if (canceled) return;
 
@@ -64,35 +61,35 @@ function select(src: any, label: string | null, multiple: boolean): Promise<Driv
 				os.api('drive/files/upload-from-url', {
 					url: url,
 					folderId: defaultStore.state.uploadFolder,
-					marker
+					marker,
 				});
 
 				os.alert({
 					title: i18n.ts.uploadFromUrlRequested,
-					text: i18n.ts.uploadFromUrlMayTakeTime
+					text: i18n.ts.uploadFromUrlMayTakeTime,
 				});
 			});
 		};
 
 		os.popupMenu([label ? {
 			text: label,
-			type: 'label'
+			type: 'label',
 		} : undefined, {
 			type: 'switch',
 			text: i18n.ts.keepOriginalUploading,
-			ref: keepOriginal
+			ref: keepOriginal,
 		}, {
 			text: i18n.ts.upload,
 			icon: 'fas fa-upload',
-			action: chooseFileFromPc
+			action: chooseFileFromPc,
 		}, {
 			text: i18n.ts.fromDrive,
 			icon: 'fas fa-cloud',
-			action: chooseFileFromDrive
+			action: chooseFileFromDrive,
 		}, {
 			text: i18n.ts.fromUrl,
 			icon: 'fas fa-link',
-			action: chooseFileFromUrl
+			action: chooseFileFromUrl,
 		}], src);
 	});
 }

packages/client/src/scripts/upload.ts

@@ -5,6 +5,7 @@ import { defaultStore } from '@/store';
 import { apiUrl } from '@/config';
 import { $i } from '@/account';
 import { alert } from '@/os';
+import { i18n } from '@/i18n';
 
 type Uploading = {
 	id: string;
@@ -80,14 +81,37 @@ export function uploadFile(
 			xhr.open('POST', apiUrl + '/drive/files/create', true);
 			xhr.onload = (ev) => {
 				if (xhr.status !== 200 || ev.target == null || ev.target.response == null) {
-					// TODO: 消すのではなくて再送できるようにしたい
+					// TODO: 消すのではなくて(ネットワーク的なエラーなら)再送できるようにしたい
 					uploads.value = uploads.value.filter(x => x.id !== id);
 
-					alert({
-						type: 'error',
-						title: 'Failed to upload',
-						text: `${JSON.stringify(ev.target?.response)}, ${JSON.stringify(xhr.response)}`,
-					});
+					if (ev.target?.response) {
+						const res = JSON.parse(ev.target.response);
+						if (res.error?.id === 'bec5bd69-fba3-43c9-b4fb-2894b66ad5d2') {
+							alert({
+								type: 'error',
+								title: i18n.ts.failedToUpload,
+								text: i18n.ts.cannotUploadBecauseInappropriate,
+							});
+						} else if (res.error?.id === 'd08dbc37-a6a9-463a-8c47-96c32ab5f064') {
+							alert({
+								type: 'error',
+								title: i18n.ts.failedToUpload,
+								text: i18n.ts.cannotUploadBecauseNoFreeSpace,
+							});
+						} else {
+							alert({
+								type: 'error',
+								title: i18n.ts.failedToUpload,
+								text: `${res.error?.message}\n${res.error?.code}\n${res.error?.id}`,
+							});
+						}
+					} else {
+						alert({
+							type: 'error',
+							title: 'Failed to upload',
+							text: `${JSON.stringify(ev.target?.response)}, ${JSON.stringify(xhr.response)}`,
+						});
+					}
 
 					reject();
 					return;

packages/client/src/style.scss

@@ -399,6 +399,16 @@ hr {
 	}
 }
 
+._beta {
+	margin-left: 0.7em;
+	font-size: 65%;
+	padding: 2px 3px;
+	color: var(--accent);
+	border: solid 1px var(--accent);
+	border-radius: 4px;
+	vertical-align: top;
+}
+
 ._table {
 	> ._row {
 		display: flex;