10.9.0

.config/example.yml

@@ -60,11 +60,6 @@ mongodb:
   user: example-misskey-user
   pass: example-misskey-pass
 
-redis:
-  host: localhost
-  port: 6379
-  pass: example-pass
-
 # Drive capacity of a local user (MB)
 localDriveCapacityMb: 256
 
@@ -122,47 +117,50 @@ drive:
 # Below settings are optional
 #
 
+# Redis
+#redis:
+#  host: localhost
+#  port: 6379
+#  pass: example-pass
+
 # Elasticsearch
-# elasticsearch:
-#   host: localhost
-#   port: 9200
-#   pass: null
+#elasticsearch:
+#  host: localhost
+#  port: 9200
+#  pass: null
 
 # reCAPTCHA
-# recaptcha:
-#   site_key: example-site-key
+#recaptcha:
+#  site_key: example-site-key
 #  secret_key: example-secret-key
 
 # ServiceWorker
-# sw:
-#   # Public key of VAPID
-#   public_key: example-sw-public-key
-
-#   # Private key of VAPID
-#   private_key: example-sw-private-key
-
-# google_maps_api_key: example-google-maps-api-key
+#sw:
+#  # Public key of VAPID
+#  public_key: example-sw-public-key
+#
+#  # Private key of VAPID
+#  private_key: example-sw-private-key
 
 # Twitter integration
 # You need to set the oauth callback url as : https://<your-misskey-instance>/api/tw/cb
-# twitter:
-#   consumer_key: example-twitter-consumer-key
-#   consumer_secret: example-twitter-consumer-secret-key
+#twitter:
+#  consumer_key: example-twitter-consumer-key
+#  consumer_secret: example-twitter-consumer-secret-key
 
 # Ghost
 # Ghost account is an account used for the purpose of delegating
 # followers when putting users in the list.
-# ghost: user-id-of-your-ghost-account
+#ghost: user-id-of-your-ghost-account
 
 # Clustering
-# clusterLimit: 1
+#clusterLimit: 1
 
 # Summaly proxy
-# summalyProxy: "http://example.com"
+#summalyProxy: "http://example.com"
 
 # User recommendation
-user_recommendation:
-  external: true
-  engine: http://vinayaka.distsn.org/cgi-bin/vinayaka-user-match-misskey-api.cgi?{{host}}+{{user}}+{{limit}}+{{offset}}
-  timeout: 300000
-
+#user_recommendation:
+#  external: true
+#  engine: http://vinayaka.distsn.org/cgi-bin/vinayaka-user-match-misskey-api.cgi?{{host}}+{{user}}+{{limit}}+{{offset}}
+#  timeout: 300000

docs/setup.en.md

@@ -24,12 +24,12 @@ Please install and setup these softwares:
 #### Dependencies :package:
 * **[Node.js](https://nodejs.org/en/)**
 * **[MongoDB](https://www.mongodb.com/)** >= 3.6
-* **[Redis](https://redis.io/)**
 
 ##### Optional
+* [Redis](https://redis.io/)
+  * Redis is optional, but we strongly recommended to install it
 * [Elasticsearch](https://www.elastic.co/) - used to provide searching feature instead of MongoDB
 
-
 *3.* Setup MongoDB
 ----------------------------------------------------------------
 In root :

docs/setup.ja.md

@@ -24,10 +24,17 @@ adduser --disabled-password --disabled-login misskey
 #### 依存関係 :package:
 * **[Node.js](https://nodejs.org/en/)**
 * **[MongoDB](https://www.mongodb.com/)** (3.6以上)
-* **[Redis](https://redis.io/)**
 
 ##### オプション
-* [Elasticsearch](https://www.elastic.co/) - 検索機能を向上させるために用います。
+* [Redis](https://redis.io/)
+	* Redisはオプションですが、インストールすることを強く推奨します。
+	* インストールしなくていいのは、あなたのインスタンスが自分専用のときだけとお考えください。
+	* 具体的には、Redisをインストールしないと、次の事が出来なくなります:
+		* Misskeyプロセスを複数起動しての負荷分散
+		* レートリミット
+		* Twitter連携
+* [Elasticsearch](https://www.elastic.co/)
+	* 検索機能を有効にするためにはインストールが必要です。
 
 *3.* MongoDBの設定
 ----------------------------------------------------------------

package.json

@@ -1,8 +1,8 @@
 {
 	"name": "misskey",
 	"author": "syuilo <i@syuilo.com>",
-	"version": "10.6.0",
-	"clientVersion": "1.0.10417",
+	"version": "10.9.0",
+	"clientVersion": "1.0.10443",
 	"codename": "nighthike",
 	"main": "./built/index.js",
 	"private": true,
@@ -60,7 +60,7 @@
 		"@types/mocha": "5.2.3",
 		"@types/mongodb": "3.1.12",
 		"@types/ms": "0.7.30",
-		"@types/node": "10.11.6",
+		"@types/node": "10.11.7",
 		"@types/portscanner": "2.1.0",
 		"@types/pug": "2.0.4",
 		"@types/qrcode": "1.3.0",
@@ -169,6 +169,7 @@
 		"parse5": "5.1.0",
 		"portscanner": "2.2.0",
 		"progress-bar-webpack-plugin": "1.11.0",
+		"promise-limit": "2.7.0",
 		"promise-sequential": "1.1.1",
 		"pug": "2.0.3",
 		"punycode": "2.1.1",
@@ -211,7 +212,7 @@
 		"v-animate-css": "0.0.2",
 		"vue": "2.5.17",
 		"vue-chartjs": "3.4.0",
-		"vue-color": "2.6.0",
+		"vue-color": "2.7.0",
 		"vue-cropperjs": "2.2.2",
 		"vue-js-modal": "1.3.26",
 		"vue-json-tree-view": "2.1.4",

src/client/app/common/scripts/stream.ts

@@ -10,7 +10,6 @@ import MiOS from '../../mios';
 export default class Stream extends EventEmitter {
 	private stream: ReconnectingWebsocket;
 	private state: string;
-	private buffer: any[];
 	private sharedConnectionPools: Pool[] = [];
 	private sharedConnections: SharedConnection[] = [];
 	private nonSharedConnections: NonSharedConnection[] = [];
@@ -19,7 +18,6 @@ export default class Stream extends EventEmitter {
 		super();
 
 		this.state = 'initializing';
-		this.buffer = [];
 
 		const user = os.store.state.i;
 
@@ -29,7 +27,8 @@ export default class Stream extends EventEmitter {
 		this.stream.addEventListener('message', this.onMessage);
 	}
 
-	public useSharedConnection = (channel: string): SharedConnection => {
+	@autobind
+	public useSharedConnection(channel: string): SharedConnection {
 		let pool = this.sharedConnectionPools.find(p => p.channel === channel);
 
 		if (pool == null) {
@@ -47,7 +46,8 @@ export default class Stream extends EventEmitter {
 		this.sharedConnections = this.sharedConnections.filter(c => c !== connection);
 	}
 
-	public connectToChannel = (channel: string, params?: any): NonSharedConnection => {
+	@autobind
+	public connectToChannel(channel: string, params?: any): NonSharedConnection {
 		const connection = new NonSharedConnection(this, channel, params);
 		this.nonSharedConnections.push(connection);
 		return connection;
@@ -68,17 +68,12 @@ export default class Stream extends EventEmitter {
 		this.state = 'connected';
 		this.emit('_connected_');
 
-		// バッファーを処理
-		const _buffer = [].concat(this.buffer); // Shallow copy
-		this.buffer = []; // Clear buffer
-		_buffer.forEach(data => {
-			this.send(data); // Resend each buffered messages
-		});
-
 		// チャンネル再接続
 		if (isReconnect) {
 			this.sharedConnectionPools.forEach(p => {
-				p.connect();
+				if (p.users > 0) {
+					p.connect();
+				}
 			});
 			this.nonSharedConnections.forEach(c => {
 				c.connect();
@@ -131,12 +126,6 @@ export default class Stream extends EventEmitter {
 			body: payload
 		};
 
-		// まだ接続が確立されていなかったらバッファリングして次に接続した時に送信する
-		if (this.state != 'connected') {
-			this.buffer.push(data);
-			return;
-		}
-
 		this.stream.send(JSON.stringify(data));
 	}
 
@@ -154,7 +143,7 @@ class Pool {
 	public channel: string;
 	public id: string;
 	protected stream: Stream;
-	private users = 0;
+	public users = 0;
 	private disposeTimerId: any;
 	private isConnected = false;
 

src/config/types.ts

@@ -93,11 +93,9 @@ export type Source = {
 		private_key: string;
 	};
 
-	google_maps_api_key: string;
-
 	clusterLimit?: number;
 
-	user_recommendation: {
+	user_recommendation?: {
 		external: boolean;
 		engine: string;
 		timeout: number;

src/db/redis.ts

@@ -1,10 +1,10 @@
 import * as redis from 'redis';
 import config from '../config';
 
-export default redis.createClient(
+export default config.redis ? redis.createClient(
 	config.redis.port,
 	config.redis.host,
 	{
 		auth_pass: config.redis.pass
 	}
-);
+) : null;

src/models/drive-file.ts

@@ -166,7 +166,7 @@ export const pack = (
 
 	// (データベースの欠損などで)ファイルがデータベース上に見つからなかったとき
 	if (_file == null) {
-		console.warn(`in packaging driveFile: driveFile not found on database: ${_file}`);
+		console.warn(`[DAMAGED DB] (missing) pkg: driveFile :: ${file}`);
 		return resolve(null);
 	}
 

src/models/favorite.ts

@@ -79,7 +79,7 @@ export const pack = (
 
 	// (データベースの不具合などで)投稿が見つからなかったら
 	if (_favorite.note == null) {
-		console.warn(`in packaging favorite: note not found on database: ${_favorite.noteId}`);
+		console.warn(`[DAMAGED DB] (missing) pkg: favorite -> note :: ${_favorite.id} (note ${_favorite.noteId})`);
 		return resolve(null);
 	}
 

src/models/note.ts

@@ -281,9 +281,9 @@ export const pack = async (
 		_note = deepcopy(note);
 	}
 
-	// 投稿がデータベース上に見つからなかったとき
+	// (データベースの欠損などで)投稿がデータベース上に見つからなかったとき
 	if (_note == null) {
-		console.warn(`note not found on database: ${note}`);
+		console.warn(`[DAMAGED DB] (missing) pkg: note :: ${note}`);
 		return null;
 	}
 
@@ -380,12 +380,25 @@ export const pack = async (
 	// resolve promises in _note object
 	_note = await rap(_note);
 
-	// (データベースの欠損などで)ユーザーがデータベース上に見つからなかったとき
+	//#region (データベースの欠損などで)参照しているデータがデータベース上に見つからなかったとき
 	if (_note.user == null) {
-		console.warn(`in packaging note: note user not found on database: note(${_note.id})`);
+		console.warn(`[DAMAGED DB] (missing) pkg: note -> user :: ${_note.id} (user ${_note.userId})`);
 		return null;
 	}
 
+	if (opts.detail) {
+		if (_note.replyId != null && _note.reply == null) {
+			console.warn(`[DAMAGED DB] (missing) pkg: note -> reply :: ${_note.id} (reply ${_note.replyId})`);
+			return null;
+		}
+
+		if (_note.renoteId != null && _note.renote == null) {
+			console.warn(`[DAMAGED DB] (missing) pkg: note -> renote :: ${_note.id} (renote ${_note.renoteId})`);
+			return null;
+		}
+	}
+	//#endregion
+
 	if (_note.user.isCat && _note.text) {
 		_note.text = _note.text.replace(/な/g, 'にゃ').replace(/ナ/g, 'ニャ').replace(/ﾅ/g, 'ﾆｬ');
 	}

src/models/notification.ts

@@ -132,7 +132,7 @@ export const pack = (notification: any) => new Promise<any>(async (resolve, reje
 
 			// (データベースの不具合などで)投稿が見つからなかったら
 			if (_notification.note == null) {
-				console.warn(`in packaging notification: note not found on database: ${_notification.noteId}`);
+				console.warn(`[DAMAGED DB] (missing) pkg: notification -> note :: ${_notification.id} (note ${_notification.noteId})`);
 				return resolve(null);
 			}
 			break;

src/server/api/endpoints/drive/files/check_existence.ts

@@ -27,7 +27,8 @@ export default (params: any, user: ILocalUser) => new Promise(async (res, rej) =
 
 	const file = await DriveFile.findOne({
 		md5: md5,
-		'metadata.userId': user._id
+		'metadata.userId': user._id,
+		'metadata.deletedAt': { $exists: false }
 	});
 
 	if (file === null) {

src/server/api/endpoints/drive/files/show.ts

@@ -22,7 +22,8 @@ export default async (params: any, user: ILocalUser) => {
 	const file = await DriveFile
 		.findOne({
 			_id: fileId,
-			'metadata.userId': user._id
+			'metadata.userId': user._id,
+			'metadata.deletedAt': { $exists: false }
 		});
 
 	if (file === null) {

src/server/api/limitter.ts

@@ -8,6 +8,12 @@ import { IUser } from '../../models/user';
 const log = debug('misskey:limitter');
 
 export default (endpoint: IEndpoint, user: IUser) => new Promise((ok, reject) => {
+	// Redisがインストールされてない場合は常に許可
+	if (limiterDB == null) {
+		ok();
+		return;
+	}
+
 	const limitation = endpoint.meta.limit;
 
 	const key = limitation.hasOwnProperty('key')

src/server/api/service/twitter.ts

@@ -55,7 +55,7 @@ router.get('/disconnect/twitter', async ctx => {
 	}));
 });
 
-if (config.twitter == null) {
+if (config.twitter == null || redis == null) {
 	router.get('/connect/twitter', ctx => {
 		ctx.body = '現在Twitterへ接続できません (このインスタンスではTwitterはサポートされていません)';
 	});

src/server/api/stream/channels/games/reversi-game.ts

@@ -1,5 +1,6 @@
 import autobind from 'autobind-decorator';
 import * as CRC32 from 'crc-32';
+import * as mongo from 'mongodb';
 import ReversiGame, { pack } from '../../../../../models/games/reversi/game';
 import { publishReversiGameStream } from '../../../../../stream';
 import Reversi from '../../../../../games/reversi/core';
@@ -7,11 +8,11 @@ import * as maps from '../../../../../games/reversi/maps';
 import Channel from '../../channel';
 
 export default class extends Channel {
-	private gameId: string;
+	private gameId: mongo.ObjectID;
 
 	@autobind
 	public async init(params: any) {
-		this.gameId = params.gameId as string;
+		this.gameId = new mongo.ObjectID(params.gameId as string);
 
 		// Subscribe game stream
 		this.subscriber.on(`reversiGameStream:${this.gameId}`, data => {

src/server/api/stream/index.ts

@@ -1,6 +1,5 @@
 import autobind from 'autobind-decorator';
 import * as websocket from 'websocket';
-import Xev from 'xev';
 import * as debug from 'debug';
 
 import User, { IUser } from '../../../models/user';
@@ -11,6 +10,7 @@ import readNote from '../../../services/note/read';
 
 import Channel from './channel';
 import channels from './channels';
+import { EventEmitter } from 'events';
 
 const log = debug('misskey');
 
@@ -21,14 +21,14 @@ export default class Connection {
 	public user?: IUser;
 	public app: IApp;
 	private wsConnection: websocket.connection;
-	public subscriber: Xev;
+	public subscriber: EventEmitter;
 	private channels: Channel[] = [];
 	private subscribingNotes: any = {};
 	public sendMessageToWsOverride: any = null; // 後方互換性のため
 
 	constructor(
 		wsConnection: websocket.connection,
-		subscriber: Xev,
+		subscriber: EventEmitter,
 		user: IUser,
 		app: IApp
 	) {

src/server/api/streaming.ts

@@ -1,11 +1,14 @@
 import * as http from 'http';
 import * as websocket from 'websocket';
+import * as redis from 'redis';
 import Xev from 'xev';
 
 import MainStreamConnection from './stream';
 import { ParsedUrlQuery } from 'querystring';
 import authenticate from './authenticate';
 import channels from './stream/channels';
+import { EventEmitter } from 'events';
+import config from '../../config';
 
 module.exports = (server: http.Server) => {
 	// Init websocket server
@@ -16,11 +19,34 @@ module.exports = (server: http.Server) => {
 	ws.on('request', async (request) => {
 		const connection = request.accept();
 
-		const ev = new Xev();
-
 		const q = request.resourceURL.query as ParsedUrlQuery;
 		const [user, app] = await authenticate(q.i as string);
 
+		let ev: EventEmitter;
+
+		if (config.redis) {
+			// Connect to Redis
+			const subscriber = redis.createClient(
+				config.redis.port, config.redis.host);
+
+			subscriber.subscribe('misskey');
+
+			ev = new EventEmitter();
+
+			subscriber.on('message', async (_, data) => {
+				const obj = JSON.parse(data);
+
+				ev.emit(obj.channel, obj.message);
+			});
+
+			connection.once('close', () => {
+				subscriber.unsubscribe();
+				subscriber.quit();
+			});
+		} else {
+			ev = new Xev();
+		}
+
 		const main = new MainStreamConnection(connection, ev, user, app);
 
 		// 後方互換性のため

src/stream.ts

@@ -1,4 +1,5 @@
 import * as mongo from 'mongodb';
+import redis from './db/redis';
 import Xev from 'xev';
 import Meta, { IMeta } from './models/meta';
 
@@ -9,7 +10,10 @@ class Publisher {
 	private meta: IMeta;
 
 	constructor() {
-		this.ev = new Xev();
+		// Redisがインストールされてないときはプロセス間通信を使う
+		if (redis == null) {
+			this.ev = new Xev();
+		}
 
 		setInterval(async () => {
 			this.meta = await Meta.findOne({});
@@ -28,7 +32,14 @@ class Publisher {
 			{ type: type, body: null } :
 			{ type: type, body: value };
 
-		this.ev.emit(channel, message);
+		if (this.ev) {
+			this.ev.emit(channel, message);
+		} else {
+			redis.publish('misskey', JSON.stringify({
+				channel: channel,
+				message: message
+			}));
+		}
 	}
 
 	public publishMainStream = (userId: ID, type: string, value?: any): void => {

src/tools/move-drive-files.ts

@@ -1,67 +1,83 @@
 import * as Minio from 'minio';
 import * as uuid from 'uuid';
-const sequential = require('promise-sequential');
-import DriveFile, { DriveFileChunk, getDriveFileBucket } from '../models/drive-file';
+import * as promiseLimit from 'promise-limit';
+import DriveFile, { DriveFileChunk, getDriveFileBucket, IDriveFile } from '../models/drive-file';
 import DriveFileThumbnail, { DriveFileThumbnailChunk } from '../models/drive-file-thumbnail';
 import config from '../config';
 
+const limit = promiseLimit(16);
+
 DriveFile.find({
 	$or: [{
 		withoutChunks: { $exists: false }
 	}, {
 		withoutChunks: false
-	}]
+	}],
+	'metadata.deletedAt': { $exists: false }
+}, {
+	fields: {
+		_id: true
+	}
 }).then(async files => {
-	await sequential(files.map(file => async () => {
-		const minio = new Minio.Client(config.drive.config);
+	console.log(`there is ${files.length} files`);
 
-		const keyDir = `${config.drive.prefix}/${uuid.v4()}`;
-		const key = `${keyDir}/${name}`;
-		const thumbnailKeyDir = `${config.drive.prefix}/${uuid.v4()}`;
-		const thumbnailKey = `${thumbnailKeyDir}/${name}.thumbnail.jpg`;
+	await Promise.all(files.map(file => limit(() => job(file))));
 
-		const baseUrl = config.drive.baseUrl
-			|| `${ config.drive.config.useSSL ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? `:${config.drive.config.port}` : '' }/${ config.drive.bucket }`;
-
-		const bucket = await getDriveFileBucket();
-		const readable = bucket.openDownloadStream(file._id);
-
-		await minio.putObject(config.drive.bucket, key, readable, file.length, {
-			'Content-Type': file.contentType,
-			'Cache-Control': 'max-age=31536000, immutable'
-		});
-
-		await DriveFile.findOneAndUpdate({ _id: file._id }, {
-			$set: {
-				'metadata.withoutChunks': true,
-				'metadata.storage': 'minio',
-				'metadata.storageProps': {
-					key: key,
-					thumbnailKey: thumbnailKey
-				},
-				'metadata.url': `${ baseUrl }/${ keyDir }/${ encodeURIComponent(name) }`,
-			}
-		});
-
-		// チャンクをすべて削除
-		await DriveFileChunk.remove({
-			files_id: file._id
-		});
-
-		//#region サムネイルもあれば削除
-		const thumbnail = await DriveFileThumbnail.findOne({
-			'metadata.originalId': file._id
-		});
-
-		if (thumbnail) {
-			await DriveFileThumbnailChunk.remove({
-				files_id: thumbnail._id
-			});
-
-			await DriveFileThumbnail.remove({ _id: thumbnail._id });
-		}
-		//#endregion
-
-		console.log('done', file._id);
-	}));
+	console.log('ALL DONE');
 });
+
+async function job(file: IDriveFile): Promise<any> {
+	file = await DriveFile.findOne({ _id: file._id });
+
+	const minio = new Minio.Client(config.drive.config);
+
+	const name = file.filename;
+	const keyDir = `${config.drive.prefix}/${uuid.v4()}`;
+	const key = `${keyDir}/${name}`;
+	const thumbnailKeyDir = `${config.drive.prefix}/${uuid.v4()}`;
+	const thumbnailKey = `${thumbnailKeyDir}/${name}.thumbnail.jpg`;
+
+	const baseUrl = config.drive.baseUrl
+		|| `${ config.drive.config.useSSL ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? `:${config.drive.config.port}` : '' }/${ config.drive.bucket }`;
+
+	const bucket = await getDriveFileBucket();
+	const readable = bucket.openDownloadStream(file._id);
+
+	await minio.putObject(config.drive.bucket, key, readable, file.length, {
+		'Content-Type': file.contentType,
+		'Cache-Control': 'max-age=31536000, immutable'
+	});
+
+	await DriveFile.findOneAndUpdate({ _id: file._id }, {
+		$set: {
+			'metadata.withoutChunks': true,
+			'metadata.storage': 'minio',
+			'metadata.storageProps': {
+				key: key,
+				thumbnailKey: thumbnailKey
+			},
+			'metadata.url': `${ baseUrl }/${ keyDir }/${ encodeURIComponent(name) }`,
+		}
+	});
+
+	// チャンクをすべて削除
+	await DriveFileChunk.remove({
+		files_id: file._id
+	});
+
+	//#region サムネイルもあれば削除
+	const thumbnail = await DriveFileThumbnail.findOne({
+		'metadata.originalId': file._id
+	});
+
+	if (thumbnail) {
+		await DriveFileThumbnailChunk.remove({
+			files_id: thumbnail._id
+		});
+
+		await DriveFileThumbnail.remove({ _id: thumbnail._id });
+	}
+	//#endregion
+
+	console.log('done', file._id);
+}