Introduce processor

This commit is contained in:
Akihiko Odaki
2018-03-29 01:20:40 +09:00
parent 68ce6d5748
commit 90f8fe7e53
582 changed files with 246 additions and 188 deletions

View File

@ -0,0 +1,92 @@
/**
* Module dependencies
*/
import $ from 'cafy';
import User from '../../models/user';
import Following from '../../models/following';
import { pack } from '../../models/user';
import getFriends from '../../common/get-friends';
/**
* Get followers of a user
*
* @param {any} params
* @param {any} me
* @return {Promise<any>}
*/
module.exports = (params, me) => new Promise(async (res, rej) => {
// Get 'user_id' parameter
const [userId, userIdErr] = $(params.user_id).id().$;
if (userIdErr) return rej('invalid user_id param');
// Get 'iknow' parameter
const [iknow = false, iknowErr] = $(params.iknow).optional.boolean().$;
if (iknowErr) return rej('invalid iknow param');
// Get 'limit' parameter
const [limit = 10, limitErr] = $(params.limit).optional.number().range(1, 100).$;
if (limitErr) return rej('invalid limit param');
// Get 'cursor' parameter
const [cursor = null, cursorErr] = $(params.cursor).optional.id().$;
if (cursorErr) return rej('invalid cursor param');
// Lookup user
const user = await User.findOne({
_id: userId
}, {
fields: {
_id: true
}
});
if (user === null) {
return rej('user not found');
}
// Construct query
const query = {
followee_id: user._id,
deleted_at: { $exists: false }
} as any;
// ログインしていてかつ iknow フラグがあるとき
if (me && iknow) {
// Get my friends
const myFriends = await getFriends(me._id);
query.follower_id = {
$in: myFriends
};
}
// カーソルが指定されている場合
if (cursor) {
query._id = {
$lt: cursor
};
}
// Get followers
const following = await Following
.find(query, {
limit: limit + 1,
sort: { _id: -1 }
});
// 「次のページ」があるかどうか
const inStock = following.length === limit + 1;
if (inStock) {
following.pop();
}
// Serialize
const users = await Promise.all(following.map(async f =>
await pack(f.follower_id, me, { detail: true })));
// Response
res({
users: users,
next: inStock ? following[following.length - 1]._id : null,
});
});

View File

@ -0,0 +1,92 @@
/**
* Module dependencies
*/
import $ from 'cafy';
import User from '../../models/user';
import Following from '../../models/following';
import { pack } from '../../models/user';
import getFriends from '../../common/get-friends';
/**
* Get following users of a user
*
* @param {any} params
* @param {any} me
* @return {Promise<any>}
*/
module.exports = (params, me) => new Promise(async (res, rej) => {
// Get 'user_id' parameter
const [userId, userIdErr] = $(params.user_id).id().$;
if (userIdErr) return rej('invalid user_id param');
// Get 'iknow' parameter
const [iknow = false, iknowErr] = $(params.iknow).optional.boolean().$;
if (iknowErr) return rej('invalid iknow param');
// Get 'limit' parameter
const [limit = 10, limitErr] = $(params.limit).optional.number().range(1, 100).$;
if (limitErr) return rej('invalid limit param');
// Get 'cursor' parameter
const [cursor = null, cursorErr] = $(params.cursor).optional.id().$;
if (cursorErr) return rej('invalid cursor param');
// Lookup user
const user = await User.findOne({
_id: userId
}, {
fields: {
_id: true
}
});
if (user === null) {
return rej('user not found');
}
// Construct query
const query = {
follower_id: user._id,
deleted_at: { $exists: false }
} as any;
// ログインしていてかつ iknow フラグがあるとき
if (me && iknow) {
// Get my friends
const myFriends = await getFriends(me._id);
query.followee_id = {
$in: myFriends
};
}
// カーソルが指定されている場合
if (cursor) {
query._id = {
$lt: cursor
};
}
// Get followers
const following = await Following
.find(query, {
limit: limit + 1,
sort: { _id: -1 }
});
// 「次のページ」があるかどうか
const inStock = following.length === limit + 1;
if (inStock) {
following.pop();
}
// Serialize
const users = await Promise.all(following.map(async f =>
await pack(f.followee_id, me, { detail: true })));
// Response
res({
users: users,
next: inStock ? following[following.length - 1]._id : null,
});
});

View File

@ -0,0 +1,99 @@
/**
* Module dependencies
*/
import $ from 'cafy';
import Post from '../../models/post';
import User, { pack } from '../../models/user';
module.exports = (params, me) => new Promise(async (res, rej) => {
// Get 'user_id' parameter
const [userId, userIdErr] = $(params.user_id).id().$;
if (userIdErr) return rej('invalid user_id param');
// Get 'limit' parameter
const [limit = 10, limitErr] = $(params.limit).optional.number().range(1, 100).$;
if (limitErr) return rej('invalid limit param');
// Lookup user
const user = await User.findOne({
_id: userId
}, {
fields: {
_id: true
}
});
if (user === null) {
return rej('user not found');
}
// Fetch recent posts
const recentPosts = await Post.find({
user_id: user._id,
reply_id: {
$exists: true,
$ne: null
}
}, {
sort: {
_id: -1
},
limit: 1000,
fields: {
_id: false,
reply_id: true
}
});
// 投稿が少なかったら中断
if (recentPosts.length === 0) {
return res([]);
}
const replyTargetPosts = await Post.find({
_id: {
$in: recentPosts.map(p => p.reply_id)
},
user_id: {
$ne: user._id
}
}, {
fields: {
_id: false,
user_id: true
}
});
const repliedUsers = {};
// Extract replies from recent posts
replyTargetPosts.forEach(post => {
const userId = post.user_id.toString();
if (repliedUsers[userId]) {
repliedUsers[userId]++;
} else {
repliedUsers[userId] = 1;
}
});
// Calc peak
let peak = 0;
Object.keys(repliedUsers).forEach(user => {
if (repliedUsers[user] > peak) peak = repliedUsers[user];
});
// Sort replies by frequency
const repliedUsersSorted = Object.keys(repliedUsers).sort((a, b) => repliedUsers[b] - repliedUsers[a]);
// Extract top replied users
const topRepliedUsers = repliedUsersSorted.slice(0, limit);
// Make replies object (includes weights)
const repliesObj = await Promise.all(topRepliedUsers.map(async (user) => ({
user: await pack(user, me, { detail: true }),
weight: repliedUsers[user] / peak
})));
// Response
res(repliesObj);
});

View File

@ -0,0 +1,137 @@
/**
* Module dependencies
*/
import $ from 'cafy';
import getHostLower from '../../common/get-host-lower';
import Post, { pack } from '../../models/post';
import User from '../../models/user';
/**
* Get posts of a user
*
* @param {any} params
* @param {any} me
* @return {Promise<any>}
*/
module.exports = (params, me) => new Promise(async (res, rej) => {
// Get 'user_id' parameter
const [userId, userIdErr] = $(params.user_id).optional.id().$;
if (userIdErr) return rej('invalid user_id param');
// Get 'username' parameter
const [username, usernameErr] = $(params.username).optional.string().$;
if (usernameErr) return rej('invalid username param');
if (userId === undefined && username === undefined) {
return rej('user_id or pair of username and host is required');
}
// Get 'host' parameter
const [host, hostErr] = $(params.host).optional.string().$;
if (hostErr) return rej('invalid host param');
if (userId === undefined && host === undefined) {
return rej('user_id or pair of username and host is required');
}
// Get 'include_replies' parameter
const [includeReplies = true, includeRepliesErr] = $(params.include_replies).optional.boolean().$;
if (includeRepliesErr) return rej('invalid include_replies param');
// Get 'with_media' parameter
const [withMedia = false, withMediaErr] = $(params.with_media).optional.boolean().$;
if (withMediaErr) return rej('invalid with_media param');
// Get 'limit' parameter
const [limit = 10, limitErr] = $(params.limit).optional.number().range(1, 100).$;
if (limitErr) return rej('invalid limit param');
// Get 'since_id' parameter
const [sinceId, sinceIdErr] = $(params.since_id).optional.id().$;
if (sinceIdErr) return rej('invalid since_id param');
// Get 'until_id' parameter
const [untilId, untilIdErr] = $(params.until_id).optional.id().$;
if (untilIdErr) return rej('invalid until_id param');
// Get 'since_date' parameter
const [sinceDate, sinceDateErr] = $(params.since_date).optional.number().$;
if (sinceDateErr) throw 'invalid since_date param';
// Get 'until_date' parameter
const [untilDate, untilDateErr] = $(params.until_date).optional.number().$;
if (untilDateErr) throw 'invalid until_date param';
// Check if only one of since_id, until_id, since_date, until_date specified
if ([sinceId, untilId, sinceDate, untilDate].filter(x => x != null).length > 1) {
throw 'only one of since_id, until_id, since_date, until_date can be specified';
}
const q = userId !== undefined
? { _id: userId }
: { username_lower: username.toLowerCase(), host_lower: getHostLower(host) } ;
// Lookup user
const user = await User.findOne(q, {
fields: {
_id: true
}
});
if (user === null) {
return rej('user not found');
}
//#region Construct query
const sort = {
_id: -1
};
const query = {
user_id: user._id
} as any;
if (sinceId) {
sort._id = 1;
query._id = {
$gt: sinceId
};
} else if (untilId) {
query._id = {
$lt: untilId
};
} else if (sinceDate) {
sort._id = 1;
query.created_at = {
$gt: new Date(sinceDate)
};
} else if (untilDate) {
query.created_at = {
$lt: new Date(untilDate)
};
}
if (!includeReplies) {
query.reply_id = null;
}
if (withMedia) {
query.media_ids = {
$exists: true,
$ne: null
};
}
//#endregion
// Issue query
const posts = await Post
.find(query, {
limit: limit,
sort: sort
});
// Serialize
res(await Promise.all(posts.map(async (post) =>
await pack(post, me)
)));
});

View File

@ -0,0 +1,53 @@
/**
* Module dependencies
*/
const ms = require('ms');
import $ from 'cafy';
import User, { pack } from '../../models/user';
import getFriends from '../../common/get-friends';
/**
* Get recommended users
*
* @param {any} params
* @param {any} me
* @return {Promise<any>}
*/
module.exports = (params, me) => new Promise(async (res, rej) => {
// Get 'limit' parameter
const [limit = 10, limitErr] = $(params.limit).optional.number().range(1, 100).$;
if (limitErr) return rej('invalid limit param');
// Get 'offset' parameter
const [offset = 0, offsetErr] = $(params.offset).optional.number().min(0).$;
if (offsetErr) return rej('invalid offset param');
// ID list of the user itself and other users who the user follows
const followingIds = await getFriends(me._id);
const users = await User
.find({
_id: {
$nin: followingIds
},
$or: [
{
'account.last_used_at': {
$gte: new Date(Date.now() - ms('7days'))
}
}, {
host: { $not: null }
}
]
}, {
limit: limit,
skip: offset,
sort: {
followers_count: -1
}
});
// Serialize
res(await Promise.all(users.map(async user =>
await pack(user, me, { detail: true }))));
});

View File

@ -0,0 +1,98 @@
/**
* Module dependencies
*/
import * as mongo from 'mongodb';
import $ from 'cafy';
import User, { pack } from '../../models/user';
import config from '../../../../conf';
const escapeRegexp = require('escape-regexp');
/**
* Search a user
*
* @param {any} params
* @param {any} me
* @return {Promise<any>}
*/
module.exports = (params, me) => new Promise(async (res, rej) => {
// Get 'query' parameter
const [query, queryError] = $(params.query).string().pipe(x => x != '').$;
if (queryError) return rej('invalid query param');
// Get 'offset' parameter
const [offset = 0, offsetErr] = $(params.offset).optional.number().min(0).$;
if (offsetErr) return rej('invalid offset param');
// Get 'max' parameter
const [max = 10, maxErr] = $(params.max).optional.number().range(1, 30).$;
if (maxErr) return rej('invalid max param');
// If Elasticsearch is available, search by $
// If not, search by MongoDB
(config.elasticsearch.enable ? byElasticsearch : byNative)
(res, rej, me, query, offset, max);
});
// Search by MongoDB
async function byNative(res, rej, me, query, offset, max) {
const escapedQuery = escapeRegexp(query);
// Search users
const users = await User
.find({
$or: [{
username_lower: new RegExp(escapedQuery.replace('@', '').toLowerCase())
}, {
name: new RegExp(escapedQuery)
}]
}, {
limit: max
});
// Serialize
res(await Promise.all(users.map(async user =>
await pack(user, me, { detail: true }))));
}
// Search by Elasticsearch
async function byElasticsearch(res, rej, me, query, offset, max) {
const es = require('../../db/elasticsearch');
es.search({
index: 'misskey',
type: 'user',
body: {
size: max,
from: offset,
query: {
simple_query_string: {
fields: ['username', 'name', 'bio'],
query: query,
default_operator: 'and'
}
}
}
}, async (error, response) => {
if (error) {
console.error(error);
return res(500);
}
if (response.hits.total === 0) {
return res([]);
}
const hits = response.hits.hits.map(hit => new mongo.ObjectID(hit._id));
const users = await User
.find({
_id: {
$in: hits
}
});
// Serialize
res(await Promise.all(users.map(async user =>
await pack(user, me, { detail: true }))));
});
}

View File

@ -0,0 +1,38 @@
/**
* Module dependencies
*/
import $ from 'cafy';
import User, { pack } from '../../models/user';
/**
* Search a user by username
*
* @param {any} params
* @param {any} me
* @return {Promise<any>}
*/
module.exports = (params, me) => new Promise(async (res, rej) => {
// Get 'query' parameter
const [query, queryError] = $(params.query).string().$;
if (queryError) return rej('invalid query param');
// Get 'offset' parameter
const [offset = 0, offsetErr] = $(params.offset).optional.number().min(0).$;
if (offsetErr) return rej('invalid offset param');
// Get 'limit' parameter
const [limit = 10, limitErr] = $(params.limit).optional.number().range(1, 100).$;
if (limitErr) return rej('invalid limit param');
const users = await User
.find({
username_lower: new RegExp(query.toLowerCase())
}, {
limit: limit,
skip: offset
});
// Serialize
res(await Promise.all(users.map(async user =>
await pack(user, me, { detail: true }))));
});

View File

@ -0,0 +1,209 @@
/**
* Module dependencies
*/
import $ from 'cafy';
import { JSDOM } from 'jsdom';
import { toUnicode, toASCII } from 'punycode';
import uploadFromUrl from '../../common/drive/upload_from_url';
import User, { pack, validateUsername, isValidName, isValidDescription } from '../../models/user';
const request = require('request-promise-native');
const WebFinger = require('webfinger.js');
const webFinger = new WebFinger({});
async function getCollectionCount(url) {
if (!url) {
return null;
}
try {
const collection = await request({ url, json: true });
return collection ? collection.totalItems : null;
} catch (exception) {
return null;
}
}
function findUser(q) {
return User.findOne(q, {
fields: {
data: false
}
});
}
function webFingerAndVerify(query, verifier) {
return new Promise((res, rej) => webFinger.lookup(query, (error, result) => {
if (error) {
return rej(error);
}
if (result.object.subject.toLowerCase().replace(/^acct:/, '') !== verifier) {
return rej('WebFinger verfification failed');
}
res(result.object);
}));
}
/**
* Show a user
*
* @param {any} params
* @param {any} me
* @return {Promise<any>}
*/
module.exports = (params, me) => new Promise(async (res, rej) => {
let user;
// Get 'user_id' parameter
const [userId, userIdErr] = $(params.user_id).optional.id().$;
if (userIdErr) return rej('invalid user_id param');
// Get 'username' parameter
const [username, usernameErr] = $(params.username).optional.string().$;
if (usernameErr) return rej('invalid username param');
// Get 'host' parameter
const [host, hostErr] = $(params.host).optional.string().$;
if (hostErr) return rej('invalid username param');
if (userId === undefined && typeof username !== 'string') {
return rej('user_id or pair of username and host is required');
}
// Lookup user
if (typeof host === 'string') {
const username_lower = username.toLowerCase();
const host_lower_ascii = toASCII(host).toLowerCase();
const host_lower = toUnicode(host_lower_ascii);
user = await findUser({ username_lower, host_lower });
if (user === null) {
const acct_lower = `${username_lower}@${host_lower_ascii}`;
let activityStreams;
let finger;
let followers_count;
let following_count;
let likes_count;
let posts_count;
if (!validateUsername(username)) {
return rej('username validation failed');
}
try {
finger = await webFingerAndVerify(acct_lower, acct_lower);
} catch (exception) {
return rej('WebFinger lookup failed');
}
const self = finger.links.find(link => link.rel && link.rel.toLowerCase() === 'self');
if (!self) {
return rej('WebFinger has no reference to self representation');
}
try {
activityStreams = await request({
url: self.href,
headers: {
Accept: 'application/activity+json, application/ld+json'
},
json: true
});
} catch (exception) {
return rej('failed to retrieve ActivityStreams representation');
}
if (!(activityStreams &&
(Array.isArray(activityStreams['@context']) ?
activityStreams['@context'].includes('https://www.w3.org/ns/activitystreams') :
activityStreams['@context'] === 'https://www.w3.org/ns/activitystreams') &&
activityStreams.type === 'Person' &&
typeof activityStreams.preferredUsername === 'string' &&
activityStreams.preferredUsername.toLowerCase() === username_lower &&
isValidName(activityStreams.name) &&
isValidDescription(activityStreams.summary)
)) {
return rej('failed ActivityStreams validation');
}
try {
[followers_count, following_count, likes_count, posts_count] = await Promise.all([
getCollectionCount(activityStreams.followers),
getCollectionCount(activityStreams.following),
getCollectionCount(activityStreams.liked),
getCollectionCount(activityStreams.outbox),
webFingerAndVerify(activityStreams.id, acct_lower),
]);
} catch (exception) {
return rej('failed to fetch assets');
}
const summaryDOM = JSDOM.fragment(activityStreams.summary);
// Create user
user = await User.insert({
avatar_id: null,
banner_id: null,
created_at: new Date(),
description: summaryDOM.textContent,
followers_count,
following_count,
name: activityStreams.name,
posts_count,
likes_count,
liked_count: 0,
drive_capacity: 1073741824, // 1GB
username: username,
username_lower,
host: toUnicode(finger.subject.replace(/^.*?@/, '')),
host_lower,
account: {
uri: activityStreams.id,
},
});
const [icon, image] = await Promise.all([
activityStreams.icon,
activityStreams.image,
].map(async image => {
if (!image || image.type !== 'Image') {
return { _id: null };
}
try {
return await uploadFromUrl(image.url, user);
} catch (exception) {
return { _id: null };
}
}));
User.update({ _id: user._id }, {
$set: {
avatar_id: icon._id,
banner_id: image._id,
},
});
user.avatar_id = icon._id;
user.banner_id = icon._id;
}
} else {
const q = userId !== undefined
? { _id: userId }
: { username_lower: username.toLowerCase(), host: null };
user = await findUser(q);
if (user === null) {
return rej('user not found');
}
}
// Send response
res(await pack(user, me, {
detail: true
}));
});