12.120.0-simkey-alpha.7-hotfix

This commit is contained in:
sim1222 2022-12-03 19:12:57 +09:00
commit cfbf3b61d3
No known key found for this signature in database
GPG Key ID: 04EF48D01BEB0298
6 changed files with 764 additions and 1 deletions

View File

@ -9,6 +9,10 @@
You should also include the user name that made the change.
-->
## 12.119.1 (2022/12/03)
### Bugfixes
- Server: Mitigate AP reference chain DoS vector @skehmatics
## 12.119.0 (2022/09/10)
### Improvements

View File

@ -1,6 +1,6 @@
{
"name": "misskey",
"version": "12.120.0-simkey-alpha.6-b1",
"version": "12.120.0-simkey-alpha.7-hotfix",
"codename": "indigo",
"repository": {
"type": "git",

View File

@ -0,0 +1,34 @@
import { CacheableRemoteUser } from '@/models/entities/user.js';
import { getApType, IUpdate, isActor } from '../../type.js';
import { apLogger } from '../../logger.js';
import { updateQuestion } from '../../models/question.js';
import Resolver from '../../resolver.js';
import { updatePerson } from '../../models/person.js';
/**
* Updateアクティビティを捌きます
*/
export default async (actor: CacheableRemoteUser, activity: IUpdate): Promise<string> => {
if ('actor' in activity && actor.uri !== activity.actor) {
return 'skip: invalid actor';
}
apLogger.debug('Update');
const resolver = new Resolver();
const object = await resolver.resolve(activity.object).catch(e => {
apLogger.error(`Resolution failed: ${e}`);
throw e;
});
if (isActor(object)) {
await updatePerson(actor.uri!, resolver, object);
return 'ok: Person updated';
} else if (getApType(object) === 'Question') {
await updateQuestion(object, resolver).catch(e => console.log(e));
return 'ok: Question updated';
} else {
return `skip: Unknown type: ${getApType(object)}`;
}
};

View File

@ -0,0 +1,504 @@
import { URL } from 'node:url';
import promiseLimit from 'promise-limit';
import config from '@/config/index.js';
import { registerOrFetchInstanceDoc } from '@/services/register-or-fetch-instance-doc.js';
import { Note } from '@/models/entities/note.js';
import { updateUsertags } from '@/services/update-hashtag.js';
import { Users, Instances, DriveFiles, Followings, UserProfiles, UserPublickeys } from '@/models/index.js';
import { User, IRemoteUser, CacheableUser } from '@/models/entities/user.js';
import { Emoji } from '@/models/entities/emoji.js';
import { UserNotePining } from '@/models/entities/user-note-pining.js';
import { genId } from '@/misc/gen-id.js';
import { instanceChart, usersChart } from '@/services/chart/index.js';
import { UserPublickey } from '@/models/entities/user-publickey.js';
import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js';
import { toPuny } from '@/misc/convert-host.js';
import { UserProfile } from '@/models/entities/user-profile.js';
import { toArray } from '@/prelude/array.js';
import { fetchInstanceMetadata } from '@/services/fetch-instance-metadata.js';
import { normalizeForSearch } from '@/misc/normalize-for-search.js';
import { truncate } from '@/misc/truncate.js';
import { StatusError } from '@/misc/fetch.js';
import { uriPersonCache } from '@/services/user-cache.js';
import { publishInternalEvent } from '@/services/stream.js';
import { db } from '@/db/postgre.js';
import { apLogger } from '../logger.js';
import { htmlToMfm } from '../misc/html-to-mfm.js';
import { fromHtml } from '../../../mfm/from-html.js';
import { isCollectionOrOrderedCollection, isCollection, IActor, getApId, getOneApHrefNullable, IObject, isPropertyValue, IApPropertyValue, getApType, isActor } from '../type.js';
import Resolver from '../resolver.js';
import { extractApHashtags } from './tag.js';
import { resolveNote, extractEmojis } from './note.js';
import { resolveImage } from './image.js';
const logger = apLogger;
const nameLength = 128;
const summaryLength = 2048;
/**
* Validate and convert to actor object
* @param x Fetched object
* @param uri Fetch target URI
*/
function validateActor(x: IObject, uri: string): IActor {
const expectHost = toPuny(new URL(uri).hostname);
if (x == null) {
throw new Error('invalid Actor: object is null');
}
if (!isActor(x)) {
throw new Error(`invalid Actor type '${x.type}'`);
}
if (!(typeof x.id === 'string' && x.id.length > 0)) {
throw new Error('invalid Actor: wrong id');
}
if (!(typeof x.inbox === 'string' && x.inbox.length > 0)) {
throw new Error('invalid Actor: wrong inbox');
}
if (!(typeof x.preferredUsername === 'string' && x.preferredUsername.length > 0 && x.preferredUsername.length <= 128 && /^\w([\w-.]*\w)?$/.test(x.preferredUsername))) {
throw new Error('invalid Actor: wrong username');
}
// These fields are only informational, and some AP software allows these
// fields to be very long. If they are too long, we cut them off. This way
// we can at least see these users and their activities.
if (x.name) {
if (!(typeof x.name === 'string' && x.name.length > 0)) {
throw new Error('invalid Actor: wrong name');
}
x.name = truncate(x.name, nameLength);
}
if (x.summary) {
if (!(typeof x.summary === 'string' && x.summary.length > 0)) {
throw new Error('invalid Actor: wrong summary');
}
x.summary = truncate(x.summary, summaryLength);
}
const idHost = toPuny(new URL(x.id!).hostname);
if (idHost !== expectHost) {
throw new Error('invalid Actor: id has different host');
}
if (x.publicKey) {
if (typeof x.publicKey.id !== 'string') {
throw new Error('invalid Actor: publicKey.id is not a string');
}
const publicKeyIdHost = toPuny(new URL(x.publicKey.id).hostname);
if (publicKeyIdHost !== expectHost) {
throw new Error('invalid Actor: publicKey.id has different host');
}
}
return x;
}
/**
* Personをフェッチします
*
* Misskeyに対象のPersonが登録されていればそれを返します
*/
export async function fetchPerson(uri: string, resolver?: Resolver): Promise<CacheableUser | null> {
if (typeof uri !== 'string') throw new Error('uri is not string');
const cached = uriPersonCache.get(uri);
if (cached) return cached;
// URIがこのサーバーを指しているならデータベースからフェッチ
if (uri.startsWith(config.url + '/')) {
const id = uri.split('/').pop();
const u = await Users.findOneBy({ id });
if (u) uriPersonCache.set(uri, u);
return u;
}
//#region このサーバーに既に登録されていたらそれを返す
const exist = await Users.findOneBy({ uri });
if (exist) {
uriPersonCache.set(uri, exist);
return exist;
}
//#endregion
return null;
}
/**
* Personを作成します
*/
export async function createPerson(uri: string, resolver?: Resolver): Promise<User> {
if (typeof uri !== 'string') throw new Error('uri is not string');
if (uri.startsWith(config.url)) {
throw new StatusError('cannot resolve local user', 400, 'cannot resolve local user');
}
if (resolver == null) resolver = new Resolver();
const object = await resolver.resolve(uri) as any;
const person = validateActor(object, uri);
logger.info(`Creating the Person: ${person.id}`);
const host = toPuny(new URL(object.id).hostname);
const { fields } = analyzeAttachments(person.attachment || []);
const tags = extractApHashtags(person.tag).map(tag => normalizeForSearch(tag)).splice(0, 32);
const isBot = getApType(object) === 'Service';
const bday = person['vcard:bday']?.match(/^\d{4}-\d{2}-\d{2}/);
// Create user
let user: IRemoteUser;
try {
// Start transaction
await db.transaction(async transactionalEntityManager => {
user = await transactionalEntityManager.save(new User({
id: genId(),
avatarId: null,
bannerId: null,
createdAt: new Date(),
lastFetchedAt: new Date(),
name: truncate(person.name, nameLength),
isLocked: !!person.manuallyApprovesFollowers,
isExplorable: !!person.discoverable,
username: person.preferredUsername,
usernameLower: person.preferredUsername!.toLowerCase(),
host,
inbox: person.inbox,
sharedInbox: person.sharedInbox || (person.endpoints ? person.endpoints.sharedInbox : undefined),
followersUri: person.followers ? getApId(person.followers) : undefined,
featured: person.featured ? getApId(person.featured) : undefined,
uri: person.id,
tags,
isBot,
isCat: (person as any).isCat === true,
showTimelineReplies: false,
})) as IRemoteUser;
await transactionalEntityManager.save(new UserProfile({
userId: user.id,
description: person.summary ? htmlToMfm(truncate(person.summary, summaryLength), person.tag) : null,
url: getOneApHrefNullable(person.url),
fields,
birthday: bday ? bday[0] : null,
location: person['vcard:Address'] || null,
userHost: host,
}));
if (person.publicKey) {
await transactionalEntityManager.save(new UserPublickey({
userId: user.id,
keyId: person.publicKey.id,
keyPem: person.publicKey.publicKeyPem,
}));
}
});
} catch (e) {
// duplicate key error
if (isDuplicateKeyValueError(e)) {
// /users/@a => /users/:id のように入力がaliasなときにエラーになることがあるのを対応
const u = await Users.findOneBy({
uri: person.id,
});
if (u) {
user = u as IRemoteUser;
} else {
throw new Error('already registered');
}
} else {
logger.error(e instanceof Error ? e : new Error(e as string));
throw e;
}
}
// Register host
registerOrFetchInstanceDoc(host).then(i => {
Instances.increment({ id: i.id }, 'usersCount', 1);
instanceChart.newUser(i.host);
fetchInstanceMetadata(i);
});
usersChart.update(user!, true);
// ハッシュタグ更新
updateUsertags(user!, tags);
//#region アバターとヘッダー画像をフェッチ
const [avatar, banner] = await Promise.all([
person.icon,
person.image,
].map(img =>
img == null
? Promise.resolve(null)
: resolveImage(user!, img).catch(() => null),
));
const avatarId = avatar ? avatar.id : null;
const bannerId = banner ? banner.id : null;
await Users.update(user!.id, {
avatarId,
bannerId,
});
user!.avatarId = avatarId;
user!.bannerId = bannerId;
//#endregion
//#region カスタム絵文字取得
const emojis = await extractEmojis(person.tag || [], host).catch(e => {
logger.info(`extractEmojis: ${e}`);
return [] as Emoji[];
});
const emojiNames = emojis.map(emoji => emoji.name);
await Users.update(user!.id, {
emojis: emojiNames,
});
//#endregion
await updateFeatured(user!.id, resolver).catch(err => logger.error(err));
return user!;
}
/**
* Personの情報を更新します
* Misskeyに対象のPersonが登録されていなければ無視します
* @param uri URI of Person
* @param resolver Resolver
* @param hint Hint of Person object (Personの場合Remote resolveをせずに更新に利用します)
*/
export async function updatePerson(uri: string, resolver?: Resolver | null, hint?: IObject): Promise<void> {
if (typeof uri !== 'string') throw new Error('uri is not string');
// URIがこのサーバーを指しているならスキップ
if (uri.startsWith(config.url + '/')) {
return;
}
//#region このサーバーに既に登録されているか
const exist = await Users.findOneBy({ uri }) as IRemoteUser;
if (exist == null) {
return;
}
//#endregion
if (resolver == null) resolver = new Resolver();
const object = hint || await resolver.resolve(uri);
const person = validateActor(object, uri);
logger.info(`Updating the Person: ${person.id}`);
// アバターとヘッダー画像をフェッチ
const [avatar, banner] = await Promise.all([
person.icon,
person.image,
].map(img =>
img == null
? Promise.resolve(null)
: resolveImage(exist, img).catch(() => null),
));
// カスタム絵文字取得
const emojis = await extractEmojis(person.tag || [], exist.host).catch(e => {
logger.info(`extractEmojis: ${e}`);
return [] as Emoji[];
});
const emojiNames = emojis.map(emoji => emoji.name);
const { fields } = analyzeAttachments(person.attachment || []);
const tags = extractApHashtags(person.tag).map(tag => normalizeForSearch(tag)).splice(0, 32);
const bday = person['vcard:bday']?.match(/^\d{4}-\d{2}-\d{2}/);
const updates = {
lastFetchedAt: new Date(),
inbox: person.inbox,
sharedInbox: person.sharedInbox || (person.endpoints ? person.endpoints.sharedInbox : undefined),
followersUri: person.followers ? getApId(person.followers) : undefined,
featured: person.featured,
emojis: emojiNames,
name: truncate(person.name, nameLength),
tags,
isBot: getApType(object) === 'Service',
isCat: (person as any).isCat === true,
isLocked: !!person.manuallyApprovesFollowers,
isExplorable: !!person.discoverable,
} as Partial<User>;
if (avatar) {
updates.avatarId = avatar.id;
}
if (banner) {
updates.bannerId = banner.id;
}
// Update user
await Users.update(exist.id, updates);
if (person.publicKey) {
await UserPublickeys.update({ userId: exist.id }, {
keyId: person.publicKey.id,
keyPem: person.publicKey.publicKeyPem,
});
}
await UserProfiles.update({ userId: exist.id }, {
url: getOneApHrefNullable(person.url),
fields,
description: person.summary ? htmlToMfm(truncate(person.summary, summaryLength), person.tag) : null,
birthday: bday ? bday[0] : null,
location: person['vcard:Address'] || null,
});
publishInternalEvent('remoteUserUpdated', { id: exist.id });
// ハッシュタグ更新
updateUsertags(exist, tags);
// 該当ユーザーが既にフォロワーになっていた場合はFollowingもアップデートする
await Followings.update({
followerId: exist.id,
}, {
followerSharedInbox: person.sharedInbox || (person.endpoints ? person.endpoints.sharedInbox : undefined),
});
await updateFeatured(exist.id, resolver).catch(err => logger.error(err));
}
/**
* Personを解決します
*
* Misskeyに対象のPersonが登録されていればそれを返し
* Misskeyに登録しそれを返します
*/
export async function resolvePerson(uri: string, resolver?: Resolver): Promise<CacheableUser> {
if (typeof uri !== 'string') throw new Error('uri is not string');
//#region このサーバーに既に登録されていたらそれを返す
const exist = await fetchPerson(uri);
if (exist) {
return exist;
}
//#endregion
// リモートサーバーからフェッチしてきて登録
if (resolver == null) resolver = new Resolver();
return await createPerson(uri, resolver);
}
const services: {
[x: string]: (id: string, username: string) => any
} = {
'misskey:authentication:twitter': (userId, screenName) => ({ userId, screenName }),
'misskey:authentication:github': (id, login) => ({ id, login }),
'misskey:authentication:discord': (id, name) => $discord(id, name),
};
const $discord = (id: string, name: string) => {
if (typeof name !== 'string') {
name = 'unknown#0000';
}
const [username, discriminator] = name.split('#');
return { id, username, discriminator };
};
function addService(target: { [x: string]: any }, source: IApPropertyValue) {
const service = services[source.name];
if (typeof source.value !== 'string') {
source.value = 'unknown';
}
const [id, username] = source.value.split('@');
if (service) {
target[source.name.split(':')[2]] = service(id, username);
}
}
export function analyzeAttachments(attachments: IObject | IObject[] | undefined) {
const fields: {
name: string,
value: string
}[] = [];
const services: { [x: string]: any } = {};
if (Array.isArray(attachments)) {
for (const attachment of attachments.filter(isPropertyValue)) {
if (isPropertyValue(attachment.identifier)) {
addService(services, attachment.identifier);
} else {
fields.push({
name: attachment.name,
value: fromHtml(attachment.value),
});
}
}
}
return { fields, services };
}
export async function updateFeatured(userId: User['id'], resolver?: Resolver) {
const user = await Users.findOneByOrFail({ id: userId });
if (!Users.isRemoteUser(user)) return;
if (!user.featured) return;
logger.info(`Updating the featured: ${user.uri}`);
if (resolver == null) resolver = new Resolver();
// Resolve to (Ordered)Collection Object
const collection = await resolver.resolveCollection(user.featured);
if (!isCollectionOrOrderedCollection(collection)) throw new Error('Object is not Collection or OrderedCollection');
// Resolve to Object(may be Note) arrays
const unresolvedItems = isCollection(collection) ? collection.items : collection.orderedItems;
const items = await Promise.all(toArray(unresolvedItems).map(x => resolver.resolve(x)));
// Resolve and regist Notes
const limit = promiseLimit<Note | null>(2);
const featuredNotes = await Promise.all(items
.filter(item => getApType(item) === 'Note') // TODO: Noteでなくてもいいかも
.slice(0, 5)
.map(item => limit(() => resolveNote(item, resolver))));
await db.transaction(async transactionalEntityManager => {
await transactionalEntityManager.delete(UserNotePining, { userId: user.id });
// とりあえずidを別の時間で生成して順番を維持
let td = 0;
for (const note of featuredNotes.filter(note => note != null)) {
td -= 1000;
transactionalEntityManager.insert(UserNotePining, {
id: genId(new Date(Date.now() + td)),
createdAt: new Date(),
userId: user.id,
noteId: note!.id,
});
}
});
}

View File

@ -0,0 +1,83 @@
import config from '@/config/index.js';
import { Notes, Polls } from '@/models/index.js';
import { IPoll } from '@/models/entities/poll.js';
import Resolver from '../resolver.js';
import { IObject, IQuestion, isQuestion } from '../type.js';
import { apLogger } from '../logger.js';
export async function extractPollFromQuestion(source: string | IObject, resolver?: Resolver): Promise<IPoll> {
if (resolver == null) resolver = new Resolver();
const question = await resolver.resolve(source);
if (!isQuestion(question)) {
throw new Error('invalid type');
}
const multiple = !question.oneOf;
const expiresAt = question.endTime ? new Date(question.endTime) : question.closed ? new Date(question.closed) : null;
if (multiple && !question.anyOf) {
throw new Error('invalid question');
}
const choices = question[multiple ? 'anyOf' : 'oneOf']!
.map((x, i) => x.name!);
const votes = question[multiple ? 'anyOf' : 'oneOf']!
.map((x, i) => x.replies && x.replies.totalItems || x._misskey_votes || 0);
return {
choices,
votes,
multiple,
expiresAt,
};
}
/**
* Update votes of Question
* @param uri URI of AP Question object
* @returns true if updated
*/
export async function updateQuestion(value: any, resolver?: Resolver) {
const uri = typeof value === 'string' ? value : value.id;
// URIがこのサーバーを指しているならスキップ
if (uri.startsWith(config.url + '/')) throw new Error('uri points local');
//#region このサーバーに既に登録されているか
const note = await Notes.findOneBy({ uri });
if (note == null) throw new Error('Question is not registed');
const poll = await Polls.findOneBy({ noteId: note.id });
if (poll == null) throw new Error('Question is not registed');
//#endregion
// resolve new Question object
if (resolver == null) resolver = new Resolver();
const question = await resolver.resolve(value) as IQuestion;
apLogger.debug(`fetched question: ${JSON.stringify(question, null, 2)}`);
if (question.type !== 'Question') throw new Error('object is not a Question');
const apChoices = question.oneOf || question.anyOf;
let changed = false;
for (const choice of poll.choices) {
const oldCount = poll.votes[poll.choices.indexOf(choice)];
const newCount = apChoices!.filter(ap => ap.name === choice)[0].replies!.totalItems;
if (oldCount !== newCount) {
changed = true;
poll.votes[poll.choices.indexOf(choice)] = newCount;
}
}
await Polls.update({ noteId: note.id }, {
votes: poll.votes,
});
return changed;
}

View File

@ -0,0 +1,138 @@
import config from '@/config/index.js';
import { getJson } from '@/misc/fetch.js';
import { ILocalUser } from '@/models/entities/user.js';
import { getInstanceActor } from '@/services/instance-actor.js';
import { fetchMeta } from '@/misc/fetch-meta.js';
import { extractDbHost, isSelfHost } from '@/misc/convert-host.js';
import { FollowRequests, Notes, NoteReactions, Polls, Users } from '@/models/index.js';
import renderNote from '@/remote/activitypub/renderer/note.js';
import { renderLike } from '@/remote/activitypub/renderer/like.js';
import { renderPerson } from '@/remote/activitypub/renderer/person.js';
import renderQuestion from '@/remote/activitypub/renderer/question.js';
import renderCreate from '@/remote/activitypub/renderer/create.js';
import { renderActivity } from '@/remote/activitypub/renderer/index.js';
import renderFollow from '@/remote/activitypub/renderer/follow.js';
import { parseUri } from './db-resolver.js';
import { IObject, isCollectionOrOrderedCollection, ICollection, IOrderedCollection } from './type.js';
import { signedGet } from './request.js';
export default class Resolver {
private history: Set<string>;
private user?: ILocalUser;
private recursionLimit?: number;
constructor(recursionLimit = 100) {
this.history = new Set();
}
public getHistory(): string[] {
return Array.from(this.history);
}
public async resolveCollection(value: string | IObject): Promise<ICollection | IOrderedCollection> {
const collection = typeof value === 'string'
? await this.resolve(value)
: value;
if (isCollectionOrOrderedCollection(collection)) {
return collection;
} else {
throw new Error(`unrecognized collection type: ${collection.type}`);
}
}
public async resolve(value: string | IObject): Promise<IObject> {
if (value == null) {
throw new Error('resolvee is null (or undefined)');
}
if (typeof value !== 'string') {
return value;
}
if (value.includes('#')) {
// URLs with fragment parts cannot be resolved correctly because
// the fragment part does not get transmitted over HTTP(S).
// Avoid strange behaviour by not trying to resolve these at all.
throw new Error(`cannot resolve URL with fragment: ${value}`);
}
if (this.history.has(value)) {
throw new Error('cannot resolve already resolved one');
}
if (this.recursionLimit && this.history.size > this.recursionLimit) {
throw new Error('hit recursion limit');
}
this.history.add(value);
const host = extractDbHost(value);
if (isSelfHost(host)) {
return await this.resolveLocal(value);
}
const meta = await fetchMeta();
if (meta.blockedHosts.includes(host)) {
throw new Error('Instance is blocked');
}
if (config.signToActivityPubGet && !this.user) {
this.user = await getInstanceActor();
}
const object = (this.user
? await signedGet(value, this.user)
: await getJson(value, 'application/activity+json, application/ld+json')) as IObject;
if (object == null || (
Array.isArray(object['@context']) ?
!(object['@context'] as unknown[]).includes('https://www.w3.org/ns/activitystreams') :
object['@context'] !== 'https://www.w3.org/ns/activitystreams'
)) {
throw new Error('invalid response');
}
return object;
}
private resolveLocal(url: string): Promise<IObject> {
const parsed = parseUri(url);
if (!parsed.local) throw new Error('resolveLocal: not local');
switch (parsed.type) {
case 'notes':
return Notes.findOneByOrFail({ id: parsed.id })
.then(note => {
if (parsed.rest === 'activity') {
// this refers to the create activity and not the note itself
return renderActivity(renderCreate(renderNote(note)));
} else {
return renderNote(note);
}
});
case 'users':
return Users.findOneByOrFail({ id: parsed.id })
.then(user => renderPerson(user as ILocalUser));
case 'questions':
// Polls are indexed by the note they are attached to.
return Promise.all([
Notes.findOneByOrFail({ id: parsed.id }),
Polls.findOneByOrFail({ noteId: parsed.id }),
])
.then(([note, poll]) => renderQuestion({ id: note.userId }, note, poll));
case 'likes':
return NoteReactions.findOneByOrFail({ id: parsed.id }).then(reaction => renderActivity(renderLike(reaction, { uri: null })));
case 'follows':
// rest should be <followee id>
if (parsed.rest == null || !/^\w+$/.test(parsed.rest)) throw new Error('resolveLocal: invalid follow URI');
return Promise.all(
[parsed.id, parsed.rest].map(id => Users.findOneByOrFail({ id })),
)
.then(([follower, followee]) => renderActivity(renderFollow(follower, followee, url)));
default:
throw new Error(`resolveLocal: type ${type} unhandled`);
}
}
}