mirror of
https://github.com/misskey-dev/summaly.git
synced 2025-08-06 16:23:57 +09:00
Compare commits
29 Commits
Author | SHA1 | Date | |
---|---|---|---|
fce03ebd3c | |||
f7b59909bd | |||
5582cc8c9b | |||
7ebfe151d5 | |||
b0818bed3e | |||
909774bc63 | |||
56e1137113 | |||
441c744a35 | |||
632cd903ee | |||
45153b4f08 | |||
9e3f23691c | |||
6d4ad44de3 | |||
23d5e03ba2 | |||
dfe6451012 | |||
1b93243ff9 | |||
ee06d841c2 | |||
e9547a556a | |||
71a6aefb8e | |||
fc9f7db477 | |||
cc7ae8d00a | |||
7fbab86441 | |||
1b541a1418 | |||
3e09d27613 | |||
509a35abe2 | |||
79d9eadd58 | |||
c261071a82 | |||
71fe234d3e | |||
718465e498 | |||
77c53be159 |
122
.eslintrc.cjs
122
.eslintrc.cjs
@ -1,122 +0,0 @@
|
||||
module.exports = {
|
||||
root: true,
|
||||
parser: '@typescript-eslint/parser',
|
||||
parserOptions: {
|
||||
tsconfigRootDir: __dirname,
|
||||
project: ['./tsconfig.json'],
|
||||
},
|
||||
plugins: [
|
||||
'@typescript-eslint',
|
||||
'import'
|
||||
],
|
||||
extends: [
|
||||
'eslint:recommended',
|
||||
'plugin:@typescript-eslint/recommended',
|
||||
'plugin:import/recommended',
|
||||
'plugin:import/typescript'
|
||||
],
|
||||
rules: {
|
||||
'indent': ['warn', 'tab', {
|
||||
'SwitchCase': 1,
|
||||
'MemberExpression': 1,
|
||||
'flatTernaryExpressions': true,
|
||||
'ArrayExpression': 'first',
|
||||
'ObjectExpression': 'first',
|
||||
}],
|
||||
'eol-last': ['error', 'always'],
|
||||
'semi': ['error', 'always'],
|
||||
'semi-spacing': ['error', { 'before': false, 'after': true }],
|
||||
'quotes': ['warn', 'single'],
|
||||
'comma-dangle': ['warn', 'always-multiline'],
|
||||
'comma-spacing': ['error', { 'before': false, 'after': true }],
|
||||
'array-bracket-spacing': ['error', 'never'],
|
||||
'keyword-spacing': ['error', {
|
||||
'before': true,
|
||||
'after': true,
|
||||
}],
|
||||
'key-spacing': ['error', {
|
||||
'beforeColon': false,
|
||||
'afterColon': true,
|
||||
}],
|
||||
'arrow-spacing': ['error', {
|
||||
'before': true,
|
||||
'after': true,
|
||||
}],
|
||||
'brace-style': ['error', '1tbs', {
|
||||
'allowSingleLine': true,
|
||||
}],
|
||||
'padded-blocks': ['error', 'never'],
|
||||
/* TODO: path aliasを使わないとwarnする
|
||||
'no-restricted-imports': ['warn', {
|
||||
'patterns': [
|
||||
]
|
||||
}],
|
||||
*/
|
||||
'eqeqeq': ['error', 'always', { 'null': 'ignore' }],
|
||||
'no-multi-spaces': ['error'],
|
||||
'no-var': ['error'],
|
||||
'prefer-arrow-callback': ['error'],
|
||||
'no-throw-literal': ['error'],
|
||||
'no-param-reassign': ['warn'],
|
||||
'no-constant-condition': ['warn'],
|
||||
'no-empty-pattern': ['warn'],
|
||||
'no-async-promise-executor': ['off'],
|
||||
'no-useless-escape': ['off'],
|
||||
'no-multiple-empty-lines': ['error', { 'max': 1 }],
|
||||
'no-control-regex': ['warn'],
|
||||
'no-empty': ['warn'],
|
||||
'no-inner-declarations': ['off'],
|
||||
'no-sparse-arrays': ['off'],
|
||||
'nonblock-statement-body-position': ['error', 'beside'],
|
||||
'object-curly-spacing': ['error', 'always'],
|
||||
'space-infix-ops': ['error'],
|
||||
'space-before-blocks': ['error', 'always'],
|
||||
'padding-line-between-statements': [
|
||||
'error',
|
||||
{ 'blankLine': 'always', 'prev': 'function', 'next': '*' },
|
||||
{ 'blankLine': 'always', 'prev': '*', 'next': 'function' },
|
||||
],
|
||||
"lines-between-class-members": "off",
|
||||
/* typescript-eslint では enforce に対応してないっぽい
|
||||
'@typescript-eslint/lines-between-class-members': ['error', {
|
||||
enforce: [{
|
||||
blankLine: 'always',
|
||||
prev: 'method',
|
||||
next: '*',
|
||||
}]
|
||||
}],
|
||||
*/
|
||||
'@typescript-eslint/func-call-spacing': ['error', 'never'],
|
||||
'@typescript-eslint/no-explicit-any': ['warn'],
|
||||
'@typescript-eslint/no-unused-vars': ['warn'],
|
||||
'@typescript-eslint/no-unnecessary-condition': ['warn'],
|
||||
'@typescript-eslint/no-var-requires': ['warn'],
|
||||
'@typescript-eslint/no-inferrable-types': ['warn'],
|
||||
'@typescript-eslint/no-empty-function': ['off'],
|
||||
'@typescript-eslint/no-non-null-assertion': ['warn'],
|
||||
'@typescript-eslint/explicit-function-return-type': ['off'],
|
||||
'@typescript-eslint/no-misused-promises': ['error', {
|
||||
'checksVoidReturn': false,
|
||||
}],
|
||||
'@typescript-eslint/consistent-type-imports': 'off',
|
||||
'@typescript-eslint/prefer-nullish-coalescing': [
|
||||
'warn',
|
||||
],
|
||||
'@typescript-eslint/naming-convention': [
|
||||
'error',
|
||||
{
|
||||
"selector": "typeLike",
|
||||
"format": ["PascalCase"]
|
||||
},
|
||||
{
|
||||
"selector": "typeParameter",
|
||||
"format": []
|
||||
}
|
||||
],
|
||||
'import/no-unresolved': ['off'],
|
||||
'import/no-default-export': ['warn'],
|
||||
'import/order': ['warn', {
|
||||
'groups': ['builtin', 'external', 'internal', 'parent', 'sibling', 'index', 'object', 'type'],
|
||||
}]
|
||||
},
|
||||
};
|
2
.github/workflows/npm-publish.yml
vendored
2
.github/workflows/npm-publish.yml
vendored
@ -30,6 +30,7 @@ jobs:
|
||||
with:
|
||||
node-version: ${{ matrix.node-version }}
|
||||
cache: 'pnpm'
|
||||
registry-url: 'https://registry.npmjs.org'
|
||||
- name: Publish package
|
||||
run: |
|
||||
corepack enable
|
||||
@ -38,3 +39,4 @@ jobs:
|
||||
pnpm publish --access public --no-git-checks --provenance
|
||||
env:
|
||||
NODE_AUTH_TOKEN: ${{ secrets.NODE_AUTH_TOKEN }}
|
||||
NPM_TOKEN: ${{ secrets.NODE_AUTH_TOKEN }}
|
||||
|
2
.github/workflows/test.yml
vendored
2
.github/workflows/test.yml
vendored
@ -31,4 +31,4 @@ jobs:
|
||||
pnpm build
|
||||
- name: Test
|
||||
run: |
|
||||
pnpm test
|
||||
SKIP_NETWORK_TEST=true pnpm test
|
||||
|
24
.swcrc
24
.swcrc
@ -1,24 +0,0 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/swcrc",
|
||||
"jsc": {
|
||||
"parser": {
|
||||
"syntax": "typescript",
|
||||
"dynamicImport": true,
|
||||
"decorators": true
|
||||
},
|
||||
"transform": {
|
||||
"legacyDecorator": true,
|
||||
"decoratorMetadata": true
|
||||
},
|
||||
"experimental": {
|
||||
"keepImportAssertions": true
|
||||
},
|
||||
"baseUrl": ".",
|
||||
"paths": {},
|
||||
"target": "es2021"
|
||||
},
|
||||
"module": {
|
||||
"type": "es6"
|
||||
},
|
||||
"minify": true
|
||||
}
|
45
CHANGELOG.md
45
CHANGELOG.md
@ -1,4 +1,47 @@
|
||||
5.0.0 / 2023-12-30
|
||||
(unreleased)
|
||||
------------------
|
||||
|
||||
5.2.3 / 2025/07/19
|
||||
------------------
|
||||
* パッケージが使用できない問題を修正
|
||||
|
||||
5.2.2 / 2025/07/06
|
||||
------------------
|
||||
* 最初のHEADリクエストにUAが反映されない問題を修正
|
||||
* 依存関係の更新
|
||||
* テストスイートをVitestに変更
|
||||
|
||||
5.2.1 / 2025/04/28
|
||||
------------------
|
||||
* セキュリティに関する修正
|
||||
|
||||
5.2.0 / 2025/02/05
|
||||
------------------
|
||||
* センシティブフラグの判定を `<meta property="rating">` および `rating` ヘッダでも行うように
|
||||
* Bluesky(bsky.app)のプレビューに対応
|
||||
* `fediverse:creator` のパースに対応
|
||||
* 依存関係の更新
|
||||
* eslintの設定を更新
|
||||
|
||||
5.1.0 / 2024-03-18
|
||||
------------------
|
||||
* GETリクエストよりも前にHEADリクエストを送信し、その結果を使用して検証するように (#22)
|
||||
* 下記のパラメータを`summaly`メソッドのオプションに追加
|
||||
- userAgent
|
||||
- responseTimeout
|
||||
- operationTimeout
|
||||
- contentLengthLimit
|
||||
- contentLengthRequired
|
||||
|
||||
5.0.3 / 2023-12-30
|
||||
------------------
|
||||
* Fix .github/workflows/npm-publish.yml
|
||||
|
||||
5.0.2 / 2023-12-30
|
||||
------------------
|
||||
* Fix .github/workflows/npm-publish.yml
|
||||
|
||||
5.0.1 / 2023-12-30
|
||||
------------------
|
||||
* Fix .github/workflows/npm-publish.yml
|
||||
|
||||
|
54
README.md
54
README.md
@ -43,12 +43,17 @@ npm run serve
|
||||
|
||||
#### opts (SummalyOptions)
|
||||
|
||||
| Property | Type | Description | Default |
|
||||
| :------------------ | :--------------------- | :------------------------------ | :------ |
|
||||
| **lang** | *string* | Accept-Language for the request | `null` |
|
||||
| **followRedirects** | *boolean* | Whether follow redirects | `true` |
|
||||
| **plugins** | *plugin[]* (see below) | Custom plugins | `null` |
|
||||
| **agent** | *Got.Agents* | Custom HTTP agent (see below) | `null` |
|
||||
| Property | Type | Description | Default |
|
||||
|:--------------------------|:-----------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-----------------------|
|
||||
| **lang** | *string* | Accept-Language for the request | `null` |
|
||||
| **followRedirects** | *boolean* | Whether follow redirects | `true` |
|
||||
| **plugins** | *plugin[]* (see below) | Custom plugins | `null` |
|
||||
| **agent** | *Got.Agents* | Custom HTTP agent (see below) | `null` |
|
||||
| **userAgent** | *string* | User-Agent for the request | `SummalyBot/[version]` |
|
||||
| **responseTimeout** | *number* | Set timeouts for each phase, such as host name resolution and socket communication. | `20000` |
|
||||
| **operationTimeout** | *number* | Set the timeout from the start to the end of the request. | `60000` |
|
||||
| **contentLengthLimit** | *number* | If set to true, an error will occur if the content-length value returned from the other server is larger than this parameter (or if the received body size exceeds this parameter). | `10485760` |
|
||||
| **contentLengthRequired** | *boolean* | If set to true, it will be an error if the other server does not return content-length. | `false` |
|
||||
|
||||
#### Plugin
|
||||
|
||||
@ -78,17 +83,18 @@ A Promise of an Object that contains properties below:
|
||||
|
||||
#### SummalyResult
|
||||
|
||||
| Property | Type | Description |
|
||||
| :-------------- | :------- | :------------------------------------------ |
|
||||
| **title** | *string* \| *null* | The title of the web page |
|
||||
| **icon** | *string* \| *null* | The url of the icon of the web page |
|
||||
| **description** | *string* \| *null* | The description of the web page |
|
||||
| **thumbnail** | *string* \| *null* | The url of the thumbnail of the web page |
|
||||
| **sitename** | *string* \| *null* | The name of the web site |
|
||||
| **player** | *Player* | The player of the web page |
|
||||
| **sensitive** | *boolean* | Whether the url is sensitive |
|
||||
| **activityPub** | *string* \| *null* | The url of the ActivityPub representation of that web page |
|
||||
| **url** | *string* | The url of the web page |
|
||||
| Property | Type | Description |
|
||||
|:----------------|:-------------------|:-----------------------------------------------------------|
|
||||
| **title** | *string* \| *null* | The title of the web page |
|
||||
| **icon** | *string* \| *null* | The url of the icon of the web page |
|
||||
| **description** | *string* \| *null* | The description of the web page |
|
||||
| **thumbnail** | *string* \| *null* | The url of the thumbnail of the web page |
|
||||
| **sitename** | *string* \| *null* | The name of the web site |
|
||||
| **player** | *Player* | The player of the web page |
|
||||
| **sensitive** | *boolean* | Whether the url is sensitive |
|
||||
| **activityPub** | *string* \| *null* | The url of the ActivityPub representation of that web page |
|
||||
| **fediverseCreator** | *string* \| *null* | The pages fediverse handle |
|
||||
| **url** | *string* | The url of the web page |
|
||||
|
||||
#### Summary
|
||||
|
||||
@ -96,12 +102,12 @@ A Promise of an Object that contains properties below:
|
||||
|
||||
#### Player
|
||||
|
||||
| Property | Type | Description |
|
||||
| :-------------- | :--------- | :---------------------------------------------- |
|
||||
| **url** | *string* \| *null* | The url of the player |
|
||||
| **width** | *number* \| *null* | The width of the player |
|
||||
| **height** | *number* \| *null* | The height of the player |
|
||||
| **allow** | *string[]* | The names of the allowed permissions for iframe |
|
||||
| Property | Type | Description |
|
||||
|:-----------|:-------------------|:------------------------------------------------|
|
||||
| **url** | *string* \| *null* | The url of the player |
|
||||
| **width** | *number* \| *null* | The width of the player |
|
||||
| **height** | *number* \| *null* | The height of the player |
|
||||
| **allow** | *string[]* | The names of the allowed permissions for iframe |
|
||||
|
||||
Currently the possible items in `allow` are:
|
||||
|
||||
@ -129,7 +135,7 @@ will be ... ↓
|
||||
```json
|
||||
{
|
||||
"title": "【アイドルマスター】「Stage Bye Stage」(歌:島村卯月、渋谷凛、本田未央)",
|
||||
"icon": "https://www.youtube.com/s/desktop/28b0985e/img/favicon.ico",
|
||||
"icon": "https://www.youtube.com/s/desktop/711fd789/img/logos/favicon.ico",
|
||||
"description": "Website▶https://columbia.jp/idolmaster/Playlist▶https://www.youtube.com/playlist?list=PL83A2998CF3BBC86D2018年7月18日発売予定THE IDOLM@STER CINDERELLA GIRLS CG STAR...",
|
||||
"thumbnail": "https://i.ytimg.com/vi/NMIEAhH_fTU/maxresdefault.jpg",
|
||||
"player": {
|
||||
|
38
eslint.config.js
Normal file
38
eslint.config.js
Normal file
@ -0,0 +1,38 @@
|
||||
import pluginMisskey from '@misskey-dev/eslint-plugin';
|
||||
import tsParser from '@typescript-eslint/parser';
|
||||
|
||||
//@ts-check
|
||||
/** @type {import('eslint').Linter.Config[]} */
|
||||
export default [ // eslint-disable-line import/no-default-export
|
||||
...pluginMisskey.configs['recommended'],
|
||||
{
|
||||
ignores: [
|
||||
'**/node_modules',
|
||||
'src/@types/package.json.d.ts',
|
||||
'built',
|
||||
'vitest.config.ts',
|
||||
'test',
|
||||
],
|
||||
},
|
||||
{
|
||||
files: ['**/*.ts', '**/*.tsx'],
|
||||
languageOptions: {
|
||||
parserOptions: {
|
||||
parser: tsParser,
|
||||
project: ['./tsconfig.json', './test/tsconfig.json'],
|
||||
sourceType: 'module',
|
||||
tsConfigRootDir: import.meta.dirname,
|
||||
},
|
||||
},
|
||||
rules: {
|
||||
// 空文字でもフォールバックしたいので無効
|
||||
'@typescript-eslint/prefer-nullish-coalescing': 'off',
|
||||
},
|
||||
},
|
||||
{
|
||||
files: ['**/*.js', '**/*.cjs'],
|
||||
rules: {
|
||||
'@typescript-eslint/no-var-requires': 'off',
|
||||
},
|
||||
},
|
||||
];
|
208
jest.config.js
208
jest.config.js
@ -1,208 +0,0 @@
|
||||
/*
|
||||
* For a detailed explanation regarding each configuration property and type check, visit:
|
||||
* https://jestjs.io/docs/en/configuration.html
|
||||
*/
|
||||
|
||||
export default {
|
||||
// All imported modules in your tests should be mocked automatically
|
||||
// automock: false,
|
||||
|
||||
// Stop running tests after `n` failures
|
||||
// bail: 0,
|
||||
|
||||
// The directory where Jest should store its cached dependency information
|
||||
// cacheDirectory: "C:\\Users\\ai\\AppData\\Local\\Temp\\jest",
|
||||
|
||||
// Automatically clear mock calls and instances between every test
|
||||
// clearMocks: false,
|
||||
|
||||
// Indicates whether the coverage information should be collected while executing the test
|
||||
// collectCoverage: false,
|
||||
|
||||
// An array of glob patterns indicating a set of files for which coverage information should be collected
|
||||
collectCoverageFrom: ['src/**/*.ts'],
|
||||
|
||||
// The directory where Jest should output its coverage files
|
||||
coverageDirectory: "coverage",
|
||||
|
||||
// An array of regexp pattern strings used to skip coverage collection
|
||||
// coveragePathIgnorePatterns: [
|
||||
// "\\\\node_modules\\\\"
|
||||
// ],
|
||||
|
||||
// Indicates which provider should be used to instrument code for coverage
|
||||
coverageProvider: "v8",
|
||||
|
||||
// A list of reporter names that Jest uses when writing coverage reports
|
||||
// coverageReporters: [
|
||||
// "json",
|
||||
// "text",
|
||||
// "lcov",
|
||||
// "clover"
|
||||
// ],
|
||||
|
||||
// An object that configures minimum threshold enforcement for coverage results
|
||||
// coverageThreshold: undefined,
|
||||
|
||||
// A path to a custom dependency extractor
|
||||
// dependencyExtractor: undefined,
|
||||
|
||||
// Make calling deprecated APIs throw helpful error messages
|
||||
// errorOnDeprecated: false,
|
||||
|
||||
// Force coverage collection from ignored files using an array of glob patterns
|
||||
// forceCoverageMatch: [],
|
||||
|
||||
// A path to a module which exports an async function that is triggered once before all test suites
|
||||
// globalSetup: undefined,
|
||||
|
||||
// A path to a module which exports an async function that is triggered once after all test suites
|
||||
// globalTeardown: undefined,
|
||||
|
||||
// A set of global variables that need to be available in all test environments
|
||||
globals: {
|
||||
},
|
||||
|
||||
// The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers.
|
||||
// maxWorkers: "50%",
|
||||
|
||||
// An array of directory names to be searched recursively up from the requiring module's location
|
||||
// moduleDirectories: [
|
||||
// "node_modules"
|
||||
// ],
|
||||
|
||||
// An array of file extensions your modules use
|
||||
// moduleFileExtensions: [
|
||||
// "js",
|
||||
// "json",
|
||||
// "jsx",
|
||||
// "ts",
|
||||
// "tsx",
|
||||
// "node"
|
||||
// ],
|
||||
|
||||
// A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
|
||||
moduleNameMapper: {
|
||||
// Do not resolve .wasm.js to .wasm by the rule below
|
||||
'^(.+)\\.wasm\\.js$': '$1.wasm.js',
|
||||
// SWC converts @/foo/bar.js to `../../src/foo/bar.js`, and then this rule
|
||||
// converts it again to `../../src/foo/bar` which then can be resolved to
|
||||
// `.ts` files.
|
||||
// See https://github.com/swc-project/jest/issues/64#issuecomment-1029753225
|
||||
// TODO: Use `--allowImportingTsExtensions` on TypeScript 5.0 so that we can
|
||||
// directly import `.ts` files without this hack.
|
||||
'^(\\.{1,2}/.*)\\.js$': '$1',
|
||||
},
|
||||
|
||||
// An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
|
||||
// modulePathIgnorePatterns: [],
|
||||
|
||||
// Activates notifications for test results
|
||||
// notify: false,
|
||||
|
||||
// An enum that specifies notification mode. Requires { notify: true }
|
||||
// notifyMode: "failure-change",
|
||||
|
||||
// A preset that is used as a base for Jest's configuration
|
||||
//preset: "ts-jest/presets/js-with-ts-esm",
|
||||
|
||||
// Run tests from one or more projects
|
||||
// projects: undefined,
|
||||
|
||||
// Use this configuration option to add custom reporters to Jest
|
||||
// reporters: undefined,
|
||||
|
||||
// Automatically reset mock state between every test
|
||||
// resetMocks: false,
|
||||
|
||||
// Reset the module registry before running each individual test
|
||||
// resetModules: false,
|
||||
|
||||
// A path to a custom resolver
|
||||
// resolver: './jest-resolver.cjs',
|
||||
|
||||
// Automatically restore mock state between every test
|
||||
restoreMocks: true,
|
||||
|
||||
// The root directory that Jest should scan for tests and modules within
|
||||
// rootDir: undefined,
|
||||
|
||||
// A list of paths to directories that Jest should use to search for files in
|
||||
roots: [
|
||||
"<rootDir>"
|
||||
],
|
||||
|
||||
// Allows you to use a custom runner instead of Jest's default test runner
|
||||
// runner: "jest-runner",
|
||||
|
||||
// The paths to modules that run some code to configure or set up the testing environment before each test
|
||||
// setupFiles: [],
|
||||
|
||||
// A list of paths to modules that run some code to configure or set up the testing framework before each test
|
||||
// setupFilesAfterEnv: [],
|
||||
|
||||
// The number of seconds after which a test is considered as slow and reported as such in the results.
|
||||
// slowTestThreshold: 5,
|
||||
|
||||
// A list of paths to snapshot serializer modules Jest should use for snapshot testing
|
||||
// snapshotSerializers: [],
|
||||
|
||||
// The test environment that will be used for testing
|
||||
testEnvironment: "node",
|
||||
|
||||
// Options that will be passed to the testEnvironment
|
||||
// testEnvironmentOptions: {},
|
||||
|
||||
// Adds a location field to test results
|
||||
// testLocationInResults: false,
|
||||
|
||||
// The glob patterns Jest uses to detect test files
|
||||
testMatch: [
|
||||
"<rootDir>/test/index.ts",
|
||||
],
|
||||
|
||||
// An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
|
||||
// testPathIgnorePatterns: [
|
||||
// "\\\\node_modules\\\\"
|
||||
// ],
|
||||
|
||||
// The regexp pattern or array of patterns that Jest uses to detect test files
|
||||
// testRegex: [],
|
||||
|
||||
// This option allows the use of a custom results processor
|
||||
// testResultsProcessor: undefined,
|
||||
|
||||
// This option allows use of a custom test runner
|
||||
// testRunner: "jasmine2",
|
||||
|
||||
// This option sets the URL for the jsdom environment. It is reflected in properties such as location.href
|
||||
// testURL: "http://localhost",
|
||||
|
||||
// Setting this value to "fake" allows the use of fake timers for functions such as "setTimeout"
|
||||
// timers: "real",
|
||||
|
||||
// A map from regular expressions to paths to transformers
|
||||
transform: {
|
||||
"^.+\\.(t|j)sx?$": ["@swc/jest"],
|
||||
},
|
||||
|
||||
// An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
|
||||
// transformIgnorePatterns: [
|
||||
// "\\\\node_modules\\\\",
|
||||
// "\\.pnp\\.[^\\\\]+$"
|
||||
// ],
|
||||
|
||||
// An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
|
||||
// unmockedModulePathPatterns: undefined,
|
||||
|
||||
// Indicates whether each individual test should be reported during the run
|
||||
// verbose: undefined,
|
||||
|
||||
// An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode
|
||||
// watchPathIgnorePatterns: [],
|
||||
|
||||
// Whether to use watchman for file crawling
|
||||
// watchman: true,
|
||||
|
||||
extensionsToTreatAsEsm: ['.ts'],
|
||||
};
|
51
package.json
51
package.json
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@misskey-dev/summaly",
|
||||
"version": "5.0.1",
|
||||
"version": "5.2.3",
|
||||
"description": "Get web page's summary",
|
||||
"author": "syuilo <syuilotan@yahoo.co.jp>",
|
||||
"license": "MIT",
|
||||
@ -9,43 +9,40 @@
|
||||
"main": "./built/index.js",
|
||||
"type": "module",
|
||||
"types": "./built/index.d.ts",
|
||||
"packageManager": "pnpm@8.13.1",
|
||||
"packageManager": "pnpm@9.12.3",
|
||||
"files": [
|
||||
"built",
|
||||
"LICENSE"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"eslint": "eslint --quiet \"src/**/*.ts\"",
|
||||
"test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --silent=false --verbose false",
|
||||
"build": "tsc && tsc-alias",
|
||||
"eslint": "eslint",
|
||||
"test": "vitest run ./test/index.test.ts",
|
||||
"serve": "fastify start ./built/index.js"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@jest/globals": "^29.7.0",
|
||||
"@swc/core": "^1.3.101",
|
||||
"@swc/jest": "^0.2.29",
|
||||
"@types/cheerio": "0.22.18",
|
||||
"@types/debug": "4.1.7",
|
||||
"@types/escape-regexp": "^0.0.1",
|
||||
"@types/node": "20.10.6",
|
||||
"@typescript-eslint/eslint-plugin": "^6.16.0",
|
||||
"@typescript-eslint/parser": "^6.16.0",
|
||||
"debug": "^4.3.4",
|
||||
"eslint": "^8.56.0",
|
||||
"eslint-plugin-import": "^2.29.1",
|
||||
"fastify": "^4.25.2",
|
||||
"fastify-cli": "^5.9.0",
|
||||
"jest": "^29.7.0",
|
||||
"typescript": "5.3.3"
|
||||
"@misskey-dev/eslint-plugin": "^2.1.0",
|
||||
"@types/debug": "4.1.12",
|
||||
"@types/escape-regexp": "^0.0.3",
|
||||
"@types/node": "22.16.0",
|
||||
"@typescript-eslint/eslint-plugin": "^8.35.1",
|
||||
"@typescript-eslint/parser": "^8.35.1",
|
||||
"debug": "^4.4.1",
|
||||
"eslint": "^9.30.1",
|
||||
"eslint-plugin-import": "^2.32.0",
|
||||
"fastify": "^5.4.0",
|
||||
"fastify-cli": "^7.4.0",
|
||||
"tsc-alias": "^1.8.16",
|
||||
"typescript": "5.8.3",
|
||||
"vitest": "^3.2.4"
|
||||
},
|
||||
"dependencies": {
|
||||
"cheerio": "1.0.0-rc.12",
|
||||
"cheerio": "1.1.0",
|
||||
"escape-regexp": "0.0.1",
|
||||
"got": "^12.6.1",
|
||||
"html-entities": "2.3.2",
|
||||
"got": "^14.4.7",
|
||||
"html-entities": "2.6.0",
|
||||
"iconv-lite": "0.6.3",
|
||||
"jschardet": "3.0.0",
|
||||
"private-ip": "2.3.3",
|
||||
"trace-redirect": "1.0.6"
|
||||
"jschardet": "3.1.4",
|
||||
"private-ip": "3.0.2"
|
||||
}
|
||||
}
|
||||
|
7344
pnpm-lock.yaml
generated
7344
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
@ -1,11 +1,10 @@
|
||||
import { URL } from 'node:url';
|
||||
import { decode as decodeHtml } from 'html-entities';
|
||||
import * as cheerio from 'cheerio';
|
||||
import clip from './utils/clip.js';
|
||||
import cleanupTitle from './utils/cleanup-title.js';
|
||||
import type { default as Summary, Player } from '@/summary.js';
|
||||
import { clip } from '@/utils/clip.js';
|
||||
import { cleanupTitle } from '@/utils/cleanup-title.js';
|
||||
|
||||
import { get, head, scpaping } from './utils/got.js';
|
||||
import type { default as Summary, Player } from './summary.js';
|
||||
import { get, head, scpaping } from '@/utils/got.js';
|
||||
|
||||
/**
|
||||
* Contains only the html snippet for a sanitized iframe as the thumbnail is
|
||||
@ -36,7 +35,7 @@ async function getOEmbedPlayer($: cheerio.CheerioAPI, pageUrl: string): Promise<
|
||||
const body = (() => {
|
||||
try {
|
||||
return JSON.parse(oEmbed);
|
||||
} catch {}
|
||||
} catch { /* empty */ }
|
||||
})();
|
||||
|
||||
if (!body || body.version !== '1.0' || !['rich', 'video'].includes(body.type)) {
|
||||
@ -73,7 +72,7 @@ async function getOEmbedPlayer($: cheerio.CheerioAPI, pageUrl: string): Promise<
|
||||
// Allow only HTTPS for best security
|
||||
return null;
|
||||
}
|
||||
} catch (e) {
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
@ -130,12 +129,49 @@ async function getOEmbedPlayer($: cheerio.CheerioAPI, pageUrl: string): Promise<
|
||||
};
|
||||
}
|
||||
|
||||
export default async (_url: URL | string, lang: string | null = null): Promise<Summary | null> => {
|
||||
export type GeneralScrapingOptions = {
|
||||
lang?: string | null;
|
||||
userAgent?: string;
|
||||
followRedirects?: boolean;
|
||||
responseTimeout?: number;
|
||||
operationTimeout?: number;
|
||||
contentLengthLimit?: number;
|
||||
contentLengthRequired?: boolean;
|
||||
};
|
||||
|
||||
export async function general(_url: URL | string, opts?: GeneralScrapingOptions): Promise<Summary | null> {
|
||||
let lang = opts?.lang;
|
||||
if (lang && !lang.match(/^[\w-]+(\s*,\s*[\w-]+)*$/)) lang = null;
|
||||
|
||||
const url = typeof _url === 'string' ? new URL(_url) : _url;
|
||||
|
||||
const res = await scpaping(url.href, { lang: lang || undefined });
|
||||
const res = await scpaping(url.href, {
|
||||
lang: lang || undefined,
|
||||
userAgent: opts?.userAgent,
|
||||
followRedirects: opts?.followRedirects,
|
||||
responseTimeout: opts?.responseTimeout,
|
||||
operationTimeout: opts?.operationTimeout,
|
||||
contentLengthLimit: opts?.contentLengthLimit,
|
||||
contentLengthRequired: opts?.contentLengthRequired,
|
||||
});
|
||||
|
||||
return await parseGeneral(url, res);
|
||||
}
|
||||
|
||||
function headerEqualValueContains(search: string, headerValue: string | string[] | undefined) {
|
||||
if (!headerValue) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (Array.isArray(headerValue)) {
|
||||
return headerValue.some(value => value.toLowerCase() === search.toLowerCase());
|
||||
}
|
||||
|
||||
return headerValue.toLowerCase() === search.toLowerCase();
|
||||
}
|
||||
|
||||
export async function parseGeneral(_url: URL | string, res: Awaited<ReturnType<typeof scpaping>>): Promise<Summary | null> {
|
||||
const url = typeof _url === 'string' ? new URL(_url) : _url;
|
||||
const $ = res.$;
|
||||
const twitterCard =
|
||||
$('meta[name="twitter:card"]').attr('content') ||
|
||||
@ -153,6 +189,7 @@ export default async (_url: URL | string, lang: string | null = null): Promise<S
|
||||
$('meta[property="twitter:title"]').attr('content') ||
|
||||
$('title').text();
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
||||
if (title === undefined || title === null) {
|
||||
return null;
|
||||
}
|
||||
@ -216,16 +253,23 @@ export default async (_url: URL | string, lang: string | null = null): Promise<S
|
||||
const activityPub =
|
||||
$('link[rel="alternate"][type="application/activity+json"]').attr('href') || null;
|
||||
|
||||
const fediverseCreator: string | null =
|
||||
$('meta[name=\'fediverse:creator\']').attr('content') || null;
|
||||
|
||||
// https://developer.mixi.co.jp/connect/mixi_plugin/mixi_check/spec_mixi_check/#toc-18-
|
||||
const sensitive =
|
||||
$('meta[property=\'mixi:content-rating\']').attr('content') === '1';
|
||||
$('meta[property=\'mixi:content-rating\']').attr('content') === '1' ||
|
||||
headerEqualValueContains('adult', res.response.headers.rating) ||
|
||||
headerEqualValueContains('RTA-5042-1996-1400-1577-RTA', res.response.headers.rating) ||
|
||||
$('meta[name=\'rating\']').attr('content') === 'adult' ||
|
||||
$('meta[name=\'rating\']').attr('content')?.toUpperCase() === 'RTA-5042-1996-1400-1577-RTA';
|
||||
|
||||
const find = async (path: string) => {
|
||||
const target = new URL(path, url.href);
|
||||
try {
|
||||
await head(target.href);
|
||||
return target;
|
||||
} catch (e) {
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
@ -260,5 +304,6 @@ export default async (_url: URL | string, lang: string | null = null): Promise<S
|
||||
sitename: siteName || null,
|
||||
sensitive,
|
||||
activityPub,
|
||||
fediverseCreator,
|
||||
};
|
||||
};
|
||||
}
|
||||
|
105
src/index.ts
105
src/index.ts
@ -3,16 +3,15 @@
|
||||
* https://github.com/misskey-dev/summaly
|
||||
*/
|
||||
|
||||
import { URL } from 'node:url';
|
||||
import tracer from 'trace-redirect';
|
||||
import { SummalyResult } from './summary.js';
|
||||
import { SummalyPlugin } from './iplugin.js';
|
||||
export * from './iplugin.js';
|
||||
import general from './general.js';
|
||||
import * as Got from 'got';
|
||||
import { setAgent } from './utils/got.js';
|
||||
import { got, type Agents as GotAgents } from 'got';
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
import { plugins as builtinPlugins } from './plugins/index.js';
|
||||
import { SummalyResult } from '@/summary.js';
|
||||
import { SummalyPlugin as _SummalyPlugin } from '@/iplugin.js';
|
||||
import { general, type GeneralScrapingOptions } from '@/general.js';
|
||||
import { DEFAULT_BOT_UA, DEFAULT_OPERATION_TIMEOUT, DEFAULT_RESPONSE_TIMEOUT, agent, setAgent } from '@/utils/got.js';
|
||||
import { plugins as builtinPlugins } from '@/plugins/index.js';
|
||||
|
||||
export type SummalyPlugin = _SummalyPlugin;
|
||||
|
||||
export type SummalyOptions = {
|
||||
/**
|
||||
@ -33,7 +32,36 @@ export type SummalyOptions = {
|
||||
/**
|
||||
* Custom HTTP agent
|
||||
*/
|
||||
agent?: Got.Agents;
|
||||
agent?: GotAgents;
|
||||
|
||||
/**
|
||||
* User-Agent for the request
|
||||
*/
|
||||
userAgent?: string;
|
||||
|
||||
/**
|
||||
* Response timeout.
|
||||
* Set timeouts for each phase, such as host name resolution and socket communication.
|
||||
*/
|
||||
responseTimeout?: number;
|
||||
|
||||
/**
|
||||
* Operation timeout.
|
||||
* Set the timeout from the start to the end of the request.
|
||||
*/
|
||||
operationTimeout?: number;
|
||||
|
||||
/**
|
||||
* Maximum content length.
|
||||
* If set to true, an error will occur if the content-length value returned from the other server is larger than this parameter (or if the received body size exceeds this parameter).
|
||||
*/
|
||||
contentLengthLimit?: number;
|
||||
|
||||
/**
|
||||
* Content length required.
|
||||
* If set to true, it will be an error if the other server does not return content-length.
|
||||
*/
|
||||
contentLengthRequired?: boolean;
|
||||
};
|
||||
|
||||
export const summalyDefaultOptions = {
|
||||
@ -56,8 +84,32 @@ export const summaly = async (url: string, options?: SummalyOptions): Promise<Su
|
||||
if (opts.followRedirects) {
|
||||
// .catch(() => url)にすればいいけど、jestにtrace-redirectを食わせるのが面倒なのでtry-catch
|
||||
try {
|
||||
actualUrl = await tracer(url);
|
||||
} catch (e) {
|
||||
const timeout = opts.responseTimeout ?? DEFAULT_RESPONSE_TIMEOUT;
|
||||
const operationTimeout = opts.operationTimeout ?? DEFAULT_OPERATION_TIMEOUT;
|
||||
actualUrl = await got
|
||||
.head(url, {
|
||||
headers: {
|
||||
accept: 'text/html,application/xhtml+xml',
|
||||
'user-agent': opts.userAgent ?? DEFAULT_BOT_UA,
|
||||
'accept-language': opts.lang ?? undefined,
|
||||
},
|
||||
timeout: {
|
||||
lookup: timeout,
|
||||
connect: timeout,
|
||||
secureConnect: timeout,
|
||||
socket: timeout, // read timeout
|
||||
response: timeout,
|
||||
send: timeout,
|
||||
request: operationTimeout, // whole operation timeout
|
||||
},
|
||||
agent,
|
||||
http2: false,
|
||||
retry: {
|
||||
limit: 0,
|
||||
},
|
||||
})
|
||||
.then(res => res.url);
|
||||
} catch {
|
||||
actualUrl = url;
|
||||
}
|
||||
}
|
||||
@ -68,28 +120,41 @@ export const summaly = async (url: string, options?: SummalyOptions): Promise<Su
|
||||
const match = plugins.filter(plugin => plugin.test(_url))[0];
|
||||
|
||||
// Get summary
|
||||
const summary = await (match ? match.summarize : general)(_url, opts.lang || undefined);
|
||||
const scrapingOptions: GeneralScrapingOptions = {
|
||||
lang: opts.lang,
|
||||
userAgent: opts.userAgent,
|
||||
responseTimeout: opts.responseTimeout,
|
||||
followRedirects: opts.followRedirects,
|
||||
operationTimeout: opts.operationTimeout,
|
||||
contentLengthLimit: opts.contentLengthLimit,
|
||||
contentLengthRequired: opts.contentLengthRequired,
|
||||
};
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
||||
const summary = await (match ? match.summarize : general)(_url, scrapingOptions);
|
||||
|
||||
if (summary == null) {
|
||||
throw new Error('failed summarize');
|
||||
}
|
||||
|
||||
return Object.assign(summary, {
|
||||
url: actualUrl
|
||||
url: actualUrl,
|
||||
});
|
||||
};
|
||||
|
||||
// eslint-disable-next-line import/no-default-export
|
||||
export default function (fastify: FastifyInstance, options: SummalyOptions, done: (err?: Error) => void) {
|
||||
fastify.get<{
|
||||
Querystring: {
|
||||
url?: string;
|
||||
lang?: string;
|
||||
};
|
||||
Querystring: {
|
||||
url?: string;
|
||||
lang?: string;
|
||||
};
|
||||
}>('/', async (req, reply) => {
|
||||
const url = req.query.url as string;
|
||||
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
||||
if (url == null) {
|
||||
return reply.status(400).send({
|
||||
error: 'url is required'
|
||||
error: 'url is required',
|
||||
});
|
||||
}
|
||||
|
||||
@ -103,7 +168,7 @@ export default function (fastify: FastifyInstance, options: SummalyOptions, done
|
||||
return summary;
|
||||
} catch (e) {
|
||||
return reply.status(500).send({
|
||||
error: e
|
||||
error: e,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
@ -1,7 +1,7 @@
|
||||
import type { URL } from 'node:url';
|
||||
import Summary from './summary.js';
|
||||
import type Summary from '@/summary.js';
|
||||
import type { GeneralScrapingOptions } from '@/general.js';
|
||||
|
||||
export interface SummalyPlugin {
|
||||
test: (url: URL) => boolean;
|
||||
summarize: (url: URL, lang?: string) => Promise<Summary | null>;
|
||||
summarize: (url: URL, opts?: GeneralScrapingOptions) => Promise<Summary | null>;
|
||||
}
|
||||
|
@ -1,6 +1,5 @@
|
||||
import { URL } from 'node:url';
|
||||
import { scpaping } from '../utils/got.js';
|
||||
import summary from '../summary.js';
|
||||
import { scpaping } from '@/utils/got.js';
|
||||
import summary from '@/summary.js';
|
||||
|
||||
export function test(url: URL): boolean {
|
||||
return url.hostname === 'www.amazon.com' ||
|
||||
@ -56,5 +55,6 @@ export async function summarize(url: URL): Promise<summary> {
|
||||
},
|
||||
sitename: 'Amazon',
|
||||
activityPub: null,
|
||||
fediverseCreator: null,
|
||||
};
|
||||
}
|
||||
|
26
src/plugins/bluesky.ts
Normal file
26
src/plugins/bluesky.ts
Normal file
@ -0,0 +1,26 @@
|
||||
import * as cheerio from 'cheerio';
|
||||
import type Summary from '@/summary.js';
|
||||
import { getResponse, getGotOptions } from '@/utils/got.js';
|
||||
import { parseGeneral, type GeneralScrapingOptions } from '@/general.js';
|
||||
|
||||
export function test(url: URL): boolean {
|
||||
return url.hostname === 'bsky.app';
|
||||
}
|
||||
|
||||
export async function summarize(url: URL, opts?: GeneralScrapingOptions): Promise<Summary | null> {
|
||||
const args = getGotOptions(url.href, opts);
|
||||
|
||||
// HEADで取ると404が返るためGETのみで取得
|
||||
const res = await getResponse({
|
||||
...args,
|
||||
method: 'GET',
|
||||
});
|
||||
const body = res.body;
|
||||
const $ = cheerio.load(body);
|
||||
|
||||
return await parseGeneral(url, {
|
||||
body,
|
||||
$,
|
||||
response: res,
|
||||
});
|
||||
}
|
@ -1,7 +1,5 @@
|
||||
import { URL } from 'node:url';
|
||||
import { scpaping } from '../utils/got.js';
|
||||
import general from '../general.js';
|
||||
import Summary from '../summary.js';
|
||||
import { general, type GeneralScrapingOptions } from '@/general.js';
|
||||
import Summary from '@/summary.js';
|
||||
|
||||
export function test(url: URL): boolean {
|
||||
// Branch.io を使用したディープリンクにマッチ
|
||||
@ -9,10 +7,10 @@ export function test(url: URL): boolean {
|
||||
url.hostname === 'spotify.link';
|
||||
}
|
||||
|
||||
export async function summarize(url: URL, lang: string | null = null): Promise<Summary | null> {
|
||||
export async function summarize(url: URL, opts?: GeneralScrapingOptions): Promise<Summary | null> {
|
||||
// https://help.branch.io/using-branch/docs/creating-a-deep-link#redirections
|
||||
// Web版に強制リダイレクトすることでbranch.ioの独自ページが開くのを防ぐ
|
||||
url.searchParams.append('$web_only', 'true');
|
||||
|
||||
return await general(url, lang);
|
||||
return await general(url, opts);
|
||||
}
|
||||
|
@ -1,10 +1,12 @@
|
||||
import { SummalyPlugin } from '@/iplugin.js';
|
||||
import * as amazon from './amazon.js';
|
||||
import * as bluesky from './bluesky.js';
|
||||
import * as wikipedia from './wikipedia.js';
|
||||
import * as branchIoDeeplinks from './branchio-deeplinks.js';
|
||||
import { SummalyPlugin } from '@/iplugin.js';
|
||||
|
||||
export const plugins: SummalyPlugin[] = [
|
||||
amazon,
|
||||
wikipedia,
|
||||
branchIoDeeplinks,
|
||||
amazon,
|
||||
bluesky,
|
||||
wikipedia,
|
||||
branchIoDeeplinks,
|
||||
];
|
||||
|
@ -1,8 +1,7 @@
|
||||
import { URL } from 'node:url';
|
||||
import { get } from '../utils/got.js';
|
||||
import debug from 'debug';
|
||||
import summary from '../summary.js';
|
||||
import clip from './../utils/clip.js';
|
||||
import { get } from '@/utils/got.js';
|
||||
import summary from '@/summary.js';
|
||||
import { clip } from '@/utils/clip.js';
|
||||
|
||||
const log = debug('summaly:plugins:wikipedia');
|
||||
|
||||
@ -20,6 +19,7 @@ export async function summarize(url: URL): Promise<summary> {
|
||||
log(`title is ${title}`);
|
||||
log(`endpoint is ${endpoint}`);
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
let body = await get(endpoint) as any;
|
||||
body = JSON.parse(body);
|
||||
log(body);
|
||||
@ -43,5 +43,6 @@ export async function summarize(url: URL): Promise<summary> {
|
||||
},
|
||||
sitename: 'Wikipedia',
|
||||
activityPub: null,
|
||||
fediverseCreator: null,
|
||||
};
|
||||
}
|
||||
|
@ -38,6 +38,11 @@ type Summary = {
|
||||
* The url of the ActivityPub representation of that web page
|
||||
*/
|
||||
activityPub: string | null;
|
||||
|
||||
/**
|
||||
* The @ handle of a fediverse user (https://blog.joinmastodon.org/2024/07/highlighting-journalism-on-mastodon/)
|
||||
*/
|
||||
fediverseCreator: string | null;
|
||||
};
|
||||
|
||||
export type SummalyResult = Summary & {
|
||||
@ -47,6 +52,7 @@ export type SummalyResult = Summary & {
|
||||
url: string;
|
||||
};
|
||||
|
||||
// eslint-disable-next-line import/no-default-export
|
||||
export default Summary;
|
||||
|
||||
export type Player = {
|
||||
|
@ -1,6 +1,7 @@
|
||||
/* eslint-disable no-param-reassign */
|
||||
import escapeRegExp from 'escape-regexp';
|
||||
|
||||
export default function(title: string, siteName?: string | null): string {
|
||||
export function cleanupTitle(title: string, siteName?: string | null): string {
|
||||
title = title.trim();
|
||||
|
||||
if (siteName) {
|
||||
@ -9,12 +10,12 @@ export default function(title: string, siteName?: string | null): string {
|
||||
const x = escapeRegExp(siteName);
|
||||
|
||||
const patterns = [
|
||||
`^(.+?)\\s?[\\-\\|:・]\\s?${x}$`
|
||||
`^(.+?)\\s?[\\-\\|:・]\\s?${x}$`,
|
||||
];
|
||||
|
||||
for (let i = 0; i < patterns.length; i++) {
|
||||
const pattern = new RegExp(patterns[i]);
|
||||
const [, match] = pattern.exec(title) || [null, null];
|
||||
const [, match] = pattern.exec(title) ?? [null, null];
|
||||
if (match) {
|
||||
return match;
|
||||
}
|
||||
|
@ -1,10 +1,11 @@
|
||||
import nullOrEmpty from './null-or-empty.js';
|
||||
import { nullOrEmpty } from './null-or-empty.js';
|
||||
|
||||
export default function(s: string, max: number): string {
|
||||
export function clip(s: string, max: number): string {
|
||||
if (nullOrEmpty(s)) {
|
||||
return s;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-param-reassign
|
||||
s = s.trim();
|
||||
|
||||
if (s.length > max) {
|
||||
|
@ -11,6 +11,7 @@ const regCharset = new RegExp(/charset\s*=\s*["']?([\w-]+)/, 'i');
|
||||
export function detectEncoding(body: Buffer): string {
|
||||
// By detection
|
||||
const detected = jschardet.detect(body, { minimumThreshold: 0.99 });
|
||||
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
||||
if (detected) {
|
||||
const candicate = detected.encoding;
|
||||
const encoding = toEncoding(candicate);
|
||||
|
131
src/utils/got.ts
131
src/utils/got.ts
@ -1,17 +1,20 @@
|
||||
import got, * as Got from 'got';
|
||||
import { StatusError } from './status-error.js';
|
||||
import { detectEncoding, toUtf8 } from './encoding.js';
|
||||
import * as cheerio from 'cheerio';
|
||||
import PrivateIp from 'private-ip';
|
||||
import { dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import got, * as Got from 'got';
|
||||
import * as cheerio from 'cheerio';
|
||||
import PrivateIp from 'private-ip';
|
||||
import type { GeneralScrapingOptions } from '@/general.js';
|
||||
import { StatusError } from '@/utils/status-error.js';
|
||||
import { detectEncoding, toUtf8 } from '@/utils/encoding.js';
|
||||
|
||||
const _filename = fileURLToPath(import.meta.url);
|
||||
const _dirname = dirname(_filename);
|
||||
|
||||
export let agent: Got.Agents = {};
|
||||
|
||||
export function setAgent(_agent: Got.Agents) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
||||
agent = _agent || {};
|
||||
}
|
||||
|
||||
@ -21,34 +24,48 @@ export type GotOptions = {
|
||||
body?: string;
|
||||
headers: Record<string, string | undefined>;
|
||||
typeFilter?: RegExp;
|
||||
}
|
||||
followRedirects?: boolean;
|
||||
responseTimeout?: number;
|
||||
operationTimeout?: number;
|
||||
contentLengthLimit?: number;
|
||||
contentLengthRequired?: boolean;
|
||||
};
|
||||
|
||||
const repo = JSON.parse(readFileSync(`${_dirname}/../../package.json`, 'utf8'));
|
||||
|
||||
const RESPONSE_TIMEOUT = 20 * 1000;
|
||||
const OPERATION_TIMEOUT = 60 * 1000;
|
||||
const MAX_RESPONSE_SIZE = 10 * 1024 * 1024;
|
||||
const BOT_UA = `SummalyBot/${repo.version}`;
|
||||
export const DEFAULT_RESPONSE_TIMEOUT = 20 * 1000;
|
||||
export const DEFAULT_OPERATION_TIMEOUT = 60 * 1000;
|
||||
export const DEFAULT_MAX_RESPONSE_SIZE = 10 * 1024 * 1024;
|
||||
export const DEFAULT_BOT_UA = `SummalyBot/${repo.version}`;
|
||||
|
||||
export async function scpaping(url: string, opts?: { lang?: string; }) {
|
||||
const response = await getResponse({
|
||||
export function getGotOptions(url: string, opts?: GeneralScrapingOptions): Omit<GotOptions, 'method'> {
|
||||
return {
|
||||
url,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'accept': 'text/html,application/xhtml+xml',
|
||||
'user-agent': BOT_UA,
|
||||
'accept-language': opts?.lang
|
||||
'user-agent': opts?.userAgent ?? DEFAULT_BOT_UA,
|
||||
'accept-language': opts?.lang ?? undefined,
|
||||
},
|
||||
typeFilter: /^(text\/html|application\/xhtml\+xml)/,
|
||||
followRedirects: opts?.followRedirects,
|
||||
responseTimeout: opts?.responseTimeout,
|
||||
operationTimeout: opts?.operationTimeout,
|
||||
contentLengthLimit: opts?.contentLengthLimit,
|
||||
contentLengthRequired: opts?.contentLengthRequired,
|
||||
};
|
||||
}
|
||||
|
||||
export async function scpaping(
|
||||
url: string,
|
||||
opts?: GeneralScrapingOptions,
|
||||
) {
|
||||
const args = getGotOptions(url, opts);
|
||||
|
||||
const response = await getResponse({
|
||||
...args,
|
||||
method: 'GET',
|
||||
});
|
||||
|
||||
// SUMMALY_ALLOW_PRIVATE_IPはテスト用
|
||||
const allowPrivateIp = process.env.SUMMALY_ALLOW_PRIVATE_IP === 'true' || Object.keys(agent).length > 0;
|
||||
|
||||
if (!allowPrivateIp && response.ip && PrivateIp(response.ip)) {
|
||||
throw new StatusError(`Private IP rejected ${response.ip}`, 400, 'Private IP Rejected');
|
||||
}
|
||||
|
||||
const encoding = detectEncoding(response.rawBody);
|
||||
const body = toUtf8(response.rawBody, encoding);
|
||||
const $ = cheerio.load(body);
|
||||
@ -69,24 +86,22 @@ export async function get(url: string) {
|
||||
},
|
||||
});
|
||||
|
||||
return await res.body;
|
||||
return res.body;
|
||||
}
|
||||
|
||||
export async function head(url: string) {
|
||||
const res = await getResponse({
|
||||
return await getResponse({
|
||||
url,
|
||||
method: 'HEAD',
|
||||
headers: {
|
||||
'accept': '*/*',
|
||||
},
|
||||
});
|
||||
|
||||
return await res;
|
||||
}
|
||||
|
||||
async function getResponse(args: GotOptions) {
|
||||
const timeout = RESPONSE_TIMEOUT;
|
||||
const operationTimeout = OPERATION_TIMEOUT;
|
||||
export async function getResponse(args: GotOptions) {
|
||||
const timeout = args.responseTimeout ?? DEFAULT_RESPONSE_TIMEOUT;
|
||||
const operationTimeout = args.operationTimeout ?? DEFAULT_OPERATION_TIMEOUT;
|
||||
|
||||
const req = got<string>(args.url, {
|
||||
method: args.method,
|
||||
@ -101,6 +116,7 @@ async function getResponse(args: GotOptions) {
|
||||
send: timeout,
|
||||
request: operationTimeout, // whole operation timeout
|
||||
},
|
||||
followRedirect: args.followRedirects,
|
||||
agent,
|
||||
http2: false,
|
||||
retry: {
|
||||
@ -108,30 +124,45 @@ async function getResponse(args: GotOptions) {
|
||||
},
|
||||
});
|
||||
|
||||
return await receiveResponse({ req, typeFilter: args.typeFilter });
|
||||
const res = await receiveResponse({ req, opts: args });
|
||||
|
||||
// SUMMALY_ALLOW_PRIVATE_IPはテスト用
|
||||
// TODO: Try moving this to receiveResponse- ATM `got` doesn't provide a means
|
||||
// to check the IP/response header data while streaming the response...
|
||||
const allowPrivateIp = process.env.SUMMALY_ALLOW_PRIVATE_IP === 'true' || Object.keys(agent).length > 0;
|
||||
if (!allowPrivateIp && res.ip && PrivateIp(res.ip)) {
|
||||
throw new StatusError(`Private IP rejected ${res.ip}`, 400, 'Private IP Rejected');
|
||||
}
|
||||
|
||||
// Check html
|
||||
const contentType = res.headers['content-type'];
|
||||
if (args.typeFilter && !contentType?.match(args.typeFilter)) {
|
||||
throw new Error(`Rejected by type filter ${contentType}`);
|
||||
}
|
||||
|
||||
// 応答ヘッダでサイズチェック
|
||||
const contentLength = res.headers['content-length'];
|
||||
if (contentLength) {
|
||||
const maxSize = args.contentLengthLimit ?? DEFAULT_MAX_RESPONSE_SIZE;
|
||||
const size = Number(contentLength);
|
||||
if (size > maxSize) {
|
||||
throw new Error(`maxSize exceeded (${size} > ${maxSize}) on response`);
|
||||
}
|
||||
} else {
|
||||
if (args.contentLengthRequired) {
|
||||
throw new Error('content-length required');
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
async function receiveResponse<T>(args: { req: Got.CancelableRequest<Got.Response<T>>, typeFilter?: RegExp }) {
|
||||
async function receiveResponse<T>(args: {
|
||||
req: Got.CancelableRequest<Got.Response<T>>,
|
||||
opts: GotOptions,
|
||||
}) {
|
||||
const req = args.req;
|
||||
const maxSize = MAX_RESPONSE_SIZE;
|
||||
|
||||
req.on('response', (res: Got.Response) => {
|
||||
// Check html
|
||||
if (args.typeFilter && !res.headers['content-type']?.match(args.typeFilter)) {
|
||||
// console.warn(res.headers['content-type']);
|
||||
req.cancel(`Rejected by type filter ${res.headers['content-type']}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// 応答ヘッダでサイズチェック
|
||||
const contentLength = res.headers['content-length'];
|
||||
if (contentLength != null) {
|
||||
const size = Number(contentLength);
|
||||
if (size > maxSize) {
|
||||
req.cancel(`maxSize exceeded (${size} > ${maxSize}) on response`);
|
||||
}
|
||||
}
|
||||
});
|
||||
const maxSize = args.opts.contentLengthLimit ?? DEFAULT_MAX_RESPONSE_SIZE;
|
||||
|
||||
// 受信中のデータでサイズチェック
|
||||
req.on('downloadProgress', (progress: Got.Progress) => {
|
||||
|
@ -1,4 +1,5 @@
|
||||
export default function(val: string): boolean {
|
||||
/* eslint-disable @typescript-eslint/no-unnecessary-condition */
|
||||
export function nullOrEmpty(val: string): boolean {
|
||||
if (val === undefined) {
|
||||
return true;
|
||||
} else if (val === null) {
|
||||
|
13
test/htmls/fediverse-creator.html
Normal file
13
test/htmls/fediverse-creator.html
Normal file
@ -0,0 +1,13 @@
|
||||
<!doctype html>
|
||||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="fediverse:creator" content="@test@example.com">
|
||||
<title>Meow</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Hellooo!</h1>
|
||||
<p>:3</p>
|
||||
</body>
|
||||
</html>
|
13
test/htmls/meta-adult-sensitive.html
Normal file
13
test/htmls/meta-adult-sensitive.html
Normal file
@ -0,0 +1,13 @@
|
||||
<!doctype html>
|
||||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="rating" content="adult">
|
||||
<title>SENSITIVE CONTENT!!</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Yo</h1>
|
||||
<p>Hey hey hey syuilo.</p>
|
||||
</body>
|
||||
</html>
|
13
test/htmls/meta-rta-sensitive.html
Normal file
13
test/htmls/meta-rta-sensitive.html
Normal file
@ -0,0 +1,13 @@
|
||||
<!doctype html>
|
||||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="rating" content="RTA-5042-1996-1400-1577-RTA">
|
||||
<title>SENSITIVE CONTENT!!</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Yo</h1>
|
||||
<p>Hey hey hey syuilo.</p>
|
||||
</body>
|
||||
</html>
|
733
test/index.test.ts
Normal file
733
test/index.test.ts
Normal file
@ -0,0 +1,733 @@
|
||||
/**
|
||||
* Tests!
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
/* dependencies below */
|
||||
|
||||
import fs, { readdirSync } from 'node:fs';
|
||||
import process from 'node:process';
|
||||
import { dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { Agent as httpAgent } from 'node:http';
|
||||
import { Agent as httpsAgent } from 'node:https';
|
||||
import { expect, test, describe, beforeEach, afterEach } from 'vitest';
|
||||
import fastify, { type FastifyInstance } from 'fastify';
|
||||
import { summaly } from '@/index.js';
|
||||
import { StatusError } from '@/utils/status-error.js';
|
||||
|
||||
const _filename = fileURLToPath(import.meta.url);
|
||||
const _dirname = dirname(_filename);
|
||||
|
||||
/* settings below */
|
||||
|
||||
Error.stackTraceLimit = Infinity;
|
||||
|
||||
// During the test the env variable is set to test
|
||||
process.env.NODE_ENV = 'test';
|
||||
process.env.SUMMALY_ALLOW_PRIVATE_IP = 'true';
|
||||
|
||||
const port = 3060;
|
||||
const host = `http://localhost:${port}`;
|
||||
|
||||
// Display detail of unhandled promise rejection
|
||||
process.on('unhandledRejection', console.dir);
|
||||
|
||||
let app: FastifyInstance | null = null;
|
||||
|
||||
function skippableTest(name: string, fn: () => void) {
|
||||
if (process.env.SKIP_NETWORK_TEST === 'true') {
|
||||
console.log(`[SKIP] ${name}`);
|
||||
test.skip(name, fn);
|
||||
} else {
|
||||
test(name, fn);
|
||||
}
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
// Allow private IPs by default, since a lot of the tests rely on old behvior
|
||||
process.env.SUMMALY_ALLOW_PRIVATE_IP = 'true';
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (app) {
|
||||
await app.close();
|
||||
app = null;
|
||||
}
|
||||
});
|
||||
|
||||
/* tests below */
|
||||
|
||||
test('basic', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/basic.html');
|
||||
reply.header('content-length', content.length);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
expect(await summaly(host)).toEqual({
|
||||
title: 'KISS principle',
|
||||
icon: null,
|
||||
description: null,
|
||||
thumbnail: null,
|
||||
player: {
|
||||
url: null,
|
||||
width: null,
|
||||
height: null,
|
||||
'allow': [
|
||||
'autoplay',
|
||||
'encrypted-media',
|
||||
'fullscreen',
|
||||
],
|
||||
},
|
||||
sitename: 'localhost:3060',
|
||||
sensitive: false,
|
||||
url: host + '/',
|
||||
activityPub: null,
|
||||
fediverseCreator: null,
|
||||
});
|
||||
});
|
||||
|
||||
skippableTest('Stage Bye Stage', async () => {
|
||||
// If this test fails, you must rewrite the result data and the example in README.md.
|
||||
|
||||
const summary = await summaly('https://www.youtube.com/watch?v=NMIEAhH_fTU');
|
||||
expect(summary).toEqual(
|
||||
{
|
||||
'title': '【アイドルマスター】「Stage Bye Stage」(歌:島村卯月、渋谷凛、本田未央)',
|
||||
'icon': 'https://www.youtube.com/s/desktop/78bc1359/img/logos/favicon.ico',
|
||||
'description': 'Website▶https://columbia.jp/idolmaster/Playlist▶https://www.youtube.com/playlist?list=PL83A2998CF3BBC86D2018年7月18日発売予定THE IDOLM@STER CINDERELLA GIRLS CG STAR...',
|
||||
'thumbnail': 'https://i.ytimg.com/vi/NMIEAhH_fTU/maxresdefault.jpg',
|
||||
'player': {
|
||||
'url': 'https://www.youtube.com/embed/NMIEAhH_fTU?feature=oembed',
|
||||
'width': 200,
|
||||
'height': 113,
|
||||
'allow': [
|
||||
'autoplay',
|
||||
'clipboard-write',
|
||||
'encrypted-media',
|
||||
'picture-in-picture',
|
||||
'web-share',
|
||||
'fullscreen',
|
||||
],
|
||||
},
|
||||
'sitename': 'YouTube',
|
||||
'sensitive': false,
|
||||
'activityPub': null,
|
||||
'fediverseCreator': null,
|
||||
'url': 'https://www.youtube.com/watch?v=NMIEAhH_fTU',
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('faviconがHTML上で指定されていないが、ルートに存在する場合、正しく設定される', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/no-favicon.html');
|
||||
reply.header('content-length', content.length);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
app.get('/favicon.ico', (_, reply) => reply.status(200).send());
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.icon).toBe(`${host}/favicon.ico`);
|
||||
});
|
||||
|
||||
test('faviconがHTML上で指定されていなくて、ルートにも存在しなかった場合 null になる', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/no-favicon.html');
|
||||
reply.header('content-length', content.length);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
app.get('*', (_, reply) => reply.status(404).send());
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.icon).toBe(null);
|
||||
});
|
||||
|
||||
test('titleがcleanupされる', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/og-title.html');
|
||||
reply.header('content-length', content.length);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.title).toBe('Strawberry Pasta');
|
||||
});
|
||||
|
||||
describe('Private IP blocking', () => {
|
||||
beforeEach(() => {
|
||||
process.env.SUMMALY_ALLOW_PRIVATE_IP = 'false';
|
||||
app = fastify();
|
||||
app.get('*', (request, reply) => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/og-title.html');
|
||||
reply.header('content-length', content.length);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
return app.listen({ port });
|
||||
});
|
||||
|
||||
test('private ipなサーバーの情報を取得できない', async () => {
|
||||
const summary = await summaly(host).catch((e: StatusError) => e);
|
||||
if (summary instanceof StatusError) {
|
||||
expect(summary.name).toBe('StatusError');
|
||||
} else {
|
||||
expect(summary).toBeInstanceOf(StatusError);
|
||||
}
|
||||
});
|
||||
|
||||
test('agentが指定されている場合はprivate ipを許可', async () => {
|
||||
const summary = await summaly(host, {
|
||||
agent: {
|
||||
http: new httpAgent({ keepAlive: true }),
|
||||
https: new httpsAgent({ keepAlive: true }),
|
||||
},
|
||||
});
|
||||
expect(summary.title).toBe('Strawberry Pasta');
|
||||
});
|
||||
|
||||
test('agentが空のオブジェクトの場合はprivate ipを許可しない', async () => {
|
||||
const summary = await summaly(host, { agent: {} }).catch((e: StatusError) => e);
|
||||
if (summary instanceof StatusError) {
|
||||
expect(summary.name).toBe('StatusError');
|
||||
} else {
|
||||
expect(summary).toBeInstanceOf(StatusError);
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.env.SUMMALY_ALLOW_PRIVATE_IP = 'true';
|
||||
});
|
||||
});
|
||||
|
||||
describe('OGP', () => {
|
||||
test('title', async () => {
|
||||
app = fastify();
|
||||
app.get('*', (request, reply) => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/og-title.html');
|
||||
reply.header('content-length', content.length);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.title).toBe('Strawberry Pasta');
|
||||
});
|
||||
|
||||
test('description', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/og-description.html');
|
||||
reply.header('content-length', content.length);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.description).toBe('Strawberry Pasta');
|
||||
});
|
||||
|
||||
test('site_name', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/og-site_name.html');
|
||||
reply.header('content-length', content.length);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.sitename).toBe('Strawberry Pasta');
|
||||
});
|
||||
|
||||
test('thumbnail', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/og-image.html');
|
||||
reply.header('content-length', content.length);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.thumbnail).toBe('https://himasaku.net/himasaku.png');
|
||||
});
|
||||
});
|
||||
|
||||
describe('TwitterCard', () => {
|
||||
test('title', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/twitter-title.html');
|
||||
reply.header('content-length', content.length);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.title).toBe('Strawberry Pasta');
|
||||
});
|
||||
|
||||
test('description', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/twitter-description.html');
|
||||
reply.header('content-length', content.length);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.description).toBe('Strawberry Pasta');
|
||||
});
|
||||
|
||||
test('thumbnail', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/twitter-image.html');
|
||||
reply.header('content-length', content.length);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.thumbnail).toBe('https://himasaku.net/himasaku.png');
|
||||
});
|
||||
|
||||
test('Player detection - PeerTube:video => video', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/player-peertube-video.html');
|
||||
reply.header('content-length', content.length);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe('https://example.com/embedurl');
|
||||
expect(summary.player.allow).toStrictEqual(['autoplay', 'encrypted-media', 'fullscreen']);
|
||||
});
|
||||
|
||||
test('Player detection - Pleroma:video => video', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/player-pleroma-video.html');
|
||||
reply.header('content-length', content.length);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe('https://example.com/embedurl');
|
||||
expect(summary.player.allow).toStrictEqual(['autoplay', 'encrypted-media', 'fullscreen']);
|
||||
});
|
||||
|
||||
test('Player detection - Pleroma:image => image', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/player-pleroma-image.html');
|
||||
reply.header('content-length', content.length);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.thumbnail).toBe('https://example.com/imageurl');
|
||||
});
|
||||
});
|
||||
|
||||
describe('oEmbed', () => {
|
||||
const setUpFastify = async (oEmbedPath: string, htmlPath = 'htmls/oembed.html') => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
const content = fs.readFileSync(new URL(htmlPath, import.meta.url));
|
||||
reply.header('content-length', content.length);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
app.get('/oembed.json', (request, reply) => {
|
||||
const content = fs.readFileSync(new URL(oEmbedPath, new URL('oembed/', import.meta.url)));
|
||||
reply.header('content-length', content.length);
|
||||
reply.header('content-type', 'application/json');
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
};
|
||||
|
||||
for (const filename of readdirSync(new URL('oembed/invalid', import.meta.url))) {
|
||||
test(`Invalidity test: ${filename}`, async () => {
|
||||
await setUpFastify(`invalid/${filename}`);
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe(null);
|
||||
});
|
||||
}
|
||||
|
||||
test('basic properties', async () => {
|
||||
await setUpFastify('oembed.json');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe('https://example.com/');
|
||||
expect(summary.player.width).toBe(500);
|
||||
expect(summary.player.height).toBe(300);
|
||||
});
|
||||
|
||||
test('type: video', async () => {
|
||||
await setUpFastify('oembed-video.json');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe('https://example.com/');
|
||||
expect(summary.player.width).toBe(500);
|
||||
expect(summary.player.height).toBe(300);
|
||||
});
|
||||
|
||||
test('max height', async () => {
|
||||
await setUpFastify('oembed-too-tall.json');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.height).toBe(1024);
|
||||
});
|
||||
|
||||
test('children are ignored', async () => {
|
||||
await setUpFastify('oembed-iframe-child.json');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe('https://example.com/');
|
||||
});
|
||||
|
||||
test('allows fullscreen', async () => {
|
||||
await setUpFastify('oembed-allow-fullscreen.json');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe('https://example.com/');
|
||||
expect(summary.player.allow).toStrictEqual(['fullscreen']);
|
||||
});
|
||||
|
||||
test('allows legacy allowfullscreen', async () => {
|
||||
await setUpFastify('oembed-allow-fullscreen-legacy.json');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe('https://example.com/');
|
||||
expect(summary.player.allow).toStrictEqual(['fullscreen']);
|
||||
});
|
||||
|
||||
test('allows safelisted permissions', async () => {
|
||||
await setUpFastify('oembed-allow-safelisted-permissions.json');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe('https://example.com/');
|
||||
expect(summary.player.allow).toStrictEqual([
|
||||
'autoplay', 'clipboard-write', 'fullscreen',
|
||||
'encrypted-media', 'picture-in-picture', 'web-share',
|
||||
]);
|
||||
});
|
||||
|
||||
test('ignores rare permissions', async () => {
|
||||
await setUpFastify('oembed-ignore-rare-permissions.json');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe('https://example.com/');
|
||||
expect(summary.player.allow).toStrictEqual(['autoplay']);
|
||||
});
|
||||
|
||||
test('oEmbed with relative path', async () => {
|
||||
await setUpFastify('oembed.json', 'htmls/oembed-relative.html');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe('https://example.com/');
|
||||
});
|
||||
|
||||
test('oEmbed with nonexistent path', async () => {
|
||||
await setUpFastify('oembed.json', 'htmls/oembed-nonexistent-path.html');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe(null);
|
||||
expect(summary.description).toBe('nonexistent');
|
||||
});
|
||||
|
||||
test('oEmbed with wrong path', async () => {
|
||||
await setUpFastify('oembed.json', 'htmls/oembed-wrong-path.html');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe(null);
|
||||
expect(summary.description).toBe('wrong url');
|
||||
});
|
||||
|
||||
test('oEmbed with OpenGraph', async () => {
|
||||
await setUpFastify('oembed.json', 'htmls/oembed-and-og.html');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe('https://example.com/');
|
||||
expect(summary.description).toBe('blobcats rule the world');
|
||||
});
|
||||
|
||||
test('Invalid oEmbed with valid OpenGraph', async () => {
|
||||
await setUpFastify('invalid/oembed-insecure.json', 'htmls/oembed-and-og.html');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe(null);
|
||||
expect(summary.description).toBe('blobcats rule the world');
|
||||
});
|
||||
|
||||
test('oEmbed with og:video', async () => {
|
||||
await setUpFastify('oembed.json', 'htmls/oembed-and-og-video.html');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe('https://example.com/');
|
||||
expect(summary.player.allow).toStrictEqual([]);
|
||||
});
|
||||
|
||||
test('width: 100%', async () => {
|
||||
await setUpFastify('oembed-percentage-width.json');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.width).toBe(null);
|
||||
expect(summary.player.height).toBe(300);
|
||||
});
|
||||
});
|
||||
|
||||
describe('ActivityPub', () => {
|
||||
test('Basic', async () => {
|
||||
app = fastify();
|
||||
app.get('*', (request, reply) => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/activitypub.html');
|
||||
reply.header('content-length', content.length);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.activityPub).toBe('https://misskey.test/notes/abcdefg');
|
||||
});
|
||||
|
||||
test('Null', async () => {
|
||||
app = fastify();
|
||||
app.get('*', (request, reply) => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/basic.html');
|
||||
reply.header('content-length', content.length);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.activityPub).toBe(null);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Fediverse Creator', () => {
|
||||
test('Basic', async () => {
|
||||
app = fastify();
|
||||
app.get('*', (request, reply) => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/fediverse-creator.html');
|
||||
reply.header('content-length', content.length);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.fediverseCreator).toBe('@test@example.com');
|
||||
});
|
||||
|
||||
test('Null', async () => {
|
||||
app = fastify();
|
||||
app.get('*', (request, reply) => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/basic.html');
|
||||
reply.header('content-length', content.length);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.fediverseCreator).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('sensitive', () => {
|
||||
test('default', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/basic.html');
|
||||
reply.header('content-length', content.length);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
expect((await summaly(host)).sensitive).toBe(false);
|
||||
});
|
||||
|
||||
test('mixi:content-rating 1', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/mixi-sensitive.html');
|
||||
reply.header('content-length', content.length);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
expect((await summaly(host)).sensitive).toBe(true);
|
||||
});
|
||||
|
||||
test('meta rating adult', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/meta-adult-sensitive.html');
|
||||
reply.header('content-length', content.length);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
expect((await summaly(host)).sensitive).toBe(true);
|
||||
});
|
||||
|
||||
test('meta rating rta', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/meta-rta-sensitive.html');
|
||||
reply.header('content-length', content.length);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
expect((await summaly(host)).sensitive).toBe(true);
|
||||
});
|
||||
|
||||
test('HTTP Header rating adult', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/basic.html');
|
||||
reply.header('content-length', content.length);
|
||||
reply.header('content-type', 'text/html');
|
||||
reply.header('rating', 'adult');
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
expect((await summaly(host)).sensitive).toBe(true);
|
||||
});
|
||||
|
||||
test('HTTP Header rating rta', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/basic.html');
|
||||
reply.header('content-length', content.length);
|
||||
reply.header('content-type', 'text/html');
|
||||
reply.header('rating', 'RTA-5042-1996-1400-1577-RTA');
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
expect((await summaly(host)).sensitive).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('UserAgent', () => {
|
||||
test('UA設定が反映されていること', async () => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/basic.html');
|
||||
let ua: string | undefined = undefined;
|
||||
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
reply.header('content-length', content.byteLength);
|
||||
reply.header('content-type', 'text/html');
|
||||
ua = request.headers['user-agent'];
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
await summaly(host, { userAgent: 'test-ua' });
|
||||
|
||||
expect(ua).toBe('test-ua');
|
||||
});
|
||||
});
|
||||
|
||||
describe('content-length limit', () => {
|
||||
test('content-lengthの上限以内であればエラーが起こらないこと', async () => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/basic.html');
|
||||
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
reply.header('content-length', content.byteLength);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
expect(await summaly(host, { contentLengthLimit: content.byteLength })).toBeDefined();
|
||||
});
|
||||
|
||||
test('content-lengthの上限を超えているとエラーになる事', async () => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/basic.html');
|
||||
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
reply.header('content-length', content.byteLength);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
await expect(summaly(host, { contentLengthLimit: content.byteLength - 1 })).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('content-length required', () => {
|
||||
test('[オプション有効化時] content-lengthが返された場合はエラーとならないこと', async () => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/basic.html');
|
||||
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
reply.header('content-length', content.byteLength);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
expect(await summaly(host, { contentLengthRequired: true, contentLengthLimit: content.byteLength })).toBeDefined();
|
||||
});
|
||||
|
||||
test('[オプション有効化時] content-lengthが返されない場合はエラーとなること', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
reply.header('content-type', 'text/html');
|
||||
// streamで渡さないとcontent-lengthを自動で設定されてしまう
|
||||
return reply.send(fs.createReadStream(_dirname + '/htmls/basic.html'));
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
await expect(summaly(host, { contentLengthRequired: true })).rejects.toThrow();
|
||||
});
|
||||
|
||||
test('[オプション無効化時] content-lengthが返された場合はエラーとならないこと', async () => {
|
||||
const content = fs.readFileSync(_dirname + '/htmls/basic.html');
|
||||
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
reply.header('content-length', content.byteLength);
|
||||
reply.header('content-type', 'text/html');
|
||||
return reply.send(content);
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
expect(await summaly(host, { contentLengthRequired: false, contentLengthLimit: content.byteLength })).toBeDefined();
|
||||
});
|
||||
|
||||
test('[オプション無効化時] content-lengthが返されなくてもエラーとならないこと', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
reply.header('content-type', 'text/html');
|
||||
// streamで渡さないとcontent-lengthを自動で設定されてしまう
|
||||
return reply.send(fs.createReadStream(_dirname + '/htmls/basic.html'));
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
expect(await summaly(host, { contentLengthRequired: false })).toBeDefined();
|
||||
});
|
||||
});
|
474
test/index.ts
474
test/index.ts
@ -1,474 +0,0 @@
|
||||
/**
|
||||
* Tests!
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
/* dependencies below */
|
||||
|
||||
import fs, { readdirSync } from 'node:fs';
|
||||
import process from 'node:process';
|
||||
import fastify from 'fastify';
|
||||
import { summaly } from '../src/index.js';
|
||||
import { dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { expect, jest, test, describe, beforeEach, afterEach } from '@jest/globals';
|
||||
import { Agent as httpAgent } from 'node:http';
|
||||
import { Agent as httpsAgent } from 'node:https';
|
||||
import { StatusError } from '../src/utils/status-error.js';
|
||||
|
||||
const _filename = fileURLToPath(import.meta.url);
|
||||
const _dirname = dirname(_filename);
|
||||
|
||||
/* settings below */
|
||||
|
||||
Error.stackTraceLimit = Infinity;
|
||||
|
||||
// During the test the env variable is set to test
|
||||
process.env.NODE_ENV = 'test';
|
||||
process.env.SUMMALY_ALLOW_PRIVATE_IP = 'true';
|
||||
|
||||
const port = 3060;
|
||||
const host = `http://localhost:${port}`;
|
||||
|
||||
// Display detail of unhandled promise rejection
|
||||
process.on('unhandledRejection', console.dir);
|
||||
|
||||
let app: ReturnType<typeof fastify> | null = null;
|
||||
let n = 0;
|
||||
|
||||
afterEach(async () => {
|
||||
if (app) {
|
||||
await app.close();
|
||||
app = null;
|
||||
}
|
||||
});
|
||||
|
||||
/* tests below */
|
||||
|
||||
test('basic', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
return reply.send(fs.createReadStream(_dirname + '/htmls/basic.html'));
|
||||
});
|
||||
await app.listen({ port });
|
||||
expect(await summaly(host)).toEqual({
|
||||
title: 'KISS principle',
|
||||
icon: null,
|
||||
description: null,
|
||||
thumbnail: null,
|
||||
player: {
|
||||
url: null,
|
||||
width: null,
|
||||
height: null,
|
||||
"allow": [
|
||||
"autoplay",
|
||||
"encrypted-media",
|
||||
"fullscreen",
|
||||
],
|
||||
},
|
||||
sitename: 'localhost:3060',
|
||||
sensitive: false,
|
||||
url: host,
|
||||
activityPub: null,
|
||||
});
|
||||
});
|
||||
|
||||
test('Stage Bye Stage', async () => {
|
||||
// If this test fails, you must rewrite the result data and the example in README.md.
|
||||
|
||||
const summary = await summaly('https://www.youtube.com/watch?v=NMIEAhH_fTU');
|
||||
expect(summary).toEqual(
|
||||
{
|
||||
"title": "【アイドルマスター】「Stage Bye Stage」(歌:島村卯月、渋谷凛、本田未央)",
|
||||
"icon": "https://www.youtube.com/s/desktop/28b0985e/img/favicon.ico",
|
||||
"description": "Website▶https://columbia.jp/idolmaster/Playlist▶https://www.youtube.com/playlist?list=PL83A2998CF3BBC86D2018年7月18日発売予定THE IDOLM@STER CINDERELLA GIRLS CG STAR...",
|
||||
"thumbnail": "https://i.ytimg.com/vi/NMIEAhH_fTU/maxresdefault.jpg",
|
||||
"player": {
|
||||
"url": "https://www.youtube.com/embed/NMIEAhH_fTU?feature=oembed",
|
||||
"width": 200,
|
||||
"height": 113,
|
||||
"allow": [
|
||||
"autoplay",
|
||||
"clipboard-write",
|
||||
"encrypted-media",
|
||||
"picture-in-picture",
|
||||
"web-share",
|
||||
"fullscreen",
|
||||
]
|
||||
},
|
||||
"sitename": "YouTube",
|
||||
"sensitive": false,
|
||||
"activityPub": null,
|
||||
"url": "https://www.youtube.com/watch?v=NMIEAhH_fTU"
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
test('faviconがHTML上で指定されていないが、ルートに存在する場合、正しく設定される', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
return reply.send(fs.createReadStream(_dirname + '/htmls/no-favicon.html'));
|
||||
});
|
||||
app.get('/favicon.ico', (_, reply) => reply.status(200).send());
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.icon).toBe(`${host}/favicon.ico`);
|
||||
});
|
||||
|
||||
test('faviconがHTML上で指定されていなくて、ルートにも存在しなかった場合 null になる', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
return reply.send(fs.createReadStream(_dirname + '/htmls/no-favicon.html'));
|
||||
});
|
||||
app.get('*', (_, reply) => reply.status(404).send());
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.icon).toBe(null);
|
||||
});
|
||||
|
||||
test('titleがcleanupされる', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
return reply.send(fs.createReadStream(_dirname + '/htmls/dirty-title.html'));
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.title).toBe('Strawberry Pasta');
|
||||
});
|
||||
|
||||
describe('Private IP blocking', () => {
|
||||
beforeEach(() => {
|
||||
process.env.SUMMALY_ALLOW_PRIVATE_IP = 'false';
|
||||
app = fastify();
|
||||
app.get('*', (request, reply) => {
|
||||
return reply.send(fs.createReadStream(_dirname + '/htmls/og-title.html'));
|
||||
});
|
||||
return app.listen({ port });
|
||||
});
|
||||
|
||||
test('private ipなサーバーの情報を取得できない', async () => {
|
||||
const summary = await summaly(host).catch((e: StatusError) => e);
|
||||
if (summary instanceof StatusError) {
|
||||
expect(summary.name).toBe('StatusError');
|
||||
} else {
|
||||
expect(summary).toBeInstanceOf(StatusError);
|
||||
}
|
||||
});
|
||||
|
||||
test('agentが指定されている場合はprivate ipを許可', async () => {
|
||||
const summary = await summaly(host, {
|
||||
agent: {
|
||||
http: new httpAgent({ keepAlive: true }),
|
||||
https: new httpsAgent({ keepAlive: true }),
|
||||
}
|
||||
});
|
||||
expect(summary.title).toBe('Strawberry Pasta');
|
||||
});
|
||||
|
||||
test('agentが空のオブジェクトの場合はprivate ipを許可しない', async () => {
|
||||
const summary = await summaly(host, { agent: {} }).catch((e: StatusError) => e);
|
||||
if (summary instanceof StatusError) {
|
||||
expect(summary.name).toBe('StatusError');
|
||||
} else {
|
||||
expect(summary).toBeInstanceOf(StatusError);
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.env.SUMMALY_ALLOW_PRIVATE_IP = 'true';
|
||||
});
|
||||
});
|
||||
|
||||
describe('OGP', () => {
|
||||
test('title', async () => {
|
||||
app = fastify();
|
||||
app.get('*', (request, reply) => {
|
||||
return reply.send(fs.createReadStream(_dirname + '/htmls/og-title.html'));
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.title).toBe('Strawberry Pasta');
|
||||
});
|
||||
|
||||
test('description', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
return reply.send(fs.createReadStream(_dirname + '/htmls/og-description.html'));
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.description).toBe('Strawberry Pasta');
|
||||
});
|
||||
|
||||
test('site_name', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
return reply.send(fs.createReadStream(_dirname + '/htmls/og-site_name.html'));
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.sitename).toBe('Strawberry Pasta');
|
||||
});
|
||||
|
||||
test('thumbnail', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
return reply.send(fs.createReadStream(_dirname + '/htmls/og-image.html'));
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.thumbnail).toBe('https://himasaku.net/himasaku.png');
|
||||
});
|
||||
});
|
||||
|
||||
describe('TwitterCard', () => {
|
||||
test('title', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
return reply.send(fs.createReadStream(_dirname + '/htmls/twitter-title.html'));
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.title).toBe('Strawberry Pasta');
|
||||
});
|
||||
|
||||
test('description', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
return reply.send(fs.createReadStream(_dirname + '/htmls/twitter-description.html'));
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.description).toBe('Strawberry Pasta');
|
||||
});
|
||||
|
||||
test('thumbnail', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
return reply.send(fs.createReadStream(_dirname + '/htmls/twitter-image.html'));
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.thumbnail).toBe('https://himasaku.net/himasaku.png');
|
||||
});
|
||||
|
||||
test('Player detection - PeerTube:video => video', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
return reply.send(fs.createReadStream(_dirname + '/htmls/player-peertube-video.html'));
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe('https://example.com/embedurl');
|
||||
expect(summary.player.allow).toStrictEqual(['autoplay', 'encrypted-media', 'fullscreen']);
|
||||
});
|
||||
|
||||
test('Player detection - Pleroma:video => video', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
return reply.send(fs.createReadStream(_dirname + '/htmls/player-pleroma-video.html'));
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe('https://example.com/embedurl');
|
||||
expect(summary.player.allow).toStrictEqual(['autoplay', 'encrypted-media', 'fullscreen']);
|
||||
});
|
||||
|
||||
test('Player detection - Pleroma:image => image', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
return reply.send(fs.createReadStream(_dirname + '/htmls/player-pleroma-image.html'));
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.thumbnail).toBe('https://example.com/imageurl');
|
||||
});
|
||||
});
|
||||
|
||||
describe("oEmbed", () => {
|
||||
const setUpFastify = async (oEmbedPath: string, htmlPath = 'htmls/oembed.html') => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
return reply.send(fs.createReadStream(new URL(htmlPath, import.meta.url)));
|
||||
});
|
||||
app.get('/oembed.json', (request, reply) => {
|
||||
return reply.send(fs.createReadStream(
|
||||
new URL(oEmbedPath, new URL('oembed/', import.meta.url))
|
||||
));
|
||||
});
|
||||
await app.listen({ port });
|
||||
}
|
||||
|
||||
for (const filename of readdirSync(new URL('oembed/invalid', import.meta.url))) {
|
||||
test(`Invalidity test: ${filename}`, async () => {
|
||||
await setUpFastify(`invalid/${filename}`);
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe(null);
|
||||
});
|
||||
}
|
||||
|
||||
test('basic properties', async () => {
|
||||
await setUpFastify('oembed.json');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe('https://example.com/');
|
||||
expect(summary.player.width).toBe(500);
|
||||
expect(summary.player.height).toBe(300);
|
||||
});
|
||||
|
||||
test('type: video', async () => {
|
||||
await setUpFastify('oembed-video.json');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe('https://example.com/');
|
||||
expect(summary.player.width).toBe(500);
|
||||
expect(summary.player.height).toBe(300);
|
||||
});
|
||||
|
||||
test('max height', async () => {
|
||||
await setUpFastify('oembed-too-tall.json');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.height).toBe(1024);
|
||||
});
|
||||
|
||||
test('children are ignored', async () => {
|
||||
await setUpFastify('oembed-iframe-child.json');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe('https://example.com/');
|
||||
});
|
||||
|
||||
test('allows fullscreen', async () => {
|
||||
await setUpFastify('oembed-allow-fullscreen.json');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe('https://example.com/');
|
||||
expect(summary.player.allow).toStrictEqual(['fullscreen']);
|
||||
});
|
||||
|
||||
test('allows legacy allowfullscreen', async () => {
|
||||
await setUpFastify('oembed-allow-fullscreen-legacy.json');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe('https://example.com/');
|
||||
expect(summary.player.allow).toStrictEqual(['fullscreen']);
|
||||
});
|
||||
|
||||
test('allows safelisted permissions', async () => {
|
||||
await setUpFastify('oembed-allow-safelisted-permissions.json');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe('https://example.com/');
|
||||
expect(summary.player.allow).toStrictEqual([
|
||||
'autoplay', 'clipboard-write', 'fullscreen',
|
||||
'encrypted-media', 'picture-in-picture', 'web-share',
|
||||
]);
|
||||
});
|
||||
|
||||
test('ignores rare permissions', async () => {
|
||||
await setUpFastify('oembed-ignore-rare-permissions.json');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe('https://example.com/');
|
||||
expect(summary.player.allow).toStrictEqual(['autoplay']);
|
||||
});
|
||||
|
||||
test('oEmbed with relative path', async () => {
|
||||
await setUpFastify('oembed.json', 'htmls/oembed-relative.html');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe('https://example.com/');
|
||||
});
|
||||
|
||||
test('oEmbed with nonexistent path', async () => {
|
||||
await setUpFastify('oembed.json', 'htmls/oembed-nonexistent-path.html');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe(null);
|
||||
expect(summary.description).toBe('nonexistent');
|
||||
});
|
||||
|
||||
test('oEmbed with wrong path', async () => {
|
||||
await setUpFastify('oembed.json', 'htmls/oembed-wrong-path.html');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe(null);
|
||||
expect(summary.description).toBe('wrong url');
|
||||
});
|
||||
|
||||
test('oEmbed with OpenGraph', async () => {
|
||||
await setUpFastify('oembed.json', 'htmls/oembed-and-og.html');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe('https://example.com/');
|
||||
expect(summary.description).toBe('blobcats rule the world');
|
||||
});
|
||||
|
||||
test('Invalid oEmbed with valid OpenGraph', async () => {
|
||||
await setUpFastify('invalid/oembed-insecure.json', 'htmls/oembed-and-og.html');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe(null);
|
||||
expect(summary.description).toBe('blobcats rule the world');
|
||||
});
|
||||
|
||||
test('oEmbed with og:video', async () => {
|
||||
await setUpFastify('oembed.json', 'htmls/oembed-and-og-video.html');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.url).toBe('https://example.com/');
|
||||
expect(summary.player.allow).toStrictEqual([]);
|
||||
});
|
||||
|
||||
test('width: 100%', async () => {
|
||||
await setUpFastify('oembed-percentage-width.json');
|
||||
const summary = await summaly(host);
|
||||
expect(summary.player.width).toBe(null);
|
||||
expect(summary.player.height).toBe(300);
|
||||
});
|
||||
});
|
||||
|
||||
describe('ActivityPub', () => {
|
||||
test('Basic', async () => {
|
||||
app = fastify();
|
||||
app.get('*', (request, reply) => {
|
||||
return reply.send(fs.createReadStream(_dirname + '/htmls/activitypub.html'));
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.activityPub).toBe('https://misskey.test/notes/abcdefg');
|
||||
});
|
||||
|
||||
test('Null', async () => {
|
||||
app = fastify();
|
||||
app.get('*', (request, reply) => {
|
||||
return reply.send(fs.createReadStream(_dirname + '/htmls/basic.html'));
|
||||
});
|
||||
await app.listen({ port });
|
||||
|
||||
const summary = await summaly(host);
|
||||
expect(summary.activityPub).toBe(null);
|
||||
});
|
||||
});
|
||||
|
||||
describe('sensitive', () => {
|
||||
test('default', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
return reply.send(fs.createReadStream(_dirname + '/htmls/basic.html'));
|
||||
});
|
||||
await app.listen({ port });
|
||||
expect((await summaly(host)).sensitive).toBe(false);
|
||||
});
|
||||
|
||||
test('mixi:content-rating 1', async () => {
|
||||
app = fastify();
|
||||
app.get('/', (request, reply) => {
|
||||
return reply.send(fs.createReadStream(_dirname + '/htmls/mixi-sensitive.html'));
|
||||
});
|
||||
await app.listen({ port });
|
||||
expect((await summaly(host)).sensitive).toBe(true);
|
||||
});
|
||||
});
|
45
test/tsconfig.json
Normal file
45
test/tsconfig.json
Normal file
@ -0,0 +1,45 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"allowJs": true,
|
||||
"noEmitOnError": false,
|
||||
"noImplicitAny": true,
|
||||
"noImplicitReturns": true,
|
||||
"noUnusedParameters": false,
|
||||
"noUnusedLocals": false,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"declaration": true,
|
||||
"sourceMap": false,
|
||||
"target": "es2021",
|
||||
"module": "nodenext",
|
||||
"moduleResolution": "nodenext",
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"removeComments": false,
|
||||
"noLib": false,
|
||||
"strict": true,
|
||||
"strictNullChecks": true,
|
||||
"strictPropertyInitialization": false,
|
||||
"skipLibCheck": true,
|
||||
"experimentalDecorators": true,
|
||||
"emitDecoratorMetadata": true,
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"baseUrl": "./../",
|
||||
"paths": {
|
||||
"@/*": [
|
||||
"./src/*"
|
||||
]
|
||||
},
|
||||
"outDir": "./built",
|
||||
"typeRoots": [
|
||||
"./node_modules/@types",
|
||||
"./src/@types"
|
||||
],
|
||||
"lib": [
|
||||
"esnext"
|
||||
]
|
||||
},
|
||||
"compileOnSave": false,
|
||||
"include": [
|
||||
"./**/*"
|
||||
],
|
||||
}
|
@ -10,8 +10,8 @@
|
||||
"declaration": true,
|
||||
"sourceMap": false,
|
||||
"target": "es2021",
|
||||
"module": "esnext",
|
||||
"moduleResolution": "node",
|
||||
"module": "nodenext",
|
||||
"moduleResolution": "nodenext",
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"removeComments": false,
|
||||
"noLib": false,
|
||||
@ -23,7 +23,6 @@
|
||||
"emitDecoratorMetadata": true,
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"rootDir": "./src",
|
||||
"baseUrl": "./",
|
||||
"paths": {
|
||||
"@/*": [
|
||||
@ -44,6 +43,9 @@
|
||||
},
|
||||
"compileOnSave": false,
|
||||
"include": [
|
||||
"./src/**/*.ts"
|
||||
"./src/**/*"
|
||||
],
|
||||
"exclude": [
|
||||
"node_modules",
|
||||
]
|
||||
}
|
||||
|
10
vitest.config.ts
Normal file
10
vitest.config.ts
Normal file
@ -0,0 +1,10 @@
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { defineConfig } from 'vitest/config';
|
||||
|
||||
export default defineConfig({
|
||||
resolve: {
|
||||
alias: {
|
||||
'@': fileURLToPath(new URL('./src', import.meta.url)),
|
||||
},
|
||||
},
|
||||
});
|
Reference in New Issue
Block a user