mirror of
https://github.com/misskey-dev/summaly.git
synced 2025-08-08 01:03:55 +09:00
Compare commits
6 Commits
Author | SHA1 | Date | |
---|---|---|---|
fce03ebd3c | |||
f7b59909bd | |||
5582cc8c9b | |||
7ebfe151d5 | |||
b0818bed3e | |||
909774bc63 |
24
.swcrc
24
.swcrc
@ -1,24 +0,0 @@
|
|||||||
{
|
|
||||||
"$schema": "https://json.schemastore.org/swcrc",
|
|
||||||
"jsc": {
|
|
||||||
"parser": {
|
|
||||||
"syntax": "typescript",
|
|
||||||
"dynamicImport": true,
|
|
||||||
"decorators": true
|
|
||||||
},
|
|
||||||
"transform": {
|
|
||||||
"legacyDecorator": true,
|
|
||||||
"decoratorMetadata": true
|
|
||||||
},
|
|
||||||
"experimental": {
|
|
||||||
"keepImportAssertions": true
|
|
||||||
},
|
|
||||||
"baseUrl": ".",
|
|
||||||
"paths": {},
|
|
||||||
"target": "es2021"
|
|
||||||
},
|
|
||||||
"module": {
|
|
||||||
"type": "es6"
|
|
||||||
},
|
|
||||||
"minify": true
|
|
||||||
}
|
|
10
CHANGELOG.md
10
CHANGELOG.md
@ -1,6 +1,16 @@
|
|||||||
(unreleased)
|
(unreleased)
|
||||||
------------------
|
------------------
|
||||||
|
|
||||||
|
5.2.3 / 2025/07/19
|
||||||
|
------------------
|
||||||
|
* パッケージが使用できない問題を修正
|
||||||
|
|
||||||
|
5.2.2 / 2025/07/06
|
||||||
|
------------------
|
||||||
|
* 最初のHEADリクエストにUAが反映されない問題を修正
|
||||||
|
* 依存関係の更新
|
||||||
|
* テストスイートをVitestに変更
|
||||||
|
|
||||||
5.2.1 / 2025/04/28
|
5.2.1 / 2025/04/28
|
||||||
------------------
|
------------------
|
||||||
* セキュリティに関する修正
|
* セキュリティに関する修正
|
||||||
|
@ -10,7 +10,7 @@ export default [ // eslint-disable-line import/no-default-export
|
|||||||
'**/node_modules',
|
'**/node_modules',
|
||||||
'src/@types/package.json.d.ts',
|
'src/@types/package.json.d.ts',
|
||||||
'built',
|
'built',
|
||||||
'jest.config.js',
|
'vitest.config.ts',
|
||||||
'test',
|
'test',
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
|
211
jest.config.js
211
jest.config.js
@ -1,211 +0,0 @@
|
|||||||
/*
|
|
||||||
* For a detailed explanation regarding each configuration property and type check, visit:
|
|
||||||
* https://jestjs.io/docs/en/configuration.html
|
|
||||||
*/
|
|
||||||
|
|
||||||
export default {
|
|
||||||
// All imported modules in your tests should be mocked automatically
|
|
||||||
// automock: false,
|
|
||||||
|
|
||||||
// Stop running tests after `n` failures
|
|
||||||
// bail: 0,
|
|
||||||
|
|
||||||
// The directory where Jest should store its cached dependency information
|
|
||||||
// cacheDirectory: "C:\\Users\\ai\\AppData\\Local\\Temp\\jest",
|
|
||||||
|
|
||||||
// Automatically clear mock calls and instances between every test
|
|
||||||
// clearMocks: false,
|
|
||||||
|
|
||||||
// Indicates whether the coverage information should be collected while executing the test
|
|
||||||
// collectCoverage: false,
|
|
||||||
|
|
||||||
// An array of glob patterns indicating a set of files for which coverage information should be collected
|
|
||||||
collectCoverageFrom: ['src/**/*.ts'],
|
|
||||||
|
|
||||||
// The directory where Jest should output its coverage files
|
|
||||||
coverageDirectory: "coverage",
|
|
||||||
|
|
||||||
// An array of regexp pattern strings used to skip coverage collection
|
|
||||||
// coveragePathIgnorePatterns: [
|
|
||||||
// "\\\\node_modules\\\\"
|
|
||||||
// ],
|
|
||||||
|
|
||||||
// Indicates which provider should be used to instrument code for coverage
|
|
||||||
coverageProvider: "v8",
|
|
||||||
|
|
||||||
// A list of reporter names that Jest uses when writing coverage reports
|
|
||||||
// coverageReporters: [
|
|
||||||
// "json",
|
|
||||||
// "text",
|
|
||||||
// "lcov",
|
|
||||||
// "clover"
|
|
||||||
// ],
|
|
||||||
|
|
||||||
// An object that configures minimum threshold enforcement for coverage results
|
|
||||||
// coverageThreshold: undefined,
|
|
||||||
|
|
||||||
// A path to a custom dependency extractor
|
|
||||||
// dependencyExtractor: undefined,
|
|
||||||
|
|
||||||
// Make calling deprecated APIs throw helpful error messages
|
|
||||||
// errorOnDeprecated: false,
|
|
||||||
|
|
||||||
// Force coverage collection from ignored files using an array of glob patterns
|
|
||||||
// forceCoverageMatch: [],
|
|
||||||
|
|
||||||
// A path to a module which exports an async function that is triggered once before all test suites
|
|
||||||
// globalSetup: undefined,
|
|
||||||
|
|
||||||
// A path to a module which exports an async function that is triggered once after all test suites
|
|
||||||
// globalTeardown: undefined,
|
|
||||||
|
|
||||||
// A set of global variables that need to be available in all test environments
|
|
||||||
globals: {
|
|
||||||
},
|
|
||||||
|
|
||||||
// The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers.
|
|
||||||
// maxWorkers: "50%",
|
|
||||||
|
|
||||||
// An array of directory names to be searched recursively up from the requiring module's location
|
|
||||||
// moduleDirectories: [
|
|
||||||
// "node_modules"
|
|
||||||
// ],
|
|
||||||
|
|
||||||
// An array of file extensions your modules use
|
|
||||||
// moduleFileExtensions: [
|
|
||||||
// "js",
|
|
||||||
// "json",
|
|
||||||
// "jsx",
|
|
||||||
// "ts",
|
|
||||||
// "tsx",
|
|
||||||
// "node"
|
|
||||||
// ],
|
|
||||||
|
|
||||||
// A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
|
|
||||||
moduleNameMapper: {
|
|
||||||
// Do not resolve .wasm.js to .wasm by the rule below
|
|
||||||
'^(.+)\\.wasm\\.js$': '$1.wasm.js',
|
|
||||||
// SWC converts @/foo/bar.js to `../../src/foo/bar.js`, and then this rule
|
|
||||||
// converts it again to `../../src/foo/bar` which then can be resolved to
|
|
||||||
// `.ts` files.
|
|
||||||
// See https://github.com/swc-project/jest/issues/64#issuecomment-1029753225
|
|
||||||
// TODO: Use `--allowImportingTsExtensions` on TypeScript 5.0 so that we can
|
|
||||||
// directly import `.ts` files without this hack.
|
|
||||||
'^(\\.{1,2}/.*)\\.js$': '$1',
|
|
||||||
|
|
||||||
// Resolve @/foo/bar to `../../src/foo/bar`
|
|
||||||
'^@/(.*)\\.js$': '<rootDir>/src/$1',
|
|
||||||
},
|
|
||||||
|
|
||||||
// An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
|
|
||||||
// modulePathIgnorePatterns: [],
|
|
||||||
|
|
||||||
// Activates notifications for test results
|
|
||||||
// notify: false,
|
|
||||||
|
|
||||||
// An enum that specifies notification mode. Requires { notify: true }
|
|
||||||
// notifyMode: "failure-change",
|
|
||||||
|
|
||||||
// A preset that is used as a base for Jest's configuration
|
|
||||||
//preset: "ts-jest/presets/js-with-ts-esm",
|
|
||||||
|
|
||||||
// Run tests from one or more projects
|
|
||||||
// projects: undefined,
|
|
||||||
|
|
||||||
// Use this configuration option to add custom reporters to Jest
|
|
||||||
// reporters: undefined,
|
|
||||||
|
|
||||||
// Automatically reset mock state between every test
|
|
||||||
// resetMocks: false,
|
|
||||||
|
|
||||||
// Reset the module registry before running each individual test
|
|
||||||
// resetModules: false,
|
|
||||||
|
|
||||||
// A path to a custom resolver
|
|
||||||
// resolver: './jest-resolver.cjs',
|
|
||||||
|
|
||||||
// Automatically restore mock state between every test
|
|
||||||
restoreMocks: true,
|
|
||||||
|
|
||||||
// The root directory that Jest should scan for tests and modules within
|
|
||||||
// rootDir: undefined,
|
|
||||||
|
|
||||||
// A list of paths to directories that Jest should use to search for files in
|
|
||||||
roots: [
|
|
||||||
"<rootDir>"
|
|
||||||
],
|
|
||||||
|
|
||||||
// Allows you to use a custom runner instead of Jest's default test runner
|
|
||||||
// runner: "jest-runner",
|
|
||||||
|
|
||||||
// The paths to modules that run some code to configure or set up the testing environment before each test
|
|
||||||
// setupFiles: [],
|
|
||||||
|
|
||||||
// A list of paths to modules that run some code to configure or set up the testing framework before each test
|
|
||||||
// setupFilesAfterEnv: [],
|
|
||||||
|
|
||||||
// The number of seconds after which a test is considered as slow and reported as such in the results.
|
|
||||||
// slowTestThreshold: 5,
|
|
||||||
|
|
||||||
// A list of paths to snapshot serializer modules Jest should use for snapshot testing
|
|
||||||
// snapshotSerializers: [],
|
|
||||||
|
|
||||||
// The test environment that will be used for testing
|
|
||||||
testEnvironment: "node",
|
|
||||||
|
|
||||||
// Options that will be passed to the testEnvironment
|
|
||||||
// testEnvironmentOptions: {},
|
|
||||||
|
|
||||||
// Adds a location field to test results
|
|
||||||
// testLocationInResults: false,
|
|
||||||
|
|
||||||
// The glob patterns Jest uses to detect test files
|
|
||||||
testMatch: [
|
|
||||||
"<rootDir>/test/index.ts",
|
|
||||||
],
|
|
||||||
|
|
||||||
// An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
|
|
||||||
// testPathIgnorePatterns: [
|
|
||||||
// "\\\\node_modules\\\\"
|
|
||||||
// ],
|
|
||||||
|
|
||||||
// The regexp pattern or array of patterns that Jest uses to detect test files
|
|
||||||
// testRegex: [],
|
|
||||||
|
|
||||||
// This option allows the use of a custom results processor
|
|
||||||
// testResultsProcessor: undefined,
|
|
||||||
|
|
||||||
// This option allows use of a custom test runner
|
|
||||||
// testRunner: "jasmine2",
|
|
||||||
|
|
||||||
// This option sets the URL for the jsdom environment. It is reflected in properties such as location.href
|
|
||||||
// testURL: "http://localhost",
|
|
||||||
|
|
||||||
// Setting this value to "fake" allows the use of fake timers for functions such as "setTimeout"
|
|
||||||
// timers: "real",
|
|
||||||
|
|
||||||
// A map from regular expressions to paths to transformers
|
|
||||||
transform: {
|
|
||||||
"^.+\\.(t|j)sx?$": ["@swc/jest"],
|
|
||||||
},
|
|
||||||
|
|
||||||
// An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
|
|
||||||
// transformIgnorePatterns: [
|
|
||||||
// "\\\\node_modules\\\\",
|
|
||||||
// "\\.pnp\\.[^\\\\]+$"
|
|
||||||
// ],
|
|
||||||
|
|
||||||
// An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
|
|
||||||
// unmockedModulePathPatterns: undefined,
|
|
||||||
|
|
||||||
// Indicates whether each individual test should be reported during the run
|
|
||||||
// verbose: undefined,
|
|
||||||
|
|
||||||
// An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode
|
|
||||||
// watchPathIgnorePatterns: [],
|
|
||||||
|
|
||||||
// Whether to use watchman for file crawling
|
|
||||||
// watchman: true,
|
|
||||||
|
|
||||||
extensionsToTreatAsEsm: ['.ts'],
|
|
||||||
};
|
|
36
package.json
36
package.json
@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@misskey-dev/summaly",
|
"name": "@misskey-dev/summaly",
|
||||||
"version": "5.2.1",
|
"version": "5.2.3",
|
||||||
"description": "Get web page's summary",
|
"description": "Get web page's summary",
|
||||||
"author": "syuilo <syuilotan@yahoo.co.jp>",
|
"author": "syuilo <syuilotan@yahoo.co.jp>",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
@ -17,34 +17,30 @@
|
|||||||
"scripts": {
|
"scripts": {
|
||||||
"build": "tsc && tsc-alias",
|
"build": "tsc && tsc-alias",
|
||||||
"eslint": "eslint",
|
"eslint": "eslint",
|
||||||
"test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --silent=false --verbose false",
|
"test": "vitest run ./test/index.test.ts",
|
||||||
"serve": "fastify start ./built/index.js"
|
"serve": "fastify start ./built/index.js"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@jest/globals": "^29.7.0",
|
|
||||||
"@misskey-dev/eslint-plugin": "^2.1.0",
|
"@misskey-dev/eslint-plugin": "^2.1.0",
|
||||||
"@swc/core": "^1.10.12",
|
|
||||||
"@swc/jest": "^0.2.37",
|
|
||||||
"@types/cheerio": "0.22.35",
|
|
||||||
"@types/debug": "4.1.12",
|
"@types/debug": "4.1.12",
|
||||||
"@types/escape-regexp": "^0.0.3",
|
"@types/escape-regexp": "^0.0.3",
|
||||||
"@types/node": "22.13.0",
|
"@types/node": "22.16.0",
|
||||||
"@typescript-eslint/eslint-plugin": "^8.22.0",
|
"@typescript-eslint/eslint-plugin": "^8.35.1",
|
||||||
"@typescript-eslint/parser": "^8.22.0",
|
"@typescript-eslint/parser": "^8.35.1",
|
||||||
"debug": "^4.4.0",
|
"debug": "^4.4.1",
|
||||||
"eslint": "^9.19.0",
|
"eslint": "^9.30.1",
|
||||||
"eslint-plugin-import": "^2.31.0",
|
"eslint-plugin-import": "^2.32.0",
|
||||||
"fastify": "^5.2.1",
|
"fastify": "^5.4.0",
|
||||||
"fastify-cli": "^7.3.0",
|
"fastify-cli": "^7.4.0",
|
||||||
"jest": "^29.7.0",
|
"tsc-alias": "^1.8.16",
|
||||||
"tsc-alias": "^1.8.10",
|
"typescript": "5.8.3",
|
||||||
"typescript": "5.7.3"
|
"vitest": "^3.2.4"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"cheerio": "1.0.0",
|
"cheerio": "1.1.0",
|
||||||
"escape-regexp": "0.0.1",
|
"escape-regexp": "0.0.1",
|
||||||
"got": "^14.4.5",
|
"got": "^14.4.7",
|
||||||
"html-entities": "2.5.2",
|
"html-entities": "2.6.0",
|
||||||
"iconv-lite": "0.6.3",
|
"iconv-lite": "0.6.3",
|
||||||
"jschardet": "3.1.4",
|
"jschardet": "3.1.4",
|
||||||
"private-ip": "3.0.2"
|
"private-ip": "3.0.2"
|
||||||
|
4133
pnpm-lock.yaml
generated
4133
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
@ -72,7 +72,7 @@ async function getOEmbedPlayer($: cheerio.CheerioAPI, pageUrl: string): Promise<
|
|||||||
// Allow only HTTPS for best security
|
// Allow only HTTPS for best security
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
} catch (e) {
|
} catch {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -269,7 +269,7 @@ export async function parseGeneral(_url: URL | string, res: Awaited<ReturnType<t
|
|||||||
try {
|
try {
|
||||||
await head(target.href);
|
await head(target.href);
|
||||||
return target;
|
return target;
|
||||||
} catch (e) {
|
} catch {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
@ -8,7 +8,7 @@ import type { FastifyInstance } from 'fastify';
|
|||||||
import { SummalyResult } from '@/summary.js';
|
import { SummalyResult } from '@/summary.js';
|
||||||
import { SummalyPlugin as _SummalyPlugin } from '@/iplugin.js';
|
import { SummalyPlugin as _SummalyPlugin } from '@/iplugin.js';
|
||||||
import { general, type GeneralScrapingOptions } from '@/general.js';
|
import { general, type GeneralScrapingOptions } from '@/general.js';
|
||||||
import { DEFAULT_OPERATION_TIMEOUT, DEFAULT_RESPONSE_TIMEOUT, agent, setAgent } from '@/utils/got.js';
|
import { DEFAULT_BOT_UA, DEFAULT_OPERATION_TIMEOUT, DEFAULT_RESPONSE_TIMEOUT, agent, setAgent } from '@/utils/got.js';
|
||||||
import { plugins as builtinPlugins } from '@/plugins/index.js';
|
import { plugins as builtinPlugins } from '@/plugins/index.js';
|
||||||
|
|
||||||
export type SummalyPlugin = _SummalyPlugin;
|
export type SummalyPlugin = _SummalyPlugin;
|
||||||
@ -88,6 +88,11 @@ export const summaly = async (url: string, options?: SummalyOptions): Promise<Su
|
|||||||
const operationTimeout = opts.operationTimeout ?? DEFAULT_OPERATION_TIMEOUT;
|
const operationTimeout = opts.operationTimeout ?? DEFAULT_OPERATION_TIMEOUT;
|
||||||
actualUrl = await got
|
actualUrl = await got
|
||||||
.head(url, {
|
.head(url, {
|
||||||
|
headers: {
|
||||||
|
accept: 'text/html,application/xhtml+xml',
|
||||||
|
'user-agent': opts.userAgent ?? DEFAULT_BOT_UA,
|
||||||
|
'accept-language': opts.lang ?? undefined,
|
||||||
|
},
|
||||||
timeout: {
|
timeout: {
|
||||||
lookup: timeout,
|
lookup: timeout,
|
||||||
connect: timeout,
|
connect: timeout,
|
||||||
@ -104,7 +109,7 @@ export const summaly = async (url: string, options?: SummalyOptions): Promise<Su
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
.then(res => res.url);
|
.then(res => res.url);
|
||||||
} catch (e) {
|
} catch {
|
||||||
actualUrl = url;
|
actualUrl = url;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -12,10 +12,10 @@ import { dirname } from 'node:path';
|
|||||||
import { fileURLToPath } from 'node:url';
|
import { fileURLToPath } from 'node:url';
|
||||||
import { Agent as httpAgent } from 'node:http';
|
import { Agent as httpAgent } from 'node:http';
|
||||||
import { Agent as httpsAgent } from 'node:https';
|
import { Agent as httpsAgent } from 'node:https';
|
||||||
import { expect, test, describe, beforeEach, afterEach, xtest } from '@jest/globals';
|
import { expect, test, describe, beforeEach, afterEach } from 'vitest';
|
||||||
import fastify, { type FastifyInstance } from 'fastify';
|
import fastify, { type FastifyInstance } from 'fastify';
|
||||||
import { summaly } from '../src/index.js';
|
import { summaly } from '@/index.js';
|
||||||
import { StatusError } from '../src/utils/status-error.js';
|
import { StatusError } from '@/utils/status-error.js';
|
||||||
|
|
||||||
const _filename = fileURLToPath(import.meta.url);
|
const _filename = fileURLToPath(import.meta.url);
|
||||||
const _dirname = dirname(_filename);
|
const _dirname = dirname(_filename);
|
||||||
@ -39,7 +39,7 @@ let app: FastifyInstance | null = null;
|
|||||||
function skippableTest(name: string, fn: () => void) {
|
function skippableTest(name: string, fn: () => void) {
|
||||||
if (process.env.SKIP_NETWORK_TEST === 'true') {
|
if (process.env.SKIP_NETWORK_TEST === 'true') {
|
||||||
console.log(`[SKIP] ${name}`);
|
console.log(`[SKIP] ${name}`);
|
||||||
xtest(name, fn);
|
test.skip(name, fn);
|
||||||
} else {
|
} else {
|
||||||
test(name, fn);
|
test(name, fn);
|
||||||
}
|
}
|
45
test/tsconfig.json
Normal file
45
test/tsconfig.json
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"allowJs": true,
|
||||||
|
"noEmitOnError": false,
|
||||||
|
"noImplicitAny": true,
|
||||||
|
"noImplicitReturns": true,
|
||||||
|
"noUnusedParameters": false,
|
||||||
|
"noUnusedLocals": false,
|
||||||
|
"noFallthroughCasesInSwitch": true,
|
||||||
|
"declaration": true,
|
||||||
|
"sourceMap": false,
|
||||||
|
"target": "es2021",
|
||||||
|
"module": "nodenext",
|
||||||
|
"moduleResolution": "nodenext",
|
||||||
|
"allowSyntheticDefaultImports": true,
|
||||||
|
"removeComments": false,
|
||||||
|
"noLib": false,
|
||||||
|
"strict": true,
|
||||||
|
"strictNullChecks": true,
|
||||||
|
"strictPropertyInitialization": false,
|
||||||
|
"skipLibCheck": true,
|
||||||
|
"experimentalDecorators": true,
|
||||||
|
"emitDecoratorMetadata": true,
|
||||||
|
"resolveJsonModule": true,
|
||||||
|
"isolatedModules": true,
|
||||||
|
"baseUrl": "./../",
|
||||||
|
"paths": {
|
||||||
|
"@/*": [
|
||||||
|
"./src/*"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"outDir": "./built",
|
||||||
|
"typeRoots": [
|
||||||
|
"./node_modules/@types",
|
||||||
|
"./src/@types"
|
||||||
|
],
|
||||||
|
"lib": [
|
||||||
|
"esnext"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"compileOnSave": false,
|
||||||
|
"include": [
|
||||||
|
"./**/*"
|
||||||
|
],
|
||||||
|
}
|
@ -47,6 +47,5 @@
|
|||||||
],
|
],
|
||||||
"exclude": [
|
"exclude": [
|
||||||
"node_modules",
|
"node_modules",
|
||||||
"test/**/*"
|
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
10
vitest.config.ts
Normal file
10
vitest.config.ts
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
import { fileURLToPath } from 'node:url';
|
||||||
|
import { defineConfig } from 'vitest/config';
|
||||||
|
|
||||||
|
export default defineConfig({
|
||||||
|
resolve: {
|
||||||
|
alias: {
|
||||||
|
'@': fileURLToPath(new URL('./src', import.meta.url)),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
Reference in New Issue
Block a user