26 Commits

Author SHA1 Message Date
e43065b426 built 2023-03-11 22:38:44 +01:00
38db975931 Merge branch 'oembed' into oembed-built 2023-03-11 22:38:17 +01:00
f30797e754 ignored permissions 2023-03-11 22:37:40 +01:00
6c7bfa8ff1 oops 2023-03-11 22:36:47 +01:00
f9bb67638e ignored permissions 2023-03-11 22:35:44 +01:00
87241994fd Merge branch 'oembed' into oembed-built 2023-03-11 22:15:41 +01:00
86977e5a12 built 2023-03-11 22:15:28 +01:00
f33b20ac8e restore max height test 2023-03-11 22:14:32 +01:00
e72b4191ea nullable width 2023-03-11 22:10:55 +01:00
d59a508190 test for type: video 2023-03-11 21:45:49 +01:00
28e0552f5c support width (for size ratio) 2023-03-11 21:40:34 +01:00
a28f408f4e built 2023-03-11 21:40:04 +01:00
6d182f919e support width (for size ratio) 2023-03-11 21:38:28 +01:00
1866d9929a built 2023-03-11 20:51:46 +01:00
e02da09f9a Merge branch 'oembed' into oembed-built 2023-03-11 20:51:29 +01:00
5126f71e0a fix type error 2023-03-11 20:50:54 +01:00
4c45ed716e playerを使うように 2023-03-11 20:48:25 +01:00
84f96c3961 names 2023-03-11 17:53:30 +01:00
61a4a17a02 permissions 2023-03-11 17:53:12 +01:00
2fdad915d2 Update README.md 2023-03-11 17:52:42 +01:00
883baf437a built 2023-03-11 15:09:32 +01:00
51148cea27 Merge branch 'oembed' into oembed-built 2023-03-11 15:09:10 +01:00
1ea7059a20 fix the syntax 2023-03-11 15:08:29 +01:00
5153305bdd more safelisted features 2023-03-11 14:59:09 +01:00
7b8a2b0913 built 2023-03-11 14:31:38 +01:00
a5a8c4437d feat: add oEmbed support 2023-03-11 14:09:00 +01:00
21 changed files with 868 additions and 905 deletions

View File

@ -1,15 +1,6 @@
4.0.2 / 2023-04-20 Unreleased
------------------
* YouTubeをフルスクリーンにできない問題を修正
4.0.1 / 2023-03-16
------------------
* oEmbedの読み込みでエラーが発生した際は、エラーにせずplayerの中身をnullにするように
4.0.0 / 2023-03-14
------------------ ------------------
* oEmbed type=richの制限的なサポート * oEmbed type=richの制限的なサポート
* プラグインの引数がWHATWG URLになりました
3.0.4 / 2023-02-12 3.0.4 / 2023-02-12
------------------ ------------------

View File

@ -51,13 +51,11 @@ npm run serve
``` typescript ``` typescript
interface IPlugin { interface IPlugin {
test: (url: URL) => boolean; test: (url: URL.Url) => boolean;
summarize: (url: URL) => Promise<Summary>; summarize: (url: URL.Url) => Promise<Summary>;
} }
``` ```
urls are WHATWG URL since v4.
### Returns ### Returns
A Promise of an Object that contains properties below: A Promise of an Object that contains properties below:
@ -103,33 +101,22 @@ import { summaly } from 'summaly';
const summary = await summaly('https://www.youtube.com/watch?v=NMIEAhH_fTU'); const summary = await summaly('https://www.youtube.com/watch?v=NMIEAhH_fTU');
console.log(summary); console.log(summary); // will be ... ↓
``` /*
will be ... ↓
```json
{ {
"title": "【アイドルマスター】「Stage Bye Stage」(歌:島村卯月、渋谷凛、本田未央)", title: '【楽曲試聴】「Stage Bye Stage」(歌:島村卯月、渋谷凛、本田未央)',
"icon": "https://www.youtube.com/s/desktop/9318de79/img/favicon.ico", icon: 'https://s.ytimg.com/yts/img/favicon-vfl8qSV2F.ico',
"description": "Website▶https://columbia.jp/idolmaster/Playlist▶https://www.youtube.com/playlist?list=PL83A2998CF3BBC86D2018年7月18日発売予定THE IDOLM@STER CINDERELLA GIRLS CG STAR...", description: 'http://columbia.jp/idolmaster/ 2018年7月18日発売予定 THE IDOLM@STER CINDERELLA GIRLS CG STAR LIVE Stage Bye Stage 歌:島村卯月、渋谷凛、本田未央 COCC-17495CD1枚組 ¥1,200税 収録内容 Tr...',
"thumbnail": "https://i.ytimg.com/vi/NMIEAhH_fTU/maxresdefault.jpg", thumbnail: 'https://i.ytimg.com/vi/NMIEAhH_fTU/maxresdefault.jpg',
"player": { player: {
"url": "https://www.youtube.com/embed/NMIEAhH_fTU?feature=oembed", url: 'https://www.youtube.com/embed/NMIEAhH_fTU',
"width": 200, width: 1280,
"height": 113, height: 720
"allow": [
"autoplay",
"clipboard-write",
"encrypted-media",
"picture-in-picture",
"web-share"
]
}, },
"sitename": "YouTube", sitename: 'YouTube',
"sensitive": false, url: 'https://www.youtube.com/watch?v=NMIEAhH_fTU'
"url": "https://www.youtube.com/watch?v=NMIEAhH_fTU"
} }
*/
``` ```
Testing Testing
@ -140,8 +127,12 @@ License
---------------------------------------------------------------- ----------------------------------------------------------------
[MIT](LICENSE) [MIT](LICENSE)
[npm-link]: https://www.npmjs.com/package/summaly
[npm-badge]: https://img.shields.io/npm/v/summaly.svg?style=flat-square
[mit]: http://opensource.org/licenses/MIT [mit]: http://opensource.org/licenses/MIT
[mit-badge]: https://img.shields.io/badge/license-MIT-444444.svg?style=flat-square [mit-badge]: https://img.shields.io/badge/license-MIT-444444.svg?style=flat-square
[travis-link]: https://travis-ci.org/syuilo/summaly
[travis-badge]: http://img.shields.io/travis/syuilo/summaly.svg?style=flat-square
[himasaku]: https://himasaku.net [himasaku]: https://himasaku.net
[himawari-badge]: https://img.shields.io/badge/%E5%8F%A4%E8%B0%B7-%E5%90%91%E6%97%A5%E8%91%B5-1684c5.svg?style=flat-square [himawari-badge]: https://img.shields.io/badge/%E5%8F%A4%E8%B0%B7-%E5%90%91%E6%97%A5%E8%91%B5-1684c5.svg?style=flat-square
[sakurako-badge]: https://img.shields.io/badge/%E5%A4%A7%E5%AE%A4-%E6%AB%BB%E5%AD%90-efb02a.svg?style=flat-square [sakurako-badge]: https://img.shields.io/badge/%E5%A4%A7%E5%AE%A4-%E6%AB%BB%E5%AD%90-efb02a.svg?style=flat-square

4
built/general.d.ts vendored
View File

@ -1,4 +1,4 @@
import { URL } from 'node:url'; import * as URL from 'node:url';
import type { default as Summary } from './summary.js'; import type { default as Summary } from './summary.js';
declare const _default: (_url: URL | string, lang?: string | null) => Promise<Summary | null>; declare const _default: (url: URL.Url, lang?: string | null) => Promise<Summary | null>;
export default _default; export default _default;

View File

@ -1,4 +1,4 @@
import { URL } from 'node:url'; import * as URL from 'node:url';
import clip from './utils/clip.js'; import clip from './utils/clip.js';
import cleanupTitle from './utils/cleanup-title.js'; import cleanupTitle from './utils/cleanup-title.js';
import { decode as decodeHtml } from 'html-entities'; import { decode as decodeHtml } from 'html-entities';
@ -15,21 +15,9 @@ async function getOEmbedPlayer($, pageUrl) {
if (!href) { if (!href) {
return null; return null;
} }
const oEmbedUrl = (() => { // XXX: Use global URL object instead of the deprecated `node:url`
try { // Disallow relative URL as no one seems to use it
return new URL(href, pageUrl); const oEmbed = await get(URL.resolve(pageUrl, href));
}
catch {
return null;
}
})();
if (!oEmbedUrl) {
return null;
}
const oEmbed = await get(oEmbedUrl.href).catch(() => null);
if (!oEmbed) {
return null;
}
const body = (() => { const body = (() => {
try { try {
return JSON.parse(oEmbed); return JSON.parse(oEmbed);
@ -59,15 +47,11 @@ async function getOEmbedPlayer($, pageUrl) {
// No src? // No src?
return null; return null;
} }
try { // XXX: Use global URL object instead of the deprecated `node:url`
if ((new URL(url)).protocol !== 'https:') { if (URL.parse(url).protocol !== 'https:') {
// Allow only HTTPS for best security // Allow only HTTPS for best security
return null; return null;
} }
}
catch (e) {
return null;
}
// Height is the most important, width is okay to be null. The implementer // Height is the most important, width is okay to be null. The implementer
// should choose fixed height instead of fixed aspect ratio if width is null. // should choose fixed height instead of fixed aspect ratio if width is null.
// //
@ -103,9 +87,6 @@ async function getOEmbedPlayer($, pageUrl) {
const allowedPermissions = (iframe.attr('allow') ?? '').split(/\s*;\s*/g) const allowedPermissions = (iframe.attr('allow') ?? '').split(/\s*;\s*/g)
.filter(s => s) .filter(s => s)
.filter(s => !ignoredList.includes(s)); .filter(s => !ignoredList.includes(s));
if (iframe.attr('allowfullscreen') === '') {
allowedPermissions.push('fullscreen');
}
if (allowedPermissions.some(allow => !safeList.includes(allow))) { if (allowedPermissions.some(allow => !safeList.includes(allow))) {
// This iframe is probably too powerful to be embedded // This iframe is probably too powerful to be embedded
return null; return null;
@ -117,10 +98,9 @@ async function getOEmbedPlayer($, pageUrl) {
allow: allowedPermissions allow: allowedPermissions
}; };
} }
export default async (_url, lang = null) => { export default async (url, lang = null) => {
if (lang && !lang.match(/^[\w-]+(\s*,\s*[\w-]+)*$/)) if (lang && !lang.match(/^[\w-]+(\s*,\s*[\w-]+)*$/))
lang = null; lang = null;
const url = typeof _url === 'string' ? new URL(_url) : _url;
const res = await scpaping(url.href, { lang: lang || undefined }); const res = await scpaping(url.href, { lang: lang || undefined });
const $ = res.$; const $ = res.$;
const twitterCard = $('meta[property="twitter:card"]').attr('content'); const twitterCard = $('meta[property="twitter:card"]').attr('content');
@ -136,7 +116,7 @@ export default async (_url, lang = null) => {
$('link[rel="image_src"]').attr('href') || $('link[rel="image_src"]').attr('href') ||
$('link[rel="apple-touch-icon"]').attr('href') || $('link[rel="apple-touch-icon"]').attr('href') ||
$('link[rel="apple-touch-icon image_src"]').attr('href'); $('link[rel="apple-touch-icon image_src"]').attr('href');
image = image ? (new URL(image, url.href)).href : null; image = image ? URL.resolve(url.href, image) : null;
const playerUrl = (twitterCard !== 'summary_large_image' && $('meta[property="twitter:player"]').attr('content')) || const playerUrl = (twitterCard !== 'summary_large_image' && $('meta[property="twitter:player"]').attr('content')) ||
(twitterCard !== 'summary_large_image' && $('meta[name="twitter:player"]').attr('content')) || (twitterCard !== 'summary_large_image' && $('meta[name="twitter:player"]').attr('content')) ||
$('meta[property="og:video"]').attr('content') || $('meta[property="og:video"]').attr('content') ||
@ -159,25 +139,40 @@ export default async (_url, lang = null) => {
if (title === description) { if (title === description) {
description = null; description = null;
} }
let siteName = decodeHtml($('meta[property="og:site_name"]').attr('content') || let siteName = $('meta[property="og:site_name"]').attr('content') ||
$('meta[name="application-name"]').attr('content') || $('meta[name="application-name"]').attr('content') ||
url.hostname); url.hostname;
siteName = siteName ? decodeHtml(siteName) : null;
const favicon = $('link[rel="shortcut icon"]').attr('href') || const favicon = $('link[rel="shortcut icon"]').attr('href') ||
$('link[rel="icon"]').attr('href') || $('link[rel="icon"]').attr('href') ||
'/favicon.ico'; '/favicon.ico';
const sensitive = $('.tweet').attr('data-possibly-sensitive') === 'true'; const sensitive = $('.tweet').attr('data-possibly-sensitive') === 'true';
const find = async (path) => { const find = async (path) => {
const target = new URL(path, url.href); const target = URL.resolve(url.href, path);
try { try {
await head(target.href); await head(target);
return target; return target;
} }
catch (e) { catch (e) {
return null; return null;
} }
}; };
// 相対的なURL (ex. test) を絶対的 (ex. /test) に変換
const toAbsolute = (relativeURLString) => {
const relativeURL = URL.parse(relativeURLString);
const isAbsolute = relativeURL.slashes || relativeURL.path !== null && relativeURL.path[0] === '/';
// 既に絶対的なら、即座に値を返却
if (isAbsolute) {
return relativeURLString;
}
// スラッシュを付けて返却
return '/' + relativeURLString;
};
const getIcon = async () => { const getIcon = async () => {
return (await find(favicon)) || null; return await find(favicon) ||
// 相対指定を絶対指定に変換し再試行
await find(toAbsolute(favicon)) ||
null;
}; };
const [icon, oEmbed] = await Promise.all([ const [icon, oEmbed] = await Promise.all([
getIcon(), getIcon(),
@ -190,14 +185,14 @@ export default async (_url, lang = null) => {
} }
return { return {
title: title || null, title: title || null,
icon: icon?.href || null, icon: icon || null,
description: description || null, description: description || null,
thumbnail: image || null, thumbnail: image || null,
player: oEmbed ?? { player: oEmbed ?? {
url: playerUrl || null, url: playerUrl || null,
width: Number.isNaN(playerWidth) ? null : playerWidth, width: Number.isNaN(playerWidth) ? null : playerWidth,
height: Number.isNaN(playerHeight) ? null : playerHeight, height: Number.isNaN(playerHeight) ? null : playerHeight,
allow: ['autoplay', 'encrypted-media', 'fullscreen'], allow: ['fullscreen', 'encrypted-media'],
}, },
sitename: siteName || null, sitename: siteName || null,
sensitive, sensitive,

View File

@ -2,7 +2,7 @@
* summaly * summaly
* https://github.com/syuilo/summaly * https://github.com/syuilo/summaly
*/ */
import { URL } from 'node:url'; import * as URL from 'node:url';
import tracer from 'trace-redirect'; import tracer from 'trace-redirect';
import general from './general.js'; import general from './general.js';
import { setAgent } from './utils/got.js'; import { setAgent } from './utils/got.js';
@ -30,7 +30,7 @@ export const summaly = async (url, options) => {
actualUrl = url; actualUrl = url;
} }
} }
const _url = new URL(actualUrl); const _url = URL.parse(actualUrl, true);
// Find matching plugin // Find matching plugin
const match = plugins.filter(plugin => plugin.test(_url))[0]; const match = plugins.filter(plugin => plugin.test(_url))[0];
// Get summary // Get summary

6
built/iplugin.d.ts vendored
View File

@ -1,7 +1,7 @@
/// <reference types="node" /> /// <reference types="node" />
import type { URL } from 'node:url'; import * as URL from 'node:url';
import Summary from './summary.js'; import Summary from './summary.js';
export interface IPlugin { export interface IPlugin {
test: (url: URL) => boolean; test: (url: URL.Url) => boolean;
summarize: (url: URL, lang?: string) => Promise<Summary>; summarize: (url: URL.Url, lang?: string) => Promise<Summary>;
} }

View File

@ -1,5 +1,5 @@
/// <reference types="node" /> /// <reference types="node" />
import { URL } from 'node:url'; import * as URL from 'node:url';
import summary from '../summary.js'; import summary from '../summary.js';
export declare function test(url: URL): boolean; export declare function test(url: URL.Url): boolean;
export declare function summarize(url: URL): Promise<summary>; export declare function summarize(url: URL.Url): Promise<summary>;

View File

@ -1,5 +1,5 @@
/// <reference types="node" /> /// <reference types="node" />
import { URL } from 'node:url'; import * as URL from 'node:url';
import summary from '../summary.js'; import summary from '../summary.js';
export declare function test(url: URL): boolean; export declare function test(url: URL.Url): boolean;
export declare function summarize(url: URL): Promise<summary>; export declare function summarize(url: URL.Url): Promise<summary>;

56
package-lock.json generated
View File

@ -1,17 +1,17 @@
{ {
"name": "summaly", "name": "summaly",
"version": "4.0.0", "version": "3.0.4",
"lockfileVersion": 2, "lockfileVersion": 2,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "summaly", "name": "summaly",
"version": "4.0.0", "version": "3.0.4",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"cheerio": "1.0.0-rc.12", "cheerio": "^1.0.0-rc.12",
"escape-regexp": "0.0.1", "escape-regexp": "0.0.1",
"got": "^12.6.0", "got": "^12.5.3",
"html-entities": "2.3.2", "html-entities": "2.3.2",
"iconv-lite": "0.6.3", "iconv-lite": "0.6.3",
"jschardet": "3.0.0", "jschardet": "3.0.0",
@ -25,6 +25,7 @@
"@types/cheerio": "0.22.18", "@types/cheerio": "0.22.18",
"@types/debug": "4.1.7", "@types/debug": "4.1.7",
"@types/escape-regexp": "^0.0.1", "@types/escape-regexp": "^0.0.1",
"@types/html-entities": "1.3.4",
"@types/node": "16.11.12", "@types/node": "16.11.12",
"debug": "^4.3.4", "debug": "^4.3.4",
"fastify": "^4.13.0", "fastify": "^4.13.0",
@ -1392,6 +1393,16 @@
"@types/node": "*" "@types/node": "*"
} }
}, },
"node_modules/@types/html-entities": {
"version": "1.3.4",
"resolved": "https://registry.npmjs.org/@types/html-entities/-/html-entities-1.3.4.tgz",
"integrity": "sha512-Ut62LV90H9tgXwyhmfR8U6yCw/6xeo26IlsbAJJfqPomaqDN2zoLb2Z+cbmy5AycJFhwNJDdH0zqjQp7Ox/eXg==",
"deprecated": "This is a stub types definition. html-entities provides its own type definitions, so you do not need this installed.",
"dev": true,
"dependencies": {
"html-entities": "*"
}
},
"node_modules/@types/http-cache-semantics": { "node_modules/@types/http-cache-semantics": {
"version": "4.0.1", "version": "4.0.1",
"resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.1.tgz", "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.1.tgz",
@ -1827,9 +1838,9 @@
} }
}, },
"node_modules/cacheable-request": { "node_modules/cacheable-request": {
"version": "10.2.8", "version": "10.2.7",
"resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-10.2.8.tgz", "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-10.2.7.tgz",
"integrity": "sha512-IDVO5MJ4LItE6HKFQTqT2ocAQsisOoCTUDu1ddCmnhyiwFQjXNPp4081Xj23N4tO+AFEFNzGuNEf/c8Gwwt15A==", "integrity": "sha512-I4SA6mKgDxcxVbSt/UmIkb9Ny8qSkg6ReBHtAAXnZHk7KOSx5g3DTiAOaYzcHCs6oOdHn+bip9T48E6tMvK9hw==",
"dependencies": { "dependencies": {
"@types/http-cache-semantics": "^4.0.1", "@types/http-cache-semantics": "^4.0.1",
"get-stream": "^6.0.1", "get-stream": "^6.0.1",
@ -2769,14 +2780,14 @@
} }
}, },
"node_modules/got": { "node_modules/got": {
"version": "12.6.0", "version": "12.5.3",
"resolved": "https://registry.npmjs.org/got/-/got-12.6.0.tgz", "resolved": "https://registry.npmjs.org/got/-/got-12.5.3.tgz",
"integrity": "sha512-WTcaQ963xV97MN3x0/CbAriXFZcXCfgxVp91I+Ze6pawQOa7SgzwSx2zIJJsX+kTajMnVs0xcFD1TxZKFqhdnQ==", "integrity": "sha512-8wKnb9MGU8IPGRIo+/ukTy9XLJBwDiCpIf5TVzQ9Cpol50eMTpBq2GAuDsuDIz7hTYmZgMgC1e9ydr6kSDWs3w==",
"dependencies": { "dependencies": {
"@sindresorhus/is": "^5.2.0", "@sindresorhus/is": "^5.2.0",
"@szmarczak/http-timer": "^5.0.1", "@szmarczak/http-timer": "^5.0.1",
"cacheable-lookup": "^7.0.0", "cacheable-lookup": "^7.0.0",
"cacheable-request": "^10.2.8", "cacheable-request": "^10.2.1",
"decompress-response": "^6.0.0", "decompress-response": "^6.0.0",
"form-data-encoder": "^2.1.2", "form-data-encoder": "^2.1.2",
"get-stream": "^6.0.1", "get-stream": "^6.0.1",
@ -6311,6 +6322,15 @@
"@types/node": "*" "@types/node": "*"
} }
}, },
"@types/html-entities": {
"version": "1.3.4",
"resolved": "https://registry.npmjs.org/@types/html-entities/-/html-entities-1.3.4.tgz",
"integrity": "sha512-Ut62LV90H9tgXwyhmfR8U6yCw/6xeo26IlsbAJJfqPomaqDN2zoLb2Z+cbmy5AycJFhwNJDdH0zqjQp7Ox/eXg==",
"dev": true,
"requires": {
"html-entities": "*"
}
},
"@types/http-cache-semantics": { "@types/http-cache-semantics": {
"version": "4.0.1", "version": "4.0.1",
"resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.1.tgz", "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.1.tgz",
@ -6636,9 +6656,9 @@
"integrity": "sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w==" "integrity": "sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w=="
}, },
"cacheable-request": { "cacheable-request": {
"version": "10.2.8", "version": "10.2.7",
"resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-10.2.8.tgz", "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-10.2.7.tgz",
"integrity": "sha512-IDVO5MJ4LItE6HKFQTqT2ocAQsisOoCTUDu1ddCmnhyiwFQjXNPp4081Xj23N4tO+AFEFNzGuNEf/c8Gwwt15A==", "integrity": "sha512-I4SA6mKgDxcxVbSt/UmIkb9Ny8qSkg6ReBHtAAXnZHk7KOSx5g3DTiAOaYzcHCs6oOdHn+bip9T48E6tMvK9hw==",
"requires": { "requires": {
"@types/http-cache-semantics": "^4.0.1", "@types/http-cache-semantics": "^4.0.1",
"get-stream": "^6.0.1", "get-stream": "^6.0.1",
@ -7340,14 +7360,14 @@
"dev": true "dev": true
}, },
"got": { "got": {
"version": "12.6.0", "version": "12.5.3",
"resolved": "https://registry.npmjs.org/got/-/got-12.6.0.tgz", "resolved": "https://registry.npmjs.org/got/-/got-12.5.3.tgz",
"integrity": "sha512-WTcaQ963xV97MN3x0/CbAriXFZcXCfgxVp91I+Ze6pawQOa7SgzwSx2zIJJsX+kTajMnVs0xcFD1TxZKFqhdnQ==", "integrity": "sha512-8wKnb9MGU8IPGRIo+/ukTy9XLJBwDiCpIf5TVzQ9Cpol50eMTpBq2GAuDsuDIz7hTYmZgMgC1e9ydr6kSDWs3w==",
"requires": { "requires": {
"@sindresorhus/is": "^5.2.0", "@sindresorhus/is": "^5.2.0",
"@szmarczak/http-timer": "^5.0.1", "@szmarczak/http-timer": "^5.0.1",
"cacheable-lookup": "^7.0.0", "cacheable-lookup": "^7.0.0",
"cacheable-request": "^10.2.8", "cacheable-request": "^10.2.1",
"decompress-response": "^6.0.0", "decompress-response": "^6.0.0",
"form-data-encoder": "^2.1.2", "form-data-encoder": "^2.1.2",
"get-stream": "^6.0.1", "get-stream": "^6.0.1",

View File

@ -1,6 +1,6 @@
{ {
"name": "summaly", "name": "summaly",
"version": "4.0.2", "version": "3.0.4",
"description": "Get web page's summary", "description": "Get web page's summary",
"author": "syuilo <syuilotan@yahoo.co.jp>", "author": "syuilo <syuilotan@yahoo.co.jp>",
"license": "MIT", "license": "MIT",
@ -9,7 +9,6 @@
"main": "./built/index.js", "main": "./built/index.js",
"type": "module", "type": "module",
"types": "./built/index.d.ts", "types": "./built/index.d.ts",
"packageManager": "pnpm@8.3.1",
"files": [ "files": [
"built", "built",
"LICENSE" "LICENSE"
@ -20,23 +19,24 @@
"serve": "fastify start ./built/index.js" "serve": "fastify start ./built/index.js"
}, },
"devDependencies": { "devDependencies": {
"@jest/globals": "^29.5.0", "@jest/globals": "^29.4.2",
"@swc/core": "^1.3.52", "@swc/core": "^1.3.35",
"@swc/jest": "^0.2.26", "@swc/jest": "^0.2.24",
"@types/cheerio": "0.22.18", "@types/cheerio": "0.22.18",
"@types/debug": "4.1.7", "@types/debug": "4.1.7",
"@types/escape-regexp": "^0.0.1", "@types/escape-regexp": "^0.0.1",
"@types/html-entities": "1.3.4",
"@types/node": "16.11.12", "@types/node": "16.11.12",
"debug": "^4.3.4", "debug": "^4.3.4",
"fastify": "^4.15.0", "fastify": "^4.13.0",
"fastify-cli": "^5.7.1", "fastify-cli": "^5.7.1",
"jest": "^29.5.0", "jest": "^29.4.2",
"typescript": "4.5.3" "typescript": "4.5.3"
}, },
"dependencies": { "dependencies": {
"cheerio": "1.0.0-rc.12", "cheerio": "^1.0.0-rc.12",
"escape-regexp": "0.0.1", "escape-regexp": "0.0.1",
"got": "^12.6.0", "got": "^12.5.3",
"html-entities": "2.3.2", "html-entities": "2.3.2",
"iconv-lite": "0.6.3", "iconv-lite": "0.6.3",
"jschardet": "3.0.0", "jschardet": "3.0.0",

1419
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

View File

@ -1,4 +1,4 @@
import { URL } from 'node:url'; import * as URL from 'node:url';
import clip from './utils/clip.js'; import clip from './utils/clip.js';
import cleanupTitle from './utils/cleanup-title.js'; import cleanupTitle from './utils/cleanup-title.js';
@ -20,20 +20,9 @@ async function getOEmbedPlayer($: cheerio.CheerioAPI, pageUrl: string): Promise<
return null; return null;
} }
const oEmbedUrl = (() => { // XXX: Use global URL object instead of the deprecated `node:url`
try { // Disallow relative URL as no one seems to use it
return new URL(href, pageUrl); const oEmbed = await get(URL.resolve(pageUrl, href));
} catch { return null }
})();
if (!oEmbedUrl) {
return null;
}
const oEmbed = await get(oEmbedUrl.href).catch(() => null);
if (!oEmbed) {
return null;
}
const body = (() => { const body = (() => {
try { try {
return JSON.parse(oEmbed); return JSON.parse(oEmbed);
@ -69,14 +58,11 @@ async function getOEmbedPlayer($: cheerio.CheerioAPI, pageUrl: string): Promise<
return null; return null;
} }
try { // XXX: Use global URL object instead of the deprecated `node:url`
if ((new URL(url)).protocol !== 'https:') { if (URL.parse(url).protocol !== 'https:') {
// Allow only HTTPS for best security // Allow only HTTPS for best security
return null; return null;
} }
} catch (e) {
return null;
}
// Height is the most important, width is okay to be null. The implementer // Height is the most important, width is okay to be null. The implementer
// should choose fixed height instead of fixed aspect ratio if width is null. // should choose fixed height instead of fixed aspect ratio if width is null.
@ -115,9 +101,6 @@ async function getOEmbedPlayer($: cheerio.CheerioAPI, pageUrl: string): Promise<
(iframe.attr('allow') ?? '').split(/\s*;\s*/g) (iframe.attr('allow') ?? '').split(/\s*;\s*/g)
.filter(s => s) .filter(s => s)
.filter(s => !ignoredList.includes(s)); .filter(s => !ignoredList.includes(s));
if (iframe.attr('allowfullscreen') === '') {
allowedPermissions.push('fullscreen');
}
if (allowedPermissions.some(allow => !safeList.includes(allow))) { if (allowedPermissions.some(allow => !safeList.includes(allow))) {
// This iframe is probably too powerful to be embedded // This iframe is probably too powerful to be embedded
return null; return null;
@ -131,11 +114,9 @@ async function getOEmbedPlayer($: cheerio.CheerioAPI, pageUrl: string): Promise<
} }
} }
export default async (_url: URL | string, lang: string | null = null): Promise<Summary | null> => { export default async (url: URL.Url, lang: string | null = null): Promise<Summary | null> => {
if (lang && !lang.match(/^[\w-]+(\s*,\s*[\w-]+)*$/)) lang = null; if (lang && !lang.match(/^[\w-]+(\s*,\s*[\w-]+)*$/)) lang = null;
const url = typeof _url === 'string' ? new URL(_url) : _url;
const res = await scpaping(url.href, { lang: lang || undefined }); const res = await scpaping(url.href, { lang: lang || undefined });
const $ = res.$; const $ = res.$;
const twitterCard = $('meta[property="twitter:card"]').attr('content'); const twitterCard = $('meta[property="twitter:card"]').attr('content');
@ -158,7 +139,7 @@ export default async (_url: URL | string, lang: string | null = null): Promise<S
$('link[rel="apple-touch-icon"]').attr('href') || $('link[rel="apple-touch-icon"]').attr('href') ||
$('link[rel="apple-touch-icon image_src"]').attr('href'); $('link[rel="apple-touch-icon image_src"]').attr('href');
image = image ? (new URL(image, url.href)).href : null; image = image ? URL.resolve(url.href, image) : null;
const playerUrl = const playerUrl =
(twitterCard !== 'summary_large_image' && $('meta[property="twitter:player"]').attr('content')) || (twitterCard !== 'summary_large_image' && $('meta[property="twitter:player"]').attr('content')) ||
@ -192,11 +173,12 @@ export default async (_url: URL | string, lang: string | null = null): Promise<S
description = null; description = null;
} }
let siteName = decodeHtml( let siteName =
$('meta[property="og:site_name"]').attr('content') || $('meta[property="og:site_name"]').attr('content') ||
$('meta[name="application-name"]').attr('content') || $('meta[name="application-name"]').attr('content') ||
url.hostname url.hostname;
);
siteName = siteName ? decodeHtml(siteName) : null;
const favicon = const favicon =
$('link[rel="shortcut icon"]').attr('href') || $('link[rel="shortcut icon"]').attr('href') ||
@ -206,17 +188,34 @@ export default async (_url: URL | string, lang: string | null = null): Promise<S
const sensitive = $('.tweet').attr('data-possibly-sensitive') === 'true' const sensitive = $('.tweet').attr('data-possibly-sensitive') === 'true'
const find = async (path: string) => { const find = async (path: string) => {
const target = new URL(path, url.href); const target = URL.resolve(url.href, path);
try { try {
await head(target.href); await head(target);
return target; return target;
} catch (e) { } catch (e) {
return null; return null;
} }
}; };
// 相対的なURL (ex. test) を絶対的 (ex. /test) に変換
const toAbsolute = (relativeURLString: string): string => {
const relativeURL = URL.parse(relativeURLString);
const isAbsolute = relativeURL.slashes || relativeURL.path !== null && relativeURL.path[0] === '/';
// 既に絶対的なら、即座に値を返却
if (isAbsolute) {
return relativeURLString;
}
// スラッシュを付けて返却
return '/' + relativeURLString;
};
const getIcon = async () => { const getIcon = async () => {
return (await find(favicon)) || null; return await find(favicon) ||
// 相対指定を絶対指定に変換し再試行
await find(toAbsolute(favicon)) ||
null;
} }
const [icon, oEmbed] = await Promise.all([ const [icon, oEmbed] = await Promise.all([
@ -233,14 +232,14 @@ export default async (_url: URL | string, lang: string | null = null): Promise<S
return { return {
title: title || null, title: title || null,
icon: icon?.href || null, icon: icon || null,
description: description || null, description: description || null,
thumbnail: image || null, thumbnail: image || null,
player: oEmbed ?? { player: oEmbed ?? {
url: playerUrl || null, url: playerUrl || null,
width: Number.isNaN(playerWidth) ? null : playerWidth, width: Number.isNaN(playerWidth) ? null : playerWidth,
height: Number.isNaN(playerHeight) ? null : playerHeight, height: Number.isNaN(playerHeight) ? null : playerHeight,
allow: ['autoplay', 'encrypted-media', 'fullscreen'], allow: ['fullscreen', 'encrypted-media'],
}, },
sitename: siteName || null, sitename: siteName || null,
sensitive, sensitive,

View File

@ -3,7 +3,7 @@
* https://github.com/syuilo/summaly * https://github.com/syuilo/summaly
*/ */
import { URL } from 'node:url'; import * as URL from 'node:url';
import tracer from 'trace-redirect'; import tracer from 'trace-redirect';
import Summary from './summary.js'; import Summary from './summary.js';
import type { IPlugin as _IPlugin } from './iplugin.js'; import type { IPlugin as _IPlugin } from './iplugin.js';
@ -69,7 +69,7 @@ export const summaly = async (url: string, options?: Options): Promise<Result> =
} }
} }
const _url = new URL(actualUrl); const _url = URL.parse(actualUrl, true);
// Find matching plugin // Find matching plugin
const match = plugins.filter(plugin => plugin.test(_url))[0]; const match = plugins.filter(plugin => plugin.test(_url))[0];

View File

@ -1,7 +1,7 @@
import type { URL } from 'node:url'; import * as URL from 'node:url';
import Summary from './summary.js'; import Summary from './summary.js';
export interface IPlugin { export interface IPlugin {
test: (url: URL) => boolean; test: (url: URL.Url) => boolean;
summarize: (url: URL, lang?: string) => Promise<Summary>; summarize: (url: URL.Url, lang?: string) => Promise<Summary>;
} }

View File

@ -1,8 +1,8 @@
import { URL } from 'node:url'; import * as URL from 'node:url';
import { scpaping } from '../utils/got.js'; import { scpaping } from '../utils/got.js';
import summary from '../summary.js'; import summary from '../summary.js';
export function test(url: URL): boolean { export function test(url: URL.Url): boolean {
return url.hostname === 'www.amazon.com' || return url.hostname === 'www.amazon.com' ||
url.hostname === 'www.amazon.co.jp' || url.hostname === 'www.amazon.co.jp' ||
url.hostname === 'www.amazon.ca' || url.hostname === 'www.amazon.ca' ||
@ -19,7 +19,7 @@ export function test(url: URL): boolean {
url.hostname === 'www.amazon.au'; url.hostname === 'www.amazon.au';
} }
export async function summarize(url: URL): Promise<summary> { export async function summarize(url: URL.Url): Promise<summary> {
const res = await scpaping(url.href); const res = await scpaping(url.href);
const $ = res.$; const $ = res.$;

View File

@ -1,4 +1,4 @@
import { URL } from 'node:url'; import * as URL from 'node:url';
import { get } from '../utils/got.js'; import { get } from '../utils/got.js';
import debug from 'debug'; import debug from 'debug';
import summary from '../summary.js'; import summary from '../summary.js';
@ -6,12 +6,12 @@ import clip from './../utils/clip.js';
const log = debug('summaly:plugins:wikipedia'); const log = debug('summaly:plugins:wikipedia');
export function test(url: URL): boolean { export function test(url: URL.Url): boolean {
if (!url.hostname) return false; if (!url.hostname) return false;
return /\.wikipedia\.org$/.test(url.hostname); return /\.wikipedia\.org$/.test(url.hostname);
} }
export async function summarize(url: URL): Promise<summary> { export async function summarize(url: URL.Url): Promise<summary> {
const lang = url.host ? url.host.split('.')[0] : null; const lang = url.host ? url.host.split('.')[0] : null;
const title = url.pathname ? url.pathname.split('/')[2] : null; const title = url.pathname ? url.pathname.split('/')[2] : null;
const endpoint = `https://${lang}.wikipedia.org/w/api.php?format=json&action=query&prop=extracts&exintro=&explaintext=&titles=${title}`; const endpoint = `https://${lang}.wikipedia.org/w/api.php?format=json&action=query&prop=extracts&exintro=&explaintext=&titles=${title}`;

View File

@ -1,3 +1,2 @@
<!DOCTYPE html> <!DOCTYPE html>
<link type="application/json+oembed" href="http://localhost:3060/oembe.json" /> <link type="application/json+oembed" href="http://localhost:3060/oembe.json" />
<meta property="og:description" content="nonexistent">

View File

@ -1,3 +0,0 @@
<!DOCTYPE html>
<link type="application/json+oembed" href="http://localhost:+3060/oembed.json" />
<meta property="og:description" content="wrong url">

View File

@ -0,0 +1,2 @@
<!DOCTYPE html>
<link type="application/json+oembed" href="http://localhost+:3060/oembed.json" />

View File

@ -213,7 +213,7 @@ describe('TwitterCard', () => {
const summary = await summaly(host); const summary = await summaly(host);
expect(summary.player.url).toBe('https://example.com/embedurl'); expect(summary.player.url).toBe('https://example.com/embedurl');
expect(summary.player.allow).toStrictEqual(['autoplay', 'encrypted-media', 'fullscreen']); expect(summary.player.allow).toStrictEqual(['fullscreen', 'encrypted-media']);
}); });
test('Player detection - Pleroma:video => video', async () => { test('Player detection - Pleroma:video => video', async () => {
@ -225,7 +225,7 @@ describe('TwitterCard', () => {
const summary = await summaly(host); const summary = await summaly(host);
expect(summary.player.url).toBe('https://example.com/embedurl'); expect(summary.player.url).toBe('https://example.com/embedurl');
expect(summary.player.allow).toStrictEqual(['autoplay', 'encrypted-media', 'fullscreen']); expect(summary.player.allow).toStrictEqual(['fullscreen', 'encrypted-media']);
}); });
test('Player detection - Pleroma:image => image', async () => { test('Player detection - Pleroma:image => image', async () => {
@ -294,14 +294,7 @@ describe("oEmbed", () => {
await setUpFastify('oembed-allow-fullscreen.json'); await setUpFastify('oembed-allow-fullscreen.json');
const summary = await summaly(host); const summary = await summaly(host);
expect(summary.player.url).toBe('https://example.com/'); expect(summary.player.url).toBe('https://example.com/');
expect(summary.player.allow).toStrictEqual(['fullscreen']); expect(summary.player.allow).toStrictEqual(['fullscreen'])
});
test('allows legacy allowfullscreen', async () => {
await setUpFastify('oembed-allow-fullscreen-legacy.json');
const summary = await summaly(host);
expect(summary.player.url).toBe('https://example.com/');
expect(summary.player.allow).toStrictEqual(['fullscreen']);
}); });
test('allows safelisted permissions', async () => { test('allows safelisted permissions', async () => {
@ -329,16 +322,12 @@ describe("oEmbed", () => {
test('oEmbed with nonexistent path', async () => { test('oEmbed with nonexistent path', async () => {
await setUpFastify('oembed.json', 'htmls/oembed-nonexistent-path.html'); await setUpFastify('oembed.json', 'htmls/oembed-nonexistent-path.html');
const summary = await summaly(host); await expect(summaly(host)).rejects.toThrow('404 Not Found');
expect(summary.player.url).toBe(null);
expect(summary.description).toBe('nonexistent');
}); });
test('oEmbed with wrong path', async () => { test('oEmbed with wrong path', async () => {
await setUpFastify('oembed.json', 'htmls/oembed-wrong-path.html'); await setUpFastify('oembed.json', 'htmls/oembed-wrong-path.html');
const summary = await summaly(host); await expect(summaly(host)).rejects.toThrow();
expect(summary.player.url).toBe(null);
expect(summary.description).toBe('wrong url');
}); });
test('oEmbed with OpenGraph', async () => { test('oEmbed with OpenGraph', async () => {

View File

@ -1,7 +0,0 @@
{
"version": "1.0",
"type": "rich",
"html": "<iframe src='https://example.com/' allowfullscreen></iframe>",
"width": 500,
"height": 300
}