built

CHANGELOG.md

@@ -1,7 +1,6 @@
-4.0.0 / 2023-03-14
+Unreleased
 ------------------
 * oEmbed type=richの制限的なサポート
-* プラグインの引数がWHATWG URLになりました
 
 3.0.4 / 2023-02-12
 ------------------

README.md

@@ -51,13 +51,11 @@ npm run serve
 
 ``` typescript
 interface IPlugin {
-	test: (url: URL) => boolean;
+	test: (url: URL.Url) => boolean;
-	summarize: (url: URL) => Promise<Summary>;
+	summarize: (url: URL.Url) => Promise<Summary>;
 }
 ```
 
-urls are WHATWG URL since v4.
-
 ### Returns
 
 A Promise of an Object that contains properties below:
@@ -129,5 +127,12 @@ License
 ----------------------------------------------------------------
 [MIT](LICENSE)
 
+[npm-link]:       https://www.npmjs.com/package/summaly
+[npm-badge]:      https://img.shields.io/npm/v/summaly.svg?style=flat-square
+[mit]:            http://opensource.org/licenses/MIT
+[mit-badge]:      https://img.shields.io/badge/license-MIT-444444.svg?style=flat-square
+[travis-link]:    https://travis-ci.org/syuilo/summaly
+[travis-badge]:   http://img.shields.io/travis/syuilo/summaly.svg?style=flat-square
+[himasaku]:       https://himasaku.net
 [himawari-badge]: https://img.shields.io/badge/%E5%8F%A4%E8%B0%B7-%E5%90%91%E6%97%A5%E8%91%B5-1684c5.svg?style=flat-square
 [sakurako-badge]: https://img.shields.io/badge/%E5%A4%A7%E5%AE%A4-%E6%AB%BB%E5%AD%90-efb02a.svg?style=flat-square

built/general.d.ts

@@ -1,4 +1,4 @@
-import { URL } from 'node:url';
+import * as URL from 'node:url';
 import type { default as Summary } from './summary.js';
-declare const _default: (_url: URL | string, lang?: string | null) => Promise<Summary | null>;
+declare const _default: (url: URL.Url, lang?: string | null) => Promise<Summary | null>;
 export default _default;

built/general.js

@@ -1,4 +1,4 @@
-import { URL } from 'node:url';
+import * as URL from 'node:url';
 import clip from './utils/clip.js';
 import cleanupTitle from './utils/cleanup-title.js';
 import { decode as decodeHtml } from 'html-entities';
@@ -15,7 +15,9 @@ async function getOEmbedPlayer($, pageUrl) {
     if (!href) {
         return null;
     }
-    const oEmbed = await get((new URL(href, pageUrl)).href);
+    // XXX: Use global URL object instead of the deprecated `node:url`
+    // Disallow relative URL as no one seems to use it
+    const oEmbed = await get(URL.resolve(pageUrl, href));
     const body = (() => {
         try {
             return JSON.parse(oEmbed);
@@ -45,15 +47,11 @@ async function getOEmbedPlayer($, pageUrl) {
         // No src?
         return null;
     }
-    try {
+    // XXX: Use global URL object instead of the deprecated `node:url`
-        if ((new URL(url)).protocol !== 'https:') {
+    if (URL.parse(url).protocol !== 'https:') {
         // Allow only HTTPS for best security
         return null;
     }
-    }
-    catch (e) {
-        return null;
-    }
     // Height is the most important, width is okay to be null. The implementer
     // should choose fixed height instead of fixed aspect ratio if width is null.
     //
@@ -100,10 +98,9 @@ async function getOEmbedPlayer($, pageUrl) {
         allow: allowedPermissions
     };
 }
-export default async (_url, lang = null) => {
+export default async (url, lang = null) => {
     if (lang && !lang.match(/^[\w-]+(\s*,\s*[\w-]+)*$/))
         lang = null;
-    const url = typeof _url === 'string' ? new URL(_url) : _url;
     const res = await scpaping(url.href, { lang: lang || undefined });
     const $ = res.$;
     const twitterCard = $('meta[property="twitter:card"]').attr('content');
@@ -119,7 +116,7 @@ export default async (_url, lang = null) => {
         $('link[rel="image_src"]').attr('href') ||
         $('link[rel="apple-touch-icon"]').attr('href') ||
         $('link[rel="apple-touch-icon image_src"]').attr('href');
-    image = image ? (new URL(image, url.href)).href : null;
+    image = image ? URL.resolve(url.href, image) : null;
     const playerUrl = (twitterCard !== 'summary_large_image' && $('meta[property="twitter:player"]').attr('content')) ||
         (twitterCard !== 'summary_large_image' && $('meta[name="twitter:player"]').attr('content')) ||
         $('meta[property="og:video"]').attr('content') ||
@@ -142,25 +139,40 @@ export default async (_url, lang = null) => {
     if (title === description) {
         description = null;
     }
-    let siteName = decodeHtml($('meta[property="og:site_name"]').attr('content') ||
+    let siteName = $('meta[property="og:site_name"]').attr('content') ||
         $('meta[name="application-name"]').attr('content') ||
-        url.hostname);
+        url.hostname;
+    siteName = siteName ? decodeHtml(siteName) : null;
     const favicon = $('link[rel="shortcut icon"]').attr('href') ||
         $('link[rel="icon"]').attr('href') ||
         '/favicon.ico';
     const sensitive = $('.tweet').attr('data-possibly-sensitive') === 'true';
     const find = async (path) => {
-        const target = new URL(path, url.href);
+        const target = URL.resolve(url.href, path);
         try {
-            await head(target.href);
+            await head(target);
             return target;
         }
         catch (e) {
             return null;
         }
     };
+    // 相対的なURL (ex. test) を絶対的 (ex. /test) に変換
+    const toAbsolute = (relativeURLString) => {
+        const relativeURL = URL.parse(relativeURLString);
+        const isAbsolute = relativeURL.slashes || relativeURL.path !== null && relativeURL.path[0] === '/';
+        // 既に絶対的なら、即座に値を返却
+        if (isAbsolute) {
+            return relativeURLString;
+        }
+        // スラッシュを付けて返却
+        return '/' + relativeURLString;
+    };
     const getIcon = async () => {
-        return (await find(favicon)) || null;
+        return await find(favicon) ||
+            // 相対指定を絶対指定に変換し再試行
+            await find(toAbsolute(favicon)) ||
+            null;
     };
     const [icon, oEmbed] = await Promise.all([
         getIcon(),
@@ -173,14 +185,14 @@ export default async (_url, lang = null) => {
     }
     return {
         title: title || null,
-        icon: icon?.href || null,
+        icon: icon || null,
         description: description || null,
         thumbnail: image || null,
         player: oEmbed ?? {
             url: playerUrl || null,
             width: Number.isNaN(playerWidth) ? null : playerWidth,
             height: Number.isNaN(playerHeight) ? null : playerHeight,
-            allow: ['autoplay', 'encrypted-media', 'fullscreen'],
+            allow: ['fullscreen', 'encrypted-media'],
         },
         sitename: siteName || null,
         sensitive,

built/index.js

@@ -2,7 +2,7 @@
  * summaly
  * https://github.com/syuilo/summaly
  */
-import { URL } from 'node:url';
+import * as URL from 'node:url';
 import tracer from 'trace-redirect';
 import general from './general.js';
 import { setAgent } from './utils/got.js';
@@ -30,7 +30,7 @@ export const summaly = async (url, options) => {
             actualUrl = url;
         }
     }
-    const _url = new URL(actualUrl);
+    const _url = URL.parse(actualUrl, true);
     // Find matching plugin
     const match = plugins.filter(plugin => plugin.test(_url))[0];
     // Get summary

built/iplugin.d.ts

@@ -1,7 +1,7 @@
 /// <reference types="node" />
-import type { URL } from 'node:url';
+import * as URL from 'node:url';
 import Summary from './summary.js';
 export interface IPlugin {
-    test: (url: URL) => boolean;
+    test: (url: URL.Url) => boolean;
-    summarize: (url: URL, lang?: string) => Promise<Summary>;
+    summarize: (url: URL.Url, lang?: string) => Promise<Summary>;
 }

built/plugins/amazon.d.ts

@@ -1,5 +1,5 @@
 /// <reference types="node" />
-import { URL } from 'node:url';
+import * as URL from 'node:url';
 import summary from '../summary.js';
-export declare function test(url: URL): boolean;
+export declare function test(url: URL.Url): boolean;
-export declare function summarize(url: URL): Promise<summary>;
+export declare function summarize(url: URL.Url): Promise<summary>;

built/plugins/wikipedia.d.ts

@@ -1,5 +1,5 @@
 /// <reference types="node" />
-import { URL } from 'node:url';
+import * as URL from 'node:url';
 import summary from '../summary.js';
-export declare function test(url: URL): boolean;
+export declare function test(url: URL.Url): boolean;
-export declare function summarize(url: URL): Promise<summary>;
+export declare function summarize(url: URL.Url): Promise<summary>;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "summaly",
-  "version": "4.0.0",
+  "version": "3.0.4",
   "description": "Get web page's summary",
   "author": "syuilo <syuilotan@yahoo.co.jp>",
   "license": "MIT",
@@ -25,6 +25,7 @@
     "@types/cheerio": "0.22.18",
     "@types/debug": "4.1.7",
     "@types/escape-regexp": "^0.0.1",
+    "@types/html-entities": "1.3.4",
     "@types/node": "16.11.12",
     "debug": "^4.3.4",
     "fastify": "^4.13.0",
@@ -33,9 +34,9 @@
     "typescript": "4.5.3"
   },
   "dependencies": {
-    "cheerio": "1.0.0-rc.12",
+    "cheerio": "^1.0.0-rc.12",
     "escape-regexp": "0.0.1",
-    "got": "^12.6.0",
+    "got": "^12.5.3",
     "html-entities": "2.3.2",
     "iconv-lite": "0.6.3",
     "jschardet": "3.0.0",

pnpm-lock.yaml

@@ -7,13 +7,14 @@ specifiers:
   '@types/cheerio': 0.22.18
   '@types/debug': 4.1.7
   '@types/escape-regexp': ^0.0.1
+  '@types/html-entities': 1.3.4
   '@types/node': 16.11.12
-  cheerio: 1.0.0-rc.12
+  cheerio: ^1.0.0-rc.12
   debug: ^4.3.4
   escape-regexp: 0.0.1
   fastify: ^4.13.0
   fastify-cli: ^5.7.1
-  got: ^12.6.0
+  got: ^12.5.3
   html-entities: 2.3.2
   iconv-lite: 0.6.3
   jest: ^29.4.2
@@ -39,6 +40,7 @@ devDependencies:
   '@types/cheerio': 0.22.18
   '@types/debug': 4.1.7
   '@types/escape-regexp': 0.0.1
+  '@types/html-entities': 1.3.4
   '@types/node': 16.11.12
   debug: 4.3.4
   fastify: 4.14.1
@@ -893,6 +895,13 @@ packages:
       '@types/node': 16.11.12
     dev: true
 
+  /@types/html-entities/1.3.4:
+    resolution: {integrity: sha512-Ut62LV90H9tgXwyhmfR8U6yCw/6xeo26IlsbAJJfqPomaqDN2zoLb2Z+cbmy5AycJFhwNJDdH0zqjQp7Ox/eXg==}
+    deprecated: This is a stub types definition. html-entities provides its own type definitions, so you do not need this installed.
+    dependencies:
+      html-entities: 2.3.2
+    dev: true
+
   /@types/http-cache-semantics/4.0.1:
     resolution: {integrity: sha512-SZs7ekbP8CN0txVG2xVRH6EgKmEm31BOxA07vkFaETzZz1xh+cbt8BcI0slpymvwhx5dlFnQG2rTlPVQn+iRPQ==}
     dev: false
@@ -1829,7 +1838,6 @@ packages:
 
   /html-entities/2.3.2:
     resolution: {integrity: sha512-c3Ab/url5ksaT0WyleslpBEthOzWhrjQbg75y7XUsfSzi3Dgzt0l8w5e7DylRn15MTlMMD58dTfzddNS2kcAjQ==}
-    dev: false
 
   /html-escaper/2.0.2:
     resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==}

src/general.ts

@@ -1,4 +1,4 @@
-import { URL } from 'node:url';
+import * as URL from 'node:url';
 import clip from './utils/clip.js';
 import cleanupTitle from './utils/cleanup-title.js';
 
@@ -20,7 +20,9 @@ async function getOEmbedPlayer($: cheerio.CheerioAPI, pageUrl: string): Promise<
 		return null;
 	}
 
-	const oEmbed = await get((new URL(href, pageUrl)).href);
+	// XXX: Use global URL object instead of the deprecated `node:url`
+	// Disallow relative URL as no one seems to use it
+	const oEmbed = await get(URL.resolve(pageUrl, href));
 	const body = (() => {
 		try {
 			return JSON.parse(oEmbed);
@@ -56,14 +58,11 @@ async function getOEmbedPlayer($: cheerio.CheerioAPI, pageUrl: string): Promise<
 		return null;
 	}
 
-	try {
+	// XXX: Use global URL object instead of the deprecated `node:url`
-		if ((new URL(url)).protocol !== 'https:') {
+	if (URL.parse(url).protocol !== 'https:') {
 		// Allow only HTTPS for best security
 		return null;
 	}
-	} catch (e) {
-		return null;
-	}
 
 	// Height is the most important, width is okay to be null. The implementer
 	// should choose fixed height instead of fixed aspect ratio if width is null.
@@ -115,11 +114,9 @@ async function getOEmbedPlayer($: cheerio.CheerioAPI, pageUrl: string): Promise<
 	}
 }
 
-export default async (_url: URL | string, lang: string | null = null): Promise<Summary | null> => {
+export default async (url: URL.Url, lang: string | null = null): Promise<Summary | null> => {
 	if (lang && !lang.match(/^[\w-]+(\s*,\s*[\w-]+)*$/)) lang = null;
 
-	const url = typeof _url === 'string' ? new URL(_url) : _url;
-
 	const res = await scpaping(url.href, { lang: lang || undefined });
 	const $ = res.$;
 	const twitterCard = $('meta[property="twitter:card"]').attr('content');
@@ -142,7 +139,7 @@ export default async (_url: URL | string, lang: string | null = null): Promise<S
 		$('link[rel="apple-touch-icon"]').attr('href') ||
 		$('link[rel="apple-touch-icon image_src"]').attr('href');
 
-	image = image ? (new URL(image, url.href)).href : null;
+	image = image ? URL.resolve(url.href, image) : null;
 
 	const playerUrl =
 		(twitterCard !== 'summary_large_image' && $('meta[property="twitter:player"]').attr('content')) ||
@@ -176,11 +173,12 @@ export default async (_url: URL | string, lang: string | null = null): Promise<S
 		description = null;
 	}
 
-	let siteName = decodeHtml(
+	let siteName =
 		$('meta[property="og:site_name"]').attr('content') ||
 		$('meta[name="application-name"]').attr('content') ||
-		url.hostname
+		url.hostname;
-	);
+
+	siteName = siteName ? decodeHtml(siteName) : null;
 
 	const favicon =
 		$('link[rel="shortcut icon"]').attr('href') ||
@@ -190,17 +188,34 @@ export default async (_url: URL | string, lang: string | null = null): Promise<S
 	const sensitive = $('.tweet').attr('data-possibly-sensitive') === 'true'
 
 	const find = async (path: string) => {
-		const target = new URL(path, url.href);
+		const target = URL.resolve(url.href, path);
 		try {
-			await head(target.href);
+			await head(target);
 			return target;
 		} catch (e) {
 			return null;
 		}
 	};
 
+	// 相対的なURL (ex. test) を絶対的 (ex. /test) に変換
+	const toAbsolute = (relativeURLString: string): string => {
+		const relativeURL = URL.parse(relativeURLString);
+		const isAbsolute = relativeURL.slashes || relativeURL.path !== null && relativeURL.path[0] === '/';
+
+		// 既に絶対的なら、即座に値を返却
+		if (isAbsolute) {
+			return relativeURLString;
+		}
+
+		// スラッシュを付けて返却
+		return '/' + relativeURLString;
+	};
+
 	const getIcon = async () => {
-		return (await find(favicon)) || null;
+		return await find(favicon) ||
+			// 相対指定を絶対指定に変換し再試行
+			await find(toAbsolute(favicon)) ||
+			null;
 	}
 
 	const [icon, oEmbed] = await Promise.all([
@@ -217,14 +232,14 @@ export default async (_url: URL | string, lang: string | null = null): Promise<S
 
 	return {
 		title: title || null,
-		icon: icon?.href || null,
+		icon: icon || null,
 		description: description || null,
 		thumbnail: image || null,
 		player: oEmbed ?? {
 			url: playerUrl || null,
 			width: Number.isNaN(playerWidth) ? null : playerWidth,
 			height: Number.isNaN(playerHeight) ? null : playerHeight,
-			allow: ['autoplay', 'encrypted-media', 'fullscreen'],
+			allow: ['fullscreen', 'encrypted-media'],
 		},
 		sitename: siteName || null,
 		sensitive,

src/index.ts

@@ -3,7 +3,7 @@
  * https://github.com/syuilo/summaly
  */
 
-import { URL } from 'node:url';
+import * as URL from 'node:url';
 import tracer from 'trace-redirect';
 import Summary from './summary.js';
 import type { IPlugin as _IPlugin } from './iplugin.js';
@@ -69,7 +69,7 @@ export const summaly = async (url: string, options?: Options): Promise<Result> =
 		}
 	}
  
-	const _url = new URL(actualUrl);
+	const _url = URL.parse(actualUrl, true);
 
 	// Find matching plugin
 	const match = plugins.filter(plugin => plugin.test(_url))[0];

src/iplugin.ts

@@ -1,7 +1,7 @@
-import type { URL } from 'node:url';
+import * as URL from 'node:url';
 import Summary from './summary.js';
 
 export interface IPlugin {
-	test: (url: URL) => boolean;
+	test: (url: URL.Url) => boolean;
-	summarize: (url: URL, lang?: string) => Promise<Summary>;
+	summarize: (url: URL.Url, lang?: string) => Promise<Summary>;
 }

src/plugins/amazon.ts

@@ -1,8 +1,8 @@
-import { URL } from 'node:url';
+import * as URL from 'node:url';
 import { scpaping } from '../utils/got.js';
 import summary from '../summary.js';
 
-export function test(url: URL): boolean {
+export function test(url: URL.Url): boolean {
 	return url.hostname === 'www.amazon.com' ||
 	url.hostname === 'www.amazon.co.jp' ||
 	url.hostname === 'www.amazon.ca' ||
@@ -19,7 +19,7 @@ export function test(url: URL): boolean {
 	url.hostname === 'www.amazon.au';
 }
 
-export async function summarize(url: URL): Promise<summary> {
+export async function summarize(url: URL.Url): Promise<summary> {
 	const res = await scpaping(url.href);
 	const $ = res.$;
 

src/plugins/wikipedia.ts

@@ -1,4 +1,4 @@
-import { URL } from 'node:url';
+import * as URL from 'node:url';
 import { get } from '../utils/got.js';
 import debug from 'debug';
 import summary from '../summary.js';
@@ -6,12 +6,12 @@ import clip from './../utils/clip.js';
 
 const log = debug('summaly:plugins:wikipedia');
 
-export function test(url: URL): boolean {
+export function test(url: URL.Url): boolean {
 	if (!url.hostname) return false;
 	return /\.wikipedia\.org$/.test(url.hostname);
 }
 
-export async function summarize(url: URL): Promise<summary> {
+export async function summarize(url: URL.Url): Promise<summary> {
 	const lang = url.host ? url.host.split('.')[0] : null;
 	const title = url.pathname ? url.pathname.split('/')[2] : null;
 	const endpoint = `https://${lang}.wikipedia.org/w/api.php?format=json&action=query&prop=extracts&exintro=&explaintext=&titles=${title}`;

test/index.ts

@@ -213,7 +213,7 @@ describe('TwitterCard', () => {
 
 		const summary = await summaly(host);
 		expect(summary.player.url).toBe('https://example.com/embedurl');
-		expect(summary.player.allow).toStrictEqual(['autoplay', 'encrypted-media', 'fullscreen']);
+		expect(summary.player.allow).toStrictEqual(['fullscreen', 'encrypted-media']);
 	});
 
 	test('Player detection - Pleroma:video => video', async () => {
@@ -225,7 +225,7 @@ describe('TwitterCard', () => {
 
 		const summary = await summaly(host);
 		expect(summary.player.url).toBe('https://example.com/embedurl');
-		expect(summary.player.allow).toStrictEqual(['autoplay', 'encrypted-media', 'fullscreen']);
+		expect(summary.player.allow).toStrictEqual(['fullscreen', 'encrypted-media']);
 	});
 
 	test('Player detection - Pleroma:image => image', async () => {