From 96ed0ea8f96851653c1a97b081eaf29fcc08ed1a Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Fri, 27 Apr 2018 17:02:58 +0000 Subject: [PATCH 1/8] fix(package): update cheerio-httpcli to version 0.7.3 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 74c64d7..9d2358f 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "mocha": "5.1.0" }, "dependencies": { - "cheerio-httpcli": "0.7.2", + "cheerio-httpcli": "0.7.3", "debug": "3.1.0", "escape-regexp": "0.0.1", "html-entities": "1.2.1", From ec3243da322f2651c27cc0a2803e9126f3831019 Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 18 May 2018 12:27:00 +0900 Subject: [PATCH 2/8] Handle error --- CHANGELOG.md | 4 ++++ package.json | 2 +- src/index.ts | 6 +----- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 712796c..147564e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +2.0.5 / 2018-05-18 +------------------ +* Fix bug + 2.0.4 / 2018-04-18 ------------------ * Dependencies update diff --git a/package.json b/package.json index 74c64d7..b976179 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "summaly", - "version": "2.0.4", + "version": "2.0.5", "description": "Get web page's summary", "author": "syuilo ", "license": "MIT", diff --git a/src/index.ts b/src/index.ts index f73244b..24d3b2a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -44,17 +44,13 @@ const defaultOptions = { /** * Summarize an web page - * @param {string} url URL of web page you want to summarize - * @param {Options?} options The options - * @return {Promise} Promised summary */ export default async (url: string, options?: Options): Promise => { const opts = Object.assign(defaultOptions, options); const plugins = builtinPlugins.concat(opts.plugins || []); - // Follow redirects - const actualUrl = opts.followRedirects ? await tracer(url) : url; + const actualUrl = opts.followRedirects ? await tracer(url).catch(() => url) : url; const _url = URL.parse(actualUrl, true); From b994fd39d9b330e4fce706737eeae5e2bb34ef16 Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 18 May 2018 16:51:55 +0900 Subject: [PATCH 3/8] Fix bug --- CHANGELOG.md | 4 ++++ package.json | 4 +++- src/plugins/amazon.ts | 4 ++-- src/plugins/wikipedia.ts | 51 +++++++++++++++++++++------------------- 4 files changed, 36 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 147564e..773d7c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +2.0.6 / 2018-05-18 +------------------ +* Fix bug + 2.0.5 / 2018-05-18 ------------------ * Fix bug diff --git a/package.json b/package.json index b976179..ee6a084 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "summaly", - "version": "2.0.5", + "version": "2.0.6", "description": "Get web page's summary", "author": "syuilo ", "license": "MIT", @@ -19,6 +19,7 @@ "@types/mocha": "5.0.0", "@types/request": "2.47.0", "@types/node": "9.6.5", + "@types/request-promise-native": "1.0.14", "express": "4.16.3", "typescript": "2.8.1", "mocha": "5.1.0" @@ -29,6 +30,7 @@ "escape-regexp": "0.0.1", "html-entities": "1.2.1", "request": "2.85.0", + "request-promise-native": "1.0.5", "require-all": "2.2.0", "trace-redirect": "1.0.6" } diff --git a/src/plugins/amazon.ts b/src/plugins/amazon.ts index 32d232d..249d462 100644 --- a/src/plugins/amazon.ts +++ b/src/plugins/amazon.ts @@ -20,7 +20,7 @@ export function test(url: URL.Url): boolean { url.hostname === 'www.amazon.cn' || url.hostname === 'www.amazon.in' || url.hostname === 'www.amazon.au'; -}; +} export async function summarize(url: URL.Url): Promise { const res = await client.fetch(url.href); @@ -41,4 +41,4 @@ export async function summarize(url: URL.Url): Promise { thumbnail: thumbnail || null, sitename: 'Amazon' }; -}; +} diff --git a/src/plugins/wikipedia.ts b/src/plugins/wikipedia.ts index 9d6ce13..b969f4c 100644 --- a/src/plugins/wikipedia.ts +++ b/src/plugins/wikipedia.ts @@ -1,5 +1,5 @@ import * as URL from 'url'; -import * as request from 'request'; +import * as request from 'request-promise-native'; import * as debug from 'debug'; import summary from '../summary'; import clip from './../utils/clip'; @@ -8,29 +8,32 @@ const log = debug('summaly:plugins:wikipedia'); export function test(url: URL.Url): boolean { return /\.wikipedia\.org$/.test(url.hostname); -}; +} -export function summarize(url: URL.Url): Promise { - return new Promise((res, rej) => { - const lang = url.host.split('.')[0]; - const title = url.pathname.split('/')[2]; - const endpoint = `https://${lang}.wikipedia.org/w/api.php?format=json&action=query&prop=extracts&exintro=&explaintext=&titles=${encodeURIComponent(title)}`; +export async function summarize(url: URL.Url): Promise { + const lang = url.host.split('.')[0]; + const title = url.pathname.split('/')[2]; + const endpoint = `https://${lang}.wikipedia.org/w/api.php?format=json&action=query&prop=extracts&exintro=&explaintext=&titles=${encodeURIComponent(title)}`; - log(`lang is ${lang}`); - log(`title is ${title}`); - log(`endpoint is ${endpoint}`); + log(`lang is ${lang}`); + log(`title is ${title}`); + log(`endpoint is ${endpoint}`); - request(endpoint, (err, _, body) => { - log(body); - body = JSON.parse(body); - const info = body.query.pages[Object.keys(body.query.pages)[0]]; - res({ - title: info.title, - icon: 'https://wikipedia.org/static/favicon/wikipedia.ico', - description: clip(info.extract, 300), - thumbnail: `https://wikipedia.org/static/images/project-logos/${lang}wiki.png`, - sitename: 'Wikipedia' - }); - }); - }); -}; + let body = await request(endpoint); + body = JSON.parse(body); + log(body); + + if (!('query' in body) || !('pages' in body.query)) { + throw 'fetch failed'; + } + + const info = body.query.pages[Object.keys(body.query.pages)[0]]; + + return { + title: info.title, + icon: 'https://wikipedia.org/static/favicon/wikipedia.ico', + description: clip(info.extract, 300), + thumbnail: `https://wikipedia.org/static/images/project-logos/${lang}wiki.png`, + sitename: 'Wikipedia' + }; +} From 52b26abfbd24bda4a70f1d97b6b4cd9af168fbbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Acid=20Chicken=20=28=E7=A1=AB=E9=85=B8=E9=B6=8F=29?= Date: Thu, 9 Aug 2018 16:29:28 +0900 Subject: [PATCH 4/8] Update general.ts --- src/general.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/general.ts b/src/general.ts index 2148cf7..f3dec8a 100644 --- a/src/general.ts +++ b/src/general.ts @@ -49,6 +49,9 @@ export default async (url: URL.Url): Promise => { image = image ? URL.resolve(url.href, image) : null; + const player = + $('meta[property="twitter:player"]').attr('content'); + let description = $('meta[property="og:description"]').attr('content') || $('meta[property="twitter:description"]').attr('content') || @@ -118,6 +121,7 @@ export default async (url: URL.Url): Promise => { icon: icon || null, description: description || null, thumbnail: image || null, + player: player || null, sitename: siteName || null }; }; From 8b534bd170c64ae307ca2bd4571807bedaf889e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Acid=20Chicken=20=28=E7=A1=AB=E9=85=B8=E9=B6=8F=29?= Date: Thu, 9 Aug 2018 16:32:22 +0900 Subject: [PATCH 5/8] Update summary.ts --- src/summary.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/summary.ts b/src/summary.ts index b40b616..2a2cb87 100644 --- a/src/summary.ts +++ b/src/summary.ts @@ -19,6 +19,11 @@ type Summary = { */ thumbnail: string; + /** + * The url of the player of that web page + */ + player: string; + /** * The title of that web page */ From e4c48c626e02df261c6cb38614ab67b1e62ef30e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Acid=20Chicken=20=28=E7=A1=AB=E9=85=B8=E9=B6=8F=29?= Date: Thu, 9 Aug 2018 16:33:46 +0900 Subject: [PATCH 6/8] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 653a022..6428d4b 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ A Promise of an Object that contains properties below: | **icon** | *string* | The url of the icon of the web page | | **sitename** | *string* | The name of the web site | | **thumbnail** | *string* | The url of the thumbnail of the web page | +| **player** | *string* | The url of the player of the web page | | **title** | *string* | The title of the web page | | **url** | *string* | The url of the web page | @@ -62,6 +63,7 @@ console.log(summary); // will be ... ↓ icon: 'http://livedoor.blogimg.jp/tmg24news/imgs/9/5/favicon.ico', description: '1：以下、名無しにかわりましてVIPがお送りします：2013/03/30(土) 14:57:29.09 ID:An34eOmY0モバＰ「反論が あるやつもいるかもしれない」 モバＰ「だが俺の主張も聞いてほしい！　お漏らしさせるならありすが一番だ！」 日菜子「むふふ……いきなりそんなことを大声で', thumbnail: 'http://livedoor.blogimg.jp/tmg24news/imgs/8/d/8df6e1a0-s.jpg', + player: null sitename: 'エレファント速報：SSまとめブログ', url: 'http://elephant.2chblog.jp/archives/52025138.html' } From 1f496d66bfb8040bf064ad0303bad61ee4d41e5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Acid=20Chicken=20=28=E7=A1=AB=E9=85=B8=E9=B6=8F=29?= Date: Thu, 9 Aug 2018 16:35:25 +0900 Subject: [PATCH 7/8] Update amazon.ts --- src/plugins/amazon.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/amazon.ts b/src/plugins/amazon.ts index 249d462..39d8db8 100644 --- a/src/plugins/amazon.ts +++ b/src/plugins/amazon.ts @@ -39,6 +39,7 @@ export async function summarize(url: URL.Url): Promise { icon: 'https://www.amazon.com/favicon.ico', description: description || null, thumbnail: thumbnail || null, + player: null, sitename: 'Amazon' }; } From 71e9606562d076e89240fd5aeb40113d47b7fea9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Acid=20Chicken=20=28=E7=A1=AB=E9=85=B8=E9=B6=8F=29?= Date: Thu, 9 Aug 2018 16:35:52 +0900 Subject: [PATCH 8/8] Update wikipedia.ts --- src/plugins/wikipedia.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/wikipedia.ts b/src/plugins/wikipedia.ts index b969f4c..fc36d1c 100644 --- a/src/plugins/wikipedia.ts +++ b/src/plugins/wikipedia.ts @@ -34,6 +34,7 @@ export async function summarize(url: URL.Url): Promise { icon: 'https://wikipedia.org/static/favicon/wikipedia.ico', description: clip(info.extract, 300), thumbnail: `https://wikipedia.org/static/images/project-logos/${lang}wiki.png`, + player: null, sitename: 'Wikipedia' }; }