commit 80b010a7f8c24d0738194f4aaab368c8834f4eda Author: syuilo Date: Tue Sep 13 05:44:51 2016 +0900 Initial commit :pizza: diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f7c816e --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/node_modules +/typings +/built +npm-debug.log diff --git a/.npmignore b/.npmignore new file mode 100644 index 0000000..29012d1 --- /dev/null +++ b/.npmignore @@ -0,0 +1,5 @@ +/node_modules +/src +/typings +/tmp +npm-debug.log diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..73f4ec9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 syuilo + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..a4cc90e --- /dev/null +++ b/README.md @@ -0,0 +1,12 @@ +webcard +======= + +Generate an html of any web page's summary. + +Installation +------------ +`$ npm install webcard` + +License +------- +[MIT](LICENSE) diff --git a/dtsm.json b/dtsm.json new file mode 100644 index 0000000..8ff0986 --- /dev/null +++ b/dtsm.json @@ -0,0 +1,20 @@ +{ + "repos": [ + { + "url": "https://github.com/borisyankov/DefinitelyTyped.git", + "ref": "master" + } + ], + "path": "typings", + "bundle": "typings/bundle.d.ts", + "link": { + "npm": { + "include": true + } + }, + "dependencies": { + "request/request.d.ts": { + "ref": "0c5c7a2d2bd0ce7dcab963a8402a9042749ca2da" + } + } +} \ No newline at end of file diff --git a/gulpfile.js b/gulpfile.js new file mode 100644 index 0000000..1bb25fe --- /dev/null +++ b/gulpfile.js @@ -0,0 +1,35 @@ +'use strict'; + +const gulp = require('gulp'); +const babel = require('gulp-babel'); +const ts = require('gulp-typescript'); +const es = require('event-stream'); + +const project = ts.createProject('tsconfig.json'); + +gulp.task('build', [ + 'build:ts', + 'build:copy' +]); + +gulp.task('build:ts', () => { + const tsResult = project + .src() + .pipe(ts(project)) + .pipe(babel({ + presets: ['es2015', 'stage-3'] + })); + + return es.merge( + tsResult.pipe(gulp.dest('./built/'))/*, + tsResult.dts.pipe(gulp.dest('./built/'))*/ + ); +}); + +gulp.task('build:copy', () => { + return es.merge( + gulp.src([ + './src/**/*.pug' + ]).pipe(gulp.dest('./built/')) + ); +}); diff --git a/package.json b/package.json new file mode 100644 index 0000000..93fa6b9 --- /dev/null +++ b/package.json @@ -0,0 +1,32 @@ +{ + "name": "webcard", + "version": "0.0.0", + "description": "Generate an html of any web page's summary", + "author": "syuilo ", + "license": "MIT", + "repository": "https://github.com/syuilo/webcard.git", + "bugs": "https://github.com/syuilo/webcard/issues", + "main": "./built/index.js", + "typings": "./built/index.d.ts", + "scripts": { + "start": "node ./built/index.js", + "build": "gulp build" + }, + "devDependencies": { + "babel-preset-es2015": "6.13.2", + "babel-preset-stage-3": "6.11.0", + "event-stream": "3.3.4", + "gulp": "3.9.1", + "gulp-babel": "6.1.2", + "gulp-typescript": "2.13.6", + "typescript": "1.8.10" + }, + "dependencies": { + "babel-core": "6.13.2", + "babel-polyfill": "6.13.0", + "cheerio-httpcli": "^0.6.9", + "html-entities": "^1.2.0", + "pug": "^2.0.0-beta6", + "request": "^2.74.0" + } +} diff --git a/src/general/index.ts b/src/general/index.ts new file mode 100644 index 0000000..c9423d2 --- /dev/null +++ b/src/general/index.ts @@ -0,0 +1,136 @@ +import * as URL from 'url'; +import * as request from 'request'; +const pug = require('pug'); +import Options from '../options'; + +const Entities = require('html-entities').AllHtmlEntities; +const entities = new Entities(); + +const client = require('cheerio-httpcli'); +client.referer = false; +client.timeout = 10000; + +export default async (url: URL.Url, opts: Options): Promise => { + const res = await client.fetch(url.href); + + if (res.error) { + throw 'something happened'; + } + + const contentType: string = res.response.headers['content-type']; + + // HTMLじゃなかった場合は中止 + if (contentType.indexOf('text/html') === -1) { + return null; + } + + const $: any = res.$; + + let title = + $('meta[property="misskey:title"]').attr('content') || + $('meta[property="og:title"]').attr('content') || + $('meta[property="twitter:title"]').attr('content') || + $('title').text(); + + if (title == null) { + return null; + } + + title = clip(entities.decode(title), 100); + + const lang: string = $('html').attr('lang'); + + const type = + $('meta[property="misskey:type"]').attr('content') || + $('meta[property="og:type"]').attr('content'); + + let image = + $('meta[property="misskey:image"]').attr('content') || + $('meta[property="og:image"]').attr('content') || + $('meta[property="twitter:image"]').attr('content') || + $('link[rel="image_src"]').attr('href') || + $('link[rel="apple-touch-icon"]').attr('href') || + $('link[rel="apple-touch-icon image_src"]').attr('href'); + + image = image ? proxy(URL.resolve(url.href, image)) : null; + + let description = + $('meta[property="misskey:summary"]').attr('content') || + $('meta[property="og:description"]').attr('content') || + $('meta[property="twitter:description"]').attr('content') || + $('meta[name="description"]').attr('content'); + + description = description + ? clip(entities.decode(description), 300) + : null; + + if (title === description) { + description = null; + } + + let siteName = + $('meta[property="misskey:site-name"]').attr('content') || + $('meta[property="og:site_name"]').attr('content') || + $('meta[name="application-name"]').attr('content'); + + siteName = siteName ? entities.decode(siteName) : null; + + let icon = + $('meta[property="misskey:site-icon"]').attr('content') || + $('link[rel="shortcut icon"]').attr('href') || + $('link[rel="icon"]').attr('href') || + '/favicon.ico'; + + icon = icon ? proxy(URL.resolve(url.href, icon)) : null; + + return pug.renderFile(`${__dirname}/summary.pug`, { + url: url, + title: title, + icon: icon, + lang: lang, + description: description, + type: type, + image: image, + siteName: siteName + }); + + function proxy(url: string): string { + return `${opts.proxy}/${url}`; + } +} + +function promisifyRequest(request: any): (x: any) => Promise { + return (x: any) => { + return new Promise((resolve) => { + request(x, (a: any, b: any, c: any) => { + resolve(c); + }); + }); + }; +} + +function nullOrEmpty(val: string): boolean { + if (val === undefined) { + return true; + } else if (val === null) { + return true; + } else if (val.trim() === '') { + return true; + } else { + return false; + } +} + +function clip(s: string, max: number): string { + if (nullOrEmpty(s)) { + return s; + } + + s = s.trim(); + + if (s.length > max) { + return s.substr(0, max) + '...'; + } else { + return s; + } +} diff --git a/src/general/summary.pug b/src/general/summary.pug new file mode 100644 index 0000000..f2bc2c3 --- /dev/null +++ b/src/general/summary.pug @@ -0,0 +1,16 @@ +a(title= url.href, href= url.href, target='_blank') + aside(lang= lang, data-type= type) + if image + div.thumbnail(style={'background-image': 'url(' + image + ')'}) + h1.title= title + if description + p.description= description + footer + p.hostname + if url.protocol == 'https:' + i.fa.fa-lock.secure + = url.hostname + if icon + img.icon(src= icon, alt='') + if siteName + p.site-name= siteName diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..e5a0759 --- /dev/null +++ b/src/index.ts @@ -0,0 +1,24 @@ +import * as URL from 'url'; +import IPlugin from './iplugin'; +import Options from './options'; +import general from './general'; + +// Init babel +require('babel-core/register'); +require('babel-polyfill'); + +const plugins: IPlugin[] = [ + require('./plugins/wikipedia') +]; + +export default async (url: string, opts: Options): Promise => { + const _url = URL.parse(url, true); + + const plugin = plugins.filter(plugin => plugin.test(_url))[0]; + + if (plugin) { + return await plugin.compile(_url, opts); + } else { + return await general(_url, opts); + } +} diff --git a/src/iplugin.ts b/src/iplugin.ts new file mode 100644 index 0000000..115afdc --- /dev/null +++ b/src/iplugin.ts @@ -0,0 +1,9 @@ +import * as URL from 'url'; +import Options from './options'; + +interface IPlugin { + test: (url: URL.Url) => boolean; + compile: (url: URL.Url, opts: Options) => Promise; +} + +export default IPlugin; \ No newline at end of file diff --git a/src/options.ts b/src/options.ts new file mode 100644 index 0000000..561b55f --- /dev/null +++ b/src/options.ts @@ -0,0 +1,5 @@ +interface Options { + proxy: string; +} + +export default Options; \ No newline at end of file diff --git a/src/plugins/wikipedia/index.ts b/src/plugins/wikipedia/index.ts new file mode 100644 index 0000000..b08d074 --- /dev/null +++ b/src/plugins/wikipedia/index.ts @@ -0,0 +1,31 @@ +import * as URL from 'url'; +const pug = require('pug'); +import Options from '../../options'; + +const client = require('cheerio-httpcli'); +client.referer = false; +client.timeout = 10000; + +exports.test = (url: URL.Url) => { + return /\.wikipedia\.org$/.test(url.hostname); +}; + +exports.compile = async (url: URL.Url, opts: Options) => { + const res = await client.fetch(url.href); + const $: any = res.$; + + const lang = url.hostname.substr(0, url.hostname.indexOf('.')); + const isDesktop = !/\.m\.wikipedia\.org$/.test(url.hostname); + const text: string = isDesktop + ? $('#mw-content-text > p:first-of-type').text() + : $('#bodyContent > div:first-of-type > p:first-of-type').text(); + + return pug.renderFile(`${__dirname}/../../general/summary.pug`, { + url: url, + title: decodeURI(url.pathname.split('/')[2]), + icon: 'https://wikipedia.org/static/favicon/wikipedia.ico', + description: text, + image: `https://wikipedia.org/static/images/project-logos/${lang}wiki.png`, + siteName: 'Wikipedia' + }); +}; diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..1f78993 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,34 @@ +{ + "compilerOptions": { + "noEmitOnError": true, + "noImplicitAny": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "declaration": true, + "sourceMap": false, + "target": "es6", + "module": "commonjs", + "removeComments": false, + "noLib": false, + "outDir": "built", + "rootDir": "src" + }, + "compileOnSave": false, + "atom": { + "rewriteTsconfig": true + }, + "filesGlob": [ + "./node_modules/typescript/lib/lib.es6.d.ts", + "./typings/bundle.d.ts", + "./src/**/*.ts" + ], + "files": [ + "./node_modules/typescript/lib/lib.es6.d.ts", + "./typings/bundle.d.ts", + "./src/index.ts", + "./src/iplugin.ts", + "./src/options.ts", + "./src/general/index.ts", + "./src/plugins/wikipedia/index.ts" + ] +} diff --git a/tslint.json b/tslint.json new file mode 100644 index 0000000..e7c21c3 --- /dev/null +++ b/tslint.json @@ -0,0 +1,86 @@ +{ + "rules": { + "align": [true, + "parameters", + "statements" + ], + "ban": false, + "class-name": true, + "comment-format": [true, + "check-upper-case" + ], + "curly": true, + "eofline": true, + "forin": false, + "indent": [true, "tabs"], + "interface-name": false, + "jsdoc-format": true, + "label-position": true, + "label-undefined": true, + "max-line-length": [true, 140], + "member-access": false, + "member-ordering": [true, + "static-before-instance", + "variables-before-functions" + ], + "no-any": false, + "no-arg": true, + "no-bitwise": true, + "no-console": [true, + "debug", + "info", + "time", + "timeEnd", + "trace" + ], + "no-consecutive-blank-lines": true, + "no-construct": true, + "no-constructor-vars": true, + "no-debugger": true, + "no-duplicate-key": true, + "no-shadowed-variable": false, + "no-duplicate-variable": true, + "no-empty": true, + "no-eval": true, + "no-internal-module": true, + "no-require-imports": false, + "no-string-literal": false, + "no-switch-case-fall-through": true, + "no-trailing-whitespace": true, + "no-unreachable": true, + "no-unused-expression": true, + "no-unused-variable": true, + "no-use-before-declare": true, + "no-var-keyword": true, + "no-var-requires": false, + "one-line": [true, + "check-catch", + "check-whitespace" + ], + "quotemark": false, + "radix": true, + "semicolon": true, + "switch-default": false, + "triple-equals": true, + "typedef": [true, + "call-signature", + "property-declaration" + ], + "typedef-whitespace": [true, { + "call-signature": "nospace", + "index-signature": "nospace", + "parameter": "nospace", + "property-declaration": "nospace", + "variable-declaration": "nospace" + }], + "use-strict": false, + "variable-name": false, + "whitespace": [true, + "check-branch", + "check-decl", + "check-operator", + "check-separator", + "check-type" + ] + } +}