chore: remove summaly dependency

package.json

@@ -11,7 +11,6 @@
     "hono": "^4.4.0",
     "html-entities": "^2.5.2",
     "jschardet": "^3.1.2",
-    "summaly": "^2.7.0",
     "whatwg-mimetype": "^4.0.0",
     "zod": "^3.23.8"
   }

src/index.ts

@@ -199,6 +199,27 @@ if (import.meta.vitest) {
           url: "https://open.spotify.com/track/5ZBqOdKlSlaYaK0Lyu982P?si=CMFj2OJfTCKnY8tbECKcpg&utm_source=copy-link&utm_medium=copy-link&nd=1&%24web_only=true",
         },
       ],
+      [
+        "too long texts on Amazon.co.jp",
+        "https://www.amazon.co.jp/dp/4065269210",
+        "text/html;charset=UTF-8",
+        {
+          description: "Amazonで業務用餅, kisui, 六志麻 あさの追放されたチート付与魔術師は気ままなセカンドライフを謳歌する。 ~俺は武器だけじゃなく、あらゆるものに『強化ポイント』を付与できるし、俺の意思でいつでも効果を解除できるけど、残った人たち大丈夫?~(1) (KCデラックス)。アマゾンならポイント還元本が多数。業務用餅, kisui, 六志麻 あさ作品ほか、お急ぎ便対象商品は当日お届けも可能。また追放されたチート付与魔術師は気ままなセカンドライフを謳歌する。 ~俺は武器だけじゃなく、あらゆるものに『強化ポイント』を付与できるし、俺の意思でいつでも効果を解除できるけど、残った人たち大丈夫?~(…",
+          icon: "https://www.amazon.co.jp/favicon.ico",
+          large: false,
+          player: {
+            allow: [],
+            height: null,
+            url: null,
+            width: null,
+          },
+          sensitive: false,
+          sitename: "www.amazon.co.jp",
+          thumbnail: "https://m.media-amazon.com/images/I/51KXmamiQKL._SY445_SX342_.jpg",
+          title: "追放されたチート付与魔術師は気ままなセカンドライフを謳歌する。 ~俺は武器だけじゃなく、あらゆるものに『強化ポイント』を付与できるし、俺の意思でいつでも効果を解除できるけど、残った人たち大丈夫?~(…",
+          url: "https://www.amazon.co.jp/dp/4065269210",
+        },
+      ],
     ])("should return summary of %s <%s>", async (_, url, contentType, expected) => {
       const request = new Request(`https://fakehost/url?${new URLSearchParams({ url })}`)
       const ctx = createExecutionContext()

src/summary/amazon/description.ts

@@ -1,5 +1,4 @@
 import { decode } from "html-entities"
-import clip from "summaly/built/utils/clip"
 import { BufferedTextHandler, assign } from "../common"
 import type { PrioritizedReference } from "../common"
 import type Context from "../../context"
@@ -43,7 +42,7 @@ export default function getDescription(context: Context) {
   })
   context.html.onDocument({
     end() {
-      resolve(result.content && clip(result.content, 300))
+      resolve(result.content?.trim() || null)
     },
   })
   return promise

src/summary/amazon/index.ts

@@ -1,4 +1,3 @@
-import cleanupTitle from "summaly/built/utils/cleanup-title"
 import getCard from "../general/card"
 import getDescription from "./description"
 import getFavicon from "../general/favicon"
@@ -11,6 +10,7 @@ import getSiteName from "../general/siteName"
 import getTitle from "./title"
 import getSensitive from "../general/sensitive"
 import type Context from "../../context"
+import type { NullPlayer, ValidPlayer } from "../common"
 
 export default function amazon(context: Context) {
   const card = getCard(context)
@@ -23,13 +23,15 @@ export default function amazon(context: Context) {
         url,
         width,
         height,
-      }
+        allow: [],
+      } satisfies ValidPlayer
     } else {
       return {
         url: null,
         width: null,
         height: null,
-      }
+        allow: [],
+      } satisfies NullPlayer
     }
   })
   const description = getDescription(context)
@@ -38,12 +40,6 @@ export default function amazon(context: Context) {
   const sensitive = getSensitive(context)
 
   return Promise.all([title, thumbnail, player, description, siteName, favicon, sensitive]).then(([title, thumbnail, player, description, siteName, favicon, sensitive]) => {
-    if (title === null) {
-      return null
-    }
-    if (siteName !== null) {
-      title = cleanupTitle(title, siteName)
-    }
     return {
       title,
       thumbnail,
@@ -52,6 +48,7 @@ export default function amazon(context: Context) {
       sitename: siteName,
       icon: favicon,
       sensitive,
+      large: false,
       url: context.url.href,
     }
   })

src/summary/amazon/title.ts

@@ -1,5 +1,4 @@
 import { decode } from "html-entities"
-import clip from "summaly/built/utils/clip"
 import { BufferedTextHandler, assign } from "../common"
 import type { PrioritizedReference } from "../common"
 import type Context from "../../context"
@@ -41,7 +40,7 @@ export default function getTitle(context: Context) {
   )
   context.html.onDocument({
     end() {
-      resolve(result.content && clip(result.content, 100))
+      resolve(result.content?.trim() || null)
     },
   })
   return promise

src/summary/common.ts

@@ -1,3 +1,29 @@
+export interface Summary {
+  title: string | null
+  description: string | null
+  thumbnail: string | null
+  player: ValidPlayer | NullPlayer
+  sensitive: boolean
+  large: boolean
+  icon: string | null
+  sitename: string | null
+  url: string
+}
+
+export interface ValidPlayer {
+  url: string
+  width: number
+  height: number
+  allow: string[]
+}
+
+export interface NullPlayer {
+  url: null
+  width: null
+  height: null
+  allow: []
+}
+
 export interface PrioritizedReference<T> {
   bits: number
   priority: number
@@ -19,6 +45,34 @@ export function toAbsoluteURL(url: string, base: string) {
   }
 }
 
+export function cleanupTitle(title: string, siteName: string) {
+  if (title.endsWith(siteName)) {
+    return title
+      .slice(0, -siteName.length)
+      .trim()
+      .replace(/[\-\|:·・]+$/, "")
+      .trim()
+  }
+  return title
+}
+
+const locales = Intl.Segmenter.supportedLocalesOf(["af", "agq", "ak", "am", "ar", "ars", "as", "asa", "ast", "az", "bas", "be", "bem", "bez", "bg", "bgc", "bho", "blo", "bm", "bn", "bo", "br", "brx", "bs", "ca", "ccp", "ce", "ceb", "cgg", "chr", "ckb", "cs", "csw", "cv", "cy", "da", "dav", "de", "dje", "doi", "dsb", "dua", "dyo", "dz", "ebu", "ee", "el", "en", "eo", "es", "et", "eu", "ewo", "fa", "ff", "fi", "fil", "fo", "fr", "fur", "fy", "ga", "gd", "gl", "gsw", "gu", "guz", "gv", "ha", "haw", "he", "hi", "hr", "hsb", "hu", "hy", "ia", "id", "ie", "ig", "ii", "is", "it", "ja", "jgo", "jmc", "jv", "ka", "kab", "kam", "kde", "kea", "kgp", "khq", "ki", "kk", "kkj", "kl", "kln", "km", "kn", "ko", "kok", "ks", "ksb", "ksf", "ksh", "ku", "kw", "kxv", "ky", "lag", "lb", "lg", "lij", "lkt", "lmo", "ln", "lo", "lrc", "lt", "lu", "luo", "luy", "lv", "mai", "mas", "mer", "mfe", "mg", "mgh", "mgo", "mi", "mk", "ml", "mn", "mni", "mr", "ms", "mt", "mua", "my", "mzn", "naq", "nb", "nd", "nds", "ne", "nl", "nmg", "nn", "nnh", "no", "nqo", "nus", "nyn", "oc", "om", "or", "os", "pa", "pcm", "pl", "prg", "ps", "pt", "qu", "raj", "rm", "rn", "ro", "rof", "ru", "rw", "rwk", "sa", "sah", "saq", "sat", "sbp", "sc", "sd", "se", "seh", "ses", "sg", "shi", "si", "sk", "sl", "smn", "sn", "so", "sq", "sr", "su", "sv", "sw", "syr", "szl", "ta", "te", "teo", "tg", "th", "ti", "tk", "to", "tok", "tr", "tt", "twq", "tzm", "ug", "uk", "ur", "uz", "vai", "vec", "vi", "vmw", "vun", "wae", "wo", "xh", "xnr", "xog", "yav", "yi", "yo", "yrl", "yue", "za", "zgh", "zh", "zu"])
+const segmenter = new Intl.Segmenter(locales, { granularity: "word", localeMatcher: "best fit" })
+const ellipsis = "…"
+
+export function clip(text: string, length: number) {
+  const segments = segmenter.segment(text)
+  let result = ""
+  for (const segment of segments) {
+    if (result.length + segment.segment.length > length - ellipsis.length) {
+      result += ellipsis
+      break
+    }
+    result += segment.segment
+  }
+  return result
+}
+
 export class BufferedTextHandler {
   private buffer = ""
 

src/summary/general/description.ts

@@ -1,5 +1,4 @@
 import { decode } from "html-entities"
-import clip from "summaly/built/utils/clip"
 import { assign } from "../common"
 import type Context from "../../context"
 import type { PrioritizedReference } from "../common"
@@ -37,7 +36,7 @@ export default function getDescription(context: Context) {
   })
   context.html.onDocument({
     end() {
-      resolve(result.content && clip(result.content, 300))
+      resolve(result.content?.trim() || null)
     },
   })
   return promise

src/summary/general/index.ts

@@ -1,4 +1,4 @@
-import cleanupTitle from "summaly/built/utils/cleanup-title"
+import { NullPlayer, ValidPlayer } from "../common"
 import getCard from "./card"
 import getDescription from "./description"
 import getFavicon from "./favicon"
@@ -6,20 +6,29 @@ import getImage from "./image"
 import getSiteName from "./siteName"
 import getTitle from "./title"
 import getSensitive from "./sensitive"
-import getPlayer, { Player } from "./player"
+import getPlayer from "./player"
 import type Context from "../../context"
 
 export default function general(context: Context) {
   const card = getCard(context)
   const title = getTitle(context)
   const image = getImage(context)
-  const player = Promise.all([card, getPlayer(context)]).then<Player>(([card, parsedPlayer]) => {
+  const player = Promise.all([card, getPlayer(context)]).then<ValidPlayer | NullPlayer>(([card, parsedPlayer]) => {
+    const url = (card !== "summary_large_image" && parsedPlayer.urlGeneral) || parsedPlayer.urlCommon
+    if (url === null || parsedPlayer.width === null || parsedPlayer.height === null) {
+      return {
+        url: null,
+        width: null,
+        height: null,
+        allow: parsedPlayer.allow as [],
+      } satisfies NullPlayer
+    }
     return {
-      url: (card !== "summary_large_image" && parsedPlayer.urlGeneral) || parsedPlayer.urlCommon,
+      url,
       width: parsedPlayer.width,
       height: parsedPlayer.height,
       allow: parsedPlayer.allow,
-    }
+    } satisfies ValidPlayer
   })
   const description = getDescription(context)
   const siteName = getSiteName(context)
@@ -27,12 +36,6 @@ export default function general(context: Context) {
   const sensitive = getSensitive(context)
 
   return Promise.all([card, title, image, player, description, siteName, favicon, sensitive]).then(([card, title, image, player, description, siteName, favicon, sensitive]) => {
-    if (title === null) {
-      return null
-    }
-    if (siteName !== null) {
-      title = cleanupTitle(title, siteName)
-    }
     return {
       title,
       thumbnail: image,

src/summary/general/player.ts

@@ -4,19 +4,20 @@ import getPlayerUrlGeneral from "./playerUrlGeneral"
 import getPlayerUrlHeight from "./playerUrlHeight"
 import getPlayerUrlWidth from "./playerUrlWidth"
 import type Context from "../../context"
+import type { NullPlayer, ValidPlayer } from "../common"
 
-export interface Player {
-  url: string | null
-  width: number | null
-  height: number | null
-  allow: string[]
-}
-
-export interface ParsedPlayer extends Omit<Player, "url"> {
+export interface ParsedValidPlayer extends Omit<ValidPlayer, "url"> {
   urlCommon: string | null
   urlGeneral: string | null
 }
 
+export interface ParsedNullPlayer extends Omit<NullPlayer, "url"> {
+  urlCommon: null
+  urlGeneral: null
+}
+
+export type ParsedPlayer = ParsedValidPlayer | ParsedNullPlayer
+
 export default function getPlayer(context: Context): Promise<ParsedPlayer> {
   const oEmbed = getPlayerOEmbed(context)
   const urlGeneral = getPlayerUrlGeneral(context)
@@ -28,12 +29,21 @@ export default function getPlayer(context: Context): Promise<ParsedPlayer> {
     if (oEmbed) {
       return oEmbed
     }
+    if (width === null || height === null) {
+      return {
+        urlCommon: null,
+        urlGeneral: null,
+        width: null,
+        height: null,
+        allow: [],
+      } satisfies ParsedNullPlayer
+    }
     return {
       urlCommon,
       urlGeneral,
       width,
       height,
       allow: [],
-    }
+    } satisfies ParsedValidPlayer
   })
 }

src/summary/general/siteName.ts

@@ -28,7 +28,7 @@ export default function getSiteName(context: Context) {
   })
   context.html.onDocument({
     end() {
-      resolve(result.content)
+      resolve(result.content?.trim() || null)
     },
   })
   return promise

src/summary/general/title.ts

@@ -1,5 +1,4 @@
 import { decode } from "html-entities"
-import clip from "summaly/built/utils/clip"
 import { BufferedTextHandler, assign } from "../common"
 import type Context from "../../context"
 import type { PrioritizedReference } from "../common"
@@ -35,7 +34,7 @@ export default function getTitle(context: Context) {
   )
   context.html.onDocument({
     end() {
-      resolve(result.content && clip(result.content, 100))
+      resolve(result.content?.trim() || null)
     },
   })
   return promise

src/summary/index.ts

@@ -1,18 +1,35 @@
 import amazon from "./amazon"
 import branchio from "./branchio"
+import { cleanupTitle, clip } from "./common"
 import general from "./general"
 import wikipedia from "./wikipedia"
 import type Context from "../context"
+import type { Summary } from "./common"
 
-export default function summary(context: Context) {
+export default async function summary(context: Context) {
   if (context.url.hostname === "www.amazon.com" || context.url.hostname === "www.amazon.co.jp" || context.url.hostname === "www.amazon.ca" || context.url.hostname === "www.amazon.com.br" || context.url.hostname === "www.amazon.com.mx" || context.url.hostname === "www.amazon.co.uk" || context.url.hostname === "www.amazon.de" || context.url.hostname === "www.amazon.fr" || context.url.hostname === "www.amazon.it" || context.url.hostname === "www.amazon.es" || context.url.hostname === "www.amazon.nl" || context.url.hostname === "www.amazon.cn" || context.url.hostname === "www.amazon.in" || context.url.hostname === "www.amazon.au") {
-    return amazon(context)
+    return postProcess(await amazon(context))
   }
   if (`.${context.url.hostname}`.endsWith(".app.link")) {
-    return branchio(context)
+    return postProcess(await branchio(context))
   }
   if (`.${context.url.hostname}`.endsWith(".wikipedia.org")) {
-    return wikipedia(context)
+    return postProcess(await wikipedia(context))
   }
-  return general(context)
+  return postProcess(await general(context))
+}
+
+function postProcess(summary: Summary | null) {
+  if (summary === null) {
+    return null
+  }
+  if (summary.title === null) {
+    return null
+  }
+  if (summary.sitename !== null) {
+    summary.title = cleanupTitle(summary.title, summary.sitename)
+  }
+  summary.title = clip(summary.title, 100)
+  summary.description = summary.description && clip(summary.description, 300)
+  return summary
 }

src/summary/wikipedia/index.ts

@@ -1,6 +1,6 @@
-import clip from "summaly/built/utils/clip"
 import { requestInit } from "../../config"
 import type Context from "../../context"
+import type { NullPlayer } from "../common"
 
 export default async function wikipedia(context: Context) {
   const lang = context.url.hostname.split(".")[0]
@@ -11,15 +11,17 @@ export default async function wikipedia(context: Context) {
   return {
     title: info.title,
     icon: "https://wikipedia.org/static/favicon/wikipedia.ico",
-    description: clip(info.extract, 300),
+    description: info.extract?.trim() || null,
     thumbnail: `https://wikipedia.org/static/images/project-logos/${lang}wiki.png`,
     player: {
       url: null,
       width: null,
       height: null,
       allow: [],
-    },
+    } satisfies NullPlayer,
     sitename: "Wikipedia",
+    sensitive: false,
+    large: false,
     url: context.url.href,
   }
 }