📦 synchronising historical changes

Author: DGuang21

.github/publish.yml

@@ -0,0 +1,24 @@
+name: Node.js Package
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20.11.1
+          registry-url: https://npm.pkg.github.com/
+      - run: npm ci
+      - run: npm install
+      - run: npm publish
+        env:
+          NODE_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}}

README.md

@@ -1,3 +1,18 @@
+# Slax-Readability.js
+
+Since the original Readability version is prone to excessive deletion of the normal content of the article, this forked version is only intended to deal with the following specific issues to avoid missing content in the article
+
+- Add a unified whitelist for clean class
+- Add a whitelist for ignoring style removal
+- Retain elements in the highlights block
+- Adjusted video's regularity
+- Retain iframes with video src
+- add unified preserve-class-tag for preserving class type
+- Added ignore handling for microsoft video numbers
+- Expand the scope of paragraph preservation
+- Preserve wechat image style, skip processing
+- For H1~H7 tags, increased inclusiveness
+
 # Readability.js
 
 A standalone version of the readability library used for [Firefox Reader View](https://support.mozilla.org/kb/firefox-reader-view-clutter-free-web-pages).

Readability.js

@@ -139,19 +139,22 @@ Readability.prototype = {
     // Readability-readerable.js. Please keep both copies in sync.
     unlikelyCandidates:
       /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i,
-    okMaybeItsACandidate: /and|article|body|column|content|main|shadow/i,
-
+    okMaybeItsACandidate: /and|article|body|column|content|main|shadow|hljs-|katex-|mp-common-videosnap|preserve-class-tag/i,
+    needSaveHeaderTitle: /H[1-7]{1}/,
+    ignoreCleanClassesWhitelist: /^(hljs|katex|mp-common-videosnap|preserve-class-tag)(-.*)?$/i,
+    stylePreserveClassCandidates: /^(katex-|mp-common-videosnap)(-.*)?$/i,
+    ignoreCleanStylesWhitelist: /^(hljs|katex|mp-common-videosnap)(-.*)?$/i,
     positive:
       /article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story/i,
     negative:
       /-ad-|hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|contact|footer|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|widget/i,
     extraneous:
       /print|archive|comment|discuss|e[\-]?mail|share|reply|all|login|sign|single|utility/i,
-    byline: /byline|author|dateline|writtenby|p-author/i,
+    byline: /byline|author|dateline|writtenby|p-author|profileBt|bio/i,
     replaceFonts: /<(\/?)font[^>]*>/gi,
     normalize: /\s{2,}/g,
     videos:
-      /\/\/(www\.)?((dailymotion|youtube|youtube-nocookie|player\.vimeo|v\.qq)\.com|(archive|upload\.wikimedia)\.org|player\.twitch\.tv)/i,
+      /\/\/(www\.)?((dailymotion|youtube|youtube-nocookie|player\.vimeo|mpvideo|qpic|v\.qq)\.com|(archive|upload\.wikimedia)\.org|player\.twitch\.tv)|.cn/i,
     shareElements: /(\b|_)(share|sharedaddy)(\b|_)/i,
     nextLink: /(next|weiter|continue|>([^\|]|$)|»([^\|]|$))/i,
     prevLink: /(prev|earl|old|new|<|«)/i,
@@ -208,7 +211,6 @@ Readability.prototype = {
     "frame",
     "hspace",
     "rules",
-    "style",
     "valign",
     "vspace",
   ],
@@ -258,6 +260,22 @@ Readability.prototype = {
     "TIME",
     "VAR",
     "WBR",
+    "SVG",
+    "PATH",
+    "G",
+    "FIGURE",
+    "FIGCAPTION",
+    "PICTURE",
+    "SOURCE",
+    "TRACK",
+    "AREA",
+    "MAP",
+    "TABLE",
+    "ARTICLE",
+    "SECTION",
+    "P",
+    "OL",
+    "UL",
   ],
 
   // These are the classes that readability sets itself.
@@ -416,9 +434,19 @@ Readability.prototype = {
    */
   _cleanClasses(node) {
     var classesToPreserve = this._classesToPreserve;
+    var ignoreCleanClassesWhitelist = this.REGEXPS.ignoreCleanClassesWhitelist
+    var hasWhitelistClass = false
+
     var className = (node.getAttribute("class") || "")
       .split(/\s+/)
-      .filter(cls => classesToPreserve.includes(cls))
+      .filter(cls => {
+        if (ignoreCleanClassesWhitelist.test(cls)) {
+          hasWhitelistClass = true
+          return true
+        }
+
+        return classesToPreserve.includes(cls)
+      })
       .join(" ");
 
     if (className) {
@@ -427,7 +455,9 @@ Readability.prototype = {
       node.removeAttribute("class");
     }
 
-    for (node = node.firstElementChild; node; node = node.nextElementSibling) {
+
+
+    for (node = !hasWhitelistClass ? node.firstElementChild : null; node; node = node.nextElementSibling) {
       this._cleanClasses(node);
     }
   },
@@ -595,12 +625,11 @@ Readability.prototype = {
     // If there's a separator in the title, first remove the final part
     if (/ [\|\-\\\/>»] /.test(curTitle)) {
       titleHadHierarchicalSeparators = / [\\\/>»] /.test(curTitle);
-      let allSeparators = Array.from(origTitle.matchAll(/ [\|\-\\\/>»] /gi));
-      curTitle = origTitle.substring(0, allSeparators.pop().index);
+      curTitle = origTitle.replace(/(.*)[\|\-\\\/>»] .*/gi, "$1");
 
       // If the resulting title is too short, remove the first part instead:
       if (wordCount(curTitle) < 3) {
-        curTitle = origTitle.replace(/^[^\|\-\\\/>»]*[\|\-\\\/>»]/gi, "");
+        curTitle = origTitle.replace(/[^\|\-\\\/>»]*[\|\-\\\/>»](.*)/gi, "$1");
       }
     } else if (curTitle.includes(": ")) {
       // Check if we have an heading containing this exact string, so we
@@ -825,6 +854,11 @@ Readability.prototype = {
     this._cleanConditionally(articleContent, "ul");
     this._cleanConditionally(articleContent, "div");
 
+    this._replaceNodeTags(
+      this._getAllNodesWithTag(articleContent, ["slax-mark"]),
+      "span"
+    );
+
     // replace H1 with H2 as H1 should be only title that is displayed separately
     this._replaceNodeTags(
       this._getAllNodesWithTag(articleContent, ["h1"]),
@@ -1064,9 +1098,11 @@ Readability.prototype = {
         var matchString = node.className + " " + node.id;
 
         if (!this._isProbablyVisible(node)) {
-          this.log("Removing hidden node - " + matchString);
-          node = this._removeAndGetNext(node);
-          continue;
+          if (!this._haveAllowedVideoTag(node)) {
+            this.log("Removing hidden node - " + matchString);
+            node = this._removeAndGetNext(node);
+            continue;
+          }
         }
 
         // User is not able to see elements applied with both "aria-modal = true" and "role = dialog"
@@ -1121,7 +1157,8 @@ Readability.prototype = {
             !this._hasAncestorTag(node, "table") &&
             !this._hasAncestorTag(node, "code") &&
             node.tagName !== "BODY" &&
-            node.tagName !== "A"
+            node.tagName !== "A" &&
+            !this.REGEXPS.needSaveHeaderTitle.test(node.tagName)
           ) {
             this.log("Removing unlikely candidate - " + matchString);
             node = this._removeAndGetNext(node);
@@ -2100,7 +2137,12 @@ Readability.prototype = {
       e.removeAttribute("height");
     }
 
-    var cur = e.firstElementChild;
+    if (!this.REGEXPS.stylePreserveClassCandidates.test(e.className)) {
+      e.removeAttribute('style')
+    }
+
+    const ignore = this.REGEXPS.ignoreCleanStylesWhitelist.test(e.className);
+    var cur = !ignore ? e.firstElementChild : null;
     while (cur !== null) {
       this._cleanStyles(cur);
       cur = cur.nextElementSibling;
@@ -2416,6 +2458,10 @@ Readability.prototype = {
     if (textLength === 0) {
       return 0;
     }
+    if (e.querySelector('h1, h2, h3, h4, h5, h6, h7') &&
+      (e.textContent.trim() || e.querySelector('a') || e.querySelector('svg'))) {
+      return 1;
+    }
     var childrenLength = 0;
     var children = this._getAllNodesWithTag(e, tags);
     this._forEachNode(
@@ -2480,6 +2526,11 @@ Readability.prototype = {
         return false;
       }
 
+      const iframe = e.querySelector('iframe')
+      if (iframe && this._allowedVideoRegex.test(iframe.src)) {
+        return false
+      }
+
       var weight = this._getClassWeight(node);
 
       this.log("Cleaning Conditionally", node);
@@ -2496,6 +2547,7 @@ Readability.prototype = {
         // ominous signs, remove the element.
         var p = node.getElementsByTagName("p").length;
         var img = node.getElementsByTagName("img").length;
+        var video = node.getElementsByTagName('video').length
         var li = node.getElementsByTagName("li").length - 100;
         var input = node.getElementsByTagName("input").length;
         var headingDensity = this._getTextDensity(node, [
@@ -2594,9 +2646,9 @@ Readability.prototype = {
               `Suspicious embed. (embedCount=${embedCount}, contentLength=${contentLength})`
             );
           }
-          if (img === 0 && textDensity === 0) {
+          if (img === 0 && textDensity === 0 && video === 0) {
             errs.push(
-              `No useful content. (img=${img}, textDensity=${textDensity})`
+              `No useful content. (img=${img}, textDensity=${textDensity}, video=${video})`
             );
           }
 
@@ -2631,6 +2683,11 @@ Readability.prototype = {
     });
   },
 
+  _haveAllowedVideoTag(e) {
+    const videos = Array.from(e.querySelectorAll('video')) || []
+    return !!videos.find(video => this._allowedVideoRegex.test(video.getAttribute('src')))
+  },
+
   /**
    * Clean out elements that match the specified conditions
    *
@@ -2680,7 +2737,7 @@ Readability.prototype = {
     }
     var heading = this._getInnerText(node, false);
     this.log("Evaluating similarity of header:", heading, this._articleTitle);
-    return this._textSimilarity(this._articleTitle, heading) > 0.75;
+    return this._textSimilarity(this._articleTitle, heading) === 1;
   },
 
   _flagIsActive(flag) {

package.json

@@ -1,6 +1,6 @@
 {
-  "name": "@mozilla/readability",
-  "version": "0.6.0",
+  "name": "@slax-lab/slax-readability",
+  "version": "0.6.1",
   "description": "A standalone version of the readability library used for Firefox Reader View.",
   "main": "index.js",
   "types": "index.d.ts",