enable adblock in order to speed up the loading process

2020-05-22 20:37:03 +02:00 · 2020-05-22 20:37:03 +02:00 · 49a3b14499
commit 49a3b14499
parent 921ceaa124
3 changed files with 167 additions and 1 deletions
--- a/index.js
+++ b/index.js
@ -1,5 +1,7 @@
 const puppeteer = require('puppeteer-extra');
-const StealthPlugin = require('puppeteer-extra-plugin-stealth')
+const StealthPlugin = require('puppeteer-extra-plugin-stealth');
+const AdblockerPlugin = require('puppeteer-extra-plugin-adblocker');
+puppeteer.use(AdblockerPlugin());
 puppeteer.use(StealthPlugin());
 const Koa = require('koa');
 const app = new Koa();
--- a/package-lock.json
+++ b/package-lock.json
@ -4,11 +4,129 @@
  "lockfileVersion": 1,
  "requires": true,
  "dependencies": {
+    "@cliqz/adblocker": {
+      "version": "1.14.4",
+      "resolved": "https://registry.npmjs.org/@cliqz/adblocker/-/adblocker-1.14.4.tgz",
+      "integrity": "sha512-3lTnstcPzNytR/NJ1VS5wt3iztt0tgiPJqXeOhbp80G9XfXsWfj18n6IJryyillbkNyRlmi7ElS3s/DGAZ17Bw==",
+      "requires": {
+        "@remusao/guess-url-type": "^1.1.2",
+        "@remusao/small": "^1.1.2",
+        "@remusao/smaz": "^1.7.1",
+        "@types/chrome": "^0.0.107",
+        "@types/firefox-webext-browser": "^70.0.1",
+        "tldts-experimental": "^5.6.21"
+      },
+      "dependencies": {
+        "@types/chrome": {
+          "version": "0.0.107",
+          "resolved": "https://registry.npmjs.org/@types/chrome/-/chrome-0.0.107.tgz",
+          "integrity": "sha512-7z/4YkJu4B8ROxZhbzIx5olbRJqzoDdAoMhSzcJlElsZzv2eTxsvxNkN/QFq0ClbAFNkFf07XVoTrWFHD12QyA==",
+          "requires": {
+            "@types/filesystem": "*",
+            "@types/har-format": "*"
+          }
+        }
+      }
+    },
+    "@cliqz/adblocker-content": {
+      "version": "1.14.4",
+      "resolved": "https://registry.npmjs.org/@cliqz/adblocker-content/-/adblocker-content-1.14.4.tgz",
+      "integrity": "sha512-xuQ93BoHKE2PbL8X1v2J7Eu9Ckgrh2rya9FnTgzQ1cyMn6vvtT4m/VSm/dt/JiRcKJbdMCyQA/1Z2GW0iVZz4A=="
+    },
+    "@cliqz/adblocker-puppeteer": {
+      "version": "1.14.4",
+      "resolved": "https://registry.npmjs.org/@cliqz/adblocker-puppeteer/-/adblocker-puppeteer-1.14.4.tgz",
+      "integrity": "sha512-FhN2eVFtHDDoCGHpqF00NLfT+HtcfnkSPWx/baEJ0gNaWdNZTI8EQYrX/HFz2oj/KD6MHxDr7opnDUNRIO78Gg==",
+      "requires": {
+        "@cliqz/adblocker": "^1.14.4",
+        "@cliqz/adblocker-content": "^1.14.4",
+        "@types/puppeteer": "^2.0.0",
+        "tldts-experimental": "^5.6.21"
+      },
+      "dependencies": {
+        "@types/puppeteer": {
+          "version": "2.1.1",
+          "resolved": "https://registry.npmjs.org/@types/puppeteer/-/puppeteer-2.1.1.tgz",
+          "integrity": "sha512-FqPZvUtnpTGrqbHvPUn76pvVcBPEVEqZftrdOjr6YRkaaxkjKQ8dQLNaQBjER7Lvd1Q6+0R0XR+N3tYGWBSzNw==",
+          "requires": {
+            "@types/node": "*"
+          }
+        }
+      }
+    },
+    "@remusao/guess-url-type": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@remusao/guess-url-type/-/guess-url-type-1.1.2.tgz",
+      "integrity": "sha512-NDcy4J7bieEU2WYSn1LZKjaIG+5ORDWXA//6dXnppVYPmmJ9zjJ1vJVKUKdZ1mjvTp7WLzQH6QD/wygaZ13G1Q=="
+    },
+    "@remusao/small": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@remusao/small/-/small-1.1.2.tgz",
+      "integrity": "sha512-cMteQVgRT+LhSTQQdR0pWlLRXOTV8FeePOxcwkrroYCKIWpqIKNvbimAEiioMlclDmSHos9Nu+Wz3iNIDsyQaQ=="
+    },
+    "@remusao/smaz": {
+      "version": "1.8.0",
+      "resolved": "https://registry.npmjs.org/@remusao/smaz/-/smaz-1.8.0.tgz",
+      "integrity": "sha512-tpyluMIHQSCOX1VHBBrn7KP+S1m/fovJg9Tizo06qR9OYopYehDqkuBJGLnUthE3uhm7tbxb1XI2IN4tUjOFhw==",
+      "requires": {
+        "@remusao/smaz-compress": "^1.8.0",
+        "@remusao/smaz-decompress": "^1.8.0"
+      }
+    },
+    "@remusao/smaz-compress": {
+      "version": "1.8.0",
+      "resolved": "https://registry.npmjs.org/@remusao/smaz-compress/-/smaz-compress-1.8.0.tgz",
+      "integrity": "sha512-gDlnWyYmDppwdVZZkBuN8/gzpcqxFwEEev8QxyArWbOn/1rIXMfnm+bwY/N80rmakHTWaybL0yNxTMJSoVXrUw==",
+      "requires": {
+        "@remusao/trie": "^1.3.0"
+      }
+    },
+    "@remusao/smaz-decompress": {
+      "version": "1.8.0",
+      "resolved": "https://registry.npmjs.org/@remusao/smaz-decompress/-/smaz-decompress-1.8.0.tgz",
+      "integrity": "sha512-DDGL8wB2L1OUPbUEashqBQzuEVk1xmVOR+35lERkwl3Qg5UoiL//9GWSq38jq8XlTqUG5R+BY2dQa4XX2NDeeQ=="
+    },
+    "@remusao/trie": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/@remusao/trie/-/trie-1.3.0.tgz",
+      "integrity": "sha512-IykMLl0vr8L1sJzvtGR4izkNh1Ous2njaeC4NEvAcfALCdCo7Y4VpISU0I8DaZFU2lGAFKyeh2Jz4WaM/3hbEw=="
+    },
+    "@types/chrome": {
+      "version": "0.0.91",
+      "resolved": "https://registry.npmjs.org/@types/chrome/-/chrome-0.0.91.tgz",
+      "integrity": "sha512-vNvo9lJkp1AvViWrUwe1bxhoMwr5dRZWlgr1DTuaNkz97LsG56lDX1sceWeZir2gRACJ5vdHtoRdVAvm8C75Ug==",
+      "requires": {
+        "@types/filesystem": "*"
+      }
+    },
    "@types/debug": {
      "version": "4.1.5",
      "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.5.tgz",
      "integrity": "sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ=="
    },
+    "@types/filesystem": {
+      "version": "0.0.29",
+      "resolved": "https://registry.npmjs.org/@types/filesystem/-/filesystem-0.0.29.tgz",
+      "integrity": "sha512-85/1KfRedmfPGsbK8YzeaQUyV1FQAvMPMTuWFQ5EkLd2w7szhNO96bk3Rh/SKmOfd9co2rCLf0Voy4o7ECBOvw==",
+      "requires": {
+        "@types/filewriter": "*"
+      }
+    },
+    "@types/filewriter": {
+      "version": "0.0.28",
+      "resolved": "https://registry.npmjs.org/@types/filewriter/-/filewriter-0.0.28.tgz",
+      "integrity": "sha1-wFTor02d11205jq8dviFFocU1LM="
+    },
+    "@types/firefox-webext-browser": {
+      "version": "70.0.1",
+      "resolved": "https://registry.npmjs.org/@types/firefox-webext-browser/-/firefox-webext-browser-70.0.1.tgz",
+      "integrity": "sha512-hjHsTR9vKs+yikWbNS/s7TVCx15M/MEn+VYx47wtT/W/wORsIZDD75gfUfP7lkzi+IxRvKMQBB/5/wMFlfgvgQ=="
+    },
+    "@types/har-format": {
+      "version": "1.2.4",
+      "resolved": "https://registry.npmjs.org/@types/har-format/-/har-format-1.2.4.tgz",
+      "integrity": "sha512-iUxzm1meBm3stxUMzRqgOVHjj4Kgpgu5w9fm4X7kPRfSgVRzythsucEN7/jtOo8SQzm+HfcxWWzJS0mJDH/3DQ=="
+    },
    "@types/node": {
      "version": "14.0.5",
      "resolved": "https://registry.npmjs.org/@types/node/-/node-14.0.5.tgz",
@ -541,6 +659,11 @@
      "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz",
      "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw=="
    },
+    "node-fetch": {
+      "version": "2.6.0",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz",
+      "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA=="
+    },
    "on-finished": {
      "version": "2.3.0",
      "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
@ -679,6 +802,33 @@
        }
      }
    },
+    "puppeteer-extra-plugin-adblocker": {
+      "version": "2.11.3",
+      "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-adblocker/-/puppeteer-extra-plugin-adblocker-2.11.3.tgz",
+      "integrity": "sha512-9upM0w1iUJ8Q8YoFveGVpQzZBCvB1EQkwz4WHYOEhMjCmI5lkAcxxwVnUQTIbxRS4kDLTK7M17d5o+YKRpfkog==",
+      "requires": {
+        "@cliqz/adblocker-puppeteer": "^1.4.0",
+        "@types/chrome": "0.0.91",
+        "debug": "^4.1.1",
+        "node-fetch": "^2.6.0",
+        "puppeteer-extra-plugin": "^3.1.4"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "4.1.1",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
+          "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
+          "requires": {
+            "ms": "^2.1.1"
+          }
+        },
+        "ms": {
+          "version": "2.1.2",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+          "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
+        }
+      }
+    },
    "puppeteer-extra-plugin-stealth": {
      "version": "2.4.9",
      "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-stealth/-/puppeteer-extra-plugin-stealth-2.4.9.tgz",
@ -805,6 +955,19 @@
      "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
      "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
    },
+    "tldts-core": {
+      "version": "5.6.25",
+      "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-5.6.25.tgz",
+      "integrity": "sha512-RejSlOmLrMofDgr94b7dqhzCmjwjaT+jzu5AEcuz+HhFVu+HX7s2/6EL1TaoHBSP4lF7Bby7XnyWDaOTNBMJqQ=="
+    },
+    "tldts-experimental": {
+      "version": "5.6.25",
+      "resolved": "https://registry.npmjs.org/tldts-experimental/-/tldts-experimental-5.6.25.tgz",
+      "integrity": "sha512-rBHoYaTyjMGrszpz6vm53+7UxWW1FEXJTKDvp38GxYPgYGcX6OT48N3vga6oxddlNz8q6u7MlnE3tthQ2mAXLQ==",
+      "requires": {
+        "tldts-core": "^5.6.25"
+      }
+    },
    "toidentifier": {
      "version": "1.0.0",
      "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
--- a/package.json
+++ b/package.json
@ -14,6 +14,7 @@
    "koa": "^2.12.0",
    "puppeteer": "^3.1.0",
    "puppeteer-extra": "^3.1.9",
+    "puppeteer-extra-plugin-adblocker": "^2.11.3",
    "puppeteer-extra-plugin-stealth": "^2.4.9"
  }
 }