[remark-images-download] Add a user-agent to requests (#507)

zestedesavoir · Mar 31, 2024 · 2b7798d · 2b7798d
1 parent b1888a9
commit 2b7798d
Show file tree

Hide file tree

Showing 4 changed files with 33 additions and 5 deletions.
diff --git a/packages/remark-images-download/README.md b/packages/remark-images-download/README.md
@@ -107,7 +107,7 @@ All options are optional.
 
   If not provided, local images will not end up in `downloadDestination`.
 
-## example
+## Example
 
 ```markdown
 Two small images:

diff --git a/packages/remark-images-download/__mock__/server.js b/packages/remark-images-download/__mock__/server.js
@@ -12,6 +12,16 @@ fd.on('data', (data) => {
   }
 })
 
+app.use((req, res, next) => {
+  /* Some websites (like Wikimedia) may refuse requests without User-Agent
+   * header, so mimic their behaviour: */
+  if (!req.get('User-Agent')) {
+    res.writeHead(403)
+    res.end()
+  }
+  next()
+})
+
 const bloat = Buffer.alloc(10 * 1024)
 
 app.get('/stream-bomb', (req, res) => {

diff --git a/packages/remark-images-download/dist/index.js b/packages/remark-images-download/dist/index.js
@@ -272,11 +272,19 @@ function plugin() {
   // Rejects with an error if headers are invalid.
   var initDownload = function initDownload(url) {
     return new Promise(function (resolve, reject) {
+      var packageInfo = require('../package.json');
+
       var parsedUrl = new URL(url);
       var proto = parsedUrl.protocol === 'https:' ? https : http;
-      var req = proto.get(parsedUrl, {
-        timeout: httpRequestTimeout
-      }, function (res) {
+      var reqOptions = {
+        timeout: httpRequestTimeout,
+        // Websites may refuse connection if there is no User-Agent
+        // (see for instance https://meta.wikimedia.org/wiki/User-Agent_policy)
+        headers: {
+          'User-Agent': "".concat(packageInfo.name, " bot/").concat(packageInfo.version, " (").concat(packageInfo.repository.url, ")")
+        }
+      };
+      var req = proto.get(parsedUrl, reqOptions, function (res) {
         var headers = res.headers,
             statusCode = res.statusCode;
         var error;

diff --git a/packages/remark-images-download/src/index.js b/packages/remark-images-download/src/index.js
@@ -172,10 +172,20 @@ function plugin ({
   // Rejects with an error if headers are invalid.
   const initDownload = url =>
     new Promise((resolve, reject) => {
+      const packageInfo = require('../package.json')
       const parsedUrl = new URL(url)
       const proto = parsedUrl.protocol === 'https:' ? https : http
+      const reqOptions = {
+        timeout: httpRequestTimeout,
+        // Websites may refuse connection if there is no User-Agent
+        // (see for instance https://meta.wikimedia.org/wiki/User-Agent_policy)
+        headers: {
+          'User-Agent': `${packageInfo.name} bot/${packageInfo.version} (${
+            packageInfo.repository.url})`,
+        },
+      }
 
-      const req = proto.get(parsedUrl, {timeout: httpRequestTimeout}, res => {
+      const req = proto.get(parsedUrl, reqOptions, res => {
         const {headers, statusCode} = res
         let error