Add support to slideshare.net

README.md

@@ -2,11 +2,11 @@
 [![License: GPLv3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
 
 ## About ##
-Scribd-dl helps downloading documents on [scribd.com](https://www.scribd.com/) without membership / sign-in.  
+Scribd-dl helps downloading documents on [scribd.com](https://www.scribd.com/) and [slideshare.net](https://www.slideshare.net/) without membership / sign-in.  
 
 ## Development Plan ##
-Scribd obfuscates the .pdf files, the texts copied from the documents will become strange garbled message.  
-De-obfuscating will be the next stage.
+Scribd obfuscates the .pdf files, the texts copied from the documents might become strange garbled message.  
+De-obfuscating will be the next stage of this project.
 
 ## Prerequisites ##
 Please make sure the following tool(s) / application(s) are properly setup and ready to use:
@@ -24,6 +24,7 @@ npm install
 ```
 
 ## Configuration ##
+Configuration can be altered in `config.ini`.  
 ```ini
 [SCRIBD]
 rendertime=100
@@ -31,27 +32,36 @@ rendertime=100
 [DIRECTORY]
 output=output
 ```
-Configuration can be altered in `config.ini`.  
-`rendertime` is the waiting time in millisecond for single page rendering, it is only applicable for `default` mode. (too short might cause missing images)  
+`rendertime` is the waiting time in millisecond for single page rendering in [scribd.com](https://www.scribd.com/), it is only applicable for `default` mode.  
 `output` is the ouput directory for generated .pdf files.
 
 ## Usage (CLI) ##
 ```console
 Usage: npm start [options] url
 Options:  
-  /d            default: generated by chromium's print function
-  /i        image-based: generated by image snapshots taken for pages
+  /i        image-based: generated by image snapshots taken for pages in scribd.com
 ```
 
 #### Example 1: Download 《The Minds of Billy Milligan》 ####
 ```console
-npm start https://www.scribd.com/doc/249398282/The-Minds-of-Billy-Milligan-Daniel-Keyes
+npm start "https://www.scribd.com/doc/249398282/The-Minds-of-Billy-Milligan-Daniel-Keyes"
+```
+
+#### Example 2: Download 《The Minds of Billy Milligan》 using `image-based` method ####
+```console
+npm start /i "https://www.scribd.com/doc/249398282/The-Minds-of-Billy-Milligan-Daniel-Keyes"
 ```
 
-#### Example 2: Download 《The Minds of Billy Milligan》 using image-based method####
+#### Example 3: Download 《Everything You Need To Know About ChatGPT》 ####
 ```console
-npm start /i https://www.scribd.com/doc/249398282/The-Minds-of-Billy-Milligan-Daniel-Keyes
+npm start "https://www.slideshare.net/slideshow/everything-you-need-to-know-about-chatgpt-8ba3/266783915"
 ```
 
+## Support URL Format ##
+- https://www.scribd.com/doc/**
+- https://www.scribd.com/embeds/**
+- https://www.slideshare.net/**
+- https://www.slideshare.net/slideshow/**
+
 ## License ##
 [GNU GPL v3.0](LICENSE.md)

package-lock.json

@@ -9,6 +9,7 @@
       "version": "1.0.0",
       "license": "GPL-3.0-or-later",
       "dependencies": {
+        "axios": "^1.6.8",
         "cli-progress": "^3.12.0",
         "ini": "^4.1.2",
         "pdfkit": "^0.15.0",
@@ -1986,14 +1987,6 @@
         "@sinonjs/commons": "^3.0.0"
       }
     },
-    "node_modules/@swc/helpers": {
-      "version": "0.3.17",
-      "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.3.17.tgz",
-      "integrity": "sha512-tb7Iu+oZ+zWJZ3HJqwx8oNwSDIU440hmVMDPhpACWQWnrZHK99Bxs70gT1L2dnr5Hg50ZRWEFkQCAnOVVV0z1Q==",
-      "dependencies": {
-        "tslib": "^2.4.0"
-      }
-    },
     "node_modules/@types/babel__core": {
       "version": "7.20.5",
       "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz",
@@ -2222,6 +2215,11 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/asynckit": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
+    },
     "node_modules/available-typed-arrays": {
       "version": "1.0.7",
       "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz",
@@ -2236,6 +2234,16 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/axios": {
+      "version": "1.6.8",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.8.tgz",
+      "integrity": "sha512-v/ZHtJDU39mDpyBoFVkETcd/uNdxrWRrg3bKpOKzXFA6Bvqopts6ALSMU3y6ijYxbw2B+wPrIv46egTzJXCLGQ==",
+      "dependencies": {
+        "follow-redirects": "^1.15.6",
+        "form-data": "^4.0.0",
+        "proxy-from-env": "^1.1.0"
+      }
+    },
     "node_modules/babel-jest": {
       "version": "29.7.0",
       "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz",
@@ -2777,6 +2785,17 @@
       "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
       "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
     },
+    "node_modules/combined-stream": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
+      "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
+      "dependencies": {
+        "delayed-stream": "~1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
     "node_modules/concat-map": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@@ -3077,6 +3096,14 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/delayed-stream": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
+      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
+      "engines": {
+        "node": ">=0.4.0"
+      }
+    },
     "node_modules/detect-libc": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz",
@@ -3664,20 +3691,23 @@
       "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==",
       "dev": true
     },
-    "node_modules/fontkit": {
-      "version": "1.9.0",
-      "resolved": "https://registry.npmjs.org/fontkit/-/fontkit-1.9.0.tgz",
-      "integrity": "sha512-HkW/8Lrk8jl18kzQHvAw9aTHe1cqsyx5sDnxncx652+CIfhawokEPkeM3BoIC+z/Xv7a0yMr0f3pRRwhGH455g==",
-      "dependencies": {
-        "@swc/helpers": "^0.3.13",
-        "brotli": "^1.3.2",
-        "clone": "^2.1.2",
-        "deep-equal": "^2.0.5",
-        "dfa": "^1.2.0",
-        "restructure": "^2.0.1",
-        "tiny-inflate": "^1.0.3",
-        "unicode-properties": "^1.3.1",
-        "unicode-trie": "^2.0.0"
+    "node_modules/follow-redirects": {
+      "version": "1.15.6",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz",
+      "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==",
+      "funding": [
+        {
+          "type": "individual",
+          "url": "https://github.com/sponsors/RubenVerborgh"
+        }
+      ],
+      "engines": {
+        "node": ">=4.0"
+      },
+      "peerDependenciesMeta": {
+        "debug": {
+          "optional": true
+        }
       }
     },
     "node_modules/for-each": {
@@ -3688,6 +3718,19 @@
         "is-callable": "^1.1.3"
       }
     },
+    "node_modules/form-data": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
+      "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+      "dependencies": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.8",
+        "mime-types": "^2.1.12"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
     "node_modules/fs-constants": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
@@ -6115,6 +6158,25 @@
         "node": ">=8.6"
       }
     },
+    "node_modules/mime-db": {
+      "version": "1.52.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
+      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/mime-types": {
+      "version": "2.1.35",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
+      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+      "dependencies": {
+        "mime-db": "1.52.0"
+      },
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
     "node_modules/mimic-fn": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz",
@@ -6434,6 +6496,35 @@
         "png-js": "^1.0.0"
       }
     },
+    "node_modules/pdfkit/node_modules/@swc/helpers": {
+      "version": "0.3.17",
+      "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.3.17.tgz",
+      "integrity": "sha512-tb7Iu+oZ+zWJZ3HJqwx8oNwSDIU440hmVMDPhpACWQWnrZHK99Bxs70gT1L2dnr5Hg50ZRWEFkQCAnOVVV0z1Q==",
+      "dependencies": {
+        "tslib": "^2.4.0"
+      }
+    },
+    "node_modules/pdfkit/node_modules/fontkit": {
+      "version": "1.9.0",
+      "resolved": "https://registry.npmjs.org/fontkit/-/fontkit-1.9.0.tgz",
+      "integrity": "sha512-HkW/8Lrk8jl18kzQHvAw9aTHe1cqsyx5sDnxncx652+CIfhawokEPkeM3BoIC+z/Xv7a0yMr0f3pRRwhGH455g==",
+      "dependencies": {
+        "@swc/helpers": "^0.3.13",
+        "brotli": "^1.3.2",
+        "clone": "^2.1.2",
+        "deep-equal": "^2.0.5",
+        "dfa": "^1.2.0",
+        "restructure": "^2.0.1",
+        "tiny-inflate": "^1.0.3",
+        "unicode-properties": "^1.3.1",
+        "unicode-trie": "^2.0.0"
+      }
+    },
+    "node_modules/pdfkit/node_modules/restructure": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/restructure/-/restructure-2.0.1.tgz",
+      "integrity": "sha512-e0dOpjm5DseomnXx2M5lpdZ5zoHqF1+bqdMJUohoYVVQa7cBdnk7fdmeI6byNWP/kiME72EeTiSypTCVnpLiDg=="
+    },
     "node_modules/pend": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
@@ -6748,11 +6839,6 @@
         "node": ">=10"
       }
     },
-    "node_modules/restructure": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/restructure/-/restructure-2.0.1.tgz",
-      "integrity": "sha512-e0dOpjm5DseomnXx2M5lpdZ5zoHqF1+bqdMJUohoYVVQa7cBdnk7fdmeI6byNWP/kiME72EeTiSypTCVnpLiDg=="
-    },
     "node_modules/reusify": {
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz",

package.json

@@ -17,6 +17,7 @@
   },
   "license": "GPL-3.0-or-later",
   "dependencies": {
+    "axios": "^1.6.8",
     "cli-progress": "^3.12.0",
     "ini": "^4.1.2",
     "pdfkit": "^0.15.0",

run.js

@@ -18,7 +18,6 @@ if (process.argv.length >= 3) {
     console.error(`
 Usage: npm start [options] url
 Options:  
-  /d            default: generated by chromium's print function
   /i        image-based: generated by image snapshots taken for pages
     `)
 }

src/App.js

@@ -1,5 +1,7 @@
 import { scribdDownloader } from "./service/ScribdDownloader.js"
+import { slideshareDownloader } from "./service/SlideshareDownloader.js"
 import * as scribdRegex from "./const/ScribdRegex.js"
+import * as slideshareRegex from "./const/SlideshareRegex.js"
 
 class App {
     constructor() {
@@ -12,6 +14,8 @@ class App {
     async execute(url, flag) {
         if (url.match(scribdRegex.DOMAIN)) {
             await scribdDownloader.execute(url, flag)
+        } else if (url.match(slideshareRegex.DOMAIN)) {
+            await slideshareDownloader.execute(url, flag)
         } else {
             throw new Error(`Unsupported URL: ${url}`)
         }

src/const/SlideshareRegex.js

@@ -0,0 +1,6 @@
+const DOMAIN = /^https:\/\/www\.slideshare\.net/
+const SLIDESHOW = /^https:\/\/www\.slideshare\.net\/slideshow\/([a-zA-z0-9_-]+)\/([0-9]+)/
+const PPT = /^https:\/\/www\.slideshare\.net\/[a-zA-z0-9_-]+\/([a-zA-z0-9_-]+)/
+const CDN = /(https:\/\/image\.slidesharecdn\.com\/[a-zA-z0-9_-]+\/[0-9]+\/[a-zA-z0-9_-]+-)[0-9]+-([0-9]+)(.[a-zA-Z]+)\ ([0-9]+)w/g
+
+export { DOMAIN, SLIDESHOW, PPT, CDN }

src/service/SlideshareDownloader.js

@@ -0,0 +1,99 @@
+import cliProgress from "cli-progress"
+import { puppeteerSg } from "../utils/request/PuppeteerSg.js";
+import { pdfGenerator } from "../utils/io/PdfGenerator.js";
+import { configLoader } from "../utils/io/ConfigLoader.js";
+import { directoryIo } from "../utils/io/DirectoryIo.js"
+import * as slideshareRegex from "../const/SlideshareRegex.js"
+import { Image } from "../object/Image.js"
+import sharp from "sharp";
+import axios from "axios";
+import fs from "fs"
+
+
+const output = configLoader.load("DIRECTORY", "output")
+
+class SlideshareDownloader {
+    constructor() {
+        if (!SlideshareDownloader.instance) {
+            SlideshareDownloader.instance = this
+        }
+        return SlideshareDownloader.instance
+    }
+
+    async execute(url) {
+        if (url.match(slideshareRegex.SLIDESHOW)) {
+            await this.slideshow(url, slideshareRegex.SLIDESHOW.exec(url)[1])
+        } else if (url.match(slideshareRegex.PPT)) {
+            await this.slideshow(url, slideshareRegex.PPT.exec(url)[1])
+        } else {
+            throw new Error(`Unsupported URL: ${url}`)
+        }
+    }
+
+    async slideshow(url, id) {
+        // prepare temp dir
+        let dir = `${output}/${id}`
+        await directoryIo.create(dir)
+
+        // navigate to scribd
+        let page = await puppeteerSg.getPage(url)
+
+        // wait rendering
+        await new Promise(resolve => setTimeout(resolve, 1000))
+
+        // get the page number
+        let span = await page.$("span[data-cy='page-number']")
+        let pageNumber = parseInt((await span.evaluate((el) => el.textContent)).split("of")[1])
+
+        // get the highest resolution
+        let image0 = await page.$("img#slide-image-0")
+        let srcset0 = await image0.evaluate((el) => el["srcset"])
+        let prefix = ""
+        let suffix = ""
+        let resolution = -1
+        let matches
+        while ((matches = slideshareRegex.CDN.exec(srcset0)) != null) {
+            if (resolution < parseInt(matches[4])) {
+                prefix = matches[1]
+                suffix = matches[3]
+                resolution = parseInt(matches[4])
+            }
+        }
+
+        // download images
+        let images = []
+        const bar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
+        bar.start(pageNumber, 0);
+        for (let i = 0; i < pageNumber; i++) {
+            let path = `${dir}/${(i + 1).toString().padStart(4, 0)}.png`
+
+            // convert the webp (even it shows jpg) to png
+            const resp = await axios.get(
+                `${prefix}${i + 1}-${resolution}${suffix}`,
+                { responseType: 'arraybuffer' }
+            )
+            const imageBuffer = await sharp(resp.data).toFormat('png').toBuffer();
+            fs.writeFileSync(path, Buffer.from(imageBuffer, 'binary'))
+
+            let metadata = await sharp(path).metadata()
+            images.push(new Image(
+                path,
+                metadata.width,
+                metadata.height
+            ))
+            bar.update(i + 1);
+        }
+        bar.stop();
+
+        // generate pdf
+        await pdfGenerator.generate(images, `${output}/${id}.pdf`)
+
+        // remove temp dir
+        directoryIo.remove(`${output}/${id}`)
+
+        await page.close()
+        await puppeteerSg.close()
+    }
+}
+
+export const slideshareDownloader = new SlideshareDownloader()