deno port

Signed-off-by: Niv Sardi <[email protected]>

Dockerfile.deno

@@ -0,0 +1,8 @@
+FROM docker.io/denoland/deno
+MAINTAINER Niv Sardi <[email protected]>
+WORKDIR /app
+
+COPY src-deno ./src
+RUN deno cache ./src/index.ts
+
+CMD deno ./src/index.ts

docker-compose.yaml

@@ -0,0 +1,31 @@
+version: "3.9"  # optional since v1.27.0
+services:
+  puppet:
+    build:
+      dockerfile: Dockerfile.deno
+      context: .
+    links:
+      - browserless
+    environment:
+      BROWSERLESS_HOST: browserless
+      BROWSERLESS_PORT: 3000
+      DEBUG: "puppet"
+    depends_on:
+      - "browserless"
+    command: "sh -c 'while echo deno; do sleep 3h; done'" #"deno run --allow-net --allow-env --allow-read --allow-write src/index.ts"
+    volumes:
+      - "./src-deno:/app/src:z"
+      - "./data:/app/data:z"
+    #restart: unless-stopped:600
+    deploy:
+      restart_policy:
+        condition: any
+        delay: 600s
+        window: 300s
+
+  browserless:
+     image: docker.io/zenika/alpine-chrome
+     entrypoint: ["sh", "-c", "while true; do chromium-browser --headless --use-gl=swiftshader --disable-software-rasterizer --disable-dev-shm-usage --no-sandbox --remote-debugging-address=0.0.0.0 --remote-debugging-port=3000; sleep 2; done"]
+     port:
+       - "3000:3000"
+

src/csv.test.ts

@@ -0,0 +1,16 @@
+import {
+  assertEquals,
+  assertObjectMatch
+} from "https://deno.land/[email protected]/testing/asserts.ts";
+import * as CSV from './csv.ts';
+
+Deno.test("ParseLine", () => {
+  assertEquals(CSV.parseLine('"test", "test, with", without'), ['test', 'test, with', 'without'])
+})
+Deno.test("ParseCSV", () => {
+  const res: object[] = []
+  const expected = { test: 'hello', case: 'world' }
+  CSV.parse('test,case\nhello,world', e => res.push(e))
+  assertObjectMatch(res[0], expected)
+  assertEquals(res, [expected])
+})

src/csv.ts

@@ -0,0 +1,28 @@
+export function parseLine(l: string) {
+  const res = l.match(/((?:\s+"[^"]+")|(?:[^,"]+))/g) || [];
+  for (let i = 0; i < res.length; i++) {
+    res[i] = res[i].replace(/^\s+/, '').replace(/^"/, '').replace(/[\r\n"]+$/, '')
+  }
+  return res;
+}
+
+export function parse(t: string, cb: (o: object) => void) {
+  const lines = t.split('\n');
+  const header = parseLine(lines[0]);
+  for (let i = 1; i < lines.length; i++) {
+    if (!lines[i].length) {
+      continue;
+    }
+    const l = parseLine(lines[i]) || []
+
+    if (l.length < header.length) {
+      console.error(`couldn't parse '${lines[i]}' yielded '${l}' of length ${l.length} expected ${header.length}: ${header}`);
+      return null;
+    }
+    const e = { [header[0]]: l[0] };
+    for (let j = 1; j < header.length; j++) {
+      e[`${header[j]}`] = l[j];
+    }
+    cb(e)
+  }
+}

src/img.ts

@@ -0,0 +1 @@
+import * as opencv from "https://deno.land/x/[email protected]/mod.ts";

src/index.ts

@@ -0,0 +1,71 @@
+import PQueue from "https://deno.land/x/[email protected]/mod.ts"
+
+import * as CSV from './csv.ts';
+import Puppet from './puppet.ts';
+import selectors from './selectors.ts';
+
+const puppet = new Puppet();
+const queue = new PQueue({
+  concurrency: 10,
+  timeout: 60000
+})
+let count = 0
+queue.addEventListener("active", () =>
+  console.log(`Working on item #${++count}.  Size: ${queue.size}  Pending: ${queue.pending}`))
+queue.addEventListener("next", () =>
+  console.log(`task finished, Size: ${queue.size} Pending: ${queue.pending}`))
+const statInterval = setInterval(() =>
+  console.log(`Size: ${queue.size}  Pending: ${queue.pending}`), 1000);
+
+queue.addEventListener("idle", async () => {
+  clearInterval(statInterval)
+  await puppet.close()
+  console.log("all done")
+})
+
+function process(o: { url: string, bco: string, name: string }): Promise<void> {
+  const promises: Promise<void>[] = [];
+
+  return puppet.run(async page => {
+    const url = o.url.replace('http:', 'https:');
+    promises.push(new Promise<void>((accept, _reject) => {
+      page.once('load', async () => {
+        try {
+          const logos = await page.$$(selectors.logo);
+          let annotations = '';
+          for (const i in logos) {
+            const bb = await logos[i].boundingBox();
+            if (!bb) continue;
+
+            try {
+              await logos[i].screenshot({ path: `./data/logos/${o.bco}.logo${i}.png` })
+              annotations +=
+                `${o.bco} ${bb.x + bb.width / 2} ${bb.y + bb.height / 2} ${bb.width} ${bb.height}\n`
+            } catch (e) {
+              console.error(`couldn't screenshot logo: ${e}`);
+            }
+          }
+          if (logos.length) {
+            await Deno.writeTextFile(`./data/${o.bco}.chrome.full.txt`, annotations);
+          }
+          await page.screenshot({ path: `./data/${o.bco}.chrome.full.png`, fullPage: true })
+          console.log(`screenshot ok for ${o.name}`);
+        } catch (err) {
+          console.error(`error in screenshot: ${err}`);
+        }
+        accept()
+      })
+    }))
+
+    try {
+      await page.goto(url)
+        .catch(() => page.goto(o.url))
+    } catch (e) {
+      console.error(`got error: ${e}`);
+    }
+    await Promise.all(promises);
+  })
+}
+
+const text = await Deno.readTextFile("./data/entidades.csv");
+CSV.parse(text, o => queue.add(() => process(o)))

src/puppet.test.ts

@@ -0,0 +1,9 @@
+import { assertEquals } from "https://deno.land/[email protected]/testing/asserts.ts";
+import Puppet from './puppet.ts'
+
+Deno.test("Puppet", async () => {
+  const P = new Puppet()
+  await P.connect()
+  await P.run(page => page.goto("https://google.com"))
+  await P.close()
+})

src/puppet.ts

@@ -0,0 +1,108 @@
+import Puppeteer from "https://deno.land/x/[email protected]/mod.ts";
+import EventEmitter from "https://deno.land/x/[email protected]/mod.ts";
+import type { Browser, Page } from "https://deno.land/x/[email protected]/mod.ts";
+
+const BROWSER_SIGNALS = [
+    'disconnected',
+    'targetchanged',
+    'targetcreated',
+    'targetdestroyed'
+];
+
+const CHROME_ARGS = [
+    '--no-sandbox',
+    '--disable-setuid-sandbox'
+];
+
+async function resolve(a: string) {
+    if (a.match(/(\d.?){4}/)) {
+        return a;
+    }
+    return await Deno.resolveDns(a, "A");
+}
+
+export default class Runner extends EventEmitter {
+    config: {
+        BROWSERLESS_HOST: string;
+        BROWSERLESS_PORT: string;
+    };
+    target: string;
+    browser: Browser | undefined;
+    connected: Promise<boolean> | undefined;
+
+    constructor(config = {
+        BROWSERLESS_HOST: Deno.env.get("BROWSERLESS_HOST") || "localhost",
+        BROWSERLESS_PORT: Deno.env.get("BROWSERLESS_PORT") || "3000",
+    }) {
+        super();
+        this.target = `ws://${config.BROWSERLESS_HOST}:${config.BROWSERLESS_PORT}`;
+        this.config = config;
+        this.connected
+    }
+    public async close() {
+        try {
+            if (this.browser) await this.browser.close();
+        } catch (err) {
+            console.error(`${err} on close`)
+        }
+    }
+    async connect() {
+        if (!this.connected)
+            this.connected = this._connect()
+        return this.connected
+    }
+    async _connect() {
+        try {
+            const host = await resolve(this.config.BROWSERLESS_HOST);
+            const ver = await fetch(`http://${host}:${this.config.BROWSERLESS_PORT}/json/version`)
+                .then(async res => await res.json())
+            this.target = ver.webSocketDebuggerUrl;
+            this.browser = this.browser || await Puppeteer.connect({
+                browserWSEndpoint: this.target
+            }).catch(() => {
+                console.error(`
+⚠ COULD NOT CONNECT TO BROWSERLESS
+🦄 will try to spawn a chromedriver instance for you to debug`)
+                return Puppeteer.launch({
+                    args: CHROME_ARGS,
+                    headless: false
+                })
+            });
+
+            if (!this.browser) {
+                console.error("couldn't init Browser");
+                return false;
+            }
+            BROWSER_SIGNALS.map(e => this.browser?.on(e, d => this.emit(`browser:${e}`, d)))
+            this.browser.on('error', e => console.error(`got browser error: ${e}`))
+
+            const pages = await this.browser.pages();
+            for (let p in pages) {
+                await pages[p].close();
+            }
+            this.emit("ready")
+        } catch (e) {
+            console.error(e);
+        }
+        return true;
+    }
+
+    public async run(fn: (page: Page) => void) {
+        await this.connect();
+
+        if (!this.browser) {
+            return;
+        }
+        try {
+            const page = await this.browser.newPage()
+            if (!page) {
+                return;
+            }
+            const ret = await fn(page)
+            await page.close()
+            return ret
+        } catch (e) {
+            return
+        }
+    }
+}

src/selectors.ts

@@ -0,0 +1,6 @@
+export default {
+  "logo": "img[src*=logo]",
+  "logosbancos": "img[src*=logosbancos]",
+  "entity_http": "p.post-pagina-interior a[target=_blank][href*=http]",
+  "entity_mailto": "p.post-pagina-interior a[target=_blank][href*=mailto]"
+}