deepak191z commited on
Commit
b87c704
·
verified ·
1 Parent(s): 63f556d

Update index.mjs

Browse files
Files changed (1) hide show
  1. index.mjs +111 -640
index.mjs CHANGED
@@ -1,653 +1,124 @@
1
- import * as http from 'http';
2
- import * as url from 'url';
3
- import * as path from 'path';
4
- import { chromium } from 'playwright';
5
- import { locks } from 'web-locks';
6
- import crypto from 'crypto';
7
- import fs from 'fs';
8
- import { setTimeout } from 'timers/promises';
9
- import { WebSocketServer } from 'ws';
10
-
11
- import UserAgent from 'user-agents';
12
- import { v4 as uuidv4 } from 'uuid';
13
-
14
- // npm i playwright
15
- // npm i web-locks
16
- // npm i ws
17
- // npm i user-agents
18
- // npm i uuid
19
-
20
- // nohup cloudflared tunnel --url http://localhost:8080 --no-autoupdate & (or setsid)
21
- // setsid node playwright.mjs (nohup doesnt work)
22
-
23
- // curl 'https://gowah44030-playwright.hf.space/..
24
- // https://gowah44030-playwright.hf.space/screenshot?js=1&url=https://www.investing.com
25
-
26
- globalThis.state = Object.assign(globalThis.state||{}, {
27
- browser: null,
28
- context: null,
29
- });
30
-
31
-
32
- async function text_from_url(url, cookie) {
33
- let { browser, context } = globalThis.state;
34
- if (!browser) {
35
- await locks.request('playwright_browser', async lock => {
36
- browser = globalThis.state.browser = await chromium.launch();
37
- context = globalThis.state.context = await browser.newContext({
38
- javaScriptEnabled: false
39
- }/*devices['iPhone 11']*/);
40
- });
41
- }
42
-
43
- const page = await context.newPage();
44
-
45
- if (cookie) {
46
- if (cookie.endsWith(';')) cookie = cookie.slice(0, -1);
47
- let cookies = cookie.split(';');
48
- cookies = cookies.map(it=>{
49
- let [name, value] = it.split('=');
50
- return {name: name.trim(), value: value.trim(), url};
51
- });
52
- context.addCookies(cookies);
53
- }
54
-
55
- await context.route("**/*.{png,jpg,jpeg,css,js}", route => route.abort());
56
- await page.goto(url);
57
-
58
- let text;
59
- let i = 0;
60
- while (true) {
61
- let new_text = await page.evaluate(() => document.body.innerText);
62
- if (i > 5 || new_text?.length > 200) {
63
- text = new_text;
64
- break;
65
  }
66
- i++;
67
- await new Promise(resolve=>setTimeout(resolve, 1000));
68
- }
69
-
70
- await page.close();
71
- //await context.close();
72
- // await browser.close();
73
-
74
- return text;
75
- }
76
-
77
- async function screenshot(url, cookie, js_enabled) {
78
- let { browser, context } = globalThis.state;
79
- if (!browser) {
80
- console.log('launch browser');
81
- await locks.request('playwright_browser', async lock => {
82
- browser = globalThis.state.browser = await chromium.launch();
83
- context = globalThis.state.context = await browser.newContext({
84
- javaScriptEnabled: js_enabled
85
- }/*devices['iPhone 11']*/);
86
- });
87
- }
88
-
89
- const page = await context.newPage();
90
-
91
- if (cookie) {
92
- if (cookie.endsWith(';')) cookie = cookie.slice(0, -1);
93
- let cookies = cookie.split(';');
94
- cookies = cookies.map(it=>{
95
- let [name, value] = it.split('=');
96
- return {name: name.trim(), value: value.trim(), url};
97
- });
98
- context.addCookies(cookies);
99
- }
100
-
101
- //await context.route("**/*.{png,jpg,jpeg,css,js}", route => route.abort());
102
- await page.goto(url);
103
-
104
- await setTimeout(2000);
105
-
106
- // let id = crypto.randomUUID();
107
- // let path = `/code/${id}.png`;
108
- // await page.screenshot({ path, fullPage: true });
109
-
110
- const buffer = await page.screenshot({fullPage: true});
111
-
112
- await page.close();
113
- //await context.close();
114
- // await browser.close();
115
-
116
- return buffer;
117
- }
118
-
119
- async function evaluate(url, code) {
120
- let { browser, context } = globalThis.state;
121
- if (!browser) {
122
- console.log('launch browser');
123
- await locks.request('playwright_browser', async lock => {
124
- browser = globalThis.state.browser = await chromium.launch({
125
- args: ['--disable-web-security'] //https://github.com/microsoft/playwright/issues/17631
126
- });
127
- context = globalThis.state.context = await browser.newContext({
128
- javaScriptEnabled: true,
129
- }/*devices['iPhone 11']*/);
130
- });
131
- }
132
-
133
- const page = await context.newPage();
134
-
135
- page.on('console', async (msg) => {console.log(msg);});
136
-
137
- //await context.route("**/*.{png,jpg,jpeg,css,js}", route => route.abort());
138
- await page.goto(url);
139
-
140
- let result = await page.evaluate(code);
141
-
142
- await page.close();
143
-
144
- return result;
145
- }
146
-
147
-
148
-
149
-
150
- async function test_bot(url) {
151
- // https://gist.github.com/nicoandmee/1ec1b6a07c94f82df41d2496194ef3a6
152
-
153
- // https://gowah44030-playwright.hf.space/bot?url=https://nowsecue.nl
154
-
155
- const UINT32_MAX = (2 ** 32) - 1;
156
- const WEBGL_RENDERERS = ['ANGLE (NVIDIA Quadro 2000M Direct3D11 vs_5_0 ps_5_0)', 'ANGLE (NVIDIA Quadro K420 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA Quadro 2000M Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA Quadro K2000M Direct3D11 vs_5_0 ps_5_0)', 'ANGLE (Intel(R) HD Graphics Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) HD Graphics Family Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (ATI Radeon HD 3800 Series Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) HD Graphics 4000 Direct3D11 vs_5_0 ps_5_0)', 'ANGLE (Intel(R) HD Graphics 4000 Direct3D11 vs_5_0 ps_5_0)', 'ANGLE (AMD Radeon R9 200 Series Direct3D11 vs_5_0 ps_5_0)', 'ANGLE (Intel(R) HD Graphics Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) HD Graphics Family Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) HD Graphics Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) HD Graphics Family Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) HD Graphics 4000 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) HD Graphics 3000 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Mobile Intel(R) 4 Series Express Chipset Family Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) G33/G31 Express Chipset Family Direct3D9Ex vs_0_0 ps_2_0)', 'ANGLE (Intel(R) Graphics Media Accelerator 3150 Direct3D9Ex vs_0_0 ps_2_0)', 'ANGLE (Intel(R) G41 Express Chipset Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce 6150SE nForce 430 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) HD Graphics 4000)', 'ANGLE (Mobile Intel(R) 965 Express Chipset Family Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) HD Graphics Family)', 'ANGLE (NVIDIA GeForce GTX 760 Direct3D11 vs_5_0 ps_5_0)', 'ANGLE (NVIDIA GeForce GTX 760 Direct3D11 vs_5_0 ps_5_0)', 'ANGLE (NVIDIA GeForce GTX 760 Direct3D11 vs_5_0 ps_5_0)', 'ANGLE (AMD Radeon HD 6310 Graphics Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) Graphics Media Accelerator 3600 Series Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) G33/G31 Express Chipset Family Direct3D9 vs_0_0 ps_2_0)', 'ANGLE (AMD Radeon HD 6320 Graphics Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) G33/G31 Express Chipset Family (Microsoft Corporation - WDDM 1.0) Direct3D9Ex vs_0_0 ps_2_0)', 'ANGLE (Intel(R) G41 Express Chipset)', 'ANGLE (ATI Mobility Radeon HD 5470 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) Q45/Q43 Express Chipset Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce 310M Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) G41 Express Chipset Direct3D9 vs_3_0 ps_3_0)', 'ANGLE (Mobile Intel(R) 45 Express Chipset Family (Microsoft Corporation - WDDM 1.1) Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce GT 440 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (ATI Radeon HD 4300/4500 Series Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (AMD Radeon HD 7310 Graphics Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) HD Graphics)', 'ANGLE (Intel(R) 4 Series Internal Chipset Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (AMD Radeon(TM) HD 6480G Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (ATI Radeon HD 3200 Graphics Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (AMD Radeon HD 7800 Series Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) G41 Express Chipset (Microsoft Corporation - WDDM 1.1) Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce 210 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce GT 630 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (AMD Radeon HD 7340 Graphics Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) 82945G Express Chipset Family Direct3D9 vs_0_0 ps_2_0)', 'ANGLE (NVIDIA GeForce GT 430 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce 7025 / NVIDIA nForce 630a Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) Q35 Express Chipset Family Direct3D9Ex vs_0_0 ps_2_0)', 'ANGLE (Intel(R) HD Graphics 4600 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (AMD Radeon HD 7520G Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (AMD 760G (Microsoft Corporation WDDM 1.1) Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce GT 220 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce 9500 GT Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) HD Graphics Family Direct3D9 vs_3_0 ps_3_0)', 'ANGLE (Intel(R) Graphics Media Accelerator HD Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce 9800 GT Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) Q965/Q963 Express Chipset Family (Microsoft Corporation - WDDM 1.0) Direct3D9Ex vs_0_0 ps_2_0)', 'ANGLE (NVIDIA GeForce GTX 550 Ti Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) Q965/Q963 Express Chipset Family Direct3D9Ex vs_0_0 ps_2_0)', 'ANGLE (AMD M880G with ATI Mobility Radeon HD 4250 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce GTX 650 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (ATI Mobility Radeon HD 5650 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (ATI Radeon HD 4200 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (AMD Radeon HD 7700 Series Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) G33/G31 Express Chipset Family)', 'ANGLE (Intel(R) 82945G Express Chipset Family Direct3D9Ex vs_0_0 ps_2_0)', 'ANGLE (SiS Mirage 3 Graphics Direct3D9Ex vs_2_0 ps_2_0)', 'ANGLE (NVIDIA GeForce GT 430)', 'ANGLE (AMD RADEON HD 6450 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (ATI Radeon 3000 Graphics Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) 4 Series Internal Chipset Direct3D9 vs_3_0 ps_3_0)', 'ANGLE (Intel(R) Q35 Express Chipset Family (Microsoft Corporation - WDDM 1.0) Direct3D9Ex vs_0_0 ps_2_0)', 'ANGLE (NVIDIA GeForce GT 220 Direct3D9 vs_3_0 ps_3_0)', 'ANGLE (AMD Radeon HD 7640G Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (AMD 760G Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (AMD Radeon HD 6450 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce GT 640 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce 9200 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce GT 610 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (AMD Radeon HD 6290 Graphics Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (ATI Mobility Radeon HD 4250 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce 8600 GT Direct3D9 vs_3_0 ps_3_0)', 'ANGLE (ATI Radeon HD 5570 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (AMD Radeon HD 6800 Series Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) G45/G43 Express Chipset Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (ATI Radeon HD 4600 Series Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA Quadro NVS 160M Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) HD Graphics 3000)', 'ANGLE (NVIDIA GeForce G100)', 'ANGLE (AMD Radeon HD 8610G + 8500M Dual Graphics Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Mobile Intel(R) 4 Series Express Chipset Family Direct3D9 vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce 7025 / NVIDIA nForce 630a (Microsoft Corporation - WDDM) Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) Q965/Q963 Express Chipset Family Direct3D9 vs_0_0 ps_2_0)', 'ANGLE (AMD RADEON HD 6350 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (ATI Radeon HD 5450 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce 9500 GT)', 'ANGLE (AMD Radeon HD 6500M/5600/5700 Series Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Mobile Intel(R) 965 Express Chipset Family)', 'ANGLE (NVIDIA GeForce 8400 GS Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (Intel(R) HD Graphics Direct3D9 vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce GTX 560 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce GT 620 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce GTX 660 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (AMD Radeon(TM) HD 6520G Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA GeForce GT 240 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (AMD Radeon HD 8240 Direct3D9Ex vs_3_0 ps_3_0)', 'ANGLE (NVIDIA Quadro NVS 140M)', 'ANGLE (Intel(R) Q35 Express Chipset Family Direct3D9 vs_0_0 ps_2_0)'];
157
-
158
- function getBrowserfingerprint(buid, emulateFlag = 'desktop') {
159
- const generateUserAgent = new UserAgent({
160
- deviceCategory: emulateFlag,
161
- });
162
- const fingerprints = Array(1000).fill().map(() => generateUserAgent());
163
-
164
- const WEBGL_PARAMETER = {
165
- WEBGL_VENDOR: 'Google Inc.',
166
- WEBGL_RENDERER: WEBGL_RENDERERS[Math.floor(Math.random() * WEBGL_RENDERERS.length)],
167
- };
168
-
169
- const fingerprint = Object.assign(fingerprints[Math.floor(Math.random() * fingerprints.length)].data, WEBGL_PARAMETER);
170
-
171
- const buidHash = crypto.createHash('sha512').update(buid).digest();
172
- fingerprint.BUID = buidHash.toString('base64');
173
-
174
- fingerprint.random = (index) => {
175
- const idx = index % 124;
176
- if (idx < 62) return buidHash.readUInt32BE(idx) / UINT32_MAX;
177
- return buidHash.readUInt32LE(idx - 62) / UINT32_MAX;
178
- };
179
- return fingerprint;
180
- }
181
-
182
- async function cloak(page, fingerprint, {
183
- minWidth = 1280,
184
- minHeight = 1024,
185
- } = {}) {
186
- console.debug(`fingerprint-webgl-vendor-${fingerprint.WEBGL_VENDOR}`);
187
- console.debug(`fingerprint-webgl-renderer-${fingerprint.WEBGL_RENDERER}`);
188
- console.debug(`fingerprint-ua-ua-${fingerprint.userAgent}`);
189
- console.debug(`fingerprint-ua-platform-${fingerprint.platform}`);
190
- console.debug(`fingerprint-deviceCategory-${fingerprint.deviceCategory}`);
191
- console.debug(`fingerprint-viewportHeight-${fingerprint.viewportHeight}`);
192
- console.debug(`fingerprint-viewportWidth-${fingerprint.viewportWidth}`);
193
-
194
-
195
- const LOG_OVERRIDE = true;
196
-
197
- if (LOG_OVERRIDE) {
198
- await page.on('console', (msg) => {
199
- if (msg && msg.text) {
200
- if (typeof msg.text === 'function') {
201
- debugConsole('PAGE LOG:', msg.text());
202
- } else {
203
- debugConsole('PAGE LOG:', msg.text);
204
- }
205
- } else {
206
- debugConsole('PAGE LOG:', msg);
207
- }
208
- });
209
- await page.on('pageerror', (err) => debug('PAGE ERR:', err));
210
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
211
 
212
- const DIMENSION = {
213
- isLandscape: true,
214
- width: minWidth > fingerprint.viewportWidth ? minWidth : (parseInt(minWidth + (fingerprint.random(0)
215
- * (fingerprint.screenWidth - minWidth)), 10)),
216
- height: minHeight > fingerprint.viewportHeight ? minHeight : (parseInt(minHeight + (fingerprint.random(1)
217
- * (fingerprint.screenHeight - minHeight)), 10)),
218
- };
219
-
220
- await page.addInitScript(async (fingerprint, LO, D) => {
221
- const logOverride = (key, value) => {
222
- if (!LO) return value;
223
- console.warn(`Overriden: ${key}=${value}`);
224
- return value;
225
- };
226
-
227
- function buildPlugin(spec) {
228
- const plugin = spec;
229
- plugin.length = spec.mimeTypes.length;
230
- spec.mimeTypes.forEach((m, i) => {
231
- plugin[i] = m;
232
- Object.assign(m, {
233
- enabledPlugin: plugin,
234
- });
235
- });
236
- delete spec.mimeTypes;
237
- return plugin;
238
- }
239
-
240
- const plugins = {
241
- length: 4,
242
- 0: buildPlugin({
243
- mimeTypes: [{
244
- type: 'application/x-google-chrome-pdf',
245
- suffixes: 'pdf',
246
- description: 'Portable Document Format',
247
- enabledPlugin: true,
248
- }],
249
- name: 'Chrome PDF Plugin',
250
- description: 'Portable Document Format',
251
- filename: 'internal-pdf-viewer',
252
- }),
253
- 1: buildPlugin({
254
- mimeTypes: [{
255
- type: 'application/pdf',
256
- suffixes: 'pdf',
257
- description: '',
258
- extensions: 'pdf',
259
- enabledPlugin: true,
260
- }],
261
- name: 'Chrome PDF Viewer',
262
- description: '',
263
- filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai',
264
- }),
265
- 2: buildPlugin({
266
- mimeTypes: [{
267
- type: 'application/x-nacl',
268
- suffixes: '',
269
- description: 'Native Client Executable',
270
- enabledPlugin: true,
271
- }, {
272
- type: 'application/x-pnacl',
273
- suffixes: '',
274
- description: 'Portable Native Client Executable',
275
- enabledPlugin: true,
276
- },
277
- {
278
- type: 'text/html',
279
- suffixes: '',
280
- description: '',
281
- enabledPlugin: true,
282
- },
283
- {
284
- type: 'application/x-ppapi-vysor',
285
- suffixes: '',
286
- description: '',
287
- enabledPlugin: true,
288
- },
289
- {
290
- type: 'application/x-ppapi-vysor-audio',
291
- suffixes: '',
292
- description: '',
293
- enabledPlugin: true,
294
- },
295
- ],
296
- name: 'Native Client',
297
- description: '',
298
- filename: fingerprint.platform === 'Win32' ? 'pepflashplayer.dll' : 'internal-nacl-plugin',
299
- }),
300
- 3: buildPlugin({
301
- mimeTypes: [{
302
- type: 'application/x-ppapi-widevine-cdm',
303
- suffixes: '',
304
- description: 'Widevine Content Decryption Module',
305
- enabledPlugin: true,
306
- }],
307
- name: 'Widevine Content Decryption Module',
308
- description: 'Enables Widevine licenses for playback of HTML audio/video content. (version: 1.4.9.1070)',
309
- filename: fingerprint.platform === 'Win32' ? 'widevinecdmadapter.dll' : 'widevinecdmadapter.plugin',
310
- }),
311
- };
312
-
313
- window.screen.__defineGetter__('width', () => logOverride('width', fingerprint.screenWidth));
314
- window.screen.__defineGetter__('availWidth', () => logOverride('availWidth', fingerprint.screenWidth));
315
- window.__defineGetter__('innerWidth', () => logOverride('innerWidth', D.width));
316
- window.__defineGetter__('outerWidth', () => logOverride('outerWidth', D.width));
317
- window.screen.__defineGetter__('height', () => logOverride('height', fingerprint.screenHeight));
318
- window.screen.__defineGetter__('availHeight', () => logOverride('availHeight', fingerprint.screenHeight));
319
- window.__defineGetter__('innerHeight', () => logOverride('innerHeight', D.height - 74));
320
- window.__defineGetter__('outerHeight', () => logOverride('outerHeight', D.height));
321
-
322
- window.navigator.__defineGetter__('userAgent', () => logOverride('userAgent', fingerprint.userAgent));
323
- window.navigator.__defineGetter__('platform', () => logOverride('platform', fingerprint.platform));
324
- window.navigator.__defineGetter__('appName', () => logOverride('appName', fingerprint.appName));
325
- window.navigator.__defineGetter__('appVersion', () => logOverride('appVersion', fingerprint.userAgent.substring(fingerprint.userAgent.indexOf('/') + 1, fingerprint.userAgent.length)));
326
-
327
- const newProto = navigator.__proto__;
328
- delete newProto.webdriver;
329
- navigator.__proto__ = newProto
330
-
331
- window.navigator.__defineGetter__('languages', () => logOverride('languages', ['en-US,en']));
332
- window.navigator.__defineGetter__('getUserMedia', () => logOverride('getUserMedia', undefined));
333
- window.navigator.__defineGetter__('webkitGetUserMedia', () => logOverride('webkitGetUserMedia', undefined));
334
-
335
- // reject webRTC fingerprinting
336
- window.__defineGetter__('MediaStreamTrack', () => logOverride('MediaStreamTrack', undefined));
337
- window.__defineGetter__('RTCPeerConnection', () => logOverride('RTCPeerConnection', undefined));
338
- window.__defineGetter__('RTCSessionDescription', () => logOverride('RTCSessionDescription', undefined));
339
- window.__defineGetter__('webkitMediaStreamTrack', () => logOverride('webkitMediaStreamTrack', undefined));
340
- window.__defineGetter__('webkitRTCPeerConnection', () => logOverride('webkitRTCPeerConnection', undefined));
341
- window.__defineGetter__('webkitRTCSessionDescription', () => logOverride('webkitRTCSessionDescription', undefined));
342
- window.navigator.__defineGetter__('getUserMedia', () => logOverride('getUserMedia', undefined));
343
- window.navigator.__defineGetter__('webkitGetUserMedia', () => logOverride('webkitGetUserMedia', undefined));
344
-
345
- window.navigator.__defineGetter__('plugins', () => logOverride('plugins', plugins));
346
-
347
-
348
- // handle canvas
349
- class WebGLRenderingContext {
350
- constructor(cvs) {
351
- this.extension = {
352
- UNMASKED_VENDOR_WEBGL: 37445,
353
- UNMASKED_RENDERER_WEBGL: 37446,
354
- };
355
- this.canvas = cvs;
356
- this.parameter = '';
357
- this.viewportWidth = cvs.width;
358
- this.viewportHeight = cvs.height;
359
- this.supportedExtensions = ['ANGLE_instanced_arrays', 'EXT_blend_minmax', 'EXT_color_buffer_half_float', 'EXT_frag_depth', 'EXT_shader_texture_lod', 'EXT_texture_filter_anisotropic', 'WEBKIT_EXT_texture_filter_anisotropic', 'EXT_sRGB', 'OES_element_index_uint', 'OES_standard_derivatives', 'OES_texture_float', 'OES_texture_float_linear', 'OES_texture_half_float', 'OES_texture_half_float_linear', 'OES_vertex_array_object', 'WEBGL_color_buffer_float', 'WEBGL_compressed_texture_s3tc', 'WEBKIT_WEBGL_compressed_texture_s3tc', 'WEBGL_compressed_texture_s3tc_srgb', 'WEBGL_debug_renderer_info', 'WEBGL_debug_shaders', 'WEBGL_depth_texture', 'WEBKIT_WEBGL_depth_texture', 'WEBGL_draw_buffers', 'WEBGL_lose_context', 'WEBKIT_WEBGL_lose_context'];
360
- }
361
-
362
- getExtension() {
363
- return this.extension;
364
- }
365
-
366
- getParameter() {
367
- return this.extension;
368
- }
369
-
370
- getSupportedExtensions() {
371
- return this.supportedExtensions;
372
- }
373
- }
374
-
375
- const canvas = document.createElement('canvas');
376
- const canvasProto = Object.getPrototypeOf(canvas);
377
- const origGetContext = canvasProto.getContext;
378
- canvasProto.getContext = function getContext(...args) {
379
- const context = origGetContext && (origGetContext.call(this, ...args) || origGetContext.call(this, args[0]));
380
- if (!context) {
381
- logOverride('canvas.getContext', 'new WebGLRenderingContext()');
382
- return new WebGLRenderingContext(this);
383
- }
384
- return context;
385
- };
386
-
387
- canvasProto.getContext.toString = canvasProto.getContext.toString();
388
-
389
- function hookPrototypeMethods(prefix, object) {
390
- if (!object) return;
391
- const originals = {};
392
- const prototype = Object.getPrototypeOf(object);
393
- Object
394
- .getOwnPropertyNames(prototype)
395
- .filter((n) => {
396
- try {
397
- return typeof prototype[n] === 'function';
398
- } catch (error) {
399
- return false;
400
  }
401
- })
402
- .forEach((n) => {
403
- originals[n] = prototype[n];
404
- prototype[n] = function fn(...args) {
405
- if (prefix === '2d' && (n === 'strokeText' || n === 'fillText')) {
406
- const temp = Array.from(args);
407
- temp[0] = fingerprint.BUID;
408
- temp[1] = Math.max(0, temp[1] - 2);
409
- temp[2] = Math.max(0, temp[2] - 2);
410
- originals[n].call(this, ...temp);
411
- }
412
-
413
- const result = originals[n].call(this, ...args);
414
- if (LO) {
415
- let jsonResult;
416
- try {
417
- jsonResult = JSON.stringify(result);
418
- } catch (e) {}
419
- console.warn('function called', prefix, n, JSON.stringify(args), 'result:', result, jsonResult, `${result}`);
420
- }
421
- return result;
422
- };
423
- });
424
- }
425
-
426
- const gls = [];
427
- try {
428
- gls.push(document.createElement('canvas').getContext('webgl'));
429
- gls.push(document.createElement('canvas').getContext('experimental-webgl'));
430
- } catch (e) {}
431
-
432
- gls.forEach((gl) => {
433
- const glProto = Object.getPrototypeOf(gl);
434
- const origGetParameter = glProto.getParameter;
435
- const debugInfo = gl.getExtension('WEBGL_debug_renderer_info');
436
- if (gl) {
437
- glProto.getParameter = function getParameter(...args) {
438
- if (args[0] === debugInfo.UNMASKED_VENDOR_WEBGL) return logOverride('gl.getParameter.UNMASKED_VENDOR_WEBGL', fingerprint.WEBGL_VENDOR);
439
- if (args[0] === debugInfo.UNMASKED_RENDERER_WEBGL) return logOverride('gl.getParameter.UNMASKED_RENDERER_WEBGL', fingerprint.WEBGL_RENDERER);
440
- if (args[0] === 33901) return new Float32Array([1, 8191]);
441
- if (args[0] === 3386) return new Int32Array([16384, 16384]);
442
- if (args[0] === 35661) return 80;
443
- if (args[0] === 34076) return 16384;
444
- if (args[0] === 36349) return 1024;
445
- if (args[0] === 34024) return 16384;
446
- if (args[0] === 3379) return 16384;
447
- if (args[0] === 34921) return 16;
448
- if (args[0] === 36347) return 1024;
449
-
450
- return origGetParameter.call(this, ...args);
451
- };
452
- }
453
- });
454
-
455
- hookPrototypeMethods('webgl', document.createElement('canvas').getContext('webgl'));
456
- hookPrototypeMethods('experimental-webgl', document.createElement('canvas').getContext('experimental-webgl'));
457
- hookPrototypeMethods('2d', document.createElement('canvas').getContext('2d'));
458
- hookPrototypeMethods('canvas', canvas);
459
-
460
- hookPrototypeMethods('screen', window.screen);
461
- hookPrototypeMethods('navigator', window.navigator);
462
- hookPrototypeMethods('history', window.history);
463
- }, fingerprint, LOG_OVERRIDE, DIMENSION);
464
-
465
- // emulate permissions
466
- await page.addInitScript(() => {
467
- const originalQuery = window.navigator.permissions.query;
468
- window.navigator.permissions.__proto__.query = (parameters) => (parameters.name === 'notifications' ?
469
- Promise.resolve({
470
- state: Notification.permission
471
- }) :
472
- originalQuery(parameters));
473
-
474
- const oldCall = Function.prototype.call;
475
-
476
- function call() {
477
- return oldCall.apply(this, arguments);
478
- }
479
- Function.prototype.call = call;
480
-
481
- const nativeToStringFunctionString = Error.toString().replace(
482
- /Error/g,
483
- 'toString',
484
- );
485
- const oldToString = Function.prototype.toString;
486
-
487
- function functionToString() {
488
- if (this === window.navigator.permissions.query) {
489
- return 'function query() { [native code] }';
490
- }
491
- if (this === functionToString) {
492
- return nativeToStringFunctionString;
493
  }
494
- return oldCall.call(oldToString, this);
495
- }
496
- Function.prototype.toString = functionToString;
497
  });
 
 
 
 
498
 
499
- // must to hook getters
500
- await page.goto('about:blank');
501
-
502
- await page.setExtraHTTPHeaders({
503
- 'Accept-Language': 'en-US,en;q=0.9',
504
- 'Accept-Encoding': 'gzip, deflate, br',
505
- });
506
 
507
- await page.setViewport(DIMENSION);
 
 
 
508
  }
509
-
510
- const args = [
511
- '--no-sandbox',
512
- '--remote-debugging-port=9222',
513
- '--disable-setuid-sandbox',
514
- '--ignore-certificate-errors',
515
- '--disk-cache-size=1',
516
- '--disable-infobars',
517
- '--disable-blink-features=AutomationControlled'
518
- ];
519
-
520
- console.log('launchPersistentContext', url);
521
-
522
- const browser = await chromium.launchPersistentContext('', {
523
- headless: false,
524
- bypassCSP: true,
525
- ignoreHTTPSErrors: true,
526
- args: args,
527
- timezoneId: 'America/Los_Angeles',
528
- });
529
-
530
- // generate a fingerprint
531
- const fingerprint = await getBrowserfingerprint(uuidv4());
532
- let page = await browser.newPage();
533
-
534
- // "cloak" our page with evasions
535
- await cloak(page, fingerprint);
536
-
537
-
538
-
539
- console.log('goto', url);
540
-
541
- //await context.route("**/*.{png,jpg,jpeg,css,js}", route => route.abort());
542
- await page.goto(url);
543
-
544
- console.log('goto end');
545
-
546
- await setTimeout(2000);
547
-
548
- // let id = crypto.randomUUID();
549
- // let path = `/code/${id}.png`;
550
- // await page.screenshot({ path, fullPage: true });
551
-
552
- const buffer = await page.screenshot({fullPage: true});
553
-
554
- await page.close();
555
- await browser.close();
556
-
557
- return buffer;
558
- }
559
-
560
-
561
-
562
-
563
-
564
- const server = http.createServer(async function(req, res) {
565
- const {query, pathname} = url.parse(req.url, true);
566
-
567
- console.log(req.url, new Date());
568
-
569
- res.setHeader('Access-Control-Allow-Origin', '*')
570
-
571
- let _url = query.url;
572
- let cookie = query.cookie;
573
- let js_enabled = query.js;
574
-
575
  try {
576
- if (pathname == '/text') {
577
- let text = await text_from_url(_url, cookie);
578
-
579
- res.end(text);
580
- } else if (pathname == '/screenshot') {
581
- let buffer = await screenshot(_url, cookie, js_enabled);
582
-
583
- res.writeHead(200,{'Content-type':'image/png'});
584
- res.end(buffer);
585
-
586
- // const readStream = fs.createReadStream(path);
587
- // res.writeHead(200,{'Content-type':'image/png'});
588
- // readStream.pipe(res);
589
- // res.end();
590
-
591
- } else if (pathname == '/bot') {
592
- let buffer = await test_bot(_url);
593
-
594
- res.writeHead(200,{'Content-type':'image/png'});
595
- res.end(buffer);
596
-
597
- } else if (pathname == '/evaluate') {
598
- const buffers = [];
599
- for await (const chunk of req) {
600
- buffers.push(chunk);
601
- }
602
- const data = Buffer.concat(buffers).toString();
603
-
604
- const {url, code} = JSON.parse(data);
605
-
606
- try {
607
- let result = await evaluate(url, code);
608
- res.end(''+result);
609
- } catch (e) {
610
- res.end(e.toString());
611
- }
612
- } else {
613
- res.end('hello');
614
- }
615
- } catch (e) {
616
- res.end(e.toString());
617
  }
618
  });
619
 
620
- const wss = new WebSocketServer({ server });
621
-
622
- wss.on('connection', function connection(ws, req) {
623
- console.log('wss.connection', req.url, req.headers);
624
-
625
- ws.isAlive = true;
626
- ws.on('pong', function heartbeat(){ //'Pong messages are automatically sent in response to ping messages as required by the spec.'
627
- //console.log('pong');
628
- this.isAlive = true;
629
- });
630
-
631
- ws.on('message', function message(data) {
632
- console.log('received: %s', data);
633
- });
634
-
635
- ws.addEventListener('close', function close() {
636
- console.log('ws.close');
637
- });
638
- //ws.send('something');
639
- });
640
-
641
- // https://github.com/websockets/ws
642
- const interval = setInterval(function ping() {
643
- wss.clients.forEach(function each(ws) {
644
- if (ws.isAlive === false) return ws.terminate();
645
-
646
- ws.isAlive = false;
647
- ws.ping();
648
-
649
- ws.send('ping');//why: used by client to detect dead connection (because browser cant access ping/pong frames...(?))
650
- });
651
- }, 30000);
652
-
653
- server.listen(7860);
 
1
+ const express = require('express');
2
+ const puppeteer = require('puppeteer');
3
+ const bodyParser = require('body-parser');
4
+ const cors = require('cors');
5
+
6
+ const app = express();
7
+ app.use(bodyParser.urlencoded({ extended: true }));
8
+ app.use(bodyParser.json());
9
+ app.use(cors());
10
+
11
+ const html = `
12
+ <!DOCTYPE html>
13
+ <html lang="en">
14
+ <head>
15
+ <meta charset="UTF-8">
16
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
17
+ <title>YouTube Transcript Generator</title>
18
+ <style>
19
+ body {
20
+ font-family: Arial, sans-serif;
21
+ max-width: 800px;
22
+ margin: 0 auto;
23
+ padding: 20px;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
24
  }
25
+ form {
26
+ display: flex;
27
+ flex-direction: column;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
  }
29
+ input, button {
30
+ margin: 10px 0;
31
+ padding: 5px;
32
+ }
33
+ #result {
34
+ white-space: pre-wrap;
35
+ background-color: #f0f0f0;
36
+ padding: 10px;
37
+ border-radius: 5px;
38
+ }
39
+ </style>
40
+ </head>
41
+ <body>
42
+ <h1>YouTube Transcript Generator</h1>
43
+ <form id="transcriptForm">
44
+ <input type="text" id="videoUrl" name="videoUrl" placeholder="YouTube Video URL" required>
45
+ <input type="text" id="videoTitle" name="videoTitle" placeholder="Video Title" required>
46
+ <button type="submit">Generate Transcript</button>
47
+ </form>
48
+ <div id="result"></div>
49
+
50
+ <script>
51
+ document.getElementById('transcriptForm').addEventListener('submit', async (e) => {
52
+ e.preventDefault();
53
+ const videoUrl = document.getElementById('videoUrl').value;
54
+ const videoTitle = document.getElementById('videoTitle').value;
55
+ const resultDiv = document.getElementById('result');
56
+
57
+ resultDiv.textContent = 'Generating transcript...';
58
 
59
+ try {
60
+ const response = await fetch('/extract-transcript', {
61
+ method: 'POST',
62
+ headers: {
63
+ 'Content-Type': 'application/json',
64
+ },
65
+ body: JSON.stringify({ videoUrl, videoTitle }),
66
+ });
67
+
68
+ if (response.ok) {
69
+ const data = await response.json();
70
+ resultDiv.textContent = data.transcript;
71
+ } else {
72
+ resultDiv.textContent = 'Error generating transcript. Please try again.';
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
73
  }
74
+ } catch (error) {
75
+ console.error('Error:', error);
76
+ resultDiv.textContent = 'An error occurred. Please try again.';
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
77
  }
 
 
 
78
  });
79
+ </script>
80
+ </body>
81
+ </html>
82
+ `;
83
 
84
+ app.get('/', (req, res) => {
85
+ res.send(html);
86
+ });
 
 
 
 
87
 
88
+ app.post('/extract-transcript', async (req, res) => {
89
+ const { videoUrl, videoTitle } = req.body;
90
+ if (!videoUrl || !videoTitle) {
91
+ return res.status(400).send('videoUrl and videoTitle are required');
92
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
93
  try {
94
+ const browser = await puppeteer.launch({ headless: true });
95
+ const page = await browser.newPage();
96
+ await page.goto(videoUrl, { waitUntil: 'networkidle2' });
97
+ await page.setViewport({ width: 1920, height: 1080 });
98
+
99
+ await page.waitForSelector('tp-yt-paper-button#expand', { timeout: 60000 });
100
+ await page.click('tp-yt-paper-button#expand');
101
+
102
+ await page.waitForSelector('button[aria-label="Show transcript"]', { timeout: 60000 });
103
+ await page.click('button[aria-label="Show transcript"]');
104
+
105
+ await page.waitForSelector('ytd-transcript-segment-list-renderer', { timeout: 60000 });
106
+
107
+ const transcript = await page.evaluate(() => {
108
+ const elements = Array.from(document.querySelectorAll('ytd-transcript-segment-renderer .segment-text'));
109
+ return elements.map(element => element.innerText).join('\n');
110
+ });
111
+
112
+ await browser.close();
113
+ res.json({ transcript });
114
+
115
+ } catch (error) {
116
+ console.error('Error extracting transcript:', error);
117
+ res.status(500).send('Error extracting transcript');
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
118
  }
119
  });
120
 
121
+ const PORT = process.env.PORT || 7860;
122
+ app.listen(PORT, () => {
123
+ console.log(`Server is running on port ${PORT}`);
124
+ });