Hugging Face's logo

Spaces:
sanjay7178
/
flaresolverr
Runtime error

App Files Community
sanjay7178 commited on
Commit
7387da9
·
1 Parent(s): a4e2aa5

chore: Update test requirements and package information

Browse files
Files changed (33) hide show
  1. .gitattributes +35 -35
  2. CHANGELOG.md +422 -422
  3. Dockerfile +78 -78
  4. README.md +324 -324
  5. docker-compose.yml +15 -15
  6. html_samples/cloudflare_captcha_hcaptcha_v1.html +218 -218
  7. html_samples/cloudflare_captcha_norobot_v1.html +169 -169
  8. html_samples/cloudflare_init_v1.html +119 -119
  9. html_samples/cloudflare_spinner_v1.html +166 -166
  10. package.json +7 -7
  11. requirements.txt +13 -13
  12. resources/flaresolverr_logo.svg +180 -180
  13. src/bottle_plugins/error_plugin.py +22 -22
  14. src/bottle_plugins/logger_plugin.py +23 -23
  15. src/bottle_plugins/prometheus_plugin.py +66 -66
  16. src/build_package.py +110 -110
  17. src/dtos.py +86 -86
  18. src/flaresolverr.py +125 -125
  19. src/flaresolverr_service.py +455 -455
  20. src/metrics.py +32 -32
  21. src/sessions.py +84 -84
  22. src/tests.py +632 -632
  23. src/tests_sites.py +102 -102
  24. src/undetected_chromedriver/__init__.py +914 -914
  25. src/undetected_chromedriver/cdp.py +112 -112
  26. src/undetected_chromedriver/devtool.py +193 -193
  27. src/undetected_chromedriver/dprocess.py +77 -77
  28. src/undetected_chromedriver/options.py +85 -85
  29. src/undetected_chromedriver/patcher.py +451 -451
  30. src/undetected_chromedriver/reactor.py +99 -99
  31. src/undetected_chromedriver/webelement.py +86 -86
  32. src/utils.py +341 -341
  33. test-requirements.txt +1 -1
.gitattributes CHANGED
@@ -1,35 +1,35 @@
1
- *.7z filter=lfs diff=lfs merge=lfs -text
2
- *.arrow filter=lfs diff=lfs merge=lfs -text
3
- *.bin filter=lfs diff=lfs merge=lfs -text
4
- *.bz2 filter=lfs diff=lfs merge=lfs -text
5
- *.ckpt filter=lfs diff=lfs merge=lfs -text
6
- *.ftz filter=lfs diff=lfs merge=lfs -text
7
- *.gz filter=lfs diff=lfs merge=lfs -text
8
- *.h5 filter=lfs diff=lfs merge=lfs -text
9
- *.joblib filter=lfs diff=lfs merge=lfs -text
10
- *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
- *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
- *.model filter=lfs diff=lfs merge=lfs -text
13
- *.msgpack filter=lfs diff=lfs merge=lfs -text
14
- *.npy filter=lfs diff=lfs merge=lfs -text
15
- *.npz filter=lfs diff=lfs merge=lfs -text
16
- *.onnx filter=lfs diff=lfs merge=lfs -text
17
- *.ot filter=lfs diff=lfs merge=lfs -text
18
- *.parquet filter=lfs diff=lfs merge=lfs -text
19
- *.pb filter=lfs diff=lfs merge=lfs -text
20
- *.pickle filter=lfs diff=lfs merge=lfs -text
21
- *.pkl filter=lfs diff=lfs merge=lfs -text
22
- *.pt filter=lfs diff=lfs merge=lfs -text
23
- *.pth filter=lfs diff=lfs merge=lfs -text
24
- *.rar filter=lfs diff=lfs merge=lfs -text
25
- *.safetensors filter=lfs diff=lfs merge=lfs -text
26
- saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
- *.tar.* filter=lfs diff=lfs merge=lfs -text
28
- *.tar filter=lfs diff=lfs merge=lfs -text
29
- *.tflite filter=lfs diff=lfs merge=lfs -text
30
- *.tgz filter=lfs diff=lfs merge=lfs -text
31
- *.wasm filter=lfs diff=lfs merge=lfs -text
32
- *.xz filter=lfs diff=lfs merge=lfs -text
33
- *.zip filter=lfs diff=lfs merge=lfs -text
34
- *.zst filter=lfs diff=lfs merge=lfs -text
35
- *tfevents* filter=lfs diff=lfs merge=lfs -text
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tar filter=lfs diff=lfs merge=lfs -text
29
+ *.tflite filter=lfs diff=lfs merge=lfs -text
30
+ *.tgz filter=lfs diff=lfs merge=lfs -text
31
+ *.wasm filter=lfs diff=lfs merge=lfs -text
32
+ *.xz filter=lfs diff=lfs merge=lfs -text
33
+ *.zip filter=lfs diff=lfs merge=lfs -text
34
+ *.zst filter=lfs diff=lfs merge=lfs -text
35
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
CHANGELOG.md CHANGED
@@ -1,422 +1,422 @@
1
- # Changelog
2
-
3
- ## v3.3.21 (2024/06/26)
4
- * Add challenge selector to catch reloading page on non-English systems
5
- * Escape values for generated form used in request.post. Thanks @mynameisbogdan
6
-
7
- ## v3.3.20 (2024/06/21)
8
- * maxTimeout should always be int
9
- * Check not running in Docker before logging version_main error
10
- * Update Cloudflare challenge and checkbox selectors. Thanks @tenettow & @21hsmw
11
-
12
- ## v3.3.19 (2024/05/23)
13
- * Fix occasional headless issue on Linux when set to "false". Thanks @21hsmw
14
-
15
- ## v3.3.18 (2024/05/20)
16
-
17
- * Fix LANG ENV for Linux
18
- * Fix Chrome v124+ not closing on Windows. Thanks @RileyXX
19
-
20
- ## v3.3.17 (2024/04/09)
21
-
22
- * Fix file descriptor leak in service on quit(). Thanks @zkulis
23
-
24
- ## v3.3.16 (2024/02/28)
25
-
26
- * Fix of the subprocess.STARTUPINFO() call. Thanks @ceconelo
27
- * Add FreeBSD support. Thanks @Asthowen
28
- * Use headless configuration properly. Thanks @hashworks
29
-
30
- ## v3.3.15 (2024/02/20)
31
-
32
- * Fix looping challenges
33
-
34
- ## v3.3.14-hotfix2 (2024/02/17)
35
-
36
- * Hotfix 2 - bad Chromium build, instances failed to terminate
37
-
38
- ## v3.3.14-hotfix (2024/02/17)
39
-
40
- * Hotfix for Linux build - some Chrome files no longer exist
41
-
42
- ## v3.3.14 (2024/02/17)
43
-
44
- * Update Chrome downloads. Thanks @opemvbs
45
-
46
- ## v3.3.13 (2024/01/07)
47
-
48
- * Fix too many open files error
49
-
50
- ## v3.3.12 (2023/12/15)
51
-
52
- * Fix looping challenges and invalid cookies
53
-
54
- ## v3.3.11 (2023/12/11)
55
-
56
- * Update UC 3.5.4 & Selenium 4.15.2. Thanks @txtsd
57
-
58
- ## v3.3.10 (2023/11/14)
59
-
60
- * Add LANG ENV - resolves issues with YGGtorrent
61
-
62
- ## v3.3.9 (2023/11/13)
63
-
64
- * Fix for Docker build, capture TypeError
65
-
66
- ## v3.3.8 (2023/11/13)
67
-
68
- * Fix headless=true for Chrome 117+. Thanks @NabiKAZ
69
- * Support running Chrome 119 from source. Thanks @koleg and @Chris7X
70
- * Fix "OSError: [WinError 6] The handle is invalid" on exit. Thanks @enesgorkemgenc
71
-
72
- ## v3.3.7 (2023/11/05)
73
-
74
- * Bump to rebuild. Thanks @JoachimDorchies
75
-
76
- ## v3.3.6 (2023/09/15)
77
-
78
- * Update checkbox selector, again
79
-
80
- ## v3.3.5 (2023/09/13)
81
-
82
- * Change checkbox selector, support languages other than English
83
-
84
- ## v3.3.4 (2023/09/02)
85
-
86
- * Update checkbox selector
87
-
88
- ## v3.3.3 (2023/08/31)
89
-
90
- * Update undetected_chromedriver to v3.5.3
91
-
92
- ## v3.3.2 (2023/08/03)
93
-
94
- * Fix URL domain in Prometheus exporter
95
-
96
- ## v3.3.1 (2023/08/03)
97
-
98
- * Fix for Cloudflare verify checkbox
99
- * Fix HEADLESS=false in Windows binary
100
- * Fix Prometheus exporter for management and health endpoints
101
- * Remove misleading stack trace when the verify checkbox is not found
102
- * Revert "Update base Docker image to Debian Bookworm" #849
103
- * Revert "Install Chromium 115 from Debian testing" #849
104
-
105
- ## v3.3.0 (2023/08/02)
106
-
107
- * Fix for new Cloudflare detection. Thanks @cedric-bour for #845
108
- * Add support for proxy authentication username/password. Thanks @jacobprice808 for #807
109
- * Implement Prometheus metrics
110
- * Fix Chromium Driver for Chrome / Chromium version > 114
111
- * Use Chromium 115 in binary packages (Windows and Linux)
112
- * Install Chromium 115 from Debian testing (Docker)
113
- * Update base Docker image to Debian Bookworm
114
- * Update Selenium 4.11.2
115
- * Update pyinstaller 5.13.0
116
- * Add more traces in build_package.py
117
-
118
- ## v3.2.2 (2023/07/16)
119
-
120
- * Workaround for updated 'verify you are human' check
121
-
122
- ## v3.2.1 (2023/06/10)
123
-
124
- * Kill dead Chrome processes in Windows
125
- * Fix Chrome GL erros in ASUSTOR NAS
126
-
127
- ## v3.2.0 (2023/05/23)
128
-
129
- * Support "proxy" param in requests and sessions
130
- * Support "cookies" param in requests
131
- * Fix Chromium exec permissions in Linux package
132
- * Update Python dependencies
133
-
134
- ## v3.1.2 (2023/04/02)
135
-
136
- * Fix headless mode in macOS
137
- * Remove redundant artifact from Windows binary package
138
- * Bump Selenium dependency
139
-
140
- ## v3.1.1 (2023/03/25)
141
-
142
- * Distribute binary executables in compressed package
143
- * Add icon for binary executable
144
- * Include information about supported architectures in the readme
145
- * Check Python version on start
146
-
147
- ## v3.1.0 (2023/03/20)
148
-
149
- * Build binaries for Linux x64 and Windows x64
150
- * Sessions with auto-creation on fetch request and TTL
151
- * Fix error trace: Crash Reports/pending No such file or directory
152
- * Fix Waitress server error with asyncore_use_poll=true
153
- * Attempt to fix Docker ARM32 build
154
- * Print platform information on start up
155
- * Add Fairlane challenge selector
156
- * Update DDOS-GUARD title
157
- * Update dependencies
158
-
159
- ## v3.0.4 (2023/03/07)
160
-
161
- * Click on the Cloudflare's 'Verify you are human' button if necessary
162
-
163
- ## v3.0.3 (2023/03/06)
164
-
165
- * Update undetected_chromedriver version to 3.4.6
166
-
167
- ## v3.0.2 (2023/01/08)
168
-
169
- * Detect Cloudflare blocked access
170
- * Check Chrome / Chromium web browser is installed correctly
171
-
172
- ## v3.0.1 (2023/01/06)
173
-
174
- * Kill Chromium processes properly to avoid defunct/zombie processes
175
- * Update undetected-chromedriver
176
- * Disable Zygote sandbox in Chromium browser
177
- * Add more selectors to detect blocked access
178
- * Include procps (ps), curl and vim packages in the Docker image
179
-
180
- ## v3.0.0 (2023/01/04)
181
-
182
- * This is the first release of FlareSolverr v3. There are some breaking changes
183
- * Docker images for linux/386, linux/amd64, linux/arm/v7 and linux/arm64/v8
184
- * Replaced Firefox with Chrome
185
- * Replaced NodeJS / Typescript with Python
186
- * Replaced Puppeter with Selenium
187
- * No binaries for Linux / Windows. You have to use the Docker image or install from Source code
188
- * No proxy support
189
- * No session support
190
-
191
- ## v2.2.10 (2022/10/22)
192
-
193
- * Detect DDoS-Guard through title content
194
-
195
- ## v2.2.9 (2022/09/25)
196
-
197
- * Detect Cloudflare Access Denied
198
- * Commit the complete changelog
199
-
200
- ## v2.2.8 (2022/09/17)
201
-
202
- * Remove 30 s delay and clean legacy code
203
-
204
- ## v2.2.7 (2022/09/12)
205
-
206
- * Temporary fix: add 30s delay
207
- * Update README.md
208
-
209
- ## v2.2.6 (2022/07/31)
210
-
211
- * Fix Cloudflare detection in POST requests
212
-
213
- ## v2.2.5 (2022/07/30)
214
-
215
- * Update GitHub actions to build executables with NodeJs 16
216
- * Update Cloudflare selectors and add HTML samples
217
- * Install Firefox 94 instead of the latest Nightly
218
- * Update dependencies
219
- * Upgrade Puppeteer (#396)
220
-
221
- ## v2.2.4 (2022/04/17)
222
-
223
- * Detect DDoS-Guard challenge
224
-
225
- ## v2.2.3 (2022/04/16)
226
-
227
- * Fix 2000 ms navigation timeout
228
- * Update README.md (libseccomp2 package in Debian)
229
- * Update README.md (clarify proxy parameter) (#307)
230
- * Update NPM dependencies
231
- * Disable Cloudflare ban detection
232
-
233
- ## v2.2.2 (2022/03/19)
234
-
235
- * Fix ban detection. Resolves #330 (#336)
236
-
237
- ## v2.2.1 (2022/02/06)
238
-
239
- * Fix max timeout error in some pages
240
- * Avoid crashing in NodeJS 17 due to Unhandled promise rejection
241
- * Improve proxy validation and debug traces
242
- * Remove @types/puppeteer dependency
243
-
244
- ## v2.2.0 (2022/01/31)
245
-
246
- * Increase default BROWSER_TIMEOUT=40000 (40 seconds)
247
- * Fix Puppeter deprecation warnings
248
- * Update base Docker image Alpine 3.15 / NodeJS 16
249
- * Build precompiled binaries with NodeJS 16
250
- * Update Puppeter and other dependencies
251
- * Add support for Custom CloudFlare challenge
252
- * Add support for DDoS-GUARD challenge
253
-
254
- ## v2.1.0 (2021/12/12)
255
-
256
- * Add aarch64 to user agents to be replaced (#248)
257
- * Fix SOCKSv4 and SOCKSv5 proxy. resolves #214 #220
258
- * Remove redundant JSON key (postData) (#242)
259
- * Make test URL configurable with TEST_URL env var. resolves #240
260
- * Bypass new Cloudflare protection
261
- * Update donation links
262
-
263
- ## v2.0.2 (2021/10/31)
264
-
265
- * Fix SOCKS5 proxy. Resolves #214
266
- * Replace Firefox ERS with a newer version
267
- * Catch startup exceptions and give some advices
268
- * Add env var BROWSER_TIMEOUT for slow systems
269
- * Fix NPM warning in Docker images
270
-
271
- ## v2.0.1 (2021/10/24)
272
-
273
- * Check user home dir before testing web browser installation
274
-
275
- ## v2.0.0 (2021/10/20)
276
-
277
- FlareSolverr 2.0.0 is out with some important changes:
278
-
279
- * It is capable of solving the automatic challenges of Cloudflare. CAPTCHAs (hCaptcha) cannot be resolved and the old solvers have been removed.
280
- * The Chrome browser has been replaced by Firefox. This has caused some functionality to be removed. Parameters: `userAgent`, `headers`, `rawHtml` and `downloadare` no longer available.
281
- * Included `proxy` support without user/password credentials. If you are writing your own integration with FlareSolverr, make sure your client uses the same User-Agent header and Proxy that FlareSolverr uses. Those values together with the Cookie are checked and detected by Cloudflare.
282
- * FlareSolverr has been rewritten from scratch. From now on it should be easier to maintain and test.
283
- * If you are using Jackett make sure you have version v0.18.1041 or higher. FlareSolverSharp v2.0.0 is out too.
284
-
285
- Complete changelog:
286
-
287
- * Bump version 2.0.0
288
- * Set puppeteer timeout half of maxTimeout param. Resolves #180
289
- * Add test for blocked IP
290
- * Avoid reloading the page in case of error
291
- * Improve Cloudflare detection
292
- * Fix version
293
- * Fix browser preferences and proxy
294
- * Fix request.post method and clean error traces
295
- * Use Firefox ESR for Docker images
296
- * Improve Firefox start time and code clean up
297
- * Improve bad request management and tests
298
- * Build native packages with Firefox
299
- * Update readme
300
- * Improve Docker image and clean TODOs
301
- * Add proxy support
302
- * Implement request.post method for Firefox
303
- * Code clean up, remove returnRawHtml, download, headers params
304
- * Remove outdated chaptcha solvers
305
- * Refactor the app to use Express server and Jest for tests
306
- * Fix Cloudflare resolver for Linux ARM builds
307
- * Fix Cloudflare resolver
308
- * Replace Chrome web browser with Firefox
309
- * Remove userAgent parameter since any modification is detected by CF
310
- * Update dependencies
311
- * Remove Puppeter steath plugin
312
-
313
- ## v1.2.9 (2021/08/01)
314
-
315
- * Improve "Execution context was destroyed" error handling
316
- * Implement returnRawHtml parameter. resolves #172 resolves #165
317
- * Capture Docker stop signal. resolves #158
318
- * Reduce Docker image size 20 MB
319
- * Fix page reload after challenge is solved. resolves #162 resolves #143
320
- * Avoid loading images/css/fonts to speed up page load
321
- * Improve Cloudflare IP ban detection
322
- * Fix vulnerabilities
323
-
324
- ## v1.2.8 (2021/06/01)
325
-
326
- * Improve old JS challenge waiting. Resolves #129
327
-
328
- ## v1.2.7 (2021/06/01)
329
-
330
- * Improvements in Cloudflare redirect detection. Resolves #140
331
- * Fix installation instructions
332
-
333
- ## v1.2.6 (2021/05/30)
334
-
335
- * Handle new Cloudflare challenge. Resolves #135 Resolves #134
336
- * Provide reference Systemd unit file. Resolves #72
337
- * Fix EACCES: permission denied, open '/tmp/flaresolverr.txt'. Resolves #120
338
- * Configure timezone with TZ env var. Resolves #109
339
- * Return the redirected URL in the response (#126)
340
- * Show an error in hcaptcha-solver. Resolves #132
341
- * Regenerate package-lock.json lockfileVersion 2
342
- * Update issue template. Resolves #130
343
- * Bump ws from 7.4.1 to 7.4.6 (#137)
344
- * Bump hosted-git-info from 2.8.8 to 2.8.9 (#124)
345
- * Bump lodash from 4.17.20 to 4.17.21 (#125)
346
-
347
- ## v1.2.5 (2021/04/05)
348
-
349
- * Fix memory regression, close test browser
350
- * Fix release-docker GitHub action
351
-
352
- ## v1.2.4 (2021/04/04)
353
-
354
- * Include license in release zips. resolves #75
355
- * Validate Chrome is working at startup
356
- * Speedup Docker image build
357
- * Add health check endpoint
358
- * Update issue template
359
- * Minor improvements in debug traces
360
- * Validate environment variables at startup. resolves #101
361
- * Add FlareSolverr logo. resolves #23
362
-
363
- ## v1.2.3 (2021/01/10)
364
-
365
- * CI/CD: Generate release changelog from commits. resolves #34
366
- * Update README.md
367
- * Add donation links
368
- * Simplify docker-compose.yml
369
- * Allow to configure "none" captcha resolver
370
- * Override docker-compose.yml variables via .env resolves #64 (#66)
371
-
372
- ## v1.2.2 (2021/01/09)
373
-
374
- * Add documentation for precompiled binaries installation
375
- * Add instructions to set environment variables in Windows
376
- * Build Windows and Linux binaries. resolves #18
377
- * Add release badge in the readme
378
- * CI/CD: Generate release changelog from commits. resolves #34
379
- * Add a notice about captcha solvers
380
- * Add Chrome flag --disable-dev-shm-usage to fix crashes. resolves #45
381
- * Fix Docker CLI documentation
382
- * Add traces with captcha solver service. resolves #39
383
- * Improve logic to detect Cloudflare captcha. resolves #48
384
- * Move Cloudflare provider logic to his own class
385
- * Simplify and document the "return only cookies" parameter
386
- * Show message when debug log is enabled
387
- * Update readme to add more clarifications. resolves #53 (#60)
388
- * issue_template: typo fix (#52)
389
-
390
- ## v1.2.1 (2020/12/20)
391
-
392
- * Change version to match release tag / 1.2.0 => v1.2.0
393
- * CI/CD Publish release in GitHub repository. resolves #34
394
- * Add welcome message in / endpoint
395
- * Rewrite request timeout handling (maxTimeout) resolves #42
396
- * Add http status for better logging
397
- * Return an error when no selectors are found, #25
398
- * Add issue template, fix #32
399
- * Moving log.html right after loading the page and add one on reload, fix #30
400
- * Update User-Agent to match chromium version, ref: #15 (#28)
401
- * Update install from source code documentation
402
- * Update readme to add Docker instructions (#20)
403
- * Clean up readme (#19)
404
- * Add docker-compose
405
- * Change default log level to info
406
-
407
- ## v1.2.0 (2020/12/20)
408
-
409
- * Fix User-Agent detected by CouldFlare (Docker ARM) resolves #15
410
- * Include exception message in error response
411
- * CI/CD: Rename GitHub Action build => publish
412
- * Bump version
413
- * Fix TypeScript compilation and bump minor version
414
- * CI/CD: Bump minor version
415
- * CI/CD: Configure GitHub Actions
416
- * CI/CD: Configure GitHub Actions
417
- * CI/CD: Bump minor version
418
- * CI/CD: Configure Build GitHub Action
419
- * CI/CD: Configure AutoTag GitHub Action (#14)
420
- * CI/CD: Build the Docker images with GitHub Actions (#13)
421
- * Update dependencies
422
- * Backport changes from Cloudproxy (#11)
 
1
+ # Changelog
2
+
3
+ ## v3.3.21 (2024/06/26)
4
+ * Add challenge selector to catch reloading page on non-English systems
5
+ * Escape values for generated form used in request.post. Thanks @mynameisbogdan
6
+
7
+ ## v3.3.20 (2024/06/21)
8
+ * maxTimeout should always be int
9
+ * Check not running in Docker before logging version_main error
10
+ * Update Cloudflare challenge and checkbox selectors. Thanks @tenettow & @21hsmw
11
+
12
+ ## v3.3.19 (2024/05/23)
13
+ * Fix occasional headless issue on Linux when set to "false". Thanks @21hsmw
14
+
15
+ ## v3.3.18 (2024/05/20)
16
+
17
+ * Fix LANG ENV for Linux
18
+ * Fix Chrome v124+ not closing on Windows. Thanks @RileyXX
19
+
20
+ ## v3.3.17 (2024/04/09)
21
+
22
+ * Fix file descriptor leak in service on quit(). Thanks @zkulis
23
+
24
+ ## v3.3.16 (2024/02/28)
25
+
26
+ * Fix of the subprocess.STARTUPINFO() call. Thanks @ceconelo
27
+ * Add FreeBSD support. Thanks @Asthowen
28
+ * Use headless configuration properly. Thanks @hashworks
29
+
30
+ ## v3.3.15 (2024/02/20)
31
+
32
+ * Fix looping challenges
33
+
34
+ ## v3.3.14-hotfix2 (2024/02/17)
35
+
36
+ * Hotfix 2 - bad Chromium build, instances failed to terminate
37
+
38
+ ## v3.3.14-hotfix (2024/02/17)
39
+
40
+ * Hotfix for Linux build - some Chrome files no longer exist
41
+
42
+ ## v3.3.14 (2024/02/17)
43
+
44
+ * Update Chrome downloads. Thanks @opemvbs
45
+
46
+ ## v3.3.13 (2024/01/07)
47
+
48
+ * Fix too many open files error
49
+
50
+ ## v3.3.12 (2023/12/15)
51
+
52
+ * Fix looping challenges and invalid cookies
53
+
54
+ ## v3.3.11 (2023/12/11)
55
+
56
+ * Update UC 3.5.4 & Selenium 4.15.2. Thanks @txtsd
57
+
58
+ ## v3.3.10 (2023/11/14)
59
+
60
+ * Add LANG ENV - resolves issues with YGGtorrent
61
+
62
+ ## v3.3.9 (2023/11/13)
63
+
64
+ * Fix for Docker build, capture TypeError
65
+
66
+ ## v3.3.8 (2023/11/13)
67
+
68
+ * Fix headless=true for Chrome 117+. Thanks @NabiKAZ
69
+ * Support running Chrome 119 from source. Thanks @koleg and @Chris7X
70
+ * Fix "OSError: [WinError 6] The handle is invalid" on exit. Thanks @enesgorkemgenc
71
+
72
+ ## v3.3.7 (2023/11/05)
73
+
74
+ * Bump to rebuild. Thanks @JoachimDorchies
75
+
76
+ ## v3.3.6 (2023/09/15)
77
+
78
+ * Update checkbox selector, again
79
+
80
+ ## v3.3.5 (2023/09/13)
81
+
82
+ * Change checkbox selector, support languages other than English
83
+
84
+ ## v3.3.4 (2023/09/02)
85
+
86
+ * Update checkbox selector
87
+
88
+ ## v3.3.3 (2023/08/31)
89
+
90
+ * Update undetected_chromedriver to v3.5.3
91
+
92
+ ## v3.3.2 (2023/08/03)
93
+
94
+ * Fix URL domain in Prometheus exporter
95
+
96
+ ## v3.3.1 (2023/08/03)
97
+
98
+ * Fix for Cloudflare verify checkbox
99
+ * Fix HEADLESS=false in Windows binary
100
+ * Fix Prometheus exporter for management and health endpoints
101
+ * Remove misleading stack trace when the verify checkbox is not found
102
+ * Revert "Update base Docker image to Debian Bookworm" #849
103
+ * Revert "Install Chromium 115 from Debian testing" #849
104
+
105
+ ## v3.3.0 (2023/08/02)
106
+
107
+ * Fix for new Cloudflare detection. Thanks @cedric-bour for #845
108
+ * Add support for proxy authentication username/password. Thanks @jacobprice808 for #807
109
+ * Implement Prometheus metrics
110
+ * Fix Chromium Driver for Chrome / Chromium version > 114
111
+ * Use Chromium 115 in binary packages (Windows and Linux)
112
+ * Install Chromium 115 from Debian testing (Docker)
113
+ * Update base Docker image to Debian Bookworm
114
+ * Update Selenium 4.11.2
115
+ * Update pyinstaller 5.13.0
116
+ * Add more traces in build_package.py
117
+
118
+ ## v3.2.2 (2023/07/16)
119
+
120
+ * Workaround for updated 'verify you are human' check
121
+
122
+ ## v3.2.1 (2023/06/10)
123
+
124
+ * Kill dead Chrome processes in Windows
125
+ * Fix Chrome GL erros in ASUSTOR NAS
126
+
127
+ ## v3.2.0 (2023/05/23)
128
+
129
+ * Support "proxy" param in requests and sessions
130
+ * Support "cookies" param in requests
131
+ * Fix Chromium exec permissions in Linux package
132
+ * Update Python dependencies
133
+
134
+ ## v3.1.2 (2023/04/02)
135
+
136
+ * Fix headless mode in macOS
137
+ * Remove redundant artifact from Windows binary package
138
+ * Bump Selenium dependency
139
+
140
+ ## v3.1.1 (2023/03/25)
141
+
142
+ * Distribute binary executables in compressed package
143
+ * Add icon for binary executable
144
+ * Include information about supported architectures in the readme
145
+ * Check Python version on start
146
+
147
+ ## v3.1.0 (2023/03/20)
148
+
149
+ * Build binaries for Linux x64 and Windows x64
150
+ * Sessions with auto-creation on fetch request and TTL
151
+ * Fix error trace: Crash Reports/pending No such file or directory
152
+ * Fix Waitress server error with asyncore_use_poll=true
153
+ * Attempt to fix Docker ARM32 build
154
+ * Print platform information on start up
155
+ * Add Fairlane challenge selector
156
+ * Update DDOS-GUARD title
157
+ * Update dependencies
158
+
159
+ ## v3.0.4 (2023/03/07)
160
+
161
+ * Click on the Cloudflare's 'Verify you are human' button if necessary
162
+
163
+ ## v3.0.3 (2023/03/06)
164
+
165
+ * Update undetected_chromedriver version to 3.4.6
166
+
167
+ ## v3.0.2 (2023/01/08)
168
+
169
+ * Detect Cloudflare blocked access
170
+ * Check Chrome / Chromium web browser is installed correctly
171
+
172
+ ## v3.0.1 (2023/01/06)
173
+
174
+ * Kill Chromium processes properly to avoid defunct/zombie processes
175
+ * Update undetected-chromedriver
176
+ * Disable Zygote sandbox in Chromium browser
177
+ * Add more selectors to detect blocked access
178
+ * Include procps (ps), curl and vim packages in the Docker image
179
+
180
+ ## v3.0.0 (2023/01/04)
181
+
182
+ * This is the first release of FlareSolverr v3. There are some breaking changes
183
+ * Docker images for linux/386, linux/amd64, linux/arm/v7 and linux/arm64/v8
184
+ * Replaced Firefox with Chrome
185
+ * Replaced NodeJS / Typescript with Python
186
+ * Replaced Puppeter with Selenium
187
+ * No binaries for Linux / Windows. You have to use the Docker image or install from Source code
188
+ * No proxy support
189
+ * No session support
190
+
191
+ ## v2.2.10 (2022/10/22)
192
+
193
+ * Detect DDoS-Guard through title content
194
+
195
+ ## v2.2.9 (2022/09/25)
196
+
197
+ * Detect Cloudflare Access Denied
198
+ * Commit the complete changelog
199
+
200
+ ## v2.2.8 (2022/09/17)
201
+
202
+ * Remove 30 s delay and clean legacy code
203
+
204
+ ## v2.2.7 (2022/09/12)
205
+
206
+ * Temporary fix: add 30s delay
207
+ * Update README.md
208
+
209
+ ## v2.2.6 (2022/07/31)
210
+
211
+ * Fix Cloudflare detection in POST requests
212
+
213
+ ## v2.2.5 (2022/07/30)
214
+
215
+ * Update GitHub actions to build executables with NodeJs 16
216
+ * Update Cloudflare selectors and add HTML samples
217
+ * Install Firefox 94 instead of the latest Nightly
218
+ * Update dependencies
219
+ * Upgrade Puppeteer (#396)
220
+
221
+ ## v2.2.4 (2022/04/17)
222
+
223
+ * Detect DDoS-Guard challenge
224
+
225
+ ## v2.2.3 (2022/04/16)
226
+
227
+ * Fix 2000 ms navigation timeout
228
+ * Update README.md (libseccomp2 package in Debian)
229
+ * Update README.md (clarify proxy parameter) (#307)
230
+ * Update NPM dependencies
231
+ * Disable Cloudflare ban detection
232
+
233
+ ## v2.2.2 (2022/03/19)
234
+
235
+ * Fix ban detection. Resolves #330 (#336)
236
+
237
+ ## v2.2.1 (2022/02/06)
238
+
239
+ * Fix max timeout error in some pages
240
+ * Avoid crashing in NodeJS 17 due to Unhandled promise rejection
241
+ * Improve proxy validation and debug traces
242
+ * Remove @types/puppeteer dependency
243
+
244
+ ## v2.2.0 (2022/01/31)
245
+
246
+ * Increase default BROWSER_TIMEOUT=40000 (40 seconds)
247
+ * Fix Puppeter deprecation warnings
248
+ * Update base Docker image Alpine 3.15 / NodeJS 16
249
+ * Build precompiled binaries with NodeJS 16
250
+ * Update Puppeter and other dependencies
251
+ * Add support for Custom CloudFlare challenge
252
+ * Add support for DDoS-GUARD challenge
253
+
254
+ ## v2.1.0 (2021/12/12)
255
+
256
+ * Add aarch64 to user agents to be replaced (#248)
257
+ * Fix SOCKSv4 and SOCKSv5 proxy. resolves #214 #220
258
+ * Remove redundant JSON key (postData) (#242)
259
+ * Make test URL configurable with TEST_URL env var. resolves #240
260
+ * Bypass new Cloudflare protection
261
+ * Update donation links
262
+
263
+ ## v2.0.2 (2021/10/31)
264
+
265
+ * Fix SOCKS5 proxy. Resolves #214
266
+ * Replace Firefox ERS with a newer version
267
+ * Catch startup exceptions and give some advices
268
+ * Add env var BROWSER_TIMEOUT for slow systems
269
+ * Fix NPM warning in Docker images
270
+
271
+ ## v2.0.1 (2021/10/24)
272
+
273
+ * Check user home dir before testing web browser installation
274
+
275
+ ## v2.0.0 (2021/10/20)
276
+
277
+ FlareSolverr 2.0.0 is out with some important changes:
278
+
279
+ * It is capable of solving the automatic challenges of Cloudflare. CAPTCHAs (hCaptcha) cannot be resolved and the old solvers have been removed.
280
+ * The Chrome browser has been replaced by Firefox. This has caused some functionality to be removed. Parameters: `userAgent`, `headers`, `rawHtml` and `downloadare` no longer available.
281
+ * Included `proxy` support without user/password credentials. If you are writing your own integration with FlareSolverr, make sure your client uses the same User-Agent header and Proxy that FlareSolverr uses. Those values together with the Cookie are checked and detected by Cloudflare.
282
+ * FlareSolverr has been rewritten from scratch. From now on it should be easier to maintain and test.
283
+ * If you are using Jackett make sure you have version v0.18.1041 or higher. FlareSolverSharp v2.0.0 is out too.
284
+
285
+ Complete changelog:
286
+
287
+ * Bump version 2.0.0
288
+ * Set puppeteer timeout half of maxTimeout param. Resolves #180
289
+ * Add test for blocked IP
290
+ * Avoid reloading the page in case of error
291
+ * Improve Cloudflare detection
292
+ * Fix version
293
+ * Fix browser preferences and proxy
294
+ * Fix request.post method and clean error traces
295
+ * Use Firefox ESR for Docker images
296
+ * Improve Firefox start time and code clean up
297
+ * Improve bad request management and tests
298
+ * Build native packages with Firefox
299
+ * Update readme
300
+ * Improve Docker image and clean TODOs
301
+ * Add proxy support
302
+ * Implement request.post method for Firefox
303
+ * Code clean up, remove returnRawHtml, download, headers params
304
+ * Remove outdated chaptcha solvers
305
+ * Refactor the app to use Express server and Jest for tests
306
+ * Fix Cloudflare resolver for Linux ARM builds
307
+ * Fix Cloudflare resolver
308
+ * Replace Chrome web browser with Firefox
309
+ * Remove userAgent parameter since any modification is detected by CF
310
+ * Update dependencies
311
+ * Remove Puppeter steath plugin
312
+
313
+ ## v1.2.9 (2021/08/01)
314
+
315
+ * Improve "Execution context was destroyed" error handling
316
+ * Implement returnRawHtml parameter. resolves #172 resolves #165
317
+ * Capture Docker stop signal. resolves #158
318
+ * Reduce Docker image size 20 MB
319
+ * Fix page reload after challenge is solved. resolves #162 resolves #143
320
+ * Avoid loading images/css/fonts to speed up page load
321
+ * Improve Cloudflare IP ban detection
322
+ * Fix vulnerabilities
323
+
324
+ ## v1.2.8 (2021/06/01)
325
+
326
+ * Improve old JS challenge waiting. Resolves #129
327
+
328
+ ## v1.2.7 (2021/06/01)
329
+
330
+ * Improvements in Cloudflare redirect detection. Resolves #140
331
+ * Fix installation instructions
332
+
333
+ ## v1.2.6 (2021/05/30)
334
+
335
+ * Handle new Cloudflare challenge. Resolves #135 Resolves #134
336
+ * Provide reference Systemd unit file. Resolves #72
337
+ * Fix EACCES: permission denied, open '/tmp/flaresolverr.txt'. Resolves #120
338
+ * Configure timezone with TZ env var. Resolves #109
339
+ * Return the redirected URL in the response (#126)
340
+ * Show an error in hcaptcha-solver. Resolves #132
341
+ * Regenerate package-lock.json lockfileVersion 2
342
+ * Update issue template. Resolves #130
343
+ * Bump ws from 7.4.1 to 7.4.6 (#137)
344
+ * Bump hosted-git-info from 2.8.8 to 2.8.9 (#124)
345
+ * Bump lodash from 4.17.20 to 4.17.21 (#125)
346
+
347
+ ## v1.2.5 (2021/04/05)
348
+
349
+ * Fix memory regression, close test browser
350
+ * Fix release-docker GitHub action
351
+
352
+ ## v1.2.4 (2021/04/04)
353
+
354
+ * Include license in release zips. resolves #75
355
+ * Validate Chrome is working at startup
356
+ * Speedup Docker image build
357
+ * Add health check endpoint
358
+ * Update issue template
359
+ * Minor improvements in debug traces
360
+ * Validate environment variables at startup. resolves #101
361
+ * Add FlareSolverr logo. resolves #23
362
+
363
+ ## v1.2.3 (2021/01/10)
364
+
365
+ * CI/CD: Generate release changelog from commits. resolves #34
366
+ * Update README.md
367
+ * Add donation links
368
+ * Simplify docker-compose.yml
369
+ * Allow to configure "none" captcha resolver
370
+ * Override docker-compose.yml variables via .env resolves #64 (#66)
371
+
372
+ ## v1.2.2 (2021/01/09)
373
+
374
+ * Add documentation for precompiled binaries installation
375
+ * Add instructions to set environment variables in Windows
376
+ * Build Windows and Linux binaries. resolves #18
377
+ * Add release badge in the readme
378
+ * CI/CD: Generate release changelog from commits. resolves #34
379
+ * Add a notice about captcha solvers
380
+ * Add Chrome flag --disable-dev-shm-usage to fix crashes. resolves #45
381
+ * Fix Docker CLI documentation
382
+ * Add traces with captcha solver service. resolves #39
383
+ * Improve logic to detect Cloudflare captcha. resolves #48
384
+ * Move Cloudflare provider logic to his own class
385
+ * Simplify and document the "return only cookies" parameter
386
+ * Show message when debug log is enabled
387
+ * Update readme to add more clarifications. resolves #53 (#60)
388
+ * issue_template: typo fix (#52)
389
+
390
+ ## v1.2.1 (2020/12/20)
391
+
392
+ * Change version to match release tag / 1.2.0 => v1.2.0
393
+ * CI/CD Publish release in GitHub repository. resolves #34
394
+ * Add welcome message in / endpoint
395
+ * Rewrite request timeout handling (maxTimeout) resolves #42
396
+ * Add http status for better logging
397
+ * Return an error when no selectors are found, #25
398
+ * Add issue template, fix #32
399
+ * Moving log.html right after loading the page and add one on reload, fix #30
400
+ * Update User-Agent to match chromium version, ref: #15 (#28)
401
+ * Update install from source code documentation
402
+ * Update readme to add Docker instructions (#20)
403
+ * Clean up readme (#19)
404
+ * Add docker-compose
405
+ * Change default log level to info
406
+
407
+ ## v1.2.0 (2020/12/20)
408
+
409
+ * Fix User-Agent detected by CouldFlare (Docker ARM) resolves #15
410
+ * Include exception message in error response
411
+ * CI/CD: Rename GitHub Action build => publish
412
+ * Bump version
413
+ * Fix TypeScript compilation and bump minor version
414
+ * CI/CD: Bump minor version
415
+ * CI/CD: Configure GitHub Actions
416
+ * CI/CD: Configure GitHub Actions
417
+ * CI/CD: Bump minor version
418
+ * CI/CD: Configure Build GitHub Action
419
+ * CI/CD: Configure AutoTag GitHub Action (#14)
420
+ * CI/CD: Build the Docker images with GitHub Actions (#13)
421
+ * Update dependencies
422
+ * Backport changes from Cloudproxy (#11)
Dockerfile CHANGED
@@ -1,78 +1,78 @@
1
- FROM python:3.11-slim-bullseye as builder
2
-
3
- # Build dummy packages to skip installing them and their dependencies
4
- RUN apt-get update \
5
- && apt-get install -y --no-install-recommends equivs \
6
- && equivs-control libgl1-mesa-dri \
7
- && printf 'Section: misc\nPriority: optional\nStandards-Version: 3.9.2\nPackage: libgl1-mesa-dri\nVersion: 99.0.0\nDescription: Dummy package for libgl1-mesa-dri\n' >> libgl1-mesa-dri \
8
- && equivs-build libgl1-mesa-dri \
9
- && mv libgl1-mesa-dri_*.deb /libgl1-mesa-dri.deb \
10
- && equivs-control adwaita-icon-theme \
11
- && printf 'Section: misc\nPriority: optional\nStandards-Version: 3.9.2\nPackage: adwaita-icon-theme\nVersion: 99.0.0\nDescription: Dummy package for adwaita-icon-theme\n' >> adwaita-icon-theme \
12
- && equivs-build adwaita-icon-theme \
13
- && mv adwaita-icon-theme_*.deb /adwaita-icon-theme.deb
14
-
15
- FROM python:3.11-slim-bullseye
16
-
17
- # Copy dummy packages
18
- COPY --from=builder /*.deb /
19
-
20
- # Install dependencies and create flaresolverr user
21
- # You can test Chromium running this command inside the container:
22
- # xvfb-run -s "-screen 0 1600x1200x24" chromium --no-sandbox
23
- # The error traces is like this: "*** stack smashing detected ***: terminated"
24
- # To check the package versions available you can use this command:
25
- # apt-cache madison chromium
26
- WORKDIR /app
27
- # Install dummy packages
28
- RUN dpkg -i /libgl1-mesa-dri.deb \
29
- && dpkg -i /adwaita-icon-theme.deb \
30
- # Install dependencies
31
- && apt-get update \
32
- && apt-get install -y --no-install-recommends chromium chromium-common chromium-driver xvfb dumb-init \
33
- procps curl vim xauth \
34
- # Remove temporary files and hardware decoding libraries
35
- && rm -rf /var/lib/apt/lists/* \
36
- && rm -f /usr/lib/x86_64-linux-gnu/libmfxhw* \
37
- && rm -f /usr/lib/x86_64-linux-gnu/mfx/* \
38
- # Create flaresolverr user
39
- && useradd --home-dir /app --shell /bin/sh flaresolverr \
40
- && mv /usr/bin/chromedriver chromedriver \
41
- && chown -R flaresolverr:flaresolverr .
42
-
43
- # Install Python dependencies
44
- COPY requirements.txt .
45
- RUN pip install -r requirements.txt \
46
- # Remove temporary files
47
- && rm -rf /root/.cache
48
-
49
- USER flaresolverr
50
-
51
- RUN mkdir -p "/app/.config/chromium/Crash Reports/pending"
52
-
53
- COPY src .
54
- COPY package.json ../
55
-
56
- EXPOSE 8191
57
- EXPOSE 8192
58
-
59
- # dumb-init avoids zombie chromium processes
60
- ENTRYPOINT ["/usr/bin/dumb-init", "--"]
61
-
62
- CMD ["/usr/local/bin/python", "-u", "/app/flaresolverr.py"]
63
-
64
- # Local build
65
- # docker build -t ngosang/flaresolverr:3.3.21 .
66
- # docker run -p 8191:8191 ngosang/flaresolverr:3.3.21
67
-
68
- # Multi-arch build
69
- # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
70
- # docker buildx create --use
71
- # docker buildx build -t ngosang/flaresolverr:3.3.21 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 .
72
- # add --push to publish in DockerHub
73
-
74
- # Test multi-arch build
75
- # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
76
- # docker buildx create --use
77
- # docker buildx build -t ngosang/flaresolverr:3.3.21 --platform linux/arm/v7 --load .
78
- # docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.21
 
1
+ FROM python:3.11-slim-bullseye as builder
2
+
3
+ # Build dummy packages to skip installing them and their dependencies
4
+ RUN apt-get update \
5
+ && apt-get install -y --no-install-recommends equivs \
6
+ && equivs-control libgl1-mesa-dri \
7
+ && printf 'Section: misc\nPriority: optional\nStandards-Version: 3.9.2\nPackage: libgl1-mesa-dri\nVersion: 99.0.0\nDescription: Dummy package for libgl1-mesa-dri\n' >> libgl1-mesa-dri \
8
+ && equivs-build libgl1-mesa-dri \
9
+ && mv libgl1-mesa-dri_*.deb /libgl1-mesa-dri.deb \
10
+ && equivs-control adwaita-icon-theme \
11
+ && printf 'Section: misc\nPriority: optional\nStandards-Version: 3.9.2\nPackage: adwaita-icon-theme\nVersion: 99.0.0\nDescription: Dummy package for adwaita-icon-theme\n' >> adwaita-icon-theme \
12
+ && equivs-build adwaita-icon-theme \
13
+ && mv adwaita-icon-theme_*.deb /adwaita-icon-theme.deb
14
+
15
+ FROM python:3.11-slim-bullseye
16
+
17
+ # Copy dummy packages
18
+ COPY --from=builder /*.deb /
19
+
20
+ # Install dependencies and create flaresolverr user
21
+ # You can test Chromium running this command inside the container:
22
+ # xvfb-run -s "-screen 0 1600x1200x24" chromium --no-sandbox
23
+ # The error traces is like this: "*** stack smashing detected ***: terminated"
24
+ # To check the package versions available you can use this command:
25
+ # apt-cache madison chromium
26
+ WORKDIR /app
27
+ # Install dummy packages
28
+ RUN dpkg -i /libgl1-mesa-dri.deb \
29
+ && dpkg -i /adwaita-icon-theme.deb \
30
+ # Install dependencies
31
+ && apt-get update \
32
+ && apt-get install -y --no-install-recommends chromium chromium-common chromium-driver xvfb dumb-init \
33
+ procps curl vim xauth \
34
+ # Remove temporary files and hardware decoding libraries
35
+ && rm -rf /var/lib/apt/lists/* \
36
+ && rm -f /usr/lib/x86_64-linux-gnu/libmfxhw* \
37
+ && rm -f /usr/lib/x86_64-linux-gnu/mfx/* \
38
+ # Create flaresolverr user
39
+ && useradd --home-dir /app --shell /bin/sh flaresolverr \
40
+ && mv /usr/bin/chromedriver chromedriver \
41
+ && chown -R flaresolverr:flaresolverr .
42
+
43
+ # Install Python dependencies
44
+ COPY requirements.txt .
45
+ RUN pip install -r requirements.txt \
46
+ # Remove temporary files
47
+ && rm -rf /root/.cache
48
+
49
+ USER flaresolverr
50
+
51
+ RUN mkdir -p "/app/.config/chromium/Crash Reports/pending"
52
+
53
+ COPY src .
54
+ COPY package.json ../
55
+
56
+ EXPOSE 8191
57
+ EXPOSE 8192
58
+
59
+ # dumb-init avoids zombie chromium processes
60
+ ENTRYPOINT ["/usr/bin/dumb-init", "--"]
61
+
62
+ CMD ["/usr/local/bin/python", "-u", "/app/flaresolverr.py"]
63
+
64
+ # Local build
65
+ # docker build -t ngosang/flaresolverr:3.3.21 .
66
+ # docker run -p 8191:8191 ngosang/flaresolverr:3.3.21
67
+
68
+ # Multi-arch build
69
+ # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
70
+ # docker buildx create --use
71
+ # docker buildx build -t ngosang/flaresolverr:3.3.21 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 .
72
+ # add --push to publish in DockerHub
73
+
74
+ # Test multi-arch build
75
+ # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
76
+ # docker buildx create --use
77
+ # docker buildx build -t ngosang/flaresolverr:3.3.21 --platform linux/arm/v7 --load .
78
+ # docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.21
README.md CHANGED
@@ -1,324 +1,324 @@
1
- # FlareSolverr
2
-
3
- [![Latest release](https://img.shields.io/github/v/release/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/releases)
4
- [![Docker Pulls](https://img.shields.io/docker/pulls/flaresolverr/flaresolverr)](https://hub.docker.com/r/flaresolverr/flaresolverr/)
5
- [![GitHub issues](https://img.shields.io/github/issues/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/issues)
6
- [![GitHub pull requests](https://img.shields.io/github/issues-pr/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/pulls)
7
- [![Donate PayPal](https://img.shields.io/badge/Donate-PayPal-yellow.svg)](https://www.paypal.com/paypalme/diegoheras0xff)
8
- [![Donate Bitcoin](https://img.shields.io/badge/Donate-Bitcoin-f7931a.svg)](https://www.blockchain.com/btc/address/13Hcv77AdnFWEUZ9qUpoPBttQsUT7q9TTh)
9
- [![Donate Ethereum](https://img.shields.io/badge/Donate-Ethereum-8c8c8c.svg)](https://www.blockchain.com/eth/address/0x0D1549BbB00926BF3D92c1A8A58695e982f1BE2E)
10
-
11
- FlareSolverr is a proxy server to bypass Cloudflare and DDoS-GUARD protection.
12
-
13
- ## How it works
14
-
15
- FlareSolverr starts a proxy server, and it waits for user requests in an idle state using few resources.
16
- When some request arrives, it uses [Selenium](https://www.selenium.dev) with the
17
- [undetected-chromedriver](https://github.com/ultrafunkamsterdam/undetected-chromedriver)
18
- to create a web browser (Chrome). It opens the URL with user parameters and waits until the Cloudflare challenge
19
- is solved (or timeout). The HTML code and the cookies are sent back to the user, and those cookies can be used to
20
- bypass Cloudflare using other HTTP clients.
21
-
22
- **NOTE**: Web browsers consume a lot of memory. If you are running FlareSolverr on a machine with few RAM, do not make
23
- many requests at once. With each request a new browser is launched.
24
-
25
- It is also possible to use a permanent session. However, if you use sessions, you should make sure to close them as
26
- soon as you are done using them.
27
-
28
- ## Installation
29
-
30
- ### Docker
31
-
32
- It is recommended to install using a Docker container because the project depends on an external browser that is
33
- already included within the image.
34
-
35
- Docker images are available in:
36
- * GitHub Registry => https://github.com/orgs/FlareSolverr/packages/container/package/flaresolverr
37
- * DockerHub => https://hub.docker.com/r/flaresolverr/flaresolverr
38
-
39
- Supported architectures are:
40
-
41
- | Architecture | Tag |
42
- |--------------|--------------|
43
- | x86 | linux/386 |
44
- | x86-64 | linux/amd64 |
45
- | ARM32 | linux/arm/v7 |
46
- | ARM64 | linux/arm64 |
47
-
48
- We provide a `docker-compose.yml` configuration file. Clone this repository and execute
49
- `docker-compose up -d` _(Compose V1)_ or `docker compose up -d` _(Compose V2)_ to start
50
- the container.
51
-
52
- If you prefer the `docker cli` execute the following command.
53
- ```bash
54
- docker run -d \
55
- --name=flaresolverr \
56
- -p 8191:8191 \
57
- -e LOG_LEVEL=info \
58
- --restart unless-stopped \
59
- ghcr.io/flaresolverr/flaresolverr:latest
60
- ```
61
-
62
- If your host OS is Debian, make sure `libseccomp2` version is 2.5.x. You can check the version with `sudo apt-cache policy libseccomp2`
63
- and update the package with `sudo apt install libseccomp2=2.5.1-1~bpo10+1` or `sudo apt install libseccomp2=2.5.1-1+deb11u1`.
64
- Remember to restart the Docker daemon and the container after the update.
65
-
66
- ### Precompiled binaries
67
-
68
- > **Warning**
69
- > Precompiled binaries are only available for x64 architecture. For other architectures see Docker images.
70
-
71
- This is the recommended way for Windows users.
72
- * Download the [FlareSolverr executable](https://github.com/FlareSolverr/FlareSolverr/releases) from the release's page. It is available for Windows x64 and Linux x64.
73
- * Execute FlareSolverr binary. In the environment variables section you can find how to change the configuration.
74
-
75
- ### From source code
76
-
77
- > **Warning**
78
- > Installing from source code only works for x64 architecture. For other architectures see Docker images.
79
-
80
- * Install [Python 3.11](https://www.python.org/downloads/).
81
- * Install [Chrome](https://www.google.com/intl/en_us/chrome/) (all OS) or [Chromium](https://www.chromium.org/getting-involved/download-chromium/) (just Linux, it doesn't work in Windows) web browser.
82
- * (Only in Linux) Install [Xvfb](https://en.wikipedia.org/wiki/Xvfb) package.
83
- * (Only in macOS) Install [XQuartz](https://www.xquartz.org/) package.
84
- * Clone this repository and open a shell in that path.
85
- * Run `pip install -r requirements.txt` command to install FlareSolverr dependencies.
86
- * Run `python src/flaresolverr.py` command to start FlareSolverr.
87
-
88
- ### From source code (FreeBSD/TrueNAS CORE)
89
-
90
- * Run `pkg install chromium python39 py39-pip xorg-vfbserver` command to install the required dependencies.
91
- * Clone this repository and open a shell in that path.
92
- * Run `python3.9 -m pip install -r requirements.txt` command to install FlareSolverr dependencies.
93
- * Run `python3.9 src/flaresolverr.py` command to start FlareSolverr.
94
-
95
- ### Systemd service
96
-
97
- We provide an example Systemd unit file `flaresolverr.service` as reference. You have to modify the file to suit your needs: paths, user and environment variables.
98
-
99
- ## Usage
100
-
101
- Example Bash request:
102
- ```bash
103
- curl -L -X POST 'http://localhost:8191/v1' \
104
- -H 'Content-Type: application/json' \
105
- --data-raw '{
106
- "cmd": "request.get",
107
- "url": "http://www.google.com/",
108
- "maxTimeout": 60000
109
- }'
110
- ```
111
-
112
- Example Python request:
113
- ```py
114
- import requests
115
-
116
- url = "http://localhost:8191/v1"
117
- headers = {"Content-Type": "application/json"}
118
- data = {
119
- "cmd": "request.get",
120
- "url": "http://www.google.com/",
121
- "maxTimeout": 60000
122
- }
123
- response = requests.post(url, headers=headers, json=data)
124
- print(response.text)
125
- ```
126
-
127
- Example PowerShell request:
128
- ```ps1
129
- $body = @{
130
- cmd = "request.get"
131
- url = "http://www.google.com/"
132
- maxTimeout = 60000
133
- } | ConvertTo-Json
134
-
135
- irm -UseBasicParsing 'http://localhost:8191/v1' -Headers @{"Content-Type"="application/json"} -Method Post -Body $body
136
- ```
137
-
138
- ### Commands
139
-
140
- #### + `sessions.create`
141
-
142
- This will launch a new browser instance which will retain cookies until you destroy it with `sessions.destroy`.
143
- This comes in handy, so you don't have to keep solving challenges over and over and you won't need to keep sending
144
- cookies for the browser to use.
145
-
146
- This also speeds up the requests since it won't have to launch a new browser instance for every request.
147
-
148
- | Parameter | Notes |
149
- |-----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
150
- | session | Optional. The session ID that you want to be assigned to the instance. If isn't set a random UUID will be assigned. |
151
- | proxy | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is supported. Eg: `"proxy": {"url": "http://127.0.0.1:8888", "username": "testuser", "password": "testpass"}` |
152
-
153
- #### + `sessions.list`
154
-
155
- Returns a list of all the active sessions. More for debugging if you are curious to see how many sessions are running.
156
- You should always make sure to properly close each session when you are done using them as too many may slow your
157
- computer down.
158
-
159
- Example response:
160
-
161
- ```json
162
- {
163
- "sessions": [
164
- "session_id_1",
165
- "session_id_2",
166
- "session_id_3..."
167
- ]
168
- }
169
- ```
170
-
171
- #### + `sessions.destroy`
172
-
173
- This will properly shutdown a browser instance and remove all files associated with it to free up resources for a new
174
- session. When you no longer need to use a session you should make sure to close it.
175
-
176
- | Parameter | Notes |
177
- |-----------|-----------------------------------------------|
178
- | session | The session ID that you want to be destroyed. |
179
-
180
- #### + `request.get`
181
-
182
- | Parameter | Notes |
183
- |---------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
184
- | url | Mandatory |
185
- | session | Optional. Will send the request from and existing browser instance. If one is not sent it will create a temporary instance that will be destroyed immediately after the request is completed. |
186
- | session_ttl_minutes | Optional. FlareSolverr will automatically rotate expired sessions based on the TTL provided in minutes. |
187
- | maxTimeout | Optional, default value 60000. Max timeout to solve the challenge in milliseconds. |
188
- | cookies | Optional. Will be used by the headless browser. Eg: `"cookies": [{"name": "cookie1", "value": "value1"}, {"name": "cookie2", "value": "value2"}]`. |
189
- | returnOnlyCookies | Optional, default false. Only returns the cookies. Response data, headers and other parts of the response are removed. |
190
- | proxy | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is not supported. (When the `session` parameter is set, the proxy is ignored; a session specific proxy can be set in `sessions.create`.) |
191
-
192
- > **Warning**
193
- > If you want to use Cloudflare clearance cookie in your scripts, make sure you use the FlareSolverr User-Agent too. If they don't match you will see the challenge.
194
-
195
- Example response from running the `curl` above:
196
-
197
- ```json
198
- {
199
- "solution": {
200
- "url": "https://www.google.com/?gws_rd=ssl",
201
- "status": 200,
202
- "headers": {
203
- "status": "200",
204
- "date": "Thu, 16 Jul 2020 04:15:49 GMT",
205
- "expires": "-1",
206
- "cache-control": "private, max-age=0",
207
- "content-type": "text/html; charset=UTF-8",
208
- "strict-transport-security": "max-age=31536000",
209
- "p3p": "CP=\"This is not a P3P policy! See g.co/p3phelp for more info.\"",
210
- "content-encoding": "br",
211
- "server": "gws",
212
- "content-length": "61587",
213
- "x-xss-protection": "0",
214
- "x-frame-options": "SAMEORIGIN",
215
- "set-cookie": "1P_JAR=2020-07-16-04; expires=Sat..."
216
- },
217
- "response":"<!DOCTYPE html>...",
218
- "cookies": [
219
- {
220
- "name": "NID",
221
- "value": "204=QE3Ocq15XalczqjuDy52HeseG3zAZuJzID3R57...",
222
- "domain": ".google.com",
223
- "path": "/",
224
- "expires": 1610684149.307722,
225
- "size": 178,
226
- "httpOnly": true,
227
- "secure": true,
228
- "session": false,
229
- "sameSite": "None"
230
- },
231
- {
232
- "name": "1P_JAR",
233
- "value": "2020-07-16-04",
234
- "domain": ".google.com",
235
- "path": "/",
236
- "expires": 1597464949.307626,
237
- "size": 19,
238
- "httpOnly": false,
239
- "secure": true,
240
- "session": false,
241
- "sameSite": "None"
242
- }
243
- ],
244
- "userAgent": "Windows NT 10.0; Win64; x64) AppleWebKit/5..."
245
- },
246
- "status": "ok",
247
- "message": "",
248
- "startTimestamp": 1594872947467,
249
- "endTimestamp": 1594872949617,
250
- "version": "1.0.0"
251
- }
252
- ```
253
-
254
- ### + `request.post`
255
-
256
- This is the same as `request.get` but it takes one more param:
257
-
258
- | Parameter | Notes |
259
- |-----------|--------------------------------------------------------------------------|
260
- | postData | Must be a string with `application/x-www-form-urlencoded`. Eg: `a=b&c=d` |
261
-
262
- ## Environment variables
263
-
264
- | Name | Default | Notes |
265
- |--------------------|------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------|
266
- | LOG_LEVEL | info | Verbosity of the logging. Use `LOG_LEVEL=debug` for more information. |
267
- | LOG_HTML | false | Only for debugging. If `true` all HTML that passes through the proxy will be logged to the console in `debug` level. |
268
- | CAPTCHA_SOLVER | none | Captcha solving method. It is used when a captcha is encountered. See the Captcha Solvers section. |
269
- | TZ | UTC | Timezone used in the logs and the web browser. Example: `TZ=Europe/London`. |
270
- | LANG | none | Language used in the web browser. Example: `LANG=en_GB`. |
271
- | HEADLESS | true | Only for debugging. To run the web browser in headless mode or visible. |
272
- | BROWSER_TIMEOUT | 40000 | If you are experiencing errors/timeouts because your system is slow, you can try to increase this value. Remember to increase the `maxTimeout` parameter too. |
273
- | TEST_URL | https://www.google.com | FlareSolverr makes a request on start to make sure the web browser is working. You can change that URL if it is blocked in your country. |
274
- | PORT | 8191 | Listening port. You don't need to change this if you are running on Docker. |
275
- | HOST | 0.0.0.0 | Listening interface. You don't need to change this if you are running on Docker. |
276
- | PROMETHEUS_ENABLED | false | Enable Prometheus exporter. See the Prometheus section below. |
277
- | PROMETHEUS_PORT | 8192 | Listening port for Prometheus exporter. See the Prometheus section below. |
278
-
279
- Environment variables are set differently depending on the operating system. Some examples:
280
- * Docker: Take a look at the Docker section in this document. Environment variables can be set in the `docker-compose.yml` file or in the Docker CLI command.
281
- * Linux: Run `export LOG_LEVEL=debug` and then run `flaresolverr` in the same shell.
282
- * Windows: Open `cmd.exe`, run `set LOG_LEVEL=debug` and then run `flaresolverr.exe` in the same shell.
283
-
284
- ## Prometheus exporter
285
-
286
- The Prometheus exporter for FlareSolverr is disabled by default. It can be enabled with the environment variable `PROMETHEUS_ENABLED`. If you are using Docker make sure you expose the `PROMETHEUS_PORT`.
287
-
288
- Example metrics:
289
- ```shell
290
- # HELP flaresolverr_request_total Total requests with result
291
- # TYPE flaresolverr_request_total counter
292
- flaresolverr_request_total{domain="nowsecure.nl",result="solved"} 1.0
293
- # HELP flaresolverr_request_created Total requests with result
294
- # TYPE flaresolverr_request_created gauge
295
- flaresolverr_request_created{domain="nowsecure.nl",result="solved"} 1.690141657157109e+09
296
- # HELP flaresolverr_request_duration Request duration in seconds
297
- # TYPE flaresolverr_request_duration histogram
298
- flaresolverr_request_duration_bucket{domain="nowsecure.nl",le="0.0"} 0.0
299
- flaresolverr_request_duration_bucket{domain="nowsecure.nl",le="10.0"} 1.0
300
- flaresolverr_request_duration_bucket{domain="nowsecure.nl",le="25.0"} 1.0
301
- flaresolverr_request_duration_bucket{domain="nowsecure.nl",le="50.0"} 1.0
302
- flaresolverr_request_duration_bucket{domain="nowsecure.nl",le="+Inf"} 1.0
303
- flaresolverr_request_duration_count{domain="nowsecure.nl"} 1.0
304
- flaresolverr_request_duration_sum{domain="nowsecure.nl"} 5.858
305
- # HELP flaresolverr_request_duration_created Request duration in seconds
306
- # TYPE flaresolverr_request_duration_created gauge
307
- flaresolverr_request_duration_created{domain="nowsecure.nl"} 1.6901416571570296e+09
308
- ```
309
-
310
- ## Captcha Solvers
311
-
312
- > **Warning**
313
- > At this time none of the captcha solvers work. You can check the status in the open issues. Any help is welcome.
314
-
315
- Sometimes CloudFlare not only gives mathematical computations and browser tests, sometimes they also require the user to
316
- solve a captcha.
317
- If this is the case, FlareSolverr will return the error `Captcha detected but no automatic solver is configured.`
318
-
319
- FlareSolverr can be customized to solve the CAPTCHA automatically by setting the environment variable `CAPTCHA_SOLVER`
320
- to the file name of one of the adapters inside the [/captcha](src/captcha) directory.
321
-
322
- ## Related projects
323
-
324
- * C# implementation => https://github.com/FlareSolverr/FlareSolverrSharp
 
1
+ # FlareSolverr
2
+
3
+ [![Latest release](https://img.shields.io/github/v/release/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/releases)
4
+ [![Docker Pulls](https://img.shields.io/docker/pulls/flaresolverr/flaresolverr)](https://hub.docker.com/r/flaresolverr/flaresolverr/)
5
+ [![GitHub issues](https://img.shields.io/github/issues/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/issues)
6
+ [![GitHub pull requests](https://img.shields.io/github/issues-pr/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/pulls)
7
+ [![Donate PayPal](https://img.shields.io/badge/Donate-PayPal-yellow.svg)](https://www.paypal.com/paypalme/diegoheras0xff)
8
+ [![Donate Bitcoin](https://img.shields.io/badge/Donate-Bitcoin-f7931a.svg)](https://www.blockchain.com/btc/address/13Hcv77AdnFWEUZ9qUpoPBttQsUT7q9TTh)
9
+ [![Donate Ethereum](https://img.shields.io/badge/Donate-Ethereum-8c8c8c.svg)](https://www.blockchain.com/eth/address/0x0D1549BbB00926BF3D92c1A8A58695e982f1BE2E)
10
+
11
+ FlareSolverr is a proxy server to bypass Cloudflare and DDoS-GUARD protection.
12
+
13
+ ## How it works
14
+
15
+ FlareSolverr starts a proxy server, and it waits for user requests in an idle state using few resources.
16
+ When some request arrives, it uses [Selenium](https://www.selenium.dev) with the
17
+ [undetected-chromedriver](https://github.com/ultrafunkamsterdam/undetected-chromedriver)
18
+ to create a web browser (Chrome). It opens the URL with user parameters and waits until the Cloudflare challenge
19
+ is solved (or timeout). The HTML code and the cookies are sent back to the user, and those cookies can be used to
20
+ bypass Cloudflare using other HTTP clients.
21
+
22
+ **NOTE**: Web browsers consume a lot of memory. If you are running FlareSolverr on a machine with few RAM, do not make
23
+ many requests at once. With each request a new browser is launched.
24
+
25
+ It is also possible to use a permanent session. However, if you use sessions, you should make sure to close them as
26
+ soon as you are done using them.
27
+
28
+ ## Installation
29
+
30
+ ### Docker
31
+
32
+ It is recommended to install using a Docker container because the project depends on an external browser that is
33
+ already included within the image.
34
+
35
+ Docker images are available in:
36
+ * GitHub Registry => https://github.com/orgs/FlareSolverr/packages/container/package/flaresolverr
37
+ * DockerHub => https://hub.docker.com/r/flaresolverr/flaresolverr
38
+
39
+ Supported architectures are:
40
+
41
+ | Architecture | Tag |
42
+ |--------------|--------------|
43
+ | x86 | linux/386 |
44
+ | x86-64 | linux/amd64 |
45
+ | ARM32 | linux/arm/v7 |
46
+ | ARM64 | linux/arm64 |
47
+
48
+ We provide a `docker-compose.yml` configuration file. Clone this repository and execute
49
+ `docker-compose up -d` _(Compose V1)_ or `docker compose up -d` _(Compose V2)_ to start
50
+ the container.
51
+
52
+ If you prefer the `docker cli` execute the following command.
53
+ ```bash
54
+ docker run -d \
55
+ --name=flaresolverr \
56
+ -p 8191:8191 \
57
+ -e LOG_LEVEL=info \
58
+ --restart unless-stopped \
59
+ ghcr.io/flaresolverr/flaresolverr:latest
60
+ ```
61
+
62
+ If your host OS is Debian, make sure `libseccomp2` version is 2.5.x. You can check the version with `sudo apt-cache policy libseccomp2`
63
+ and update the package with `sudo apt install libseccomp2=2.5.1-1~bpo10+1` or `sudo apt install libseccomp2=2.5.1-1+deb11u1`.
64
+ Remember to restart the Docker daemon and the container after the update.
65
+
66
+ ### Precompiled binaries
67
+
68
+ > **Warning**
69
+ > Precompiled binaries are only available for x64 architecture. For other architectures see Docker images.
70
+
71
+ This is the recommended way for Windows users.
72
+ * Download the [FlareSolverr executable](https://github.com/FlareSolverr/FlareSolverr/releases) from the release's page. It is available for Windows x64 and Linux x64.
73
+ * Execute FlareSolverr binary. In the environment variables section you can find how to change the configuration.
74
+
75
+ ### From source code
76
+
77
+ > **Warning**
78
+ > Installing from source code only works for x64 architecture. For other architectures see Docker images.
79
+
80
+ * Install [Python 3.11](https://www.python.org/downloads/).
81
+ * Install [Chrome](https://www.google.com/intl/en_us/chrome/) (all OS) or [Chromium](https://www.chromium.org/getting-involved/download-chromium/) (just Linux, it doesn't work in Windows) web browser.
82
+ * (Only in Linux) Install [Xvfb](https://en.wikipedia.org/wiki/Xvfb) package.
83
+ * (Only in macOS) Install [XQuartz](https://www.xquartz.org/) package.
84
+ * Clone this repository and open a shell in that path.
85
+ * Run `pip install -r requirements.txt` command to install FlareSolverr dependencies.
86
+ * Run `python src/flaresolverr.py` command to start FlareSolverr.
87
+
88
+ ### From source code (FreeBSD/TrueNAS CORE)
89
+
90
+ * Run `pkg install chromium python39 py39-pip xorg-vfbserver` command to install the required dependencies.
91
+ * Clone this repository and open a shell in that path.
92
+ * Run `python3.9 -m pip install -r requirements.txt` command to install FlareSolverr dependencies.
93
+ * Run `python3.9 src/flaresolverr.py` command to start FlareSolverr.
94
+
95
+ ### Systemd service
96
+
97
+ We provide an example Systemd unit file `flaresolverr.service` as reference. You have to modify the file to suit your needs: paths, user and environment variables.
98
+
99
+ ## Usage
100
+
101
+ Example Bash request:
102
+ ```bash
103
+ curl -L -X POST 'http://localhost:8191/v1' \
104
+ -H 'Content-Type: application/json' \
105
+ --data-raw '{
106
+ "cmd": "request.get",
107
+ "url": "http://www.google.com/",
108
+ "maxTimeout": 60000
109
+ }'
110
+ ```
111
+
112
+ Example Python request:
113
+ ```py
114
+ import requests
115
+
116
+ url = "http://localhost:8191/v1"
117
+ headers = {"Content-Type": "application/json"}
118
+ data = {
119
+ "cmd": "request.get",
120
+ "url": "http://www.google.com/",
121
+ "maxTimeout": 60000
122
+ }
123
+ response = requests.post(url, headers=headers, json=data)
124
+ print(response.text)
125
+ ```
126
+
127
+ Example PowerShell request:
128
+ ```ps1
129
+ $body = @{
130
+ cmd = "request.get"
131
+ url = "http://www.google.com/"
132
+ maxTimeout = 60000
133
+ } | ConvertTo-Json
134
+
135
+ irm -UseBasicParsing 'http://localhost:8191/v1' -Headers @{"Content-Type"="application/json"} -Method Post -Body $body
136
+ ```
137
+
138
+ ### Commands
139
+
140
+ #### + `sessions.create`
141
+
142
+ This will launch a new browser instance which will retain cookies until you destroy it with `sessions.destroy`.
143
+ This comes in handy, so you don't have to keep solving challenges over and over and you won't need to keep sending
144
+ cookies for the browser to use.
145
+
146
+ This also speeds up the requests since it won't have to launch a new browser instance for every request.
147
+
148
+ | Parameter | Notes |
149
+ |-----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
150
+ | session | Optional. The session ID that you want to be assigned to the instance. If isn't set a random UUID will be assigned. |
151
+ | proxy | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is supported. Eg: `"proxy": {"url": "http://127.0.0.1:8888", "username": "testuser", "password": "testpass"}` |
152
+
153
+ #### + `sessions.list`
154
+
155
+ Returns a list of all the active sessions. More for debugging if you are curious to see how many sessions are running.
156
+ You should always make sure to properly close each session when you are done using them as too many may slow your
157
+ computer down.
158
+
159
+ Example response:
160
+
161
+ ```json
162
+ {
163
+ "sessions": [
164
+ "session_id_1",
165
+ "session_id_2",
166
+ "session_id_3..."
167
+ ]
168
+ }
169
+ ```
170
+
171
+ #### + `sessions.destroy`
172
+
173
+ This will properly shutdown a browser instance and remove all files associated with it to free up resources for a new
174
+ session. When you no longer need to use a session you should make sure to close it.
175
+
176
+ | Parameter | Notes |
177
+ |-----------|-----------------------------------------------|
178
+ | session | The session ID that you want to be destroyed. |
179
+
180
+ #### + `request.get`
181
+
182
+ | Parameter | Notes |
183
+ |---------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
184
+ | url | Mandatory |
185
+ | session | Optional. Will send the request from and existing browser instance. If one is not sent it will create a temporary instance that will be destroyed immediately after the request is completed. |
186
+ | session_ttl_minutes | Optional. FlareSolverr will automatically rotate expired sessions based on the TTL provided in minutes. |
187
+ | maxTimeout | Optional, default value 60000. Max timeout to solve the challenge in milliseconds. |
188
+ | cookies | Optional. Will be used by the headless browser. Eg: `"cookies": [{"name": "cookie1", "value": "value1"}, {"name": "cookie2", "value": "value2"}]`. |
189
+ | returnOnlyCookies | Optional, default false. Only returns the cookies. Response data, headers and other parts of the response are removed. |
190
+ | proxy | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is not supported. (When the `session` parameter is set, the proxy is ignored; a session specific proxy can be set in `sessions.create`.) |
191
+
192
+ > **Warning**
193
+ > If you want to use Cloudflare clearance cookie in your scripts, make sure you use the FlareSolverr User-Agent too. If they don't match you will see the challenge.
194
+
195
+ Example response from running the `curl` above:
196
+
197
+ ```json
198
+ {
199
+ "solution": {
200
+ "url": "https://www.google.com/?gws_rd=ssl",
201
+ "status": 200,
202
+ "headers": {
203
+ "status": "200",
204
+ "date": "Thu, 16 Jul 2020 04:15:49 GMT",
205
+ "expires": "-1",
206
+ "cache-control": "private, max-age=0",
207
+ "content-type": "text/html; charset=UTF-8",
208
+ "strict-transport-security": "max-age=31536000",
209
+ "p3p": "CP=\"This is not a P3P policy! See g.co/p3phelp for more info.\"",
210
+ "content-encoding": "br",
211
+ "server": "gws",
212
+ "content-length": "61587",
213
+ "x-xss-protection": "0",
214
+ "x-frame-options": "SAMEORIGIN",
215
+ "set-cookie": "1P_JAR=2020-07-16-04; expires=Sat..."
216
+ },
217
+ "response":"<!DOCTYPE html>...",
218
+ "cookies": [
219
+ {
220
+ "name": "NID",
221
+ "value": "204=QE3Ocq15XalczqjuDy52HeseG3zAZuJzID3R57...",
222
+ "domain": ".google.com",
223
+ "path": "/",
224
+ "expires": 1610684149.307722,
225
+ "size": 178,
226
+ "httpOnly": true,
227
+ "secure": true,
228
+ "session": false,
229
+ "sameSite": "None"
230
+ },
231
+ {
232
+ "name": "1P_JAR",
233
+ "value": "2020-07-16-04",
234
+ "domain": ".google.com",
235
+ "path": "/",
236
+ "expires": 1597464949.307626,
237
+ "size": 19,
238
+ "httpOnly": false,
239
+ "secure": true,
240
+ "session": false,
241
+ "sameSite": "None"
242
+ }
243
+ ],
244
+ "userAgent": "Windows NT 10.0; Win64; x64) AppleWebKit/5..."
245
+ },
246
+ "status": "ok",
247
+ "message": "",
248
+ "startTimestamp": 1594872947467,
249
+ "endTimestamp": 1594872949617,
250
+ "version": "1.0.0"
251
+ }
252
+ ```
253
+
254
+ ### + `request.post`
255
+
256
+ This is the same as `request.get` but it takes one more param:
257
+
258
+ | Parameter | Notes |
259
+ |-----------|--------------------------------------------------------------------------|
260
+ | postData | Must be a string with `application/x-www-form-urlencoded`. Eg: `a=b&c=d` |
261
+
262
+ ## Environment variables
263
+
264
+ | Name | Default | Notes |
265
+ |--------------------|------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------|
266
+ | LOG_LEVEL | info | Verbosity of the logging. Use `LOG_LEVEL=debug` for more information. |
267
+ | LOG_HTML | false | Only for debugging. If `true` all HTML that passes through the proxy will be logged to the console in `debug` level. |
268
+ | CAPTCHA_SOLVER | none | Captcha solving method. It is used when a captcha is encountered. See the Captcha Solvers section. |
269
+ | TZ | UTC | Timezone used in the logs and the web browser. Example: `TZ=Europe/London`. |
270
+ | LANG | none | Language used in the web browser. Example: `LANG=en_GB`. |
271
+ | HEADLESS | true | Only for debugging. To run the web browser in headless mode or visible. |
272
+ | BROWSER_TIMEOUT | 40000 | If you are experiencing errors/timeouts because your system is slow, you can try to increase this value. Remember to increase the `maxTimeout` parameter too. |
273
+ | TEST_URL | https://www.google.com | FlareSolverr makes a request on start to make sure the web browser is working. You can change that URL if it is blocked in your country. |
274
+ | PORT | 8191 | Listening port. You don't need to change this if you are running on Docker. |
275
+ | HOST | 0.0.0.0 | Listening interface. You don't need to change this if you are running on Docker. |
276
+ | PROMETHEUS_ENABLED | false | Enable Prometheus exporter. See the Prometheus section below. |
277
+ | PROMETHEUS_PORT | 8192 | Listening port for Prometheus exporter. See the Prometheus section below. |
278
+
279
+ Environment variables are set differently depending on the operating system. Some examples:
280
+ * Docker: Take a look at the Docker section in this document. Environment variables can be set in the `docker-compose.yml` file or in the Docker CLI command.
281
+ * Linux: Run `export LOG_LEVEL=debug` and then run `flaresolverr` in the same shell.
282
+ * Windows: Open `cmd.exe`, run `set LOG_LEVEL=debug` and then run `flaresolverr.exe` in the same shell.
283
+
284
+ ## Prometheus exporter
285
+
286
+ The Prometheus exporter for FlareSolverr is disabled by default. It can be enabled with the environment variable `PROMETHEUS_ENABLED`. If you are using Docker make sure you expose the `PROMETHEUS_PORT`.
287
+
288
+ Example metrics:
289
+ ```shell
290
+ # HELP flaresolverr_request_total Total requests with result
291
+ # TYPE flaresolverr_request_total counter
292
+ flaresolverr_request_total{domain="nowsecure.nl",result="solved"} 1.0
293
+ # HELP flaresolverr_request_created Total requests with result
294
+ # TYPE flaresolverr_request_created gauge
295
+ flaresolverr_request_created{domain="nowsecure.nl",result="solved"} 1.690141657157109e+09
296
+ # HELP flaresolverr_request_duration Request duration in seconds
297
+ # TYPE flaresolverr_request_duration histogram
298
+ flaresolverr_request_duration_bucket{domain="nowsecure.nl",le="0.0"} 0.0
299
+ flaresolverr_request_duration_bucket{domain="nowsecure.nl",le="10.0"} 1.0
300
+ flaresolverr_request_duration_bucket{domain="nowsecure.nl",le="25.0"} 1.0
301
+ flaresolverr_request_duration_bucket{domain="nowsecure.nl",le="50.0"} 1.0
302
+ flaresolverr_request_duration_bucket{domain="nowsecure.nl",le="+Inf"} 1.0
303
+ flaresolverr_request_duration_count{domain="nowsecure.nl"} 1.0
304
+ flaresolverr_request_duration_sum{domain="nowsecure.nl"} 5.858
305
+ # HELP flaresolverr_request_duration_created Request duration in seconds
306
+ # TYPE flaresolverr_request_duration_created gauge
307
+ flaresolverr_request_duration_created{domain="nowsecure.nl"} 1.6901416571570296e+09
308
+ ```
309
+
310
+ ## Captcha Solvers
311
+
312
+ > **Warning**
313
+ > At this time none of the captcha solvers work. You can check the status in the open issues. Any help is welcome.
314
+
315
+ Sometimes CloudFlare not only gives mathematical computations and browser tests, sometimes they also require the user to
316
+ solve a captcha.
317
+ If this is the case, FlareSolverr will return the error `Captcha detected but no automatic solver is configured.`
318
+
319
+ FlareSolverr can be customized to solve the CAPTCHA automatically by setting the environment variable `CAPTCHA_SOLVER`
320
+ to the file name of one of the adapters inside the [/captcha](src/captcha) directory.
321
+
322
+ ## Related projects
323
+
324
+ * C# implementation => https://github.com/FlareSolverr/FlareSolverrSharp
docker-compose.yml CHANGED
@@ -1,15 +1,15 @@
1
- ---
2
- version: "2.1"
3
- services:
4
- flaresolverr:
5
- # DockerHub mirror flaresolverr/flaresolverr:latest
6
- image: ghcr.io/flaresolverr/flaresolverr:latest
7
- container_name: flaresolverr
8
- environment:
9
- - LOG_LEVEL=${LOG_LEVEL:-info}
10
- - LOG_HTML=${LOG_HTML:-false}
11
- - CAPTCHA_SOLVER=${CAPTCHA_SOLVER:-none}
12
- - TZ=Europe/London
13
- ports:
14
- - "${PORT:-8191}:8191"
15
- restart: unless-stopped
 
1
+ ---
2
+ version: "2.1"
3
+ services:
4
+ flaresolverr:
5
+ # DockerHub mirror flaresolverr/flaresolverr:latest
6
+ image: ghcr.io/flaresolverr/flaresolverr:latest
7
+ container_name: flaresolverr
8
+ environment:
9
+ - LOG_LEVEL=${LOG_LEVEL:-info}
10
+ - LOG_HTML=${LOG_HTML:-false}
11
+ - CAPTCHA_SOLVER=${CAPTCHA_SOLVER:-none}
12
+ - TZ=Europe/London
13
+ ports:
14
+ - "${PORT:-8191}:8191"
15
+ restart: unless-stopped
html_samples/cloudflare_captcha_hcaptcha_v1.html CHANGED
@@ -1,219 +1,219 @@
1
- <!DOCTYPE html>
2
- <html lang="en-US">
3
-
4
- <head>
5
- <title>Just a moment...</title>
6
- <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
7
- <meta http-equiv="X-UA-Compatible" content="IE=Edge">
8
- <meta name="robots" content="noindex,nofollow">
9
- <meta name="viewport" content="width=device-width,initial-scale=1">
10
- <link href="Just%20a%20moment_files/cf-errors.css" rel="stylesheet">
11
-
12
- <script>
13
- (function () {
14
- window._cf_chl_opt = {
15
- cvId: '2',
16
- cType: 'managed',
17
- cNounce: '67839',
18
- cRay: '732fbc436ab471ed',
19
- cHash: 'dce5bd920f3aa51',
20
- cUPMDTk: "\/search?q=2022&__cf_chl_tk=lkycIb1jDXlmFqiB7AXTwy38_EzYPvu79CCQyU9lhUE-1659201316-0-gaNycGzNCf0",
21
- cFPWv: 'g',
22
- cTTimeMs: '1000',
23
- cTplV: 2,
24
- cRq: {
25
- ru: 'aHR0cHM6Ly8wbWFnbmV0LmNvbS9zZWFyY2g/cT0yMDIy',
26
- ra: 'TW96aWxsYS81LjAgKFgxMTsgTGludXggeDg2XzY0OyBydjoxMDUuMCkgR2Vja28vMjAxMDAxMDEgRmlyZWZveC8xMDUuMA==',
27
- rm: 'R0VU',
28
- d: 'MqxNbbGfWazPaVMZ7GQRz02TV/pSUL9POWx0y4e7HFRwP1RTAxLc1RZRuHg+N/bGMuPj08kSx0UpcjEjMkSOqiU6I/64IDYbCJvey5rY07fkkljpZaYGTDZIoWdOWlgP3ky15ybZ42xMK4tfI1yJ+iFZCVgR6VBjJzi5I56j9Ijog2AvsoQW2TrguGpgKaT1LkhxWNElzBbvXWt1uyRgE19UQ9J/5vtxEwoh5wodHh7WE297n8uI1hpDgge2bDYQvwe+RDq3QAyhQOmymg+IIlt1y115v9R8k5ehT9TFY3vYvYnoJu9cOyHYprf9Z0jTNGxSTvLHYJbfq30Samu5fKfE0oZREZizvPUgUsJm2rRKkCY9VCdBkpO8vaUgIwIYkeWavtqdudjb3zEDBCD4cAH/xv3Bl1VRy2Qf7XlcbpElCOq06TDTQ1uGjyCqbVbvjesrOy0Dp2nXTjdfbkWvnN7mWpFlPUD7/41MUo9lc6V1Aj1Kjg6AKfVV4DUHpq6ZVnMHzrcPQLy4qD7CptcMpQKArZtJCRsUpgq8GWKJcU4dU8ZmyROAA+l+JEVnGbh2bsRdif4azh57OdjZfEKSa5c+AL3i66vyWAZCw9Wl6CAQdFTA+ixkbl8zKbCm8ulv',
29
- t: 'MTY1OTIwMTMxNi4zOTIwMDA=',
30
- m: '3l81qRkXiMTbjTzBtc0v1XwSheF46UfagbXVhYgbAVw=',
31
- i1: 'Iu5a1gH3p9igzqBwncow9g==',
32
- i2: 'PmNXozjc73unhnp/X0+kUQ==',
33
- zh: 'qP4bnGc6j96JlnjNSE7HmQci3S9L50bHFtm4bQRjjKU=',
34
- uh: 'SK3PXNkeRzZtkRARhJpbmZpCIiWQw6+5gpOE7vojWx4=',
35
- hh: 'azXzJl8Ou22g0nN/9idVUoB9EqZ7fLmkSdDRHM3Lkmw=',
36
- }
37
- }
38
- window._cf_chl_enter = function () { window._cf_chl_opt.p = 1 };
39
- })();
40
- </script>
41
-
42
- <script src="Just%20a%20moment_files/v1.js"></script>
43
- <script type="text/javascript" src="Just%20a%20moment_files/api.js"></script>
44
- </head>
45
-
46
- <body class="no-js">
47
-
48
- <div class="privacy-pass">
49
- <a rel="noopener noreferrer" href="https://addons.mozilla.org/en-US/firefox/addon/privacy-pass/"
50
- target="_blank">
51
- Privacy Pass
52
- <span class="privacy-pass-icon-wrapper">
53
- <div class="privacy-pass-icon"></div>
54
- </span>
55
- </a>
56
- </div>
57
-
58
- <div class="main-wrapper" role="main">
59
- <div class="main-content">
60
- <h1 class="zone-name-title h1">
61
- <img class="heading-favicon" src="Just%20a%20moment_files/favicon.ico"
62
- onerror="this.onerror=null;this.parentNode.removeChild(this)">
63
- 0MAGNET.COM
64
- </h1>
65
- <h2 class="h2" id="cf-challenge-running">
66
- Checking if the site connection is secure
67
- </h2>
68
- <div id="cf-challenge-stage" style="display: block;">
69
- <div id="cf-challenge-hcaptcha-wrapper" class="captcha-prompt spacer">
70
- <div style="display: none;" class="hcaptcha-box"><iframe src="Just%20a%20moment_files/hcaptcha.html"
71
- title="widget containing checkbox for hCaptcha security challenge" tabindex="0"
72
- scrolling="no" data-hcaptcha-widget-id="0tiueg8lyuj" data-hcaptcha-response=""
73
- style="width: 303px; height: 78px; overflow: hidden;" frameborder="0"></iframe><textarea
74
- id="h-captcha-response-0tiueg8lyuj" name="h-captcha-response"
75
- style="display: none;"></textarea></div>
76
- <div class="hcaptcha-box"><iframe src="Just%20a%20moment_files/hcaptcha_002.html"
77
- title="widget containing checkbox for hCaptcha security challenge" tabindex="0"
78
- scrolling="no" data-hcaptcha-widget-id="10tlmhzz0qyq" data-hcaptcha-response=""
79
- style="width: 303px; height: 78px; overflow: hidden;" frameborder="0"></iframe><textarea
80
- id="h-captcha-response-10tlmhzz0qyq" name="h-captcha-response"
81
- style="display: none;"></textarea></div>
82
- </div>
83
- </div>
84
- <div id="cf-challenge-spinner" class="spacer loading-spinner" style="display: none; visibility: hidden;">
85
- <div class="lds-ring">
86
- <div></div>
87
- <div></div>
88
- <div></div>
89
- <div></div>
90
- </div>
91
- </div>
92
- <noscript>
93
- <div id="cf-challenge-error-title">
94
- <div class="h2">
95
- <span class="icon-wrapper">
96
- <div class="heading-icon warning-icon"></div>
97
- </span>
98
- <span id="cf-challenge-error-text">
99
- Enable JavaScript and cookies to continue
100
- </span>
101
- </div>
102
- </div>
103
- </noscript>
104
- <div
105
- style="display:none;background-image:url('/cdn-cgi/images/trace/captcha/nojs/transparent.gif?ray=732fbc436ab471ed')">
106
- </div>
107
- <div id="cf-challenge-body-text" class="core-msg spacer">
108
- 0magnet.com needs to review the security of your connection before
109
- proceeding.
110
- </div>
111
- <div id="cf-challenge-fact-wrapper" style="display: block; visibility: visible;" class="fact spacer hidden">
112
- <span class="fact-title">Did you know</span> <span id="cf-challenge-fact" class="body-text">the first
113
- botnet in 2003 took over 500-1000 devices? Today, botnets take over millions of devices at
114
- once.</span>
115
- </div>
116
- <div id="cf-challenge-explainer-expandable" class="hidden expandable body-text spacer"
117
- style="display: block; visibility: visible;">
118
- <div class="expandable-title" id="cf-challenge-explainer-summary"><button class="expandable-summary-btn"
119
- id="cf-challenge-explainer-btn" type="button"> Why am I seeing this page? <span
120
- class="caret-icon-wrapper">
121
- <div class="caret-icon"></div>
122
- </span> </button> </div>
123
- <div class="expandable-details" id="cf-challenge-explainer-details">
124
- Requests from malicious bots can pose as legitimate traffic.
125
- Occasionally, you may see this page while the site ensures that the
126
- connection is secure.</div>
127
- </div>
128
- <div id="cf-challenge-success" style="display: none;">
129
- <div class="h2"><span class="icon-wrapper"><img class="heading-icon" alt="Success icon"
130
- src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADQAAAA0CAMAAADypuvZAAAANlBMVEUAAAAxMTEwMDAxMTExMTEwMDAwMDAwMDAxMTExMTExMTEwMDAwMDAxMTExMTEwMDAwMDAxMTHB9N+uAAAAEXRSTlMA3zDvfyBAEJC/n3BQz69gX7VMkcMAAAGySURBVEjHnZZbFoMgDEQJiDzVuv/NtgbtFGuQ4/zUKpeMIQbUhXSKE5l1XSn4pFWHRm/WShT1HRLWC01LGxFEVkCc30eYkLJ1Sjk9pvkw690VY6k8DWP9OM9yMG0Koi+mi8XA36NXmW0UXra4eJ3iwHfrfXVlgL0NqqGBHdqfeQhMmyJ48WDuKP81h3+SMPeRKkJcSXiLUK4XTHCjESOnz1VUXQoc6lgi2x4cI5aTQ201Mt8wHysI5fc05M5c81uZEtHcMKhxZ7iYEty1GfhLvGKpm+EYkdGxm1F5axmcB93DoORIbXfdN7f+hlFuyxtDP+sxtBnF43cIYwaZAWRgzxIoiXEMESoPlMhwLRDXeK772CAzXEdBRV7cmnoVBp0OSlyGidEzJTFq5hhcsA5388oSGM6b5p+qjpZrBlMS9xj4AwXmz108ukU1IomM3ceiW0CDwHCqp1NjAqXlFrbga+xuloQJ+tuyfbIBPNpqnmxqT7dPaOnZqBfhSBCteJAxWj58zLk2xgg+SPGYM6dRO6WczSnIxxwEExRaO+UyCUhbOp7CGQ+kxSUfNtLQFC+Po29vvy7jj4y0yAAAAABJRU5ErkJggg=="></span>Connection
131
- is secure</div>
132
- <div class="core-msg spacer">Proceeding...</div>
133
- </div>
134
- <form id="challenge-form"
135
- action="/search?q=2022&amp;__cf_chl_f_tk=lkycIb1jDXlmFqiB7AXTwy38_EzYPvu79CCQyU9lhUE-1659201316-0-gaNycGzNCf0"
136
- method="POST" enctype="application/x-www-form-urlencoded">
137
- <input type="hidden" name="md"
138
- value="P4fDbSohR3e3VZmGdBSN0Gd8t8ueht.ZVgSdQYwa45Y-1659201316-0-AesEKnKN8eJLiLESJle3R0T3fwKbVMlX09CR0sIU1LruDXen0nSlT2a5OpMUFYR7HQMGcF9Ja227n2p2D2ffUlWHPVeFX-YSNiewLZA3XuAQmOn-1DyWKA-SaMH_MW2vOSC7PCHAdJDhoRWjM_o3MyKziopj3WmDcaCI_ikk68bJTIValZ_e9tO7hmHC8zjsxDC8kXmI0tbrhyW5nyS2hRlx_ZVRcRHbHsVRN0-FGtEbCoaHmnp-q0N4AYhCJXofYRunPcSG_Y1iWMk-7ofOXON_gO7oGG_8-WWD5EG1jaz2ldpNO1RTkS7dQvTiC1Io1qAsVnQtokEaDR2zoWK_MF-hz6tOmuJIDgnAoH6vPFAa9EyJOUiG2RV-3q1CKTUgr82XRJw5CaXpN0QeBq0xHxFl5mzkFO8xqQsRnPkGUKtxBQ58syPIhR4AvNp8HA028gUNmaztJZ9i2UcWydut4VghHsoJjS5DEKTamjJhNrrkargjXUekXTfKXMVKCXxo0NFObTmKwzsNB5hrk3M43KzZCOOgTnqsrVUk54bAeDsr4qmTVW2wVk-0u78QpV2JFFOIJxRLikPmqo9CUokgUJ_IPsEjA5Q3kjrf9yq2OHU0MkwzLFNOAyc5N3A4WSYp91kESwxM98qFetpAZ0R3LID2c2-MraHnpOI2Xn4bxbDIdUPmjy6VB8Huuuf6M-o3Tw">
139
- <input type="hidden" name="r"
140
- value="bdZ7.nm8dGOZxq3EDOv_Kx7nKVv68q7b0RARXAlR9kQ-1659201316-0-AawyK3x4GgWasA2OtBBEp9Ea52qs8zEWwnQJxLWUnC+1jqlxaKHTIHeVQjvrTl/ccu6QA41yrSTKvazKiv6zQEiDj/6ziYkhldx+oJ7SqgMzPozzza1jofsGpCCPAzIlDicF+7sh4WKOxUJOeHgHCgfEZF/MPNsaahvbQ10U8Ei9tmvj8c2tkoybya75Bj5XHPPu0S9hnOH7S24ltm9vmyHlttI7uuI962FzPCTGjuAl4R/5+06WVAzBCJrS4biDNIuyYe22PtLl4b3Yf55eW7AFgyzKgddsohZJuNNliKyD6cusHDhm7MYpnXc5zwTdCbt6KGK/tBaylNyYwH/WBAUhyRYN5EVt9/iIKHrb+P6Z0RL4nO3BtQE/Zwx1VC3g1Wy4PPQJjqLixQptzl5eIzu43JIO/LBvT/mWuheH4eoPlghvyMYwfHcs7B4d7FCv1Tj9Skp9Fcj6HBAZlq/ss/eIwk7oOcTviQs+EUF9/yYatgtpXX9RCyvhMU6/ghOLfXRmOpAzsmoGnVqEpc2IMlZegYtieLveXU35cGJMI6wCR2ciCJIX995vLuL/4BdCAMEhyMAUWxtaCD2ZfRHyOWKNuf80w9k6/Ofhu7RevCr2mjQJAVTyE2OWWgOUuYJ4pZim93J7slMXieL3S5/JM08Q8g179Of7dzpN/oG7s80ljxAiCprpUAwpEmNiqNJN//v0e9KxknhCHeAWSAe8IeXbp5PSEQHXTmsqOFRkpud1pTsETcNbdonk8XMyv8mZRcFPVWRRWUb8hupn/d+x9r6mOdKdJkH8ZZ0R30LG0SLPYEvsVr2yU9o+uCZrRWkuE3SP3Lq3BIx+0vtm0DOvj6cODxy5/4Zm4x7LIpSa9wr69Rs2x+t+U5ydUupZ7oiAbWfYZSXHpmB0zJYOLMPJZcut50J/IgWuTMda8QBcTG3jRr4BTwpcmBZRmddfOJYgD7EMpOi1HgwLnS7l5QELafaMn0Hl6G774GVy4lEK2jURG9IEE3PV1m5Y903pqldFkJQsMxdisJWOzVjbtf41fxxnt4cQgiDQhktqCwg8xP6ijzPeWgvQHL5fMq61cQ5/4HB+yt9wKWMlBfUJR+ocI2MYx2nUWz+0BwCnTU29D9bx1xkir9bsnUnfOlfRDO2OEvI0iTe82666rVQO9XqTEz3POxrJYzLcSC9fTHpHfmCVwT2zWGGLi6pW5kqZh/uzSQ12MSvF5+dwvhe7yRks5gwMhnDHMQFyxKw3Xxm+dq2Ix/1uUucOhCu2L72j/NIwkF3Z7O7afY9nIu+NqMe8PbPJjq5ovEluosQfAMzWJH5Va8iur8o6K1y6hm7XFdNYAR+uCtxMw6WzF58QWVXXrDvfPeBMaNz+VVCnGP9elAwv62tc4Uh2SCbKbWdZchtLHgrJgYgQtCMhBDh0AXzE6ubbtfm9jE2vWcPj5jbo8U72i1pL2j8Xfr562Xc2WrQ7tKvSFQepGxfu2XgF7q55XKVqrnrBeXxZViUkB/gyXxI26CfrVfPLW+sYUo3JS+eCjyn2K7phv+630ixdpKrRJCTmkP3G8tcoLTJCB67/pbz+dXiNSB4JlHf4i3FVRkr8TAWS2zuMjJhB+ZyxnrGq/m7KSwpEEqgSCpOrQ5nkeoKIOyITfe9EPBSy9QtYDK+SAhUiLnICVURK7kGgrhZuKyK5/nyK9l7ffg16aaChJBisPBeiYsTDHlAeq0GbW7VR/jQDAVtVldeyD/dM5rJ4X+wl3A+faYD1OUxYT3n8dMs+E/1jLnYixXJpo7iXCqlTV3phOatg4XDQ5Bj6EYQIljVI4x2e8XHspcETIa0WepLsZF7WUtY7KbN8ZyDBFXgTMb4lzPmWyY8hZ05uX8EBKqUJhWh91AUob/OpJdf+u3axDDeRgsjl8K6CFM/5uQKo93co3KPqGZiqx0JoVj1t6KGxkrzYsgwlrTyeL44cEgr0zRQz5oFExuwHGYyogbHZ8EvU1eoiJ3IuQFxUH/1ULidfGtB371RYfz9gqONi1KiVzJ+zLjw+4HgMKXOV+ra09Dyg+eyUNfHillLXkhKWVoDpUhc+r46W6vXFp3oMKUWTRM0dE7iHofo+0tHb73d3ID2blRXgUeoMCQwOptoAYlFBIUYjggrIhd1AMC8TiZmiNULyP5imDePwcfq+ZjGH3o8VKRI2FcoFmChQegGco6pEbB5DxCguDuJbFRwGH4t9T0y74ZhlZiTNKA4xXsQnfIBEC5qz3mkcDAWoe73zqFAjp35JRVBjo3UDvehJppxzuoCXt9UbeuNEGll5/YJR4lfbUsEai0U6TFVleTTY53ofYCWEM6EnNDIToTFbm514YFTUSc4h8Qlq2fPeqC3IcCmirNT4Kf0FCO7MQrtGNFPJme2cpb/pZguS3pxxkKb4lOS+eGiUBGcSs1v3zHroJ+hum4wTJFRG0Yb99aCVQU44wgV3nKW7FZkXzwO3QY7nnkFI2kaAXerCPF4+Ho463g==">
141
-
142
- <span style="display: none;"><span class="text-gray-600" data-translate="error">error code:
143
- 1020</span></span>
144
- </form>
145
- </div>
146
- </div>
147
- <script>
148
- (function () {
149
- var trkjs = document.createElement('img');
150
- trkjs.setAttribute('src', '/cdn-cgi/images/trace/captcha/js/transparent.gif?ray=732fbc436ab471ed');
151
- trkjs.setAttribute('style', 'display: none');
152
- document.body.appendChild(trkjs);
153
- var cpo = document.createElement('script');
154
- cpo.src = '/cdn-cgi/challenge-platform/h/g/orchestrate/managed/v1?ray=732fbc436ab471ed';
155
- window._cf_chl_opt.cOgUHash = location.hash === '' && location.href.indexOf('#') !== -1 ? '#' : location.hash;
156
- window._cf_chl_opt.cOgUQuery = location.search === '' && location.href.slice(0, -window._cf_chl_opt.cOgUHash.length).indexOf('?') !== -1 ? '?' : location.search;
157
- if (window.history && window.history.replaceState) {
158
- var ogU = location.pathname + window._cf_chl_opt.cOgUQuery + window._cf_chl_opt.cOgUHash;
159
- history.replaceState(null, null, "\/search?q=2022&__cf_chl_rt_tk=lkycIb1jDXlmFqiB7AXTwy38_EzYPvu79CCQyU9lhUE-1659201316-0-gaNycGzNCf0" + window._cf_chl_opt.cOgUHash);
160
- cpo.onload = function () {
161
- history.replaceState(null, null, ogU);
162
- };
163
- }
164
- document.getElementsByTagName('head')[0].appendChild(cpo);
165
- }());
166
- </script><img src="Just%20a%20moment_files/transparent.gif" style="display: none">
167
-
168
- <div class="footer" role="contentinfo">
169
- <div class="footer-inner">
170
- <div class="clearfix diagnostic-wrapper">
171
- <div class="ray-id">Ray ID: <code>732fbc436ab471ed</code></div>
172
- </div>
173
- <div class="text-center">
174
- Performance &amp; security by
175
- <a rel="noopener noreferrer" href="https://www.cloudflare.com/" target="_blank">Cloudflare</a>
176
- </div>
177
- </div>
178
- </div>
179
-
180
-
181
- <div style="background-color: rgb(255, 255, 255); border: 1px solid rgb(215, 215, 215); box-shadow: rgba(0, 0, 0, 0.1) 0px 0px 4px; border-radius: 4px; left: -10000px; top: -10000px; z-index: -2147483648; position: absolute; transition: opacity 0.15s ease-out 0s; opacity: 0; visibility: hidden;"
182
- aria-hidden="true">
183
- <div style="position: relative; z-index: 1;"><iframe src="Just%20a%20moment_files/hcaptcha_003.html"
184
- title="Main content of the hCaptcha challenge" scrolling="no"
185
- style="border: 0px none; z-index: 2000000000; position: relative;" frameborder="0"></iframe></div>
186
- <div
187
- style="width: 100%; height: 100%; position: fixed; pointer-events: none; top: 0px; left: 0px; z-index: 0; background-color: rgb(255, 255, 255); opacity: 0.05;">
188
- </div>
189
- <div
190
- style="border-width: 11px; position: absolute; pointer-events: none; margin-top: -11px; z-index: 1; right: 100%;">
191
- <div
192
- style="border-width: 10px; border-style: solid; border-color: transparent rgb(255, 255, 255) transparent transparent; position: relative; top: 10px; z-index: 1;">
193
- </div>
194
- <div
195
- style="border-width: 11px; border-style: solid; border-color: transparent rgb(215, 215, 215) transparent transparent; position: relative; top: -11px; z-index: 0;">
196
- </div>
197
- </div>
198
- </div>
199
- <div style="background-color: rgb(255, 255, 255); border: 1px solid rgb(215, 215, 215); box-shadow: rgba(0, 0, 0, 0.1) 0px 0px 4px; border-radius: 4px; left: -10000px; top: -10000px; z-index: -2147483648; position: absolute; transition: opacity 0.15s ease-out 0s; opacity: 0; visibility: hidden;"
200
- aria-hidden="true">
201
- <div style="position: relative; z-index: 1;"><iframe src="Just%20a%20moment_files/hcaptcha_004.html"
202
- title="Main content of the hCaptcha challenge" scrolling="no"
203
- style="border: 0px none; z-index: 2000000000; position: relative;" frameborder="0"></iframe></div>
204
- <div
205
- style="width: 100%; height: 100%; position: fixed; pointer-events: none; top: 0px; left: 0px; z-index: 0; background-color: rgb(255, 255, 255); opacity: 0.05;">
206
- </div>
207
- <div
208
- style="border-width: 11px; position: absolute; pointer-events: none; margin-top: -11px; z-index: 1; right: 100%;">
209
- <div
210
- style="border-width: 10px; border-style: solid; border-color: transparent rgb(255, 255, 255) transparent transparent; position: relative; top: 10px; z-index: 1;">
211
- </div>
212
- <div
213
- style="border-width: 11px; border-style: solid; border-color: transparent rgb(215, 215, 215) transparent transparent; position: relative; top: -11px; z-index: 0;">
214
- </div>
215
- </div>
216
- </div>
217
- </body>
218
-
219
  </html>
 
1
+ <!DOCTYPE html>
2
+ <html lang="en-US">
3
+
4
+ <head>
5
+ <title>Just a moment...</title>
6
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
7
+ <meta http-equiv="X-UA-Compatible" content="IE=Edge">
8
+ <meta name="robots" content="noindex,nofollow">
9
+ <meta name="viewport" content="width=device-width,initial-scale=1">
10
+ <link href="Just%20a%20moment_files/cf-errors.css" rel="stylesheet">
11
+
12
+ <script>
13
+ (function () {
14
+ window._cf_chl_opt = {
15
+ cvId: '2',
16
+ cType: 'managed',
17
+ cNounce: '67839',
18
+ cRay: '732fbc436ab471ed',
19
+ cHash: 'dce5bd920f3aa51',
20
+ cUPMDTk: "\/search?q=2022&__cf_chl_tk=lkycIb1jDXlmFqiB7AXTwy38_EzYPvu79CCQyU9lhUE-1659201316-0-gaNycGzNCf0",
21
+ cFPWv: 'g',
22
+ cTTimeMs: '1000',
23
+ cTplV: 2,
24
+ cRq: {
25
+ ru: 'aHR0cHM6Ly8wbWFnbmV0LmNvbS9zZWFyY2g/cT0yMDIy',
26
+ ra: 'TW96aWxsYS81LjAgKFgxMTsgTGludXggeDg2XzY0OyBydjoxMDUuMCkgR2Vja28vMjAxMDAxMDEgRmlyZWZveC8xMDUuMA==',
27
+ rm: 'R0VU',
28
+ d: 'MqxNbbGfWazPaVMZ7GQRz02TV/pSUL9POWx0y4e7HFRwP1RTAxLc1RZRuHg+N/bGMuPj08kSx0UpcjEjMkSOqiU6I/64IDYbCJvey5rY07fkkljpZaYGTDZIoWdOWlgP3ky15ybZ42xMK4tfI1yJ+iFZCVgR6VBjJzi5I56j9Ijog2AvsoQW2TrguGpgKaT1LkhxWNElzBbvXWt1uyRgE19UQ9J/5vtxEwoh5wodHh7WE297n8uI1hpDgge2bDYQvwe+RDq3QAyhQOmymg+IIlt1y115v9R8k5ehT9TFY3vYvYnoJu9cOyHYprf9Z0jTNGxSTvLHYJbfq30Samu5fKfE0oZREZizvPUgUsJm2rRKkCY9VCdBkpO8vaUgIwIYkeWavtqdudjb3zEDBCD4cAH/xv3Bl1VRy2Qf7XlcbpElCOq06TDTQ1uGjyCqbVbvjesrOy0Dp2nXTjdfbkWvnN7mWpFlPUD7/41MUo9lc6V1Aj1Kjg6AKfVV4DUHpq6ZVnMHzrcPQLy4qD7CptcMpQKArZtJCRsUpgq8GWKJcU4dU8ZmyROAA+l+JEVnGbh2bsRdif4azh57OdjZfEKSa5c+AL3i66vyWAZCw9Wl6CAQdFTA+ixkbl8zKbCm8ulv',
29
+ t: 'MTY1OTIwMTMxNi4zOTIwMDA=',
30
+ m: '3l81qRkXiMTbjTzBtc0v1XwSheF46UfagbXVhYgbAVw=',
31
+ i1: 'Iu5a1gH3p9igzqBwncow9g==',
32
+ i2: 'PmNXozjc73unhnp/X0+kUQ==',
33
+ zh: 'qP4bnGc6j96JlnjNSE7HmQci3S9L50bHFtm4bQRjjKU=',
34
+ uh: 'SK3PXNkeRzZtkRARhJpbmZpCIiWQw6+5gpOE7vojWx4=',
35
+ hh: 'azXzJl8Ou22g0nN/9idVUoB9EqZ7fLmkSdDRHM3Lkmw=',
36
+ }
37
+ }
38
+ window._cf_chl_enter = function () { window._cf_chl_opt.p = 1 };
39
+ })();
40
+ </script>
41
+
42
+ <script src="Just%20a%20moment_files/v1.js"></script>
43
+ <script type="text/javascript" src="Just%20a%20moment_files/api.js"></script>
44
+ </head>
45
+
46
+ <body class="no-js">
47
+
48
+ <div class="privacy-pass">
49
+ <a rel="noopener noreferrer" href="https://addons.mozilla.org/en-US/firefox/addon/privacy-pass/"
50
+ target="_blank">
51
+ Privacy Pass
52
+ <span class="privacy-pass-icon-wrapper">
53
+ <div class="privacy-pass-icon"></div>
54
+ </span>
55
+ </a>
56
+ </div>
57
+
58
+ <div class="main-wrapper" role="main">
59
+ <div class="main-content">
60
+ <h1 class="zone-name-title h1">
61
+ <img class="heading-favicon" src="Just%20a%20moment_files/favicon.ico"
62
+ onerror="this.onerror=null;this.parentNode.removeChild(this)">
63
+ 0MAGNET.COM
64
+ </h1>
65
+ <h2 class="h2" id="cf-challenge-running">
66
+ Checking if the site connection is secure
67
+ </h2>
68
+ <div id="cf-challenge-stage" style="display: block;">
69
+ <div id="cf-challenge-hcaptcha-wrapper" class="captcha-prompt spacer">
70
+ <div style="display: none;" class="hcaptcha-box"><iframe src="Just%20a%20moment_files/hcaptcha.html"
71
+ title="widget containing checkbox for hCaptcha security challenge" tabindex="0"
72
+ scrolling="no" data-hcaptcha-widget-id="0tiueg8lyuj" data-hcaptcha-response=""
73
+ style="width: 303px; height: 78px; overflow: hidden;" frameborder="0"></iframe><textarea
74
+ id="h-captcha-response-0tiueg8lyuj" name="h-captcha-response"
75
+ style="display: none;"></textarea></div>
76
+ <div class="hcaptcha-box"><iframe src="Just%20a%20moment_files/hcaptcha_002.html"
77
+ title="widget containing checkbox for hCaptcha security challenge" tabindex="0"
78
+ scrolling="no" data-hcaptcha-widget-id="10tlmhzz0qyq" data-hcaptcha-response=""
79
+ style="width: 303px; height: 78px; overflow: hidden;" frameborder="0"></iframe><textarea
80
+ id="h-captcha-response-10tlmhzz0qyq" name="h-captcha-response"
81
+ style="display: none;"></textarea></div>
82
+ </div>
83
+ </div>
84
+ <div id="cf-challenge-spinner" class="spacer loading-spinner" style="display: none; visibility: hidden;">
85
+ <div class="lds-ring">
86
+ <div></div>
87
+ <div></div>
88
+ <div></div>
89
+ <div></div>
90
+ </div>
91
+ </div>
92
+ <noscript>
93
+ <div id="cf-challenge-error-title">
94
+ <div class="h2">
95
+ <span class="icon-wrapper">
96
+ <div class="heading-icon warning-icon"></div>
97
+ </span>
98
+ <span id="cf-challenge-error-text">
99
+ Enable JavaScript and cookies to continue
100
+ </span>
101
+ </div>
102
+ </div>
103
+ </noscript>
104
+ <div
105
+ style="display:none;background-image:url('/cdn-cgi/images/trace/captcha/nojs/transparent.gif?ray=732fbc436ab471ed')">
106
+ </div>
107
+ <div id="cf-challenge-body-text" class="core-msg spacer">
108
+ 0magnet.com needs to review the security of your connection before
109
+ proceeding.
110
+ </div>
111
+ <div id="cf-challenge-fact-wrapper" style="display: block; visibility: visible;" class="fact spacer hidden">
112
+ <span class="fact-title">Did you know</span> <span id="cf-challenge-fact" class="body-text">the first
113
+ botnet in 2003 took over 500-1000 devices? Today, botnets take over millions of devices at
114
+ once.</span>
115
+ </div>
116
+ <div id="cf-challenge-explainer-expandable" class="hidden expandable body-text spacer"
117
+ style="display: block; visibility: visible;">
118
+ <div class="expandable-title" id="cf-challenge-explainer-summary"><button class="expandable-summary-btn"
119
+ id="cf-challenge-explainer-btn" type="button"> Why am I seeing this page? <span
120
+ class="caret-icon-wrapper">
121
+ <div class="caret-icon"></div>
122
+ </span> </button> </div>
123
+ <div class="expandable-details" id="cf-challenge-explainer-details">
124
+ Requests from malicious bots can pose as legitimate traffic.
125
+ Occasionally, you may see this page while the site ensures that the
126
+ connection is secure.</div>
127
+ </div>
128
+ <div id="cf-challenge-success" style="display: none;">
129
+ <div class="h2"><span class="icon-wrapper"><img class="heading-icon" alt="Success icon"
130
+ src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADQAAAA0CAMAAADypuvZAAAANlBMVEUAAAAxMTEwMDAxMTExMTEwMDAwMDAwMDAxMTExMTExMTEwMDAwMDAxMTExMTEwMDAwMDAxMTHB9N+uAAAAEXRSTlMA3zDvfyBAEJC/n3BQz69gX7VMkcMAAAGySURBVEjHnZZbFoMgDEQJiDzVuv/NtgbtFGuQ4/zUKpeMIQbUhXSKE5l1XSn4pFWHRm/WShT1HRLWC01LGxFEVkCc30eYkLJ1Sjk9pvkw690VY6k8DWP9OM9yMG0Koi+mi8XA36NXmW0UXra4eJ3iwHfrfXVlgL0NqqGBHdqfeQhMmyJ48WDuKP81h3+SMPeRKkJcSXiLUK4XTHCjESOnz1VUXQoc6lgi2x4cI5aTQ201Mt8wHysI5fc05M5c81uZEtHcMKhxZ7iYEty1GfhLvGKpm+EYkdGxm1F5axmcB93DoORIbXfdN7f+hlFuyxtDP+sxtBnF43cIYwaZAWRgzxIoiXEMESoPlMhwLRDXeK772CAzXEdBRV7cmnoVBp0OSlyGidEzJTFq5hhcsA5388oSGM6b5p+qjpZrBlMS9xj4AwXmz108ukU1IomM3ceiW0CDwHCqp1NjAqXlFrbga+xuloQJ+tuyfbIBPNpqnmxqT7dPaOnZqBfhSBCteJAxWj58zLk2xgg+SPGYM6dRO6WczSnIxxwEExRaO+UyCUhbOp7CGQ+kxSUfNtLQFC+Po29vvy7jj4y0yAAAAABJRU5ErkJggg=="></span>Connection
131
+ is secure</div>
132
+ <div class="core-msg spacer">Proceeding...</div>
133
+ </div>
134
+ <form id="challenge-form"
135
+ action="/search?q=2022&amp;__cf_chl_f_tk=lkycIb1jDXlmFqiB7AXTwy38_EzYPvu79CCQyU9lhUE-1659201316-0-gaNycGzNCf0"
136
+ method="POST" enctype="application/x-www-form-urlencoded">
137
+ <input type="hidden" name="md"
138
+ value="P4fDbSohR3e3VZmGdBSN0Gd8t8ueht.ZVgSdQYwa45Y-1659201316-0-AesEKnKN8eJLiLESJle3R0T3fwKbVMlX09CR0sIU1LruDXen0nSlT2a5OpMUFYR7HQMGcF9Ja227n2p2D2ffUlWHPVeFX-YSNiewLZA3XuAQmOn-1DyWKA-SaMH_MW2vOSC7PCHAdJDhoRWjM_o3MyKziopj3WmDcaCI_ikk68bJTIValZ_e9tO7hmHC8zjsxDC8kXmI0tbrhyW5nyS2hRlx_ZVRcRHbHsVRN0-FGtEbCoaHmnp-q0N4AYhCJXofYRunPcSG_Y1iWMk-7ofOXON_gO7oGG_8-WWD5EG1jaz2ldpNO1RTkS7dQvTiC1Io1qAsVnQtokEaDR2zoWK_MF-hz6tOmuJIDgnAoH6vPFAa9EyJOUiG2RV-3q1CKTUgr82XRJw5CaXpN0QeBq0xHxFl5mzkFO8xqQsRnPkGUKtxBQ58syPIhR4AvNp8HA028gUNmaztJZ9i2UcWydut4VghHsoJjS5DEKTamjJhNrrkargjXUekXTfKXMVKCXxo0NFObTmKwzsNB5hrk3M43KzZCOOgTnqsrVUk54bAeDsr4qmTVW2wVk-0u78QpV2JFFOIJxRLikPmqo9CUokgUJ_IPsEjA5Q3kjrf9yq2OHU0MkwzLFNOAyc5N3A4WSYp91kESwxM98qFetpAZ0R3LID2c2-MraHnpOI2Xn4bxbDIdUPmjy6VB8Huuuf6M-o3Tw">
139
+ <input type="hidden" name="r"
140
+ value="bdZ7.nm8dGOZxq3EDOv_Kx7nKVv68q7b0RARXAlR9kQ-1659201316-0-AawyK3x4GgWasA2OtBBEp9Ea52qs8zEWwnQJxLWUnC+1jqlxaKHTIHeVQjvrTl/ccu6QA41yrSTKvazKiv6zQEiDj/6ziYkhldx+oJ7SqgMzPozzza1jofsGpCCPAzIlDicF+7sh4WKOxUJOeHgHCgfEZF/MPNsaahvbQ10U8Ei9tmvj8c2tkoybya75Bj5XHPPu0S9hnOH7S24ltm9vmyHlttI7uuI962FzPCTGjuAl4R/5+06WVAzBCJrS4biDNIuyYe22PtLl4b3Yf55eW7AFgyzKgddsohZJuNNliKyD6cusHDhm7MYpnXc5zwTdCbt6KGK/tBaylNyYwH/WBAUhyRYN5EVt9/iIKHrb+P6Z0RL4nO3BtQE/Zwx1VC3g1Wy4PPQJjqLixQptzl5eIzu43JIO/LBvT/mWuheH4eoPlghvyMYwfHcs7B4d7FCv1Tj9Skp9Fcj6HBAZlq/ss/eIwk7oOcTviQs+EUF9/yYatgtpXX9RCyvhMU6/ghOLfXRmOpAzsmoGnVqEpc2IMlZegYtieLveXU35cGJMI6wCR2ciCJIX995vLuL/4BdCAMEhyMAUWxtaCD2ZfRHyOWKNuf80w9k6/Ofhu7RevCr2mjQJAVTyE2OWWgOUuYJ4pZim93J7slMXieL3S5/JM08Q8g179Of7dzpN/oG7s80ljxAiCprpUAwpEmNiqNJN//v0e9KxknhCHeAWSAe8IeXbp5PSEQHXTmsqOFRkpud1pTsETcNbdonk8XMyv8mZRcFPVWRRWUb8hupn/d+x9r6mOdKdJkH8ZZ0R30LG0SLPYEvsVr2yU9o+uCZrRWkuE3SP3Lq3BIx+0vtm0DOvj6cODxy5/4Zm4x7LIpSa9wr69Rs2x+t+U5ydUupZ7oiAbWfYZSXHpmB0zJYOLMPJZcut50J/IgWuTMda8QBcTG3jRr4BTwpcmBZRmddfOJYgD7EMpOi1HgwLnS7l5QELafaMn0Hl6G774GVy4lEK2jURG9IEE3PV1m5Y903pqldFkJQsMxdisJWOzVjbtf41fxxnt4cQgiDQhktqCwg8xP6ijzPeWgvQHL5fMq61cQ5/4HB+yt9wKWMlBfUJR+ocI2MYx2nUWz+0BwCnTU29D9bx1xkir9bsnUnfOlfRDO2OEvI0iTe82666rVQO9XqTEz3POxrJYzLcSC9fTHpHfmCVwT2zWGGLi6pW5kqZh/uzSQ12MSvF5+dwvhe7yRks5gwMhnDHMQFyxKw3Xxm+dq2Ix/1uUucOhCu2L72j/NIwkF3Z7O7afY9nIu+NqMe8PbPJjq5ovEluosQfAMzWJH5Va8iur8o6K1y6hm7XFdNYAR+uCtxMw6WzF58QWVXXrDvfPeBMaNz+VVCnGP9elAwv62tc4Uh2SCbKbWdZchtLHgrJgYgQtCMhBDh0AXzE6ubbtfm9jE2vWcPj5jbo8U72i1pL2j8Xfr562Xc2WrQ7tKvSFQepGxfu2XgF7q55XKVqrnrBeXxZViUkB/gyXxI26CfrVfPLW+sYUo3JS+eCjyn2K7phv+630ixdpKrRJCTmkP3G8tcoLTJCB67/pbz+dXiNSB4JlHf4i3FVRkr8TAWS2zuMjJhB+ZyxnrGq/m7KSwpEEqgSCpOrQ5nkeoKIOyITfe9EPBSy9QtYDK+SAhUiLnICVURK7kGgrhZuKyK5/nyK9l7ffg16aaChJBisPBeiYsTDHlAeq0GbW7VR/jQDAVtVldeyD/dM5rJ4X+wl3A+faYD1OUxYT3n8dMs+E/1jLnYixXJpo7iXCqlTV3phOatg4XDQ5Bj6EYQIljVI4x2e8XHspcETIa0WepLsZF7WUtY7KbN8ZyDBFXgTMb4lzPmWyY8hZ05uX8EBKqUJhWh91AUob/OpJdf+u3axDDeRgsjl8K6CFM/5uQKo93co3KPqGZiqx0JoVj1t6KGxkrzYsgwlrTyeL44cEgr0zRQz5oFExuwHGYyogbHZ8EvU1eoiJ3IuQFxUH/1ULidfGtB371RYfz9gqONi1KiVzJ+zLjw+4HgMKXOV+ra09Dyg+eyUNfHillLXkhKWVoDpUhc+r46W6vXFp3oMKUWTRM0dE7iHofo+0tHb73d3ID2blRXgUeoMCQwOptoAYlFBIUYjggrIhd1AMC8TiZmiNULyP5imDePwcfq+ZjGH3o8VKRI2FcoFmChQegGco6pEbB5DxCguDuJbFRwGH4t9T0y74ZhlZiTNKA4xXsQnfIBEC5qz3mkcDAWoe73zqFAjp35JRVBjo3UDvehJppxzuoCXt9UbeuNEGll5/YJR4lfbUsEai0U6TFVleTTY53ofYCWEM6EnNDIToTFbm514YFTUSc4h8Qlq2fPeqC3IcCmirNT4Kf0FCO7MQrtGNFPJme2cpb/pZguS3pxxkKb4lOS+eGiUBGcSs1v3zHroJ+hum4wTJFRG0Yb99aCVQU44wgV3nKW7FZkXzwO3QY7nnkFI2kaAXerCPF4+Ho463g==">
141
+
142
+ <span style="display: none;"><span class="text-gray-600" data-translate="error">error code:
143
+ 1020</span></span>
144
+ </form>
145
+ </div>
146
+ </div>
147
+ <script>
148
+ (function () {
149
+ var trkjs = document.createElement('img');
150
+ trkjs.setAttribute('src', '/cdn-cgi/images/trace/captcha/js/transparent.gif?ray=732fbc436ab471ed');
151
+ trkjs.setAttribute('style', 'display: none');
152
+ document.body.appendChild(trkjs);
153
+ var cpo = document.createElement('script');
154
+ cpo.src = '/cdn-cgi/challenge-platform/h/g/orchestrate/managed/v1?ray=732fbc436ab471ed';
155
+ window._cf_chl_opt.cOgUHash = location.hash === '' && location.href.indexOf('#') !== -1 ? '#' : location.hash;
156
+ window._cf_chl_opt.cOgUQuery = location.search === '' && location.href.slice(0, -window._cf_chl_opt.cOgUHash.length).indexOf('?') !== -1 ? '?' : location.search;
157
+ if (window.history && window.history.replaceState) {
158
+ var ogU = location.pathname + window._cf_chl_opt.cOgUQuery + window._cf_chl_opt.cOgUHash;
159
+ history.replaceState(null, null, "\/search?q=2022&__cf_chl_rt_tk=lkycIb1jDXlmFqiB7AXTwy38_EzYPvu79CCQyU9lhUE-1659201316-0-gaNycGzNCf0" + window._cf_chl_opt.cOgUHash);
160
+ cpo.onload = function () {
161
+ history.replaceState(null, null, ogU);
162
+ };
163
+ }
164
+ document.getElementsByTagName('head')[0].appendChild(cpo);
165
+ }());
166
+ </script><img src="Just%20a%20moment_files/transparent.gif" style="display: none">
167
+
168
+ <div class="footer" role="contentinfo">
169
+ <div class="footer-inner">
170
+ <div class="clearfix diagnostic-wrapper">
171
+ <div class="ray-id">Ray ID: <code>732fbc436ab471ed</code></div>
172
+ </div>
173
+ <div class="text-center">
174
+ Performance &amp; security by
175
+ <a rel="noopener noreferrer" href="https://www.cloudflare.com/" target="_blank">Cloudflare</a>
176
+ </div>
177
+ </div>
178
+ </div>
179
+
180
+
181
+ <div style="background-color: rgb(255, 255, 255); border: 1px solid rgb(215, 215, 215); box-shadow: rgba(0, 0, 0, 0.1) 0px 0px 4px; border-radius: 4px; left: -10000px; top: -10000px; z-index: -2147483648; position: absolute; transition: opacity 0.15s ease-out 0s; opacity: 0; visibility: hidden;"
182
+ aria-hidden="true">
183
+ <div style="position: relative; z-index: 1;"><iframe src="Just%20a%20moment_files/hcaptcha_003.html"
184
+ title="Main content of the hCaptcha challenge" scrolling="no"
185
+ style="border: 0px none; z-index: 2000000000; position: relative;" frameborder="0"></iframe></div>
186
+ <div
187
+ style="width: 100%; height: 100%; position: fixed; pointer-events: none; top: 0px; left: 0px; z-index: 0; background-color: rgb(255, 255, 255); opacity: 0.05;">
188
+ </div>
189
+ <div
190
+ style="border-width: 11px; position: absolute; pointer-events: none; margin-top: -11px; z-index: 1; right: 100%;">
191
+ <div
192
+ style="border-width: 10px; border-style: solid; border-color: transparent rgb(255, 255, 255) transparent transparent; position: relative; top: 10px; z-index: 1;">
193
+ </div>
194
+ <div
195
+ style="border-width: 11px; border-style: solid; border-color: transparent rgb(215, 215, 215) transparent transparent; position: relative; top: -11px; z-index: 0;">
196
+ </div>
197
+ </div>
198
+ </div>
199
+ <div style="background-color: rgb(255, 255, 255); border: 1px solid rgb(215, 215, 215); box-shadow: rgba(0, 0, 0, 0.1) 0px 0px 4px; border-radius: 4px; left: -10000px; top: -10000px; z-index: -2147483648; position: absolute; transition: opacity 0.15s ease-out 0s; opacity: 0; visibility: hidden;"
200
+ aria-hidden="true">
201
+ <div style="position: relative; z-index: 1;"><iframe src="Just%20a%20moment_files/hcaptcha_004.html"
202
+ title="Main content of the hCaptcha challenge" scrolling="no"
203
+ style="border: 0px none; z-index: 2000000000; position: relative;" frameborder="0"></iframe></div>
204
+ <div
205
+ style="width: 100%; height: 100%; position: fixed; pointer-events: none; top: 0px; left: 0px; z-index: 0; background-color: rgb(255, 255, 255); opacity: 0.05;">
206
+ </div>
207
+ <div
208
+ style="border-width: 11px; position: absolute; pointer-events: none; margin-top: -11px; z-index: 1; right: 100%;">
209
+ <div
210
+ style="border-width: 10px; border-style: solid; border-color: transparent rgb(255, 255, 255) transparent transparent; position: relative; top: 10px; z-index: 1;">
211
+ </div>
212
+ <div
213
+ style="border-width: 11px; border-style: solid; border-color: transparent rgb(215, 215, 215) transparent transparent; position: relative; top: -11px; z-index: 0;">
214
+ </div>
215
+ </div>
216
+ </div>
217
+ </body>
218
+
219
  </html>
html_samples/cloudflare_captcha_norobot_v1.html CHANGED
@@ -1,170 +1,170 @@
1
- <!DOCTYPE html>
2
- <html lang="en-US">
3
-
4
- <head>
5
- <title>Just a moment...</title>
6
- <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
7
- <meta http-equiv="X-UA-Compatible" content="IE=Edge">
8
- <meta name="robots" content="noindex,nofollow">
9
- <meta name="viewport" content="width=device-width,initial-scale=1">
10
- <link href="Just%20a%20moment2_files/cf-errors.css" rel="stylesheet">
11
-
12
- <script>
13
- (function () {
14
- window._cf_chl_opt = {
15
- cvId: '2',
16
- cType: 'managed',
17
- cNounce: '94250',
18
- cRay: '732fc1c74f757330',
19
- cHash: '8c4978fa93c1751',
20
- cUPMDTk: "\/search?q=2022&__cf_chl_tk=6E3KpS5eCzuCMJG64ch2shvOMHdwQ8ioliqACpoQqM8-1659201542-0-gaNycGzNCeU",
21
- cFPWv: 'g',
22
- cTTimeMs: '1000',
23
- cTplV: 2,
24
- cRq: {
25
- ru: 'aHR0cHM6Ly8wbWFnbmV0LmNvbS9zZWFyY2g/cT0yMDIy',
26
- ra: 'TW96aWxsYS81LjAgKFgxMTsgTGludXggeDg2XzY0OyBydjoxMDUuMCkgR2Vja28vMjAxMDAxMDEgRmlyZWZveC8xMDUuMA==',
27
- rm: 'R0VU',
28
- d: 'C4CtJo9JDMtUWZ0r+/s2CwjYdSTdqGYK3qFo1OXpvSc9v7/3d5QuMwmvG3e5oV1BpjlQb8eJJ23gVRxavjw/gpPp1brmKoHuvcJEmAP3Sof38vqcpF91/9NHe3JbmCM2xshiGvJdbpJXb5wXdYKYPMqy7NUHL1VU4hupa3Da3tBq9zyuMa1NcZaiyeE6piSl7n96m+VziRdwyG+SBUldIG/Fsv9J1yl+Gj19wbX1XEneMXChcClGgRrSe1MTd9thLkq2NGFqROnsUmpA8b+2Eqi+IPYQfkPcydWkHmJqQixN9ZFTIBChIC60hGHOQ7O354ju65tVGAhB/nBRREpdqvwoYzgufgg83+dbPHVdQasiuLRHvftOtHhS5/iaBOVoEBH+rElTSk/OYjU2Yh6gkQj0FjkbebEBptFeVAxgqoYZljOrhamWYYZ14tOKeonzc1rz/FXNTM5qVtrWCwAlt9SsXDjM/GYXZMTbOdNLnLZGlLNQCx+l6hMC0OQC45sWFzZECljbjXwiYfodKobeqe11lUXnskj8AN5Qc7O8OqtALsxoNCLZ7ou+ORY0lauremeuu3U3WqadgSGFGA+TZZw2VcCA3BIUKCGlsNLBlJ8wQS2UAGJfGLOVuhErmtsM',
29
- t: 'MTY1OTIwMTU0Mi4yOTUwMDA=',
30
- m: 'eWHHJ28v6yOyvSePVqcdyHxAYkkc3xq3VJ8YiDCk5nk=',
31
- i1: 'M3dMvem+HcwSbNQrJbaYdQ==',
32
- i2: 'ebY327qYCu6NZKHSQXkbaQ==',
33
- zh: 'qP4bnGc6j96JlnjNSE7HmQci3S9L50bHFtm4bQRjjKU=',
34
- uh: 'SK3PXNkeRzZtkRARhJpbmZpCIiWQw6+5gpOE7vojWx4=',
35
- hh: 'azXzJl8Ou22g0nN/9idVUoB9EqZ7fLmkSdDRHM3Lkmw=',
36
- }
37
- }
38
- window._cf_chl_enter = function () { window._cf_chl_opt.p = 1 };
39
- })();
40
- </script>
41
-
42
- <script src="Just%20a%20moment2_files/v1.js"></script>
43
- <script type="text/javascript" src="Just%20a%20moment2_files/api.js"></script>
44
- </head>
45
-
46
- <body class="no-js">
47
-
48
- <div class="privacy-pass">
49
- <a rel="noopener noreferrer" href="https://addons.mozilla.org/en-US/firefox/addon/privacy-pass/"
50
- target="_blank">
51
- Privacy Pass
52
- <span class="privacy-pass-icon-wrapper">
53
- <div class="privacy-pass-icon"></div>
54
- </span>
55
- </a>
56
- </div>
57
-
58
- <div class="main-wrapper" role="main">
59
- <div class="main-content">
60
- <h1 class="zone-name-title h1">
61
- <img class="heading-favicon" src="Just%20a%20moment2_files/favicon.ico"
62
- onerror="this.onerror=null;this.parentNode.removeChild(this)">
63
- 0MAGNET.COM
64
- </h1>
65
- <h2 class="h2" id="cf-challenge-running">
66
- Checking if the site connection is secure
67
- </h2>
68
- <div id="cf-challenge-stage" style="display: block;">
69
- <div id="cf-norobot-container" style="display: flex;"><input type="button" value="Verify you are human"
70
- class="big-button pow-button" style="cursor: pointer;"></div>
71
- </div>
72
- <div id="cf-challenge-spinner" class="spacer loading-spinner" style="display: none; visibility: hidden;">
73
- <div class="lds-ring">
74
- <div></div>
75
- <div></div>
76
- <div></div>
77
- <div></div>
78
- </div>
79
- </div>
80
- <noscript>
81
- <div id="cf-challenge-error-title">
82
- <div class="h2">
83
- <span class="icon-wrapper">
84
- <div class="heading-icon warning-icon"></div>
85
- </span>
86
- <span id="cf-challenge-error-text">
87
- Enable JavaScript and cookies to continue
88
- </span>
89
- </div>
90
- </div>
91
- </noscript>
92
- <div
93
- style="display:none;background-image:url('/cdn-cgi/images/trace/captcha/nojs/transparent.gif?ray=732fc1c74f757330')">
94
- </div>
95
- <div id="cf-challenge-body-text" class="core-msg spacer">
96
- 0magnet.com needs to review the security of your connection before
97
- proceeding.
98
- </div>
99
- <div id="cf-challenge-fact-wrapper" style="display: block; visibility: visible;" class="fact spacer hidden">
100
- <span class="fact-title">Did you know</span> <span id="cf-challenge-fact" class="body-text">botnets can
101
- be used to shutdown popular websites?</span>
102
- </div>
103
- <div id="cf-challenge-explainer-expandable" class="hidden expandable body-text spacer"
104
- style="display: block; visibility: visible;">
105
- <div class="expandable-title" id="cf-challenge-explainer-summary"><button class="expandable-summary-btn"
106
- id="cf-challenge-explainer-btn" type="button"> Why am I seeing this page? <span
107
- class="caret-icon-wrapper">
108
- <div class="caret-icon"></div>
109
- </span> </button> </div>
110
- <div class="expandable-details" id="cf-challenge-explainer-details">
111
- Requests from malicious bots can pose as legitimate traffic.
112
- Occasionally, you may see this page while the site ensures that the
113
- connection is secure.</div>
114
- </div>
115
- <div id="cf-challenge-success" style="display: none;">
116
- <div class="h2"><span class="icon-wrapper"><img class="heading-icon" alt="Success icon"
117
- src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADQAAAA0CAMAAADypuvZAAAANlBMVEUAAAAxMTEwMDAxMTExMTEwMDAwMDAwMDAxMTExMTExMTEwMDAwMDAxMTExMTEwMDAwMDAxMTHB9N+uAAAAEXRSTlMA3zDvfyBAEJC/n3BQz69gX7VMkcMAAAGySURBVEjHnZZbFoMgDEQJiDzVuv/NtgbtFGuQ4/zUKpeMIQbUhXSKE5l1XSn4pFWHRm/WShT1HRLWC01LGxFEVkCc30eYkLJ1Sjk9pvkw690VY6k8DWP9OM9yMG0Koi+mi8XA36NXmW0UXra4eJ3iwHfrfXVlgL0NqqGBHdqfeQhMmyJ48WDuKP81h3+SMPeRKkJcSXiLUK4XTHCjESOnz1VUXQoc6lgi2x4cI5aTQ201Mt8wHysI5fc05M5c81uZEtHcMKhxZ7iYEty1GfhLvGKpm+EYkdGxm1F5axmcB93DoORIbXfdN7f+hlFuyxtDP+sxtBnF43cIYwaZAWRgzxIoiXEMESoPlMhwLRDXeK772CAzXEdBRV7cmnoVBp0OSlyGidEzJTFq5hhcsA5388oSGM6b5p+qjpZrBlMS9xj4AwXmz108ukU1IomM3ceiW0CDwHCqp1NjAqXlFrbga+xuloQJ+tuyfbIBPNpqnmxqT7dPaOnZqBfhSBCteJAxWj58zLk2xgg+SPGYM6dRO6WczSnIxxwEExRaO+UyCUhbOp7CGQ+kxSUfNtLQFC+Po29vvy7jj4y0yAAAAABJRU5ErkJggg=="></span>Connection
118
- is secure</div>
119
- <div class="core-msg spacer">Proceeding...</div>
120
- </div>
121
- <form id="challenge-form"
122
- action="/search?q=2022&amp;__cf_chl_f_tk=6E3KpS5eCzuCMJG64ch2shvOMHdwQ8ioliqACpoQqM8-1659201542-0-gaNycGzNCeU"
123
- method="POST" enctype="application/x-www-form-urlencoded">
124
- <input type="hidden" name="md"
125
- value="UPeuijc1TS5ZQ21GIY6wjg6HHN_jWKH9sqolcSJABwg-1659201542-0-AR_ZxgiwVB4GwEgAjllIrmnGAumHNwuvfpFBddySYLh6CWexrUnxVYlX_wlB19Yndm45fs-KngMxbYB4dEOuf4MOJ_yL_BsNG3_cIPybV0bNn9WQXecJg3FfFrIBuMFIappZOX4hdDjLtRo9f4JsVsU6FzD9sUoKJRd4BTkjTAm25yFbqmPgV15XZhnJ5HRux044u0IIOVZCwTTzgRLCqToVb-OfiuUcHBzt4W7_wNlF1ObUi2oEr00DA1zZvzzY2KnXdZVN8m2OaNY_f2zkk9uDlLQRob_Ti6MHPNDr4eRkyMqZMZ1XDCxe-9lBkcEfpqtg6_4yac9ZiIEoNdJnJVE6cuNzb59DcBooXAq3IWp6fK4y4UIBStjqOXk4bxQb5yt1COfdPuQ9iLE_7yYOPG_t7n5I-4mjwvG7_U337A17oeEemXHfJkGC88Vm3SQdEHiW96VJuOA_X-rb7p3iOMlLYB5DKJ5DaBoPnP86uAWhoHWE6nrVzeAxeQ1y0uBHYPioJba5Kn9d-e2HsTMuAi7ZgSKuk90ApclIiW3owI4bLc4wxO5cu3ZIz7sZfbdvIKDhf9ESZhpQrITU_4Hgqjz0s3lt-MVeNP_0bz31XSeA--pdiulzUpQWLx1jhC4s7Av6STUb9bmbHpE41283KbbpuzBbmHN1UczNiaaquYZiEXRHKYyEMhKD782nWTJwQA">
126
- <input type="hidden" name="r"
127
- value="i1ShtnCs9Zs8QexeFnp6EFtrWs3WbGEVQGXbVfYwpRI-1659201542-0-AbDM6G9qkbgoH+BqDdr1tzCDHr/DU9Sdxelapvp2/FZN6VqYfpDkJGv+HxhBQng6aVktcEobxp2ouOxJZxPQrR6tVFIhOW6uPOAdy5kh2BBJWUHfER13aq8LQ86fvDyRh3AThEHj6bgs2udacfvOrDrHT2j/KHBPePlGKbh8rzDTJBKw0ejUleHk8eKX/BQ1bVULgxT+ZZY721lyn2wrjsde1j1OAsiiCDkVvQ4Rs+Bas7UApD5HeWzyrCu2VFk/Qf+Rk+6spM+StYenQUAKXXrekJoIeNxPf/W9ZRsJfwUoY0JUK2thOWiwQOtw21nVDpiCFB9nhhOsmBzBoRQGjckZyu/O5U7jMIFdS9ThCFC0Kffg0MEr5xTkmgw+CNSwN7AlI9v3GS2XdTFOPXe29b68fZXzYfbm2CjqYhmxomZjCGTAmkzXWaVnMOs9Vl/8VurCUEu8SAt5k9Za/vFrEurX1edXNCviVuTOBSLHjqBiLui9FbufzGLq6BaHYi3WIFA1nMkoxduxbErP+Eqyi8UNvzvmEqUbj2COalXcQzkbHkyyLo33MNHZEi1zhhHjwCm1lp6mm4BRe60kRgTHb8X7oxBpY4vEcMz4jQQdsW15xBPAjsH8m9cj1H2ujpd7kfo8JGTyZ7FcoxOzGOuZr8XRpGkH72HaWYz7M+GIb3BBZ1v2Za7sSrzinNLFjHCCVXq68MqOmZ6RhgeexGoKJzcMHsHvgGXB8CisyyNTtA3OQOujybNUnNzlW7vJ/wDreTHkko6jQ/Lm/X2GnLg85BIg6IeROzt3eInAYsCaNKpST/h5bvSGCyzRoOW46oO8ZzZrV2FI2rEr0xLTIVWzQ//K2iGOCz58RisCfxWiF2n+fzj/5nE/0cjTPzYP68TM5BxB058EO7ZEFbgqhUji8IR9V2ahy7kI9dUhwd2S4IyjL+O6hCNPwjpRohkt93wXUCZDMgNoxi1BIylqqtAxYBfodyjFz8mB8GgcqBaCHN3tI0BINENVfvSJwKniYxL73frTX5KEqniT9GdT15o4F7QLf4S1atwYzF6ezJTYgLf6fOWUZKpaFMSRzEsxmZDmOFZeiss8lj7bKS6drOpkaOYzZiSgp5t5VwLKT0yQ+PDWQmqkpZ5WOa9/ayXLyOCunzk1IUO6VkvgFe0P2LZC9XEZUfwAFakYemej8/SZx0EknoPob1il3MMsbfHNAvcvUJK9xDbdAQ7rz34r4D5zO2aPnmYw1yv9K36z78I2dZpjVT9kpiKFwaOTkuSDUDtcmnhKM1XE+goG/C66G6PsChpGKLCeaDw4Rp7BxlumiSGB4Mp/bs8pTz3gez7pSu1oNodr7Tr1wJvCK8T5nVJ5GRO/tQ+Ff2K2s67udoV0CFtKufJyRsGCEv/0u5sArg3uwtwIz1W0JtAVjhe+J2nUihLa0Gqm7AwcCwfhsLHOhMG28V2NAw19iVq8RuMN7A2kGg5PH6bUeilWUxxZvWyDfyRSJZYMQytwAJdt4gQ++Qnl3mcaSk1N3pSiltVUDpfLcYb5gd35m+mKQWtPnIDlJMAtGoBeqROQPLNDg+LYdI/dnJzIOHjI3J+pTWhbAlF7B7NtccZOHmI9Cl3vS6Fpqs5aSPEDoDENTap6JN1kgm5NszMay9tAm66AcKF95W6QhwgQsyRrwScgRaPUtCx9ZJcbav6T/CAulcBB85MjwAd8+HF1g+UZT9VvChZoxh7NzfMoR53pVbxvW6acO8oVN5ITTP8mNAIisRvWi2KVdi4KqaLjYtLFNN8AMzjAC0vBIaFyGZlIbFsB44MRiMufD64b/66dqeC0l0WrUlUG/DgrnSQr6lgK2gONJKPQZGXoaK0Ga8O8xMOkaFLNaqH5UH5KpHvIQ8nwhuXk/MS/7Gdp1W02OEB4l0hhKFytgWdo9QmCquSatvOjuFyRPa6tV8ceGmuDnQw22bJM9BwzdKlHn/2/mHjCz7gcEA3Hb/CbeP8V8mF1mc5R8HEEdz/rx+BTESmGiRivv+WQYpRKNh77iqbYvCvkduK4b3UErNbvcS10aTt8zDFF/oIwjDpsniJsrIUcC0FdQRs2dqPIfkoSHvs7YGmOjx9QThCAiTkPKUE9C5C4YPY4CWRV3nYAFJrTq0F047PkzDYm0AJMCahWK7Vq/Ra3l3nRHu9yI+P0HiruUbkzLgiEJYnAuUtxvpC/Vj0uhr+A0R9Obs1MHkwtDuMs/ETh3ZymeFtWLj70StkslJxTzKGimZSsqQXRFYGHY6CHqHwIXGrArYNjTty48VIfbfaEu58KQp6roOdFmx90AcK2lV0V5UdyuzDJeH/V5ERAmxWLrXQKWgiDrY4ZqecnRk5XEAVMq/ChPts9gR7xsQK5WsHtQNKLfltkL8YvAoS+jZvxzfUUBg99YSC4J/HzQS+FQAnkDxCgeroahXysNN1bgDASXOrn3NsC3LYpUiZ2AVTLPkj1roR9r65O">
128
-
129
- <span style="display: none;"><span style="display: none;" class="text-gray-600"
130
- data-translate="error">error code: 1020</span></span>
131
- </form>
132
- </div>
133
- </div>
134
- <script>
135
- (function () {
136
- var trkjs = document.createElement('img');
137
- trkjs.setAttribute('src', '/cdn-cgi/images/trace/captcha/js/transparent.gif?ray=732fc1c74f757330');
138
- trkjs.setAttribute('style', 'display: none');
139
- document.body.appendChild(trkjs);
140
- var cpo = document.createElement('script');
141
- cpo.src = '/cdn-cgi/challenge-platform/h/g/orchestrate/managed/v1?ray=732fc1c74f757330';
142
- window._cf_chl_opt.cOgUHash = location.hash === '' && location.href.indexOf('#') !== -1 ? '#' : location.hash;
143
- window._cf_chl_opt.cOgUQuery = location.search === '' && location.href.slice(0, -window._cf_chl_opt.cOgUHash.length).indexOf('?') !== -1 ? '?' : location.search;
144
- if (window.history && window.history.replaceState) {
145
- var ogU = location.pathname + window._cf_chl_opt.cOgUQuery + window._cf_chl_opt.cOgUHash;
146
- history.replaceState(null, null, "\/search?q=2022&__cf_chl_rt_tk=6E3KpS5eCzuCMJG64ch2shvOMHdwQ8ioliqACpoQqM8-1659201542-0-gaNycGzNCeU" + window._cf_chl_opt.cOgUHash);
147
- cpo.onload = function () {
148
- history.replaceState(null, null, ogU);
149
- };
150
- }
151
- document.getElementsByTagName('head')[0].appendChild(cpo);
152
- }());
153
- </script><img src="Just%20a%20moment2_files/transparent.gif" style="display: none">
154
-
155
- <div class="footer" role="contentinfo">
156
- <div class="footer-inner">
157
- <div class="clearfix diagnostic-wrapper">
158
- <div class="ray-id">Ray ID: <code>732fc1c74f757330</code></div>
159
- </div>
160
- <div class="text-center">
161
- Performance &amp; security by
162
- <a rel="noopener noreferrer" href="https://www.cloudflare.com/" target="_blank">Cloudflare</a>
163
- </div>
164
- </div>
165
- </div>
166
-
167
-
168
- </body>
169
-
170
  </html>
 
1
+ <!DOCTYPE html>
2
+ <html lang="en-US">
3
+
4
+ <head>
5
+ <title>Just a moment...</title>
6
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
7
+ <meta http-equiv="X-UA-Compatible" content="IE=Edge">
8
+ <meta name="robots" content="noindex,nofollow">
9
+ <meta name="viewport" content="width=device-width,initial-scale=1">
10
+ <link href="Just%20a%20moment2_files/cf-errors.css" rel="stylesheet">
11
+
12
+ <script>
13
+ (function () {
14
+ window._cf_chl_opt = {
15
+ cvId: '2',
16
+ cType: 'managed',
17
+ cNounce: '94250',
18
+ cRay: '732fc1c74f757330',
19
+ cHash: '8c4978fa93c1751',
20
+ cUPMDTk: "\/search?q=2022&__cf_chl_tk=6E3KpS5eCzuCMJG64ch2shvOMHdwQ8ioliqACpoQqM8-1659201542-0-gaNycGzNCeU",
21
+ cFPWv: 'g',
22
+ cTTimeMs: '1000',
23
+ cTplV: 2,
24
+ cRq: {
25
+ ru: 'aHR0cHM6Ly8wbWFnbmV0LmNvbS9zZWFyY2g/cT0yMDIy',
26
+ ra: 'TW96aWxsYS81LjAgKFgxMTsgTGludXggeDg2XzY0OyBydjoxMDUuMCkgR2Vja28vMjAxMDAxMDEgRmlyZWZveC8xMDUuMA==',
27
+ rm: 'R0VU',
28
+ d: 'C4CtJo9JDMtUWZ0r+/s2CwjYdSTdqGYK3qFo1OXpvSc9v7/3d5QuMwmvG3e5oV1BpjlQb8eJJ23gVRxavjw/gpPp1brmKoHuvcJEmAP3Sof38vqcpF91/9NHe3JbmCM2xshiGvJdbpJXb5wXdYKYPMqy7NUHL1VU4hupa3Da3tBq9zyuMa1NcZaiyeE6piSl7n96m+VziRdwyG+SBUldIG/Fsv9J1yl+Gj19wbX1XEneMXChcClGgRrSe1MTd9thLkq2NGFqROnsUmpA8b+2Eqi+IPYQfkPcydWkHmJqQixN9ZFTIBChIC60hGHOQ7O354ju65tVGAhB/nBRREpdqvwoYzgufgg83+dbPHVdQasiuLRHvftOtHhS5/iaBOVoEBH+rElTSk/OYjU2Yh6gkQj0FjkbebEBptFeVAxgqoYZljOrhamWYYZ14tOKeonzc1rz/FXNTM5qVtrWCwAlt9SsXDjM/GYXZMTbOdNLnLZGlLNQCx+l6hMC0OQC45sWFzZECljbjXwiYfodKobeqe11lUXnskj8AN5Qc7O8OqtALsxoNCLZ7ou+ORY0lauremeuu3U3WqadgSGFGA+TZZw2VcCA3BIUKCGlsNLBlJ8wQS2UAGJfGLOVuhErmtsM',
29
+ t: 'MTY1OTIwMTU0Mi4yOTUwMDA=',
30
+ m: 'eWHHJ28v6yOyvSePVqcdyHxAYkkc3xq3VJ8YiDCk5nk=',
31
+ i1: 'M3dMvem+HcwSbNQrJbaYdQ==',
32
+ i2: 'ebY327qYCu6NZKHSQXkbaQ==',
33
+ zh: 'qP4bnGc6j96JlnjNSE7HmQci3S9L50bHFtm4bQRjjKU=',
34
+ uh: 'SK3PXNkeRzZtkRARhJpbmZpCIiWQw6+5gpOE7vojWx4=',
35
+ hh: 'azXzJl8Ou22g0nN/9idVUoB9EqZ7fLmkSdDRHM3Lkmw=',
36
+ }
37
+ }
38
+ window._cf_chl_enter = function () { window._cf_chl_opt.p = 1 };
39
+ })();
40
+ </script>
41
+
42
+ <script src="Just%20a%20moment2_files/v1.js"></script>
43
+ <script type="text/javascript" src="Just%20a%20moment2_files/api.js"></script>
44
+ </head>
45
+
46
+ <body class="no-js">
47
+
48
+ <div class="privacy-pass">
49
+ <a rel="noopener noreferrer" href="https://addons.mozilla.org/en-US/firefox/addon/privacy-pass/"
50
+ target="_blank">
51
+ Privacy Pass
52
+ <span class="privacy-pass-icon-wrapper">
53
+ <div class="privacy-pass-icon"></div>
54
+ </span>
55
+ </a>
56
+ </div>
57
+
58
+ <div class="main-wrapper" role="main">
59
+ <div class="main-content">
60
+ <h1 class="zone-name-title h1">
61
+ <img class="heading-favicon" src="Just%20a%20moment2_files/favicon.ico"
62
+ onerror="this.onerror=null;this.parentNode.removeChild(this)">
63
+ 0MAGNET.COM
64
+ </h1>
65
+ <h2 class="h2" id="cf-challenge-running">
66
+ Checking if the site connection is secure
67
+ </h2>
68
+ <div id="cf-challenge-stage" style="display: block;">
69
+ <div id="cf-norobot-container" style="display: flex;"><input type="button" value="Verify you are human"
70
+ class="big-button pow-button" style="cursor: pointer;"></div>
71
+ </div>
72
+ <div id="cf-challenge-spinner" class="spacer loading-spinner" style="display: none; visibility: hidden;">
73
+ <div class="lds-ring">
74
+ <div></div>
75
+ <div></div>
76
+ <div></div>
77
+ <div></div>
78
+ </div>
79
+ </div>
80
+ <noscript>
81
+ <div id="cf-challenge-error-title">
82
+ <div class="h2">
83
+ <span class="icon-wrapper">
84
+ <div class="heading-icon warning-icon"></div>
85
+ </span>
86
+ <span id="cf-challenge-error-text">
87
+ Enable JavaScript and cookies to continue
88
+ </span>
89
+ </div>
90
+ </div>
91
+ </noscript>
92
+ <div
93
+ style="display:none;background-image:url('/cdn-cgi/images/trace/captcha/nojs/transparent.gif?ray=732fc1c74f757330')">
94
+ </div>
95
+ <div id="cf-challenge-body-text" class="core-msg spacer">
96
+ 0magnet.com needs to review the security of your connection before
97
+ proceeding.
98
+ </div>
99
+ <div id="cf-challenge-fact-wrapper" style="display: block; visibility: visible;" class="fact spacer hidden">
100
+ <span class="fact-title">Did you know</span> <span id="cf-challenge-fact" class="body-text">botnets can
101
+ be used to shutdown popular websites?</span>
102
+ </div>
103
+ <div id="cf-challenge-explainer-expandable" class="hidden expandable body-text spacer"
104
+ style="display: block; visibility: visible;">
105
+ <div class="expandable-title" id="cf-challenge-explainer-summary"><button class="expandable-summary-btn"
106
+ id="cf-challenge-explainer-btn" type="button"> Why am I seeing this page? <span
107
+ class="caret-icon-wrapper">
108
+ <div class="caret-icon"></div>
109
+ </span> </button> </div>
110
+ <div class="expandable-details" id="cf-challenge-explainer-details">
111
+ Requests from malicious bots can pose as legitimate traffic.
112
+ Occasionally, you may see this page while the site ensures that the
113
+ connection is secure.</div>
114
+ </div>
115
+ <div id="cf-challenge-success" style="display: none;">
116
+ <div class="h2"><span class="icon-wrapper"><img class="heading-icon" alt="Success icon"
117
+ src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADQAAAA0CAMAAADypuvZAAAANlBMVEUAAAAxMTEwMDAxMTExMTEwMDAwMDAwMDAxMTExMTExMTEwMDAwMDAxMTExMTEwMDAwMDAxMTHB9N+uAAAAEXRSTlMA3zDvfyBAEJC/n3BQz69gX7VMkcMAAAGySURBVEjHnZZbFoMgDEQJiDzVuv/NtgbtFGuQ4/zUKpeMIQbUhXSKE5l1XSn4pFWHRm/WShT1HRLWC01LGxFEVkCc30eYkLJ1Sjk9pvkw690VY6k8DWP9OM9yMG0Koi+mi8XA36NXmW0UXra4eJ3iwHfrfXVlgL0NqqGBHdqfeQhMmyJ48WDuKP81h3+SMPeRKkJcSXiLUK4XTHCjESOnz1VUXQoc6lgi2x4cI5aTQ201Mt8wHysI5fc05M5c81uZEtHcMKhxZ7iYEty1GfhLvGKpm+EYkdGxm1F5axmcB93DoORIbXfdN7f+hlFuyxtDP+sxtBnF43cIYwaZAWRgzxIoiXEMESoPlMhwLRDXeK772CAzXEdBRV7cmnoVBp0OSlyGidEzJTFq5hhcsA5388oSGM6b5p+qjpZrBlMS9xj4AwXmz108ukU1IomM3ceiW0CDwHCqp1NjAqXlFrbga+xuloQJ+tuyfbIBPNpqnmxqT7dPaOnZqBfhSBCteJAxWj58zLk2xgg+SPGYM6dRO6WczSnIxxwEExRaO+UyCUhbOp7CGQ+kxSUfNtLQFC+Po29vvy7jj4y0yAAAAABJRU5ErkJggg=="></span>Connection
118
+ is secure</div>
119
+ <div class="core-msg spacer">Proceeding...</div>
120
+ </div>
121
+ <form id="challenge-form"
122
+ action="/search?q=2022&amp;__cf_chl_f_tk=6E3KpS5eCzuCMJG64ch2shvOMHdwQ8ioliqACpoQqM8-1659201542-0-gaNycGzNCeU"
123
+ method="POST" enctype="application/x-www-form-urlencoded">
124
+ <input type="hidden" name="md"
125
+ value="UPeuijc1TS5ZQ21GIY6wjg6HHN_jWKH9sqolcSJABwg-1659201542-0-AR_ZxgiwVB4GwEgAjllIrmnGAumHNwuvfpFBddySYLh6CWexrUnxVYlX_wlB19Yndm45fs-KngMxbYB4dEOuf4MOJ_yL_BsNG3_cIPybV0bNn9WQXecJg3FfFrIBuMFIappZOX4hdDjLtRo9f4JsVsU6FzD9sUoKJRd4BTkjTAm25yFbqmPgV15XZhnJ5HRux044u0IIOVZCwTTzgRLCqToVb-OfiuUcHBzt4W7_wNlF1ObUi2oEr00DA1zZvzzY2KnXdZVN8m2OaNY_f2zkk9uDlLQRob_Ti6MHPNDr4eRkyMqZMZ1XDCxe-9lBkcEfpqtg6_4yac9ZiIEoNdJnJVE6cuNzb59DcBooXAq3IWp6fK4y4UIBStjqOXk4bxQb5yt1COfdPuQ9iLE_7yYOPG_t7n5I-4mjwvG7_U337A17oeEemXHfJkGC88Vm3SQdEHiW96VJuOA_X-rb7p3iOMlLYB5DKJ5DaBoPnP86uAWhoHWE6nrVzeAxeQ1y0uBHYPioJba5Kn9d-e2HsTMuAi7ZgSKuk90ApclIiW3owI4bLc4wxO5cu3ZIz7sZfbdvIKDhf9ESZhpQrITU_4Hgqjz0s3lt-MVeNP_0bz31XSeA--pdiulzUpQWLx1jhC4s7Av6STUb9bmbHpE41283KbbpuzBbmHN1UczNiaaquYZiEXRHKYyEMhKD782nWTJwQA">
126
+ <input type="hidden" name="r"
127
+ value="i1ShtnCs9Zs8QexeFnp6EFtrWs3WbGEVQGXbVfYwpRI-1659201542-0-AbDM6G9qkbgoH+BqDdr1tzCDHr/DU9Sdxelapvp2/FZN6VqYfpDkJGv+HxhBQng6aVktcEobxp2ouOxJZxPQrR6tVFIhOW6uPOAdy5kh2BBJWUHfER13aq8LQ86fvDyRh3AThEHj6bgs2udacfvOrDrHT2j/KHBPePlGKbh8rzDTJBKw0ejUleHk8eKX/BQ1bVULgxT+ZZY721lyn2wrjsde1j1OAsiiCDkVvQ4Rs+Bas7UApD5HeWzyrCu2VFk/Qf+Rk+6spM+StYenQUAKXXrekJoIeNxPf/W9ZRsJfwUoY0JUK2thOWiwQOtw21nVDpiCFB9nhhOsmBzBoRQGjckZyu/O5U7jMIFdS9ThCFC0Kffg0MEr5xTkmgw+CNSwN7AlI9v3GS2XdTFOPXe29b68fZXzYfbm2CjqYhmxomZjCGTAmkzXWaVnMOs9Vl/8VurCUEu8SAt5k9Za/vFrEurX1edXNCviVuTOBSLHjqBiLui9FbufzGLq6BaHYi3WIFA1nMkoxduxbErP+Eqyi8UNvzvmEqUbj2COalXcQzkbHkyyLo33MNHZEi1zhhHjwCm1lp6mm4BRe60kRgTHb8X7oxBpY4vEcMz4jQQdsW15xBPAjsH8m9cj1H2ujpd7kfo8JGTyZ7FcoxOzGOuZr8XRpGkH72HaWYz7M+GIb3BBZ1v2Za7sSrzinNLFjHCCVXq68MqOmZ6RhgeexGoKJzcMHsHvgGXB8CisyyNTtA3OQOujybNUnNzlW7vJ/wDreTHkko6jQ/Lm/X2GnLg85BIg6IeROzt3eInAYsCaNKpST/h5bvSGCyzRoOW46oO8ZzZrV2FI2rEr0xLTIVWzQ//K2iGOCz58RisCfxWiF2n+fzj/5nE/0cjTPzYP68TM5BxB058EO7ZEFbgqhUji8IR9V2ahy7kI9dUhwd2S4IyjL+O6hCNPwjpRohkt93wXUCZDMgNoxi1BIylqqtAxYBfodyjFz8mB8GgcqBaCHN3tI0BINENVfvSJwKniYxL73frTX5KEqniT9GdT15o4F7QLf4S1atwYzF6ezJTYgLf6fOWUZKpaFMSRzEsxmZDmOFZeiss8lj7bKS6drOpkaOYzZiSgp5t5VwLKT0yQ+PDWQmqkpZ5WOa9/ayXLyOCunzk1IUO6VkvgFe0P2LZC9XEZUfwAFakYemej8/SZx0EknoPob1il3MMsbfHNAvcvUJK9xDbdAQ7rz34r4D5zO2aPnmYw1yv9K36z78I2dZpjVT9kpiKFwaOTkuSDUDtcmnhKM1XE+goG/C66G6PsChpGKLCeaDw4Rp7BxlumiSGB4Mp/bs8pTz3gez7pSu1oNodr7Tr1wJvCK8T5nVJ5GRO/tQ+Ff2K2s67udoV0CFtKufJyRsGCEv/0u5sArg3uwtwIz1W0JtAVjhe+J2nUihLa0Gqm7AwcCwfhsLHOhMG28V2NAw19iVq8RuMN7A2kGg5PH6bUeilWUxxZvWyDfyRSJZYMQytwAJdt4gQ++Qnl3mcaSk1N3pSiltVUDpfLcYb5gd35m+mKQWtPnIDlJMAtGoBeqROQPLNDg+LYdI/dnJzIOHjI3J+pTWhbAlF7B7NtccZOHmI9Cl3vS6Fpqs5aSPEDoDENTap6JN1kgm5NszMay9tAm66AcKF95W6QhwgQsyRrwScgRaPUtCx9ZJcbav6T/CAulcBB85MjwAd8+HF1g+UZT9VvChZoxh7NzfMoR53pVbxvW6acO8oVN5ITTP8mNAIisRvWi2KVdi4KqaLjYtLFNN8AMzjAC0vBIaFyGZlIbFsB44MRiMufD64b/66dqeC0l0WrUlUG/DgrnSQr6lgK2gONJKPQZGXoaK0Ga8O8xMOkaFLNaqH5UH5KpHvIQ8nwhuXk/MS/7Gdp1W02OEB4l0hhKFytgWdo9QmCquSatvOjuFyRPa6tV8ceGmuDnQw22bJM9BwzdKlHn/2/mHjCz7gcEA3Hb/CbeP8V8mF1mc5R8HEEdz/rx+BTESmGiRivv+WQYpRKNh77iqbYvCvkduK4b3UErNbvcS10aTt8zDFF/oIwjDpsniJsrIUcC0FdQRs2dqPIfkoSHvs7YGmOjx9QThCAiTkPKUE9C5C4YPY4CWRV3nYAFJrTq0F047PkzDYm0AJMCahWK7Vq/Ra3l3nRHu9yI+P0HiruUbkzLgiEJYnAuUtxvpC/Vj0uhr+A0R9Obs1MHkwtDuMs/ETh3ZymeFtWLj70StkslJxTzKGimZSsqQXRFYGHY6CHqHwIXGrArYNjTty48VIfbfaEu58KQp6roOdFmx90AcK2lV0V5UdyuzDJeH/V5ERAmxWLrXQKWgiDrY4ZqecnRk5XEAVMq/ChPts9gR7xsQK5WsHtQNKLfltkL8YvAoS+jZvxzfUUBg99YSC4J/HzQS+FQAnkDxCgeroahXysNN1bgDASXOrn3NsC3LYpUiZ2AVTLPkj1roR9r65O">
128
+
129
+ <span style="display: none;"><span style="display: none;" class="text-gray-600"
130
+ data-translate="error">error code: 1020</span></span>
131
+ </form>
132
+ </div>
133
+ </div>
134
+ <script>
135
+ (function () {
136
+ var trkjs = document.createElement('img');
137
+ trkjs.setAttribute('src', '/cdn-cgi/images/trace/captcha/js/transparent.gif?ray=732fc1c74f757330');
138
+ trkjs.setAttribute('style', 'display: none');
139
+ document.body.appendChild(trkjs);
140
+ var cpo = document.createElement('script');
141
+ cpo.src = '/cdn-cgi/challenge-platform/h/g/orchestrate/managed/v1?ray=732fc1c74f757330';
142
+ window._cf_chl_opt.cOgUHash = location.hash === '' && location.href.indexOf('#') !== -1 ? '#' : location.hash;
143
+ window._cf_chl_opt.cOgUQuery = location.search === '' && location.href.slice(0, -window._cf_chl_opt.cOgUHash.length).indexOf('?') !== -1 ? '?' : location.search;
144
+ if (window.history && window.history.replaceState) {
145
+ var ogU = location.pathname + window._cf_chl_opt.cOgUQuery + window._cf_chl_opt.cOgUHash;
146
+ history.replaceState(null, null, "\/search?q=2022&__cf_chl_rt_tk=6E3KpS5eCzuCMJG64ch2shvOMHdwQ8ioliqACpoQqM8-1659201542-0-gaNycGzNCeU" + window._cf_chl_opt.cOgUHash);
147
+ cpo.onload = function () {
148
+ history.replaceState(null, null, ogU);
149
+ };
150
+ }
151
+ document.getElementsByTagName('head')[0].appendChild(cpo);
152
+ }());
153
+ </script><img src="Just%20a%20moment2_files/transparent.gif" style="display: none">
154
+
155
+ <div class="footer" role="contentinfo">
156
+ <div class="footer-inner">
157
+ <div class="clearfix diagnostic-wrapper">
158
+ <div class="ray-id">Ray ID: <code>732fc1c74f757330</code></div>
159
+ </div>
160
+ <div class="text-center">
161
+ Performance &amp; security by
162
+ <a rel="noopener noreferrer" href="https://www.cloudflare.com/" target="_blank">Cloudflare</a>
163
+ </div>
164
+ </div>
165
+ </div>
166
+
167
+
168
+ </body>
169
+
170
  </html>
html_samples/cloudflare_init_v1.html CHANGED
@@ -1,120 +1,120 @@
1
- <!DOCTYPE html>
2
- <html lang="en-US">
3
-
4
- <head>
5
- <title>Just a moment...</title>
6
- <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
7
- <meta http-equiv="X-UA-Compatible" content="IE=Edge" />
8
- <meta name="robots" content="noindex,nofollow" />
9
- <meta name="viewport" content="width=device-width,initial-scale=1" />
10
- <link href="/cdn-cgi/styles/cf-errors.css" rel="stylesheet" />
11
-
12
- <script>
13
- (function () {
14
- window._cf_chl_opt = {
15
- cvId: '2',
16
- cType: 'managed',
17
- cNounce: '46449',
18
- cRay: '732fd3bc9c1d72de',
19
- cHash: '8838fcad2a7f56c',
20
- cUPMDTk: "\/search?q=2022&__cf_chl_tk=y4XnN88eYeUiXmFkQeqEipve1VuK0jJA.G4Hz6xztsM-1659202277-0-gaNycGzNBz0",
21
- cFPWv: 'g',
22
- cTTimeMs: '1000',
23
- cTplV: 2,
24
- cRq: {
25
- ru: 'aHR0cHM6Ly8wbWFnbmV0LmNvbS9zZWFyY2g/cT0yMDIy',
26
- ra: 'Y3VybC83Ljg0LjA=',
27
- rm: 'R0VU',
28
- d: '+SdFLvm4kJf8Z9BVci1ZbUOY6ab/Dm5Zzyb0IvscIzmY9PnAAcvPfJ/3TD9YJViBxB/ArnbCQrOUfbSkq4odyaZmW19gm+exRuL8Z3POm1ABs7y6jwMshM19q4Gr3eFY/MUO/IYWuyA2F9q94hRCI6ZNb7dLEh9yh6hORbKRd62pdn59h1xCx8tNdKDtP7VXPXo85nYmJJPLOdXTnII+YxZ03a4isAmBHbi+lGoQN/bCV0K006VmpfPElAfAO9jm45o7pc1NgPQhZSKWpTyI/nHMueH6wacPREzN5RtREoQfKuwYpV++Gq56qr5bAe/SKeF+rI0x7OSqC4HQvrNwbA+kHZzaxgOKeiMFjDxmro/GyC/+sxeZmrxnSIAh4BScjPxEl1FLLkg/6D0JH6HmxoT8N/Jgpi9447Am4WeX+WQxJ9+uDs5WrFIahx7pWrgcZUTRPh+UCu3allJ2Q3cAfwK6BclhES/HhBBbJv0pnR1R2RfKDM/gr1MpLuhaK4mFEO/kSyNUjOnCjOfd+5d7Qb0DZn7sHpF2SVc+zNv5OWSvCRDUcNHjIOV6fq0datVyVWmxD6unPS0MMUFO+ZZNiB4ionrhVCiLrb2FjPQ8tzyCqXg+tnV7WtZ0h4+JuK3rxcaQ8PQy60/As8dKHqVTnw==',
29
- t: 'MTY1OTIwMjI3Ny44NjMwMDA=',
30
- m: 'zvAOPvfoONkW1BzH+jMnKOPtDpPpZijRP52DVDWH+i8=',
31
- i1: 'dDlQDNhOEuHzFEPo/etoAA==',
32
- i2: '+LTK9hchBRjTTQk1WQU1Vw==',
33
- zh: 'qP4bnGc6j96JlnjNSE7HmQci3S9L50bHFtm4bQRjjKU=',
34
- uh: 'IdIU2i4FhVxxcYhzSFWdjoBuQm7qnyVK65JGofJuWV4=',
35
- hh: 'azXzJl8Ou22g0nN/9idVUoB9EqZ7fLmkSdDRHM3Lkmw=',
36
- }
37
- }
38
- window._cf_chl_enter = function () { window._cf_chl_opt.p = 1 };
39
- })();
40
- </script>
41
-
42
- </head>
43
-
44
- <body class="no-js">
45
-
46
- <div class="main-wrapper" role="main">
47
- <div class="main-content">
48
- <h1 class="zone-name-title h1">
49
- <img class="heading-favicon" src="/favicon.ico"
50
- onerror="this.onerror=null;this.parentNode.removeChild(this)" />
51
- 0MAGNET.COM
52
- </h1>
53
- <h2 class="h2" id="cf-challenge-running">
54
- Checking if the site connection is secure
55
- </h2>
56
- <noscript>
57
- <div id="cf-challenge-error-title">
58
- <div class="h2">
59
- <span class="icon-wrapper">
60
- <div class="heading-icon warning-icon"></div>
61
- </span>
62
- <span id="cf-challenge-error-text">
63
- Enable JavaScript and cookies to continue
64
- </span>
65
- </div>
66
- </div>
67
- </noscript>
68
- <div
69
- style="display:none;background-image:url('/cdn-cgi/images/trace/captcha/nojs/transparent.gif?ray=732fd3bc9c1d72de')">
70
- </div>
71
- <div id="cf-challenge-body-text" class="core-msg spacer">
72
- 0magnet.com needs to review the security of your connection before
73
- proceeding.
74
- </div>
75
- <form id="challenge-form"
76
- action="/search?q=2022&amp;__cf_chl_f_tk=y4XnN88eYeUiXmFkQeqEipve1VuK0jJA.G4Hz6xztsM-1659202277-0-gaNycGzNBz0"
77
- method="POST" enctype="application/x-www-form-urlencoded">
78
- <input type="hidden" name="md"
79
- value="DpGhFnuVRfDhqsQNASrgdT4WiiJ8m6lqTIs03.l6RLc-1659202277-0-AfUEAk9DsJ4rmpVI_Al7-eogy2CmM3YgWe4-31iw0oG2CcDIbYvauEW2IvK9m27_gq1FvdH-UPaGHR0q6Q2haXlX4pgmQK5rlQUSEd5HquGdtWMasHWqL_Q_TZGdOKz30bE2FEk8wLHRErHJRJloDRj0tiG8MreT2La_GLvovNK1XbMXDxFZT2Cc-DThBvxbgbDffw3okYfdl1ECXhLw9G6L4o8xgLsz3QZQG3dNZNhm5n4mf55-BBsFDzDTEN1_1BgORVw3mtbsodedktcACsVBCRupyBpTev9MML1jHzk06ZT9dhcCP4zXvsMS4-gG212LFu79Cpl0MHifKvPk0DTJQja1ulaT4gVuIvmLPihPh1IYMGbEcdX4MFH0Wu_RL6UPINE6esf-oAx8-imKhKITB_R4974rpq9XJk65Kf9R6AJhu072CyOqW1YcmYMkUCqFjdZnRyNgHRT2Q5bMEJ8fv0DwfFV6ynG7n6JGMd_pEnZp0nEvjWXpK6Ft8ZZGOXtFMfmW4vNgFhs6xJ1wnaJWuLXae3V6gTZYxMkeIsyMzlvRSzYBz_rgRBNkvvAwbNvOZ369tKbaElS39hOI1WTaoOsnY2d0Z4mDe4AVbSs3fVJGikzZSa3Ctr1RnqqOztVIRYL1Q7IYRJ02P6egL7sn7RniJ6znNAoPhaWJLYzynWXeQF5YO5U0Zf779qkm3A" />
80
- <input type="hidden" name="r"
81
- value="QJznOl.RWpNvkdG1Pf6TAzaNhRIFpH8DJ0w1yAuwRLw-1659202277-0-AcOBapBisncM3qf1RYdkTNlIXCth/TmoAMnk3vJozFlG8/vYeLPpjG389mhQu01aSlpJqFWn0VQf9c/7w3yh85jHmrpaxJtpxTiSL9k+AWm61kE6DkHgJBl5jUc7gu4W3oHdmP4FyOUzhbBpIkOAntSkVJJmgu6SaIE3I9fRAFu7bPxBveT8zZGyVUJSPKpwx/w4rNPzs2VnCeEVL1eOdbLInHYR1kqC8M4JyBynwdVXxIX+j5o/rTrNK8E/W4UZMhuWqIaOnX7FzmceglyBSjDqJFLCt0TOhc66m82Y25Obi8Gvsqn34bjwPA2G8qOvgrHA2RFH6lEQFSdGMzLrF4qU5P9j9FzU1CPSTfGtkKbsMnGcMrtzmyQ7LdMIfghYvnCBXTi82iIzaSwzY3sEnW9KZs24Akxu/AV1E03sqW1CAA1UCRURpX4GKXvD6UYpSgc6++q8naLdRozkLP81T/CvHyIRdQx8vylmVN9u/rPvMbW1jWtniDmuAjBQDUd058YH+IRmm4lREG5JN2yeX083h/BG6tssEQVdTcIgwZRNDB+kK8vtOmywmo5qTAX1VE/sgfPCw5+3Xxu+hhZON3C7VGfrCQI5ZSb6+YvBLXmO26Nlp3fSOyeBwZy3pVuGwv/TrEo+e8USIlIs1T6MQJYQeX/4vOdy89npo6KBqY23giTFDh8EMZo//93hfBsRUbHrY/It6kp42qzsnWTjbkyiqd1zBSpQhuMyuMPeKpQ63oVI2tlGyioLg3HcfhbHQcdpUAWDn8lZ4+GTFVMix+20fGbErkVeBs7WvFSLlZ1YtYpCXrgVaomj7WCr8Icb7ASXKfvEuqC1ZnZgn6Lb6x3dUBGiDtnSFnixHFElIF6nPedVIV0+TxccjlV/LJeyNM58GHtRo4NcmIo1a6kN3vzPAjTUhgDJe4aYP6oVKCRNDcrHlGlLubu6XuIvBFM5Sq401xxahOe3VP2u7JovkzXwfl+yUxQOYaoq1LR+wnDhXgVbBNbM2QfIhez578zu2TN5bu5H14UXZ1E78KA6Op9b/PUgA1AsgTJVRk4M6OQSpa5wRIKkXzxpGIRz6+YBxSjIaX2I220GH4s6Te4CBpq77g6V4CVIkEvqZwbN9hIoAoljWVbEEdb3WmYZqoPxN/8ZIjU7uwQUyDgnCOlc7Z52TgG6nVvj7RVyxv5ugskW+fcOI12o35iYNNpXTh1boHyn7nlPG7wtSsl9UlTss27nd04AIzbH0qyX3kn77yPsobMDYUJ3IGhOujV8Cg08XHFIlYSYGPbqqpog+CuuWtzvwyk5mmHXkJNPyFEZL/irApJbatpGNqgGNnQL+5KYp+/U8/kROLTOWa8tG5609MF+wdrScsfPT9eE+HYh7tEFURnwm8kJtdAadcxYjzO60PFcUI1R5SMGHRflAnpY2gvAzbsSssk1WIF+6eHSe6FLHMXCHMp0w1XkNKpny5Ce3YTKhJ4TRg7HfN1pvet2Duj4G04A328uYUppPlU7Spz0fj5N/FHJf3sPaqJC8jn74L0mT92ecGaxS3ZGvytw51ulA00wgzfZDWL4pirzgYVjUQTqVl9FzWYua4Vk4l3BX0opWKA4FloLTP3ekrvmO/zkztMBV4fvK+F8JIOzLOs4AuoCv8uXl7Ny9wLQI3a0hJAdbXJpI3WV/iuV7da4fQao2Z2HiatQh3ZtdLqWmGtqQlcVtsBrac82eo7mKAfwltTfLlX9Drtp4ohwoFe0Upm+YsfY6DK7zHrk3k9GN7gm6cMi1neNFaqWZR9s8ABDBg==" />
82
-
83
- </form>
84
- </div>
85
- </div>
86
- <script>
87
- (function () {
88
- var trkjs = document.createElement('img');
89
- trkjs.setAttribute('src', '/cdn-cgi/images/trace/captcha/js/transparent.gif?ray=732fd3bc9c1d72de');
90
- trkjs.setAttribute('style', 'display: none');
91
- document.body.appendChild(trkjs);
92
- var cpo = document.createElement('script');
93
- cpo.src = '/cdn-cgi/challenge-platform/h/g/orchestrate/managed/v1?ray=732fd3bc9c1d72de';
94
- window._cf_chl_opt.cOgUHash = location.hash === '' && location.href.indexOf('#') !== -1 ? '#' : location.hash;
95
- window._cf_chl_opt.cOgUQuery = location.search === '' && location.href.slice(0, -window._cf_chl_opt.cOgUHash.length).indexOf('?') !== -1 ? '?' : location.search;
96
- if (window.history && window.history.replaceState) {
97
- var ogU = location.pathname + window._cf_chl_opt.cOgUQuery + window._cf_chl_opt.cOgUHash;
98
- history.replaceState(null, null, "\/search?q=2022&__cf_chl_rt_tk=y4XnN88eYeUiXmFkQeqEipve1VuK0jJA.G4Hz6xztsM-1659202277-0-gaNycGzNBz0" + window._cf_chl_opt.cOgUHash);
99
- cpo.onload = function () {
100
- history.replaceState(null, null, ogU);
101
- };
102
- }
103
- document.getElementsByTagName('head')[0].appendChild(cpo);
104
- }());
105
- </script>
106
-
107
- <div class="footer" role="contentinfo">
108
- <div class="footer-inner">
109
- <div class="clearfix diagnostic-wrapper">
110
- <div class="ray-id">Ray ID: <code>732fd3bc9c1d72de</code></div>
111
- </div>
112
- <div class="text-center">
113
- Performance &amp; security by
114
- <a rel="noopener noreferrer" href="https://www.cloudflare.com" target="_blank">Cloudflare</a>
115
- </div>
116
- </div>
117
- </div>
118
- </body>
119
-
120
  </html>
 
1
+ <!DOCTYPE html>
2
+ <html lang="en-US">
3
+
4
+ <head>
5
+ <title>Just a moment...</title>
6
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
7
+ <meta http-equiv="X-UA-Compatible" content="IE=Edge" />
8
+ <meta name="robots" content="noindex,nofollow" />
9
+ <meta name="viewport" content="width=device-width,initial-scale=1" />
10
+ <link href="/cdn-cgi/styles/cf-errors.css" rel="stylesheet" />
11
+
12
+ <script>
13
+ (function () {
14
+ window._cf_chl_opt = {
15
+ cvId: '2',
16
+ cType: 'managed',
17
+ cNounce: '46449',
18
+ cRay: '732fd3bc9c1d72de',
19
+ cHash: '8838fcad2a7f56c',
20
+ cUPMDTk: "\/search?q=2022&__cf_chl_tk=y4XnN88eYeUiXmFkQeqEipve1VuK0jJA.G4Hz6xztsM-1659202277-0-gaNycGzNBz0",
21
+ cFPWv: 'g',
22
+ cTTimeMs: '1000',
23
+ cTplV: 2,
24
+ cRq: {
25
+ ru: 'aHR0cHM6Ly8wbWFnbmV0LmNvbS9zZWFyY2g/cT0yMDIy',
26
+ ra: 'Y3VybC83Ljg0LjA=',
27
+ rm: 'R0VU',
28
+ d: '+SdFLvm4kJf8Z9BVci1ZbUOY6ab/Dm5Zzyb0IvscIzmY9PnAAcvPfJ/3TD9YJViBxB/ArnbCQrOUfbSkq4odyaZmW19gm+exRuL8Z3POm1ABs7y6jwMshM19q4Gr3eFY/MUO/IYWuyA2F9q94hRCI6ZNb7dLEh9yh6hORbKRd62pdn59h1xCx8tNdKDtP7VXPXo85nYmJJPLOdXTnII+YxZ03a4isAmBHbi+lGoQN/bCV0K006VmpfPElAfAO9jm45o7pc1NgPQhZSKWpTyI/nHMueH6wacPREzN5RtREoQfKuwYpV++Gq56qr5bAe/SKeF+rI0x7OSqC4HQvrNwbA+kHZzaxgOKeiMFjDxmro/GyC/+sxeZmrxnSIAh4BScjPxEl1FLLkg/6D0JH6HmxoT8N/Jgpi9447Am4WeX+WQxJ9+uDs5WrFIahx7pWrgcZUTRPh+UCu3allJ2Q3cAfwK6BclhES/HhBBbJv0pnR1R2RfKDM/gr1MpLuhaK4mFEO/kSyNUjOnCjOfd+5d7Qb0DZn7sHpF2SVc+zNv5OWSvCRDUcNHjIOV6fq0datVyVWmxD6unPS0MMUFO+ZZNiB4ionrhVCiLrb2FjPQ8tzyCqXg+tnV7WtZ0h4+JuK3rxcaQ8PQy60/As8dKHqVTnw==',
29
+ t: 'MTY1OTIwMjI3Ny44NjMwMDA=',
30
+ m: 'zvAOPvfoONkW1BzH+jMnKOPtDpPpZijRP52DVDWH+i8=',
31
+ i1: 'dDlQDNhOEuHzFEPo/etoAA==',
32
+ i2: '+LTK9hchBRjTTQk1WQU1Vw==',
33
+ zh: 'qP4bnGc6j96JlnjNSE7HmQci3S9L50bHFtm4bQRjjKU=',
34
+ uh: 'IdIU2i4FhVxxcYhzSFWdjoBuQm7qnyVK65JGofJuWV4=',
35
+ hh: 'azXzJl8Ou22g0nN/9idVUoB9EqZ7fLmkSdDRHM3Lkmw=',
36
+ }
37
+ }
38
+ window._cf_chl_enter = function () { window._cf_chl_opt.p = 1 };
39
+ })();
40
+ </script>
41
+
42
+ </head>
43
+
44
+ <body class="no-js">
45
+
46
+ <div class="main-wrapper" role="main">
47
+ <div class="main-content">
48
+ <h1 class="zone-name-title h1">
49
+ <img class="heading-favicon" src="/favicon.ico"
50
+ onerror="this.onerror=null;this.parentNode.removeChild(this)" />
51
+ 0MAGNET.COM
52
+ </h1>
53
+ <h2 class="h2" id="cf-challenge-running">
54
+ Checking if the site connection is secure
55
+ </h2>
56
+ <noscript>
57
+ <div id="cf-challenge-error-title">
58
+ <div class="h2">
59
+ <span class="icon-wrapper">
60
+ <div class="heading-icon warning-icon"></div>
61
+ </span>
62
+ <span id="cf-challenge-error-text">
63
+ Enable JavaScript and cookies to continue
64
+ </span>
65
+ </div>
66
+ </div>
67
+ </noscript>
68
+ <div
69
+ style="display:none;background-image:url('/cdn-cgi/images/trace/captcha/nojs/transparent.gif?ray=732fd3bc9c1d72de')">
70
+ </div>
71
+ <div id="cf-challenge-body-text" class="core-msg spacer">
72
+ 0magnet.com needs to review the security of your connection before
73
+ proceeding.
74
+ </div>
75
+ <form id="challenge-form"
76
+ action="/search?q=2022&amp;__cf_chl_f_tk=y4XnN88eYeUiXmFkQeqEipve1VuK0jJA.G4Hz6xztsM-1659202277-0-gaNycGzNBz0"
77
+ method="POST" enctype="application/x-www-form-urlencoded">
78
+ <input type="hidden" name="md"
79
+ value="DpGhFnuVRfDhqsQNASrgdT4WiiJ8m6lqTIs03.l6RLc-1659202277-0-AfUEAk9DsJ4rmpVI_Al7-eogy2CmM3YgWe4-31iw0oG2CcDIbYvauEW2IvK9m27_gq1FvdH-UPaGHR0q6Q2haXlX4pgmQK5rlQUSEd5HquGdtWMasHWqL_Q_TZGdOKz30bE2FEk8wLHRErHJRJloDRj0tiG8MreT2La_GLvovNK1XbMXDxFZT2Cc-DThBvxbgbDffw3okYfdl1ECXhLw9G6L4o8xgLsz3QZQG3dNZNhm5n4mf55-BBsFDzDTEN1_1BgORVw3mtbsodedktcACsVBCRupyBpTev9MML1jHzk06ZT9dhcCP4zXvsMS4-gG212LFu79Cpl0MHifKvPk0DTJQja1ulaT4gVuIvmLPihPh1IYMGbEcdX4MFH0Wu_RL6UPINE6esf-oAx8-imKhKITB_R4974rpq9XJk65Kf9R6AJhu072CyOqW1YcmYMkUCqFjdZnRyNgHRT2Q5bMEJ8fv0DwfFV6ynG7n6JGMd_pEnZp0nEvjWXpK6Ft8ZZGOXtFMfmW4vNgFhs6xJ1wnaJWuLXae3V6gTZYxMkeIsyMzlvRSzYBz_rgRBNkvvAwbNvOZ369tKbaElS39hOI1WTaoOsnY2d0Z4mDe4AVbSs3fVJGikzZSa3Ctr1RnqqOztVIRYL1Q7IYRJ02P6egL7sn7RniJ6znNAoPhaWJLYzynWXeQF5YO5U0Zf779qkm3A" />
80
+ <input type="hidden" name="r"
81
+ value="QJznOl.RWpNvkdG1Pf6TAzaNhRIFpH8DJ0w1yAuwRLw-1659202277-0-AcOBapBisncM3qf1RYdkTNlIXCth/TmoAMnk3vJozFlG8/vYeLPpjG389mhQu01aSlpJqFWn0VQf9c/7w3yh85jHmrpaxJtpxTiSL9k+AWm61kE6DkHgJBl5jUc7gu4W3oHdmP4FyOUzhbBpIkOAntSkVJJmgu6SaIE3I9fRAFu7bPxBveT8zZGyVUJSPKpwx/w4rNPzs2VnCeEVL1eOdbLInHYR1kqC8M4JyBynwdVXxIX+j5o/rTrNK8E/W4UZMhuWqIaOnX7FzmceglyBSjDqJFLCt0TOhc66m82Y25Obi8Gvsqn34bjwPA2G8qOvgrHA2RFH6lEQFSdGMzLrF4qU5P9j9FzU1CPSTfGtkKbsMnGcMrtzmyQ7LdMIfghYvnCBXTi82iIzaSwzY3sEnW9KZs24Akxu/AV1E03sqW1CAA1UCRURpX4GKXvD6UYpSgc6++q8naLdRozkLP81T/CvHyIRdQx8vylmVN9u/rPvMbW1jWtniDmuAjBQDUd058YH+IRmm4lREG5JN2yeX083h/BG6tssEQVdTcIgwZRNDB+kK8vtOmywmo5qTAX1VE/sgfPCw5+3Xxu+hhZON3C7VGfrCQI5ZSb6+YvBLXmO26Nlp3fSOyeBwZy3pVuGwv/TrEo+e8USIlIs1T6MQJYQeX/4vOdy89npo6KBqY23giTFDh8EMZo//93hfBsRUbHrY/It6kp42qzsnWTjbkyiqd1zBSpQhuMyuMPeKpQ63oVI2tlGyioLg3HcfhbHQcdpUAWDn8lZ4+GTFVMix+20fGbErkVeBs7WvFSLlZ1YtYpCXrgVaomj7WCr8Icb7ASXKfvEuqC1ZnZgn6Lb6x3dUBGiDtnSFnixHFElIF6nPedVIV0+TxccjlV/LJeyNM58GHtRo4NcmIo1a6kN3vzPAjTUhgDJe4aYP6oVKCRNDcrHlGlLubu6XuIvBFM5Sq401xxahOe3VP2u7JovkzXwfl+yUxQOYaoq1LR+wnDhXgVbBNbM2QfIhez578zu2TN5bu5H14UXZ1E78KA6Op9b/PUgA1AsgTJVRk4M6OQSpa5wRIKkXzxpGIRz6+YBxSjIaX2I220GH4s6Te4CBpq77g6V4CVIkEvqZwbN9hIoAoljWVbEEdb3WmYZqoPxN/8ZIjU7uwQUyDgnCOlc7Z52TgG6nVvj7RVyxv5ugskW+fcOI12o35iYNNpXTh1boHyn7nlPG7wtSsl9UlTss27nd04AIzbH0qyX3kn77yPsobMDYUJ3IGhOujV8Cg08XHFIlYSYGPbqqpog+CuuWtzvwyk5mmHXkJNPyFEZL/irApJbatpGNqgGNnQL+5KYp+/U8/kROLTOWa8tG5609MF+wdrScsfPT9eE+HYh7tEFURnwm8kJtdAadcxYjzO60PFcUI1R5SMGHRflAnpY2gvAzbsSssk1WIF+6eHSe6FLHMXCHMp0w1XkNKpny5Ce3YTKhJ4TRg7HfN1pvet2Duj4G04A328uYUppPlU7Spz0fj5N/FHJf3sPaqJC8jn74L0mT92ecGaxS3ZGvytw51ulA00wgzfZDWL4pirzgYVjUQTqVl9FzWYua4Vk4l3BX0opWKA4FloLTP3ekrvmO/zkztMBV4fvK+F8JIOzLOs4AuoCv8uXl7Ny9wLQI3a0hJAdbXJpI3WV/iuV7da4fQao2Z2HiatQh3ZtdLqWmGtqQlcVtsBrac82eo7mKAfwltTfLlX9Drtp4ohwoFe0Upm+YsfY6DK7zHrk3k9GN7gm6cMi1neNFaqWZR9s8ABDBg==" />
82
+
83
+ </form>
84
+ </div>
85
+ </div>
86
+ <script>
87
+ (function () {
88
+ var trkjs = document.createElement('img');
89
+ trkjs.setAttribute('src', '/cdn-cgi/images/trace/captcha/js/transparent.gif?ray=732fd3bc9c1d72de');
90
+ trkjs.setAttribute('style', 'display: none');
91
+ document.body.appendChild(trkjs);
92
+ var cpo = document.createElement('script');
93
+ cpo.src = '/cdn-cgi/challenge-platform/h/g/orchestrate/managed/v1?ray=732fd3bc9c1d72de';
94
+ window._cf_chl_opt.cOgUHash = location.hash === '' && location.href.indexOf('#') !== -1 ? '#' : location.hash;
95
+ window._cf_chl_opt.cOgUQuery = location.search === '' && location.href.slice(0, -window._cf_chl_opt.cOgUHash.length).indexOf('?') !== -1 ? '?' : location.search;
96
+ if (window.history && window.history.replaceState) {
97
+ var ogU = location.pathname + window._cf_chl_opt.cOgUQuery + window._cf_chl_opt.cOgUHash;
98
+ history.replaceState(null, null, "\/search?q=2022&__cf_chl_rt_tk=y4XnN88eYeUiXmFkQeqEipve1VuK0jJA.G4Hz6xztsM-1659202277-0-gaNycGzNBz0" + window._cf_chl_opt.cOgUHash);
99
+ cpo.onload = function () {
100
+ history.replaceState(null, null, ogU);
101
+ };
102
+ }
103
+ document.getElementsByTagName('head')[0].appendChild(cpo);
104
+ }());
105
+ </script>
106
+
107
+ <div class="footer" role="contentinfo">
108
+ <div class="footer-inner">
109
+ <div class="clearfix diagnostic-wrapper">
110
+ <div class="ray-id">Ray ID: <code>732fd3bc9c1d72de</code></div>
111
+ </div>
112
+ <div class="text-center">
113
+ Performance &amp; security by
114
+ <a rel="noopener noreferrer" href="https://www.cloudflare.com" target="_blank">Cloudflare</a>
115
+ </div>
116
+ </div>
117
+ </div>
118
+ </body>
119
+
120
  </html>
html_samples/cloudflare_spinner_v1.html CHANGED
@@ -1,167 +1,167 @@
1
- <html lang="en-US">
2
-
3
- <head>
4
- <title>Just a moment...</title>
5
- <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
6
- <meta http-equiv="X-UA-Compatible" content="IE=Edge">
7
- <meta name="robots" content="noindex,nofollow">
8
- <meta name="viewport" content="width=device-width,initial-scale=1">
9
- <link href="/cdn-cgi/styles/cf-errors.css" rel="stylesheet">
10
-
11
- <script>
12
- (function () {
13
- window._cf_chl_opt = {
14
- cvId: '2',
15
- cType: 'managed',
16
- cNounce: '52875',
17
- cRay: '732fa2449b567521',
18
- cHash: '79cce74ebb92671',
19
- cUPMDTk: "\/search?q=2022&__cf_chl_tk=1qWQAgl8.irfEoDb73Rb0pUm1SXbis3ZamDAIoTcPks-1659200251-0-gaNycGzNCFE",
20
- cFPWv: 'g',
21
- cTTimeMs: '1000',
22
- cTplV: 2,
23
- cRq: {
24
- ru: 'aHR0cHM6Ly8wbWFnbmV0LmNvbS9zZWFyY2g/cT0yMDIy',
25
- ra: 'TW96aWxsYS81LjAgKFgxMTsgTGludXggeDg2XzY0KSBBcHBsZVdlYktpdC81MzcuMzYgKEtIVE1MLCBsaWtlIEdlY2tvKSBDaHJvbWUvMTAzLjAuNTA2MC4xMzQgU2FmYXJpLzUzNy4zNg==',
26
- rm: 'R0VU',
27
- d: 'UfK0k9mFeKGEdqoWAUIbk3OXbXe9DOHoYXdKLPyxbICSIQBS4GSNYar0DtbPI7+UQ7UeBZ2XCdQinvgH0pgzJCF1qB0nkXtu0qlLk6EwkrGAKD/pMGFFQF2EaCw3m00/xoRCDgLZRl/wUkRGz3HUOkTuPeKgZjsFyPoPv7MbYSMUtH7QU6ruIh+O3hvDOT2oA/BOKbRMSTnFedTIXADXL6GE8ZyNZ33wJlef5KzT0MHlN+3eZTAt6urCvJaY3MdTXKVye6fwyjqGEksaJ6B85vwrifLTYEU4/bORwXx8mTQTqjo3kh1rATlmthQwBpcQtWXmgDUcJ5gPrOk1fzhqrhO4b++HiIx3P5YZ9Ko2D0NNWeg1AYIwDjh9rZg5m0MmCXh1VqXDbnpseQW1vPkkZAADxyvLf/eEc1o2EpYGpK+qSpMZ4RcngnU0o8A2nS+j/CNsid0315OrYVyOIZcw6L3ovu6yfAAAALyOmg5ctXCqjzRthoibUb58u+myxOtfX1ew9IzNq8Z6t6RlomjR7Iy/7BJiJQNCF98dllNbODHz//TymlI1m8D9w+CYlZFIpiWJVH1M4h+tabH5YrqDVbkJgY6yVAfnr/NI6d6NHrhN+eSW30jkvAmZ6JRMhVWW',
28
- t: 'MTY1OTIwMDI1MS42MjAwMDA=',
29
- m: '/e8nTBb03IHZzN/DSkoHPRu0Ndm3ynYs8g6ZC+VxHcc=',
30
- i1: 'tx+ntPfeE2Gv81s52vIOlA==',
31
- i2: 'fpw8a/EO+Fo2t/ZiNKxEcg==',
32
- zh: 'qP4bnGc6j96JlnjNSE7HmQci3S9L50bHFtm4bQRjjKU=',
33
- uh: 'Eex9UQDjphKtV6LyVQ95F/MC5kBA3Rj4lC6CudiU3Vs=',
34
- hh: 'azXzJl8Ou22g0nN/9idVUoB9EqZ7fLmkSdDRHM3Lkmw=',
35
- }
36
- }
37
- window._cf_chl_enter = function () { window._cf_chl_opt.p = 1 };
38
- })();
39
- </script>
40
-
41
- <script src="/cdn-cgi/challenge-platform/h/g/orchestrate/managed/v1?ray=732fa2449b567521"></script>
42
- <script type="text/javascript"
43
- src="https://cloudflare.hcaptcha.com/1/api.js?endpoint=https%3A%2F%2Fcloudflare.hcaptcha.com&amp;assethost=https%3A%2F%2Fcf-assets.hcaptcha.com&amp;imghost=https%3A%2F%2Fcf-imgs.hcaptcha.com&amp;render=explicit&amp;recaptchacompat=off&amp;onload=_cf_chl_hload"></script>
44
- </head>
45
-
46
- <body class="no-js">
47
-
48
- <div class="privacy-pass">
49
- <a rel="noopener noreferrer"
50
- href="https://chrome.google.com/webstore/detail/privacy-pass/ajhmfdgkijocedmfjonnpjfojldioehi"
51
- target="_blank">
52
- Privacy Pass
53
- <span class="privacy-pass-icon-wrapper">
54
- <div class="privacy-pass-icon"></div>
55
- </span>
56
- </a>
57
- </div>
58
-
59
- <div class="main-wrapper" role="main">
60
- <div class="main-content">
61
- <h1 class="zone-name-title h1">
62
- <img class="heading-favicon" src="/favicon.ico"
63
- onerror="this.onerror=null;this.parentNode.removeChild(this)">
64
- 0MAGNET.COM
65
- </h1>
66
- <h2 class="h2" id="cf-challenge-running">
67
- Checking if the site connection is secure
68
- </h2>
69
- <div id="cf-challenge-stage" style="display: none;"></div>
70
- <div id="cf-challenge-spinner" class="spacer loading-spinner" style="display: block; visibility: visible;">
71
- <div class="lds-ring">
72
- <div></div>
73
- <div></div>
74
- <div></div>
75
- <div></div>
76
- </div>
77
- </div>
78
- <noscript>
79
- <div id="cf-challenge-error-title">
80
- <div class="h2">
81
- <span class="icon-wrapper">
82
- <div class="heading-icon warning-icon"></div>
83
- </span>
84
- <span id="cf-challenge-error-text">
85
- Enable JavaScript and cookies to continue
86
- </span>
87
- </div>
88
- </div>
89
- </noscript>
90
- <div
91
- style="display:none;background-image:url('/cdn-cgi/images/trace/captcha/nojs/transparent.gif?ray=732fa2449b567521')">
92
- </div>
93
- <div id="cf-challenge-body-text" class="core-msg spacer">
94
- 0magnet.com needs to review the security of your connection before
95
- proceeding.
96
- </div>
97
- <div id="cf-challenge-fact-wrapper" class="fact spacer hidden" style="display: block; visibility: visible;">
98
- <span class="fact-title">Did you know</span> <span id="cf-challenge-fact" class="body-text">bots
99
- historically made up nearly 40% of all internet traffic?</span>
100
- </div>
101
- <div id="cf-challenge-explainer-expandable" class="hidden expandable body-text spacer"
102
- style="display: none;">
103
- <div class="expandable-title" id="cf-challenge-explainer-summary"><button class="expandable-summary-btn"
104
- id="cf-challenge-explainer-btn" type="button"> Why am I seeing this page? <span
105
- class="caret-icon-wrapper">
106
- <div class="caret-icon"></div>
107
- </span> </button> </div>
108
- <div class="expandable-details" id="cf-challenge-explainer-details"> Requests from malicious bots can
109
- pose as legitimate traffic. Occasionally, you may see this page while the site ensures that the
110
- connection is secure.</div>
111
- </div>
112
- <div id="cf-challenge-success" style="display: none;">
113
- <div class="h2"><span class="icon-wrapper"><img class="heading-icon" alt="Success icon"
114
- src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADQAAAA0CAMAAADypuvZAAAANlBMVEUAAAAxMTEwMDAxMTExMTEwMDAwMDAwMDAxMTExMTExMTEwMDAwMDAxMTExMTEwMDAwMDAxMTHB9N+uAAAAEXRSTlMA3zDvfyBAEJC/n3BQz69gX7VMkcMAAAGySURBVEjHnZZbFoMgDEQJiDzVuv/NtgbtFGuQ4/zUKpeMIQbUhXSKE5l1XSn4pFWHRm/WShT1HRLWC01LGxFEVkCc30eYkLJ1Sjk9pvkw690VY6k8DWP9OM9yMG0Koi+mi8XA36NXmW0UXra4eJ3iwHfrfXVlgL0NqqGBHdqfeQhMmyJ48WDuKP81h3+SMPeRKkJcSXiLUK4XTHCjESOnz1VUXQoc6lgi2x4cI5aTQ201Mt8wHysI5fc05M5c81uZEtHcMKhxZ7iYEty1GfhLvGKpm+EYkdGxm1F5axmcB93DoORIbXfdN7f+hlFuyxtDP+sxtBnF43cIYwaZAWRgzxIoiXEMESoPlMhwLRDXeK772CAzXEdBRV7cmnoVBp0OSlyGidEzJTFq5hhcsA5388oSGM6b5p+qjpZrBlMS9xj4AwXmz108ukU1IomM3ceiW0CDwHCqp1NjAqXlFrbga+xuloQJ+tuyfbIBPNpqnmxqT7dPaOnZqBfhSBCteJAxWj58zLk2xgg+SPGYM6dRO6WczSnIxxwEExRaO+UyCUhbOp7CGQ+kxSUfNtLQFC+Po29vvy7jj4y0yAAAAABJRU5ErkJggg=="></span>Connection
115
- is secure</div>
116
- <div class="core-msg spacer">Proceeding...</div>
117
- </div>
118
- <form id="challenge-form"
119
- action="/search?q=2022&amp;__cf_chl_f_tk=1qWQAgl8.irfEoDb73Rb0pUm1SXbis3ZamDAIoTcPks-1659200251-0-gaNycGzNCFE"
120
- method="POST" enctype="application/x-www-form-urlencoded">
121
- <input type="hidden" name="md"
122
- value="OghUU_ltYW6I0fpWl7rE4yHBGBPHfpZQIKZRSEpJKjE-1659200251-0-AWB-KR-MabhObmvYa3mR5-xDk3qZVV73547wjnl-QtfPoTxe017AXt4WUskEcVzEIUKC7dsJoiy8ec1NA0fxdnI8X9OfPhtynl00ReWBVZc_3Gba_wigWMmM_9e8PX9vpVDcXpCRbz1BJ5_YLsba9TJM1sp14U9RtIce-tRBB53qoxLxJRz9QFmckEVBvsba4RfoycOvYPMMsfAqSkq13qtsA3Kd6RDB5Rb5-qF8674DsB4AMvd9xu_fBplQqKjOpEtrThCUtw8M2DHY8FUr_owUo1NIS1s6fSBEyHh6ehz9CidJ7zpRwYZFwgz_Pq9i8LmQG_AajozOJJhLp-tox0dptbUZnRNGt3hGQgrNu3jlCfwPC2XVp7xgLvmZoPYrzzrZoi_wErnIvVgyGCw9-sDPblPdvLBUz6uXNreWwThEW6PeRtMXnePO9UwcZmj_2awhwcVSHSLz1t1z22LtVsQ8xNpMbiE7xDvI2D5LNHAPIUC7Wp4AcehWD-fEm0w5jnVTWOmFlVRxtcnYZSMfDSaRUxsZ3hg5B1-ghVMEX6M-r_hAd6pLKNmjIfdl_Nvdm6veQvV-gTFaULbfuhmQQjYEb9G2IptDiNTZs5S7FtmjqVBAA7PmvwBTQwxw86J0cV3v_4pT1Oj8tigwiPny35HMTrKRmRZWaAZudCmWxDZkJIW8Eir7KQ57ba-u9cHh0A">
123
- <input type="hidden" name="r"
124
- value="q.UUtPBFcFi4IkcVw3l4U_xJJKDIbHJj7xmuB43IIAI-1659200251-0-Aa4lU5RipD+d4of3hcdQ0rVmZ4ulb3siZYKwhm1jGNiA+/9b8IW1HL8k1GrsYEVexDW7ycP5UINQZ1sYJvZBTCQe3lhyGLHdLZ7KdI9RXKEbPx1NUOR/HthCD0Wbo7H41jbAf7l+HhH0zTLjm77/6NpJZHcgfsbBwwubl4R3oLarzPSByV2PVBnkuMyKCYgibriuMUt2iJHoMLx7Cr+Bmjx1KEFCrPYP0t7vgQs2APTylhL7ebP77XB9ndxU6Of3r4eHnTwLIcomFJ3+jqL6pzFaNoXdUBHrv9oZs/33KZjf2NB8cu5KUpAdM2lp3t5oTSQE19fJVroxmf91hcTdele3F2DAeawFGDwncm/Jo725SlyNk4TqsmR+il7DLkS/FTcCNzQe4cQM6DRWdmF9I1OohAl1/uGXYqUJSK1F45n3gec/pyPTQyZI0OLc7sCGYXfn3VPFsGATkg5mxE9rgZIB2b6ID9JggzIlDdYxlQRWecpruu07KOgk3m7g95lyHNZTohqemo4T8Z2MOZECjmXGMAuwvvk4d5sakVHr39kmAY6aSfXrRB+iONCOKkahbumrVmjLsnMvrpTb0DFE5pRAxwANPZKzb6Ikmlvxh7oJIPOB0mG9hDeoc/AJVlvZJV4CrpDLulNjHetAWXMwMptZuYJGEhcDXxmYj0ybntTCU4Y3JJQc5K+7ehSdnluTvMueWfs628854r4PcOONZzsO337j+3lUxrP5vDUCzYD25FNxvs8jGfqRivqHMOq2z9iOs0sHQTlHroLLSt2G7M50yRJBGTfxrIsvLq+ML3e/mRIkYIQxOcp8ugoPoT4c9gex3OyY0cnnA2/9OibQs9kevwf9DSnutMRRcbIXZI0XO6FY07+MykWqUcXygwMHs1vQxhaQ26NFYwolEWfOL7EQpp4GKyN30nL4nPNil/7GsXIr5SC+o55KI0l3AOEYE1jirVx2G0U7Br7SW80Ih4Fn5U/+4qFfW57GAJrpuk9qjFfJehe7wFBu5bHghEGRhKAu0wvpY7UTc9AiacMfP7ujVWi4DIbTCfOzOgVT8E0T6KaUurBppPJflLQE41c8n29ULyKmki9t8lIKvxYmv/3/AauhXFAExh+JrdnaeSDxhJYjWEUDJiaNvnkDCHMxPs/bePhSg4DYRMh4ngcOHCRkkRlDjipUUgeCrwNBY0qu2DIqLZXI1ZMwU+R0nuWnwc5xJuMHtrLkWbziP0FQcGaF0B6SaFIcOLnWG7YjJZFzxjFpvLb8GnZxk7i2YHCDTn0Stq3JDZHCkjJQjaPMmuK+5KYzfaSHcKOcaQbkbyDjn3t/XQX3a7lknngVchIJVsVn8osqgKvOx3aAdCicYKR6QaukrXHhR9uIEbPdoBYqZPKFz0uvVOShsUx2f65CaI8wWMjOBRWxTK1xUPNsetOiyYSvNwjeULaCXPKLi2qv/cZRRbsr3g5ghdHvNTpD/O0/xUgiziev3/9CpNopyr6VzLar9dJ/s++imXY1w1TCRJ2uCI2H70XGBGWxSZdbnfxU+j3zNCL0dBuabwhDd4ZnOZmlFZjGBiOUpsWdRrHd3c+QpwXdxB3QurRwX6J+LhmkqcWsPhP7LlMnN7dr2HUFZ5FS4LASl5AOf8hjCnO06FT8fWLl1eKVVjCugx9w54qjGqOV8A0v/PdWr7Ic0WfriyYbmwn/XnH8t0ri3bqDZsDkfhQMMF9JSWHEdoGD60a7McGDxr4g9s3LZhq5KozgSvyG+RUBPla8g2zB253hR7amWE5WO4IChl7AXmRB89F9u2+AoDIbefseb3pwG7GfkYpSBwmgJ4Ju4LAWoSfBhZSPMQadHZOCg36R11KesUy+NAy9bvD1bE3UMx9e2NbFohu6sXlilpnxINHp0sFEeulreEjWSQreri1eZeKxV2QfKIzWUiMoNdyT0JzM+/brYzddBpO2DrlnK5bEPWgtu0D7d4Kfm+0T7S//Fq+hxf40lSMPP8cBlan6sEd2iWmZ6gW3z43wNbJaPQIUDgb58ELxaEKQN4tOOy75/XXfISNnhG0K8M79a175WUb8v0A=">
125
-
126
- <span style="display: none;"><span class="text-gray-600" data-translate="error">error code:
127
- 1020</span></span>
128
- </form>
129
- </div>
130
- </div>
131
- <script>
132
- (function () {
133
- var trkjs = document.createElement('img');
134
- trkjs.setAttribute('src', '/cdn-cgi/images/trace/captcha/js/transparent.gif?ray=732fa2449b567521');
135
- trkjs.setAttribute('style', 'display: none');
136
- document.body.appendChild(trkjs);
137
- var cpo = document.createElement('script');
138
- cpo.src = '/cdn-cgi/challenge-platform/h/g/orchestrate/managed/v1?ray=732fa2449b567521';
139
- window._cf_chl_opt.cOgUHash = location.hash === '' && location.href.indexOf('#') !== -1 ? '#' : location.hash;
140
- window._cf_chl_opt.cOgUQuery = location.search === '' && location.href.slice(0, -window._cf_chl_opt.cOgUHash.length).indexOf('?') !== -1 ? '?' : location.search;
141
- if (window.history && window.history.replaceState) {
142
- var ogU = location.pathname + window._cf_chl_opt.cOgUQuery + window._cf_chl_opt.cOgUHash;
143
- history.replaceState(null, null, "\/search?q=2022&__cf_chl_rt_tk=1qWQAgl8.irfEoDb73Rb0pUm1SXbis3ZamDAIoTcPks-1659200251-0-gaNycGzNCFE" + window._cf_chl_opt.cOgUHash);
144
- cpo.onload = function () {
145
- history.replaceState(null, null, ogU);
146
- };
147
- }
148
- document.getElementsByTagName('head')[0].appendChild(cpo);
149
- }());
150
- </script><img src="/cdn-cgi/images/trace/captcha/js/transparent.gif?ray=732fa2449b567521" style="display: none">
151
-
152
- <div class="footer" role="contentinfo">
153
- <div class="footer-inner">
154
- <div class="clearfix diagnostic-wrapper">
155
- <div class="ray-id">Ray ID: <code>732fa2449b567521</code></div>
156
- </div>
157
- <div class="text-center">
158
- Performance &amp; security by
159
- <a rel="noopener noreferrer" href="https://www.cloudflare.com" target="_blank">Cloudflare</a>
160
- </div>
161
- </div>
162
- </div>
163
-
164
-
165
- </body>
166
-
167
  </html>
 
1
+ <html lang="en-US">
2
+
3
+ <head>
4
+ <title>Just a moment...</title>
5
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
6
+ <meta http-equiv="X-UA-Compatible" content="IE=Edge">
7
+ <meta name="robots" content="noindex,nofollow">
8
+ <meta name="viewport" content="width=device-width,initial-scale=1">
9
+ <link href="/cdn-cgi/styles/cf-errors.css" rel="stylesheet">
10
+
11
+ <script>
12
+ (function () {
13
+ window._cf_chl_opt = {
14
+ cvId: '2',
15
+ cType: 'managed',
16
+ cNounce: '52875',
17
+ cRay: '732fa2449b567521',
18
+ cHash: '79cce74ebb92671',
19
+ cUPMDTk: "\/search?q=2022&__cf_chl_tk=1qWQAgl8.irfEoDb73Rb0pUm1SXbis3ZamDAIoTcPks-1659200251-0-gaNycGzNCFE",
20
+ cFPWv: 'g',
21
+ cTTimeMs: '1000',
22
+ cTplV: 2,
23
+ cRq: {
24
+ ru: 'aHR0cHM6Ly8wbWFnbmV0LmNvbS9zZWFyY2g/cT0yMDIy',
25
+ ra: 'TW96aWxsYS81LjAgKFgxMTsgTGludXggeDg2XzY0KSBBcHBsZVdlYktpdC81MzcuMzYgKEtIVE1MLCBsaWtlIEdlY2tvKSBDaHJvbWUvMTAzLjAuNTA2MC4xMzQgU2FmYXJpLzUzNy4zNg==',
26
+ rm: 'R0VU',
27
+ d: 'UfK0k9mFeKGEdqoWAUIbk3OXbXe9DOHoYXdKLPyxbICSIQBS4GSNYar0DtbPI7+UQ7UeBZ2XCdQinvgH0pgzJCF1qB0nkXtu0qlLk6EwkrGAKD/pMGFFQF2EaCw3m00/xoRCDgLZRl/wUkRGz3HUOkTuPeKgZjsFyPoPv7MbYSMUtH7QU6ruIh+O3hvDOT2oA/BOKbRMSTnFedTIXADXL6GE8ZyNZ33wJlef5KzT0MHlN+3eZTAt6urCvJaY3MdTXKVye6fwyjqGEksaJ6B85vwrifLTYEU4/bORwXx8mTQTqjo3kh1rATlmthQwBpcQtWXmgDUcJ5gPrOk1fzhqrhO4b++HiIx3P5YZ9Ko2D0NNWeg1AYIwDjh9rZg5m0MmCXh1VqXDbnpseQW1vPkkZAADxyvLf/eEc1o2EpYGpK+qSpMZ4RcngnU0o8A2nS+j/CNsid0315OrYVyOIZcw6L3ovu6yfAAAALyOmg5ctXCqjzRthoibUb58u+myxOtfX1ew9IzNq8Z6t6RlomjR7Iy/7BJiJQNCF98dllNbODHz//TymlI1m8D9w+CYlZFIpiWJVH1M4h+tabH5YrqDVbkJgY6yVAfnr/NI6d6NHrhN+eSW30jkvAmZ6JRMhVWW',
28
+ t: 'MTY1OTIwMDI1MS42MjAwMDA=',
29
+ m: '/e8nTBb03IHZzN/DSkoHPRu0Ndm3ynYs8g6ZC+VxHcc=',
30
+ i1: 'tx+ntPfeE2Gv81s52vIOlA==',
31
+ i2: 'fpw8a/EO+Fo2t/ZiNKxEcg==',
32
+ zh: 'qP4bnGc6j96JlnjNSE7HmQci3S9L50bHFtm4bQRjjKU=',
33
+ uh: 'Eex9UQDjphKtV6LyVQ95F/MC5kBA3Rj4lC6CudiU3Vs=',
34
+ hh: 'azXzJl8Ou22g0nN/9idVUoB9EqZ7fLmkSdDRHM3Lkmw=',
35
+ }
36
+ }
37
+ window._cf_chl_enter = function () { window._cf_chl_opt.p = 1 };
38
+ })();
39
+ </script>
40
+
41
+ <script src="/cdn-cgi/challenge-platform/h/g/orchestrate/managed/v1?ray=732fa2449b567521"></script>
42
+ <script type="text/javascript"
43
+ src="https://cloudflare.hcaptcha.com/1/api.js?endpoint=https%3A%2F%2Fcloudflare.hcaptcha.com&amp;assethost=https%3A%2F%2Fcf-assets.hcaptcha.com&amp;imghost=https%3A%2F%2Fcf-imgs.hcaptcha.com&amp;render=explicit&amp;recaptchacompat=off&amp;onload=_cf_chl_hload"></script>
44
+ </head>
45
+
46
+ <body class="no-js">
47
+
48
+ <div class="privacy-pass">
49
+ <a rel="noopener noreferrer"
50
+ href="https://chrome.google.com/webstore/detail/privacy-pass/ajhmfdgkijocedmfjonnpjfojldioehi"
51
+ target="_blank">
52
+ Privacy Pass
53
+ <span class="privacy-pass-icon-wrapper">
54
+ <div class="privacy-pass-icon"></div>
55
+ </span>
56
+ </a>
57
+ </div>
58
+
59
+ <div class="main-wrapper" role="main">
60
+ <div class="main-content">
61
+ <h1 class="zone-name-title h1">
62
+ <img class="heading-favicon" src="/favicon.ico"
63
+ onerror="this.onerror=null;this.parentNode.removeChild(this)">
64
+ 0MAGNET.COM
65
+ </h1>
66
+ <h2 class="h2" id="cf-challenge-running">
67
+ Checking if the site connection is secure
68
+ </h2>
69
+ <div id="cf-challenge-stage" style="display: none;"></div>
70
+ <div id="cf-challenge-spinner" class="spacer loading-spinner" style="display: block; visibility: visible;">
71
+ <div class="lds-ring">
72
+ <div></div>
73
+ <div></div>
74
+ <div></div>
75
+ <div></div>
76
+ </div>
77
+ </div>
78
+ <noscript>
79
+ <div id="cf-challenge-error-title">
80
+ <div class="h2">
81
+ <span class="icon-wrapper">
82
+ <div class="heading-icon warning-icon"></div>
83
+ </span>
84
+ <span id="cf-challenge-error-text">
85
+ Enable JavaScript and cookies to continue
86
+ </span>
87
+ </div>
88
+ </div>
89
+ </noscript>
90
+ <div
91
+ style="display:none;background-image:url('/cdn-cgi/images/trace/captcha/nojs/transparent.gif?ray=732fa2449b567521')">
92
+ </div>
93
+ <div id="cf-challenge-body-text" class="core-msg spacer">
94
+ 0magnet.com needs to review the security of your connection before
95
+ proceeding.
96
+ </div>
97
+ <div id="cf-challenge-fact-wrapper" class="fact spacer hidden" style="display: block; visibility: visible;">
98
+ <span class="fact-title">Did you know</span> <span id="cf-challenge-fact" class="body-text">bots
99
+ historically made up nearly 40% of all internet traffic?</span>
100
+ </div>
101
+ <div id="cf-challenge-explainer-expandable" class="hidden expandable body-text spacer"
102
+ style="display: none;">
103
+ <div class="expandable-title" id="cf-challenge-explainer-summary"><button class="expandable-summary-btn"
104
+ id="cf-challenge-explainer-btn" type="button"> Why am I seeing this page? <span
105
+ class="caret-icon-wrapper">
106
+ <div class="caret-icon"></div>
107
+ </span> </button> </div>
108
+ <div class="expandable-details" id="cf-challenge-explainer-details"> Requests from malicious bots can
109
+ pose as legitimate traffic. Occasionally, you may see this page while the site ensures that the
110
+ connection is secure.</div>
111
+ </div>
112
+ <div id="cf-challenge-success" style="display: none;">
113
+ <div class="h2"><span class="icon-wrapper"><img class="heading-icon" alt="Success icon"
114
+ src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADQAAAA0CAMAAADypuvZAAAANlBMVEUAAAAxMTEwMDAxMTExMTEwMDAwMDAwMDAxMTExMTExMTEwMDAwMDAxMTExMTEwMDAwMDAxMTHB9N+uAAAAEXRSTlMA3zDvfyBAEJC/n3BQz69gX7VMkcMAAAGySURBVEjHnZZbFoMgDEQJiDzVuv/NtgbtFGuQ4/zUKpeMIQbUhXSKE5l1XSn4pFWHRm/WShT1HRLWC01LGxFEVkCc30eYkLJ1Sjk9pvkw690VY6k8DWP9OM9yMG0Koi+mi8XA36NXmW0UXra4eJ3iwHfrfXVlgL0NqqGBHdqfeQhMmyJ48WDuKP81h3+SMPeRKkJcSXiLUK4XTHCjESOnz1VUXQoc6lgi2x4cI5aTQ201Mt8wHysI5fc05M5c81uZEtHcMKhxZ7iYEty1GfhLvGKpm+EYkdGxm1F5axmcB93DoORIbXfdN7f+hlFuyxtDP+sxtBnF43cIYwaZAWRgzxIoiXEMESoPlMhwLRDXeK772CAzXEdBRV7cmnoVBp0OSlyGidEzJTFq5hhcsA5388oSGM6b5p+qjpZrBlMS9xj4AwXmz108ukU1IomM3ceiW0CDwHCqp1NjAqXlFrbga+xuloQJ+tuyfbIBPNpqnmxqT7dPaOnZqBfhSBCteJAxWj58zLk2xgg+SPGYM6dRO6WczSnIxxwEExRaO+UyCUhbOp7CGQ+kxSUfNtLQFC+Po29vvy7jj4y0yAAAAABJRU5ErkJggg=="></span>Connection
115
+ is secure</div>
116
+ <div class="core-msg spacer">Proceeding...</div>
117
+ </div>
118
+ <form id="challenge-form"
119
+ action="/search?q=2022&amp;__cf_chl_f_tk=1qWQAgl8.irfEoDb73Rb0pUm1SXbis3ZamDAIoTcPks-1659200251-0-gaNycGzNCFE"
120
+ method="POST" enctype="application/x-www-form-urlencoded">
121
+ <input type="hidden" name="md"
122
+ value="OghUU_ltYW6I0fpWl7rE4yHBGBPHfpZQIKZRSEpJKjE-1659200251-0-AWB-KR-MabhObmvYa3mR5-xDk3qZVV73547wjnl-QtfPoTxe017AXt4WUskEcVzEIUKC7dsJoiy8ec1NA0fxdnI8X9OfPhtynl00ReWBVZc_3Gba_wigWMmM_9e8PX9vpVDcXpCRbz1BJ5_YLsba9TJM1sp14U9RtIce-tRBB53qoxLxJRz9QFmckEVBvsba4RfoycOvYPMMsfAqSkq13qtsA3Kd6RDB5Rb5-qF8674DsB4AMvd9xu_fBplQqKjOpEtrThCUtw8M2DHY8FUr_owUo1NIS1s6fSBEyHh6ehz9CidJ7zpRwYZFwgz_Pq9i8LmQG_AajozOJJhLp-tox0dptbUZnRNGt3hGQgrNu3jlCfwPC2XVp7xgLvmZoPYrzzrZoi_wErnIvVgyGCw9-sDPblPdvLBUz6uXNreWwThEW6PeRtMXnePO9UwcZmj_2awhwcVSHSLz1t1z22LtVsQ8xNpMbiE7xDvI2D5LNHAPIUC7Wp4AcehWD-fEm0w5jnVTWOmFlVRxtcnYZSMfDSaRUxsZ3hg5B1-ghVMEX6M-r_hAd6pLKNmjIfdl_Nvdm6veQvV-gTFaULbfuhmQQjYEb9G2IptDiNTZs5S7FtmjqVBAA7PmvwBTQwxw86J0cV3v_4pT1Oj8tigwiPny35HMTrKRmRZWaAZudCmWxDZkJIW8Eir7KQ57ba-u9cHh0A">
123
+ <input type="hidden" name="r"
124
+ value="q.UUtPBFcFi4IkcVw3l4U_xJJKDIbHJj7xmuB43IIAI-1659200251-0-Aa4lU5RipD+d4of3hcdQ0rVmZ4ulb3siZYKwhm1jGNiA+/9b8IW1HL8k1GrsYEVexDW7ycP5UINQZ1sYJvZBTCQe3lhyGLHdLZ7KdI9RXKEbPx1NUOR/HthCD0Wbo7H41jbAf7l+HhH0zTLjm77/6NpJZHcgfsbBwwubl4R3oLarzPSByV2PVBnkuMyKCYgibriuMUt2iJHoMLx7Cr+Bmjx1KEFCrPYP0t7vgQs2APTylhL7ebP77XB9ndxU6Of3r4eHnTwLIcomFJ3+jqL6pzFaNoXdUBHrv9oZs/33KZjf2NB8cu5KUpAdM2lp3t5oTSQE19fJVroxmf91hcTdele3F2DAeawFGDwncm/Jo725SlyNk4TqsmR+il7DLkS/FTcCNzQe4cQM6DRWdmF9I1OohAl1/uGXYqUJSK1F45n3gec/pyPTQyZI0OLc7sCGYXfn3VPFsGATkg5mxE9rgZIB2b6ID9JggzIlDdYxlQRWecpruu07KOgk3m7g95lyHNZTohqemo4T8Z2MOZECjmXGMAuwvvk4d5sakVHr39kmAY6aSfXrRB+iONCOKkahbumrVmjLsnMvrpTb0DFE5pRAxwANPZKzb6Ikmlvxh7oJIPOB0mG9hDeoc/AJVlvZJV4CrpDLulNjHetAWXMwMptZuYJGEhcDXxmYj0ybntTCU4Y3JJQc5K+7ehSdnluTvMueWfs628854r4PcOONZzsO337j+3lUxrP5vDUCzYD25FNxvs8jGfqRivqHMOq2z9iOs0sHQTlHroLLSt2G7M50yRJBGTfxrIsvLq+ML3e/mRIkYIQxOcp8ugoPoT4c9gex3OyY0cnnA2/9OibQs9kevwf9DSnutMRRcbIXZI0XO6FY07+MykWqUcXygwMHs1vQxhaQ26NFYwolEWfOL7EQpp4GKyN30nL4nPNil/7GsXIr5SC+o55KI0l3AOEYE1jirVx2G0U7Br7SW80Ih4Fn5U/+4qFfW57GAJrpuk9qjFfJehe7wFBu5bHghEGRhKAu0wvpY7UTc9AiacMfP7ujVWi4DIbTCfOzOgVT8E0T6KaUurBppPJflLQE41c8n29ULyKmki9t8lIKvxYmv/3/AauhXFAExh+JrdnaeSDxhJYjWEUDJiaNvnkDCHMxPs/bePhSg4DYRMh4ngcOHCRkkRlDjipUUgeCrwNBY0qu2DIqLZXI1ZMwU+R0nuWnwc5xJuMHtrLkWbziP0FQcGaF0B6SaFIcOLnWG7YjJZFzxjFpvLb8GnZxk7i2YHCDTn0Stq3JDZHCkjJQjaPMmuK+5KYzfaSHcKOcaQbkbyDjn3t/XQX3a7lknngVchIJVsVn8osqgKvOx3aAdCicYKR6QaukrXHhR9uIEbPdoBYqZPKFz0uvVOShsUx2f65CaI8wWMjOBRWxTK1xUPNsetOiyYSvNwjeULaCXPKLi2qv/cZRRbsr3g5ghdHvNTpD/O0/xUgiziev3/9CpNopyr6VzLar9dJ/s++imXY1w1TCRJ2uCI2H70XGBGWxSZdbnfxU+j3zNCL0dBuabwhDd4ZnOZmlFZjGBiOUpsWdRrHd3c+QpwXdxB3QurRwX6J+LhmkqcWsPhP7LlMnN7dr2HUFZ5FS4LASl5AOf8hjCnO06FT8fWLl1eKVVjCugx9w54qjGqOV8A0v/PdWr7Ic0WfriyYbmwn/XnH8t0ri3bqDZsDkfhQMMF9JSWHEdoGD60a7McGDxr4g9s3LZhq5KozgSvyG+RUBPla8g2zB253hR7amWE5WO4IChl7AXmRB89F9u2+AoDIbefseb3pwG7GfkYpSBwmgJ4Ju4LAWoSfBhZSPMQadHZOCg36R11KesUy+NAy9bvD1bE3UMx9e2NbFohu6sXlilpnxINHp0sFEeulreEjWSQreri1eZeKxV2QfKIzWUiMoNdyT0JzM+/brYzddBpO2DrlnK5bEPWgtu0D7d4Kfm+0T7S//Fq+hxf40lSMPP8cBlan6sEd2iWmZ6gW3z43wNbJaPQIUDgb58ELxaEKQN4tOOy75/XXfISNnhG0K8M79a175WUb8v0A=">
125
+
126
+ <span style="display: none;"><span class="text-gray-600" data-translate="error">error code:
127
+ 1020</span></span>
128
+ </form>
129
+ </div>
130
+ </div>
131
+ <script>
132
+ (function () {
133
+ var trkjs = document.createElement('img');
134
+ trkjs.setAttribute('src', '/cdn-cgi/images/trace/captcha/js/transparent.gif?ray=732fa2449b567521');
135
+ trkjs.setAttribute('style', 'display: none');
136
+ document.body.appendChild(trkjs);
137
+ var cpo = document.createElement('script');
138
+ cpo.src = '/cdn-cgi/challenge-platform/h/g/orchestrate/managed/v1?ray=732fa2449b567521';
139
+ window._cf_chl_opt.cOgUHash = location.hash === '' && location.href.indexOf('#') !== -1 ? '#' : location.hash;
140
+ window._cf_chl_opt.cOgUQuery = location.search === '' && location.href.slice(0, -window._cf_chl_opt.cOgUHash.length).indexOf('?') !== -1 ? '?' : location.search;
141
+ if (window.history && window.history.replaceState) {
142
+ var ogU = location.pathname + window._cf_chl_opt.cOgUQuery + window._cf_chl_opt.cOgUHash;
143
+ history.replaceState(null, null, "\/search?q=2022&__cf_chl_rt_tk=1qWQAgl8.irfEoDb73Rb0pUm1SXbis3ZamDAIoTcPks-1659200251-0-gaNycGzNCFE" + window._cf_chl_opt.cOgUHash);
144
+ cpo.onload = function () {
145
+ history.replaceState(null, null, ogU);
146
+ };
147
+ }
148
+ document.getElementsByTagName('head')[0].appendChild(cpo);
149
+ }());
150
+ </script><img src="/cdn-cgi/images/trace/captcha/js/transparent.gif?ray=732fa2449b567521" style="display: none">
151
+
152
+ <div class="footer" role="contentinfo">
153
+ <div class="footer-inner">
154
+ <div class="clearfix diagnostic-wrapper">
155
+ <div class="ray-id">Ray ID: <code>732fa2449b567521</code></div>
156
+ </div>
157
+ <div class="text-center">
158
+ Performance &amp; security by
159
+ <a rel="noopener noreferrer" href="https://www.cloudflare.com" target="_blank">Cloudflare</a>
160
+ </div>
161
+ </div>
162
+ </div>
163
+
164
+
165
+ </body>
166
+
167
  </html>
package.json CHANGED
@@ -1,7 +1,7 @@
1
- {
2
- "name": "flaresolverr",
3
- "version": "3.3.21",
4
- "description": "Proxy server to bypass Cloudflare protection",
5
- "author": "Diego Heras (ngosang / [email protected])",
6
- "license": "MIT"
7
- }
 
1
+ {
2
+ "name": "flaresolverr",
3
+ "version": "3.3.21",
4
+ "description": "Proxy server to bypass Cloudflare protection",
5
+ "author": "Diego Heras (ngosang / [email protected])",
6
+ "license": "MIT"
7
+ }
requirements.txt CHANGED
@@ -1,13 +1,13 @@
1
- bottle==0.12.25
2
- waitress==2.1.2
3
- selenium==4.15.2
4
- func-timeout==4.3.5
5
- prometheus-client==0.17.1
6
- # required by undetected_chromedriver
7
- requests==2.32.3
8
- certifi==2024.07.04
9
- websockets==11.0.3
10
- # only required for linux and macos
11
- xvfbwrapper==0.2.9; platform_system != "Windows"
12
- # only required for windows
13
- pefile==2023.2.7; platform_system == "Windows"
 
1
+ bottle==0.12.25
2
+ waitress==2.1.2
3
+ selenium==4.15.2
4
+ func-timeout==4.3.5
5
+ prometheus-client==0.17.1
6
+ # required by undetected_chromedriver
7
+ requests==2.32.3
8
+ certifi==2024.07.04
9
+ websockets==11.0.3
10
+ # only required for linux and macos
11
+ xvfbwrapper==0.2.9; platform_system != "Windows"
12
+ # only required for windows
13
+ pefile==2023.2.7; platform_system == "Windows"
resources/flaresolverr_logo.svg CHANGED
src/bottle_plugins/error_plugin.py CHANGED
@@ -1,22 +1,22 @@
1
- from bottle import response
2
- import logging
3
-
4
-
5
- def error_plugin(callback):
6
- """
7
- Bottle plugin to handle exceptions
8
- https://stackoverflow.com/a/32764250
9
- """
10
-
11
- def wrapper(*args, **kwargs):
12
- try:
13
- actual_response = callback(*args, **kwargs)
14
- except Exception as e:
15
- logging.error(str(e))
16
- actual_response = {
17
- "error": str(e)
18
- }
19
- response.status = 500
20
- return actual_response
21
-
22
- return wrapper
 
1
+ from bottle import response
2
+ import logging
3
+
4
+
5
+ def error_plugin(callback):
6
+ """
7
+ Bottle plugin to handle exceptions
8
+ https://stackoverflow.com/a/32764250
9
+ """
10
+
11
+ def wrapper(*args, **kwargs):
12
+ try:
13
+ actual_response = callback(*args, **kwargs)
14
+ except Exception as e:
15
+ logging.error(str(e))
16
+ actual_response = {
17
+ "error": str(e)
18
+ }
19
+ response.status = 500
20
+ return actual_response
21
+
22
+ return wrapper
src/bottle_plugins/logger_plugin.py CHANGED
@@ -1,23 +1,23 @@
1
- from bottle import request, response
2
- import logging
3
-
4
-
5
- def logger_plugin(callback):
6
- """
7
- Bottle plugin to use logging module
8
- http://bottlepy.org/docs/dev/plugindev.html
9
-
10
- Wrap a Bottle request so that a log line is emitted after it's handled.
11
- (This decorator can be extended to take the desired logger as a param.)
12
- """
13
-
14
- def wrapper(*args, **kwargs):
15
- actual_response = callback(*args, **kwargs)
16
- if not request.url.endswith("/health"):
17
- logging.info('%s %s %s %s' % (request.remote_addr,
18
- request.method,
19
- request.url,
20
- response.status))
21
- return actual_response
22
-
23
- return wrapper
 
1
+ from bottle import request, response
2
+ import logging
3
+
4
+
5
+ def logger_plugin(callback):
6
+ """
7
+ Bottle plugin to use logging module
8
+ http://bottlepy.org/docs/dev/plugindev.html
9
+
10
+ Wrap a Bottle request so that a log line is emitted after it's handled.
11
+ (This decorator can be extended to take the desired logger as a param.)
12
+ """
13
+
14
+ def wrapper(*args, **kwargs):
15
+ actual_response = callback(*args, **kwargs)
16
+ if not request.url.endswith("/health"):
17
+ logging.info('%s %s %s %s' % (request.remote_addr,
18
+ request.method,
19
+ request.url,
20
+ response.status))
21
+ return actual_response
22
+
23
+ return wrapper
src/bottle_plugins/prometheus_plugin.py CHANGED
@@ -1,66 +1,66 @@
1
- import logging
2
- import os
3
- import urllib.parse
4
-
5
- from bottle import request
6
- from dtos import V1RequestBase, V1ResponseBase
7
- from metrics import start_metrics_http_server, REQUEST_COUNTER, REQUEST_DURATION
8
-
9
- PROMETHEUS_ENABLED = os.environ.get('PROMETHEUS_ENABLED', 'false').lower() == 'true'
10
- PROMETHEUS_PORT = int(os.environ.get('PROMETHEUS_PORT', 8192))
11
-
12
-
13
- def setup():
14
- if PROMETHEUS_ENABLED:
15
- start_metrics_http_server(PROMETHEUS_PORT)
16
-
17
-
18
- def prometheus_plugin(callback):
19
- """
20
- Bottle plugin to expose Prometheus metrics
21
- http://bottlepy.org/docs/dev/plugindev.html
22
- """
23
- def wrapper(*args, **kwargs):
24
- actual_response = callback(*args, **kwargs)
25
-
26
- if PROMETHEUS_ENABLED:
27
- try:
28
- export_metrics(actual_response)
29
- except Exception as e:
30
- logging.warning("Error exporting metrics: " + str(e))
31
-
32
- return actual_response
33
-
34
- def export_metrics(actual_response):
35
- res = V1ResponseBase(actual_response)
36
-
37
- if res.startTimestamp is None or res.endTimestamp is None:
38
- # skip management and healthcheck endpoints
39
- return
40
-
41
- domain = "unknown"
42
- if res.solution and res.solution.url:
43
- domain = parse_domain_url(res.solution.url)
44
- else:
45
- # timeout error
46
- req = V1RequestBase(request.json)
47
- if req.url:
48
- domain = parse_domain_url(req.url)
49
-
50
- run_time = (res.endTimestamp - res.startTimestamp) / 1000
51
- REQUEST_DURATION.labels(domain=domain).observe(run_time)
52
-
53
- result = "unknown"
54
- if res.message == "Challenge solved!":
55
- result = "solved"
56
- elif res.message == "Challenge not detected!":
57
- result = "not_detected"
58
- elif res.message.startswith("Error"):
59
- result = "error"
60
- REQUEST_COUNTER.labels(domain=domain, result=result).inc()
61
-
62
- def parse_domain_url(url):
63
- parsed_url = urllib.parse.urlparse(url)
64
- return parsed_url.hostname
65
-
66
- return wrapper
 
1
+ import logging
2
+ import os
3
+ import urllib.parse
4
+
5
+ from bottle import request
6
+ from dtos import V1RequestBase, V1ResponseBase
7
+ from metrics import start_metrics_http_server, REQUEST_COUNTER, REQUEST_DURATION
8
+
9
+ PROMETHEUS_ENABLED = os.environ.get('PROMETHEUS_ENABLED', 'false').lower() == 'true'
10
+ PROMETHEUS_PORT = int(os.environ.get('PROMETHEUS_PORT', 8192))
11
+
12
+
13
+ def setup():
14
+ if PROMETHEUS_ENABLED:
15
+ start_metrics_http_server(PROMETHEUS_PORT)
16
+
17
+
18
+ def prometheus_plugin(callback):
19
+ """
20
+ Bottle plugin to expose Prometheus metrics
21
+ http://bottlepy.org/docs/dev/plugindev.html
22
+ """
23
+ def wrapper(*args, **kwargs):
24
+ actual_response = callback(*args, **kwargs)
25
+
26
+ if PROMETHEUS_ENABLED:
27
+ try:
28
+ export_metrics(actual_response)
29
+ except Exception as e:
30
+ logging.warning("Error exporting metrics: " + str(e))
31
+
32
+ return actual_response
33
+
34
+ def export_metrics(actual_response):
35
+ res = V1ResponseBase(actual_response)
36
+
37
+ if res.startTimestamp is None or res.endTimestamp is None:
38
+ # skip management and healthcheck endpoints
39
+ return
40
+
41
+ domain = "unknown"
42
+ if res.solution and res.solution.url:
43
+ domain = parse_domain_url(res.solution.url)
44
+ else:
45
+ # timeout error
46
+ req = V1RequestBase(request.json)
47
+ if req.url:
48
+ domain = parse_domain_url(req.url)
49
+
50
+ run_time = (res.endTimestamp - res.startTimestamp) / 1000
51
+ REQUEST_DURATION.labels(domain=domain).observe(run_time)
52
+
53
+ result = "unknown"
54
+ if res.message == "Challenge solved!":
55
+ result = "solved"
56
+ elif res.message == "Challenge not detected!":
57
+ result = "not_detected"
58
+ elif res.message.startswith("Error"):
59
+ result = "error"
60
+ REQUEST_COUNTER.labels(domain=domain, result=result).inc()
61
+
62
+ def parse_domain_url(url):
63
+ parsed_url = urllib.parse.urlparse(url)
64
+ return parsed_url.hostname
65
+
66
+ return wrapper
src/build_package.py CHANGED
@@ -1,110 +1,110 @@
1
- import os
2
- import platform
3
- import shutil
4
- import subprocess
5
- import sys
6
- import zipfile
7
-
8
- import requests
9
-
10
-
11
- def clean_files():
12
- try:
13
- shutil.rmtree(os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'build'))
14
- except Exception:
15
- pass
16
- try:
17
- shutil.rmtree(os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist'))
18
- except Exception:
19
- pass
20
- try:
21
- shutil.rmtree(os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist_chrome'))
22
- except Exception:
23
- pass
24
-
25
-
26
- def download_chromium():
27
- # https://commondatastorage.googleapis.com/chromium-browser-snapshots/index.html?prefix=Linux_x64/
28
- revision = "1260008" if os.name == 'nt' else '1260015'
29
- arch = 'Win_x64' if os.name == 'nt' else 'Linux_x64'
30
- dl_file = 'chrome-win' if os.name == 'nt' else 'chrome-linux'
31
- dl_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist_chrome')
32
- dl_path_folder = os.path.join(dl_path, dl_file)
33
- dl_path_zip = dl_path_folder + '.zip'
34
-
35
- # response = requests.get(
36
- # f'https://commondatastorage.googleapis.com/chromium-browser-snapshots/{arch}/LAST_CHANGE',
37
- # timeout=30)
38
- # revision = response.text.strip()
39
- print("Downloading revision: " + revision)
40
-
41
- os.mkdir(dl_path)
42
- with requests.get(
43
- f'https://commondatastorage.googleapis.com/chromium-browser-snapshots/{arch}/{revision}/{dl_file}.zip',
44
- stream=True) as r:
45
- r.raise_for_status()
46
- with open(dl_path_zip, 'wb') as f:
47
- for chunk in r.iter_content(chunk_size=8192):
48
- f.write(chunk)
49
- print("File downloaded: " + dl_path_zip)
50
- with zipfile.ZipFile(dl_path_zip, 'r') as zip_ref:
51
- zip_ref.extractall(dl_path)
52
- os.remove(dl_path_zip)
53
-
54
- chrome_path = os.path.join(dl_path, "chrome")
55
- shutil.move(dl_path_folder, chrome_path)
56
- print("Extracted in: " + chrome_path)
57
-
58
- if os.name != 'nt':
59
- # Give executable permissions for *nix
60
- # file * | grep executable | cut -d: -f1
61
- print("Giving executable permissions...")
62
- execs = ['chrome', 'chrome_crashpad_handler', 'chrome_sandbox', 'chrome-wrapper', 'xdg-mime', 'xdg-settings']
63
- for exec_file in execs:
64
- exec_path = os.path.join(chrome_path, exec_file)
65
- os.chmod(exec_path, 0o755)
66
-
67
-
68
- def run_pyinstaller():
69
- sep = ';' if os.name == 'nt' else ':'
70
- result = subprocess.run([sys.executable, "-m", "PyInstaller",
71
- "--icon", "resources/flaresolverr_logo.ico",
72
- "--add-data", f"package.json{sep}.",
73
- "--add-data", f"{os.path.join('dist_chrome', 'chrome')}{sep}chrome",
74
- os.path.join("src", "flaresolverr.py")],
75
- cwd=os.pardir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
76
- if result.returncode != 0:
77
- print(result.stderr.decode('utf-8'))
78
- raise Exception("Error running pyInstaller")
79
-
80
-
81
- def compress_package():
82
- dist_folder = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist')
83
- package_folder = os.path.join(dist_folder, 'package')
84
- shutil.move(os.path.join(dist_folder, 'flaresolverr'), os.path.join(package_folder, 'flaresolverr'))
85
- print("Package folder: " + package_folder)
86
-
87
- compr_format = 'zip' if os.name == 'nt' else 'gztar'
88
- compr_file_name = 'flaresolverr_windows_x64' if os.name == 'nt' else 'flaresolverr_linux_x64'
89
- compr_file_path = os.path.join(dist_folder, compr_file_name)
90
- shutil.make_archive(compr_file_path, compr_format, package_folder)
91
- print("Compressed file path: " + compr_file_path)
92
-
93
-
94
- if __name__ == "__main__":
95
- print("Building package...")
96
- print("Platform: " + platform.platform())
97
-
98
- print("Cleaning previous build...")
99
- clean_files()
100
-
101
- print("Downloading Chromium...")
102
- download_chromium()
103
-
104
- print("Building pyinstaller executable... ")
105
- run_pyinstaller()
106
-
107
- print("Compressing package... ")
108
- compress_package()
109
-
110
- # NOTE: python -m pip install pyinstaller
 
1
+ import os
2
+ import platform
3
+ import shutil
4
+ import subprocess
5
+ import sys
6
+ import zipfile
7
+
8
+ import requests
9
+
10
+
11
+ def clean_files():
12
+ try:
13
+ shutil.rmtree(os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'build'))
14
+ except Exception:
15
+ pass
16
+ try:
17
+ shutil.rmtree(os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist'))
18
+ except Exception:
19
+ pass
20
+ try:
21
+ shutil.rmtree(os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist_chrome'))
22
+ except Exception:
23
+ pass
24
+
25
+
26
+ def download_chromium():
27
+ # https://commondatastorage.googleapis.com/chromium-browser-snapshots/index.html?prefix=Linux_x64/
28
+ revision = "1260008" if os.name == 'nt' else '1260015'
29
+ arch = 'Win_x64' if os.name == 'nt' else 'Linux_x64'
30
+ dl_file = 'chrome-win' if os.name == 'nt' else 'chrome-linux'
31
+ dl_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist_chrome')
32
+ dl_path_folder = os.path.join(dl_path, dl_file)
33
+ dl_path_zip = dl_path_folder + '.zip'
34
+
35
+ # response = requests.get(
36
+ # f'https://commondatastorage.googleapis.com/chromium-browser-snapshots/{arch}/LAST_CHANGE',
37
+ # timeout=30)
38
+ # revision = response.text.strip()
39
+ print("Downloading revision: " + revision)
40
+
41
+ os.mkdir(dl_path)
42
+ with requests.get(
43
+ f'https://commondatastorage.googleapis.com/chromium-browser-snapshots/{arch}/{revision}/{dl_file}.zip',
44
+ stream=True) as r:
45
+ r.raise_for_status()
46
+ with open(dl_path_zip, 'wb') as f:
47
+ for chunk in r.iter_content(chunk_size=8192):
48
+ f.write(chunk)
49
+ print("File downloaded: " + dl_path_zip)
50
+ with zipfile.ZipFile(dl_path_zip, 'r') as zip_ref:
51
+ zip_ref.extractall(dl_path)
52
+ os.remove(dl_path_zip)
53
+
54
+ chrome_path = os.path.join(dl_path, "chrome")
55
+ shutil.move(dl_path_folder, chrome_path)
56
+ print("Extracted in: " + chrome_path)
57
+
58
+ if os.name != 'nt':
59
+ # Give executable permissions for *nix
60
+ # file * | grep executable | cut -d: -f1
61
+ print("Giving executable permissions...")
62
+ execs = ['chrome', 'chrome_crashpad_handler', 'chrome_sandbox', 'chrome-wrapper', 'xdg-mime', 'xdg-settings']
63
+ for exec_file in execs:
64
+ exec_path = os.path.join(chrome_path, exec_file)
65
+ os.chmod(exec_path, 0o755)
66
+
67
+
68
+ def run_pyinstaller():
69
+ sep = ';' if os.name == 'nt' else ':'
70
+ result = subprocess.run([sys.executable, "-m", "PyInstaller",
71
+ "--icon", "resources/flaresolverr_logo.ico",
72
+ "--add-data", f"package.json{sep}.",
73
+ "--add-data", f"{os.path.join('dist_chrome', 'chrome')}{sep}chrome",
74
+ os.path.join("src", "flaresolverr.py")],
75
+ cwd=os.pardir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
76
+ if result.returncode != 0:
77
+ print(result.stderr.decode('utf-8'))
78
+ raise Exception("Error running pyInstaller")
79
+
80
+
81
+ def compress_package():
82
+ dist_folder = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist')
83
+ package_folder = os.path.join(dist_folder, 'package')
84
+ shutil.move(os.path.join(dist_folder, 'flaresolverr'), os.path.join(package_folder, 'flaresolverr'))
85
+ print("Package folder: " + package_folder)
86
+
87
+ compr_format = 'zip' if os.name == 'nt' else 'gztar'
88
+ compr_file_name = 'flaresolverr_windows_x64' if os.name == 'nt' else 'flaresolverr_linux_x64'
89
+ compr_file_path = os.path.join(dist_folder, compr_file_name)
90
+ shutil.make_archive(compr_file_path, compr_format, package_folder)
91
+ print("Compressed file path: " + compr_file_path)
92
+
93
+
94
+ if __name__ == "__main__":
95
+ print("Building package...")
96
+ print("Platform: " + platform.platform())
97
+
98
+ print("Cleaning previous build...")
99
+ clean_files()
100
+
101
+ print("Downloading Chromium...")
102
+ download_chromium()
103
+
104
+ print("Building pyinstaller executable... ")
105
+ run_pyinstaller()
106
+
107
+ print("Compressing package... ")
108
+ compress_package()
109
+
110
+ # NOTE: python -m pip install pyinstaller
src/dtos.py CHANGED
@@ -1,86 +1,86 @@
1
-
2
- STATUS_OK = "ok"
3
- STATUS_ERROR = "error"
4
-
5
-
6
- class ChallengeResolutionResultT:
7
- url: str = None
8
- status: int = None
9
- headers: list = None
10
- response: str = None
11
- cookies: list = None
12
- userAgent: str = None
13
-
14
- def __init__(self, _dict):
15
- self.__dict__.update(_dict)
16
-
17
-
18
- class ChallengeResolutionT:
19
- status: str = None
20
- message: str = None
21
- result: ChallengeResolutionResultT = None
22
-
23
- def __init__(self, _dict):
24
- self.__dict__.update(_dict)
25
- if self.result is not None:
26
- self.result = ChallengeResolutionResultT(self.result)
27
-
28
-
29
- class V1RequestBase(object):
30
- # V1RequestBase
31
- cmd: str = None
32
- cookies: list = None
33
- maxTimeout: int = None
34
- proxy: dict = None
35
- session: str = None
36
- session_ttl_minutes: int = None
37
- headers: list = None # deprecated v2.0.0, not used
38
- userAgent: str = None # deprecated v2.0.0, not used
39
-
40
- # V1Request
41
- url: str = None
42
- postData: str = None
43
- returnOnlyCookies: bool = None
44
- download: bool = None # deprecated v2.0.0, not used
45
- returnRawHtml: bool = None # deprecated v2.0.0, not used
46
-
47
- def __init__(self, _dict):
48
- self.__dict__.update(_dict)
49
-
50
-
51
- class V1ResponseBase(object):
52
- # V1ResponseBase
53
- status: str = None
54
- message: str = None
55
- session: str = None
56
- sessions: list[str] = None
57
- startTimestamp: int = None
58
- endTimestamp: int = None
59
- version: str = None
60
-
61
- # V1ResponseSolution
62
- solution: ChallengeResolutionResultT = None
63
-
64
- # hidden vars
65
- __error_500__: bool = False
66
-
67
- def __init__(self, _dict):
68
- self.__dict__.update(_dict)
69
- if self.solution is not None:
70
- self.solution = ChallengeResolutionResultT(self.solution)
71
-
72
-
73
- class IndexResponse(object):
74
- msg: str = None
75
- version: str = None
76
- userAgent: str = None
77
-
78
- def __init__(self, _dict):
79
- self.__dict__.update(_dict)
80
-
81
-
82
- class HealthResponse(object):
83
- status: str = None
84
-
85
- def __init__(self, _dict):
86
- self.__dict__.update(_dict)
 
1
+
2
+ STATUS_OK = "ok"
3
+ STATUS_ERROR = "error"
4
+
5
+
6
+ class ChallengeResolutionResultT:
7
+ url: str = None
8
+ status: int = None
9
+ headers: list = None
10
+ response: str = None
11
+ cookies: list = None
12
+ userAgent: str = None
13
+
14
+ def __init__(self, _dict):
15
+ self.__dict__.update(_dict)
16
+
17
+
18
+ class ChallengeResolutionT:
19
+ status: str = None
20
+ message: str = None
21
+ result: ChallengeResolutionResultT = None
22
+
23
+ def __init__(self, _dict):
24
+ self.__dict__.update(_dict)
25
+ if self.result is not None:
26
+ self.result = ChallengeResolutionResultT(self.result)
27
+
28
+
29
+ class V1RequestBase(object):
30
+ # V1RequestBase
31
+ cmd: str = None
32
+ cookies: list = None
33
+ maxTimeout: int = None
34
+ proxy: dict = None
35
+ session: str = None
36
+ session_ttl_minutes: int = None
37
+ headers: list = None # deprecated v2.0.0, not used
38
+ userAgent: str = None # deprecated v2.0.0, not used
39
+
40
+ # V1Request
41
+ url: str = None
42
+ postData: str = None
43
+ returnOnlyCookies: bool = None
44
+ download: bool = None # deprecated v2.0.0, not used
45
+ returnRawHtml: bool = None # deprecated v2.0.0, not used
46
+
47
+ def __init__(self, _dict):
48
+ self.__dict__.update(_dict)
49
+
50
+
51
+ class V1ResponseBase(object):
52
+ # V1ResponseBase
53
+ status: str = None
54
+ message: str = None
55
+ session: str = None
56
+ sessions: list[str] = None
57
+ startTimestamp: int = None
58
+ endTimestamp: int = None
59
+ version: str = None
60
+
61
+ # V1ResponseSolution
62
+ solution: ChallengeResolutionResultT = None
63
+
64
+ # hidden vars
65
+ __error_500__: bool = False
66
+
67
+ def __init__(self, _dict):
68
+ self.__dict__.update(_dict)
69
+ if self.solution is not None:
70
+ self.solution = ChallengeResolutionResultT(self.solution)
71
+
72
+
73
+ class IndexResponse(object):
74
+ msg: str = None
75
+ version: str = None
76
+ userAgent: str = None
77
+
78
+ def __init__(self, _dict):
79
+ self.__dict__.update(_dict)
80
+
81
+
82
+ class HealthResponse(object):
83
+ status: str = None
84
+
85
+ def __init__(self, _dict):
86
+ self.__dict__.update(_dict)
src/flaresolverr.py CHANGED
@@ -1,125 +1,125 @@
1
- import json
2
- import logging
3
- import os
4
- import sys
5
-
6
- import certifi
7
- from bottle import run, response, Bottle, request, ServerAdapter
8
-
9
- from bottle_plugins.error_plugin import error_plugin
10
- from bottle_plugins.logger_plugin import logger_plugin
11
- from bottle_plugins import prometheus_plugin
12
- from dtos import V1RequestBase
13
- import flaresolverr_service
14
- import utils
15
-
16
-
17
- class JSONErrorBottle(Bottle):
18
- """
19
- Handle 404 errors
20
- """
21
- def default_error_handler(self, res):
22
- response.content_type = 'application/json'
23
- return json.dumps(dict(error=res.body, status_code=res.status_code))
24
-
25
-
26
- app = JSONErrorBottle()
27
-
28
-
29
- @app.route('/')
30
- def index():
31
- """
32
- Show welcome message
33
- """
34
- res = flaresolverr_service.index_endpoint()
35
- return utils.object_to_dict(res)
36
-
37
-
38
- @app.route('/health')
39
- def health():
40
- """
41
- Healthcheck endpoint.
42
- This endpoint is special because it doesn't print traces
43
- """
44
- res = flaresolverr_service.health_endpoint()
45
- return utils.object_to_dict(res)
46
-
47
-
48
- @app.post('/v1')
49
- def controller_v1():
50
- """
51
- Controller v1
52
- """
53
- req = V1RequestBase(request.json)
54
- res = flaresolverr_service.controller_v1_endpoint(req)
55
- if res.__error_500__:
56
- response.status = 500
57
- return utils.object_to_dict(res)
58
-
59
-
60
- if __name__ == "__main__":
61
- # check python version
62
- if sys.version_info < (3, 9):
63
- raise Exception("The Python version is less than 3.9, a version equal to or higher is required.")
64
-
65
- # fix for HEADLESS=false in Windows binary
66
- # https://stackoverflow.com/a/27694505
67
- if os.name == 'nt':
68
- import multiprocessing
69
- multiprocessing.freeze_support()
70
-
71
- # fix ssl certificates for compiled binaries
72
- # https://github.com/pyinstaller/pyinstaller/issues/7229
73
- # https://stackoverflow.com/questions/55736855/how-to-change-the-cafile-argument-in-the-ssl-module-in-python3
74
- os.environ["REQUESTS_CA_BUNDLE"] = certifi.where()
75
- os.environ["SSL_CERT_FILE"] = certifi.where()
76
-
77
- # validate configuration
78
- log_level = os.environ.get('LOG_LEVEL', 'info').upper()
79
- log_html = utils.get_config_log_html()
80
- headless = utils.get_config_headless()
81
- server_host = os.environ.get('HOST', '0.0.0.0')
82
- server_port = int(os.environ.get('PORT', 8191))
83
-
84
- # configure logger
85
- logger_format = '%(asctime)s %(levelname)-8s %(message)s'
86
- if log_level == 'DEBUG':
87
- logger_format = '%(asctime)s %(levelname)-8s ReqId %(thread)s %(message)s'
88
- logging.basicConfig(
89
- format=logger_format,
90
- level=log_level,
91
- datefmt='%Y-%m-%d %H:%M:%S',
92
- handlers=[
93
- logging.StreamHandler(sys.stdout)
94
- ]
95
- )
96
- # disable warning traces from urllib3
97
- logging.getLogger('urllib3').setLevel(logging.ERROR)
98
- logging.getLogger('selenium.webdriver.remote.remote_connection').setLevel(logging.WARNING)
99
- logging.getLogger('undetected_chromedriver').setLevel(logging.WARNING)
100
-
101
- logging.info(f'FlareSolverr {utils.get_flaresolverr_version()}')
102
- logging.debug('Debug log enabled')
103
-
104
- # Get current OS for global variable
105
- utils.get_current_platform()
106
-
107
- # test browser installation
108
- flaresolverr_service.test_browser_installation()
109
-
110
- # start bootle plugins
111
- # plugin order is important
112
- app.install(logger_plugin)
113
- app.install(error_plugin)
114
- prometheus_plugin.setup()
115
- app.install(prometheus_plugin.prometheus_plugin)
116
-
117
- # start webserver
118
- # default server 'wsgiref' does not support concurrent requests
119
- # https://github.com/FlareSolverr/FlareSolverr/issues/680
120
- # https://github.com/Pylons/waitress/issues/31
121
- class WaitressServerPoll(ServerAdapter):
122
- def run(self, handler):
123
- from waitress import serve
124
- serve(handler, host=self.host, port=self.port, asyncore_use_poll=True)
125
- run(app, host=server_host, port=server_port, quiet=True, server=WaitressServerPoll)
 
1
+ import json
2
+ import logging
3
+ import os
4
+ import sys
5
+
6
+ import certifi
7
+ from bottle import run, response, Bottle, request, ServerAdapter
8
+
9
+ from bottle_plugins.error_plugin import error_plugin
10
+ from bottle_plugins.logger_plugin import logger_plugin
11
+ from bottle_plugins import prometheus_plugin
12
+ from dtos import V1RequestBase
13
+ import flaresolverr_service
14
+ import utils
15
+
16
+
17
+ class JSONErrorBottle(Bottle):
18
+ """
19
+ Handle 404 errors
20
+ """
21
+ def default_error_handler(self, res):
22
+ response.content_type = 'application/json'
23
+ return json.dumps(dict(error=res.body, status_code=res.status_code))
24
+
25
+
26
+ app = JSONErrorBottle()
27
+
28
+
29
+ @app.route('/')
30
+ def index():
31
+ """
32
+ Show welcome message
33
+ """
34
+ res = flaresolverr_service.index_endpoint()
35
+ return utils.object_to_dict(res)
36
+
37
+
38
+ @app.route('/health')
39
+ def health():
40
+ """
41
+ Healthcheck endpoint.
42
+ This endpoint is special because it doesn't print traces
43
+ """
44
+ res = flaresolverr_service.health_endpoint()
45
+ return utils.object_to_dict(res)
46
+
47
+
48
+ @app.post('/v1')
49
+ def controller_v1():
50
+ """
51
+ Controller v1
52
+ """
53
+ req = V1RequestBase(request.json)
54
+ res = flaresolverr_service.controller_v1_endpoint(req)
55
+ if res.__error_500__:
56
+ response.status = 500
57
+ return utils.object_to_dict(res)
58
+
59
+
60
+ if __name__ == "__main__":
61
+ # check python version
62
+ if sys.version_info < (3, 9):
63
+ raise Exception("The Python version is less than 3.9, a version equal to or higher is required.")
64
+
65
+ # fix for HEADLESS=false in Windows binary
66
+ # https://stackoverflow.com/a/27694505
67
+ if os.name == 'nt':
68
+ import multiprocessing
69
+ multiprocessing.freeze_support()
70
+
71
+ # fix ssl certificates for compiled binaries
72
+ # https://github.com/pyinstaller/pyinstaller/issues/7229
73
+ # https://stackoverflow.com/questions/55736855/how-to-change-the-cafile-argument-in-the-ssl-module-in-python3
74
+ os.environ["REQUESTS_CA_BUNDLE"] = certifi.where()
75
+ os.environ["SSL_CERT_FILE"] = certifi.where()
76
+
77
+ # validate configuration
78
+ log_level = os.environ.get('LOG_LEVEL', 'info').upper()
79
+ log_html = utils.get_config_log_html()
80
+ headless = utils.get_config_headless()
81
+ server_host = os.environ.get('HOST', '0.0.0.0')
82
+ server_port = int(os.environ.get('PORT', 8191))
83
+
84
+ # configure logger
85
+ logger_format = '%(asctime)s %(levelname)-8s %(message)s'
86
+ if log_level == 'DEBUG':
87
+ logger_format = '%(asctime)s %(levelname)-8s ReqId %(thread)s %(message)s'
88
+ logging.basicConfig(
89
+ format=logger_format,
90
+ level=log_level,
91
+ datefmt='%Y-%m-%d %H:%M:%S',
92
+ handlers=[
93
+ logging.StreamHandler(sys.stdout)
94
+ ]
95
+ )
96
+ # disable warning traces from urllib3
97
+ logging.getLogger('urllib3').setLevel(logging.ERROR)
98
+ logging.getLogger('selenium.webdriver.remote.remote_connection').setLevel(logging.WARNING)
99
+ logging.getLogger('undetected_chromedriver').setLevel(logging.WARNING)
100
+
101
+ logging.info(f'FlareSolverr {utils.get_flaresolverr_version()}')
102
+ logging.debug('Debug log enabled')
103
+
104
+ # Get current OS for global variable
105
+ utils.get_current_platform()
106
+
107
+ # test browser installation
108
+ flaresolverr_service.test_browser_installation()
109
+
110
+ # start bootle plugins
111
+ # plugin order is important
112
+ app.install(logger_plugin)
113
+ app.install(error_plugin)
114
+ prometheus_plugin.setup()
115
+ app.install(prometheus_plugin.prometheus_plugin)
116
+
117
+ # start webserver
118
+ # default server 'wsgiref' does not support concurrent requests
119
+ # https://github.com/FlareSolverr/FlareSolverr/issues/680
120
+ # https://github.com/Pylons/waitress/issues/31
121
+ class WaitressServerPoll(ServerAdapter):
122
+ def run(self, handler):
123
+ from waitress import serve
124
+ serve(handler, host=self.host, port=self.port, asyncore_use_poll=True)
125
+ run(app, host=server_host, port=server_port, quiet=True, server=WaitressServerPoll)
src/flaresolverr_service.py CHANGED
@@ -1,455 +1,455 @@
1
- import logging
2
- import platform
3
- import sys
4
- import time
5
- from datetime import timedelta
6
- from html import escape
7
- from urllib.parse import unquote, quote
8
-
9
- from func_timeout import FunctionTimedOut, func_timeout
10
- from selenium.common import TimeoutException
11
- from selenium.webdriver.chrome.webdriver import WebDriver
12
- from selenium.webdriver.common.by import By
13
- from selenium.webdriver.support.expected_conditions import (
14
- presence_of_element_located, staleness_of, title_is)
15
- from selenium.webdriver.common.action_chains import ActionChains
16
- from selenium.webdriver.support.wait import WebDriverWait
17
-
18
- import utils
19
- from dtos import (STATUS_ERROR, STATUS_OK, ChallengeResolutionResultT,
20
- ChallengeResolutionT, HealthResponse, IndexResponse,
21
- V1RequestBase, V1ResponseBase)
22
- from sessions import SessionsStorage
23
-
24
- ACCESS_DENIED_TITLES = [
25
- # Cloudflare
26
- 'Access denied',
27
- # Cloudflare http://bitturk.net/ Firefox
28
- 'Attention Required! | Cloudflare'
29
- ]
30
- ACCESS_DENIED_SELECTORS = [
31
- # Cloudflare
32
- 'div.cf-error-title span.cf-code-label span',
33
- # Cloudflare http://bitturk.net/ Firefox
34
- '#cf-error-details div.cf-error-overview h1'
35
- ]
36
- CHALLENGE_TITLES = [
37
- # Cloudflare
38
- 'Just a moment...',
39
- # DDoS-GUARD
40
- 'DDoS-Guard'
41
- ]
42
- CHALLENGE_SELECTORS = [
43
- # Cloudflare
44
- '#cf-challenge-running', '.ray_id', '.attack-box', '#cf-please-wait', '#challenge-spinner', '#trk_jschal_js', '#turnstile-wrapper', '.lds-ring',
45
- # Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
46
- 'td.info #js_info',
47
- # Fairlane / pararius.com
48
- 'div.vc div.text-box h2'
49
- ]
50
- SHORT_TIMEOUT = 1
51
- SESSIONS_STORAGE = SessionsStorage()
52
-
53
-
54
- def test_browser_installation():
55
- logging.info("Testing web browser installation...")
56
- logging.info("Platform: " + platform.platform())
57
-
58
- chrome_exe_path = utils.get_chrome_exe_path()
59
- if chrome_exe_path is None:
60
- logging.error("Chrome / Chromium web browser not installed!")
61
- sys.exit(1)
62
- else:
63
- logging.info("Chrome / Chromium path: " + chrome_exe_path)
64
-
65
- chrome_major_version = utils.get_chrome_major_version()
66
- if chrome_major_version == '':
67
- logging.error("Chrome / Chromium version not detected!")
68
- sys.exit(1)
69
- else:
70
- logging.info("Chrome / Chromium major version: " + chrome_major_version)
71
-
72
- logging.info("Launching web browser...")
73
- user_agent = utils.get_user_agent()
74
- logging.info("FlareSolverr User-Agent: " + user_agent)
75
- logging.info("Test successful!")
76
-
77
-
78
- def index_endpoint() -> IndexResponse:
79
- res = IndexResponse({})
80
- res.msg = "FlareSolverr is ready!"
81
- res.version = utils.get_flaresolverr_version()
82
- res.userAgent = utils.get_user_agent()
83
- return res
84
-
85
-
86
- def health_endpoint() -> HealthResponse:
87
- res = HealthResponse({})
88
- res.status = STATUS_OK
89
- return res
90
-
91
-
92
- def controller_v1_endpoint(req: V1RequestBase) -> V1ResponseBase:
93
- start_ts = int(time.time() * 1000)
94
- logging.info(f"Incoming request => POST /v1 body: {utils.object_to_dict(req)}")
95
- res: V1ResponseBase
96
- try:
97
- res = _controller_v1_handler(req)
98
- except Exception as e:
99
- res = V1ResponseBase({})
100
- res.__error_500__ = True
101
- res.status = STATUS_ERROR
102
- res.message = "Error: " + str(e)
103
- logging.error(res.message)
104
-
105
- res.startTimestamp = start_ts
106
- res.endTimestamp = int(time.time() * 1000)
107
- res.version = utils.get_flaresolverr_version()
108
- logging.debug(f"Response => POST /v1 body: {utils.object_to_dict(res)}")
109
- logging.info(f"Response in {(res.endTimestamp - res.startTimestamp) / 1000} s")
110
- return res
111
-
112
-
113
- def _controller_v1_handler(req: V1RequestBase) -> V1ResponseBase:
114
- # do some validations
115
- if req.cmd is None:
116
- raise Exception("Request parameter 'cmd' is mandatory.")
117
- if req.headers is not None:
118
- logging.warning("Request parameter 'headers' was removed in FlareSolverr v2.")
119
- if req.userAgent is not None:
120
- logging.warning("Request parameter 'userAgent' was removed in FlareSolverr v2.")
121
-
122
- # set default values
123
- if req.maxTimeout is None or int(req.maxTimeout) < 1:
124
- req.maxTimeout = 60000
125
-
126
- # execute the command
127
- res: V1ResponseBase
128
- if req.cmd == 'sessions.create':
129
- res = _cmd_sessions_create(req)
130
- elif req.cmd == 'sessions.list':
131
- res = _cmd_sessions_list(req)
132
- elif req.cmd == 'sessions.destroy':
133
- res = _cmd_sessions_destroy(req)
134
- elif req.cmd == 'request.get':
135
- res = _cmd_request_get(req)
136
- elif req.cmd == 'request.post':
137
- res = _cmd_request_post(req)
138
- else:
139
- raise Exception(f"Request parameter 'cmd' = '{req.cmd}' is invalid.")
140
-
141
- return res
142
-
143
-
144
- def _cmd_request_get(req: V1RequestBase) -> V1ResponseBase:
145
- # do some validations
146
- if req.url is None:
147
- raise Exception("Request parameter 'url' is mandatory in 'request.get' command.")
148
- if req.postData is not None:
149
- raise Exception("Cannot use 'postBody' when sending a GET request.")
150
- if req.returnRawHtml is not None:
151
- logging.warning("Request parameter 'returnRawHtml' was removed in FlareSolverr v2.")
152
- if req.download is not None:
153
- logging.warning("Request parameter 'download' was removed in FlareSolverr v2.")
154
-
155
- challenge_res = _resolve_challenge(req, 'GET')
156
- res = V1ResponseBase({})
157
- res.status = challenge_res.status
158
- res.message = challenge_res.message
159
- res.solution = challenge_res.result
160
- return res
161
-
162
-
163
- def _cmd_request_post(req: V1RequestBase) -> V1ResponseBase:
164
- # do some validations
165
- if req.postData is None:
166
- raise Exception("Request parameter 'postData' is mandatory in 'request.post' command.")
167
- if req.returnRawHtml is not None:
168
- logging.warning("Request parameter 'returnRawHtml' was removed in FlareSolverr v2.")
169
- if req.download is not None:
170
- logging.warning("Request parameter 'download' was removed in FlareSolverr v2.")
171
-
172
- challenge_res = _resolve_challenge(req, 'POST')
173
- res = V1ResponseBase({})
174
- res.status = challenge_res.status
175
- res.message = challenge_res.message
176
- res.solution = challenge_res.result
177
- return res
178
-
179
-
180
- def _cmd_sessions_create(req: V1RequestBase) -> V1ResponseBase:
181
- logging.debug("Creating new session...")
182
-
183
- session, fresh = SESSIONS_STORAGE.create(session_id=req.session, proxy=req.proxy)
184
- session_id = session.session_id
185
-
186
- if not fresh:
187
- return V1ResponseBase({
188
- "status": STATUS_OK,
189
- "message": "Session already exists.",
190
- "session": session_id
191
- })
192
-
193
- return V1ResponseBase({
194
- "status": STATUS_OK,
195
- "message": "Session created successfully.",
196
- "session": session_id
197
- })
198
-
199
-
200
- def _cmd_sessions_list(req: V1RequestBase) -> V1ResponseBase:
201
- session_ids = SESSIONS_STORAGE.session_ids()
202
-
203
- return V1ResponseBase({
204
- "status": STATUS_OK,
205
- "message": "",
206
- "sessions": session_ids
207
- })
208
-
209
-
210
- def _cmd_sessions_destroy(req: V1RequestBase) -> V1ResponseBase:
211
- session_id = req.session
212
- existed = SESSIONS_STORAGE.destroy(session_id)
213
-
214
- if not existed:
215
- raise Exception("The session doesn't exist.")
216
-
217
- return V1ResponseBase({
218
- "status": STATUS_OK,
219
- "message": "The session has been removed."
220
- })
221
-
222
-
223
- def _resolve_challenge(req: V1RequestBase, method: str) -> ChallengeResolutionT:
224
- timeout = int(req.maxTimeout) / 1000
225
- driver = None
226
- try:
227
- if req.session:
228
- session_id = req.session
229
- ttl = timedelta(minutes=req.session_ttl_minutes) if req.session_ttl_minutes else None
230
- session, fresh = SESSIONS_STORAGE.get(session_id, ttl)
231
-
232
- if fresh:
233
- logging.debug(f"new session created to perform the request (session_id={session_id})")
234
- else:
235
- logging.debug(f"existing session is used to perform the request (session_id={session_id}, "
236
- f"lifetime={str(session.lifetime())}, ttl={str(ttl)})")
237
-
238
- driver = session.driver
239
- else:
240
- driver = utils.get_webdriver(req.proxy)
241
- logging.debug('New instance of webdriver has been created to perform the request')
242
- return func_timeout(timeout, _evil_logic, (req, driver, method))
243
- except FunctionTimedOut:
244
- raise Exception(f'Error solving the challenge. Timeout after {timeout} seconds.')
245
- except Exception as e:
246
- raise Exception('Error solving the challenge. ' + str(e).replace('\n', '\\n'))
247
- finally:
248
- if not req.session and driver is not None:
249
- if utils.PLATFORM_VERSION == "nt":
250
- driver.close()
251
- driver.quit()
252
- logging.debug('A used instance of webdriver has been destroyed')
253
-
254
-
255
- def click_verify(driver: WebDriver):
256
- try:
257
- logging.debug("Try to find the Cloudflare verify checkbox...")
258
- iframe = driver.find_element(By.XPATH, "//iframe[starts-with(@id, 'cf-chl-widget-')]")
259
- driver.switch_to.frame(iframe)
260
- checkbox = driver.find_element(
261
- by=By.XPATH,
262
- value='//*[@id="content"]/div/div/label/input',
263
- )
264
- if checkbox:
265
- actions = ActionChains(driver)
266
- actions.move_to_element_with_offset(checkbox, 5, 7)
267
- actions.click(checkbox)
268
- actions.perform()
269
- logging.debug("Cloudflare verify checkbox found and clicked!")
270
- except Exception:
271
- logging.debug("Cloudflare verify checkbox not found on the page.")
272
- finally:
273
- driver.switch_to.default_content()
274
-
275
- try:
276
- logging.debug("Try to find the Cloudflare 'Verify you are human' button...")
277
- button = driver.find_element(
278
- by=By.XPATH,
279
- value="//input[@type='button' and @value='Verify you are human']",
280
- )
281
- if button:
282
- actions = ActionChains(driver)
283
- actions.move_to_element_with_offset(button, 5, 7)
284
- actions.click(button)
285
- actions.perform()
286
- logging.debug("The Cloudflare 'Verify you are human' button found and clicked!")
287
- except Exception:
288
- logging.debug("The Cloudflare 'Verify you are human' button not found on the page.")
289
-
290
- time.sleep(2)
291
-
292
-
293
- def get_correct_window(driver: WebDriver) -> WebDriver:
294
- if len(driver.window_handles) > 1:
295
- for window_handle in driver.window_handles:
296
- driver.switch_to.window(window_handle)
297
- current_url = driver.current_url
298
- if not current_url.startswith("devtools://devtools"):
299
- return driver
300
- return driver
301
-
302
-
303
- def access_page(driver: WebDriver, url: str) -> None:
304
- driver.get(url)
305
- driver.start_session()
306
- driver.start_session() # required to bypass Cloudflare
307
-
308
-
309
- def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> ChallengeResolutionT:
310
- res = ChallengeResolutionT({})
311
- res.status = STATUS_OK
312
- res.message = ""
313
-
314
-
315
- # navigate to the page
316
- logging.debug(f'Navigating to... {req.url}')
317
- if method == 'POST':
318
- _post_request(req, driver)
319
- else:
320
- access_page(driver, req.url)
321
- driver = get_correct_window(driver)
322
-
323
- # set cookies if required
324
- if req.cookies is not None and len(req.cookies) > 0:
325
- logging.debug(f'Setting cookies...')
326
- for cookie in req.cookies:
327
- driver.delete_cookie(cookie['name'])
328
- driver.add_cookie(cookie)
329
- # reload the page
330
- if method == 'POST':
331
- _post_request(req, driver)
332
- else:
333
- access_page(driver, req.url)
334
- driver = get_correct_window(driver)
335
-
336
- # wait for the page
337
- if utils.get_config_log_html():
338
- logging.debug(f"Response HTML:\n{driver.page_source}")
339
- html_element = driver.find_element(By.TAG_NAME, "html")
340
- page_title = driver.title
341
-
342
- # find access denied titles
343
- for title in ACCESS_DENIED_TITLES:
344
- if title == page_title:
345
- raise Exception('Cloudflare has blocked this request. '
346
- 'Probably your IP is banned for this site, check in your web browser.')
347
- # find access denied selectors
348
- for selector in ACCESS_DENIED_SELECTORS:
349
- found_elements = driver.find_elements(By.CSS_SELECTOR, selector)
350
- if len(found_elements) > 0:
351
- raise Exception('Cloudflare has blocked this request. '
352
- 'Probably your IP is banned for this site, check in your web browser.')
353
-
354
- # find challenge by title
355
- challenge_found = False
356
- for title in CHALLENGE_TITLES:
357
- if title.lower() == page_title.lower():
358
- challenge_found = True
359
- logging.info("Challenge detected. Title found: " + page_title)
360
- break
361
- if not challenge_found:
362
- # find challenge by selectors
363
- for selector in CHALLENGE_SELECTORS:
364
- found_elements = driver.find_elements(By.CSS_SELECTOR, selector)
365
- if len(found_elements) > 0:
366
- challenge_found = True
367
- logging.info("Challenge detected. Selector found: " + selector)
368
- break
369
-
370
- attempt = 0
371
- if challenge_found:
372
- while True:
373
- try:
374
- attempt = attempt + 1
375
- # wait until the title changes
376
- for title in CHALLENGE_TITLES:
377
- logging.debug("Waiting for title (attempt " + str(attempt) + "): " + title)
378
- WebDriverWait(driver, SHORT_TIMEOUT).until_not(title_is(title))
379
-
380
- # then wait until all the selectors disappear
381
- for selector in CHALLENGE_SELECTORS:
382
- logging.debug("Waiting for selector (attempt " + str(attempt) + "): " + selector)
383
- WebDriverWait(driver, SHORT_TIMEOUT).until_not(
384
- presence_of_element_located((By.CSS_SELECTOR, selector)))
385
-
386
- # all elements not found
387
- break
388
-
389
- except TimeoutException:
390
- logging.debug("Timeout waiting for selector")
391
-
392
- click_verify(driver)
393
-
394
- # update the html (cloudflare reloads the page every 5 s)
395
- html_element = driver.find_element(By.TAG_NAME, "html")
396
-
397
- # waits until cloudflare redirection ends
398
- logging.debug("Waiting for redirect")
399
- # noinspection PyBroadException
400
- try:
401
- WebDriverWait(driver, SHORT_TIMEOUT).until(staleness_of(html_element))
402
- except Exception:
403
- logging.debug("Timeout waiting for redirect")
404
-
405
- logging.info("Challenge solved!")
406
- res.message = "Challenge solved!"
407
- else:
408
- logging.info("Challenge not detected!")
409
- res.message = "Challenge not detected!"
410
-
411
- challenge_res = ChallengeResolutionResultT({})
412
- challenge_res.url = driver.current_url
413
- challenge_res.status = 200 # todo: fix, selenium not provides this info
414
- challenge_res.cookies = driver.get_cookies()
415
- challenge_res.userAgent = utils.get_user_agent(driver)
416
-
417
- if not req.returnOnlyCookies:
418
- challenge_res.headers = {} # todo: fix, selenium not provides this info
419
- challenge_res.response = driver.page_source
420
-
421
- res.result = challenge_res
422
- return res
423
-
424
-
425
- def _post_request(req: V1RequestBase, driver: WebDriver):
426
- post_form = f'<form id="hackForm" action="{req.url}" method="POST">'
427
- query_string = req.postData if req.postData[0] != '?' else req.postData[1:]
428
- pairs = query_string.split('&')
429
- for pair in pairs:
430
- parts = pair.split('=')
431
- # noinspection PyBroadException
432
- try:
433
- name = unquote(parts[0])
434
- except Exception:
435
- name = parts[0]
436
- if name == 'submit':
437
- continue
438
- # noinspection PyBroadException
439
- try:
440
- value = unquote(parts[1])
441
- except Exception:
442
- value = parts[1]
443
- post_form += f'<input type="text" name="{escape(quote(name))}" value="{escape(quote(value))}"><br>'
444
- post_form += '</form>'
445
- html_content = f"""
446
- <!DOCTYPE html>
447
- <html>
448
- <body>
449
- {post_form}
450
- <script>document.getElementById('hackForm').submit();</script>
451
- </body>
452
- </html>"""
453
- driver.get("data:text/html;charset=utf-8,{html_content}".format(html_content=html_content))
454
- driver.start_session()
455
- driver.start_session() # required to bypass Cloudflare
 
1
+ import logging
2
+ import platform
3
+ import sys
4
+ import time
5
+ from datetime import timedelta
6
+ from html import escape
7
+ from urllib.parse import unquote, quote
8
+
9
+ from func_timeout import FunctionTimedOut, func_timeout
10
+ from selenium.common import TimeoutException
11
+ from selenium.webdriver.chrome.webdriver import WebDriver
12
+ from selenium.webdriver.common.by import By
13
+ from selenium.webdriver.support.expected_conditions import (
14
+ presence_of_element_located, staleness_of, title_is)
15
+ from selenium.webdriver.common.action_chains import ActionChains
16
+ from selenium.webdriver.support.wait import WebDriverWait
17
+
18
+ import utils
19
+ from dtos import (STATUS_ERROR, STATUS_OK, ChallengeResolutionResultT,
20
+ ChallengeResolutionT, HealthResponse, IndexResponse,
21
+ V1RequestBase, V1ResponseBase)
22
+ from sessions import SessionsStorage
23
+
24
+ ACCESS_DENIED_TITLES = [
25
+ # Cloudflare
26
+ 'Access denied',
27
+ # Cloudflare http://bitturk.net/ Firefox
28
+ 'Attention Required! | Cloudflare'
29
+ ]
30
+ ACCESS_DENIED_SELECTORS = [
31
+ # Cloudflare
32
+ 'div.cf-error-title span.cf-code-label span',
33
+ # Cloudflare http://bitturk.net/ Firefox
34
+ '#cf-error-details div.cf-error-overview h1'
35
+ ]
36
+ CHALLENGE_TITLES = [
37
+ # Cloudflare
38
+ 'Just a moment...',
39
+ # DDoS-GUARD
40
+ 'DDoS-Guard'
41
+ ]
42
+ CHALLENGE_SELECTORS = [
43
+ # Cloudflare
44
+ '#cf-challenge-running', '.ray_id', '.attack-box', '#cf-please-wait', '#challenge-spinner', '#trk_jschal_js', '#turnstile-wrapper', '.lds-ring',
45
+ # Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
46
+ 'td.info #js_info',
47
+ # Fairlane / pararius.com
48
+ 'div.vc div.text-box h2'
49
+ ]
50
+ SHORT_TIMEOUT = 1
51
+ SESSIONS_STORAGE = SessionsStorage()
52
+
53
+
54
+ def test_browser_installation():
55
+ logging.info("Testing web browser installation...")
56
+ logging.info("Platform: " + platform.platform())
57
+
58
+ chrome_exe_path = utils.get_chrome_exe_path()
59
+ if chrome_exe_path is None:
60
+ logging.error("Chrome / Chromium web browser not installed!")
61
+ sys.exit(1)
62
+ else:
63
+ logging.info("Chrome / Chromium path: " + chrome_exe_path)
64
+
65
+ chrome_major_version = utils.get_chrome_major_version()
66
+ if chrome_major_version == '':
67
+ logging.error("Chrome / Chromium version not detected!")
68
+ sys.exit(1)
69
+ else:
70
+ logging.info("Chrome / Chromium major version: " + chrome_major_version)
71
+
72
+ logging.info("Launching web browser...")
73
+ user_agent = utils.get_user_agent()
74
+ logging.info("FlareSolverr User-Agent: " + user_agent)
75
+ logging.info("Test successful!")
76
+
77
+
78
+ def index_endpoint() -> IndexResponse:
79
+ res = IndexResponse({})
80
+ res.msg = "FlareSolverr is ready!"
81
+ res.version = utils.get_flaresolverr_version()
82
+ res.userAgent = utils.get_user_agent()
83
+ return res
84
+
85
+
86
+ def health_endpoint() -> HealthResponse:
87
+ res = HealthResponse({})
88
+ res.status = STATUS_OK
89
+ return res
90
+
91
+
92
+ def controller_v1_endpoint(req: V1RequestBase) -> V1ResponseBase:
93
+ start_ts = int(time.time() * 1000)
94
+ logging.info(f"Incoming request => POST /v1 body: {utils.object_to_dict(req)}")
95
+ res: V1ResponseBase
96
+ try:
97
+ res = _controller_v1_handler(req)
98
+ except Exception as e:
99
+ res = V1ResponseBase({})
100
+ res.__error_500__ = True
101
+ res.status = STATUS_ERROR
102
+ res.message = "Error: " + str(e)
103
+ logging.error(res.message)
104
+
105
+ res.startTimestamp = start_ts
106
+ res.endTimestamp = int(time.time() * 1000)
107
+ res.version = utils.get_flaresolverr_version()
108
+ logging.debug(f"Response => POST /v1 body: {utils.object_to_dict(res)}")
109
+ logging.info(f"Response in {(res.endTimestamp - res.startTimestamp) / 1000} s")
110
+ return res
111
+
112
+
113
+ def _controller_v1_handler(req: V1RequestBase) -> V1ResponseBase:
114
+ # do some validations
115
+ if req.cmd is None:
116
+ raise Exception("Request parameter 'cmd' is mandatory.")
117
+ if req.headers is not None:
118
+ logging.warning("Request parameter 'headers' was removed in FlareSolverr v2.")
119
+ if req.userAgent is not None:
120
+ logging.warning("Request parameter 'userAgent' was removed in FlareSolverr v2.")
121
+
122
+ # set default values
123
+ if req.maxTimeout is None or int(req.maxTimeout) < 1:
124
+ req.maxTimeout = 60000
125
+
126
+ # execute the command
127
+ res: V1ResponseBase
128
+ if req.cmd == 'sessions.create':
129
+ res = _cmd_sessions_create(req)
130
+ elif req.cmd == 'sessions.list':
131
+ res = _cmd_sessions_list(req)
132
+ elif req.cmd == 'sessions.destroy':
133
+ res = _cmd_sessions_destroy(req)
134
+ elif req.cmd == 'request.get':
135
+ res = _cmd_request_get(req)
136
+ elif req.cmd == 'request.post':
137
+ res = _cmd_request_post(req)
138
+ else:
139
+ raise Exception(f"Request parameter 'cmd' = '{req.cmd}' is invalid.")
140
+
141
+ return res
142
+
143
+
144
+ def _cmd_request_get(req: V1RequestBase) -> V1ResponseBase:
145
+ # do some validations
146
+ if req.url is None:
147
+ raise Exception("Request parameter 'url' is mandatory in 'request.get' command.")
148
+ if req.postData is not None:
149
+ raise Exception("Cannot use 'postBody' when sending a GET request.")
150
+ if req.returnRawHtml is not None:
151
+ logging.warning("Request parameter 'returnRawHtml' was removed in FlareSolverr v2.")
152
+ if req.download is not None:
153
+ logging.warning("Request parameter 'download' was removed in FlareSolverr v2.")
154
+
155
+ challenge_res = _resolve_challenge(req, 'GET')
156
+ res = V1ResponseBase({})
157
+ res.status = challenge_res.status
158
+ res.message = challenge_res.message
159
+ res.solution = challenge_res.result
160
+ return res
161
+
162
+
163
+ def _cmd_request_post(req: V1RequestBase) -> V1ResponseBase:
164
+ # do some validations
165
+ if req.postData is None:
166
+ raise Exception("Request parameter 'postData' is mandatory in 'request.post' command.")
167
+ if req.returnRawHtml is not None:
168
+ logging.warning("Request parameter 'returnRawHtml' was removed in FlareSolverr v2.")
169
+ if req.download is not None:
170
+ logging.warning("Request parameter 'download' was removed in FlareSolverr v2.")
171
+
172
+ challenge_res = _resolve_challenge(req, 'POST')
173
+ res = V1ResponseBase({})
174
+ res.status = challenge_res.status
175
+ res.message = challenge_res.message
176
+ res.solution = challenge_res.result
177
+ return res
178
+
179
+
180
+ def _cmd_sessions_create(req: V1RequestBase) -> V1ResponseBase:
181
+ logging.debug("Creating new session...")
182
+
183
+ session, fresh = SESSIONS_STORAGE.create(session_id=req.session, proxy=req.proxy)
184
+ session_id = session.session_id
185
+
186
+ if not fresh:
187
+ return V1ResponseBase({
188
+ "status": STATUS_OK,
189
+ "message": "Session already exists.",
190
+ "session": session_id
191
+ })
192
+
193
+ return V1ResponseBase({
194
+ "status": STATUS_OK,
195
+ "message": "Session created successfully.",
196
+ "session": session_id
197
+ })
198
+
199
+
200
+ def _cmd_sessions_list(req: V1RequestBase) -> V1ResponseBase:
201
+ session_ids = SESSIONS_STORAGE.session_ids()
202
+
203
+ return V1ResponseBase({
204
+ "status": STATUS_OK,
205
+ "message": "",
206
+ "sessions": session_ids
207
+ })
208
+
209
+
210
+ def _cmd_sessions_destroy(req: V1RequestBase) -> V1ResponseBase:
211
+ session_id = req.session
212
+ existed = SESSIONS_STORAGE.destroy(session_id)
213
+
214
+ if not existed:
215
+ raise Exception("The session doesn't exist.")
216
+
217
+ return V1ResponseBase({
218
+ "status": STATUS_OK,
219
+ "message": "The session has been removed."
220
+ })
221
+
222
+
223
+ def _resolve_challenge(req: V1RequestBase, method: str) -> ChallengeResolutionT:
224
+ timeout = int(req.maxTimeout) / 1000
225
+ driver = None
226
+ try:
227
+ if req.session:
228
+ session_id = req.session
229
+ ttl = timedelta(minutes=req.session_ttl_minutes) if req.session_ttl_minutes else None
230
+ session, fresh = SESSIONS_STORAGE.get(session_id, ttl)
231
+
232
+ if fresh:
233
+ logging.debug(f"new session created to perform the request (session_id={session_id})")
234
+ else:
235
+ logging.debug(f"existing session is used to perform the request (session_id={session_id}, "
236
+ f"lifetime={str(session.lifetime())}, ttl={str(ttl)})")
237
+
238
+ driver = session.driver
239
+ else:
240
+ driver = utils.get_webdriver(req.proxy)
241
+ logging.debug('New instance of webdriver has been created to perform the request')
242
+ return func_timeout(timeout, _evil_logic, (req, driver, method))
243
+ except FunctionTimedOut:
244
+ raise Exception(f'Error solving the challenge. Timeout after {timeout} seconds.')
245
+ except Exception as e:
246
+ raise Exception('Error solving the challenge. ' + str(e).replace('\n', '\\n'))
247
+ finally:
248
+ if not req.session and driver is not None:
249
+ if utils.PLATFORM_VERSION == "nt":
250
+ driver.close()
251
+ driver.quit()
252
+ logging.debug('A used instance of webdriver has been destroyed')
253
+
254
+
255
+ def click_verify(driver: WebDriver):
256
+ try:
257
+ logging.debug("Try to find the Cloudflare verify checkbox...")
258
+ iframe = driver.find_element(By.XPATH, "//iframe[starts-with(@id, 'cf-chl-widget-')]")
259
+ driver.switch_to.frame(iframe)
260
+ checkbox = driver.find_element(
261
+ by=By.XPATH,
262
+ value='//*[@id="content"]/div/div/label/input',
263
+ )
264
+ if checkbox:
265
+ actions = ActionChains(driver)
266
+ actions.move_to_element_with_offset(checkbox, 5, 7)
267
+ actions.click(checkbox)
268
+ actions.perform()
269
+ logging.debug("Cloudflare verify checkbox found and clicked!")
270
+ except Exception:
271
+ logging.debug("Cloudflare verify checkbox not found on the page.")
272
+ finally:
273
+ driver.switch_to.default_content()
274
+
275
+ try:
276
+ logging.debug("Try to find the Cloudflare 'Verify you are human' button...")
277
+ button = driver.find_element(
278
+ by=By.XPATH,
279
+ value="//input[@type='button' and @value='Verify you are human']",
280
+ )
281
+ if button:
282
+ actions = ActionChains(driver)
283
+ actions.move_to_element_with_offset(button, 5, 7)
284
+ actions.click(button)
285
+ actions.perform()
286
+ logging.debug("The Cloudflare 'Verify you are human' button found and clicked!")
287
+ except Exception:
288
+ logging.debug("The Cloudflare 'Verify you are human' button not found on the page.")
289
+
290
+ time.sleep(2)
291
+
292
+
293
+ def get_correct_window(driver: WebDriver) -> WebDriver:
294
+ if len(driver.window_handles) > 1:
295
+ for window_handle in driver.window_handles:
296
+ driver.switch_to.window(window_handle)
297
+ current_url = driver.current_url
298
+ if not current_url.startswith("devtools://devtools"):
299
+ return driver
300
+ return driver
301
+
302
+
303
+ def access_page(driver: WebDriver, url: str) -> None:
304
+ driver.get(url)
305
+ driver.start_session()
306
+ driver.start_session() # required to bypass Cloudflare
307
+
308
+
309
+ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> ChallengeResolutionT:
310
+ res = ChallengeResolutionT({})
311
+ res.status = STATUS_OK
312
+ res.message = ""
313
+
314
+
315
+ # navigate to the page
316
+ logging.debug(f'Navigating to... {req.url}')
317
+ if method == 'POST':
318
+ _post_request(req, driver)
319
+ else:
320
+ access_page(driver, req.url)
321
+ driver = get_correct_window(driver)
322
+
323
+ # set cookies if required
324
+ if req.cookies is not None and len(req.cookies) > 0:
325
+ logging.debug(f'Setting cookies...')
326
+ for cookie in req.cookies:
327
+ driver.delete_cookie(cookie['name'])
328
+ driver.add_cookie(cookie)
329
+ # reload the page
330
+ if method == 'POST':
331
+ _post_request(req, driver)
332
+ else:
333
+ access_page(driver, req.url)
334
+ driver = get_correct_window(driver)
335
+
336
+ # wait for the page
337
+ if utils.get_config_log_html():
338
+ logging.debug(f"Response HTML:\n{driver.page_source}")
339
+ html_element = driver.find_element(By.TAG_NAME, "html")
340
+ page_title = driver.title
341
+
342
+ # find access denied titles
343
+ for title in ACCESS_DENIED_TITLES:
344
+ if title == page_title:
345
+ raise Exception('Cloudflare has blocked this request. '
346
+ 'Probably your IP is banned for this site, check in your web browser.')
347
+ # find access denied selectors
348
+ for selector in ACCESS_DENIED_SELECTORS:
349
+ found_elements = driver.find_elements(By.CSS_SELECTOR, selector)
350
+ if len(found_elements) > 0:
351
+ raise Exception('Cloudflare has blocked this request. '
352
+ 'Probably your IP is banned for this site, check in your web browser.')
353
+
354
+ # find challenge by title
355
+ challenge_found = False
356
+ for title in CHALLENGE_TITLES:
357
+ if title.lower() == page_title.lower():
358
+ challenge_found = True
359
+ logging.info("Challenge detected. Title found: " + page_title)
360
+ break
361
+ if not challenge_found:
362
+ # find challenge by selectors
363
+ for selector in CHALLENGE_SELECTORS:
364
+ found_elements = driver.find_elements(By.CSS_SELECTOR, selector)
365
+ if len(found_elements) > 0:
366
+ challenge_found = True
367
+ logging.info("Challenge detected. Selector found: " + selector)
368
+ break
369
+
370
+ attempt = 0
371
+ if challenge_found:
372
+ while True:
373
+ try:
374
+ attempt = attempt + 1
375
+ # wait until the title changes
376
+ for title in CHALLENGE_TITLES:
377
+ logging.debug("Waiting for title (attempt " + str(attempt) + "): " + title)
378
+ WebDriverWait(driver, SHORT_TIMEOUT).until_not(title_is(title))
379
+
380
+ # then wait until all the selectors disappear
381
+ for selector in CHALLENGE_SELECTORS:
382
+ logging.debug("Waiting for selector (attempt " + str(attempt) + "): " + selector)
383
+ WebDriverWait(driver, SHORT_TIMEOUT).until_not(
384
+ presence_of_element_located((By.CSS_SELECTOR, selector)))
385
+
386
+ # all elements not found
387
+ break
388
+
389
+ except TimeoutException:
390
+ logging.debug("Timeout waiting for selector")
391
+
392
+ click_verify(driver)
393
+
394
+ # update the html (cloudflare reloads the page every 5 s)
395
+ html_element = driver.find_element(By.TAG_NAME, "html")
396
+
397
+ # waits until cloudflare redirection ends
398
+ logging.debug("Waiting for redirect")
399
+ # noinspection PyBroadException
400
+ try:
401
+ WebDriverWait(driver, SHORT_TIMEOUT).until(staleness_of(html_element))
402
+ except Exception:
403
+ logging.debug("Timeout waiting for redirect")
404
+
405
+ logging.info("Challenge solved!")
406
+ res.message = "Challenge solved!"
407
+ else:
408
+ logging.info("Challenge not detected!")
409
+ res.message = "Challenge not detected!"
410
+
411
+ challenge_res = ChallengeResolutionResultT({})
412
+ challenge_res.url = driver.current_url
413
+ challenge_res.status = 200 # todo: fix, selenium not provides this info
414
+ challenge_res.cookies = driver.get_cookies()
415
+ challenge_res.userAgent = utils.get_user_agent(driver)
416
+
417
+ if not req.returnOnlyCookies:
418
+ challenge_res.headers = {} # todo: fix, selenium not provides this info
419
+ challenge_res.response = driver.page_source
420
+
421
+ res.result = challenge_res
422
+ return res
423
+
424
+
425
+ def _post_request(req: V1RequestBase, driver: WebDriver):
426
+ post_form = f'<form id="hackForm" action="{req.url}" method="POST">'
427
+ query_string = req.postData if req.postData[0] != '?' else req.postData[1:]
428
+ pairs = query_string.split('&')
429
+ for pair in pairs:
430
+ parts = pair.split('=')
431
+ # noinspection PyBroadException
432
+ try:
433
+ name = unquote(parts[0])
434
+ except Exception:
435
+ name = parts[0]
436
+ if name == 'submit':
437
+ continue
438
+ # noinspection PyBroadException
439
+ try:
440
+ value = unquote(parts[1])
441
+ except Exception:
442
+ value = parts[1]
443
+ post_form += f'<input type="text" name="{escape(quote(name))}" value="{escape(quote(value))}"><br>'
444
+ post_form += '</form>'
445
+ html_content = f"""
446
+ <!DOCTYPE html>
447
+ <html>
448
+ <body>
449
+ {post_form}
450
+ <script>document.getElementById('hackForm').submit();</script>
451
+ </body>
452
+ </html>"""
453
+ driver.get("data:text/html;charset=utf-8,{html_content}".format(html_content=html_content))
454
+ driver.start_session()
455
+ driver.start_session() # required to bypass Cloudflare
src/metrics.py CHANGED
@@ -1,32 +1,32 @@
1
- import logging
2
-
3
- from prometheus_client import Counter, Histogram, start_http_server
4
- import time
5
-
6
- REQUEST_COUNTER = Counter(
7
- name='flaresolverr_request',
8
- documentation='Total requests with result',
9
- labelnames=['domain', 'result']
10
- )
11
- REQUEST_DURATION = Histogram(
12
- name='flaresolverr_request_duration',
13
- documentation='Request duration in seconds',
14
- labelnames=['domain'],
15
- buckets=[0, 10, 25, 50]
16
- )
17
-
18
-
19
- def serve(port):
20
- start_http_server(port=port)
21
- while True:
22
- time.sleep(600)
23
-
24
-
25
- def start_metrics_http_server(prometheus_port: int):
26
- logging.info(f"Serving Prometheus exporter on http://0.0.0.0:{prometheus_port}/metrics")
27
- from threading import Thread
28
- Thread(
29
- target=serve,
30
- kwargs=dict(port=prometheus_port),
31
- daemon=True,
32
- ).start()
 
1
+ import logging
2
+
3
+ from prometheus_client import Counter, Histogram, start_http_server
4
+ import time
5
+
6
+ REQUEST_COUNTER = Counter(
7
+ name='flaresolverr_request',
8
+ documentation='Total requests with result',
9
+ labelnames=['domain', 'result']
10
+ )
11
+ REQUEST_DURATION = Histogram(
12
+ name='flaresolverr_request_duration',
13
+ documentation='Request duration in seconds',
14
+ labelnames=['domain'],
15
+ buckets=[0, 10, 25, 50]
16
+ )
17
+
18
+
19
+ def serve(port):
20
+ start_http_server(port=port)
21
+ while True:
22
+ time.sleep(600)
23
+
24
+
25
+ def start_metrics_http_server(prometheus_port: int):
26
+ logging.info(f"Serving Prometheus exporter on http://0.0.0.0:{prometheus_port}/metrics")
27
+ from threading import Thread
28
+ Thread(
29
+ target=serve,
30
+ kwargs=dict(port=prometheus_port),
31
+ daemon=True,
32
+ ).start()
src/sessions.py CHANGED
@@ -1,84 +1,84 @@
1
- import logging
2
- from dataclasses import dataclass
3
- from datetime import datetime, timedelta
4
- from typing import Optional, Tuple
5
- from uuid import uuid1
6
-
7
- from selenium.webdriver.chrome.webdriver import WebDriver
8
-
9
- import utils
10
-
11
-
12
- @dataclass
13
- class Session:
14
- session_id: str
15
- driver: WebDriver
16
- created_at: datetime
17
-
18
- def lifetime(self) -> timedelta:
19
- return datetime.now() - self.created_at
20
-
21
-
22
- class SessionsStorage:
23
- """SessionsStorage creates, stores and process all the sessions"""
24
-
25
- def __init__(self):
26
- self.sessions = {}
27
-
28
- def create(self, session_id: Optional[str] = None, proxy: Optional[dict] = None,
29
- force_new: Optional[bool] = False) -> Tuple[Session, bool]:
30
- """create creates new instance of WebDriver if necessary,
31
- assign defined (or newly generated) session_id to the instance
32
- and returns the session object. If a new session has been created
33
- second argument is set to True.
34
-
35
- Note: The function is idempotent, so in case if session_id
36
- already exists in the storage a new instance of WebDriver won't be created
37
- and existing session will be returned. Second argument defines if
38
- new session has been created (True) or an existing one was used (False).
39
- """
40
- session_id = session_id or str(uuid1())
41
-
42
- if force_new:
43
- self.destroy(session_id)
44
-
45
- if self.exists(session_id):
46
- return self.sessions[session_id], False
47
-
48
- driver = utils.get_webdriver(proxy)
49
- created_at = datetime.now()
50
- session = Session(session_id, driver, created_at)
51
-
52
- self.sessions[session_id] = session
53
-
54
- return session, True
55
-
56
- def exists(self, session_id: str) -> bool:
57
- return session_id in self.sessions
58
-
59
- def destroy(self, session_id: str) -> bool:
60
- """destroy closes the driver instance and removes session from the storage.
61
- The function is noop if session_id doesn't exist.
62
- The function returns True if session was found and destroyed,
63
- and False if session_id wasn't found.
64
- """
65
- if not self.exists(session_id):
66
- return False
67
-
68
- session = self.sessions.pop(session_id)
69
- if utils.PLATFORM_VERSION == "nt":
70
- session.driver.close()
71
- session.driver.quit()
72
- return True
73
-
74
- def get(self, session_id: str, ttl: Optional[timedelta] = None) -> Tuple[Session, bool]:
75
- session, fresh = self.create(session_id)
76
-
77
- if ttl is not None and not fresh and session.lifetime() > ttl:
78
- logging.debug(f'session\'s lifetime has expired, so the session is recreated (session_id={session_id})')
79
- session, fresh = self.create(session_id, force_new=True)
80
-
81
- return session, fresh
82
-
83
- def session_ids(self) -> list[str]:
84
- return list(self.sessions.keys())
 
1
+ import logging
2
+ from dataclasses import dataclass
3
+ from datetime import datetime, timedelta
4
+ from typing import Optional, Tuple
5
+ from uuid import uuid1
6
+
7
+ from selenium.webdriver.chrome.webdriver import WebDriver
8
+
9
+ import utils
10
+
11
+
12
+ @dataclass
13
+ class Session:
14
+ session_id: str
15
+ driver: WebDriver
16
+ created_at: datetime
17
+
18
+ def lifetime(self) -> timedelta:
19
+ return datetime.now() - self.created_at
20
+
21
+
22
+ class SessionsStorage:
23
+ """SessionsStorage creates, stores and process all the sessions"""
24
+
25
+ def __init__(self):
26
+ self.sessions = {}
27
+
28
+ def create(self, session_id: Optional[str] = None, proxy: Optional[dict] = None,
29
+ force_new: Optional[bool] = False) -> Tuple[Session, bool]:
30
+ """create creates new instance of WebDriver if necessary,
31
+ assign defined (or newly generated) session_id to the instance
32
+ and returns the session object. If a new session has been created
33
+ second argument is set to True.
34
+
35
+ Note: The function is idempotent, so in case if session_id
36
+ already exists in the storage a new instance of WebDriver won't be created
37
+ and existing session will be returned. Second argument defines if
38
+ new session has been created (True) or an existing one was used (False).
39
+ """
40
+ session_id = session_id or str(uuid1())
41
+
42
+ if force_new:
43
+ self.destroy(session_id)
44
+
45
+ if self.exists(session_id):
46
+ return self.sessions[session_id], False
47
+
48
+ driver = utils.get_webdriver(proxy)
49
+ created_at = datetime.now()
50
+ session = Session(session_id, driver, created_at)
51
+
52
+ self.sessions[session_id] = session
53
+
54
+ return session, True
55
+
56
+ def exists(self, session_id: str) -> bool:
57
+ return session_id in self.sessions
58
+
59
+ def destroy(self, session_id: str) -> bool:
60
+ """destroy closes the driver instance and removes session from the storage.
61
+ The function is noop if session_id doesn't exist.
62
+ The function returns True if session was found and destroyed,
63
+ and False if session_id wasn't found.
64
+ """
65
+ if not self.exists(session_id):
66
+ return False
67
+
68
+ session = self.sessions.pop(session_id)
69
+ if utils.PLATFORM_VERSION == "nt":
70
+ session.driver.close()
71
+ session.driver.quit()
72
+ return True
73
+
74
+ def get(self, session_id: str, ttl: Optional[timedelta] = None) -> Tuple[Session, bool]:
75
+ session, fresh = self.create(session_id)
76
+
77
+ if ttl is not None and not fresh and session.lifetime() > ttl:
78
+ logging.debug(f'session\'s lifetime has expired, so the session is recreated (session_id={session_id})')
79
+ session, fresh = self.create(session_id, force_new=True)
80
+
81
+ return session, fresh
82
+
83
+ def session_ids(self) -> list[str]:
84
+ return list(self.sessions.keys())
src/tests.py CHANGED
@@ -1,632 +1,632 @@
1
- import unittest
2
- from typing import Optional
3
-
4
- from webtest import TestApp
5
-
6
- from dtos import IndexResponse, HealthResponse, V1ResponseBase, STATUS_OK, STATUS_ERROR
7
- import flaresolverr
8
- import utils
9
-
10
-
11
- def _find_obj_by_key(key: str, value: str, _list: list) -> Optional[dict]:
12
- for obj in _list:
13
- if obj[key] == value:
14
- return obj
15
- return None
16
-
17
-
18
- class TestFlareSolverr(unittest.TestCase):
19
-
20
- proxy_url = "http://127.0.0.1:8888"
21
- proxy_socks_url = "socks5://127.0.0.1:1080"
22
- google_url = "https://www.google.com"
23
- post_url = "https://httpbin.org/post"
24
- cloudflare_url = "https://nowsecure.nl"
25
- cloudflare_url_2 = "https://idope.se/torrent-list/harry/"
26
- ddos_guard_url = "https://anidex.info/"
27
- fairlane_url = "https://www.pararius.com/apartments/amsterdam"
28
- custom_cloudflare_url = "https://www.muziekfabriek.org"
29
- cloudflare_blocked_url = "https://cpasbiens3.fr/index.php?do=search&subaction=search"
30
-
31
- app = TestApp(flaresolverr.app)
32
- # wait until the server is ready
33
- app.get('/')
34
-
35
- def test_wrong_endpoint(self):
36
- res = self.app.get('/wrong', status=404)
37
- self.assertEqual(res.status_code, 404)
38
-
39
- body = res.json
40
- self.assertEqual("Not found: '/wrong'", body['error'])
41
- self.assertEqual(404, body['status_code'])
42
-
43
- def test_index_endpoint(self):
44
- res = self.app.get('/')
45
- self.assertEqual(res.status_code, 200)
46
-
47
- body = IndexResponse(res.json)
48
- self.assertEqual("FlareSolverr is ready!", body.msg)
49
- self.assertEqual(utils.get_flaresolverr_version(), body.version)
50
- self.assertIn("Chrome/", body.userAgent)
51
-
52
- def test_health_endpoint(self):
53
- res = self.app.get('/health')
54
- self.assertEqual(res.status_code, 200)
55
-
56
- body = HealthResponse(res.json)
57
- self.assertEqual(STATUS_OK, body.status)
58
-
59
- def test_v1_endpoint_wrong_cmd(self):
60
- res = self.app.post_json('/v1', {
61
- "cmd": "request.bad",
62
- "url": self.google_url
63
- }, status=500)
64
- self.assertEqual(res.status_code, 500)
65
-
66
- body = V1ResponseBase(res.json)
67
- self.assertEqual(STATUS_ERROR, body.status)
68
- self.assertEqual("Error: Request parameter 'cmd' = 'request.bad' is invalid.", body.message)
69
- self.assertGreater(body.startTimestamp, 10000)
70
- self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
71
- self.assertEqual(utils.get_flaresolverr_version(), body.version)
72
-
73
- def test_v1_endpoint_request_get_no_cloudflare(self):
74
- res = self.app.post_json('/v1', {
75
- "cmd": "request.get",
76
- "url": self.google_url
77
- })
78
- self.assertEqual(res.status_code, 200)
79
-
80
- body = V1ResponseBase(res.json)
81
- self.assertEqual(STATUS_OK, body.status)
82
- self.assertEqual("Challenge not detected!", body.message)
83
- self.assertGreater(body.startTimestamp, 10000)
84
- self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
85
- self.assertEqual(utils.get_flaresolverr_version(), body.version)
86
-
87
- solution = body.solution
88
- self.assertIn(self.google_url, solution.url)
89
- self.assertEqual(solution.status, 200)
90
- self.assertIs(len(solution.headers), 0)
91
- self.assertIn("<title>Google</title>", solution.response)
92
- self.assertGreater(len(solution.cookies), 0)
93
- self.assertIn("Chrome/", solution.userAgent)
94
-
95
- def test_v1_endpoint_request_get_cloudflare_js_1(self):
96
- res = self.app.post_json('/v1', {
97
- "cmd": "request.get",
98
- "url": self.cloudflare_url
99
- })
100
- self.assertEqual(res.status_code, 200)
101
-
102
- body = V1ResponseBase(res.json)
103
- self.assertEqual(STATUS_OK, body.status)
104
- self.assertEqual("Challenge solved!", body.message)
105
- self.assertGreater(body.startTimestamp, 10000)
106
- self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
107
- self.assertEqual(utils.get_flaresolverr_version(), body.version)
108
-
109
- solution = body.solution
110
- self.assertIn(self.cloudflare_url, solution.url)
111
- self.assertEqual(solution.status, 200)
112
- self.assertIs(len(solution.headers), 0)
113
- self.assertIn("<title>nowSecure</title>", solution.response)
114
- self.assertGreater(len(solution.cookies), 0)
115
- self.assertIn("Chrome/", solution.userAgent)
116
-
117
- cf_cookie = _find_obj_by_key("name", "cf_clearance", solution.cookies)
118
- self.assertIsNotNone(cf_cookie, "Cloudflare cookie not found")
119
- self.assertGreater(len(cf_cookie["value"]), 30)
120
-
121
- def test_v1_endpoint_request_get_cloudflare_js_2(self):
122
- res = self.app.post_json('/v1', {
123
- "cmd": "request.get",
124
- "url": self.cloudflare_url_2
125
- })
126
- self.assertEqual(res.status_code, 200)
127
-
128
- body = V1ResponseBase(res.json)
129
- self.assertEqual(STATUS_OK, body.status)
130
- self.assertEqual("Challenge solved!", body.message)
131
- self.assertGreater(body.startTimestamp, 10000)
132
- self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
133
- self.assertEqual(utils.get_flaresolverr_version(), body.version)
134
-
135
- solution = body.solution
136
- self.assertIn(self.cloudflare_url_2, solution.url)
137
- self.assertEqual(solution.status, 200)
138
- self.assertIs(len(solution.headers), 0)
139
- self.assertIn("<title>harry - idope torrent search</title>", solution.response)
140
- self.assertGreater(len(solution.cookies), 0)
141
- self.assertIn("Chrome/", solution.userAgent)
142
-
143
- cf_cookie = _find_obj_by_key("name", "cf_clearance", solution.cookies)
144
- self.assertIsNotNone(cf_cookie, "Cloudflare cookie not found")
145
- self.assertGreater(len(cf_cookie["value"]), 30)
146
-
147
- def test_v1_endpoint_request_get_ddos_guard_js(self):
148
- res = self.app.post_json('/v1', {
149
- "cmd": "request.get",
150
- "url": self.ddos_guard_url
151
- })
152
- self.assertEqual(res.status_code, 200)
153
-
154
- body = V1ResponseBase(res.json)
155
- self.assertEqual(STATUS_OK, body.status)
156
- self.assertEqual("Challenge solved!", body.message)
157
- self.assertGreater(body.startTimestamp, 10000)
158
- self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
159
- self.assertEqual(utils.get_flaresolverr_version(), body.version)
160
-
161
- solution = body.solution
162
- self.assertIn(self.ddos_guard_url, solution.url)
163
- self.assertEqual(solution.status, 200)
164
- self.assertIs(len(solution.headers), 0)
165
- self.assertIn("<title>AniDex</title>", solution.response)
166
- self.assertGreater(len(solution.cookies), 0)
167
- self.assertIn("Chrome/", solution.userAgent)
168
-
169
- cf_cookie = _find_obj_by_key("name", "__ddg1_", solution.cookies)
170
- self.assertIsNotNone(cf_cookie, "DDOS-Guard cookie not found")
171
- self.assertGreater(len(cf_cookie["value"]), 10)
172
-
173
- def test_v1_endpoint_request_get_fairlane_js(self):
174
- res = self.app.post_json('/v1', {
175
- "cmd": "request.get",
176
- "url": self.fairlane_url
177
- })
178
- self.assertEqual(res.status_code, 200)
179
-
180
- body = V1ResponseBase(res.json)
181
- self.assertEqual(STATUS_OK, body.status)
182
- self.assertEqual("Challenge solved!", body.message)
183
- self.assertGreater(body.startTimestamp, 10000)
184
- self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
185
- self.assertEqual(utils.get_flaresolverr_version(), body.version)
186
-
187
- solution = body.solution
188
- self.assertIn(self.fairlane_url, solution.url)
189
- self.assertEqual(solution.status, 200)
190
- self.assertIs(len(solution.headers), 0)
191
- self.assertIn("<title>Rental Apartments Amsterdam</title>", solution.response)
192
- self.assertGreater(len(solution.cookies), 0)
193
- self.assertIn("Chrome/", solution.userAgent)
194
-
195
- cf_cookie = _find_obj_by_key("name", "fl_pass_v2_b", solution.cookies)
196
- self.assertIsNotNone(cf_cookie, "Fairlane cookie not found")
197
- self.assertGreater(len(cf_cookie["value"]), 50)
198
-
199
- def test_v1_endpoint_request_get_custom_cloudflare_js(self):
200
- res = self.app.post_json('/v1', {
201
- "cmd": "request.get",
202
- "url": self.custom_cloudflare_url
203
- })
204
- self.assertEqual(res.status_code, 200)
205
-
206
- body = V1ResponseBase(res.json)
207
- self.assertEqual(STATUS_OK, body.status)
208
- self.assertEqual("Challenge solved!", body.message)
209
- self.assertGreater(body.startTimestamp, 10000)
210
- self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
211
- self.assertEqual(utils.get_flaresolverr_version(), body.version)
212
-
213
- solution = body.solution
214
- self.assertIn(self.custom_cloudflare_url, solution.url)
215
- self.assertEqual(solution.status, 200)
216
- self.assertIs(len(solution.headers), 0)
217
- self.assertIn("<title>MuziekFabriek : Aanmelden</title>", solution.response)
218
- self.assertGreater(len(solution.cookies), 0)
219
- self.assertIn("Chrome/", solution.userAgent)
220
-
221
- cf_cookie = _find_obj_by_key("name", "ct_anti_ddos_key", solution.cookies)
222
- self.assertIsNotNone(cf_cookie, "Custom Cloudflare cookie not found")
223
- self.assertGreater(len(cf_cookie["value"]), 10)
224
-
225
- # todo: test Cmd 'request.get' should return fail with Cloudflare CAPTCHA
226
-
227
- def test_v1_endpoint_request_get_cloudflare_blocked(self):
228
- res = self.app.post_json('/v1', {
229
- "cmd": "request.get",
230
- "url": self.cloudflare_blocked_url
231
- }, status=500)
232
- self.assertEqual(res.status_code, 500)
233
-
234
- body = V1ResponseBase(res.json)
235
- self.assertEqual(STATUS_ERROR, body.status)
236
- self.assertEqual("Error: Error solving the challenge. Cloudflare has blocked this request. "
237
- "Probably your IP is banned for this site, check in your web browser.", body.message)
238
- self.assertGreater(body.startTimestamp, 10000)
239
- self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
240
- self.assertEqual(utils.get_flaresolverr_version(), body.version)
241
-
242
- def test_v1_endpoint_request_get_cookies_param(self):
243
- res = self.app.post_json('/v1', {
244
- "cmd": "request.get",
245
- "url": self.google_url,
246
- "cookies": [
247
- {
248
- "name": "testcookie1",
249
- "value": "testvalue1"
250
- },
251
- {
252
- "name": "testcookie2",
253
- "value": "testvalue2"
254
- }
255
- ]
256
- })
257
- self.assertEqual(res.status_code, 200)
258
-
259
- body = V1ResponseBase(res.json)
260
- self.assertEqual(STATUS_OK, body.status)
261
- self.assertEqual("Challenge not detected!", body.message)
262
- self.assertGreater(body.startTimestamp, 10000)
263
- self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
264
- self.assertEqual(utils.get_flaresolverr_version(), body.version)
265
-
266
- solution = body.solution
267
- self.assertIn(self.google_url, solution.url)
268
- self.assertEqual(solution.status, 200)
269
- self.assertIs(len(solution.headers), 0)
270
- self.assertIn("<title>Google</title>", solution.response)
271
- self.assertGreater(len(solution.cookies), 1)
272
- self.assertIn("Chrome/", solution.userAgent)
273
-
274
- user_cookie1 = _find_obj_by_key("name", "testcookie1", solution.cookies)
275
- self.assertIsNotNone(user_cookie1, "User cookie 1 not found")
276
- self.assertEqual("testvalue1", user_cookie1["value"])
277
-
278
- user_cookie2 = _find_obj_by_key("name", "testcookie2", solution.cookies)
279
- self.assertIsNotNone(user_cookie2, "User cookie 2 not found")
280
- self.assertEqual("testvalue2", user_cookie2["value"])
281
-
282
- def test_v1_endpoint_request_get_returnOnlyCookies_param(self):
283
- res = self.app.post_json('/v1', {
284
- "cmd": "request.get",
285
- "url": self.google_url,
286
- "returnOnlyCookies": True
287
- })
288
- self.assertEqual(res.status_code, 200)
289
-
290
- body = V1ResponseBase(res.json)
291
- self.assertEqual(STATUS_OK, body.status)
292
- self.assertEqual("Challenge not detected!", body.message)
293
- self.assertGreater(body.startTimestamp, 10000)
294
- self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
295
- self.assertEqual(utils.get_flaresolverr_version(), body.version)
296
-
297
- solution = body.solution
298
- self.assertIn(self.google_url, solution.url)
299
- self.assertEqual(solution.status, 200)
300
- self.assertIsNone(solution.headers)
301
- self.assertIsNone(solution.response)
302
- self.assertGreater(len(solution.cookies), 0)
303
- self.assertIn("Chrome/", solution.userAgent)
304
-
305
- def test_v1_endpoint_request_get_proxy_http_param(self):
306
- """
307
- To configure TinyProxy in local:
308
- * sudo vim /etc/tinyproxy/tinyproxy.conf
309
- * edit => LogFile "/tmp/tinyproxy.log"
310
- * edit => Syslog Off
311
- * sudo tinyproxy -d
312
- * sudo tail -f /tmp/tinyproxy.log
313
- """
314
- res = self.app.post_json('/v1', {
315
- "cmd": "request.get",
316
- "url": self.google_url,
317
- "proxy": {
318
- "url": self.proxy_url
319
- }
320
- })
321
- self.assertEqual(res.status_code, 200)
322
-
323
- body = V1ResponseBase(res.json)
324
- self.assertEqual(STATUS_OK, body.status)
325
- self.assertEqual("Challenge not detected!", body.message)
326
- self.assertGreater(body.startTimestamp, 10000)
327
- self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
328
- self.assertEqual(utils.get_flaresolverr_version(), body.version)
329
-
330
- solution = body.solution
331
- self.assertIn(self.google_url, solution.url)
332
- self.assertEqual(solution.status, 200)
333
- self.assertIs(len(solution.headers), 0)
334
- self.assertIn("<title>Google</title>", solution.response)
335
- self.assertGreater(len(solution.cookies), 0)
336
- self.assertIn("Chrome/", solution.userAgent)
337
-
338
- def test_v1_endpoint_request_get_proxy_http_param_with_credentials(self):
339
- """
340
- To configure TinyProxy in local:
341
- * sudo vim /etc/tinyproxy/tinyproxy.conf
342
- * edit => LogFile "/tmp/tinyproxy.log"
343
- * edit => Syslog Off
344
- * add => BasicAuth testuser testpass
345
- * sudo tinyproxy -d
346
- * sudo tail -f /tmp/tinyproxy.log
347
- """
348
- res = self.app.post_json('/v1', {
349
- "cmd": "request.get",
350
- "url": self.google_url,
351
- "proxy": {
352
- "url": self.proxy_url,
353
- "username": "testuser",
354
- "password": "testpass"
355
- }
356
- })
357
- self.assertEqual(res.status_code, 200)
358
-
359
- body = V1ResponseBase(res.json)
360
- self.assertEqual(STATUS_OK, body.status)
361
- self.assertEqual("Challenge not detected!", body.message)
362
- self.assertGreater(body.startTimestamp, 10000)
363
- self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
364
- self.assertEqual(utils.get_flaresolverr_version(), body.version)
365
-
366
- solution = body.solution
367
- self.assertIn(self.google_url, solution.url)
368
- self.assertEqual(solution.status, 200)
369
- self.assertIs(len(solution.headers), 0)
370
- self.assertIn("<title>Google</title>", solution.response)
371
- self.assertGreater(len(solution.cookies), 0)
372
- self.assertIn("Chrome/", solution.userAgent)
373
-
374
- def test_v1_endpoint_request_get_proxy_socks_param(self):
375
- """
376
- To configure Dante in local:
377
- * https://linuxhint.com/set-up-a-socks5-proxy-on-ubuntu-with-dante/
378
- * sudo vim /etc/sockd.conf
379
- * sudo systemctl restart sockd.service
380
- * curl --socks5 socks5://127.0.0.1:1080 https://www.google.com
381
- """
382
- res = self.app.post_json('/v1', {
383
- "cmd": "request.get",
384
- "url": self.google_url,
385
- "proxy": {
386
- "url": self.proxy_socks_url
387
- }
388
- })
389
- self.assertEqual(res.status_code, 200)
390
-
391
- body = V1ResponseBase(res.json)
392
- self.assertEqual(STATUS_OK, body.status)
393
- self.assertEqual("Challenge not detected!", body.message)
394
- self.assertGreater(body.startTimestamp, 10000)
395
- self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
396
- self.assertEqual(utils.get_flaresolverr_version(), body.version)
397
-
398
- solution = body.solution
399
- self.assertIn(self.google_url, solution.url)
400
- self.assertEqual(solution.status, 200)
401
- self.assertIs(len(solution.headers), 0)
402
- self.assertIn("<title>Google</title>", solution.response)
403
- self.assertGreater(len(solution.cookies), 0)
404
- self.assertIn("Chrome/", solution.userAgent)
405
-
406
- def test_v1_endpoint_request_get_proxy_wrong_param(self):
407
- res = self.app.post_json('/v1', {
408
- "cmd": "request.get",
409
- "url": self.google_url,
410
- "proxy": {
411
- "url": "http://127.0.0.1:43210"
412
- }
413
- }, status=500)
414
- self.assertEqual(res.status_code, 500)
415
-
416
- body = V1ResponseBase(res.json)
417
- self.assertEqual(STATUS_ERROR, body.status)
418
- self.assertIn("Error: Error solving the challenge. Message: unknown error: net::ERR_PROXY_CONNECTION_FAILED",
419
- body.message)
420
- self.assertGreater(body.startTimestamp, 10000)
421
- self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
422
- self.assertEqual(utils.get_flaresolverr_version(), body.version)
423
-
424
- def test_v1_endpoint_request_get_fail_timeout(self):
425
- res = self.app.post_json('/v1', {
426
- "cmd": "request.get",
427
- "url": self.google_url,
428
- "maxTimeout": 10
429
- }, status=500)
430
- self.assertEqual(res.status_code, 500)
431
-
432
- body = V1ResponseBase(res.json)
433
- self.assertEqual(STATUS_ERROR, body.status)
434
- self.assertEqual("Error: Error solving the challenge. Timeout after 0.01 seconds.", body.message)
435
- self.assertGreater(body.startTimestamp, 10000)
436
- self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
437
- self.assertEqual(utils.get_flaresolverr_version(), body.version)
438
-
439
- def test_v1_endpoint_request_get_fail_bad_domain(self):
440
- res = self.app.post_json('/v1', {
441
- "cmd": "request.get",
442
- "url": "https://www.google.combad"
443
- }, status=500)
444
- self.assertEqual(res.status_code, 500)
445
-
446
- body = V1ResponseBase(res.json)
447
- self.assertEqual(STATUS_ERROR, body.status)
448
- self.assertIn("Message: unknown error: net::ERR_NAME_NOT_RESOLVED", body.message)
449
-
450
- def test_v1_endpoint_request_get_deprecated_param(self):
451
- res = self.app.post_json('/v1', {
452
- "cmd": "request.get",
453
- "url": self.google_url,
454
- "userAgent": "Test User-Agent" # was removed in v2, not used
455
- })
456
- self.assertEqual(res.status_code, 200)
457
-
458
- body = V1ResponseBase(res.json)
459
- self.assertEqual(STATUS_OK, body.status)
460
- self.assertEqual("Challenge not detected!", body.message)
461
-
462
- def test_v1_endpoint_request_post_no_cloudflare(self):
463
- res = self.app.post_json('/v1', {
464
- "cmd": "request.post",
465
- "url": self.post_url,
466
- "postData": "param1=value1&param2=value2"
467
- })
468
- self.assertEqual(res.status_code, 200)
469
-
470
- body = V1ResponseBase(res.json)
471
- self.assertEqual(STATUS_OK, body.status)
472
- self.assertEqual("Challenge not detected!", body.message)
473
- self.assertGreater(body.startTimestamp, 10000)
474
- self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
475
- self.assertEqual(utils.get_flaresolverr_version(), body.version)
476
-
477
- solution = body.solution
478
- self.assertIn(self.post_url, solution.url)
479
- self.assertEqual(solution.status, 200)
480
- self.assertIs(len(solution.headers), 0)
481
- self.assertIn('"form": {\n "param1": "value1", \n "param2": "value2"\n }', solution.response)
482
- self.assertEqual(len(solution.cookies), 0)
483
- self.assertIn("Chrome/", solution.userAgent)
484
-
485
- def test_v1_endpoint_request_post_cloudflare(self):
486
- res = self.app.post_json('/v1', {
487
- "cmd": "request.post",
488
- "url": self.cloudflare_url,
489
- "postData": "param1=value1&param2=value2"
490
- })
491
- self.assertEqual(res.status_code, 200)
492
-
493
- body = V1ResponseBase(res.json)
494
- self.assertEqual(STATUS_OK, body.status)
495
- self.assertEqual("Challenge solved!", body.message)
496
- self.assertGreater(body.startTimestamp, 10000)
497
- self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
498
- self.assertEqual(utils.get_flaresolverr_version(), body.version)
499
-
500
- solution = body.solution
501
- self.assertIn(self.cloudflare_url, solution.url)
502
- self.assertEqual(solution.status, 200)
503
- self.assertIs(len(solution.headers), 0)
504
- self.assertIn("<title>405 Not Allowed</title>", solution.response)
505
- self.assertGreater(len(solution.cookies), 0)
506
- self.assertIn("Chrome/", solution.userAgent)
507
-
508
- cf_cookie = _find_obj_by_key("name", "cf_clearance", solution.cookies)
509
- self.assertIsNotNone(cf_cookie, "Cloudflare cookie not found")
510
- self.assertGreater(len(cf_cookie["value"]), 30)
511
-
512
- def test_v1_endpoint_request_post_fail_no_post_data(self):
513
- res = self.app.post_json('/v1', {
514
- "cmd": "request.post",
515
- "url": self.google_url
516
- }, status=500)
517
- self.assertEqual(res.status_code, 500)
518
-
519
- body = V1ResponseBase(res.json)
520
- self.assertEqual(STATUS_ERROR, body.status)
521
- self.assertIn("Request parameter 'postData' is mandatory in 'request.post' command", body.message)
522
-
523
- def test_v1_endpoint_request_post_deprecated_param(self):
524
- res = self.app.post_json('/v1', {
525
- "cmd": "request.post",
526
- "url": self.google_url,
527
- "postData": "param1=value1&param2=value2",
528
- "userAgent": "Test User-Agent" # was removed in v2, not used
529
- })
530
- self.assertEqual(res.status_code, 200)
531
-
532
- body = V1ResponseBase(res.json)
533
- self.assertEqual(STATUS_OK, body.status)
534
- self.assertEqual("Challenge not detected!", body.message)
535
-
536
- def test_v1_endpoint_sessions_create_without_session(self):
537
- res = self.app.post_json('/v1', {
538
- "cmd": "sessions.create"
539
- })
540
- self.assertEqual(res.status_code, 200)
541
-
542
- body = V1ResponseBase(res.json)
543
- self.assertEqual(STATUS_OK, body.status)
544
- self.assertEqual("Session created successfully.", body.message)
545
- self.assertIsNotNone(body.session)
546
-
547
- def test_v1_endpoint_sessions_create_with_session(self):
548
- res = self.app.post_json('/v1', {
549
- "cmd": "sessions.create",
550
- "session": "test_create_session"
551
- })
552
- self.assertEqual(res.status_code, 200)
553
-
554
- body = V1ResponseBase(res.json)
555
- self.assertEqual(STATUS_OK, body.status)
556
- self.assertEqual("Session created successfully.", body.message)
557
- self.assertEqual(body.session, "test_create_session")
558
-
559
- def test_v1_endpoint_sessions_create_with_proxy(self):
560
- res = self.app.post_json('/v1', {
561
- "cmd": "sessions.create",
562
- "proxy": {
563
- "url": self.proxy_url
564
- }
565
- })
566
- self.assertEqual(res.status_code, 200)
567
-
568
- body = V1ResponseBase(res.json)
569
- self.assertEqual(STATUS_OK, body.status)
570
- self.assertEqual("Session created successfully.", body.message)
571
- self.assertIsNotNone(body.session)
572
-
573
- def test_v1_endpoint_sessions_list(self):
574
- self.app.post_json('/v1', {
575
- "cmd": "sessions.create",
576
- "session": "test_list_sessions"
577
- })
578
- res = self.app.post_json('/v1', {
579
- "cmd": "sessions.list"
580
- })
581
- self.assertEqual(res.status_code, 200)
582
-
583
- body = V1ResponseBase(res.json)
584
- self.assertEqual(STATUS_OK, body.status)
585
- self.assertEqual("", body.message)
586
- self.assertGreaterEqual(len(body.sessions), 1)
587
- self.assertIn("test_list_sessions", body.sessions)
588
-
589
- def test_v1_endpoint_sessions_destroy_existing_session(self):
590
- self.app.post_json('/v1', {
591
- "cmd": "sessions.create",
592
- "session": "test_destroy_sessions"
593
- })
594
- res = self.app.post_json('/v1', {
595
- "cmd": "sessions.destroy",
596
- "session": "test_destroy_sessions"
597
- })
598
- self.assertEqual(res.status_code, 200)
599
-
600
- body = V1ResponseBase(res.json)
601
- self.assertEqual(STATUS_OK, body.status)
602
- self.assertEqual("The session has been removed.", body.message)
603
-
604
- def test_v1_endpoint_sessions_destroy_non_existing_session(self):
605
- res = self.app.post_json('/v1', {
606
- "cmd": "sessions.destroy",
607
- "session": "non_existing_session_name"
608
- }, status=500)
609
- self.assertEqual(res.status_code, 500)
610
-
611
- body = V1ResponseBase(res.json)
612
- self.assertEqual(STATUS_ERROR, body.status)
613
- self.assertEqual("Error: The session doesn't exist.", body.message)
614
-
615
- def test_v1_endpoint_request_get_with_session(self):
616
- self.app.post_json('/v1', {
617
- "cmd": "sessions.create",
618
- "session": "test_request_sessions"
619
- })
620
- res = self.app.post_json('/v1', {
621
- "cmd": "request.get",
622
- "session": "test_request_sessions",
623
- "url": self.google_url
624
- })
625
- self.assertEqual(res.status_code, 200)
626
-
627
- body = V1ResponseBase(res.json)
628
- self.assertEqual(STATUS_OK, body.status)
629
-
630
-
631
- if __name__ == '__main__':
632
- unittest.main()
 
1
+ import unittest
2
+ from typing import Optional
3
+
4
+ from webtest import TestApp
5
+
6
+ from dtos import IndexResponse, HealthResponse, V1ResponseBase, STATUS_OK, STATUS_ERROR
7
+ import flaresolverr
8
+ import utils
9
+
10
+
11
+ def _find_obj_by_key(key: str, value: str, _list: list) -> Optional[dict]:
12
+ for obj in _list:
13
+ if obj[key] == value:
14
+ return obj
15
+ return None
16
+
17
+
18
+ class TestFlareSolverr(unittest.TestCase):
19
+
20
+ proxy_url = "http://127.0.0.1:8888"
21
+ proxy_socks_url = "socks5://127.0.0.1:1080"
22
+ google_url = "https://www.google.com"
23
+ post_url = "https://httpbin.org/post"
24
+ cloudflare_url = "https://nowsecure.nl"
25
+ cloudflare_url_2 = "https://idope.se/torrent-list/harry/"
26
+ ddos_guard_url = "https://anidex.info/"
27
+ fairlane_url = "https://www.pararius.com/apartments/amsterdam"
28
+ custom_cloudflare_url = "https://www.muziekfabriek.org"
29
+ cloudflare_blocked_url = "https://cpasbiens3.fr/index.php?do=search&subaction=search"
30
+
31
+ app = TestApp(flaresolverr.app)
32
+ # wait until the server is ready
33
+ app.get('/')
34
+
35
+ def test_wrong_endpoint(self):
36
+ res = self.app.get('/wrong', status=404)
37
+ self.assertEqual(res.status_code, 404)
38
+
39
+ body = res.json
40
+ self.assertEqual("Not found: '/wrong'", body['error'])
41
+ self.assertEqual(404, body['status_code'])
42
+
43
+ def test_index_endpoint(self):
44
+ res = self.app.get('/')
45
+ self.assertEqual(res.status_code, 200)
46
+
47
+ body = IndexResponse(res.json)
48
+ self.assertEqual("FlareSolverr is ready!", body.msg)
49
+ self.assertEqual(utils.get_flaresolverr_version(), body.version)
50
+ self.assertIn("Chrome/", body.userAgent)
51
+
52
+ def test_health_endpoint(self):
53
+ res = self.app.get('/health')
54
+ self.assertEqual(res.status_code, 200)
55
+
56
+ body = HealthResponse(res.json)
57
+ self.assertEqual(STATUS_OK, body.status)
58
+
59
+ def test_v1_endpoint_wrong_cmd(self):
60
+ res = self.app.post_json('/v1', {
61
+ "cmd": "request.bad",
62
+ "url": self.google_url
63
+ }, status=500)
64
+ self.assertEqual(res.status_code, 500)
65
+
66
+ body = V1ResponseBase(res.json)
67
+ self.assertEqual(STATUS_ERROR, body.status)
68
+ self.assertEqual("Error: Request parameter 'cmd' = 'request.bad' is invalid.", body.message)
69
+ self.assertGreater(body.startTimestamp, 10000)
70
+ self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
71
+ self.assertEqual(utils.get_flaresolverr_version(), body.version)
72
+
73
+ def test_v1_endpoint_request_get_no_cloudflare(self):
74
+ res = self.app.post_json('/v1', {
75
+ "cmd": "request.get",
76
+ "url": self.google_url
77
+ })
78
+ self.assertEqual(res.status_code, 200)
79
+
80
+ body = V1ResponseBase(res.json)
81
+ self.assertEqual(STATUS_OK, body.status)
82
+ self.assertEqual("Challenge not detected!", body.message)
83
+ self.assertGreater(body.startTimestamp, 10000)
84
+ self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
85
+ self.assertEqual(utils.get_flaresolverr_version(), body.version)
86
+
87
+ solution = body.solution
88
+ self.assertIn(self.google_url, solution.url)
89
+ self.assertEqual(solution.status, 200)
90
+ self.assertIs(len(solution.headers), 0)
91
+ self.assertIn("<title>Google</title>", solution.response)
92
+ self.assertGreater(len(solution.cookies), 0)
93
+ self.assertIn("Chrome/", solution.userAgent)
94
+
95
+ def test_v1_endpoint_request_get_cloudflare_js_1(self):
96
+ res = self.app.post_json('/v1', {
97
+ "cmd": "request.get",
98
+ "url": self.cloudflare_url
99
+ })
100
+ self.assertEqual(res.status_code, 200)
101
+
102
+ body = V1ResponseBase(res.json)
103
+ self.assertEqual(STATUS_OK, body.status)
104
+ self.assertEqual("Challenge solved!", body.message)
105
+ self.assertGreater(body.startTimestamp, 10000)
106
+ self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
107
+ self.assertEqual(utils.get_flaresolverr_version(), body.version)
108
+
109
+ solution = body.solution
110
+ self.assertIn(self.cloudflare_url, solution.url)
111
+ self.assertEqual(solution.status, 200)
112
+ self.assertIs(len(solution.headers), 0)
113
+ self.assertIn("<title>nowSecure</title>", solution.response)
114
+ self.assertGreater(len(solution.cookies), 0)
115
+ self.assertIn("Chrome/", solution.userAgent)
116
+
117
+ cf_cookie = _find_obj_by_key("name", "cf_clearance", solution.cookies)
118
+ self.assertIsNotNone(cf_cookie, "Cloudflare cookie not found")
119
+ self.assertGreater(len(cf_cookie["value"]), 30)
120
+
121
+ def test_v1_endpoint_request_get_cloudflare_js_2(self):
122
+ res = self.app.post_json('/v1', {
123
+ "cmd": "request.get",
124
+ "url": self.cloudflare_url_2
125
+ })
126
+ self.assertEqual(res.status_code, 200)
127
+
128
+ body = V1ResponseBase(res.json)
129
+ self.assertEqual(STATUS_OK, body.status)
130
+ self.assertEqual("Challenge solved!", body.message)
131
+ self.assertGreater(body.startTimestamp, 10000)
132
+ self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
133
+ self.assertEqual(utils.get_flaresolverr_version(), body.version)
134
+
135
+ solution = body.solution
136
+ self.assertIn(self.cloudflare_url_2, solution.url)
137
+ self.assertEqual(solution.status, 200)
138
+ self.assertIs(len(solution.headers), 0)
139
+ self.assertIn("<title>harry - idope torrent search</title>", solution.response)
140
+ self.assertGreater(len(solution.cookies), 0)
141
+ self.assertIn("Chrome/", solution.userAgent)
142
+
143
+ cf_cookie = _find_obj_by_key("name", "cf_clearance", solution.cookies)
144
+ self.assertIsNotNone(cf_cookie, "Cloudflare cookie not found")
145
+ self.assertGreater(len(cf_cookie["value"]), 30)
146
+
147
+ def test_v1_endpoint_request_get_ddos_guard_js(self):
148
+ res = self.app.post_json('/v1', {
149
+ "cmd": "request.get",
150
+ "url": self.ddos_guard_url
151
+ })
152
+ self.assertEqual(res.status_code, 200)
153
+
154
+ body = V1ResponseBase(res.json)
155
+ self.assertEqual(STATUS_OK, body.status)
156
+ self.assertEqual("Challenge solved!", body.message)
157
+ self.assertGreater(body.startTimestamp, 10000)
158
+ self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
159
+ self.assertEqual(utils.get_flaresolverr_version(), body.version)
160
+
161
+ solution = body.solution
162
+ self.assertIn(self.ddos_guard_url, solution.url)
163
+ self.assertEqual(solution.status, 200)
164
+ self.assertIs(len(solution.headers), 0)
165
+ self.assertIn("<title>AniDex</title>", solution.response)
166
+ self.assertGreater(len(solution.cookies), 0)
167
+ self.assertIn("Chrome/", solution.userAgent)
168
+
169
+ cf_cookie = _find_obj_by_key("name", "__ddg1_", solution.cookies)
170
+ self.assertIsNotNone(cf_cookie, "DDOS-Guard cookie not found")
171
+ self.assertGreater(len(cf_cookie["value"]), 10)
172
+
173
+ def test_v1_endpoint_request_get_fairlane_js(self):
174
+ res = self.app.post_json('/v1', {
175
+ "cmd": "request.get",
176
+ "url": self.fairlane_url
177
+ })
178
+ self.assertEqual(res.status_code, 200)
179
+
180
+ body = V1ResponseBase(res.json)
181
+ self.assertEqual(STATUS_OK, body.status)
182
+ self.assertEqual("Challenge solved!", body.message)
183
+ self.assertGreater(body.startTimestamp, 10000)
184
+ self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
185
+ self.assertEqual(utils.get_flaresolverr_version(), body.version)
186
+
187
+ solution = body.solution
188
+ self.assertIn(self.fairlane_url, solution.url)
189
+ self.assertEqual(solution.status, 200)
190
+ self.assertIs(len(solution.headers), 0)
191
+ self.assertIn("<title>Rental Apartments Amsterdam</title>", solution.response)
192
+ self.assertGreater(len(solution.cookies), 0)
193
+ self.assertIn("Chrome/", solution.userAgent)
194
+
195
+ cf_cookie = _find_obj_by_key("name", "fl_pass_v2_b", solution.cookies)
196
+ self.assertIsNotNone(cf_cookie, "Fairlane cookie not found")
197
+ self.assertGreater(len(cf_cookie["value"]), 50)
198
+
199
+ def test_v1_endpoint_request_get_custom_cloudflare_js(self):
200
+ res = self.app.post_json('/v1', {
201
+ "cmd": "request.get",
202
+ "url": self.custom_cloudflare_url
203
+ })
204
+ self.assertEqual(res.status_code, 200)
205
+
206
+ body = V1ResponseBase(res.json)
207
+ self.assertEqual(STATUS_OK, body.status)
208
+ self.assertEqual("Challenge solved!", body.message)
209
+ self.assertGreater(body.startTimestamp, 10000)
210
+ self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
211
+ self.assertEqual(utils.get_flaresolverr_version(), body.version)
212
+
213
+ solution = body.solution
214
+ self.assertIn(self.custom_cloudflare_url, solution.url)
215
+ self.assertEqual(solution.status, 200)
216
+ self.assertIs(len(solution.headers), 0)
217
+ self.assertIn("<title>MuziekFabriek : Aanmelden</title>", solution.response)
218
+ self.assertGreater(len(solution.cookies), 0)
219
+ self.assertIn("Chrome/", solution.userAgent)
220
+
221
+ cf_cookie = _find_obj_by_key("name", "ct_anti_ddos_key", solution.cookies)
222
+ self.assertIsNotNone(cf_cookie, "Custom Cloudflare cookie not found")
223
+ self.assertGreater(len(cf_cookie["value"]), 10)
224
+
225
+ # todo: test Cmd 'request.get' should return fail with Cloudflare CAPTCHA
226
+
227
+ def test_v1_endpoint_request_get_cloudflare_blocked(self):
228
+ res = self.app.post_json('/v1', {
229
+ "cmd": "request.get",
230
+ "url": self.cloudflare_blocked_url
231
+ }, status=500)
232
+ self.assertEqual(res.status_code, 500)
233
+
234
+ body = V1ResponseBase(res.json)
235
+ self.assertEqual(STATUS_ERROR, body.status)
236
+ self.assertEqual("Error: Error solving the challenge. Cloudflare has blocked this request. "
237
+ "Probably your IP is banned for this site, check in your web browser.", body.message)
238
+ self.assertGreater(body.startTimestamp, 10000)
239
+ self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
240
+ self.assertEqual(utils.get_flaresolverr_version(), body.version)
241
+
242
+ def test_v1_endpoint_request_get_cookies_param(self):
243
+ res = self.app.post_json('/v1', {
244
+ "cmd": "request.get",
245
+ "url": self.google_url,
246
+ "cookies": [
247
+ {
248
+ "name": "testcookie1",
249
+ "value": "testvalue1"
250
+ },
251
+ {
252
+ "name": "testcookie2",
253
+ "value": "testvalue2"
254
+ }
255
+ ]
256
+ })
257
+ self.assertEqual(res.status_code, 200)
258
+
259
+ body = V1ResponseBase(res.json)
260
+ self.assertEqual(STATUS_OK, body.status)
261
+ self.assertEqual("Challenge not detected!", body.message)
262
+ self.assertGreater(body.startTimestamp, 10000)
263
+ self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
264
+ self.assertEqual(utils.get_flaresolverr_version(), body.version)
265
+
266
+ solution = body.solution
267
+ self.assertIn(self.google_url, solution.url)
268
+ self.assertEqual(solution.status, 200)
269
+ self.assertIs(len(solution.headers), 0)
270
+ self.assertIn("<title>Google</title>", solution.response)
271
+ self.assertGreater(len(solution.cookies), 1)
272
+ self.assertIn("Chrome/", solution.userAgent)
273
+
274
+ user_cookie1 = _find_obj_by_key("name", "testcookie1", solution.cookies)
275
+ self.assertIsNotNone(user_cookie1, "User cookie 1 not found")
276
+ self.assertEqual("testvalue1", user_cookie1["value"])
277
+
278
+ user_cookie2 = _find_obj_by_key("name", "testcookie2", solution.cookies)
279
+ self.assertIsNotNone(user_cookie2, "User cookie 2 not found")
280
+ self.assertEqual("testvalue2", user_cookie2["value"])
281
+
282
+ def test_v1_endpoint_request_get_returnOnlyCookies_param(self):
283
+ res = self.app.post_json('/v1', {
284
+ "cmd": "request.get",
285
+ "url": self.google_url,
286
+ "returnOnlyCookies": True
287
+ })
288
+ self.assertEqual(res.status_code, 200)
289
+
290
+ body = V1ResponseBase(res.json)
291
+ self.assertEqual(STATUS_OK, body.status)
292
+ self.assertEqual("Challenge not detected!", body.message)
293
+ self.assertGreater(body.startTimestamp, 10000)
294
+ self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
295
+ self.assertEqual(utils.get_flaresolverr_version(), body.version)
296
+
297
+ solution = body.solution
298
+ self.assertIn(self.google_url, solution.url)
299
+ self.assertEqual(solution.status, 200)
300
+ self.assertIsNone(solution.headers)
301
+ self.assertIsNone(solution.response)
302
+ self.assertGreater(len(solution.cookies), 0)
303
+ self.assertIn("Chrome/", solution.userAgent)
304
+
305
+ def test_v1_endpoint_request_get_proxy_http_param(self):
306
+ """
307
+ To configure TinyProxy in local:
308
+ * sudo vim /etc/tinyproxy/tinyproxy.conf
309
+ * edit => LogFile "/tmp/tinyproxy.log"
310
+ * edit => Syslog Off
311
+ * sudo tinyproxy -d
312
+ * sudo tail -f /tmp/tinyproxy.log
313
+ """
314
+ res = self.app.post_json('/v1', {
315
+ "cmd": "request.get",
316
+ "url": self.google_url,
317
+ "proxy": {
318
+ "url": self.proxy_url
319
+ }
320
+ })
321
+ self.assertEqual(res.status_code, 200)
322
+
323
+ body = V1ResponseBase(res.json)
324
+ self.assertEqual(STATUS_OK, body.status)
325
+ self.assertEqual("Challenge not detected!", body.message)
326
+ self.assertGreater(body.startTimestamp, 10000)
327
+ self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
328
+ self.assertEqual(utils.get_flaresolverr_version(), body.version)
329
+
330
+ solution = body.solution
331
+ self.assertIn(self.google_url, solution.url)
332
+ self.assertEqual(solution.status, 200)
333
+ self.assertIs(len(solution.headers), 0)
334
+ self.assertIn("<title>Google</title>", solution.response)
335
+ self.assertGreater(len(solution.cookies), 0)
336
+ self.assertIn("Chrome/", solution.userAgent)
337
+
338
+ def test_v1_endpoint_request_get_proxy_http_param_with_credentials(self):
339
+ """
340
+ To configure TinyProxy in local:
341
+ * sudo vim /etc/tinyproxy/tinyproxy.conf
342
+ * edit => LogFile "/tmp/tinyproxy.log"
343
+ * edit => Syslog Off
344
+ * add => BasicAuth testuser testpass
345
+ * sudo tinyproxy -d
346
+ * sudo tail -f /tmp/tinyproxy.log
347
+ """
348
+ res = self.app.post_json('/v1', {
349
+ "cmd": "request.get",
350
+ "url": self.google_url,
351
+ "proxy": {
352
+ "url": self.proxy_url,
353
+ "username": "testuser",
354
+ "password": "testpass"
355
+ }
356
+ })
357
+ self.assertEqual(res.status_code, 200)
358
+
359
+ body = V1ResponseBase(res.json)
360
+ self.assertEqual(STATUS_OK, body.status)
361
+ self.assertEqual("Challenge not detected!", body.message)
362
+ self.assertGreater(body.startTimestamp, 10000)
363
+ self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
364
+ self.assertEqual(utils.get_flaresolverr_version(), body.version)
365
+
366
+ solution = body.solution
367
+ self.assertIn(self.google_url, solution.url)
368
+ self.assertEqual(solution.status, 200)
369
+ self.assertIs(len(solution.headers), 0)
370
+ self.assertIn("<title>Google</title>", solution.response)
371
+ self.assertGreater(len(solution.cookies), 0)
372
+ self.assertIn("Chrome/", solution.userAgent)
373
+
374
+ def test_v1_endpoint_request_get_proxy_socks_param(self):
375
+ """
376
+ To configure Dante in local:
377
+ * https://linuxhint.com/set-up-a-socks5-proxy-on-ubuntu-with-dante/
378
+ * sudo vim /etc/sockd.conf
379
+ * sudo systemctl restart sockd.service
380
+ * curl --socks5 socks5://127.0.0.1:1080 https://www.google.com
381
+ """
382
+ res = self.app.post_json('/v1', {
383
+ "cmd": "request.get",
384
+ "url": self.google_url,
385
+ "proxy": {
386
+ "url": self.proxy_socks_url
387
+ }
388
+ })
389
+ self.assertEqual(res.status_code, 200)
390
+
391
+ body = V1ResponseBase(res.json)
392
+ self.assertEqual(STATUS_OK, body.status)
393
+ self.assertEqual("Challenge not detected!", body.message)
394
+ self.assertGreater(body.startTimestamp, 10000)
395
+ self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
396
+ self.assertEqual(utils.get_flaresolverr_version(), body.version)
397
+
398
+ solution = body.solution
399
+ self.assertIn(self.google_url, solution.url)
400
+ self.assertEqual(solution.status, 200)
401
+ self.assertIs(len(solution.headers), 0)
402
+ self.assertIn("<title>Google</title>", solution.response)
403
+ self.assertGreater(len(solution.cookies), 0)
404
+ self.assertIn("Chrome/", solution.userAgent)
405
+
406
+ def test_v1_endpoint_request_get_proxy_wrong_param(self):
407
+ res = self.app.post_json('/v1', {
408
+ "cmd": "request.get",
409
+ "url": self.google_url,
410
+ "proxy": {
411
+ "url": "http://127.0.0.1:43210"
412
+ }
413
+ }, status=500)
414
+ self.assertEqual(res.status_code, 500)
415
+
416
+ body = V1ResponseBase(res.json)
417
+ self.assertEqual(STATUS_ERROR, body.status)
418
+ self.assertIn("Error: Error solving the challenge. Message: unknown error: net::ERR_PROXY_CONNECTION_FAILED",
419
+ body.message)
420
+ self.assertGreater(body.startTimestamp, 10000)
421
+ self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
422
+ self.assertEqual(utils.get_flaresolverr_version(), body.version)
423
+
424
+ def test_v1_endpoint_request_get_fail_timeout(self):
425
+ res = self.app.post_json('/v1', {
426
+ "cmd": "request.get",
427
+ "url": self.google_url,
428
+ "maxTimeout": 10
429
+ }, status=500)
430
+ self.assertEqual(res.status_code, 500)
431
+
432
+ body = V1ResponseBase(res.json)
433
+ self.assertEqual(STATUS_ERROR, body.status)
434
+ self.assertEqual("Error: Error solving the challenge. Timeout after 0.01 seconds.", body.message)
435
+ self.assertGreater(body.startTimestamp, 10000)
436
+ self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
437
+ self.assertEqual(utils.get_flaresolverr_version(), body.version)
438
+
439
+ def test_v1_endpoint_request_get_fail_bad_domain(self):
440
+ res = self.app.post_json('/v1', {
441
+ "cmd": "request.get",
442
+ "url": "https://www.google.combad"
443
+ }, status=500)
444
+ self.assertEqual(res.status_code, 500)
445
+
446
+ body = V1ResponseBase(res.json)
447
+ self.assertEqual(STATUS_ERROR, body.status)
448
+ self.assertIn("Message: unknown error: net::ERR_NAME_NOT_RESOLVED", body.message)
449
+
450
+ def test_v1_endpoint_request_get_deprecated_param(self):
451
+ res = self.app.post_json('/v1', {
452
+ "cmd": "request.get",
453
+ "url": self.google_url,
454
+ "userAgent": "Test User-Agent" # was removed in v2, not used
455
+ })
456
+ self.assertEqual(res.status_code, 200)
457
+
458
+ body = V1ResponseBase(res.json)
459
+ self.assertEqual(STATUS_OK, body.status)
460
+ self.assertEqual("Challenge not detected!", body.message)
461
+
462
+ def test_v1_endpoint_request_post_no_cloudflare(self):
463
+ res = self.app.post_json('/v1', {
464
+ "cmd": "request.post",
465
+ "url": self.post_url,
466
+ "postData": "param1=value1&param2=value2"
467
+ })
468
+ self.assertEqual(res.status_code, 200)
469
+
470
+ body = V1ResponseBase(res.json)
471
+ self.assertEqual(STATUS_OK, body.status)
472
+ self.assertEqual("Challenge not detected!", body.message)
473
+ self.assertGreater(body.startTimestamp, 10000)
474
+ self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
475
+ self.assertEqual(utils.get_flaresolverr_version(), body.version)
476
+
477
+ solution = body.solution
478
+ self.assertIn(self.post_url, solution.url)
479
+ self.assertEqual(solution.status, 200)
480
+ self.assertIs(len(solution.headers), 0)
481
+ self.assertIn('"form": {\n "param1": "value1", \n "param2": "value2"\n }', solution.response)
482
+ self.assertEqual(len(solution.cookies), 0)
483
+ self.assertIn("Chrome/", solution.userAgent)
484
+
485
+ def test_v1_endpoint_request_post_cloudflare(self):
486
+ res = self.app.post_json('/v1', {
487
+ "cmd": "request.post",
488
+ "url": self.cloudflare_url,
489
+ "postData": "param1=value1&param2=value2"
490
+ })
491
+ self.assertEqual(res.status_code, 200)
492
+
493
+ body = V1ResponseBase(res.json)
494
+ self.assertEqual(STATUS_OK, body.status)
495
+ self.assertEqual("Challenge solved!", body.message)
496
+ self.assertGreater(body.startTimestamp, 10000)
497
+ self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
498
+ self.assertEqual(utils.get_flaresolverr_version(), body.version)
499
+
500
+ solution = body.solution
501
+ self.assertIn(self.cloudflare_url, solution.url)
502
+ self.assertEqual(solution.status, 200)
503
+ self.assertIs(len(solution.headers), 0)
504
+ self.assertIn("<title>405 Not Allowed</title>", solution.response)
505
+ self.assertGreater(len(solution.cookies), 0)
506
+ self.assertIn("Chrome/", solution.userAgent)
507
+
508
+ cf_cookie = _find_obj_by_key("name", "cf_clearance", solution.cookies)
509
+ self.assertIsNotNone(cf_cookie, "Cloudflare cookie not found")
510
+ self.assertGreater(len(cf_cookie["value"]), 30)
511
+
512
+ def test_v1_endpoint_request_post_fail_no_post_data(self):
513
+ res = self.app.post_json('/v1', {
514
+ "cmd": "request.post",
515
+ "url": self.google_url
516
+ }, status=500)
517
+ self.assertEqual(res.status_code, 500)
518
+
519
+ body = V1ResponseBase(res.json)
520
+ self.assertEqual(STATUS_ERROR, body.status)
521
+ self.assertIn("Request parameter 'postData' is mandatory in 'request.post' command", body.message)
522
+
523
+ def test_v1_endpoint_request_post_deprecated_param(self):
524
+ res = self.app.post_json('/v1', {
525
+ "cmd": "request.post",
526
+ "url": self.google_url,
527
+ "postData": "param1=value1&param2=value2",
528
+ "userAgent": "Test User-Agent" # was removed in v2, not used
529
+ })
530
+ self.assertEqual(res.status_code, 200)
531
+
532
+ body = V1ResponseBase(res.json)
533
+ self.assertEqual(STATUS_OK, body.status)
534
+ self.assertEqual("Challenge not detected!", body.message)
535
+
536
+ def test_v1_endpoint_sessions_create_without_session(self):
537
+ res = self.app.post_json('/v1', {
538
+ "cmd": "sessions.create"
539
+ })
540
+ self.assertEqual(res.status_code, 200)
541
+
542
+ body = V1ResponseBase(res.json)
543
+ self.assertEqual(STATUS_OK, body.status)
544
+ self.assertEqual("Session created successfully.", body.message)
545
+ self.assertIsNotNone(body.session)
546
+
547
+ def test_v1_endpoint_sessions_create_with_session(self):
548
+ res = self.app.post_json('/v1', {
549
+ "cmd": "sessions.create",
550
+ "session": "test_create_session"
551
+ })
552
+ self.assertEqual(res.status_code, 200)
553
+
554
+ body = V1ResponseBase(res.json)
555
+ self.assertEqual(STATUS_OK, body.status)
556
+ self.assertEqual("Session created successfully.", body.message)
557
+ self.assertEqual(body.session, "test_create_session")
558
+
559
+ def test_v1_endpoint_sessions_create_with_proxy(self):
560
+ res = self.app.post_json('/v1', {
561
+ "cmd": "sessions.create",
562
+ "proxy": {
563
+ "url": self.proxy_url
564
+ }
565
+ })
566
+ self.assertEqual(res.status_code, 200)
567
+
568
+ body = V1ResponseBase(res.json)
569
+ self.assertEqual(STATUS_OK, body.status)
570
+ self.assertEqual("Session created successfully.", body.message)
571
+ self.assertIsNotNone(body.session)
572
+
573
+ def test_v1_endpoint_sessions_list(self):
574
+ self.app.post_json('/v1', {
575
+ "cmd": "sessions.create",
576
+ "session": "test_list_sessions"
577
+ })
578
+ res = self.app.post_json('/v1', {
579
+ "cmd": "sessions.list"
580
+ })
581
+ self.assertEqual(res.status_code, 200)
582
+
583
+ body = V1ResponseBase(res.json)
584
+ self.assertEqual(STATUS_OK, body.status)
585
+ self.assertEqual("", body.message)
586
+ self.assertGreaterEqual(len(body.sessions), 1)
587
+ self.assertIn("test_list_sessions", body.sessions)
588
+
589
+ def test_v1_endpoint_sessions_destroy_existing_session(self):
590
+ self.app.post_json('/v1', {
591
+ "cmd": "sessions.create",
592
+ "session": "test_destroy_sessions"
593
+ })
594
+ res = self.app.post_json('/v1', {
595
+ "cmd": "sessions.destroy",
596
+ "session": "test_destroy_sessions"
597
+ })
598
+ self.assertEqual(res.status_code, 200)
599
+
600
+ body = V1ResponseBase(res.json)
601
+ self.assertEqual(STATUS_OK, body.status)
602
+ self.assertEqual("The session has been removed.", body.message)
603
+
604
+ def test_v1_endpoint_sessions_destroy_non_existing_session(self):
605
+ res = self.app.post_json('/v1', {
606
+ "cmd": "sessions.destroy",
607
+ "session": "non_existing_session_name"
608
+ }, status=500)
609
+ self.assertEqual(res.status_code, 500)
610
+
611
+ body = V1ResponseBase(res.json)
612
+ self.assertEqual(STATUS_ERROR, body.status)
613
+ self.assertEqual("Error: The session doesn't exist.", body.message)
614
+
615
+ def test_v1_endpoint_request_get_with_session(self):
616
+ self.app.post_json('/v1', {
617
+ "cmd": "sessions.create",
618
+ "session": "test_request_sessions"
619
+ })
620
+ res = self.app.post_json('/v1', {
621
+ "cmd": "request.get",
622
+ "session": "test_request_sessions",
623
+ "url": self.google_url
624
+ })
625
+ self.assertEqual(res.status_code, 200)
626
+
627
+ body = V1ResponseBase(res.json)
628
+ self.assertEqual(STATUS_OK, body.status)
629
+
630
+
631
+ if __name__ == '__main__':
632
+ unittest.main()
src/tests_sites.py CHANGED
@@ -1,102 +1,102 @@
1
- import unittest
2
-
3
- from webtest import TestApp
4
-
5
- from dtos import V1ResponseBase, STATUS_OK
6
- import flaresolverr
7
- import utils
8
-
9
-
10
- def _find_obj_by_key(key: str, value: str, _list: list) -> dict | None:
11
- for obj in _list:
12
- if obj[key] == value:
13
- return obj
14
- return None
15
-
16
-
17
- def asset_cloudflare_solution(self, res, site_url, site_text):
18
- self.assertEqual(res.status_code, 200)
19
-
20
- body = V1ResponseBase(res.json)
21
- self.assertEqual(STATUS_OK, body.status)
22
- self.assertEqual("Challenge solved!", body.message)
23
- self.assertGreater(body.startTimestamp, 10000)
24
- self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
25
- self.assertEqual(utils.get_flaresolverr_version(), body.version)
26
-
27
- solution = body.solution
28
- self.assertIn(site_url, solution.url)
29
- self.assertEqual(solution.status, 200)
30
- self.assertIs(len(solution.headers), 0)
31
- self.assertIn(site_text, solution.response)
32
- self.assertGreater(len(solution.cookies), 0)
33
- self.assertIn("Chrome/", solution.userAgent)
34
-
35
- cf_cookie = _find_obj_by_key("name", "cf_clearance", solution.cookies)
36
- self.assertIsNotNone(cf_cookie, "Cloudflare cookie not found")
37
- self.assertGreater(len(cf_cookie["value"]), 30)
38
-
39
-
40
- class TestFlareSolverr(unittest.TestCase):
41
- app = TestApp(flaresolverr.app)
42
- # wait until the server is ready
43
- app.get('/')
44
-
45
- def test_v1_endpoint_request_get_cloudflare(self):
46
- sites_get = [
47
- ('nowsecure', 'https://nowsecure.nl', '<title>nowSecure</title>'),
48
- ('0magnet', 'https://0magnet.com/search?q=2022', 'Torrent Search - ØMagnet'),
49
- ('1337x', 'https://1337x.unblockit.cat/cat/Movies/time/desc/1/', ''),
50
- ('avistaz', 'https://avistaz.to/api/v1/jackett/torrents?in=1&type=0&search=',
51
- '<title>Access denied</title>'),
52
- ('badasstorrents', 'https://badasstorrents.com/torrents/search/720p/date/desc',
53
- '<title>Latest Torrents - BadassTorrents</title>'),
54
- ('bt4g', 'https://bt4g.org/search/2022', '<title>Download 2022 Torrents - BT4G</title>'),
55
- ('cinemaz', 'https://cinemaz.to/api/v1/jackett/torrents?in=1&type=0&search=',
56
- '<title>Access denied</title>'),
57
- ('epublibre', 'https://epublibre.unblockit.cat/catalogo/index/0/nuevo/todos/sin/todos/--/ajax',
58
- '<title>epublibre - catálogo</title>'),
59
- ('ext', 'https://ext.to/latest/?order=age&sort=desc',
60
- '<title>Download Latest Torrents - EXT Torrents</title>'),
61
- ('extratorrent', 'https://extratorrent.st/search/?srt=added&order=desc&search=720p&new=1&x=0&y=0',
62
- 'Page 1 - ExtraTorrent'),
63
- ('idope', 'https://idope.se/browse.html', '<title>Recent Torrents</title>'),
64
- ('limetorrents', 'https://limetorrents.unblockninja.com/latest100',
65
- '<title>Latest 100 torrents - LimeTorrents</title>'),
66
- ('privatehd', 'https://privatehd.to/api/v1/jackett/torrents?in=1&type=0&search=',
67
- '<title>Access denied</title>'),
68
- ('torrentcore', 'https://torrentcore.xyz/index', '<title>Torrent[CORE] - Torrent community.</title>'),
69
- ('torrentqq223', 'https://torrentqq223.com/torrent/newest.html', 'https://torrentqq223.com/ads/'),
70
- ('36dm', 'https://www.36dm.club/1.html', 'https://www.36dm.club/yesterday-1.html'),
71
- ('erai-raws', 'https://www.erai-raws.info/feed/?type=magnet', '403 Forbidden'),
72
- ('teamos', 'https://www.teamos.xyz/torrents/?filename=&freeleech=',
73
- '<title>Log in | Team OS : Your Only Destination To Custom OS !!</title>'),
74
- ('yts', 'https://yts.unblockninja.com/api/v2/list_movies.json?query_term=&limit=50&sort=date_added',
75
- '{"movie_count":')
76
- ]
77
- for site_name, site_url, site_text in sites_get:
78
- with self.subTest(msg=site_name):
79
- res = self.app.post_json('/v1', {
80
- "cmd": "request.get",
81
- "url": site_url
82
- })
83
- asset_cloudflare_solution(self, res, site_url, site_text)
84
-
85
- def test_v1_endpoint_request_post_cloudflare(self):
86
- sites_post = [
87
- ('nnmclub', 'https://nnmclub.to/forum/tracker.php', '<title>Трекер :: NNM-Club</title>',
88
- 'prev_sd=0&prev_a=0&prev_my=0&prev_n=0&prev_shc=0&prev_shf=1&prev_sha=1&prev_shs=0&prev_shr=0&prev_sht=0&f%5B%5D=-1&o=1&s=2&tm=-1&shf=1&sha=1&ta=-1&sns=-1&sds=-1&nm=&pn=&submit=%CF%EE%E8%F1%EA')
89
- ]
90
-
91
- for site_name, site_url, site_text, post_data in sites_post:
92
- with self.subTest(msg=site_name):
93
- res = self.app.post_json('/v1', {
94
- "cmd": "request.post",
95
- "url": site_url,
96
- "postData": post_data
97
- })
98
- asset_cloudflare_solution(self, res, site_url, site_text)
99
-
100
-
101
- if __name__ == '__main__':
102
- unittest.main()
 
1
+ import unittest
2
+
3
+ from webtest import TestApp
4
+
5
+ from dtos import V1ResponseBase, STATUS_OK
6
+ import flaresolverr
7
+ import utils
8
+
9
+
10
+ def _find_obj_by_key(key: str, value: str, _list: list) -> dict | None:
11
+ for obj in _list:
12
+ if obj[key] == value:
13
+ return obj
14
+ return None
15
+
16
+
17
+ def asset_cloudflare_solution(self, res, site_url, site_text):
18
+ self.assertEqual(res.status_code, 200)
19
+
20
+ body = V1ResponseBase(res.json)
21
+ self.assertEqual(STATUS_OK, body.status)
22
+ self.assertEqual("Challenge solved!", body.message)
23
+ self.assertGreater(body.startTimestamp, 10000)
24
+ self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
25
+ self.assertEqual(utils.get_flaresolverr_version(), body.version)
26
+
27
+ solution = body.solution
28
+ self.assertIn(site_url, solution.url)
29
+ self.assertEqual(solution.status, 200)
30
+ self.assertIs(len(solution.headers), 0)
31
+ self.assertIn(site_text, solution.response)
32
+ self.assertGreater(len(solution.cookies), 0)
33
+ self.assertIn("Chrome/", solution.userAgent)
34
+
35
+ cf_cookie = _find_obj_by_key("name", "cf_clearance", solution.cookies)
36
+ self.assertIsNotNone(cf_cookie, "Cloudflare cookie not found")
37
+ self.assertGreater(len(cf_cookie["value"]), 30)
38
+
39
+
40
+ class TestFlareSolverr(unittest.TestCase):
41
+ app = TestApp(flaresolverr.app)
42
+ # wait until the server is ready
43
+ app.get('/')
44
+
45
+ def test_v1_endpoint_request_get_cloudflare(self):
46
+ sites_get = [
47
+ ('nowsecure', 'https://nowsecure.nl', '<title>nowSecure</title>'),
48
+ ('0magnet', 'https://0magnet.com/search?q=2022', 'Torrent Search - ØMagnet'),
49
+ ('1337x', 'https://1337x.unblockit.cat/cat/Movies/time/desc/1/', ''),
50
+ ('avistaz', 'https://avistaz.to/api/v1/jackett/torrents?in=1&type=0&search=',
51
+ '<title>Access denied</title>'),
52
+ ('badasstorrents', 'https://badasstorrents.com/torrents/search/720p/date/desc',
53
+ '<title>Latest Torrents - BadassTorrents</title>'),
54
+ ('bt4g', 'https://bt4g.org/search/2022', '<title>Download 2022 Torrents - BT4G</title>'),
55
+ ('cinemaz', 'https://cinemaz.to/api/v1/jackett/torrents?in=1&type=0&search=',
56
+ '<title>Access denied</title>'),
57
+ ('epublibre', 'https://epublibre.unblockit.cat/catalogo/index/0/nuevo/todos/sin/todos/--/ajax',
58
+ '<title>epublibre - catálogo</title>'),
59
+ ('ext', 'https://ext.to/latest/?order=age&sort=desc',
60
+ '<title>Download Latest Torrents - EXT Torrents</title>'),
61
+ ('extratorrent', 'https://extratorrent.st/search/?srt=added&order=desc&search=720p&new=1&x=0&y=0',
62
+ 'Page 1 - ExtraTorrent'),
63
+ ('idope', 'https://idope.se/browse.html', '<title>Recent Torrents</title>'),
64
+ ('limetorrents', 'https://limetorrents.unblockninja.com/latest100',
65
+ '<title>Latest 100 torrents - LimeTorrents</title>'),
66
+ ('privatehd', 'https://privatehd.to/api/v1/jackett/torrents?in=1&type=0&search=',
67
+ '<title>Access denied</title>'),
68
+ ('torrentcore', 'https://torrentcore.xyz/index', '<title>Torrent[CORE] - Torrent community.</title>'),
69
+ ('torrentqq223', 'https://torrentqq223.com/torrent/newest.html', 'https://torrentqq223.com/ads/'),
70
+ ('36dm', 'https://www.36dm.club/1.html', 'https://www.36dm.club/yesterday-1.html'),
71
+ ('erai-raws', 'https://www.erai-raws.info/feed/?type=magnet', '403 Forbidden'),
72
+ ('teamos', 'https://www.teamos.xyz/torrents/?filename=&freeleech=',
73
+ '<title>Log in | Team OS : Your Only Destination To Custom OS !!</title>'),
74
+ ('yts', 'https://yts.unblockninja.com/api/v2/list_movies.json?query_term=&limit=50&sort=date_added',
75
+ '{"movie_count":')
76
+ ]
77
+ for site_name, site_url, site_text in sites_get:
78
+ with self.subTest(msg=site_name):
79
+ res = self.app.post_json('/v1', {
80
+ "cmd": "request.get",
81
+ "url": site_url
82
+ })
83
+ asset_cloudflare_solution(self, res, site_url, site_text)
84
+
85
+ def test_v1_endpoint_request_post_cloudflare(self):
86
+ sites_post = [
87
+ ('nnmclub', 'https://nnmclub.to/forum/tracker.php', '<title>Трекер :: NNM-Club</title>',
88
+ 'prev_sd=0&prev_a=0&prev_my=0&prev_n=0&prev_shc=0&prev_shf=1&prev_sha=1&prev_shs=0&prev_shr=0&prev_sht=0&f%5B%5D=-1&o=1&s=2&tm=-1&shf=1&sha=1&ta=-1&sns=-1&sds=-1&nm=&pn=&submit=%CF%EE%E8%F1%EA')
89
+ ]
90
+
91
+ for site_name, site_url, site_text, post_data in sites_post:
92
+ with self.subTest(msg=site_name):
93
+ res = self.app.post_json('/v1', {
94
+ "cmd": "request.post",
95
+ "url": site_url,
96
+ "postData": post_data
97
+ })
98
+ asset_cloudflare_solution(self, res, site_url, site_text)
99
+
100
+
101
+ if __name__ == '__main__':
102
+ unittest.main()
src/undetected_chromedriver/__init__.py CHANGED
@@ -1,914 +1,914 @@
1
- #!/usr/bin/env python3
2
-
3
- """
4
-
5
- 888 888 d8b
6
- 888 888 Y8P
7
- 888 888
8
- .d8888b 88888b. 888d888 .d88b. 88888b.d88b. .d88b. .d88888 888d888 888 888 888 .d88b. 888d888
9
- d88P" 888 "88b 888P" d88""88b 888 "888 "88b d8P Y8b d88" 888 888P" 888 888 888 d8P Y8b 888P"
10
- 888 888 888 888 888 888 888 888 888 88888888 888 888 888 888 Y88 88P 88888888 888
11
- Y88b. 888 888 888 Y88..88P 888 888 888 Y8b. Y88b 888 888 888 Y8bd8P Y8b. 888
12
- "Y8888P 888 888 888 "Y88P" 888 888 888 "Y8888 "Y88888 888 888 Y88P "Y8888 888 88888888
13
-
14
- by UltrafunkAmsterdam (https://github.com/ultrafunkamsterdam)
15
-
16
- """
17
- from __future__ import annotations
18
-
19
-
20
- __version__ = "3.5.5"
21
-
22
- import json
23
- import logging
24
- import os
25
- import pathlib
26
- import re
27
- import shutil
28
- import subprocess
29
- import sys
30
- import tempfile
31
- import time
32
- from weakref import finalize
33
-
34
- import selenium.webdriver.chrome.service
35
- import selenium.webdriver.chrome.webdriver
36
- from selenium.webdriver.common.by import By
37
- import selenium.webdriver.chromium.service
38
- import selenium.webdriver.remote.command
39
- import selenium.webdriver.remote.webdriver
40
-
41
- from .cdp import CDP
42
- from .dprocess import start_detached
43
- from .options import ChromeOptions
44
- from .patcher import IS_POSIX
45
- from .patcher import Patcher
46
- from .reactor import Reactor
47
- from .webelement import UCWebElement
48
- from .webelement import WebElement
49
-
50
-
51
- __all__ = (
52
- "Chrome",
53
- "ChromeOptions",
54
- "Patcher",
55
- "Reactor",
56
- "CDP",
57
- "find_chrome_executable",
58
- )
59
-
60
- logger = logging.getLogger("uc")
61
- logger.setLevel(logging.getLogger().getEffectiveLevel())
62
-
63
-
64
- class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
65
- """
66
-
67
- Controls the ChromeDriver and allows you to drive the browser.
68
-
69
- The webdriver file will be downloaded by this module automatically,
70
- you do not need to specify this. however, you may if you wish.
71
-
72
- Attributes
73
- ----------
74
-
75
- Methods
76
- -------
77
-
78
- reconnect()
79
-
80
- this can be useful in case of heavy detection methods
81
- -stops the chromedriver service which runs in the background
82
- -starts the chromedriver service which runs in the background
83
- -recreate session
84
-
85
-
86
- start_session(capabilities=None, browser_profile=None)
87
-
88
- differentiates from the regular method in that it does not
89
- require a capabilities argument. The capabilities are automatically
90
- recreated from the options at creation time.
91
-
92
- --------------------------------------------------------------------------
93
- NOTE:
94
- Chrome has everything included to work out of the box.
95
- it does not `need` customizations.
96
- any customizations MAY lead to trigger bot migitation systems.
97
-
98
- --------------------------------------------------------------------------
99
- """
100
-
101
- _instances = set()
102
- session_id = None
103
- debug = False
104
-
105
- def __init__(
106
- self,
107
- options=None,
108
- user_data_dir=None,
109
- driver_executable_path=None,
110
- browser_executable_path=None,
111
- port=0,
112
- enable_cdp_events=False,
113
- # service_args=None,
114
- # service_creationflags=None,
115
- desired_capabilities=None,
116
- advanced_elements=False,
117
- # service_log_path=None,
118
- keep_alive=True,
119
- log_level=0,
120
- headless=False,
121
- version_main=None,
122
- patcher_force_close=False,
123
- suppress_welcome=True,
124
- use_subprocess=False,
125
- debug=False,
126
- no_sandbox=True,
127
- windows_headless=False,
128
- user_multi_procs: bool = False,
129
- **kw,
130
- ):
131
- """
132
- Creates a new instance of the chrome driver.
133
-
134
- Starts the service and then creates new instance of chrome driver.
135
-
136
- Parameters
137
- ----------
138
-
139
- options: ChromeOptions, optional, default: None - automatic useful defaults
140
- this takes an instance of ChromeOptions, mainly to customize browser behavior.
141
- anything other dan the default, for example extensions or startup options
142
- are not supported in case of failure, and can probably lowers your undetectability.
143
-
144
-
145
- user_data_dir: str , optional, default: None (creates temp profile)
146
- if user_data_dir is a path to a valid chrome profile directory, use it,
147
- and turn off automatic removal mechanism at exit.
148
-
149
- driver_executable_path: str, optional, default: None(=downloads and patches new binary)
150
-
151
- browser_executable_path: str, optional, default: None - use find_chrome_executable
152
- Path to the browser executable.
153
- If not specified, make sure the executable's folder is in $PATH
154
-
155
- port: int, optional, default: 0
156
- port to be used by the chromedriver executable, this is NOT the debugger port.
157
- leave it at 0 unless you know what you are doing.
158
- the default value of 0 automatically picks an available port.
159
-
160
- enable_cdp_events: bool, default: False
161
- :: currently for chrome only
162
- this enables the handling of wire messages
163
- when enabled, you can subscribe to CDP events by using:
164
-
165
- driver.add_cdp_listener("Network.dataReceived", yourcallback)
166
- # yourcallback is an callable which accepts exactly 1 dict as parameter
167
-
168
-
169
- service_args: list of str, optional, default: None
170
- arguments to pass to the driver service
171
-
172
- desired_capabilities: dict, optional, default: None - auto from config
173
- Dictionary object with non-browser specific capabilities only, such as "item" or "loggingPref".
174
-
175
- advanced_elements: bool, optional, default: False
176
- makes it easier to recognize elements like you know them from html/browser inspection, especially when working
177
- in an interactive environment
178
-
179
- default webelement repr:
180
- <selenium.webdriver.remote.webelement.WebElement (session="85ff0f671512fa535630e71ee951b1f2", element="6357cb55-92c3-4c0f-9416-b174f9c1b8c4")>
181
-
182
- advanced webelement repr
183
- <WebElement(<a class="mobile-show-inline-block mc-update-infos init-ok" href="#" id="main-cat-switcher-mobile">)>
184
-
185
- note: when retrieving large amounts of elements ( example: find_elements_by_tag("*") ) and print them, it does take a little more time.
186
-
187
-
188
- service_log_path: str, optional, default: None
189
- path to log information from the driver.
190
-
191
- keep_alive: bool, optional, default: True
192
- Whether to configure ChromeRemoteConnection to use HTTP keep-alive.
193
-
194
- log_level: int, optional, default: adapts to python global log level
195
-
196
- headless: bool, optional, default: False
197
- can also be specified in the options instance.
198
- Specify whether you want to use the browser in headless mode.
199
- warning: this lowers undetectability and not fully supported.
200
-
201
- version_main: int, optional, default: None (=auto)
202
- if you, for god knows whatever reason, use
203
- an older version of Chrome. You can specify it's full rounded version number
204
- here. Example: 87 for all versions of 87
205
-
206
- patcher_force_close: bool, optional, default: False
207
- instructs the patcher to do whatever it can to access the chromedriver binary
208
- if the file is locked, it will force shutdown all instances.
209
- setting it is not recommended, unless you know the implications and think
210
- you might need it.
211
-
212
- suppress_welcome: bool, optional , default: True
213
- a "welcome" alert might show up on *nix-like systems asking whether you want to set
214
- chrome as your default browser, and if you want to send even more data to google.
215
- now, in case you are nag-fetishist, or a diagnostics data feeder to google, you can set this to False.
216
- Note: if you don't handle the nag screen in time, the browser loses it's connection and throws an Exception.
217
-
218
- use_subprocess: bool, optional , default: True,
219
-
220
- False (the default) makes sure Chrome will get it's own process (so no subprocess of chromedriver.exe or python
221
- This fixes a LOT of issues, like multithreaded run, but mst importantly. shutting corectly after
222
- program exits or using .quit()
223
- you should be knowing what you're doing, and know how python works.
224
-
225
- unfortunately, there is always an edge case in which one would like to write an single script with the only contents being:
226
- --start script--
227
- import undetected_chromedriver as uc
228
- d = uc.Chrome()
229
- d.get('https://somesite/')
230
- ---end script --
231
-
232
- and will be greeted with an error, since the program exists before chrome has a change to launch.
233
- in that case you can set this to `True`. The browser will start via subprocess, and will keep running most of times.
234
- ! setting it to True comes with NO support when being detected. !
235
-
236
- no_sandbox: bool, optional, default=True
237
- uses the --no-sandbox option, and additionally does suppress the "unsecure option" status bar
238
- this option has a default of True since many people seem to run this as root (....) , and chrome does not start
239
- when running as root without using --no-sandbox flag.
240
-
241
- user_multi_procs:
242
- set to true when you are using multithreads/multiprocessing
243
- ensures not all processes are trying to modify a binary which is in use by another.
244
- for this to work. YOU MUST HAVE AT LEAST 1 UNDETECTED_CHROMEDRIVER BINARY IN YOUR ROAMING DATA FOLDER.
245
- this requirement can be easily satisfied, by just running this program "normal" and close/kill it.
246
-
247
-
248
- """
249
-
250
- finalize(self, self._ensure_close, self)
251
- self.debug = debug
252
- self.patcher = Patcher(
253
- executable_path=driver_executable_path,
254
- force=patcher_force_close,
255
- version_main=version_main,
256
- user_multi_procs=user_multi_procs,
257
- )
258
- # self.patcher.auto(user_multiprocess = user_multi_num_procs)
259
- self.patcher.auto()
260
-
261
- # self.patcher = patcher
262
- if not options:
263
- options = ChromeOptions()
264
-
265
- try:
266
- if hasattr(options, "_session") and options._session is not None:
267
- # prevent reuse of options,
268
- # as it just appends arguments, not replace them
269
- # you'll get conflicts starting chrome
270
- raise RuntimeError("you cannot reuse the ChromeOptions object")
271
- except AttributeError:
272
- pass
273
-
274
- options._session = self
275
-
276
- if not options.debugger_address:
277
- debug_port = (
278
- port
279
- if port != 0
280
- else selenium.webdriver.common.service.utils.free_port()
281
- )
282
- debug_host = "127.0.0.1"
283
- options.debugger_address = "%s:%d" % (debug_host, debug_port)
284
- else:
285
- debug_host, debug_port = options.debugger_address.split(":")
286
- debug_port = int(debug_port)
287
-
288
- if enable_cdp_events:
289
- options.set_capability(
290
- "goog:loggingPrefs", {"performance": "ALL", "browser": "ALL"}
291
- )
292
-
293
- options.add_argument("--remote-debugging-host=%s" % debug_host)
294
- options.add_argument("--remote-debugging-port=%s" % debug_port)
295
-
296
- if user_data_dir:
297
- options.add_argument("--user-data-dir=%s" % user_data_dir)
298
-
299
- language, keep_user_data_dir = None, bool(user_data_dir)
300
-
301
- # see if a custom user profile is specified in options
302
- for arg in options.arguments:
303
-
304
- if any([_ in arg for _ in ("--headless", "headless")]):
305
- options.arguments.remove(arg)
306
- options.headless = True
307
-
308
- if "lang" in arg:
309
- m = re.search("(?:--)?lang(?:[ =])?(.*)", arg)
310
- try:
311
- language = m[1]
312
- except IndexError:
313
- logger.debug("will set the language to en-US,en;q=0.9")
314
- language = "en-US,en;q=0.9"
315
-
316
- if "user-data-dir" in arg:
317
- m = re.search("(?:--)?user-data-dir(?:[ =])?(.*)", arg)
318
- try:
319
- user_data_dir = m[1]
320
- logger.debug(
321
- "user-data-dir found in user argument %s => %s" % (arg, m[1])
322
- )
323
- keep_user_data_dir = True
324
-
325
- except IndexError:
326
- logger.debug(
327
- "no user data dir could be extracted from supplied argument %s "
328
- % arg
329
- )
330
-
331
- if not user_data_dir:
332
- # backward compatiblity
333
- # check if an old uc.ChromeOptions is used, and extract the user data dir
334
-
335
- if hasattr(options, "user_data_dir") and getattr(
336
- options, "user_data_dir", None
337
- ):
338
- import warnings
339
-
340
- warnings.warn(
341
- "using ChromeOptions.user_data_dir might stop working in future versions."
342
- "use uc.Chrome(user_data_dir='/xyz/some/data') in case you need existing profile folder"
343
- )
344
- options.add_argument("--user-data-dir=%s" % options.user_data_dir)
345
- keep_user_data_dir = True
346
- logger.debug(
347
- "user_data_dir property found in options object: %s" % user_data_dir
348
- )
349
-
350
- else:
351
- user_data_dir = os.path.normpath(tempfile.mkdtemp())
352
- keep_user_data_dir = False
353
- arg = "--user-data-dir=%s" % user_data_dir
354
- options.add_argument(arg)
355
- logger.debug(
356
- "created a temporary folder in which the user-data (profile) will be stored during this\n"
357
- "session, and added it to chrome startup arguments: %s" % arg
358
- )
359
-
360
- if not language:
361
- try:
362
- import locale
363
-
364
- language = locale.getdefaultlocale()[0].replace("_", "-")
365
- except Exception:
366
- pass
367
- if not language:
368
- language = "en-US"
369
-
370
- options.add_argument("--lang=%s" % language)
371
-
372
- if not options.binary_location:
373
- options.binary_location = (
374
- browser_executable_path or find_chrome_executable()
375
- )
376
-
377
- if not options.binary_location or not \
378
- pathlib.Path(options.binary_location).exists():
379
- raise FileNotFoundError(
380
- "\n---------------------\n"
381
- "Could not determine browser executable."
382
- "\n---------------------\n"
383
- "Make sure your browser is installed in the default location (path).\n"
384
- "If you are sure about the browser executable, you can specify it using\n"
385
- "the `browser_executable_path='{}` parameter.\n\n"
386
- .format("/path/to/browser/executable" if IS_POSIX else "c:/path/to/your/browser.exe")
387
- )
388
-
389
- self._delay = 3
390
-
391
- self.user_data_dir = user_data_dir
392
- self.keep_user_data_dir = keep_user_data_dir
393
-
394
- if suppress_welcome:
395
- options.arguments.extend(["--no-default-browser-check", "--no-first-run"])
396
- if no_sandbox:
397
- options.arguments.extend(["--no-sandbox", "--test-type"])
398
-
399
- if headless or getattr(options, 'headless', None):
400
- #workaround until a better checking is found
401
- try:
402
- v_main = int(self.patcher.version_main) if self.patcher.version_main else 108
403
- if v_main < 108:
404
- options.add_argument("--headless=chrome")
405
- elif v_main >= 108:
406
- options.add_argument("--headless=new")
407
- except:
408
- logger.warning("could not detect version_main."
409
- "therefore, we are assuming it is chrome 108 or higher")
410
- options.add_argument("--headless=new")
411
-
412
- options.add_argument("--window-size=1920,1080")
413
- options.add_argument("--start-maximized")
414
- options.add_argument("--no-sandbox")
415
- # fixes "could not connect to chrome" error when running
416
- # on linux using privileged user like root (which i don't recommend)
417
-
418
- options.add_argument(
419
- "--log-level=%d" % log_level
420
- or divmod(logging.getLogger().getEffectiveLevel(), 10)[0]
421
- )
422
-
423
- if hasattr(options, "handle_prefs"):
424
- options.handle_prefs(user_data_dir)
425
-
426
- # fix exit_type flag to prevent tab-restore nag
427
- try:
428
- with open(
429
- os.path.join(user_data_dir, "Default/Preferences"),
430
- encoding="latin1",
431
- mode="r+",
432
- ) as fs:
433
- config = json.load(fs)
434
- if config["profile"]["exit_type"] is not None:
435
- # fixing the restore-tabs-nag
436
- config["profile"]["exit_type"] = None
437
- fs.seek(0, 0)
438
- json.dump(config, fs)
439
- fs.truncate() # the file might be shorter
440
- logger.debug("fixed exit_type flag")
441
- except Exception as e:
442
- logger.debug("did not find a bad exit_type flag ")
443
-
444
- self.options = options
445
-
446
- if not desired_capabilities:
447
- desired_capabilities = options.to_capabilities()
448
-
449
- if not use_subprocess and not windows_headless:
450
- self.browser_pid = start_detached(
451
- options.binary_location, *options.arguments
452
- )
453
- else:
454
- startupinfo = None
455
- if os.name == 'nt' and windows_headless:
456
- # STARTUPINFO() is Windows only
457
- startupinfo = subprocess.STARTUPINFO()
458
- startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
459
- browser = subprocess.Popen(
460
- [options.binary_location, *options.arguments],
461
- stdin=subprocess.PIPE,
462
- stdout=subprocess.PIPE,
463
- stderr=subprocess.PIPE,
464
- close_fds=IS_POSIX,
465
- startupinfo=startupinfo
466
- )
467
- self.browser_pid = browser.pid
468
-
469
-
470
- service = selenium.webdriver.chromium.service.ChromiumService(
471
- self.patcher.executable_path
472
- )
473
-
474
- super(Chrome, self).__init__(
475
- service=service,
476
- options=options,
477
- keep_alive=keep_alive,
478
- )
479
-
480
- self.reactor = None
481
-
482
- if enable_cdp_events:
483
- if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
484
- logging.getLogger(
485
- "selenium.webdriver.remote.remote_connection"
486
- ).setLevel(20)
487
- reactor = Reactor(self)
488
- reactor.start()
489
- self.reactor = reactor
490
-
491
- if advanced_elements:
492
- self._web_element_cls = UCWebElement
493
- else:
494
- self._web_element_cls = WebElement
495
-
496
- if headless or getattr(options, 'headless', None):
497
- self._configure_headless()
498
-
499
- def _configure_headless(self):
500
- orig_get = self.get
501
- logger.info("setting properties for headless")
502
-
503
- def get_wrapped(*args, **kwargs):
504
- if self.execute_script("return navigator.webdriver"):
505
- logger.info("patch navigator.webdriver")
506
- self.execute_cdp_cmd(
507
- "Page.addScriptToEvaluateOnNewDocument",
508
- {
509
- "source": """
510
-
511
- Object.defineProperty(window, "navigator", {
512
- Object.defineProperty(window, "navigator", {
513
- value: new Proxy(navigator, {
514
- has: (target, key) => (key === "webdriver" ? false : key in target),
515
- get: (target, key) =>
516
- key === "webdriver"
517
- ? false
518
- : typeof target[key] === "function"
519
- ? target[key].bind(target)
520
- : target[key],
521
- }),
522
- });
523
- """
524
- },
525
- )
526
-
527
- logger.info("patch user-agent string")
528
- self.execute_cdp_cmd(
529
- "Network.setUserAgentOverride",
530
- {
531
- "userAgent": self.execute_script(
532
- "return navigator.userAgent"
533
- ).replace("Headless", "")
534
- },
535
- )
536
- self.execute_cdp_cmd(
537
- "Page.addScriptToEvaluateOnNewDocument",
538
- {
539
- "source": """
540
- Object.defineProperty(navigator, 'maxTouchPoints', {get: () => 1});
541
- Object.defineProperty(navigator.connection, 'rtt', {get: () => 100});
542
-
543
- // https://github.com/microlinkhq/browserless/blob/master/packages/goto/src/evasions/chrome-runtime.js
544
- window.chrome = {
545
- app: {
546
- isInstalled: false,
547
- InstallState: {
548
- DISABLED: 'disabled',
549
- INSTALLED: 'installed',
550
- NOT_INSTALLED: 'not_installed'
551
- },
552
- RunningState: {
553
- CANNOT_RUN: 'cannot_run',
554
- READY_TO_RUN: 'ready_to_run',
555
- RUNNING: 'running'
556
- }
557
- },
558
- runtime: {
559
- OnInstalledReason: {
560
- CHROME_UPDATE: 'chrome_update',
561
- INSTALL: 'install',
562
- SHARED_MODULE_UPDATE: 'shared_module_update',
563
- UPDATE: 'update'
564
- },
565
- OnRestartRequiredReason: {
566
- APP_UPDATE: 'app_update',
567
- OS_UPDATE: 'os_update',
568
- PERIODIC: 'periodic'
569
- },
570
- PlatformArch: {
571
- ARM: 'arm',
572
- ARM64: 'arm64',
573
- MIPS: 'mips',
574
- MIPS64: 'mips64',
575
- X86_32: 'x86-32',
576
- X86_64: 'x86-64'
577
- },
578
- PlatformNaclArch: {
579
- ARM: 'arm',
580
- MIPS: 'mips',
581
- MIPS64: 'mips64',
582
- X86_32: 'x86-32',
583
- X86_64: 'x86-64'
584
- },
585
- PlatformOs: {
586
- ANDROID: 'android',
587
- CROS: 'cros',
588
- LINUX: 'linux',
589
- MAC: 'mac',
590
- OPENBSD: 'openbsd',
591
- WIN: 'win'
592
- },
593
- RequestUpdateCheckStatus: {
594
- NO_UPDATE: 'no_update',
595
- THROTTLED: 'throttled',
596
- UPDATE_AVAILABLE: 'update_available'
597
- }
598
- }
599
- }
600
-
601
- // https://github.com/microlinkhq/browserless/blob/master/packages/goto/src/evasions/navigator-permissions.js
602
- if (!window.Notification) {
603
- window.Notification = {
604
- permission: 'denied'
605
- }
606
- }
607
-
608
- const originalQuery = window.navigator.permissions.query
609
- window.navigator.permissions.__proto__.query = parameters =>
610
- parameters.name === 'notifications'
611
- ? Promise.resolve({ state: window.Notification.permission })
612
- : originalQuery(parameters)
613
-
614
- const oldCall = Function.prototype.call
615
- function call() {
616
- return oldCall.apply(this, arguments)
617
- }
618
- Function.prototype.call = call
619
-
620
- const nativeToStringFunctionString = Error.toString().replace(/Error/g, 'toString')
621
- const oldToString = Function.prototype.toString
622
-
623
- function functionToString() {
624
- if (this === window.navigator.permissions.query) {
625
- return 'function query() { [native code] }'
626
- }
627
- if (this === functionToString) {
628
- return nativeToStringFunctionString
629
- }
630
- return oldCall.call(oldToString, this)
631
- }
632
- // eslint-disable-next-line
633
- Function.prototype.toString = functionToString
634
- """
635
- },
636
- )
637
- return orig_get(*args, **kwargs)
638
-
639
- self.get = get_wrapped
640
-
641
- # def _get_cdc_props(self):
642
- # return self.execute_script(
643
- # """
644
- # let objectToInspect = window,
645
- # result = [];
646
- # while(objectToInspect !== null)
647
- # { result = result.concat(Object.getOwnPropertyNames(objectToInspect));
648
- # objectToInspect = Object.getPrototypeOf(objectToInspect); }
649
- #
650
- # return result.filter(i => i.match(/^([a-zA-Z]){27}(Array|Promise|Symbol)$/ig))
651
- # """
652
- # )
653
- #
654
- # def _hook_remove_cdc_props(self):
655
- # self.execute_cdp_cmd(
656
- # "Page.addScriptToEvaluateOnNewDocument",
657
- # {
658
- # "source": """
659
- # let objectToInspect = window,
660
- # result = [];
661
- # while(objectToInspect !== null)
662
- # { result = result.concat(Object.getOwnPropertyNames(objectToInspect));
663
- # objectToInspect = Object.getPrototypeOf(objectToInspect); }
664
- # result.forEach(p => p.match(/^([a-zA-Z]){27}(Array|Promise|Symbol)$/ig)
665
- # &&delete window[p]&&console.log('removed',p))
666
- # """
667
- # },
668
- # )
669
-
670
- def get(self, url):
671
- # if self._get_cdc_props():
672
- # self._hook_remove_cdc_props()
673
- return super().get(url)
674
-
675
- def add_cdp_listener(self, event_name, callback):
676
- if (
677
- self.reactor
678
- and self.reactor is not None
679
- and isinstance(self.reactor, Reactor)
680
- ):
681
- self.reactor.add_event_handler(event_name, callback)
682
- return self.reactor.handlers
683
- return False
684
-
685
- def clear_cdp_listeners(self):
686
- if self.reactor and isinstance(self.reactor, Reactor):
687
- self.reactor.handlers.clear()
688
-
689
- def window_new(self):
690
- self.execute(
691
- selenium.webdriver.remote.command.Command.NEW_WINDOW, {"type": "window"}
692
- )
693
-
694
- def tab_new(self, url: str):
695
- """
696
- this opens a url in a new tab.
697
- apparently, that passes all tests directly!
698
-
699
- Parameters
700
- ----------
701
- url
702
-
703
- Returns
704
- -------
705
-
706
- """
707
- if not hasattr(self, "cdp"):
708
- from .cdp import CDP
709
-
710
- cdp = CDP(self.options)
711
- cdp.tab_new(url)
712
-
713
- def reconnect(self, timeout=0.1):
714
- try:
715
- self.service.stop()
716
- except Exception as e:
717
- logger.debug(e)
718
- time.sleep(timeout)
719
- try:
720
- self.service.start()
721
- except Exception as e:
722
- logger.debug(e)
723
-
724
- try:
725
- self.start_session()
726
- except Exception as e:
727
- logger.debug(e)
728
-
729
- def start_session(self, capabilities=None, browser_profile=None):
730
- if not capabilities:
731
- capabilities = self.options.to_capabilities()
732
- super(selenium.webdriver.chrome.webdriver.WebDriver, self).start_session(
733
- capabilities
734
- )
735
- # super(Chrome, self).start_session(capabilities, browser_profile)
736
-
737
- def find_elements_recursive(self, by, value):
738
- """
739
- find elements in all frames
740
- this is a generator function, which is needed
741
- since if it would return a list of elements, they
742
- will be stale on arrival.
743
- using generator, when the element is returned we are in the correct frame
744
- to use it directly
745
- Args:
746
- by: By
747
- value: str
748
- Returns: Generator[webelement.WebElement]
749
- """
750
- def search_frame(f=None):
751
- if not f:
752
- # ensure we are on main content frame
753
- self.switch_to.default_content()
754
- else:
755
- self.switch_to.frame(f)
756
- for elem in self.find_elements(by, value):
757
- yield elem
758
- # switch back to main content, otherwise we will get StaleElementReferenceException
759
- self.switch_to.default_content()
760
-
761
- # search root frame
762
- for elem in search_frame():
763
- yield elem
764
- # get iframes
765
- frames = self.find_elements('css selector', 'iframe')
766
-
767
- # search per frame
768
- for f in frames:
769
- for elem in search_frame(f):
770
- yield elem
771
-
772
- def quit(self):
773
- try:
774
- self.service.stop()
775
- self.service.process.kill()
776
- self.command_executor.close()
777
- self.service.process.wait(5)
778
- logger.debug("webdriver process ended")
779
- except (AttributeError, RuntimeError, OSError):
780
- pass
781
- try:
782
- self.reactor.event.set()
783
- logger.debug("shutting down reactor")
784
- except AttributeError:
785
- pass
786
- try:
787
- os.kill(self.browser_pid, 15)
788
- logger.debug("gracefully closed browser")
789
- except Exception as e: # noqa
790
- pass
791
- if (
792
- hasattr(self, "keep_user_data_dir")
793
- and hasattr(self, "user_data_dir")
794
- and not self.keep_user_data_dir
795
- ):
796
- for _ in range(5):
797
- try:
798
- shutil.rmtree(self.user_data_dir, ignore_errors=False)
799
- except FileNotFoundError:
800
- pass
801
- except (RuntimeError, OSError, PermissionError) as e:
802
- logger.debug(
803
- "When removing the temp profile, a %s occured: %s\nretrying..."
804
- % (e.__class__.__name__, e)
805
- )
806
- else:
807
- logger.debug("successfully removed %s" % self.user_data_dir)
808
- break
809
-
810
- try:
811
- time.sleep(0.1)
812
- except OSError:
813
- pass
814
-
815
- # dereference patcher, so patcher can start cleaning up as well.
816
- # this must come last, otherwise it will throw 'in use' errors
817
- self.patcher = None
818
-
819
- def __getattribute__(self, item):
820
- if not super().__getattribute__("debug"):
821
- return super().__getattribute__(item)
822
- else:
823
- import inspect
824
-
825
- original = super().__getattribute__(item)
826
- if inspect.ismethod(original) and not inspect.isclass(original):
827
-
828
- def newfunc(*args, **kwargs):
829
- logger.debug(
830
- "calling %s with args %s and kwargs %s\n"
831
- % (original.__qualname__, args, kwargs)
832
- )
833
- return original(*args, **kwargs)
834
-
835
- return newfunc
836
- return original
837
-
838
- def __enter__(self):
839
- return self
840
-
841
- def __exit__(self, exc_type, exc_val, exc_tb):
842
- self.service.stop()
843
- time.sleep(self._delay)
844
- self.service.start()
845
- self.start_session()
846
-
847
- def __hash__(self):
848
- return hash(self.options.debugger_address)
849
-
850
- def __dir__(self):
851
- return object.__dir__(self)
852
-
853
- def __del__(self):
854
- try:
855
- self.service.process.kill()
856
- except: # noqa
857
- pass
858
- self.quit()
859
-
860
- @classmethod
861
- def _ensure_close(cls, self):
862
- # needs to be a classmethod so finalize can find the reference
863
- logger.info("ensuring close")
864
- if (
865
- hasattr(self, "service")
866
- and hasattr(self.service, "process")
867
- and hasattr(self.service.process, "kill")
868
- ):
869
- self.service.process.kill()
870
-
871
-
872
- def find_chrome_executable():
873
- """
874
- Finds the chrome, chrome beta, chrome canary, chromium executable
875
-
876
- Returns
877
- -------
878
- executable_path : str
879
- the full file path to found executable
880
-
881
- """
882
- candidates = set()
883
- if IS_POSIX:
884
- for item in os.environ.get("PATH").split(os.pathsep):
885
- for subitem in (
886
- "google-chrome",
887
- "chromium",
888
- "chromium-browser",
889
- "chrome",
890
- "google-chrome-stable",
891
- ):
892
- candidates.add(os.sep.join((item, subitem)))
893
- if "darwin" in sys.platform:
894
- candidates.update(
895
- [
896
- "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
897
- "/Applications/Chromium.app/Contents/MacOS/Chromium",
898
- ]
899
- )
900
- else:
901
- for item in map(
902
- os.environ.get,
903
- ("PROGRAMFILES", "PROGRAMFILES(X86)", "LOCALAPPDATA", "PROGRAMW6432"),
904
- ):
905
- if item is not None:
906
- for subitem in (
907
- "Google/Chrome/Application",
908
- ):
909
- candidates.add(os.sep.join((item, subitem, "chrome.exe")))
910
- for candidate in candidates:
911
- logger.debug('checking if %s exists and is executable' % candidate)
912
- if os.path.exists(candidate) and os.access(candidate, os.X_OK):
913
- logger.debug('found! using %s' % candidate)
914
- return os.path.normpath(candidate)
 
1
+ #!/usr/bin/env python3
2
+
3
+ """
4
+
5
+ 888 888 d8b
6
+ 888 888 Y8P
7
+ 888 888
8
+ .d8888b 88888b. 888d888 .d88b. 88888b.d88b. .d88b. .d88888 888d888 888 888 888 .d88b. 888d888
9
+ d88P" 888 "88b 888P" d88""88b 888 "888 "88b d8P Y8b d88" 888 888P" 888 888 888 d8P Y8b 888P"
10
+ 888 888 888 888 888 888 888 888 888 88888888 888 888 888 888 Y88 88P 88888888 888
11
+ Y88b. 888 888 888 Y88..88P 888 888 888 Y8b. Y88b 888 888 888 Y8bd8P Y8b. 888
12
+ "Y8888P 888 888 888 "Y88P" 888 888 888 "Y8888 "Y88888 888 888 Y88P "Y8888 888 88888888
13
+
14
+ by UltrafunkAmsterdam (https://github.com/ultrafunkamsterdam)
15
+
16
+ """
17
+ from __future__ import annotations
18
+
19
+
20
+ __version__ = "3.5.5"
21
+
22
+ import json
23
+ import logging
24
+ import os
25
+ import pathlib
26
+ import re
27
+ import shutil
28
+ import subprocess
29
+ import sys
30
+ import tempfile
31
+ import time
32
+ from weakref import finalize
33
+
34
+ import selenium.webdriver.chrome.service
35
+ import selenium.webdriver.chrome.webdriver
36
+ from selenium.webdriver.common.by import By
37
+ import selenium.webdriver.chromium.service
38
+ import selenium.webdriver.remote.command
39
+ import selenium.webdriver.remote.webdriver
40
+
41
+ from .cdp import CDP
42
+ from .dprocess import start_detached
43
+ from .options import ChromeOptions
44
+ from .patcher import IS_POSIX
45
+ from .patcher import Patcher
46
+ from .reactor import Reactor
47
+ from .webelement import UCWebElement
48
+ from .webelement import WebElement
49
+
50
+
51
+ __all__ = (
52
+ "Chrome",
53
+ "ChromeOptions",
54
+ "Patcher",
55
+ "Reactor",
56
+ "CDP",
57
+ "find_chrome_executable",
58
+ )
59
+
60
+ logger = logging.getLogger("uc")
61
+ logger.setLevel(logging.getLogger().getEffectiveLevel())
62
+
63
+
64
+ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
65
+ """
66
+
67
+ Controls the ChromeDriver and allows you to drive the browser.
68
+
69
+ The webdriver file will be downloaded by this module automatically,
70
+ you do not need to specify this. however, you may if you wish.
71
+
72
+ Attributes
73
+ ----------
74
+
75
+ Methods
76
+ -------
77
+
78
+ reconnect()
79
+
80
+ this can be useful in case of heavy detection methods
81
+ -stops the chromedriver service which runs in the background
82
+ -starts the chromedriver service which runs in the background
83
+ -recreate session
84
+
85
+
86
+ start_session(capabilities=None, browser_profile=None)
87
+
88
+ differentiates from the regular method in that it does not
89
+ require a capabilities argument. The capabilities are automatically
90
+ recreated from the options at creation time.
91
+
92
+ --------------------------------------------------------------------------
93
+ NOTE:
94
+ Chrome has everything included to work out of the box.
95
+ it does not `need` customizations.
96
+ any customizations MAY lead to trigger bot migitation systems.
97
+
98
+ --------------------------------------------------------------------------
99
+ """
100
+
101
+ _instances = set()
102
+ session_id = None
103
+ debug = False
104
+
105
+ def __init__(
106
+ self,
107
+ options=None,
108
+ user_data_dir=None,
109
+ driver_executable_path=None,
110
+ browser_executable_path=None,
111
+ port=0,
112
+ enable_cdp_events=False,
113
+ # service_args=None,
114
+ # service_creationflags=None,
115
+ desired_capabilities=None,
116
+ advanced_elements=False,
117
+ # service_log_path=None,
118
+ keep_alive=True,
119
+ log_level=0,
120
+ headless=False,
121
+ version_main=None,
122
+ patcher_force_close=False,
123
+ suppress_welcome=True,
124
+ use_subprocess=False,
125
+ debug=False,
126
+ no_sandbox=True,
127
+ windows_headless=False,
128
+ user_multi_procs: bool = False,
129
+ **kw,
130
+ ):
131
+ """
132
+ Creates a new instance of the chrome driver.
133
+
134
+ Starts the service and then creates new instance of chrome driver.
135
+
136
+ Parameters
137
+ ----------
138
+
139
+ options: ChromeOptions, optional, default: None - automatic useful defaults
140
+ this takes an instance of ChromeOptions, mainly to customize browser behavior.
141
+ anything other dan the default, for example extensions or startup options
142
+ are not supported in case of failure, and can probably lowers your undetectability.
143
+
144
+
145
+ user_data_dir: str , optional, default: None (creates temp profile)
146
+ if user_data_dir is a path to a valid chrome profile directory, use it,
147
+ and turn off automatic removal mechanism at exit.
148
+
149
+ driver_executable_path: str, optional, default: None(=downloads and patches new binary)
150
+
151
+ browser_executable_path: str, optional, default: None - use find_chrome_executable
152
+ Path to the browser executable.
153
+ If not specified, make sure the executable's folder is in $PATH
154
+
155
+ port: int, optional, default: 0
156
+ port to be used by the chromedriver executable, this is NOT the debugger port.
157
+ leave it at 0 unless you know what you are doing.
158
+ the default value of 0 automatically picks an available port.
159
+
160
+ enable_cdp_events: bool, default: False
161
+ :: currently for chrome only
162
+ this enables the handling of wire messages
163
+ when enabled, you can subscribe to CDP events by using:
164
+
165
+ driver.add_cdp_listener("Network.dataReceived", yourcallback)
166
+ # yourcallback is an callable which accepts exactly 1 dict as parameter
167
+
168
+
169
+ service_args: list of str, optional, default: None
170
+ arguments to pass to the driver service
171
+
172
+ desired_capabilities: dict, optional, default: None - auto from config
173
+ Dictionary object with non-browser specific capabilities only, such as "item" or "loggingPref".
174
+
175
+ advanced_elements: bool, optional, default: False
176
+ makes it easier to recognize elements like you know them from html/browser inspection, especially when working
177
+ in an interactive environment
178
+
179
+ default webelement repr:
180
+ <selenium.webdriver.remote.webelement.WebElement (session="85ff0f671512fa535630e71ee951b1f2", element="6357cb55-92c3-4c0f-9416-b174f9c1b8c4")>
181
+
182
+ advanced webelement repr
183
+ <WebElement(<a class="mobile-show-inline-block mc-update-infos init-ok" href="#" id="main-cat-switcher-mobile">)>
184
+
185
+ note: when retrieving large amounts of elements ( example: find_elements_by_tag("*") ) and print them, it does take a little more time.
186
+
187
+
188
+ service_log_path: str, optional, default: None
189
+ path to log information from the driver.
190
+
191
+ keep_alive: bool, optional, default: True
192
+ Whether to configure ChromeRemoteConnection to use HTTP keep-alive.
193
+
194
+ log_level: int, optional, default: adapts to python global log level
195
+
196
+ headless: bool, optional, default: False
197
+ can also be specified in the options instance.
198
+ Specify whether you want to use the browser in headless mode.
199
+ warning: this lowers undetectability and not fully supported.
200
+
201
+ version_main: int, optional, default: None (=auto)
202
+ if you, for god knows whatever reason, use
203
+ an older version of Chrome. You can specify it's full rounded version number
204
+ here. Example: 87 for all versions of 87
205
+
206
+ patcher_force_close: bool, optional, default: False
207
+ instructs the patcher to do whatever it can to access the chromedriver binary
208
+ if the file is locked, it will force shutdown all instances.
209
+ setting it is not recommended, unless you know the implications and think
210
+ you might need it.
211
+
212
+ suppress_welcome: bool, optional , default: True
213
+ a "welcome" alert might show up on *nix-like systems asking whether you want to set
214
+ chrome as your default browser, and if you want to send even more data to google.
215
+ now, in case you are nag-fetishist, or a diagnostics data feeder to google, you can set this to False.
216
+ Note: if you don't handle the nag screen in time, the browser loses it's connection and throws an Exception.
217
+
218
+ use_subprocess: bool, optional , default: True,
219
+
220
+ False (the default) makes sure Chrome will get it's own process (so no subprocess of chromedriver.exe or python
221
+ This fixes a LOT of issues, like multithreaded run, but mst importantly. shutting corectly after
222
+ program exits or using .quit()
223
+ you should be knowing what you're doing, and know how python works.
224
+
225
+ unfortunately, there is always an edge case in which one would like to write an single script with the only contents being:
226
+ --start script--
227
+ import undetected_chromedriver as uc
228
+ d = uc.Chrome()
229
+ d.get('https://somesite/')
230
+ ---end script --
231
+
232
+ and will be greeted with an error, since the program exists before chrome has a change to launch.
233
+ in that case you can set this to `True`. The browser will start via subprocess, and will keep running most of times.
234
+ ! setting it to True comes with NO support when being detected. !
235
+
236
+ no_sandbox: bool, optional, default=True
237
+ uses the --no-sandbox option, and additionally does suppress the "unsecure option" status bar
238
+ this option has a default of True since many people seem to run this as root (....) , and chrome does not start
239
+ when running as root without using --no-sandbox flag.
240
+
241
+ user_multi_procs:
242
+ set to true when you are using multithreads/multiprocessing
243
+ ensures not all processes are trying to modify a binary which is in use by another.
244
+ for this to work. YOU MUST HAVE AT LEAST 1 UNDETECTED_CHROMEDRIVER BINARY IN YOUR ROAMING DATA FOLDER.
245
+ this requirement can be easily satisfied, by just running this program "normal" and close/kill it.
246
+
247
+
248
+ """
249
+
250
+ finalize(self, self._ensure_close, self)
251
+ self.debug = debug
252
+ self.patcher = Patcher(
253
+ executable_path=driver_executable_path,
254
+ force=patcher_force_close,
255
+ version_main=version_main,
256
+ user_multi_procs=user_multi_procs,
257
+ )
258
+ # self.patcher.auto(user_multiprocess = user_multi_num_procs)
259
+ self.patcher.auto()
260
+
261
+ # self.patcher = patcher
262
+ if not options:
263
+ options = ChromeOptions()
264
+
265
+ try:
266
+ if hasattr(options, "_session") and options._session is not None:
267
+ # prevent reuse of options,
268
+ # as it just appends arguments, not replace them
269
+ # you'll get conflicts starting chrome
270
+ raise RuntimeError("you cannot reuse the ChromeOptions object")
271
+ except AttributeError:
272
+ pass
273
+
274
+ options._session = self
275
+
276
+ if not options.debugger_address:
277
+ debug_port = (
278
+ port
279
+ if port != 0
280
+ else selenium.webdriver.common.service.utils.free_port()
281
+ )
282
+ debug_host = "127.0.0.1"
283
+ options.debugger_address = "%s:%d" % (debug_host, debug_port)
284
+ else:
285
+ debug_host, debug_port = options.debugger_address.split(":")
286
+ debug_port = int(debug_port)
287
+
288
+ if enable_cdp_events:
289
+ options.set_capability(
290
+ "goog:loggingPrefs", {"performance": "ALL", "browser": "ALL"}
291
+ )
292
+
293
+ options.add_argument("--remote-debugging-host=%s" % debug_host)
294
+ options.add_argument("--remote-debugging-port=%s" % debug_port)
295
+
296
+ if user_data_dir:
297
+ options.add_argument("--user-data-dir=%s" % user_data_dir)
298
+
299
+ language, keep_user_data_dir = None, bool(user_data_dir)
300
+
301
+ # see if a custom user profile is specified in options
302
+ for arg in options.arguments:
303
+
304
+ if any([_ in arg for _ in ("--headless", "headless")]):
305
+ options.arguments.remove(arg)
306
+ options.headless = True
307
+
308
+ if "lang" in arg:
309
+ m = re.search("(?:--)?lang(?:[ =])?(.*)", arg)
310
+ try:
311
+ language = m[1]
312
+ except IndexError:
313
+ logger.debug("will set the language to en-US,en;q=0.9")
314
+ language = "en-US,en;q=0.9"
315
+
316
+ if "user-data-dir" in arg:
317
+ m = re.search("(?:--)?user-data-dir(?:[ =])?(.*)", arg)
318
+ try:
319
+ user_data_dir = m[1]
320
+ logger.debug(
321
+ "user-data-dir found in user argument %s => %s" % (arg, m[1])
322
+ )
323
+ keep_user_data_dir = True
324
+
325
+ except IndexError:
326
+ logger.debug(
327
+ "no user data dir could be extracted from supplied argument %s "
328
+ % arg
329
+ )
330
+
331
+ if not user_data_dir:
332
+ # backward compatiblity
333
+ # check if an old uc.ChromeOptions is used, and extract the user data dir
334
+
335
+ if hasattr(options, "user_data_dir") and getattr(
336
+ options, "user_data_dir", None
337
+ ):
338
+ import warnings
339
+
340
+ warnings.warn(
341
+ "using ChromeOptions.user_data_dir might stop working in future versions."
342
+ "use uc.Chrome(user_data_dir='/xyz/some/data') in case you need existing profile folder"
343
+ )
344
+ options.add_argument("--user-data-dir=%s" % options.user_data_dir)
345
+ keep_user_data_dir = True
346
+ logger.debug(
347
+ "user_data_dir property found in options object: %s" % user_data_dir
348
+ )
349
+
350
+ else:
351
+ user_data_dir = os.path.normpath(tempfile.mkdtemp())
352
+ keep_user_data_dir = False
353
+ arg = "--user-data-dir=%s" % user_data_dir
354
+ options.add_argument(arg)
355
+ logger.debug(
356
+ "created a temporary folder in which the user-data (profile) will be stored during this\n"
357
+ "session, and added it to chrome startup arguments: %s" % arg
358
+ )
359
+
360
+ if not language:
361
+ try:
362
+ import locale
363
+
364
+ language = locale.getdefaultlocale()[0].replace("_", "-")
365
+ except Exception:
366
+ pass
367
+ if not language:
368
+ language = "en-US"
369
+
370
+ options.add_argument("--lang=%s" % language)
371
+
372
+ if not options.binary_location:
373
+ options.binary_location = (
374
+ browser_executable_path or find_chrome_executable()
375
+ )
376
+
377
+ if not options.binary_location or not \
378
+ pathlib.Path(options.binary_location).exists():
379
+ raise FileNotFoundError(
380
+ "\n---------------------\n"
381
+ "Could not determine browser executable."
382
+ "\n---------------------\n"
383
+ "Make sure your browser is installed in the default location (path).\n"
384
+ "If you are sure about the browser executable, you can specify it using\n"
385
+ "the `browser_executable_path='{}` parameter.\n\n"
386
+ .format("/path/to/browser/executable" if IS_POSIX else "c:/path/to/your/browser.exe")
387
+ )
388
+
389
+ self._delay = 3
390
+
391
+ self.user_data_dir = user_data_dir
392
+ self.keep_user_data_dir = keep_user_data_dir
393
+
394
+ if suppress_welcome:
395
+ options.arguments.extend(["--no-default-browser-check", "--no-first-run"])
396
+ if no_sandbox:
397
+ options.arguments.extend(["--no-sandbox", "--test-type"])
398
+
399
+ if headless or getattr(options, 'headless', None):
400
+ #workaround until a better checking is found
401
+ try:
402
+ v_main = int(self.patcher.version_main) if self.patcher.version_main else 108
403
+ if v_main < 108:
404
+ options.add_argument("--headless=chrome")
405
+ elif v_main >= 108:
406
+ options.add_argument("--headless=new")
407
+ except:
408
+ logger.warning("could not detect version_main."
409
+ "therefore, we are assuming it is chrome 108 or higher")
410
+ options.add_argument("--headless=new")
411
+
412
+ options.add_argument("--window-size=1920,1080")
413
+ options.add_argument("--start-maximized")
414
+ options.add_argument("--no-sandbox")
415
+ # fixes "could not connect to chrome" error when running
416
+ # on linux using privileged user like root (which i don't recommend)
417
+
418
+ options.add_argument(
419
+ "--log-level=%d" % log_level
420
+ or divmod(logging.getLogger().getEffectiveLevel(), 10)[0]
421
+ )
422
+
423
+ if hasattr(options, "handle_prefs"):
424
+ options.handle_prefs(user_data_dir)
425
+
426
+ # fix exit_type flag to prevent tab-restore nag
427
+ try:
428
+ with open(
429
+ os.path.join(user_data_dir, "Default/Preferences"),
430
+ encoding="latin1",
431
+ mode="r+",
432
+ ) as fs:
433
+ config = json.load(fs)
434
+ if config["profile"]["exit_type"] is not None:
435
+ # fixing the restore-tabs-nag
436
+ config["profile"]["exit_type"] = None
437
+ fs.seek(0, 0)
438
+ json.dump(config, fs)
439
+ fs.truncate() # the file might be shorter
440
+ logger.debug("fixed exit_type flag")
441
+ except Exception as e:
442
+ logger.debug("did not find a bad exit_type flag ")
443
+
444
+ self.options = options
445
+
446
+ if not desired_capabilities:
447
+ desired_capabilities = options.to_capabilities()
448
+
449
+ if not use_subprocess and not windows_headless:
450
+ self.browser_pid = start_detached(
451
+ options.binary_location, *options.arguments
452
+ )
453
+ else:
454
+ startupinfo = None
455
+ if os.name == 'nt' and windows_headless:
456
+ # STARTUPINFO() is Windows only
457
+ startupinfo = subprocess.STARTUPINFO()
458
+ startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
459
+ browser = subprocess.Popen(
460
+ [options.binary_location, *options.arguments],
461
+ stdin=subprocess.PIPE,
462
+ stdout=subprocess.PIPE,
463
+ stderr=subprocess.PIPE,
464
+ close_fds=IS_POSIX,
465
+ startupinfo=startupinfo
466
+ )
467
+ self.browser_pid = browser.pid
468
+
469
+
470
+ service = selenium.webdriver.chromium.service.ChromiumService(
471
+ self.patcher.executable_path
472
+ )
473
+
474
+ super(Chrome, self).__init__(
475
+ service=service,
476
+ options=options,
477
+ keep_alive=keep_alive,
478
+ )
479
+
480
+ self.reactor = None
481
+
482
+ if enable_cdp_events:
483
+ if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
484
+ logging.getLogger(
485
+ "selenium.webdriver.remote.remote_connection"
486
+ ).setLevel(20)
487
+ reactor = Reactor(self)
488
+ reactor.start()
489
+ self.reactor = reactor
490
+
491
+ if advanced_elements:
492
+ self._web_element_cls = UCWebElement
493
+ else:
494
+ self._web_element_cls = WebElement
495
+
496
+ if headless or getattr(options, 'headless', None):
497
+ self._configure_headless()
498
+
499
+ def _configure_headless(self):
500
+ orig_get = self.get
501
+ logger.info("setting properties for headless")
502
+
503
+ def get_wrapped(*args, **kwargs):
504
+ if self.execute_script("return navigator.webdriver"):
505
+ logger.info("patch navigator.webdriver")
506
+ self.execute_cdp_cmd(
507
+ "Page.addScriptToEvaluateOnNewDocument",
508
+ {
509
+ "source": """
510
+
511
+ Object.defineProperty(window, "navigator", {
512
+ Object.defineProperty(window, "navigator", {
513
+ value: new Proxy(navigator, {
514
+ has: (target, key) => (key === "webdriver" ? false : key in target),
515
+ get: (target, key) =>
516
+ key === "webdriver"
517
+ ? false
518
+ : typeof target[key] === "function"
519
+ ? target[key].bind(target)
520
+ : target[key],
521
+ }),
522
+ });
523
+ """
524
+ },
525
+ )
526
+
527
+ logger.info("patch user-agent string")
528
+ self.execute_cdp_cmd(
529
+ "Network.setUserAgentOverride",
530
+ {
531
+ "userAgent": self.execute_script(
532
+ "return navigator.userAgent"
533
+ ).replace("Headless", "")
534
+ },
535
+ )
536
+ self.execute_cdp_cmd(
537
+ "Page.addScriptToEvaluateOnNewDocument",
538
+ {
539
+ "source": """
540
+ Object.defineProperty(navigator, 'maxTouchPoints', {get: () => 1});
541
+ Object.defineProperty(navigator.connection, 'rtt', {get: () => 100});
542
+
543
+ // https://github.com/microlinkhq/browserless/blob/master/packages/goto/src/evasions/chrome-runtime.js
544
+ window.chrome = {
545
+ app: {
546
+ isInstalled: false,
547
+ InstallState: {
548
+ DISABLED: 'disabled',
549
+ INSTALLED: 'installed',
550
+ NOT_INSTALLED: 'not_installed'
551
+ },
552
+ RunningState: {
553
+ CANNOT_RUN: 'cannot_run',
554
+ READY_TO_RUN: 'ready_to_run',
555
+ RUNNING: 'running'
556
+ }
557
+ },
558
+ runtime: {
559
+ OnInstalledReason: {
560
+ CHROME_UPDATE: 'chrome_update',
561
+ INSTALL: 'install',
562
+ SHARED_MODULE_UPDATE: 'shared_module_update',
563
+ UPDATE: 'update'
564
+ },
565
+ OnRestartRequiredReason: {
566
+ APP_UPDATE: 'app_update',
567
+ OS_UPDATE: 'os_update',
568
+ PERIODIC: 'periodic'
569
+ },
570
+ PlatformArch: {
571
+ ARM: 'arm',
572
+ ARM64: 'arm64',
573
+ MIPS: 'mips',
574
+ MIPS64: 'mips64',
575
+ X86_32: 'x86-32',
576
+ X86_64: 'x86-64'
577
+ },
578
+ PlatformNaclArch: {
579
+ ARM: 'arm',
580
+ MIPS: 'mips',
581
+ MIPS64: 'mips64',
582
+ X86_32: 'x86-32',
583
+ X86_64: 'x86-64'
584
+ },
585
+ PlatformOs: {
586
+ ANDROID: 'android',
587
+ CROS: 'cros',
588
+ LINUX: 'linux',
589
+ MAC: 'mac',
590
+ OPENBSD: 'openbsd',
591
+ WIN: 'win'
592
+ },
593
+ RequestUpdateCheckStatus: {
594
+ NO_UPDATE: 'no_update',
595
+ THROTTLED: 'throttled',
596
+ UPDATE_AVAILABLE: 'update_available'
597
+ }
598
+ }
599
+ }
600
+
601
+ // https://github.com/microlinkhq/browserless/blob/master/packages/goto/src/evasions/navigator-permissions.js
602
+ if (!window.Notification) {
603
+ window.Notification = {
604
+ permission: 'denied'
605
+ }
606
+ }
607
+
608
+ const originalQuery = window.navigator.permissions.query
609
+ window.navigator.permissions.__proto__.query = parameters =>
610
+ parameters.name === 'notifications'
611
+ ? Promise.resolve({ state: window.Notification.permission })
612
+ : originalQuery(parameters)
613
+
614
+ const oldCall = Function.prototype.call
615
+ function call() {
616
+ return oldCall.apply(this, arguments)
617
+ }
618
+ Function.prototype.call = call
619
+
620
+ const nativeToStringFunctionString = Error.toString().replace(/Error/g, 'toString')
621
+ const oldToString = Function.prototype.toString
622
+
623
+ function functionToString() {
624
+ if (this === window.navigator.permissions.query) {
625
+ return 'function query() { [native code] }'
626
+ }
627
+ if (this === functionToString) {
628
+ return nativeToStringFunctionString
629
+ }
630
+ return oldCall.call(oldToString, this)
631
+ }
632
+ // eslint-disable-next-line
633
+ Function.prototype.toString = functionToString
634
+ """
635
+ },
636
+ )
637
+ return orig_get(*args, **kwargs)
638
+
639
+ self.get = get_wrapped
640
+
641
+ # def _get_cdc_props(self):
642
+ # return self.execute_script(
643
+ # """
644
+ # let objectToInspect = window,
645
+ # result = [];
646
+ # while(objectToInspect !== null)
647
+ # { result = result.concat(Object.getOwnPropertyNames(objectToInspect));
648
+ # objectToInspect = Object.getPrototypeOf(objectToInspect); }
649
+ #
650
+ # return result.filter(i => i.match(/^([a-zA-Z]){27}(Array|Promise|Symbol)$/ig))
651
+ # """
652
+ # )
653
+ #
654
+ # def _hook_remove_cdc_props(self):
655
+ # self.execute_cdp_cmd(
656
+ # "Page.addScriptToEvaluateOnNewDocument",
657
+ # {
658
+ # "source": """
659
+ # let objectToInspect = window,
660
+ # result = [];
661
+ # while(objectToInspect !== null)
662
+ # { result = result.concat(Object.getOwnPropertyNames(objectToInspect));
663
+ # objectToInspect = Object.getPrototypeOf(objectToInspect); }
664
+ # result.forEach(p => p.match(/^([a-zA-Z]){27}(Array|Promise|Symbol)$/ig)
665
+ # &&delete window[p]&&console.log('removed',p))
666
+ # """
667
+ # },
668
+ # )
669
+
670
+ def get(self, url):
671
+ # if self._get_cdc_props():
672
+ # self._hook_remove_cdc_props()
673
+ return super().get(url)
674
+
675
+ def add_cdp_listener(self, event_name, callback):
676
+ if (
677
+ self.reactor
678
+ and self.reactor is not None
679
+ and isinstance(self.reactor, Reactor)
680
+ ):
681
+ self.reactor.add_event_handler(event_name, callback)
682
+ return self.reactor.handlers
683
+ return False
684
+
685
+ def clear_cdp_listeners(self):
686
+ if self.reactor and isinstance(self.reactor, Reactor):
687
+ self.reactor.handlers.clear()
688
+
689
+ def window_new(self):
690
+ self.execute(
691
+ selenium.webdriver.remote.command.Command.NEW_WINDOW, {"type": "window"}
692
+ )
693
+
694
+ def tab_new(self, url: str):
695
+ """
696
+ this opens a url in a new tab.
697
+ apparently, that passes all tests directly!
698
+
699
+ Parameters
700
+ ----------
701
+ url
702
+
703
+ Returns
704
+ -------
705
+
706
+ """
707
+ if not hasattr(self, "cdp"):
708
+ from .cdp import CDP
709
+
710
+ cdp = CDP(self.options)
711
+ cdp.tab_new(url)
712
+
713
+ def reconnect(self, timeout=0.1):
714
+ try:
715
+ self.service.stop()
716
+ except Exception as e:
717
+ logger.debug(e)
718
+ time.sleep(timeout)
719
+ try:
720
+ self.service.start()
721
+ except Exception as e:
722
+ logger.debug(e)
723
+
724
+ try:
725
+ self.start_session()
726
+ except Exception as e:
727
+ logger.debug(e)
728
+
729
+ def start_session(self, capabilities=None, browser_profile=None):
730
+ if not capabilities:
731
+ capabilities = self.options.to_capabilities()
732
+ super(selenium.webdriver.chrome.webdriver.WebDriver, self).start_session(
733
+ capabilities
734
+ )
735
+ # super(Chrome, self).start_session(capabilities, browser_profile)
736
+
737
+ def find_elements_recursive(self, by, value):
738
+ """
739
+ find elements in all frames
740
+ this is a generator function, which is needed
741
+ since if it would return a list of elements, they
742
+ will be stale on arrival.
743
+ using generator, when the element is returned we are in the correct frame
744
+ to use it directly
745
+ Args:
746
+ by: By
747
+ value: str
748
+ Returns: Generator[webelement.WebElement]
749
+ """
750
+ def search_frame(f=None):
751
+ if not f:
752
+ # ensure we are on main content frame
753
+ self.switch_to.default_content()
754
+ else:
755
+ self.switch_to.frame(f)
756
+ for elem in self.find_elements(by, value):
757
+ yield elem
758
+ # switch back to main content, otherwise we will get StaleElementReferenceException
759
+ self.switch_to.default_content()
760
+
761
+ # search root frame
762
+ for elem in search_frame():
763
+ yield elem
764
+ # get iframes
765
+ frames = self.find_elements('css selector', 'iframe')
766
+
767
+ # search per frame
768
+ for f in frames:
769
+ for elem in search_frame(f):
770
+ yield elem
771
+
772
+ def quit(self):
773
+ try:
774
+ self.service.stop()
775
+ self.service.process.kill()
776
+ self.command_executor.close()
777
+ self.service.process.wait(5)
778
+ logger.debug("webdriver process ended")
779
+ except (AttributeError, RuntimeError, OSError):
780
+ pass
781
+ try:
782
+ self.reactor.event.set()
783
+ logger.debug("shutting down reactor")
784
+ except AttributeError:
785
+ pass
786
+ try:
787
+ os.kill(self.browser_pid, 15)
788
+ logger.debug("gracefully closed browser")
789
+ except Exception as e: # noqa
790
+ pass
791
+ if (
792
+ hasattr(self, "keep_user_data_dir")
793
+ and hasattr(self, "user_data_dir")
794
+ and not self.keep_user_data_dir
795
+ ):
796
+ for _ in range(5):
797
+ try:
798
+ shutil.rmtree(self.user_data_dir, ignore_errors=False)
799
+ except FileNotFoundError:
800
+ pass
801
+ except (RuntimeError, OSError, PermissionError) as e:
802
+ logger.debug(
803
+ "When removing the temp profile, a %s occured: %s\nretrying..."
804
+ % (e.__class__.__name__, e)
805
+ )
806
+ else:
807
+ logger.debug("successfully removed %s" % self.user_data_dir)
808
+ break
809
+
810
+ try:
811
+ time.sleep(0.1)
812
+ except OSError:
813
+ pass
814
+
815
+ # dereference patcher, so patcher can start cleaning up as well.
816
+ # this must come last, otherwise it will throw 'in use' errors
817
+ self.patcher = None
818
+
819
+ def __getattribute__(self, item):
820
+ if not super().__getattribute__("debug"):
821
+ return super().__getattribute__(item)
822
+ else:
823
+ import inspect
824
+
825
+ original = super().__getattribute__(item)
826
+ if inspect.ismethod(original) and not inspect.isclass(original):
827
+
828
+ def newfunc(*args, **kwargs):
829
+ logger.debug(
830
+ "calling %s with args %s and kwargs %s\n"
831
+ % (original.__qualname__, args, kwargs)
832
+ )
833
+ return original(*args, **kwargs)
834
+
835
+ return newfunc
836
+ return original
837
+
838
+ def __enter__(self):
839
+ return self
840
+
841
+ def __exit__(self, exc_type, exc_val, exc_tb):
842
+ self.service.stop()
843
+ time.sleep(self._delay)
844
+ self.service.start()
845
+ self.start_session()
846
+
847
+ def __hash__(self):
848
+ return hash(self.options.debugger_address)
849
+
850
+ def __dir__(self):
851
+ return object.__dir__(self)
852
+
853
+ def __del__(self):
854
+ try:
855
+ self.service.process.kill()
856
+ except: # noqa
857
+ pass
858
+ self.quit()
859
+
860
+ @classmethod
861
+ def _ensure_close(cls, self):
862
+ # needs to be a classmethod so finalize can find the reference
863
+ logger.info("ensuring close")
864
+ if (
865
+ hasattr(self, "service")
866
+ and hasattr(self.service, "process")
867
+ and hasattr(self.service.process, "kill")
868
+ ):
869
+ self.service.process.kill()
870
+
871
+
872
+ def find_chrome_executable():
873
+ """
874
+ Finds the chrome, chrome beta, chrome canary, chromium executable
875
+
876
+ Returns
877
+ -------
878
+ executable_path : str
879
+ the full file path to found executable
880
+
881
+ """
882
+ candidates = set()
883
+ if IS_POSIX:
884
+ for item in os.environ.get("PATH").split(os.pathsep):
885
+ for subitem in (
886
+ "google-chrome",
887
+ "chromium",
888
+ "chromium-browser",
889
+ "chrome",
890
+ "google-chrome-stable",
891
+ ):
892
+ candidates.add(os.sep.join((item, subitem)))
893
+ if "darwin" in sys.platform:
894
+ candidates.update(
895
+ [
896
+ "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
897
+ "/Applications/Chromium.app/Contents/MacOS/Chromium",
898
+ ]
899
+ )
900
+ else:
901
+ for item in map(
902
+ os.environ.get,
903
+ ("PROGRAMFILES", "PROGRAMFILES(X86)", "LOCALAPPDATA", "PROGRAMW6432"),
904
+ ):
905
+ if item is not None:
906
+ for subitem in (
907
+ "Google/Chrome/Application",
908
+ ):
909
+ candidates.add(os.sep.join((item, subitem, "chrome.exe")))
910
+ for candidate in candidates:
911
+ logger.debug('checking if %s exists and is executable' % candidate)
912
+ if os.path.exists(candidate) and os.access(candidate, os.X_OK):
913
+ logger.debug('found! using %s' % candidate)
914
+ return os.path.normpath(candidate)
src/undetected_chromedriver/cdp.py CHANGED
@@ -1,112 +1,112 @@
1
- #!/usr/bin/env python3
2
- # this module is part of undetected_chromedriver
3
-
4
- import json
5
- import logging
6
-
7
- import requests
8
- import websockets
9
-
10
-
11
- log = logging.getLogger(__name__)
12
-
13
-
14
- class CDPObject(dict):
15
- def __init__(self, *a, **k):
16
- super().__init__(*a, **k)
17
- self.__dict__ = self
18
- for k in self.__dict__:
19
- if isinstance(self.__dict__[k], dict):
20
- self.__dict__[k] = CDPObject(self.__dict__[k])
21
- elif isinstance(self.__dict__[k], list):
22
- for i in range(len(self.__dict__[k])):
23
- if isinstance(self.__dict__[k][i], dict):
24
- self.__dict__[k][i] = CDPObject(self)
25
-
26
- def __repr__(self):
27
- tpl = f"{self.__class__.__name__}(\n\t{{}}\n\t)"
28
- return tpl.format("\n ".join(f"{k} = {v}" for k, v in self.items()))
29
-
30
-
31
- class PageElement(CDPObject):
32
- pass
33
-
34
-
35
- class CDP:
36
- log = logging.getLogger("CDP")
37
-
38
- endpoints = CDPObject(
39
- {
40
- "json": "/json",
41
- "protocol": "/json/protocol",
42
- "list": "/json/list",
43
- "new": "/json/new?{url}",
44
- "activate": "/json/activate/{id}",
45
- "close": "/json/close/{id}",
46
- }
47
- )
48
-
49
- def __init__(self, options: "ChromeOptions"): # noqa
50
- self.server_addr = "http://{0}:{1}".format(*options.debugger_address.split(":"))
51
-
52
- self._reqid = 0
53
- self._session = requests.Session()
54
- self._last_resp = None
55
- self._last_json = None
56
-
57
- resp = self.get(self.endpoints.json) # noqa
58
- self.sessionId = resp[0]["id"]
59
- self.wsurl = resp[0]["webSocketDebuggerUrl"]
60
-
61
- def tab_activate(self, id=None):
62
- if not id:
63
- active_tab = self.tab_list()[0]
64
- id = active_tab.id # noqa
65
- self.wsurl = active_tab.webSocketDebuggerUrl # noqa
66
- return self.post(self.endpoints["activate"].format(id=id))
67
-
68
- def tab_list(self):
69
- retval = self.get(self.endpoints["list"])
70
- return [PageElement(o) for o in retval]
71
-
72
- def tab_new(self, url):
73
- return self.post(self.endpoints["new"].format(url=url))
74
-
75
- def tab_close_last_opened(self):
76
- sessions = self.tab_list()
77
- opentabs = [s for s in sessions if s["type"] == "page"]
78
- return self.post(self.endpoints["close"].format(id=opentabs[-1]["id"]))
79
-
80
- async def send(self, method: str, params: dict):
81
- self._reqid += 1
82
- async with websockets.connect(self.wsurl) as ws:
83
- await ws.send(
84
- json.dumps({"method": method, "params": params, "id": self._reqid})
85
- )
86
- self._last_resp = await ws.recv()
87
- self._last_json = json.loads(self._last_resp)
88
- self.log.info(self._last_json)
89
-
90
- def get(self, uri):
91
- resp = self._session.get(self.server_addr + uri)
92
- try:
93
- self._last_resp = resp
94
- self._last_json = resp.json()
95
- except Exception:
96
- return
97
- else:
98
- return self._last_json
99
-
100
- def post(self, uri, data: dict = None):
101
- if not data:
102
- data = {}
103
- resp = self._session.post(self.server_addr + uri, json=data)
104
- try:
105
- self._last_resp = resp
106
- self._last_json = resp.json()
107
- except Exception:
108
- return self._last_resp
109
-
110
- @property
111
- def last_json(self):
112
- return self._last_json
 
1
+ #!/usr/bin/env python3
2
+ # this module is part of undetected_chromedriver
3
+
4
+ import json
5
+ import logging
6
+
7
+ import requests
8
+ import websockets
9
+
10
+
11
+ log = logging.getLogger(__name__)
12
+
13
+
14
+ class CDPObject(dict):
15
+ def __init__(self, *a, **k):
16
+ super().__init__(*a, **k)
17
+ self.__dict__ = self
18
+ for k in self.__dict__:
19
+ if isinstance(self.__dict__[k], dict):
20
+ self.__dict__[k] = CDPObject(self.__dict__[k])
21
+ elif isinstance(self.__dict__[k], list):
22
+ for i in range(len(self.__dict__[k])):
23
+ if isinstance(self.__dict__[k][i], dict):
24
+ self.__dict__[k][i] = CDPObject(self)
25
+
26
+ def __repr__(self):
27
+ tpl = f"{self.__class__.__name__}(\n\t{{}}\n\t)"
28
+ return tpl.format("\n ".join(f"{k} = {v}" for k, v in self.items()))
29
+
30
+
31
+ class PageElement(CDPObject):
32
+ pass
33
+
34
+
35
+ class CDP:
36
+ log = logging.getLogger("CDP")
37
+
38
+ endpoints = CDPObject(
39
+ {
40
+ "json": "/json",
41
+ "protocol": "/json/protocol",
42
+ "list": "/json/list",
43
+ "new": "/json/new?{url}",
44
+ "activate": "/json/activate/{id}",
45
+ "close": "/json/close/{id}",
46
+ }
47
+ )
48
+
49
+ def __init__(self, options: "ChromeOptions"): # noqa
50
+ self.server_addr = "http://{0}:{1}".format(*options.debugger_address.split(":"))
51
+
52
+ self._reqid = 0
53
+ self._session = requests.Session()
54
+ self._last_resp = None
55
+ self._last_json = None
56
+
57
+ resp = self.get(self.endpoints.json) # noqa
58
+ self.sessionId = resp[0]["id"]
59
+ self.wsurl = resp[0]["webSocketDebuggerUrl"]
60
+
61
+ def tab_activate(self, id=None):
62
+ if not id:
63
+ active_tab = self.tab_list()[0]
64
+ id = active_tab.id # noqa
65
+ self.wsurl = active_tab.webSocketDebuggerUrl # noqa
66
+ return self.post(self.endpoints["activate"].format(id=id))
67
+
68
+ def tab_list(self):
69
+ retval = self.get(self.endpoints["list"])
70
+ return [PageElement(o) for o in retval]
71
+
72
+ def tab_new(self, url):
73
+ return self.post(self.endpoints["new"].format(url=url))
74
+
75
+ def tab_close_last_opened(self):
76
+ sessions = self.tab_list()
77
+ opentabs = [s for s in sessions if s["type"] == "page"]
78
+ return self.post(self.endpoints["close"].format(id=opentabs[-1]["id"]))
79
+
80
+ async def send(self, method: str, params: dict):
81
+ self._reqid += 1
82
+ async with websockets.connect(self.wsurl) as ws:
83
+ await ws.send(
84
+ json.dumps({"method": method, "params": params, "id": self._reqid})
85
+ )
86
+ self._last_resp = await ws.recv()
87
+ self._last_json = json.loads(self._last_resp)
88
+ self.log.info(self._last_json)
89
+
90
+ def get(self, uri):
91
+ resp = self._session.get(self.server_addr + uri)
92
+ try:
93
+ self._last_resp = resp
94
+ self._last_json = resp.json()
95
+ except Exception:
96
+ return
97
+ else:
98
+ return self._last_json
99
+
100
+ def post(self, uri, data: dict = None):
101
+ if not data:
102
+ data = {}
103
+ resp = self._session.post(self.server_addr + uri, json=data)
104
+ try:
105
+ self._last_resp = resp
106
+ self._last_json = resp.json()
107
+ except Exception:
108
+ return self._last_resp
109
+
110
+ @property
111
+ def last_json(self):
112
+ return self._last_json
src/undetected_chromedriver/devtool.py CHANGED
@@ -1,193 +1,193 @@
1
- import asyncio
2
- from collections.abc import Mapping
3
- from collections.abc import Sequence
4
- from functools import wraps
5
- import os
6
- import logging
7
- import threading
8
- import time
9
- import traceback
10
- from typing import Any
11
- from typing import Awaitable
12
- from typing import Callable
13
- from typing import List
14
- from typing import Optional
15
-
16
-
17
- class Structure(dict):
18
- """
19
- This is a dict-like object structure, which you should subclass
20
- Only properties defined in the class context are used on initialization.
21
-
22
- See example
23
- """
24
-
25
- _store = {}
26
-
27
- def __init__(self, *a, **kw):
28
- """
29
- Instantiate a new instance.
30
-
31
- :param a:
32
- :param kw:
33
- """
34
-
35
- super().__init__()
36
-
37
- # auxiliar dict
38
- d = dict(*a, **kw)
39
- for k, v in d.items():
40
- if isinstance(v, Mapping):
41
- self[k] = self.__class__(v)
42
- elif isinstance(v, Sequence) and not isinstance(v, (str, bytes)):
43
- self[k] = [self.__class__(i) for i in v]
44
- else:
45
- self[k] = v
46
- super().__setattr__("__dict__", self)
47
-
48
- def __getattr__(self, item):
49
- return getattr(super(), item)
50
-
51
- def __getitem__(self, item):
52
- return super().__getitem__(item)
53
-
54
- def __setattr__(self, key, value):
55
- self.__setitem__(key, value)
56
-
57
- def __setitem__(self, key, value):
58
- super().__setitem__(key, value)
59
-
60
- def update(self, *a, **kw):
61
- super().update(*a, **kw)
62
-
63
- def __eq__(self, other):
64
- return frozenset(other.items()) == frozenset(self.items())
65
-
66
- def __hash__(self):
67
- return hash(frozenset(self.items()))
68
-
69
- @classmethod
70
- def __init_subclass__(cls, **kwargs):
71
- cls._store = {}
72
-
73
- def _normalize_strings(self):
74
- for k, v in self.copy().items():
75
- if isinstance(v, (str)):
76
- self[k] = v.strip()
77
-
78
-
79
- def timeout(seconds=3, on_timeout: Optional[Callable[[callable], Any]] = None):
80
- def wrapper(func):
81
- @wraps(func)
82
- def wrapped(*args, **kwargs):
83
- def function_reached_timeout():
84
- if on_timeout:
85
- on_timeout(func)
86
- else:
87
- raise TimeoutError("function call timed out")
88
-
89
- t = threading.Timer(interval=seconds, function=function_reached_timeout)
90
- t.start()
91
- try:
92
- return func(*args, **kwargs)
93
- except:
94
- t.cancel()
95
- raise
96
- finally:
97
- t.cancel()
98
-
99
- return wrapped
100
-
101
- return wrapper
102
-
103
-
104
- def test():
105
- import sys, os
106
-
107
- sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))
108
- import undetected_chromedriver as uc
109
- import threading
110
-
111
- def collector(
112
- driver: uc.Chrome,
113
- stop_event: threading.Event,
114
- on_event_coro: Optional[Callable[[List[str]], Awaitable[Any]]] = None,
115
- listen_events: Sequence = ("browser", "network", "performance"),
116
- ):
117
- def threaded(driver, stop_event, on_event_coro):
118
- async def _ensure_service_started():
119
- while (
120
- getattr(driver, "service", False)
121
- and getattr(driver.service, "process", False)
122
- and driver.service.process.poll()
123
- ):
124
- print("waiting for driver service to come back on")
125
- await asyncio.sleep(0.05)
126
- # await asyncio.sleep(driver._delay or .25)
127
-
128
- async def get_log_lines(typ):
129
- await _ensure_service_started()
130
- return driver.get_log(typ)
131
-
132
- async def looper():
133
- while not stop_event.is_set():
134
- log_lines = []
135
- try:
136
- for _ in listen_events:
137
- try:
138
- log_lines += await get_log_lines(_)
139
- except:
140
- if logging.getLogger().getEffectiveLevel() <= 10:
141
- traceback.print_exc()
142
- continue
143
- if log_lines and on_event_coro:
144
- await on_event_coro(log_lines)
145
- except Exception as e:
146
- if logging.getLogger().getEffectiveLevel() <= 10:
147
- traceback.print_exc()
148
-
149
- loop = asyncio.new_event_loop()
150
- asyncio.set_event_loop(loop)
151
- loop.run_until_complete(looper())
152
-
153
- t = threading.Thread(target=threaded, args=(driver, stop_event, on_event_coro))
154
- t.start()
155
-
156
- async def on_event(data):
157
- print("on_event")
158
- print("data:", data)
159
-
160
- def func_called(fn):
161
- def wrapped(*args, **kwargs):
162
- print(
163
- "func called! %s (args: %s, kwargs: %s)" % (fn.__name__, args, kwargs)
164
- )
165
- while driver.service.process and driver.service.process.poll() is not None:
166
- time.sleep(0.1)
167
- res = fn(*args, **kwargs)
168
- print("func completed! (result: %s)" % res)
169
- return res
170
-
171
- return wrapped
172
-
173
- logging.basicConfig(level=10)
174
-
175
- options = uc.ChromeOptions()
176
- options.set_capability(
177
- "goog:loggingPrefs", {"performance": "ALL", "browser": "ALL", "network": "ALL"}
178
- )
179
-
180
- driver = uc.Chrome(version_main=96, options=options)
181
-
182
- # driver.command_executor._request = timeout(seconds=1)(driver.command_executor._request)
183
- driver.command_executor._request = func_called(driver.command_executor._request)
184
- collector_stop = threading.Event()
185
- collector(driver, collector_stop, on_event)
186
-
187
- driver.get("https://nowsecure.nl")
188
-
189
- time.sleep(10)
190
-
191
- if os.name == "nt":
192
- driver.close()
193
- driver.quit()
 
1
+ import asyncio
2
+ from collections.abc import Mapping
3
+ from collections.abc import Sequence
4
+ from functools import wraps
5
+ import os
6
+ import logging
7
+ import threading
8
+ import time
9
+ import traceback
10
+ from typing import Any
11
+ from typing import Awaitable
12
+ from typing import Callable
13
+ from typing import List
14
+ from typing import Optional
15
+
16
+
17
+ class Structure(dict):
18
+ """
19
+ This is a dict-like object structure, which you should subclass
20
+ Only properties defined in the class context are used on initialization.
21
+
22
+ See example
23
+ """
24
+
25
+ _store = {}
26
+
27
+ def __init__(self, *a, **kw):
28
+ """
29
+ Instantiate a new instance.
30
+
31
+ :param a:
32
+ :param kw:
33
+ """
34
+
35
+ super().__init__()
36
+
37
+ # auxiliar dict
38
+ d = dict(*a, **kw)
39
+ for k, v in d.items():
40
+ if isinstance(v, Mapping):
41
+ self[k] = self.__class__(v)
42
+ elif isinstance(v, Sequence) and not isinstance(v, (str, bytes)):
43
+ self[k] = [self.__class__(i) for i in v]
44
+ else:
45
+ self[k] = v
46
+ super().__setattr__("__dict__", self)
47
+
48
+ def __getattr__(self, item):
49
+ return getattr(super(), item)
50
+
51
+ def __getitem__(self, item):
52
+ return super().__getitem__(item)
53
+
54
+ def __setattr__(self, key, value):
55
+ self.__setitem__(key, value)
56
+
57
+ def __setitem__(self, key, value):
58
+ super().__setitem__(key, value)
59
+
60
+ def update(self, *a, **kw):
61
+ super().update(*a, **kw)
62
+
63
+ def __eq__(self, other):
64
+ return frozenset(other.items()) == frozenset(self.items())
65
+
66
+ def __hash__(self):
67
+ return hash(frozenset(self.items()))
68
+
69
+ @classmethod
70
+ def __init_subclass__(cls, **kwargs):
71
+ cls._store = {}
72
+
73
+ def _normalize_strings(self):
74
+ for k, v in self.copy().items():
75
+ if isinstance(v, (str)):
76
+ self[k] = v.strip()
77
+
78
+
79
+ def timeout(seconds=3, on_timeout: Optional[Callable[[callable], Any]] = None):
80
+ def wrapper(func):
81
+ @wraps(func)
82
+ def wrapped(*args, **kwargs):
83
+ def function_reached_timeout():
84
+ if on_timeout:
85
+ on_timeout(func)
86
+ else:
87
+ raise TimeoutError("function call timed out")
88
+
89
+ t = threading.Timer(interval=seconds, function=function_reached_timeout)
90
+ t.start()
91
+ try:
92
+ return func(*args, **kwargs)
93
+ except:
94
+ t.cancel()
95
+ raise
96
+ finally:
97
+ t.cancel()
98
+
99
+ return wrapped
100
+
101
+ return wrapper
102
+
103
+
104
+ def test():
105
+ import sys, os
106
+
107
+ sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))
108
+ import undetected_chromedriver as uc
109
+ import threading
110
+
111
+ def collector(
112
+ driver: uc.Chrome,
113
+ stop_event: threading.Event,
114
+ on_event_coro: Optional[Callable[[List[str]], Awaitable[Any]]] = None,
115
+ listen_events: Sequence = ("browser", "network", "performance"),
116
+ ):
117
+ def threaded(driver, stop_event, on_event_coro):
118
+ async def _ensure_service_started():
119
+ while (
120
+ getattr(driver, "service", False)
121
+ and getattr(driver.service, "process", False)
122
+ and driver.service.process.poll()
123
+ ):
124
+ print("waiting for driver service to come back on")
125
+ await asyncio.sleep(0.05)
126
+ # await asyncio.sleep(driver._delay or .25)
127
+
128
+ async def get_log_lines(typ):
129
+ await _ensure_service_started()
130
+ return driver.get_log(typ)
131
+
132
+ async def looper():
133
+ while not stop_event.is_set():
134
+ log_lines = []
135
+ try:
136
+ for _ in listen_events:
137
+ try:
138
+ log_lines += await get_log_lines(_)
139
+ except:
140
+ if logging.getLogger().getEffectiveLevel() <= 10:
141
+ traceback.print_exc()
142
+ continue
143
+ if log_lines and on_event_coro:
144
+ await on_event_coro(log_lines)
145
+ except Exception as e:
146
+ if logging.getLogger().getEffectiveLevel() <= 10:
147
+ traceback.print_exc()
148
+
149
+ loop = asyncio.new_event_loop()
150
+ asyncio.set_event_loop(loop)
151
+ loop.run_until_complete(looper())
152
+
153
+ t = threading.Thread(target=threaded, args=(driver, stop_event, on_event_coro))
154
+ t.start()
155
+
156
+ async def on_event(data):
157
+ print("on_event")
158
+ print("data:", data)
159
+
160
+ def func_called(fn):
161
+ def wrapped(*args, **kwargs):
162
+ print(
163
+ "func called! %s (args: %s, kwargs: %s)" % (fn.__name__, args, kwargs)
164
+ )
165
+ while driver.service.process and driver.service.process.poll() is not None:
166
+ time.sleep(0.1)
167
+ res = fn(*args, **kwargs)
168
+ print("func completed! (result: %s)" % res)
169
+ return res
170
+
171
+ return wrapped
172
+
173
+ logging.basicConfig(level=10)
174
+
175
+ options = uc.ChromeOptions()
176
+ options.set_capability(
177
+ "goog:loggingPrefs", {"performance": "ALL", "browser": "ALL", "network": "ALL"}
178
+ )
179
+
180
+ driver = uc.Chrome(version_main=96, options=options)
181
+
182
+ # driver.command_executor._request = timeout(seconds=1)(driver.command_executor._request)
183
+ driver.command_executor._request = func_called(driver.command_executor._request)
184
+ collector_stop = threading.Event()
185
+ collector(driver, collector_stop, on_event)
186
+
187
+ driver.get("https://nowsecure.nl")
188
+
189
+ time.sleep(10)
190
+
191
+ if os.name == "nt":
192
+ driver.close()
193
+ driver.quit()
src/undetected_chromedriver/dprocess.py CHANGED
@@ -1,77 +1,77 @@
1
- import atexit
2
- import logging
3
- import multiprocessing
4
- import os
5
- import platform
6
- import signal
7
- from subprocess import PIPE
8
- from subprocess import Popen
9
- import sys
10
-
11
-
12
- CREATE_NEW_PROCESS_GROUP = 0x00000200
13
- DETACHED_PROCESS = 0x00000008
14
-
15
- REGISTERED = []
16
-
17
-
18
- def start_detached(executable, *args):
19
- """
20
- Starts a fully independent subprocess (with no parent)
21
- :param executable: executable
22
- :param args: arguments to the executable, eg: ['--param1_key=param1_val', '-vvv' ...]
23
- :return: pid of the grandchild process
24
- """
25
-
26
- # create pipe
27
- reader, writer = multiprocessing.Pipe(False)
28
-
29
- # do not keep reference
30
- process = multiprocessing.Process(
31
- target=_start_detached,
32
- args=(executable, *args),
33
- kwargs={"writer": writer},
34
- daemon=True,
35
- )
36
- process.start()
37
- process.join()
38
- # receive pid from pipe
39
- pid = reader.recv()
40
- REGISTERED.append(pid)
41
- # close pipes
42
- writer.close()
43
- reader.close()
44
- process.close()
45
-
46
- return pid
47
-
48
-
49
- def _start_detached(executable, *args, writer: multiprocessing.Pipe = None):
50
- # configure launch
51
- kwargs = {}
52
- if platform.system() == "Windows":
53
- kwargs.update(creationflags=DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP)
54
- elif sys.version_info < (3, 2):
55
- # assume posix
56
- kwargs.update(preexec_fn=os.setsid)
57
- else: # Python 3.2+ and Unix
58
- kwargs.update(start_new_session=True)
59
-
60
- # run
61
- p = Popen([executable, *args], stdin=PIPE, stdout=PIPE, stderr=PIPE, **kwargs)
62
-
63
- # send pid to pipe
64
- writer.send(p.pid)
65
- sys.exit()
66
-
67
-
68
- def _cleanup():
69
- for pid in REGISTERED:
70
- try:
71
- logging.getLogger(__name__).debug("cleaning up pid %d " % pid)
72
- os.kill(pid, signal.SIGTERM)
73
- except: # noqa
74
- pass
75
-
76
-
77
- atexit.register(_cleanup)
 
1
+ import atexit
2
+ import logging
3
+ import multiprocessing
4
+ import os
5
+ import platform
6
+ import signal
7
+ from subprocess import PIPE
8
+ from subprocess import Popen
9
+ import sys
10
+
11
+
12
+ CREATE_NEW_PROCESS_GROUP = 0x00000200
13
+ DETACHED_PROCESS = 0x00000008
14
+
15
+ REGISTERED = []
16
+
17
+
18
+ def start_detached(executable, *args):
19
+ """
20
+ Starts a fully independent subprocess (with no parent)
21
+ :param executable: executable
22
+ :param args: arguments to the executable, eg: ['--param1_key=param1_val', '-vvv' ...]
23
+ :return: pid of the grandchild process
24
+ """
25
+
26
+ # create pipe
27
+ reader, writer = multiprocessing.Pipe(False)
28
+
29
+ # do not keep reference
30
+ process = multiprocessing.Process(
31
+ target=_start_detached,
32
+ args=(executable, *args),
33
+ kwargs={"writer": writer},
34
+ daemon=True,
35
+ )
36
+ process.start()
37
+ process.join()
38
+ # receive pid from pipe
39
+ pid = reader.recv()
40
+ REGISTERED.append(pid)
41
+ # close pipes
42
+ writer.close()
43
+ reader.close()
44
+ process.close()
45
+
46
+ return pid
47
+
48
+
49
+ def _start_detached(executable, *args, writer: multiprocessing.Pipe = None):
50
+ # configure launch
51
+ kwargs = {}
52
+ if platform.system() == "Windows":
53
+ kwargs.update(creationflags=DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP)
54
+ elif sys.version_info < (3, 2):
55
+ # assume posix
56
+ kwargs.update(preexec_fn=os.setsid)
57
+ else: # Python 3.2+ and Unix
58
+ kwargs.update(start_new_session=True)
59
+
60
+ # run
61
+ p = Popen([executable, *args], stdin=PIPE, stdout=PIPE, stderr=PIPE, **kwargs)
62
+
63
+ # send pid to pipe
64
+ writer.send(p.pid)
65
+ sys.exit()
66
+
67
+
68
+ def _cleanup():
69
+ for pid in REGISTERED:
70
+ try:
71
+ logging.getLogger(__name__).debug("cleaning up pid %d " % pid)
72
+ os.kill(pid, signal.SIGTERM)
73
+ except: # noqa
74
+ pass
75
+
76
+
77
+ atexit.register(_cleanup)
src/undetected_chromedriver/options.py CHANGED
@@ -1,85 +1,85 @@
1
- #!/usr/bin/env python3
2
- # this module is part of undetected_chromedriver
3
-
4
-
5
- import json
6
- import os
7
-
8
- from selenium.webdriver.chromium.options import ChromiumOptions as _ChromiumOptions
9
-
10
-
11
- class ChromeOptions(_ChromiumOptions):
12
- _session = None
13
- _user_data_dir = None
14
-
15
- @property
16
- def user_data_dir(self):
17
- return self._user_data_dir
18
-
19
- @user_data_dir.setter
20
- def user_data_dir(self, path: str):
21
- """
22
- Sets the browser profile folder to use, or creates a new profile
23
- at given <path>.
24
-
25
- Parameters
26
- ----------
27
- path: str
28
- the path to a chrome profile folder
29
- if it does not exist, a new profile will be created at given location
30
- """
31
- apath = os.path.abspath(path)
32
- self._user_data_dir = os.path.normpath(apath)
33
-
34
- @staticmethod
35
- def _undot_key(key, value):
36
- """turn a (dotted key, value) into a proper nested dict"""
37
- if "." in key:
38
- key, rest = key.split(".", 1)
39
- value = ChromeOptions._undot_key(rest, value)
40
- return {key: value}
41
-
42
- @staticmethod
43
- def _merge_nested(a, b):
44
- """
45
- merges b into a
46
- leaf values in a are overwritten with values from b
47
- """
48
- for key in b:
49
- if key in a:
50
- if isinstance(a[key], dict) and isinstance(b[key], dict):
51
- ChromeOptions._merge_nested(a[key], b[key])
52
- continue
53
- a[key] = b[key]
54
- return a
55
-
56
- def handle_prefs(self, user_data_dir):
57
- prefs = self.experimental_options.get("prefs")
58
- if prefs:
59
- user_data_dir = user_data_dir or self._user_data_dir
60
- default_path = os.path.join(user_data_dir, "Default")
61
- os.makedirs(default_path, exist_ok=True)
62
-
63
- # undot prefs dict keys
64
- undot_prefs = {}
65
- for key, value in prefs.items():
66
- undot_prefs = self._merge_nested(
67
- undot_prefs, self._undot_key(key, value)
68
- )
69
-
70
- prefs_file = os.path.join(default_path, "Preferences")
71
- if os.path.exists(prefs_file):
72
- with open(prefs_file, encoding="latin1", mode="r") as f:
73
- undot_prefs = self._merge_nested(json.load(f), undot_prefs)
74
-
75
- with open(prefs_file, encoding="latin1", mode="w") as f:
76
- json.dump(undot_prefs, f)
77
-
78
- # remove the experimental_options to avoid an error
79
- del self._experimental_options["prefs"]
80
-
81
- @classmethod
82
- def from_options(cls, options):
83
- o = cls()
84
- o.__dict__.update(options.__dict__)
85
- return o
 
1
+ #!/usr/bin/env python3
2
+ # this module is part of undetected_chromedriver
3
+
4
+
5
+ import json
6
+ import os
7
+
8
+ from selenium.webdriver.chromium.options import ChromiumOptions as _ChromiumOptions
9
+
10
+
11
+ class ChromeOptions(_ChromiumOptions):
12
+ _session = None
13
+ _user_data_dir = None
14
+
15
+ @property
16
+ def user_data_dir(self):
17
+ return self._user_data_dir
18
+
19
+ @user_data_dir.setter
20
+ def user_data_dir(self, path: str):
21
+ """
22
+ Sets the browser profile folder to use, or creates a new profile
23
+ at given <path>.
24
+
25
+ Parameters
26
+ ----------
27
+ path: str
28
+ the path to a chrome profile folder
29
+ if it does not exist, a new profile will be created at given location
30
+ """
31
+ apath = os.path.abspath(path)
32
+ self._user_data_dir = os.path.normpath(apath)
33
+
34
+ @staticmethod
35
+ def _undot_key(key, value):
36
+ """turn a (dotted key, value) into a proper nested dict"""
37
+ if "." in key:
38
+ key, rest = key.split(".", 1)
39
+ value = ChromeOptions._undot_key(rest, value)
40
+ return {key: value}
41
+
42
+ @staticmethod
43
+ def _merge_nested(a, b):
44
+ """
45
+ merges b into a
46
+ leaf values in a are overwritten with values from b
47
+ """
48
+ for key in b:
49
+ if key in a:
50
+ if isinstance(a[key], dict) and isinstance(b[key], dict):
51
+ ChromeOptions._merge_nested(a[key], b[key])
52
+ continue
53
+ a[key] = b[key]
54
+ return a
55
+
56
+ def handle_prefs(self, user_data_dir):
57
+ prefs = self.experimental_options.get("prefs")
58
+ if prefs:
59
+ user_data_dir = user_data_dir or self._user_data_dir
60
+ default_path = os.path.join(user_data_dir, "Default")
61
+ os.makedirs(default_path, exist_ok=True)
62
+
63
+ # undot prefs dict keys
64
+ undot_prefs = {}
65
+ for key, value in prefs.items():
66
+ undot_prefs = self._merge_nested(
67
+ undot_prefs, self._undot_key(key, value)
68
+ )
69
+
70
+ prefs_file = os.path.join(default_path, "Preferences")
71
+ if os.path.exists(prefs_file):
72
+ with open(prefs_file, encoding="latin1", mode="r") as f:
73
+ undot_prefs = self._merge_nested(json.load(f), undot_prefs)
74
+
75
+ with open(prefs_file, encoding="latin1", mode="w") as f:
76
+ json.dump(undot_prefs, f)
77
+
78
+ # remove the experimental_options to avoid an error
79
+ del self._experimental_options["prefs"]
80
+
81
+ @classmethod
82
+ def from_options(cls, options):
83
+ o = cls()
84
+ o.__dict__.update(options.__dict__)
85
+ return o
src/undetected_chromedriver/patcher.py CHANGED
@@ -1,451 +1,451 @@
1
- #!/usr/bin/env python3
2
- # this module is part of undetected_chromedriver
3
-
4
- from distutils.version import LooseVersion
5
- import io
6
- import json
7
- import logging
8
- import os
9
- import pathlib
10
- import platform
11
- import random
12
- import re
13
- import shutil
14
- import string
15
- import sys
16
- import time
17
- from urllib.request import urlopen
18
- from urllib.request import urlretrieve
19
- import zipfile
20
- from multiprocessing import Lock
21
-
22
- logger = logging.getLogger(__name__)
23
-
24
- IS_POSIX = sys.platform.startswith(("darwin", "cygwin", "linux", "linux2", "freebsd"))
25
-
26
-
27
- class Patcher(object):
28
- lock = Lock()
29
- exe_name = "chromedriver%s"
30
-
31
- platform = sys.platform
32
- if platform.endswith("win32"):
33
- d = "~/appdata/roaming/undetected_chromedriver"
34
- elif "LAMBDA_TASK_ROOT" in os.environ:
35
- d = "/tmp/undetected_chromedriver"
36
- elif platform.startswith(("linux", "linux2")):
37
- d = "~/.local/share/undetected_chromedriver"
38
- elif platform.endswith("darwin"):
39
- d = "~/Library/Application Support/undetected_chromedriver"
40
- else:
41
- d = "~/.undetected_chromedriver"
42
- data_path = os.path.abspath(os.path.expanduser(d))
43
-
44
- def __init__(
45
- self,
46
- executable_path=None,
47
- force=False,
48
- version_main: int = 0,
49
- user_multi_procs=False,
50
- ):
51
- """
52
- Args:
53
- executable_path: None = automatic
54
- a full file path to the chromedriver executable
55
- force: False
56
- terminate processes which are holding lock
57
- version_main: 0 = auto
58
- specify main chrome version (rounded, ex: 82)
59
- """
60
- self.force = force
61
- self._custom_exe_path = False
62
- prefix = "undetected"
63
- self.user_multi_procs = user_multi_procs
64
-
65
- try:
66
- # Try to convert version_main into an integer
67
- version_main_int = int(version_main)
68
- # check if version_main_int is less than or equal to e.g 114
69
- self.is_old_chromedriver = version_main and version_main_int <= 114
70
- except (ValueError,TypeError):
71
- # Check not running inside Docker
72
- if not os.path.exists("/app/chromedriver"):
73
- # If the conversion fails, log an error message
74
- logging.info("version_main cannot be converted to an integer")
75
- # Set self.is_old_chromedriver to False if the conversion fails
76
- self.is_old_chromedriver = False
77
-
78
- # Needs to be called before self.exe_name is accessed
79
- self._set_platform_name()
80
-
81
- if not os.path.exists(self.data_path):
82
- os.makedirs(self.data_path, exist_ok=True)
83
-
84
- if not executable_path:
85
- if sys.platform.startswith("freebsd"):
86
- self.executable_path = os.path.join(
87
- self.data_path, self.exe_name
88
- )
89
- else:
90
- self.executable_path = os.path.join(
91
- self.data_path, "_".join([prefix, self.exe_name])
92
- )
93
-
94
- if not IS_POSIX:
95
- if executable_path:
96
- if not executable_path[-4:] == ".exe":
97
- executable_path += ".exe"
98
-
99
- self.zip_path = os.path.join(self.data_path, prefix)
100
-
101
- if not executable_path:
102
- if not self.user_multi_procs:
103
- self.executable_path = os.path.abspath(
104
- os.path.join(".", self.executable_path)
105
- )
106
-
107
- if executable_path:
108
- self._custom_exe_path = True
109
- self.executable_path = executable_path
110
-
111
- # Set the correct repository to download the Chromedriver from
112
- if self.is_old_chromedriver:
113
- self.url_repo = "https://chromedriver.storage.googleapis.com"
114
- else:
115
- self.url_repo = "https://googlechromelabs.github.io/chrome-for-testing"
116
-
117
- self.version_main = version_main
118
- self.version_full = None
119
-
120
- def _set_platform_name(self):
121
- """
122
- Set the platform and exe name based on the platform undetected_chromedriver is running on
123
- in order to download the correct chromedriver.
124
- """
125
- if self.platform.endswith("win32"):
126
- self.platform_name = "win32"
127
- self.exe_name %= ".exe"
128
- if self.platform.endswith(("linux", "linux2")):
129
- self.platform_name = "linux64"
130
- self.exe_name %= ""
131
- if self.platform.endswith("darwin"):
132
- if self.is_old_chromedriver:
133
- self.platform_name = "mac64"
134
- else:
135
- self.platform_name = "mac-x64"
136
- self.exe_name %= ""
137
- if self.platform.startswith("freebsd"):
138
- self.platform_name = "freebsd"
139
- self.exe_name %= ""
140
-
141
- def auto(self, executable_path=None, force=False, version_main=None, _=None):
142
- """
143
-
144
- Args:
145
- executable_path:
146
- force:
147
- version_main:
148
-
149
- Returns:
150
-
151
- """
152
- p = pathlib.Path(self.data_path)
153
- if self.user_multi_procs:
154
- with Lock():
155
- files = list(p.rglob("*chromedriver*"))
156
- most_recent = max(files, key=lambda f: f.stat().st_mtime)
157
- files.remove(most_recent)
158
- list(map(lambda f: f.unlink(), files))
159
- if self.is_binary_patched(most_recent):
160
- self.executable_path = str(most_recent)
161
- return True
162
-
163
- if executable_path:
164
- self.executable_path = executable_path
165
- self._custom_exe_path = True
166
-
167
- if self._custom_exe_path:
168
- ispatched = self.is_binary_patched(self.executable_path)
169
- if not ispatched:
170
- return self.patch_exe()
171
- else:
172
- return
173
-
174
- if version_main:
175
- self.version_main = version_main
176
- if force is True:
177
- self.force = force
178
-
179
-
180
- if self.platform_name == "freebsd":
181
- chromedriver_path = shutil.which("chromedriver")
182
-
183
- if not os.path.isfile(chromedriver_path) or not os.access(chromedriver_path, os.X_OK):
184
- logging.error("Chromedriver not installed!")
185
- return
186
-
187
- version_path = os.path.join(os.path.dirname(self.executable_path), "version.txt")
188
-
189
- process = os.popen(f'"{chromedriver_path}" --version')
190
- chromedriver_version = process.read().split(' ')[1].split(' ')[0]
191
- process.close()
192
-
193
- current_version = None
194
- if os.path.isfile(version_path) or os.access(version_path, os.X_OK):
195
- with open(version_path, 'r') as f:
196
- current_version = f.read()
197
-
198
- if current_version != chromedriver_version:
199
- logging.info("Copying chromedriver executable...")
200
- shutil.copy(chromedriver_path, self.executable_path)
201
- os.chmod(self.executable_path, 0o755)
202
-
203
- with open(version_path, 'w') as f:
204
- f.write(chromedriver_version)
205
-
206
- logging.info("Chromedriver executable copied!")
207
- else:
208
- try:
209
- os.unlink(self.executable_path)
210
- except PermissionError:
211
- if self.force:
212
- self.force_kill_instances(self.executable_path)
213
- return self.auto(force=not self.force)
214
- try:
215
- if self.is_binary_patched():
216
- # assumes already running AND patched
217
- return True
218
- except PermissionError:
219
- pass
220
- # return False
221
- except FileNotFoundError:
222
- pass
223
-
224
- release = self.fetch_release_number()
225
- self.version_main = release.version[0]
226
- self.version_full = release
227
- self.unzip_package(self.fetch_package())
228
-
229
- return self.patch()
230
-
231
- def driver_binary_in_use(self, path: str = None) -> bool:
232
- """
233
- naive test to check if a found chromedriver binary is
234
- currently in use
235
-
236
- Args:
237
- path: a string or PathLike object to the binary to check.
238
- if not specified, we check use this object's executable_path
239
- """
240
- if not path:
241
- path = self.executable_path
242
- p = pathlib.Path(path)
243
-
244
- if not p.exists():
245
- raise OSError("file does not exist: %s" % p)
246
- try:
247
- with open(p, mode="a+b") as fs:
248
- exc = []
249
- try:
250
-
251
- fs.seek(0, 0)
252
- except PermissionError as e:
253
- exc.append(e) # since some systems apprently allow seeking
254
- # we conduct another test
255
- try:
256
- fs.readline()
257
- except PermissionError as e:
258
- exc.append(e)
259
-
260
- if exc:
261
-
262
- return True
263
- return False
264
- # ok safe to assume this is in use
265
- except Exception as e:
266
- # logger.exception("whoops ", e)
267
- pass
268
-
269
- def cleanup_unused_files(self):
270
- p = pathlib.Path(self.data_path)
271
- items = list(p.glob("*undetected*"))
272
- for item in items:
273
- try:
274
- item.unlink()
275
- except:
276
- pass
277
-
278
- def patch(self):
279
- self.patch_exe()
280
- return self.is_binary_patched()
281
-
282
- def fetch_release_number(self):
283
- """
284
- Gets the latest major version available, or the latest major version of self.target_version if set explicitly.
285
- :return: version string
286
- :rtype: LooseVersion
287
- """
288
- # Endpoint for old versions of Chromedriver (114 and below)
289
- if self.is_old_chromedriver:
290
- path = f"/latest_release_{self.version_main}"
291
- path = path.upper()
292
- logger.debug("getting release number from %s" % path)
293
- return LooseVersion(urlopen(self.url_repo + path).read().decode())
294
-
295
- # Endpoint for new versions of Chromedriver (115+)
296
- if not self.version_main:
297
- # Fetch the latest version
298
- path = "/last-known-good-versions-with-downloads.json"
299
- logger.debug("getting release number from %s" % path)
300
- with urlopen(self.url_repo + path) as conn:
301
- response = conn.read().decode()
302
-
303
- last_versions = json.loads(response)
304
- return LooseVersion(last_versions["channels"]["Stable"]["version"])
305
-
306
- # Fetch the latest minor version of the major version provided
307
- path = "/latest-versions-per-milestone-with-downloads.json"
308
- logger.debug("getting release number from %s" % path)
309
- with urlopen(self.url_repo + path) as conn:
310
- response = conn.read().decode()
311
-
312
- major_versions = json.loads(response)
313
- return LooseVersion(major_versions["milestones"][str(self.version_main)]["version"])
314
-
315
- def parse_exe_version(self):
316
- with io.open(self.executable_path, "rb") as f:
317
- for line in iter(lambda: f.readline(), b""):
318
- match = re.search(rb"platform_handle\x00content\x00([0-9.]*)", line)
319
- if match:
320
- return LooseVersion(match[1].decode())
321
-
322
- def fetch_package(self):
323
- """
324
- Downloads ChromeDriver from source
325
-
326
- :return: path to downloaded file
327
- """
328
- zip_name = f"chromedriver_{self.platform_name}.zip"
329
- if self.is_old_chromedriver:
330
- download_url = "%s/%s/%s" % (self.url_repo, self.version_full.vstring, zip_name)
331
- else:
332
- zip_name = zip_name.replace("_", "-", 1)
333
- download_url = "https://storage.googleapis.com/chrome-for-testing-public/%s/%s/%s"
334
- download_url %= (self.version_full.vstring, self.platform_name, zip_name)
335
-
336
- logger.debug("downloading from %s" % download_url)
337
- return urlretrieve(download_url)[0]
338
-
339
- def unzip_package(self, fp):
340
- """
341
- Does what it says
342
-
343
- :return: path to unpacked executable
344
- """
345
- exe_path = self.exe_name
346
- if not self.is_old_chromedriver:
347
- # The new chromedriver unzips into its own folder
348
- zip_name = f"chromedriver-{self.platform_name}"
349
- exe_path = os.path.join(zip_name, self.exe_name)
350
-
351
- logger.debug("unzipping %s" % fp)
352
- try:
353
- os.unlink(self.zip_path)
354
- except (FileNotFoundError, OSError):
355
- pass
356
-
357
- os.makedirs(self.zip_path, mode=0o755, exist_ok=True)
358
- with zipfile.ZipFile(fp, mode="r") as zf:
359
- zf.extractall(self.zip_path)
360
- os.rename(os.path.join(self.zip_path, exe_path), self.executable_path)
361
- os.remove(fp)
362
- shutil.rmtree
363
- os.chmod(self.executable_path, 0o755)
364
- return self.executable_path
365
-
366
- @staticmethod
367
- def force_kill_instances(exe_name):
368
- """
369
- kills running instances.
370
- :param: executable name to kill, may be a path as well
371
-
372
- :return: True on success else False
373
- """
374
- exe_name = os.path.basename(exe_name)
375
- if IS_POSIX:
376
- r = os.system("kill -f -9 $(pidof %s)" % exe_name)
377
- else:
378
- r = os.system("taskkill /f /im %s" % exe_name)
379
- return not r
380
-
381
- @staticmethod
382
- def gen_random_cdc():
383
- cdc = random.choices(string.ascii_letters, k=27)
384
- return "".join(cdc).encode()
385
-
386
- def is_binary_patched(self, executable_path=None):
387
- executable_path = executable_path or self.executable_path
388
- try:
389
- with io.open(executable_path, "rb") as fh:
390
- return fh.read().find(b"undetected chromedriver") != -1
391
- except FileNotFoundError:
392
- return False
393
-
394
- def patch_exe(self):
395
- start = time.perf_counter()
396
- logger.info("patching driver executable %s" % self.executable_path)
397
- with io.open(self.executable_path, "r+b") as fh:
398
- content = fh.read()
399
- # match_injected_codeblock = re.search(rb"{window.*;}", content)
400
- match_injected_codeblock = re.search(rb"\{window\.cdc.*?;\}", content)
401
- if match_injected_codeblock:
402
- target_bytes = match_injected_codeblock[0]
403
- new_target_bytes = (
404
- b'{console.log("undetected chromedriver 1337!")}'.ljust(
405
- len(target_bytes), b" "
406
- )
407
- )
408
- new_content = content.replace(target_bytes, new_target_bytes)
409
- if new_content == content:
410
- logger.warning(
411
- "something went wrong patching the driver binary. could not find injection code block"
412
- )
413
- else:
414
- logger.debug(
415
- "found block:\n%s\nreplacing with:\n%s"
416
- % (target_bytes, new_target_bytes)
417
- )
418
- fh.seek(0)
419
- fh.write(new_content)
420
- logger.debug(
421
- "patching took us {:.2f} seconds".format(time.perf_counter() - start)
422
- )
423
-
424
- def __repr__(self):
425
- return "{0:s}({1:s})".format(
426
- self.__class__.__name__,
427
- self.executable_path,
428
- )
429
-
430
- def __del__(self):
431
- if self._custom_exe_path:
432
- # if the driver binary is specified by user
433
- # we assume it is important enough to not delete it
434
- return
435
- else:
436
- timeout = 3 # stop trying after this many seconds
437
- t = time.monotonic()
438
- now = lambda: time.monotonic()
439
- while now() - t > timeout:
440
- # we don't want to wait until the end of time
441
- try:
442
- if self.user_multi_procs:
443
- break
444
- os.unlink(self.executable_path)
445
- logger.debug("successfully unlinked %s" % self.executable_path)
446
- break
447
- except (OSError, RuntimeError, PermissionError):
448
- time.sleep(0.01)
449
- continue
450
- except FileNotFoundError:
451
- break
 
1
+ #!/usr/bin/env python3
2
+ # this module is part of undetected_chromedriver
3
+
4
+ from distutils.version import LooseVersion
5
+ import io
6
+ import json
7
+ import logging
8
+ import os
9
+ import pathlib
10
+ import platform
11
+ import random
12
+ import re
13
+ import shutil
14
+ import string
15
+ import sys
16
+ import time
17
+ from urllib.request import urlopen
18
+ from urllib.request import urlretrieve
19
+ import zipfile
20
+ from multiprocessing import Lock
21
+
22
+ logger = logging.getLogger(__name__)
23
+
24
+ IS_POSIX = sys.platform.startswith(("darwin", "cygwin", "linux", "linux2", "freebsd"))
25
+
26
+
27
+ class Patcher(object):
28
+ lock = Lock()
29
+ exe_name = "chromedriver%s"
30
+
31
+ platform = sys.platform
32
+ if platform.endswith("win32"):
33
+ d = "~/appdata/roaming/undetected_chromedriver"
34
+ elif "LAMBDA_TASK_ROOT" in os.environ:
35
+ d = "/tmp/undetected_chromedriver"
36
+ elif platform.startswith(("linux", "linux2")):
37
+ d = "~/.local/share/undetected_chromedriver"
38
+ elif platform.endswith("darwin"):
39
+ d = "~/Library/Application Support/undetected_chromedriver"
40
+ else:
41
+ d = "~/.undetected_chromedriver"
42
+ data_path = os.path.abspath(os.path.expanduser(d))
43
+
44
+ def __init__(
45
+ self,
46
+ executable_path=None,
47
+ force=False,
48
+ version_main: int = 0,
49
+ user_multi_procs=False,
50
+ ):
51
+ """
52
+ Args:
53
+ executable_path: None = automatic
54
+ a full file path to the chromedriver executable
55
+ force: False
56
+ terminate processes which are holding lock
57
+ version_main: 0 = auto
58
+ specify main chrome version (rounded, ex: 82)
59
+ """
60
+ self.force = force
61
+ self._custom_exe_path = False
62
+ prefix = "undetected"
63
+ self.user_multi_procs = user_multi_procs
64
+
65
+ try:
66
+ # Try to convert version_main into an integer
67
+ version_main_int = int(version_main)
68
+ # check if version_main_int is less than or equal to e.g 114
69
+ self.is_old_chromedriver = version_main and version_main_int <= 114
70
+ except (ValueError,TypeError):
71
+ # Check not running inside Docker
72
+ if not os.path.exists("/app/chromedriver"):
73
+ # If the conversion fails, log an error message
74
+ logging.info("version_main cannot be converted to an integer")
75
+ # Set self.is_old_chromedriver to False if the conversion fails
76
+ self.is_old_chromedriver = False
77
+
78
+ # Needs to be called before self.exe_name is accessed
79
+ self._set_platform_name()
80
+
81
+ if not os.path.exists(self.data_path):
82
+ os.makedirs(self.data_path, exist_ok=True)
83
+
84
+ if not executable_path:
85
+ if sys.platform.startswith("freebsd"):
86
+ self.executable_path = os.path.join(
87
+ self.data_path, self.exe_name
88
+ )
89
+ else:
90
+ self.executable_path = os.path.join(
91
+ self.data_path, "_".join([prefix, self.exe_name])
92
+ )
93
+
94
+ if not IS_POSIX:
95
+ if executable_path:
96
+ if not executable_path[-4:] == ".exe":
97
+ executable_path += ".exe"
98
+
99
+ self.zip_path = os.path.join(self.data_path, prefix)
100
+
101
+ if not executable_path:
102
+ if not self.user_multi_procs:
103
+ self.executable_path = os.path.abspath(
104
+ os.path.join(".", self.executable_path)
105
+ )
106
+
107
+ if executable_path:
108
+ self._custom_exe_path = True
109
+ self.executable_path = executable_path
110
+
111
+ # Set the correct repository to download the Chromedriver from
112
+ if self.is_old_chromedriver:
113
+ self.url_repo = "https://chromedriver.storage.googleapis.com"
114
+ else:
115
+ self.url_repo = "https://googlechromelabs.github.io/chrome-for-testing"
116
+
117
+ self.version_main = version_main
118
+ self.version_full = None
119
+
120
+ def _set_platform_name(self):
121
+ """
122
+ Set the platform and exe name based on the platform undetected_chromedriver is running on
123
+ in order to download the correct chromedriver.
124
+ """
125
+ if self.platform.endswith("win32"):
126
+ self.platform_name = "win32"
127
+ self.exe_name %= ".exe"
128
+ if self.platform.endswith(("linux", "linux2")):
129
+ self.platform_name = "linux64"
130
+ self.exe_name %= ""
131
+ if self.platform.endswith("darwin"):
132
+ if self.is_old_chromedriver:
133
+ self.platform_name = "mac64"
134
+ else:
135
+ self.platform_name = "mac-x64"
136
+ self.exe_name %= ""
137
+ if self.platform.startswith("freebsd"):
138
+ self.platform_name = "freebsd"
139
+ self.exe_name %= ""
140
+
141
+ def auto(self, executable_path=None, force=False, version_main=None, _=None):
142
+ """
143
+
144
+ Args:
145
+ executable_path:
146
+ force:
147
+ version_main:
148
+
149
+ Returns:
150
+
151
+ """
152
+ p = pathlib.Path(self.data_path)
153
+ if self.user_multi_procs:
154
+ with Lock():
155
+ files = list(p.rglob("*chromedriver*"))
156
+ most_recent = max(files, key=lambda f: f.stat().st_mtime)
157
+ files.remove(most_recent)
158
+ list(map(lambda f: f.unlink(), files))
159
+ if self.is_binary_patched(most_recent):
160
+ self.executable_path = str(most_recent)
161
+ return True
162
+
163
+ if executable_path:
164
+ self.executable_path = executable_path
165
+ self._custom_exe_path = True
166
+
167
+ if self._custom_exe_path:
168
+ ispatched = self.is_binary_patched(self.executable_path)
169
+ if not ispatched:
170
+ return self.patch_exe()
171
+ else:
172
+ return
173
+
174
+ if version_main:
175
+ self.version_main = version_main
176
+ if force is True:
177
+ self.force = force
178
+
179
+
180
+ if self.platform_name == "freebsd":
181
+ chromedriver_path = shutil.which("chromedriver")
182
+
183
+ if not os.path.isfile(chromedriver_path) or not os.access(chromedriver_path, os.X_OK):
184
+ logging.error("Chromedriver not installed!")
185
+ return
186
+
187
+ version_path = os.path.join(os.path.dirname(self.executable_path), "version.txt")
188
+
189
+ process = os.popen(f'"{chromedriver_path}" --version')
190
+ chromedriver_version = process.read().split(' ')[1].split(' ')[0]
191
+ process.close()
192
+
193
+ current_version = None
194
+ if os.path.isfile(version_path) or os.access(version_path, os.X_OK):
195
+ with open(version_path, 'r') as f:
196
+ current_version = f.read()
197
+
198
+ if current_version != chromedriver_version:
199
+ logging.info("Copying chromedriver executable...")
200
+ shutil.copy(chromedriver_path, self.executable_path)
201
+ os.chmod(self.executable_path, 0o755)
202
+
203
+ with open(version_path, 'w') as f:
204
+ f.write(chromedriver_version)
205
+
206
+ logging.info("Chromedriver executable copied!")
207
+ else:
208
+ try:
209
+ os.unlink(self.executable_path)
210
+ except PermissionError:
211
+ if self.force:
212
+ self.force_kill_instances(self.executable_path)
213
+ return self.auto(force=not self.force)
214
+ try:
215
+ if self.is_binary_patched():
216
+ # assumes already running AND patched
217
+ return True
218
+ except PermissionError:
219
+ pass
220
+ # return False
221
+ except FileNotFoundError:
222
+ pass
223
+
224
+ release = self.fetch_release_number()
225
+ self.version_main = release.version[0]
226
+ self.version_full = release
227
+ self.unzip_package(self.fetch_package())
228
+
229
+ return self.patch()
230
+
231
+ def driver_binary_in_use(self, path: str = None) -> bool:
232
+ """
233
+ naive test to check if a found chromedriver binary is
234
+ currently in use
235
+
236
+ Args:
237
+ path: a string or PathLike object to the binary to check.
238
+ if not specified, we check use this object's executable_path
239
+ """
240
+ if not path:
241
+ path = self.executable_path
242
+ p = pathlib.Path(path)
243
+
244
+ if not p.exists():
245
+ raise OSError("file does not exist: %s" % p)
246
+ try:
247
+ with open(p, mode="a+b") as fs:
248
+ exc = []
249
+ try:
250
+
251
+ fs.seek(0, 0)
252
+ except PermissionError as e:
253
+ exc.append(e) # since some systems apprently allow seeking
254
+ # we conduct another test
255
+ try:
256
+ fs.readline()
257
+ except PermissionError as e:
258
+ exc.append(e)
259
+
260
+ if exc:
261
+
262
+ return True
263
+ return False
264
+ # ok safe to assume this is in use
265
+ except Exception as e:
266
+ # logger.exception("whoops ", e)
267
+ pass
268
+
269
+ def cleanup_unused_files(self):
270
+ p = pathlib.Path(self.data_path)
271
+ items = list(p.glob("*undetected*"))
272
+ for item in items:
273
+ try:
274
+ item.unlink()
275
+ except:
276
+ pass
277
+
278
+ def patch(self):
279
+ self.patch_exe()
280
+ return self.is_binary_patched()
281
+
282
+ def fetch_release_number(self):
283
+ """
284
+ Gets the latest major version available, or the latest major version of self.target_version if set explicitly.
285
+ :return: version string
286
+ :rtype: LooseVersion
287
+ """
288
+ # Endpoint for old versions of Chromedriver (114 and below)
289
+ if self.is_old_chromedriver:
290
+ path = f"/latest_release_{self.version_main}"
291
+ path = path.upper()
292
+ logger.debug("getting release number from %s" % path)
293
+ return LooseVersion(urlopen(self.url_repo + path).read().decode())
294
+
295
+ # Endpoint for new versions of Chromedriver (115+)
296
+ if not self.version_main:
297
+ # Fetch the latest version
298
+ path = "/last-known-good-versions-with-downloads.json"
299
+ logger.debug("getting release number from %s" % path)
300
+ with urlopen(self.url_repo + path) as conn:
301
+ response = conn.read().decode()
302
+
303
+ last_versions = json.loads(response)
304
+ return LooseVersion(last_versions["channels"]["Stable"]["version"])
305
+
306
+ # Fetch the latest minor version of the major version provided
307
+ path = "/latest-versions-per-milestone-with-downloads.json"
308
+ logger.debug("getting release number from %s" % path)
309
+ with urlopen(self.url_repo + path) as conn:
310
+ response = conn.read().decode()
311
+
312
+ major_versions = json.loads(response)
313
+ return LooseVersion(major_versions["milestones"][str(self.version_main)]["version"])
314
+
315
+ def parse_exe_version(self):
316
+ with io.open(self.executable_path, "rb") as f:
317
+ for line in iter(lambda: f.readline(), b""):
318
+ match = re.search(rb"platform_handle\x00content\x00([0-9.]*)", line)
319
+ if match:
320
+ return LooseVersion(match[1].decode())
321
+
322
+ def fetch_package(self):
323
+ """
324
+ Downloads ChromeDriver from source
325
+
326
+ :return: path to downloaded file
327
+ """
328
+ zip_name = f"chromedriver_{self.platform_name}.zip"
329
+ if self.is_old_chromedriver:
330
+ download_url = "%s/%s/%s" % (self.url_repo, self.version_full.vstring, zip_name)
331
+ else:
332
+ zip_name = zip_name.replace("_", "-", 1)
333
+ download_url = "https://storage.googleapis.com/chrome-for-testing-public/%s/%s/%s"
334
+ download_url %= (self.version_full.vstring, self.platform_name, zip_name)
335
+
336
+ logger.debug("downloading from %s" % download_url)
337
+ return urlretrieve(download_url)[0]
338
+
339
+ def unzip_package(self, fp):
340
+ """
341
+ Does what it says
342
+
343
+ :return: path to unpacked executable
344
+ """
345
+ exe_path = self.exe_name
346
+ if not self.is_old_chromedriver:
347
+ # The new chromedriver unzips into its own folder
348
+ zip_name = f"chromedriver-{self.platform_name}"
349
+ exe_path = os.path.join(zip_name, self.exe_name)
350
+
351
+ logger.debug("unzipping %s" % fp)
352
+ try:
353
+ os.unlink(self.zip_path)
354
+ except (FileNotFoundError, OSError):
355
+ pass
356
+
357
+ os.makedirs(self.zip_path, mode=0o755, exist_ok=True)
358
+ with zipfile.ZipFile(fp, mode="r") as zf:
359
+ zf.extractall(self.zip_path)
360
+ os.rename(os.path.join(self.zip_path, exe_path), self.executable_path)
361
+ os.remove(fp)
362
+ shutil.rmtree
363
+ os.chmod(self.executable_path, 0o755)
364
+ return self.executable_path
365
+
366
+ @staticmethod
367
+ def force_kill_instances(exe_name):
368
+ """
369
+ kills running instances.
370
+ :param: executable name to kill, may be a path as well
371
+
372
+ :return: True on success else False
373
+ """
374
+ exe_name = os.path.basename(exe_name)
375
+ if IS_POSIX:
376
+ r = os.system("kill -f -9 $(pidof %s)" % exe_name)
377
+ else:
378
+ r = os.system("taskkill /f /im %s" % exe_name)
379
+ return not r
380
+
381
+ @staticmethod
382
+ def gen_random_cdc():
383
+ cdc = random.choices(string.ascii_letters, k=27)
384
+ return "".join(cdc).encode()
385
+
386
+ def is_binary_patched(self, executable_path=None):
387
+ executable_path = executable_path or self.executable_path
388
+ try:
389
+ with io.open(executable_path, "rb") as fh:
390
+ return fh.read().find(b"undetected chromedriver") != -1
391
+ except FileNotFoundError:
392
+ return False
393
+
394
+ def patch_exe(self):
395
+ start = time.perf_counter()
396
+ logger.info("patching driver executable %s" % self.executable_path)
397
+ with io.open(self.executable_path, "r+b") as fh:
398
+ content = fh.read()
399
+ # match_injected_codeblock = re.search(rb"{window.*;}", content)
400
+ match_injected_codeblock = re.search(rb"\{window\.cdc.*?;\}", content)
401
+ if match_injected_codeblock:
402
+ target_bytes = match_injected_codeblock[0]
403
+ new_target_bytes = (
404
+ b'{console.log("undetected chromedriver 1337!")}'.ljust(
405
+ len(target_bytes), b" "
406
+ )
407
+ )
408
+ new_content = content.replace(target_bytes, new_target_bytes)
409
+ if new_content == content:
410
+ logger.warning(
411
+ "something went wrong patching the driver binary. could not find injection code block"
412
+ )
413
+ else:
414
+ logger.debug(
415
+ "found block:\n%s\nreplacing with:\n%s"
416
+ % (target_bytes, new_target_bytes)
417
+ )
418
+ fh.seek(0)
419
+ fh.write(new_content)
420
+ logger.debug(
421
+ "patching took us {:.2f} seconds".format(time.perf_counter() - start)
422
+ )
423
+
424
+ def __repr__(self):
425
+ return "{0:s}({1:s})".format(
426
+ self.__class__.__name__,
427
+ self.executable_path,
428
+ )
429
+
430
+ def __del__(self):
431
+ if self._custom_exe_path:
432
+ # if the driver binary is specified by user
433
+ # we assume it is important enough to not delete it
434
+ return
435
+ else:
436
+ timeout = 3 # stop trying after this many seconds
437
+ t = time.monotonic()
438
+ now = lambda: time.monotonic()
439
+ while now() - t > timeout:
440
+ # we don't want to wait until the end of time
441
+ try:
442
+ if self.user_multi_procs:
443
+ break
444
+ os.unlink(self.executable_path)
445
+ logger.debug("successfully unlinked %s" % self.executable_path)
446
+ break
447
+ except (OSError, RuntimeError, PermissionError):
448
+ time.sleep(0.01)
449
+ continue
450
+ except FileNotFoundError:
451
+ break
src/undetected_chromedriver/reactor.py CHANGED
@@ -1,99 +1,99 @@
1
- #!/usr/bin/env python3
2
- # this module is part of undetected_chromedriver
3
-
4
- import asyncio
5
- import json
6
- import logging
7
- import threading
8
-
9
-
10
- logger = logging.getLogger(__name__)
11
-
12
-
13
- class Reactor(threading.Thread):
14
- def __init__(self, driver: "Chrome"):
15
- super().__init__()
16
-
17
- self.driver = driver
18
- self.loop = asyncio.new_event_loop()
19
-
20
- self.lock = threading.Lock()
21
- self.event = threading.Event()
22
- self.daemon = True
23
- self.handlers = {}
24
-
25
- def add_event_handler(self, method_name, callback: callable):
26
- """
27
-
28
- Parameters
29
- ----------
30
- event_name: str
31
- example "Network.responseReceived"
32
-
33
- callback: callable
34
- callable which accepts 1 parameter: the message object dictionary
35
-
36
- Returns
37
- -------
38
-
39
- """
40
- with self.lock:
41
- self.handlers[method_name.lower()] = callback
42
-
43
- @property
44
- def running(self):
45
- return not self.event.is_set()
46
-
47
- def run(self):
48
- try:
49
- asyncio.set_event_loop(self.loop)
50
- self.loop.run_until_complete(self.listen())
51
- except Exception as e:
52
- logger.warning("Reactor.run() => %s", e)
53
-
54
- async def _wait_service_started(self):
55
- while True:
56
- with self.lock:
57
- if (
58
- getattr(self.driver, "service", None)
59
- and getattr(self.driver.service, "process", None)
60
- and self.driver.service.process.poll()
61
- ):
62
- await asyncio.sleep(self.driver._delay or 0.25)
63
- else:
64
- break
65
-
66
- async def listen(self):
67
- while self.running:
68
- await self._wait_service_started()
69
- await asyncio.sleep(1)
70
-
71
- try:
72
- with self.lock:
73
- log_entries = self.driver.get_log("performance")
74
-
75
- for entry in log_entries:
76
- try:
77
- obj_serialized: str = entry.get("message")
78
- obj = json.loads(obj_serialized)
79
- message = obj.get("message")
80
- method = message.get("method")
81
-
82
- if "*" in self.handlers:
83
- await self.loop.run_in_executor(
84
- None, self.handlers["*"], message
85
- )
86
- elif method.lower() in self.handlers:
87
- await self.loop.run_in_executor(
88
- None, self.handlers[method.lower()], message
89
- )
90
-
91
- # print(type(message), message)
92
- except Exception as e:
93
- raise e from None
94
-
95
- except Exception as e:
96
- if "invalid session id" in str(e):
97
- pass
98
- else:
99
- logging.debug("exception ignored :", e)
 
1
+ #!/usr/bin/env python3
2
+ # this module is part of undetected_chromedriver
3
+
4
+ import asyncio
5
+ import json
6
+ import logging
7
+ import threading
8
+
9
+
10
+ logger = logging.getLogger(__name__)
11
+
12
+
13
+ class Reactor(threading.Thread):
14
+ def __init__(self, driver: "Chrome"):
15
+ super().__init__()
16
+
17
+ self.driver = driver
18
+ self.loop = asyncio.new_event_loop()
19
+
20
+ self.lock = threading.Lock()
21
+ self.event = threading.Event()
22
+ self.daemon = True
23
+ self.handlers = {}
24
+
25
+ def add_event_handler(self, method_name, callback: callable):
26
+ """
27
+
28
+ Parameters
29
+ ----------
30
+ event_name: str
31
+ example "Network.responseReceived"
32
+
33
+ callback: callable
34
+ callable which accepts 1 parameter: the message object dictionary
35
+
36
+ Returns
37
+ -------
38
+
39
+ """
40
+ with self.lock:
41
+ self.handlers[method_name.lower()] = callback
42
+
43
+ @property
44
+ def running(self):
45
+ return not self.event.is_set()
46
+
47
+ def run(self):
48
+ try:
49
+ asyncio.set_event_loop(self.loop)
50
+ self.loop.run_until_complete(self.listen())
51
+ except Exception as e:
52
+ logger.warning("Reactor.run() => %s", e)
53
+
54
+ async def _wait_service_started(self):
55
+ while True:
56
+ with self.lock:
57
+ if (
58
+ getattr(self.driver, "service", None)
59
+ and getattr(self.driver.service, "process", None)
60
+ and self.driver.service.process.poll()
61
+ ):
62
+ await asyncio.sleep(self.driver._delay or 0.25)
63
+ else:
64
+ break
65
+
66
+ async def listen(self):
67
+ while self.running:
68
+ await self._wait_service_started()
69
+ await asyncio.sleep(1)
70
+
71
+ try:
72
+ with self.lock:
73
+ log_entries = self.driver.get_log("performance")
74
+
75
+ for entry in log_entries:
76
+ try:
77
+ obj_serialized: str = entry.get("message")
78
+ obj = json.loads(obj_serialized)
79
+ message = obj.get("message")
80
+ method = message.get("method")
81
+
82
+ if "*" in self.handlers:
83
+ await self.loop.run_in_executor(
84
+ None, self.handlers["*"], message
85
+ )
86
+ elif method.lower() in self.handlers:
87
+ await self.loop.run_in_executor(
88
+ None, self.handlers[method.lower()], message
89
+ )
90
+
91
+ # print(type(message), message)
92
+ except Exception as e:
93
+ raise e from None
94
+
95
+ except Exception as e:
96
+ if "invalid session id" in str(e):
97
+ pass
98
+ else:
99
+ logging.debug("exception ignored :", e)
src/undetected_chromedriver/webelement.py CHANGED
@@ -1,86 +1,86 @@
1
- from typing import List
2
-
3
- from selenium.webdriver.common.by import By
4
- import selenium.webdriver.remote.webelement
5
-
6
-
7
- class WebElement(selenium.webdriver.remote.webelement.WebElement):
8
- def click_safe(self):
9
- super().click()
10
- self._parent.reconnect(0.1)
11
-
12
- def children(
13
- self, tag=None, recursive=False
14
- ) -> List[selenium.webdriver.remote.webelement.WebElement]:
15
- """
16
- returns direct child elements of current element
17
- :param tag: str, if supplied, returns <tag> nodes only
18
- """
19
- script = "return [... arguments[0].children]"
20
- if tag:
21
- script += ".filter( node => node.tagName === '%s')" % tag.upper()
22
- if recursive:
23
- return list(_recursive_children(self, tag))
24
- return list(self._parent.execute_script(script, self))
25
-
26
-
27
- class UCWebElement(WebElement):
28
- """
29
- Custom WebElement class which makes it easier to view elements when
30
- working in an interactive environment.
31
-
32
- standard webelement repr:
33
- <selenium.webdriver.remote.webelement.WebElement (session="85ff0f671512fa535630e71ee951b1f2", element="6357cb55-92c3-4c0f-9416-b174f9c1b8c4")>
34
-
35
- using this WebElement class:
36
- <WebElement(<a class="mobile-show-inline-block mc-update-infos init-ok" href="#" id="main-cat-switcher-mobile">)>
37
-
38
- """
39
-
40
- def __init__(self, parent, id_):
41
- super().__init__(parent, id_)
42
- self._attrs = None
43
-
44
- @property
45
- def attrs(self):
46
- if not self._attrs:
47
- self._attrs = self._parent.execute_script(
48
- """
49
- var items = {};
50
- for (index = 0; index < arguments[0].attributes.length; ++index)
51
- {
52
- items[arguments[0].attributes[index].name] = arguments[0].attributes[index].value
53
- };
54
- return items;
55
- """,
56
- self,
57
- )
58
- return self._attrs
59
-
60
- def __repr__(self):
61
- strattrs = " ".join([f'{k}="{v}"' for k, v in self.attrs.items()])
62
- if strattrs:
63
- strattrs = " " + strattrs
64
- return f"{self.__class__.__name__} <{self.tag_name}{strattrs}>"
65
-
66
-
67
- def _recursive_children(element, tag: str = None, _results=None):
68
- """
69
- returns all children of <element> recursively
70
-
71
- :param element: `WebElement` object.
72
- find children below this <element>
73
-
74
- :param tag: str = None.
75
- if provided, return only <tag> elements. example: 'a', or 'img'
76
- :param _results: do not use!
77
- """
78
- results = _results or set()
79
- for element in element.children():
80
- if tag:
81
- if element.tag_name == tag:
82
- results.add(element)
83
- else:
84
- results.add(element)
85
- results |= _recursive_children(element, tag, results)
86
- return results
 
1
+ from typing import List
2
+
3
+ from selenium.webdriver.common.by import By
4
+ import selenium.webdriver.remote.webelement
5
+
6
+
7
+ class WebElement(selenium.webdriver.remote.webelement.WebElement):
8
+ def click_safe(self):
9
+ super().click()
10
+ self._parent.reconnect(0.1)
11
+
12
+ def children(
13
+ self, tag=None, recursive=False
14
+ ) -> List[selenium.webdriver.remote.webelement.WebElement]:
15
+ """
16
+ returns direct child elements of current element
17
+ :param tag: str, if supplied, returns <tag> nodes only
18
+ """
19
+ script = "return [... arguments[0].children]"
20
+ if tag:
21
+ script += ".filter( node => node.tagName === '%s')" % tag.upper()
22
+ if recursive:
23
+ return list(_recursive_children(self, tag))
24
+ return list(self._parent.execute_script(script, self))
25
+
26
+
27
+ class UCWebElement(WebElement):
28
+ """
29
+ Custom WebElement class which makes it easier to view elements when
30
+ working in an interactive environment.
31
+
32
+ standard webelement repr:
33
+ <selenium.webdriver.remote.webelement.WebElement (session="85ff0f671512fa535630e71ee951b1f2", element="6357cb55-92c3-4c0f-9416-b174f9c1b8c4")>
34
+
35
+ using this WebElement class:
36
+ <WebElement(<a class="mobile-show-inline-block mc-update-infos init-ok" href="#" id="main-cat-switcher-mobile">)>
37
+
38
+ """
39
+
40
+ def __init__(self, parent, id_):
41
+ super().__init__(parent, id_)
42
+ self._attrs = None
43
+
44
+ @property
45
+ def attrs(self):
46
+ if not self._attrs:
47
+ self._attrs = self._parent.execute_script(
48
+ """
49
+ var items = {};
50
+ for (index = 0; index < arguments[0].attributes.length; ++index)
51
+ {
52
+ items[arguments[0].attributes[index].name] = arguments[0].attributes[index].value
53
+ };
54
+ return items;
55
+ """,
56
+ self,
57
+ )
58
+ return self._attrs
59
+
60
+ def __repr__(self):
61
+ strattrs = " ".join([f'{k}="{v}"' for k, v in self.attrs.items()])
62
+ if strattrs:
63
+ strattrs = " " + strattrs
64
+ return f"{self.__class__.__name__} <{self.tag_name}{strattrs}>"
65
+
66
+
67
+ def _recursive_children(element, tag: str = None, _results=None):
68
+ """
69
+ returns all children of <element> recursively
70
+
71
+ :param element: `WebElement` object.
72
+ find children below this <element>
73
+
74
+ :param tag: str = None.
75
+ if provided, return only <tag> elements. example: 'a', or 'img'
76
+ :param _results: do not use!
77
+ """
78
+ results = _results or set()
79
+ for element in element.children():
80
+ if tag:
81
+ if element.tag_name == tag:
82
+ results.add(element)
83
+ else:
84
+ results.add(element)
85
+ results |= _recursive_children(element, tag, results)
86
+ return results
src/utils.py CHANGED
@@ -1,341 +1,341 @@
1
- import json
2
- import logging
3
- import os
4
- import re
5
- import shutil
6
- import urllib.parse
7
- import tempfile
8
- import sys
9
-
10
- from selenium.webdriver.chrome.webdriver import WebDriver
11
- import undetected_chromedriver as uc
12
-
13
- FLARESOLVERR_VERSION = None
14
- PLATFORM_VERSION = None
15
- CHROME_EXE_PATH = None
16
- CHROME_MAJOR_VERSION = None
17
- USER_AGENT = None
18
- XVFB_DISPLAY = None
19
- PATCHED_DRIVER_PATH = None
20
-
21
-
22
- def get_config_log_html() -> bool:
23
- return os.environ.get('LOG_HTML', 'false').lower() == 'true'
24
-
25
-
26
- def get_config_headless() -> bool:
27
- return os.environ.get('HEADLESS', 'true').lower() == 'true'
28
-
29
-
30
- def get_flaresolverr_version() -> str:
31
- global FLARESOLVERR_VERSION
32
- if FLARESOLVERR_VERSION is not None:
33
- return FLARESOLVERR_VERSION
34
-
35
- package_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'package.json')
36
- if not os.path.isfile(package_path):
37
- package_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'package.json')
38
- with open(package_path) as f:
39
- FLARESOLVERR_VERSION = json.loads(f.read())['version']
40
- return FLARESOLVERR_VERSION
41
-
42
- def get_current_platform() -> str:
43
- global PLATFORM_VERSION
44
- if PLATFORM_VERSION is not None:
45
- return PLATFORM_VERSION
46
- PLATFORM_VERSION = os.name
47
- return PLATFORM_VERSION
48
-
49
-
50
- def create_proxy_extension(proxy: dict) -> str:
51
- parsed_url = urllib.parse.urlparse(proxy['url'])
52
- scheme = parsed_url.scheme
53
- host = parsed_url.hostname
54
- port = parsed_url.port
55
- username = proxy['username']
56
- password = proxy['password']
57
- manifest_json = """
58
- {
59
- "version": "1.0.0",
60
- "manifest_version": 2,
61
- "name": "Chrome Proxy",
62
- "permissions": [
63
- "proxy",
64
- "tabs",
65
- "unlimitedStorage",
66
- "storage",
67
- "<all_urls>",
68
- "webRequest",
69
- "webRequestBlocking"
70
- ],
71
- "background": {"scripts": ["background.js"]},
72
- "minimum_chrome_version": "76.0.0"
73
- }
74
- """
75
-
76
- background_js = """
77
- var config = {
78
- mode: "fixed_servers",
79
- rules: {
80
- singleProxy: {
81
- scheme: "%s",
82
- host: "%s",
83
- port: %d
84
- },
85
- bypassList: ["localhost"]
86
- }
87
- };
88
-
89
- chrome.proxy.settings.set({value: config, scope: "regular"}, function() {});
90
-
91
- function callbackFn(details) {
92
- return {
93
- authCredentials: {
94
- username: "%s",
95
- password: "%s"
96
- }
97
- };
98
- }
99
-
100
- chrome.webRequest.onAuthRequired.addListener(
101
- callbackFn,
102
- { urls: ["<all_urls>"] },
103
- ['blocking']
104
- );
105
- """ % (
106
- scheme,
107
- host,
108
- port,
109
- username,
110
- password
111
- )
112
-
113
- proxy_extension_dir = tempfile.mkdtemp()
114
-
115
- with open(os.path.join(proxy_extension_dir, "manifest.json"), "w") as f:
116
- f.write(manifest_json)
117
-
118
- with open(os.path.join(proxy_extension_dir, "background.js"), "w") as f:
119
- f.write(background_js)
120
-
121
- return proxy_extension_dir
122
-
123
-
124
- def get_webdriver(proxy: dict = None) -> WebDriver:
125
- global PATCHED_DRIVER_PATH, USER_AGENT
126
- logging.debug('Launching web browser...')
127
-
128
- # undetected_chromedriver
129
- options = uc.ChromeOptions()
130
- options.add_argument('--no-sandbox')
131
- options.add_argument('--window-size=1920,1080')
132
- # todo: this param shows a warning in chrome head-full
133
- options.add_argument('--disable-setuid-sandbox')
134
- options.add_argument('--disable-dev-shm-usage')
135
- # this option removes the zygote sandbox (it seems that the resolution is a bit faster)
136
- options.add_argument('--no-zygote')
137
- # attempt to fix Docker ARM32 build
138
- options.add_argument('--disable-gpu-sandbox')
139
- options.add_argument('--disable-software-rasterizer')
140
- options.add_argument('--ignore-certificate-errors')
141
- options.add_argument('--ignore-ssl-errors')
142
- # fix GL errors in ASUSTOR NAS
143
- # https://github.com/FlareSolverr/FlareSolverr/issues/782
144
- # https://github.com/microsoft/vscode/issues/127800#issuecomment-873342069
145
- # https://peter.sh/experiments/chromium-command-line-switches/#use-gl
146
- options.add_argument('--use-gl=swiftshader')
147
-
148
- language = os.environ.get('LANG', None)
149
- if language is not None:
150
- options.add_argument('--accept-lang=%s' % language)
151
-
152
- # Fix for Chrome 117 | https://github.com/FlareSolverr/FlareSolverr/issues/910
153
- if USER_AGENT is not None:
154
- options.add_argument('--user-agent=%s' % USER_AGENT)
155
-
156
- proxy_extension_dir = None
157
- if proxy and all(key in proxy for key in ['url', 'username', 'password']):
158
- proxy_extension_dir = create_proxy_extension(proxy)
159
- options.add_argument("--load-extension=%s" % os.path.abspath(proxy_extension_dir))
160
- elif proxy and 'url' in proxy:
161
- proxy_url = proxy['url']
162
- logging.debug("Using webdriver proxy: %s", proxy_url)
163
- options.add_argument('--proxy-server=%s' % proxy_url)
164
-
165
- # note: headless mode is detected (headless = True)
166
- # we launch the browser in head-full mode with the window hidden
167
- windows_headless = False
168
- if get_config_headless():
169
- if os.name == 'nt':
170
- windows_headless = True
171
- else:
172
- start_xvfb_display()
173
- # For normal headless mode:
174
- # options.add_argument('--headless')
175
-
176
- options.add_argument("--auto-open-devtools-for-tabs")
177
-
178
- # if we are inside the Docker container, we avoid downloading the driver
179
- driver_exe_path = None
180
- version_main = None
181
- if os.path.exists("/app/chromedriver"):
182
- # running inside Docker
183
- driver_exe_path = "/app/chromedriver"
184
- else:
185
- version_main = get_chrome_major_version()
186
- if PATCHED_DRIVER_PATH is not None:
187
- driver_exe_path = PATCHED_DRIVER_PATH
188
-
189
- # detect chrome path
190
- browser_executable_path = get_chrome_exe_path()
191
-
192
- # downloads and patches the chromedriver
193
- # if we don't set driver_executable_path it downloads, patches, and deletes the driver each time
194
- try:
195
- driver = uc.Chrome(options=options, browser_executable_path=browser_executable_path,
196
- driver_executable_path=driver_exe_path, version_main=version_main,
197
- windows_headless=windows_headless, headless=get_config_headless())
198
- except Exception as e:
199
- logging.error("Error starting Chrome: %s" % e)
200
-
201
- # save the patched driver to avoid re-downloads
202
- if driver_exe_path is None:
203
- PATCHED_DRIVER_PATH = os.path.join(driver.patcher.data_path, driver.patcher.exe_name)
204
- if PATCHED_DRIVER_PATH != driver.patcher.executable_path:
205
- shutil.copy(driver.patcher.executable_path, PATCHED_DRIVER_PATH)
206
-
207
- # clean up proxy extension directory
208
- if proxy_extension_dir is not None:
209
- shutil.rmtree(proxy_extension_dir)
210
-
211
- # selenium vanilla
212
- # options = webdriver.ChromeOptions()
213
- # options.add_argument('--no-sandbox')
214
- # options.add_argument('--window-size=1920,1080')
215
- # options.add_argument('--disable-setuid-sandbox')
216
- # options.add_argument('--disable-dev-shm-usage')
217
- # driver = webdriver.Chrome(options=options)
218
-
219
- return driver
220
-
221
-
222
- def get_chrome_exe_path() -> str:
223
- global CHROME_EXE_PATH
224
- if CHROME_EXE_PATH is not None:
225
- return CHROME_EXE_PATH
226
- # linux pyinstaller bundle
227
- chrome_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'chrome', "chrome")
228
- if os.path.exists(chrome_path):
229
- if not os.access(chrome_path, os.X_OK):
230
- raise Exception(f'Chrome binary "{chrome_path}" is not executable. '
231
- f'Please, extract the archive with "tar xzf <file.tar.gz>".')
232
- CHROME_EXE_PATH = chrome_path
233
- return CHROME_EXE_PATH
234
- # windows pyinstaller bundle
235
- chrome_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'chrome', "chrome.exe")
236
- if os.path.exists(chrome_path):
237
- CHROME_EXE_PATH = chrome_path
238
- return CHROME_EXE_PATH
239
- # system
240
- CHROME_EXE_PATH = uc.find_chrome_executable()
241
- return CHROME_EXE_PATH
242
-
243
-
244
- def get_chrome_major_version() -> str:
245
- global CHROME_MAJOR_VERSION
246
- if CHROME_MAJOR_VERSION is not None:
247
- return CHROME_MAJOR_VERSION
248
-
249
- if os.name == 'nt':
250
- # Example: '104.0.5112.79'
251
- try:
252
- complete_version = extract_version_nt_executable(get_chrome_exe_path())
253
- except Exception:
254
- try:
255
- complete_version = extract_version_nt_registry()
256
- except Exception:
257
- # Example: '104.0.5112.79'
258
- complete_version = extract_version_nt_folder()
259
- else:
260
- chrome_path = get_chrome_exe_path()
261
- process = os.popen(f'"{chrome_path}" --version')
262
- # Example 1: 'Chromium 104.0.5112.79 Arch Linux\n'
263
- # Example 2: 'Google Chrome 104.0.5112.79 Arch Linux\n'
264
- complete_version = process.read()
265
- process.close()
266
-
267
- CHROME_MAJOR_VERSION = complete_version.split('.')[0].split(' ')[-1]
268
- return CHROME_MAJOR_VERSION
269
-
270
-
271
- def extract_version_nt_executable(exe_path: str) -> str:
272
- import pefile
273
- pe = pefile.PE(exe_path, fast_load=True)
274
- pe.parse_data_directories(
275
- directories=[pefile.DIRECTORY_ENTRY["IMAGE_DIRECTORY_ENTRY_RESOURCE"]]
276
- )
277
- return pe.FileInfo[0][0].StringTable[0].entries[b"FileVersion"].decode('utf-8')
278
-
279
-
280
- def extract_version_nt_registry() -> str:
281
- stream = os.popen(
282
- 'reg query "HKLM\\SOFTWARE\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\Google Chrome"')
283
- output = stream.read()
284
- google_version = ''
285
- for letter in output[output.rindex('DisplayVersion REG_SZ') + 24:]:
286
- if letter != '\n':
287
- google_version += letter
288
- else:
289
- break
290
- return google_version.strip()
291
-
292
-
293
- def extract_version_nt_folder() -> str:
294
- # Check if the Chrome folder exists in the x32 or x64 Program Files folders.
295
- for i in range(2):
296
- path = 'C:\\Program Files' + (' (x86)' if i else '') + '\\Google\\Chrome\\Application'
297
- if os.path.isdir(path):
298
- paths = [f.path for f in os.scandir(path) if f.is_dir()]
299
- for path in paths:
300
- filename = os.path.basename(path)
301
- pattern = '\d+\.\d+\.\d+\.\d+'
302
- match = re.search(pattern, filename)
303
- if match and match.group():
304
- # Found a Chrome version.
305
- return match.group(0)
306
- return ''
307
-
308
-
309
- def get_user_agent(driver=None) -> str:
310
- global USER_AGENT
311
- if USER_AGENT is not None:
312
- return USER_AGENT
313
-
314
- try:
315
- if driver is None:
316
- driver = get_webdriver()
317
- USER_AGENT = driver.execute_script("return navigator.userAgent")
318
- # Fix for Chrome 117 | https://github.com/FlareSolverr/FlareSolverr/issues/910
319
- USER_AGENT = re.sub('HEADLESS', '', USER_AGENT, flags=re.IGNORECASE)
320
- return USER_AGENT
321
- except Exception as e:
322
- raise Exception("Error getting browser User-Agent. " + str(e))
323
- finally:
324
- if driver is not None:
325
- if PLATFORM_VERSION == "nt":
326
- driver.close()
327
- driver.quit()
328
-
329
-
330
- def start_xvfb_display():
331
- global XVFB_DISPLAY
332
- if XVFB_DISPLAY is None:
333
- from xvfbwrapper import Xvfb
334
- XVFB_DISPLAY = Xvfb()
335
- XVFB_DISPLAY.start()
336
-
337
-
338
- def object_to_dict(_object):
339
- json_dict = json.loads(json.dumps(_object, default=lambda o: o.__dict__))
340
- # remove hidden fields
341
- return {k: v for k, v in json_dict.items() if not k.startswith('__')}
 
1
+ import json
2
+ import logging
3
+ import os
4
+ import re
5
+ import shutil
6
+ import urllib.parse
7
+ import tempfile
8
+ import sys
9
+
10
+ from selenium.webdriver.chrome.webdriver import WebDriver
11
+ import undetected_chromedriver as uc
12
+
13
+ FLARESOLVERR_VERSION = None
14
+ PLATFORM_VERSION = None
15
+ CHROME_EXE_PATH = None
16
+ CHROME_MAJOR_VERSION = None
17
+ USER_AGENT = None
18
+ XVFB_DISPLAY = None
19
+ PATCHED_DRIVER_PATH = None
20
+
21
+
22
+ def get_config_log_html() -> bool:
23
+ return os.environ.get('LOG_HTML', 'false').lower() == 'true'
24
+
25
+
26
+ def get_config_headless() -> bool:
27
+ return os.environ.get('HEADLESS', 'true').lower() == 'true'
28
+
29
+
30
+ def get_flaresolverr_version() -> str:
31
+ global FLARESOLVERR_VERSION
32
+ if FLARESOLVERR_VERSION is not None:
33
+ return FLARESOLVERR_VERSION
34
+
35
+ package_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'package.json')
36
+ if not os.path.isfile(package_path):
37
+ package_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'package.json')
38
+ with open(package_path) as f:
39
+ FLARESOLVERR_VERSION = json.loads(f.read())['version']
40
+ return FLARESOLVERR_VERSION
41
+
42
+ def get_current_platform() -> str:
43
+ global PLATFORM_VERSION
44
+ if PLATFORM_VERSION is not None:
45
+ return PLATFORM_VERSION
46
+ PLATFORM_VERSION = os.name
47
+ return PLATFORM_VERSION
48
+
49
+
50
+ def create_proxy_extension(proxy: dict) -> str:
51
+ parsed_url = urllib.parse.urlparse(proxy['url'])
52
+ scheme = parsed_url.scheme
53
+ host = parsed_url.hostname
54
+ port = parsed_url.port
55
+ username = proxy['username']
56
+ password = proxy['password']
57
+ manifest_json = """
58
+ {
59
+ "version": "1.0.0",
60
+ "manifest_version": 2,
61
+ "name": "Chrome Proxy",
62
+ "permissions": [
63
+ "proxy",
64
+ "tabs",
65
+ "unlimitedStorage",
66
+ "storage",
67
+ "<all_urls>",
68
+ "webRequest",
69
+ "webRequestBlocking"
70
+ ],
71
+ "background": {"scripts": ["background.js"]},
72
+ "minimum_chrome_version": "76.0.0"
73
+ }
74
+ """
75
+
76
+ background_js = """
77
+ var config = {
78
+ mode: "fixed_servers",
79
+ rules: {
80
+ singleProxy: {
81
+ scheme: "%s",
82
+ host: "%s",
83
+ port: %d
84
+ },
85
+ bypassList: ["localhost"]
86
+ }
87
+ };
88
+
89
+ chrome.proxy.settings.set({value: config, scope: "regular"}, function() {});
90
+
91
+ function callbackFn(details) {
92
+ return {
93
+ authCredentials: {
94
+ username: "%s",
95
+ password: "%s"
96
+ }
97
+ };
98
+ }
99
+
100
+ chrome.webRequest.onAuthRequired.addListener(
101
+ callbackFn,
102
+ { urls: ["<all_urls>"] },
103
+ ['blocking']
104
+ );
105
+ """ % (
106
+ scheme,
107
+ host,
108
+ port,
109
+ username,
110
+ password
111
+ )
112
+
113
+ proxy_extension_dir = tempfile.mkdtemp()
114
+
115
+ with open(os.path.join(proxy_extension_dir, "manifest.json"), "w") as f:
116
+ f.write(manifest_json)
117
+
118
+ with open(os.path.join(proxy_extension_dir, "background.js"), "w") as f:
119
+ f.write(background_js)
120
+
121
+ return proxy_extension_dir
122
+
123
+
124
+ def get_webdriver(proxy: dict = None) -> WebDriver:
125
+ global PATCHED_DRIVER_PATH, USER_AGENT
126
+ logging.debug('Launching web browser...')
127
+
128
+ # undetected_chromedriver
129
+ options = uc.ChromeOptions()
130
+ options.add_argument('--no-sandbox')
131
+ options.add_argument('--window-size=1920,1080')
132
+ # todo: this param shows a warning in chrome head-full
133
+ options.add_argument('--disable-setuid-sandbox')
134
+ options.add_argument('--disable-dev-shm-usage')
135
+ # this option removes the zygote sandbox (it seems that the resolution is a bit faster)
136
+ options.add_argument('--no-zygote')
137
+ # attempt to fix Docker ARM32 build
138
+ options.add_argument('--disable-gpu-sandbox')
139
+ options.add_argument('--disable-software-rasterizer')
140
+ options.add_argument('--ignore-certificate-errors')
141
+ options.add_argument('--ignore-ssl-errors')
142
+ # fix GL errors in ASUSTOR NAS
143
+ # https://github.com/FlareSolverr/FlareSolverr/issues/782
144
+ # https://github.com/microsoft/vscode/issues/127800#issuecomment-873342069
145
+ # https://peter.sh/experiments/chromium-command-line-switches/#use-gl
146
+ options.add_argument('--use-gl=swiftshader')
147
+
148
+ language = os.environ.get('LANG', None)
149
+ if language is not None:
150
+ options.add_argument('--accept-lang=%s' % language)
151
+
152
+ # Fix for Chrome 117 | https://github.com/FlareSolverr/FlareSolverr/issues/910
153
+ if USER_AGENT is not None:
154
+ options.add_argument('--user-agent=%s' % USER_AGENT)
155
+
156
+ proxy_extension_dir = None
157
+ if proxy and all(key in proxy for key in ['url', 'username', 'password']):
158
+ proxy_extension_dir = create_proxy_extension(proxy)
159
+ options.add_argument("--load-extension=%s" % os.path.abspath(proxy_extension_dir))
160
+ elif proxy and 'url' in proxy:
161
+ proxy_url = proxy['url']
162
+ logging.debug("Using webdriver proxy: %s", proxy_url)
163
+ options.add_argument('--proxy-server=%s' % proxy_url)
164
+
165
+ # note: headless mode is detected (headless = True)
166
+ # we launch the browser in head-full mode with the window hidden
167
+ windows_headless = False
168
+ if get_config_headless():
169
+ if os.name == 'nt':
170
+ windows_headless = True
171
+ else:
172
+ start_xvfb_display()
173
+ # For normal headless mode:
174
+ # options.add_argument('--headless')
175
+
176
+ options.add_argument("--auto-open-devtools-for-tabs")
177
+
178
+ # if we are inside the Docker container, we avoid downloading the driver
179
+ driver_exe_path = None
180
+ version_main = None
181
+ if os.path.exists("/app/chromedriver"):
182
+ # running inside Docker
183
+ driver_exe_path = "/app/chromedriver"
184
+ else:
185
+ version_main = get_chrome_major_version()
186
+ if PATCHED_DRIVER_PATH is not None:
187
+ driver_exe_path = PATCHED_DRIVER_PATH
188
+
189
+ # detect chrome path
190
+ browser_executable_path = get_chrome_exe_path()
191
+
192
+ # downloads and patches the chromedriver
193
+ # if we don't set driver_executable_path it downloads, patches, and deletes the driver each time
194
+ try:
195
+ driver = uc.Chrome(options=options, browser_executable_path=browser_executable_path,
196
+ driver_executable_path=driver_exe_path, version_main=version_main,
197
+ windows_headless=windows_headless, headless=get_config_headless())
198
+ except Exception as e:
199
+ logging.error("Error starting Chrome: %s" % e)
200
+
201
+ # save the patched driver to avoid re-downloads
202
+ if driver_exe_path is None:
203
+ PATCHED_DRIVER_PATH = os.path.join(driver.patcher.data_path, driver.patcher.exe_name)
204
+ if PATCHED_DRIVER_PATH != driver.patcher.executable_path:
205
+ shutil.copy(driver.patcher.executable_path, PATCHED_DRIVER_PATH)
206
+
207
+ # clean up proxy extension directory
208
+ if proxy_extension_dir is not None:
209
+ shutil.rmtree(proxy_extension_dir)
210
+
211
+ # selenium vanilla
212
+ # options = webdriver.ChromeOptions()
213
+ # options.add_argument('--no-sandbox')
214
+ # options.add_argument('--window-size=1920,1080')
215
+ # options.add_argument('--disable-setuid-sandbox')
216
+ # options.add_argument('--disable-dev-shm-usage')
217
+ # driver = webdriver.Chrome(options=options)
218
+
219
+ return driver
220
+
221
+
222
+ def get_chrome_exe_path() -> str:
223
+ global CHROME_EXE_PATH
224
+ if CHROME_EXE_PATH is not None:
225
+ return CHROME_EXE_PATH
226
+ # linux pyinstaller bundle
227
+ chrome_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'chrome', "chrome")
228
+ if os.path.exists(chrome_path):
229
+ if not os.access(chrome_path, os.X_OK):
230
+ raise Exception(f'Chrome binary "{chrome_path}" is not executable. '
231
+ f'Please, extract the archive with "tar xzf <file.tar.gz>".')
232
+ CHROME_EXE_PATH = chrome_path
233
+ return CHROME_EXE_PATH
234
+ # windows pyinstaller bundle
235
+ chrome_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'chrome', "chrome.exe")
236
+ if os.path.exists(chrome_path):
237
+ CHROME_EXE_PATH = chrome_path
238
+ return CHROME_EXE_PATH
239
+ # system
240
+ CHROME_EXE_PATH = uc.find_chrome_executable()
241
+ return CHROME_EXE_PATH
242
+
243
+
244
+ def get_chrome_major_version() -> str:
245
+ global CHROME_MAJOR_VERSION
246
+ if CHROME_MAJOR_VERSION is not None:
247
+ return CHROME_MAJOR_VERSION
248
+
249
+ if os.name == 'nt':
250
+ # Example: '104.0.5112.79'
251
+ try:
252
+ complete_version = extract_version_nt_executable(get_chrome_exe_path())
253
+ except Exception:
254
+ try:
255
+ complete_version = extract_version_nt_registry()
256
+ except Exception:
257
+ # Example: '104.0.5112.79'
258
+ complete_version = extract_version_nt_folder()
259
+ else:
260
+ chrome_path = get_chrome_exe_path()
261
+ process = os.popen(f'"{chrome_path}" --version')
262
+ # Example 1: 'Chromium 104.0.5112.79 Arch Linux\n'
263
+ # Example 2: 'Google Chrome 104.0.5112.79 Arch Linux\n'
264
+ complete_version = process.read()
265
+ process.close()
266
+
267
+ CHROME_MAJOR_VERSION = complete_version.split('.')[0].split(' ')[-1]
268
+ return CHROME_MAJOR_VERSION
269
+
270
+
271
+ def extract_version_nt_executable(exe_path: str) -> str:
272
+ import pefile
273
+ pe = pefile.PE(exe_path, fast_load=True)
274
+ pe.parse_data_directories(
275
+ directories=[pefile.DIRECTORY_ENTRY["IMAGE_DIRECTORY_ENTRY_RESOURCE"]]
276
+ )
277
+ return pe.FileInfo[0][0].StringTable[0].entries[b"FileVersion"].decode('utf-8')
278
+
279
+
280
+ def extract_version_nt_registry() -> str:
281
+ stream = os.popen(
282
+ 'reg query "HKLM\\SOFTWARE\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\Google Chrome"')
283
+ output = stream.read()
284
+ google_version = ''
285
+ for letter in output[output.rindex('DisplayVersion REG_SZ') + 24:]:
286
+ if letter != '\n':
287
+ google_version += letter
288
+ else:
289
+ break
290
+ return google_version.strip()
291
+
292
+
293
+ def extract_version_nt_folder() -> str:
294
+ # Check if the Chrome folder exists in the x32 or x64 Program Files folders.
295
+ for i in range(2):
296
+ path = 'C:\\Program Files' + (' (x86)' if i else '') + '\\Google\\Chrome\\Application'
297
+ if os.path.isdir(path):
298
+ paths = [f.path for f in os.scandir(path) if f.is_dir()]
299
+ for path in paths:
300
+ filename = os.path.basename(path)
301
+ pattern = '\d+\.\d+\.\d+\.\d+'
302
+ match = re.search(pattern, filename)
303
+ if match and match.group():
304
+ # Found a Chrome version.
305
+ return match.group(0)
306
+ return ''
307
+
308
+
309
+ def get_user_agent(driver=None) -> str:
310
+ global USER_AGENT
311
+ if USER_AGENT is not None:
312
+ return USER_AGENT
313
+
314
+ try:
315
+ if driver is None:
316
+ driver = get_webdriver()
317
+ USER_AGENT = driver.execute_script("return navigator.userAgent")
318
+ # Fix for Chrome 117 | https://github.com/FlareSolverr/FlareSolverr/issues/910
319
+ USER_AGENT = re.sub('HEADLESS', '', USER_AGENT, flags=re.IGNORECASE)
320
+ return USER_AGENT
321
+ except Exception as e:
322
+ raise Exception("Error getting browser User-Agent. " + str(e))
323
+ finally:
324
+ if driver is not None:
325
+ if PLATFORM_VERSION == "nt":
326
+ driver.close()
327
+ driver.quit()
328
+
329
+
330
+ def start_xvfb_display():
331
+ global XVFB_DISPLAY
332
+ if XVFB_DISPLAY is None:
333
+ from xvfbwrapper import Xvfb
334
+ XVFB_DISPLAY = Xvfb()
335
+ XVFB_DISPLAY.start()
336
+
337
+
338
+ def object_to_dict(_object):
339
+ json_dict = json.loads(json.dumps(_object, default=lambda o: o.__dict__))
340
+ # remove hidden fields
341
+ return {k: v for k, v in json_dict.items() if not k.startswith('__')}
test-requirements.txt CHANGED
@@ -1 +1 @@
1
- WebTest==3.0.0
 
1
+ WebTest==3.0.0