diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000000000000000000000000000000000..67c221fb82b4c04c871992ab3ad97423e556e448 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +env +venv +db_data +.ash_history +.DS_Store diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..e3a11c1d8a7d89bfc42a98a94bbf9d2e999e9da7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +assets/instagram.png filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..6b3facfce41f3a182ab34a4041c4b835ca921e3c --- /dev/null +++ b/.gitignore @@ -0,0 +1,175 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ +/.idea/modules.xml +/.idea/inspectionProfiles/profiles_settings.xml +/.idea/inspectionProfiles/Project_Default.xml +/.idea/vcs.xml +/.idea/ytdl-bot.iml +/.idea/misc.xml +/.idea/workspace.xml +/.idea/jsonSchemas.xml +/*.session +/.idea/ytdlbot.iml +/*.sqlite +/.idea/dataSources.xml +/.idea/sqldialects.xml +/.idea/.gitignore +/.idea/dataSources/bf75f0a6-c774-4ecf-9448-2086f57b70df.xml +/.idea/dataSources.local.xml +/.idea/dataSources/bf75f0a6-c774-4ecf-9448-2086f57b70df/entities/entities.dat +/.idea/dataSources/bf75f0a6-c774-4ecf-9448-2086f57b70df/entities/entities.dat.len +/.idea/dataSources/bf75f0a6-c774-4ecf-9448-2086f57b70df/entities/entities.dat.values +/.idea/dataSources/bf75f0a6-c774-4ecf-9448-2086f57b70df/entities/entities.dat.values.at +/.idea/dataSources/bf75f0a6-c774-4ecf-9448-2086f57b70df/entities/entities.dat.values.s +/.idea/dataSources/bf75f0a6-c774-4ecf-9448-2086f57b70df/entities/entities.dat_i +/.idea/dataSources/bf75f0a6-c774-4ecf-9448-2086f57b70df/entities/entities.dat_i.len +/.idea/dataSources/bf75f0a6-c774-4ecf-9448-2086f57b70df/storage_v2/_src_/schema/main.uQUzAA.meta +db_data/* +env/* +.ash_history +.DS_Store +ytdlbot/ytdl.session +data/* +upgrade_worker.sh +ytdl.session +reinforcement/* +/ytdlbot/session/celery.session +/.idea/prettier.xml +/.idea/watcherTasks.xml +/ytdlbot/session/ytdl.session-journal +/ytdlbot/unknown_errors.txt +/ytdlbot/ytdl.session-journal +/ytdlbot/ytdl-main.session-journal +/ytdlbot/ytdl-main.session +/ytdlbot/ytdl-celery.session-journal +/ytdlbot/ytdl-celery.session +/ytdlbot/main.session +/ytdlbot/tasks.session +/ytdlbot/tasks.session-journal diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..e2bc8f761a940d33d4b2bf12b610aac4ff2fa188 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.10 + +WORKDIR /code + +COPY ./requirements.txt /code/requirements.txt + +RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt + +COPY . . + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"] \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..299f31fa8d7739af49c4302658ffcef2dfda7054 --- /dev/null +++ b/Makefile @@ -0,0 +1,52 @@ +define NOLOGGING + + logging: + driver: none +endef +export NOLOGGING + +default: + docker pull bennythink/ytdlbot + +bot: + make + docker-compose up -d + docker system prune -a --volumes -f + +worker: + make + docker-compose -f worker.yml up -d + docker system prune -a --volumes -f + sleep 5 + +weak-worker: + make + docker-compose --compatibility -f worker.yml up -d + docker system prune -a --volumes -f + sleep 5 + +upgrade-all-worker: + bash upgrade_worker.sh + +tag: + git tag -a v$(shell date "+%Y-%m-%d")_$(shell git rev-parse --short HEAD) -m v$(shell date "+%Y-%m-%d") + git push --tags + +nolog: + echo "$$NOLOGGING">> worker.yml + +flower: + echo 'import dbm;dbm.open("data/flower","n");exit()'| python3 + +up: + docker build -t bennythink/ytdlbot:latest . + docker-compose -f docker-compose.yml -f worker.yml up -d + +ps: + docker-compose -f docker-compose.yml -f worker.yml ps + +down: + docker-compose -f docker-compose.yml -f worker.yml down + +logs: + docker-compose -f docker-compose.yml -f worker.yml logs -f worker ytdl \ No newline at end of file diff --git a/Procfile b/Procfile new file mode 100644 index 0000000000000000000000000000000000000000..8be22ed7863e2cb582f3a9cdab5c323ad6e83827 --- /dev/null +++ b/Procfile @@ -0,0 +1 @@ +worker: python ytdlbot/ytdl_bot.py \ No newline at end of file diff --git a/app.json b/app.json new file mode 100644 index 0000000000000000000000000000000000000000..f5f5fc379e4ab259bb3ce3998480a7fc483bc2b6 --- /dev/null +++ b/app.json @@ -0,0 +1,43 @@ +{ + "name": "YouTube-Downloader", + "description": "A Telegrambot to download youtube video", + "repository": "https://github.com/tgbot-collection/ytdlbot", + "logo": "https://avatars.githubusercontent.com/u/73354211?s=200&v=4", + "keywords": [ + "telegram", + "youtube-dl" + ], + "env": { + "TOKEN": { + "description": "Bot token", + "value": "token" + }, + "APP_ID": { + "description": "APP ID", + "value": "12345" + }, + "APP_HASH": { + "description": "APP HASH", + "value": "12345abc" + }, + "OWNER": { + "description": "Your telegram username", + "value": "username", + "required": false + } + }, + "formation": { + "worker": { + "quantity": 1, + "size": "eco" + } + }, + "buildpacks": [ + { + "url": "https://github.com/heroku/heroku-buildpack-python.git" + }, + { + "url": "https://github.com/jonathanong/heroku-buildpack-ffmpeg-latest.git" + } + ] +} diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..f488bea2d94d6c3f9a2c803a67557c7125253d87 --- /dev/null +++ b/app.py @@ -0,0 +1,26 @@ +import os +import streamlit as st + +def run_python_file(file_path): + try: + st.text(f"Running {file_path}...") + os.system(f"python {file_path}") + st.success("Script executed successfully!") + except Exception as e: + st.error(f"Error: {e}") + +def main(): + st.title("YTDLBot Runner") + + # Specify the directory and file name + directory = "ytdlbot" + file_name = "ytdl_bot.py" + file_path = os.path.join(directory, file_name) + + st.text(f"Selected file: {file_path}") + + # Run the Python file automatically when the app starts + run_python_file(file_path) + +if __name__ == "__main__": + main() diff --git a/assets/1.jpeg b/assets/1.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..88334b52e0404688eabe5c73f4108431741d5b55 Binary files /dev/null and b/assets/1.jpeg differ diff --git a/assets/2.jpeg b/assets/2.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..65c36cd44b10d9611ee0bedd798c1ee94b6381cb Binary files /dev/null and b/assets/2.jpeg differ diff --git a/assets/CNY.png b/assets/CNY.png new file mode 100644 index 0000000000000000000000000000000000000000..0bf5838020330e6b9c028bed3c10e470fd11d10d Binary files /dev/null and b/assets/CNY.png differ diff --git a/assets/USD.png b/assets/USD.png new file mode 100644 index 0000000000000000000000000000000000000000..108fee1c474db96119513a805f269fb8e179dbbb Binary files /dev/null and b/assets/USD.png differ diff --git a/assets/instagram.png b/assets/instagram.png new file mode 100644 index 0000000000000000000000000000000000000000..50f842b080aa30c3f485d3f72b74767051c2f081 --- /dev/null +++ b/assets/instagram.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:403808b9b818ec3ad934a4b7b4b1689c179d318eb34a3cabbe5e00b1b90fb14a +size 1581864 diff --git a/assets/tron.png b/assets/tron.png new file mode 100644 index 0000000000000000000000000000000000000000..fb3fea3ef6f00d1f726569d4ba7aec429b67b78c Binary files /dev/null and b/assets/tron.png differ diff --git a/conf/YouTube Download Celery.json b/conf/YouTube Download Celery.json new file mode 100644 index 0000000000000000000000000000000000000000..34393887ac94521c99c67fa5f309d947f53c5089 --- /dev/null +++ b/conf/YouTube Download Celery.json @@ -0,0 +1,794 @@ +{ + "__inputs": [ + { + "name": "DS_CELERY", + "label": "celery", + "description": "", + "type": "datasource", + "pluginId": "influxdb", + "pluginName": "InfluxDB" + } + ], + "__elements": [], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "8.3.1" + }, + { + "type": "datasource", + "id": "influxdb", + "name": "InfluxDB", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "iteration": 1644554238421, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "influxdb", + "uid": "${DS_CELERY}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "Active", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "active", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"active\") FROM \"active\" WHERE $timeFilter GROUP BY time($__interval) ", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "active" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + }, + { + "alias": "$tag_hostname", + "hide": false, + "query": "\nSELECT \nmean(\"active\") AS active\nFROM \"tasks\" WHERE (\"hostname\" =~ /^$hostname$/) AND $timeFilter GROUP BY time($__interval) ,* ORDER BY asc ", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series" + } + ], + "title": "Active Jobs", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_CELERY}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "$col", + "datasource": { + "type": "influxdb", + "uid": "${DS_CELERY}" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "metrics", + "orderByTime": "ASC", + "policy": "default", + "query": "\nSELECT \nmean(\"today_audio_success\")/mean(\"today_audio_request\")*100 as audio_success,\nmean(\"today_video_success\")/mean(\"today_video_request\")*100 as video_success\n\nFROM \"metrics\" WHERE $timeFilter GROUP BY time($__interval), * ORDER BY asc ", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "today_audio_success" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "title": "Video & Audio Success Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_CELERY}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "$tag_hostname:$col", + "query": "SELECT mean(\"load1\") AS load1,mean(\"load5\") AS load5,mean(\"load15\") AS load15\nFROM \"tasks\" WHERE (\"hostname\" =~ /^$hostname$/) AND $timeFilter GROUP BY time($__interval) ,* ORDER BY asc \n\n", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "title": "Load Average", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_CELERY}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "$tag_hostname:$col", + "datasource": { + "type": "influxdb", + "uid": "${DS_CELERY}" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "tasks", + "orderByTime": "ASC", + "policy": "default", + "query": "\nSELECT mean(\"task-succeeded\")/mean(\"task-received\")*100 AS success_rate, mean(\"task-failed\")/mean(\"task-received\")*100 AS fail_rate\n\nFROM \"tasks\" WHERE (\"hostname\" =~ /^$hostname$/) AND $timeFilter GROUP BY time($__interval) ,* ORDER BY asc ", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "task-received" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "hostname", + "operator": "=~", + "value": "/^$hostname$/" + } + ] + } + ], + "title": "Task Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_CELERY}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "$tag_hostname:$col", + "datasource": { + "type": "influxdb", + "uid": "${DS_CELERY}" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "tasks", + "orderByTime": "ASC", + "policy": "default", + "query": "\nSELECT mean(\"task-received\") AS received, mean(\"task-started\") AS started,mean(\"task-succeeded\") AS succeeded,mean(\"task-failed\") AS failed\n\nFROM \"tasks\" WHERE (\"hostname\" =~ /^$hostname$/) AND $timeFilter GROUP BY time($__interval) ,* ORDER BY asc ", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "task-received" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "hostname", + "operator": "=~", + "value": "/^$hostname$/" + } + ] + } + ], + "title": "Task Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_CELERY}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "$col", + "datasource": { + "type": "influxdb", + "uid": "${DS_CELERY}" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "metrics", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT \nmean(\"today_audio_request\") as audio_request,\nmean(\"today_audio_success\") as audio_success,\n\nmean(\"today_bad_request\") as bad_request,\n\nmean(\"today_video_request\") as video_request,\nmean(\"today_video_success\") as video_success\nFROM \"metrics\" WHERE $timeFilter GROUP BY time($__interval), * ORDER BY asc ", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "today_audio_success" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "title": "Video & Audio", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 33, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "type": "influxdb", + "uid": "${DS_CELERY}" + }, + "definition": "show tag values with KEY=\"hostname\"", + "hide": 0, + "includeAll": true, + "label": "hostname", + "multi": true, + "name": "hostname", + "options": [], + "query": "show tag values with KEY=\"hostname\"", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "YouTube Download Celery", + "uid": "9yXGmc1nk", + "version": 14, + "weekStart": "" +} \ No newline at end of file diff --git a/conf/supervisor_main.conf b/conf/supervisor_main.conf new file mode 100644 index 0000000000000000000000000000000000000000..dbae1442bc9caa5e2674becc44a61094d5b74cbe --- /dev/null +++ b/conf/supervisor_main.conf @@ -0,0 +1,34 @@ +[supervisord] +nodaemon=true +logfile=/dev/null +logfile_maxbytes=0 +user=root + + +[program:vnstat] +command=vnstatd -n +autorestart=true + + +[program:ytdl] +directory=/ytdlbot/ytdlbot/ +command=python ytdl_bot.py +autorestart=true +priority=900 +stopasgroup=true +startsecs = 30 +startretries = 2 + +redirect_stderr=true +stdout_logfile_maxbytes = 50MB +stdout_logfile_backups = 2 +stdout_logfile = /var/log/ytdl.log + +[program:log] +command=tail -f /var/log/ytdl.log +autorestart=true +priority=999 + +redirect_stderr=true +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 diff --git a/conf/supervisor_worker.conf b/conf/supervisor_worker.conf new file mode 100644 index 0000000000000000000000000000000000000000..dbad141c1b7850c66c26a99c7b1fa6f3df5fc3f3 --- /dev/null +++ b/conf/supervisor_worker.conf @@ -0,0 +1,33 @@ +[supervisord] +nodaemon=true +logfile=/dev/null +logfile_maxbytes=0 +user=root + + +[program:vnstat] +command=vnstatd -n +autorestart=true + +[program:worker] +directory=/ytdlbot/ytdlbot/ +command=python tasks.py +autorestart=true +priority=900 +stopasgroup=true +startsecs = 5 +startretries = 5 + +redirect_stderr=true +stdout_logfile_maxbytes = 50MB +stdout_logfile_backups = 2 +stdout_logfile = /var/log/ytdl.log + +[program:log] +command=tail -f /var/log/ytdl.log +autorestart=true +priority=999 + +redirect_stderr=true +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..3db5eeb5005a58cdd06dbc12d98fd73345c3a22c --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,60 @@ +version: '3.1' + +services: + socat: + image: bennythink/socat + restart: always + volumes: + - /var/run/docker.sock:/var/run/docker.sock + entrypoint: [ "socat", "tcp-listen:2375,fork,reuseaddr","unix-connect:/var/run/docker.sock" ] + + redis: + image: redis:7-alpine + restart: always + logging: + driver: none + + mysql: + image: ubuntu/mysql:8.0-22.04_beta + restart: always + volumes: + - ./db_data:/var/lib/mysql + environment: + MYSQL_ROOT_PASSWORD: 'root' + command: --default-authentication-plugin=mysql_native_password + logging: + driver: none + + ytdl: + image: bennythink/ytdlbot + env_file: + - env/ytdl.env + restart: always + depends_on: + - socat + - redis + volumes: + - ./data/vnstat/:/var/lib/vnstat/ + labels: + - "com.centurylinklabs.watchtower.enable=true" + + flower: + image: bennythink/ytdlbot + env_file: + - env/ytdl.env + restart: unless-stopped + command: [ "/usr/local/bin/celery", + "-A", "flower_tasks", "flower", + "--basic_auth=benny:123456", + "--address=0.0.0.0", "--persistent","--purge_offline_workers=3600" ] + volumes: + - ./data/flower:/ytdlbot/ytdlbot/flower + ports: + - "127.0.0.1:15555:5555" + + instagram: + image: bennythink/ytdlbot + env_file: + - env/ytdl.env + restart: always + command: [ "/usr/local/bin/python", "/ytdlbot/ytdlbot/instagram.py" ] diff --git a/k8s.md b/k8s.md new file mode 100644 index 0000000000000000000000000000000000000000..61b0146b6771b04354c9eec444a201059e3fbac7 --- /dev/null +++ b/k8s.md @@ -0,0 +1,200 @@ +## Kubernetes + +Kubernetes, also known as K8s, is an open-source system for automating deployment, scaling, and management of +containerized applications + +# Complete deployment guide for k8s deloyment + +* contains every functionality +* compatible with amd64, arm64 and armv7l + +## First. Get all file in k8s folder + +Download `k8s` file to a directory on your k8s server and go to this folder + +## 1. Create Redis deloyment + +```shell +kubectl apply -f 01.redis.yml +``` + +This command will create ytdl namespace, redis pod and redis service + +## 2. Creat MariaDB deloyment + +```shell +kubectl apply -f 02.mariadb.yml +``` + +This deloyment will claim 10GB storage from storageClassName: longhorn. Please replace longhorn with your +storageClassName before apply. + +## 3. Set environment variables + +Create configMap for env + +### 3.1 Edit configmap.yml + +```shell +vim 03.configmap.yml +``` + +you can configure all the following environment variables: + +* PYRO_WORKERS: number of workers for pyrogram, default is 100 +* WORKERS: workers count for celery +* APP_ID: **REQUIRED**, get it from https://core.telegram.org/ +* APP_HASH: **REQUIRED** +* TOKEN: **REQUIRED** +* REDIS: **REQUIRED if you need VIP mode and cache** ⚠️ Don't publish your redis server on the internet. ⚠️ + +* OWNER: owner username +* QUOTA: quota in bytes +* EX: quota expire time +* MULTIPLY: vip quota comparing to normal quota +* USD2CNY: exchange rate +* VIP: VIP mode, default: disable +* AFD_LINK +* COFFEE_LINK +* COFFEE_TOKEN +* AFD_TOKEN +* AFD_USER_ID + +* AUTHORIZED_USER: users that could use this bot, user_id, separated with `,` +* REQUIRED_MEMBERSHIP: group or channel username, user must join this group to use the bot. Could be use with + above `AUTHORIZED_USER` + +* ENABLE_CELERY: Distribution mode, default: disable. You'll can setup workers in different locations. +* ENABLE_FFMPEG: enable ffmpeg so Telegram can stream +* MYSQL_HOST: you'll have to setup MySQL if you enable VIP mode +* MYSQL_USER +* MYSQL_PASS +* GOOGLE_API_KEY: YouTube API key, required for YouTube video subscription. +* AUDIO_FORMAT: audio format, default is m4a. You can set to any known and supported format for ffmpeg. For + example,`mp3`, `flac`, etc. ⚠️ m4a is the fastest. Other formats may affect performance. +* ARCHIVE_ID: group or channel id/username. All downloads will send to this group first and then forward to end user. + **Inline button will be lost during the forwarding.** + +### 3.2 Apply configMap for environment variables + +```shell +kubectl apply -f 03.configmap.yml +``` + +## 4. Run Master Celery + +```shell +kubectl apply -f 04.ytdl-master.yml +``` + +This deloyment will create ytdl-pvc PersistentVolumeClaim on storageClassName: longhorn. This clain will contain vnstat, +cookies folder and flower database. Please replace longhorn with your storageClassName before apply + +### 4.1 Setup instagram cookies + +Required if you want to support instagram. + +You can use this extension +[Get cookies.txt](https://chrome.google.com/webstore/detail/get-cookiestxt/bgaddhkoddajcdgocldbbfleckgcbcid) +to get instagram cookies + +Get pod running ytdl master: + +```shell +kubectl get pods --namespace ytdl +``` + +Name should be ytdl-xxxxxxxx + +Access to pod + +```shell +kubectl --namespace=ytdl exec --stdin --tty ytdl-xxx -- sh +``` + +(replace ytdl-xxx by your pod name) + +Go to ytdl-pvc mounted folder + +```shell +cd /ytdlbot/ytdlbot/data/ +vim instagram.com_cookies.txt +# paste your cookies +``` + +## 5. Run Worker Celery + +```shell +kubectl apply -f 05.ytdl-worker.yml +``` + +## 6. Run Flower image (OPTIONAL) + +### 6.1 Setup flower db + +Get pod running ytdl master: + +```shell +kubectl get pods --namespace ytdl +``` + +Name should be ytdl-xxxxxxxx + +Access to pod + +```shell +kubectl --namespace=ytdl exec --stdin --tty ytdl-xxx -- sh +``` + +(replace ytdl-xxx by your pod name) + +Go to ytdl-pvc mounted folder + +```shel +cd /var/lib/vnstat/ +``` + +Create flower database file + +```shell +{} ~ python3 +Python 3.9.9 (main, Nov 21 2021, 03:22:47) +[Clang 12.0.0 (clang-1200.0.32.29)] on darwin +Type "help", "copyright", "credits" or "license" for more information. +>>> import dbm;dbm.open("flower","n");exit() +``` + +### 6.2 Config Flower Ingress + +This step need config ingress from line 51 of file 06.flower.yml with your ingress service. Need for access from +internet. +YML file should be adjusted depending on your load balancing, ingress and network system + +For active SSL + +```yml +cert-manager.io/cluster-issuer: letsencrypt-prod +``` + +Replace nginx by your ingress service + +```yml +ingressClassName: nginx +``` + +Add your domain, example + +```yml +tls: + - hosts: + - flower.benny.com + secretName: flower-tls + rules: + - host: flower.benny.com +``` + +### 6.3 Apply Flower deloyment + +```shell +kubectl apply -f 06.flower.yml +``` diff --git a/k8s/01.redis.yml b/k8s/01.redis.yml new file mode 100644 index 0000000000000000000000000000000000000000..da52fc3c7ada0beb07d8ef9f688bd86af3b65d70 --- /dev/null +++ b/k8s/01.redis.yml @@ -0,0 +1,53 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ytdl + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + ytdl: redis + name: redis + namespace: ytdl +spec: + replicas: 1 + selector: + matchLabels: + ytdl: redis + strategy: {} + template: + metadata: + creationTimestamp: null + labels: + ytdl: redis + spec: + containers: + - image: redis:7-alpine + name: redis + ports: + - containerPort: 6379 + resources: {} + restartPolicy: Always +status: {} + +--- +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + labels: + ytdl: redis + name: redis + namespace: ytdl +spec: + ports: + - name: "6379" + port: 6379 + targetPort: 6379 + selector: + ytdl: redis +status: + loadBalancer: {} \ No newline at end of file diff --git a/k8s/02.mariadb.yml b/k8s/02.mariadb.yml new file mode 100644 index 0000000000000000000000000000000000000000..c89dbc752f29caf11fdc2f60d34e689c9f259d75 --- /dev/null +++ b/k8s/02.mariadb.yml @@ -0,0 +1,80 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + creationTimestamp: null + labels: + ytdl: mariadb-pvc + name: mariadb-pvc + namespace: ytdl +spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn + resources: + requests: + storage: 10Gi +status: {} + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + creationTimestamp: null + labels: + ytdl: mariadb + name: mariadb + namespace: ytdl +spec: + replicas: 1 + selector: + matchLabels: + ytdl: mariadb + strategy: + type: Recreate + template: + metadata: + creationTimestamp: null + labels: + ytdl: mariadb + spec: + containers: + - env: + - name: MYSQL_ROOT_PASSWORD + value: ro0tP4sSworD + - name: MYSQL_DATABASE + value: ytdl + image: mariadb:latest + name: mariadb + ports: + - containerPort: 3306 + resources: {} + volumeMounts: + - mountPath: /var/lib/mysql + name: "mariadb-persistent-storage" + restartPolicy: Always + volumes: + - name: mariadb-persistent-storage + persistentVolumeClaim: + claimName: mariadb-pvc +status: {} + +--- +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + labels: + ytdl: mariadb + name: mariadb-svc + namespace: ytdl +spec: + ports: + - name: "3306" + port: 3306 + targetPort: 3306 + selector: + ytdl: mariadb +status: + loadBalancer: {} + diff --git a/k8s/03.configmap.yml b/k8s/03.configmap.yml new file mode 100644 index 0000000000000000000000000000000000000000..90ec84a80dbc33fac04c54fe61f749190b52304b --- /dev/null +++ b/k8s/03.configmap.yml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ytdlenv + namespace: ytdl + annotations: +data: + APP_HASH: + APP_ID: + TOKEN: + ARCHIVE_ID: + ENABLE_CELERY: 'True' + ENABLE_FFMPEG: 'True' + MYSQL_HOST: mariadb-svc + MYSQL_PASS: ro0tP4sSworD + MYSQL_USER: root + REDIS: redis \ No newline at end of file diff --git a/k8s/04.ytdl-master.yml b/k8s/04.ytdl-master.yml new file mode 100644 index 0000000000000000000000000000000000000000..f17579f69a8f59cef1357aa5953e684f3ce9bde3 --- /dev/null +++ b/k8s/04.ytdl-master.yml @@ -0,0 +1,65 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ytdl-pvc + namespace: ytdl + creationTimestamp: null + labels: + ytdl: ytdl-pvc +spec: + accessModes: + - ReadWriteMany + storageClassName: longhorn + resources: + requests: + storage: 10Gi +status: {} + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ytdl + namespace: ytdl + creationTimestamp: null + labels: + ytdl: ytdl +spec: + replicas: 1 + selector: + matchLabels: + ytdl: ytdl + template: + metadata: + creationTimestamp: null + labels: + ytdl: ytdl + spec: + volumes: + - name: ytdl-pvc + persistentVolumeClaim: + claimName: ytdl-pvc + containers: + - name: ytdl + image: bennythink/ytdlbot + envFrom: + - configMapRef: + name: ytdlenv + resources: {} + volumeMounts: + - name: ytdl-pvc + mountPath: /var/lib/vnstat/ + subPath: vnstat/ + - name: ytdl-pvc + mountPath: /ytdlbot/ytdlbot/data/ + subPath: data/ + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + imagePullPolicy: Always + restartPolicy: Always + terminationGracePeriodSeconds: 30 + dnsPolicy: ClusterFirst + securityContext: {} + schedulerName: default-scheduler +status: {} diff --git a/k8s/05.ytdl-worker.yml b/k8s/05.ytdl-worker.yml new file mode 100644 index 0000000000000000000000000000000000000000..ca154655ecd9d430f144a794cc7edc45022ba71d --- /dev/null +++ b/k8s/05.ytdl-worker.yml @@ -0,0 +1,47 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + ytdl: ytdl-worker + name: ytdl-worker + namespace: ytdl +spec: + replicas: 4 + selector: + matchLabels: + ytdl: ytdl-worker + template: + metadata: + creationTimestamp: null + labels: + ytdl: ytdl-worker + spec: + volumes: + - name: ytdl-pvc + persistentVolumeClaim: + claimName: ytdl-pvc + containers: + - name: ytdl-worker + image: bennythink/ytdlbot + args: + - /usr/local/bin/supervisord + - '-c' + - /ytdlbot/conf/supervisor_worker.conf + envFrom: + - configMapRef: + name: ytdlenv + resources: {} + volumeMounts: + - name: ytdl-pvc + mountPath: /ytdlbot/ytdlbot/data/ + subPath: data/ + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + imagePullPolicy: Always + restartPolicy: Always + terminationGracePeriodSeconds: 30 + dnsPolicy: ClusterFirst + securityContext: {} + schedulerName: default-scheduler +status: {} diff --git a/k8s/06.flower.yml b/k8s/06.flower.yml new file mode 100644 index 0000000000000000000000000000000000000000..e7c01c73323196a394c1f592052f0e289fcc6cf7 --- /dev/null +++ b/k8s/06.flower.yml @@ -0,0 +1,101 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + ytdl: flower + name: flower + namespace: ytdl +spec: + replicas: 1 + selector: + matchLabels: + ytdl: flower + strategy: + type: Recreate + template: + metadata: + creationTimestamp: null + labels: + ytdl: flower + spec: + containers: + - envFrom: + - configMapRef: + name: ytdlenv + args: + - /usr/local/bin/celery + - -A + - flower_tasks + - flower + - --basic_auth=bennythink:123456 + - --address=0.0.0.0 + - --persistent + - --purge_offline_workers=3600 + image: bennythink/ytdlbot + name: flower + ports: + - containerPort: 5555 + resources: {} + volumeMounts: + - name: ytdl-pvc + mountPath: /ytdlbot/ytdlbot/flower + subPath: vnstat/flower + restartPolicy: Always + volumes: + - name: ytdl-pvc + persistentVolumeClaim: + claimName: ytdl-pvc +status: {} + +# THIS IS OPTION IF YOU WANT PUBLIC FLOWER PAGE TO INTERNET. +# should be adjusted depending on your load balancing system machine +--- +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + labels: + ytdl: flower + name: flower-svc + namespace: ytdl +spec: + type: NodePort + ports: + - name: "5555" + protocol: TCP + port: 5555 + targetPort: 5555 + selector: + ytdl: flower +status: + loadBalancer: {} + +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: nginx-flower-ingress + namespace: ytdl + annotations: + # cert-manager.io/cluster-issuer: letsencrypt-prod + nginx.ingress.kubernetes.io/rewrite-target: / + # nginx.ingress.kubernetes.io/whitelist-source-range: 14.161.27.151 limit by ipaddresss + +spec: + ingressClassName: nginx + tls: + - hosts: + - your-domain + secretName: flower-tls + rules: + - host: your-domain + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: flower-svc + port: + number: 5555 \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 0000000000000000000000000000000000000000..67a3f11e38f39733dd0ad3e2e1a25835095d26d7 --- /dev/null +++ b/main.py @@ -0,0 +1,21 @@ +from fastapi import FastAPI, HTTPException +import subprocess + +def run_python_verbose(): + try: + # Run the 'python -v' command in a subprocess + print("okk") + subprocess.run(['python', 'ytdlbot/ytdl_bot.py'], check=True) + except subprocess.CalledProcessError as e: + print(f"Error running 'python -v': {e}") + +# Call the function + +app = FastAPI() + +@app.get("/") +async def root(): + return {"message": "Hello World"} +@app.get("/okk") +async def okk(): + run_python_verbose() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..2ae37e24fbec39d096acce56c062ebc53087189e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,31 @@ +pyrogram +tgcrypto +yt-dlp +APScheduler +beautifultable +ffmpeg-python +PyMySQL +celery +filetype +flower +psutil +influxdb +beautifulsoup4 +fakeredis +supervisor +tgbot-ping +redis +requests +tqdm +requests-toolbelt +ffpb +youtube-search-python +token-bucket +coloredlogs +tronpy +mnemonic +qrcode +blinker +flask +fastapi +uvicorn diff --git a/scripts/low_id.sh b/scripts/low_id.sh new file mode 100644 index 0000000000000000000000000000000000000000..16fc2cef94d1749579e200a602fea10cd16fd50b --- /dev/null +++ b/scripts/low_id.sh @@ -0,0 +1,12 @@ +#!/bin/bash +export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/usr/local/go/bin:/opt/bin + +# Check the logs for the given string +if docker-compose logs --tail=100 ytdl | grep -q "The msg_id is too low"; then + # If the string is found, stop the ytdl service + echo "ytdl service stopped due to 'The msg_id is too low' found in logs." + docker-compose stop ytdl && docker-compose rm ytdl && docker-compose up -d + +else + echo "String not found in logs." +fi diff --git a/scripts/migrate_to_mysql.py b/scripts/migrate_to_mysql.py new file mode 100644 index 0000000000000000000000000000000000000000..436fb78a848571ac1edefd05aa989ba2671dea4e --- /dev/null +++ b/scripts/migrate_to_mysql.py @@ -0,0 +1,27 @@ +#!/usr/local/bin/python3 +# coding: utf-8 + +# ytdlbot - migrate_to_mysql.py +# 12/29/21 15:28 +# + +__author__ = "Benny " + +import sqlite3 + +import pymysql + +mysql_con = pymysql.connect(host='localhost', user='root', passwd='root', db='vip', charset='utf8mb4') +sqlite_con = sqlite3.connect('vip.sqlite') + +vips = sqlite_con.execute('SELECT * FROM VIP').fetchall() + +for vip in vips: + mysql_con.cursor().execute('INSERT INTO vip VALUES (%s, %s, %s, %s, %s, %s)', vip) + +settings = sqlite_con.execute('SELECT * FROM settings').fetchall() + +for setting in settings: + mysql_con.cursor().execute("INSERT INTO settings VALUES (%s,%s,%s)", setting) + +mysql_con.commit() diff --git a/scripts/start.sh b/scripts/start.sh new file mode 100644 index 0000000000000000000000000000000000000000..242f207fc4a641c41c626e3171aa80e2c0fe62e3 --- /dev/null +++ b/scripts/start.sh @@ -0,0 +1,13 @@ +docker run -d --restart unless-stopped --name ytdl \ + --net host \ + -e TOKEN=12345 \ + -e APP_ID=123123 \ + -e APP_HASH=4990 \ + -e ENABLE_CELERY=True \ + -e REDIS=192.168.6.1 \ + -e MYSQL_HOST=192.168.6.1 \ + -e WORKERS=4 \ + -e VIP=True \ + -e CUSTOM_TEXT=#StandWithUkraine \ + bennythink/ytdlbot \ + /usr/local/bin/supervisord -c "/ytdlbot/conf/supervisor_worker.conf" diff --git a/scripts/transfer.py b/scripts/transfer.py new file mode 100644 index 0000000000000000000000000000000000000000..2ace122f9d676eeeda726d1c023dc645baf57286 --- /dev/null +++ b/scripts/transfer.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# coding: utf-8 + +# ytdlbot - transfer.py +# 2023-12-07 18:21 +from tronpy import Tron +from tronpy.hdwallet import seed_from_mnemonic, key_from_seed +from tronpy.keys import PrivateKey + +mnemonic = "web horse smile ramp olive slush blue property world physical donkey pumpkin" + +client = Tron(network="nile") + +from_ = client.generate_address_from_mnemonic(mnemonic, account_path="m/44'/195'/0'/0/0")["base58check_address"] +balance = client.get_account_balance(from_) +print("my addr: ", from_, "balance: ", balance) +to = input("to: ") +amount = int(input("amount in TRX: ")) + + +def mnemonic_to_private_key(): + seed = seed_from_mnemonic(mnemonic, passphrase="") + private_key = key_from_seed(seed, account_path="m/44'/195'/0'/0/0") + return PrivateKey(private_key) + + +t = client.trx.transfer(from_, to, amount * 1_000_000).build().sign(mnemonic_to_private_key()).broadcast() + +print(t.wait()) diff --git a/worker.yml b/worker.yml new file mode 100644 index 0000000000000000000000000000000000000000..990e24533c9f84a79e4689eeb79dc7487399b9f2 --- /dev/null +++ b/worker.yml @@ -0,0 +1,15 @@ +version: '3.1' + +services: + worker: + image: bennythink/ytdlbot + env_file: + - env/ytdl.env + restart: always + command: [ "/usr/local/bin/supervisord", "-c" ,"/ytdlbot/conf/supervisor_worker.conf" ] +# network_mode: "host" +# deploy: +# resources: +# limits: +# cpus: '0.3' +# memory: 1500M diff --git a/ytdlbot/__pycache__/channel.cpython-310.pyc b/ytdlbot/__pycache__/channel.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f9bb3dcafbfa025c546d2da3a9a99650eb60cffc Binary files /dev/null and b/ytdlbot/__pycache__/channel.cpython-310.pyc differ diff --git a/ytdlbot/__pycache__/channel.cpython-38.pyc b/ytdlbot/__pycache__/channel.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..40aececb6626210d508fbe1a694dcb87d03d0f0e Binary files /dev/null and b/ytdlbot/__pycache__/channel.cpython-38.pyc differ diff --git a/ytdlbot/__pycache__/client_init.cpython-310.pyc b/ytdlbot/__pycache__/client_init.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8d7cb05cf344dc35d18416836ad8a253e6055683 Binary files /dev/null and b/ytdlbot/__pycache__/client_init.cpython-310.pyc differ diff --git a/ytdlbot/__pycache__/client_init.cpython-38.pyc b/ytdlbot/__pycache__/client_init.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e5e2d1aed5d29ac656c7862ffe122192eb20678a Binary files /dev/null and b/ytdlbot/__pycache__/client_init.cpython-38.pyc differ diff --git a/ytdlbot/__pycache__/config.cpython-310.pyc b/ytdlbot/__pycache__/config.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f7ecd818088121dd68dc102564f9efbd9cf485f2 Binary files /dev/null and b/ytdlbot/__pycache__/config.cpython-310.pyc differ diff --git a/ytdlbot/__pycache__/config.cpython-38.pyc b/ytdlbot/__pycache__/config.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d5d1553263b575d1ed897f6a43e0f270ecf54e45 Binary files /dev/null and b/ytdlbot/__pycache__/config.cpython-38.pyc differ diff --git a/ytdlbot/__pycache__/constant.cpython-310.pyc b/ytdlbot/__pycache__/constant.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..18e8257149be9cc8a1ae50c738d0a8e54b34c0f2 Binary files /dev/null and b/ytdlbot/__pycache__/constant.cpython-310.pyc differ diff --git a/ytdlbot/__pycache__/constant.cpython-38.pyc b/ytdlbot/__pycache__/constant.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..adc0d063d3ddf17997d3b7c06ad4e94bf3ecb2b8 Binary files /dev/null and b/ytdlbot/__pycache__/constant.cpython-38.pyc differ diff --git a/ytdlbot/__pycache__/database.cpython-310.pyc b/ytdlbot/__pycache__/database.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f49cebcfc3ba6317bfd2d08dbac7a26d96428835 Binary files /dev/null and b/ytdlbot/__pycache__/database.cpython-310.pyc differ diff --git a/ytdlbot/__pycache__/database.cpython-38.pyc b/ytdlbot/__pycache__/database.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b495091077f2785127302443a976e9255247941e Binary files /dev/null and b/ytdlbot/__pycache__/database.cpython-38.pyc differ diff --git a/ytdlbot/__pycache__/downloader.cpython-310.pyc b/ytdlbot/__pycache__/downloader.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..06395c9fec06a3c0726252e8b5608983610404ed Binary files /dev/null and b/ytdlbot/__pycache__/downloader.cpython-310.pyc differ diff --git a/ytdlbot/__pycache__/downloader.cpython-38.pyc b/ytdlbot/__pycache__/downloader.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f97b06fa38b2fc2e74dd04f8493455a2215e0bb0 Binary files /dev/null and b/ytdlbot/__pycache__/downloader.cpython-38.pyc differ diff --git a/ytdlbot/__pycache__/flower_tasks.cpython-310.pyc b/ytdlbot/__pycache__/flower_tasks.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8e3021611d74d67311b60df7be2e4e645e22c12f Binary files /dev/null and b/ytdlbot/__pycache__/flower_tasks.cpython-310.pyc differ diff --git a/ytdlbot/__pycache__/flower_tasks.cpython-38.pyc b/ytdlbot/__pycache__/flower_tasks.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..334819dae969d7b21916af5181d0f645c0acc97d Binary files /dev/null and b/ytdlbot/__pycache__/flower_tasks.cpython-38.pyc differ diff --git a/ytdlbot/__pycache__/limit.cpython-310.pyc b/ytdlbot/__pycache__/limit.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8e74633a3a76a91108c4b122b101b19befd0ed38 Binary files /dev/null and b/ytdlbot/__pycache__/limit.cpython-310.pyc differ diff --git a/ytdlbot/__pycache__/limit.cpython-38.pyc b/ytdlbot/__pycache__/limit.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..81e93c1f34e312df4337f1803180c958225f0731 Binary files /dev/null and b/ytdlbot/__pycache__/limit.cpython-38.pyc differ diff --git a/ytdlbot/__pycache__/tasks.cpython-310.pyc b/ytdlbot/__pycache__/tasks.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7f3308e53aa21c65af3964683c631dae70c814ca Binary files /dev/null and b/ytdlbot/__pycache__/tasks.cpython-310.pyc differ diff --git a/ytdlbot/__pycache__/tasks.cpython-38.pyc b/ytdlbot/__pycache__/tasks.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c8b9abe895148e7574d59f75f2113f948c348d86 Binary files /dev/null and b/ytdlbot/__pycache__/tasks.cpython-38.pyc differ diff --git a/ytdlbot/__pycache__/utils.cpython-310.pyc b/ytdlbot/__pycache__/utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..821c08cf0a0abba600cd14d371a9540b0438ed20 Binary files /dev/null and b/ytdlbot/__pycache__/utils.cpython-310.pyc differ diff --git a/ytdlbot/__pycache__/utils.cpython-38.pyc b/ytdlbot/__pycache__/utils.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d2a23300080e4ed07eb6f22c7aeaff4273006e0b Binary files /dev/null and b/ytdlbot/__pycache__/utils.cpython-38.pyc differ diff --git a/ytdlbot/channel.py b/ytdlbot/channel.py new file mode 100644 index 0000000000000000000000000000000000000000..52d2236b8d7d3959059dc98ace6fa79885f5bbcc --- /dev/null +++ b/ytdlbot/channel.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +# coding: utf-8 +import http +import logging +import os +import re + +import requests +from bs4 import BeautifulSoup + +from config import ENABLE_VIP +from limit import Payment + + +class Channel(Payment): + def subscribe_channel(self, user_id: int, share_link: str) -> str: + if not re.findall(r"youtube\.com|youtu\.be", share_link): + raise ValueError("Is this a valid YouTube Channel link?") + if ENABLE_VIP: + self.cur.execute("select count(user_id) from subscribe where user_id=%s", (user_id,)) + usage = int(self.cur.fetchone()[0]) + if usage >= 10: + logging.warning("User %s has subscribed %s channels", user_id, usage) + return "You have subscribed too many channels. Maximum 5 channels." + + data = self.get_channel_info(share_link) + channel_id = data["channel_id"] + + self.cur.execute("select user_id from subscribe where user_id=%s and channel_id=%s", (user_id, channel_id)) + if self.cur.fetchall(): + raise ValueError("You have already subscribed this channel.") + + self.cur.execute( + "INSERT IGNORE INTO channel values" + "(%(link)s,%(title)s,%(description)s,%(channel_id)s,%(playlist)s,%(last_video)s)", + data, + ) + self.cur.execute("INSERT INTO subscribe values(%s,%s, NULL)", (user_id, channel_id)) + self.con.commit() + logging.info("User %s subscribed channel %s", user_id, data["title"]) + return "Subscribed to {}".format(data["title"]) + + def unsubscribe_channel(self, user_id: int, channel_id: str) -> int: + affected_rows = self.cur.execute( + "DELETE FROM subscribe WHERE user_id=%s AND channel_id=%s", (user_id, channel_id) + ) + self.con.commit() + logging.info("User %s tried to unsubscribe channel %s", user_id, channel_id) + return affected_rows + + @staticmethod + def extract_canonical_link(url: str) -> str: + # canonic link works for many websites. It will strip out unnecessary stuff + props = ["canonical", "alternate", "shortlinkUrl"] + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36" + } + cookie = {"CONSENT": "PENDING+197"} + # send head request first + r = requests.head(url, headers=headers, allow_redirects=True, cookies=cookie) + if r.status_code != http.HTTPStatus.METHOD_NOT_ALLOWED and "text/html" not in r.headers.get("content-type", ""): + # get content-type, if it's not text/html, there's no need to issue a GET request + logging.warning("%s Content-type is not text/html, no need to GET for extract_canonical_link", url) + return url + + html_doc = requests.get(url, headers=headers, cookies=cookie, timeout=5).text + soup = BeautifulSoup(html_doc, "html.parser") + for prop in props: + element = soup.find("link", rel=prop) + try: + href = element["href"] + if href not in ["null", "", None, "https://consent.youtube.com/m"]: + return href + except Exception as e: + logging.debug("Canonical exception %s %s e", url, e) + + return url + + def get_channel_info(self, url: str) -> dict: + api_key = os.getenv("GOOGLE_API_KEY") + canonical_link = self.extract_canonical_link(url) + try: + channel_id = canonical_link.split("youtube.com/channel/")[1] + except IndexError: + channel_id = canonical_link.split("/")[-1] + channel_api = ( + f"https://www.googleapis.com/youtube/v3/channels?part=snippet,contentDetails&id={channel_id}&key={api_key}" + ) + + data = requests.get(channel_api).json() + snippet = data["items"][0]["snippet"] + title = snippet["title"] + description = snippet["description"] + playlist = data["items"][0]["contentDetails"]["relatedPlaylists"]["uploads"] + + return { + "link": url, + "title": title, + "description": description, + "channel_id": channel_id, + "playlist": playlist, + "last_video": self.get_latest_video(playlist), + } + + @staticmethod + def get_latest_video(playlist_id: str) -> str: + api_key = os.getenv("GOOGLE_API_KEY") + video_api = ( + f"https://www.googleapis.com/youtube/v3/playlistItems?part=snippet&maxResults=1&" + f"playlistId={playlist_id}&key={api_key}" + ) + data = requests.get(video_api).json() + video_id = data["items"][0]["snippet"]["resourceId"]["videoId"] + logging.info(f"Latest video %s from %s", video_id, data["items"][0]["snippet"]["channelTitle"]) + return f"https://www.youtube.com/watch?v={video_id}" + + def has_newer_update(self, channel_id: str) -> str: + self.cur.execute("SELECT playlist,latest_video FROM channel WHERE channel_id=%s", (channel_id,)) + data = self.cur.fetchone() + playlist_id = data[0] + old_video = data[1] + newest_video = self.get_latest_video(playlist_id) + if old_video != newest_video: + logging.info("Newer update found for %s %s", channel_id, newest_video) + self.cur.execute("UPDATE channel SET latest_video=%s WHERE channel_id=%s", (newest_video, channel_id)) + self.con.commit() + return newest_video + + def get_user_subscription(self, user_id: int) -> str: + self.cur.execute( + """ + select title, link, channel.channel_id from channel, subscribe + where subscribe.user_id = %s and channel.channel_id = subscribe.channel_id + """, + (user_id,), + ) + data = self.cur.fetchall() + text = "" + for item in data: + text += "[{}]({}) `{}\n`".format(*item) + return text + + def group_subscriber(self) -> dict: + # {"channel_id": [user_id, user_id, ...]} + self.cur.execute("select * from subscribe where is_valid=1") + data = self.cur.fetchall() + group = {} + for item in data: + group.setdefault(item[1], []).append(item[0]) + logging.info("Checking periodic subscriber...") + return group + + def deactivate_user_subscription(self, user_id: int): + self.cur.execute("UPDATE subscribe set is_valid=0 WHERE user_id=%s", (user_id,)) + self.con.commit() + + def sub_count(self) -> str: + sql = """ + select user_id, channel.title, channel.link + from subscribe, channel where subscribe.channel_id = channel.channel_id + """ + self.cur.execute(sql) + data = self.cur.fetchall() + text = f"Total {len(data)} subscriptions found.\n\n" + for item in data: + text += "{} ==> [{}]({})\n".format(*item) + return text + + def del_cache(self, user_link: str) -> int: + unique = self.extract_canonical_link(user_link) + caches = self.r.hgetall("cache") + count = 0 + for key in caches: + if key.startswith(unique): + count += self.del_send_cache(key) + return count + + +if __name__ == "__main__": + s = Channel.extract_canonical_link("https://www.youtube.com/shorts/KkbYbknjPBM") + print(s) diff --git a/ytdlbot/client_init.py b/ytdlbot/client_init.py new file mode 100644 index 0000000000000000000000000000000000000000..3fa8c2056e1798ec0d77a48794e1de8421aa0902 --- /dev/null +++ b/ytdlbot/client_init.py @@ -0,0 +1,23 @@ +#!/usr/local/bin/python3 +# coding: utf-8 + +# ytdlbot - client_init.py +# 12/29/21 16:20 +# + +__author__ = "Benny " + +from pyrogram import Client + +from config import APP_HASH, APP_ID, PYRO_WORKERS, TOKEN, IPv6 + + +def create_app(name: str, workers: int = PYRO_WORKERS) -> Client: + return Client( + name, + APP_ID, + APP_HASH, + bot_token=TOKEN, + workers=workers, + ipv6=IPv6, + ) diff --git a/ytdlbot/config.py b/ytdlbot/config.py new file mode 100644 index 0000000000000000000000000000000000000000..6661a6bd9546cb22affd6fc60203d5ef3814afac --- /dev/null +++ b/ytdlbot/config.py @@ -0,0 +1,79 @@ +#!/usr/local/bin/python3 +# coding: utf-8 + +# ytdlbot - config.py +# 8/28/21 15:01 +# + +__author__ = "Benny " + +import os + +from blinker import signal + +# general settings +WORKERS: int = int(os.getenv("WORKERS", 10)) +PYRO_WORKERS: int = int(os.getenv("PYRO_WORKERS", 100)) +APP_ID: int = int(20295350) +APP_HASH = str("805a0a86f3b382d904617d0a2fd4fb6f") +TOKEN = str("6718500924:AAHljBZcqaegxtDMOODLomnbeR625E-s5uw") + +REDIS = os.getenv("REDIS", "redis") + +TG_MAX_SIZE = 2000 * 1024 * 1024 +# TG_MAX_SIZE = 10 * 1024 * 1024 + +EXPIRE = 24 * 3600 + +ENABLE_VIP = os.getenv("VIP", False) +OWNER = str("navpan18") + +# limitation settings +AUTHORIZED_USER: str = os.getenv("AUTHORIZED_USER", "") +# membership requires: the format could be username(without @ sign)/chat_id of channel or group. +# You need to add the bot to this group/channel as admin +REQUIRED_MEMBERSHIP: str = os.getenv("REQUIRED_MEMBERSHIP", "") + +# celery related +IS_BACKUP_BOT = os.getenv("IS_BACKUP_BOT") +ENABLE_CELERY = os.getenv("ENABLE_CELERY", False) +if IS_BACKUP_BOT: + BROKER = os.getenv("BROKER", f"redis://{REDIS}:6379/1") +else: + BROKER = os.getenv("BROKER", f"redis://{REDIS}:6379/0") + +MYSQL_HOST = os.getenv("MYSQL_HOST", "mysql") +MYSQL_USER = os.getenv("MYSQL_USER", "root") +MYSQL_PASS = os.getenv("MYSQL_PASS", "root") + +AUDIO_FORMAT = os.getenv("AUDIO_FORMAT") +ARCHIVE_ID = -1002047782676 + +IPv6 = os.getenv("IPv6", False) +ENABLE_FFMPEG = os.getenv("ENABLE_FFMPEG", False) + +PLAYLIST_SUPPORT = True +M3U8_SUPPORT = True +ENABLE_ARIA2 = os.getenv("ENABLE_ARIA2", False) + +RATE_LIMIT = os.getenv("RATE_LIMIT", 120) +RCLONE_PATH = os.getenv("RCLONE") +# This will set the value for the tmpfile path(download path) if it is set. +# If TMPFILE is not set, it will return None and use system’s default temporary file path. +# Please ensure that the directory exists and you have necessary permissions to write to it. +# If you don't know what this is just leave it as it is. +TMPFILE_PATH = os.getenv("TMPFILE") + +# payment settings +AFD_LINK = os.getenv("AFD_LINK", "https://afdian.net/@BennyThink") +COFFEE_LINK = os.getenv("COFFEE_LINK", "https://www.buymeacoffee.com/bennythink") +COFFEE_TOKEN = os.getenv("COFFEE_TOKEN") +AFD_TOKEN = os.getenv("AFD_TOKEN") +AFD_USER_ID = os.getenv("AFD_USER_ID") +PROVIDER_TOKEN = os.getenv("PROVIDER_TOKEN") or "1234" +FREE_DOWNLOAD = 20000 +TOKEN_PRICE = os.getenv("BUY_UNIT", 20) # one USD=20 credits +TRONGRID_KEY = os.getenv("TRONGRID_KEY", "").split(",") +# the default mnemonic is for nile testnet +TRON_MNEMONIC = os.getenv("TRON_MNEMONIC", "cram floor today legend service drill pitch leaf car govern harvest soda") +TRX_SIGNAL = signal("trx_received") diff --git a/ytdlbot/constant.py b/ytdlbot/constant.py new file mode 100644 index 0000000000000000000000000000000000000000..14750a1d16532773588cded0823294c6f0e75bda --- /dev/null +++ b/ytdlbot/constant.py @@ -0,0 +1,117 @@ +#!/usr/local/bin/python3 +# coding: utf-8 + +# ytdlbot - constant.py +# 8/16/21 16:59 +# + +__author__ = "Benny " + +import os + +from config import ( + AFD_LINK, + COFFEE_LINK, + ENABLE_CELERY, + FREE_DOWNLOAD, + REQUIRED_MEMBERSHIP, + TOKEN_PRICE, +) +from database import InfluxDB +from utils import get_func_queue + + +class BotText: + start = """ + Welcome to YouTube Download bot. Type /help for more information. + Backup bot: @benny_2ytdlbot + Join https://t.me/+OGRC8tp9-U9mZDZl for updates.""" + + help = f""" +1. If the bot doesn't work, try again or join https://t.me/+OGRC8tp9-U9mZDZl for updates. + +2. Source code: https://github.com/tgbot-collection/ytdlbot + """ + + about = "YouTube Downloader by @BennyThink.\n\nOpen source on GitHub: https://github.com/tgbot-collection/ytdlbot" + + buy = f""" +**Terms:** +1. You can use this bot to download video for {FREE_DOWNLOAD} times within a 24-hour period. + +2. You can buy additional download tokens, valid permanently. + +3. Refunds are possible, contact me if you need that @BennyThink + +4. Download for paid user will be automatically changed to Local mode to avoid queuing. + +**Price:** +valid permanently +1. 1 USD == {TOKEN_PRICE} tokens +2. 7 CNY == {TOKEN_PRICE} tokens +3. 10 TRX == {TOKEN_PRICE} tokens + +**Payment options:** +Pay any amount you want. For example you can send 20 TRX for {TOKEN_PRICE * 2} tokens. +1. AFDIAN(AliPay, WeChat Pay and PayPal): {AFD_LINK} +2. Buy me a coffee: {COFFEE_LINK} +3. Telegram Bot Payment(Stripe), please click Bot Payment button. +4. TRON(TRX), please click TRON(TRX) button. + +**After payment:** +1. Afdian: attach order number with /redeem command (e.g., `/redeem 123456`). +2. Buy Me a Coffee: attach email with /redeem command (e.g., `/redeem 123@x.com`). **Use different email each time.** +3. Telegram Payment & Tron(TRX): automatically activated within 60s. Check /start to see your balance. + +Want to buy more token with Telegram payment? Let's say 100? Here you go! `/buy 123` + """ + + private = "This bot is for private use" + + membership_require = f"You need to join this group or channel to use this bot\n\nhttps://t.me/{REQUIRED_MEMBERSHIP}" + + settings = """ +Please choose the preferred format and video quality for your video. These settings only **apply to YouTube videos**. + +High quality is recommended. Medium quality aims to 720P, while low quality is 480P. + +If you choose to send the video as a document, it will not be possible to stream it. + +Your current settings: +Video quality: **{0}** +Sending format: **{1}** +""" + custom_text = os.getenv("CUSTOM_TEXT", "") + + @staticmethod + def get_receive_link_text() -> str: + reserved = get_func_queue("reserved") + if ENABLE_CELERY and reserved: + text = f"Your tasks was added to the reserved queue {reserved}. Processing...\n\n" + else: + text = "Your task was added to active queue.\nProcessing...\n\n" + + return text + + @staticmethod + def ping_worker() -> str: + from tasks import app as celery_app + + workers = InfluxDB().extract_dashboard_data() + # [{'celery@BennyのMBP': 'abc'}, {'celery@BennyのMBP': 'abc'}] + response = celery_app.control.broadcast("ping_revision", reply=True) + revision = {} + for item in response: + revision.update(item) + + text = "" + for worker in workers: + fields = worker["fields"] + hostname = worker["tags"]["hostname"] + status = {True: "✅"}.get(fields["status"], "❌") + active = fields["active"] + load = "{},{},{}".format(fields["load1"], fields["load5"], fields["load15"]) + rev = revision.get(hostname, "") + text += f"{status}{hostname} **{active}** {load} {rev}\n" + + return text diff --git a/ytdlbot/database.py b/ytdlbot/database.py new file mode 100644 index 0000000000000000000000000000000000000000..7243a7f058023e2afd9c604b71b0420ba227ca0b --- /dev/null +++ b/ytdlbot/database.py @@ -0,0 +1,375 @@ +#!/usr/local/bin/python3 +# coding: utf-8 + +# ytdlbot - database.py +# 12/7/21 16:57 +# + +__author__ = "Benny " + +import base64 +import contextlib +import datetime +import logging +import os +import re +import sqlite3 +import subprocess +import time +from io import BytesIO + +import fakeredis +import pymysql +import redis +import requests +from beautifultable import BeautifulTable +from influxdb import InfluxDBClient + +from config import MYSQL_HOST, MYSQL_PASS, MYSQL_USER, REDIS, IS_BACKUP_BOT + +init_con = sqlite3.connect(":memory:", check_same_thread=False) + + +class FakeMySQL: + @staticmethod + def cursor() -> "Cursor": + return Cursor() + + def commit(self): + pass + + def close(self): + pass + + def ping(self, reconnect): + pass + + +class Cursor: + def __init__(self): + self.con = init_con + self.cur = self.con.cursor() + + def execute(self, *args, **kwargs): + sql = self.sub(args[0]) + new_args = (sql,) + args[1:] + with contextlib.suppress(sqlite3.OperationalError): + return self.cur.execute(*new_args, **kwargs) + + def fetchall(self): + return self.cur.fetchall() + + def fetchone(self): + return self.cur.fetchone() + + @staticmethod + def sub(sql): + sql = re.sub(r"CHARSET.*|charset.*", "", sql, re.IGNORECASE) + sql = sql.replace("%s", "?") + return sql + + +class Redis: + def __init__(self): + db = 1 if IS_BACKUP_BOT else 0 + try: + self.r = redis.StrictRedis(host=REDIS, db=db, decode_responses=True) + self.r.ping() + except Exception: + self.r = fakeredis.FakeStrictRedis(host=REDIS, db=db, decode_responses=True) + + db_banner = "=" * 20 + "DB data" + "=" * 20 + quota_banner = "=" * 20 + "Celery" + "=" * 20 + metrics_banner = "=" * 20 + "Metrics" + "=" * 20 + usage_banner = "=" * 20 + "Usage" + "=" * 20 + vnstat_banner = "=" * 20 + "vnstat" + "=" * 20 + self.final_text = f""" +{db_banner} +%s + + +{vnstat_banner} +%s + + +{quota_banner} +%s + + +{metrics_banner} +%s + + +{usage_banner} +%s + """ + super().__init__() + + def __del__(self): + self.r.close() + + def update_metrics(self, metrics: str): + logging.info(f"Setting metrics: {metrics}") + all_ = f"all_{metrics}" + today = f"today_{metrics}" + self.r.hincrby("metrics", all_) + self.r.hincrby("metrics", today) + + @staticmethod + def generate_table(header, all_data: list): + table = BeautifulTable() + for data in all_data: + table.rows.append(data) + table.columns.header = header + table.rows.header = [str(i) for i in range(1, len(all_data) + 1)] + return table + + def show_usage(self): + db = MySQL() + db.cur.execute("select user_id,payment_amount,old_user,token from payment") + data = db.cur.fetchall() + fd = [] + for item in data: + fd.append([item[0], item[1], item[2], item[3]]) + db_text = self.generate_table(["ID", "pay amount", "old user", "token"], fd) + + fd = [] + hash_keys = self.r.hgetall("metrics") + for key, value in hash_keys.items(): + if re.findall(r"^today|all", key): + fd.append([key, value]) + fd.sort(key=lambda x: x[0]) + metrics_text = self.generate_table(["name", "count"], fd) + + fd = [] + for key, value in hash_keys.items(): + if re.findall(r"\d+", key): + fd.append([key, value]) + fd.sort(key=lambda x: int(x[-1]), reverse=True) + usage_text = self.generate_table(["UserID", "count"], fd) + + worker_data = InfluxDB.get_worker_data() + fd = [] + for item in worker_data["data"]: + fd.append( + [ + item.get("hostname", 0), + item.get("status", 0), + item.get("active", 0), + item.get("processed", 0), + item.get("task-failed", 0), + item.get("task-succeeded", 0), + ",".join(str(i) for i in item.get("loadavg", [])), + ] + ) + + worker_text = self.generate_table( + ["worker name", "status", "active", "processed", "failed", "succeeded", "Load Average"], fd + ) + + # vnstat + if os.uname().sysname == "Darwin": + cmd = "/opt/homebrew/bin/vnstat -i en0".split() + else: + cmd = "/usr/bin/vnstat -i eth0".split() + vnstat_text = subprocess.check_output(cmd).decode("u8") + return self.final_text % (db_text, vnstat_text, worker_text, metrics_text, usage_text) + + def reset_today(self): + pairs = self.r.hgetall("metrics") + for k in pairs: + if k.startswith("today"): + self.r.hdel("metrics", k) + + def user_count(self, user_id): + self.r.hincrby("metrics", user_id) + + def generate_file(self): + text = self.show_usage() + file = BytesIO() + file.write(text.encode("u8")) + date = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time())) + file.name = f"{date}.txt" + return file + + def add_send_cache(self, unique: str, file_id: str): + self.r.hset("cache", unique, file_id) + + def get_send_cache(self, unique) -> str: + return self.r.hget("cache", unique) + + def del_send_cache(self, unique): + return self.r.hdel("cache", unique) + + +class MySQL: + vip_sql = """ + CREATE TABLE if not exists `payment` + ( + `user_id` bigint NOT NULL, + `payment_amount` float DEFAULT NULL, + `payment_id` varchar(256) DEFAULT NULL, + `old_user` tinyint(1) DEFAULT NULL, + `token` int DEFAULT NULL, + UNIQUE KEY `payment_id` (`payment_id`) + ) CHARSET = utf8mb4 + """ + + settings_sql = """ + create table if not exists settings + ( + user_id bigint not null, + resolution varchar(128) null, + method varchar(64) null, + mode varchar(32) default 'Celery' null, + constraint settings_pk + primary key (user_id) + ); + """ + + channel_sql = """ + create table if not exists channel + ( + link varchar(256) null, + title varchar(256) null, + description text null, + channel_id varchar(256), + playlist varchar(256) null, + latest_video varchar(256) null, + constraint channel_pk + primary key (channel_id) + ) CHARSET=utf8mb4; + """ + + subscribe_sql = """ + create table if not exists subscribe + ( + user_id bigint null, + channel_id varchar(256) null, + is_valid boolean default 1 null + ) CHARSET=utf8mb4; + """ + + def __init__(self): + try: + self.con = pymysql.connect( + host=MYSQL_HOST, user=MYSQL_USER, passwd=MYSQL_PASS, db="ytdl", charset="utf8mb4" + ) + except Exception: + + self.con = FakeMySQL() + + self.con.ping(reconnect=True) + self.cur = self.con.cursor() + self.init_db() + super().__init__() + + def init_db(self): + self.cur.execute(self.vip_sql) + self.cur.execute(self.settings_sql) + self.cur.execute(self.channel_sql) + self.cur.execute(self.subscribe_sql) + self.con.commit() + + def __del__(self): + self.con.close() + + def get_user_settings(self, user_id: int) -> tuple: + self.cur.execute("SELECT * FROM settings WHERE user_id = %s", (user_id,)) + data = self.cur.fetchone() + if data is None: + return 100, "high", "video", "Celery" + return data + + def set_user_settings(self, user_id: int, field: str, value: str): + cur = self.con.cursor() + cur.execute("SELECT * FROM settings WHERE user_id = %s", (user_id,)) + data = cur.fetchone() + if data is None: + resolution = method = "" + if field == "resolution": + method = "video" + resolution = value + if field == "method": + method = value + resolution = "high" + cur.execute("INSERT INTO settings VALUES (%s,%s,%s,%s)", (user_id, resolution, method, "Celery")) + else: + cur.execute(f"UPDATE settings SET {field} =%s WHERE user_id = %s", (value, user_id)) + self.con.commit() + + +class InfluxDB: + def __init__(self): + self.client = InfluxDBClient(host=os.getenv("INFLUX_HOST", "192.168.7.233"), database="celery") + self.data = None + + def __del__(self): + self.client.close() + + @staticmethod + def get_worker_data() -> dict: + username = os.getenv("FLOWER_USERNAME", "benny") + password = os.getenv("FLOWER_PASSWORD", "123456abc") + token = base64.b64encode(f"{username}:{password}".encode()).decode() + headers = {"Authorization": f"Basic {token}"} + r = requests.get("https://celery.dmesg.app/dashboard?json=1", headers=headers) + if r.status_code != 200: + return dict(data=[]) + return r.json() + + def extract_dashboard_data(self): + self.data = self.get_worker_data() + json_body = [] + for worker in self.data["data"]: + load1, load5, load15 = worker["loadavg"] + t = { + "measurement": "tasks", + "tags": { + "hostname": worker["hostname"], + }, + "time": datetime.datetime.utcnow(), + "fields": { + "task-received": worker.get("task-received", 0), + "task-started": worker.get("task-started", 0), + "task-succeeded": worker.get("task-succeeded", 0), + "task-failed": worker.get("task-failed", 0), + "active": worker.get("active", 0), + "status": worker.get("status", False), + "load1": load1, + "load5": load5, + "load15": load15, + }, + } + json_body.append(t) + return json_body + + def __fill_worker_data(self): + json_body = self.extract_dashboard_data() + self.client.write_points(json_body) + + def __fill_overall_data(self): + active = sum([i["active"] for i in self.data["data"]]) + json_body = [{"measurement": "active", "time": datetime.datetime.utcnow(), "fields": {"active": active}}] + self.client.write_points(json_body) + + def __fill_redis_metrics(self): + json_body = [{"measurement": "metrics", "time": datetime.datetime.utcnow(), "fields": {}}] + r = Redis().r + hash_keys = r.hgetall("metrics") + for key, value in hash_keys.items(): + if re.findall(r"^today", key): + json_body[0]["fields"][key] = int(value) + + self.client.write_points(json_body) + + def collect_data(self): + if os.getenv("INFLUX_HOST") is None: + return + + with contextlib.suppress(Exception): + self.data = self.get_worker_data() + self.__fill_worker_data() + self.__fill_overall_data() + self.__fill_redis_metrics() + logging.debug("InfluxDB data was collected.") diff --git a/ytdlbot/downloader.py b/ytdlbot/downloader.py new file mode 100644 index 0000000000000000000000000000000000000000..8fecc3bc463a115b12e31699a6a46d48b4cf2883 --- /dev/null +++ b/ytdlbot/downloader.py @@ -0,0 +1,281 @@ +#!/usr/local/bin/python3 +# coding: utf-8 + +# ytdlbot - downloader.py +# 8/14/21 16:53 +# + +__author__ = "Benny " + +import logging +import os +import pathlib +import random +import re +import subprocess +import time +import traceback +from io import StringIO +from unittest.mock import MagicMock + +import fakeredis +import ffmpeg +import ffpb +import filetype +import requests +import yt_dlp as ytdl +from pyrogram import types +from tqdm import tqdm + +from config import AUDIO_FORMAT, ENABLE_ARIA2, ENABLE_FFMPEG, TG_MAX_SIZE, IPv6 +from limit import Payment +from utils import adjust_formats, apply_log_formatter, current_time, sizeof_fmt + +r = fakeredis.FakeStrictRedis() +apply_log_formatter() + + +def edit_text(bot_msg: types.Message, text: str): + key = f"{bot_msg.chat.id}-{bot_msg.id}" + # if the key exists, we shouldn't send edit message + if not r.exists(key): + time.sleep(random.random()) + r.set(key, "ok", ex=3) + bot_msg.edit_text(text) + + +def tqdm_progress(desc, total, finished, speed="", eta=""): + def more(title, initial): + if initial: + return f"{title} {initial}" + else: + return "" + + f = StringIO() + tqdm( + total=total, + initial=finished, + file=f, + ascii=False, + unit_scale=True, + ncols=30, + bar_format="{l_bar}{bar} |{n_fmt}/{total_fmt} ", + ) + raw_output = f.getvalue() + tqdm_output = raw_output.split("|") + progress = f"`[{tqdm_output[1]}]`" + detail = tqdm_output[2].replace("[A", "") + text = f""" +{desc} + +{progress} +{detail} +{more("Speed:", speed)} +{more("ETA:", eta)} + """ + f.close() + return text + + +def remove_bash_color(text): + return re.sub(r"\u001b|\[0;94m|\u001b\[0m|\[0;32m|\[0m|\[0;33m", "", text) + + +def download_hook(d: dict, bot_msg): + # since we're using celery, server location may be located in different region. + # Therefore, we can't trigger the hook very often. + # the key is user_id + download_link + original_url = d["info_dict"]["original_url"] + key = f"{bot_msg.chat.id}-{original_url}" + + if d["status"] == "downloading": + downloaded = d.get("downloaded_bytes", 0) + total = d.get("total_bytes") or d.get("total_bytes_estimate", 0) + if total > TG_MAX_SIZE: + raise Exception(f"Your download file size {sizeof_fmt(total)} is too large for Telegram.") + + # percent = remove_bash_color(d.get("_percent_str", "N/A")) + speed = remove_bash_color(d.get("_speed_str", "N/A")) + eta = remove_bash_color(d.get("_eta_str", d.get("eta"))) + text = tqdm_progress("Downloading...", total, downloaded, speed, eta) + edit_text(bot_msg, text) + r.set(key, "ok", ex=5) + + +def upload_hook(current, total, bot_msg): + text = tqdm_progress("Uploading...", total, current) + edit_text(bot_msg, text) + + +def convert_to_mp4(video_paths: list, bot_msg): + default_type = ["video/x-flv", "video/webm"] + # all_converted = [] + for path in video_paths: + # if we can't guess file type, we assume it's video/mp4 + mime = getattr(filetype.guess(path), "mime", "video/mp4") + if mime in default_type: + if not can_convert_mp4(path, bot_msg.chat.id): + logging.warning("Conversion abort for %s", bot_msg.chat.id) + bot_msg._client.send_message(bot_msg.chat.id, "Can't convert your video. ffmpeg has been disabled.") + break + edit_text(bot_msg, f"{current_time()}: Converting {path.name} to mp4. Please wait.") + new_file_path = path.with_suffix(".mp4") + logging.info("Detected %s, converting to mp4...", mime) + run_ffmpeg_progressbar(["ffmpeg", "-y", "-i", path, new_file_path], bot_msg) + index = video_paths.index(path) + video_paths[index] = new_file_path + + +class ProgressBar(tqdm): + b = None + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.bot_msg = self.b + + def update(self, n=1): + super().update(n) + t = tqdm_progress("Converting...", self.total, self.n) + edit_text(self.bot_msg, t) + + +def run_ffmpeg_progressbar(cmd_list: list, bm): + cmd_list = cmd_list.copy()[1:] + ProgressBar.b = bm + ffpb.main(cmd_list, tqdm=ProgressBar) + + +def can_convert_mp4(video_path, uid): + if not ENABLE_FFMPEG: + return False + return True + + +def ytdl_download(url: str, tempdir: str, bm, **kwargs) -> list: + payment = Payment() + chat_id = bm.chat.id + hijack = kwargs.get("hijack") + output = pathlib.Path(tempdir, "%(title).70s.%(ext)s").as_posix() + ydl_opts = { + "progress_hooks": [lambda d: download_hook(d, bm)], + "outtmpl": output, + "restrictfilenames": False, + "quiet": True, + } + if ENABLE_ARIA2: + ydl_opts["external_downloader"] = "aria2c" + ydl_opts["external_downloader_args"] = [ + "--min-split-size=1M", + "--max-connection-per-server=16", + "--max-concurrent-downloads=16", + "--split=16", + ] + if url.startswith("https://drive.google.com"): + # Always use the `source` format for Google Drive URLs. + formats = ["source"] + else: + # Use the default formats for other URLs. + formats = [ + # webm , vp9 and av01 are not streamable on telegram, so we'll extract only mp4 + "bestvideo[ext=mp4][vcodec!*=av01][vcodec!*=vp09]+bestaudio[ext=m4a]/bestvideo+bestaudio", + "bestvideo[vcodec^=avc]+bestaudio[acodec^=mp4a]/best[vcodec^=avc]/best", + None, + ] + adjust_formats(chat_id, url, formats, hijack) + if download_instagram(url, tempdir): + return list(pathlib.Path(tempdir).glob("*")) + + address = ["::", "0.0.0.0"] if IPv6 else [None] + error = None + video_paths = None + for format_ in formats: + ydl_opts["format"] = format_ + for addr in address: + # IPv6 goes first in each format + ydl_opts["source_address"] = addr + try: + logging.info("Downloading for %s with format %s", url, format_) + with ytdl.YoutubeDL(ydl_opts) as ydl: + ydl.download([url]) + video_paths = list(pathlib.Path(tempdir).glob("*")) + break + except Exception: + error = traceback.format_exc() + logging.error("Download failed for %s - %s, try another way", format_, url) + if error is None: + break + + if not video_paths: + raise Exception(error) + + # convert format if necessary + settings = payment.get_user_settings(chat_id) + if settings[2] == "video" or isinstance(settings[2], MagicMock): + # only convert if send type is video + convert_to_mp4(video_paths, bm) + if settings[2] == "audio" or hijack == "bestaudio[ext=m4a]": + convert_audio_format(video_paths, bm) + # split_large_video(video_paths) + return video_paths + + +def convert_audio_format(video_paths: list, bm): + # 1. file is audio, default format + # 2. file is video, default format + # 3. non default format + + for path in video_paths: + streams = ffmpeg.probe(path)["streams"] + if AUDIO_FORMAT is None and len(streams) == 1 and streams[0]["codec_type"] == "audio": + logging.info("%s is audio, default format, no need to convert", path) + elif AUDIO_FORMAT is None and len(streams) >= 2: + logging.info("%s is video, default format, need to extract audio", path) + audio_stream = {"codec_name": "m4a"} + for stream in streams: + if stream["codec_type"] == "audio": + audio_stream = stream + break + ext = audio_stream["codec_name"] + new_path = path.with_suffix(f".{ext}") + run_ffmpeg_progressbar(["ffmpeg", "-y", "-i", path, "-vn", "-acodec", "copy", new_path], bm) + path.unlink() + index = video_paths.index(path) + video_paths[index] = new_path + else: + logging.info("Not default format, converting %s to %s", path, AUDIO_FORMAT) + new_path = path.with_suffix(f".{AUDIO_FORMAT}") + run_ffmpeg_progressbar(["ffmpeg", "-y", "-i", path, new_path], bm) + path.unlink() + index = video_paths.index(path) + video_paths[index] = new_path + + +def split_large_video(video_paths: list): + original_video = None + split = False + for original_video in video_paths: + size = os.stat(original_video).st_size + if size > TG_MAX_SIZE: + split = True + logging.warning("file is too large %s, splitting...", size) + subprocess.check_output(f"sh split-video.sh {original_video} {TG_MAX_SIZE * 0.95} ".split()) + os.remove(original_video) + + if split and original_video: + return [i for i in pathlib.Path(original_video).parent.glob("*")] + + +def download_instagram(url: str, tempdir: str): + if not url.startswith("https://www.instagram.com"): + return False + + resp = requests.get(f"http://192.168.6.1:15000/?url={url}").json() + if url_results := resp.get("data"): + for link in url_results: + content = requests.get(link, stream=True).content + ext = filetype.guess_extension(content) + save_path = pathlib.Path(tempdir, f"{id(link)}.{ext}") + with open(save_path, "wb") as f: + f.write(content) + + return True diff --git a/ytdlbot/flower_tasks.py b/ytdlbot/flower_tasks.py new file mode 100644 index 0000000000000000000000000000000000000000..c791421a48a1b14b3a1e35cb2a5b4cfb0aab85c2 --- /dev/null +++ b/ytdlbot/flower_tasks.py @@ -0,0 +1,14 @@ +#!/usr/local/bin/python3 +# coding: utf-8 + +# ytdlbot - flower_tasks.py +# 1/2/22 10:17 +# + +__author__ = "Benny " + +from celery import Celery + +from config import BROKER + +app = Celery("tasks", broker=BROKER, timezone="Europe/London") diff --git a/ytdlbot/limit.py b/ytdlbot/limit.py new file mode 100644 index 0000000000000000000000000000000000000000..819978dbd067e215e6a159036510f451779c79a9 --- /dev/null +++ b/ytdlbot/limit.py @@ -0,0 +1,260 @@ +#!/usr/local/bin/python3 +# coding: utf-8 + +# ytdlbot - limit.py +# 8/15/21 18:23 +# + +__author__ = "Benny " + +import hashlib +import logging +import time + +import requests +from tronpy import Tron +from tronpy.exceptions import TransactionError, ValidationError +from tronpy.hdwallet import key_from_seed, seed_from_mnemonic +from tronpy.keys import PrivateKey +from tronpy.providers import HTTPProvider + +from config import ( + AFD_TOKEN, + AFD_USER_ID, + COFFEE_TOKEN, + EXPIRE, + FREE_DOWNLOAD, + OWNER, + TOKEN_PRICE, + TRON_MNEMONIC, + TRONGRID_KEY, + TRX_SIGNAL, +) +from database import MySQL, Redis +from utils import apply_log_formatter, current_time + +apply_log_formatter() + + +class BuyMeACoffee: + def __init__(self): + self._token = COFFEE_TOKEN + self._url = "https://developers.buymeacoffee.com/api/v1/supporters" + self._data = [] + + def _get_data(self, url): + d = requests.get(url, headers={"Authorization": f"Bearer {self._token}"}).json() + self._data.extend(d["data"]) + next_page = d["next_page_url"] + if next_page: + self._get_data(next_page) + + def _get_bmac_status(self, email: str) -> dict: + self._get_data(self._url) + for user in self._data: + if user["payer_email"] == email or user["support_email"] == email: + return user + return {} + + def get_user_payment(self, email: str) -> (int, "float", str): + order = self._get_bmac_status(email) + price = float(order.get("support_coffee_price", 0)) + cups = float(order.get("support_coffees", 1)) + amount = price * cups + return amount, email + + +class Afdian: + def __init__(self): + self._token = AFD_TOKEN + self._user_id = AFD_USER_ID + self._url = "https://afdian.net/api/open/query-order" + + def _generate_signature(self): + data = { + "user_id": self._user_id, + "params": '{"x":0}', + "ts": int(time.time()), + } + sign_text = "{token}params{params}ts{ts}user_id{user_id}".format( + token=self._token, params=data["params"], ts=data["ts"], user_id=data["user_id"] + ) + + md5 = hashlib.md5(sign_text.encode("u8")) + md5 = md5.hexdigest() + data["sign"] = md5 + + return data + + def _get_afdian_status(self, trade_no: str) -> dict: + req_data = self._generate_signature() + data = requests.post(self._url, json=req_data).json() + # latest 50 + for order in data["data"]["list"]: + if order["out_trade_no"] == trade_no: + return order + + return {} + + def get_user_payment(self, trade_no: str) -> (int, float, str): + order = self._get_afdian_status(trade_no) + amount = float(order.get("show_amount", 0)) + # convert to USD + return amount / 7, trade_no + + +class TronTrx: + def __init__(self): + if TRON_MNEMONIC == "cram floor today legend service drill pitch leaf car govern harvest soda": + logging.warning("Using nile testnet") + provider = HTTPProvider(endpoint_uri="https://nile.trongrid.io") + network = "nile" + else: + provider = HTTPProvider(api_key=TRONGRID_KEY) + network = "mainnet" + self.client = Tron(provider, network=network) + + def central_transfer(self, from_, index, amount: int): + logging.info("Generated key with index %s", index) + seed = seed_from_mnemonic(TRON_MNEMONIC, passphrase="") + key = PrivateKey(key_from_seed(seed, account_path=f"m/44'/195'/1'/0/{index}")) + central = self.central_wallet() + logging.info("Transfer %s TRX from %s to %s", amount, from_, central) + try: + self.client.trx.transfer(from_, central, amount).build().sign(key).broadcast() + except (TransactionError, ValidationError): + logging.error("Failed to transfer %s TRX to %s. Lower and try again.", amount, from_) + if amount > 1_100_000: + # 1.1 trx transfer fee + self.client.trx.transfer(from_, central, amount - 1_100_000).build().sign(key).broadcast() + + def central_wallet(self): + wallet = self.client.generate_address_from_mnemonic(TRON_MNEMONIC, account_path="m/44'/195'/0'/0/0") + return wallet["base58check_address"] + + def get_payment_address(self, user_id): + # payment_id is like tron,0,TN8Mn9KKv3cSrKyrt6Xx5L18nmezbpiW31,index where 0 means unpaid + db = MySQL() + con = db.con + cur = db.cur + cur.execute("select user_id from payment where payment_id like 'tron,%'") + data = cur.fetchall() + index = len(data) + path = f"m/44'/195'/1'/0/{index}" + logging.info("Generating address for user %s with path %s", user_id, path) + addr = self.client.generate_address_from_mnemonic(TRON_MNEMONIC, account_path=path)["base58check_address"] + # add row in db, unpaid + cur.execute("insert into payment values (%s,%s,%s,%s,%s)", (user_id, 0, f"tron,0,{addr},{index}", 0, 0)) + con.commit() + return addr + + def check_payment(self): + db = MySQL() + con = db.con + cur = db.cur + + cur.execute("select user_id, payment_id from payment where payment_id like 'tron,0,T%'") + data = cur.fetchall() + for row in data: + logging.info("Checking user payment %s", row) + user_id = row[0] + addr, index = row[1].split(",")[2:] + try: + balance = self.client.get_account_balance(addr) + except: + balance = 0 + if balance: + logging.info("User %s has %s TRX", user_id, balance) + # paid, calc token count + token_count = int(balance / 10 * TOKEN_PRICE) + cur.execute( + "update payment set token=%s,payment_id=%s where user_id=%s and payment_id like %s", + (token_count, f"tron,1,{addr},{index}", user_id, f"tron,%{addr}%"), + ) + cur.execute("UPDATE settings SET mode='Local' WHERE user_id=%s", (user_id,)) + con.commit() + self.central_transfer(addr, index, int(balance * 1_000_000)) + logging.debug("Dispatch signal now....") + TRX_SIGNAL.send("cron", user_id=user_id, text=f"{balance} TRX received, {token_count} tokens added.") + + +class Payment(Redis, MySQL): + def check_old_user(self, user_id: int) -> tuple: + self.cur.execute("SELECT * FROM payment WHERE user_id=%s AND old_user=1", (user_id,)) + data = self.cur.fetchone() + return data + + def get_pay_token(self, user_id: int) -> int: + self.cur.execute("SELECT token, old_user FROM payment WHERE user_id=%s", (user_id,)) + data = self.cur.fetchall() or [(0, False)] + number = sum([i[0] for i in data if i[0]]) + if number == 0 and data[0][1] != 1: + # not old user, no token + logging.warning("User %s has no token, set download mode to Celery", user_id) + # change download mode to Celery + self.set_user_settings(user_id, "mode", "Celery") + return number + + def get_free_token(self, user_id: int) -> int: + if self.r.exists(user_id): + return int(self.r.get(user_id)) + else: + # set and return + self.r.set(user_id, FREE_DOWNLOAD, ex=EXPIRE) + return FREE_DOWNLOAD + + def get_token(self, user_id: int): + ttl = self.r.ttl(user_id) + return self.get_free_token(user_id), self.get_pay_token(user_id), current_time(time.time() + ttl) + + def use_free_token(self, user_id: int): + if self.r.exists(user_id): + self.r.decr(user_id, 1) + else: + # first time download + self.r.set(user_id, 5 - 1, ex=EXPIRE) + + def use_pay_token(self, user_id: int): + # a user may pay multiple times, so we'll need to filter the first payment with valid token + self.cur.execute("SELECT payment_id FROM payment WHERE user_id=%s AND token>0", (user_id,)) + data = self.cur.fetchone() + payment_id = data[0] + logging.info("User %s use pay token with payment_id %s", user_id, payment_id) + self.cur.execute("UPDATE payment SET token=token-1 WHERE payment_id=%s", (payment_id,)) + self.con.commit() + + def use_token(self, user_id: int): + free = self.get_free_token(user_id) + if free > 0: + self.use_free_token(user_id) + else: + self.use_pay_token(user_id) + + def add_pay_user(self, pay_data: list): + self.cur.execute("INSERT INTO payment VALUES (%s,%s,%s,%s,%s)", pay_data) + self.set_user_settings(pay_data[0], "mode", "Local") + self.con.commit() + + def verify_payment(self, user_id: int, unique: str) -> str: + pay = BuyMeACoffee() if "@" in unique else Afdian() + self.cur.execute("SELECT * FROM payment WHERE payment_id=%s ", (unique,)) + data = self.cur.fetchone() + if data: + # TODO what if a user pay twice with the same email address? + return ( + f"Failed. Payment has been verified by other users. Please contact @{OWNER} if you have any questions." + ) + + amount, pay_id = pay.get_user_payment(unique) + logging.info("User %s paid %s, identifier is %s", user_id, amount, unique) + # amount is already in USD + if amount == 0: + return "Payment not found. Please check your payment ID or email address" + self.add_pay_user([user_id, amount, pay_id, 0, amount * TOKEN_PRICE]) + return "Thanks! Your payment has been verified. /start to get your token details" + + +if __name__ == "__main__": + a = TronTrx() + # a.central_wallet() + a.check_payment() diff --git a/ytdlbot/main.session b/ytdlbot/main.session new file mode 100644 index 0000000000000000000000000000000000000000..0fd856c52989c0915c736c838d51067777354a03 Binary files /dev/null and b/ytdlbot/main.session differ diff --git a/ytdlbot/split-video.sh b/ytdlbot/split-video.sh new file mode 100644 index 0000000000000000000000000000000000000000..5265ea1954b154b084a98bed8c5e09e4c25ea5a8 --- /dev/null +++ b/ytdlbot/split-video.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Short script to split videos by filesize using ffmpeg by LukeLR + +if [ $# -ne 2 ]; then + echo 'Illegal number of parameters. Needs 2 parameters:' + echo 'Usage:' + echo './split-video.sh FILE SIZELIMIT "FFMPEG_ARGS' + echo + echo 'Parameters:' + echo ' - FILE: Name of the video file to split' + echo ' - SIZELIMIT: Maximum file size of each part (in bytes)' + echo ' - FFMPEG_ARGS: Additional arguments to pass to each ffmpeg-call' + echo ' (video format and quality options etc.)' + exit 1 +fi + +FILE="$1" +SIZELIMIT="$2" +FFMPEG_ARGS="$3" + +# Duration of the source video +DURATION=$(ffprobe -i "$FILE" -show_entries format=duration -v quiet -of default=noprint_wrappers=1:nokey=1|cut -d. -f1) + +# Duration that has been encoded so far +CUR_DURATION=0 + +# Filename of the source video (without extension) +BASENAME="${FILE%.*}" + +# Extension for the video parts +#EXTENSION="${FILE##*.}" +EXTENSION="mp4" + +# Number of the current video part +i=1 + +# Filename of the next video part +NEXTFILENAME="$BASENAME-$i.$EXTENSION" + +echo "Duration of source video: $DURATION" + +# Until the duration of all partial videos has reached the duration of the source video +while [[ $CUR_DURATION -lt $DURATION ]]; do + # Encode next part + echo ffmpeg -i "$FILE" -ss "$CUR_DURATION" -fs "$SIZELIMIT" $FFMPEG_ARGS "$NEXTFILENAME" + ffmpeg -ss "$CUR_DURATION" -i "$FILE" -fs "$SIZELIMIT" $FFMPEG_ARGS "$NEXTFILENAME" + + # Duration of the new part + NEW_DURATION=$(ffprobe -i "$NEXTFILENAME" -show_entries format=duration -v quiet -of default=noprint_wrappers=1:nokey=1|cut -d. -f1) + + # Total duration encoded so far + CUR_DURATION=$((CUR_DURATION + NEW_DURATION)) + + i=$((i + 1)) + + echo "Duration of $NEXTFILENAME: $NEW_DURATION" + echo "Part No. $i starts at $CUR_DURATION" + + NEXTFILENAME="$BASENAME-$i.$EXTENSION" +done \ No newline at end of file diff --git a/ytdlbot/tasks.py b/ytdlbot/tasks.py new file mode 100644 index 0000000000000000000000000000000000000000..be65f848cc1d8df61eae3f0af82ec80f7908671f --- /dev/null +++ b/ytdlbot/tasks.py @@ -0,0 +1,491 @@ +#!/usr/local/bin/python3 +# coding: utf-8 + +# ytdlbot - tasks.py +# 12/29/21 14:57 +# + +__author__ = "Benny " + +import asyncio +import logging +import os +import pathlib +import re +import shutil +import subprocess +import tempfile +import threading +import time +import traceback +import typing +from typing import Any +from urllib.parse import quote_plus + +import filetype +import psutil +import pyrogram.errors +import requests +from apscheduler.schedulers.background import BackgroundScheduler +from celery import Celery +from celery.worker.control import Panel +from pyrogram import Client, enums, idle, types + +from channel import Channel +from client_init import create_app +from config import ( + ARCHIVE_ID, + BROKER, + ENABLE_CELERY, + ENABLE_VIP, + OWNER, + RATE_LIMIT, + RCLONE_PATH, + TMPFILE_PATH, + WORKERS, +) +from constant import BotText +from database import Redis +from downloader import edit_text, tqdm_progress, upload_hook, ytdl_download +from limit import Payment +from utils import ( + apply_log_formatter, + auto_restart, + customize_logger, + get_metadata, + get_revision, + sizeof_fmt, +) + +customize_logger(["pyrogram.client", "pyrogram.session.session", "pyrogram.connection.connection"]) +apply_log_formatter() +bot_text = BotText() +logging.getLogger("apscheduler.executors.default").propagate = False + +app = Celery("tasks", broker=BROKER) +bot = create_app("tasks") +channel = Channel() + + +def retrieve_message(chat_id: int, message_id: int) -> types.Message | Any: + # this should only be called by celery tasks + try: + return bot.get_messages(chat_id, message_id) + except ConnectionError as e: + logging.critical("BOT IS NOT STARTED YET: %s", e) + bot.start() + return bot.get_messages(chat_id, message_id) + + +@app.task(rate_limit=f"{RATE_LIMIT}/m") +def ytdl_download_task(chat_id: int, message_id: int, url: str): + logging.info("YouTube celery tasks started for %s", url) + bot_msg = retrieve_message(chat_id, message_id) + ytdl_normal_download(bot, bot_msg, url) + logging.info("YouTube celery tasks ended.") + + +@app.task() +def audio_task(chat_id: int, message_id: int): + logging.info("Audio celery tasks started for %s-%s", chat_id, message_id) + bot_msg = retrieve_message(chat_id, message_id) + normal_audio(bot, bot_msg) + logging.info("Audio celery tasks ended.") + + +@app.task() +def direct_download_task(chat_id: int, message_id: int, url: str): + logging.info("Direct download celery tasks started for %s", url) + bot_msg = retrieve_message(chat_id, message_id) + direct_normal_download(bot, bot_msg, url) + logging.info("Direct download celery tasks ended.") + + +def get_unique_clink(original_url: str, user_id: int): + payment = Payment() + settings = payment.get_user_settings(user_id) + clink = channel.extract_canonical_link(original_url) + try: + # different user may have different resolution settings + unique = "{}?p={}{}".format(clink, *settings[1:]) + except IndexError: + unique = clink + return unique + + +def forward_video(client, bot_msg: types.Message | Any, url: str, cached_fid: str): + res_msg = upload_processor(client, bot_msg, url, cached_fid) + obj = res_msg.document or res_msg.video or res_msg.audio or res_msg.animation or res_msg.photo + + caption, _ = gen_cap(bot_msg, url, obj) + res_msg.edit_text(caption, reply_markup=gen_video_markup()) + bot_msg.edit_text(f"Download success!✅") + return True + + +def ytdl_download_entrance(client: Client, bot_msg: types.Message, url: str, mode=None): + # in Local node and forward mode, we pass client from main + # in celery mode, we need to use our own client called bot + payment = Payment() + redis = Redis() + chat_id = bot_msg.chat.id + unique = get_unique_clink(url, chat_id) + cached_fid = redis.get_send_cache(unique) + + try: + if cached_fid: + forward_video(client, bot_msg, url, cached_fid) + redis.update_metrics("cache_hit") + return + redis.update_metrics("cache_miss") + mode = mode or payment.get_user_settings(chat_id)[-1] + if ENABLE_CELERY and mode in [None, "Celery"]: + ytdl_download_task.delay(chat_id, bot_msg.id, url) + else: + ytdl_normal_download(client, bot_msg, url) + except Exception as e: + logging.error("Failed to download %s, error: %s", url, e) + bot_msg.edit_text(f"Download failed!❌\n\n`{traceback.format_exc()[0:4000]}`", disable_web_page_preview=True) + + +def direct_download_entrance(client: Client, bot_msg: typing.Union[types.Message, typing.Coroutine], url: str): + if ENABLE_CELERY: + direct_normal_download(client, bot_msg, url) + # direct_download_task.delay(bot_msg.chat.id, bot_msg.id, url) + else: + direct_normal_download(client, bot_msg, url) + + +def audio_entrance(client: Client, bot_msg: types.Message): + if ENABLE_CELERY: + audio_task.delay(bot_msg.chat.id, bot_msg.id) + else: + normal_audio(client, bot_msg) + + +def direct_normal_download(client: Client, bot_msg: typing.Union[types.Message, typing.Coroutine], url: str): + chat_id = bot_msg.chat.id + headers = { + "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.3987.149 Safari/537.36" + } + length = 0 + + req = None + try: + req = requests.get(url, headers=headers, stream=True) + length = int(req.headers.get("content-length")) + filename = re.findall("filename=(.+)", req.headers.get("content-disposition"))[0] + except TypeError: + filename = getattr(req, "url", "").rsplit("/")[-1] + except Exception as e: + bot_msg.edit_text(f"Download failed!❌\n\n```{e}```", disable_web_page_preview=True) + return + + if not filename: + filename = quote_plus(url) + + with tempfile.TemporaryDirectory(prefix="ytdl-", dir=TMPFILE_PATH) as f: + filepath = f"{f}/{filename}" + # consume the req.content + downloaded = 0 + for chunk in req.iter_content(1024 * 1024): + text = tqdm_progress("Downloading...", length, downloaded) + edit_text(bot_msg, text) + with open(filepath, "ab") as fp: + fp.write(chunk) + downloaded += len(chunk) + logging.info("Downloaded file %s", filename) + st_size = os.stat(filepath).st_size + + client.send_chat_action(chat_id, enums.ChatAction.UPLOAD_DOCUMENT) + client.send_document( + bot_msg.chat.id, + filepath, + caption=f"filesize: {sizeof_fmt(st_size)}", + progress=upload_hook, + progress_args=(bot_msg,), + ) + bot_msg.edit_text("Download success!✅") + + +def normal_audio(client: Client, bot_msg: typing.Union[types.Message, typing.Coroutine]): + chat_id = bot_msg.chat.id + # fn = getattr(bot_msg.video, "file_name", None) or getattr(bot_msg.document, "file_name", None) + status_msg: typing.Union[types.Message, typing.Coroutine] = bot_msg.reply_text( + "Converting to audio...please wait patiently", quote=True + ) + orig_url: str = re.findall(r"https?://.*", bot_msg.caption)[0] + with tempfile.TemporaryDirectory(prefix="ytdl-", dir=TMPFILE_PATH) as tmp: + client.send_chat_action(chat_id, enums.ChatAction.RECORD_AUDIO) + # just try to download the audio using yt-dlp + filepath = ytdl_download(orig_url, tmp, status_msg, hijack="bestaudio[ext=m4a]") + status_msg.edit_text("Sending audio now...") + client.send_chat_action(chat_id, enums.ChatAction.UPLOAD_AUDIO) + for f in filepath: + client.send_audio(chat_id, f) + status_msg.edit_text("✅ Conversion complete.") + Redis().update_metrics("audio_success") + + +def ytdl_normal_download(client: Client, bot_msg: types.Message | typing.Any, url: str): + """ + This function is called by celery task or directly by bot + :param client: bot client, either from main or bot(celery) + :param bot_msg: bot message + :param url: url to download + """ + chat_id = bot_msg.chat.id + temp_dir = tempfile.TemporaryDirectory(prefix="ytdl-", dir=TMPFILE_PATH) + + video_paths = ytdl_download(url, temp_dir.name, bot_msg) + logging.info("Download complete.") + client.send_chat_action(chat_id, enums.ChatAction.UPLOAD_DOCUMENT) + bot_msg.edit_text("Download complete. Sending now...") + try: + upload_processor(client, bot_msg, url, video_paths) + except pyrogram.errors.Flood as e: + logging.critical("FloodWait from Telegram: %s", e) + client.send_message( + chat_id, + f"I'm being rate limited by Telegram. Your video will come after {e} seconds. Please wait patiently.", + ) + client.send_message(OWNER, f"CRITICAL INFO: {e}") + time.sleep(e.value) + upload_processor(client, bot_msg, url, video_paths) + + bot_msg.edit_text("Download success!✅") + + # setup rclone environment var to back up the downloaded file + if RCLONE_PATH: + for item in os.listdir(temp_dir.name): + logging.info("Copying %s to %s", item, RCLONE_PATH) + shutil.copy(os.path.join(temp_dir.name, item), RCLONE_PATH) + temp_dir.cleanup() + + +def generate_input_media(file_paths: list, cap: str) -> list: + input_media = [] + for path in file_paths: + mime = filetype.guess_mime(path) + if "video" in mime: + input_media.append(pyrogram.types.InputMediaVideo(media=path)) + elif "image" in mime: + input_media.append(pyrogram.types.InputMediaPhoto(media=path)) + elif "audio" in mime: + input_media.append(pyrogram.types.InputMediaAudio(media=path)) + else: + input_media.append(pyrogram.types.InputMediaDocument(media=path)) + + input_media[0].caption = cap + return input_media + + +def upload_processor(client: Client, bot_msg: types.Message, url: str, vp_or_fid: str | list): + redis = Redis() + # raise pyrogram.errors.exceptions.FloodWait(13) + # if is str, it's a file id; else it's a list of paths + payment = Payment() + chat_id = bot_msg.chat.id + markup = gen_video_markup() + if isinstance(vp_or_fid, list) and len(vp_or_fid) > 1: + # just generate the first for simplicity, send as media group(2-20) + cap, meta = gen_cap(bot_msg, url, vp_or_fid[0]) + res_msg: list["types.Message"] | Any = client.send_media_group(chat_id, generate_input_media(vp_or_fid, cap)) + # TODO no cache for now + return res_msg[0] + elif isinstance(vp_or_fid, list) and len(vp_or_fid) == 1: + # normal download, just contains one file in video_paths + vp_or_fid = vp_or_fid[0] + cap, meta = gen_cap(bot_msg, url, vp_or_fid) + else: + # just a file id as string + cap, meta = gen_cap(bot_msg, url, vp_or_fid) + + settings = payment.get_user_settings(chat_id) + if ARCHIVE_ID and isinstance(vp_or_fid, pathlib.Path): + chat_id = ARCHIVE_ID + + if settings[2] == "document": + logging.info("Sending as document") + try: + # send as document could be sent as video even if it's a document + res_msg = client.send_document( + chat_id, + vp_or_fid, + caption=cap, + progress=upload_hook, + progress_args=(bot_msg,), + reply_markup=markup, + thumb=meta["thumb"], + force_document=True, + ) + except ValueError: + logging.error("Retry to send as video") + res_msg = client.send_video( + chat_id, + vp_or_fid, + supports_streaming=True, + caption=cap, + progress=upload_hook, + progress_args=(bot_msg,), + reply_markup=markup, + **meta, + ) + elif settings[2] == "audio": + logging.info("Sending as audio") + res_msg = client.send_audio( + chat_id, + vp_or_fid, + caption=cap, + progress=upload_hook, + progress_args=(bot_msg,), + ) + else: + # settings==video + logging.info("Sending as video") + try: + res_msg = client.send_video( + chat_id, + vp_or_fid, + supports_streaming=True, + caption=cap, + progress=upload_hook, + progress_args=(bot_msg,), + reply_markup=markup, + **meta, + ) + except Exception: + # try to send as annimation, photo + try: + logging.warning("Retry to send as animation") + res_msg = client.send_animation( + chat_id, + vp_or_fid, + caption=cap, + progress=upload_hook, + progress_args=(bot_msg,), + reply_markup=markup, + **meta, + ) + except Exception: + # this is likely a photo + logging.warning("Retry to send as photo") + res_msg = client.send_photo( + chat_id, + vp_or_fid, + caption=cap, + progress=upload_hook, + progress_args=(bot_msg,), + ) + + unique = get_unique_clink(url, bot_msg.chat.id) + obj = res_msg.document or res_msg.video or res_msg.audio or res_msg.animation or res_msg.photo + redis.add_send_cache(unique, getattr(obj, "file_id", None)) + redis.update_metrics("video_success") + if ARCHIVE_ID and isinstance(vp_or_fid, pathlib.Path): + client.forward_messages(bot_msg.chat.id, ARCHIVE_ID, res_msg.id) + return res_msg + + +def gen_cap(bm, url, video_path): + payment = Payment() + chat_id = bm.chat.id + user = bm.chat + try: + user_info = "@{}({})-{}".format(user.username or "N/A", user.first_name or "" + user.last_name or "", user.id) + except Exception: + user_info = "" + + if isinstance(video_path, pathlib.Path): + meta = get_metadata(video_path) + file_name = video_path.name + file_size = sizeof_fmt(os.stat(video_path).st_size) + else: + file_name = getattr(video_path, "file_name", "") + file_size = sizeof_fmt(getattr(video_path, "file_size", (2 << 2) + ((2 << 2) + 1) + (2 << 5))) + meta = dict( + width=getattr(video_path, "width", 0), + height=getattr(video_path, "height", 0), + duration=getattr(video_path, "duration", 0), + thumb=getattr(video_path, "thumb", None), + ) + free = payment.get_free_token(chat_id) + pay = payment.get_pay_token(chat_id) + if ENABLE_VIP: + remain = f"Download token count: free {free}, pay {pay}" + else: + remain = "" + + if worker_name := os.getenv("WORKER_NAME"): + worker = f"Downloaded by {worker_name}" + else: + worker = "" + cap = ( + f"{user_info}\n{file_name}\n\n{url}\n\nInfo: {meta['width']}x{meta['height']} {file_size}\t" + f"{meta['duration']}s\n{remain}\n{worker}\n{bot_text.custom_text}" + ) + return cap, meta + + +def gen_video_markup(): + markup = types.InlineKeyboardMarkup( + [ + [ # First row + types.InlineKeyboardButton( # Generates a callback query when pressed + "convert to audio", callback_data="convert" + ) + ] + ] + ) + return markup + + +@Panel.register +def ping_revision(*args): + return get_revision() + + +@Panel.register +def hot_patch(*args): + app_path = pathlib.Path().cwd().parent + logging.info("Hot patching on path %s...", app_path) + + pip_install = "pip install -r requirements.txt" + unset = "git config --unset http.https://github.com/.extraheader" + pull_unshallow = "git pull origin --unshallow" + pull = "git pull" + + subprocess.call(unset, shell=True, cwd=app_path) + if subprocess.call(pull_unshallow, shell=True, cwd=app_path) != 0: + logging.info("Already unshallow, pulling now...") + subprocess.call(pull, shell=True, cwd=app_path) + + logging.info("Code is updated, applying hot patch now...") + subprocess.call(pip_install, shell=True, cwd=app_path) + psutil.Process().kill() + + +def purge_tasks(): + count = app.control.purge() + return f"purged {count} tasks." + + +def run_celery(): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + worker_name = os.getenv("WORKER_NAME", "") + argv = ["-A", "tasks", "worker", "--loglevel=info", "--pool=threads", f"--concurrency={WORKERS}", "-n", worker_name] + app.worker_main(argv) + + +if __name__ == "__main__": + print("Bootstrapping Celery worker now.....") + time.sleep(5) + threading.Thread(target=run_celery, daemon=True).start() + + scheduler = BackgroundScheduler(timezone="Europe/London") + scheduler.add_job(auto_restart, "interval", seconds=900) + scheduler.start() + + idle() + bot.stop() diff --git a/ytdlbot/utils.py b/ytdlbot/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..349cbed69c8a4bae39326ca726061ffc69c89755 --- /dev/null +++ b/ytdlbot/utils.py @@ -0,0 +1,216 @@ +#!/usr/local/bin/python3 +# coding: utf-8 + +# ytdlbot - utils.py +# 9/1/21 22:50 +# + +__author__ = "Benny " + +import contextlib +import inspect as pyinspect +import logging +import os +import pathlib +import shutil +import subprocess +import tempfile +import time +import uuid + +import coloredlogs +import ffmpeg +import psutil + +from config import TMPFILE_PATH +from flower_tasks import app + +inspect = app.control.inspect() + + +def apply_log_formatter(): + coloredlogs.install( + level=logging.INFO, + fmt="[%(asctime)s %(filename)s:%(lineno)d %(levelname).1s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + + +def customize_logger(logger: list): + for log in logger: + logging.getLogger(log).setLevel(level=logging.INFO) + + +def sizeof_fmt(num: int, suffix="B"): + for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]: + if abs(num) < 1024.0: + return "%3.1f%s%s" % (num, unit, suffix) + num /= 1024.0 + return "%.1f%s%s" % (num, "Yi", suffix) + + +def is_youtube(url: str): + if url.startswith("https://www.youtube.com/") or url.startswith("https://youtu.be/"): + return True + + +def adjust_formats(user_id: int, url: str, formats: list, hijack=None): + from database import MySQL + + # high: best quality 1080P, 2K, 4K, 8K + # medium: 720P + # low: 480P + if hijack: + formats.insert(0, hijack) + return + + mapping = {"high": [], "medium": [720], "low": [480]} + settings = MySQL().get_user_settings(user_id) + if settings and is_youtube(url): + for m in mapping.get(settings[1], []): + formats.insert(0, f"bestvideo[ext=mp4][height={m}]+bestaudio[ext=m4a]") + formats.insert(1, f"bestvideo[vcodec^=avc][height={m}]+bestaudio[acodec^=mp4a]/best[vcodec^=avc]/best") + + if settings[2] == "audio": + formats.insert(0, "bestaudio[ext=m4a]") + + +def get_metadata(video_path): + width, height, duration = 1280, 720, 0 + try: + video_streams = ffmpeg.probe(video_path, select_streams="v") + for item in video_streams.get("streams", []): + height = item["height"] + width = item["width"] + duration = int(float(video_streams["format"]["duration"])) + except Exception as e: + logging.error(e) + try: + thumb = pathlib.Path(video_path).parent.joinpath(f"{uuid.uuid4().hex}-thunmnail.png").as_posix() + ffmpeg.input(video_path, ss=duration / 2).filter("scale", width, -1).output(thumb, vframes=1).run() + except ffmpeg._run.Error: + thumb = None + + return dict(height=height, width=width, duration=duration, thumb=thumb) + + +def current_time(ts=None): + return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ts)) + + +def get_revision(): + with contextlib.suppress(subprocess.SubprocessError): + return subprocess.check_output("git -C ../ rev-parse --short HEAD".split()).decode("u8").replace("\n", "") + return "unknown" + + +def get_func_queue(func) -> int: + try: + count = 0 + data = getattr(inspect, func)() or {} + for _, task in data.items(): + count += len(task) + return count + except Exception: + return 0 + + +def tail_log(f, lines=1, _buffer=4098): + """Tail a file and get X lines from the end""" + # placeholder for the lines found + lines_found = [] + + # block counter will be multiplied by buffer + # to get the block size from the end + block_counter = -1 + + # loop until we find X lines + while len(lines_found) < lines: + try: + f.seek(block_counter * _buffer, os.SEEK_END) + except IOError: # either file is too small, or too many lines requested + f.seek(0) + lines_found = f.readlines() + break + + lines_found = f.readlines() + + # we found enough lines, get out + # Removed this line because it was redundant the while will catch + # it, I left it for history + # if len(lines_found) > lines: + # break + + # decrement the block counter to get the + # next X bytes + block_counter -= 1 + + return lines_found[-lines:] + + +class Detector: + def __init__(self, logs: str): + self.logs = logs + + @staticmethod + def func_name(): + with contextlib.suppress(Exception): + return pyinspect.stack()[1][3] + return "N/A" + + def updates_too_long_detector(self): + # If you're seeing this, that means you have logged more than 10 device + # and the earliest account was kicked out. Restart the program could get you back in. + indicators = [ + "types.UpdatesTooLong", + "Got shutdown from remote", + "Code is updated", + "OSError: Connection lost", + "[Errno -3] Try again", + "MISCONF", + ] + for indicator in indicators: + if indicator in self.logs: + logging.critical("kick out crash: %s", self.func_name()) + return True + logging.debug("No crash detected.") + + def next_salt_detector(self): + text = "Next salt in" + if self.logs.count(text) >= 5: + logging.critical("Next salt crash: %s", self.func_name()) + return True + + def connection_reset_detector(self): + text = "Send exception: ConnectionResetError Connection lost" + if text in self.logs: + logging.critical("connection lost: %s ", self.func_name()) + return True + + +def auto_restart(): + log_path = "/var/log/ytdl.log" + if not os.path.exists(log_path): + return + with open(log_path) as f: + logs = "".join(tail_log(f, lines=100)) + + det = Detector(logs) + method_list = [getattr(det, func) for func in dir(det) if func.endswith("_detector")] + for method in method_list: + if method(): + logging.critical("%s bye bye world!☠️", method) + for item in pathlib.Path(TMPFILE_PATH or tempfile.gettempdir()).glob("ytdl-*"): + shutil.rmtree(item, ignore_errors=True) + time.sleep(5) + psutil.Process().kill() + + +def clean_tempfile(): + for item in pathlib.Path(TMPFILE_PATH or tempfile.gettempdir()).glob("ytdl-*"): + if time.time() - item.stat().st_ctime > 3600: + shutil.rmtree(item, ignore_errors=True) + + +if __name__ == "__main__": + auto_restart() diff --git a/ytdlbot/ytdl_bot.py b/ytdlbot/ytdl_bot.py new file mode 100644 index 0000000000000000000000000000000000000000..30e30c783d79ce4f58061f46eb949cdac367f321 --- /dev/null +++ b/ytdlbot/ytdl_bot.py @@ -0,0 +1,562 @@ +#!/usr/local/bin/python3 +# coding: utf-8 + +# ytdlbot - new.py +# 8/14/21 14:37 +# + +__author__ = "Benny " + +import contextlib +import logging +import os +import random +import re +import tempfile +import time +import traceback +from io import BytesIO +from typing import Any + +import pyrogram.errors +import qrcode +import yt_dlp +from apscheduler.schedulers.background import BackgroundScheduler +from pyrogram import Client, enums, filters, types +from pyrogram.errors.exceptions.bad_request_400 import UserNotParticipant +from pyrogram.raw import functions +from pyrogram.raw import types as raw_types +from tgbot_ping import get_runtime +from youtubesearchpython import VideosSearch + +from channel import Channel +from client_init import create_app +from config import ( + AUTHORIZED_USER, + ENABLE_CELERY, + ENABLE_FFMPEG, + ENABLE_VIP, + IS_BACKUP_BOT, + M3U8_SUPPORT, + OWNER, + PLAYLIST_SUPPORT, + PROVIDER_TOKEN, + REQUIRED_MEMBERSHIP, + TOKEN_PRICE, + TRX_SIGNAL, +) +from constant import BotText +from database import InfluxDB, MySQL, Redis +from limit import Payment, TronTrx +from tasks import app as celery_app +from tasks import ( + audio_entrance, + direct_download_entrance, + hot_patch, + purge_tasks, + ytdl_download_entrance, +) +from utils import auto_restart, clean_tempfile, customize_logger, get_revision + +logging.info("Authorized users are %s", AUTHORIZED_USER) +customize_logger(["pyrogram.client", "pyrogram.session.session", "pyrogram.connection.connection"]) +logging.getLogger("apscheduler.executors.default").propagate = False + +app = create_app("main") +channel = Channel() + + +def private_use(func): + def wrapper(client: Client, message: types.Message): + chat_id = getattr(message.from_user, "id", None) + + # message type check + if message.chat.type != enums.ChatType.PRIVATE and not message.text.lower().startswith("/ytdl"): + logging.debug("%s, it's annoying me...🙄️ ", message.text) + return + + # authorized users check + if AUTHORIZED_USER: + users = [int(i) for i in AUTHORIZED_USER.split(",")] + else: + users = [] + + if users and chat_id and chat_id not in users: + message.reply_text(BotText.private, quote=True) + return + + if REQUIRED_MEMBERSHIP: + try: + member: types.ChatMember | Any = app.get_chat_member(REQUIRED_MEMBERSHIP, chat_id) + if member.status not in [ + enums.ChatMemberStatus.ADMINISTRATOR, + enums.ChatMemberStatus.MEMBER, + enums.ChatMemberStatus.OWNER, + ]: + raise UserNotParticipant() + else: + logging.info("user %s check passed for group/channel %s.", chat_id, REQUIRED_MEMBERSHIP) + except UserNotParticipant: + logging.warning("user %s is not a member of group/channel %s", chat_id, REQUIRED_MEMBERSHIP) + message.reply_text(BotText.membership_require, quote=True) + return + + return func(client, message) + + return wrapper + + +@app.on_message(filters.command(["start"])) +def start_handler(client: Client, message: types.Message): + payment = Payment() + from_id = message.from_user.id + logging.info("%s welcome to youtube-dl bot!", message.from_user.id) + client.send_chat_action(from_id, enums.ChatAction.TYPING) + is_old_user = payment.check_old_user(from_id) + if is_old_user: + info = "" + if ENABLE_VIP: + free_token, pay_token, reset = payment.get_token(from_id) + info = f"Free token: {free_token}, Pay token: {pay_token}, Reset: {reset}" + else: + info = "" + text = f"{BotText.start}\n\n{info}\n{BotText.custom_text}" + client.send_message(message.chat.id, text, disable_web_page_preview=True) + + +@app.on_message(filters.command(["help"])) +def help_handler(client: Client, message: types.Message): + chat_id = message.chat.id + client.send_chat_action(chat_id, enums.ChatAction.TYPING) + client.send_message(chat_id, BotText.help, disable_web_page_preview=True) + + +@app.on_message(filters.command(["about"])) +def about_handler(client: Client, message: types.Message): + chat_id = message.chat.id + client.send_chat_action(chat_id, enums.ChatAction.TYPING) + client.send_message(chat_id, BotText.about) + + +@app.on_message(filters.command(["sub"])) +def subscribe_handler(client: Client, message: types.Message): + chat_id = message.chat.id + client.send_chat_action(chat_id, enums.ChatAction.TYPING) + if message.text == "/sub": + result = channel.get_user_subscription(chat_id) + else: + link = message.text.split()[1] + try: + result = channel.subscribe_channel(chat_id, link) + except (IndexError, ValueError): + result = f"Error: \n{traceback.format_exc()}" + client.send_message(chat_id, result or "You have no subscription.", disable_web_page_preview=True) + + +@app.on_message(filters.command(["unsub"])) +def unsubscribe_handler(client: Client, message: types.Message): + chat_id = message.chat.id + client.send_chat_action(chat_id, enums.ChatAction.TYPING) + text = message.text.split(" ") + if len(text) == 1: + client.send_message(chat_id, "/unsub channel_id", disable_web_page_preview=True) + return + + rows = channel.unsubscribe_channel(chat_id, text[1]) + if rows: + text = f"Unsubscribed from {text[1]}" + else: + text = "Unable to find the channel." + client.send_message(chat_id, text, disable_web_page_preview=True) + + +@app.on_message(filters.command(["patch"])) +def patch_handler(client: Client, message: types.Message): + username = message.from_user.username + chat_id = message.chat.id + if username == OWNER: + celery_app.control.broadcast("hot_patch") + client.send_chat_action(chat_id, enums.ChatAction.TYPING) + client.send_message(chat_id, "Oorah!") + hot_patch() + + +@app.on_message(filters.command(["uncache"])) +def uncache_handler(client: Client, message: types.Message): + username = message.from_user.username + link = message.text.split()[1] + if username == OWNER: + count = channel.del_cache(link) + message.reply_text(f"{count} cache(s) deleted.", quote=True) + + +@app.on_message(filters.command(["purge"])) +def purge_handler(client: Client, message: types.Message): + username = message.from_user.username + if username == OWNER: + message.reply_text(purge_tasks(), quote=True) + + +@app.on_message(filters.command(["ping"])) +def ping_handler(client: Client, message: types.Message): + redis = Redis() + chat_id = message.chat.id + client.send_chat_action(chat_id, enums.ChatAction.TYPING) + if os.uname().sysname == "Darwin" or ".heroku" in os.getenv("PYTHONHOME", ""): + bot_info = "ping unavailable." + else: + bot_info = get_runtime("ytdlbot_ytdl_1", "YouTube-dl") + if message.chat.username == OWNER: + stats = BotText.ping_worker()[:1000] + client.send_document(chat_id, redis.generate_file(), caption=f"{bot_info}\n\n{stats}") + else: + client.send_message(chat_id, f"{bot_info.split('CPU')[0]}") + + +@app.on_message(filters.command(["sub_count"])) +def sub_count_handler(client: Client, message: types.Message): + username = message.from_user.username + chat_id = message.chat.id + if username == OWNER: + with BytesIO() as f: + f.write(channel.sub_count().encode("u8")) + f.name = "subscription count.txt" + client.send_document(chat_id, f) + + +@app.on_message(filters.command(["direct"])) +def direct_handler(client: Client, message: types.Message): + redis = Redis() + chat_id = message.from_user.id + client.send_chat_action(chat_id, enums.ChatAction.TYPING) + url = re.sub(r"/direct\s*", "", message.text) + logging.info("direct start %s", url) + if not re.findall(r"^https?://", url.lower()): + redis.update_metrics("bad_request") + message.reply_text("Send me a DIRECT LINK.", quote=True) + return + + bot_msg = message.reply_text("Request received.", quote=True) + redis.update_metrics("direct_request") + direct_download_entrance(client, bot_msg, url) + + +@app.on_message(filters.command(["settings"])) +def settings_handler(client: Client, message: types.Message): + chat_id = message.chat.id + payment = Payment() + client.send_chat_action(chat_id, enums.ChatAction.TYPING) + data = MySQL().get_user_settings(chat_id) + set_mode = data[-1] + text = {"Local": "Celery", "Celery": "Local"}.get(set_mode, "Local") + mode_text = f"Download mode: **{set_mode}**" + if message.chat.username == OWNER or payment.get_pay_token(chat_id): + extra = [types.InlineKeyboardButton(f"Change download mode to {text}", callback_data=text)] + else: + extra = [] + + markup = types.InlineKeyboardMarkup( + [ + [ # First row + types.InlineKeyboardButton("send as document", callback_data="document"), + types.InlineKeyboardButton("send as video", callback_data="video"), + types.InlineKeyboardButton("send as audio", callback_data="audio"), + ], + [ # second row + types.InlineKeyboardButton("High Quality", callback_data="high"), + types.InlineKeyboardButton("Medium Quality", callback_data="medium"), + types.InlineKeyboardButton("Low Quality", callback_data="low"), + ], + extra, + ] + ) + + try: + client.send_message(chat_id, BotText.settings.format(data[1], data[2]) + mode_text, reply_markup=markup) + except: + client.send_message( + chat_id, BotText.settings.format(data[1] + ".", data[2] + ".") + mode_text, reply_markup=markup + ) + + +@app.on_message(filters.command(["buy"])) +def buy_handler(client: Client, message: types.Message): + # process as chat.id, not from_user.id + chat_id = message.chat.id + client.send_chat_action(chat_id, enums.ChatAction.TYPING) + # currency USD + token_count = message.text.replace("/buy", "").strip() + if token_count.isdigit(): + price = int(int(token_count) / TOKEN_PRICE * 100) + else: + price = 100 + + markup = types.InlineKeyboardMarkup( + [ + [ + types.InlineKeyboardButton("Bot Payments", callback_data=f"bot-payments-{price}"), + types.InlineKeyboardButton("TRON(TRX)", callback_data="tron-trx"), + ], + ] + ) + client.send_message(chat_id, BotText.buy, disable_web_page_preview=True, reply_markup=markup) + + +@app.on_callback_query(filters.regex(r"tron-trx")) +def tronpayment_btn_calback(client: Client, callback_query: types.CallbackQuery): + callback_query.answer("Generating QR code...") + chat_id = callback_query.message.chat.id + client.send_chat_action(chat_id, enums.ChatAction.TYPING) + + addr = TronTrx().get_payment_address(chat_id) + with BytesIO() as bio: + qr = qrcode.make(addr) + qr.save(bio) + client.send_photo(chat_id, bio, caption=f"Send any amount of TRX to `{addr}`") + + +@app.on_callback_query(filters.regex(r"bot-payments-.*")) +def bot_payment_btn_calback(client: Client, callback_query: types.CallbackQuery): + callback_query.answer("Generating invoice...") + chat_id = callback_query.message.chat.id + client.send_chat_action(chat_id, enums.ChatAction.TYPING) + + data = callback_query.data + price = int(data.split("-")[-1]) + payload = f"{chat_id}-buy" + invoice = generate_invoice(price, f"Buy {TOKEN_PRICE} download tokens", "Pay by card", payload) + app.invoke( + functions.messages.SendMedia( + peer=(raw_types.InputPeerUser(user_id=chat_id, access_hash=0)), + media=invoice, + random_id=app.rnd_id(), + message="Buy more download token", + ) + ) + + +@app.on_message(filters.command(["redeem"])) +def redeem_handler(client: Client, message: types.Message): + payment = Payment() + chat_id = message.chat.id + text = message.text.strip() + unique = text.replace("/redeem", "").strip() + msg = payment.verify_payment(chat_id, unique) + message.reply_text(msg, quote=True) + + +def generate_invoice(amount: int, title: str, description: str, payload: str): + invoice = raw_types.input_media_invoice.InputMediaInvoice( + invoice=raw_types.invoice.Invoice( + currency="USD", prices=[raw_types.LabeledPrice(label="price", amount=amount)] + ), + title=title, + description=description, + provider=PROVIDER_TOKEN, + provider_data=raw_types.DataJSON(data="{}"), + payload=payload.encode(), + start_param=payload, + ) + return invoice + + +def link_checker(url: str) -> str: + if url.startswith("https://www.instagram.com"): + return "" + ytdl = yt_dlp.YoutubeDL() + + if not PLAYLIST_SUPPORT and ( + re.findall(r"^https://www\.youtube\.com/channel/", Channel.extract_canonical_link(url)) or "list" in url + ): + return "Playlist or channel links are disabled." + + if not M3U8_SUPPORT and (re.findall(r"m3u8|\.m3u8|\.m3u$", url.lower())): + return "m3u8 links are disabled." + + with contextlib.suppress(yt_dlp.utils.DownloadError): + if ytdl.extract_info(url, download=False).get("live_status") == "is_live": + return "Live stream links are disabled. Please download it after the stream ends." + + +def search_ytb(kw: str): + videos_search = VideosSearch(kw, limit=10) + text = "" + results = videos_search.result()["result"] + for item in results: + title = item.get("title") + link = item.get("link") + index = results.index(item) + 1 + text += f"{index}. {title}\n{link}\n\n" + return text + + +@app.on_message(filters.incoming & (filters.text | filters.document)) +@private_use +def download_handler(client: Client, message: types.Message): + redis = Redis() + payment = Payment() + chat_id = message.from_user.id + client.send_chat_action(chat_id, enums.ChatAction.TYPING) + redis.user_count(chat_id) + if message.document: + with tempfile.NamedTemporaryFile(mode="r+") as tf: + logging.info("Downloading file to %s", tf.name) + message.download(tf.name) + contents = open(tf.name, "r").read() # don't know why + urls = contents.split() + else: + urls = [re.sub(r"/ytdl\s*", "", message.text)] + logging.info("start %s", urls) + + for url in urls: + # check url + if not re.findall(r"^https?://", url.lower()): + redis.update_metrics("bad_request") + text = search_ytb(url) + message.reply_text(text, quote=True, disable_web_page_preview=True) + return + + if text := link_checker(url): + message.reply_text(text, quote=True) + redis.update_metrics("reject_link_checker") + return + + # old user is not limited by token + if ENABLE_VIP and not payment.check_old_user(chat_id): + free, pay, reset = payment.get_token(chat_id) + if free + pay <= 0: + message.reply_text(f"You don't have enough token. Please wait until {reset} or /buy .", quote=True) + redis.update_metrics("reject_token") + return + else: + payment.use_token(chat_id) + + redis.update_metrics("video_request") + + text = BotText.get_receive_link_text() + try: + # raise pyrogram.errors.exceptions.FloodWait(10) + bot_msg: types.Message | Any = message.reply_text(text, quote=True) + except pyrogram.errors.Flood as e: + f = BytesIO() + f.write(str(e).encode()) + f.write(b"Your job will be done soon. Just wait! Don't rush.") + f.name = "Please don't flood me.txt" + bot_msg = message.reply_document( + f, caption=f"Flood wait! Please wait {e} seconds...." f"Your job will start automatically", quote=True + ) + f.close() + client.send_message(OWNER, f"Flood wait! 🙁 {e} seconds....") + time.sleep(e.value) + + client.send_chat_action(chat_id, enums.ChatAction.UPLOAD_VIDEO) + bot_msg.chat = message.chat + ytdl_download_entrance(client, bot_msg, url) + + +@app.on_callback_query(filters.regex(r"document|video|audio")) +def send_method_callback(client: Client, callback_query: types.CallbackQuery): + chat_id = callback_query.message.chat.id + data = callback_query.data + logging.info("Setting %s file type to %s", chat_id, data) + MySQL().set_user_settings(chat_id, "method", data) + callback_query.answer(f"Your send type was set to {callback_query.data}") + + +@app.on_callback_query(filters.regex(r"high|medium|low")) +def download_resolution_callback(client: Client, callback_query: types.CallbackQuery): + chat_id = callback_query.message.chat.id + data = callback_query.data + logging.info("Setting %s file type to %s", chat_id, data) + MySQL().set_user_settings(chat_id, "resolution", data) + callback_query.answer(f"Your default download quality was set to {callback_query.data}") + + +@app.on_callback_query(filters.regex(r"convert")) +def audio_callback(client: Client, callback_query: types.CallbackQuery): + redis = Redis() + if not ENABLE_FFMPEG: + callback_query.answer("Request rejected.") + callback_query.message.reply_text("Audio conversion is disabled now.") + return + + callback_query.answer(f"Converting to audio...please wait patiently") + redis.update_metrics("audio_request") + audio_entrance(client, callback_query.message) + + +@app.on_callback_query(filters.regex(r"Local|Celery")) +def owner_local_callback(client: Client, callback_query: types.CallbackQuery): + chat_id = callback_query.message.chat.id + MySQL().set_user_settings(chat_id, "mode", callback_query.data) + callback_query.answer(f"Download mode was changed to {callback_query.data}") + + +def periodic_sub_check(): + exceptions = pyrogram.errors.exceptions + for cid, uids in channel.group_subscriber().items(): + video_url = channel.has_newer_update(cid) + if video_url: + logging.info(f"periodic update:{video_url} - {uids}") + for uid in uids: + try: + app.send_message(uid, f"{video_url} is out. Watch it on YouTube") + except (exceptions.bad_request_400.PeerIdInvalid, exceptions.bad_request_400.UserIsBlocked) as e: + logging.warning("User is blocked or deleted. %s", e) + channel.deactivate_user_subscription(uid) + except Exception: + logging.error("Unknown error when sending message to user. %s", traceback.format_exc()) + finally: + time.sleep(random.random() * 3) + + +@app.on_raw_update() +def raw_update(client: Client, update, users, chats): + payment = Payment() + action = getattr(getattr(update, "message", None), "action", None) + if update.QUALNAME == "types.UpdateBotPrecheckoutQuery": + client.invoke( + functions.messages.SetBotPrecheckoutResults( + query_id=update.query_id, + success=True, + ) + ) + elif action and action.QUALNAME == "types.MessageActionPaymentSentMe": + logging.info("Payment received. %s", action) + uid = update.message.peer_id.user_id + amount = action.total_amount / 100 + payment.add_pay_user([uid, amount, action.charge.provider_charge_id, 0, amount * TOKEN_PRICE]) + client.send_message(uid, f"Thank you {uid}. Payment received: {amount} {action.currency}") + + +def trx_notify(_, **kwargs): + user_id = kwargs.get("user_id") + text = kwargs.get("text") + logging.info("Sending trx notification to %s", user_id) + app.send_message(user_id, text) + + +if __name__ == "__main__": + MySQL() + TRX_SIGNAL.connect(trx_notify) + scheduler = BackgroundScheduler(timezone="Europe/London", job_defaults={"max_instances": 6}) + scheduler.add_job(auto_restart, "interval", seconds=600) + scheduler.add_job(clean_tempfile, "interval", seconds=120) + if not IS_BACKUP_BOT: + scheduler.add_job(Redis().reset_today, "cron", hour=0, minute=0) + scheduler.add_job(InfluxDB().collect_data, "interval", seconds=120) + scheduler.add_job(TronTrx().check_payment, "interval", seconds=60, max_instances=1) + # default quota allocation of 10,000 units per day + scheduler.add_job(periodic_sub_check, "interval", seconds=3600) + scheduler.start() + banner = f""" +▌ ▌ ▀▛▘ ▌ ▛▀▖ ▜ ▌ +▝▞ ▞▀▖ ▌ ▌ ▌ ▌ ▌ ▛▀▖ ▞▀▖ ▌ ▌ ▞▀▖ ▌ ▌ ▛▀▖ ▐ ▞▀▖ ▝▀▖ ▞▀▌ + ▌ ▌ ▌ ▌ ▌ ▌ ▌ ▌ ▌ ▌ ▛▀ ▌ ▌ ▌ ▌ ▐▐▐ ▌ ▌ ▐ ▌ ▌ ▞▀▌ ▌ ▌ + ▘ ▝▀ ▝▀▘ ▘ ▝▀▘ ▀▀ ▝▀▘ ▀▀ ▝▀ ▘▘ ▘ ▘ ▘ ▝▀ ▝▀▘ ▝▀▘ + +By @BennyThink, VIP mode: {ENABLE_VIP}, Celery Mode: {ENABLE_CELERY} +Version: {get_revision()} + """ + print(banner) + app.run()