Upload 65 files

.dockerignore

@@ -0,0 +1,5 @@
+env
+venv
+db_data
+.ash_history
+.DS_Store

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/instagram.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,175 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+/.idea/modules.xml
+/.idea/inspectionProfiles/profiles_settings.xml
+/.idea/inspectionProfiles/Project_Default.xml
+/.idea/vcs.xml
+/.idea/ytdl-bot.iml
+/.idea/misc.xml
+/.idea/workspace.xml
+/.idea/jsonSchemas.xml
+/*.session
+/.idea/ytdlbot.iml
+/*.sqlite
+/.idea/dataSources.xml
+/.idea/sqldialects.xml
+/.idea/.gitignore
+/.idea/dataSources/bf75f0a6-c774-4ecf-9448-2086f57b70df.xml
+/.idea/dataSources.local.xml
+/.idea/dataSources/bf75f0a6-c774-4ecf-9448-2086f57b70df/entities/entities.dat
+/.idea/dataSources/bf75f0a6-c774-4ecf-9448-2086f57b70df/entities/entities.dat.len
+/.idea/dataSources/bf75f0a6-c774-4ecf-9448-2086f57b70df/entities/entities.dat.values
+/.idea/dataSources/bf75f0a6-c774-4ecf-9448-2086f57b70df/entities/entities.dat.values.at
+/.idea/dataSources/bf75f0a6-c774-4ecf-9448-2086f57b70df/entities/entities.dat.values.s
+/.idea/dataSources/bf75f0a6-c774-4ecf-9448-2086f57b70df/entities/entities.dat_i
+/.idea/dataSources/bf75f0a6-c774-4ecf-9448-2086f57b70df/entities/entities.dat_i.len
+/.idea/dataSources/bf75f0a6-c774-4ecf-9448-2086f57b70df/storage_v2/_src_/schema/main.uQUzAA.meta
+db_data/*
+env/*
+.ash_history
+.DS_Store
+ytdlbot/ytdl.session
+data/*
+upgrade_worker.sh
+ytdl.session
+reinforcement/*
+/ytdlbot/session/celery.session
+/.idea/prettier.xml
+/.idea/watcherTasks.xml
+/ytdlbot/session/ytdl.session-journal
+/ytdlbot/unknown_errors.txt
+/ytdlbot/ytdl.session-journal
+/ytdlbot/ytdl-main.session-journal
+/ytdlbot/ytdl-main.session
+/ytdlbot/ytdl-celery.session-journal
+/ytdlbot/ytdl-celery.session
+/ytdlbot/main.session
+/ytdlbot/tasks.session
+/ytdlbot/tasks.session-journal

Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.10
+
+WORKDIR /code
+
+COPY ./requirements.txt /code/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+COPY . .
+
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"]

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Makefile

@@ -0,0 +1,52 @@
+define NOLOGGING
+
+    logging:
+      driver: none
+endef
+export NOLOGGING
+
+default:
+	docker pull bennythink/ytdlbot
+
+bot:
+	make
+	docker-compose up -d
+	docker system prune -a --volumes -f
+
+worker:
+	make
+	docker-compose -f worker.yml up -d
+	docker system prune -a --volumes -f
+	sleep 5
+
+weak-worker:
+	make
+	docker-compose --compatibility -f worker.yml up -d
+	docker system prune -a --volumes -f
+	sleep 5
+
+upgrade-all-worker:
+	bash upgrade_worker.sh
+
+tag:
+	git tag -a v$(shell date "+%Y-%m-%d")_$(shell git rev-parse --short HEAD) -m v$(shell date "+%Y-%m-%d")
+	git push --tags
+
+nolog:
+	echo "$$NOLOGGING">> worker.yml
+
+flower:
+	echo  'import dbm;dbm.open("data/flower","n");exit()'| python3
+
+up:
+	docker build -t bennythink/ytdlbot:latest .
+	docker-compose -f docker-compose.yml -f worker.yml up -d
+
+ps:
+	docker-compose -f docker-compose.yml -f worker.yml ps
+
+down:
+	docker-compose -f docker-compose.yml -f worker.yml down
+
+logs:
+	docker-compose -f docker-compose.yml -f worker.yml logs -f worker ytdl

Procfile

@@ -0,0 +1 @@
+worker: python ytdlbot/ytdl_bot.py

app.json

@@ -0,0 +1,43 @@
+{
+  "name": "YouTube-Downloader",
+  "description": "A Telegrambot to download youtube video",
+  "repository": "https://github.com/tgbot-collection/ytdlbot",
+  "logo": "https://avatars.githubusercontent.com/u/73354211?s=200&v=4",
+  "keywords": [
+    "telegram",
+    "youtube-dl"
+  ],
+  "env": {
+    "TOKEN": {
+      "description": "Bot token",
+      "value": "token"
+    },
+    "APP_ID": {
+      "description": "APP ID",
+      "value": "12345"
+    },
+    "APP_HASH": {
+      "description": "APP HASH",
+      "value": "12345abc"
+    },
+    "OWNER": {
+      "description": "Your telegram username",
+      "value": "username",
+      "required": false
+    }
+  },
+  "formation": {
+    "worker": {
+      "quantity": 1,
+      "size": "eco"
+    }
+  },
+  "buildpacks": [
+    {
+      "url": "https://github.com/heroku/heroku-buildpack-python.git"
+    },
+    {
+      "url": "https://github.com/jonathanong/heroku-buildpack-ffmpeg-latest.git"
+    }
+  ]
+}

app.py

@@ -0,0 +1,26 @@
+import os
+import streamlit as st
+
+def run_python_file(file_path):
+    try:
+        st.text(f"Running {file_path}...")
+        os.system(f"python {file_path}")
+        st.success("Script executed successfully!")
+    except Exception as e:
+        st.error(f"Error: {e}")
+
+def main():
+    st.title("YTDLBot Runner")
+
+    # Specify the directory and file name
+    directory = "ytdlbot"
+    file_name = "ytdl_bot.py"
+    file_path = os.path.join(directory, file_name)
+
+    st.text(f"Selected file: {file_path}")
+
+    # Run the Python file automatically when the app starts
+    run_python_file(file_path)
+
+if __name__ == "__main__":
+    main()

conf/YouTube Download Celery.json

@@ -0,0 +1,794 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_CELERY",
+      "label": "celery",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "influxdb",
+      "pluginName": "InfluxDB"
+    }
+  ],
+  "__elements": [],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "8.3.1"
+    },
+    {
+      "type": "datasource",
+      "id": "influxdb",
+      "name": "InfluxDB",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "timeseries",
+      "name": "Time series",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "iteration": 1644554238421,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "influxdb",
+        "uid": "${DS_CELERY}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 5,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "targets": [
+        {
+          "alias": "Active",
+          "groupBy": [
+            {
+              "params": [
+                "$__interval"
+              ],
+              "type": "time"
+            },
+            {
+              "params": [
+                "null"
+              ],
+              "type": "fill"
+            }
+          ],
+          "measurement": "active",
+          "orderByTime": "ASC",
+          "policy": "default",
+          "query": "SELECT mean(\"active\") FROM \"active\" WHERE $timeFilter GROUP BY time($__interval) ",
+          "rawQuery": true,
+          "refId": "A",
+          "resultFormat": "time_series",
+          "select": [
+            [
+              {
+                "params": [
+                  "active"
+                ],
+                "type": "field"
+              },
+              {
+                "params": [],
+                "type": "mean"
+              }
+            ]
+          ],
+          "tags": []
+        },
+        {
+          "alias": "$tag_hostname",
+          "hide": false,
+          "query": "\nSELECT \nmean(\"active\") AS active\nFROM \"tasks\" WHERE (\"hostname\" =~ /^$hostname$/) AND $timeFilter GROUP BY time($__interval) ,* ORDER BY asc ",
+          "rawQuery": true,
+          "refId": "B",
+          "resultFormat": "time_series"
+        }
+      ],
+      "title": "Active Jobs",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "influxdb",
+        "uid": "${DS_CELERY}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 5,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 10,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "targets": [
+        {
+          "alias": "$col",
+          "datasource": {
+            "type": "influxdb",
+            "uid": "${DS_CELERY}"
+          },
+          "groupBy": [
+            {
+              "params": [
+                "$__interval"
+              ],
+              "type": "time"
+            },
+            {
+              "params": [
+                "null"
+              ],
+              "type": "fill"
+            }
+          ],
+          "measurement": "metrics",
+          "orderByTime": "ASC",
+          "policy": "default",
+          "query": "\nSELECT \nmean(\"today_audio_success\")/mean(\"today_audio_request\")*100 as audio_success,\nmean(\"today_video_success\")/mean(\"today_video_request\")*100 as video_success\n\nFROM \"metrics\" WHERE $timeFilter GROUP BY time($__interval), * ORDER BY asc ",
+          "rawQuery": true,
+          "refId": "A",
+          "resultFormat": "time_series",
+          "select": [
+            [
+              {
+                "params": [
+                  "today_audio_success"
+                ],
+                "type": "field"
+              },
+              {
+                "params": [],
+                "type": "mean"
+              }
+            ]
+          ],
+          "tags": []
+        }
+      ],
+      "title": "Video & Audio Success Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "influxdb",
+        "uid": "${DS_CELERY}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 5,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "targets": [
+        {
+          "alias": "$tag_hostname:$col",
+          "query": "SELECT mean(\"load1\") AS load1,mean(\"load5\") AS load5,mean(\"load15\") AS load15\nFROM \"tasks\" WHERE (\"hostname\" =~ /^$hostname$/) AND $timeFilter GROUP BY time($__interval) ,* ORDER BY asc \n\n",
+          "rawQuery": true,
+          "refId": "A",
+          "resultFormat": "time_series"
+        }
+      ],
+      "title": "Load Average",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "influxdb",
+        "uid": "${DS_CELERY}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 5,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 9,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "targets": [
+        {
+          "alias": "$tag_hostname:$col",
+          "datasource": {
+            "type": "influxdb",
+            "uid": "${DS_CELERY}"
+          },
+          "groupBy": [
+            {
+              "params": [
+                "$__interval"
+              ],
+              "type": "time"
+            },
+            {
+              "params": [
+                "null"
+              ],
+              "type": "fill"
+            }
+          ],
+          "measurement": "tasks",
+          "orderByTime": "ASC",
+          "policy": "default",
+          "query": "\nSELECT mean(\"task-succeeded\")/mean(\"task-received\")*100  AS success_rate,  mean(\"task-failed\")/mean(\"task-received\")*100 AS fail_rate\n\nFROM \"tasks\" WHERE (\"hostname\" =~ /^$hostname$/) AND $timeFilter GROUP BY time($__interval) ,* ORDER BY asc ",
+          "rawQuery": true,
+          "refId": "A",
+          "resultFormat": "time_series",
+          "select": [
+            [
+              {
+                "params": [
+                  "task-received"
+                ],
+                "type": "field"
+              }
+            ]
+          ],
+          "tags": [
+            {
+              "key": "hostname",
+              "operator": "=~",
+              "value": "/^$hostname$/"
+            }
+          ]
+        }
+      ],
+      "title": "Task Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "influxdb",
+        "uid": "${DS_CELERY}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 5,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "id": 13,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "targets": [
+        {
+          "alias": "$tag_hostname:$col",
+          "datasource": {
+            "type": "influxdb",
+            "uid": "${DS_CELERY}"
+          },
+          "groupBy": [
+            {
+              "params": [
+                "$__interval"
+              ],
+              "type": "time"
+            },
+            {
+              "params": [
+                "null"
+              ],
+              "type": "fill"
+            }
+          ],
+          "measurement": "tasks",
+          "orderByTime": "ASC",
+          "policy": "default",
+          "query": "\nSELECT mean(\"task-received\") AS received,  mean(\"task-started\") AS started,mean(\"task-succeeded\") AS succeeded,mean(\"task-failed\") AS failed\n\nFROM \"tasks\" WHERE (\"hostname\" =~ /^$hostname$/) AND $timeFilter GROUP BY time($__interval) ,* ORDER BY asc ",
+          "rawQuery": true,
+          "refId": "A",
+          "resultFormat": "time_series",
+          "select": [
+            [
+              {
+                "params": [
+                  "task-received"
+                ],
+                "type": "field"
+              }
+            ]
+          ],
+          "tags": [
+            {
+              "key": "hostname",
+              "operator": "=~",
+              "value": "/^$hostname$/"
+            }
+          ]
+        }
+      ],
+      "title": "Task Status",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "influxdb",
+        "uid": "${DS_CELERY}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 5,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "targets": [
+        {
+          "alias": "$col",
+          "datasource": {
+            "type": "influxdb",
+            "uid": "${DS_CELERY}"
+          },
+          "groupBy": [
+            {
+              "params": [
+                "$__interval"
+              ],
+              "type": "time"
+            },
+            {
+              "params": [
+                "null"
+              ],
+              "type": "fill"
+            }
+          ],
+          "measurement": "metrics",
+          "orderByTime": "ASC",
+          "policy": "default",
+          "query": "SELECT \nmean(\"today_audio_request\") as audio_request,\nmean(\"today_audio_success\") as audio_success,\n\nmean(\"today_bad_request\") as bad_request,\n\nmean(\"today_video_request\") as video_request,\nmean(\"today_video_success\") as video_success\nFROM \"metrics\" WHERE $timeFilter GROUP BY time($__interval), * ORDER BY asc ",
+          "rawQuery": true,
+          "refId": "A",
+          "resultFormat": "time_series",
+          "select": [
+            [
+              {
+                "params": [
+                  "today_audio_success"
+                ],
+                "type": "field"
+              },
+              {
+                "params": [],
+                "type": "mean"
+              }
+            ]
+          ],
+          "tags": []
+        }
+      ],
+      "title": "Video & Audio",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "",
+  "schemaVersion": 33,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {},
+        "datasource": {
+          "type": "influxdb",
+          "uid": "${DS_CELERY}"
+        },
+        "definition": "show tag values with  KEY=\"hostname\"",
+        "hide": 0,
+        "includeAll": true,
+        "label": "hostname",
+        "multi": true,
+        "name": "hostname",
+        "options": [],
+        "query": "show tag values with  KEY=\"hostname\"",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-15m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "YouTube Download Celery",
+  "uid": "9yXGmc1nk",
+  "version": 14,
+  "weekStart": ""
+}

conf/supervisor_main.conf

@@ -0,0 +1,34 @@
+[supervisord]
+nodaemon=true
+logfile=/dev/null
+logfile_maxbytes=0
+user=root
+
+
+[program:vnstat]
+command=vnstatd -n
+autorestart=true
+
+
+[program:ytdl]
+directory=/ytdlbot/ytdlbot/
+command=python ytdl_bot.py
+autorestart=true
+priority=900
+stopasgroup=true
+startsecs = 30
+startretries = 2
+
+redirect_stderr=true
+stdout_logfile_maxbytes = 50MB
+stdout_logfile_backups = 2
+stdout_logfile = /var/log/ytdl.log
+
+[program:log]
+command=tail -f /var/log/ytdl.log
+autorestart=true
+priority=999
+
+redirect_stderr=true
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0

conf/supervisor_worker.conf

@@ -0,0 +1,33 @@
+[supervisord]
+nodaemon=true
+logfile=/dev/null
+logfile_maxbytes=0
+user=root
+
+
+[program:vnstat]
+command=vnstatd -n
+autorestart=true
+
+[program:worker]
+directory=/ytdlbot/ytdlbot/
+command=python tasks.py
+autorestart=true
+priority=900
+stopasgroup=true
+startsecs = 5
+startretries = 5
+
+redirect_stderr=true
+stdout_logfile_maxbytes = 50MB
+stdout_logfile_backups = 2
+stdout_logfile = /var/log/ytdl.log
+
+[program:log]
+command=tail -f /var/log/ytdl.log
+autorestart=true
+priority=999
+
+redirect_stderr=true
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0

docker-compose.yml

@@ -0,0 +1,60 @@
+version: '3.1'
+
+services:
+  socat:
+    image: bennythink/socat
+    restart: always
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    entrypoint: [ "socat", "tcp-listen:2375,fork,reuseaddr","unix-connect:/var/run/docker.sock" ]
+
+  redis:
+    image: redis:7-alpine
+    restart: always
+    logging:
+      driver: none
+
+  mysql:
+    image: ubuntu/mysql:8.0-22.04_beta
+    restart: always
+    volumes:
+      - ./db_data:/var/lib/mysql
+    environment:
+      MYSQL_ROOT_PASSWORD: 'root'
+    command: --default-authentication-plugin=mysql_native_password
+    logging:
+      driver: none
+
+  ytdl:
+    image: bennythink/ytdlbot
+    env_file:
+      - env/ytdl.env
+    restart: always
+    depends_on:
+      - socat
+      - redis
+    volumes:
+      - ./data/vnstat/:/var/lib/vnstat/
+    labels:
+      - "com.centurylinklabs.watchtower.enable=true"
+
+  flower:
+    image: bennythink/ytdlbot
+    env_file:
+      - env/ytdl.env
+    restart: unless-stopped
+    command: [ "/usr/local/bin/celery",
+               "-A", "flower_tasks", "flower",
+               "--basic_auth=benny:123456",
+               "--address=0.0.0.0", "--persistent","--purge_offline_workers=3600" ]
+    volumes:
+      - ./data/flower:/ytdlbot/ytdlbot/flower
+    ports:
+      - "127.0.0.1:15555:5555"
+
+  instagram:
+    image: bennythink/ytdlbot
+    env_file:
+      - env/ytdl.env
+    restart: always
+    command: [ "/usr/local/bin/python",  "/ytdlbot/ytdlbot/instagram.py" ]

k8s.md

@@ -0,0 +1,200 @@
+## Kubernetes
+
+Kubernetes, also known as K8s, is an open-source system for automating deployment, scaling, and management of
+containerized applications
+
+# Complete deployment guide for k8s deloyment
+
+* contains every functionality
+* compatible with amd64, arm64 and armv7l
+
+## First. Get all file in k8s folder
+
+Download `k8s` file to a directory on your k8s server and go to this folder
+
+## 1. Create Redis deloyment
+
+```shell
+kubectl apply -f 01.redis.yml
+```
+
+This command will create ytdl namespace, redis pod and redis service
+
+## 2. Creat MariaDB deloyment
+
+```shell
+kubectl apply -f 02.mariadb.yml
+```
+
+This deloyment will claim 10GB storage from storageClassName: longhorn. Please replace longhorn with your
+storageClassName before apply.
+
+## 3. Set environment variables
+
+Create configMap for env
+
+### 3.1 Edit configmap.yml
+
+```shell
+vim 03.configmap.yml
+```
+
+you can configure all the following environment variables:
+
+* PYRO_WORKERS: number of workers for pyrogram, default is 100
+* WORKERS: workers count for celery
+* APP_ID: **REQUIRED**, get it from https://core.telegram.org/
+* APP_HASH: **REQUIRED**
+* TOKEN: **REQUIRED**
+* REDIS: **REQUIRED if you need VIP mode and cache** ⚠️ Don't publish your redis server on the internet. ⚠️
+
+* OWNER: owner username
+* QUOTA: quota in bytes
+* EX: quota expire time
+* MULTIPLY: vip quota comparing to normal quota
+* USD2CNY: exchange rate
+* VIP: VIP mode, default: disable
+* AFD_LINK
+* COFFEE_LINK
+* COFFEE_TOKEN
+* AFD_TOKEN
+* AFD_USER_ID
+
+* AUTHORIZED_USER: users that could use this bot, user_id, separated with `,`
+* REQUIRED_MEMBERSHIP: group or channel username, user must join this group to use the bot. Could be use with
+  above `AUTHORIZED_USER`
+
+* ENABLE_CELERY: Distribution mode, default: disable. You'll can setup workers in different locations.
+* ENABLE_FFMPEG: enable ffmpeg so Telegram can stream
+* MYSQL_HOST: you'll have to setup MySQL if you enable VIP mode
+* MYSQL_USER
+* MYSQL_PASS
+* GOOGLE_API_KEY: YouTube API key, required for YouTube video subscription.
+* AUDIO_FORMAT: audio format, default is m4a. You can set to any known and supported format for ffmpeg. For
+  example,`mp3`, `flac`, etc. ⚠️ m4a is the fastest. Other formats may affect performance.
+* ARCHIVE_ID: group or channel id/username. All downloads will send to this group first and then forward to end user.
+  **Inline button will be lost during the forwarding.**
+
+### 3.2 Apply configMap for environment variables
+
+```shell
+kubectl apply -f 03.configmap.yml
+```
+
+## 4. Run Master Celery
+
+```shell
+kubectl apply -f 04.ytdl-master.yml
+```
+
+This deloyment will create ytdl-pvc PersistentVolumeClaim on storageClassName: longhorn. This clain will contain vnstat,
+cookies folder and flower database. Please replace longhorn with your storageClassName before apply
+
+### 4.1 Setup instagram cookies
+
+Required if you want to support instagram.
+
+You can use this extension
+[Get cookies.txt](https://chrome.google.com/webstore/detail/get-cookiestxt/bgaddhkoddajcdgocldbbfleckgcbcid)
+to get instagram cookies
+
+Get pod running ytdl master:
+
+```shell
+kubectl get pods --namespace ytdl
+```
+
+Name should be ytdl-xxxxxxxx
+
+Access to pod
+
+```shell
+kubectl --namespace=ytdl exec --stdin --tty ytdl-xxx -- sh
+```
+
+(replace ytdl-xxx by your pod name)
+
+Go to ytdl-pvc mounted folder
+
+```shell
+cd /ytdlbot/ytdlbot/data/
+vim  instagram.com_cookies.txt
+# paste your cookies
+```
+
+## 5. Run Worker Celery
+
+```shell
+kubectl apply -f 05.ytdl-worker.yml
+```
+
+## 6. Run Flower image (OPTIONAL)
+
+### 6.1 Setup flower db
+
+Get pod running ytdl master:
+
+```shell
+kubectl get pods --namespace ytdl
+```
+
+Name should be ytdl-xxxxxxxx
+
+Access to pod
+
+```shell
+kubectl --namespace=ytdl exec --stdin --tty ytdl-xxx -- sh
+```
+
+(replace ytdl-xxx by your pod name)
+
+Go to ytdl-pvc mounted folder
+
+```shel
+cd /var/lib/vnstat/
+```
+
+Create flower database file
+
+```shell
+{} ~ python3
+Python 3.9.9 (main, Nov 21 2021, 03:22:47)
+[Clang 12.0.0 (clang-1200.0.32.29)] on darwin
+Type "help", "copyright", "credits" or "license" for more information.
+>>> import dbm;dbm.open("flower","n");exit()
+```
+
+### 6.2 Config Flower Ingress
+
+This step need config ingress from line 51 of file 06.flower.yml with your ingress service. Need for access from
+internet.
+YML file should be adjusted depending on your load balancing, ingress and network system
+
+For active SSL
+
+```yml
+cert-manager.io/cluster-issuer: letsencrypt-prod
+```
+
+Replace nginx by your ingress service
+
+```yml
+ingressClassName: nginx
+```
+
+Add your domain, example
+
+```yml
+tls:
+  - hosts:
+      - flower.benny.com
+    secretName: flower-tls
+  rules:
+    - host: flower.benny.com
+```
+
+### 6.3 Apply Flower deloyment
+
+```shell
+kubectl apply -f 06.flower.yml
+```

k8s/01.redis.yml

@@ -0,0 +1,53 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ytdl
+  
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  creationTimestamp: null
+  labels:
+    ytdl: redis
+  name: redis
+  namespace: ytdl
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      ytdl: redis
+  strategy: {}
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        ytdl: redis
+    spec:
+      containers:
+        - image: redis:7-alpine
+          name: redis
+          ports:
+            - containerPort: 6379
+          resources: {}
+      restartPolicy: Always
+status: {}
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  creationTimestamp: null
+  labels:
+    ytdl: redis
+  name: redis
+  namespace: ytdl
+spec:
+  ports:
+    - name: "6379"
+      port: 6379
+      targetPort: 6379
+  selector:
+    ytdl: redis
+status:
+  loadBalancer: {}

k8s/02.mariadb.yml

@@ -0,0 +1,80 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  creationTimestamp: null
+  labels:
+    ytdl: mariadb-pvc
+  name: mariadb-pvc
+  namespace: ytdl
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: longhorn
+  resources:
+    requests:
+      storage: 10Gi
+status: {}
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+  creationTimestamp: null
+  labels:
+    ytdl: mariadb
+  name: mariadb
+  namespace: ytdl
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      ytdl: mariadb
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        ytdl: mariadb
+    spec:
+      containers:
+        - env:
+            - name: MYSQL_ROOT_PASSWORD
+              value: ro0tP4sSworD
+            - name: MYSQL_DATABASE
+              value: ytdl
+          image: mariadb:latest
+          name: mariadb
+          ports:
+            - containerPort: 3306
+          resources: {}
+          volumeMounts:
+            - mountPath: /var/lib/mysql
+              name: "mariadb-persistent-storage"
+      restartPolicy: Always
+      volumes:
+        - name: mariadb-persistent-storage
+          persistentVolumeClaim:
+            claimName: mariadb-pvc
+status: {}
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  creationTimestamp: null
+  labels:
+    ytdl: mariadb
+  name: mariadb-svc
+  namespace: ytdl
+spec:
+  ports:
+    - name: "3306"
+      port: 3306
+      targetPort: 3306
+  selector:
+    ytdl: mariadb
+status:
+  loadBalancer: {}
+

k8s/03.configmap.yml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ytdlenv
+  namespace: ytdl
+  annotations:
+data:
+  APP_HASH: 
+  APP_ID: 
+  TOKEN:
+  ARCHIVE_ID: 
+  ENABLE_CELERY: 'True'
+  ENABLE_FFMPEG: 'True'
+  MYSQL_HOST: mariadb-svc
+  MYSQL_PASS: ro0tP4sSworD
+  MYSQL_USER: root
+  REDIS: redis

k8s/04.ytdl-master.yml

@@ -0,0 +1,65 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ytdl-pvc
+  namespace: ytdl
+  creationTimestamp: null
+  labels:
+    ytdl: ytdl-pvc
+spec:
+  accessModes:
+    - ReadWriteMany
+  storageClassName: longhorn
+  resources:
+    requests:
+      storage: 10Gi
+status: {}
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ytdl
+  namespace: ytdl
+  creationTimestamp: null
+  labels:
+    ytdl: ytdl
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      ytdl: ytdl
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        ytdl: ytdl
+    spec:
+      volumes:
+        - name: ytdl-pvc
+          persistentVolumeClaim:
+            claimName: ytdl-pvc
+      containers:
+        - name: ytdl
+          image: bennythink/ytdlbot
+          envFrom:
+            - configMapRef:
+                name: ytdlenv
+          resources: {}
+          volumeMounts:
+            - name: ytdl-pvc
+              mountPath: /var/lib/vnstat/
+              subPath: vnstat/
+            - name: ytdl-pvc
+              mountPath: /ytdlbot/ytdlbot/data/
+              subPath: data/
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+          imagePullPolicy: Always
+      restartPolicy: Always
+      terminationGracePeriodSeconds: 30
+      dnsPolicy: ClusterFirst
+      securityContext: {}
+      schedulerName: default-scheduler
+status: {}

k8s/05.ytdl-worker.yml

@@ -0,0 +1,47 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  creationTimestamp: null
+  labels:
+    ytdl: ytdl-worker
+  name: ytdl-worker
+  namespace: ytdl
+spec:
+  replicas: 4
+  selector:
+    matchLabels:
+      ytdl: ytdl-worker
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        ytdl: ytdl-worker
+    spec:
+      volumes:
+        - name: ytdl-pvc
+          persistentVolumeClaim:
+            claimName: ytdl-pvc
+      containers:
+        - name: ytdl-worker
+          image: bennythink/ytdlbot
+          args:
+            - /usr/local/bin/supervisord
+            - '-c'
+            - /ytdlbot/conf/supervisor_worker.conf
+          envFrom:
+            - configMapRef:
+                name: ytdlenv
+          resources: {}
+          volumeMounts:
+            - name: ytdl-pvc
+              mountPath: /ytdlbot/ytdlbot/data/
+              subPath: data/
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+          imagePullPolicy: Always
+      restartPolicy: Always
+      terminationGracePeriodSeconds: 30
+      dnsPolicy: ClusterFirst
+      securityContext: {}
+      schedulerName: default-scheduler
+status: {}

k8s/06.flower.yml

@@ -0,0 +1,101 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  creationTimestamp: null
+  labels:
+    ytdl: flower
+  name: flower
+  namespace: ytdl
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      ytdl: flower
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        ytdl: flower
+    spec:
+      containers:
+        - envFrom:
+            - configMapRef:
+                name: ytdlenv
+          args:
+           - /usr/local/bin/celery
+           - -A
+           - flower_tasks
+           - flower
+           - --basic_auth=bennythink:123456
+           - --address=0.0.0.0
+           - --persistent
+           - --purge_offline_workers=3600
+          image: bennythink/ytdlbot
+          name: flower
+          ports:
+            - containerPort: 5555
+          resources: {}
+          volumeMounts:
+            - name: ytdl-pvc
+              mountPath: /ytdlbot/ytdlbot/flower
+              subPath: vnstat/flower
+      restartPolicy: Always
+      volumes:
+        - name: ytdl-pvc
+          persistentVolumeClaim:
+            claimName: ytdl-pvc
+status: {}
+
+# THIS IS OPTION IF YOU WANT PUBLIC FLOWER PAGE TO INTERNET.
+# should be adjusted depending on your load balancing system machine
+---
+apiVersion: v1
+kind: Service
+metadata:
+  creationTimestamp: null
+  labels:
+    ytdl: flower
+  name: flower-svc
+  namespace: ytdl
+spec:
+  type: NodePort
+  ports:
+    - name: "5555"
+      protocol: TCP
+      port: 5555
+      targetPort: 5555
+  selector:
+    ytdl: flower
+status:
+  loadBalancer: {}
+
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: nginx-flower-ingress
+  namespace: ytdl
+  annotations:
+    # cert-manager.io/cluster-issuer: letsencrypt-prod
+    nginx.ingress.kubernetes.io/rewrite-target: /
+    # nginx.ingress.kubernetes.io/whitelist-source-range: 14.161.27.151 limit by ipaddresss
+ 
+spec:
+  ingressClassName: nginx
+  tls:
+    - hosts:
+        - your-domain
+      secretName: flower-tls
+  rules:
+    - host: your-domain
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: flower-svc
+                port:
+                  number: 5555

main.py

@@ -0,0 +1,21 @@
+from fastapi import FastAPI, HTTPException
+import subprocess
+
+def run_python_verbose():
+    try:
+        # Run the 'python -v' command in a subprocess
+        print("okk")
+        subprocess.run(['python', 'ytdlbot/ytdl_bot.py'], check=True)
+    except subprocess.CalledProcessError as e:
+        print(f"Error running 'python -v': {e}")
+
+# Call the function
+
+app = FastAPI()
+
+@app.get("/")
+async def root():
+    return {"message": "Hello World"}
+@app.get("/okk")
+async def okk():
+    run_python_verbose()

requirements.txt

@@ -0,0 +1,31 @@
+pyrogram
+tgcrypto
+yt-dlp
+APScheduler
+beautifultable
+ffmpeg-python
+PyMySQL
+celery
+filetype
+flower
+psutil
+influxdb
+beautifulsoup4
+fakeredis
+supervisor
+tgbot-ping
+redis
+requests
+tqdm
+requests-toolbelt
+ffpb
+youtube-search-python
+token-bucket
+coloredlogs
+tronpy
+mnemonic
+qrcode
+blinker
+flask
+fastapi
+uvicorn

scripts/low_id.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/usr/local/go/bin:/opt/bin
+
+# Check the logs for the given string
+if docker-compose logs --tail=100 ytdl | grep -q "The msg_id is too low"; then
+    # If the string is found, stop the ytdl service
+    echo "ytdl service stopped due to 'The msg_id is too low' found in logs."
+    docker-compose stop ytdl && docker-compose rm ytdl && docker-compose up -d
+
+else
+    echo "String not found in logs."
+fi

scripts/migrate_to_mysql.py

@@ -0,0 +1,27 @@
+#!/usr/local/bin/python3
+# coding: utf-8
+
+# ytdlbot - migrate_to_mysql.py
+# 12/29/21 15:28
+#
+
+__author__ = "Benny <[email protected]>"
+
+import sqlite3
+
+import pymysql
+
+mysql_con = pymysql.connect(host='localhost', user='root', passwd='root', db='vip', charset='utf8mb4')
+sqlite_con = sqlite3.connect('vip.sqlite')
+
+vips = sqlite_con.execute('SELECT * FROM VIP').fetchall()
+
+for vip in vips:
+    mysql_con.cursor().execute('INSERT INTO vip  VALUES (%s, %s, %s, %s, %s, %s)', vip)
+
+settings = sqlite_con.execute('SELECT * FROM settings').fetchall()
+
+for setting in settings:
+    mysql_con.cursor().execute("INSERT INTO settings VALUES (%s,%s,%s)", setting)
+
+mysql_con.commit()

scripts/start.sh

@@ -0,0 +1,13 @@
+docker run -d --restart unless-stopped --name ytdl \
+   --net host \
+     -e TOKEN=12345 \
+     -e APP_ID=123123 \
+     -e APP_HASH=4990 \
+     -e ENABLE_CELERY=True \
+     -e REDIS=192.168.6.1 \
+     -e MYSQL_HOST=192.168.6.1 \
+     -e WORKERS=4 \
+     -e VIP=True \
+     -e CUSTOM_TEXT=#StandWithUkraine \
+     bennythink/ytdlbot \
+     /usr/local/bin/supervisord -c "/ytdlbot/conf/supervisor_worker.conf"

scripts/transfer.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+# coding: utf-8
+
+# ytdlbot - transfer.py
+# 2023-12-07  18:21
+from tronpy import Tron
+from tronpy.hdwallet import seed_from_mnemonic, key_from_seed
+from tronpy.keys import PrivateKey
+
+mnemonic = "web horse smile ramp olive slush blue property world physical donkey pumpkin"
+
+client = Tron(network="nile")
+
+from_ = client.generate_address_from_mnemonic(mnemonic, account_path="m/44'/195'/0'/0/0")["base58check_address"]
+balance = client.get_account_balance(from_)
+print("my addr: ", from_, "balance: ", balance)
+to = input("to: ")
+amount = int(input("amount in TRX: "))
+
+
+def mnemonic_to_private_key():
+    seed = seed_from_mnemonic(mnemonic, passphrase="")
+    private_key = key_from_seed(seed, account_path="m/44'/195'/0'/0/0")
+    return PrivateKey(private_key)
+
+
+t = client.trx.transfer(from_, to, amount * 1_000_000).build().sign(mnemonic_to_private_key()).broadcast()
+
+print(t.wait())

worker.yml

@@ -0,0 +1,15 @@
+version: '3.1'
+
+services:
+  worker:
+    image: bennythink/ytdlbot
+    env_file:
+      - env/ytdl.env
+    restart: always
+    command: [ "/usr/local/bin/supervisord", "-c" ,"/ytdlbot/conf/supervisor_worker.conf" ]
+#    network_mode: "host"
+#    deploy:
+#      resources:
+#        limits:
+#          cpus: '0.3'
+#          memory: 1500M