Upload 42 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/instagram.png filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -0,0 +1,20 @@
+FROM python:3.10
+
+# Set up a new user named "user" with user ID 1000
+RUN useradd -m -u 1000 user
+
+# Switch to the "user" user
+USER user
+WORKDIR /code
+
+COPY --chown=user ./requirements.txt /code/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+COPY --chown=user . .
+
+# Expose the port on which Streamlit will run
+EXPOSE 7860
+
+# Set the entry point for the Streamlit app
+CMD ["streamlit", "run", "app.py", "--server.port", "7860"]

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Makefile

@@ -0,0 +1,52 @@
+define NOLOGGING
+
+    logging:
+      driver: none
+endef
+export NOLOGGING
+
+default:
+	docker pull bennythink/ytdlbot
+
+bot:
+	make
+	docker-compose up -d
+	docker system prune -a --volumes -f
+
+worker:
+	make
+	docker-compose -f worker.yml up -d
+	docker system prune -a --volumes -f
+	sleep 5
+
+weak-worker:
+	make
+	docker-compose --compatibility -f worker.yml up -d
+	docker system prune -a --volumes -f
+	sleep 5
+
+upgrade-all-worker:
+	bash upgrade_worker.sh
+
+tag:
+	git tag -a v$(shell date "+%Y-%m-%d")_$(shell git rev-parse --short HEAD) -m v$(shell date "+%Y-%m-%d")
+	git push --tags
+
+nolog:
+	echo "$$NOLOGGING">> worker.yml
+
+flower:
+	echo  'import dbm;dbm.open("data/flower","n");exit()'| python3
+
+up:
+	docker build -t bennythink/ytdlbot:latest .
+	docker-compose -f docker-compose.yml -f worker.yml up -d
+
+ps:
+	docker-compose -f docker-compose.yml -f worker.yml ps
+
+down:
+	docker-compose -f docker-compose.yml -f worker.yml down
+
+logs:
+	docker-compose -f docker-compose.yml -f worker.yml logs -f worker ytdl

Procfile

@@ -0,0 +1 @@
+worker: python ytdlbot/ytdl_bot.py

app.json

@@ -0,0 +1,43 @@
+{
+  "name": "YouTube-Downloader",
+  "description": "A Telegrambot to download youtube video",
+  "repository": "https://github.com/tgbot-collection/ytdlbot",
+  "logo": "https://avatars.githubusercontent.com/u/73354211?s=200&v=4",
+  "keywords": [
+    "telegram",
+    "youtube-dl"
+  ],
+  "env": {
+    "TOKEN": {
+      "description": "Bot token",
+      "value": "token"
+    },
+    "APP_ID": {
+      "description": "APP ID",
+      "value": "12345"
+    },
+    "APP_HASH": {
+      "description": "APP HASH",
+      "value": "12345abc"
+    },
+    "OWNER": {
+      "description": "Your telegram username",
+      "value": "username",
+      "required": false
+    }
+  },
+  "formation": {
+    "worker": {
+      "quantity": 1,
+      "size": "eco"
+    }
+  },
+  "buildpacks": [
+    {
+      "url": "https://github.com/heroku/heroku-buildpack-python.git"
+    },
+    {
+      "url": "https://github.com/jonathanong/heroku-buildpack-ffmpeg-latest.git"
+    }
+  ]
+}

app.py

@@ -0,0 +1,26 @@
+import os
+import streamlit as st
+
+def run_python_file(file_path):
+    try:
+        st.text(f"Running {file_path}...")
+        os.system(f"python {file_path}")
+        st.success("Script executed successfully!")
+    except Exception as e:
+        st.error(f"Error: {e}")
+
+def main():
+    st.title("YTDLBot Runner")
+
+    # Specify the directory and file name
+    directory = "ytdlbot"
+    file_name = "ytdl_bot.py"
+    file_path = os.path.join(directory, file_name)
+
+    st.text(f"Selected file: {file_path}")
+
+    # Run the Python file automatically when the app starts
+    run_python_file(file_path)
+
+if __name__ == "__main__":
+    main()

conf/YouTube Download Celery.json

@@ -0,0 +1,794 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_CELERY",
+      "label": "celery",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "influxdb",
+      "pluginName": "InfluxDB"
+    }
+  ],
+  "__elements": [],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "8.3.1"
+    },
+    {
+      "type": "datasource",
+      "id": "influxdb",
+      "name": "InfluxDB",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "timeseries",
+      "name": "Time series",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "iteration": 1644554238421,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "influxdb",
+        "uid": "${DS_CELERY}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 5,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "targets": [
+        {
+          "alias": "Active",
+          "groupBy": [
+            {
+              "params": [
+                "$__interval"
+              ],
+              "type": "time"
+            },
+            {
+              "params": [
+                "null"
+              ],
+              "type": "fill"
+            }
+          ],
+          "measurement": "active",
+          "orderByTime": "ASC",
+          "policy": "default",
+          "query": "SELECT mean(\"active\") FROM \"active\" WHERE $timeFilter GROUP BY time($__interval) ",
+          "rawQuery": true,
+          "refId": "A",
+          "resultFormat": "time_series",
+          "select": [
+            [
+              {
+                "params": [
+                  "active"
+                ],
+                "type": "field"
+              },
+              {
+                "params": [],
+                "type": "mean"
+              }
+            ]
+          ],
+          "tags": []
+        },
+        {
+          "alias": "$tag_hostname",
+          "hide": false,
+          "query": "\nSELECT \nmean(\"active\") AS active\nFROM \"tasks\" WHERE (\"hostname\" =~ /^$hostname$/) AND $timeFilter GROUP BY time($__interval) ,* ORDER BY asc ",
+          "rawQuery": true,
+          "refId": "B",
+          "resultFormat": "time_series"
+        }
+      ],
+      "title": "Active Jobs",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "influxdb",
+        "uid": "${DS_CELERY}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 5,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 10,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "targets": [
+        {
+          "alias": "$col",
+          "datasource": {
+            "type": "influxdb",
+            "uid": "${DS_CELERY}"
+          },
+          "groupBy": [
+            {
+              "params": [
+                "$__interval"
+              ],
+              "type": "time"
+            },
+            {
+              "params": [
+                "null"
+              ],
+              "type": "fill"
+            }
+          ],
+          "measurement": "metrics",
+          "orderByTime": "ASC",
+          "policy": "default",
+          "query": "\nSELECT \nmean(\"today_audio_success\")/mean(\"today_audio_request\")*100 as audio_success,\nmean(\"today_video_success\")/mean(\"today_video_request\")*100 as video_success\n\nFROM \"metrics\" WHERE $timeFilter GROUP BY time($__interval), * ORDER BY asc ",
+          "rawQuery": true,
+          "refId": "A",
+          "resultFormat": "time_series",
+          "select": [
+            [
+              {
+                "params": [
+                  "today_audio_success"
+                ],
+                "type": "field"
+              },
+              {
+                "params": [],
+                "type": "mean"
+              }
+            ]
+          ],
+          "tags": []
+        }
+      ],
+      "title": "Video & Audio Success Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "influxdb",
+        "uid": "${DS_CELERY}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 5,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "targets": [
+        {
+          "alias": "$tag_hostname:$col",
+          "query": "SELECT mean(\"load1\") AS load1,mean(\"load5\") AS load5,mean(\"load15\") AS load15\nFROM \"tasks\" WHERE (\"hostname\" =~ /^$hostname$/) AND $timeFilter GROUP BY time($__interval) ,* ORDER BY asc \n\n",
+          "rawQuery": true,
+          "refId": "A",
+          "resultFormat": "time_series"
+        }
+      ],
+      "title": "Load Average",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "influxdb",
+        "uid": "${DS_CELERY}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 5,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 9,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "targets": [
+        {
+          "alias": "$tag_hostname:$col",
+          "datasource": {
+            "type": "influxdb",
+            "uid": "${DS_CELERY}"
+          },
+          "groupBy": [
+            {
+              "params": [
+                "$__interval"
+              ],
+              "type": "time"
+            },
+            {
+              "params": [
+                "null"
+              ],
+              "type": "fill"
+            }
+          ],
+          "measurement": "tasks",
+          "orderByTime": "ASC",
+          "policy": "default",
+          "query": "\nSELECT mean(\"task-succeeded\")/mean(\"task-received\")*100  AS success_rate,  mean(\"task-failed\")/mean(\"task-received\")*100 AS fail_rate\n\nFROM \"tasks\" WHERE (\"hostname\" =~ /^$hostname$/) AND $timeFilter GROUP BY time($__interval) ,* ORDER BY asc ",
+          "rawQuery": true,
+          "refId": "A",
+          "resultFormat": "time_series",
+          "select": [
+            [
+              {
+                "params": [
+                  "task-received"
+                ],
+                "type": "field"
+              }
+            ]
+          ],
+          "tags": [
+            {
+              "key": "hostname",
+              "operator": "=~",
+              "value": "/^$hostname$/"
+            }
+          ]
+        }
+      ],
+      "title": "Task Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "influxdb",
+        "uid": "${DS_CELERY}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 5,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "id": 13,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "targets": [
+        {
+          "alias": "$tag_hostname:$col",
+          "datasource": {
+            "type": "influxdb",
+            "uid": "${DS_CELERY}"
+          },
+          "groupBy": [
+            {
+              "params": [
+                "$__interval"
+              ],
+              "type": "time"
+            },
+            {
+              "params": [
+                "null"
+              ],
+              "type": "fill"
+            }
+          ],
+          "measurement": "tasks",
+          "orderByTime": "ASC",
+          "policy": "default",
+          "query": "\nSELECT mean(\"task-received\") AS received,  mean(\"task-started\") AS started,mean(\"task-succeeded\") AS succeeded,mean(\"task-failed\") AS failed\n\nFROM \"tasks\" WHERE (\"hostname\" =~ /^$hostname$/) AND $timeFilter GROUP BY time($__interval) ,* ORDER BY asc ",
+          "rawQuery": true,
+          "refId": "A",
+          "resultFormat": "time_series",
+          "select": [
+            [
+              {
+                "params": [
+                  "task-received"
+                ],
+                "type": "field"
+              }
+            ]
+          ],
+          "tags": [
+            {
+              "key": "hostname",
+              "operator": "=~",
+              "value": "/^$hostname$/"
+            }
+          ]
+        }
+      ],
+      "title": "Task Status",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "influxdb",
+        "uid": "${DS_CELERY}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 5,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "targets": [
+        {
+          "alias": "$col",
+          "datasource": {
+            "type": "influxdb",
+            "uid": "${DS_CELERY}"
+          },
+          "groupBy": [
+            {
+              "params": [
+                "$__interval"
+              ],
+              "type": "time"
+            },
+            {
+              "params": [
+                "null"
+              ],
+              "type": "fill"
+            }
+          ],
+          "measurement": "metrics",
+          "orderByTime": "ASC",
+          "policy": "default",
+          "query": "SELECT \nmean(\"today_audio_request\") as audio_request,\nmean(\"today_audio_success\") as audio_success,\n\nmean(\"today_bad_request\") as bad_request,\n\nmean(\"today_video_request\") as video_request,\nmean(\"today_video_success\") as video_success\nFROM \"metrics\" WHERE $timeFilter GROUP BY time($__interval), * ORDER BY asc ",
+          "rawQuery": true,
+          "refId": "A",
+          "resultFormat": "time_series",
+          "select": [
+            [
+              {
+                "params": [
+                  "today_audio_success"
+                ],
+                "type": "field"
+              },
+              {
+                "params": [],
+                "type": "mean"
+              }
+            ]
+          ],
+          "tags": []
+        }
+      ],
+      "title": "Video & Audio",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "",
+  "schemaVersion": 33,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {},
+        "datasource": {
+          "type": "influxdb",
+          "uid": "${DS_CELERY}"
+        },
+        "definition": "show tag values with  KEY=\"hostname\"",
+        "hide": 0,
+        "includeAll": true,
+        "label": "hostname",
+        "multi": true,
+        "name": "hostname",
+        "options": [],
+        "query": "show tag values with  KEY=\"hostname\"",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-15m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "YouTube Download Celery",
+  "uid": "9yXGmc1nk",
+  "version": 14,
+  "weekStart": ""
+}

conf/supervisor_main.conf

@@ -0,0 +1,34 @@
+[supervisord]
+nodaemon=true
+logfile=/dev/null
+logfile_maxbytes=0
+user=root
+
+
+[program:vnstat]
+command=vnstatd -n
+autorestart=true
+
+
+[program:ytdl]
+directory=/ytdlbot/ytdlbot/
+command=python ytdl_bot.py
+autorestart=true
+priority=900
+stopasgroup=true
+startsecs = 30
+startretries = 2
+
+redirect_stderr=true
+stdout_logfile_maxbytes = 50MB
+stdout_logfile_backups = 2
+stdout_logfile = /var/log/ytdl.log
+
+[program:log]
+command=tail -f /var/log/ytdl.log
+autorestart=true
+priority=999
+
+redirect_stderr=true
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0

conf/supervisor_worker.conf

@@ -0,0 +1,33 @@
+[supervisord]
+nodaemon=true
+logfile=/dev/null
+logfile_maxbytes=0
+user=root
+
+
+[program:vnstat]
+command=vnstatd -n
+autorestart=true
+
+[program:worker]
+directory=/ytdlbot/ytdlbot/
+command=python tasks.py
+autorestart=true
+priority=900
+stopasgroup=true
+startsecs = 5
+startretries = 5
+
+redirect_stderr=true
+stdout_logfile_maxbytes = 50MB
+stdout_logfile_backups = 2
+stdout_logfile = /var/log/ytdl.log
+
+[program:log]
+command=tail -f /var/log/ytdl.log
+autorestart=true
+priority=999
+
+redirect_stderr=true
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0

k8s.md

@@ -0,0 +1,200 @@
+## Kubernetes
+
+Kubernetes, also known as K8s, is an open-source system for automating deployment, scaling, and management of
+containerized applications
+
+# Complete deployment guide for k8s deloyment
+
+* contains every functionality
+* compatible with amd64, arm64 and armv7l
+
+## First. Get all file in k8s folder
+
+Download `k8s` file to a directory on your k8s server and go to this folder
+
+## 1. Create Redis deloyment
+
+```shell
+kubectl apply -f 01.redis.yml
+```
+
+This command will create ytdl namespace, redis pod and redis service
+
+## 2. Creat MariaDB deloyment
+
+```shell
+kubectl apply -f 02.mariadb.yml
+```
+
+This deloyment will claim 10GB storage from storageClassName: longhorn. Please replace longhorn with your
+storageClassName before apply.
+
+## 3. Set environment variables
+
+Create configMap for env
+
+### 3.1 Edit configmap.yml
+
+```shell
+vim 03.configmap.yml
+```
+
+you can configure all the following environment variables:
+
+* PYRO_WORKERS: number of workers for pyrogram, default is 100
+* WORKERS: workers count for celery
+* APP_ID: **REQUIRED**, get it from https://core.telegram.org/
+* APP_HASH: **REQUIRED**
+* TOKEN: **REQUIRED**
+* REDIS: **REQUIRED if you need VIP mode and cache** ⚠️ Don't publish your redis server on the internet. ⚠️
+
+* OWNER: owner username
+* QUOTA: quota in bytes
+* EX: quota expire time
+* MULTIPLY: vip quota comparing to normal quota
+* USD2CNY: exchange rate
+* VIP: VIP mode, default: disable
+* AFD_LINK
+* COFFEE_LINK
+* COFFEE_TOKEN
+* AFD_TOKEN
+* AFD_USER_ID
+
+* AUTHORIZED_USER: users that could use this bot, user_id, separated with `,`
+* REQUIRED_MEMBERSHIP: group or channel username, user must join this group to use the bot. Could be use with
+  above `AUTHORIZED_USER`
+
+* ENABLE_CELERY: Distribution mode, default: disable. You'll can setup workers in different locations.
+* ENABLE_FFMPEG: enable ffmpeg so Telegram can stream
+* MYSQL_HOST: you'll have to setup MySQL if you enable VIP mode
+* MYSQL_USER
+* MYSQL_PASS
+* GOOGLE_API_KEY: YouTube API key, required for YouTube video subscription.
+* AUDIO_FORMAT: audio format, default is m4a. You can set to any known and supported format for ffmpeg. For
+  example,`mp3`, `flac`, etc. ⚠️ m4a is the fastest. Other formats may affect performance.
+* ARCHIVE_ID: group or channel id/username. All downloads will send to this group first and then forward to end user.
+  **Inline button will be lost during the forwarding.**
+
+### 3.2 Apply configMap for environment variables
+
+```shell
+kubectl apply -f 03.configmap.yml
+```
+
+## 4. Run Master Celery
+
+```shell
+kubectl apply -f 04.ytdl-master.yml
+```
+
+This deloyment will create ytdl-pvc PersistentVolumeClaim on storageClassName: longhorn. This clain will contain vnstat,
+cookies folder and flower database. Please replace longhorn with your storageClassName before apply
+
+### 4.1 Setup instagram cookies
+
+Required if you want to support instagram.
+
+You can use this extension
+[Get cookies.txt](https://chrome.google.com/webstore/detail/get-cookiestxt/bgaddhkoddajcdgocldbbfleckgcbcid)
+to get instagram cookies
+
+Get pod running ytdl master:
+
+```shell
+kubectl get pods --namespace ytdl
+```
+
+Name should be ytdl-xxxxxxxx
+
+Access to pod
+
+```shell
+kubectl --namespace=ytdl exec --stdin --tty ytdl-xxx -- sh
+```
+
+(replace ytdl-xxx by your pod name)
+
+Go to ytdl-pvc mounted folder
+
+```shell
+cd /ytdlbot/ytdlbot/data/
+vim  instagram.com_cookies.txt
+# paste your cookies
+```
+
+## 5. Run Worker Celery
+
+```shell
+kubectl apply -f 05.ytdl-worker.yml
+```
+
+## 6. Run Flower image (OPTIONAL)
+
+### 6.1 Setup flower db
+
+Get pod running ytdl master:
+
+```shell
+kubectl get pods --namespace ytdl
+```
+
+Name should be ytdl-xxxxxxxx
+
+Access to pod
+
+```shell
+kubectl --namespace=ytdl exec --stdin --tty ytdl-xxx -- sh
+```
+
+(replace ytdl-xxx by your pod name)
+
+Go to ytdl-pvc mounted folder
+
+```shel
+cd /var/lib/vnstat/
+```
+
+Create flower database file
+
+```shell
+{} ~ python3
+Python 3.9.9 (main, Nov 21 2021, 03:22:47)
+[Clang 12.0.0 (clang-1200.0.32.29)] on darwin
+Type "help", "copyright", "credits" or "license" for more information.
+>>> import dbm;dbm.open("flower","n");exit()
+```
+
+### 6.2 Config Flower Ingress
+
+This step need config ingress from line 51 of file 06.flower.yml with your ingress service. Need for access from
+internet.
+YML file should be adjusted depending on your load balancing, ingress and network system
+
+For active SSL
+
+```yml
+cert-manager.io/cluster-issuer: letsencrypt-prod
+```
+
+Replace nginx by your ingress service
+
+```yml
+ingressClassName: nginx
+```
+
+Add your domain, example
+
+```yml
+tls:
+  - hosts:
+      - flower.benny.com
+    secretName: flower-tls
+  rules:
+    - host: flower.benny.com
+```
+
+### 6.3 Apply Flower deloyment
+
+```shell
+kubectl apply -f 06.flower.yml
+```

k8s/01.redis.yml

@@ -0,0 +1,53 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ytdl
+  
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  creationTimestamp: null
+  labels:
+    ytdl: redis
+  name: redis
+  namespace: ytdl
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      ytdl: redis
+  strategy: {}
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        ytdl: redis
+    spec:
+      containers:
+        - image: redis:7-alpine
+          name: redis
+          ports:
+            - containerPort: 6379
+          resources: {}
+      restartPolicy: Always
+status: {}
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  creationTimestamp: null
+  labels:
+    ytdl: redis
+  name: redis
+  namespace: ytdl
+spec:
+  ports:
+    - name: "6379"
+      port: 6379
+      targetPort: 6379
+  selector:
+    ytdl: redis
+status:
+  loadBalancer: {}

k8s/02.mariadb.yml

@@ -0,0 +1,80 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  creationTimestamp: null
+  labels:
+    ytdl: mariadb-pvc
+  name: mariadb-pvc
+  namespace: ytdl
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: longhorn
+  resources:
+    requests:
+      storage: 10Gi
+status: {}
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+  creationTimestamp: null
+  labels:
+    ytdl: mariadb
+  name: mariadb
+  namespace: ytdl
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      ytdl: mariadb
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        ytdl: mariadb
+    spec:
+      containers:
+        - env:
+            - name: MYSQL_ROOT_PASSWORD
+              value: ro0tP4sSworD
+            - name: MYSQL_DATABASE
+              value: ytdl
+          image: mariadb:latest
+          name: mariadb
+          ports:
+            - containerPort: 3306
+          resources: {}
+          volumeMounts:
+            - mountPath: /var/lib/mysql
+              name: "mariadb-persistent-storage"
+      restartPolicy: Always
+      volumes:
+        - name: mariadb-persistent-storage
+          persistentVolumeClaim:
+            claimName: mariadb-pvc
+status: {}
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  creationTimestamp: null
+  labels:
+    ytdl: mariadb
+  name: mariadb-svc
+  namespace: ytdl
+spec:
+  ports:
+    - name: "3306"
+      port: 3306
+      targetPort: 3306
+  selector:
+    ytdl: mariadb
+status:
+  loadBalancer: {}
+

k8s/03.configmap.yml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ytdlenv
+  namespace: ytdl
+  annotations:
+data:
+  APP_HASH: 
+  APP_ID: 
+  TOKEN:
+  ARCHIVE_ID: 
+  ENABLE_CELERY: 'True'
+  ENABLE_FFMPEG: 'True'
+  MYSQL_HOST: mariadb-svc
+  MYSQL_PASS: ro0tP4sSworD
+  MYSQL_USER: root
+  REDIS: redis

k8s/04.ytdl-master.yml

@@ -0,0 +1,65 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ytdl-pvc
+  namespace: ytdl
+  creationTimestamp: null
+  labels:
+    ytdl: ytdl-pvc
+spec:
+  accessModes:
+    - ReadWriteMany
+  storageClassName: longhorn
+  resources:
+    requests:
+      storage: 10Gi
+status: {}
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ytdl
+  namespace: ytdl
+  creationTimestamp: null
+  labels:
+    ytdl: ytdl
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      ytdl: ytdl
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        ytdl: ytdl
+    spec:
+      volumes:
+        - name: ytdl-pvc
+          persistentVolumeClaim:
+            claimName: ytdl-pvc
+      containers:
+        - name: ytdl
+          image: bennythink/ytdlbot
+          envFrom:
+            - configMapRef:
+                name: ytdlenv
+          resources: {}
+          volumeMounts:
+            - name: ytdl-pvc
+              mountPath: /var/lib/vnstat/
+              subPath: vnstat/
+            - name: ytdl-pvc
+              mountPath: /ytdlbot/ytdlbot/data/
+              subPath: data/
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+          imagePullPolicy: Always
+      restartPolicy: Always
+      terminationGracePeriodSeconds: 30
+      dnsPolicy: ClusterFirst
+      securityContext: {}
+      schedulerName: default-scheduler
+status: {}

k8s/05.ytdl-worker.yml

@@ -0,0 +1,47 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  creationTimestamp: null
+  labels:
+    ytdl: ytdl-worker
+  name: ytdl-worker
+  namespace: ytdl
+spec:
+  replicas: 4
+  selector:
+    matchLabels:
+      ytdl: ytdl-worker
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        ytdl: ytdl-worker
+    spec:
+      volumes:
+        - name: ytdl-pvc
+          persistentVolumeClaim:
+            claimName: ytdl-pvc
+      containers:
+        - name: ytdl-worker
+          image: bennythink/ytdlbot
+          args:
+            - /usr/local/bin/supervisord
+            - '-c'
+            - /ytdlbot/conf/supervisor_worker.conf
+          envFrom:
+            - configMapRef:
+                name: ytdlenv
+          resources: {}
+          volumeMounts:
+            - name: ytdl-pvc
+              mountPath: /ytdlbot/ytdlbot/data/
+              subPath: data/
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+          imagePullPolicy: Always
+      restartPolicy: Always
+      terminationGracePeriodSeconds: 30
+      dnsPolicy: ClusterFirst
+      securityContext: {}
+      schedulerName: default-scheduler
+status: {}

k8s/06.flower.yml

@@ -0,0 +1,101 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  creationTimestamp: null
+  labels:
+    ytdl: flower
+  name: flower
+  namespace: ytdl
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      ytdl: flower
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        ytdl: flower
+    spec:
+      containers:
+        - envFrom:
+            - configMapRef:
+                name: ytdlenv
+          args:
+           - /usr/local/bin/celery
+           - -A
+           - flower_tasks
+           - flower
+           - --basic_auth=bennythink:123456
+           - --address=0.0.0.0
+           - --persistent
+           - --purge_offline_workers=3600
+          image: bennythink/ytdlbot
+          name: flower
+          ports:
+            - containerPort: 5555
+          resources: {}
+          volumeMounts:
+            - name: ytdl-pvc
+              mountPath: /ytdlbot/ytdlbot/flower
+              subPath: vnstat/flower
+      restartPolicy: Always
+      volumes:
+        - name: ytdl-pvc
+          persistentVolumeClaim:
+            claimName: ytdl-pvc
+status: {}
+
+# THIS IS OPTION IF YOU WANT PUBLIC FLOWER PAGE TO INTERNET.
+# should be adjusted depending on your load balancing system machine
+---
+apiVersion: v1
+kind: Service
+metadata:
+  creationTimestamp: null
+  labels:
+    ytdl: flower
+  name: flower-svc
+  namespace: ytdl
+spec:
+  type: NodePort
+  ports:
+    - name: "5555"
+      protocol: TCP
+      port: 5555
+      targetPort: 5555
+  selector:
+    ytdl: flower
+status:
+  loadBalancer: {}
+
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: nginx-flower-ingress
+  namespace: ytdl
+  annotations:
+    # cert-manager.io/cluster-issuer: letsencrypt-prod
+    nginx.ingress.kubernetes.io/rewrite-target: /
+    # nginx.ingress.kubernetes.io/whitelist-source-range: 14.161.27.151 limit by ipaddresss
+ 
+spec:
+  ingressClassName: nginx
+  tls:
+    - hosts:
+        - your-domain
+      secretName: flower-tls
+  rules:
+    - host: your-domain
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: flower-svc
+                port:
+                  number: 5555

main.py

@@ -0,0 +1,21 @@
+from fastapi import FastAPI, HTTPException
+import subprocess
+
+def run_python_verbose():
+    try:
+        # Run the 'python -v' command in a subprocess
+        print("okk")
+        subprocess.run(['python', 'ytdlbot/ytdl_bot.py'], check=True)
+    except subprocess.CalledProcessError as e:
+        print(f"Error running 'python -v': {e}")
+
+# Call the function
+
+app = FastAPI()
+
+@app.get("/")
+async def root():
+    return {"message": "Hello World"}
+@app.get("/okk")
+async def okk():
+    run_python_verbose()

requirements.txt

@@ -0,0 +1,30 @@
+pyrogram
+tgcrypto
+yt-dlp
+APScheduler
+beautifultable
+ffmpeg-python
+PyMySQL
+celery
+filetype
+flower
+psutil
+influxdb
+beautifulsoup4
+fakeredis
+supervisor
+tgbot-ping
+redis
+requests
+tqdm
+requests-toolbelt
+ffpb
+youtube-search-python
+token-bucket
+coloredlogs
+tronpy
+mnemonic
+qrcode
+blinker
+flask
+streamlit

scripts/low_id.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/usr/local/go/bin:/opt/bin
+
+# Check the logs for the given string
+if docker-compose logs --tail=100 ytdl | grep -q "The msg_id is too low"; then
+    # If the string is found, stop the ytdl service
+    echo "ytdl service stopped due to 'The msg_id is too low' found in logs."
+    docker-compose stop ytdl && docker-compose rm ytdl && docker-compose up -d
+
+else
+    echo "String not found in logs."
+fi

scripts/migrate_to_mysql.py

@@ -0,0 +1,27 @@
+#!/usr/local/bin/python3
+# coding: utf-8
+
+# ytdlbot - migrate_to_mysql.py
+# 12/29/21 15:28
+#
+
+__author__ = "Benny <[email protected]>"
+
+import sqlite3
+
+import pymysql
+
+mysql_con = pymysql.connect(host='localhost', user='root', passwd='root', db='vip', charset='utf8mb4')
+sqlite_con = sqlite3.connect('vip.sqlite')
+
+vips = sqlite_con.execute('SELECT * FROM VIP').fetchall()
+
+for vip in vips:
+    mysql_con.cursor().execute('INSERT INTO vip  VALUES (%s, %s, %s, %s, %s, %s)', vip)
+
+settings = sqlite_con.execute('SELECT * FROM settings').fetchall()
+
+for setting in settings:
+    mysql_con.cursor().execute("INSERT INTO settings VALUES (%s,%s,%s)", setting)
+
+mysql_con.commit()

scripts/start.sh

@@ -0,0 +1,13 @@
+docker run -d --restart unless-stopped --name ytdl \
+   --net host \
+     -e TOKEN=12345 \
+     -e APP_ID=123123 \
+     -e APP_HASH=4990 \
+     -e ENABLE_CELERY=True \
+     -e REDIS=192.168.6.1 \
+     -e MYSQL_HOST=192.168.6.1 \
+     -e WORKERS=4 \
+     -e VIP=True \
+     -e CUSTOM_TEXT=#StandWithUkraine \
+     bennythink/ytdlbot \
+     /usr/local/bin/supervisord -c "/ytdlbot/conf/supervisor_worker.conf"

scripts/transfer.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+# coding: utf-8
+
+# ytdlbot - transfer.py
+# 2023-12-07  18:21
+from tronpy import Tron
+from tronpy.hdwallet import seed_from_mnemonic, key_from_seed
+from tronpy.keys import PrivateKey
+
+mnemonic = "web horse smile ramp olive slush blue property world physical donkey pumpkin"
+
+client = Tron(network="nile")
+
+from_ = client.generate_address_from_mnemonic(mnemonic, account_path="m/44'/195'/0'/0/0")["base58check_address"]
+balance = client.get_account_balance(from_)
+print("my addr: ", from_, "balance: ", balance)
+to = input("to: ")
+amount = int(input("amount in TRX: "))
+
+
+def mnemonic_to_private_key():
+    seed = seed_from_mnemonic(mnemonic, passphrase="")
+    private_key = key_from_seed(seed, account_path="m/44'/195'/0'/0/0")
+    return PrivateKey(private_key)
+
+
+t = client.trx.transfer(from_, to, amount * 1_000_000).build().sign(mnemonic_to_private_key()).broadcast()
+
+print(t.wait())

worker.yml

@@ -0,0 +1,15 @@
+version: '3.1'
+
+services:
+  worker:
+    image: bennythink/ytdlbot
+    env_file:
+      - env/ytdl.env
+    restart: always
+    command: [ "/usr/local/bin/supervisord", "-c" ,"/ytdlbot/conf/supervisor_worker.conf" ]
+#    network_mode: "host"
+#    deploy:
+#      resources:
+#        limits:
+#          cpus: '0.3'
+#          memory: 1500M

ytdlbot/channel.py

@@ -0,0 +1,181 @@
+#!/usr/bin/env python3
+# coding: utf-8
+import http
+import logging
+import os
+import re
+
+import requests
+from bs4 import BeautifulSoup
+
+from config import ENABLE_VIP
+from limit import Payment
+
+
+class Channel(Payment):
+    def subscribe_channel(self, user_id: int, share_link: str) -> str:
+        if not re.findall(r"youtube\.com|youtu\.be", share_link):
+            raise ValueError("Is this a valid YouTube Channel link?")
+        if ENABLE_VIP:
+            self.cur.execute("select count(user_id) from subscribe where user_id=%s", (user_id,))
+            usage = int(self.cur.fetchone()[0])
+            if usage >= 10:
+                logging.warning("User %s has subscribed %s channels", user_id, usage)
+                return "You have subscribed too many channels. Maximum 5 channels."
+
+        data = self.get_channel_info(share_link)
+        channel_id = data["channel_id"]
+
+        self.cur.execute("select user_id from subscribe where user_id=%s and channel_id=%s", (user_id, channel_id))
+        if self.cur.fetchall():
+            raise ValueError("You have already subscribed this channel.")
+
+        self.cur.execute(
+            "INSERT IGNORE INTO channel values"
+            "(%(link)s,%(title)s,%(description)s,%(channel_id)s,%(playlist)s,%(last_video)s)",
+            data,
+        )
+        self.cur.execute("INSERT INTO subscribe values(%s,%s, NULL)", (user_id, channel_id))
+        self.con.commit()
+        logging.info("User %s subscribed channel %s", user_id, data["title"])
+        return "Subscribed to {}".format(data["title"])
+
+    def unsubscribe_channel(self, user_id: int, channel_id: str) -> int:
+        affected_rows = self.cur.execute(
+            "DELETE FROM subscribe WHERE user_id=%s AND channel_id=%s", (user_id, channel_id)
+        )
+        self.con.commit()
+        logging.info("User %s tried to unsubscribe channel %s", user_id, channel_id)
+        return affected_rows
+
+    @staticmethod
+    def extract_canonical_link(url: str) -> str:
+        # canonic link works for many websites. It will strip out unnecessary stuff
+        props = ["canonical", "alternate", "shortlinkUrl"]
+        headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36"
+        }
+        cookie = {"CONSENT": "PENDING+197"}
+        # send head request first
+        r = requests.head(url, headers=headers, allow_redirects=True, cookies=cookie)
+        if r.status_code != http.HTTPStatus.METHOD_NOT_ALLOWED and "text/html" not in r.headers.get("content-type", ""):
+            # get content-type, if it's not text/html, there's no need to issue a GET request
+            logging.warning("%s Content-type is not text/html, no need to GET for extract_canonical_link", url)
+            return url
+
+        html_doc = requests.get(url, headers=headers, cookies=cookie, timeout=5).text
+        soup = BeautifulSoup(html_doc, "html.parser")
+        for prop in props:
+            element = soup.find("link", rel=prop)
+            try:
+                href = element["href"]
+                if href not in ["null", "", None, "https://consent.youtube.com/m"]:
+                    return href
+            except Exception as e:
+                logging.debug("Canonical exception %s %s e", url, e)
+
+        return url
+
+    def get_channel_info(self, url: str) -> dict:
+        api_key = os.getenv("GOOGLE_API_KEY")
+        canonical_link = self.extract_canonical_link(url)
+        try:
+            channel_id = canonical_link.split("youtube.com/channel/")[1]
+        except IndexError:
+            channel_id = canonical_link.split("/")[-1]
+        channel_api = (
+            f"https://www.googleapis.com/youtube/v3/channels?part=snippet,contentDetails&id={channel_id}&key={api_key}"
+        )
+
+        data = requests.get(channel_api).json()
+        snippet = data["items"][0]["snippet"]
+        title = snippet["title"]
+        description = snippet["description"]
+        playlist = data["items"][0]["contentDetails"]["relatedPlaylists"]["uploads"]
+
+        return {
+            "link": url,
+            "title": title,
+            "description": description,
+            "channel_id": channel_id,
+            "playlist": playlist,
+            "last_video": self.get_latest_video(playlist),
+        }
+
+    @staticmethod
+    def get_latest_video(playlist_id: str) -> str:
+        api_key = os.getenv("GOOGLE_API_KEY")
+        video_api = (
+            f"https://www.googleapis.com/youtube/v3/playlistItems?part=snippet&maxResults=1&"
+            f"playlistId={playlist_id}&key={api_key}"
+        )
+        data = requests.get(video_api).json()
+        video_id = data["items"][0]["snippet"]["resourceId"]["videoId"]
+        logging.info(f"Latest video %s from %s", video_id, data["items"][0]["snippet"]["channelTitle"])
+        return f"https://www.youtube.com/watch?v={video_id}"
+
+    def has_newer_update(self, channel_id: str) -> str:
+        self.cur.execute("SELECT playlist,latest_video FROM channel WHERE channel_id=%s", (channel_id,))
+        data = self.cur.fetchone()
+        playlist_id = data[0]
+        old_video = data[1]
+        newest_video = self.get_latest_video(playlist_id)
+        if old_video != newest_video:
+            logging.info("Newer update found for %s %s", channel_id, newest_video)
+            self.cur.execute("UPDATE channel SET latest_video=%s WHERE channel_id=%s", (newest_video, channel_id))
+            self.con.commit()
+            return newest_video
+
+    def get_user_subscription(self, user_id: int) -> str:
+        self.cur.execute(
+            """
+               select title, link, channel.channel_id from channel, subscribe 
+               where subscribe.user_id = %s and channel.channel_id = subscribe.channel_id
+            """,
+            (user_id,),
+        )
+        data = self.cur.fetchall()
+        text = ""
+        for item in data:
+            text += "[{}]({}) `{}\n`".format(*item)
+        return text
+
+    def group_subscriber(self) -> dict:
+        # {"channel_id": [user_id, user_id, ...]}
+        self.cur.execute("select * from subscribe where is_valid=1")
+        data = self.cur.fetchall()
+        group = {}
+        for item in data:
+            group.setdefault(item[1], []).append(item[0])
+        logging.info("Checking periodic subscriber...")
+        return group
+
+    def deactivate_user_subscription(self, user_id: int):
+        self.cur.execute("UPDATE subscribe set is_valid=0 WHERE user_id=%s", (user_id,))
+        self.con.commit()
+
+    def sub_count(self) -> str:
+        sql = """
+        select user_id, channel.title, channel.link
+        from subscribe, channel where subscribe.channel_id = channel.channel_id
+        """
+        self.cur.execute(sql)
+        data = self.cur.fetchall()
+        text = f"Total {len(data)} subscriptions found.\n\n"
+        for item in data:
+            text += "{} ==> [{}]({})\n".format(*item)
+        return text
+
+    def del_cache(self, user_link: str) -> int:
+        unique = self.extract_canonical_link(user_link)
+        caches = self.r.hgetall("cache")
+        count = 0
+        for key in caches:
+            if key.startswith(unique):
+                count += self.del_send_cache(key)
+        return count
+
+
+if __name__ == "__main__":
+    s = Channel.extract_canonical_link("https://www.youtube.com/shorts/KkbYbknjPBM")
+    print(s)

ytdlbot/client_init.py

@@ -0,0 +1,23 @@
+#!/usr/local/bin/python3
+# coding: utf-8
+
+# ytdlbot - client_init.py
+# 12/29/21 16:20
+#
+
+__author__ = "Benny <[email protected]>"
+
+from pyrogram import Client
+
+from config import APP_HASH, APP_ID, PYRO_WORKERS, TOKEN, IPv6
+
+
+def create_app(name: str, workers: int = PYRO_WORKERS) -> Client:
+    return Client(
+        name,
+        APP_ID,
+        APP_HASH,
+        bot_token=TOKEN,
+        workers=workers,
+        ipv6=IPv6,
+    )

ytdlbot/config.py

@@ -0,0 +1,79 @@
+#!/usr/local/bin/python3
+# coding: utf-8
+
+# ytdlbot - config.py
+# 8/28/21 15:01
+#
+
+__author__ = "Benny <[email protected]>"
+
+import os
+
+from blinker import signal
+
+# general settings
+WORKERS: int = int(os.getenv("WORKERS", 10))
+PYRO_WORKERS: int = int(os.getenv("PYRO_WORKERS", 100))
+APP_ID: int = int(20295350)
+APP_HASH = str("805a0a86f3b382d904617d0a2fd4fb6f")
+TOKEN = str("6718500924:AAHljBZcqaegxtDMOODLomnbeR625E-s5uw")
+
+REDIS = os.getenv("REDIS", "redis")
+
+TG_MAX_SIZE = 2000 * 1024 * 1024
+# TG_MAX_SIZE = 10 * 1024 * 1024
+
+EXPIRE = 24 * 3600
+
+ENABLE_VIP = os.getenv("VIP", False)
+OWNER = str("navpan18")
+
+# limitation settings
+AUTHORIZED_USER: str = os.getenv("AUTHORIZED_USER", "")
+# membership requires: the format could be username(without @ sign)/chat_id of channel or group.
+# You need to add the bot to this group/channel as admin
+REQUIRED_MEMBERSHIP: str = os.getenv("REQUIRED_MEMBERSHIP", "")
+
+# celery related
+IS_BACKUP_BOT = os.getenv("IS_BACKUP_BOT")
+ENABLE_CELERY = os.getenv("ENABLE_CELERY", False)
+if IS_BACKUP_BOT:
+    BROKER = os.getenv("BROKER", f"redis://{REDIS}:6379/1")
+else:
+    BROKER = os.getenv("BROKER", f"redis://{REDIS}:6379/0")
+
+MYSQL_HOST = os.getenv("MYSQL_HOST", "mysql")
+MYSQL_USER = os.getenv("MYSQL_USER", "root")
+MYSQL_PASS = os.getenv("MYSQL_PASS", "root")
+
+AUDIO_FORMAT = os.getenv("AUDIO_FORMAT")
+ARCHIVE_ID = -1002047782676
+
+IPv6 = os.getenv("IPv6", False)
+ENABLE_FFMPEG = os.getenv("ENABLE_FFMPEG", False)
+
+PLAYLIST_SUPPORT = True
+M3U8_SUPPORT = True
+ENABLE_ARIA2 = os.getenv("ENABLE_ARIA2", False)
+
+RATE_LIMIT = os.getenv("RATE_LIMIT", 120)
+RCLONE_PATH = os.getenv("RCLONE")
+# This will set the value for the tmpfile path(download path) if it is set.
+# If TMPFILE is not set, it will return None and use system’s default temporary file path.
+# Please ensure that the directory exists and you have necessary permissions to write to it.
+# If you don't know what this is just leave it as it is.
+TMPFILE_PATH = os.getenv("TMPFILE")
+
+# payment settings
+AFD_LINK = os.getenv("AFD_LINK", "https://afdian.net/@BennyThink")
+COFFEE_LINK = os.getenv("COFFEE_LINK", "https://www.buymeacoffee.com/bennythink")
+COFFEE_TOKEN = os.getenv("COFFEE_TOKEN")
+AFD_TOKEN = os.getenv("AFD_TOKEN")
+AFD_USER_ID = os.getenv("AFD_USER_ID")
+PROVIDER_TOKEN = os.getenv("PROVIDER_TOKEN") or "1234"
+FREE_DOWNLOAD = 20000
+TOKEN_PRICE = os.getenv("BUY_UNIT", 20)  # one USD=20 credits
+TRONGRID_KEY = os.getenv("TRONGRID_KEY", "").split(",")
+# the default mnemonic is for nile testnet
+TRON_MNEMONIC = os.getenv("TRON_MNEMONIC", "cram floor today legend service drill pitch leaf car govern harvest soda")
+TRX_SIGNAL = signal("trx_received")

ytdlbot/constant.py

@@ -0,0 +1,117 @@
+#!/usr/local/bin/python3
+# coding: utf-8
+
+# ytdlbot - constant.py
+# 8/16/21 16:59
+#
+
+__author__ = "Benny <[email protected]>"
+
+import os
+
+from config import (
+    AFD_LINK,
+    COFFEE_LINK,
+    ENABLE_CELERY,
+    FREE_DOWNLOAD,
+    REQUIRED_MEMBERSHIP,
+    TOKEN_PRICE,
+)
+from database import InfluxDB
+from utils import get_func_queue
+
+
+class BotText:
+    start = """
+    Welcome to YouTube Download bot. Type /help for more information. 
+    Backup bot: @benny_2ytdlbot
+    Join https://t.me/+OGRC8tp9-U9mZDZl for updates."""
+
+    help = f"""
+1. If the bot doesn't work, try again or join https://t.me/+OGRC8tp9-U9mZDZl for updates.
+
+2. Source code: https://github.com/tgbot-collection/ytdlbot
+    """
+
+    about = "YouTube Downloader by @BennyThink.\n\nOpen source on GitHub: https://github.com/tgbot-collection/ytdlbot"
+
+    buy = f"""
+**Terms:**
+1. You can use this bot to download video for {FREE_DOWNLOAD} times within a 24-hour period.
+
+2. You can buy additional download tokens, valid permanently.
+
+3. Refunds are possible, contact me if you need that @BennyThink
+
+4. Download for paid user will be automatically changed to Local mode to avoid queuing.
+
+**Price:**
+valid permanently
+1. 1 USD == {TOKEN_PRICE} tokens
+2. 7 CNY == {TOKEN_PRICE} tokens
+3. 10 TRX == {TOKEN_PRICE} tokens
+
+**Payment options:**
+Pay any amount you want. For example you can send 20 TRX for {TOKEN_PRICE * 2} tokens.
+1. AFDIAN(AliPay, WeChat Pay and PayPal): {AFD_LINK}
+2. Buy me a coffee: {COFFEE_LINK}
+3. Telegram Bot Payment(Stripe), please click Bot Payment button.
+4. TRON(TRX), please click TRON(TRX) button.
+
+**After payment:**
+1. Afdian: attach order number with /redeem command (e.g., `/redeem 123456`).
+2. Buy Me a Coffee: attach email with /redeem command (e.g., `/redeem [email protected]`). **Use different email each time.**
+3. Telegram Payment & Tron(TRX): automatically activated within 60s. Check /start to see your balance.
+
+Want to buy more token with Telegram payment? Let's say 100? Here you go! `/buy 123`
+    """
+
+    private = "This bot is for private use"
+
+    membership_require = f"You need to join this group or channel to use this bot\n\nhttps://t.me/{REQUIRED_MEMBERSHIP}"
+
+    settings = """
+Please choose the preferred format and video quality for your video. These settings only **apply to YouTube videos**.
+
+High quality is recommended. Medium quality aims to 720P, while low quality is 480P.
+
+If you choose to send the video as a document, it will not be possible to stream it.
+
+Your current settings:
+Video quality: **{0}**
+Sending format: **{1}**
+"""
+    custom_text = os.getenv("CUSTOM_TEXT", "")
+
+    @staticmethod
+    def get_receive_link_text() -> str:
+        reserved = get_func_queue("reserved")
+        if ENABLE_CELERY and reserved:
+            text = f"Your tasks was added to the reserved queue {reserved}. Processing...\n\n"
+        else:
+            text = "Your task was added to active queue.\nProcessing...\n\n"
+
+        return text
+
+    @staticmethod
+    def ping_worker() -> str:
+        from tasks import app as celery_app
+
+        workers = InfluxDB().extract_dashboard_data()
+        # [{'celery@BennyのMBP': 'abc'}, {'celery@BennyのMBP': 'abc'}]
+        response = celery_app.control.broadcast("ping_revision", reply=True)
+        revision = {}
+        for item in response:
+            revision.update(item)
+
+        text = ""
+        for worker in workers:
+            fields = worker["fields"]
+            hostname = worker["tags"]["hostname"]
+            status = {True: "✅"}.get(fields["status"], "❌")
+            active = fields["active"]
+            load = "{},{},{}".format(fields["load1"], fields["load5"], fields["load15"])
+            rev = revision.get(hostname, "")
+            text += f"{status}{hostname} **{active}** {load} {rev}\n"
+
+        return text

ytdlbot/database.py

@@ -0,0 +1,375 @@
+#!/usr/local/bin/python3
+# coding: utf-8
+
+# ytdlbot - database.py
+# 12/7/21 16:57
+#
+
+__author__ = "Benny <[email protected]>"
+
+import base64
+import contextlib
+import datetime
+import logging
+import os
+import re
+import sqlite3
+import subprocess
+import time
+from io import BytesIO
+
+import fakeredis
+import pymysql
+import redis
+import requests
+from beautifultable import BeautifulTable
+from influxdb import InfluxDBClient
+
+from config import MYSQL_HOST, MYSQL_PASS, MYSQL_USER, REDIS, IS_BACKUP_BOT
+
+init_con = sqlite3.connect(":memory:", check_same_thread=False)
+
+
+class FakeMySQL:
+    @staticmethod
+    def cursor() -> "Cursor":
+        return Cursor()
+
+    def commit(self):
+        pass
+
+    def close(self):
+        pass
+
+    def ping(self, reconnect):
+        pass
+
+
+class Cursor:
+    def __init__(self):
+        self.con = init_con
+        self.cur = self.con.cursor()
+
+    def execute(self, *args, **kwargs):
+        sql = self.sub(args[0])
+        new_args = (sql,) + args[1:]
+        with contextlib.suppress(sqlite3.OperationalError):
+            return self.cur.execute(*new_args, **kwargs)
+
+    def fetchall(self):
+        return self.cur.fetchall()
+
+    def fetchone(self):
+        return self.cur.fetchone()
+
+    @staticmethod
+    def sub(sql):
+        sql = re.sub(r"CHARSET.*|charset.*", "", sql, re.IGNORECASE)
+        sql = sql.replace("%s", "?")
+        return sql
+
+
+class Redis:
+    def __init__(self):
+        db = 1 if IS_BACKUP_BOT else 0
+        try:
+            self.r = redis.StrictRedis(host=REDIS, db=db, decode_responses=True)
+            self.r.ping()
+        except Exception:
+            self.r = fakeredis.FakeStrictRedis(host=REDIS, db=db, decode_responses=True)
+
+        db_banner = "=" * 20 + "DB data" + "=" * 20
+        quota_banner = "=" * 20 + "Celery" + "=" * 20
+        metrics_banner = "=" * 20 + "Metrics" + "=" * 20
+        usage_banner = "=" * 20 + "Usage" + "=" * 20
+        vnstat_banner = "=" * 20 + "vnstat" + "=" * 20
+        self.final_text = f"""
+{db_banner}
+%s
+
+
+{vnstat_banner}
+%s
+
+
+{quota_banner}
+%s
+
+
+{metrics_banner}
+%s
+
+
+{usage_banner}
+%s
+        """
+        super().__init__()
+
+    def __del__(self):
+        self.r.close()
+
+    def update_metrics(self, metrics: str):
+        logging.info(f"Setting metrics: {metrics}")
+        all_ = f"all_{metrics}"
+        today = f"today_{metrics}"
+        self.r.hincrby("metrics", all_)
+        self.r.hincrby("metrics", today)
+
+    @staticmethod
+    def generate_table(header, all_data: list):
+        table = BeautifulTable()
+        for data in all_data:
+            table.rows.append(data)
+        table.columns.header = header
+        table.rows.header = [str(i) for i in range(1, len(all_data) + 1)]
+        return table
+
+    def show_usage(self):
+        db = MySQL()
+        db.cur.execute("select user_id,payment_amount,old_user,token from payment")
+        data = db.cur.fetchall()
+        fd = []
+        for item in data:
+            fd.append([item[0], item[1], item[2], item[3]])
+        db_text = self.generate_table(["ID", "pay amount", "old user", "token"], fd)
+
+        fd = []
+        hash_keys = self.r.hgetall("metrics")
+        for key, value in hash_keys.items():
+            if re.findall(r"^today|all", key):
+                fd.append([key, value])
+        fd.sort(key=lambda x: x[0])
+        metrics_text = self.generate_table(["name", "count"], fd)
+
+        fd = []
+        for key, value in hash_keys.items():
+            if re.findall(r"\d+", key):
+                fd.append([key, value])
+        fd.sort(key=lambda x: int(x[-1]), reverse=True)
+        usage_text = self.generate_table(["UserID", "count"], fd)
+
+        worker_data = InfluxDB.get_worker_data()
+        fd = []
+        for item in worker_data["data"]:
+            fd.append(
+                [
+                    item.get("hostname", 0),
+                    item.get("status", 0),
+                    item.get("active", 0),
+                    item.get("processed", 0),
+                    item.get("task-failed", 0),
+                    item.get("task-succeeded", 0),
+                    ",".join(str(i) for i in item.get("loadavg", [])),
+                ]
+            )
+
+        worker_text = self.generate_table(
+            ["worker name", "status", "active", "processed", "failed", "succeeded", "Load Average"], fd
+        )
+
+        # vnstat
+        if os.uname().sysname == "Darwin":
+            cmd = "/opt/homebrew/bin/vnstat -i en0".split()
+        else:
+            cmd = "/usr/bin/vnstat -i eth0".split()
+        vnstat_text = subprocess.check_output(cmd).decode("u8")
+        return self.final_text % (db_text, vnstat_text, worker_text, metrics_text, usage_text)
+
+    def reset_today(self):
+        pairs = self.r.hgetall("metrics")
+        for k in pairs:
+            if k.startswith("today"):
+                self.r.hdel("metrics", k)
+
+    def user_count(self, user_id):
+        self.r.hincrby("metrics", user_id)
+
+    def generate_file(self):
+        text = self.show_usage()
+        file = BytesIO()
+        file.write(text.encode("u8"))
+        date = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time()))
+        file.name = f"{date}.txt"
+        return file
+
+    def add_send_cache(self, unique: str, file_id: str):
+        self.r.hset("cache", unique, file_id)
+
+    def get_send_cache(self, unique) -> str:
+        return self.r.hget("cache", unique)
+
+    def del_send_cache(self, unique):
+        return self.r.hdel("cache", unique)
+
+
+class MySQL:
+    vip_sql = """
+    CREATE TABLE if not exists `payment`
+    (
+        `user_id`        bigint NOT NULL,
+        `payment_amount` float        DEFAULT NULL,
+        `payment_id`     varchar(256) DEFAULT NULL,
+        `old_user`       tinyint(1)   DEFAULT NULL,
+        `token`          int          DEFAULT NULL,
+        UNIQUE KEY `payment_id` (`payment_id`)
+    ) CHARSET = utf8mb4
+                """
+
+    settings_sql = """
+    create table if not exists  settings
+    (
+        user_id    bigint          not null,
+        resolution varchar(128) null,
+        method     varchar(64)  null,
+        mode varchar(32) default 'Celery' null,
+        constraint settings_pk
+            primary key (user_id)
+    );
+            """
+
+    channel_sql = """
+    create table if not exists channel
+    (
+        link              varchar(256) null,
+        title             varchar(256) null,
+        description       text         null,
+        channel_id        varchar(256),
+        playlist          varchar(256) null,
+        latest_video varchar(256) null,
+        constraint channel_pk
+            primary key (channel_id)
+    ) CHARSET=utf8mb4;
+    """
+
+    subscribe_sql = """
+    create table if not exists subscribe
+    (
+        user_id    bigint       null,
+        channel_id varchar(256) null,
+        is_valid boolean default 1 null
+    ) CHARSET=utf8mb4;
+    """
+
+    def __init__(self):
+        try:
+            self.con = pymysql.connect(
+                host=MYSQL_HOST, user=MYSQL_USER, passwd=MYSQL_PASS, db="ytdl", charset="utf8mb4"
+            )
+        except Exception:
+            
+            self.con = FakeMySQL()
+
+        self.con.ping(reconnect=True)
+        self.cur = self.con.cursor()
+        self.init_db()
+        super().__init__()
+
+    def init_db(self):
+        self.cur.execute(self.vip_sql)
+        self.cur.execute(self.settings_sql)
+        self.cur.execute(self.channel_sql)
+        self.cur.execute(self.subscribe_sql)
+        self.con.commit()
+
+    def __del__(self):
+        self.con.close()
+
+    def get_user_settings(self, user_id: int) -> tuple:
+        self.cur.execute("SELECT * FROM settings WHERE user_id = %s", (user_id,))
+        data = self.cur.fetchone()
+        if data is None:
+            return 100, "high", "video", "Celery"
+        return data
+
+    def set_user_settings(self, user_id: int, field: str, value: str):
+        cur = self.con.cursor()
+        cur.execute("SELECT * FROM settings WHERE user_id = %s", (user_id,))
+        data = cur.fetchone()
+        if data is None:
+            resolution = method = ""
+            if field == "resolution":
+                method = "video"
+                resolution = value
+            if field == "method":
+                method = value
+                resolution = "high"
+            cur.execute("INSERT INTO settings VALUES (%s,%s,%s,%s)", (user_id, resolution, method, "Celery"))
+        else:
+            cur.execute(f"UPDATE settings SET {field} =%s WHERE user_id = %s", (value, user_id))
+        self.con.commit()
+
+
+class InfluxDB:
+    def __init__(self):
+        self.client = InfluxDBClient(host=os.getenv("INFLUX_HOST", "192.168.7.233"), database="celery")
+        self.data = None
+
+    def __del__(self):
+        self.client.close()
+
+    @staticmethod
+    def get_worker_data() -> dict:
+        username = os.getenv("FLOWER_USERNAME", "benny")
+        password = os.getenv("FLOWER_PASSWORD", "123456abc")
+        token = base64.b64encode(f"{username}:{password}".encode()).decode()
+        headers = {"Authorization": f"Basic {token}"}
+        r = requests.get("https://celery.dmesg.app/dashboard?json=1", headers=headers)
+        if r.status_code != 200:
+            return dict(data=[])
+        return r.json()
+
+    def extract_dashboard_data(self):
+        self.data = self.get_worker_data()
+        json_body = []
+        for worker in self.data["data"]:
+            load1, load5, load15 = worker["loadavg"]
+            t = {
+                "measurement": "tasks",
+                "tags": {
+                    "hostname": worker["hostname"],
+                },
+                "time": datetime.datetime.utcnow(),
+                "fields": {
+                    "task-received": worker.get("task-received", 0),
+                    "task-started": worker.get("task-started", 0),
+                    "task-succeeded": worker.get("task-succeeded", 0),
+                    "task-failed": worker.get("task-failed", 0),
+                    "active": worker.get("active", 0),
+                    "status": worker.get("status", False),
+                    "load1": load1,
+                    "load5": load5,
+                    "load15": load15,
+                },
+            }
+            json_body.append(t)
+        return json_body
+
+    def __fill_worker_data(self):
+        json_body = self.extract_dashboard_data()
+        self.client.write_points(json_body)
+
+    def __fill_overall_data(self):
+        active = sum([i["active"] for i in self.data["data"]])
+        json_body = [{"measurement": "active", "time": datetime.datetime.utcnow(), "fields": {"active": active}}]
+        self.client.write_points(json_body)
+
+    def __fill_redis_metrics(self):
+        json_body = [{"measurement": "metrics", "time": datetime.datetime.utcnow(), "fields": {}}]
+        r = Redis().r
+        hash_keys = r.hgetall("metrics")
+        for key, value in hash_keys.items():
+            if re.findall(r"^today", key):
+                json_body[0]["fields"][key] = int(value)
+
+        self.client.write_points(json_body)
+
+    def collect_data(self):
+        if os.getenv("INFLUX_HOST") is None:
+            return
+
+        with contextlib.suppress(Exception):
+            self.data = self.get_worker_data()
+            self.__fill_worker_data()
+            self.__fill_overall_data()
+            self.__fill_redis_metrics()
+            logging.debug("InfluxDB data was collected.")

ytdlbot/downloader.py

@@ -0,0 +1,281 @@
+#!/usr/local/bin/python3
+# coding: utf-8
+
+# ytdlbot - downloader.py
+# 8/14/21 16:53
+#
+
+__author__ = "Benny <[email protected]>"
+
+import logging
+import os
+import pathlib
+import random
+import re
+import subprocess
+import time
+import traceback
+from io import StringIO
+from unittest.mock import MagicMock
+
+import fakeredis
+import ffmpeg
+import ffpb
+import filetype
+import requests
+import yt_dlp as ytdl
+from pyrogram import types
+from tqdm import tqdm
+
+from config import AUDIO_FORMAT, ENABLE_ARIA2, ENABLE_FFMPEG, TG_MAX_SIZE, IPv6
+from limit import Payment
+from utils import adjust_formats, apply_log_formatter, current_time, sizeof_fmt
+
+r = fakeredis.FakeStrictRedis()
+apply_log_formatter()
+
+
+def edit_text(bot_msg: types.Message, text: str):
+    key = f"{bot_msg.chat.id}-{bot_msg.id}"
+    # if the key exists, we shouldn't send edit message
+    if not r.exists(key):
+        time.sleep(random.random())
+        r.set(key, "ok", ex=3)
+        bot_msg.edit_text(text)
+
+
+def tqdm_progress(desc, total, finished, speed="", eta=""):
+    def more(title, initial):
+        if initial:
+            return f"{title} {initial}"
+        else:
+            return ""
+
+    f = StringIO()
+    tqdm(
+        total=total,
+        initial=finished,
+        file=f,
+        ascii=False,
+        unit_scale=True,
+        ncols=30,
+        bar_format="{l_bar}{bar} |{n_fmt}/{total_fmt} ",
+    )
+    raw_output = f.getvalue()
+    tqdm_output = raw_output.split("|")
+    progress = f"`[{tqdm_output[1]}]`"
+    detail = tqdm_output[2].replace("[A", "")
+    text = f"""
+{desc}
+
+{progress}
+{detail}
+{more("Speed:", speed)}
+{more("ETA:", eta)}
+    """
+    f.close()
+    return text
+
+
+def remove_bash_color(text):
+    return re.sub(r"\u001b|\[0;94m|\u001b\[0m|\[0;32m|\[0m|\[0;33m", "", text)
+
+
+def download_hook(d: dict, bot_msg):
+    # since we're using celery, server location may be located in different region.
+    # Therefore, we can't trigger the hook very often.
+    # the key is user_id + download_link
+    original_url = d["info_dict"]["original_url"]
+    key = f"{bot_msg.chat.id}-{original_url}"
+
+    if d["status"] == "downloading":
+        downloaded = d.get("downloaded_bytes", 0)
+        total = d.get("total_bytes") or d.get("total_bytes_estimate", 0)
+        if total > TG_MAX_SIZE:
+            raise Exception(f"Your download file size {sizeof_fmt(total)} is too large for Telegram.")
+
+        # percent = remove_bash_color(d.get("_percent_str", "N/A"))
+        speed = remove_bash_color(d.get("_speed_str", "N/A"))
+        eta = remove_bash_color(d.get("_eta_str", d.get("eta")))
+        text = tqdm_progress("Downloading...", total, downloaded, speed, eta)
+        edit_text(bot_msg, text)
+        r.set(key, "ok", ex=5)
+
+
+def upload_hook(current, total, bot_msg):
+    text = tqdm_progress("Uploading...", total, current)
+    edit_text(bot_msg, text)
+
+
+def convert_to_mp4(video_paths: list, bot_msg):
+    default_type = ["video/x-flv", "video/webm"]
+    # all_converted = []
+    for path in video_paths:
+        # if we can't guess file type, we assume it's video/mp4
+        mime = getattr(filetype.guess(path), "mime", "video/mp4")
+        if mime in default_type:
+            if not can_convert_mp4(path, bot_msg.chat.id):
+                logging.warning("Conversion abort for %s", bot_msg.chat.id)
+                bot_msg._client.send_message(bot_msg.chat.id, "Can't convert your video. ffmpeg has been disabled.")
+                break
+            edit_text(bot_msg, f"{current_time()}: Converting {path.name} to mp4. Please wait.")
+            new_file_path = path.with_suffix(".mp4")
+            logging.info("Detected %s, converting to mp4...", mime)
+            run_ffmpeg_progressbar(["ffmpeg", "-y", "-i", path, new_file_path], bot_msg)
+            index = video_paths.index(path)
+            video_paths[index] = new_file_path
+
+
+class ProgressBar(tqdm):
+    b = None
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.bot_msg = self.b
+
+    def update(self, n=1):
+        super().update(n)
+        t = tqdm_progress("Converting...", self.total, self.n)
+        edit_text(self.bot_msg, t)
+
+
+def run_ffmpeg_progressbar(cmd_list: list, bm):
+    cmd_list = cmd_list.copy()[1:]
+    ProgressBar.b = bm
+    ffpb.main(cmd_list, tqdm=ProgressBar)
+
+
+def can_convert_mp4(video_path, uid):
+    if not ENABLE_FFMPEG:
+        return False
+    return True
+
+
+def ytdl_download(url: str, tempdir: str, bm, **kwargs) -> list:
+    payment = Payment()
+    chat_id = bm.chat.id
+    hijack = kwargs.get("hijack")
+    output = pathlib.Path(tempdir, "%(title).70s.%(ext)s").as_posix()
+    ydl_opts = {
+        "progress_hooks": [lambda d: download_hook(d, bm)],
+        "outtmpl": output,
+        "restrictfilenames": False,
+        "quiet": True,
+    }
+    if ENABLE_ARIA2:
+        ydl_opts["external_downloader"] = "aria2c"
+        ydl_opts["external_downloader_args"] = [
+            "--min-split-size=1M",
+            "--max-connection-per-server=16",
+            "--max-concurrent-downloads=16",
+            "--split=16",
+        ]
+    if url.startswith("https://drive.google.com"):
+        # Always use the `source` format for Google Drive URLs.
+        formats = ["source"]
+    else:
+        # Use the default formats for other URLs.
+        formats = [
+            # webm , vp9 and av01 are not streamable on telegram, so we'll extract only mp4
+            "bestvideo[ext=mp4][vcodec!*=av01][vcodec!*=vp09]+bestaudio[ext=m4a]/bestvideo+bestaudio",
+            "bestvideo[vcodec^=avc]+bestaudio[acodec^=mp4a]/best[vcodec^=avc]/best",
+            None,
+        ]
+    adjust_formats(chat_id, url, formats, hijack)
+    if download_instagram(url, tempdir):
+        return list(pathlib.Path(tempdir).glob("*"))
+
+    address = ["::", "0.0.0.0"] if IPv6 else [None]
+    error = None
+    video_paths = None
+    for format_ in formats:
+        ydl_opts["format"] = format_
+        for addr in address:
+            # IPv6 goes first in each format
+            ydl_opts["source_address"] = addr
+            try:
+                logging.info("Downloading for %s with format %s", url, format_)
+                with ytdl.YoutubeDL(ydl_opts) as ydl:
+                    ydl.download([url])
+                video_paths = list(pathlib.Path(tempdir).glob("*"))
+                break
+            except Exception:
+                error = traceback.format_exc()
+                logging.error("Download failed for %s - %s, try another way", format_, url)
+        if error is None:
+            break
+
+    if not video_paths:
+        raise Exception(error)
+
+    # convert format if necessary
+    settings = payment.get_user_settings(chat_id)
+    if settings[2] == "video" or isinstance(settings[2], MagicMock):
+        # only convert if send type is video
+        convert_to_mp4(video_paths, bm)
+    if settings[2] == "audio" or hijack == "bestaudio[ext=m4a]":
+        convert_audio_format(video_paths, bm)
+    # split_large_video(video_paths)
+    return video_paths
+
+
+def convert_audio_format(video_paths: list, bm):
+    # 1. file is audio, default format
+    # 2. file is video, default format
+    # 3. non default format
+
+    for path in video_paths:
+        streams = ffmpeg.probe(path)["streams"]
+        if AUDIO_FORMAT is None and len(streams) == 1 and streams[0]["codec_type"] == "audio":
+            logging.info("%s is audio, default format, no need to convert", path)
+        elif AUDIO_FORMAT is None and len(streams) >= 2:
+            logging.info("%s is video, default format, need to extract audio", path)
+            audio_stream = {"codec_name": "m4a"}
+            for stream in streams:
+                if stream["codec_type"] == "audio":
+                    audio_stream = stream
+                    break
+            ext = audio_stream["codec_name"]
+            new_path = path.with_suffix(f".{ext}")
+            run_ffmpeg_progressbar(["ffmpeg", "-y", "-i", path, "-vn", "-acodec", "copy", new_path], bm)
+            path.unlink()
+            index = video_paths.index(path)
+            video_paths[index] = new_path
+        else:
+            logging.info("Not default format, converting %s to %s", path, AUDIO_FORMAT)
+            new_path = path.with_suffix(f".{AUDIO_FORMAT}")
+            run_ffmpeg_progressbar(["ffmpeg", "-y", "-i", path, new_path], bm)
+            path.unlink()
+            index = video_paths.index(path)
+            video_paths[index] = new_path
+
+
+def split_large_video(video_paths: list):
+    original_video = None
+    split = False
+    for original_video in video_paths:
+        size = os.stat(original_video).st_size
+        if size > TG_MAX_SIZE:
+            split = True
+            logging.warning("file is too large %s, splitting...", size)
+            subprocess.check_output(f"sh split-video.sh {original_video} {TG_MAX_SIZE * 0.95} ".split())
+            os.remove(original_video)
+
+    if split and original_video:
+        return [i for i in pathlib.Path(original_video).parent.glob("*")]
+
+
+def download_instagram(url: str, tempdir: str):
+    if not url.startswith("https://www.instagram.com"):
+        return False
+
+    resp = requests.get(f"http://192.168.6.1:15000/?url={url}").json()
+    if url_results := resp.get("data"):
+        for link in url_results:
+            content = requests.get(link, stream=True).content
+            ext = filetype.guess_extension(content)
+            save_path = pathlib.Path(tempdir, f"{id(link)}.{ext}")
+            with open(save_path, "wb") as f:
+                f.write(content)
+
+        return True

ytdlbot/flower_tasks.py

@@ -0,0 +1,14 @@
+#!/usr/local/bin/python3
+# coding: utf-8
+
+# ytdlbot - flower_tasks.py
+# 1/2/22 10:17
+#
+
+__author__ = "Benny <[email protected]>"
+
+from celery import Celery
+
+from config import BROKER
+
+app = Celery("tasks", broker=BROKER, timezone="Europe/London")

ytdlbot/limit.py

@@ -0,0 +1,260 @@
+#!/usr/local/bin/python3
+# coding: utf-8
+
+# ytdlbot - limit.py
+# 8/15/21 18:23
+#
+
+__author__ = "Benny <[email protected]>"
+
+import hashlib
+import logging
+import time
+
+import requests
+from tronpy import Tron
+from tronpy.exceptions import TransactionError, ValidationError
+from tronpy.hdwallet import key_from_seed, seed_from_mnemonic
+from tronpy.keys import PrivateKey
+from tronpy.providers import HTTPProvider
+
+from config import (
+    AFD_TOKEN,
+    AFD_USER_ID,
+    COFFEE_TOKEN,
+    EXPIRE,
+    FREE_DOWNLOAD,
+    OWNER,
+    TOKEN_PRICE,
+    TRON_MNEMONIC,
+    TRONGRID_KEY,
+    TRX_SIGNAL,
+)
+from database import MySQL, Redis
+from utils import apply_log_formatter, current_time
+
+apply_log_formatter()
+
+
+class BuyMeACoffee:
+    def __init__(self):
+        self._token = COFFEE_TOKEN
+        self._url = "https://developers.buymeacoffee.com/api/v1/supporters"
+        self._data = []
+
+    def _get_data(self, url):
+        d = requests.get(url, headers={"Authorization": f"Bearer {self._token}"}).json()
+        self._data.extend(d["data"])
+        next_page = d["next_page_url"]
+        if next_page:
+            self._get_data(next_page)
+
+    def _get_bmac_status(self, email: str) -> dict:
+        self._get_data(self._url)
+        for user in self._data:
+            if user["payer_email"] == email or user["support_email"] == email:
+                return user
+        return {}
+
+    def get_user_payment(self, email: str) -> (int, "float", str):
+        order = self._get_bmac_status(email)
+        price = float(order.get("support_coffee_price", 0))
+        cups = float(order.get("support_coffees", 1))
+        amount = price * cups
+        return amount, email
+
+
+class Afdian:
+    def __init__(self):
+        self._token = AFD_TOKEN
+        self._user_id = AFD_USER_ID
+        self._url = "https://afdian.net/api/open/query-order"
+
+    def _generate_signature(self):
+        data = {
+            "user_id": self._user_id,
+            "params": '{"x":0}',
+            "ts": int(time.time()),
+        }
+        sign_text = "{token}params{params}ts{ts}user_id{user_id}".format(
+            token=self._token, params=data["params"], ts=data["ts"], user_id=data["user_id"]
+        )
+
+        md5 = hashlib.md5(sign_text.encode("u8"))
+        md5 = md5.hexdigest()
+        data["sign"] = md5
+
+        return data
+
+    def _get_afdian_status(self, trade_no: str) -> dict:
+        req_data = self._generate_signature()
+        data = requests.post(self._url, json=req_data).json()
+        # latest 50
+        for order in data["data"]["list"]:
+            if order["out_trade_no"] == trade_no:
+                return order
+
+        return {}
+
+    def get_user_payment(self, trade_no: str) -> (int, float, str):
+        order = self._get_afdian_status(trade_no)
+        amount = float(order.get("show_amount", 0))
+        # convert to USD
+        return amount / 7, trade_no
+
+
+class TronTrx:
+    def __init__(self):
+        if TRON_MNEMONIC == "cram floor today legend service drill pitch leaf car govern harvest soda":
+            logging.warning("Using nile testnet")
+            provider = HTTPProvider(endpoint_uri="https://nile.trongrid.io")
+            network = "nile"
+        else:
+            provider = HTTPProvider(api_key=TRONGRID_KEY)
+            network = "mainnet"
+        self.client = Tron(provider, network=network)
+
+    def central_transfer(self, from_, index, amount: int):
+        logging.info("Generated key with index %s", index)
+        seed = seed_from_mnemonic(TRON_MNEMONIC, passphrase="")
+        key = PrivateKey(key_from_seed(seed, account_path=f"m/44'/195'/1'/0/{index}"))
+        central = self.central_wallet()
+        logging.info("Transfer %s TRX from %s to %s", amount, from_, central)
+        try:
+            self.client.trx.transfer(from_, central, amount).build().sign(key).broadcast()
+        except (TransactionError, ValidationError):
+            logging.error("Failed to transfer %s TRX to %s. Lower and try again.", amount, from_)
+            if amount > 1_100_000:
+                # 1.1 trx transfer fee
+                self.client.trx.transfer(from_, central, amount - 1_100_000).build().sign(key).broadcast()
+
+    def central_wallet(self):
+        wallet = self.client.generate_address_from_mnemonic(TRON_MNEMONIC, account_path="m/44'/195'/0'/0/0")
+        return wallet["base58check_address"]
+
+    def get_payment_address(self, user_id):
+        # payment_id is like tron,0,TN8Mn9KKv3cSrKyrt6Xx5L18nmezbpiW31,index where 0 means unpaid
+        db = MySQL()
+        con = db.con
+        cur = db.cur
+        cur.execute("select user_id from payment where payment_id like 'tron,%'")
+        data = cur.fetchall()
+        index = len(data)
+        path = f"m/44'/195'/1'/0/{index}"
+        logging.info("Generating address for user %s with path %s", user_id, path)
+        addr = self.client.generate_address_from_mnemonic(TRON_MNEMONIC, account_path=path)["base58check_address"]
+        # add row in db, unpaid
+        cur.execute("insert into payment values (%s,%s,%s,%s,%s)", (user_id, 0, f"tron,0,{addr},{index}", 0, 0))
+        con.commit()
+        return addr
+
+    def check_payment(self):
+        db = MySQL()
+        con = db.con
+        cur = db.cur
+
+        cur.execute("select user_id, payment_id from payment where payment_id like 'tron,0,T%'")
+        data = cur.fetchall()
+        for row in data:
+            logging.info("Checking user payment %s", row)
+            user_id = row[0]
+            addr, index = row[1].split(",")[2:]
+            try:
+                balance = self.client.get_account_balance(addr)
+            except:
+                balance = 0
+            if balance:
+                logging.info("User %s has %s TRX", user_id, balance)
+                # paid, calc token count
+                token_count = int(balance / 10 * TOKEN_PRICE)
+                cur.execute(
+                    "update payment set token=%s,payment_id=%s where user_id=%s and payment_id like %s",
+                    (token_count, f"tron,1,{addr},{index}", user_id, f"tron,%{addr}%"),
+                )
+                cur.execute("UPDATE settings SET mode='Local' WHERE user_id=%s", (user_id,))
+                con.commit()
+                self.central_transfer(addr, index, int(balance * 1_000_000))
+                logging.debug("Dispatch signal now....")
+                TRX_SIGNAL.send("cron", user_id=user_id, text=f"{balance} TRX received, {token_count} tokens added.")
+
+
+class Payment(Redis, MySQL):
+    def check_old_user(self, user_id: int) -> tuple:
+        self.cur.execute("SELECT * FROM payment WHERE user_id=%s AND old_user=1", (user_id,))
+        data = self.cur.fetchone()
+        return data
+
+    def get_pay_token(self, user_id: int) -> int:
+        self.cur.execute("SELECT token, old_user FROM payment WHERE user_id=%s", (user_id,))
+        data = self.cur.fetchall() or [(0, False)]
+        number = sum([i[0] for i in data if i[0]])
+        if number == 0 and data[0][1] != 1:
+            # not old user, no token
+            logging.warning("User %s has no token, set download mode to Celery", user_id)
+            # change download mode to Celery
+            self.set_user_settings(user_id, "mode", "Celery")
+        return number
+
+    def get_free_token(self, user_id: int) -> int:
+        if self.r.exists(user_id):
+            return int(self.r.get(user_id))
+        else:
+            # set and return
+            self.r.set(user_id, FREE_DOWNLOAD, ex=EXPIRE)
+            return FREE_DOWNLOAD
+
+    def get_token(self, user_id: int):
+        ttl = self.r.ttl(user_id)
+        return self.get_free_token(user_id), self.get_pay_token(user_id), current_time(time.time() + ttl)
+
+    def use_free_token(self, user_id: int):
+        if self.r.exists(user_id):
+            self.r.decr(user_id, 1)
+        else:
+            # first time download
+            self.r.set(user_id, 5 - 1, ex=EXPIRE)
+
+    def use_pay_token(self, user_id: int):
+        # a user may pay multiple times, so we'll need to filter the first payment with valid token
+        self.cur.execute("SELECT payment_id FROM payment WHERE user_id=%s AND token>0", (user_id,))
+        data = self.cur.fetchone()
+        payment_id = data[0]
+        logging.info("User %s use pay token with payment_id %s", user_id, payment_id)
+        self.cur.execute("UPDATE payment SET token=token-1 WHERE payment_id=%s", (payment_id,))
+        self.con.commit()
+
+    def use_token(self, user_id: int):
+        free = self.get_free_token(user_id)
+        if free > 0:
+            self.use_free_token(user_id)
+        else:
+            self.use_pay_token(user_id)
+
+    def add_pay_user(self, pay_data: list):
+        self.cur.execute("INSERT INTO payment VALUES (%s,%s,%s,%s,%s)", pay_data)
+        self.set_user_settings(pay_data[0], "mode", "Local")
+        self.con.commit()
+
+    def verify_payment(self, user_id: int, unique: str) -> str:
+        pay = BuyMeACoffee() if "@" in unique else Afdian()
+        self.cur.execute("SELECT * FROM payment WHERE payment_id=%s ", (unique,))
+        data = self.cur.fetchone()
+        if data:
+            # TODO what if a user pay twice with the same email address?
+            return (
+                f"Failed. Payment has been verified by other users. Please contact @{OWNER} if you have any questions."
+            )
+
+        amount, pay_id = pay.get_user_payment(unique)
+        logging.info("User %s paid %s, identifier is %s", user_id, amount, unique)
+        # amount is already in USD
+        if amount == 0:
+            return "Payment not found. Please check your payment ID or email address"
+        self.add_pay_user([user_id, amount, pay_id, 0, amount * TOKEN_PRICE])
+        return "Thanks! Your payment has been verified. /start to get your token details"
+
+
+if __name__ == "__main__":
+    a = TronTrx()
+    # a.central_wallet()
+    a.check_payment()

ytdlbot/split-video.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+# Short script to split videos by filesize using ffmpeg by LukeLR
+
+if [ $# -ne 2 ]; then
+    echo 'Illegal number of parameters. Needs 2 parameters:'
+    echo 'Usage:'
+    echo './split-video.sh FILE SIZELIMIT "FFMPEG_ARGS'
+    echo 
+    echo 'Parameters:'
+    echo '    - FILE:        Name of the video file to split'
+    echo '    - SIZELIMIT:   Maximum file size of each part (in bytes)'
+    echo '    - FFMPEG_ARGS: Additional arguments to pass to each ffmpeg-call'
+    echo '                   (video format and quality options etc.)'
+    exit 1
+fi
+
+FILE="$1"
+SIZELIMIT="$2"
+FFMPEG_ARGS="$3"
+
+# Duration of the source video
+DURATION=$(ffprobe -i "$FILE" -show_entries format=duration -v quiet -of default=noprint_wrappers=1:nokey=1|cut -d. -f1)
+
+# Duration that has been encoded so far
+CUR_DURATION=0
+
+# Filename of the source video (without extension)
+BASENAME="${FILE%.*}"
+
+# Extension for the video parts
+#EXTENSION="${FILE##*.}"
+EXTENSION="mp4"
+
+# Number of the current video part
+i=1
+
+# Filename of the next video part
+NEXTFILENAME="$BASENAME-$i.$EXTENSION"
+
+echo "Duration of source video: $DURATION"
+
+# Until the duration of all partial videos has reached the duration of the source video
+while [[ $CUR_DURATION -lt $DURATION ]]; do
+    # Encode next part
+    echo ffmpeg -i "$FILE" -ss "$CUR_DURATION" -fs "$SIZELIMIT" $FFMPEG_ARGS "$NEXTFILENAME"
+    ffmpeg -ss "$CUR_DURATION" -i "$FILE" -fs "$SIZELIMIT" $FFMPEG_ARGS "$NEXTFILENAME"
+
+    # Duration of the new part
+    NEW_DURATION=$(ffprobe -i "$NEXTFILENAME" -show_entries format=duration -v quiet -of default=noprint_wrappers=1:nokey=1|cut -d. -f1)
+
+    # Total duration encoded so far
+    CUR_DURATION=$((CUR_DURATION + NEW_DURATION))
+
+    i=$((i + 1))
+
+    echo "Duration of $NEXTFILENAME: $NEW_DURATION"
+    echo "Part No. $i starts at $CUR_DURATION"
+
+    NEXTFILENAME="$BASENAME-$i.$EXTENSION"
+done

ytdlbot/tasks.py

@@ -0,0 +1,491 @@
+#!/usr/local/bin/python3
+# coding: utf-8
+
+# ytdlbot - tasks.py
+# 12/29/21 14:57
+#
+
+__author__ = "Benny <[email protected]>"
+
+import asyncio
+import logging
+import os
+import pathlib
+import re
+import shutil
+import subprocess
+import tempfile
+import threading
+import time
+import traceback
+import typing
+from typing import Any
+from urllib.parse import quote_plus
+
+import filetype
+import psutil
+import pyrogram.errors
+import requests
+from apscheduler.schedulers.background import BackgroundScheduler
+from celery import Celery
+from celery.worker.control import Panel
+from pyrogram import Client, enums, idle, types
+
+from channel import Channel
+from client_init import create_app
+from config import (
+    ARCHIVE_ID,
+    BROKER,
+    ENABLE_CELERY,
+    ENABLE_VIP,
+    OWNER,
+    RATE_LIMIT,
+    RCLONE_PATH,
+    TMPFILE_PATH,
+    WORKERS,
+)
+from constant import BotText
+from database import Redis
+from downloader import edit_text, tqdm_progress, upload_hook, ytdl_download
+from limit import Payment
+from utils import (
+    apply_log_formatter,
+    auto_restart,
+    customize_logger,
+    get_metadata,
+    get_revision,
+    sizeof_fmt,
+)
+
+customize_logger(["pyrogram.client", "pyrogram.session.session", "pyrogram.connection.connection"])
+apply_log_formatter()
+bot_text = BotText()
+logging.getLogger("apscheduler.executors.default").propagate = False
+
+app = Celery("tasks", broker=BROKER)
+bot = create_app("tasks")
+channel = Channel()
+
+
+def retrieve_message(chat_id: int, message_id: int) -> types.Message | Any:
+    # this should only be called by celery tasks
+    try:
+        return bot.get_messages(chat_id, message_id)
+    except ConnectionError as e:
+        logging.critical("BOT IS NOT STARTED YET: %s", e)
+        bot.start()
+        return bot.get_messages(chat_id, message_id)
+
+
+@app.task(rate_limit=f"{RATE_LIMIT}/m")
+def ytdl_download_task(chat_id: int, message_id: int, url: str):
+    logging.info("YouTube celery tasks started for %s", url)
+    bot_msg = retrieve_message(chat_id, message_id)
+    ytdl_normal_download(bot, bot_msg, url)
+    logging.info("YouTube celery tasks ended.")
+
+
+@app.task()
+def audio_task(chat_id: int, message_id: int):
+    logging.info("Audio celery tasks started for %s-%s", chat_id, message_id)
+    bot_msg = retrieve_message(chat_id, message_id)
+    normal_audio(bot, bot_msg)
+    logging.info("Audio celery tasks ended.")
+
+
+@app.task()
+def direct_download_task(chat_id: int, message_id: int, url: str):
+    logging.info("Direct download celery tasks started for %s", url)
+    bot_msg = retrieve_message(chat_id, message_id)
+    direct_normal_download(bot, bot_msg, url)
+    logging.info("Direct download celery tasks ended.")
+
+
+def get_unique_clink(original_url: str, user_id: int):
+    payment = Payment()
+    settings = payment.get_user_settings(user_id)
+    clink = channel.extract_canonical_link(original_url)
+    try:
+        # different user may have different resolution settings
+        unique = "{}?p={}{}".format(clink, *settings[1:])
+    except IndexError:
+        unique = clink
+    return unique
+
+
+def forward_video(client, bot_msg: types.Message | Any, url: str, cached_fid: str):
+    res_msg = upload_processor(client, bot_msg, url, cached_fid)
+    obj = res_msg.document or res_msg.video or res_msg.audio or res_msg.animation or res_msg.photo
+
+    caption, _ = gen_cap(bot_msg, url, obj)
+    res_msg.edit_text(caption, reply_markup=gen_video_markup())
+    bot_msg.edit_text(f"Download success!✅")
+    return True
+
+
+def ytdl_download_entrance(client: Client, bot_msg: types.Message, url: str, mode=None):
+    # in Local node and forward mode, we pass client from main
+    # in celery mode, we need to use our own client called bot
+    payment = Payment()
+    redis = Redis()
+    chat_id = bot_msg.chat.id
+    unique = get_unique_clink(url, chat_id)
+    cached_fid = redis.get_send_cache(unique)
+
+    try:
+        if cached_fid:
+            forward_video(client, bot_msg, url, cached_fid)
+            redis.update_metrics("cache_hit")
+            return
+        redis.update_metrics("cache_miss")
+        mode = mode or payment.get_user_settings(chat_id)[-1]
+        if ENABLE_CELERY and mode in [None, "Celery"]:
+            ytdl_download_task.delay(chat_id, bot_msg.id, url)
+        else:
+            ytdl_normal_download(client, bot_msg, url)
+    except Exception as e:
+        logging.error("Failed to download %s, error: %s", url, e)
+        bot_msg.edit_text(f"Download failed!❌\n\n`{traceback.format_exc()[0:4000]}`", disable_web_page_preview=True)
+
+
+def direct_download_entrance(client: Client, bot_msg: typing.Union[types.Message, typing.Coroutine], url: str):
+    if ENABLE_CELERY:
+        direct_normal_download(client, bot_msg, url)
+        # direct_download_task.delay(bot_msg.chat.id, bot_msg.id, url)
+    else:
+        direct_normal_download(client, bot_msg, url)
+
+
+def audio_entrance(client: Client, bot_msg: types.Message):
+    if ENABLE_CELERY:
+        audio_task.delay(bot_msg.chat.id, bot_msg.id)
+    else:
+        normal_audio(client, bot_msg)
+
+
+def direct_normal_download(client: Client, bot_msg: typing.Union[types.Message, typing.Coroutine], url: str):
+    chat_id = bot_msg.chat.id
+    headers = {
+        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.3987.149 Safari/537.36"
+    }
+    length = 0
+
+    req = None
+    try:
+        req = requests.get(url, headers=headers, stream=True)
+        length = int(req.headers.get("content-length"))
+        filename = re.findall("filename=(.+)", req.headers.get("content-disposition"))[0]
+    except TypeError:
+        filename = getattr(req, "url", "").rsplit("/")[-1]
+    except Exception as e:
+        bot_msg.edit_text(f"Download failed!❌\n\n```{e}```", disable_web_page_preview=True)
+        return
+
+    if not filename:
+        filename = quote_plus(url)
+
+    with tempfile.TemporaryDirectory(prefix="ytdl-", dir=TMPFILE_PATH) as f:
+        filepath = f"{f}/{filename}"
+        # consume the req.content
+        downloaded = 0
+        for chunk in req.iter_content(1024 * 1024):
+            text = tqdm_progress("Downloading...", length, downloaded)
+            edit_text(bot_msg, text)
+            with open(filepath, "ab") as fp:
+                fp.write(chunk)
+            downloaded += len(chunk)
+        logging.info("Downloaded file %s", filename)
+        st_size = os.stat(filepath).st_size
+
+        client.send_chat_action(chat_id, enums.ChatAction.UPLOAD_DOCUMENT)
+        client.send_document(
+            bot_msg.chat.id,
+            filepath,
+            caption=f"filesize: {sizeof_fmt(st_size)}",
+            progress=upload_hook,
+            progress_args=(bot_msg,),
+        )
+        bot_msg.edit_text("Download success!✅")
+
+
+def normal_audio(client: Client, bot_msg: typing.Union[types.Message, typing.Coroutine]):
+    chat_id = bot_msg.chat.id
+    # fn = getattr(bot_msg.video, "file_name", None) or getattr(bot_msg.document, "file_name", None)
+    status_msg: typing.Union[types.Message, typing.Coroutine] = bot_msg.reply_text(
+        "Converting to audio...please wait patiently", quote=True
+    )
+    orig_url: str = re.findall(r"https?://.*", bot_msg.caption)[0]
+    with tempfile.TemporaryDirectory(prefix="ytdl-", dir=TMPFILE_PATH) as tmp:
+        client.send_chat_action(chat_id, enums.ChatAction.RECORD_AUDIO)
+        # just try to download the audio using yt-dlp
+        filepath = ytdl_download(orig_url, tmp, status_msg, hijack="bestaudio[ext=m4a]")
+        status_msg.edit_text("Sending audio now...")
+        client.send_chat_action(chat_id, enums.ChatAction.UPLOAD_AUDIO)
+        for f in filepath:
+            client.send_audio(chat_id, f)
+        status_msg.edit_text("✅ Conversion complete.")
+        Redis().update_metrics("audio_success")
+
+
+def ytdl_normal_download(client: Client, bot_msg: types.Message | typing.Any, url: str):
+    """
+    This function is called by celery task or directly by bot
+    :param client: bot client, either from main or bot(celery)
+    :param bot_msg: bot message
+    :param url: url to download
+    """
+    chat_id = bot_msg.chat.id
+    temp_dir = tempfile.TemporaryDirectory(prefix="ytdl-", dir=TMPFILE_PATH)
+
+    video_paths = ytdl_download(url, temp_dir.name, bot_msg)
+    logging.info("Download complete.")
+    client.send_chat_action(chat_id, enums.ChatAction.UPLOAD_DOCUMENT)
+    bot_msg.edit_text("Download complete. Sending now...")
+    try:
+        upload_processor(client, bot_msg, url, video_paths)
+    except pyrogram.errors.Flood as e:
+        logging.critical("FloodWait from Telegram: %s", e)
+        client.send_message(
+            chat_id,
+            f"I'm being rate limited by Telegram. Your video will come after {e} seconds. Please wait patiently.",
+        )
+        client.send_message(OWNER, f"CRITICAL INFO: {e}")
+        time.sleep(e.value)
+        upload_processor(client, bot_msg, url, video_paths)
+
+    bot_msg.edit_text("Download success!✅")
+
+    # setup rclone environment var to back up the downloaded file
+    if RCLONE_PATH:
+        for item in os.listdir(temp_dir.name):
+            logging.info("Copying %s to %s", item, RCLONE_PATH)
+            shutil.copy(os.path.join(temp_dir.name, item), RCLONE_PATH)
+    temp_dir.cleanup()
+
+
+def generate_input_media(file_paths: list, cap: str) -> list:
+    input_media = []
+    for path in file_paths:
+        mime = filetype.guess_mime(path)
+        if "video" in mime:
+            input_media.append(pyrogram.types.InputMediaVideo(media=path))
+        elif "image" in mime:
+            input_media.append(pyrogram.types.InputMediaPhoto(media=path))
+        elif "audio" in mime:
+            input_media.append(pyrogram.types.InputMediaAudio(media=path))
+        else:
+            input_media.append(pyrogram.types.InputMediaDocument(media=path))
+
+    input_media[0].caption = cap
+    return input_media
+
+
+def upload_processor(client: Client, bot_msg: types.Message, url: str, vp_or_fid: str | list):
+    redis = Redis()
+    # raise pyrogram.errors.exceptions.FloodWait(13)
+    # if is str, it's a file id; else it's a list of paths
+    payment = Payment()
+    chat_id = bot_msg.chat.id
+    markup = gen_video_markup()
+    if isinstance(vp_or_fid, list) and len(vp_or_fid) > 1:
+        # just generate the first for simplicity, send as media group(2-20)
+        cap, meta = gen_cap(bot_msg, url, vp_or_fid[0])
+        res_msg: list["types.Message"] | Any = client.send_media_group(chat_id, generate_input_media(vp_or_fid, cap))
+        # TODO no cache for now
+        return res_msg[0]
+    elif isinstance(vp_or_fid, list) and len(vp_or_fid) == 1:
+        # normal download, just contains one file in video_paths
+        vp_or_fid = vp_or_fid[0]
+        cap, meta = gen_cap(bot_msg, url, vp_or_fid)
+    else:
+        # just a file id as string
+        cap, meta = gen_cap(bot_msg, url, vp_or_fid)
+
+    settings = payment.get_user_settings(chat_id)
+    if ARCHIVE_ID and isinstance(vp_or_fid, pathlib.Path):
+        chat_id = ARCHIVE_ID
+
+    if settings[2] == "document":
+        logging.info("Sending as document")
+        try:
+            # send as document could be sent as video even if it's a document
+            res_msg = client.send_document(
+                chat_id,
+                vp_or_fid,
+                caption=cap,
+                progress=upload_hook,
+                progress_args=(bot_msg,),
+                reply_markup=markup,
+                thumb=meta["thumb"],
+                force_document=True,
+            )
+        except ValueError:
+            logging.error("Retry to send as video")
+            res_msg = client.send_video(
+                chat_id,
+                vp_or_fid,
+                supports_streaming=True,
+                caption=cap,
+                progress=upload_hook,
+                progress_args=(bot_msg,),
+                reply_markup=markup,
+                **meta,
+            )
+    elif settings[2] == "audio":
+        logging.info("Sending as audio")
+        res_msg = client.send_audio(
+            chat_id,
+            vp_or_fid,
+            caption=cap,
+            progress=upload_hook,
+            progress_args=(bot_msg,),
+        )
+    else:
+        # settings==video
+        logging.info("Sending as video")
+        try:
+            res_msg = client.send_video(
+                chat_id,
+                vp_or_fid,
+                supports_streaming=True,
+                caption=cap,
+                progress=upload_hook,
+                progress_args=(bot_msg,),
+                reply_markup=markup,
+                **meta,
+            )
+        except Exception:
+            # try to send as annimation, photo
+            try:
+                logging.warning("Retry to send as animation")
+                res_msg = client.send_animation(
+                    chat_id,
+                    vp_or_fid,
+                    caption=cap,
+                    progress=upload_hook,
+                    progress_args=(bot_msg,),
+                    reply_markup=markup,
+                    **meta,
+                )
+            except Exception:
+                # this is likely a photo
+                logging.warning("Retry to send as photo")
+                res_msg = client.send_photo(
+                    chat_id,
+                    vp_or_fid,
+                    caption=cap,
+                    progress=upload_hook,
+                    progress_args=(bot_msg,),
+                )
+
+    unique = get_unique_clink(url, bot_msg.chat.id)
+    obj = res_msg.document or res_msg.video or res_msg.audio or res_msg.animation or res_msg.photo
+    redis.add_send_cache(unique, getattr(obj, "file_id", None))
+    redis.update_metrics("video_success")
+    if ARCHIVE_ID and isinstance(vp_or_fid, pathlib.Path):
+        client.forward_messages(bot_msg.chat.id, ARCHIVE_ID, res_msg.id)
+    return res_msg
+
+
+def gen_cap(bm, url, video_path):
+    payment = Payment()
+    chat_id = bm.chat.id
+    user = bm.chat
+    try:
+        user_info = "@{}({})-{}".format(user.username or "N/A", user.first_name or "" + user.last_name or "", user.id)
+    except Exception:
+        user_info = ""
+
+    if isinstance(video_path, pathlib.Path):
+        meta = get_metadata(video_path)
+        file_name = video_path.name
+        file_size = sizeof_fmt(os.stat(video_path).st_size)
+    else:
+        file_name = getattr(video_path, "file_name", "")
+        file_size = sizeof_fmt(getattr(video_path, "file_size", (2 << 2) + ((2 << 2) + 1) + (2 << 5)))
+        meta = dict(
+            width=getattr(video_path, "width", 0),
+            height=getattr(video_path, "height", 0),
+            duration=getattr(video_path, "duration", 0),
+            thumb=getattr(video_path, "thumb", None),
+        )
+    free = payment.get_free_token(chat_id)
+    pay = payment.get_pay_token(chat_id)
+    if ENABLE_VIP:
+        remain = f"Download token count: free {free}, pay {pay}"
+    else:
+        remain = ""
+
+    if worker_name := os.getenv("WORKER_NAME"):
+        worker = f"Downloaded by  {worker_name}"
+    else:
+        worker = ""
+    cap = (
+        f"{user_info}\n{file_name}\n\n{url}\n\nInfo: {meta['width']}x{meta['height']} {file_size}\t"
+        f"{meta['duration']}s\n{remain}\n{worker}\n{bot_text.custom_text}"
+    )
+    return cap, meta
+
+
+def gen_video_markup():
+    markup = types.InlineKeyboardMarkup(
+        [
+            [  # First row
+                types.InlineKeyboardButton(  # Generates a callback query when pressed
+                    "convert to audio", callback_data="convert"
+                )
+            ]
+        ]
+    )
+    return markup
+
+
+@Panel.register
+def ping_revision(*args):
+    return get_revision()
+
+
+@Panel.register
+def hot_patch(*args):
+    app_path = pathlib.Path().cwd().parent
+    logging.info("Hot patching on path %s...", app_path)
+
+    pip_install = "pip install -r requirements.txt"
+    unset = "git config --unset http.https://github.com/.extraheader"
+    pull_unshallow = "git pull origin --unshallow"
+    pull = "git pull"
+
+    subprocess.call(unset, shell=True, cwd=app_path)
+    if subprocess.call(pull_unshallow, shell=True, cwd=app_path) != 0:
+        logging.info("Already unshallow, pulling now...")
+        subprocess.call(pull, shell=True, cwd=app_path)
+
+    logging.info("Code is updated, applying hot patch now...")
+    subprocess.call(pip_install, shell=True, cwd=app_path)
+    psutil.Process().kill()
+
+
+def purge_tasks():
+    count = app.control.purge()
+    return f"purged {count} tasks."
+
+
+def run_celery():
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    worker_name = os.getenv("WORKER_NAME", "")
+    argv = ["-A", "tasks", "worker", "--loglevel=info", "--pool=threads", f"--concurrency={WORKERS}", "-n", worker_name]
+    app.worker_main(argv)
+
+
+if __name__ == "__main__":
+    print("Bootstrapping Celery worker now.....")
+    time.sleep(5)
+    threading.Thread(target=run_celery, daemon=True).start()
+
+    scheduler = BackgroundScheduler(timezone="Europe/London")
+    scheduler.add_job(auto_restart, "interval", seconds=900)
+    scheduler.start()
+
+    idle()
+    bot.stop()

ytdlbot/utils.py

@@ -0,0 +1,216 @@
+#!/usr/local/bin/python3
+# coding: utf-8
+
+# ytdlbot - utils.py
+# 9/1/21 22:50
+#
+
+__author__ = "Benny <[email protected]>"
+
+import contextlib
+import inspect as pyinspect
+import logging
+import os
+import pathlib
+import shutil
+import subprocess
+import tempfile
+import time
+import uuid
+
+import coloredlogs
+import ffmpeg
+import psutil
+
+from config import TMPFILE_PATH
+from flower_tasks import app
+
+inspect = app.control.inspect()
+
+
+def apply_log_formatter():
+    coloredlogs.install(
+        level=logging.INFO,
+        fmt="[%(asctime)s %(filename)s:%(lineno)d %(levelname).1s] %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+
+
+def customize_logger(logger: list):
+    for log in logger:
+        logging.getLogger(log).setLevel(level=logging.INFO)
+
+
+def sizeof_fmt(num: int, suffix="B"):
+    for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]:
+        if abs(num) < 1024.0:
+            return "%3.1f%s%s" % (num, unit, suffix)
+        num /= 1024.0
+    return "%.1f%s%s" % (num, "Yi", suffix)
+
+
+def is_youtube(url: str):
+    if url.startswith("https://www.youtube.com/") or url.startswith("https://youtu.be/"):
+        return True
+
+
+def adjust_formats(user_id: int, url: str, formats: list, hijack=None):
+    from database import MySQL
+
+    # high: best quality 1080P, 2K, 4K, 8K
+    # medium: 720P
+    # low: 480P
+    if hijack:
+        formats.insert(0, hijack)
+        return
+
+    mapping = {"high": [], "medium": [720], "low": [480]}
+    settings = MySQL().get_user_settings(user_id)
+    if settings and is_youtube(url):
+        for m in mapping.get(settings[1], []):
+            formats.insert(0, f"bestvideo[ext=mp4][height={m}]+bestaudio[ext=m4a]")
+            formats.insert(1, f"bestvideo[vcodec^=avc][height={m}]+bestaudio[acodec^=mp4a]/best[vcodec^=avc]/best")
+
+    if settings[2] == "audio":
+        formats.insert(0, "bestaudio[ext=m4a]")
+
+
+def get_metadata(video_path):
+    width, height, duration = 1280, 720, 0
+    try:
+        video_streams = ffmpeg.probe(video_path, select_streams="v")
+        for item in video_streams.get("streams", []):
+            height = item["height"]
+            width = item["width"]
+        duration = int(float(video_streams["format"]["duration"]))
+    except Exception as e:
+        logging.error(e)
+    try:
+        thumb = pathlib.Path(video_path).parent.joinpath(f"{uuid.uuid4().hex}-thunmnail.png").as_posix()
+        ffmpeg.input(video_path, ss=duration / 2).filter("scale", width, -1).output(thumb, vframes=1).run()
+    except ffmpeg._run.Error:
+        thumb = None
+
+    return dict(height=height, width=width, duration=duration, thumb=thumb)
+
+
+def current_time(ts=None):
+    return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ts))
+
+
+def get_revision():
+    with contextlib.suppress(subprocess.SubprocessError):
+        return subprocess.check_output("git -C ../ rev-parse --short HEAD".split()).decode("u8").replace("\n", "")
+    return "unknown"
+
+
+def get_func_queue(func) -> int:
+    try:
+        count = 0
+        data = getattr(inspect, func)() or {}
+        for _, task in data.items():
+            count += len(task)
+        return count
+    except Exception:
+        return 0
+
+
+def tail_log(f, lines=1, _buffer=4098):
+    """Tail a file and get X lines from the end"""
+    # placeholder for the lines found
+    lines_found = []
+
+    # block counter will be multiplied by buffer
+    # to get the block size from the end
+    block_counter = -1
+
+    # loop until we find X lines
+    while len(lines_found) < lines:
+        try:
+            f.seek(block_counter * _buffer, os.SEEK_END)
+        except IOError:  # either file is too small, or too many lines requested
+            f.seek(0)
+            lines_found = f.readlines()
+            break
+
+        lines_found = f.readlines()
+
+        # we found enough lines, get out
+        # Removed this line because it was redundant the while will catch
+        # it, I left it for history
+        # if len(lines_found) > lines:
+        #    break
+
+        # decrement the block counter to get the
+        # next X bytes
+        block_counter -= 1
+
+    return lines_found[-lines:]
+
+
+class Detector:
+    def __init__(self, logs: str):
+        self.logs = logs
+
+    @staticmethod
+    def func_name():
+        with contextlib.suppress(Exception):
+            return pyinspect.stack()[1][3]
+        return "N/A"
+
+    def updates_too_long_detector(self):
+        # If you're seeing this, that means you have logged more than 10 device
+        # and the earliest account was kicked out. Restart the program could get you back in.
+        indicators = [
+            "types.UpdatesTooLong",
+            "Got shutdown from remote",
+            "Code is updated",
+            "OSError: Connection lost",
+            "[Errno -3] Try again",
+            "MISCONF",
+        ]
+        for indicator in indicators:
+            if indicator in self.logs:
+                logging.critical("kick out crash: %s", self.func_name())
+                return True
+        logging.debug("No crash detected.")
+
+    def next_salt_detector(self):
+        text = "Next salt in"
+        if self.logs.count(text) >= 5:
+            logging.critical("Next salt crash: %s", self.func_name())
+            return True
+
+    def connection_reset_detector(self):
+        text = "Send exception: ConnectionResetError Connection lost"
+        if text in self.logs:
+            logging.critical("connection lost: %s ", self.func_name())
+            return True
+
+
+def auto_restart():
+    log_path = "/var/log/ytdl.log"
+    if not os.path.exists(log_path):
+        return
+    with open(log_path) as f:
+        logs = "".join(tail_log(f, lines=100))
+
+    det = Detector(logs)
+    method_list = [getattr(det, func) for func in dir(det) if func.endswith("_detector")]
+    for method in method_list:
+        if method():
+            logging.critical("%s bye bye world!☠️", method)
+            for item in pathlib.Path(TMPFILE_PATH or tempfile.gettempdir()).glob("ytdl-*"):
+                shutil.rmtree(item, ignore_errors=True)
+            time.sleep(5)
+            psutil.Process().kill()
+
+
+def clean_tempfile():
+    for item in pathlib.Path(TMPFILE_PATH or tempfile.gettempdir()).glob("ytdl-*"):
+        if time.time() - item.stat().st_ctime > 3600:
+            shutil.rmtree(item, ignore_errors=True)
+
+
+if __name__ == "__main__":
+    auto_restart()

ytdlbot/ytdl_bot.py

@@ -0,0 +1,562 @@
+#!/usr/local/bin/python3
+# coding: utf-8
+
+# ytdlbot - new.py
+# 8/14/21 14:37
+#
+
+__author__ = "Benny <[email protected]>"
+
+import contextlib
+import logging
+import os
+import random
+import re
+import tempfile
+import time
+import traceback
+from io import BytesIO
+from typing import Any
+
+import pyrogram.errors
+import qrcode
+import yt_dlp
+from apscheduler.schedulers.background import BackgroundScheduler
+from pyrogram import Client, enums, filters, types
+from pyrogram.errors.exceptions.bad_request_400 import UserNotParticipant
+from pyrogram.raw import functions
+from pyrogram.raw import types as raw_types
+from tgbot_ping import get_runtime
+from youtubesearchpython import VideosSearch
+
+from channel import Channel
+from client_init import create_app
+from config import (
+    AUTHORIZED_USER,
+    ENABLE_CELERY,
+    ENABLE_FFMPEG,
+    ENABLE_VIP,
+    IS_BACKUP_BOT,
+    M3U8_SUPPORT,
+    OWNER,
+    PLAYLIST_SUPPORT,
+    PROVIDER_TOKEN,
+    REQUIRED_MEMBERSHIP,
+    TOKEN_PRICE,
+    TRX_SIGNAL,
+)
+from constant import BotText
+from database import InfluxDB, MySQL, Redis
+from limit import Payment, TronTrx
+from tasks import app as celery_app
+from tasks import (
+    audio_entrance,
+    direct_download_entrance,
+    hot_patch,
+    purge_tasks,
+    ytdl_download_entrance,
+)
+from utils import auto_restart, clean_tempfile, customize_logger, get_revision
+
+logging.info("Authorized users are %s", AUTHORIZED_USER)
+customize_logger(["pyrogram.client", "pyrogram.session.session", "pyrogram.connection.connection"])
+logging.getLogger("apscheduler.executors.default").propagate = False
+
+app = create_app("main")
+channel = Channel()
+
+
+def private_use(func):
+    def wrapper(client: Client, message: types.Message):
+        chat_id = getattr(message.from_user, "id", None)
+
+        # message type check
+        if message.chat.type != enums.ChatType.PRIVATE and not message.text.lower().startswith("/ytdl"):
+            logging.debug("%s, it's annoying me...🙄️ ", message.text)
+            return
+
+        # authorized users check
+        if AUTHORIZED_USER:
+            users = [int(i) for i in AUTHORIZED_USER.split(",")]
+        else:
+            users = []
+
+        if users and chat_id and chat_id not in users:
+            message.reply_text(BotText.private, quote=True)
+            return
+
+        if REQUIRED_MEMBERSHIP:
+            try:
+                member: types.ChatMember | Any = app.get_chat_member(REQUIRED_MEMBERSHIP, chat_id)
+                if member.status not in [
+                    enums.ChatMemberStatus.ADMINISTRATOR,
+                    enums.ChatMemberStatus.MEMBER,
+                    enums.ChatMemberStatus.OWNER,
+                ]:
+                    raise UserNotParticipant()
+                else:
+                    logging.info("user %s check passed for group/channel %s.", chat_id, REQUIRED_MEMBERSHIP)
+            except UserNotParticipant:
+                logging.warning("user %s is not a member of group/channel %s", chat_id, REQUIRED_MEMBERSHIP)
+                message.reply_text(BotText.membership_require, quote=True)
+                return
+
+        return func(client, message)
+
+    return wrapper
+
+
+@app.on_message(filters.command(["start"]))
+def start_handler(client: Client, message: types.Message):
+    payment = Payment()
+    from_id = message.from_user.id
+    logging.info("%s welcome to youtube-dl bot!", message.from_user.id)
+    client.send_chat_action(from_id, enums.ChatAction.TYPING)
+    is_old_user = payment.check_old_user(from_id)
+    if is_old_user:
+        info = ""
+    if ENABLE_VIP:
+        free_token, pay_token, reset = payment.get_token(from_id)
+        info = f"Free token: {free_token}, Pay token: {pay_token}, Reset: {reset}"
+    else:
+        info = ""
+    text = f"{BotText.start}\n\n{info}\n{BotText.custom_text}"
+    client.send_message(message.chat.id, text, disable_web_page_preview=True)
+
+
+@app.on_message(filters.command(["help"]))
+def help_handler(client: Client, message: types.Message):
+    chat_id = message.chat.id
+    client.send_chat_action(chat_id, enums.ChatAction.TYPING)
+    client.send_message(chat_id, BotText.help, disable_web_page_preview=True)
+
+
+@app.on_message(filters.command(["about"]))
+def about_handler(client: Client, message: types.Message):
+    chat_id = message.chat.id
+    client.send_chat_action(chat_id, enums.ChatAction.TYPING)
+    client.send_message(chat_id, BotText.about)
+
+
+@app.on_message(filters.command(["sub"]))
+def subscribe_handler(client: Client, message: types.Message):
+    chat_id = message.chat.id
+    client.send_chat_action(chat_id, enums.ChatAction.TYPING)
+    if message.text == "/sub":
+        result = channel.get_user_subscription(chat_id)
+    else:
+        link = message.text.split()[1]
+        try:
+            result = channel.subscribe_channel(chat_id, link)
+        except (IndexError, ValueError):
+            result = f"Error: \n{traceback.format_exc()}"
+    client.send_message(chat_id, result or "You have no subscription.", disable_web_page_preview=True)
+
+
+@app.on_message(filters.command(["unsub"]))
+def unsubscribe_handler(client: Client, message: types.Message):
+    chat_id = message.chat.id
+    client.send_chat_action(chat_id, enums.ChatAction.TYPING)
+    text = message.text.split(" ")
+    if len(text) == 1:
+        client.send_message(chat_id, "/unsub channel_id", disable_web_page_preview=True)
+        return
+
+    rows = channel.unsubscribe_channel(chat_id, text[1])
+    if rows:
+        text = f"Unsubscribed from {text[1]}"
+    else:
+        text = "Unable to find the channel."
+    client.send_message(chat_id, text, disable_web_page_preview=True)
+
+
+@app.on_message(filters.command(["patch"]))
+def patch_handler(client: Client, message: types.Message):
+    username = message.from_user.username
+    chat_id = message.chat.id
+    if username == OWNER:
+        celery_app.control.broadcast("hot_patch")
+        client.send_chat_action(chat_id, enums.ChatAction.TYPING)
+        client.send_message(chat_id, "Oorah!")
+        hot_patch()
+
+
+@app.on_message(filters.command(["uncache"]))
+def uncache_handler(client: Client, message: types.Message):
+    username = message.from_user.username
+    link = message.text.split()[1]
+    if username == OWNER:
+        count = channel.del_cache(link)
+        message.reply_text(f"{count} cache(s) deleted.", quote=True)
+
+
+@app.on_message(filters.command(["purge"]))
+def purge_handler(client: Client, message: types.Message):
+    username = message.from_user.username
+    if username == OWNER:
+        message.reply_text(purge_tasks(), quote=True)
+
+
+@app.on_message(filters.command(["ping"]))
+def ping_handler(client: Client, message: types.Message):
+    redis = Redis()
+    chat_id = message.chat.id
+    client.send_chat_action(chat_id, enums.ChatAction.TYPING)
+    if os.uname().sysname == "Darwin" or ".heroku" in os.getenv("PYTHONHOME", ""):
+        bot_info = "ping unavailable."
+    else:
+        bot_info = get_runtime("ytdlbot_ytdl_1", "YouTube-dl")
+    if message.chat.username == OWNER:
+        stats = BotText.ping_worker()[:1000]
+        client.send_document(chat_id, redis.generate_file(), caption=f"{bot_info}\n\n{stats}")
+    else:
+        client.send_message(chat_id, f"{bot_info.split('CPU')[0]}")
+
+
+@app.on_message(filters.command(["sub_count"]))
+def sub_count_handler(client: Client, message: types.Message):
+    username = message.from_user.username
+    chat_id = message.chat.id
+    if username == OWNER:
+        with BytesIO() as f:
+            f.write(channel.sub_count().encode("u8"))
+            f.name = "subscription count.txt"
+            client.send_document(chat_id, f)
+
+
+@app.on_message(filters.command(["direct"]))
+def direct_handler(client: Client, message: types.Message):
+    redis = Redis()
+    chat_id = message.from_user.id
+    client.send_chat_action(chat_id, enums.ChatAction.TYPING)
+    url = re.sub(r"/direct\s*", "", message.text)
+    logging.info("direct start %s", url)
+    if not re.findall(r"^https?://", url.lower()):
+        redis.update_metrics("bad_request")
+        message.reply_text("Send me a DIRECT LINK.", quote=True)
+        return
+
+    bot_msg = message.reply_text("Request received.", quote=True)
+    redis.update_metrics("direct_request")
+    direct_download_entrance(client, bot_msg, url)
+
+
+@app.on_message(filters.command(["settings"]))
+def settings_handler(client: Client, message: types.Message):
+    chat_id = message.chat.id
+    payment = Payment()
+    client.send_chat_action(chat_id, enums.ChatAction.TYPING)
+    data = MySQL().get_user_settings(chat_id)
+    set_mode = data[-1]
+    text = {"Local": "Celery", "Celery": "Local"}.get(set_mode, "Local")
+    mode_text = f"Download mode: **{set_mode}**"
+    if message.chat.username == OWNER or payment.get_pay_token(chat_id):
+        extra = [types.InlineKeyboardButton(f"Change download mode to {text}", callback_data=text)]
+    else:
+        extra = []
+
+    markup = types.InlineKeyboardMarkup(
+        [
+            [  # First row
+                types.InlineKeyboardButton("send as document", callback_data="document"),
+                types.InlineKeyboardButton("send as video", callback_data="video"),
+                types.InlineKeyboardButton("send as audio", callback_data="audio"),
+            ],
+            [  # second row
+                types.InlineKeyboardButton("High Quality", callback_data="high"),
+                types.InlineKeyboardButton("Medium Quality", callback_data="medium"),
+                types.InlineKeyboardButton("Low Quality", callback_data="low"),
+            ],
+            extra,
+        ]
+    )
+
+    try:
+        client.send_message(chat_id, BotText.settings.format(data[1], data[2]) + mode_text, reply_markup=markup)
+    except:
+        client.send_message(
+            chat_id, BotText.settings.format(data[1] + ".", data[2] + ".") + mode_text, reply_markup=markup
+        )
+
+
+@app.on_message(filters.command(["buy"]))
+def buy_handler(client: Client, message: types.Message):
+    # process as chat.id, not from_user.id
+    chat_id = message.chat.id
+    client.send_chat_action(chat_id, enums.ChatAction.TYPING)
+    # currency USD
+    token_count = message.text.replace("/buy", "").strip()
+    if token_count.isdigit():
+        price = int(int(token_count) / TOKEN_PRICE * 100)
+    else:
+        price = 100
+
+    markup = types.InlineKeyboardMarkup(
+        [
+            [
+                types.InlineKeyboardButton("Bot Payments", callback_data=f"bot-payments-{price}"),
+                types.InlineKeyboardButton("TRON(TRX)", callback_data="tron-trx"),
+            ],
+        ]
+    )
+    client.send_message(chat_id, BotText.buy, disable_web_page_preview=True, reply_markup=markup)
+
+
+@app.on_callback_query(filters.regex(r"tron-trx"))
+def tronpayment_btn_calback(client: Client, callback_query: types.CallbackQuery):
+    callback_query.answer("Generating QR code...")
+    chat_id = callback_query.message.chat.id
+    client.send_chat_action(chat_id, enums.ChatAction.TYPING)
+
+    addr = TronTrx().get_payment_address(chat_id)
+    with BytesIO() as bio:
+        qr = qrcode.make(addr)
+        qr.save(bio)
+        client.send_photo(chat_id, bio, caption=f"Send any amount of TRX to `{addr}`")
+
+
+@app.on_callback_query(filters.regex(r"bot-payments-.*"))
+def bot_payment_btn_calback(client: Client, callback_query: types.CallbackQuery):
+    callback_query.answer("Generating invoice...")
+    chat_id = callback_query.message.chat.id
+    client.send_chat_action(chat_id, enums.ChatAction.TYPING)
+
+    data = callback_query.data
+    price = int(data.split("-")[-1])
+    payload = f"{chat_id}-buy"
+    invoice = generate_invoice(price, f"Buy {TOKEN_PRICE} download tokens", "Pay by card", payload)
+    app.invoke(
+        functions.messages.SendMedia(
+            peer=(raw_types.InputPeerUser(user_id=chat_id, access_hash=0)),
+            media=invoice,
+            random_id=app.rnd_id(),
+            message="Buy more download token",
+        )
+    )
+
+
+@app.on_message(filters.command(["redeem"]))
+def redeem_handler(client: Client, message: types.Message):
+    payment = Payment()
+    chat_id = message.chat.id
+    text = message.text.strip()
+    unique = text.replace("/redeem", "").strip()
+    msg = payment.verify_payment(chat_id, unique)
+    message.reply_text(msg, quote=True)
+
+
+def generate_invoice(amount: int, title: str, description: str, payload: str):
+    invoice = raw_types.input_media_invoice.InputMediaInvoice(
+        invoice=raw_types.invoice.Invoice(
+            currency="USD", prices=[raw_types.LabeledPrice(label="price", amount=amount)]
+        ),
+        title=title,
+        description=description,
+        provider=PROVIDER_TOKEN,
+        provider_data=raw_types.DataJSON(data="{}"),
+        payload=payload.encode(),
+        start_param=payload,
+    )
+    return invoice
+
+
+def link_checker(url: str) -> str:
+    if url.startswith("https://www.instagram.com"):
+        return ""
+    ytdl = yt_dlp.YoutubeDL()
+
+    if not PLAYLIST_SUPPORT and (
+        re.findall(r"^https://www\.youtube\.com/channel/", Channel.extract_canonical_link(url)) or "list" in url
+    ):
+        return "Playlist or channel links are disabled."
+
+    if not M3U8_SUPPORT and (re.findall(r"m3u8|\.m3u8|\.m3u$", url.lower())):
+        return "m3u8 links are disabled."
+
+    with contextlib.suppress(yt_dlp.utils.DownloadError):
+        if ytdl.extract_info(url, download=False).get("live_status") == "is_live":
+            return "Live stream links are disabled. Please download it after the stream ends."
+
+
+def search_ytb(kw: str):
+    videos_search = VideosSearch(kw, limit=10)
+    text = ""
+    results = videos_search.result()["result"]
+    for item in results:
+        title = item.get("title")
+        link = item.get("link")
+        index = results.index(item) + 1
+        text += f"{index}. {title}\n{link}\n\n"
+    return text
+
+
+@app.on_message(filters.incoming & (filters.text | filters.document))
+@private_use
+def download_handler(client: Client, message: types.Message):
+    redis = Redis()
+    payment = Payment()
+    chat_id = message.from_user.id
+    client.send_chat_action(chat_id, enums.ChatAction.TYPING)
+    redis.user_count(chat_id)
+    if message.document:
+        with tempfile.NamedTemporaryFile(mode="r+") as tf:
+            logging.info("Downloading file to %s", tf.name)
+            message.download(tf.name)
+            contents = open(tf.name, "r").read()  # don't know why
+        urls = contents.split()
+    else:
+        urls = [re.sub(r"/ytdl\s*", "", message.text)]
+        logging.info("start %s", urls)
+
+    for url in urls:
+        # check url
+        if not re.findall(r"^https?://", url.lower()):
+            redis.update_metrics("bad_request")
+            text = search_ytb(url)
+            message.reply_text(text, quote=True, disable_web_page_preview=True)
+            return
+
+        if text := link_checker(url):
+            message.reply_text(text, quote=True)
+            redis.update_metrics("reject_link_checker")
+            return
+
+        # old user is not limited by token
+        if ENABLE_VIP and not payment.check_old_user(chat_id):
+            free, pay, reset = payment.get_token(chat_id)
+            if free + pay <= 0:
+                message.reply_text(f"You don't have enough token. Please wait until {reset} or /buy .", quote=True)
+                redis.update_metrics("reject_token")
+                return
+            else:
+                payment.use_token(chat_id)
+
+        redis.update_metrics("video_request")
+
+        text = BotText.get_receive_link_text()
+        try:
+            # raise pyrogram.errors.exceptions.FloodWait(10)
+            bot_msg: types.Message | Any = message.reply_text(text, quote=True)
+        except pyrogram.errors.Flood as e:
+            f = BytesIO()
+            f.write(str(e).encode())
+            f.write(b"Your job will be done soon. Just wait! Don't rush.")
+            f.name = "Please don't flood me.txt"
+            bot_msg = message.reply_document(
+                f, caption=f"Flood wait! Please wait {e} seconds...." f"Your job will start automatically", quote=True
+            )
+            f.close()
+            client.send_message(OWNER, f"Flood wait! 🙁 {e} seconds....")
+            time.sleep(e.value)
+
+        client.send_chat_action(chat_id, enums.ChatAction.UPLOAD_VIDEO)
+        bot_msg.chat = message.chat
+        ytdl_download_entrance(client, bot_msg, url)
+
+
+@app.on_callback_query(filters.regex(r"document|video|audio"))
+def send_method_callback(client: Client, callback_query: types.CallbackQuery):
+    chat_id = callback_query.message.chat.id
+    data = callback_query.data
+    logging.info("Setting %s file type to %s", chat_id, data)
+    MySQL().set_user_settings(chat_id, "method", data)
+    callback_query.answer(f"Your send type was set to {callback_query.data}")
+
+
+@app.on_callback_query(filters.regex(r"high|medium|low"))
+def download_resolution_callback(client: Client, callback_query: types.CallbackQuery):
+    chat_id = callback_query.message.chat.id
+    data = callback_query.data
+    logging.info("Setting %s file type to %s", chat_id, data)
+    MySQL().set_user_settings(chat_id, "resolution", data)
+    callback_query.answer(f"Your default download quality was set to {callback_query.data}")
+
+
+@app.on_callback_query(filters.regex(r"convert"))
+def audio_callback(client: Client, callback_query: types.CallbackQuery):
+    redis = Redis()
+    if not ENABLE_FFMPEG:
+        callback_query.answer("Request rejected.")
+        callback_query.message.reply_text("Audio conversion is disabled now.")
+        return
+
+    callback_query.answer(f"Converting to audio...please wait patiently")
+    redis.update_metrics("audio_request")
+    audio_entrance(client, callback_query.message)
+
+
+@app.on_callback_query(filters.regex(r"Local|Celery"))
+def owner_local_callback(client: Client, callback_query: types.CallbackQuery):
+    chat_id = callback_query.message.chat.id
+    MySQL().set_user_settings(chat_id, "mode", callback_query.data)
+    callback_query.answer(f"Download mode was changed to {callback_query.data}")
+
+
+def periodic_sub_check():
+    exceptions = pyrogram.errors.exceptions
+    for cid, uids in channel.group_subscriber().items():
+        video_url = channel.has_newer_update(cid)
+        if video_url:
+            logging.info(f"periodic update:{video_url} - {uids}")
+            for uid in uids:
+                try:
+                    app.send_message(uid, f"{video_url} is out. Watch it on YouTube")
+                except (exceptions.bad_request_400.PeerIdInvalid, exceptions.bad_request_400.UserIsBlocked) as e:
+                    logging.warning("User is blocked or deleted. %s", e)
+                    channel.deactivate_user_subscription(uid)
+                except Exception:
+                    logging.error("Unknown error when sending message to user. %s", traceback.format_exc())
+                finally:
+                    time.sleep(random.random() * 3)
+
+
+@app.on_raw_update()
+def raw_update(client: Client, update, users, chats):
+    payment = Payment()
+    action = getattr(getattr(update, "message", None), "action", None)
+    if update.QUALNAME == "types.UpdateBotPrecheckoutQuery":
+        client.invoke(
+            functions.messages.SetBotPrecheckoutResults(
+                query_id=update.query_id,
+                success=True,
+            )
+        )
+    elif action and action.QUALNAME == "types.MessageActionPaymentSentMe":
+        logging.info("Payment received. %s", action)
+        uid = update.message.peer_id.user_id
+        amount = action.total_amount / 100
+        payment.add_pay_user([uid, amount, action.charge.provider_charge_id, 0, amount * TOKEN_PRICE])
+        client.send_message(uid, f"Thank you {uid}. Payment received: {amount} {action.currency}")
+
+
+def trx_notify(_, **kwargs):
+    user_id = kwargs.get("user_id")
+    text = kwargs.get("text")
+    logging.info("Sending trx notification to %s", user_id)
+    app.send_message(user_id, text)
+
+
+if __name__ == "__main__":
+    MySQL()
+    TRX_SIGNAL.connect(trx_notify)
+    scheduler = BackgroundScheduler(timezone="Europe/London", job_defaults={"max_instances": 6})
+    scheduler.add_job(auto_restart, "interval", seconds=600)
+    scheduler.add_job(clean_tempfile, "interval", seconds=120)
+    if not IS_BACKUP_BOT:
+        scheduler.add_job(Redis().reset_today, "cron", hour=0, minute=0)
+        scheduler.add_job(InfluxDB().collect_data, "interval", seconds=120)
+        scheduler.add_job(TronTrx().check_payment, "interval", seconds=60, max_instances=1)
+        #  default quota allocation of 10,000 units per day
+        scheduler.add_job(periodic_sub_check, "interval", seconds=3600)
+    scheduler.start()
+    banner = f"""
+▌ ▌         ▀▛▘     ▌       ▛▀▖              ▜            ▌
+▝▞  ▞▀▖ ▌ ▌  ▌  ▌ ▌ ▛▀▖ ▞▀▖ ▌ ▌ ▞▀▖ ▌  ▌ ▛▀▖ ▐  ▞▀▖ ▝▀▖ ▞▀▌
+ ▌  ▌ ▌ ▌ ▌  ▌  ▌ ▌ ▌ ▌ ▛▀  ▌ ▌ ▌ ▌ ▐▐▐  ▌ ▌ ▐  ▌ ▌ ▞▀▌ ▌ ▌
+ ▘  ▝▀  ▝▀▘  ▘  ▝▀▘ ▀▀  ▝▀▘ ▀▀  ▝▀   ▘▘  ▘ ▘  ▘ ▝▀  ▝▀▘ ▝▀▘
+
+By @BennyThink, VIP mode: {ENABLE_VIP}, Celery Mode: {ENABLE_CELERY}
+Version: {get_revision()}
+    """
+    print(banner)
+    app.run()