new data and removing old info script

data/all_trades_profitability.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3068c54295d43c0d40f331cf3ad988fb8bf150bed0c948d3103161d7d7065f38
-size 3292156
+oid sha256:223f85e66279e8e12547e53f16efb0af7c9c902578b1cc529c878f7ee7379ce6
+size 3551233

data/fpmmTrades.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:59bd9c58e15de9dcb4ae76cd8adca7750b460abfa2bdf79ee5042d3e3b5c396e
-size 13934569
+oid sha256:4a55fcd7ac6fe8e8008fd8395edd39cc3ac0a05ab202f4296c22943f42f33470
+size 18283162

data/fpmms.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:16b0570a9c07e0ef5d137ede96584fcfe1645a784a7380a83b9bdfa5829ad3e2
-size 515347
+oid sha256:61ce94d0b622f72f695a244890bf91ae6286e793bd65cb5393a3d8bc810ff1a5
+size 525473

data/invalid_trades.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:18f4b47e3c764b8c7f157b4b408d0c97e3436f58d86eb39edecf2a7cf2748a21
-size 84033
+oid sha256:4400412e24b55e93d070b2a153e5a8c1a13f2f6f4c22c15eeb1aa7c2cce2e151
+size 101019

data/new_fpmmTrades.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:84442b8ab800d01ec66e0c78efebfc31a5d954d3fdddfaf5fab41a75030a3967
-size 3267040
+oid sha256:2179890c22497ae3572341fed5eb8b73bddc90f27c7b5c60037edd8170e74e51
+size 4542500

data/new_tools.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:67da51ef342b56056a5bfb49f78ecab7354731e2cd88d16ccbffeaa141e175ec
-size 64443733
+oid sha256:6afba2f5c6e14a81abbf1f936ddb0f83ada76c4515a6daf86fbc01ceba1a6e32
+size 79645959

data/outliers.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:72326e188a845663048e6ebf368045dfc387eac9a54a38303e9020f5ca112ad6
-size 18966
+oid sha256:494d9e9d0a20fcac4699dd4bf7bc658a50a124e8e33060997dd5eed0ce4863bc
+size 18711

data/summary_profitability.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:522003ca4b76df815bf662ebc92478bc103652ac9f82dc82718578c26c650509
-size 87497
+oid sha256:8ef6ed929a47113c2afac034356a094b8af089bb03f9e5a673c6cc2f27978120
+size 94924

data/t_map.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:69d6fcd0360c5bbd646fa748b3f5a1e4bcccae358f32c85aa96509cdb6319c76
-size 24153722
+oid sha256:ed3bcdd7a1c5ed48707a5fdb0dd16775d2ba347c7416fe6d4b5dc3a71dfea212
+size 25526067

data/tools.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aa6d6407da787ae9d2ed80233d939f57feae82cd66a8193937b861c601f24828
-size 406224765
+oid sha256:5e4acfa4aa1c8c80150269caf40fe6ff16e7aff3c90e2c4ca809d29357a6c36b
+size 448917044

data/tools_accuracy.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b0113d5cbc3c5eb981b5f1c5a7776f616fc52f44b15b5f96880a16989fa07d16
-size 1240
+oid sha256:174f3deae7eb082b6623e409771f17f538e94f74bb4fcfa62ea211c60ff3404a
+size 1339

scripts/cleaning_old_info.py

@@ -63,4 +63,4 @@ def clean_old_data_from_parquet_files(cutoff_date: str):
 
 
 if __name__ == "__main__":
-    clean_old_data_from_parquet_files("2024-09-15")
+    clean_old_data_from_parquet_files("2024-09-22")

scripts/get_mech_info.py

@@ -330,6 +330,8 @@ def get_mech_events_since_last_run():
         to_block=last_block_number,
         filename="new_mech_delivers.json",
     )
+    if delivers_dict is None:
+        return None
     # clean delivers
     clean_mech_delivers("new_mech_requests.json", "new_mech_delivers.json")
 

scripts/mech_request_utils.py

@@ -21,6 +21,7 @@
 import json
 import time
 import pickle
+from random import uniform
 from typing import Any, Dict, List, Tuple
 from pathlib import Path
 import requests
@@ -124,9 +125,10 @@ def collect_all_mech_requests(from_block: int, to_block: int, filename: str) ->
             "blockNumber_lte": str(to_block),  # str
         }
         try:
-            response = client.execute(
-                gql(REQUESTS_QUERY_FILTER), variable_values=variables
-            )
+            # response = client.execute(
+            #     gql(REQUESTS_QUERY_FILTER), variable_values=variables
+            # )
+            response = fetch_with_retry(client, REQUESTS_QUERY_FILTER, variables)
 
             items = response.get("requests", [])
 
@@ -154,10 +156,21 @@ def collect_all_mech_requests(from_block: int, to_block: int, filename: str) ->
     return mech_requests, duplicated_reqIds
 
 
+def fetch_with_retry(client, query, variables, max_retries=5):
+    for attempt in range(max_retries):
+        try:
+            return client.execute(gql(query), variable_values=variables)
+        except Exception as e:
+            if attempt == max_retries - 1:
+                raise e
+            wait_time = (2**attempt) + uniform(0, 1)  # exponential backoff with jitter
+            time.sleep(wait_time)
+
+
 def collect_all_mech_delivers(from_block: int, to_block: int, filename: str) -> Tuple:
 
     print(f"Fetching all mech delivers from {from_block} to {to_block}")
-    # TODO save as new json file, check how to merge later json files
+
     mech_delivers = {}
     duplicated_requestIds = []
     transport = RequestsHTTPTransport(url=THEGRAPH_ENDPOINT)
@@ -173,9 +186,10 @@ def collect_all_mech_delivers(from_block: int, to_block: int, filename: str) ->
             "blockNumber_lte": str(to_block),  # str
         }
         try:
-            response = client.execute(
-                gql(DELIVERS_QUERY_NO_FILTER), variable_values=variables
-            )
+            # response = client.execute(
+            #     gql(DELIVERS_QUERY_NO_FILTER), variable_values=variables
+            # )
+            response = fetch_with_retry(client, DELIVERS_QUERY_NO_FILTER, variables)
             items = response.get("delivers", [])
 
             if not items:
@@ -187,10 +201,9 @@ def collect_all_mech_delivers(from_block: int, to_block: int, filename: str) ->
                 else:
                     duplicated_requestIds.append(mech_deliver["requestId"])
                     # we will handle the duplicated later
-                    mech_delivers[mech_deliver["requestId"]].append(mech_deliver)
         except Exception as e:
             print(f"Error while getting the response: {e}")
-            return
+            return None, None
 
         id_gt = items[-1]["id"]
         time.sleep(IPFS_POLL_INTERVAL)
@@ -219,9 +232,7 @@ def collect_missing_delivers(request_id: int, block_number: int) -> Dict[str, An
         "blockNumber_lte": str(to_block),  # str
     }
     try:
-        response = client.execute(
-            gql(MISSING_DELIVERS_QUERY), variable_values=variables
-        )
+        response = fetch_with_retry(client, MISSING_DELIVERS_QUERY, variables)
         items = response.get("delivers", [])
         # If the user sends requests with the same values (tool, prompt, nonce) it
         # will generate the same requestId. Therefore, multiple items can be retrieved

scripts/pull_data.py

@@ -21,6 +21,7 @@ from get_mech_info import (
     update_json_files,
 )
 from update_tools_accuracy import compute_tools_accuracy
+from cleaning_old_info import clean_old_data_from_parquet_files
 import gc
 
 logging.basicConfig(level=logging.INFO)
@@ -131,6 +132,9 @@ def only_new_weekly_analysis():
     logging.info("Generating the mech json files")
     # get only new data
     latest_timestamp = get_mech_events_since_last_run()
+    if latest_timestamp == None:
+        print("Error while getting the mech events")
+        return
     logging.info(f"Finished generating the mech json files from {latest_timestamp}")
 
     # Run tools ETL
@@ -162,6 +166,8 @@ def only_new_weekly_analysis():
         logging.error("Error while updating timestamps of tools")
         print(e)
 
+    clean_old_data_from_parquet_files("2024-09-22")
+
     compute_tools_accuracy()
 
     logging.info("Weekly analysis files generated and saved")

tabs/trades.py

@@ -105,7 +105,7 @@ def get_overall_winning_by_market_and_trader_type(
 
 
 def plot_trades_by_week(trades_df: pd.DataFrame) -> gr.BarPlot:
-    """Plots the trades data for the given tools and calculates the winning percentage."""
+    """Plots the weekly trades data ."""
     return gr.BarPlot(
         value=trades_df,
         x="month_year_week",