new weekly data and historical folder

data/all_trades_profitability.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:52f72088df284d5addddb5d6dd3e2226c8fc98c58b9048c0cf145d52016da783
-size 3136886
+oid sha256:a68b38ee2dbaf6add318f6637d921f991641b1f2e1c6b91b34e4848c30d41bd1
+size 3225038

data/daily_info.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aa46e26ddf79ae4565e8572d489d377884e071f20eb19ea3c8d99684c2c00548
-size 611323
+oid sha256:195d5d8c5252f2c663eb5226e70074515124ac3dce174fbc153eb7dd85b2ba00
+size 499139

data/fpmmTrades.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:82b8e872c454048229fd0fc85ddd3b848d9d52b6e336113f239b8ed90d745141
-size 19295640
+oid sha256:6e3791d1bd7596439089a61667b72f5054b5086016af69d43996a2d8d04cacff
+size 21345678

data/fpmms.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bcee66dc9b6b8f6be7e8cd33f57344f126af01c503c9e08f05d571b0157109f5
-size 529819
+oid sha256:6fdcaf956aa1fecb53fbb5b34bb7c3b200c4e3df14477c82392c73acfab44e47
+size 536235

data/new_fpmmTrades.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f8bf44ce187a57486f821f934fbb8f6676b14b4c4a1a1d25806c5cf6255614aa
-size 2265950
+oid sha256:65482c09468f61f676d0fddddc4afe6818fedb5df55d37b5b292693be5ce657e
+size 1907525

data/new_tools.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6b4d38de15b4da119a9706cc5addd45f1979bac11eac70442f45155805a9d5bc
-size 25083815
+oid sha256:1e2bd4aacc316286d95b7c2d304aa22fa56a53f6bf586a7836954ad5d41b0f8a
+size 24727423

data/summary_profitability.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3047c4a75c6c90fb8132d625f7501d979ebb1fb5711ede766fa66009c6dad5e3
-size 92478
+oid sha256:cee5c37645de66824cb74faaa07bae45673086664518f6fd127afe9ba5a74a65
+size 93516

data/tools.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:87d50073d2f75f1a5f09353ac46747180046ade9be74cf8bb52167e30da2085d
-size 446483909
+oid sha256:255f264c677007a32fbba217dd4a7835290732dbf46502eaec66a861930c639a
+size 455022141

data/tools_accuracy.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c6c7f4a8a992798d4920949a1e1d839474512a6b291b3fe19138d8f921574253
-size 1325
+oid sha256:b3a0887662471990e6e43c7ef0ba0f1734dd1a81270e0b6c3af2ab3474838fd8
+size 1339

historical_data/all_trades_profitability_20241129_153007.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a68b38ee2dbaf6add318f6637d921f991641b1f2e1c6b91b34e4848c30d41bd1
+size 3225038

historical_data/all_trades_profitability_20241202_091215.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a68b38ee2dbaf6add318f6637d921f991641b1f2e1c6b91b34e4848c30d41bd1
+size 3225038

historical_data/tools_20241129_153007.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf99c5cd3d48c602d269f81962f9c0b8f08cf7d604f40d1a6af7463086ac5b25
+size 451439409

historical_data/tools_20241202_091215.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e99cd1d2db2dcd649ce1eabda3c2fd61162bad2afba71fa28070691e5058299d
+size 455141895

scripts/cleaning_old_info.py

@@ -1,7 +1,4 @@
-import requests
-import json
 import pandas as pd
-import numpy as np
 from profitability import DATA_DIR, summary_analyse
 from staking import label_trades_by_staking
 

scripts/profitability.py

@@ -599,16 +599,22 @@ def run_profitability_analysis(
     all_trades_df.to_parquet(JSON_DATA_DIR / "all_trades_df.parquet")
     # filter invalid markets. Condition: "is_invalid" is True
     invalid_trades = all_trades_df.loc[all_trades_df["is_invalid"] == True]
-    if merge:
-        try:
-            old_invalid_trades = pd.read_parquet(DATA_DIR / "invalid_trades.parquet")
-            merge_df = pd.concat(
-                [old_invalid_trades, invalid_trades], ignore_index=True
-            )
-            invalid_trades = merge_df.drop_duplicates()
-        except Exception as e:
-            print(f"Error updating the invalid trades parquet {e}")
-    invalid_trades.to_parquet(DATA_DIR / "invalid_trades.parquet", index=False)
+    if len(invalid_trades) == 0:
+        print("No new invalid trades")
+    else:
+        if merge:
+            try:
+                print("Merging invalid trades parquet file")
+                old_invalid_trades = pd.read_parquet(
+                    DATA_DIR / "invalid_trades.parquet"
+                )
+                merge_df = pd.concat(
+                    [old_invalid_trades, invalid_trades], ignore_index=True
+                )
+                invalid_trades = merge_df.drop_duplicates()
+            except Exception as e:
+                print(f"Error updating the invalid trades parquet {e}")
+        invalid_trades.to_parquet(DATA_DIR / "invalid_trades.parquet", index=False)
 
     all_trades_df = all_trades_df.loc[all_trades_df["is_invalid"] == False]