diff --git "a/test.ipynb" "b/test.ipynb" new file mode 100644--- /dev/null +++ "b/test.ipynb" @@ -0,0 +1,3386 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "from datetime import datetime\n", + "from tqdm import tqdm\n", + "\n", + "import time\n", + "import requests\n", + "import datetime\n", + "import pandas as pd\n", + "from collections import defaultdict\n", + "from typing import Any, Union\n", + "from string import Template\n", + "from enum import Enum\n", + "from tqdm import tqdm\n", + "import numpy as np\n", + "from pathlib import Path" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "tools = pd.read_parquet('/Users/arshath/play/openautonomy/olas-prediction-live-dashboard/data/tools.parquet')\n", + "# trades = pd.read_parquet('/Users/arshath/play/openautonomy/olas-prediction-live-dashboard/data/all_trades_profitability.parquet')\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "IRRELEVANT_TOOLS = [\n", + " \"openai-text-davinci-002\",\n", + " \"openai-text-davinci-003\",\n", + " \"openai-gpt-3.5-turbo\",\n", + " \"openai-gpt-4\",\n", + " \"stabilityai-stable-diffusion-v1-5\",\n", + " \"stabilityai-stable-diffusion-xl-beta-v2-2-2\",\n", + " \"stabilityai-stable-diffusion-512-v2-1\",\n", + " \"stabilityai-stable-diffusion-768-v2-1\",\n", + " \"deepmind-optimization-strong\",\n", + " \"deepmind-optimization\",\n", + "]\n", + "QUERY_BATCH_SIZE = 1000\n", + "DUST_THRESHOLD = 10000000000000\n", + "INVALID_ANSWER_HEX = (\n", + " \"0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\"\n", + ")\n", + "INVALID_ANSWER = -1\n", + "FPMM_CREATOR = \"0x89c5cc945dd550bcffb72fe42bff002429f46fec\"\n", + "DEFAULT_FROM_DATE = \"1970-01-01T00:00:00\"\n", + "DEFAULT_TO_DATE = \"2038-01-19T03:14:07\"\n", + "DEFAULT_FROM_TIMESTAMP = 0\n", + "DEFAULT_TO_TIMESTAMP = 2147483647\n", + "WXDAI_CONTRACT_ADDRESS = \"0xe91D153E0b41518A2Ce8Dd3D7944Fa863463a97d\"\n", + "DEFAULT_MECH_FEE = 0.01\n", + "DUST_THRESHOLD = 10000000000000\n", + "SCRIPTS_DIR = Path('/Users/arshath/play/openautonomy/olas-prediction-live-dashboard/scripts')\n", + "ROOT_DIR = SCRIPTS_DIR.parent\n", + "DATA_DIR = ROOT_DIR / \"data\"\n", + "\n", + "class MarketState(Enum):\n", + " \"\"\"Market state\"\"\"\n", + "\n", + " OPEN = 1\n", + " PENDING = 2\n", + " FINALIZING = 3\n", + " ARBITRATING = 4\n", + " CLOSED = 5\n", + "\n", + " def __str__(self) -> str:\n", + " \"\"\"Prints the market status.\"\"\"\n", + " return self.name.capitalize()\n", + "\n", + "\n", + "class MarketAttribute(Enum):\n", + " \"\"\"Attribute\"\"\"\n", + "\n", + " NUM_TRADES = \"Num_trades\"\n", + " WINNER_TRADES = \"Winner_trades\"\n", + " NUM_REDEEMED = \"Num_redeemed\"\n", + " INVESTMENT = \"Investment\"\n", + " FEES = \"Fees\"\n", + " MECH_CALLS = \"Mech_calls\"\n", + " MECH_FEES = \"Mech_fees\"\n", + " EARNINGS = \"Earnings\"\n", + " NET_EARNINGS = \"Net_earnings\"\n", + " REDEMPTIONS = \"Redemptions\"\n", + " ROI = \"ROI\"\n", + "\n", + " def __str__(self) -> str:\n", + " \"\"\"Prints the attribute.\"\"\"\n", + " return self.value\n", + "\n", + " def __repr__(self) -> str:\n", + " \"\"\"Prints the attribute representation.\"\"\"\n", + " return self.name\n", + "\n", + " @staticmethod\n", + " def argparse(s: str) -> \"MarketAttribute\":\n", + " \"\"\"Performs string conversion to MarketAttribute.\"\"\"\n", + " try:\n", + " return MarketAttribute[s.upper()]\n", + " except KeyError as e:\n", + " raise ValueError(f\"Invalid MarketAttribute: {s}\") from e\n", + "\n", + "\n", + "ALL_TRADES_STATS_DF_COLS = [\n", + " \"trader_address\",\n", + " \"trade_id\",\n", + " \"creation_timestamp\",\n", + " \"title\",\n", + " \"market_status\",\n", + " \"collateral_amount\",\n", + " \"outcome_index\",\n", + " \"trade_fee_amount\",\n", + " \"outcomes_tokens_traded\",\n", + " \"current_answer\",\n", + " \"is_invalid\",\n", + " \"winning_trade\",\n", + " \"earnings\",\n", + " \"redeemed\",\n", + " \"redeemed_amount\",\n", + " \"num_mech_calls\",\n", + " \"mech_fee_amount\",\n", + " \"net_earnings\",\n", + " \"roi\",\n", + "]\n", + "\n", + "SUMMARY_STATS_DF_COLS = [\n", + " \"trader_address\",\n", + " \"num_trades\",\n", + " \"num_winning_trades\",\n", + " \"num_redeemed\",\n", + " \"total_investment\",\n", + " \"total_trade_fees\",\n", + " \"num_mech_calls\",\n", + " \"total_mech_fees\",\n", + " \"total_earnings\",\n", + " \"total_redeemed_amount\",\n", + " \"total_net_earnings\",\n", + " \"total_net_earnings_wo_mech_fees\",\n", + " \"total_roi\",\n", + " \"total_roi_wo_mech_fees\",\n", + " \"mean_mech_calls_per_trade\",\n", + " \"mean_mech_fee_amount_per_trade\",\n", + "]\n", + "headers = {\n", + " \"Accept\": \"application/json, multipart/mixed\",\n", + " \"Content-Type\": \"application/json\",\n", + "}\n", + "\n", + "\n", + "omen_xdai_trades_query = Template(\n", + " \"\"\"\n", + " {\n", + " fpmmTrades(\n", + " where: {\n", + " type: Buy,\n", + " fpmm_: {\n", + " creator: \"${fpmm_creator}\"\n", + " creationTimestamp_gte: \"${fpmm_creationTimestamp_gte}\",\n", + " creationTimestamp_lt: \"${fpmm_creationTimestamp_lte}\"\n", + " },\n", + " creationTimestamp_gte: \"${creationTimestamp_gte}\",\n", + " creationTimestamp_lte: \"${creationTimestamp_lte}\"\n", + " id_gt: \"${id_gt}\"\n", + " }\n", + " first: ${first}\n", + " orderBy: id\n", + " orderDirection: asc\n", + " ) {\n", + " id\n", + " title\n", + " collateralToken\n", + " outcomeTokenMarginalPrice\n", + " oldOutcomeTokenMarginalPrice\n", + " type\n", + " creator {\n", + " id\n", + " }\n", + " creationTimestamp\n", + " collateralAmount\n", + " collateralAmountUSD\n", + " feeAmount\n", + " outcomeIndex\n", + " outcomeTokensTraded\n", + " transactionHash\n", + " fpmm {\n", + " id\n", + " outcomes\n", + " title\n", + " answerFinalizedTimestamp\n", + " currentAnswer\n", + " isPendingArbitration\n", + " arbitrationOccurred\n", + " openingTimestamp\n", + " condition {\n", + " id\n", + " }\n", + " }\n", + " }\n", + " }\n", + " \"\"\"\n", + ")\n", + "\n", + "\n", + "conditional_tokens_gc_user_query = Template(\n", + " \"\"\"\n", + " {\n", + " user(id: \"${id}\") {\n", + " userPositions(\n", + " first: ${first}\n", + " where: {\n", + " id_gt: \"${userPositions_id_gt}\"\n", + " }\n", + " orderBy: id\n", + " ) {\n", + " balance\n", + " id\n", + " position {\n", + " id\n", + " conditionIds\n", + " }\n", + " totalBalance\n", + " wrappedBalance\n", + " }\n", + " }\n", + " }\n", + " \"\"\"\n", + ")\n", + "\n", + "\n", + "def _to_content(q: str) -> dict[str, Any]:\n", + " \"\"\"Convert the given query string to payload content, i.e., add it under a `queries` key and convert it to bytes.\"\"\"\n", + " finalized_query = {\n", + " \"query\": q,\n", + " \"variables\": None,\n", + " \"extensions\": {\"headers\": None},\n", + " }\n", + " return finalized_query\n", + "\n", + "\n", + "def _query_omen_xdai_subgraph(\n", + " from_timestamp: float,\n", + " to_timestamp: float,\n", + " fpmm_from_timestamp: float,\n", + " fpmm_to_timestamp: float,\n", + ") -> dict[str, Any]:\n", + " \"\"\"Query the subgraph.\"\"\"\n", + " url = \"https://api.thegraph.com/subgraphs/name/protofire/omen-xdai\"\n", + "\n", + " grouped_results = defaultdict(list)\n", + " id_gt = \"\"\n", + "\n", + " while True:\n", + " query = omen_xdai_trades_query.substitute(\n", + " fpmm_creator=FPMM_CREATOR.lower(),\n", + " creationTimestamp_gte=int(from_timestamp),\n", + " creationTimestamp_lte=int(to_timestamp),\n", + " fpmm_creationTimestamp_gte=int(fpmm_from_timestamp),\n", + " fpmm_creationTimestamp_lte=int(fpmm_to_timestamp),\n", + " first=QUERY_BATCH_SIZE,\n", + " id_gt=id_gt,\n", + " )\n", + " content_json = _to_content(query)\n", + " res = requests.post(url, headers=headers, json=content_json)\n", + " result_json = res.json()\n", + " user_trades = result_json.get(\"data\", {}).get(\"fpmmTrades\", [])\n", + "\n", + " if not user_trades:\n", + " break\n", + "\n", + " for trade in user_trades:\n", + " fpmm_id = trade.get(\"fpmm\", {}).get(\"id\")\n", + " grouped_results[fpmm_id].append(trade)\n", + "\n", + " id_gt = user_trades[len(user_trades) - 1][\"id\"]\n", + "\n", + " all_results = {\n", + " \"data\": {\n", + " \"fpmmTrades\": [\n", + " trade\n", + " for trades_list in grouped_results.values()\n", + " for trade in trades_list\n", + " ]\n", + " }\n", + " }\n", + "\n", + " return all_results\n", + "\n", + "\n", + "def _query_conditional_tokens_gc_subgraph(creator: str) -> dict[str, Any]:\n", + " \"\"\"Query the subgraph.\"\"\"\n", + " url = \"https://api.thegraph.com/subgraphs/name/gnosis/conditional-tokens-gc\"\n", + "\n", + " all_results: dict[str, Any] = {\"data\": {\"user\": {\"userPositions\": []}}}\n", + " userPositions_id_gt = \"\"\n", + " while True:\n", + " query = conditional_tokens_gc_user_query.substitute(\n", + " id=creator.lower(),\n", + " first=QUERY_BATCH_SIZE,\n", + " userPositions_id_gt=userPositions_id_gt,\n", + " )\n", + " content_json = {\"query\": query}\n", + " res = requests.post(url, headers=headers, json=content_json)\n", + " result_json = res.json()\n", + " user_data = result_json.get(\"data\", {}).get(\"user\", {})\n", + "\n", + " if not user_data:\n", + " break\n", + "\n", + " user_positions = user_data.get(\"userPositions\", [])\n", + "\n", + " if user_positions:\n", + " all_results[\"data\"][\"user\"][\"userPositions\"].extend(user_positions)\n", + " userPositions_id_gt = user_positions[len(user_positions) - 1][\"id\"]\n", + " else:\n", + " break\n", + "\n", + " if len(all_results[\"data\"][\"user\"][\"userPositions\"]) == 0:\n", + " return {\"data\": {\"user\": None}}\n", + "\n", + " return all_results\n", + "\n", + "\n", + "def convert_hex_to_int(x: Union[str, float]) -> Union[int, float]:\n", + " \"\"\"Convert hex to int\"\"\"\n", + " if isinstance(x, float):\n", + " return np.nan\n", + " elif isinstance(x, str):\n", + " if x == INVALID_ANSWER_HEX:\n", + " return -1\n", + " else:\n", + " return int(x, 16)\n", + "\n", + "\n", + "def wei_to_unit(wei: int) -> float:\n", + " \"\"\"Converts wei to currency unit.\"\"\"\n", + " return wei / 10**18\n", + "\n", + "\n", + "def _is_redeemed(user_json: dict[str, Any], fpmmTrade: dict[str, Any]) -> bool:\n", + " \"\"\"Returns whether the user has redeemed the position.\"\"\"\n", + " user_positions = user_json[\"data\"][\"user\"][\"userPositions\"]\n", + " outcomes_tokens_traded = int(fpmmTrade[\"outcomeTokensTraded\"])\n", + " condition_id = fpmmTrade[\"fpmm.condition.id\"]\n", + "\n", + " for position in user_positions:\n", + " position_condition_ids = position[\"position\"][\"conditionIds\"]\n", + " balance = int(position[\"balance\"])\n", + "\n", + " if condition_id in position_condition_ids:\n", + " if balance == 0:\n", + " return True\n", + " # return early\n", + " return False\n", + " return False\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "def determine_market_status(trade, current_answer):\n", + " \"\"\"Determine the market status of a trade.\"\"\"\n", + " if current_answer is np.nan and time.time() >= int(trade[\"fpmm.openingTimestamp\"]):\n", + " return MarketState.PENDING\n", + " elif current_answer == np.nan:\n", + " return MarketState.OPEN\n", + " elif trade[\"fpmm.isPendingArbitration\"]:\n", + " return MarketState.ARBITRATING\n", + " elif time.time() < int(trade[\"fpmm.answerFinalizedTimestamp\"]):\n", + " return MarketState.FINALIZING\n", + " return MarketState.CLOSED" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Analysing creators: 0%| | 0/280 [00:00