First decent attempt, seems to generate valid queries after pre-training. Needs evaluation still

fine-tuned-model/{checkpoint-236 → checkpoint-295}/adapter_config.json

@@ -24,10 +24,10 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "q_proj",
     "k_proj",
-    "v_proj",
-    "o_proj"
+    "q_proj",
+    "o_proj",
+    "v_proj"
   ],
   "task_type": "CAUSAL_LM",
   "trainable_token_indices": null,

fine-tuned-model/{checkpoint-236 → checkpoint-295}/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c1675aadfd9bd5995a73de58148ef251da4d532994321a4944157eb47e23efe1
+oid sha256:5dbc52cfab26c0dfe8ab09edbd2f3c33aa892b2ea1e668a652968307cf887d90
 size 25191536

fine-tuned-model/{checkpoint-236 → checkpoint-295}/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4154204936127b67041d1fa14b3b47592d28098e3af1fbc90ccb186d4b7aa920
+oid sha256:bc34cbc76e28ac17c0afc1333f5d7b30a8beebce84a6f3e73589b31ff00733f2
 size 50492858

fine-tuned-model/{checkpoint-236 → checkpoint-295}/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9594002d0ce67a88462882fb88058ad890c8624e05cfe9253cbfdb98dbcea4d7
+oid sha256:47500e947b0979f4187ced89844b0b41af88c14cc3ed27ad8cb01fdb1072cb88
 size 14244

fine-tuned-model/{checkpoint-236 → checkpoint-295}/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:713b6197e1c6fd2587869ada366ebb7d57d251ebefa1a0bc3b17bf2ed26a407b
+oid sha256:a0cac5ada32db5a3f9944a8e52bf38c14646b18fcd0190b4fb8155b619d8f5ab
 size 988

fine-tuned-model/{checkpoint-236 → checkpoint-295}/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:babe16327608a66b40524b5a3c6405ef24c518e11abf91e1eb89f529de80b3db
+oid sha256:4ee2a611a1529c905642325059ae97e660232f5907273b45d09703eb7bc5fa03
 size 1064

fine-tuned-model/{checkpoint-236 → checkpoint-295}/trainer_state.json

@@ -2,62 +2,93 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 2.0,
+  "epoch": 5.0,
   "eval_steps": 500,
-  "global_step": 236,
+  "global_step": 295,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.423728813559322,
-      "grad_norm": 1.3252296447753906,
-      "learning_rate": 0.0004307909604519774,
-      "loss": 3.0243,
+      "epoch": 0.847457627118644,
+      "grad_norm": 3.6365857124328613,
+      "learning_rate": 4.152542372881356e-05,
+      "loss": 8.5103,
       "step": 50
     },
     {
-      "epoch": 0.847457627118644,
-      "grad_norm": 1.302614450454712,
-      "learning_rate": 0.00036016949152542374,
-      "loss": 1.2356,
+      "epoch": 1.0,
+      "eval_loss": 1.4866021871566772,
+      "eval_runtime": 5.4305,
+      "eval_samples_per_second": 19.335,
+      "eval_steps_per_second": 1.289,
+      "step": 59
+    },
+    {
+      "epoch": 1.694915254237288,
+      "grad_norm": 3.137465715408325,
+      "learning_rate": 3.305084745762712e-05,
+      "loss": 1.7098,
       "step": 100
     },
     {
-      "epoch": 1.0,
-      "eval_loss": 0.9709166288375854,
-      "eval_runtime": 5.194,
-      "eval_samples_per_second": 20.216,
-      "eval_steps_per_second": 2.695,
+      "epoch": 2.0,
+      "eval_loss": 1.2273037433624268,
+      "eval_runtime": 5.362,
+      "eval_samples_per_second": 19.582,
+      "eval_steps_per_second": 1.305,
       "step": 118
     },
     {
-      "epoch": 1.271186440677966,
-      "grad_norm": 0.6076271533966064,
-      "learning_rate": 0.0002895480225988701,
-      "loss": 1.2005,
+      "epoch": 2.542372881355932,
+      "grad_norm": 1.6243258714675903,
+      "learning_rate": 2.457627118644068e-05,
+      "loss": 1.5421,
       "step": 150
     },
     {
-      "epoch": 1.694915254237288,
-      "grad_norm": 1.1516226530075073,
-      "learning_rate": 0.0002189265536723164,
-      "loss": 1.2331,
+      "epoch": 3.0,
+      "eval_loss": 1.1611202955245972,
+      "eval_runtime": 5.348,
+      "eval_samples_per_second": 19.634,
+      "eval_steps_per_second": 1.309,
+      "step": 177
+    },
+    {
+      "epoch": 3.389830508474576,
+      "grad_norm": 1.7812302112579346,
+      "learning_rate": 1.6101694915254237e-05,
+      "loss": 1.4875,
       "step": 200
     },
     {
-      "epoch": 2.0,
-      "eval_loss": 0.9370157718658447,
-      "eval_runtime": 5.1425,
-      "eval_samples_per_second": 20.418,
-      "eval_steps_per_second": 2.722,
+      "epoch": 4.0,
+      "eval_loss": 1.153254508972168,
+      "eval_runtime": 5.347,
+      "eval_samples_per_second": 19.637,
+      "eval_steps_per_second": 1.309,
       "step": 236
+    },
+    {
+      "epoch": 4.237288135593221,
+      "grad_norm": 2.1582489013671875,
+      "learning_rate": 7.627118644067798e-06,
+      "loss": 1.3883,
+      "step": 250
+    },
+    {
+      "epoch": 5.0,
+      "eval_loss": 1.1216797828674316,
+      "eval_runtime": 5.3095,
+      "eval_samples_per_second": 19.776,
+      "eval_steps_per_second": 1.318,
+      "step": 295
     }
   ],
   "logging_steps": 50,
-  "max_steps": 354,
+  "max_steps": 295,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 3,
+  "num_train_epochs": 5,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -66,13 +97,13 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 3711634067423232.0,
-  "train_batch_size": 8,
+  "total_flos": 9279085168558080.0,
+  "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null
 }

fine-tuned-model/{checkpoint-236 → checkpoint-295}/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:067bd988483e44c78bdd5214e17314d730fc21ddb68a86665b5964dfa1dff3cf
+oid sha256:145881f21d543612154f9ea67fda44fa38754eb20bd4eb13e1492be30ee670d7
 size 5368

fine-tuned-model/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d7d5c722eaa522148cdd50895d0b066e1cdeb624bf1cf7c77f0de1b647d74ad4
+oid sha256:9062451f577293bdd513174888e96a053a05c6f763eabe4c1196ad82987caf2c
 size 1480793144

fine-tuned-model/runs/Apr03_12-32-10_DESKTOP-SMJC97K/events.out.tfevents.1743708731.DESKTOP-SMJC97K.4788.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e13d3e6b9cb02db1498428eecc437b77881de7829603019b54d53fcc6c0a9c7
+size 5656

fine-tuned-model/runs/Apr03_12-35-47_DESKTOP-SMJC97K/events.out.tfevents.1743708948.DESKTOP-SMJC97K.21476.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5d2c6a4697d12bb25ba105c3c95241aa9d53ab1ea6c5a0116ec7a5416885586
+size 8402

fine-tuned-model/runs/Apr03_14-34-10_DESKTOP-SMJC97K/events.out.tfevents.1743716051.DESKTOP-SMJC97K.9248.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1c85607a6a7da30d998111f662d1777c8550adb9a1d046bfae143ea1947ce2a
+size 8402

finetune_model.ipynb

@@ -251,13 +251,396 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Map: 100%|██████████| 1044/1044 [00:01<00:00, 543.61 examples/s]"
+      "Map:   0%|          | 0/1044 [00:00<?, ? examples/s]"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
+      "You are an AI assistant that converts natural language queries into valid SQLite queries.\n",
+      "Database Schema and Explanations\n",
+      "\n",
+      "team Table\n",
+      "Stores information about NBA teams.\n",
+      "CREATE TABLE IF NOT EXISTS \"team\" (\n",
+      "  \"id\" TEXT PRIMARY KEY,      -- Unique identifier for the team\n",
+      "  \"full_name\" TEXT,           -- Full official name of the team (e.g., \"Los Angeles Lakers\")\n",
+      "  \"abbreviation\" TEXT,        -- Shortened team name (e.g., \"LAL\")\n",
+      "  \"nickname\" TEXT,            -- Commonly used nickname for the team (e.g., \"Lakers\")\n",
+      "  \"city\" TEXT,                -- City where the team is based\n",
+      "  \"state\" TEXT,               -- State where the team is located\n",
+      "  \"year_founded\" REAL         -- Year the team was established\n",
+      ");\n",
+      "\n",
+      "game Table\n",
+      "Contains detailed statistics for each NBA game, including home and away team performance.\n",
+      "CREATE TABLE IF NOT EXISTS \"game\" (\n",
+      "  \"season_id\" TEXT,            -- Season identifier, formatted as \"2YYYY\" (e.g., \"21970\" for the 1970 season)\n",
+      "  \"team_id_home\" TEXT,         -- ID of the home team (matches \"id\" in team table)\n",
+      "  \"team_abbreviation_home\" TEXT, -- Abbreviation of the home team\n",
+      "  \"team_name_home\" TEXT,       -- Full name of the home team\n",
+      "  \"game_id\" TEXT PRIMARY KEY,  -- Unique identifier for the game\n",
+      "  \"game_date\" TIMESTAMP,       -- Date the game was played (YYYY-MM-DD format)\n",
+      "  \"matchup_home\" TEXT,         -- Matchup details including opponent (e.g., \"LAL vs. BOS\")\n",
+      "  \"wl_home\" TEXT,              -- \"W\" if the home team won, \"L\" if they lost\n",
+      "  \"min\" INTEGER,               -- Total minutes played in the game\n",
+      "  \"fgm_home\" REAL,             -- Field goals made by the home team\n",
+      "  \"fga_home\" REAL,             -- Field goals attempted by the home team\n",
+      "  \"fg_pct_home\" REAL,          -- Field goal percentage of the home team\n",
+      "  \"fg3m_home\" REAL,            -- Three-point field goals made by the home team\n",
+      "  \"fg3a_home\" REAL,            -- Three-point attempts by the home team\n",
+      "  \"fg3_pct_home\" REAL,         -- Three-point field goal percentage of the home team\n",
+      "  \"ftm_home\" REAL,             -- Free throws made by the home team\n",
+      "  \"fta_home\" REAL,             -- Free throws attempted by the home team\n",
+      "  \"ft_pct_home\" REAL,          -- Free throw percentage of the home team\n",
+      "  \"oreb_home\" REAL,            -- Offensive rebounds by the home team\n",
+      "  \"dreb_home\" REAL,            -- Defensive rebounds by the home team\n",
+      "  \"reb_home\" REAL,             -- Total rebounds by the home team\n",
+      "  \"ast_home\" REAL,             -- Assists by the home team\n",
+      "  \"stl_home\" REAL,             -- Steals by the home team\n",
+      "  \"blk_home\" REAL,             -- Blocks by the home team\n",
+      "  \"tov_home\" REAL,             -- Turnovers by the home team\n",
+      "  \"pf_home\" REAL,              -- Personal fouls by the home team\n",
+      "  \"pts_home\" REAL,             -- Total points scored by the home team\n",
+      "  \"plus_minus_home\" INTEGER,   -- Plus/minus rating for the home team\n",
+      "  \"video_available_home\" INTEGER, -- Indicates whether video is available (1 = Yes, 0 = No)\n",
+      "  \"team_id_away\" TEXT,         -- ID of the away team\n",
+      "  \"team_abbreviation_away\" TEXT, -- Abbreviation of the away team\n",
+      "  \"team_name_away\" TEXT,       -- Full name of the away team\n",
+      "  \"matchup_away\" TEXT,         -- Matchup details from the away team’s perspective\n",
+      "  \"wl_away\" TEXT,              -- \"W\" if the away team won, \"L\" if they lost\n",
+      "  \"fgm_away\" REAL,             -- Field goals made by the away team\n",
+      "  \"fga_away\" REAL,             -- Field goals attempted by the away team\n",
+      "  \"fg_pct_away\" REAL,          -- Field goal percentage of the away team\n",
+      "  \"fg3m_away\" REAL,            -- Three-point field goals made by the away team\n",
+      "  \"fg3a_away\" REAL,            -- Three-point attempts by the away team\n",
+      "  \"fg3_pct_away\" REAL,         -- Three-point field goal percentage of the away team\n",
+      "  \"ftm_away\" REAL,             -- Free throws made by the away team\n",
+      "  \"fta_away\" REAL,             -- Free throws attempted by the away team\n",
+      "  \"ft_pct_away\" REAL,          -- Free throw percentage of the away team\n",
+      "  \"oreb_away\" REAL,            -- Offensive rebounds by the away team\n",
+      "  \"dreb_away\" REAL,            -- Defensive rebounds by the away team\n",
+      "  \"reb_away\" REAL,             -- Total rebounds by the away team\n",
+      "  \"ast_away\" REAL,             -- Assists by the away team\n",
+      "  \"stl_away\" REAL,             -- Steals by the away team\n",
+      "  \"blk_away\" REAL,             -- Blocks by the away team\n",
+      "  \"tov_away\" REAL,             -- Turnovers by the away team\n",
+      "  \"pf_away\" REAL,              -- Personal fouls by the away team\n",
+      "  \"pts_away\" REAL,             -- Total points scored by the away team\n",
+      "  \"plus_minus_away\" INTEGER,   -- Plus/minus rating for the away team\n",
+      "  \"video_available_away\" INTEGER, -- Indicates whether video is available (1 = Yes, 0 = No)\n",
+      "  \"season_type\" TEXT           -- Regular season or playoffs\n",
+      ");\n",
+      "\n",
+      "other_stats Table\n",
+      "Stores additional statistics, linked to the game table via game_id.\n",
+      "CREATE TABLE IF NOT EXISTS \"other_stats\" (\n",
+      "  \"game_id\" TEXT,             -- Unique game identifier, matches id column from game table\n",
+      "  \"league_id\" TEXT,           -- League identifier\n",
+      "  \"team_id_home\" TEXT,        -- Home team identifier\n",
+      "  \"team_abbreviation_home\" TEXT, -- Home team abbreviation\n",
+      "  \"team_city_home\" TEXT,      -- Home team city\n",
+      "  \"pts_paint_home\" INTEGER,   -- Points in the paint by the home team\n",
+      "  \"pts_2nd_chance_home\" INTEGER, -- Second chance points by the home team\n",
+      "  \"pts_fb_home\" INTEGER,      -- Fast break points by the home team\n",
+      "  \"largest_lead_home\" INTEGER,-- Largest lead by the home team\n",
+      "  \"lead_changes\" INTEGER,     -- Number of lead changes \n",
+      "  \"times_tied\" INTEGER,       -- Number of times the score was tied\n",
+      "  \"team_turnovers_home\" INTEGER, -- Home team turnovers\n",
+      "  \"total_turnovers_home\" INTEGER, -- Total turnovers by the home team\n",
+      "  \"team_rebounds_home\" INTEGER, -- Home team rebounds\n",
+      "  \"pts_off_to_home\" INTEGER,  -- Points off turnovers by the home team\n",
+      "  \"team_id_away\" TEXT,        -- Away team identifier\n",
+      "  \"team_abbreviation_away\" TEXT,  -- Away team abbreviation\n",
+      "  \"pts_paint_away\" INTEGER,   -- Points in the paint by the away team\n",
+      "  \"pts_2nd_chance_away\" INTEGER, -- Second chance points by the away team\n",
+      "  \"pts_fb_away\" INTEGER,      -- Fast break points by the away team\n",
+      "  \"largest_lead_away\" INTEGER,-- Largest lead by the away team\n",
+      "  \"team_turnovers_away\" INTEGER, -- Away team turnovers\n",
+      "  \"total_turnovers_away\" INTEGER, -- Total turnovers by the away team\n",
+      "  \"team_rebounds_away\" INTEGER, -- Away team rebounds\n",
+      "  \"pts_off_to_away\" INTEGER   -- Points off turnovers by the away team\n",
+      ");\n",
+      "\n",
+      "\n",
+      "Team Name Information\n",
+      "In the plaintext user questions, only the full team names will be used, but in the queries you may use the full team names or the abbreviations. \n",
+      "The full team names can be used with the game table, while the abbreviations should be used with the other_stats table.\n",
+      "Notice they are separated by the | character in the following list:\n",
+      "\n",
+      "Atlanta Hawks|ATL\n",
+      "Boston Celtics|BOS\n",
+      "Cleveland Cavaliers|CLE\n",
+      "New Orleans Pelicans|NOP\n",
+      "Chicago Bulls|CHI\n",
+      "Dallas Mavericks|DAL\n",
+      "Denver Nuggets|DEN\n",
+      "Golden State Warriors|GSW\n",
+      "Houston Rockets|HOU\n",
+      "Los Angeles Clippers|LAC\n",
+      "Los Angeles Lakers|LAL\n",
+      "Miami Heat|MIA\n",
+      "Milwaukee Bucks|MIL\n",
+      "Minnesota Timberwolves|MIN\n",
+      "Brooklyn Nets|BKN\n",
+      "New York Knicks|NYK\n",
+      "Orlando Magic|ORL\n",
+      "Indiana Pacers|IND\n",
+      "Philadelphia 76ers|PHI\n",
+      "Phoenix Suns|PHX\n",
+      "Portland Trail Blazers|POR\n",
+      "Sacramento Kings|SAC\n",
+      "San Antonio Spurs|SAS\n",
+      "Oklahoma City Thunder|OKC\n",
+      "Toronto Raptors|TOR\n",
+      "Utah Jazz|UTA\n",
+      "Memphis Grizzlies|MEM\n",
+      "Washington Wizards|WAS\n",
+      "Detroit Pistons|DET\n",
+      "Charlotte Hornets|CHA\n",
+      "\n",
+      "Query Guidelines\n",
+      "Use team_name_home and team_name_away to match teams to the game table. Use team_abbreviation_home and team_abbreviation away to match teams to the other_stats table.\n",
+      "\n",
+      "To filter by season, use season_id = '2YYYY'.\n",
+      "\n",
+      "Example: To get statistics from 2005, use a statement like: season_id = '22005'. To get statistics from 1972, use a statement like: season_id = \"21972\". To get statistics from 2015, use a statement like: season_id = \"22015\".\n",
+      "\n",
+      "Ensure queries return relevant columns and avoid unnecessary joins.\n",
+      "\n",
+      "Example User Requests and SQLite Queries\n",
+      "Request:\n",
+      "\"What is the most points the Los Angeles Lakers have ever scored at home?\"\n",
+      "SQLite:\n",
+      "SELECT MAX(pts_home) \n",
+      "FROM game \n",
+      "WHERE team_name_home = 'Los Angeles Lakers';\n",
+      "\n",
+      "Request:\n",
+      "\"Which teams are located in the state of California?\"\n",
+      "SQLite:\n",
+      "SELECT full_name FROM team WHERE state = 'California';\n",
+      "\n",
+      "Request:\n",
+      "\"Which team had the highest number of team turnovers in an away game?\"\n",
+      "SQLite:\n",
+      "SELECT team_abbreviation_away FROM other_stats ORDER BY team_turnovers_away DESC LIMIT 1;\n",
+      "\n",
+      "Request:\n",
+      "\"Which teams were founded before 1979?\"\n",
+      "SQLite:\n",
+      "SELECT full_name FROM team WHERE year_founded < 1979;\n",
+      "\n",
+      "Request:\n",
+      "\"Find the Boston Celtics largest home victory margin in the 2008 season.\"\n",
+      "SQLite:\n",
+      "SELECT MAX(pts_home - pts_away) AS biggest_win\n",
+      "FROM game\n",
+      "WHERE team_name_home = 'Boston Celtics' AND season_id = '22008';\n",
+      "\n",
+      "Generate only the SQLite query prefaced by SQLite: and no other text, do not output an explanation of the query. Now generate an SQLite query for the following user request. Request:\n",
+      "Which NBA teams were established after the year 2000? List their names and founding years, sorted from newest to oldest\n",
+      "SQLite: \n",
+      "SELECT full_name FROM team WHERE year_founded > 2000 ORDER BY year_founded DESC;\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Map: 100%|██████████| 1044/1044 [00:01<00:00, 552.67 examples/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "You are an AI assistant that converts natural language queries into valid SQLite queries.\n",
+      "Database Schema and Explanations\n",
+      "\n",
+      "team Table\n",
+      "Stores information about NBA teams.\n",
+      "CREATE TABLE IF NOT EXISTS \"team\" (\n",
+      "  \"id\" TEXT PRIMARY KEY,      -- Unique identifier for the team\n",
+      "  \"full_name\" TEXT,           -- Full official name of the team (e.g., \"Los Angeles Lakers\")\n",
+      "  \"abbreviation\" TEXT,        -- Shortened team name (e.g., \"LAL\")\n",
+      "  \"nickname\" TEXT,            -- Commonly used nickname for the team (e.g., \"Lakers\")\n",
+      "  \"city\" TEXT,                -- City where the team is based\n",
+      "  \"state\" TEXT,               -- State where the team is located\n",
+      "  \"year_founded\" REAL         -- Year the team was established\n",
+      ");\n",
+      "\n",
+      "game Table\n",
+      "Contains detailed statistics for each NBA game, including home and away team performance.\n",
+      "CREATE TABLE IF NOT EXISTS \"game\" (\n",
+      "  \"season_id\" TEXT,            -- Season identifier, formatted as \"2YYYY\" (e.g., \"21970\" for the 1970 season)\n",
+      "  \"team_id_home\" TEXT,         -- ID of the home team (matches \"id\" in team table)\n",
+      "  \"team_abbreviation_home\" TEXT, -- Abbreviation of the home team\n",
+      "  \"team_name_home\" TEXT,       -- Full name of the home team\n",
+      "  \"game_id\" TEXT PRIMARY KEY,  -- Unique identifier for the game\n",
+      "  \"game_date\" TIMESTAMP,       -- Date the game was played (YYYY-MM-DD format)\n",
+      "  \"matchup_home\" TEXT,         -- Matchup details including opponent (e.g., \"LAL vs. BOS\")\n",
+      "  \"wl_home\" TEXT,              -- \"W\" if the home team won, \"L\" if they lost\n",
+      "  \"min\" INTEGER,               -- Total minutes played in the game\n",
+      "  \"fgm_home\" REAL,             -- Field goals made by the home team\n",
+      "  \"fga_home\" REAL,             -- Field goals attempted by the home team\n",
+      "  \"fg_pct_home\" REAL,          -- Field goal percentage of the home team\n",
+      "  \"fg3m_home\" REAL,            -- Three-point field goals made by the home team\n",
+      "  \"fg3a_home\" REAL,            -- Three-point attempts by the home team\n",
+      "  \"fg3_pct_home\" REAL,         -- Three-point field goal percentage of the home team\n",
+      "  \"ftm_home\" REAL,             -- Free throws made by the home team\n",
+      "  \"fta_home\" REAL,             -- Free throws attempted by the home team\n",
+      "  \"ft_pct_home\" REAL,          -- Free throw percentage of the home team\n",
+      "  \"oreb_home\" REAL,            -- Offensive rebounds by the home team\n",
+      "  \"dreb_home\" REAL,            -- Defensive rebounds by the home team\n",
+      "  \"reb_home\" REAL,             -- Total rebounds by the home team\n",
+      "  \"ast_home\" REAL,             -- Assists by the home team\n",
+      "  \"stl_home\" REAL,             -- Steals by the home team\n",
+      "  \"blk_home\" REAL,             -- Blocks by the home team\n",
+      "  \"tov_home\" REAL,             -- Turnovers by the home team\n",
+      "  \"pf_home\" REAL,              -- Personal fouls by the home team\n",
+      "  \"pts_home\" REAL,             -- Total points scored by the home team\n",
+      "  \"plus_minus_home\" INTEGER,   -- Plus/minus rating for the home team\n",
+      "  \"video_available_home\" INTEGER, -- Indicates whether video is available (1 = Yes, 0 = No)\n",
+      "  \"team_id_away\" TEXT,         -- ID of the away team\n",
+      "  \"team_abbreviation_away\" TEXT, -- Abbreviation of the away team\n",
+      "  \"team_name_away\" TEXT,       -- Full name of the away team\n",
+      "  \"matchup_away\" TEXT,         -- Matchup details from the away team’s perspective\n",
+      "  \"wl_away\" TEXT,              -- \"W\" if the away team won, \"L\" if they lost\n",
+      "  \"fgm_away\" REAL,             -- Field goals made by the away team\n",
+      "  \"fga_away\" REAL,             -- Field goals attempted by the away team\n",
+      "  \"fg_pct_away\" REAL,          -- Field goal percentage of the away team\n",
+      "  \"fg3m_away\" REAL,            -- Three-point field goals made by the away team\n",
+      "  \"fg3a_away\" REAL,            -- Three-point attempts by the away team\n",
+      "  \"fg3_pct_away\" REAL,         -- Three-point field goal percentage of the away team\n",
+      "  \"ftm_away\" REAL,             -- Free throws made by the away team\n",
+      "  \"fta_away\" REAL,             -- Free throws attempted by the away team\n",
+      "  \"ft_pct_away\" REAL,          -- Free throw percentage of the away team\n",
+      "  \"oreb_away\" REAL,            -- Offensive rebounds by the away team\n",
+      "  \"dreb_away\" REAL,            -- Defensive rebounds by the away team\n",
+      "  \"reb_away\" REAL,             -- Total rebounds by the away team\n",
+      "  \"ast_away\" REAL,             -- Assists by the away team\n",
+      "  \"stl_away\" REAL,             -- Steals by the away team\n",
+      "  \"blk_away\" REAL,             -- Blocks by the away team\n",
+      "  \"tov_away\" REAL,             -- Turnovers by the away team\n",
+      "  \"pf_away\" REAL,              -- Personal fouls by the away team\n",
+      "  \"pts_away\" REAL,             -- Total points scored by the away team\n",
+      "  \"plus_minus_away\" INTEGER,   -- Plus/minus rating for the away team\n",
+      "  \"video_available_away\" INTEGER, -- Indicates whether video is available (1 = Yes, 0 = No)\n",
+      "  \"season_type\" TEXT           -- Regular season or playoffs\n",
+      ");\n",
+      "\n",
+      "other_stats Table\n",
+      "Stores additional statistics, linked to the game table via game_id.\n",
+      "CREATE TABLE IF NOT EXISTS \"other_stats\" (\n",
+      "  \"game_id\" TEXT,             -- Unique game identifier, matches id column from game table\n",
+      "  \"league_id\" TEXT,           -- League identifier\n",
+      "  \"team_id_home\" TEXT,        -- Home team identifier\n",
+      "  \"team_abbreviation_home\" TEXT, -- Home team abbreviation\n",
+      "  \"team_city_home\" TEXT,      -- Home team city\n",
+      "  \"pts_paint_home\" INTEGER,   -- Points in the paint by the home team\n",
+      "  \"pts_2nd_chance_home\" INTEGER, -- Second chance points by the home team\n",
+      "  \"pts_fb_home\" INTEGER,      -- Fast break points by the home team\n",
+      "  \"largest_lead_home\" INTEGER,-- Largest lead by the home team\n",
+      "  \"lead_changes\" INTEGER,     -- Number of lead changes \n",
+      "  \"times_tied\" INTEGER,       -- Number of times the score was tied\n",
+      "  \"team_turnovers_home\" INTEGER, -- Home team turnovers\n",
+      "  \"total_turnovers_home\" INTEGER, -- Total turnovers by the home team\n",
+      "  \"team_rebounds_home\" INTEGER, -- Home team rebounds\n",
+      "  \"pts_off_to_home\" INTEGER,  -- Points off turnovers by the home team\n",
+      "  \"team_id_away\" TEXT,        -- Away team identifier\n",
+      "  \"team_abbreviation_away\" TEXT,  -- Away team abbreviation\n",
+      "  \"pts_paint_away\" INTEGER,   -- Points in the paint by the away team\n",
+      "  \"pts_2nd_chance_away\" INTEGER, -- Second chance points by the away team\n",
+      "  \"pts_fb_away\" INTEGER,      -- Fast break points by the away team\n",
+      "  \"largest_lead_away\" INTEGER,-- Largest lead by the away team\n",
+      "  \"team_turnovers_away\" INTEGER, -- Away team turnovers\n",
+      "  \"total_turnovers_away\" INTEGER, -- Total turnovers by the away team\n",
+      "  \"team_rebounds_away\" INTEGER, -- Away team rebounds\n",
+      "  \"pts_off_to_away\" INTEGER   -- Points off turnovers by the away team\n",
+      ");\n",
+      "\n",
+      "\n",
+      "Team Name Information\n",
+      "In the plaintext user questions, only the full team names will be used, but in the queries you may use the full team names or the abbreviations. \n",
+      "The full team names can be used with the game table, while the abbreviations should be used with the other_stats table.\n",
+      "Notice they are separated by the | character in the following list:\n",
+      "\n",
+      "Atlanta Hawks|ATL\n",
+      "Boston Celtics|BOS\n",
+      "Cleveland Cavaliers|CLE\n",
+      "New Orleans Pelicans|NOP\n",
+      "Chicago Bulls|CHI\n",
+      "Dallas Mavericks|DAL\n",
+      "Denver Nuggets|DEN\n",
+      "Golden State Warriors|GSW\n",
+      "Houston Rockets|HOU\n",
+      "Los Angeles Clippers|LAC\n",
+      "Los Angeles Lakers|LAL\n",
+      "Miami Heat|MIA\n",
+      "Milwaukee Bucks|MIL\n",
+      "Minnesota Timberwolves|MIN\n",
+      "Brooklyn Nets|BKN\n",
+      "New York Knicks|NYK\n",
+      "Orlando Magic|ORL\n",
+      "Indiana Pacers|IND\n",
+      "Philadelphia 76ers|PHI\n",
+      "Phoenix Suns|PHX\n",
+      "Portland Trail Blazers|POR\n",
+      "Sacramento Kings|SAC\n",
+      "San Antonio Spurs|SAS\n",
+      "Oklahoma City Thunder|OKC\n",
+      "Toronto Raptors|TOR\n",
+      "Utah Jazz|UTA\n",
+      "Memphis Grizzlies|MEM\n",
+      "Washington Wizards|WAS\n",
+      "Detroit Pistons|DET\n",
+      "Charlotte Hornets|CHA\n",
+      "\n",
+      "Query Guidelines\n",
+      "Use team_name_home and team_name_away to match teams to the game table. Use team_abbreviation_home and team_abbreviation away to match teams to the other_stats table.\n",
+      "\n",
+      "To filter by season, use season_id = '2YYYY'.\n",
+      "\n",
+      "Example: To get statistics from 2005, use a statement like: season_id = '22005'. To get statistics from 1972, use a statement like: season_id = \"21972\". To get statistics from 2015, use a statement like: season_id = \"22015\".\n",
+      "\n",
+      "Ensure queries return relevant columns and avoid unnecessary joins.\n",
+      "\n",
+      "Example User Requests and SQLite Queries\n",
+      "Request:\n",
+      "\"What is the most points the Los Angeles Lakers have ever scored at home?\"\n",
+      "SQLite:\n",
+      "SELECT MAX(pts_home) \n",
+      "FROM game \n",
+      "WHERE team_name_home = 'Los Angeles Lakers';\n",
+      "\n",
+      "Request:\n",
+      "\"Which teams are located in the state of California?\"\n",
+      "SQLite:\n",
+      "SELECT full_name FROM team WHERE state = 'California';\n",
+      "\n",
+      "Request:\n",
+      "\"Which team had the highest number of team turnovers in an away game?\"\n",
+      "SQLite:\n",
+      "SELECT team_abbreviation_away FROM other_stats ORDER BY team_turnovers_away DESC LIMIT 1;\n",
+      "\n",
+      "Request:\n",
+      "\"Which teams were founded before 1979?\"\n",
+      "SQLite:\n",
+      "SELECT full_name FROM team WHERE year_founded < 1979;\n",
+      "\n",
+      "Request:\n",
+      "\"Find the Boston Celtics largest home victory margin in the 2008 season.\"\n",
+      "SQLite:\n",
+      "SELECT MAX(pts_home - pts_away) AS biggest_win\n",
+      "FROM game\n",
+      "WHERE team_name_home = 'Boston Celtics' AND season_id = '22008';\n",
+      "\n",
+      "Generate only the SQLite query prefaced by SQLite: and no other text, do not output an explanation of the query. Now generate an SQLite query for the following user request. Request:\n",
+      "How many points did the Golden State Warriors score in their first game of the 2005 season?\n",
+      "SQLite: \n",
+      "SELECT pts_home  FROM game  WHERE team_abbreviation_home = 'GSW'  AND season_id = '22005'  ORDER BY game_date ASC  LIMIT 1;\n",
       "939\n",
       "105\n"
      ]
@@ -296,7 +679,10 @@
     "    Tokenizes input natural language queries and corresponding SQL queries.\n",
     "    \"\"\"\n",
     "    inputs = [input_prompt + q for q in examples[\"natural_query\"]]\n",
-    "    targets = examples[\"sql_query\"]\n",
+    "    targets = [\"SQLite: \\n\" + q for q in examples[\"sql_query\"]]\n",
+    "\n",
+    "    print(inputs[0])\n",
+    "    print(targets[0])\n",
     "\n",
     "    model_inputs = tokenizer(inputs, padding=\"max_length\", truncation=True, max_length=256)\n",
     "    labels = tokenizer(targets, padding=\"max_length\", truncation=True, max_length=256)\n",
@@ -396,7 +782,7 @@
      "text": [
       "c:\\Users\\Dean\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\site-packages\\transformers\\training_args.py:1611: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
       "  warnings.warn(\n",
-      "C:\\Users\\Dean\\AppData\\Local\\Temp\\ipykernel_10280\\3557190339.py:17: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
+      "C:\\Users\\Dean\\AppData\\Local\\Temp\\ipykernel_9248\\2737143648.py:17: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
       "  trainer = Trainer(\n",
       "No label_names provided for model class `PeftModelForCausalLM`. Since `PeftModel` hides base models input arguments, if label_names is not given, label_names can't be set automatically within `Trainer`. Note that empty label_names list will be used instead.\n"
      ]
@@ -407,10 +793,10 @@
     "    output_dir=\"./fine-tuned-model\",\n",
     "    evaluation_strategy=\"epoch\",  # Evaluate at the end of each epoch\n",
     "    save_strategy=\"epoch\",  # Save model every epoch\n",
-    "    per_device_train_batch_size=8,  # LoRA allows higher batch size\n",
-    "    per_device_eval_batch_size=8,\n",
-    "    num_train_epochs=3,  # Increase if needed\n",
-    "    learning_rate=5e-4,  # Higher LR since we're only training LoRA layers\n",
+    "    per_device_train_batch_size=16,  # LoRA allows higher batch size\n",
+    "    per_device_eval_batch_size=16,\n",
+    "    num_train_epochs=5,  # Increase if needed\n",
+    "    learning_rate=5e-5,  # Higher LR since we're only training LoRA layers\n",
     "    weight_decay=0.01,\n",
     "    logging_steps=50,  # Print loss every 50 steps\n",
     "    save_total_limit=2,  # Keep last 2 checkpoints\n",
@@ -454,8 +840,8 @@
        "\n",
        "    <div>\n",
        "      \n",
-       "      <progress value='354' max='354' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
-       "      [354/354 05:21, Epoch 3/3]\n",
+       "      <progress value='295' max='295' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [295/295 1:04:01, Epoch 5/5]\n",
        "    </div>\n",
        "    <table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
@@ -468,18 +854,28 @@
        "  <tbody>\n",
        "    <tr>\n",
        "      <td>1</td>\n",
-       "      <td>1.235600</td>\n",
-       "      <td>0.970917</td>\n",
+       "      <td>8.510300</td>\n",
+       "      <td>1.486602</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <td>2</td>\n",
-       "      <td>1.233100</td>\n",
-       "      <td>0.937016</td>\n",
+       "      <td>1.709800</td>\n",
+       "      <td>1.227304</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <td>3</td>\n",
-       "      <td>1.157600</td>\n",
-       "      <td>0.940143</td>\n",
+       "      <td>1.542100</td>\n",
+       "      <td>1.161120</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>4</td>\n",
+       "      <td>1.487500</td>\n",
+       "      <td>1.153255</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>5</td>\n",
+       "      <td>1.388300</td>\n",
+       "      <td>1.121680</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table><p>"
@@ -531,22 +927,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "c:\\Users\\Dean\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\site-packages\\bitsandbytes\\autograd\\_functions.py:315: UserWarning: MatMul8bitLt: inputs will be cast from torch.bfloat16 to float16 during quantization\n",
-      "  warnings.warn(f\"MatMul8bitLt: inputs will be cast from {A.dtype} to float16 during quantization\")\n"
-     ]
-    },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Generated SQL: SELECT_________________________________________________________________________________________________________\n"
+      "Generated SQL: SQLite: SELECT AVG(pts_home) FROM game WHERE team_name_home = 'Los Angeles Lakers';\n",
+      "\n"
      ]
     }
    ],
@@ -555,7 +944,7 @@
     "tokenizer = AutoTokenizer.from_pretrained(\"./fine-tuned-model\")\n",
     "\n",
     "# Prepare query with the same prompt\n",
-    "input_text = \"Show the top 5 highest-scoring NBA games in history.\"\n",
+    "input_text = \"How many points to the Los Angeles Lakers average at home?\"\n",
     "message = [{'role': 'user', 'content': input_prompt + input_text}]\n",
     "inputs = tokenizer.apply_chat_template(message, add_generation_prompt=True, return_tensors=\"pt\").to(model.device)\n",
     "\n",