Refactor data extraction logic to improve category handling and streamline data sanitization

.gitignore

@@ -1,8 +1,5 @@
 /.cache
 /__pycache__
 .env
-.env.local
-.env.development.local
-.env.test.local
-.env.production.local
 .DS_Store
+*.json

arvix.py

@@ -51,18 +51,30 @@ def extract_data(category):
     new_data = extract_new_data(category)
     recent_data = extract_recent_data(category)
     data = list(set(new_data + recent_data))
-    for id in data:
-        if len(sanitized_data) >= 12:
-            break
-        if category in ["hep-ex", "hep-lat", "hep-ph", "hep-th"]:
-            id = id[:3]
-        if tools.check_data_in_file(id, 'arxiv.txt'):
-            continue
-        else:
+    if category in ["hep-ex", "hep-lat", "hep-ph", "hep-th"]:
+        category_list = []
+        for id in data:
+            if len(category_list) >= 3:
+                break
+            if tools.check_data_in_file(id, 'arxiv.txt'):
+                continue
+            else:
+                category_list.append(id)
+        for category_id in category_list:
+            sanitized_data.append(category_id)
             tools.write_data_to_file(id, 'arxiv.txt')
-            sanitized_data.append(id)
+    else:
+        for id in data:
+            if len(sanitized_data) >= 12:
+                break
+            if tools.check_data_in_file(id, 'arxiv.txt'):
+                continue
+            else:
+                tools.write_data_to_file(id, 'arxiv.txt')
+                sanitized_data.append(id)
     random.shuffle(sanitized_data)
-    return sanitized_data[:12]
+    print(len(sanitized_data))
+    return sanitized_data
 
 def extract_arxiv_data():
     if not tools.download_datafile('arxiv.txt'):