Upload 13 files

arxiv_corpus/dataset_dict.json

@@ -0,0 +1 @@
+{"splits": ["train"]}

arxiv_corpus/train/data-00000-of-00009.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:592316b955cd556a7f857b34ae6d3cbc2d41607ebc7edf0f4936dcbc72d9b210
+size 466235848

arxiv_corpus/train/data-00001-of-00009.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8dc80f91831e6ec8d887220fa58bd7fc9a24a45f3f8f700386ea5f51465566c
+size 457426648

arxiv_corpus/train/data-00002-of-00009.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5c83ff443d42f500930beff99b7ebcb99c6b13dbd4a2706d511871b846d5743
+size 458475864

arxiv_corpus/train/data-00003-of-00009.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a57e350cef8e17b0fc039d731803470f6984161edfff0ef563268dd8ab34238
+size 459603008

arxiv_corpus/train/data-00004-of-00009.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:290e397fbf9dc69f69bdc253b639f141c55923def25de26634a5bf32979b64cb
+size 458884624

arxiv_corpus/train/data-00005-of-00009.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:532dc21466633c66ddf9f836211ecf5d8418facf9770e697567f4f86185df6e7
+size 465391392

arxiv_corpus/train/data-00006-of-00009.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea8418f4d85e2c51dab08103274b7129a0e18577f85101aab89f6eaae8a388ce
+size 454939008

arxiv_corpus/train/data-00007-of-00009.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e3b772eb9d0190efcac9852ee98f381d015f231bffcc45704415d347fc5624c
+size 448746432

arxiv_corpus/train/data-00008-of-00009.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3cd1cabc942ca520de26ad97f87bb9e491225d548096414eb9387da8b7e2c1fc
+size 459532024

arxiv_corpus/train/dataset_info.json

@@ -0,0 +1,204 @@
+{
+  "builder_name": "parquet",
+  "citation": "",
+  "config_name": "default",
+  "dataset_name": "astro_paper_corpus",
+  "dataset_size": 4128813829,
+  "description": "",
+  "download_checksums": {
+    "hf://datasets/JSALT2024-Astro-LLMs/astro_paper_corpus@b957a28700badb3b5f5c7af06ea77a2560ab6e46/data/train-00000-of-00009.parquet": {
+      "num_bytes": 240072323,
+      "checksum": null
+    },
+    "hf://datasets/JSALT2024-Astro-LLMs/astro_paper_corpus@b957a28700badb3b5f5c7af06ea77a2560ab6e46/data/train-00001-of-00009.parquet": {
+      "num_bytes": 235851056,
+      "checksum": null
+    },
+    "hf://datasets/JSALT2024-Astro-LLMs/astro_paper_corpus@b957a28700badb3b5f5c7af06ea77a2560ab6e46/data/train-00002-of-00009.parquet": {
+      "num_bytes": 236413937,
+      "checksum": null
+    },
+    "hf://datasets/JSALT2024-Astro-LLMs/astro_paper_corpus@b957a28700badb3b5f5c7af06ea77a2560ab6e46/data/train-00003-of-00009.parquet": {
+      "num_bytes": 237728419,
+      "checksum": null
+    },
+    "hf://datasets/JSALT2024-Astro-LLMs/astro_paper_corpus@b957a28700badb3b5f5c7af06ea77a2560ab6e46/data/train-00004-of-00009.parquet": {
+      "num_bytes": 236710419,
+      "checksum": null
+    },
+    "hf://datasets/JSALT2024-Astro-LLMs/astro_paper_corpus@b957a28700badb3b5f5c7af06ea77a2560ab6e46/data/train-00005-of-00009.parquet": {
+      "num_bytes": 239567004,
+      "checksum": null
+    },
+    "hf://datasets/JSALT2024-Astro-LLMs/astro_paper_corpus@b957a28700badb3b5f5c7af06ea77a2560ab6e46/data/train-00006-of-00009.parquet": {
+      "num_bytes": 234863979,
+      "checksum": null
+    },
+    "hf://datasets/JSALT2024-Astro-LLMs/astro_paper_corpus@b957a28700badb3b5f5c7af06ea77a2560ab6e46/data/train-00007-of-00009.parquet": {
+      "num_bytes": 232662046,
+      "checksum": null
+    },
+    "hf://datasets/JSALT2024-Astro-LLMs/astro_paper_corpus@b957a28700badb3b5f5c7af06ea77a2560ab6e46/data/train-00008-of-00009.parquet": {
+      "num_bytes": 237444927,
+      "checksum": null
+    }
+  },
+  "download_size": 2131314110,
+  "features": {
+    "id": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "author": {
+      "feature": {
+        "dtype": "string",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "bibcode": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "title": {
+      "feature": {
+        "dtype": "string",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "citation_count": {
+      "dtype": "int64",
+      "_type": "Value"
+    },
+    "aff": {
+      "feature": {
+        "dtype": "string",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "citation": {
+      "feature": {
+        "dtype": "string",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "database": {
+      "feature": {
+        "dtype": "string",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "read_count": {
+      "dtype": "int64",
+      "_type": "Value"
+    },
+    "keyword": {
+      "feature": {
+        "dtype": "string",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "reference": {
+      "feature": {
+        "dtype": "string",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "doi": {
+      "feature": {
+        "dtype": "string",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "subfolder": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "filename": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "introduction": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "conclusions": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "year": {
+      "dtype": "int64",
+      "_type": "Value"
+    },
+    "month": {
+      "dtype": "int64",
+      "_type": "Value"
+    },
+    "arxiv_id": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "abstract": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "failed_ids": {
+      "dtype": "bool",
+      "_type": "Value"
+    },
+    "keyword_search": {
+      "feature": {
+        "dtype": "string",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "umap_x": {
+      "dtype": "float32",
+      "_type": "Value"
+    },
+    "umap_y": {
+      "dtype": "float32",
+      "_type": "Value"
+    },
+    "clust_id": {
+      "dtype": "int64",
+      "_type": "Value"
+    }
+  },
+  "homepage": "",
+  "license": "",
+  "size_in_bytes": 6260127939,
+  "splits": {
+    "train": {
+      "name": "train",
+      "num_bytes": 4128813829,
+      "num_examples": 271544,
+      "shard_lengths": [
+        33172,
+        33172,
+        33172,
+        33172,
+        33172,
+        33171,
+        34171,
+        34171,
+        4171
+      ],
+      "dataset_name": "astro_paper_corpus"
+    }
+  },
+  "version": {
+    "version_str": "0.0.0",
+    "major": 0,
+    "minor": 0,
+    "patch": 0
+  }
+}

arxiv_corpus/train/state.json

@@ -0,0 +1,37 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00009.arrow"
+    },
+    {
+      "filename": "data-00001-of-00009.arrow"
+    },
+    {
+      "filename": "data-00002-of-00009.arrow"
+    },
+    {
+      "filename": "data-00003-of-00009.arrow"
+    },
+    {
+      "filename": "data-00004-of-00009.arrow"
+    },
+    {
+      "filename": "data-00005-of-00009.arrow"
+    },
+    {
+      "filename": "data-00006-of-00009.arrow"
+    },
+    {
+      "filename": "data-00007-of-00009.arrow"
+    },
+    {
+      "filename": "data-00008-of-00009.arrow"
+    }
+  ],
+  "_fingerprint": "b9db3ec46232aa87",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": "train"
+}

requirements.txt

@@ -9,6 +9,7 @@ langchain_openai
 langchain_community
 langchain_core
 openai
+anthropic
 feedparser
 tiktoken
 chromadb