Update FlexRAG retriever

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+corpus.jsonl filter=lfs diff=lfs merge=lfs -text
+corpus.mmindex.json filter=lfs diff=lfs merge=lfs -text
+vocab.index.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,40 @@
+---
+language: en
+library_name: FlexRAG
+tags:
+- FlexRAG
+- retrieval
+- search
+- lexical
+- RAG
+---
+
+# FlexRAG Retriever
+
+This is a BM25SRetriever created with the [`FlexRAG`](https://github.com/ictnlp/flexrag) library (version `0.1.8`).
+
+## Installation
+
+You can install the `FlexRAG` library with `pip`:
+
+```bash
+pip install flexrag
+```
+
+## Loading a `FlexRAG` retriever
+
+You can use this retriever for information retrieval tasks. Here is an example:
+
+```python
+from flexrag.retriever import LocalRetriever
+
+# Load the retriever from the HuggingFace Hub
+retriever = LocalRetriever.load_from_hub("FlexRAG/wiki2021_atlas_bm25s")
+
+# You can retrieve now
+results = retriever.search("Who is Bruce Wayne?")
+```
+
+FlexRAG Related Links:
+* 📚[Documentation](https://flexrag.readthedocs.io/en/latest/)
+* 💻[GitHub Repository](https://github.com/ictnlp/flexrag)

config.yaml

@@ -0,0 +1,43 @@
+log_interval: 100
+top_k: 10
+batch_size: 32
+query_preprocess_pipeline:
+  processor_type: []
+  length_filter_config:
+    max_tokens: null
+    min_tokens: null
+    max_chars: null
+    min_chars: null
+    max_bytes: null
+    min_bytes: null
+    tokenizer_config:
+      tokenizer_type: moses
+      hf_tokenizer_path: null
+      tiktok_tokenizer_name: null
+      lang: null
+  token_normalize_config:
+    lang: en
+    penn: true
+    norm_quote_commas: true
+    norm_numbers: true
+    pre_replace_unicode_punct: false
+    post_remove_control_chars: false
+    perl_parity: false
+  truncate_config:
+    max_chars: null
+    max_bytes: null
+    max_tokens: null
+    tokenizer_config:
+      tokenizer_type: moses
+      hf_tokenizer_path: null
+      tiktok_tokenizer_name: null
+      lang: null
+database_path: ./bm25s_lucene
+method: lucene
+idf_method: null
+backend: auto
+k1: 1.5
+b: 0.75
+delta: 0.5
+lang: english
+indexed_fields: null

corpus.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ef7c8f44c67ed805f0c41ee9cd267d76e310247664aea7a5cedaab4329fec71
+size 22992342304

corpus.mmindex.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f61677c4e0b18a994bbe28d5e5fd54ed57a56108accf9219bb9a97259fb735d1
+size 432224577

data.csc.index.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45bf59f9178ac72e917c20895c416bdadb6837b66539494f3db836ad1713bfaa
+size 7294946272

indices.csc.index.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16865d9412728283ea48ec9e2409ff7fa8506ec9c40a3a18918ad3f31a377df9
+size 7294946272

indptr.csc.index.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:732794e91b5565488702dcdf2aba1e1cd05e2d22ffd6b4dada666966cf4462d4
+size 47865776

params.index.json

@@ -0,0 +1,12 @@
+{
+    "k1": 1.5,
+    "b": 0.75,
+    "delta": 0.5,
+    "method": "lucene",
+    "idf_method": "lucene",
+    "dtype": "float32",
+    "int_dtype": "int32",
+    "num_docs": 37507469,
+    "version": "0.2.1",
+    "backend": "numba"
+}

retriever.id

@@ -0,0 +1 @@
+BM25SRetriever

vocab.index.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f8b43e07c3cc260486127f8cea0821d3c0b01de842158cd796f9980a86139d5
+size 246450673