upload

README.md

@@ -0,0 +1,90 @@
+# Natural Questions Models
+[Google's Natural Questions dataset](https://ai.google.com/research/NaturalQuestions) constists of about 100k real search queries from Google with the respective, relevant passage from Wikipedia. Models trained on this dataset work well for question-answer retrieval.
+
+## Usage (Sentence Transformers)
+Using this model becomes more convenient when you have [sentence-transformers](https://github.com/UKPLab/sentence-transformers) installed:
+```
+pip install -U sentence-transformers
+```
+
+Then you can use the model like this:
+
+```python
+from sentence_transformers import SentenceTransformer, util
+model = SentenceTransformer('nq-distilbert-base-v1')
+
+query_embedding = model.encode('How many people live in London?')
+
+#The passages are encoded as [ [title1, text1], [title2, text2], ...]
+passage_embedding = model.encode([['London', 'London has 9,787,426 inhabitants at the 2011 census.']])
+
+print("Similarity:", util.pytorch_cos_sim(query_embedding, passage_embedding))
+```
+
+Note: For the passage, we have to encode the Wikipedia article title together with a text paragraph from that article.
+
+
+
+## Usage (HuggingFace Models Repository)
+
+You can use the model directly from the model repository to compute sentence embeddings:
+```python
+from transformers import AutoTokenizer, AutoModel
+import torch
+
+
+#Mean Pooling - Take attention mask into account for correct averaging
+def mean_pooling(model_output, attention_mask):
+    token_embeddings = model_output[0] #First element of model_output contains all token embeddings
+    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+    sum_embeddings = torch.sum(token_embeddings * input_mask_expanded, 1)
+    sum_mask = torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+    return sum_embeddings / sum_mask
+
+
+
+# Queries we want embeddings for
+queries = ['What is the capital of France?', 'How many people live in New York City?']
+
+# Passages that provide answers
+titles = ['Paris', 'New York City']
+passages = ['Paris is the capital of France', 'New York City is the most populous city in the United States, with an estimated 8,336,817 people living in the city, according to U.S. Census estimates dating July 1, 2019']
+
+#Load AutoModel from huggingface model repository
+tokenizer = AutoTokenizer.from_pretrained("model_name")
+model = AutoModel.from_pretrained("model_name")
+
+def compute_embeddings(sentences, titles=None):
+	#Tokenize sentences
+	if titles is not None:
+		encoded_input = tokenizer(titles, sentences, padding=True, truncation=True, return_tensors='pt')
+	else:
+		encoded_input = tokenizer(sentences, padding=True, truncation=True, return_tensors='pt')
+
+	#Compute query embeddings
+	with torch.no_grad():
+		model_output = model(**encoded_input)
+
+	#Perform pooling. In this case, mean pooling
+	return mean_pooling(model_output, encoded_input['attention_mask'])
+
+query_embeddings = compute_embeddings(queries)
+passage_embeddings = compute_embeddings(passages, titles)
+```
+
+## Performance
+For performance details, see: [SBERT.net - Pre-Trained Models - Natural Questions](https://www.sbert.net/docs/pretrained-models/nq-v1.html)
+
+## Citing & Authors
+If you find this model helpful, feel free to cite our publication [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084):
+``` 
+@inproceedings{reimers-2019-sentence-bert,
+    title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks",
+    author = "Reimers, Nils and Gurevych, Iryna",
+    booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing",
+    month = "11",
+    year = "2019",
+    publisher = "Association for Computational Linguistics",
+    url = "http://arxiv.org/abs/1908.10084",
+}
+```

config.json

@@ -0,0 +1,22 @@
+{
+  "_name_or_path": "/home/ukp-reimers/msmarco/output/distilbert-base-uncased-mined_hard_neg-mean-pooling-no_identifier-epoch10-batchsize75-NTXentLossTriplet-2021-01-08_17-47-51/0_Transformer",
+  "activation": "gelu",
+  "architectures": [
+    "DistilBertModel"
+  ],
+  "attention_dropout": 0.1,
+  "dim": 768,
+  "dropout": 0.1,
+  "hidden_dim": 3072,
+  "initializer_range": 0.02,
+  "max_position_embeddings": 512,
+  "model_type": "distilbert",
+  "n_heads": 12,
+  "n_layers": 6,
+  "pad_token_id": 0,
+  "qa_dropout": 0.1,
+  "seq_classif_dropout": 0.2,
+  "sinusoidal_pos_embds": false,
+  "tie_weights_": true,
+  "vocab_size": 30522
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f12c8e5381d61116632b98a428a45916e46b5e49350b9f26fb31ccfbfbfcc642
+size 265491187

sentence_bert_config.json

@@ -0,0 +1,3 @@
+{
+  "max_seq_length": 512
+}

special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": true, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "model_max_length": 512, "name_or_path": "/home/ukp-reimers/msmarco/output/distilbert-base-uncased-mined_hard_neg-mean-pooling-no_identifier-epoch10-batchsize75-NTXentLossTriplet-2021-01-08_17-47-51/0_Transformer", "special_tokens_map_file": "/home/ukp-reimers/msmarco/output/distilbert-base-uncased-mined_hard_neg-mean-pooling-no_identifier-epoch10-batchsize75-NTXentLossTriplet-2021-01-08_17-47-51/0_Transformer/special_tokens_map.json", "do_basic_tokenize": true, "never_split": null}