File size: 2,685 Bytes
507f62f 6886972 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 |
---
license: apache-2.0
---
## Generate training data
```
# Function to convert dataframe to list of InputExample
def df_to_input_examples(df):
return [
InputExample(texts=[row['query'],
row['document']],
label=float(row['relevance_score']))
for _, row in df.iterrows()
]
train_samples = df_to_input_examples(train_df)
val_samples = df_to_input_examples(val_df)
# Create a DataLoader for training
train_dataloader = DataLoader(train_samples, shuffle=True, batch_size=16)
```
## Create Evaluator class
```
# Custom evaluator for CrossEncoder
class CrossEncoderEvaluator:
def __init__(self, eval_samples):
self.eval_samples = eval_samples
def __call__(self, model, **kwargs): # Add **kwargs to catch extra arguments
predictions = model.predict([[sample.texts[0], sample.texts[1]] for sample in self.eval_samples])
labels = [sample.label for sample in self.eval_samples]
pearson_corr, _ = pearsonr(predictions, labels)
spearman_corr, _ = spearmanr(predictions, labels)
return (pearson_corr + spearman_corr) / 2 # Average of Pearson and Spearman correlations
# Prepare the evaluator
evaluator = CrossEncoderEvaluator(val_samples)
```
## Train the model
```
# Initialize the cross-encoder model
model = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2', num_labels=1)
# Train the model
model.fit(
train_dataloader=train_dataloader,
evaluator=evaluator,
epochs=100,
warmup_steps=100,
evaluation_steps=500,
output_path='fine_tuned_reranker'
)
```
## Usage
```
# Load the fine-tuned reranker
reranker_model = CrossEncoder('fine_tuned_reranker')
def search_and_rerank(query, documents, top_k=10):
# Prepare pairs for reranking
pairs = [(query, doc) for doc in documents]
# Rerank using fine-tuned cross-encoder
rerank_scores = reranker_model.predict(pairs)
# Sort results by reranker scores
reranked_results = sorted(
zip(documents, rerank_scores.tolist()),
key=lambda x: x[1], reverse=True
)
return reranked_results
query = "OPPO 8GB 128G"
documents = [
"OPPO Reno11F 5G 8GB-256GB",
"OPPO Reno11F 5G 8GB-32GB",
"OPPO Reno11F 5G 16GB-128GB",
"Samsung galaxy 128GB",
"Samsung S24 128GB",
# ...
]
start_time = time.time()
results = search_and_rerank(query, documents, len(documents)-1)
end_time = time.time()
execution_time = (end_time - start_time)*1000
print(f"Execution time: {execution_time:.4f} mili seconds")
print(f"Query: \t\t\t\t{query}")
for res in results:
print(f"Score: {res[-1]:.4f} | Document: {res[0]}")
```
Credit goes to: [email protected] |