Update README.md
#1
by
MassMin
- opened
README.md
CHANGED
@@ -118,44 +118,44 @@ The model's performance is evaluated using the F1 score for NER. The predictions
|
|
118 |
[More Information Needed]
|
119 |
|
120 |
## Evaluation
|
121 |
-
'''python
|
122 |
-
|
123 |
-
|
124 |
-
|
125 |
-
|
126 |
-
|
127 |
-
|
128 |
-
|
129 |
-
|
130 |
-
|
131 |
-
|
132 |
-
|
133 |
-
|
134 |
-
|
135 |
-
|
136 |
-
|
137 |
-
|
138 |
-
|
139 |
-
|
140 |
-
|
141 |
-
|
142 |
-
|
143 |
-
|
144 |
-
|
145 |
-
|
146 |
-
|
147 |
-
|
148 |
-
|
149 |
-
|
150 |
-
|
151 |
-
|
152 |
-
|
153 |
-
|
154 |
-
|
155 |
-
|
156 |
-
|
157 |
-
|
158 |
-
|
159 |
|
160 |
|
161 |
<!-- This section describes the evaluation protocols and provides the results. -->
|
|
|
118 |
[More Information Needed]
|
119 |
|
120 |
## Evaluation
|
121 |
+
#'''python
|
122 |
+
import torch
|
123 |
+
|
124 |
+
from transformers import AutoTokenizer, AutoModelForTokenClassification, pipeline
|
125 |
+
|
126 |
+
import pandas as pd
|
127 |
+
|
128 |
+
|
129 |
+
model_checkpoint = "MassMin/xlm-roberta-base-finetuned-panx-de" # Replace with your Hugging Face model name
|
130 |
+
|
131 |
+
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
132 |
+
|
133 |
+
|
134 |
+
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
|
135 |
+
|
136 |
+
model = AutoModelForTokenClassification.from_pretrained(model_checkpoint).to(device)
|
137 |
+
|
138 |
+
|
139 |
+
ner_pipeline = pipeline("ner", model=model, tokenizer=tokenizer, framework="pt", device=0 if torch.cuda.is_available() else -1)
|
140 |
+
|
141 |
+
|
142 |
+
def tag_text_with_pipeline(text, ner_pipeline):
|
143 |
+
|
144 |
+
# Use the NER pipeline to get predictions
|
145 |
+
results = ner_pipeline(text)
|
146 |
+
|
147 |
+
# Convert results to a DataFrame for easy viewing
|
148 |
+
df = pd.DataFrame(results)
|
149 |
+
df = df[['word', 'entity', 'score']]
|
150 |
+
df.columns = ['Tokens', 'Tags', 'Score'] # Rename columns for clarity
|
151 |
+
return df
|
152 |
+
|
153 |
+
|
154 |
+
text = "Jeff Dean works at Google in California."
|
155 |
+
|
156 |
+
result = tag_text_with_pipeline(text, ner_pipeline)
|
157 |
+
|
158 |
+
print(result)
|
159 |
|
160 |
|
161 |
<!-- This section describes the evaluation protocols and provides the results. -->
|