arubenruben
/

NER-PT-BERT-CRF-Conll2003

Token Classification

Model card Files Files and versions Community

arubenruben commited on May 29, 2023

Commit

fa92478

·

1 Parent(s): d200bec

Update deploy_pipeline.py

Files changed (1) hide show

deploy_pipeline.py +1 -1

deploy_pipeline.py CHANGED Viewed

@@ -14,7 +14,7 @@ class TokenizeAndAlignLabelsStep():
     # Adapted From : https://huggingface.co/docs/transformers/tasks/token_classification
     def tokenize_and_align_labels(self, examples, tokenizer):
-        tokenized_inputs = tokenizer(examples, padding='max_length', truncation=True, max_length=512)
         # Map tokens to their respective word.
         word_ids = tokenized_inputs.word_ids()

     # Adapted From : https://huggingface.co/docs/transformers/tasks/token_classification
     def tokenize_and_align_labels(self, examples, tokenizer):
+        tokenized_inputs = tokenizer(examples, padding='max_length', truncation=True, max_length=128)
         # Map tokens to their respective word.
         word_ids = tokenized_inputs.word_ids()