vives commited on
Commit
cc3cff0
·
1 Parent(s): 77ff185

Create app.py

Browse files
Files changed (1) hide show
  1. app.py +16 -0
app.py ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from transformers import AutoModelForMaskedLM
2
+ from transformers import AutoTokenizer
3
+ import spacy
4
+ import pytextrank
5
+
6
+ model_checkpoint = "vives/distilbert-base-uncased-finetuned-cvent-2019_2022"
7
+ model = AutoModelForMaskedLM.from_pretrained(model_checkpoint, output_hidden_states=True)
8
+ tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
9
+
10
+
11
+ FILT_GROUPS = ["CARDINAL", "TIME", "DATE", "PERCENT", "MONEY", "QUANTITY", "ORDINAL"]
12
+ POS = ["NOUN", "PROPN", "VERB"]
13
+
14
+ nlp = spacy.load("en_core_web_sm")
15
+ nlp.add_pipe("textrank", last=True, config={"pos_kept": POS, "token_lookback": 3})
16
+ all_stopwords = nlp.Defaults.stop_words