Commit
·
afa6b01
1
Parent(s):
2e4daec
Update functions.py
Browse files- functions.py +1 -1
functions.py
CHANGED
@@ -2,7 +2,7 @@ from transformers import AutoTokenizer, AutoConfig, AutoModelForSequenceClassifi
|
|
2 |
from scipy.special import softmax
|
3 |
|
4 |
# Define the preprocess function
|
5 |
-
def
|
6 |
new_text = []
|
7 |
for t in text.split(" "):
|
8 |
# Check if the token is an emoji
|
|
|
2 |
from scipy.special import softmax
|
3 |
|
4 |
# Define the preprocess function
|
5 |
+
def preprocess(text):
|
6 |
new_text = []
|
7 |
for t in text.split(" "):
|
8 |
# Check if the token is an emoji
|