amank
commited on
Commit
·
7007f93
1
Parent(s):
7839b8e
Minor Code cleanup in utils file
Browse files
utils.py
CHANGED
@@ -42,15 +42,8 @@ def keep_devnagri_hf_doc(document):
|
|
42 |
return cleaned_text
|
43 |
|
44 |
if batched:
|
45 |
-
|
46 |
-
cleaned_text_ls = []
|
47 |
-
for text in text_ls:
|
48 |
-
cleaned_text = get_clean_text(text)
|
49 |
-
cleaned_text_ls.append(cleaned_text)
|
50 |
-
document['text'] = cleaned_text_ls
|
51 |
else:
|
52 |
-
text = document['text']
|
53 |
-
cleaned_text = get_clean_text(text)
|
54 |
-
document['text'] = cleaned_text
|
55 |
|
56 |
return document
|
|
|
42 |
return cleaned_text
|
43 |
|
44 |
if batched:
|
45 |
+
document['text'] = [get_clean_text(text) for text in document['text']]
|
|
|
|
|
|
|
|
|
|
|
46 |
else:
|
47 |
+
document['text'] = get_clean_text(document['text'])
|
|
|
|
|
48 |
|
49 |
return document
|