Update utils/cleaning.py
Browse files- utils/cleaning.py +2 -2
utils/cleaning.py
CHANGED
|
@@ -193,8 +193,8 @@ def chunk_data(data):
|
|
| 193 |
opinions_split = pd.DataFrame(data_dict)
|
| 194 |
chunks = split_text(data)
|
| 195 |
for chunk in chunks:
|
| 196 |
-
if len(chunk) < 1000:
|
| 197 |
-
|
| 198 |
tmp = pd.DataFrame({"label": [200], "text": [chunk]})
|
| 199 |
opinions_split = pd.concat([opinions_split, tmp])
|
| 200 |
return opinions_split
|
|
|
|
| 193 |
opinions_split = pd.DataFrame(data_dict)
|
| 194 |
chunks = split_text(data)
|
| 195 |
for chunk in chunks:
|
| 196 |
+
# if len(chunk) < 1000:
|
| 197 |
+
# continue
|
| 198 |
tmp = pd.DataFrame({"label": [200], "text": [chunk]})
|
| 199 |
opinions_split = pd.concat([opinions_split, tmp])
|
| 200 |
return opinions_split
|