Spaces:
Runtime error
Runtime error
portalniy-dev
commited on
Commit
β’
fc46763
1
Parent(s):
f6e4f14
Update app.py
Browse files
app.py
CHANGED
@@ -9,7 +9,7 @@ dataset_names = {
|
|
9 |
'ag_news': None,
|
10 |
'squad': None,
|
11 |
'cnn_dailymail': '1.0.0', # Specify configuration for cnn_dailymail
|
12 |
-
'wiki40b': '
|
13 |
}
|
14 |
|
15 |
# Global variables for model and tokenizer
|
@@ -32,6 +32,7 @@ def load_and_prepare_datasets():
|
|
32 |
|
33 |
for ds in datasets:
|
34 |
if 'train' in ds:
|
|
|
35 |
if 'text' in ds['train'].features:
|
36 |
train_datasets.append(ds['train'].map(lambda x: {'text': x['text']}))
|
37 |
elif 'content' in ds['train'].features: # Example for CNN/DailyMail
|
@@ -40,6 +41,7 @@ def load_and_prepare_datasets():
|
|
40 |
print(f"Warning: No suitable text field found in {ds['train'].features}")
|
41 |
|
42 |
if 'test' in ds:
|
|
|
43 |
if 'text' in ds['test'].features:
|
44 |
eval_datasets.append(ds['test'].map(lambda x: {'text': x['text']}))
|
45 |
elif 'content' in ds['test'].features: # Example for CNN/DailyMail
|
|
|
9 |
'ag_news': None,
|
10 |
'squad': None,
|
11 |
'cnn_dailymail': '1.0.0', # Specify configuration for cnn_dailymail
|
12 |
+
'wiki40b': 'en' # Specify language for wiki40b
|
13 |
}
|
14 |
|
15 |
# Global variables for model and tokenizer
|
|
|
32 |
|
33 |
for ds in datasets:
|
34 |
if 'train' in ds:
|
35 |
+
# Extract text field based on available keys
|
36 |
if 'text' in ds['train'].features:
|
37 |
train_datasets.append(ds['train'].map(lambda x: {'text': x['text']}))
|
38 |
elif 'content' in ds['train'].features: # Example for CNN/DailyMail
|
|
|
41 |
print(f"Warning: No suitable text field found in {ds['train'].features}")
|
42 |
|
43 |
if 'test' in ds:
|
44 |
+
# Extract text field based on available keys
|
45 |
if 'text' in ds['test'].features:
|
46 |
eval_datasets.append(ds['test'].map(lambda x: {'text': x['text']}))
|
47 |
elif 'content' in ds['test'].features: # Example for CNN/DailyMail
|