portalniy-dev commited on
Commit
fc46763
β€’
1 Parent(s): f6e4f14

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +3 -1
app.py CHANGED
@@ -9,7 +9,7 @@ dataset_names = {
9
  'ag_news': None,
10
  'squad': None,
11
  'cnn_dailymail': '1.0.0', # Specify configuration for cnn_dailymail
12
- 'wiki40b': 'ru' # Specify language for wiki40b
13
  }
14
 
15
  # Global variables for model and tokenizer
@@ -32,6 +32,7 @@ def load_and_prepare_datasets():
32
 
33
  for ds in datasets:
34
  if 'train' in ds:
 
35
  if 'text' in ds['train'].features:
36
  train_datasets.append(ds['train'].map(lambda x: {'text': x['text']}))
37
  elif 'content' in ds['train'].features: # Example for CNN/DailyMail
@@ -40,6 +41,7 @@ def load_and_prepare_datasets():
40
  print(f"Warning: No suitable text field found in {ds['train'].features}")
41
 
42
  if 'test' in ds:
 
43
  if 'text' in ds['test'].features:
44
  eval_datasets.append(ds['test'].map(lambda x: {'text': x['text']}))
45
  elif 'content' in ds['test'].features: # Example for CNN/DailyMail
 
9
  'ag_news': None,
10
  'squad': None,
11
  'cnn_dailymail': '1.0.0', # Specify configuration for cnn_dailymail
12
+ 'wiki40b': 'en' # Specify language for wiki40b
13
  }
14
 
15
  # Global variables for model and tokenizer
 
32
 
33
  for ds in datasets:
34
  if 'train' in ds:
35
+ # Extract text field based on available keys
36
  if 'text' in ds['train'].features:
37
  train_datasets.append(ds['train'].map(lambda x: {'text': x['text']}))
38
  elif 'content' in ds['train'].features: # Example for CNN/DailyMail
 
41
  print(f"Warning: No suitable text field found in {ds['train'].features}")
42
 
43
  if 'test' in ds:
44
+ # Extract text field based on available keys
45
  if 'text' in ds['test'].features:
46
  eval_datasets.append(ds['test'].map(lambda x: {'text': x['text']}))
47
  elif 'content' in ds['test'].features: # Example for CNN/DailyMail