Spaces:
Build error
Build error
Commit
·
a3ff196
1
Parent(s):
6e1398d
Update app.py
Browse files
app.py
CHANGED
@@ -4,23 +4,19 @@ import json
|
|
4 |
|
5 |
# Read requirements.txt file
|
6 |
with open('requirements.txt', 'r') as req_file:
|
7 |
-
|
8 |
|
9 |
-
# Install the required libraries
|
10 |
-
for requirement in requirements:
|
11 |
-
# Use your preferred method to install the libraries
|
12 |
-
# e.g., subprocess, pip, etc.
|
13 |
|
14 |
# Load and preprocess the IMDB dataset in JSON format
|
15 |
with open('IMDB Dataset.json', 'r') as json_file:
|
16 |
-
|
17 |
|
18 |
# Select only 30 words from the dataset
|
19 |
preprocessed_data = []
|
20 |
for entry in imdb_data:
|
21 |
-
|
22 |
-
|
23 |
-
|
24 |
'text': ' '.join(words),
|
25 |
'label': entry['label']
|
26 |
}
|
@@ -32,7 +28,7 @@ dataset = load_dataset('json', data=preprocessed_data)
|
|
32 |
# Tokenize the dataset
|
33 |
tokenizer = AutoTokenizer.from_pretrained("bigscience/bloom-560m")
|
34 |
def tokenize_function(examples):
|
35 |
-
|
36 |
|
37 |
tokenized_datasets = dataset.map(tokenize_function, batched=True)
|
38 |
|
|
|
4 |
|
5 |
# Read requirements.txt file
|
6 |
with open('requirements.txt', 'r') as req_file:
|
7 |
+
requirements = req_file.read().splitlines()
|
8 |
|
|
|
|
|
|
|
|
|
9 |
|
10 |
# Load and preprocess the IMDB dataset in JSON format
|
11 |
with open('IMDB Dataset.json', 'r') as json_file:
|
12 |
+
imdb_data = json.load(json_file)
|
13 |
|
14 |
# Select only 30 words from the dataset
|
15 |
preprocessed_data = []
|
16 |
for entry in imdb_data:
|
17 |
+
text = entry['text']
|
18 |
+
words = text.split()[:30]
|
19 |
+
preprocessed_entry = {
|
20 |
'text': ' '.join(words),
|
21 |
'label': entry['label']
|
22 |
}
|
|
|
28 |
# Tokenize the dataset
|
29 |
tokenizer = AutoTokenizer.from_pretrained("bigscience/bloom-560m")
|
30 |
def tokenize_function(examples):
|
31 |
+
return tokenizer(examples["text"], padding="max_length", truncation=True)
|
32 |
|
33 |
tokenized_datasets = dataset.map(tokenize_function, batched=True)
|
34 |
|