Tristan Thrush
commited on
Commit
·
82b7df7
1
Parent(s):
bce177f
bugfixes
Browse files- app.py +6 -2
- collect.py +3 -3
- requirements.txt +2 -1
app.py
CHANGED
@@ -8,12 +8,16 @@ import gradio as gr
|
|
8 |
import requests
|
9 |
from transformers import pipeline
|
10 |
from huggingface_hub import Repository
|
|
|
|
|
11 |
|
12 |
# These variables are for storing the mturk HITs in a Hugging Face dataset.
|
|
|
|
|
|
|
|
|
13 |
DATA_FILENAME = "data.jsonl"
|
14 |
DATA_FILE = os.path.join("data", DATA_FILENAME)
|
15 |
-
DATASET_REPO_URL = os.environ.get(DATASET_REPO_URL)
|
16 |
-
HF_TOKEN = os.environ.get("HF_TOKEN")
|
17 |
repo = Repository(
|
18 |
local_dir="data", clone_from=DATASET_REPO_URL, use_auth_token=HF_TOKEN
|
19 |
)
|
|
|
8 |
import requests
|
9 |
from transformers import pipeline
|
10 |
from huggingface_hub import Repository
|
11 |
+
from dotenv import load_dotenv
|
12 |
+
from pathlib import Path
|
13 |
|
14 |
# These variables are for storing the mturk HITs in a Hugging Face dataset.
|
15 |
+
if Path(".env").is_file():
|
16 |
+
load_dotenv(".env")
|
17 |
+
DATASET_REPO_URL = os.getenv("DATASET_REPO_URL")
|
18 |
+
HF_TOKEN = os.getenv("HF_TOKEN")
|
19 |
DATA_FILENAME = "data.jsonl"
|
20 |
DATA_FILE = os.path.join("data", DATA_FILENAME)
|
|
|
|
|
21 |
repo = Repository(
|
22 |
local_dir="data", clone_from=DATASET_REPO_URL, use_auth_token=HF_TOKEN
|
23 |
)
|
collect.py
CHANGED
@@ -20,13 +20,13 @@ parser.add_argument("--live_mode", action="store_true", help="""
|
|
20 |
|
21 |
args = parser.parse_args()
|
22 |
|
23 |
-
MTURK_URL = f"https://mturk-requester{
|
24 |
|
25 |
mturk = boto3.client(
|
26 |
"mturk",
|
27 |
aws_access_key_id=MTURK_KEY,
|
28 |
aws_secret_access_key=MTURK_SECRET,
|
29 |
-
region_name=
|
30 |
endpoint_url=MTURK_URL,
|
31 |
)
|
32 |
|
@@ -48,6 +48,6 @@ new_hit = mturk.create_hit(
|
|
48 |
)
|
49 |
|
50 |
print(
|
51 |
-
f"Link: https://worker{
|
52 |
+ new_hit["HIT"]["HITGroupId"]
|
53 |
)
|
|
|
20 |
|
21 |
args = parser.parse_args()
|
22 |
|
23 |
+
MTURK_URL = f"https://mturk-requester{'' if args.live_mode else '-sandbox'}.{args.mturk_region}.amazonaws.com"
|
24 |
|
25 |
mturk = boto3.client(
|
26 |
"mturk",
|
27 |
aws_access_key_id=MTURK_KEY,
|
28 |
aws_secret_access_key=MTURK_SECRET,
|
29 |
+
region_name=args.mturk_region,
|
30 |
endpoint_url=MTURK_URL,
|
31 |
)
|
32 |
|
|
|
48 |
)
|
49 |
|
50 |
print(
|
51 |
+
f"Link: https://worker{'' if args.live_mode else 'sandbox'}.mturk.com/mturk/preview?groupId="
|
52 |
+ new_hit["HIT"]["HITGroupId"]
|
53 |
)
|
requirements.txt
CHANGED
@@ -3,4 +3,5 @@ torch
|
|
3 |
transformers
|
4 |
gradio
|
5 |
boto3
|
6 |
-
huggingface_hub
|
|
|
|
3 |
transformers
|
4 |
gradio
|
5 |
boto3
|
6 |
+
huggingface_hub
|
7 |
+
python-dotenv
|