| import argparse | |
| import jsonlines | |
| import json | |
| from tqdm import tqdm | |
| import uuid | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--in-file", type=str, default="flan1m-alpaca-uncensored.jsonl") | |
| parser.add_argument("--out-file", type=str, default="flan1m-sharegpt-deduped.json") | |
| args = parser.parse_args() | |
| in_file = args.in_file | |
| out_file = args.out_file | |
| f = open(out_file, "w", encoding="utf-8") | |
| questions = {} | |
| out = [] | |
| with jsonlines.open(in_file) as reader: | |
| for obj in tqdm(reader): | |
| if questions.get(obj["instruction"] + obj["input"]) is None: | |
| questions[obj["instruction"] + obj["input"]] = True | |
| out.append( | |
| { | |
| "id": f"{uuid.uuid4()}", | |
| "bot": "dolphin", | |
| "training": obj["instruction"], | |
| "conversations": [ | |
| {"from": "human", "value": obj["input"]}, | |
| {"from": "gpt", "value": obj["output"]}, | |
| ], | |
| } | |
| ) | |
| json.dump(out, f, ensure_ascii=False) | |
| f.close() | |