Spaces:
Runtime error
Runtime error
File size: 3,227 Bytes
50e6fbc 82935d8 4c20fbb cf5eed6 3e4a220 50e6fbc 898b5fd 50e6fbc 3e4a220 4b039b3 82935d8 4b039b3 3e4a220 cf5eed6 4b039b3 cf5eed6 4b039b3 4c20fbb cf5eed6 4c20fbb cf5eed6 4c20fbb cf5eed6 4c20fbb 50e6fbc 4c20fbb 4b039b3 cf5eed6 4c20fbb 4b039b3 4c20fbb 50e6fbc 4b039b3 cf5eed6 4c20fbb 4b039b3 cf5eed6 4c20fbb cf5eed6 82935d8 4b039b3 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 |
import os
import gradio as gr
import matplotlib.pyplot as plt
import numpy as np
from functools import partial
from datasets import load_dataset
from pathlib import Path
# get secret environment variable
token = os.environ["HF_TOKEN"]
# write to disk
path = Path("./huggingface")
path.mkdir(parents=True, exist_ok=True)
with open(path/"token", "w") as f:
f.write(token)
dataset_names = [
"AI4Code",
"AMPS",
"ASFPublicMail",
"CPDataset",
"DMMath",
"Discourse",
"Enwiki",
"EuroParliamentProceedings",
"FreeLaw_Options",
"GithubDiff",
"GithubIssues",
"Gutenberg",
"LeetCode",
"PileOfLaw",
"PubMed",
"S2ORC",
"StackExchange",
"USENET",
"USPTO",
"UbuntuIRC",
"arXiv",
]
dataset_data = {}
for name in dataset_names:
path = f"data/{name}/data.json"
ds = load_dataset(
"CarperAI/pilev2_smol_metadata",
data_files=path,
use_auth_token=True,
split="train",
# download_mode="force_redownload",
)
dataset_data[name] = {
"ds": ds,
"word_rep_ratios": np.random.randn(len(ds)),
"char_rep_ratios": np.array(ds["check_char_repetition_criteria"]),
"flagged_word_ratios": np.array(ds["check_flagged_words_criteria"]),
}
def plt_plot(ratio, dataset, threshold):
plt.close("all")
x = dataset_data[dataset][ratio]
# calculate percentage of data that will be removed given threshold
perc = np.sum(x > threshold) / len(x)
# create a figure
fig = plt.figure()
# add a subplot
ax = fig.add_subplot(111)
# plot some data using black
ax.hist(x, bins=50, color="black")
# plot red dashed line at threshold
ax.axvline(threshold, color='r', linestyle='dashed', linewidth=2)
# set title
# add percentage of data removed
ax.set_title(f"{dataset} (removed {perc:.2%})")
plt.xlabel("Value")
plt.ylabel("Frequency")
# make it look nice
plt.tight_layout()
return fig
def check_filtered():
...
with gr.Blocks() as demo:
dataset = gr.Radio(dataset_names, label="Dataset", value="arXiv")
print(dataset.value)
with gr.Tab("Character Repetition Ratio"):
# plot some random data
plot = gr.Plot()
threshold = gr.Slider(minimum=0, maximum=1, label="Threshold")
calculate = gr.Button("Calculate")
check = gr.Button("Check Filtered Data")
plot_fn = partial(plt_plot, "char_rep_ratios")
calculate.click(plot_fn, [dataset, threshold], plot)
with gr.Tab("Word Repetition Ratio"):# plot some random data
plot = gr.Plot()
threshold = gr.Slider(minimum=0, maximum=1, label="Threshold")
calculate = gr.Button("Calculate")
plot_fn = partial(plt_plot, "word_rep_ratios")
calculate.click(plot_fn, [dataset, threshold], plot)
with gr.Tab("Flagged Word Ratio"):# plot some random data
plot = gr.Plot()
threshold = gr.Slider(minimum=0, maximum=1, label="Threshold")
calculate = gr.Button("Calculate")
plot_fn = partial(plt_plot, "flagged_word_ratios")
calculate.click(plot_fn, [dataset, threshold], plot)
if __name__ == "__main__":
demo.launch() |