Spaces:
Runtime error
Runtime error
File size: 3,040 Bytes
50e6fbc 82935d8 4c20fbb cf5eed6 3e4a220 50e6fbc 4b039b3 82935d8 4b039b3 4671e61 4b039b3 3e4a220 cf5eed6 4b039b3 cf5eed6 4b039b3 4c20fbb cf5eed6 4c20fbb cf5eed6 4c20fbb cf5eed6 4c20fbb 50e6fbc 4c20fbb 4b039b3 cf5eed6 4c20fbb 4b039b3 4c20fbb 50e6fbc 4b039b3 cf5eed6 4c20fbb 4b039b3 cf5eed6 4c20fbb cf5eed6 82935d8 4b039b3 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 |
import os
import gradio as gr
import matplotlib.pyplot as plt
import numpy as np
from functools import partial
from datasets import load_dataset
from pathlib import Path
dataset_names = [
"AI4Code",
"AMPS",
"ASFPublicMail",
"CPDataset",
"DMMath",
"Discourse",
"Enwiki",
"EuroParliamentProceedings",
"FreeLaw_Options",
"GithubDiff",
"GithubIssues",
"Gutenberg",
"LeetCode",
"PileOfLaw",
"PubMed",
"S2ORC",
"StackExchange",
"USENET",
"USPTO",
"UbuntuIRC",
"arXiv",
]
dataset_data = {}
for name in dataset_names:
path = f"data/{name}/data.json"
ds = load_dataset(
"CarperAI/pilev2_smol_metadata",
data_files=path,
use_auth_token=os.environ["HF_TOKEN"],
split="train",
# download_mode="force_redownload",
)
dataset_data[name] = {
"ds": ds,
"word_rep_ratios": np.random.randn(len(ds)),
"char_rep_ratios": np.array(ds["check_char_repetition_criteria"]),
"flagged_word_ratios": np.array(ds["check_flagged_words_criteria"]),
}
def plt_plot(ratio, dataset, threshold):
plt.close("all")
x = dataset_data[dataset][ratio]
# calculate percentage of data that will be removed given threshold
perc = np.sum(x > threshold) / len(x)
# create a figure
fig = plt.figure()
# add a subplot
ax = fig.add_subplot(111)
# plot some data using black
ax.hist(x, bins=50, color="black")
# plot red dashed line at threshold
ax.axvline(threshold, color='r', linestyle='dashed', linewidth=2)
# set title
# add percentage of data removed
ax.set_title(f"{dataset} (removed {perc:.2%})")
plt.xlabel("Value")
plt.ylabel("Frequency")
# make it look nice
plt.tight_layout()
return fig
def check_filtered():
...
with gr.Blocks() as demo:
dataset = gr.Radio(dataset_names, label="Dataset", value="arXiv")
print(dataset.value)
with gr.Tab("Character Repetition Ratio"):
# plot some random data
plot = gr.Plot()
threshold = gr.Slider(minimum=0, maximum=1, label="Threshold")
calculate = gr.Button("Calculate")
check = gr.Button("Check Filtered Data")
plot_fn = partial(plt_plot, "char_rep_ratios")
calculate.click(plot_fn, [dataset, threshold], plot)
with gr.Tab("Word Repetition Ratio"):# plot some random data
plot = gr.Plot()
threshold = gr.Slider(minimum=0, maximum=1, label="Threshold")
calculate = gr.Button("Calculate")
plot_fn = partial(plt_plot, "word_rep_ratios")
calculate.click(plot_fn, [dataset, threshold], plot)
with gr.Tab("Flagged Word Ratio"):# plot some random data
plot = gr.Plot()
threshold = gr.Slider(minimum=0, maximum=1, label="Threshold")
calculate = gr.Button("Calculate")
plot_fn = partial(plt_plot, "flagged_word_ratios")
calculate.click(plot_fn, [dataset, threshold], plot)
if __name__ == "__main__":
demo.launch() |