app.py

import gradio as gr
from langchain.text_splitter import (
    CharacterTextSplitter,
    RecursiveCharacterTextSplitter,
)

LABEL_TEXTSPLITTER = "LangChain's CharacterTextSplitter"
LABEL_RECURSIVE = "Langchain's RecursiveCharacterTextSplitter"

bert_tokenizer = AutoTokenizer.from_pretrained('google-bert/bert-base-uncased')

def extract_separators_from_string(separators_str):
    try:
        separators = separators_str[1:-1].split(", ")
        return [separator.replace('"', "").replace("'", "") for separator in separators]
    except Exception as e:
        print(e)
        raise gr.Error(f"""
        Did not succeed in extracting seperators from string: {separator_str}.
        Please type it in the correct format: "['separator_1', 'separator_2', etc]"
        """)

def change_split_selection(text, slider_count, split_selection, separator_selection, length_unit_selection):
    print("Updating separator selection interactivity:")
    return (
        gr.Textbox.update(visible=(split_selection==LABEL_RECURSIVE)),
        chunk(text, slider_count, split_selection, separator_selection, length_unit_selection)
    )

def chunk(text, length, splitter_selection, separators_str, length_unit_selection):
    separators = extract_separators_from_string(separators_str)
    
    if splitter_selection == LABEL_TEXTSPLITTER:
        if "token" in length_unit_selection.lower():
            text_splitter = CharacterTextSplitter.from_huggingface_tokenizer(
                AutoTokenizer.from_pretrained(tokenizer_name),
                separator="",
                chunk_size=length,
                chunk_overlap=0,
                length_function=len,
                is_separator_regex=False,
            )
        else:
            text_splitter = CharacterTextSplitter(
                separator="",
                chunk_size=length,
                chunk_overlap=0,
                length_function=len,
                is_separator_regex=False,
            )
    elif splitter_selection == LABEL_RECURSIVE:
        if "token" in length_unit_selection.lower():
            text_splitter = RecursiveCharacterTextSplitter.from_huggingface_tokenizer(
                AutoTokenizer.from_pretrained(tokenizer_name),
                chunk_size=chunk_size,
                chunk_overlap=0,
                add_start_index=True,
                strip_whitespace=False,
                separators=separators,
            )
        else:
            text_splitter = RecursiveCharacterTextSplitter(
                chunk_size=length,
                chunk_overlap=0,
                length_function=len,
                add_start_index=True,
                strip_whitespace=False,
                separators=separators,
            )
    splits = text_splitter.create_documents([text])
    text_splits = [split.page_content for split in splits]

    output = [(split, str(i)) for i, split in enumerate(text_splits)]
    return output


ESSAY = """Chapter 6

WHAT SORT OF DESPOTISM DEMOCRATIC NATIONS HAVE TO FEAR

I had remarked during my stay in the United States that a democratic state of society, similar to that of the Americans, might offer singular facilities for the establishment of despotism; and I perceived, upon my return to Europe, how much use had already been made, by most of our rulers, of the notions, the sentiments, and the wants created by this same social condition, for the purpose of extending the circle of their power. This led me to think that the nations of Christendom would perhaps eventually undergo some oppression like that which hung over several of the nations of the ancient world.
A more accurate examination of the subject, and five years of further meditation, have not diminished my fears, but have changed their object.
No sovereign ever lived in former ages so absolute or so powerful as to undertake to administer by his own agency, and without the assistance of intermediate powers, all the parts of a great empire; none ever attempted to subject all his subjects indiscriminately to strict uniformity of regulation and personally to tutor and direct every member of the community. The notion of such an undertaking never occurred to the human mind; and if any man had conceived it, the want of information, the imperfection of the administrative system, and, above all, the natural obstacles caused by the inequality of conditions would speedily have checked the execution of so vast a design.
When the Roman emperors were at the height of their power, the different nations of the empire still preserved usages and customs of great diversity; although they were subject to the same monarch, most of the provinces were separately administered; they abounded in powerful and active municipalities; and although the whole government of the empire was centered in the hands of the Emperor alone and he always remained, in case of need, the supreme arbiter in all matters, yet the details of social life and private occupations lay for the most part beyond his control. The emperors possessed, it is true, an immense and unchecked power, which allowed them to gratify all their whimsical tastes and to employ for that purpose the whole strength of the state. They frequently abused that power arbitrarily to deprive their subjects of property or of life; their tyranny was extremely onerous to the few, but it did not reach the many; it was confined to some few main objects and neglected the rest; it was violent, but its range was limited.

---

Then you can [Create a dataset repository](../huggingface_hub/quick-start#create-a-repository), for example using:

```python
from huggingface_hub import HfApi
HfApi().create_repo(repo_id="username/my_dataset", repo_type="dataset")
```
Finally, you can use [Hugging Face paths]([Hugging Face paths](https://huggingface.co/docs/huggingface_hub/guides/hf_file_system#integrations)) in Pandas:
```python
import pandas as pd
df.to_parquet("hf://datasets/username/my_dataset/data.parquet")
# or write in separate files if the dataset has train/validation/test splits
df_train.to_parquet("hf://datasets/username/my_dataset/train.parquet")
df_valid.to_parquet("hf://datasets/username/my_dataset/validation.parquet")
df_test .to_parquet("hf://datasets/username/my_dataset/test.parquet")
```
"""


with gr.Blocks(theme=gr.themes.Soft(), css="#textbox_id {color: red; font-samily:monospace}") as demo:
    text = gr.Textbox(label="Your text 🪶", value=ESSAY)
    with gr.Row():
        split_selection = gr.Dropdown(
            choices=[
                LABEL_TEXTSPLITTER,
                LABEL_RECURSIVE,
            ],
            value=LABEL_TEXTSPLITTER,
            label="Chunking method ",
        )
        separator_selection = gr.Textbox(
            elem_id="textbox_id",
            value=["\n\n", "\n", ".", " ", ""],
            label="Separators used in RecursiveCharacterTextSplitter",
            visible=False,
        )
    with gr.Row():
        length_unit_selection = gr.Dropdown(
            choices=[
                "Character count",
                "Token count (BERT tokens)",
            ],
            value="Character count",
            label="Length function",
            info="How should we count our chunk lengths?",
        )
        slider_count = gr.Slider(
            20, 500, value=50, label="Count 🧮", info="Chunk size, in the chosen unit."
        )
    out = gr.HighlightedText(
        label="Output",
        show_legend=True,
        show_label=False,
    )
    text.change(
        fn=chunk,
        inputs=[text, slider_count, split_selection, separator_selection, length_unit_selection],
        outputs=out,
    )
    length_unit_selection.change(
        fn=chunk,
        inputs=[text, slider_count, split_selection, separator_selection, length_unit_selection],
        outputs=out,
    )
    split_selection.change(
        fn=change_split_selection,
        inputs=[text, slider_count, split_selection, separator_selection, length_unit_selection],
        outputs=[separator_selection, out],
    )
    slider_count.change(
        fn=chunk,
        inputs=[text, slider_count, split_selection, separator_selection, length_unit_selection],
        outputs=out,
    )
demo.launch()