app.py

import gradio as gr
import pandas as pd
import requests
from io import BytesIO

def convert_parquet_to_jsonl(parquet_file=None, parquet_url=None):
    # Read Parquet from file or URL
    if parquet_file is not None:
        df = pd.read_parquet(parquet_file.name)
    elif parquet_url is not None:
        response = requests.get(parquet_url)
        response.raise_for_status()
        df = pd.read_parquet(BytesIO(response.content))
    else:
        raise ValueError("Either parquet_file or parquet_url must be provided")

    # Function to clean any invalid UTF-8 sequences by replacing them
    def clean_string(val):
        if isinstance(val, str):
            # Re-encode string to UTF-8, replacing invalid characters
            return val.encode("utf-8", errors="replace").decode("utf-8", errors="replace")
        return val

    # Apply cleaning to every cell in the DataFrame
    df = df.applymap(clean_string)

    # Now safely convert to JSON Lines
    jsonl_data = df.to_json(orient="records", lines=True)

    # Write out to a file
    output_file_path = "output.jsonl"
    with open(output_file_path, "w", encoding="utf-8") as f:
        f.write(jsonl_data)

    return output_file_path

demo = gr.Interface(
    fn=convert_parquet_to_jsonl,
    inputs=[gr.File(label="Parquet File"), gr.Textbox(label="Parquet File URL")],
    outputs=[gr.File(label="JSONL Output")],
    title="Parquet to JSONL Converter",
    description="Convert a Parquet file to JSONL format from a downloadable link or file upload"
)

if __name__ == "__main__":
    demo.launch()