Update app.py

app.py

@@ -4,18 +4,32 @@ import requests
 from io import BytesIO
 
 def convert_parquet_to_jsonl(parquet_file=None, parquet_url=None):
+    # Read Parquet from file or URL
     if parquet_file is not None:
         df = pd.read_parquet(parquet_file.name)
     elif parquet_url is not None:
         response = requests.get(parquet_url)
-        response.raise_for_status()  # Ensure the request was successful
+        response.raise_for_status()
         df = pd.read_parquet(BytesIO(response.content))
     else:
         raise ValueError("Either parquet_file or parquet_url must be provided")
 
-    jsonl_data = df.to_json(orient='records', lines=True)
+    # Function to clean any invalid UTF-8 sequences by replacing them
+    def clean_string(val):
+        if isinstance(val, str):
+            # Re-encode string to UTF-8, replacing invalid characters
+            return val.encode("utf-8", errors="replace").decode("utf-8", errors="replace")
+        return val
+
+    # Apply cleaning to every cell in the DataFrame
+    df = df.applymap(clean_string)
+
+    # Now safely convert to JSON Lines
+    jsonl_data = df.to_json(orient="records", lines=True)
+
+    # Write out to a file
     output_file_path = "output.jsonl"
-    with open(output_file_path, "w") as f:
+    with open(output_file_path, "w", encoding="utf-8") as f:
         f.write(jsonl_data)
 
     return output_file_path