Spaces:
Running
Running
| import PyPDF2 | |
| from openpyxl import load_workbook | |
| from pptx import Presentation | |
| import gradio as gr | |
| import io | |
| import re | |
| import zipfile | |
| import xml.etree.ElementTree as ET | |
| import filetype | |
| import requests | |
| import os | |
| import mimetypes | |
| from bs4 import BeautifulSoup | |
| from urllib.parse import urljoin | |
| # Constants | |
| CHUNK_SIZE = 32000 | |
| # --- Utility Functions --- | |
| def xml2text(xml): | |
| """Extracts text from XML data.""" | |
| text = u'' | |
| root = ET.fromstring(xml) | |
| for child in root.iter(): | |
| text += child.text + " " if child.text is not None else '' | |
| return text | |
| def clean_text(content): | |
| """Cleans text content based on the 'clean' parameter.""" | |
| content = content.replace('\n', ' ') | |
| content = content.replace('\r', ' ') | |
| content = content.replace('\t', ' ') | |
| content = re.sub(r'\s+', ' ', content) | |
| return content | |
| def split_content(content, chunk_size=CHUNK_SIZE): | |
| """Splits content into chunks of a specified size.""" | |
| chunks = [] | |
| for i in range(0, len(content), chunk_size): | |
| chunks.append(content[i:i + chunk_size]) | |
| return chunks | |
| # --- Document Reading Functions --- | |
| def extract_text_from_docx(docx_data, clean=True): | |
| """Extracts text from DOCX files.""" | |
| text = u'' | |
| zipf = zipfile.ZipFile(io.BytesIO(docx_data)) | |
| filelist = zipf.namelist() | |
| header_xmls = 'word/header[0-9]*.xml' | |
| for fname in filelist: | |
| if re.match(header_xmls, fname): | |
| text += xml2text(zipf.read(fname)) | |
| doc_xml = 'word/document.xml' | |
| text += xml2text(zipf.read(doc_xml)) | |
| footer_xmls = 'word/footer[0-9]*.xml' | |
| for fname in filelist: | |
| if re.match(footer_xmls, fname): | |
| text += xml2text(zipf.read(fname)) | |
| zipf.close() | |
| if clean: | |
| text = clean_text(text) | |
| return text, len(text) | |
| def extract_text_from_pptx(pptx_data, clean=True): | |
| """Extracts text from PPT files.""" | |
| text = u'' | |
| zipf = zipfile.ZipFile(io.BytesIO(pptx_data)) | |
| filelist = zipf.namelist() | |
| # Extract text from slide notes | |
| notes_xmls = 'ppt/notesSlides/notesSlide[0-9]*.xml' | |
| for fname in filelist: | |
| if re.match(notes_xmls, fname): | |
| text += xml2text(zipf.read(fname)) | |
| # Extract text from slide content (shapes and text boxes) | |
| slide_xmls = 'ppt/slides/slide[0-9]*.xml' | |
| for fname in filelist: | |
| if re.match(slide_xmls, fname): | |
| text += xml2text(zipf.read(fname)) | |
| zipf.close() | |
| if clean: | |
| text = clean_text(text) | |
| return text, len(text) | |
| def read_document(file_path, clean=True, url=""): | |
| with open(file_path, "rb") as f: | |
| file_content = f.read() | |
| kind = filetype.guess(file_content) | |
| if kind is None: | |
| mime = "text/html" | |
| else: | |
| mime = kind.mime | |
| if mime == "application/pdf": | |
| try: | |
| pdf_reader = PyPDF2.PdfReader(io.BytesIO(file_content)) | |
| content = '' | |
| for page in range(len(pdf_reader.pages)): | |
| content += pdf_reader.pages[page].extract_text() | |
| if clean: | |
| content = clean_text(content) | |
| return content, len(repr(content)) | |
| except Exception as e: | |
| return f"Error reading PDF: {e}", 0 | |
| elif mime == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": | |
| try: | |
| wb = load_workbook(io.BytesIO(file_content)) | |
| content = '' | |
| for sheet in wb.worksheets: | |
| for row in sheet.rows: | |
| for cell in row: | |
| if cell.value is not None: | |
| content += str(cell.value) + ' ' | |
| if clean: | |
| content = clean_text(content) | |
| return content, len(repr(content)) | |
| except Exception as e: | |
| return f"Error reading XLSX: {e}", 0 | |
| elif mime == "text/plain": | |
| try: | |
| content = file_content.decode('utf-8') | |
| if clean: | |
| content = clean_text(content) | |
| return content, len(repr(content)) | |
| except Exception as e: | |
| return f"Error reading TXT file: {e}", 0 | |
| elif mime == "text/csv": | |
| try: | |
| content = file_content.decode('utf-8') | |
| if clean: | |
| content = clean_text(content) | |
| return content, len(repr(content)) | |
| except Exception as e: | |
| return f"Error reading CSV file: {e}", 0 | |
| elif mime == "application/vnd.openxmlformats-officedocument.wordprocessingml.document": | |
| try: | |
| return extract_text_from_docx(file_content, clean) | |
| except Exception as e: | |
| return f"Error reading DOCX: {e}", 0 | |
| elif mime == "application/vnd.openxmlformats-officedocument.presentationml.presentation": | |
| try: | |
| return extract_text_from_pptx(file_content, clean) | |
| except Exception as e: | |
| return f"Error reading PPTX: {e}", 0 | |
| elif mime == "text/html": # Handle HTML content | |
| try: | |
| soup = BeautifulSoup(file_content, 'html.parser') | |
| structured_data = { | |
| "Texts": extract_texts(soup), | |
| "Links": extract_links(soup, url), | |
| "Images": extract_images(soup, url) | |
| } | |
| return format_detailed_output(structured_data), 0 | |
| except Exception as e: | |
| return f"Error parsing HTML content: {e}", 0 | |
| else: | |
| try: | |
| content = file_content.decode('utf-8') | |
| if clean: | |
| content = clean_text(content) | |
| return content, len(repr(content)) | |
| except Exception as e: | |
| return f"Error reading file: {e}", 0 | |
| def download_and_process_file(url, clean=True): | |
| """Downloads a file from a URL and returns the local file path.""" | |
| if not url.startswith("http://") and not url.startswith("https://"): | |
| url = "http://" + url # Prepend "http://" if not present | |
| try: | |
| response = requests.get(url, stream=True, timeout=10) | |
| response.raise_for_status() # Raise an exception for bad status codes | |
| # Generate a safe and unique temporary filename | |
| original_filename = os.path.basename(url) | |
| # Remove invalid characters from filename | |
| safe_filename = re.sub(r'[^\w\-_\. ]', '_', original_filename) | |
| temp_filename = f"{safe_filename}" | |
| # Infer file extension from content type | |
| content_type = response.headers['content-type'] | |
| ext = mimetypes.guess_extension(content_type) | |
| if ext and not temp_filename.endswith(ext): # Append extension if not already present | |
| temp_filename += ext | |
| with open(temp_filename, 'wb') as f: | |
| for chunk in response.iter_content(chunk_size=8192000): | |
| f.write(chunk) | |
| # Check if it's an image type | |
| kind = filetype.guess(temp_filename) | |
| if kind and kind.mime.startswith('image/'): | |
| return f"", 0 # Return markdown image syntax if it's an image | |
| else: | |
| return read_document(temp_filename, clean, url) # Otherwise, process as a document | |
| except requests.exceptions.MissingSchema: | |
| return "Error: Invalid URL format. Even after adding 'http://', the URL is still invalid.", 0 | |
| except requests.exceptions.ConnectionError: | |
| return "Error: Could not connect to the server. Please check your internet connection.", 0 | |
| except requests.exceptions.Timeout: | |
| return "Error: Connection timed out while trying to fetch the URL.", 0 | |
| except requests.exceptions.RequestException as e: | |
| return f"Error downloading file: {e}", 0 | |
| # --- Web Page Content Extraction Functions (from previous code) --- | |
| def extract_texts(soup): | |
| """Extracts all text content from the soup.""" | |
| return [text for text in soup.stripped_strings] | |
| def extract_links(soup, base_url): | |
| """Extracts all valid links from the soup.""" | |
| links = [] | |
| for link in soup.find_all('a', href=True): | |
| href = link['href'] | |
| # Use urljoin to create an absolute URL | |
| full_url = urljoin(base_url, href) if not href.startswith(("http://", "https://")) else href | |
| link_text = link.get_text(strip=True) or "No Text" | |
| links.append({"Text": link_text, "URL": full_url}) | |
| return links | |
| def extract_images(soup, base_url): | |
| """Extracts all valid image URLs and their alt text from the soup.""" | |
| images = [] | |
| for img in soup.find_all('img', src=True): | |
| img_url = img['src'] | |
| # Use urljoin to create an absolute URL | |
| full_img_url = urljoin(base_url, img_url) if not img_url.startswith(("http://", "https://")) else img_url | |
| alt_text = img.get('alt', 'No Alt Text') | |
| images.append({"Alt Text": alt_text, "Image URL": full_img_url}) | |
| return images | |
| def fetch_page_content(url): | |
| """Fetches the content of the page at the given URL.""" | |
| try: | |
| response = requests.get(url, timeout=10) | |
| response.raise_for_status() | |
| return response.text | |
| except requests.exceptions.RequestException as e: | |
| return f"Error fetching the URL: {e}" | |
| def format_detailed_output(structured_data): | |
| """Formats the structured data into a Markdown string.""" | |
| result = "### Structured Page Content\n\n" | |
| result += "**Texts:**\n" + (" ".join(structured_data["Texts"]) if structured_data["Texts"] else "No textual content found.") + "\n\n" | |
| result += "**Links:**\n" | |
| if structured_data["Links"]: | |
| result += "\n".join(f"[{link['Text']}]({link['URL']})" for link in structured_data["Links"]) + "\n" | |
| else: | |
| result += "No links found.\n" | |
| result += "**Images:**\n" | |
| if structured_data["Images"]: | |
| result += "\n".join(f"![{img['Alt Text']}]({img['Image URL']})" for img in structured_data["Images"]) + "\n" | |
| else: | |
| result += "No images found.\n" | |
| return result | |
| def extract_page_content(url): | |
| """Extracts and formats the content of the page at the given URL.""" | |
| page_content = fetch_page_content(url) | |
| if "Error" in page_content: | |
| return page_content | |
| soup = BeautifulSoup(page_content, 'html.parser') | |
| structured_data = { | |
| "Texts": extract_texts(soup), | |
| "Links": extract_links(soup, url), # Pass the base URL | |
| "Images": extract_images(soup, url) # Pass the base URL | |
| } | |
| return format_detailed_output(structured_data) | |
| # --- Gradio Interface --- | |
| iface = gr.Interface( | |
| fn=download_and_process_file, | |
| inputs=[ | |
| gr.Textbox(lines=1, placeholder="Enter URL of the file"), | |
| gr.Checkbox(label="Clean Text", value=True), | |
| ], | |
| outputs=[ | |
| gr.Markdown(label="Document Content/Image Markdown/Web Page Content"), | |
| gr.Number(label="Document Length (characters)"), | |
| ], | |
| title="Enhanced File and Web Page Processor for Hugging Face Chat Tools", | |
| description="Enter the URL of an image, video, document, or web page. The tool will handle it accordingly: images will be displayed as Markdown, documents will have their text extracted, and web pages will have their content structured and displayed. This tool is designed for use with Hugging Face Chat Tools. \n [https://hf.co/chat/tools/66f1a8159d41ad4398ebb711](https://hf.co/chat/tools/66f1a8159d41ad4398ebb711)", | |
| concurrency_limit=None | |
| ) | |
| iface.launch() |