Spaces:
Running
Running
| import logging | |
| import json | |
| import re | |
| import math | |
| from deep_translator import GoogleTranslator | |
| from gematria import calculate_gematria # Stelle sicher, dass diese Funktion korrekt definiert ist | |
| # Konfiguriere den Logger | |
| #logging.basicConfig(level=logging.DEBUG, format='%(levelname)s:%(message)s') | |
| logger = logging.getLogger(__name__) | |
| def process_json_files(start=1, end=10, step=1, rounds="1", length=0, tlang="en", strip_spaces=True, | |
| strip_in_braces=True, strip_diacritics=True, translate=False): | |
| """ | |
| Processes Rigveda JSON files and performs various text manipulations. | |
| Parameters: | |
| - start (int): Start number of the Mandala. | |
| - end (int): End number of the Mandala. | |
| - step (int): Step size for character selection. | |
| - rounds (str): Comma-separated list of round numbers (can include floats). | |
| - length (int): Maximum length of the result text. | |
| - tlang (str): Target language for translation. | |
| - strip_spaces (bool): Whether to remove spaces. | |
| - strip_in_braces (bool): Whether to remove text within brackets. | |
| - strip_diacritics (bool): Whether to remove diacritics. | |
| - translate (bool): Whether to translate the result text. | |
| Returns: | |
| - list: A list of dictionaries with the processed data or error messages. | |
| """ | |
| base_path = "texts/rigveda" | |
| translator = GoogleTranslator(source='sa', target=tlang) if translate else None # Correct source language for Sanskrit | |
| results = [] | |
| for i in range(start, end + 1): | |
| file_name = f"{base_path}/rigveda_mandala_{i:02}.json" | |
| # Optional, if needed: | |
| # file_name = f"{base_path}/mahabharata_book_{i:02}.json" | |
| try: | |
| with open(file_name, 'r', encoding='utf-8') as file: | |
| data = json.load(file) | |
| # Concatenate all Suktas for the current Mandala | |
| full_text = " ".join(sukta.get("text", "") for sukta in data) | |
| clean_text = full_text | |
| if strip_in_braces: | |
| # Remove content within [], {}, <> brackets | |
| clean_text = re.sub(r"\[.*?\]|\{.*?\}|\<.*?\>", "", clean_text, flags=re.DOTALL) | |
| if strip_diacritics: | |
| # Keep only Devanagari characters and spaces | |
| clean_text = re.sub(r'[^\u0900-\u097F\s]', '', clean_text) | |
| # Remove specific diacritics and punctuation | |
| clean_text = re.sub(r'[\u0951-\u0954\u0964\u0965]+', '', clean_text) | |
| # Remove Devanagari numerals | |
| clean_text = re.sub(r'[०१२३४५६७८९]+', '', clean_text) | |
| # Remove additional unwanted characters | |
| clean_text = re.sub(r'[:?!\'\-]', '', clean_text) | |
| # Normalize spaces | |
| clean_text = clean_text.replace("\n\n ", " ") | |
| clean_text = clean_text.replace("\n", " ") | |
| clean_text = re.sub(r'\s+', ' ', clean_text).strip() | |
| if strip_spaces: | |
| clean_text = clean_text.replace(" ", "") | |
| text_length = len(clean_text) | |
| logger.debug(f"Mandala {i}: Clean text length = {text_length}") | |
| if text_length == 0: | |
| logger.warning(f"Mandala {i}: No text available after cleaning.") | |
| continue # Skip processing if no text is available | |
| try: | |
| rounds_list = list(map(float, rounds.split(','))) | |
| except ValueError as e: | |
| logger.error(f"Mandala {i}: Invalid 'rounds' format: {rounds}") | |
| results.append({"error": f"Mandala {i}: Invalid 'rounds' format: {rounds}"}) | |
| continue # Skip this Mandala due to invalid 'rounds' input | |
| result_text = "" | |
| for r in rounds_list: | |
| abs_r = abs(r) | |
| # Determine the number of full passes and the remainder. | |
| full_passes = math.floor(abs_r) | |
| remainder = abs_r - full_passes | |
| # Base number of characters per pass | |
| base_chars = text_length // step | |
| if base_chars == 0: | |
| if abs_r > 1: # Changed from >=1 to >1 | |
| # When step > text_length and rounds >1, pick 1 character per full pass | |
| chars_per_full_pass = 1 | |
| logger.debug(f"Mandala {i}: step > text_length ({step} > {text_length}), selecting 1 character per full pass.") | |
| else: | |
| # No characters to pick | |
| chars_per_full_pass = 0 | |
| logger.debug(f"Mandala {i}: step > text_length ({step} > {text_length}) and rounds <=1, no characters selected.") | |
| # For remainder, since base_chars=0, no remainder characters | |
| chars_for_remainder = 0 | |
| else: | |
| # Normal case | |
| chars_per_full_pass = base_chars | |
| chars_for_remainder = math.floor(base_chars * remainder) # Partial pass | |
| logger.debug(f"Mandala {i}: Normal case, chars_per_full_pass = {chars_per_full_pass}, chars_for_remainder = {chars_for_remainder}") | |
| if r > 0: | |
| current_index = (step - 1) % text_length | |
| direction = 1 | |
| else: | |
| current_index = (text_length - step) % text_length | |
| direction = -1 | |
| pass_result = "" | |
| # Full passes, keep only the last pass | |
| for pass_num in range(1, full_passes + 1): | |
| current_pass_chars = "" | |
| for _ in range(chars_per_full_pass): | |
| if chars_per_full_pass == 0: | |
| break | |
| current_pass_chars += clean_text[current_index] | |
| current_index = (current_index + direction * step) % text_length | |
| # Keep only the last full pass | |
| if pass_num == full_passes: | |
| pass_result = current_pass_chars | |
| logger.debug(f"Mandala {i}: Pass {pass_num}, pass_result = {pass_result}") | |
| # Remainder pass for fractional rounds | |
| if remainder > 0 and chars_for_remainder > 0: | |
| current_pass_chars = "" | |
| for _ in range(chars_for_remainder): | |
| current_pass_chars += clean_text[current_index] | |
| current_index = (current_index + direction * step) % text_length | |
| pass_result += current_pass_chars | |
| logger.debug(f"Mandala {i}: Remainder pass_result = {pass_result}") | |
| # Handle cases where step > text_length and chars_per_full_pass=1 | |
| if base_chars == 0 and chars_per_full_pass == 1 and full_passes > 0: | |
| # pass_result already contains the last character picked | |
| pass | |
| elif base_chars == 0 and chars_per_full_pass == 0 and full_passes > 0: | |
| # When no characters are picked, skip appending | |
| pass | |
| result_text += pass_result | |
| if length != 0: | |
| result_text = result_text[:length] | |
| logger.debug(f"Mandala {i}: Result text truncated to length {length}.") | |
| # Translate the result text if required | |
| try: | |
| translated_text = translator.translate(result_text) if translator and result_text else "" | |
| except Exception as e: | |
| logger.error(f"Mandala {i}: Translation error: {e}") | |
| translated_text = "" | |
| # Calculate the Gematria sum | |
| try: | |
| result_sum = calculate_gematria(result_text) | |
| except Exception as e: | |
| logger.error(f"Mandala {i}: Gematria calculation error: {e}") | |
| result_sum = None | |
| if result_text: | |
| results.append({ | |
| "book": f"Rigveda {i}.", | |
| "title": f"Mandala {i}", | |
| "result_text": result_text, | |
| "result_sum": result_sum, | |
| "translated_text": translated_text, | |
| "source_language": "sa", | |
| }) | |
| except FileNotFoundError: | |
| logger.error(f"Mandala {i}: File not found: {file_name}") | |
| results.append({"error": f"Mandala {i}: File not found: {file_name}"}) | |
| except json.JSONDecodeError as e: | |
| logger.error(f"Mandala {i}: JSON error in file {file_name}: {e}") | |
| results.append({"error": f"Mandala {i}: JSON error in file {file_name}: {e}"}) | |
| except KeyError as e: | |
| logger.error(f"Mandala {i}: Missing key in file {file_name}: {e}") | |
| results.append({"error": f"Mandala {i}: Missing key in file {file_name}: {e}"}) | |
| except Exception as e: | |
| logger.error(f"Mandala {i}: Unexpected error in file {file_name}: {e}") | |
| results.append({"error": f"Mandala {i}: Unexpected error in file {file_name}: {e}"}) | |
| return results if results else None | |