import logging import json import re import math import glob from deep_translator import GoogleTranslator from gematria import calculate_gematria # Ensure this function is correctly defined import csv # Configure the logger # You can uncomment the next line to enable debugging logs # logging.basicConfig(level=logging.DEBUG, format='%(levelname)s:%(message)s') logger = logging.getLogger(__name__) def process_json_files(start=1, end=52, step=1, rounds="1,-1", length=0, tlang="en", strip_spaces=True, strip_in_braces=True, strip_diacritics=True, translate=False): """ Processes Tripitaka JSON files and performs various text manipulations. Parameters: - start (int): Start number of the Tripitaka book. - end (int): End number of the Tripitaka book. - step (int): Step size for character selection. - rounds (str): Comma-separated list of round numbers (can include negative values). - length (int): Maximum length of the result text. - tlang (str): Target language for translation. - strip_spaces (bool): Whether to remove spaces from the text. - strip_in_braces (bool): Whether to remove text within braces. - strip_diacritics (bool): Whether to remove diacritics from the text. - translate (bool): Whether to translate the result text. Returns: - list: A list of dictionaries containing processed data or error messages. """ base_path = "texts/tripitaka" translator = GoogleTranslator(source='sa', target=tlang) if translate else None # Correct source language for Sanskrit results = [] for i in range(start, end + 1): file_pattern = f"{base_path}/{i:02}*.json" # Adjust to Tripitaka file naming matched_files = glob.glob(file_pattern) if not matched_files: logger.warning(f"Mandala {i}: No files found for pattern {file_pattern}.") results.append({"error": f"Mandala {i}: No files found for pattern {file_pattern}."}) continue for file_name in matched_files: try: with open(file_name, 'r', encoding='utf-8') as file: data = json.load(file) full_text = "" for gatha in data.get("gathas", []): # Safe access to 'gathas' full_text += gatha + " " clean_text = full_text if strip_in_braces: # Remove content within [], {}, <> brackets clean_text = re.sub(r"\[.*?\]|\{.*?\}|\<.*?\>", "", clean_text, flags=re.DOTALL) if strip_diacritics: # Keep only Devanagari characters and spaces clean_text = re.sub(r'[^\u0900-\u097F\s]', '', clean_text) # Remove specific diacritics and punctuation clean_text = re.sub(r'[\u0951-\u0954\u0964\u0965]+', '', clean_text) # Remove Devanagari numerals clean_text = re.sub(r'[०१२३४५६७८९]+', '', clean_text) # Remove additional unwanted characters in one step clean_text = re.sub(r'[:?!\'\-]', '', clean_text) # Normalize spaces clean_text = clean_text.replace("\n\n ", " ") clean_text = clean_text.replace("\n", " ") clean_text = re.sub(r'\s+', ' ', clean_text).strip() if strip_spaces: clean_text = clean_text.replace(" ", "") text_length = len(clean_text) logger.debug(f"Mandala {i}: Clean text length = {text_length}") if text_length == 0: logger.warning(f"Mandala {i}: No text available after cleaning.") continue # Skip processing if no text is available # Parse 'rounds', allow floats try: rounds_list = list(map(float, rounds.split(','))) except ValueError as e: logger.error(f"Mandala {i}: Invalid 'rounds' format: {rounds}") results.append({"error": f"Mandala {i}: Invalid 'rounds' format: {rounds}"}) continue # Skip this Mandala due to invalid 'rounds' input result_text = "" for r in rounds_list: abs_r = abs(r) # Determine the number of full passes and the remainder. full_passes = math.floor(abs_r) remainder = abs_r - full_passes # Base number of characters per pass base_chars = text_length // step if base_chars == 0: if abs_r > 1: # Changed from >=1 to >1 # When step > text_length and rounds >1, pick 1 character per full pass chars_per_full_pass = 1 logger.debug(f"Mandala {i}: step > text_length ({step} > {text_length}), selecting 1 character per full pass.") else: # No characters to pick chars_per_full_pass = 0 logger.debug(f"Mandala {i}: step > text_length ({step} > {text_length}) and rounds <=1, no characters selected.") # For remainder, since base_chars=0, no remainder characters chars_for_remainder = 0 else: # Normal case chars_per_full_pass = base_chars chars_for_remainder = math.floor(base_chars * remainder) # Partial pass logger.debug(f"Mandala {i}: Normal case, chars_per_full_pass = {chars_per_full_pass}, chars_for_remainder = {chars_for_remainder}") if r > 0: current_index = (step - 1) % text_length direction = 1 else: current_index = (text_length - step) % text_length direction = -1 pass_result = "" # Full passes, keep only the last pass for pass_num in range(1, full_passes + 1): current_pass_chars = "" for _ in range(chars_per_full_pass): if chars_per_full_pass == 0: break current_pass_chars += clean_text[current_index] current_index = (current_index + direction * step) % text_length # Keep only the last full pass if pass_num == full_passes: pass_result = current_pass_chars logger.debug(f"Mandala {i}: Pass {pass_num}, pass_result = {pass_result}") # Remainder pass for fractional rounds if remainder > 0 and chars_for_remainder > 0: current_pass_chars = "" for _ in range(chars_for_remainder): current_pass_chars += clean_text[current_index] current_index = (current_index + direction * step) % text_length pass_result += current_pass_chars logger.debug(f"Mandala {i}: Remainder pass_result = {pass_result}") # Handle cases where step > text_length and chars_per_full_pass=1 if base_chars == 0 and chars_per_full_pass == 1 and full_passes > 0: # pass_result already contains the last character picked pass elif base_chars == 0 and chars_per_full_pass == 0 and full_passes > 0: # When no characters are picked, skip appending pass result_text += pass_result logger.debug(f"Result text for Mandala {i}: {result_text}") if length != 0: result_text = result_text[:length] logger.debug(f"Mandala {i}: Result text truncated to length {length}.") # Translate the result text if required try: translated_text = translator.translate(result_text) if translator and result_text else "" except Exception as e: logger.error(f"Translation error: {e}") translated_text = "" # Calculate the Gematria sum try: result_sum = calculate_gematria(result_text) except Exception as e: logger.error(f"Mandala {i}: Gematria calculation error: {e}") result_sum = None if result_text: result = { 'book': f"Tripitaka {i}.", 'title': f'{data.get("title", "Unknown Title")} {data.get("book_name", "Unknown Book")} {data.get("chapter", "")}', 'result_text': result_text, 'result_sum': result_sum, 'translated_text': translated_text, 'source_language': "sa", } results.append(result) except (FileNotFoundError, json.JSONDecodeError, KeyError) as e: logger.error(f"Mandala {i}: Error processing {file_name}: {e}") results.append({"error": f"Mandala {i}: Error processing {file_name}: {e}"}) except Exception as e: logger.error(f"Mandala {i}: Unexpected error processing {file_name}: {e}") results.append({"error": f"Mandala {i}: Unexpected error processing {file_name}: {e}"}) return results if results else None # Tests test_results = [ (process_json_files(1,1,386,"1,-1", translate=True),"अअमुोच्टेकोसचण") ] # Function to run tests def run_tests(test_results): all_tests_passed = True for idx, (result, expected) in enumerate(test_results): if expected is None: # Check if no result is expected if not result: logger.warning(f"Test {idx}: Passed (Expected no results, got no results.)") else: # Extract result_text from the first result result_text = result[0].get('result_text', None) logger.error(f"Test {idx}: Failed (Expected no results, but got: {result_text})") all_tests_passed = False else: # Check if result is not empty before accessing elements if result: result_text = result[0].get("result_text") if result_text == expected: logger.warning(f"Test {idx}: Passed (Expected: '{expected}', Got: '{result_text}')") else: logger.error(f"Test {idx}: Failed (Expected: '{expected}', but got: '{result_text}')") all_tests_passed = False else: logger.error(f"Test {idx}: Failed (Expected: '{expected}', but got no results)") all_tests_passed = False if all_tests_passed: logger.info("All round tests passed.") else: logger.info("Some tests failed. Please check the error messages above.") # Run the tests run_tests(test_results)