add util

util.py

@@ -1,26 +1,16 @@
 import json
 import re
 
-def process_json_files(start, end, step, strip_in_braces=False, strip_diacritics=False):
+def process_json_files(start, end):
     base_path = "texts"
     results = []
 
-    for i in range(start, end + 1, step):
+    for i in range(start, end + 1):
         file_name = f"{base_path}/{i:02}.json"
         try:
             with open(file_name, 'r', encoding='utf-8') as file:
                 data = json.load(file)
-                text_blocks = data.get("text", [])
-
-                full_text = " ".join([' '.join(block) for block in text_blocks])
-
-                if strip_in_braces:
-                    full_text = re.sub(r"\[.*?\]", "", full_text, flags=re.DOTALL)
-                if strip_diacritics:
-                    full_text = re.sub(r"[^\u05D0-\u05EA ]+", "", full_text)
-
-                # Check if the full text is not empty after processing
-                if full_text.strip():
+                if data:
                     results.append({
                         "book": i,
                         "title": data.get("title", "No title"),