Spaces:
Running
Running
File size: 2,416 Bytes
2dc7757 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 |
import requests
import bz2
import xml.etree.ElementTree as ET
import os
import pickle
from tqdm import tqdm
import mwparserfromhell
# Step 1: Download the latest dump
DUMP_URL = "https://dumps.wikimedia.org/tewiktionary/latest/tewiktionary-latest-pages-articles.xml.bz2"
response = requests.get(DUMP_URL, stream=True)
print("Downloading the latest dump...")
total_size = int(response.headers.get('content-length', 0))
progress_bar = tqdm(total=total_size, unit='B', unit_scale=True)
dump_file = "tewiktionary-latest-pages-articles.xml.bz2"
with open(dump_file, 'wb') as file:
for chunk in response.iter_content(chunk_size=8192):
progress_bar.update(len(chunk))
file.write(chunk)
progress_bar.close()
# Step 2: Extract the dump
print("\nExtracting the dump...")
with bz2.open(dump_file, 'rb') as source, open(dump_file[:-4], 'wb') as dest:
for line in source:
dest.write(line)
# Step 3: Parse the XML dump and extract translations
print("Parsing the XML dump to extract translations...")
tree = ET.parse(dump_file[:-4])
root = tree.getroot()
ns = {'ns': 'http://www.mediawiki.org/xml/export-0.10/'}
translations = {}
for page in root.findall('ns:page', ns):
title = page.find('ns:title', ns).text
revision = page.find('ns:revision', ns)
if revision:
text_data = revision.find('ns:text', ns)
if text_data and text_data.text:
# Parse the wikitext
wikicode = mwparserfromhell.parse(text_data.text)
links = [link.title for link in wikicode.filter_wikilinks() if link.title.startswith("en:")]
if links:
english_translations = [str(link.split(':')[1]) for link in links]
translations[title] = english_translations
# Display the first 1000 translations
print("\nDisplaying the first 1000 translations:")
for i, (telugu_word, english_words) in enumerate(translations.items()):
if i >= 1000:
break
print(f"Telugu Word: {telugu_word}, English Translations: {', '.join(english_words)}")
# Save the translations to a pickle file
print("\nSaving translations to pickle file...")
pickle_filename = "telugu_english_translations.pkl"
with open(pickle_filename, 'wb') as file:
pickle.dump(translations, file)
print(f"Translations saved to {pickle_filename}")
# Optional: Remove the downloaded files if you want
# os.remove(dump_file)
# os.remove(dump_file[:-4])
|