File size: 1,367 Bytes
dc02386 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 |
import re
from html import unescape
def clean_text(t):
t = clean_markdown(t)
t = t.replace("\n"," ")
t = t.replace("\t"," ")
t = t.replace("^M"," ")
t = t.replace("\r"," ")
t = t.replace(" ,", ",")
t = re.sub(" +", " ", t)
return t
def clean_markdown(md_text):
# Remove code blocks
md_text = re.sub(r'```.*?```', '', md_text, flags=re.DOTALL)
# Remove inline code
md_text = re.sub(r'`[^`]*`', '', md_text)
# Remove images
md_text = re.sub(r'!\[.*?\]\(.*?\)', '', md_text)
# Remove links but keep link text
md_text = re.sub(r'\[([^\]]+)\]\(.*?\)', r'\1', md_text)
# Remove bold and italic (groups of *, _)
md_text = re.sub(r'(\*\*|__)(.*?)\1', r'\2', md_text)
md_text = re.sub(r'(\*|_)(.*?)\1', r'\2', md_text)
# Remove headings
md_text = re.sub(r'#+ ', '', md_text)
# Remove blockquotes
md_text = re.sub(r'^>.*$', '', md_text, flags=re.MULTILINE)
# Remove list markers
md_text = re.sub(r'^(\s*[-*+]|\d+\.)\s+', '', md_text, flags=re.MULTILINE)
# Remove horizontal rules
md_text = re.sub(r'^\s*[-*_]{3,}\s*$', '', md_text, flags=re.MULTILINE)
# Remove tables
md_text = re.sub(r'\|.*?\|', '', md_text)
# Remove raw HTML tags
md_text = re.sub(r'<.*?>', '', md_text)
# Decode HTML entities
md_text = unescape(md_text)
return md_text |