mh_parser/main_2.py
2023-12-19 06:01:43 -05:00

580 lines
20 KiB
Python

import collections
import itertools
import re
import string
import sys
from html.parser import HTMLParser
from pathlib import Path
import pythonbible as bible
from bs4 import BeautifulSoup
from loguru import logger
from tqdm import tqdm
import difflib
base = Path('./scraps_consolidated').absolute()
logger.remove(0)
logger.add('mixing.log')
from functools import partialmethod
logger.__class__.foobar = partialmethod(logger.__class__.log, "foobar", level='WARNING')
logger.add('log.txt', level='WARNING')
shorts = dict(
[
("Nahum", "Nah"),
("Colossians", "Col"),
("Esther", "Esth"),
("James", "Jas"),
("John", "John"),
("3John", "3John"),
("Revelation", "Rev"),
("Titus", "Titus"),
("Zechariah", "Zech"),
("Malachi", "Mal"),
("Lamentations", "Lam"),
("Matthew", "Matt"),
("Jonah", "Jonah"),
("Hosea", "Hos"),
("Joel", "Joel"),
("Genesis", "Gen"),
("Proverbs", "Prov"),
("Romans", "Rom"),
("Micah", "Mic"),
("Jeremiah", "Jer"),
("Ecclesiastes", "Eccl"),
("Philippians", "Phil"),
("Galatians", "Gal"),
("Acts", "Acts"),
("Psalms", "Ps"),
("Isaiah", "Isa"),
("2Chronicles", "2Chr"),
("Job", "Job"),
("Deuteronomy", "Deut"),
("Luke", "Luke"),
("Numbers", "Num"),
("Daniel", "Dan"),
("Hebrews", "Heb"),
("Nehemiah", "Neh"),
("1John", "1John"),
("Haggai", "Hag"),
("1Kings", "1Kgs"),
("Joshua", "Josh"),
("1Corinthians", "1Cor"),
("Mark", "Mark"),
("2Kings", "2Kgs"),
("Obadiah", "Obad"),
("Ephesians", "Eph"),
("1Timothy", "1Tim"),
("Philemon", "Phlm"),
("1Chronicles", "1Chr"),
("2Samuel", "2Sam"),
("1Samuel", "1Sam"),
("2Peter", "2Pet"),
("2John", "2John"),
("Song of Solomon", "Song"),
("Jude", "Jude"),
("Amos", "Amos"),
("1Peter", "1Pet"),
("Leviticus", "Lev"),
("Habakkuk", "Hab"),
("Ruth", "Ruth"),
("1Thessalonians", "1Thess"),
("Ezra", "Ezra"),
("Ezekiel", "Ezek"),
("2Thessalonians", "2Thess"),
("2Timothy", "2Tim"),
("Exodus", "Exod"),
("2Corinthians", "2Cor"),
("Judges", "Judg"),
("Zephaniah", "Zeph"),
]
)
end_verses = dict([
('Obadiah', 21),
('Philemon', 25),
('2John', 13),
('3John', 15),
('Jude', 25),
])
class MyHTMLParser(HTMLParser):
file = None
passage_verse = None
tag = None
tag_type = None # 'start' or 'end'
skip_re = False
attrs = dict()
# All the patterns are for re.sub(). This is specifically for getting
# list/sublist into the correct position. The first item is for the pattern
# to find and the second is the replacement pattern. The replacement pattern
# also ids the list so that when the self.clean_file() we can properly add
# the correct amount of `\t` to the line.
patterns = [
(r"\[(\d+\.)\]", lambda x: f'\n{x.group(1)}~4'),
(r"\((\d+\.)\)", lambda x: f'\n{x.group(1)}~3'),
# These share the same id. There are times were the two are a mixed
# within the parent lists.
(r"^\b(\d+\.)", lambda x: f'\n{x.group(1)}~2'),
(r"(\S\s)(\d+\.)(\s\b|\W)",
lambda x: f'{x.group(1)}\n{x.group(2)}~2{x.group(3)}'),
(r"(?=[MDCLXVI])M*(C[MD]|D?C{0,3})(X[CL]|L?X{0,3})(I[XV]|V?I{0,3})\.",
lambda x: f'\n{parser.roman_to_int(x.group())}.~1'),
]
@staticmethod
def roman_to_int(number: str) -> int:
if number is None:
return 0
# Sometimes the roman numeral comes with a '.'. Striping to not cause
# issues
number = number.strip('.')
roman = {'I': 1, 'V': 5, 'X': 10, 'L': 50, 'C': 100, 'D': 500, 'M': 1000}
total = 0
for i in range(len(number) - 1, -1, -1):
num = roman[number[i]]
if 3 * num < total:
total -= num
else:
total += num
return total
def create_md_file(self, base_dir: Path, folder: str, file: str):
""" Create the path of the md file that will be written to as the
program process the xml file.
"""
md_file = Path(f'{base_dir}/{folder}/{file}.md').absolute()
md_file.parent.mkdir(parents=True, exist_ok=True)
md_file.unlink(missing_ok=True)
# logger.info(md_file.absolute())
self.file = md_file
return md_file
def write_to_file(self, data, skip_re=False):
# if self.file is None:
# raise ValueError('No File specified')
if not skip_re:
# Here the list/sublist are searched for and altered for later
# processing in self.clean_file()
for pattern, sub in self.patterns:
# logger.debug(f'Running Pattern: {pattern} on {data[:50]}')
data = re.sub(pattern, sub, data)
# logger.info(f'writing: {data!r}')
if self.file:
# raise ValueError('No File specified')
with open(self.file, 'a+') as file:
file.write(data)
else:
logger.info(data)
def clean_file(self):
logger.info(' CLEANING FILE')
if self.file is None:
raise ValueError('No File specified')
# Read all the lines
with open(self.file, 'r') as file:
lines = file.readlines()
indent = 0 # Current number of '\t' to add in front of the list number
sublist = dict() # Track the indent number based on the id.
with open(self.file, 'w') as file:
file.write('# Chapter Introduction\n\n')
header = None
for line in lines:
# Do not write blank lines
if not line.strip():
continue
# Main header, if a header has already been written skip it
if line.strip().startswith('# '):
if header == line.strip():
continue
header = line.strip()
# Add a `\n` if line is a heading.
if line.startswith('#'):
line = f'\n{line.strip()}'
# If line is not a list then add a `\n` after the paragraph,
# and reset the indent tracker
if line.strip()[0] not in string.digits:
line = f'{line}\n'
# logger.info('resetting indent')
indent = 0
sublist.clear()
else:
# Get the list id
start = line.find('~')
end = start + 2
list_id = line[start:end]
# Get the number of indents based on the list id, if it does
# not exist they set the value to the max value +1. If this
# is the first it will be set to 0 (-1 +1 = 0)
indents = "\t" * sublist.setdefault(
list_id, max(sublist.values(), default=-1)+1
)
line = f'{indents}{line.replace(list_id, "")}'
if line.startswith('Passage: '):
line = line.removeprefix('Passage: ')
file.write(line)
def normalize_osis_verses(self, verses):
"""Takes this
'Bible:John.1.3 Bible:John.1.10 Bible:Eph.3.9 Bible:Col.1.16 Bible:Heb.1.2 Bible:Acts.17.24-Acts.17.25 Bible:Exod.5.26 Bible:Gen.5'
and turns it into somthing more readable"""
normalized_refs = list()
for chunks in verses.removeprefix("Bible:").split('Bible:'):
spanned_verses = list()
for ref in chunks.split('-'):
verse = ref.replace('.', ' ', 1).replace('.', ':')
book = ref.split('.')[0]
spanned_verses.append(
bible.format_scripture_references(
bible.get_references(verse.strip())
)
)
try:
normalized = bible.format_scripture_references(
bible.get_references('-'.join(spanned_verses))
)
except:
logger.warning(f"Error with: {verses=}: {spanned_verses=}")
raise
if normalized is False:
logger.info(f"Error with: {verses=}: {spanned_verses=}")
sys.exit()
normalized_refs.append(normalized)
return ';'.join(normalized_refs)
# @staticmethod
def old_normalize_osis_verses(self, verses):
"""Takes this
'Bible:John.1.3 Bible:John.1.10 Bible:Eph.3.9 Bible:Acts.17.24-Acts.17.25 Bible:Exod.5.26 Bible:Gen.5'
and turns it into somthing more readable"""
normalized_refs = list()
for chunks in verses.removeprefix("Bible:").split('Bible:'):
spanned_verses = chunks.split('-')
if len(spanned_verses) < 2:
spanned_verses = list(itertools.repeat(spanned_verses[0], 2))
book, start_chapter, start_verse = spanned_verses[0].split('.')
_, end_chapter, end_verse = spanned_verses[1].split('.')
book = bible.Book[convert[book].upper()]
normalized = bible.NormalizedReference(
book=book,
start_chapter=start_chapter,
start_verse=start_verse,
end_chapter=end_chapter,
end_verse=end_verse,
)
if normalized is False:
logger.foobar(f'{self.args=}')
normalized_refs.append(
bible.format_scripture_references([normalized]).strip()
)
return ';'.join(normalized_refs)
@staticmethod
@logger.catch(reraise=False)
def obsidian_links(verses):
"""Take `Book Chapter:Ver-Ver and turn it into a obsidian bible link"""
links = []
for verse in verses.split(';'):
verse = verse.replace(':', '#').replace('-', '..')
if verse[0] in string.digits:
verse.replace(' ', '', 1)
links.append(f"[[{verse}]]")
return '; '.join(links)
def convert_passage(self, verses):
book = self.file.parts[-2]
chapter = None
ref = []
for ver in verses.split(','):
try:
chapter, v = ver.split(':')
except:
v = ver
r = []
for i in v.split('-'):
r.append(f'{book}.{chapter}.{i}')
ref.append(f'Bible:{"-".join(r)}')
x = ' '.join(ref)
return f'osisRef="{x}"'
def process_tag(self, test=None):
match self.tag:
case 'b': # Bold
self.write_to_file('**')
case 'i': # Italics
self.write_to_file('*')
case 'scripcom':
# This saves the verse for the section.
if self.tag_type == 'start':
try:
verse = bible.get_references(
self.normalize_osis_verses(
self.attrs['osisref'])
)[0]
except:
logger.debug(self.passage_verse)
raise
if verse.start_verse == verse.end_verse:
self.write_to_file('## Verse: ')
self.write_to_file(
f'{verse.start_verse}'
)
else:
self.write_to_file('## Verses: ')
self.write_to_file(
f'{verse.start_verse} - {verse.end_verse}'
)
self.write_to_file('\n\n')
case 'h4': # this is the section header. Need to keep track of it per file.
if self.tag_type == 'start':
self.write_to_file('\n# ',)
self.skip_re = True
if self.tag_type == 'end':
self.write_to_file('\n')
self.skip_re = False
# case 'scripref': # Scripture ref
# # get attr 'osisref' and parse [..., ('passage', 'Bible:Rev.14.6-Rev.14.7')]
# # bible.format_scripture_references(bible.get_references(attrs['osisref']))
# if self.tag_type == 'start':
# if 'osisref' in self.attrs:
# verses = self.normalize_osis_verses(
# self.attrs['osisref']
# )
# else:
# # verses = self.normalize_osis_verses(
# # self.convert_passage(self.attrs['passage'])
# # )
# raise ValueError('No `osisref` in tag attrs')
#
# self.write_to_file(self.obsidian_links(verses))
case 'p': # Paragraph
if self.tag_type == 'start':
if self.attrs.get('class', False) == 'passage':
# Need this so the regex in self.clean does not pick
# this up. Will be also cleaned in the same function.
self.write_to_file('Passage: ')
if self.tag_type == 'end':
self.write_to_file('\n\n')
case 'li':
if self.tag_type == 'start':
self.write_to_file('- ')
else:
self.write_to_file('\n')
# case _:
# # logger.info(tag.upper())
# pass
def handle_starttag(self, tag, attrs):
# Set taf trackers
self.tag_type = 'start'
self.tag = tag
self.attrs = dict(attrs)
self.process_tag()
def handle_endtag(self, tag):
# logger.info(f'End: {tag}')
self.tag = tag
self.tag_type = 'end'
self.process_tag()
# Clear tag tracker
self.tag = None
self.tag_type = None
self.attrs.clear()
self.skip_re = False
def handle_data(self, data):
# logger.info(f'Data: {data!r}')
if self.tag in ['scripref', 'h2', 'h3', 'h5']:
return
if data == '\n':
return
data = data.replace('', '-- ')
data = data.replace('GOD', 'God')
data = data.replace('\n', ' ')
self.write_to_file(data, self.skip_re)
def create_file(base_dir: Path, folder: str, file: str, file_type: str) -> Path:
""" Create the path of the md file that will be written to as the
program process the xml file.
"""
new_file = Path(f'{base_dir}/{folder}/{file}.{file_type}').absolute()
new_file.parent.mkdir(parents=True, exist_ok=True)
new_file.unlink(missing_ok=True)
# logger.info(md_file.absolute())
return new_file
def write_file(file, data):
with open(file, 'a+') as f:
for d in data:
f.write(d)
f.write('\n')
if __name__ == '__main__':
parser = MyHTMLParser()
# Process each volume file.
scraps = Path('./scraps')
files_used = list()
missing = list()
for book in Path('./vol_split').glob('*'):
book_name = book.name.split(' - ')[-1]
logger.info(book_name)
print(book_name)
for chapter in book.glob('*.xml'):
chapter_name = chapter.stem
chapter_num = chapter_name.split(' ')[-1]
logger.info(f"{chapter_name}")
file = create_file(base, book_name, chapter_name, 'html')
# book intro file
if 'INTRODUCTION' in chapter_name.upper():
book_intro = scraps / f"{book_name}.html"
if book_intro.exists() is False:
logger.info(book_intro.name, book_intro.exists())
files_used.append(book_intro.name)
write_file(file, f'<!-- Gotten from {book_intro}-->')
write_file(file, book_intro.read_text())
parser.create_md_file('./mh_scraps/', book_name, file.stem)
parser.feed(file.read_text())
parser.clean_file()
continue
# Chapter intro
book_short = shorts[book_name]
ci = bible.get_references(f"{book_name} {chapter_num}")[0]
if ev := end_verses.get(book_name, False):
ci.end_verse = ev
start = f'{book_short}_{chapter_num}_{ci.start_verse}'
end = f'{book_short}_{chapter_num}_{ci.end_verse}'
chapter_into = scraps / f"chapter_{start}-{end}.html"
if chapter_into.exists() is False:
logger.info(chapter_into.name, chapter_into.exists())
raise ValueError
files_used.append(chapter_into.name)
# write(file, '<h1>Chapter Introduction</h1>')
write_file(file, f'<!-- Chapter Intro: {chapter_into} -->')
write_file(file, chapter_into.read_text())
sections = BeautifulSoup(chapter.read_text(), 'xml')
# Parse the text of the chapter
# May need to add a loop to look in all <p class='passage> and
# unrap any scripref that is in them. the place Dan is getting confused
# with the book Daniel.
for passage in sections.find_all('p', {'class': 'passage'}):
# print(passage.text)
new_t = sections.new_tag('p', attrs=passage.attrs)
new_t.string = passage.text
passage.replace_with(new_t)
# Clearing all scripRef of internal text. That way if it
# had other tags in it would not be processed
for scripture in sections.find_all('scripRef'):
scripture.clear()
write_file(file, '<!-- Chapter Sections -->')
for section in sections.find_all('div', {'class': 'Commentary'}):
if header := section.find('h4'):
write_file(file, str(header))
# Scripture ref, should be the section!
scripture_ref = section.previous_sibling
# write_file(file, '\n')
write_file(file, str(scripture_ref))
# Passage text
if passage := section.find('p', {'class': 'passage'}):
write_file(file, str(passage))
logger.info(scripture_ref)
verses_id = str(scripture_ref['osisRef'])
verses = verses_id.replace('Bible:', '').replace('.', '_')
# Sanity Check verses are legit
for v in verses.split('-'):
v = v.replace('_', ' ', 1).replace('_', ':')
b_ref = bible.get_references(v)
if not b_ref:
raise ValueError(f"{v} is in valid")
if int(b_ref[0].start_chapter) != int(chapter_num):
raise ValueError('Wrong Chapter')
verse_section = scraps / f"{verses}.html"
logger.info('\t\t',verse_section.name, verse_section.exists())
if verse_section.exists() is False:
missing.append((book_name, chapter_num, verse_section.name))
if verse_section.name in files_used:
raise FileExistsError(f'{verse_section} has been already used.')
files_used.append(verse_section.name)
write_file(file, f'<!-- {verse_section} -->')
write_file(file, verse_section.read_text())
parser.create_md_file('./mh_scraps/', book_name, file.stem)
parser.feed(file.read_text())
parser.clean_file()