""" Module regrouping baseline and basic extraction functions. """ # pylint:disable-msg=E0611 import json from typing import Any, Tuple from lxml.etree import _Element, Element, SubElement from lxml.html import HtmlElement from .settings import BASIC_CLEAN_XPATH from .utils import load_html, trim from .xml import delete_element def basic_cleaning(tree: HtmlElement) -> HtmlElement: "Remove a few section types from the document." for elem in BASIC_CLEAN_XPATH(tree): delete_element(elem) return tree def baseline(filecontent: Any) -> Tuple[_Element, str, int]: """Use baseline extraction function targeting text paragraphs and/or JSON metadata. Args: filecontent: HTML code as binary string or string. Returns: A LXML element containing the extracted paragraphs, the main text as string, and its length as integer. """ tree = load_html(filecontent) postbody = Element('body') if tree is None: return postbody, '', 0 # scrape from json text temp_text = "" for elem in tree.iterfind('.//script[@type="application/ld+json"]'): if elem.text and 'articleBody' in elem.text: try: json_body = json.loads(elem.text).get("articleBody", "") except Exception: # JSONDecodeError or 'list' object has no attribute 'get' json_body = "" if json_body: if "

" in json_body: parsed = load_html(json_body) text = trim(parsed.text_content()) if parsed is not None else "" else: text = trim(json_body) SubElement(postbody, 'p').text = text temp_text += " " + text if temp_text else text # return postbody, elem.text, len(elem.text) if len(temp_text) > 100: return postbody, temp_text, len(temp_text) tree = basic_cleaning(tree) # scrape from article tag temp_text = "" for article_elem in tree.iterfind('.//article'): text = trim(article_elem.text_content()) if len(text) > 100: SubElement(postbody, 'p').text = text temp_text += " " + text if temp_text else text if len(postbody) > 0: # temp_text = trim('\n'.join(postbody.itertext())) return postbody, temp_text, len(temp_text) # scrape from text paragraphs results = set() temp_text = "" # postbody = Element('body') for element in tree.iter('blockquote', 'code', 'p', 'pre', 'q', 'quote'): entry = trim(element.text_content()) if entry not in results: SubElement(postbody, 'p').text = entry temp_text += " " + entry if temp_text else entry results.add(entry) # temp_text = trim('\n'.join(postbody.itertext())) if len(temp_text) > 100: return postbody, temp_text, len(temp_text) # default strategy: clean the tree and take everything postbody = Element('body') body_elem = tree.find('.//body') if body_elem is not None: p_elem = SubElement(postbody, 'p') # todo: sanitize? text_elems = [trim(e) for e in body_elem.itertext()] p_elem.text = '\n'.join([e for e in text_elems if e]) return postbody, p_elem.text, len(p_elem.text) # new fallback text = html2txt(tree, clean=False) SubElement(postbody, 'p').text = text return postbody, text, len(text) def html2txt(content: Any, clean: bool = True) -> str: """Run basic html2txt on a document. Args: content: HTML document as string or LXML element. clean: remove potentially undesirable elements. Returns: The extracted text in the form of a string or an empty string. """ tree = load_html(content) if tree is None: return "" body = tree.find(".//body") if body is None: return "" if clean: body = basic_cleaning(body) return " ".join(body.text_content().split()).strip()