Move files

src/great_ai/utilities/__init__.py

@@ -0,0 +1,14 @@
+from .clean import clean
+from .config_file import ConfigFile
+from .evaluate_ranking import evaluate_ranking
+from .get_sentences import get_sentences
+from .language import english_name_of_language, is_english, predict_language
+from .lemmatize_text import lemmatize_text
+from .lemmatize_token import lemmatize_token
+from .logger import get_logger
+from .match_names import match_names
+from .matcher import fast_tokenize, filter_sentences, normalize
+from .nlp import nlp
+from .parallel_map import parallel_map
+from .publication_tei import PublicationTEI
+from .unique import unique

src/great_ai/utilities/clean.py

@@ -0,0 +1,68 @@
+import html
+import re
+import unicodedata
+
+import unidecode
+
+from .data import left_regular_punctuations, right_regular_punctuations
+from .external.pylatexenc.latex2text import LatexNodes2Text
+from .logger import get_logger
+
+logger = get_logger("clean")
+latex = LatexNodes2Text()
+
+
+joined_left_punctuations = "".join(left_regular_punctuations).replace("]", r"\]")
+joined_right_punctuations = "".join(right_regular_punctuations).replace("[", r"\[")
+
+
+def clean(
+    text: str,
+    ignore_xml: bool = False,
+    ignore_latex: bool = False,
+    remove_brackets: bool = False,
+    convert_to_ascii: bool = False,
+) -> str:
+    if not ignore_xml:
+        text = re.sub(r"<[^>]*>", " ", text)
+        text = html.unescape(text)
+
+    if not ignore_latex:
+        text = text.replace("%", "\\%")  # escape LaTeX comments before parsing as LaTeX
+
+        try:
+            text = latex.latex_to_text(text, tolerant_parsing=True, strict_braces=False)
+            text = text.replace("%s", " ")
+        except:
+            logger.exception("Latex parsing error")
+
+    if convert_to_ascii:
+        text = unicodedata.normalize("NFKD", text)
+
+        try:
+            text.encode("ASCII", errors="strict")
+        except UnicodeEncodeError:
+            text = "".join([c for c in text if not unicodedata.combining(c)])
+            text = unidecode.unidecode(text)
+
+    text = re.sub(
+        r"\b[a-zA-Z](?:[\t ]+[a-zA-Z]\b)+", lambda m: re.sub(r"[\t ]", "", m[0]), text
+    )  # A R T I C L E => ARTICLE
+
+    if remove_brackets:
+        text = re.sub(r"\[[^\]]*\]", " ", text)
+
+    # fix hypens: break- word => break-word
+    text = re.sub(r"(\S)-\s+", r"\1-", text)
+    text = re.sub(r"\s+-(\S)", r"-\1", text)
+
+    # collapse whitespace
+    text = re.sub(r"\s+", " ", text)
+
+    # fix punctuation
+    text = re.sub(rf" ([{joined_left_punctuations}])", r"\1", text)
+    text = re.sub(rf"([{joined_right_punctuations}]) ", r"\1", text)
+
+    text = text.strip()
+
+    return text

src/great_ai/utilities/config_file/__init__.py

@@ -0,0 +1,2 @@
+from .config_file import ConfigFile
+from .parse_error import ParseError

src/great_ai/utilities/config_file/config_file.py

@@ -0,0 +1,87 @@
+import os
+from pathlib import Path
+from typing import Dict, Iterable, Tuple, Union
+
+from ..logger import get_logger
+from .parse_error import ParseError
+from .pattern import pattern
+
+ENVIRONMENT_VARIABLE_KEY_PREFIX = "ENV"
+
+logger = get_logger("ConfigFile")
+
+
+class ConfigFile:
+    def __init__(
+        self, path: Union[Path, str], ignore_missing_environment_variables: bool = False
+    ) -> None:
+        if not isinstance(path, Path):
+            path = Path(path)
+
+        if not path.exists():
+            raise FileNotFoundError(path.absolute())
+
+        self._path = path
+        self._ignore_missing_environment_variables = (
+            ignore_missing_environment_variables
+        )
+        self._key_values: Dict[str, str] = {}
+
+        self._parse()
+
+    def _parse(self):
+        with open(self._path, encoding="utf-8") as f:
+            lines: str = f.read()
+
+        matches = pattern.findall(lines)
+        for key, *values in matches:
+            if key in self._key_values:
+                raise KeyError(
+                    f"Key `{key}` has been already defined and its value is `{self._key_values[key]}`"
+                )
+
+            try:
+                value = next(v for v in values if v)
+            except StopIteration:
+                raise ParseError(
+                    f"Cannot parse config file ({self._path.absolute()}), error at key `{key}`"
+                )
+
+            if value.startswith(f"{ENVIRONMENT_VARIABLE_KEY_PREFIX}:"):
+                _, value = value.split(":")
+                if value not in os.environ:
+                    issue = f'The value of `{key}` contains the "{ENVIRONMENT_VARIABLE_KEY_PREFIX}` prefix but `{value}` is not defined as an environment variable'
+                    if self._ignore_missing_environment_variables:
+                        logger.warning(f"{issue}, defaulting to `None`")
+                    else:
+                        raise KeyError(issue)
+                value = os.environ[value]
+
+            self._key_values[key] = value
+
+    def __getattr__(self, key: str) -> str:
+        if key in self._key_values:
+            return self._key_values[key]
+        raise KeyError(
+            f"Key `{key}` is not found in configuration file ({self._path.absolute()})"
+        )
+
+    __getitem__ = __getattr__
+
+    def __iter__(self) -> Iterable[Tuple[str, str]]:
+        return iter(self._key_values)
+
+    def __len__(self) -> int:
+        return len(self._key_values)
+
+    def keys(self):
+        return self._key_values.keys()
+
+    def values(self):
+        return self._key_values.values()
+
+    def items(self):
+        return self._key_values.items()
+
+    def __repr__(self):
+        return f"{type(self).__name__}(\n  path={self._path},\n  ignore_missing_environment_variables={self._ignore_missing_environment_variables}\n) {{{self._key_values}}}"

src/great_ai/utilities/config_file/parse_error.py

@@ -0,0 +1,2 @@
+class ParseError(Exception):
+    pass

src/great_ai/utilities/config_file/pattern.py

@@ -0,0 +1,20 @@
+import re
+
+pattern = re.compile(
+    r"""
+        (?:^|\n)        # new key-value pairs must start on a new line
+        \s*             # leading whitespace is allowed
+        (?!\#)          # the key cannot start with a `#` symbol
+        (\w+?)          # then comes the key
+        \s*=\s*         # the key and value are separated by an equal sign
+        (?:             # then comes the value
+              "([^"]*)"   # the value can be surrounded by quotes: "value"
+            | '([^']*)'   # the value can be surrounded by quotes: 'value'
+            | `([^`]*)`   # the value can be surrounded by quotes: `value`
+            | ([^#\n]*?)  # or it is bare, in that case, the trailing whitespace is ignored
+        )
+        \s*(?:\#.*)?    # comments can be added with the `#` symbol
+        (?=\n|$)        # a key-value pairs are separated by new lines
+    """,
+    flags=re.UNICODE | re.VERBOSE,
+)

src/great_ai/utilities/data/__init__.py

@@ -0,0 +1,6 @@
+from .american_spellings import american_spellings
+from .punctuations import (
+    left_regular_punctuations,
+    right_regular_punctuations,
+    sentence_ending_punctuations,
+)

src/great_ai/utilities/data/punctuations.py

@@ -0,0 +1,22 @@
+sentence_ending_punctuations = [".", "?", "!", ":", ";"]
+
+# punctuations that usually have no space between them and the character to their left
+left_regular_punctuations = [
+    *sentence_ending_punctuations,
+    ",",
+    "'",
+    "%",
+    ")",
+    "}",
+    "]",
+]
+
+# punctuations that usually have no space between them and the character to their right
+right_regular_punctuations = [
+    "(",
+    "{",
+    "[",
+    "$",
+    "€",
+    "#",
+]

src/great_ai/utilities/evaluate_ranking/__init__.py

@@ -0,0 +1,2 @@
+from .draw_f1_iso_lines import draw_f1_iso_lines
+from .evaluate_ranking import evaluate_ranking

src/great_ai/utilities/evaluate_ranking/draw_f1_iso_lines.py

@@ -0,0 +1,28 @@
+from typing import Optional
+
+import matplotlib.pyplot as plt
+import numpy as np
+from matplotlib.axes import Axes
+
+
+def draw_f1_iso_lines(
+    resolution: int = 1000,
+    min: float = 0.2,
+    max: float = 0.8,
+    steps: int = 4,
+    axes: Optional[Axes] = None,
+) -> None:
+    if axes is None:
+        axes = plt.axes()
+
+    for f_score in np.linspace(min, max, num=steps):
+        x = np.linspace(f_score / (2 - f_score), 1, num=resolution)
+        y = f_score * x / (2 * x - f_score)
+
+        axes.plot(x[y >= 0], y[y >= 0], color="gray", alpha=0.2)
+
+        axes.annotate(
+            f"f1={f_score:0.1f}",
+            backgroundcolor="w",
+            xy=(0.9, y[int(resolution * 0.9)] + 0.02),
+        )

src/great_ai/utilities/evaluate_ranking/evaluate_ranking.py

@@ -0,0 +1,90 @@
+from pathlib import Path
+from typing import Dict, List, Optional, Union
+
+import matplotlib
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.metrics import average_precision_score, precision_recall_curve
+
+from ..unique import unique
+from .draw_f1_iso_lines import draw_f1_iso_lines
+
+
+def evaluate_ranking(
+    expected: List[Union[str, float]],
+    actual_scores: List[float],
+    target_recall: float,
+    title: Optional[str] = "",
+    disable_interpolation: bool = False,
+    axes: Optional[plt.Axes] = None,
+    output_svg: Optional[Path] = None,
+    reverse_order: bool = False,
+    plot: bool = True,
+) -> Dict[Union[str, float], float]:
+    assert 0 <= target_recall <= 1
+
+    if plot and axes is None:
+        fig = plt.figure(figsize=(20, 10))
+        fig.patch.set_facecolor("white")
+        ax = plt.axes()
+    else:
+        ax = axes
+
+    classes = sorted(unique(expected), reverse=reverse_order)
+    str_classes = [str(c) for c in classes]
+
+    with matplotlib.rc_context({"font.size": 20}):
+        if plot:
+            ax.set_xmargin(0)
+
+            draw_f1_iso_lines(axes=ax)
+
+        results: Dict[Union[str, float], float] = {}
+        for i in range(len(classes) - 1):
+            binarized_expected = [
+                (v < classes[i]) if reverse_order else (v > classes[i])
+                for v in expected
+            ]
+            precision, recall, _ = precision_recall_curve(
+                binarized_expected, actual_scores
+            )
+
+            if not disable_interpolation:
+                for j in range(1, len(precision)):
+                    precision[j] = max(precision[j - 1], precision[j])
+
+            closest_recall_index = np.argmin(np.abs(recall - target_recall))
+            precision_at_closest_recall = precision[closest_recall_index]
+            average_precision = average_precision_score(
+                binarized_expected, actual_scores
+            )
+            results[classes[i]] = precision_at_closest_recall
+
+            if plot:
+                ax.plot(
+                    recall,
+                    precision,
+                    label=f"{'|'.join(str_classes[:i + 1])} ↔ {'|'.join(str_classes[i+1:])} (P@{target_recall:.2f}={precision_at_closest_recall:.2f}, AP={average_precision:.2f})",
+                )
+
+        if plot:
+            ax.legend(loc="upper right")
+            ax.axvline(x=target_recall, linestyle="--", color="#55c6bb", linewidth=2.0)
+
+            if title is None:
+                title = "Ranking evaluation"
+
+            ax.set_title(f'{title} ({" < ".join(str_classes)})', pad=20)
+
+            ax.set_xlabel("Recall")
+            ax.set_ylabel("Precision")
+
+            ax.set_xticks([target_recall] + sorted(ax.get_xticks()))
+
+        if plot and output_svg is None:
+            if axes is None:
+                plt.show()
+        elif output_svg:
+            plt.savefig(output_svg, format="svg")
+
+    return results

src/great_ai/utilities/external/negspacy/README.md

@@ -0,0 +1 @@
+https://github.com/jenojp/negspacy

src/great_ai/utilities/external/negspacy/negation.py

@@ -0,0 +1,222 @@
+import logging
+
+from spacy.language import Language
+from spacy.matcher import PhraseMatcher
+from spacy.tokens import Token
+
+from .termsets import termset
+
+default_ts = termset("en").get_patterns()
+
+
+@Language.factory(
+    "negex",
+    default_config={
+        "neg_termset": default_ts,
+        "extension_name": "negex",
+        "chunk_prefix": list(),
+    },
+)
+class Negex:
+    """
+        A spaCy pipeline component which identifies negated tokens in text.
+
+        Based on: NegEx - A Simple Algorithm for Identifying Negated Findings and Diseasesin Discharge Summaries
+    Chapman, Bridewell, Hanbury, Cooper, Buchanan
+
+    Parameters
+    ----------
+    nlp: object
+        spaCy language object
+    termset_lang: str
+        language code, if using default termsets (e.g. "en" for english)
+    extension_name: str
+        defaults to "negex"; whether entity is negated is then available as ent._.negex
+    pseudo_negations: list
+        list of phrases that cancel out a negation, if empty, defaults are used
+    preceding_negations: list
+        negations that appear before an entity, if empty, defaults are used
+    following_negations: list
+        negations that appear after an entity, if empty, defaults are used
+    termination: list
+        phrases that "terminate" a sentence for processing purposes such as "but". If empty, defaults are used
+
+    """
+
+    def __init__(
+        self,
+        nlp: Language,
+        name: str,
+        neg_termset: dict,
+        extension_name: str,
+        chunk_prefix: list,
+    ):
+        if not Token.has_extension(extension_name):
+            Token.set_extension(extension_name, default=False, force=True)
+
+        ts = neg_termset
+        expected_keys = [
+            "pseudo_negations",
+            "preceding_negations",
+            "following_negations",
+            "termination",
+        ]
+        if not set(ts.keys()) == set(expected_keys):
+            raise KeyError(
+                f"Unexpected or missing keys in 'neg_termset', expected: {expected_keys}, instead got: {list(ts.keys())}"
+            )
+
+        self.pseudo_negations = ts["pseudo_negations"]
+        self.preceding_negations = ts["preceding_negations"]
+        self.following_negations = ts["following_negations"]
+        self.termination = ts["termination"]
+
+        self.nlp = nlp
+        self.extension_name = extension_name
+        self.build_patterns()
+        self.chunk_prefix = list(nlp.tokenizer.pipe(chunk_prefix))
+
+    def build_patterns(self):
+        # efficiently build spaCy matcher patterns
+        self.matcher = PhraseMatcher(self.nlp.vocab, attr="LOWER")
+
+        self.pseudo_patterns = list(self.nlp.tokenizer.pipe(self.pseudo_negations))
+        self.matcher.add("pseudo", None, *self.pseudo_patterns)
+
+        self.preceding_patterns = list(
+            self.nlp.tokenizer.pipe(self.preceding_negations)
+        )
+        self.matcher.add("Preceding", None, *self.preceding_patterns)
+
+        self.following_patterns = list(
+            self.nlp.tokenizer.pipe(self.following_negations)
+        )
+        self.matcher.add("Following", None, *self.following_patterns)
+
+        self.termination_patterns = list(self.nlp.tokenizer.pipe(self.termination))
+        self.matcher.add("Termination", None, *self.termination_patterns)
+
+    def process_negations(self, doc):
+        """
+        Find negations in doc and clean candidate negations to remove pseudo negations
+
+        Parameters
+        ----------
+        doc: object
+            spaCy Doc object
+
+        Returns
+        -------
+        preceding: list
+            list of tuples for preceding negations
+        following: list
+            list of tuples for following negations
+        terminating: list
+            list of tuples of terminating phrases
+
+        """
+        ###
+        # does not work properly in spacy 2.1.8. Will incorporate after 2.2.
+        # Relying on user to use NER in meantime
+        # see https://github.com/jenojp/negspacy/issues/7
+        ###
+        # if not doc.is_nered:
+        #     raise ValueError(
+        #         "Negations are evaluated for Named Entities found in text. "
+        #         "Your SpaCy pipeline does not included Named Entity resolution. "
+        #         "Please ensure it is enabled or choose a different language model that includes it."
+        #     )
+        preceding = list()
+        following = list()
+        terminating = list()
+
+        matches = self.matcher(doc)
+        pseudo = [
+            (match_id, start, end)
+            for match_id, start, end in matches
+            if self.nlp.vocab.strings[match_id] == "pseudo"
+        ]
+
+        for match_id, start, end in matches:
+            if self.nlp.vocab.strings[match_id] == "pseudo":
+                continue
+            pseudo_flag = False
+            for p in pseudo:
+                if start >= p[1] and start <= p[2]:
+                    pseudo_flag = True
+                    continue
+            if not pseudo_flag:
+                if self.nlp.vocab.strings[match_id] == "Preceding":
+                    preceding.append((match_id, start, end))
+                elif self.nlp.vocab.strings[match_id] == "Following":
+                    following.append((match_id, start, end))
+                elif self.nlp.vocab.strings[match_id] == "Termination":
+                    terminating.append((match_id, start, end))
+                else:
+                    logging.warnings(
+                        f"phrase {doc[start:end].text} not in one of the expected matcher types."
+                    )
+        return preceding, following, terminating
+
+    def termination_boundaries(self, doc, terminating):
+        """
+        Create sub sentences based on terminations found in text.
+
+        Parameters
+        ----------
+        doc: object
+            spaCy Doc object
+        terminating: list
+            list of tuples with (match_id, start, end)
+
+        returns
+        -------
+        boundaries: list
+            list of tuples with (start, end) of spans
+
+        """
+        sent_starts = [sent.start for sent in doc.sents]
+        terminating_starts = [t[1] for t in terminating]
+        starts = sent_starts + terminating_starts + [len(doc)]
+        starts.sort()
+        boundaries = list()
+        index = 0
+        for i, start in enumerate(starts):
+            if not i == 0:
+                boundaries.append((index, start))
+            index = start
+        return boundaries
+
+    def negex(self, doc):
+        """
+        Negates entities of interest
+
+        Parameters
+        ----------
+        doc: object
+            spaCy Doc object
+
+        """
+        preceding, following, terminating = self.process_negations(doc)
+        boundaries = self.termination_boundaries(doc, terminating)
+        for b in boundaries:
+            sub_preceding = [i for i in preceding if b[0] <= i[1] < b[1]]
+            sub_following = [i for i in following if b[0] <= i[1] < b[1]]
+
+            for e in doc[b[0] : b[1]]:
+                if any(pre < e.i for pre in [i[1] for i in sub_preceding]):
+                    e._.set(self.extension_name, True)
+                    continue
+                if any(fol > e.i for fol in [i[2] for i in sub_following]):
+                    e._.set(self.extension_name, True)
+                    continue
+                if self.chunk_prefix:
+                    if any(
+                        e.text.lower().startswith(c.text.lower())
+                        for c in self.chunk_prefix
+                    ):
+                        e._.set(self.extension_name, True)
+        return doc
+
+    def __call__(self, doc):
+        return self.negex(doc)

src/great_ai/utilities/external/negspacy/termsets.py

@@ -0,0 +1,229 @@
+"""
+Default termsets for various languages
+"""
+
+LANGUAGES = dict()
+
+# english termset dictionary
+en = dict()
+pseudo = [
+    "no further",
+    "not able to be",
+    "not certain if",
+    "not certain whether",
+    "not necessarily",
+    "without any further",
+    "without difficulty",
+    "without further",
+    "might not",
+    "not only",
+    "no increase",
+    "no significant change",
+    "no change",
+    "no definite change",
+    "not extend",
+    "not cause",
+]
+en["pseudo_negations"] = pseudo
+
+preceding = [
+    "absence of",
+    "declined",
+    "denied",
+    "denies",
+    "denying",
+    "no sign of",
+    "no signs of",
+    "not",
+    "not demonstrate",
+    "symptoms atypical",
+    "doubt",
+    "negative for",
+    "no",
+    "versus",
+    "without",
+    "doesn't",
+    "doesnt",
+    "don't",
+    "dont",
+    "didn't",
+    "didnt",
+    "wasn't",
+    "wasnt",
+    "weren't",
+    "werent",
+    "isn't",
+    "isnt",
+    "aren't",
+    "arent",
+    "cannot",
+    "can't",
+    "cant",
+    "couldn't",
+    "couldnt",
+    "never",
+]
+en["preceding_negations"] = preceding
+
+following = [
+    "declined",
+    "unlikely",
+    "was not",
+    "were not",
+    "wasn't",
+    "wasnt",
+    "weren't",
+    "werent",
+]
+en["following_negations"] = following
+
+termination = [
+    "although",
+    "apart from",
+    "as there are",
+    "aside from",
+    "but",
+    "except",
+    "however",
+    "involving",
+    "nevertheless",
+    "still",
+    "though",
+    "which",
+    "yet",
+]
+en["termination"] = termination
+
+LANGUAGES["en"] = en
+
+# en_clinical builds upon en
+en_clinical = dict()
+pseudo_clinical = pseudo + [
+    "gram negative",
+    "not rule out",
+    "not ruled out",
+    "not been ruled out",
+    "not drain",
+    "no suspicious change",
+    "no interval change",
+    "no significant interval change",
+]
+en_clinical["pseudo_negations"] = pseudo_clinical
+
+preceding_clinical = preceding + [
+    "patient was not",
+    "without indication of",
+    "without sign of",
+    "without signs of",
+    "without any reactions or signs of",
+    "no complaints of",
+    "no evidence of",
+    "no cause of",
+    "evaluate for",
+    "fails to reveal",
+    "free of",
+    "never developed",
+    "never had",
+    "did not exhibit",
+    "rules out",
+    "rule out",
+    "rule him out",
+    "rule her out",
+    "rule patient out",
+    "rule the patient out",
+    "ruled out",
+    "ruled him out",
+    "ruled her out",
+    "ruled patient out",
+    "ruled the patient out",
+    "r/o",
+    "ro",
+]
+en_clinical["preceding_negations"] = preceding_clinical
+
+following_clinical = following + ["was ruled out", "were ruled out", "free"]
+en_clinical["following_negations"] = following_clinical
+
+termination_clinical = termination + [
+    "cause for",
+    "cause of",
+    "causes for",
+    "causes of",
+    "etiology for",
+    "etiology of",
+    "origin for",
+    "origin of",
+    "origins for",
+    "origins of",
+    "other possibilities of",
+    "reason for",
+    "reason of",
+    "reasons for",
+    "reasons of",
+    "secondary to",
+    "source for",
+    "source of",
+    "sources for",
+    "sources of",
+    "trigger event for",
+]
+en_clinical["termination"] = termination_clinical
+LANGUAGES["en_clinical"] = en_clinical
+
+en_clinical_sensitive = dict()
+
+preceding_clinical_sensitive = preceding_clinical + [
+    "concern for",
+    "supposed",
+    "which causes",
+    "leads to",
+    "h/o",
+    "history of",
+    "instead of",
+    "if you experience",
+    "if you get",
+    "teaching the patient",
+    "taught the patient",
+    "teach the patient",
+    "educated the patient",
+    "educate the patient",
+    "educating the patient",
+    "monitored for",
+    "monitor for",
+    "test for",
+    "tested for",
+]
+en_clinical_sensitive["pseudo_negations"] = pseudo_clinical
+en_clinical_sensitive["preceding_negations"] = preceding_clinical_sensitive
+en_clinical_sensitive["following_negations"] = following_clinical
+en_clinical_sensitive["termination"] = termination_clinical
+
+LANGUAGES["en_clinical_sensitive"] = en_clinical_sensitive
+
+
+class termset:
+    def __init__(self, termset_lang):
+        self.pattern_types = [
+            "pseudo_negations",
+            "preceding_negations",
+            "following_negations",
+            "termination",
+        ]
+        self.terms = LANGUAGES[termset_lang]
+
+    def get_patterns(self):
+        return self.terms
+
+    def remove_patterns(self, pattern_dict):
+        for key, value in pattern_dict.items():
+            if key in self.pattern_types:
+                self.terms[key] = [i for i in self.terms[key] if i not in value]
+            else:
+                raise ValueError(f"Unexpected key: {key} not in {self.pattern_types}")
+
+    def add_patterns(self, pattern_dict):
+        for key, value in pattern_dict.items():
+            if key in self.pattern_types:
+                self.terms[key] = list(set(self.terms[key] + value))
+            else:
+                raise ValueError(f"Unexpected key: {key} not in {self.pattern_types}")

src/great_ai/utilities/external/pylatexenc/README.md

@@ -0,0 +1 @@
+https://github.com/phfaist/pylatexenc

src/great_ai/utilities/external/pylatexenc/__init__.py

@@ -0,0 +1,37 @@
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2015 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+
+"""
+Utilities for LaTeX to/from Unicode Text conversion.
+
+Main Site:
+
+    https://github.com/phfaist/pylatexenc/
+
+"""
+
+from .version import version_str as _version_str
+
+__version__ = _version_str

src/great_ai/utilities/external/pylatexenc/_util.py

@@ -0,0 +1,161 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2019 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+
+# Internal module. Internal API may move, disappear or otherwise change at any
+# time and without notice.
+
+
+try:
+    # Python >= 3.3
+    from collections.abc import MutableMapping
+except ImportError:
+    from collections import MutableMapping
+
+import bisect
+import warnings
+
+# ------------------------------------------------------------------------------
+
+
+def pylatexenc_deprecated_ver(ver, msg, stacklevel=2):
+    warnings.warn(
+        "Deprecated (pylatexenc {}): {} ".format(ver, msg.strip()),
+        DeprecationWarning,
+        stacklevel=stacklevel + 1,
+    )
+
+
+def pylatexenc_deprecated_2(msg, stacklevel=2):
+    warnings.warn(
+        (
+            "Deprecated (pylatexenc 2.0): {} "
+            "[see https://pylatexenc.readthedocs.io/en/latest/new-in-pylatexenc-2/]"
+        ).format(msg.strip()),
+        DeprecationWarning,
+        stacklevel=stacklevel + 1,
+    )
+
+
+# ------------------------------------------------------------------------------
+
+
+class LazyDict(MutableMapping):
+    r"""
+    A lazy dictionary that loads its data when it is first queried.
+
+    This is used to store the legacy
+    :py:data:`pylatexenc.latexwalker.default_macro_dict` as well as
+    :py:data:`pylatexenc.latex2text.default_macro_dict` etc.  Such that these
+    "dictionaries" are still exposed at the module-level, but the data is loaded
+    only if they are actually queried.
+    """
+
+    def __init__(self, generate_dict_fn):
+        self._full_dict = None
+        self._generate_dict_fn = generate_dict_fn
+
+    def _ensure_instance(self):
+        if self._full_dict is not None:
+            return
+        self._full_dict = self._generate_dict_fn()
+
+    def __getitem__(self, key):
+        self._ensure_instance()
+        return self._full_dict.__getitem__(key)
+
+    def __setitem__(self, key, val):
+        self._ensure_instance()
+        return self._full_dict.__setitem__(key, val)
+
+    def __delitem__(self, key):
+        self._ensure_instance()
+        return self._full_dict.__delitem__(key)
+
+    def __iter__(self):
+        self._ensure_instance()
+        return iter(self._full_dict)
+
+    def __len__(self):
+        self._ensure_instance()
+        return len(self._full_dict)
+
+    def copy(self):
+        self._ensure_instance()
+        return self._full_dict.copy()
+
+    def clear(self):
+        self._ensure_instance()
+        return self._full_dict.clear()
+
+
+# ------------------------------------------------------------------------------
+
+
+class LineNumbersCalculator(object):
+    r"""
+    Utility to calculate line numbers.
+    """
+
+    def __init__(self, s):
+        super(LineNumbersCalculator, self).__init__()
+
+        def find_all_new_lines(x):
+            # first line starts at the beginning of the string
+            yield 0
+            k = 0
+            while k < len(x):
+                k = x.find("\n", k)
+                if k == -1:
+                    return
+                k += 1
+                # s[k] is the character after the newline, i.e., the 0-th column
+                # of the new line
+                yield k
+
+        self._pos_new_lines = list(find_all_new_lines(s))
+
+    def pos_to_lineno_colno(self, pos, as_dict=False):
+        r"""
+        Return the line and column number corresponding to the given `pos`.
+
+        Return a tuple `(lineno, colno)` giving line number and column number.
+        Line numbers start at 1 and column number start at zero, i.e., the
+        beginning of the document (`pos=0`) has line and column number `(1,0)`.
+        If `as_dict=True`, then a dictionary with keys 'lineno', 'colno' is
+        returned instead of a tuple.
+        """
+
+        # find line number in list
+
+        # line_no is the index of the last item in self._pos_new_lines that is <= pos.
+        line_no = bisect.bisect_right(self._pos_new_lines, pos) - 1
+        assert line_no >= 0 and line_no < len(self._pos_new_lines)
+
+        col_no = pos - self._pos_new_lines[line_no]
+        # 1+... so that line and column numbers start at 1
+        if as_dict:
+            return {"lineno": 1 + line_no, "colno": col_no}
+        return (1 + line_no, col_no)