Add files

Signed-off-by: András Schmelczer <andras@schmelczer.dev>

good_ai/src/sus/external/negspacy/README.md

@@ -0,0 +1 @@
+https://github.com/jenojp/negspacy

good_ai/src/sus/external/negspacy/negation.py

@@ -0,0 +1,222 @@
+import logging
+
+from spacy.language import Language
+from spacy.matcher import PhraseMatcher
+from spacy.tokens import Token
+
+from .termsets import termset
+
+default_ts = termset("en").get_patterns()
+
+
+@Language.factory(
+    "negex",
+    default_config={
+        "neg_termset": default_ts,
+        "extension_name": "negex",
+        "chunk_prefix": list(),
+    },
+)
+class Negex:
+    """
+        A spaCy pipeline component which identifies negated tokens in text.
+
+        Based on: NegEx - A Simple Algorithm for Identifying Negated Findings and Diseasesin Discharge Summaries
+    Chapman, Bridewell, Hanbury, Cooper, Buchanan
+
+    Parameters
+    ----------
+    nlp: object
+        spaCy language object
+    termset_lang: str
+        language code, if using default termsets (e.g. "en" for english)
+    extension_name: str
+        defaults to "negex"; whether entity is negated is then available as ent._.negex
+    pseudo_negations: list
+        list of phrases that cancel out a negation, if empty, defaults are used
+    preceding_negations: list
+        negations that appear before an entity, if empty, defaults are used
+    following_negations: list
+        negations that appear after an entity, if empty, defaults are used
+    termination: list
+        phrases that "terminate" a sentence for processing purposes such as "but". If empty, defaults are used
+
+    """
+
+    def __init__(
+        self,
+        nlp: Language,
+        name: str,
+        neg_termset: dict,
+        extension_name: str,
+        chunk_prefix: list,
+    ):
+        if not Token.has_extension(extension_name):
+            Token.set_extension(extension_name, default=False, force=True)
+
+        ts = neg_termset
+        expected_keys = [
+            "pseudo_negations",
+            "preceding_negations",
+            "following_negations",
+            "termination",
+        ]
+        if not set(ts.keys()) == set(expected_keys):
+            raise KeyError(
+                f"Unexpected or missing keys in 'neg_termset', expected: {expected_keys}, instead got: {list(ts.keys())}"
+            )
+
+        self.pseudo_negations = ts["pseudo_negations"]
+        self.preceding_negations = ts["preceding_negations"]
+        self.following_negations = ts["following_negations"]
+        self.termination = ts["termination"]
+
+        self.nlp = nlp
+        self.extension_name = extension_name
+        self.build_patterns()
+        self.chunk_prefix = list(nlp.tokenizer.pipe(chunk_prefix))
+
+    def build_patterns(self):
+        # efficiently build spaCy matcher patterns
+        self.matcher = PhraseMatcher(self.nlp.vocab, attr="LOWER")
+
+        self.pseudo_patterns = list(self.nlp.tokenizer.pipe(self.pseudo_negations))
+        self.matcher.add("pseudo", None, *self.pseudo_patterns)
+
+        self.preceding_patterns = list(
+            self.nlp.tokenizer.pipe(self.preceding_negations)
+        )
+        self.matcher.add("Preceding", None, *self.preceding_patterns)
+
+        self.following_patterns = list(
+            self.nlp.tokenizer.pipe(self.following_negations)
+        )
+        self.matcher.add("Following", None, *self.following_patterns)
+
+        self.termination_patterns = list(self.nlp.tokenizer.pipe(self.termination))
+        self.matcher.add("Termination", None, *self.termination_patterns)
+
+    def process_negations(self, doc):
+        """
+        Find negations in doc and clean candidate negations to remove pseudo negations
+
+        Parameters
+        ----------
+        doc: object
+            spaCy Doc object
+
+        Returns
+        -------
+        preceding: list
+            list of tuples for preceding negations
+        following: list
+            list of tuples for following negations
+        terminating: list
+            list of tuples of terminating phrases
+
+        """
+        ###
+        # does not work properly in spacy 2.1.8. Will incorporate after 2.2.
+        # Relying on user to use NER in meantime
+        # see https://github.com/jenojp/negspacy/issues/7
+        ###
+        # if not doc.is_nered:
+        #     raise ValueError(
+        #         "Negations are evaluated for Named Entities found in text. "
+        #         "Your SpaCy pipeline does not included Named Entity resolution. "
+        #         "Please ensure it is enabled or choose a different language model that includes it."
+        #     )
+        preceding = list()
+        following = list()
+        terminating = list()
+
+        matches = self.matcher(doc)
+        pseudo = [
+            (match_id, start, end)
+            for match_id, start, end in matches
+            if self.nlp.vocab.strings[match_id] == "pseudo"
+        ]
+
+        for match_id, start, end in matches:
+            if self.nlp.vocab.strings[match_id] == "pseudo":
+                continue
+            pseudo_flag = False
+            for p in pseudo:
+                if start >= p[1] and start <= p[2]:
+                    pseudo_flag = True
+                    continue
+            if not pseudo_flag:
+                if self.nlp.vocab.strings[match_id] == "Preceding":
+                    preceding.append((match_id, start, end))
+                elif self.nlp.vocab.strings[match_id] == "Following":
+                    following.append((match_id, start, end))
+                elif self.nlp.vocab.strings[match_id] == "Termination":
+                    terminating.append((match_id, start, end))
+                else:
+                    logging.warnings(
+                        f"phrase {doc[start:end].text} not in one of the expected matcher types."
+                    )
+        return preceding, following, terminating
+
+    def termination_boundaries(self, doc, terminating):
+        """
+        Create sub sentences based on terminations found in text.
+
+        Parameters
+        ----------
+        doc: object
+            spaCy Doc object
+        terminating: list
+            list of tuples with (match_id, start, end)
+
+        returns
+        -------
+        boundaries: list
+            list of tuples with (start, end) of spans
+
+        """
+        sent_starts = [sent.start for sent in doc.sents]
+        terminating_starts = [t[1] for t in terminating]
+        starts = sent_starts + terminating_starts + [len(doc)]
+        starts.sort()
+        boundaries = list()
+        index = 0
+        for i, start in enumerate(starts):
+            if not i == 0:
+                boundaries.append((index, start))
+            index = start
+        return boundaries
+
+    def negex(self, doc):
+        """
+        Negates entities of interest
+
+        Parameters
+        ----------
+        doc: object
+            spaCy Doc object
+
+        """
+        preceding, following, terminating = self.process_negations(doc)
+        boundaries = self.termination_boundaries(doc, terminating)
+        for b in boundaries:
+            sub_preceding = [i for i in preceding if b[0] <= i[1] < b[1]]
+            sub_following = [i for i in following if b[0] <= i[1] < b[1]]
+
+            for e in doc[b[0] : b[1]]:
+                if any(pre < e.i for pre in [i[1] for i in sub_preceding]):
+                    e._.set(self.extension_name, True)
+                    continue
+                if any(fol > e.i for fol in [i[2] for i in sub_following]):
+                    e._.set(self.extension_name, True)
+                    continue
+                if self.chunk_prefix:
+                    if any(
+                        e.text.lower().startswith(c.text.lower())
+                        for c in self.chunk_prefix
+                    ):
+                        e._.set(self.extension_name, True)
+        return doc
+
+    def __call__(self, doc):
+        return self.negex(doc)

good_ai/src/sus/external/negspacy/termsets.py

@@ -0,0 +1,229 @@
+"""
+Default termsets for various languages
+"""
+
+LANGUAGES = dict()
+
+# english termset dictionary
+en = dict()
+pseudo = [
+    "no further",
+    "not able to be",
+    "not certain if",
+    "not certain whether",
+    "not necessarily",
+    "without any further",
+    "without difficulty",
+    "without further",
+    "might not",
+    "not only",
+    "no increase",
+    "no significant change",
+    "no change",
+    "no definite change",
+    "not extend",
+    "not cause",
+]
+en["pseudo_negations"] = pseudo
+
+preceding = [
+    "absence of",
+    "declined",
+    "denied",
+    "denies",
+    "denying",
+    "no sign of",
+    "no signs of",
+    "not",
+    "not demonstrate",
+    "symptoms atypical",
+    "doubt",
+    "negative for",
+    "no",
+    "versus",
+    "without",
+    "doesn't",
+    "doesnt",
+    "don't",
+    "dont",
+    "didn't",
+    "didnt",
+    "wasn't",
+    "wasnt",
+    "weren't",
+    "werent",
+    "isn't",
+    "isnt",
+    "aren't",
+    "arent",
+    "cannot",
+    "can't",
+    "cant",
+    "couldn't",
+    "couldnt",
+    "never",
+]
+en["preceding_negations"] = preceding
+
+following = [
+    "declined",
+    "unlikely",
+    "was not",
+    "were not",
+    "wasn't",
+    "wasnt",
+    "weren't",
+    "werent",
+]
+en["following_negations"] = following
+
+termination = [
+    "although",
+    "apart from",
+    "as there are",
+    "aside from",
+    "but",
+    "except",
+    "however",
+    "involving",
+    "nevertheless",
+    "still",
+    "though",
+    "which",
+    "yet",
+]
+en["termination"] = termination
+
+LANGUAGES["en"] = en
+
+# en_clinical builds upon en
+en_clinical = dict()
+pseudo_clinical = pseudo + [
+    "gram negative",
+    "not rule out",
+    "not ruled out",
+    "not been ruled out",
+    "not drain",
+    "no suspicious change",
+    "no interval change",
+    "no significant interval change",
+]
+en_clinical["pseudo_negations"] = pseudo_clinical
+
+preceding_clinical = preceding + [
+    "patient was not",
+    "without indication of",
+    "without sign of",
+    "without signs of",
+    "without any reactions or signs of",
+    "no complaints of",
+    "no evidence of",
+    "no cause of",
+    "evaluate for",
+    "fails to reveal",
+    "free of",
+    "never developed",
+    "never had",
+    "did not exhibit",
+    "rules out",
+    "rule out",
+    "rule him out",
+    "rule her out",
+    "rule patient out",
+    "rule the patient out",
+    "ruled out",
+    "ruled him out",
+    "ruled her out",
+    "ruled patient out",
+    "ruled the patient out",
+    "r/o",
+    "ro",
+]
+en_clinical["preceding_negations"] = preceding_clinical
+
+following_clinical = following + ["was ruled out", "were ruled out", "free"]
+en_clinical["following_negations"] = following_clinical
+
+termination_clinical = termination + [
+    "cause for",
+    "cause of",
+    "causes for",
+    "causes of",
+    "etiology for",
+    "etiology of",
+    "origin for",
+    "origin of",
+    "origins for",
+    "origins of",
+    "other possibilities of",
+    "reason for",
+    "reason of",
+    "reasons for",
+    "reasons of",
+    "secondary to",
+    "source for",
+    "source of",
+    "sources for",
+    "sources of",
+    "trigger event for",
+]
+en_clinical["termination"] = termination_clinical
+LANGUAGES["en_clinical"] = en_clinical
+
+en_clinical_sensitive = dict()
+
+preceding_clinical_sensitive = preceding_clinical + [
+    "concern for",
+    "supposed",
+    "which causes",
+    "leads to",
+    "h/o",
+    "history of",
+    "instead of",
+    "if you experience",
+    "if you get",
+    "teaching the patient",
+    "taught the patient",
+    "teach the patient",
+    "educated the patient",
+    "educate the patient",
+    "educating the patient",
+    "monitored for",
+    "monitor for",
+    "test for",
+    "tested for",
+]
+en_clinical_sensitive["pseudo_negations"] = pseudo_clinical
+en_clinical_sensitive["preceding_negations"] = preceding_clinical_sensitive
+en_clinical_sensitive["following_negations"] = following_clinical
+en_clinical_sensitive["termination"] = termination_clinical
+
+LANGUAGES["en_clinical_sensitive"] = en_clinical_sensitive
+
+
+class termset:
+    def __init__(self, termset_lang):
+        self.pattern_types = [
+            "pseudo_negations",
+            "preceding_negations",
+            "following_negations",
+            "termination",
+        ]
+        self.terms = LANGUAGES[termset_lang]
+
+    def get_patterns(self):
+        return self.terms
+
+    def remove_patterns(self, pattern_dict):
+        for key, value in pattern_dict.items():
+            if key in self.pattern_types:
+                self.terms[key] = [i for i in self.terms[key] if i not in value]
+            else:
+                raise ValueError(f"Unexpected key: {key} not in {self.pattern_types}")
+
+    def add_patterns(self, pattern_dict):
+        for key, value in pattern_dict.items():
+            if key in self.pattern_types:
+                self.terms[key] = list(set(self.terms[key] + value))
+            else:
+                raise ValueError(f"Unexpected key: {key} not in {self.pattern_types}")

good_ai/src/sus/external/pylatexenc/README.md

@@ -0,0 +1 @@
+https://github.com/phfaist/pylatexenc

good_ai/src/sus/external/pylatexenc/__init__.py

@@ -0,0 +1,37 @@
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2015 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+
+"""
+Utilities for LaTeX to/from Unicode Text conversion.
+
+Main Site:
+
+    https://github.com/phfaist/pylatexenc/
+
+"""
+
+from .version import version_str as _version_str
+
+__version__ = _version_str

good_ai/src/sus/external/pylatexenc/_util.py

@@ -0,0 +1,161 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2019 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+
+# Internal module. Internal API may move, disappear or otherwise change at any
+# time and without notice.
+
+
+try:
+    # Python >= 3.3
+    from collections.abc import MutableMapping
+except ImportError:
+    from collections import MutableMapping
+
+import bisect
+import warnings
+
+# ------------------------------------------------------------------------------
+
+
+def pylatexenc_deprecated_ver(ver, msg, stacklevel=2):
+    warnings.warn(
+        "Deprecated (pylatexenc {}): {} ".format(ver, msg.strip()),
+        DeprecationWarning,
+        stacklevel=stacklevel + 1,
+    )
+
+
+def pylatexenc_deprecated_2(msg, stacklevel=2):
+    warnings.warn(
+        (
+            "Deprecated (pylatexenc 2.0): {} "
+            "[see https://pylatexenc.readthedocs.io/en/latest/new-in-pylatexenc-2/]"
+        ).format(msg.strip()),
+        DeprecationWarning,
+        stacklevel=stacklevel + 1,
+    )
+
+
+# ------------------------------------------------------------------------------
+
+
+class LazyDict(MutableMapping):
+    r"""
+    A lazy dictionary that loads its data when it is first queried.
+
+    This is used to store the legacy
+    :py:data:`pylatexenc.latexwalker.default_macro_dict` as well as
+    :py:data:`pylatexenc.latex2text.default_macro_dict` etc.  Such that these
+    "dictionaries" are still exposed at the module-level, but the data is loaded
+    only if they are actually queried.
+    """
+
+    def __init__(self, generate_dict_fn):
+        self._full_dict = None
+        self._generate_dict_fn = generate_dict_fn
+
+    def _ensure_instance(self):
+        if self._full_dict is not None:
+            return
+        self._full_dict = self._generate_dict_fn()
+
+    def __getitem__(self, key):
+        self._ensure_instance()
+        return self._full_dict.__getitem__(key)
+
+    def __setitem__(self, key, val):
+        self._ensure_instance()
+        return self._full_dict.__setitem__(key, val)
+
+    def __delitem__(self, key):
+        self._ensure_instance()
+        return self._full_dict.__delitem__(key)
+
+    def __iter__(self):
+        self._ensure_instance()
+        return iter(self._full_dict)
+
+    def __len__(self):
+        self._ensure_instance()
+        return len(self._full_dict)
+
+    def copy(self):
+        self._ensure_instance()
+        return self._full_dict.copy()
+
+    def clear(self):
+        self._ensure_instance()
+        return self._full_dict.clear()
+
+
+# ------------------------------------------------------------------------------
+
+
+class LineNumbersCalculator(object):
+    r"""
+    Utility to calculate line numbers.
+    """
+
+    def __init__(self, s):
+        super(LineNumbersCalculator, self).__init__()
+
+        def find_all_new_lines(x):
+            # first line starts at the beginning of the string
+            yield 0
+            k = 0
+            while k < len(x):
+                k = x.find("\n", k)
+                if k == -1:
+                    return
+                k += 1
+                # s[k] is the character after the newline, i.e., the 0-th column
+                # of the new line
+                yield k
+
+        self._pos_new_lines = list(find_all_new_lines(s))
+
+    def pos_to_lineno_colno(self, pos, as_dict=False):
+        r"""
+        Return the line and column number corresponding to the given `pos`.
+
+        Return a tuple `(lineno, colno)` giving line number and column number.
+        Line numbers start at 1 and column number start at zero, i.e., the
+        beginning of the document (`pos=0`) has line and column number `(1,0)`.
+        If `as_dict=True`, then a dictionary with keys 'lineno', 'colno' is
+        returned instead of a tuple.
+        """
+
+        # find line number in list
+
+        # line_no is the index of the last item in self._pos_new_lines that is <= pos.
+        line_no = bisect.bisect_right(self._pos_new_lines, pos) - 1
+        assert line_no >= 0 and line_no < len(self._pos_new_lines)
+
+        col_no = pos - self._pos_new_lines[line_no]
+        # 1+... so that line and column numbers start at 1
+        if as_dict:
+            return {"lineno": 1 + line_no, "colno": col_no}
+        return (1 + line_no, col_no)

good_ai/src/sus/external/pylatexenc/latex2text/__main__.py

@@ -0,0 +1,325 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2018 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+import argparse
+import fileinput
+import logging
+import sys
+
+from .. import latexwalker
+from ..latex2text import LatexNodes2Text, _strict_latex_spaces_predef
+from ..version import version_str
+
+
+def main(argv=None):
+
+    if argv is None:
+        argv = sys.argv[1:]
+
+    parser = argparse.ArgumentParser(prog="latex2text", add_help=False)
+
+    codegroup = parser.add_argument_group("Input options")
+
+    codegroup.add_argument(
+        "--code",
+        "-c",
+        action="store",
+        default=None,
+        metavar="LATEX_CODE",
+        help="Convert the given LATEX_CODE to unicode text instead of reading "
+        "from FILE or standard input.  You cannot specify FILEs if you use this "
+        "option, and any standard input is ignored.",
+    )
+
+    codegroup.add_argument(
+        "files",
+        metavar="FILE",
+        nargs="*",
+        help="Input files to read LaTeX code from. If no FILE(s) is/are specified, "
+        "LaTeX code is read from standard input unless --code is specified",
+    )
+
+    group = parser.add_argument_group("LatexWalker options")
+
+    group.add_argument(
+        "--parser-keep-inline-math",
+        action="store_const",
+        const=True,
+        dest="parser_keep_inline_math",
+        default=None,
+        help=argparse.SUPPRESS,
+    )
+    group.add_argument(
+        "--no-parser-keep-inline-math",
+        action="store_const",
+        const=False,
+        dest="parser_keep_inline_math",
+        help=argparse.SUPPRESS,
+    )
+
+    group.add_argument(
+        "--tolerant-parsing",
+        action="store_const",
+        const=True,
+        dest="tolerant_parsing",
+        default=True,
+    )
+    group.add_argument(
+        "--no-tolerant-parsing",
+        action="store_const",
+        const=False,
+        dest="tolerant_parsing",
+        help="Tolerate syntax errors when parsing, and attempt to continue (default yes)",
+    )
+
+    # I'm not sure this flag is useful and if it should be exposed at all.
+    # Accept it, but make it hidden.
+    parser.add_argument(
+        "--strict-braces",
+        action="store_const",
+        const=True,
+        dest="strict_braces",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--no-strict-braces",
+        action="store_const",
+        const=False,
+        dest="strict_braces",
+        # help="Report errors for mismatching LaTeX braces (default no)"
+        help=argparse.SUPPRESS,
+    )
+
+    group = parser.add_argument_group("LatexNodes2Text options")
+
+    group.add_argument(
+        "--text-keep-inline-math",
+        action="store_const",
+        const=True,
+        dest="text_keep_inline_math",
+        default=None,
+        help=argparse.SUPPRESS,
+    )
+    group.add_argument(
+        "--no-text-keep-inline-math",
+        action="store_const",
+        const=False,
+        dest="text_keep_inline_math",
+        help=argparse.SUPPRESS,
+    )
+
+    group.add_argument(
+        "--math-mode",
+        action="store",
+        dest="math_mode",
+        choices=["text", "with-delimiters", "verbatim", "remove"],
+        default="text",
+        help="How to handle chunks of math mode LaTeX code. 'text' = convert "
+        "to text like the rest; 'with-delimiters' = same as 'text' but retain "
+        "the original math mode delimiters; 'verbatim' = keep verbatim LaTeX code; "
+        "'remove' = remove from input entirely",
+    )
+
+    group.add_argument(
+        "--fill-text",
+        dest="fill_text",
+        action="store",
+        nargs="?",
+        default=-1,
+        help="Attempt to wrap text to the given width, or 80 columns if option is "
+        "specified with no argument",
+    )
+
+    group.add_argument(
+        "--keep-comments",
+        action="store_const",
+        const=True,
+        dest="keep_comments",
+        default=False,
+    )
+    group.add_argument(
+        "--no-keep-comments",
+        action="store_const",
+        const=False,
+        dest="keep_comments",
+        help="Keep LaTeX comments in text output (default no)",
+    )
+
+    class ListWithHiddenItems(list):
+        def __init__(self, thelist, hiddenitems):
+            super(ListWithHiddenItems, self).__init__(thelist)
+            self.hiddenitems = hiddenitems
+
+        def __contains__(self, value):
+            return (
+                super(ListWithHiddenItems, self).__contains__(value)
+                or value in self.hiddenitems
+            )
+
+    strict_latex_spaces_choices = ListWithHiddenItems(
+        # the list
+        ["off", "on"]
+        + list(k for k in _strict_latex_spaces_predef.keys() if k != "default"),
+        # hidden items: Value is accepted, but not shown in list of choices
+        ["default"],
+    )
+    group.add_argument(
+        "--strict-latex-spaces",
+        choices=strict_latex_spaces_choices,
+        dest="strict_latex_spaces",
+        default="macros",
+        help="How to handle whitespace. See documentation for the class "
+        "LatexNodes2Text().",
+    )
+
+    group.add_argument(
+        "--keep-braced-groups",
+        action="store_const",
+        const=True,
+        dest="keep_braced_groups",
+        default=False,
+    )
+    group.add_argument(
+        "--no-keep-braced-groups",
+        action="store_const",
+        const=False,
+        dest="keep_braced_groups",
+        help="Keep LaTeX {braced groups} in text output (default no)",
+    )
+
+    group.add_argument(
+        "--keep-braced-groups-minlen",
+        type=int,
+        default=2,
+        dest="keep_braced_groups_minlen",
+        help="Only apply --keep-braced-groups to groups that contain at least "
+        "this many characters",
+    )
+
+    group = parser.add_argument_group("General options")
+
+    group.add_argument(
+        "-q",
+        "--quiet",
+        dest="logging_level",
+        action="store_const",
+        const=logging.ERROR,
+        default=logging.INFO,
+        help="Suppress warning messages",
+    )
+    group.add_argument(
+        "-v",
+        "--verbose",
+        dest="logging_level",
+        action="store_const",
+        const=logging.DEBUG,
+        help="Verbose output",
+    )
+    group.add_argument(
+        "--version",
+        action="version",
+        version="pylatexenc {}".format(version_str),
+        help="Show version information and exit",
+    )
+    group.add_argument(
+        "--help", action="help", help="Show this help information and exit"
+    )
+
+    args = parser.parse_args(argv)
+
+    logging.basicConfig()
+    logging.getLogger().setLevel(args.logging_level)
+    logger = logging.getLogger(__name__)
+
+    if (
+        args.parser_keep_inline_math is not None
+        or args.text_keep_inline_math is not None
+    ):
+        logger.warning(
+            "Options --parser-keep-inline-math and --text-keep-inline-math are "
+            "deprecated and no longer have any effect.  Please use "
+            "--math-mode=... instead."
+        )
+
+    latex = ""
+    if args.code:
+        if args.files:
+            logger.error(
+                "Cannot specify both FILEs and --code option. "
+                "Use --help option for more information."
+            )
+            sys.exit(1)
+        latex = args.code
+    else:
+        for line in fileinput.input(files=args.files):
+            latex += line
+
+    if args.fill_text != -1:
+        if args.fill_text is not None and len(args.fill_text):
+            fill_text = int(args.fill_text)
+        else:
+            fill_text = True
+    else:
+        fill_text = None
+
+    lw = latexwalker.LatexWalker(
+        latex, tolerant_parsing=args.tolerant_parsing, strict_braces=args.strict_braces
+    )
+
+    (nodelist, pos, len_) = lw.get_latex_nodes()
+
+    ln2t = LatexNodes2Text(
+        math_mode=args.math_mode,
+        keep_comments=args.keep_comments,
+        strict_latex_spaces=args.strict_latex_spaces,
+        keep_braced_groups=args.keep_braced_groups,
+        keep_braced_groups_minlen=args.keep_braced_groups_minlen,
+        fill_text=fill_text,
+    )
+
+    print(ln2t.nodelist_to_text(nodelist))
+
+
+def run_main():
+
+    try:
+
+        main()
+
+    except SystemExit:
+        raise
+    except:  # lgtm [py/catch-base-exception]
+        import pdb
+        import traceback
+
+        traceback.print_exc()
+        pdb.post_mortem()
+
+
+if __name__ == "__main__":
+
+    main()
+    # run_main() # debug

good_ai/src/sus/external/pylatexenc/latexencode/__init__.py

@@ -0,0 +1,331 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2018 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+r"""
+The `latexencode` module provides a set of routines that allows you to
+convert a unicode string to LaTeX escape sequences.
+
+For basic usage you can use the :py:func:`unicode_to_latex()` function
+directly::
+
+  >>> from pylatexenc.latexencode import unicode_to_latex
+  >>> print(unicode_to_latex('À votre santé'))
+  \`A votre sant\'e
+  >>> print(unicode_to_latex('The length of samples #3 & #4 is 3μm'))
+  The length of samples \#3 \& \#4 is 3\ensuremath{\mu}m
+
+The conversion is handled by the class :py:class:`UnicodeToLatexEncoder`.  If
+you are converting multiple strings, you may create an instance with the flags
+you like and invoke its method
+:py:meth:`~UnicodeToLatexEncoder.unicode_to_latex()` as many times as necessary::
+
+  >>> from pylatexenc.latexencode import UnicodeToLatexEncoder
+  >>> u = UnicodeToLatexEncoder(unknown_char_policy='replace')
+  >>> print(u.unicode_to_latex('À votre santé'))
+  \`A votre sant\'e
+  >>> print(u.unicode_to_latex('The length of samples #3 & #4 is 3μm'))
+  The length of samples \#3 \& \#4 is 3\ensuremath{\mu}m
+  >>> print(u.unicode_to_latex('À votre santé: 乾杯'))
+  No known latex representation for character: U+4E7E - ‘乾’
+  No known latex representation for character: U+676F - ‘杯’
+  \`A votre sant\'e: {\bfseries ?}{\bfseries ?}
+
+Example using custom conversion rules::
+
+  >>> from pylatexenc.latexencode import UnicodeToLatexEncoder, \
+  ...     UnicodeToLatexConversionRule, RULE_REGEX
+  >>> u = UnicodeToLatexEncoder(
+  ...     conversion_rules=[
+  ...         UnicodeToLatexConversionRule(rule_type=RULE_REGEX, rule=[
+  ...             (re.compile(r'-->'), r'\\textrightarrow'),
+  ...             (re.compile(r'<--'), r'\\textleftarrow'),
+  ...         ]),
+  ...         'defaults'
+  ...     ]
+  ... )
+  >>> print(u.unicode_to_latex("Cheers --> À votre santé"))
+  Cheers {\textrightarrow} \`A votre sant\'e
+
+See :py:class:`UnicodeToLatexEncoder` and
+:py:class:`UnicodeToLatexConversionRule`.  Note for regex rules, the replacement
+text is expanded like the second argument of `re.sub()` and backslashes need to
+be escaped even inside raw strings.
+
+.. versionadded:: 2.0
+
+   The class :py:class:`UnicodeToLatexEncoder` along with its helper functions
+   and classes were introduced in `pylatexenc 2.0`.
+
+   The earlier function :py:func:`utf8tolatex()` that was available in
+   `pylatexenc 1.x` is still provided unchanged, so code written for `pylatexenc
+   1.x` should work without changes.  New code is however strongly encouraged to
+   employ the new API.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import functools
+import itertools
+import logging
+import sys
+import unicodedata
+
+if sys.version_info.major > 2:
+    unicode = str  # need to support unicode() w/ no arguments
+    basestring = str
+    # use MappingProxyType for keeping
+    # inspect function argument names
+    from inspect import getfullargspec
+    from types import MappingProxyType as _MappingProxyType
+else:
+    _MappingProxyType = dict
+    # inspect function argument names -- simulate getfullargspec with getargspec (argh...)
+    from inspect import getargspec as getfullargspec
+
+logger = logging.getLogger(__name__)
+
+
+from .. import _util
+from ._partial_latex_encoder import PartialLatexToLatexEncoder
+from ._unicode_to_latex_encoder import (
+    RULE_CALLABLE,
+    RULE_DICT,
+    RULE_REGEX,
+    UnicodeToLatexConversionRule,
+    UnicodeToLatexEncoder,
+    get_builtin_conversion_rules,
+    get_builtin_uni2latex_dict,
+)
+
+# ------------------------------------------------
+
+
+# ------------------------------------------------
+
+
+# ------------------------------------------------
+
+
+_u2l_obj_cache = {}
+
+
+def unicode_to_latex(
+    s,
+    non_ascii_only=False,
+    replacement_latex_protection="braces",
+    unknown_char_policy="keep",
+    unknown_char_warning=True,
+):
+    r"""
+    Shorthand for constructing a :py:class:`UnicodeToLatexEncoder` instance and
+    calling its :py:meth:`~UnicodeToLatexEncoder.unicode_to_latex()` method.
+
+    The :py:class:`UnicodeToLatexEncoder` instances for given option settings
+    are cached, making repeated calls to :py:func:`unicode_to_latex()` possible
+    without creating a new instance upon each call.
+
+    The parameters `non_ascii_only`, `replacement_latex_protection`,
+    `unknown_char_policy`, and `unknown_char_warning` are directly passed on to
+    the :py:class:`UnicodeToLatexEncoder` constructor.  See the class doc for
+    :py:class:`UnicodeToLatexEncoder` for more information about what they do.
+
+    You may only use arguments to this function that are python hashable (like
+    `True`, `False`, or simple strings) to help us keep a cache of previously
+    constructed :py:class:`UnicodeToLatexEncoder` instances.  For instance, it
+    is not possible to provide a callable to `unknown_char_policy`.  It is also
+    not possible to specify custom conversion rules with this helper function.
+    If you need any of these features, simply create a
+    :py:class:`UnicodeToLatexEncoder` instance directly.
+    """
+
+    key = (
+        non_ascii_only,
+        replacement_latex_protection,
+        unknown_char_policy,
+        unknown_char_warning,
+    )
+
+    if key in _u2l_obj_cache:
+        u = _u2l_obj_cache[key]
+    else:
+        u = UnicodeToLatexEncoder(
+            non_ascii_only=non_ascii_only,
+            replacement_latex_protection=replacement_latex_protection,
+            unknown_char_policy=unknown_char_policy,
+            unknown_char_warning=unknown_char_warning,
+        )
+        _u2l_obj_cache[key] = u
+
+    return u.unicode_to_latex(s)
+
+
+# ------------------------------------------------------------------------------
+
+# Don't change pylatexenc 1.x function:
+
+
+def _get_deprecated_utf82latex():
+    #
+    # Don't issue a deprecation warning, because utf8tolatex() uses the
+    # `utf82latex` dict even if it isn't modified by the user.
+    #
+    #     _util.pylatexenc_deprecated_2(
+    #         "The module-level dictionary `pylatexenc.latexencode.utf82latex` is deprecated "
+    #         "and might be removed in a future version of `pylatexenc`.",
+    #     )
+
+    # return a copy of the dict so that the user can modify the module-level
+    # `utf82latex` dict without influencing the behavior of the new
+    # `unicode_to_latex()` routines. (E.g., if two python modules use
+    # pylatexenc.latexencode, we don't want one python module's use of
+    # `utf2tolatex()` to influence the behavior of another module's use of
+    # `unicode_to_latex()`.  If both modules use `utf8tolatex()`, we can't avoid
+    # this influence.)
+    from ._uni2latexmap import uni2latex as _uni2latex
+
+    return _uni2latex.copy()
+
+
+utf82latex = _util.LazyDict(generate_dict_fn=_get_deprecated_utf82latex)
+"""
+.. deprecated:: 2.0
+
+   Pylatexenc 1.x exposed the module-level dictionary `utf82latex` that could be
+   modified to alter the behavior of `utf8tolatex()`.
+
+   If you would like to obtain a copy of the built-in unicode to text
+   dictionary, see :py:func:`get_builtin_uni2latex_dict()`.  If you would like
+   to alter the behavior of :py:func:`utf8tolatex()`, you should use
+   :py:class:`UnicodeToLatexEncoder` which provides a rich interface for
+   specifying rules how to convert chars to LaTeX escapes.
+
+   For backwards compatibility, you can still modify the module-level dictionary
+   `utf82latex` (but you can't assign a new object to it) and this will directly
+   modify the global built-in dictionary of known latex escapes.  This is not
+   recommended however, and the `utf82latex` module-level dictionary might be
+   removed in the future.
+
+   .. warning::
+
+      Modifying the `utf82latex` module-level dictionary is not recommended.
+      Doing so will alter the behavior of the `utf8tolatex()` function also for
+      all other modules that also use `pylatexenc`!
+"""
+
+
+def utf8tolatex(
+    s,
+    non_ascii_only=False,
+    brackets=True,
+    substitute_bad_chars=False,
+    fail_bad_chars=False,
+):
+    """
+    .. note::
+
+       Since `pylatexenc 2.0`, it is recommended to use the the
+       :py:func:`unicode_to_latex()` function or the
+       :py:class:`UnicodeToLatexEncoder` class instead of the earlier function
+       `utf8tolatex()`.
+
+       The new routines provide much more flexibility and versatility.  For
+       instance, you can specify custom escape sequences for certain characters.
+       Some cheap benchmarks seem to indicate that the new routines are not
+       significantly slower than the `utf8tolatex()` function.  Also, the name
+       `utf8tolatex()` was poorly chosen, since the argument is in fact not
+       'utf-8'-encoded but rather a Python unicode string object.
+
+       The function `utf8tolatex()` is still provided unchanged from `pylatexenc
+       1.x`.  We do not plan to remove this function in the near future so it is
+       not (yet) considered as deprecated and we will continue to provide it in
+       near future versions of `pylatexenc`.  Bug reports, improvements, and new
+       features will however be directed to :py:func:`UnicodeToLatexEncoder()`.
+
+    Encode a UTF-8 string to a LaTeX snippet.
+
+    If `non_ascii_only` is set to `True`, then usual (ascii) characters such as ``#``,
+    ``{``, ``}`` etc. will not be escaped.  If set to `False` (the default), they are
+    escaped to their respective LaTeX escape sequences.
+
+    If `brackets` is set to `True` (the default), then LaTeX macros are enclosed in
+    brackets.  For example, ``sant\N{LATIN SMALL LETTER E WITH ACUTE}`` is replaced by
+    ``sant{\\'e}`` if `brackets=True` and by ``sant\\'e`` if `brackets=False`.
+
+    .. warning::
+        Using `brackets=False` might give you an invalid LaTeX string, so avoid
+        it! (for instance, ``ma\N{LATIN SMALL LETTER I WITH CIRCUMFLEX}tre`` will be
+        replaced incorrectly by ``ma\\^\\itre`` resulting in an unknown macro ``\\itre``).
+
+    If `substitute_bad_chars=True`, then any non-ascii character for which no LaTeX escape
+    sequence is known is replaced by a question mark in boldface. Otherwise (by default),
+    the character is left as it is.
+
+    If `fail_bad_chars=True`, then a `ValueError` is raised if we cannot find a
+    character substitution for any non-ascii character.
+
+    .. versionchanged:: 1.3
+
+        Added `fail_bad_chars` switch
+    """
+
+    s = unicode(s)  # make sure s is unicode
+    s = unicodedata.normalize("NFC", s)
+
+    if not s:
+        return ""
+
+    result = ""
+    for ch in s:
+        # logger.longdebug("Encoding char %r", ch)
+        if non_ascii_only and ord(ch) < 127:
+            result += ch
+        else:
+            # use the `utf82latex` dict -- not `_uni2latex` which should NOT be
+            # modified externally even for backwards-compatible code
+            lch = utf82latex.get(ord(ch), None)
+            if lch is not None:
+                # add brackets if needed, i.e. if we have a substituting macro.
+                # note: in condition, beware, that lch might be of zero length.
+                result += "{" + lch + "}" if brackets and lch[0:1] == "\\" else lch
+            elif (ord(ch) >= 32 and ord(ch) <= 127) or (ch in "\n\r\t"):
+                # ordinary printable ascii char, just add it
+                result += ch
+            else:
+                # non-ascii char
+                msg = "Character cannot be encoded into LaTeX: U+%04X - `%s'" % (
+                    ord(ch),
+                    ch,
+                )
+                if fail_bad_chars:
+                    raise ValueError(msg)
+
+                logger.warning(msg)
+                if substitute_bad_chars:
+                    result += r"{\bfseries ?}"
+                else:
+                    # keep unescaped char
+                    result += ch
+
+    return result

good_ai/src/sus/external/pylatexenc/latexencode/__main__.py

@@ -0,0 +1,148 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2019 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+
+import argparse
+import fileinput
+import logging
+import sys
+
+from ..latexencode import unicode_to_latex
+from ..version import version_str
+
+
+def main(argv=None):
+
+    if argv is None:
+        argv = sys.argv[1:]
+
+    parser = argparse.ArgumentParser(prog="latexencode", add_help=False)
+    parser.add_argument(
+        "files",
+        metavar="FILE",
+        nargs="*",
+        help="Input files (if none specified, read from stdandard input)",
+    )
+
+    parser.add_argument(
+        "--non-ascii-only",
+        action="store_const",
+        const=True,
+        dest="non_ascii_only",
+        default=False,
+    )
+    parser.add_argument(
+        "--no-non-ascii-only",
+        action="store_const",
+        const=False,
+        dest="non_ascii_only",
+        help="The option --non-ascii-only specifies that only non-ascii characters "
+        "are to be encoded into LaTeX sequences, and not characters like '$' "
+        "even though they might have a special LaTeX meaning.",
+    )
+
+    parser.add_argument(
+        "--replacement-latex-protection",
+        choices=(
+            "braces",
+            "braces-all",
+            "braces-almost-all",
+            "braces-after-macro",
+            "none",
+        ),
+        dest="replacement_latex_protection",
+        default="braces",
+        help=r"How to protect replacement latex code from producing invalid latex code "
+        r"when concatenated in a longer string.  One of 'braces', 'braces-all', "
+        r"'braces-almost-all', 'braces-after-macro', 'none'.  Example: using "
+        r"choice 'braces' we avoid the invalid replacement 'a→b' -> 'a\tob' "
+        r"with instead 'a{\to}b'.",
+    )
+
+    parser.add_argument(
+        "--unknown-char-policy",
+        choices=("keep", "replace", "ignore", "fail"),
+        dest="unknown_char_policy",
+        default="keep",
+        help="How to deal with nonascii characters with no known latex code equivalent.",
+    )
+
+    parser.add_argument(
+        "-q",
+        "--quiet",
+        dest="logging_level",
+        action="store_const",
+        const=logging.ERROR,
+        default=logging.INFO,
+        help="Suppress warning messages",
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="pylatexenc {}".format(version_str),
+        help="Show version information and exit",
+    )
+    parser.add_argument(
+        "--help", action="help", help="Show this help information and exit"
+    )
+
+    args = parser.parse_args(argv)
+
+    logging.basicConfig()
+    logging.getLogger().setLevel(args.logging_level)
+
+    latex = ""
+    for line in fileinput.input(files=args.files):
+        latex += line
+
+    result = unicode_to_latex(
+        latex,
+        non_ascii_only=args.non_ascii_only,
+        replacement_latex_protection=args.replacement_latex_protection,
+        unknown_char_policy=args.unknown_char_policy,
+    )
+
+    sys.stdout.write(result)
+
+
+def run_main():
+    try:
+
+        main()
+
+    except SystemExit:
+        raise
+    except:  # lgtm [py/catch-base-exception]
+        import pdb
+        import traceback
+
+        traceback.print_exc()
+        pdb.post_mortem()
+
+
+if __name__ == "__main__":
+
+    # run_main()  ## DEBUG
+    main()

good_ai/src/sus/external/pylatexenc/latexencode/_partial_latex_encoder.py

@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2021 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+# import sys
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+from ._unicode_to_latex_encoder import (
+    RULE_CALLABLE,
+    UnicodeToLatexConversionRule,
+    UnicodeToLatexEncoder,
+)
+
+# if sys.version_info.major == 2:
+#     bytes = str
+#     str = unicode
+
+
+class PartialLatexToLatexEncoder(UnicodeToLatexEncoder):
+    r"""
+    Encode a string while preserving some (fuzzily detected) LaTeX constructs
+    that the input string already has (e.g. accent macros or inline math modes).
+
+    Sometimes you need to fully LaTeX-encode a string that already has some
+    LaTeX constructs.  For instance, titles of bibliographic entries might
+    include some inline math or accents, but they might also include unicode
+    characters that need to be encoded.  Using a
+    :py:class:`UnicodeToLatexEncoder` on such strings would result in ugly
+    doubly-escaped strings such as ``\textbackslash{}'\{e\}``.  Instead,
+    constructs such as ``\'{e}`` should be preserved while other characters
+    and/or constructs (say '&' or '%') as well as unicode characters should be
+    encoded.
+
+    This class offers a simple partial solution: Characters are encoded as per
+    the given `conversion_rules` (or the default conversion rules of
+    :py:class:`UnicodeToLatexEncoder` objects), except that the characters in
+    `keep_latex_chars` are to be interpreted as LaTeX and are not to be further
+    encoded.
+
+    .. versionadded: 2.10
+    """
+
+    def __init__(
+        self,
+        # keyword arguments:
+        keep_latex_chars=r"\${}^_",
+        conversion_rules=None,
+        **kwargs
+    ):
+
+        base_conversion_rules = conversion_rules
+        if base_conversion_rules is None:
+            base_conversion_rules = ["defaults"]
+
+        super(PartialLatexToLatexEncoder, self).__init__(
+            # only a single rule, our own special method that tries to parse
+            # partial latex.
+            conversion_rules=[
+                UnicodeToLatexConversionRule(
+                    rule_type=RULE_CALLABLE,
+                    rule=self._do_partial_latex_encode_step,
+                    replacement_latex_protection="none",
+                )
+            ]
+            + base_conversion_rules,
+            **kwargs
+        )
+
+        self.keep_latex_chars = keep_latex_chars
+
+    def _do_partial_latex_encode_step(self, s, pos):
+        r"""
+        This method is used as a "callable rule" for the
+        :py:class:`UnicodeToLatexEncoder` object.
+
+        The strategy is to see if we have something that looks like a LaTeX char
+        we want to keep.  If so, keep it as is; if not, return `None` so that
+        further rules can be considered by the base unicode encoder.
+        """
+
+        from ..latexwalker import LatexWalker
+
+        if s[pos] in self.keep_latex_chars:
+            # Read a token and if it is a macro, keep the full macro!
+            lw = LatexWalker(s, tolerant_parsing=False)
+
+            tok = lw.get_token(pos, environments=False)
+
+            tok_as_latex = tok.pre_space + s[tok.pos : tok.pos + tok.len]
+
+            # keep the LaTeX token as-is
+            return (tok.pos + tok.len - pos, tok_as_latex)
+
+        return None

good_ai/src/sus/external/pylatexenc/latexencode/_unicode_to_latex_encoder.py

@@ -0,0 +1,686 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2021 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import functools
+import itertools
+import logging
+import sys
+import unicodedata
+
+if sys.version_info.major > 2:
+    unicode = str  # need to support unicode() w/ no arguments
+    basestring = str
+    # use MappingProxyType for keeping
+    # inspect function argument names
+    from inspect import getfullargspec
+    from types import MappingProxyType as _MappingProxyType
+else:
+    _MappingProxyType = dict
+    # inspect function argument names -- simulate getfullargspec with getargspec (argh...)
+    from inspect import getargspec as getfullargspec
+
+logger = logging.getLogger(__name__)
+
+
+def get_builtin_uni2latex_dict():
+    r"""
+    Return a dictionary that contains the default collection of known LaTeX
+    escape sequences for unicode characters.
+
+    The keys of the dictionary are integers that correspond to unicode code
+    points (i.e., `ord(char)`).  The values are the corresponding LaTeX
+    replacement strings.
+
+    The returned dictionary may not be modified.  To alter the behavior of
+    :py:func:`unicode_to_latex()`, you should specify custom rules to a new
+    instance of :py:class:`UnicodeToLatexEncoder`.
+
+    .. versionadded:: 2.0
+
+       This function was introduced in `pylatexenc 2.0`.
+    """
+    from ._uni2latexmap import uni2latex as _uni2latex
+
+    return _MappingProxyType(_uni2latex)
+
+
+RULE_DICT = 0
+r"""
+Indicates a rule type that is a dictionary of unicode point values to
+replacement strings. See :py:class:`UnicodeToLatexConversionRule`.
+
+.. versionadded:: 2.0
+
+   This member was introduced in pylatexenc version 2.0.
+"""
+
+RULE_REGEX = 1
+r"""
+Indicates a rule type that is a list (or iterable) of pairs
+`(compiled_regular_expression, replacement_string)`.  See
+:py:class:`UnicodeToLatexConversionRule`.
+
+.. versionadded:: 2.0
+
+   This member was introduced in pylatexenc version 2.0.
+"""
+
+RULE_CALLABLE = 2
+r"""
+Indicates a rule type that is a custom callable.  See
+:py:class:`UnicodeToLatexConversionRule`.
+
+.. versionadded:: 2.0
+
+   This member was introduced in pylatexenc version 2.0.
+"""
+
+
+class UnicodeToLatexConversionRule:
+    r"""
+    Specify a rule how to convert unicode characters into LaTeX escapes.
+
+    .. py:attribute:: rule_type
+    
+       One of :py:data:`RULE_DICT`, :py:data:`RULE_REGEX`, or
+       :py:data:`RULE_CALLABLE`.
+
+    .. py:attribute:: rule
+
+       A specification of the rule itself.  The `rule` attribute is an object
+       that depends on what `rule_type` is set to.  See below.
+
+    .. py:attribute:: replacement_latex_protection
+
+       If non-`None`, then the setting here will override any
+       `replacement_latex_protection` set on
+       :py:class:`UnicodeToLatexConversionRule` objects.  By default the value
+       is `None`, and you can set a replacement_latex_protection globally for
+       all rules on the :py:class:`UnicodeToLatexEncoder` object.
+
+       The use of this attribute is mainly in case you have a fancy rule in
+       which you already guarantee that whatever you output is valid LaTeX even
+       if concatenated with the remainder of the string; in this case you can
+       set `replacement_latex_protection='none'` to avoid unnecessary or
+       unwanted braces around the generated code.
+
+       .. versionadded:: 2.10
+
+          The `replacement_latex_protection` attribute was introduced in
+          `pylatexenc 2.10`.
+
+
+    Constructor syntax::
+    
+        UnicodeToLatexConversionRule(RULE_XXX, <...>)
+        UnicodeToLatexConversionRule(rule_type=RULE_XXX, rule=<...>)
+
+        UnicodeToLatexConversionRule(..., replacement_latex_protection='none')
+
+    Note that you can get some built-in rules via the
+    :py:func:`get_builtin_conversion_rules()` function::
+
+        conversion_rules = get_builtin_conversion_rules('defaults') # all defaults
+
+
+    Rules types:
+    
+      - `RULE_DICT`: If `rule_type` is `RULE_DICT`, then `rule` should be a
+        dictionary whose keys are integers representing unicode code points
+        (e.g., `0x210F`), and whose values are corresponding replacement strings
+        (e.g., ``r'\hbar'``).  See :py:func:`get_builtin_uni2latex_dict()` for
+        an example.
+
+      - `RULE_REGEX`: If `rule_type` is `RULE_REGEX`, then `rule` should be an
+        iterable of tuple pairs `(compiled_regular_expression,
+        replacement_string)` where `compiled_regular_expression` was obtained
+        with `re.compile(...)` and `replacement_string` is anything that can be
+        specified as the second (`repl`) argument of `re.sub(...)`.  This can be
+        a replacement string that includes escapes (like ``\1, \2, \g<name>``)
+        for captured sub-expressions or a callable that takes a match object as
+        argument.
+
+        .. note::
+    
+           The replacement string is parsed like the second argument to
+           `re.sub()` and backslashes have a special meaning because they can
+           refer to captured sub-expressions.  For a literal backslash, use two
+           backslashes ``\\`` in raw strings, four backslashes in normal
+           strings.
+
+        Example::
+
+          regex_conversion_rule = UnicodeToLatexConversionRule(
+              rule_type=RULE_REGEX,
+              rule=[
+                  # protect acronyms of capital letters with braces,
+                  # e.g.: ABC -> {ABC}
+                  (re.compile(r'[A-Z]{2,}'), r'{\1}'),
+                  # Additional rules, e.g., "..." -> "\ldots"
+                  (re.compile(r'...'), r'\\ldots'), # note double \\
+              ]
+          )
+
+      - `RULE_CALLABLE`: If `rule_type` is `RULE_CALLABLE`, then `rule` should
+        be a callable that accepts two arguments, the unicode string and the
+        position in the string (an integer).  The callable will be called with
+        the original unicode string as argument and the position of the
+        character that needs to be encoded.  If this rule can encode the given
+        character at the given position, it should return a tuple
+        `(consumed_length, replacement_string)` where `consumed_length` is the
+        number of characters in the unicode string that `replacement_string`
+        represents.  If the character(s) at the given position can't be encoded
+        by this rule, the callable should return `None` to indicate that further
+        rules should be attempted.
+
+        If the callable accepts an additional argument called `u2lobj`, then the
+        :py:class:`UnicodeToLatexEncoder` instance is provided to that argument.
+
+        For example, the following callable should achieve the same effect as
+        the previous example with regexes::
+
+          def convert_stuff(s, pos):
+              m = re.match(r'[A-Z]{2,}', s, pos)
+              if m is not None:
+                  return (m.end()-m.start(), '{'+m.group()+'}')
+              if s.startswith('...', pos): # or  s[pos:pos+3] == '...'
+                  return (3, r'\ldots')
+              return None
+
+
+    .. versionadded:: 2.0
+
+       This class was introduced in `pylatexenc 2.0`.
+    """
+
+    def __init__(
+        self,
+        rule_type,
+        rule=None,
+        # keyword-only, please:
+        replacement_latex_protection=None,
+    ):
+        self.rule_type = rule_type
+        self.rule = rule
+        self.replacement_latex_protection = replacement_latex_protection
+
+    def __repr__(self):
+        return "{}(rule_type={!r}, rule=<{}>, replacement_latex_protection={})".format(
+            self.__class__.__name__,
+            self.rule_type,
+            type(self.rule).__name__,
+            repr(self.replacement_latex_protection),
+        )
+
+
+def get_builtin_conversion_rules(builtin_name):
+    r"""
+    Return a built-in set of conversion rules specified by a given name
+    `builtin_name`.
+
+    There are two builtin conversion rules, with the following names:
+
+      - `'defaults'`: the default conversion rules, a custom-curated list of
+        unicode chars to LaTeX escapes.
+
+      - `'unicode-xml'`: the conversion rules derived from the `unicode.xml` file
+        maintained at https://www.w3.org/TR/xml-entity-names/#source by David
+        Carlisle.
+
+    The return value is a list of :py:class:`UnicodeToLatexConversionRule`
+    objects that can be either directly specified to the `conversion_rules=`
+    argument of :py:class:`UnicodeToLatexEncoder`, or included in a larger list
+    that can be provided to that argument.
+
+    .. versionadded:: 2.0
+
+       This function was introduced in `pylatexenc 2.0`.
+    """
+    if builtin_name == "defaults":
+        return [
+            UnicodeToLatexConversionRule(
+                rule_type=RULE_DICT, rule=get_builtin_uni2latex_dict()
+            )
+        ]
+    if builtin_name == "unicode-xml":
+        from . import _uni2latexmap_xml
+
+        return [
+            UnicodeToLatexConversionRule(
+                rule_type=RULE_DICT, rule=_uni2latexmap_xml.uni2latex
+            )
+        ]
+    raise ValueError("Unknown builtin rule set: {}".format(builtin_name))
+
+
+class UnicodeToLatexEncoder(object):
+    r"""
+    Encode a string with unicode characters into a LaTeX snippet.
+
+    The following general attributes can be specified as keyword arguments to
+    the constructor.  Note: These attributes must be specified to the
+    constructor and may NOT be subsequently modified.  This is because in the
+    constructor we pre-compile some rules and flags to optimize calls to
+    :py:meth:`unicode_to_text()`.
+
+    .. py:attribute:: non_ascii_only
+
+       Whether we should convert only non-ascii characters into LaTeX sequences,
+       or also all known ascii characters with special LaTeX meaning such as
+       '\\\\', '$', '&', etc.
+
+       If `non_ascii_only` is set to `True` (the default is `False`), then
+       conversion rules are not applied at positions in the string where an
+       ASCII character is encountered.
+
+    .. py:attribute:: conversion_rules
+
+       The conversion rules, specified as a list of
+       :py:class:`UnicodeToLatexConversionRule` objects.  For each position in
+       the string, the rules will be applied in the given sequence until a
+       replacement string is found.
+
+       Instead of a :py:class:`UnicodeToLatexConversionRule` object you may also
+       specify a string specifying a built-in rule (e.g., 'defaults'), which
+       will be expanded to the corresponding rules according to
+       :py:func:`get_builtin_conversion_rules()`.
+
+       If you specify your own list of rules using this argument, you will
+       probably want to include presumably at the end of your list the element
+       'defaults' to include all built-in default conversion rules.  To override
+       built-in rules, simply add your custom rules earlier in the list.
+       Example::
+
+         conversion_rules = [
+             # our custom rules
+             UnicodeToLatexConversionRule(RULE_REGEX, [
+                 # double \\ needed, see UnicodeToLatexConversionRule
+                 ( re.compile(r'...'), r'\\ldots' ),
+                 ( re.compile(r'î'), r'\\^i' ),
+             ]),
+             # plus all the default rules
+             'defaults'
+         ]
+         u = UnicodeToLatexEncoder(conversion_rules=conversion_rules)
+
+    .. py:attribute:: replacement_latex_protection
+
+       How to "protect" LaTeX replacement text that looks like it could be
+       interpreted differently if concatenated to arbitrary strings before and
+       after.
+
+       Currently in the default scheme only one situation is recognized: if the
+       replacement string ends with a latex macro invocation with a non-symbol
+       macro name, e.g. ``\textemdash`` or ``\^\i``.  Indeed, if we naively
+       replace these texts in an arbitrary string (like ``maître``), we might
+       get an invalid macro invocation (like ``ma\^\itre`` which causes un known
+       macro name ``\itre``).
+
+       Possible protection schemes are:
+
+         - 'braces' (the default):  Any suspicious replacement text (that
+           might look fragile) is placed in curly braces ``{...}``.
+
+         - 'braces-all':  All replacement latex escapes are surrounded in
+           protective curly braces ``{...}``, regardless of whether or not they
+           might be deemed "fragile" or "unsafe".
+
+         - 'braces-almost-all':  Almost all replacement latex escapes are
+           surrounded in protective curly braces ``{...}``.  This option
+           emulates closely the behavior of `brackets=True` of the function
+           `utf8tolatex()` in `pylatexenc 1.x`, though I'm not sure it is really
+           useful.  [Specifically, all those replacement strings that start with
+           a backslash are surrounded by curly braces].
+
+         - 'braces-after-macro':  In the situation where the replacement latex
+           code ends with a string-named macro, then a pair of empty braces is
+           added at the end of the replacement text to protect the macro.
+
+         - 'none': No protection is applied, even in "unsafe" cases.  This is
+           not recommended, as this will likely result in invalid LaTeX
+           code. (Note this is the string 'none', not Python's built-in `None`.)
+
+         - any callable object: The callable should take a single argument, the
+           replacement latex string associated with a piece of the input (maybe
+           a special character) that has been encoded; it should return the
+           actual string to append to the output string.
+
+         .. versionadded:: 2.10
+
+            You can specify a callable object to `replacement_latex_protection`
+            since `pylatexenc 2.10`.
+
+    .. py:attribute:: unknown_char_policy
+
+       What to do when a non-ascii character is encountered without any known
+       substitution macro.  The attribute `unknown_char_policy` can be set to one of:
+
+         - 'keep': keep the character as is;
+
+         - 'replace': replace the character by a boldface question mark;
+
+         - 'ignore': ignore the character from the input entirely and don't
+           output anything for it;
+
+         - 'fail': raise a `ValueError` exception;
+
+         - 'unihex': output the unicode hexadecimal code (U+XXXX) of the
+           character in typewriter font;
+
+         - a Python callable --- will be called with argument the character that
+           could not be encoded.  (If the callable accepts a second argument
+           called 'u2lobj', then the `UnicodeToLatexEncoder` instance is
+           provided to that argument.)  The return value of the callable is used
+           as LaTeX replacement code.
+
+    .. py:attribute:: unknown_char_warning
+
+       In addition to the `unknown_char_policy`, this attribute indicates
+       whether or not (`True` or `False`) one should generate a warning when a
+       nonascii character without any known latex representation is
+       encountered. (Default: True)
+
+    .. py:attribute:: latex_string_class
+
+       The return type of :py:meth:`unicode_to_latex()`.  Normally this is a
+       simple unicode string (`str` on `Python 3` or `unicode` on `Python 2`).
+
+       But you can specify your custom string type via the `latex_string_class`
+       argument.  The `latex_string_class` will be invoked with no arguments to
+       construct an empty object (so `latex_string_class` can be either an
+       object that can be constructed with no arguments or it can be a function
+       with no arguments that return a fresh object instance).  The object must
+       support the operation "+=", i.e., you should overload the ``__iadd__()``
+       method.
+
+       For instance, you can record the chunks that would have been appended
+       into a single string as follows::
+
+           class LatexChunkList:
+               def __init__(self):
+                   self.chunks = []
+
+               def __iadd__(self, s):
+                   self.chunks.append(s)
+                   return self
+
+           u = UnicodeToLatexEncoder(latex_string_class=LatexChunkList,
+                                     replacement_latex_protection='none')
+           result = u.unicode_to_latex("é → α")
+           # result.chunks == [ r"\'e", ' ', r'\textrightarrow', ' ',
+           #                    r'\ensuremath{\alpha}' ]
+
+    .. warning::
+
+       None of the above attributes should be modified after constructing the
+       object.  The values specified to the class constructor are final and
+       cannot be changed.  [Indeed, the class constructor "compiles" these
+       attribute values into a data structure that makes
+       :py:meth:`unicode_to_text()` slightly more efficient.]
+
+    .. versionadded:: 2.0
+
+       This class was introduced in `pylatexenc 2.0`.
+    """
+
+    def __init__(self, **kwargs):
+        self.non_ascii_only = kwargs.pop("non_ascii_only", False)
+        self.conversion_rules = kwargs.pop("conversion_rules", ["defaults"])
+        self.replacement_latex_protection = kwargs.pop(
+            "replacement_latex_protection", "braces"
+        )
+        self.unknown_char_policy = kwargs.pop("unknown_char_policy", "keep")
+        self.unknown_char_warning = kwargs.pop("unknown_char_warning", True)
+        self.latex_string_class = kwargs.pop("latex_string_class", unicode)
+
+        if kwargs:
+            logger.warning(
+                "Ignoring unknown keyword arguments: %s", ",".join(kwargs.keys())
+            )
+
+        super(UnicodeToLatexEncoder, self).__init__(**kwargs)
+
+        # build generator that expands built-in conversion rules
+        expanded_conversion_rules = itertools.chain.from_iterable(
+            (get_builtin_conversion_rules(r) if isinstance(r, basestring) else [r])
+            for r in self.conversion_rules
+        )
+
+        #
+        # now "pre-compile" some stuff so that calls to unicode_to_latex() can
+        # hopefully execute faster
+        #
+
+        # "pre-compile" rules and check rule types:
+        self._compiled_rules = []
+        for rule in expanded_conversion_rules:
+            if rule.rule_type == RULE_DICT:
+                self._compiled_rules.append(
+                    functools.partial(self._apply_rule_dict, rule.rule, rule)
+                )
+            elif rule.rule_type == RULE_REGEX:
+                self._compiled_rules.append(
+                    functools.partial(self._apply_rule_regex, rule.rule, rule)
+                )
+            elif rule.rule_type == RULE_CALLABLE:
+                thecallable = rule.rule
+                if "u2lobj" in getfullargspec(thecallable)[0]:
+                    thecallable = functools.partial(rule.rule, u2lobj=self)
+                self._compiled_rules.append(
+                    functools.partial(self._apply_rule_callable, thecallable, rule)
+                )
+            else:
+                raise TypeError("Invalid rule type: {}".format(rule.rule_type))
+
+        # bad char policy:
+        if isinstance(self.unknown_char_policy, basestring):
+            self._do_unknown_char = self._get_method_fn(
+                "do_unknown_char", self.unknown_char_policy, what="unknown_char_policy"
+            )
+        elif callable(self.unknown_char_policy):
+            fn = self.unknown_char_policy
+            if "u2lobj" in getfullargspec(fn)[0]:
+                self._do_unknown_char = functools.partial(
+                    self.unknown_char_policy, u2lobj=self
+                )
+            else:
+                self._do_unknown_char = self.unknown_char_policy
+        else:
+            raise TypeError(
+                "Invalid argument for unknown_char_policy: {!r}".format(
+                    self.unknown_char_policy
+                )
+            )
+
+        # bad char warning:
+        if not self.unknown_char_warning:
+            self._do_warn_unknown_char = lambda ch: None  # replace method by no-op
+
+        # set a method that will skip ascii characters if required:
+        if self.non_ascii_only:
+            self._maybe_skip_ascii = self._check_do_skip_ascii
+        else:
+            self._maybe_skip_ascii = lambda s, p: False
+
+        # set a method to protect replacement latex code, if necessary:
+        self._apply_protection = self._get_replacement_latex_fn(
+            self.replacement_latex_protection
+        )
+
+    def _get_method_fn(self, base, name, what):
+        selfmethname = "_" + base + "_" + name.replace("-", "_")
+        if not hasattr(self, selfmethname):
+            raise ValueError("Invalid {}: {}".format(what, name))
+        return getattr(self, selfmethname)
+
+    def _get_replacement_latex_fn(self, replacement_latex_protection):
+        if callable(replacement_latex_protection):
+            return replacement_latex_protection
+        return self._get_method_fn(
+            "apply_protection",
+            replacement_latex_protection,
+            what="replacement_latex_protection",
+        )
+
+    def unicode_to_latex(self, s):
+        """
+        Convert unicode characters in the string `s` into latex escape sequences,
+        according to the rules and options given to the constructor.
+        """
+
+        s = unicode(s)  # make sure s is unicode
+        s = unicodedata.normalize("NFC", s)
+
+        class _NS:
+            pass
+
+        p = _NS()
+        p.latex = self.latex_string_class()
+        p.pos = 0
+
+        while p.pos < len(s):
+
+            if self._maybe_skip_ascii(s, p):
+                continue
+
+            for compiledrule in self._compiled_rules:
+                if compiledrule(s, p):
+                    break
+            else:
+                # for-else, see
+                # https://docs.python.org/2/tutorial/controlflow.html\
+                # #break-and-continue-statements-and-else-clauses-on-loops
+                ch = s[p.pos]
+                o = ord(ch)
+                if (o >= 32 and o <= 127) or (ch in "\n\r\t"):
+                    p.latex += ch
+                    p.pos += 1
+                else:
+                    self._do_warn_unknown_char(ch)
+                    p.latex += self._do_unknown_char(ch)
+                    p.pos += 1
+
+        return p.latex
+
+    def _check_do_skip_ascii(self, s, p):
+        if ord(s[p.pos]) < 127:
+            # skip, we only want to convert non-ascii chars
+            p.latex += s[p.pos]
+            p.pos += 1
+            return True
+        return False
+
+    def _apply_rule_dict(self, ruledict, rule, s, p):
+        o = ord(s[p.pos])
+        if o in ruledict:
+            self._apply_replacement(p, ruledict[o], 1, rule)
+            return True
+        return None
+
+    def _apply_rule_regex(self, ruleregexes, rule, s, p):
+        for regex, repl in ruleregexes:
+            m = regex.match(s, p.pos)
+            if m is not None:
+                if callable(repl):
+                    replstr = repl(m)
+                else:
+                    replstr = m.expand(repl)
+                self._apply_replacement(p, replstr, m.end() - m.start(), rule)
+                return True
+        return None
+
+    def _apply_rule_callable(self, rulecallable, rule, s, p):
+        res = rulecallable(s, p.pos)
+        if res is None:
+            return None
+        (consumed, repl) = res
+        self._apply_replacement(p, repl, consumed, rule)
+        return True
+
+    def _apply_replacement(self, p, repl, numchars, ruleobj):
+        # check for possible replacement latex protection, like braces.
+
+        protect_fn = self._apply_protection
+
+        # maybe the rule object has overridden the replacement_latex_protection to use.
+        if ruleobj.replacement_latex_protection is not None:
+            protect_fn = self._get_replacement_latex_fn(
+                ruleobj.replacement_latex_protection
+            )
+
+        repl = protect_fn(repl)
+        p.latex += repl
+        p.pos += numchars
+
+    def _apply_protection_none(self, repl):
+        # no protection
+        return repl
+
+    def _apply_protection_braces(self, repl):
+        k = repl.rfind("\\")
+        if k >= 0 and repl[k + 1 :].isalpha():
+            # has dangling named macro, apply protection.
+            return "{" + repl + "}"
+        return repl
+
+    def _apply_protection_braces_almost_all(self, repl):
+        if repl[0:1] == "\\":
+            return "{" + repl + "}"
+        return repl
+
+    def _apply_protection_braces_all(self, repl):
+        return "{" + repl + "}"
+
+    def _apply_protection_braces_after_macro(self, repl):
+        k = repl.rfind("\\")
+        if k >= 0 and repl[k + 1 :].isalpha():
+            # has dangling named macro, apply protection.
+            return repl + "{}"
+        return repl
+
+    # policies for "bad chars":
+    def _do_unknown_char_keep(self, ch):
+        return ch
+
+    def _do_unknown_char_replace(self, ch):
+        return r"{\bfseries ?}"
+
+    def _do_unknown_char_ignore(self, ch):
+        return ""
+
+    def _do_unknown_char_fail(self, ch):
+        raise ValueError(
+            "No known latex representation for character: U+%04X - ‘%s’" % (ord(ch), ch)
+        )
+
+    def _do_unknown_char_unihex(self, ch):
+        return r"\ensuremath{\langle}\texttt{U+%04X}\ensuremath{\rangle}" % (ord(ch))
+
+    def _do_warn_unknown_char(self, ch):
+        logger.warning(
+            "No known latex representation for character: U+%04X - ‘%s’", ord(ch), ch
+        )

good_ai/src/sus/external/pylatexenc/latexwalker/__main__.py

@@ -0,0 +1,230 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2018 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+import argparse
+import fileinput
+import json
+import logging
+import sys
+
+from ..latexwalker import LatexWalker, disp_node, make_json_encoder
+from ..version import version_str
+
+
+def main(argv=None):
+
+    if argv is None:
+        argv = sys.argv[1:]
+
+    parser = argparse.ArgumentParser(prog="latexwalker", add_help=False)
+
+    parser.add_argument(
+        "--output-format",
+        metavar="FORMAT",
+        dest="output_format",
+        choices=["human", "json"],
+        default="human",
+        help='Requested output format for the node tree ("human" or "json")',
+    )
+    parser.add_argument(
+        "--json-indent",
+        metavar="NUMSPACES",
+        dest="json_indent",
+        type=int,
+        default=2,
+        help="Indentation in JSON output (specify number of spaces "
+        "per indentation level)",
+    )
+    parser.add_argument(
+        "--json-compact",
+        dest="json_indent",
+        action="store_const",
+        const=None,
+        help="Output compact JSON",
+    )
+
+    parser.add_argument(
+        "--keep-inline-math",
+        action="store_const",
+        const=True,
+        dest="keep_inline_math",
+        default=True,
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--no-keep-inline-math",
+        action="store_const",
+        const=False,
+        dest="keep_inline_math",
+        help=argparse.SUPPRESS,
+    )
+
+    parser.add_argument(
+        "--tolerant-parsing",
+        action="store_const",
+        const=True,
+        dest="tolerant_parsing",
+        default=True,
+    )
+    parser.add_argument(
+        "--no-tolerant-parsing",
+        action="store_const",
+        const=False,
+        dest="tolerant_parsing",
+        help="Tolerate syntax errors when parsing, and attempt "
+        "to continue (default yes)",
+    )
+
+    # I'm not sure this flag is useful and if it should be exposed at all.
+    # Accept it, but make it hidden.
+    parser.add_argument(
+        "--strict-braces",
+        action="store_const",
+        const=True,
+        dest="strict_braces",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--no-strict-braces",
+        action="store_const",
+        const=False,
+        dest="strict_braces",
+        # help="Report errors for mismatching LaTeX braces (default no)"
+        help=argparse.SUPPRESS,
+    )
+
+    parser.add_argument(
+        "-q",
+        "--quiet",
+        dest="logging_level",
+        action="store_const",
+        const=logging.ERROR,
+        default=logging.INFO,
+        help="Suppress warning messages",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        dest="logging_level",
+        action="store_const",
+        const=logging.DEBUG,
+        help="Verbose output",
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="pylatexenc {}".format(version_str),
+        help="Show version information and exit",
+    )
+    parser.add_argument(
+        "--help", action="help", help="Show this help information and exit"
+    )
+
+    parser.add_argument(
+        "--code",
+        "-c",
+        action="store",
+        default=None,
+        metavar="LATEX_CODE",
+        help="Convert the given LATEX_CODE to unicode text instead of reading "
+        "from FILE or standard input.  You cannot specify FILEs if you use this "
+        "option, and any standard input is ignored.",
+    )
+
+    parser.add_argument(
+        "files",
+        metavar="FILE",
+        nargs="*",
+        help="Input files (if none specified, read from stdandard input)",
+    )
+
+    args = parser.parse_args(argv)
+
+    logging.basicConfig()
+    logging.getLogger().setLevel(args.logging_level)
+    logger = logging.getLogger(__name__)
+
+    latex = ""
+    if args.code:
+        if args.files:
+            logger.error(
+                "Cannot specify both FILEs and --code option. "
+                "Use --help option for more information."
+            )
+            sys.exit(1)
+        latex = args.code
+    else:
+        for line in fileinput.input(files=args.files):
+            latex += line
+
+    latexwalker = LatexWalker(
+        latex, tolerant_parsing=args.tolerant_parsing, strict_braces=args.strict_braces
+    )
+
+    (nodelist, pos, len_) = latexwalker.get_latex_nodes()
+
+    if args.output_format == "human":
+        print("\n--- NODES ---\n")
+        for n in nodelist:
+            disp_node(n)
+        print("\n-------------\n")
+        return
+
+    if args.output_format == "json":
+        json.dump(
+            {
+                "nodelist": nodelist,
+            },
+            sys.stdout,
+            cls=make_json_encoder(latexwalker),
+            indent=args.json_indent,
+        )
+        sys.stdout.write("\n")
+        return
+
+    raise ValueError("Invalid output format: " + args.output_format)
+
+
+def run_main():
+
+    try:
+
+        main()
+
+    except SystemExit:
+        raise
+    except:  # lgtm [py/catch-base-exception]
+        import pdb
+        import traceback
+
+        traceback.print_exc()
+        pdb.post_mortem()
+
+
+if __name__ == "__main__":
+
+    # run_main() # debug
+    main()

good_ai/src/sus/external/pylatexenc/latexwalker/_defaultspecs.py

@@ -0,0 +1,459 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2019 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+
+# Internal module. May change without notice.
+
+
+from ..macrospec import (
+    EnvironmentSpec,
+    MacroSpec,
+    MacroStandardArgsParser,
+    VerbatimArgsParser,
+    std_environment,
+    std_macro,
+    std_specials,
+)
+
+specs = [
+    #
+    # CATEGORY: latex-base
+    #
+    (
+        "latex-base",
+        {
+            "macros": [
+                std_macro("documentclass", True, 1),
+                std_macro("usepackage", True, 1),
+                std_macro("RequirePackage", True, 1),
+                std_macro("selectlanguage", True, 1),
+                std_macro("setlength", True, 2),
+                std_macro("addlength", True, 2),
+                std_macro("setcounter", True, 2),
+                std_macro("addcounter", True, 2),
+                std_macro("newcommand", "*{[[{"),
+                std_macro("renewcommand", "*{[[{"),
+                std_macro("providecommand", "*{[[{"),
+                std_macro("newenvironment", "*{[[{{"),
+                std_macro("renewenvironment", "*{[[{{"),
+                std_macro("provideenvironment", "*{[[{{"),
+                std_macro("DeclareMathOperator", "*{{"),
+                std_macro("hspace", "*{"),
+                std_macro("vspace", "*{"),
+                MacroSpec(
+                    "mbox",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                # \title, \author, \date
+                MacroSpec("title", "{"),
+                MacroSpec("author", "{"),
+                MacroSpec("date", "{"),
+                # (Note: single backslash) end of line with optional no-break ('*') and
+                # additional vertical spacing, e.g. \\*[2mm]
+                #
+                # Special for this command: don't allow an optional spacing argument
+                # [2mm] to be separated by spaces from the rest of the macro.  This
+                # emulates the behavior in AMS environments, and avoids some errors;
+                # e.g. in "\begin{align} A=0 \\ [C,D]=0 \end{align}" the "[C,D]"
+                # does not get captured as an optional macro argument.
+                MacroSpec(
+                    "\\",
+                    args_parser=MacroStandardArgsParser(
+                        "*[", optional_arg_no_space=True
+                    ),
+                ),
+                std_macro("item", True, 0),
+                # \input{someotherfile}
+                std_macro("input", False, 1),
+                std_macro("include", False, 1),
+                std_macro("includegraphics", True, 1),
+                std_macro("chapter", "*[{"),
+                std_macro("section", "*[{"),
+                std_macro("subsection", "*[{"),
+                std_macro("subsubsection", "*[{"),
+                std_macro("pagagraph", "*[{"),
+                std_macro("subparagraph", "*[{"),
+                std_macro("bibliography", "{"),
+                std_macro("emph", False, 1),
+                MacroSpec(
+                    "textrm",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textit",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textbf",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textmd",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textsc",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textsf",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textsl",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "texttt",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "textup",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                MacroSpec(
+                    "text",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[False]),
+                ),
+                std_macro("mathrm", False, 1),  # only allowed in math mode anyway
+                std_macro("mathbb", False, 1),  # only allowed in math mode anyway
+                std_macro("mathbf", False, 1),
+                std_macro("mathit", False, 1),
+                std_macro("mathsf", False, 1),
+                std_macro("mathtt", False, 1),
+                std_macro("mathcal", False, 1),
+                std_macro("mathscr", False, 1),
+                std_macro("mathfrak", False, 1),
+                std_macro("label", False, 1),
+                std_macro("ref", False, 1),
+                std_macro("autoref", False, 1),
+                std_macro("cref", False, 1),
+                std_macro("Cref", False, 1),
+                std_macro("eqref", False, 1),
+                std_macro("url", False, 1),
+                std_macro("hypersetup", False, 1),
+                std_macro("footnote", True, 1),
+                std_macro("keywords", False, 1),
+                std_macro("hphantom", True, 1),
+                std_macro("vphantom", True, 1),
+                std_macro("'", False, 1),
+                std_macro("`", False, 1),
+                std_macro('"', False, 1),
+                std_macro("c", False, 1),
+                std_macro("^", False, 1),
+                std_macro("~", False, 1),
+                std_macro("H", False, 1),
+                std_macro("k", False, 1),
+                std_macro("=", False, 1),
+                std_macro("b", False, 1),
+                std_macro(".", False, 1),
+                std_macro("d", False, 1),
+                std_macro("r", False, 1),
+                std_macro("u", False, 1),
+                std_macro("v", False, 1),
+                MacroSpec(
+                    "ensuremath",
+                    args_parser=MacroStandardArgsParser("{", args_math_mode=[True]),
+                ),
+                std_macro("not", False, 1),
+                std_macro("vec", False, 1),
+                std_macro("dot", False, 1),
+                std_macro("hat", False, 1),
+                std_macro("check", False, 1),
+                std_macro("breve", False, 1),
+                std_macro("acute", False, 1),
+                std_macro("grave", False, 1),
+                std_macro("tilde", False, 1),
+                std_macro("bar", False, 1),
+                std_macro("ddot", False, 1),
+                std_macro("frac", False, 2),
+                std_macro("nicefrac", False, 2),
+                std_macro("sqrt", True, 1),
+                MacroSpec("overline", "{"),
+                MacroSpec("underline", "{"),
+                MacroSpec("widehat", "{"),
+                MacroSpec("widetilde", "{"),
+                MacroSpec("wideparen", "{"),
+                MacroSpec("overleftarrow", "{"),
+                MacroSpec("overrightarrow", "{"),
+                MacroSpec("overleftrightarrow", "{"),
+                MacroSpec("underleftarrow", "{"),
+                MacroSpec("underrightarrow", "{"),
+                MacroSpec("underleftrightarrow", "{"),
+                MacroSpec("overbrace", "{"),
+                MacroSpec("underbrace", "{"),
+                MacroSpec("overgroup", "{"),
+                MacroSpec("undergroup", "{"),
+                MacroSpec("overbracket", "{"),
+                MacroSpec("underbracket", "{"),
+                MacroSpec("overlinesegment", "{"),
+                MacroSpec("underlinesegment", "{"),
+                MacroSpec("overleftharpoon", "{"),
+                MacroSpec("overrightharpoon", "{"),
+                MacroSpec("xleftarrow", "[{"),
+                MacroSpec("xrightarrow", "[{"),
+                std_macro("ket", False, 1),
+                std_macro("bra", False, 1),
+                std_macro("braket", False, 2),
+                std_macro("ketbra", False, 2),
+                std_macro("texorpdfstring", False, 2),
+                # xcolor commands
+                MacroSpec("definecolor", "[{{{"),
+                MacroSpec("providecolor", "[{{{"),
+                MacroSpec("colorlet", "[{[{"),
+                MacroSpec("color", "[{"),
+                MacroSpec("textcolor", "[{{"),
+                MacroSpec("pagecolor", "[{"),
+                MacroSpec("nopagecolor", ""),
+                MacroSpec("colorbox", "[{{"),
+                MacroSpec("fcolorbox", "[{[{{"),
+                MacroSpec("boxframe", "{{{"),
+                MacroSpec("rowcolors", "*[{{{"),
+            ],
+            "environments": [
+                # NOTE: Starred variants (as in \begin{equation*}) are not specified as
+                # for macros with an argspec='*'.  Rather, we need to define a separate
+                # spec for the starred variant as the star really is part of the
+                # environment name.  If you specify argspec='*', the parser will try to
+                # look for an expression of the form '\begin{equation}*'
+                std_environment("figure", "["),
+                std_environment("figure*", "["),
+                std_environment("table", "["),
+                std_environment("table*", "["),
+                std_environment("abstract", None),
+                std_environment("tabular", "{"),
+                std_environment("tabular*", "{{"),
+                std_environment("tabularx", "{[{"),
+                std_environment("array", "[{"),
+                std_environment("equation", None, is_math_mode=True),
+                std_environment("equation*", None, is_math_mode=True),
+                std_environment("eqnarray", None, is_math_mode=True),
+                std_environment("eqnarray*", None, is_math_mode=True),
+                # AMS environments
+                std_environment("align", None, is_math_mode=True),
+                std_environment("align*", None, is_math_mode=True),
+                std_environment("gather", None, is_math_mode=True),
+                std_environment("gather*", None, is_math_mode=True),
+                std_environment("flalign", None, is_math_mode=True),
+                std_environment("flalign*", None, is_math_mode=True),
+                std_environment("multline", None, is_math_mode=True),
+                std_environment("multline*", None, is_math_mode=True),
+                std_environment("alignat", "{", is_math_mode=True),
+                std_environment("alignat*", "{", is_math_mode=True),
+                std_environment("split", None, is_math_mode=True),
+            ],
+            "specials": [
+                std_specials("&"),
+                # TODO --- for this, we need to parse their argument but don't use
+                #          the standard args parser because we need to be able to
+                #          accept arguments like "x_\mathrm{initial}"
+                #
+                # std_specials('^'),
+                # std_specials('_'),
+            ],
+        },
+    ),
+    #
+    # CATEGORY: nonascii-specials
+    #
+    (
+        "nonascii-specials",
+        {
+            "macros": [],
+            "environments": [],
+            "specials": [
+                std_specials("~"),
+                # cf. https://tex.stackexchange.com/a/439652/32188 "fake ligatures":
+                std_specials("``"),
+                std_specials("''"),
+                std_specials("--"),
+                std_specials("---"),
+                std_specials("!`"),
+                std_specials("?`"),
+            ],
+        },
+    ),
+    #
+    # CATEGORY: verbatim
+    #
+    (
+        "verbatim",
+        {
+            "macros": [
+                MacroSpec(
+                    "verb",
+                    args_parser=VerbatimArgsParser(verbatim_arg_type="verb-macro"),
+                ),
+            ],
+            "environments": [
+                EnvironmentSpec(
+                    "verbatim",
+                    args_parser=VerbatimArgsParser(
+                        verbatim_arg_type="verbatim-environment"
+                    ),
+                ),
+            ],
+            "specials": [
+                # optionally users could include the specials "|" like in latex-doc
+                # for verbatim |\like \this|...
+            ],
+        },
+    ),
+    #
+    # CATEGORY: theorems
+    #
+    (
+        "theorems",
+        {
+            "macros": [],
+            "environments": [
+                std_environment("theorem", "["),
+                std_environment("proposition", "["),
+                std_environment("lemma", "["),
+                std_environment("corollary", "["),
+                std_environment("definition", "["),
+                std_environment("conjecture", "["),
+                std_environment("remark", "["),
+                #
+                std_environment("proof", "["),
+                # short names
+                std_environment("thm", "["),
+                std_environment("prop", "["),
+                std_environment("lem", "["),
+                std_environment("cor", "["),
+                std_environment("conj", "["),
+                std_environment("rem", "["),
+                std_environment("defn", "["),
+            ],
+            "specials": [],
+        },
+    ),
+    #
+    # CATEGORY: enumitem
+    #
+    (
+        "enumitem",
+        {
+            "macros": [],
+            "environments": [
+                std_environment("enumerate", "["),
+                std_environment("itemize", "["),
+                std_environment("description", "["),
+            ],
+            "specials": [],
+        },
+    ),
+    #
+    # CATEGORY: natbib
+    #
+    (
+        "natbib",
+        {
+            "macros": [
+                std_macro("cite", "*[[{"),
+                std_macro("citet", "*[[{"),
+                std_macro("citep", "*[[{"),
+                std_macro("citealt", "*[[{"),
+                std_macro("citealp", "*[[{"),
+                std_macro("citeauthor", "*[[{"),
+                std_macro("citefullauthor", "[[{"),
+                std_macro("citeyear", "[[{"),
+                std_macro("citeyearpar", "[[{"),
+                std_macro("Citet", "*[[{"),
+                std_macro("Citep", "*[[{"),
+                std_macro("Citealt", "*[[{"),
+                std_macro("Citealp", "*[[{"),
+                std_macro("Citeauthor", "*[[{"),
+                std_macro("citetext", "{"),
+                std_macro("citenum", "{"),
+                std_macro("defcitealias", "{{"),
+                std_macro("citetalias", "[[{"),
+                std_macro("citepalias", "[[{"),
+            ],
+            "environments": [],
+            "specials": [],
+        },
+    ),
+    #
+    # CATEGORY: latex-ethuebung
+    #
+    (
+        "latex-ethuebung",
+        {
+            "macros": [
+                # ethuebung
+                std_macro("UebungLoesungFont", False, 1),
+                std_macro("UebungHinweisFont", False, 1),
+                std_macro("UebungExTitleFont", False, 1),
+                std_macro("UebungSubExTitleFont", False, 1),
+                std_macro("UebungTipsFont", False, 1),
+                std_macro("UebungLabel", False, 1),
+                std_macro("UebungSubLabel", False, 1),
+                std_macro("UebungLabelEnum", False, 1),
+                std_macro("UebungLabelEnumSub", False, 1),
+                std_macro("UebungSolLabel", False, 1),
+                std_macro("UebungHinweisLabel", False, 1),
+                std_macro("UebungHinweiseLabel", False, 1),
+                std_macro("UebungSolEquationLabel", False, 1),
+                std_macro("UebungTipsLabel", False, 1),
+                std_macro("UebungTipsEquationLabel", False, 1),
+                std_macro("UebungsblattTitleSeries", False, 1),
+                std_macro("UebungsblattTitleSolutions", False, 1),
+                std_macro("UebungsblattTitleTips", False, 1),
+                std_macro("UebungsblattNumber", False, 1),
+                std_macro("UebungsblattTitleFont", False, 1),
+                std_macro("UebungTitleCenterVSpacing", False, 1),
+                std_macro("UebungAttachedSolutionTitleTop", False, 1),
+                std_macro("UebungAttachedSolutionTitleFont", False, 1),
+                std_macro("UebungAttachedSolutionTitle", False, 1),
+                std_macro("UebungTextAttachedSolution", False, 1),
+                std_macro("UebungDueByLabel", False, 1),
+                std_macro("UebungDueBy", False, 1),
+                std_macro("UebungLecture", False, 1),
+                std_macro("UebungProf", False, 1),
+                std_macro("UebungLecturer", False, 1),
+                std_macro("UebungSemester", False, 1),
+                std_macro("UebungLogoFile", False, 1),
+                std_macro("UebungLanguage", False, 1),
+                std_macro("UebungStyle", False, 1),
+                #
+                std_macro("uebung", "{["),
+                std_macro("exercise", "{["),
+                std_macro("keywords", False, 1),
+                std_macro("subuebung", False, 1),
+                std_macro("subexercise", False, 1),
+                std_macro("pdfloesung", True, 1),
+                std_macro("pdfsolution", True, 1),
+                std_macro("exenumfulllabel", False, 1),
+                std_macro("hint", False, 1),
+                std_macro("hints", False, 1),
+                std_macro("hinweis", False, 1),
+                std_macro("hinweise", False, 1),
+            ],
+            "environments": [],
+            "specials": [],
+        },
+    ),
+]