Add sus updates

great_ai/README.md

@@ -9,6 +9,7 @@
 - [parallel_map](src/sus/parallel_map.py)
 - [match_names](src/sus/match_names/match_names.py)
 - [evaluate_ranking](src/sus/evaluate_ranking/evaluate_ranking.py)
+- [get_sentences](src/sus/get_sentences.py)
 
 ### Requires loading spacy model
 
@@ -17,14 +18,14 @@
 > Add this to the Dockerfile for caching the spaCy model:
 >
 > ```docker
-> RUN pip install --no-cache-dir en-core-web-lg@https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.2.0/en_core_web_lg-3.2.0-py3-none-any.whl
+> RUN pip install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl
 > ```
 
-- [spacy model (nlp)](src/sus/nlp.py)
+- [publication TEI](src/sus/publication_tei/publication_tei.py)
-- [get_sentences](src/sus/get_sentences.py)
 - [lemmatize_text](src/sus/lemmatize_text.py)
 - [lemmatize_token](src/sus/lemmatize_token.py)
-- [publication TEI](src/sus/publication_tei/publication_tei.py)
+- [spacy model (nlp)](src/sus/nlp.py)
+- [filter_sentences](src/sus/matcher/filter_sentences.py)
 
 ## Development
 

great_ai/setup.cfg

@@ -1,14 +1,14 @@
 [metadata]
-name = sus
+name = great-ai
-version = 0.0.3
+version = 0.0.1
-author = ScoutinScience B.V.
+author = András Schmelczer
 author_email = andras@scoutinscience.com
-description = [S]coutinScience [u]tilitie[s]: reusable utilities for text processing
+description = 
 long_description = file: README.md
 long_description_content_type = text/markdown
-url = https://github.com/ScoutinScience/platform
+url = https://github.com/ScoutinScience/great-ai
 project_urls =
-    Bug Tracker = https://github.com/ScoutinScience/platform/issues
+    Bug Tracker = https://github.com/ScoutinScience/great-ai/issues
 classifiers =
     Programming Language :: Python :: 3
     Operating System :: OS Independent
@@ -27,13 +27,17 @@ install_requires =
     psutil >= 5.9.0
     beautifulsoup4 >= 4.10.0
     lxml >= 4.6.0
-    spacy >= 3.2.0
+    spacy >= 3.3.0
     pydantic >= 1.8.0
     scikit-learn >= 1.0.0
     matplotlib >= 3.5.0
     numpy >= 1.22.0
     langcodes[data] >= 3.3.0
+    segtok >= 1.5.11
     langdetect >= 1.0.9
 
+[options.package_data]
+* = *.json, *.yaml, *.yml
+
 [options.packages.find]
 where = src

great_ai/src/great_ai/utilities/clean.py

@@ -9,7 +9,6 @@ from .external.pylatexenc.latex2text import LatexNodes2Text
 from .logger.create_logger import create_logger
 
 logger = create_logger("clean")
-
 latex = LatexNodes2Text()
 
 
@@ -46,6 +45,10 @@ def clean(
             text = "".join([c for c in text if not unicodedata.combining(c)])
             text = unidecode.unidecode(text)
 
+    text = re.sub(
+        r"\b[a-zA-Z](?:[\t ]+[a-zA-Z]\b)+", lambda m: re.sub(r"[\t ]", "", m[0]), text
+    )  # A R T I C L E => ARTICLE
+
     if remove_brackets:
         text = re.sub(r"\[[^\]]*\]", " ", text)
 

great_ai/src/great_ai/utilities/external/negspacy/termsets.py

@@ -40,7 +40,7 @@ preceding = [
     "doubt",
     "negative for",
     "no",
-    "vergreat_ai.utilities",
+    "versus",
     "without",
     "doesn't",
     "doesnt",
@@ -104,7 +104,7 @@ pseudo_clinical = pseudo + [
     "not ruled out",
     "not been ruled out",
     "not drain",
-    "no great_ai.utilitiespicious change",
+    "no suspicious change",
     "no interval change",
     "no significant interval change",
 ]

great_ai/src/great_ai/utilities/external/pylatexenc/latex2text/__init__.py

@@ -287,7 +287,7 @@ def placeholder_node_formatter(placeholdertext, block=True):
     r"""
     This function returns a callable that can be used in
     :py:class:`MacroTextSpec`, :py:class:`EnvironmentTextSpec`, or
-    :py:class:`SpecialsTextSpec` for latex nodes that do not have a great textual
+    :py:class:`SpecialsTextSpec` for latex nodes that do not have a good textual
     representation, providing as text replacement the simple placeholder text
     ``'< P L A C E H O L D E R   T E X T >'``.
 
@@ -316,7 +316,7 @@ def fmt_placeholder_node(node, l2tobj):
     r"""
     This function can be used as callable in :py:class:`MacroTextSpec`,
     :py:class:`EnvironmentTextSpec`, or :py:class:`SpecialsTextSpec` for latex
-    nodes that do not have a great textual representation.  The text replacement
+    nodes that do not have a good textual representation.  The text replacement
     is the placeholder text
     ``'< N A M E   O F   T H E   M A C R O   O R   E N V I R O N M E N T >'``.
 

great_ai/src/great_ai/utilities/external/pylatexenc/latex2text/_defaultspecs.py

@@ -266,7 +266,7 @@ _latex_specs_approximations = {
             ("fcolorbox", "%(5)s"),
             ("hspace", ""),
             ("vspace", "\n"),
-            # \\ is treated as an "approximation" because a great text renderer would
+            # \\ is treated as an "approximation" because a good text renderer would
             # have to actually note that this is a end-of-line marker which is not
             # to be confused with other newlines in the paragraph (which can be
             # reflowed)

great_ai/src/great_ai/utilities/external/pylatexenc/latexencode/_unicode_to_latex_encoder.py

@@ -342,7 +342,7 @@ class UnicodeToLatexEncoder(object):
 
        Possible protection schemes are:
 
-         - 'braces' (the default):  Any great_ai.utilitiespicious replacement text (that
+         - 'braces' (the default):  Any suspicious replacement text (that
            might look fragile) is placed in curly braces ``{...}``.
 
          - 'braces-all':  All replacement latex escapes are surrounded in

great_ai/src/great_ai/utilities/external/pylatexenc/latexwalker/__main__.py

@@ -0,0 +1,230 @@
+# -*- coding: utf-8 -*-
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2018 Philippe Faist
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+import argparse
+import fileinput
+import json
+import logging
+import sys
+
+from ..latexwalker import LatexWalker, disp_node, make_json_encoder
+from ..version import version_str
+
+
+def main(argv=None):
+
+    if argv is None:
+        argv = sys.argv[1:]
+
+    parser = argparse.ArgumentParser(prog="latexwalker", add_help=False)
+
+    parser.add_argument(
+        "--output-format",
+        metavar="FORMAT",
+        dest="output_format",
+        choices=["human", "json"],
+        default="human",
+        help='Requested output format for the node tree ("human" or "json")',
+    )
+    parser.add_argument(
+        "--json-indent",
+        metavar="NUMSPACES",
+        dest="json_indent",
+        type=int,
+        default=2,
+        help="Indentation in JSON output (specify number of spaces "
+        "per indentation level)",
+    )
+    parser.add_argument(
+        "--json-compact",
+        dest="json_indent",
+        action="store_const",
+        const=None,
+        help="Output compact JSON",
+    )
+
+    parser.add_argument(
+        "--keep-inline-math",
+        action="store_const",
+        const=True,
+        dest="keep_inline_math",
+        default=True,
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--no-keep-inline-math",
+        action="store_const",
+        const=False,
+        dest="keep_inline_math",
+        help=argparse.SUPPRESS,
+    )
+
+    parser.add_argument(
+        "--tolerant-parsing",
+        action="store_const",
+        const=True,
+        dest="tolerant_parsing",
+        default=True,
+    )
+    parser.add_argument(
+        "--no-tolerant-parsing",
+        action="store_const",
+        const=False,
+        dest="tolerant_parsing",
+        help="Tolerate syntax errors when parsing, and attempt "
+        "to continue (default yes)",
+    )
+
+    # I'm not sure this flag is useful and if it should be exposed at all.
+    # Accept it, but make it hidden.
+    parser.add_argument(
+        "--strict-braces",
+        action="store_const",
+        const=True,
+        dest="strict_braces",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--no-strict-braces",
+        action="store_const",
+        const=False,
+        dest="strict_braces",
+        # help="Report errors for mismatching LaTeX braces (default no)"
+        help=argparse.SUPPRESS,
+    )
+
+    parser.add_argument(
+        "-q",
+        "--quiet",
+        dest="logging_level",
+        action="store_const",
+        const=logging.ERROR,
+        default=logging.INFO,
+        help="Suppress warning messages",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        dest="logging_level",
+        action="store_const",
+        const=logging.DEBUG,
+        help="Verbose output",
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="pylatexenc {}".format(version_str),
+        help="Show version information and exit",
+    )
+    parser.add_argument(
+        "--help", action="help", help="Show this help information and exit"
+    )
+
+    parser.add_argument(
+        "--code",
+        "-c",
+        action="store",
+        default=None,
+        metavar="LATEX_CODE",
+        help="Convert the given LATEX_CODE to unicode text instead of reading "
+        "from FILE or standard input.  You cannot specify FILEs if you use this "
+        "option, and any standard input is ignored.",
+    )
+
+    parser.add_argument(
+        "files",
+        metavar="FILE",
+        nargs="*",
+        help="Input files (if none specified, read from stdandard input)",
+    )
+
+    args = parser.parse_args(argv)
+
+    logging.basicConfig()
+    logging.getLogger().setLevel(args.logging_level)
+    logger = logging.getLogger(__name__)
+
+    latex = ""
+    if args.code:
+        if args.files:
+            logger.error(
+                "Cannot specify both FILEs and --code option. "
+                "Use --help option for more information."
+            )
+            sys.exit(1)
+        latex = args.code
+    else:
+        for line in fileinput.input(files=args.files):
+            latex += line
+
+    latexwalker = LatexWalker(
+        latex, tolerant_parsing=args.tolerant_parsing, strict_braces=args.strict_braces
+    )
+
+    (nodelist, pos, len_) = latexwalker.get_latex_nodes()
+
+    if args.output_format == "human":
+        print("\n--- NODES ---\n")
+        for n in nodelist:
+            disp_node(n)
+        print("\n-------------\n")
+        return
+
+    if args.output_format == "json":
+        json.dump(
+            {
+                "nodelist": nodelist,
+            },
+            sys.stdout,
+            cls=make_json_encoder(latexwalker),
+            indent=args.json_indent,
+        )
+        sys.stdout.write("\n")
+        return
+
+    raise ValueError("Invalid output format: " + args.output_format)
+
+
+def run_main():
+
+    try:
+
+        main()
+
+    except SystemExit:
+        raise
+    except:  # lgtm [py/catch-base-exception]
+        import pdb
+        import traceback
+
+        traceback.print_exc()
+        pdb.post_mortem()
+
+
+if __name__ == "__main__":
+
+    # run_main() # debug
+    main()

great_ai/src/great_ai/utilities/external/pylatexenc/version.py

@@ -33,7 +33,7 @@
 #
 #       - tests pass: https://travis-ci.org/github/phfaist/pylatexenc
 #
-#       - LGTM looks great: https://lgtm.com/projects/g/phfaist/pylatexenc/
+#       - LGTM looks good: https://lgtm.com/projects/g/phfaist/pylatexenc/
 #
 #       - python package creation works: (python setup.py sdist, pip install
 #         dist/pylatexenc-xxx.tar.gz)

great_ai/src/great_ai/utilities/get_sentences.py

@@ -1,12 +1,15 @@
 from typing import List
 
+from segtok.segmenter import split_multi
+
 from .data import sentence_ending_punctuations
-from .nlp import nlp
 
 
 def get_sentences(text: str, ignore_partial: bool = False) -> List[str]:
-    doc = nlp(text)
+    if text.strip() == "":
-    possible_sentences = [s.text for s in doc.sents]
+        return []
+
+    possible_sentences = [s for s in split_multi(text) if s]
 
     if ignore_partial:
         possible_sentences = [

great_ai/src/great_ai/utilities/matcher/__init__.py

@@ -0,0 +1,3 @@
+from .fast_tokenize import fast_tokenize
+from .filter_sentences import filter_sentences
+from .normalize import normalize

great_ai/src/great_ai/utilities/matcher/fast_tokenize.py

@@ -0,0 +1,30 @@
+import re
+from typing import List, Union
+
+from segtok.tokenizer import word_tokenizer
+
+from ..get_sentences import get_sentences
+from .normalize import normalize
+
+
+def fast_tokenize(
+    text: Union[List[str], str], ignore_partial: bool = False
+) -> List[List[str]]:
+    if isinstance(text, str):
+        text = normalize(text)
+        text = get_sentences(text, ignore_partial=ignore_partial)
+
+    results: List[List[str]] = []
+
+    for sentence in text:
+        sentence = re.sub(r"\bare\b", "is", sentence)
+        sentence = re.sub(r"\ban\b", "a", sentence)
+        sentence = re.sub(r"\bthese\b", "this", sentence)
+        results.append(
+            [
+                token.lower() if token not in {"CITATION", "NUMBER"} else token
+                for token in word_tokenizer(sentence)
+            ]
+        )
+
+    return results

great_ai/src/great_ai/utilities/matcher/filter_sentences.py

@@ -0,0 +1,66 @@
+from pathlib import Path
+from typing import Dict, List, Union
+
+import yaml
+from spacy.matcher import Matcher
+
+from ..get_sentences import get_sentences
+from ..nlp import nlp
+from .fast_tokenize import fast_tokenize
+from .normalize import normalize
+
+rules_cache: Dict[str, Matcher] = {}
+
+
+def filter_sentences(
+    sentences: str,
+    rules_file: Path,
+    inverse: bool = False,
+    ignore_partial: bool = False,
+) -> List[str]:
+    if str(rules_file) not in rules_cache:
+        with open(rules_file, encoding="utf-8") as f:
+            rule_patterns = yaml.safe_load(f).keys()
+
+        matcher = Matcher(nlp.vocab)
+        rules = [_pattern_to_rule(p) for p in rule_patterns]
+        matcher.add("", rules)
+        rules_cache[str(rules_file)] = matcher
+
+    matcher = rules_cache[str(rules_file)]
+
+    original_sentences = get_sentences(sentences, ignore_partial=ignore_partial)
+
+    tokenized = fast_tokenize(original_sentences, ignore_partial=ignore_partial)
+
+    results: List[str] = []
+    for original_sentence, sentence in zip(original_sentences, tokenized):
+        doc = nlp(normalize(" ".join(sentence)))
+        matches = matcher(doc)
+        if matches:
+            # _, start, end = max(
+            #     matches,
+            #     key=lambda v: v[2] - v[1],
+            # )
+            # print(str(doc[start:end]))
+
+            if not inverse:
+                results.append(original_sentence)
+        elif inverse:
+            results.append(original_sentence)
+
+    return results
+
+
+def _pattern_to_rule(pattern: str) -> List[Dict[str, Union[bool, str]]]:
+    result: List[Dict[str, Union[bool, str]]] = []
+    for t in pattern.split():
+        if t == "*":
+            result.extend([{"OP": "?"}, {"OP": "?"}])
+        elif t == "CITATION":
+            result.append({"ORTH": "CITATION"})
+        elif t == "NUMBER":
+            result.append({"ORTH": "NUMBER"})
+        else:
+            result.append({"LOWER": t})
+    return result

great_ai/src/great_ai/utilities/matcher/normalize.py

@@ -0,0 +1,22 @@
+import re
+
+from ..clean import clean
+
+
+def normalize(text: str) -> str:
+    text = re.sub(
+        r"""
+            ([A-Z]\w+\W+(et\ al.))     # inline reference: Bank et al.
+            | (\[[0-9-, ]+\])          # IEEE style: [1], [2-4], [3, 5]
+            | (\(.*?,?\W+?\d+\))       # APA style: (Bank, 2020)
+            | ([A-Z]\w+ \(?\d+\))      # APA style: Bank (2020)
+        """,
+        " CITATION ",
+        text,
+        flags=re.VERBOSE,
+    )
+    text = re.sub(r"\d[\d,. -]*(st|nd|rd|th)?", " NUMBER ", text)
+    text = clean(text, convert_to_ascii=True)
+    text = re.sub(r"[^a-zA-Z.?!,:;'\" -]", "", text)
+
+    return text

great_ai/src/great_ai/utilities/nlp.py

@@ -1,21 +1,21 @@
 try:
-    import en_core_web_lg
+    import en_core_web_sm
 except ImportError:
     import subprocess
 
-    print("Spacy model en_core_web_lg not found locally, downloading...")
+    print("Spacy model en_core_web_sm not found locally, downloading...")
 
     subprocess.call(
         [
             "pip",
             "install",
-            "https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.2.0/en_core_web_lg-3.2.0-py3-none-any.whl",
+            "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl",
         ]
     )
-    import en_core_web_lg
+    import en_core_web_sm
 
 from .external.negspacy import negation  # noqa: F401 it's important to import this
 
-nlp = en_core_web_lg.load()
+nlp = en_core_web_sm.load()
 
 nlp.add_pipe("negex")

great_ai/src/great_ai/utilities/parallel_map.py

@@ -9,16 +9,13 @@ from .logger import create_logger
 
 logger = create_logger("parallel_map")
 
-
 def parallel_map(
     function: Callable[[Any], Any],
     values: Iterable[Any],
     chunk_size: Optional[int] = None,
-    concurrency: Optional[int] = None,
+    concurrency: int = psutil.cpu_count(),
     disable_progress: bool = False,
 ) -> List[Any]:
-    if concurrency is None:
-        concurrency = psutil.cpu_count()
     assert concurrency > 0
     assert chunk_size is None or chunk_size > 0
 

great_ai/src/great_ai/utilities/publication_tei/models/__init__.py

@@ -1,5 +1,7 @@
 from .affiliation import Affiliation
 from .author import Author
+from .bookmark import Bookmark
+from .bookmark_title import BookmarkTitle
 from .element import Element, Meta, MetaType, Paragraph, Title
 from .publication_metadata import PublicationMetadata
 from .text import Text

great_ai/src/great_ai/utilities/publication_tei/models/bookmark.py

@@ -0,0 +1,10 @@
+from pydantic import BaseModel
+
+from .bookmark_title import BookmarkTitle
+
+
+class Bookmark(BaseModel):
+    title: BookmarkTitle
+    original_title: str
+    document_order: int
+    coordinates: str

great_ai/src/great_ai/utilities/publication_tei/models/bookmark_title.py

@@ -0,0 +1,16 @@
+from typing import Literal
+
+BookmarkTitle = Literal[
+    "Abstract",
+    "Author contribution",
+    "Introduction",
+    "Background",
+    "Methods",
+    "Results",
+    "Discussion",
+    "Outlook",
+    "Conclusion",
+    "Conflict of interest",
+    "Acknowledgement",
+    "Annex",
+]

great_ai/src/great_ai/utilities/publication_tei/models/text.py

@@ -1,7 +1,9 @@
+from typing import Optional
+
 from pydantic import BaseModel
 
 
 class Text(BaseModel):
     content: str
     document_order: int
-    coordinates: str
+    coordinates: Optional[str]