Move files

src/great_ai/utilities/matcher/__init__.py

@@ -0,0 +1,3 @@
+from .fast_tokenize import fast_tokenize
+from .filter_sentences import filter_sentences
+from .normalize import normalize

src/great_ai/utilities/matcher/fast_tokenize.py

@@ -0,0 +1,30 @@
+import re
+from typing import List, Union
+
+from segtok.tokenizer import word_tokenizer
+
+from ..get_sentences import get_sentences
+from .normalize import normalize
+
+
+def fast_tokenize(
+    text: Union[List[str], str], ignore_partial: bool = False
+) -> List[List[str]]:
+    if isinstance(text, str):
+        text = normalize(text)
+        text = get_sentences(text, ignore_partial=ignore_partial)
+
+    results: List[List[str]] = []
+
+    for sentence in text:
+        sentence = re.sub(r"\bare\b", "is", sentence)
+        sentence = re.sub(r"\ban\b", "a", sentence)
+        sentence = re.sub(r"\bthese\b", "this", sentence)
+        results.append(
+            [
+                token.lower() if token not in {"CITATION", "NUMBER"} else token
+                for token in word_tokenizer(sentence)
+            ]
+        )
+
+    return results

src/great_ai/utilities/matcher/filter_sentences.py

@@ -0,0 +1,66 @@
+from pathlib import Path
+from typing import Dict, List, Union
+
+import yaml
+from spacy.matcher import Matcher
+
+from ..get_sentences import get_sentences
+from ..nlp import nlp
+from .fast_tokenize import fast_tokenize
+from .normalize import normalize
+
+rules_cache: Dict[str, Matcher] = {}
+
+
+def filter_sentences(
+    sentences: str,
+    rules_file: Path,
+    inverse: bool = False,
+    ignore_partial: bool = False,
+) -> List[str]:
+    if str(rules_file) not in rules_cache:
+        with open(rules_file, encoding="utf-8") as f:
+            rule_patterns = yaml.safe_load(f).keys()
+
+        matcher = Matcher(nlp.vocab)
+        rules = [_pattern_to_rule(p) for p in rule_patterns]
+        matcher.add("", rules)
+        rules_cache[str(rules_file)] = matcher
+
+    matcher = rules_cache[str(rules_file)]
+
+    original_sentences = get_sentences(sentences, ignore_partial=ignore_partial)
+
+    tokenized = fast_tokenize(original_sentences, ignore_partial=ignore_partial)
+
+    results: List[str] = []
+    for original_sentence, sentence in zip(original_sentences, tokenized):
+        doc = nlp(normalize(" ".join(sentence)))
+        matches = matcher(doc)
+        if matches:
+            # _, start, end = max(
+            #     matches,
+            #     key=lambda v: v[2] - v[1],
+            # )
+            # print(str(doc[start:end]))
+
+            if not inverse:
+                results.append(original_sentence)
+        elif inverse:
+            results.append(original_sentence)
+
+    return results
+
+
+def _pattern_to_rule(pattern: str) -> List[Dict[str, Union[bool, str]]]:
+    result: List[Dict[str, Union[bool, str]]] = []
+    for t in pattern.split():
+        if t == "*":
+            result.extend([{"OP": "?"}, {"OP": "?"}])
+        elif t == "CITATION":
+            result.append({"ORTH": "CITATION"})
+        elif t == "NUMBER":
+            result.append({"ORTH": "NUMBER"})
+        else:
+            result.append({"LOWER": t})
+    return result

src/great_ai/utilities/matcher/normalize.py

@@ -0,0 +1,22 @@
+import re
+
+from ..clean import clean
+
+
+def normalize(text: str) -> str:
+    text = re.sub(
+        r"""
+            ([A-Z]\w+\W+(et\ al.))     # inline reference: Bank et al.
+            | (\[[0-9-, ]+\])          # IEEE style: [1], [2-4], [3, 5]
+            | (\(.*?,?\W+?\d+\))       # APA style: (Bank, 2020)
+            | ([A-Z]\w+ \(?\d+\))      # APA style: Bank (2020)
+        """,
+        " CITATION ",
+        text,
+        flags=re.VERBOSE,
+    )
+    text = re.sub(r"\d[\d,. -]*(st|nd|rd|th)?", " NUMBER ", text)
+    text = clean(text, convert_to_ascii=True)
+    text = re.sub(r"[^a-zA-Z.?!,:;'\" -]", "", text)
+
+    return text