andras/great-ai
andras/great-ai
Watch
1
0
Fork
You've already forked great-ai
 0
Code Issues Pull requests 2 Projects Releases 4 Packages 1 Wiki Activity Actions

Remove Spacy and Publication TEI

Browse source
This commit is contained in:
Andras Schmelczer 2022-06-26 10:46:40 +02:00
commit 46c99a06f9
30 changed files with 5 additions and 2412 deletions
Download patch file Download diff file Expand all files Collapse all files

3
Dockerfile
View file

@ -19,9 +19,6 @@ RUN python3 -m pip --no-cache-dir install --upgrade pip &&\
pip install --no-cache-dir ./great_ai &&\ pip install --no-cache-dir ./great_ai &&\
rm -rf great_ai rm -rf great_ai
# great_ai.utilities.nlp depends on this
RUN pip3 install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl
HEALTHCHECK \ HEALTHCHECK \
--interval=60s \ --interval=60s \
--timeout=60s \ --timeout=60s \

20
docs/README.md
View file

@ -1,32 +1,14 @@
# **S**coutinScience **U**tilitie**S** for text processing [![Lint and test ScoutinScience utilities](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml/badge.svg)](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml) # **S**coutinScience **U**tilitie**S** for text processing [![Lint and test ScoutinScience utilities](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml/badge.svg)](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml)
> amogus
## Exports ## Exports
- [clean](src/sus/clean.py) - [clean](src/sus/clean.py)
- [unique](src/sus/unique.py) - [unique](src/sus/unique.py)
- [parallel_map](src/sus/parallel_map.py) - [parallel_map](src/sus/parallel_map.py)
- [match_names](src/sus/match_names/match_names.py) - [lemmatize](src/sus/lemmatize.py)
- [evaluate_ranking](src/sus/evaluate_ranking/evaluate_ranking.py) - [evaluate_ranking](src/sus/evaluate_ranking/evaluate_ranking.py)
- [get_sentences](src/sus/get_sentences.py) - [get_sentences](src/sus/get_sentences.py)
### Requires loading spacy model
> This is automatic but will require some time.
> Add this to the Dockerfile for caching the spaCy model:
>
> ```docker
> RUN pip install --no-cache-dir en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl
> ```
- [publication TEI](src/sus/publication_tei/publication_tei.py)
- [lemmatize_text](src/sus/lemmatize_text.py)
- [lemmatize_token](src/sus/lemmatize_token.py)
- [spacy model (nlp)](src/sus/nlp.py)
- [filter_sentences](src/sus/matcher/filter_sentences.py)
## Development ## Development
- Optional booleans must have a default value of `False`. - Optional booleans must have a default value of `False`.

4
setup.cfg
View file

@ -23,9 +23,6 @@ install_requires =
unidecode >= 1.3.0 unidecode >= 1.3.0
multiprocess >= 0.70.0.0 multiprocess >= 0.70.0.0
tqdm >= 4.0.0 tqdm >= 4.0.0
beautifulsoup4 >= 4.10.0
lxml >= 4.6.0
spacy >= 3.3.0
scikit-learn == 1.1.1 scikit-learn == 1.1.1
matplotlib >= 3.5.0 matplotlib >= 3.5.0
numpy >= 1.22.0 numpy >= 1.22.0
@ -34,7 +31,6 @@ install_requires =
langdetect >= 1.0.9 langdetect >= 1.0.9
tinydb >= 4.7.0 tinydb >= 4.7.0
pandas >= 1.4.0 pandas >= 1.4.0
pyaml >= 21.0.0
boto3 >= 1.23.0 boto3 >= 1.23.0
fastapi >= 0.70.0 fastapi >= 0.70.0
plotly >= 5.8.0 plotly >= 5.8.0

12
src/great_ai/utilities/__init__.py
View file

@ -1,14 +1,10 @@
from .clean import clean from .clean import clean
from .parallel_map import parallel_map
from .unique import unique
from .config_file import ConfigFile, ParseError from .config_file import ConfigFile, ParseError
from .evaluate_ranking import evaluate_ranking
from .get_sentences import get_sentences from .get_sentences import get_sentences
from .language import english_name_of_language, is_english, predict_language from .language import english_name_of_language, is_english, predict_language
from .lemmatize_text import lemmatize_text from .lemmatize import lemmatize
from .lemmatize_token import lemmatize_token
from .logger import get_logger from .logger import get_logger
from .evaluate_ranking import evaluate_ranking
from .match_names import match_names from .match_names import match_names
from .matcher import fast_tokenize, filter_sentences, normalize
from .nlp import nlp
from .parallel_map import parallel_map
from .publication_tei import PublicationTEI
from .unique import unique

1
src/great_ai/utilities/external/negspacy/README.md vendored
View file

@ -1 +0,0 @@
https://github.com/jenojp/negspacy

0
src/great_ai/utilities/external/negspacy/__init__.py vendored
View file

222
src/great_ai/utilities/external/negspacy/negation.py vendored
View file

@ -1,222 +0,0 @@
import logging
from spacy.language import Language
from spacy.matcher import PhraseMatcher
from spacy.tokens import Token
from .termsets import termset
default_ts = termset("en").get_patterns()
@Language.factory(
"negex",
default_config={
"neg_termset": default_ts,
"extension_name": "negex",
"chunk_prefix": list(),
},
)
class Negex:
"""
A spaCy pipeline component which identifies negated tokens in text.
Based on: NegEx - A Simple Algorithm for Identifying Negated Findings and Diseasesin Discharge Summaries
Chapman, Bridewell, Hanbury, Cooper, Buchanan
Parameters
----------
nlp: object
spaCy language object
termset_lang: str
language code, if using default termsets (e.g. "en" for english)
extension_name: str
defaults to "negex"; whether entity is negated is then available as ent._.negex
pseudo_negations: list
list of phrases that cancel out a negation, if empty, defaults are used
preceding_negations: list
negations that appear before an entity, if empty, defaults are used
following_negations: list
negations that appear after an entity, if empty, defaults are used
termination: list
phrases that "terminate" a sentence for processing purposes such as "but". If empty, defaults are used
"""
def __init__(
self,
nlp: Language,
name: str,
neg_termset: dict,
extension_name: str,
chunk_prefix: list,
):
if not Token.has_extension(extension_name):
Token.set_extension(extension_name, default=False, force=True)
ts = neg_termset
expected_keys = [
"pseudo_negations",
"preceding_negations",
"following_negations",
"termination",
]
if not set(ts.keys()) == set(expected_keys):
raise KeyError(
f"Unexpected or missing keys in 'neg_termset', expected: {expected_keys}, instead got: {list(ts.keys())}"
)
self.pseudo_negations = ts["pseudo_negations"]
self.preceding_negations = ts["preceding_negations"]
self.following_negations = ts["following_negations"]
self.termination = ts["termination"]
self.nlp = nlp
self.extension_name = extension_name
self.build_patterns()
self.chunk_prefix = list(nlp.tokenizer.pipe(chunk_prefix))
def build_patterns(self):
# efficiently build spaCy matcher patterns
self.matcher = PhraseMatcher(self.nlp.vocab, attr="LOWER")
self.pseudo_patterns = list(self.nlp.tokenizer.pipe(self.pseudo_negations))
self.matcher.add("pseudo", None, *self.pseudo_patterns)
self.preceding_patterns = list(
self.nlp.tokenizer.pipe(self.preceding_negations)
)
self.matcher.add("Preceding", None, *self.preceding_patterns)
self.following_patterns = list(
self.nlp.tokenizer.pipe(self.following_negations)
)
self.matcher.add("Following", None, *self.following_patterns)
self.termination_patterns = list(self.nlp.tokenizer.pipe(self.termination))
self.matcher.add("Termination", None, *self.termination_patterns)
def process_negations(self, doc):
"""
Find negations in doc and clean candidate negations to remove pseudo negations
Parameters
----------
doc: object
spaCy Doc object
Returns
-------
preceding: list
list of tuples for preceding negations
following: list
list of tuples for following negations
terminating: list
list of tuples of terminating phrases
"""
###
# does not work properly in spacy 2.1.8. Will incorporate after 2.2.
# Relying on user to use NER in meantime
# see https://github.com/jenojp/negspacy/issues/7
###
# if not doc.is_nered:
# raise ValueError(
# "Negations are evaluated for Named Entities found in text. "
# "Your SpaCy pipeline does not included Named Entity resolution. "
# "Please ensure it is enabled or choose a different language model that includes it."
# )
preceding = list()
following = list()
terminating = list()
matches = self.matcher(doc)
pseudo = [
(match_id, start, end)
for match_id, start, end in matches
if self.nlp.vocab.strings[match_id] == "pseudo"
]
for match_id, start, end in matches:
if self.nlp.vocab.strings[match_id] == "pseudo":
continue
pseudo_flag = False
for p in pseudo:
if start >= p[1] and start <= p[2]:
pseudo_flag = True
continue
if not pseudo_flag:
if self.nlp.vocab.strings[match_id] == "Preceding":
preceding.append((match_id, start, end))
elif self.nlp.vocab.strings[match_id] == "Following":
following.append((match_id, start, end))
elif self.nlp.vocab.strings[match_id] == "Termination":
terminating.append((match_id, start, end))
else:
logging.warnings(
f"phrase {doc[start:end].text} not in one of the expected matcher types."
)
return preceding, following, terminating
def termination_boundaries(self, doc, terminating):
"""
Create sub sentences based on terminations found in text.
Parameters
----------
doc: object
spaCy Doc object
terminating: list
list of tuples with (match_id, start, end)
returns
-------
boundaries: list
list of tuples with (start, end) of spans
"""
sent_starts = [sent.start for sent in doc.sents]
terminating_starts = [t[1] for t in terminating]
starts = sent_starts + terminating_starts + [len(doc)]
starts.sort()
boundaries = list()
index = 0
for i, start in enumerate(starts):
if not i == 0:
boundaries.append((index, start))
index = start
return boundaries
def negex(self, doc):
"""
Negates entities of interest
Parameters
----------
doc: object
spaCy Doc object
"""
preceding, following, terminating = self.process_negations(doc)
boundaries = self.termination_boundaries(doc, terminating)
for b in boundaries:
sub_preceding = [i for i in preceding if b[0] <= i[1] < b[1]]
sub_following = [i for i in following if b[0] <= i[1] < b[1]]
for e in doc[b[0] : b[1]]:
if any(pre < e.i for pre in [i[1] for i in sub_preceding]):
e._.set(self.extension_name, True)
continue
if any(fol > e.i for fol in [i[2] for i in sub_following]):
e._.set(self.extension_name, True)
continue
if self.chunk_prefix:
if any(
e.text.lower().startswith(c.text.lower())
for c in self.chunk_prefix
):
e._.set(self.extension_name, True)
return doc
def __call__(self, doc):
return self.negex(doc)

229
src/great_ai/utilities/external/negspacy/termsets.py vendored
View file

@ -1,229 +0,0 @@
"""
Default termsets for various languages
"""
LANGUAGES = dict()
# english termset dictionary
en = dict()
pseudo = [
"no further",
"not able to be",
"not certain if",
"not certain whether",
"not necessarily",
"without any further",
"without difficulty",
"without further",
"might not",
"not only",
"no increase",
"no significant change",
"no change",
"no definite change",
"not extend",
"not cause",
]
en["pseudo_negations"] = pseudo
preceding = [
"absence of",
"declined",
"denied",
"denies",
"denying",
"no sign of",
"no signs of",
"not",
"not demonstrate",
"symptoms atypical",
"doubt",
"negative for",
"no",
"versus",
"without",
"doesn't",
"doesnt",
"don't",
"dont",
"didn't",
"didnt",
"wasn't",
"wasnt",
"weren't",
"werent",
"isn't",
"isnt",
"aren't",
"arent",
"cannot",
"can't",
"cant",
"couldn't",
"couldnt",
"never",
]
en["preceding_negations"] = preceding
following = [
"declined",
"unlikely",
"was not",
"were not",
"wasn't",
"wasnt",
"weren't",
"werent",
]
en["following_negations"] = following
termination = [
"although",
"apart from",
"as there are",
"aside from",
"but",
"except",
"however",
"involving",
"nevertheless",
"still",
"though",
"which",
"yet",
]
en["termination"] = termination
LANGUAGES["en"] = en
# en_clinical builds upon en
en_clinical = dict()
pseudo_clinical = pseudo + [
"gram negative",
"not rule out",
"not ruled out",
"not been ruled out",
"not drain",
"no suspicious change",
"no interval change",
"no significant interval change",
]
en_clinical["pseudo_negations"] = pseudo_clinical
preceding_clinical = preceding + [
"patient was not",
"without indication of",
"without sign of",
"without signs of",
"without any reactions or signs of",
"no complaints of",
"no evidence of",
"no cause of",
"evaluate for",
"fails to reveal",
"free of",
"never developed",
"never had",
"did not exhibit",
"rules out",
"rule out",
"rule him out",
"rule her out",
"rule patient out",
"rule the patient out",
"ruled out",
"ruled him out",
"ruled her out",
"ruled patient out",
"ruled the patient out",
"r/o",
"ro",
]
en_clinical["preceding_negations"] = preceding_clinical
following_clinical = following + ["was ruled out", "were ruled out", "free"]
en_clinical["following_negations"] = following_clinical
termination_clinical = termination + [
"cause for",
"cause of",
"causes for",
"causes of",
"etiology for",
"etiology of",
"origin for",
"origin of",
"origins for",
"origins of",
"other possibilities of",
"reason for",
"reason of",
"reasons for",
"reasons of",
"secondary to",
"source for",
"source of",
"sources for",
"sources of",
"trigger event for",
]
en_clinical["termination"] = termination_clinical
LANGUAGES["en_clinical"] = en_clinical
en_clinical_sensitive = dict()
preceding_clinical_sensitive = preceding_clinical + [
"concern for",
"supposed",
"which causes",
"leads to",
"h/o",
"history of",
"instead of",
"if you experience",
"if you get",
"teaching the patient",
"taught the patient",
"teach the patient",
"educated the patient",
"educate the patient",
"educating the patient",
"monitored for",
"monitor for",
"test for",
"tested for",
]
en_clinical_sensitive["pseudo_negations"] = pseudo_clinical
en_clinical_sensitive["preceding_negations"] = preceding_clinical_sensitive
en_clinical_sensitive["following_negations"] = following_clinical
en_clinical_sensitive["termination"] = termination_clinical
LANGUAGES["en_clinical_sensitive"] = en_clinical_sensitive
class termset:
def __init__(self, termset_lang):
self.pattern_types = [
"pseudo_negations",
"preceding_negations",
"following_negations",
"termination",
]
self.terms = LANGUAGES[termset_lang]
def get_patterns(self):
return self.terms
def remove_patterns(self, pattern_dict):
for key, value in pattern_dict.items():
if key in self.pattern_types:
self.terms[key] = [i for i in self.terms[key] if i not in value]
else:
raise ValueError(f"Unexpected key: {key} not in {self.pattern_types}")
def add_patterns(self, pattern_dict):
for key, value in pattern_dict.items():
if key in self.pattern_types:
self.terms[key] = list(set(self.terms[key] + value))
else:
raise ValueError(f"Unexpected key: {key} not in {self.pattern_types}")

21
src/great_ai/utilities/lemmatize_text.py
View file

@ -1,21 +0,0 @@
from typing import List
from .lemmatize_token import lemmatize_token
from .nlp import nlp
def lemmatize_text(
text: str,
add_negation: bool = False,
add_part_of_speech: bool = False,
) -> List[str]:
doc = nlp(text)
return [
lemmatize_token(
t,
add_negation=add_negation,
add_part_of_speech=add_part_of_speech,
)
for t in doc
]

20
src/great_ai/utilities/lemmatize_token.py
View file

@ -1,20 +0,0 @@
from spacy.tokens import Token
from .data import american_spellings
def lemmatize_token(
token: Token,
add_negation: bool = False,
add_part_of_speech: bool = False,
) -> str:
lemma = token.lemma_.lower()
lemma = american_spellings.get(lemma, lemma)
if add_part_of_speech:
lemma = f"{lemma}_{token.pos_}"
if add_negation and token._.negex:
lemma = f"NOT_{lemma}"
return lemma

3
src/great_ai/utilities/matcher/__init__.py
View file

@ -1,3 +0,0 @@
from .fast_tokenize import fast_tokenize
from .filter_sentences import filter_sentences
from .normalize import normalize

30
src/great_ai/utilities/matcher/fast_tokenize.py
View file

@ -1,30 +0,0 @@
import re
from typing import List, Union
from segtok.tokenizer import word_tokenizer
from ..get_sentences import get_sentences
from .normalize import normalize
def fast_tokenize(
text: Union[List[str], str], ignore_partial: bool = False
) -> List[List[str]]:
if isinstance(text, str):
text = normalize(text)
text = get_sentences(text, ignore_partial=ignore_partial)
results: List[List[str]] = []
for sentence in text:
sentence = re.sub(r"\bare\b", "is", sentence)
sentence = re.sub(r"\ban\b", "a", sentence)
sentence = re.sub(r"\bthese\b", "this", sentence)
results.append(
[
token.lower() if token not in {"CITATION", "NUMBER"} else token
for token in word_tokenizer(sentence)
]
)
return results

66
src/great_ai/utilities/matcher/filter_sentences.py
View file

@ -1,66 +0,0 @@
from pathlib import Path
from typing import Dict, List, Union
import yaml
from spacy.matcher import Matcher
from ..get_sentences import get_sentences
from ..nlp import nlp
from .fast_tokenize import fast_tokenize
from .normalize import normalize
rules_cache: Dict[str, Matcher] = {}
def filter_sentences(
sentences: str,
rules_file: Path,
inverse: bool = False,
ignore_partial: bool = False,
) -> List[str]:
if str(rules_file) not in rules_cache:
with open(rules_file, encoding="utf-8") as f:
rule_patterns = yaml.safe_load(f).keys()
matcher = Matcher(nlp.vocab)
rules = [_pattern_to_rule(p) for p in rule_patterns]
matcher.add("", rules)
rules_cache[str(rules_file)] = matcher
matcher = rules_cache[str(rules_file)]
original_sentences = get_sentences(sentences, ignore_partial=ignore_partial)
tokenized = fast_tokenize(original_sentences, ignore_partial=ignore_partial)
results: List[str] = []
for original_sentence, sentence in zip(original_sentences, tokenized):
doc = nlp(normalize(" ".join(sentence)))
matches = matcher(doc)
if matches:
# _, start, end = max(
# matches,
# key=lambda v: v[2] - v[1],
# )
# print(str(doc[start:end]))
if not inverse:
results.append(original_sentence)
elif inverse:
results.append(original_sentence)
return results
def _pattern_to_rule(pattern: str) -> List[Dict[str, Union[bool, str]]]:
result: List[Dict[str, Union[bool, str]]] = []
for t in pattern.split():
if t == "*":
result.extend([{"OP": "?"}, {"OP": "?"}])
elif t == "CITATION":
result.append({"ORTH": "CITATION"})
elif t == "NUMBER":
result.append({"ORTH": "NUMBER"})
else:
result.append({"LOWER": t})
return result

22
src/great_ai/utilities/matcher/normalize.py
View file

@ -1,22 +0,0 @@
import re
from ..clean import clean
def normalize(text: str) -> str:
text = re.sub(
r"""
([A-Z]\w+\W+(et\ al.)) # inline reference: Bank et al.
| (\[[0-9-, ]+\]) # IEEE style: [1], [2-4], [3, 5]
| (\(.*?,?\W+?\d+\)) # APA style: (Bank, 2020)
| ([A-Z]\w+ \(?\d+\)) # APA style: Bank (2020)
""",
" CITATION ",
text,
flags=re.VERBOSE,
)
text = re.sub(r"\d[\d,. -]*(st|nd|rd|th)?", " NUMBER ", text)
text = clean(text, convert_to_ascii=True)
text = re.sub(r"[^a-zA-Z.?!,:;'\" -]", "", text)
return text

21
src/great_ai/utilities/nlp.py
View file

@ -1,21 +0,0 @@
try:
import en_core_web_sm
except ImportError:
import subprocess
print("Spacy model en_core_web_sm not found locally, downloading...")
subprocess.call(
[
"pip",
"install",
"https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl",
]
)
import en_core_web_sm
from .external.negspacy import negation # noqa: F401 it's important to import this
nlp = en_core_web_sm.load()
nlp.add_pipe("negex")

2
src/great_ai/utilities/publication_tei/__init__.py
View file

@ -1,2 +0,0 @@
from .models import *
from .publication_tei import PublicationTEI

7
src/great_ai/utilities/publication_tei/models/__init__.py
View file

@ -1,7 +0,0 @@
from .affiliation import Affiliation
from .author import Author
from .bookmark import Bookmark
from .bookmark_title import BookmarkTitle
from .element import Element, Meta, MetaType, Paragraph, Title
from .publication_metadata import PublicationMetadata
from .text import Text

11
src/great_ai/utilities/publication_tei/models/affiliation.py
View file

@ -1,11 +0,0 @@
from typing import Optional, Tuple
from pydantic import BaseModel
class Affiliation(BaseModel):
institutions: Tuple[str, ...]
departments: Tuple[str, ...]
laboratories: Tuple[str, ...]
country: Optional[str]
settlement: Optional[str]

14
src/great_ai/utilities/publication_tei/models/author.py
View file

@ -1,14 +0,0 @@
from typing import List, Optional
from pydantic import BaseModel
from .affiliation import Affiliation
class Author(BaseModel):
name: Optional[str]
orcid: Optional[str]
email: Optional[str]
corresponding: bool
affiliations: List[Affiliation]
coordinates: Optional[str]

10
src/great_ai/utilities/publication_tei/models/bookmark.py
View file

@ -1,10 +0,0 @@
from pydantic import BaseModel
from .bookmark_title import BookmarkTitle
class Bookmark(BaseModel):
title: BookmarkTitle
original_title: str
document_order: int
coordinates: str

Some files were not shown because too many files have changed in this diff Show more