andras/great-ai
andras/great-ai
Watch
1
0
Fork
You've already forked great-ai
 0
Code Issues Pull requests 2 Projects Releases 4 Packages 1 Wiki Activity Actions

Filter sentence completeness before changing case or punctuation

Browse source
This commit is contained in:
Andras Schmelczer 2026-10-27 20:03:16 +00:00
commit 4b156e1286
2 changed files with 15 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

8
great_ai/utilities/get_sentences.py
View file

@ -55,14 +55,16 @@ def get_sentences(
return sentence.strip()
sentences = [
process(tokenizer.to_text(sentence)) for sentence in segment(token_stream)
tokenizer.to_text(sentence).strip() for sentence in segment(token_stream)
]
if ignore_partial:
sentences = [
sentence
for sentence in sentences
if sentence[0].isupper() and sentence[-1] in sentence_ending_punctuations
if sentence
and sentence[0].isupper()
and sentence[-1] in sentence_ending_punctuations
]
return sentences
return [process(sentence) for sentence in sentences]

10
tests/utilities/test_get_sentences.py
View file

@ -57,3 +57,13 @@ def test_empty() -> None:
assert get_sentences("", ignore_partial=True) == []
assert get_sentences("", true_case=True) == []
assert get_sentences("", remove_punctuation=True) == []
def test_completeness_is_checked_before_optional_transformations() -> None:
assert get_sentences("Hello world.", ignore_partial=True, true_case=True) == [
"hello world."
]
assert get_sentences(
"Hello world.", ignore_partial=True, remove_punctuation=True
) == ["Hello world"]
assert get_sentences("...", ignore_partial=True, remove_punctuation=True) == []