From 4b156e1286445e9bb2931c062cae13de546ffa0f Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Tue, 27 Oct 2026 20:03:16 +0000 Subject: [PATCH] Filter sentence completeness before changing case or punctuation --- great_ai/utilities/get_sentences.py | 8 +++++--- tests/utilities/test_get_sentences.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/great_ai/utilities/get_sentences.py b/great_ai/utilities/get_sentences.py index 8cb05f3..3fcd355 100644 --- a/great_ai/utilities/get_sentences.py +++ b/great_ai/utilities/get_sentences.py @@ -55,14 +55,16 @@ def get_sentences( return sentence.strip() sentences = [ - process(tokenizer.to_text(sentence)) for sentence in segment(token_stream) + tokenizer.to_text(sentence).strip() for sentence in segment(token_stream) ] if ignore_partial: sentences = [ sentence for sentence in sentences - if sentence[0].isupper() and sentence[-1] in sentence_ending_punctuations + if sentence + and sentence[0].isupper() + and sentence[-1] in sentence_ending_punctuations ] - return sentences + return [process(sentence) for sentence in sentences] diff --git a/tests/utilities/test_get_sentences.py b/tests/utilities/test_get_sentences.py index 43810d1..6fa39e5 100644 --- a/tests/utilities/test_get_sentences.py +++ b/tests/utilities/test_get_sentences.py @@ -57,3 +57,13 @@ def test_empty() -> None: assert get_sentences("", ignore_partial=True) == [] assert get_sentences("", true_case=True) == [] assert get_sentences("", remove_punctuation=True) == [] + + +def test_completeness_is_checked_before_optional_transformations() -> None: + assert get_sentences("Hello world.", ignore_partial=True, true_case=True) == [ + "hello world." + ] + assert get_sentences( + "Hello world.", ignore_partial=True, remove_punctuation=True + ) == ["Hello world"] + assert get_sentences("...", ignore_partial=True, remove_punctuation=True) == []