andras/great-ai
andras/great-ai
Watch
1
0
Fork
You've already forked great-ai
 0
Code Issues Pull requests 2 Projects Releases 4 Packages 1 Wiki Activity Actions

Filter sentence completeness before changing case or punctuation

Browse source
This commit is contained in:
Andras Schmelczer 2026-10-27 20:03:16 +00:00
commit 4b156e1286
1 changed files with 15 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

8
great_ai/utilities/get_sentences.py
View file

@ -55,14 +55,16 @@ def get_sentences(
return sentence.strip()
sentences = [
process(tokenizer.to_text(sentence)) for sentence in segment(token_stream)
tokenizer.to_text(sentence).strip() for sentence in segment(token_stream)
]
if ignore_partial:
sentences = [
sentence
for sentence in sentences
if sentence[0].isupper() and sentence[-1] in sentence_ending_punctuations
if sentence
and sentence[0].isupper()
and sentence[-1] in sentence_ending_punctuations
]
return sentences
return [process(sentence) for sentence in sentences]