63 lines
1.8 KiB
Python
63 lines
1.8 KiB
Python
import re
|
|
|
|
JUNK_PATTERNS = [
|
|
r'all rights reserved',
|
|
r'copyright\s*©',
|
|
r'isbn[\s\-]',
|
|
r'published by',
|
|
r'first published',
|
|
r'printed in',
|
|
r'no part of this',
|
|
r'table of contents',
|
|
r'this is a work of fiction',
|
|
r'any resemblance to',
|
|
]
|
|
JUNK_RE = re.compile('|'.join(JUNK_PATTERNS), re.IGNORECASE)
|
|
|
|
def _is_junk(text):
|
|
if len(text.split()) < 30:
|
|
return True
|
|
if JUNK_RE.search(text):
|
|
return True
|
|
return False
|
|
|
|
def split_scenes(chapters, min_sentences=3, max_sentences=6):
|
|
scenes = []
|
|
for chapter_text in chapters:
|
|
paragraphs = [p.strip() for p in re.split(r'\n{2,}', chapter_text) if p.strip()]
|
|
buf = []
|
|
buf_sentences = 0
|
|
|
|
for para in paragraphs:
|
|
sentences = re.split(r'(?<=[.!?])\s+', para.strip())
|
|
sentences = [s for s in sentences if s]
|
|
|
|
for sentence in sentences:
|
|
buf.append(sentence)
|
|
buf_sentences += 1
|
|
|
|
if buf_sentences >= max_sentences:
|
|
candidate = " ".join(buf)
|
|
if not _is_junk(candidate):
|
|
scenes.append(candidate)
|
|
buf = []
|
|
buf_sentences = 0
|
|
|
|
if buf_sentences >= min_sentences:
|
|
candidate = " ".join(buf)
|
|
if not _is_junk(candidate):
|
|
scenes.append(candidate)
|
|
buf = []
|
|
buf_sentences = 0
|
|
|
|
if buf:
|
|
candidate = " ".join(buf)
|
|
if scenes and _is_junk(candidate):
|
|
pass
|
|
elif scenes:
|
|
scenes[-1] = scenes[-1] + " " + candidate
|
|
elif not _is_junk(candidate):
|
|
scenes.append(candidate)
|
|
|
|
return [s for s in scenes if len(s.strip()) > 20]
|