Drop non English tags using the system dictionary
This commit is contained in:
1 parent
faa3d501d1
commit
d9fc18fcca
2 files changed
+59
-1
No files matched your search
@@ -885,6 +885,59 @@ Return JSON only in this shape:
|
|||||||
raise RuntimeError(f"Could not generate a valid script: {last}")
|
raise RuntimeError(f"Could not generate a valid script: {last}")
|
||||||
|
|
||||||
|
|
||||||
|
_english_words = None
|
||||||
|
|
||||||
|
|
||||||
|
def english_words():
|
||||||
|
global _english_words
|
||||||
|
if _english_words is None:
|
||||||
|
words = set()
|
||||||
|
for f in ("/usr/share/dict/words", "/usr/share/dict/american-english", "/usr/share/dict/british-english"):
|
||||||
|
try:
|
||||||
|
words |= {w.strip().lower() for w in open(f, encoding="utf-8", errors="ignore") if w.strip()}
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
_english_words = words
|
||||||
|
return _english_words
|
||||||
|
|
||||||
|
|
||||||
|
def is_english_word(w, extra):
|
||||||
|
w = w.lower().strip("'")
|
||||||
|
if not w or w.isdigit() or w in extra:
|
||||||
|
return True
|
||||||
|
vocab = english_words()
|
||||||
|
if not vocab:
|
||||||
|
return True
|
||||||
|
if w in vocab:
|
||||||
|
return True
|
||||||
|
for suffix in ("s", "es", "'s", "ers", "er", "ing", "ed", "ies", "ly"):
|
||||||
|
if w.endswith(suffix) and len(w) > len(suffix) + 2:
|
||||||
|
stem = w[: -len(suffix)]
|
||||||
|
if stem in vocab or stem + "e" in vocab or (suffix == "ies" and stem + "y" in vocab):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def english_tags(tags, script):
|
||||||
|
extra = set(re.findall(r"[a-z]+", script_text(script).lower()))
|
||||||
|
extra |= {
|
||||||
|
"youtube", "shorts", "kid", "kids", "kiddie", "preschool", "preschooler", "preschoolers", "toddler", "toddlers",
|
||||||
|
"storytime", "storybook", "bedtime", "cartoon", "cartoons", "animated", "animation", "teamwork", "video", "videos",
|
||||||
|
"online", "sel", "abc", "abcs", "diy", "fun", "3d", "2d", "short", "film",
|
||||||
|
}
|
||||||
|
kept, dropped = [], []
|
||||||
|
for tag in tags:
|
||||||
|
if not tag.isascii():
|
||||||
|
dropped.append(tag)
|
||||||
|
continue
|
||||||
|
words = re.findall(r"[A-Za-z']+|\d+", tag)
|
||||||
|
if words and all(is_english_word(w, extra) for w in words):
|
||||||
|
kept.append(tag)
|
||||||
|
else:
|
||||||
|
dropped.append(tag)
|
||||||
|
return kept, dropped
|
||||||
|
|
||||||
|
|
||||||
def clean_tags(tags):
|
def clean_tags(tags):
|
||||||
out, seen, total = [], set(), 0
|
out, seen, total = [], set(), 0
|
||||||
for t in tags:
|
for t in tags:
|
||||||
@@ -908,6 +961,7 @@ def make_tags(s, script):
|
|||||||
prompt = f"""Generate YouTube tags for this Short.
|
prompt = f"""Generate YouTube tags for this Short.
|
||||||
Include specific tags for its exact subject plus broader category, genre, and audience tags that help discovery.
|
Include specific tags for its exact subject plus broader category, genre, and audience tags that help discovery.
|
||||||
Every tag must accurately describe the video. No unrelated trending terms, no names of other creators, people, brands, or copyrighted characters.
|
Every tag must accurately describe the video. No unrelated trending terms, no names of other creators, people, brands, or copyrighted characters.
|
||||||
|
Write every tag in plain English words only.
|
||||||
{niche_line}{kids_line}
|
{niche_line}{kids_line}
|
||||||
Title: {script['title']}
|
Title: {script['title']}
|
||||||
Description: {script['description']}
|
Description: {script['description']}
|
||||||
@@ -922,6 +976,10 @@ Return JSON only: {{"tags": ["15 to 25 tags, most specific first"]}}"""
|
|||||||
if isinstance(raw, str):
|
if isinstance(raw, str):
|
||||||
raw = raw.split(",")
|
raw = raw.split(",")
|
||||||
tags = clean_tags(raw)
|
tags = clean_tags(raw)
|
||||||
|
if s["source_language"].strip().lower() in ("", "en"):
|
||||||
|
tags, dropped = english_tags(tags, script)
|
||||||
|
if dropped:
|
||||||
|
logger.info("Dropped non-English tags: %s", ", ".join(dropped))
|
||||||
if tags:
|
if tags:
|
||||||
return tags
|
return tags
|
||||||
except Cancelled:
|
except Cancelled:
|
||||||
|
|||||||
+1
-1
@@ -128,7 +128,7 @@ const FIELDS=[
|
|||||||
["region_code","Region","select:US=United States|GB=United Kingdom|CA=Canada|AU=Australia|NZ=New Zealand|IE=Ireland|IN=India|PH=Philippines|ZA=South Africa|DE=Germany|FR=France|ES=Spain|MX=Mexico|BR=Brazil|JP=Japan","","p"],
|
["region_code","Region","select:US=United States|GB=United Kingdom|CA=Canada|AU=Australia|NZ=New Zealand|IE=Ireland|IN=India|PH=Philippines|ZA=South Africa|DE=Germany|FR=France|ES=Spain|MX=Mexico|BR=Brazil|JP=Japan","","p"],
|
||||||
["source_language","Source language","select:en=English|es=Spanish|fr=French|de=German|pt=Portuguese|it=Italian|=Any language","Skips source videos in other languages","p"],
|
["source_language","Source language","select:en=English|es=Spanish|fr=French|de=German|pt=Portuguese|it=Italian|=Any language","Skips source videos in other languages","p"],
|
||||||
["trending_count","Videos to scan","select:10|25|50","","p"],
|
["trending_count","Videos to scan","select:10|25|50","","p"],
|
||||||
["videos_per_day","Videos per day","select:0=Unlimited|1|2|3|4|5|6","YouTube API quota allows about 6 uploads a day","p"],
|
["videos_per_day","Videos per day","select:0=Unlimited|1|2|3|4|5|6|8|10|12|15|20","The default YouTube API quota allows about 6 uploads a day. Higher values need a quota increase from Google","p"],
|
||||||
["minutes_between_videos","Minutes between videos","select:0=No gap|30|60|120|180|240|360","","p"],
|
["minutes_between_videos","Minutes between videos","select:0=No gap|30|60|120|180|240|360","","p"],
|
||||||
["","Folders (shared by all profiles)","header","",""],
|
["","Folders (shared by all profiles)","header","",""],
|
||||||
["storage_dir","Storage folder","wide","Full path on a large disk, e.g. /Volumes/iDrive/shorts-autopilot. Jobs are built in output/ and moved to published/","g"],
|
["storage_dir","Storage folder","wide","Full path on a large disk, e.g. /Volumes/iDrive/shorts-autopilot. Jobs are built in output/ and moved to published/","g"],
|
||||||
|
|||||||
Reference in new issue
Block a user