#!/usr/bin/env python3
"""Generate the site's tag set from tag_schema.py (D49): tags grouped by WHERE THE FACT
COMES FROM, a verification chip per tag, and a per-piece EVIDENCE line for every
assignment so the collection grid can show the relevant data without opening a piece.
Emits db/tags.sql (re-runnable: clears AUTO tags/assignments, keeps hand-added ones)."""
import json, os, sys, re, collections, importlib
SRC = "/Users/michaelconnery/Desktop/bitgan_puzzle_work"
OUT = os.path.join(os.path.dirname(__file__), "..", "db", "tags.sql")
sys.path.insert(0, SRC); S = importlib.import_module("tag_schema"); PF = importlib.import_module("primary_facts")

def q(s):
    if s is None: return "NULL"
    return "'" + str(s).replace("'", "''") + "'"

pieces = {r["g"]: r for r in json.load(open(f"{SRC}/pieces.json"))["rows"] if isinstance(r.get("g"), int)}
sheet  = json.load(open(f"{SRC}/sheet.json"))
deck   = {r["GAN"]: r for r in json.load(open(f"{SRC}/deck_512.json")) if isinstance(r.get("GAN"), int)}
tx     = json.load(open(f"{SRC}/taxonomy.json"))
led    = json.load(open(f"{SRC}/theories/anomaly_ledger.json"))
pairs  = json.load(open(f"{SRC}/pairs.json"))["pairs"]
M      = json.load(open(f"{SRC}/lore_catalog_runs/interpretation_matrix.json"))
TRIBUTE_HOW = {228:"on-chain description: 'bitGAN portrait of Kevin Abosch'", 246:"on-chain description: 'reproduction of an early SuperRare Hackatao'", 264:"on-chain description: 'Killer Acid provided a portrait'", 282:"on-chain description: 'Eclectic Method and I were talking'",
    447:"name: Brinkman (Bryan Brinkman, artist) — name only", 448:"name: Crypto Stella (collector) — name only", 446:"name: SuperRare (the platform) — name only", 445:"name: Magic Internet Money (the meme) — name only", 482:"name: Testy McTest — a joke name, not a person",
    210:"lore: noohp, solver of the byteGAN puzzle — not in the data", 211:"lore: mazzone, a past solver — not in the data", 1:"lore: The Moirai (the Fates) — mythology, not a person", 2:"lore: The Artist — presumed Pindar himself; not in the data", 3:"lore: The Geometer — not in the data", 130:"not validated: no person or project in the name, traits or description"}
NUMCLASS = {"fib":"Fibonacci","prime":"prime (not Fibonacci)","sq":"square","oddcomp":"odd composite","even":"even"}
STD    = {'skull':41,'robo':55,'spooky':55,'solstice':55,'ghost':55,'review':55}

alias2c = {al: c["id"] for c in tx["concepts"] for al in c.get("aliases", [])}
recs = {}   # g -> list of (type, concept, desc)
for k, v in led.items():
    if not k.isdigit() or not isinstance(v, dict): continue
    g = int(k)
    if not (1 <= g <= 512): continue
    recs[g] = [(a["type"], alias2c.get(a["type"]), a.get("desc") or "") for a in v.get("anomalies", []) if not a.get("hidden")]

tags = {}; ptags = {}   # id -> meta ; (g, tag_id) -> evidence
def add_tag(tid, category, subcat, label, desc, status, cid=None):
    tags.setdefault(tid, dict(category=category, subcat=subcat, label=label, description=desc, status=status, concept_id=cid))
def assign(g, tid, ev=None):
    if (g, tid) not in ptags or (ev and not ptags[(g, tid)]): ptags[(g, tid)] = (ev or "")[:220]
def short(s, n=200):
    s = re.sub(r"\s+", " ", str(s or "")).strip()
    return s if len(s) <= n else s[:n-1] + "…"

# ---------------- drop + published traits (one tag per value) ----------------
DROP_LABELS = {"skull":"Skull drop","robo":"Robo drop","spooky":"Spooky drop","solstice":"Solstice drop","ghost":"Ghost drop","review":"Act 1 Review drop"}
PUB = [("gang","GANg (species)", lambda g: sheet.get(str(g),{}).get("GANg")),
       ("type","Type", lambda g: sheet.get(str(g),{}).get("Type")),
       ("subtype","SubType", lambda g: pieces.get(g,{}).get("sub")),
       ("eyes","Number of eyes", lambda g: sheet.get(str(g),{}).get("Eyes")),
       ("dimensions","Dimensions", lambda g: pieces.get(g,{}).get("dim"))]
for g in range(1, 513):
    d = pieces.get(g, {}).get("drop")
    if d:
        add_tag(f"drop:{d}", "drop", None, DROP_LABELS.get(d, d), f"Piece from the {DROP_LABELS.get(d,d)}.", "data"); assign(g, f"drop:{d}")
    for field, sub, getter in PUB:
        v = getter(g)
        if v in (None, "", "None"): continue
        add_tag(f"{field}:{v}", "published", sub, f"{sub}: {v}", f"Published {sub} is {v}.", "data"); assign(g, f"{field}:{v}")
    if deck.get(g, {}).get("Name"):
        add_tag("published:named", "published", "Named", "Named piece", "Carries a Name trait: one of the 38 pieces Pindar named individually.", "data")
        assign(g, "published:named", f"Name: {deck[g]['Name']}")

# ---------------- membership helpers ----------------
def concept_members(cids):
    return {g for g, rs in recs.items() if any((c in cids) or (t in cids) for t, c, _ in rs)}
def ledger_members(ts):
    return {g for g, rs in recs.items() if any(t in ts for t, _, _ in rs)}
def field_members(col):
    return {int(g) for g, r in sheet.items() if g.isdigit() and r.get(col) not in (None, "", False, "None")}
def rec_desc(g, cids=(), types=()):
    for t, c, dsc in recs.get(g, []):
        if (c in cids) or (t in cids) or (t in types):
            if dsc: return short(dsc)
    return ""
def _kinds(g):
    """kinds of difference for a piece, ignoring the systematic Twin->Twins rename (tagged separately)"""
    dl = [(f, sv, cv) for f, sv, cv in PF.DIFFS.get(g, []) if not (f == 'SubType' and sv.lower() == 'twin' and cv.lower() == 'twins')]
    if not dl: return set()
    fields = {f: (sv, cv) for f, sv, cv in dl}
    if 'Type' in fields and 'SubType' in fields and fields['Type'][0].lower() == fields['SubType'][1].lower() and fields['SubType'][0].lower() == fields['Type'][1].lower(): return {'swap'}
    return {PF.kind_of(f, sv, cv) for f, sv, cv in dl}
KINDS = {g: _kinds(g) for g in PF.DIFFS}
def _verb_set(kind): return {g for g, ks in KINDS.items() if kind in ks}
# Order breaks Pindar's own records show (D73).
def _not_in_longest_run(pairs):
    n = len(pairs); best = [1] * n; prev = [-1] * n
    for a in range(n):
        for b in range(a):
            if pairs[b][1] < pairs[a][1] and best[b] + 1 > best[a]: best[a], prev[a] = best[b] + 1, b
    k = max(range(n), key=lambda x: best[x]); keep = set()
    while k != -1: keep.add(pairs[k][0]); k = prev[k]
    return [g for g, _ in pairs if g not in keep]
def _save_breaks(lo, hi): return _not_in_longest_run([(g, PF.SAVE_TIME[g]) for g in range(lo, hi + 1) if g in PF.SAVE_TIME])
SAVE_OOO = set(_save_breaks(129, 192)) | set(_save_breaks(193, 208))
MINT_OOO = set(_not_in_longest_run([(g, PF.TI[g]) for g in range(1, 513) if g in PF.TI]))
REVIEW_OOO = {467: "reprisetest group is split in two runs (#467-469 and #481-484); this file was saved {d}, three months before the rest of the Review drop and ten weeks before the Ghost drop's testGANs",
              468: None, 469: None,
              470: "group reprise11 is numbered ahead of reprise09 and reprise10 (#473-478)", 471: None, 472: None,
              477: "filename says reprise11 but the piece sits among the reprise09 and reprise10 files"}
for _a, _b in ((468, 467), (469, 467), (471, 470), (472, 470)): REVIEW_OOO[_a] = REVIEW_OOO[_b]
# Instruction words (D88): essences that read as a command you could carry out on a list of words. Our reading; the collector reviews it.
INSTRUCTION_WORDS = {
 "put in a different order": ["sort", "swap", "flip", "arrange", "move", "cross", "mirror", "jump", "advance"],
 "add, join or repeat": ["add", "double", "twice", "again", "multiply", "extend", "combine", "couple", "bind", "collect", "pair"],
 "remove or split": ["divide", "half", "erase", "cancel", "abandon", "avoid", "destroy", "recycle"],
 "change": ["alter", "adjust", "adapt", "correct", "derive"],
 "solve or reveal": ["solve", "reveal", "unlock", "verify", "execute", "explain", "define", "remember"],
 "start or stop": ["begin", "exit", "escape", "expire"]}
_IW = {w: fam for fam, ws in INSTRUCTION_WORDS.items() for w in ws}
POSITION_WORDS = ["above", "across", "ahead", "among", "apart", "around", "away", "before", "behind", "below", "between", "beyond", "bottom", "over", "under", "upper", "south", "off", "edge", "border", "base", "axis", "position", "index"]
def _ess(g): return PF.SHEET[g]['Essence'].strip().lower()
# Pieces that share their exact name with exactly one other piece (the collector's suggestion, D91).
_stem = lambda t: re.sub(r"\s*\(.*$", "", t).strip().lower()
_bt = collections.defaultdict(list)
for _g in range(1, 513): _bt[_stem(PF.SHEET[_g]['Title'])].append(_g)
SAME_NAME = {}
for _k, _v in _bt.items():
    if len(_v) == 2: SAME_NAME[_v[0]] = _v[1]; SAME_NAME[_v[1]] = _v[0]
_VA = json.load(open(f"{SRC}/lore_catalog_runs/visual_audit_tags.json"))
_VA_NOTES = {int(k): v['note'].strip() for k, v in json.load(open(f"{SRC}/lore_catalog_runs/visual_audit_2026-09-27c.json"))['pieces'].items()}
_VA_EXT = {k: set(v) for k, v in _VA.get("_extend", {}).items()}
# Named pieces whose invented xxxGAN word hides a seed word the free-standing match misses (acidGAN -> acid). D69.
FUSED = {}
for _g, _r in PF.SHEET.items():
    if not (_r.get('Name') or '').strip(): continue
    _w = [m.lower() for m in re.findall(r'([A-Za-z]+)GAN', _r['Title'] + ' ' + _r['Name']) if m.lower() in set(PF.bip) and m.lower() not in PF.NAMED_WITH_SEED_WORD.get(_g, [])]
    _w = [w for w in dict.fromkeys(_w) if w != 'skull']
    if _w: FUSED[_g] = _w
FN = {"corrupt_verb:transpose": _verb_set('transpose'), "corrupt_verb:reverse": _verb_set('reverse'), "corrupt_verb:delete": _verb_set('delete'),
      "corrupt_verb:swap": _verb_set('swap'), "corrupt_verb:substitute": _verb_set('substitute'), "corrupt_verb:insert": _verb_set('insert'),
      "corrupt_verb:number": _verb_set('number'), "corrupt_verb:case": _verb_set('case'), "twins_rename": set(PF.TWINS_RENAME),
      "chain_differs_other": set(),
      "description_anomaly": set(PF.DESC_ANOMALY), "no_essence_on_chain": set(PF.NO_ESSENCE_ON_CHAIN), "named_on_chain_only": set(PF.NAMED_ON_CHAIN_ONLY),
      "trait_key_misspelled": set(PF.TRAIT_KEY_ANOMALY), "no_chain_record": set(PF.NO_CHAIN), "name_stray": set(PF.NAME_STRAY), "name_odd_numbering": set(PF.ODD_NUMBERING),
      "name_typo": set(PF.NAME_TYPO), "trait_value_typo": set(PF.TRAIT_VALUE_TYPO), "filename_typo": set(PF.FILENAME_TYPO), "essence_not_plain": set(PF.ESSENCE_NOT_PLAIN),
      "chain_wrong": set(PF.CHAIN_WRONG), "substitution_art": set(PF.SUBSTITUTION_ART),
      "filename_square_mark": set(PF.SQUARE_MARK), "filename_square_unmarked": set(PF.SQUARE_UNMARKED), "filename_smooth": set(PF.SMOOTH_MARK), "sheet_wrong": set(PF.SHEET_WRONG), "undecided_side": set(PF.UNDECIDED),
      "point_release": set(PF.POINT_RELEASE), "solo_session": set(PF.SOLO_SESSION), "named_with_seed_word": set(PF.NAMED_WITH_SEED_WORD), "named_fused_seed_word": set(FUSED), "essence_solution_block": {450,453,458,463,471,479,484,487,494,499,502,508}, "essence_ex_block": {301,313,325,337,349,361,373,385,398,410,422,434}, "essence_own_number": {307,322,348,368,372,380,389,404,407,429,444}, "essence_neighbour_pair": {323,324,350,351,369,370,392,393,414,415,435,436}, "robo_last_row_break": set(range(185, 193)), "superrare_stroke_pointer": {51, 69, 105, 135, 151, 153, 155, 172, 175, 186, 358}, "typed_mark_bit_mirror": {32,119,161,187,193}, "essence_position_word": {g for g in range(1, 513) if _ess(g) in POSITION_WORDS}, "same_name_pair": set(SAME_NAME), "season_frame_out_of_order": {240, 241, 242}, "season_date_mismatch": {240, 297}, "filename_type_mismatch": {28}, "larger_eyes_audit": {18,23,25,27,41,48,66,97,109}, "mint_out_of_order": MINT_OOO, "save_out_of_order": SAVE_OOO, "review_group_out_of_order": set(REVIEW_OOO)}
# audit 2026-09-16: the transposed pair (#259/#268) leaves 'out of sequence' for its own tag; Pindar's index column for 1-300
FN["essence_out_of_sequence"] = ledger_members({"flag:Essence out of sequence"}) - ledger_members({"essence_transposed"})
try:
    _BIP = {int(k): v for k, v in json.load(open(f"{SRC}/../bitganspuzzlebook/pages/data/page015_jan1/bip39_index.json"))["index"].items()}
except Exception:
    _BIP = {}
FN["pindar_index"] = {g for g in _BIP if g <= 300}
FN["robo_every_fifth"] = {129 + p for p in range(4, 64, 5)}
FN["robo_one_hot"] = {129 + (1 << b) for b in range(6)}
FN["skull_robo_hybrid"] = {161, 175, 187, 191, 206}
def _cls(n):
    if n in {0, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89}: return "fib"
    if n > 1 and all(n % d for d in range(2, int(n ** .5) + 1)): return "prime"
    if int(n ** .5) ** 2 == n: return "square"
    return "odd" if n % 2 else "even"
FN["odd_remaining_flaming"] = {g for g in range(1, 129) if _cls(g) == "odd"}
FN["square_silver_smoke"] = {g for g in range(1, 129) if _cls(g) == "square" and (PF.SHEET[g]["SubType"] == "Silver" or PF.SHEET[g]["Type"] == "Smoke")} | {g for g in range(129, 193) if _cls(g - 129) == "square" and (PF.SHEET[g]["SubType"] == "Silver" or PF.SHEET[g]["Type"] == "Smoke")}
FN["number_look_break"] = {7, 8, 16, 31, 41, 55, 89, 145, 165, 174, 185}
# swarm of 4 October 2026 (D121): members are facts checked by script or the collector's own recorded eye answers
FN["number_32_every_drop"] = {32, 161, 241, 332, 465}
FN["so_repeats_one_ghost"] = {450, 484, 494, 499}
FN["lying_label_eleven"] = {32, 119, 161, 187, 193, 240, 297, 371, 380, 429, 430}
FN["speed_up_binary_eleven"] = {129, 133, 147, 159, 162, 180, 192, 457, 462, 463, 464}
FN["colour_flash_twelve"] = {129, 130, 133, 141, 161, 174, 329, 462, 463, 464, 491, 497}
FN["colour_flash_closer_four"] = {130, 133, 141, 174}
FN["named_ghost_speed_up"] = {327, 370, 373, 374, 383, 401, 404, 407, 420, 441}
# solving system, cycle 001, 5 October 2026 (D123): a fact checked by script, and the collector's own eye answers of that day
FN["third_family_puzzle_to_random"] = {130, 134, 153, 170, 317, 367, 438, 454, 468}
FN["label_swapped_by_eye"] = {145, 150, 339, 340, 429, 430}
# solving system, cycle 003, 9 October 2026: the collector's own eye answer (five Review pieces he marked behind their binary)
FN["behind_binary_by_eye"] = {459, 460, 462, 463, 464}
def _odd_one_out():
    """every title family of 3+ pieces across 2+ drops in which exactly one member is alone in its drop"""
    out = {}
    for _k, _v in _bt.items():
        if len(_v) < 3: continue
        _by = collections.defaultdict(list)
        for _g in _v: _by[PF.DROP(_g)].append(_g)
        if len(_by) < 2: continue
        _lone = [_x[0] for _x in _by.values() if len(_x) == 1]
        if len(_lone) == 1: out[_lone[0]] = (re.sub(r"\s*\(.*$", "", PF.SHEET[_lone[0]]['Title']).strip(), len(_v), PF.DROP(_lone[0]), {_d: len(_x) for _d, _x in _by.items()})
    return out
ODD_ONE_OUT = _odd_one_out()
FN["odd_one_out_title_family"] = set(ODD_ONE_OUT)
_N32 = {32: "skull drop: piece number 32", 161: "robo drop: printed counter 100000 = 32", 241: "Solstice drop: place 32, counting from Sol #209 as 0 (Pindar's 1 January 2022 table)", 332: "Ghost drop: printed 32/144", 465: "Review drop: printed 0x20 = 32 in hexadecimal"}
_SO = {450: "'solution' repeats Ghost #411 'key' (same title, same Type and SubType, and the collector's note)", 484: "'soon' repeats Ghost #301 'exact', head of the 'ex' column (same title, same Type and SubType)", 494: "'sort' repeats Ghost #404 'crash', which holds word 404 (same Type and SubType)", 499: "'sound' repeats Ghost #407 Mint Bot 'crazy', which holds word 407 (the collector's note; file group repriseaa)"}
_LIE = {32: "hidden-type letter written '32' where a letter belongs", 119: "hidden-type letter 'e' where 'd' belongs (journal 002: 'only Odd number of Type E')", 161: "hidden-type letter written '32' where a letter belongs", 187: "hidden-type letter 'C' where 'E' belongs", 193: "hidden-type letter 'F' where 'A' belongs", 240: "dated June 18 but cut from the summer video", 297: "dated September 14 but cut from the autumn video", 371: "labelled Glowing; the collector: 'Lots of green'", 380: "labelled Glowing; the collector: 'Purple/Magenta'", 429: "labelled Glowing; the collector: 'Purple'", 430: "labelled Ultra; the collector: 'This is very red and orange for an Ultra.'"}
SWARM_EV = {
    "number_32_every_drop": lambda g: _N32.get(g, ""),
    "so_repeats_one_ghost": lambda g: _SO.get(g, ""),
    "lying_label_eleven": lambda g: _LIE.get(g, "") + "; the collector's eye settled the eleven on 4 October 2026",
    "speed_up_binary_eleven": lambda g: f"the collector's eye: speeds up (title {PF.SHEET[g]['Title']})" + ("; confirmed in chat 4 October 2026" if g == 457 else "; from his saved notes"),
    "colour_flash_twelve": lambda g: "the collector's eye: colour flash, one kind (4 October 2026)" + ("; one of the closer four" if g in (130, 133, 141, 174) else ""),
    "colour_flash_closer_four": lambda g: "the collector's eye, 4 October 2026: 'clearly a more closely related subset'",
    "named_ghost_speed_up": lambda g: f"the collector's eye, 4 October 2026: Name '{(PF.SHEET[g].get('Name') or '').strip() or PF.ctraits(g).get('Name')}' speeds up" + (" and flashes colour" if g == 407 else ""),
    "odd_one_out_title_family": lambda g: f"name '{ODD_ONE_OUT[g][0]}' is shared by {ODD_ONE_OUT[g][1]} pieces ({', '.join(f'{_d} {_n}' for _d, _n in ODD_ONE_OUT[g][3].items())}); this is its only member in the {ODD_ONE_OUT[g][2]} drop",
    "third_family_puzzle_to_random": lambda g: f"essence '{_ess(g)}' = wordlist word {PF.bip.index(_ess(g))+1} = 2,049 minus {2049 - (PF.bip.index(_ess(g))+1)} ('{PF.bip[2049 - (PF.bip.index(_ess(g))+1) - 1]}' turned over); checked by script, 5 October 2026",
    "behind_binary_by_eye": lambda g: f"the collector, 9 October 2026, on the eye page: sits behind its binary (title {PF.SHEET[g]['Title']}); 'In the Cyborgs, teh binary is the eyes/mouth. It\'s part of the face. In Bot Rainbow, that is one of the abstract/corrupt GANs so there is no consistency.'",
    "label_swapped_by_eye": lambda g: {145: "labelled Silver; the collector, 5 October 2026: 'Each looks like it carries the other's label' (with #150 Flaming)", 150: "labelled Flaming; the collector, 5 October 2026: 'Each looks like it carries the other's label' (with #145 Silver)",
                                      339: "labelled Vivid; the collector, 5 October 2026: 'both pairs look swapped' (with #340 Shy)", 340: "labelled Shy; the collector, 5 October 2026: 'both pairs look swapped' (with #339 Vivid)",
                                      429: "labelled Glowing; the collector, 5 October 2026: 'both pairs look swapped' (with #430 Ultra)", 430: "labelled Ultra; the collector, 5 October 2026: 'both pairs look swapped' (with #429 Glowing); 'It's like a glowing/ultra hybrid'"}.get(g, ""),
}
_WL = [w.strip() for w in open(f"{SRC}/../bitganspuzzlebook/pages/data/bip39.txt") if w.strip()]
FN["sheet_chain_any"] = {g for g in PF.DIFFS if any(k for k in KINDS.get(g, set()))}
FN["chain_differs_other"] = {g for g in PF.DIFFS if not any(g in v for k, v in FN.items() if k.startswith('corrupt_verb') or k == 'twins_rename')}
ADJ = {264:"dog", 268:"bird", 159:"dog", 147:"cat", 180:"wolf"}   # collector-adjudicated (D38); #412 is a monster = species, not listed
SPECIFIC = {"whale","spirit","heart","warrior","shark","blob","cat","dog","wolf","rat","sprite","clown","bird"}   # not species labels (skull, ghost, robot, monster...)
def creature_of(g):
    if g in ADJ: return ADJ[g]
    r = sheet.get(str(g), {}); w = r.get("Visible creature")
    if not w or str(w).startswith("("): return None
    blob = " ".join(str(r.get(k) or "") for k in ("Title (sheet)","Chain name (exact case)","GANg","Type","SubType","Essence")).lower()
    return w if (w.lower() in blob and w.lower() in SPECIFIC) else None
pair_of = collections.defaultdict(list)
for p in pairs:
    if str(p.get("status", "")).startswith("withdrawn"): continue
    a, b = p["members"]; pair_of[a].append((b, p["id"], p["reason"])); pair_of[b].append((a, p["id"], p["reason"]))

def diff_text(g):
    s = deck.get(g, {}).get("G: Chain contradicts sheet") or ""
    m = re.match(r"\s*([^:]+):\s*sheet='([^']*)'\s*chain='([^']*)'", s)
    if m: return f"{m.group(1)}: {m.group(2) or '(blank)'} (sheet) · {m.group(3) or '(blank)'} (chain)"
    if str(g) in M:
        v = M[str(g)]; return f"{v['field']}: {v['correct']} (sheet) · {v['corrupt']} (chain)"
    return short(s)

def evidence(tid, g, src):
    r = sheet.get(str(g), {})
    PINDAR_SAYS = {39: " Pindar's journal 003 calls this piece 'the cyclopsGAN', so the chain's GANg matches his own words and the sheet is the odd one.",
                   32: " Pindar's journal 002: '(32/128) is Type 32 (typo or important?)'.", 119: " Pindar's journal 002: '(119/128) is only Odd number of Type E (typo or important?)'."}
    if tid.startswith("corrupt_") or tid in ("chain_differs_other", "twins_rename", "sheet_chain_any"): return PF.diff_lines(g) + PINDAR_SAYS.get(g, "")
    if tid in ("chain_wrong", "sheet_wrong", "undecided_side", "substitution_art"): return PF.WITNESS[g][1] + " | " + PF.diff_lines(g)
    if tid in ("filename_square_mark", "filename_square_unmarked", "filename_smooth", "filename_primordial", "review_group"): return f"file {PF.FILENAME[g]}" + (f"; hidden-type letter {PF.HIDDENTYPE.get(g)}" if g in PF.HIDDENTYPE else "")
    if tid == "first_transfer": return f"first transfer out of Pindar's wallet {PF.FIRST_TRANSFER[g]} UTC" + (" — into the prize wallet 0x18f8…478d" if g == 445 else "")
    if tid == "description_anomaly": return "description begins: " + short(PF.DESC_ANOMALY.get(g, ""), 120)
    if tid == "no_essence_on_chain": return f"sheet essence '{PF.SHEET[g]['Essence']}'; chain record has no Essence trait"
    if tid == "named_on_chain_only": return f"chain Name: {PF.ctraits(g).get('Name')}"
    if tid == "trait_key_misspelled": return "chain trait keys: " + ", ".join(PF.TRAIT_KEY_ANOMALY.get(g, []))
    if tid == "no_chain_record": return f"sheet counter reads '{PF.SHEET[g]['Number']}'"
    if tid == "name_stray": return f"chain name: {PF.CNAME.get(g)}"
    if tid == "name_typo": return PF.diff_lines(g)
    if tid == "trait_value_typo": return "; ".join(PF.TRAIT_VALUE_TYPO[g])
    if tid == "filename_typo": return PF.FILENAME_TYPO[g]
    if tid == "essence_not_plain": return f"essence written '{PF.ESSENCE_NOT_PLAIN[g]}'"
    if tid in ("point_release", "solo_session"): return PF.session_note(g)
    if tid == "save_out_of_order": return f"saved {PF.SAVE_TIME[g].strftime('%d %b %Y %H:%M:%S')} UTC; #{g-1} saved {PF.SAVE_TIME[g-1].strftime('%H:%M:%S') if g-1 in PF.SAVE_TIME else '?'}, #{g+1} saved {PF.SAVE_TIME[g+1].strftime('%H:%M:%S') if g+1 in PF.SAVE_TIME else '?'}"
    if tid in ("essence_ex_block", "essence_own_number", "essence_neighbour_pair", "robo_last_row_break"):
        w = PF.SHEET[g]['Essence'].strip().lower(); w = {'err0r': 'error'}.get(w, w)
        return f"essence '{w}' = wordlist word {PF.bip.index(w)+1}; " + (f"place {g-300} of 144" if g > 300 else f"robo position {g-129}")
    if tid == "superrare_stroke_pointer": return {153: "Bit Mouth 11,400 (0x01): 1400 'puzzle'", 155: "Cyber Zombie 10,366 (0x02): 366 'color'", 135: "The Five Eyes 10,781 (0x03): 781 'ghost'", 51: "Skull Dot 20,051 (0x04): 51 'alien'", 105: "The Extra Terrestrial 10,105 (0x05): 105 'artist'", 172: "Ink Bot 11,498 (0x06): 1498 'robot'", 151: "Glitch Skull 11,532 (0x07): 1532 'satoshi'", 358: "Robo Bobo 10,482 (0x08): 482 'destroy'", 186: "The Twins 20,590 (0x09): 590 'endless'", 175: "The Sun Dancers 21,623 (0x0A): 1623 'skull'", 69: "Triclops 10,069 (0x0B): 69 'ancient'"}.get(g, "")
    if tid == "typed_mark_bit_mirror": return {32: "0100000 read backwards = 2", 119: "1110111 is a palindrome; complemented = 8", 161: "robo position 32 = 100000, read backwards = 1", 187: "robo position 58 = 111010, complemented = 5", 193: "spooky position 0"}.get(g, "")
    if tid == "essence_solution_block": return f"essence '{PF.SHEET[g]['Essence']}' = wordlist word {PF.bip.index(PF.SHEET[g]['Essence'].strip().lower())+1}; Review group {PF.REVIEW_GROUP.get(g)}"
    if tid == "essence_instruction": return f"essence '{PF.SHEET[g]['Essence']}'"
    if tid == "essence_position_word": return f"essence '{PF.SHEET[g]['Essence']}'"
    if tid == "collector_call": return PF.diff_lines(g)
    if tid == "same_name_pair":
        nm = PF.SHEET[g]['Title'].split(' (')[0]; o = SAME_NAME[g]
        return "'" + nm + "' is also the name of #" + str(o) + " (essence '" + PF.SHEET[o]['Essence'] + "'); this piece's essence is '" + PF.SHEET[g]['Essence'] + "'"
    if src.startswith("fn:visual_audit:") or g in _VA_EXT.get(tid, ()): return "the collector: " + _VA_NOTES.get(g, "")[:200]
    if tid in ("season_frame_out_of_order", "season_date_mismatch", "filename_type_mismatch"): return f"file {PF.FILENAME[g]}; essence '{PF.SHEET[g]['Essence']}'" + (f"; date {PF.ctraits(g).get('Date')}" if 209 <= g <= 300 else "")
    if tid == "mint_out_of_order": return f"mint slot {PF.TI[g]}; piece #{g-1} is slot {PF.TI.get(g-1)}, piece #{g+1} is slot {PF.TI.get(g+1)}"
    if tid == "review_group_out_of_order": return f"{PF.FILENAME.get(g)}: " + REVIEW_OOO[g].format(d=PF.SAVE_TIME[g].strftime('%d %b %Y') if g in PF.SAVE_TIME else '?')
    if tid == "out_of_order": return f"saved {PF.SAVE_TIME[g].strftime('%d %b %Y %H:%M')} UTC, filename {PF.FILENAME.get(g)}; the spooky drop files were saved 15 Oct 2021 and the Ghost drop's testGANs on 6 Jan 2022. Collector's note: testGANs should sit with the other Ghosts in the Machine pieces in this drop"
    if tid == "name_bip_fused": return f"title '{PF.SHEET[g]['Title']}'; seed word fused to GAN: {', '.join(FUSED[g])}"
    if tid == "named_with_seed_word": return f"Name '{PF.SHEET[g]['Name']}'; seed word(s): {', '.join(PF.NAMED_WITH_SEED_WORD[g])}"
    if tid == "tributes": return TRIBUTE_HOW.get(g, "not validated")
    if tid == "quantum_group": return ("vocabulary in " + ", ".join(PF.QUANTUM_VOCAB[g])) if g in PF.QUANTUM_VOCAB else ("by eye: pair of qubitGAN #130 (same sine-wave flash)" if g == 133 else "by eye: the rainbow morph that previews #512's Quantum Glitch" if g in (501,502,503,504) else "storyline (see findings)")
    if tid == "promise_piece": return "journal page 002: 'Type 32 (typo or important?)'" if g == 32 else "journal page 002: '(119/128) is only Odd number of Type E (typo or important?)'"
    if tid == "hiddentype_palindrome": return (f"F on binary-palindrome position {format(PF.native_pos(g),'06b')} with palindrome name '{PF.strip_num(PF.CNAME[g])}'" if PF.DROP(g)=='robo' else f"F on spooky position 0 (0o0, binary 0000); name is not a palindrome; the other spooky binary palindromes (#199, #202, #208) are not F; 0 is Fibonacci so the letter scheme would give A")
    if tid == "name_odd_numbering": return f"sheet Number '{PF.SHEET[g]['Number']}', title '{PF.SHEET[g]['Title']}'"
    if tid == "name_whitespace" or tid == "name_case" or tid == "name_leet": return f"chain name: {PF.CNAME.get(g, PF.SHEET[g]['Title'])}"
    if tid == "name_radix": return f"title: {PF.SHEET[g]['Title']}"
    if tid == "name_species_mismatch": return f"title says {PF.title_species(PF.SHEET[g]['Title'])}, GANg is {PF.SHEET[g]['GANg']}"
    if tid == "double_mint": return f"mint slots {PF.DOUBLE_MINT[g]} both carry '{PF.CNAME.get(g)}'"
    if tid == "coburned":
        how, others = PF.burnt_with(g); return f"{how} as {', '.join('#'+str(x) for x in others)} ({PF.BURNT[g]['ts'][:10]})"
    if tid == "burnt": return f"burnt {PF.BURNT[g]['ts'][:10]} from {PF.BURNT[g]['from'][:10]}…"
    if tid == "frame_count": return f"{PF.FRAME_COUNT_ODD[g]} frames (drop standard {PF.STD_FRAMES[PF.DROP(g)]})"
    if tid == "wrong_speed": return "frame delays " + ", ".join(f"{ms} ms ×{n}" for ms, n in PF.WRONG_SPEED[g]) + f" (drop norm: every frame {PF._pat[PF.DROP(g)][0]} ms)"
    if tid == "held_frame": return f"frame {PF.HELD_FRAME[g][0]} held {PF.HELD_FRAME[g][1]} ms; every other frame 0 ms (drop norm: every frame 100 ms)"
    if tid == "byte_identity": return "identical bytes to " + ", ".join('#'+str(x) for x in PF.BYTE_IDENTICAL[g])
    if tid == "negative_frames": fr = PF.NEGATIVE_FRAMES[g]; return f"{len(fr)} colour-negative frames: {', '.join(map(str, fr))}" + (" (the same list as #231, #256 and #274)" if g in (231,256,274) else "")
    if tid == "hiddentype_misfile": return PF.HT_MISFILE[g]
    if tid == "hiddentype_palindrome": return f"HiddenType F; native position {PF.native_pos(g)} = {format(PF.native_pos(g), '06b') if PF.DROP(g)=='robo' else PF.native_pos(g)}"
    if tid == "frame_count":
        return f"{r.get('FRAME COUNT (measured)')} frames (drop standard {STD.get(pieces.get(g,{}).get('drop'))})"
    if tid == "filename_word": return f"file {PF.FILENAME[g]} in Arweave archive {PF.ARWEAVE_FOLDER[g]}; words: {', '.join(PF.FILENAME_WORDS[g])}"
    if tid == "burnt": return "burnt to the Forge wallet" + (f"; same transaction as {', '.join('#'+str(x) for x in r['Co-burned with (same tx)'])}" if r.get("Co-burned with (same tx)") else "")
    if tid == "coburned": return "same transaction as " + ", ".join("#"+str(x) for x in r.get("Co-burned with (same tx)", []))
    if tid == "creature": return creature_of(g) or ""
    if tid == "forge_pair":
        full = next((dsc for t, c, dsc in recs.get(g, []) if t == "forge_pair" or c == "forge_pair"), "")
        m = re.search(r"together with (.+?) \(burn date ([0-9-]+)\).*?became: (.+?)\s*\(", full)
        if m: return f"forged with {m.group(1).strip()} on {m.group(2)}; together they became {m.group(3).strip()}"
    if tid == "chain_hiddentype": return f"HiddenType '{PF.HIDDENTYPE.get(g)}' = {PF.ht_meaning(g)}"
    if tid == "hiddentype_meaning": return f"letter '{PF.HIDDENTYPE.get(g)}' → {PF.ht_meaning(g)} (position {PF.native_pos(g)}, {PF.native_base(g)})"
    if tid == "numclass_gan": return f"piece #{g} is {PF.numclass(g)}"
    if tid == "numclass_mint": return f"mint #{PF.TI.get(g)} is {PF.numclass(PF.TI[g]) if g in PF.TI else '?'}"
    if tid == "deliberate_pair": return " | ".join(f"with #{b} ({pid}): {short(why, 110)}" for b, pid, why in pair_of.get(g, []))
    if tid == "collector_notes": return rec_desc(g, cids=("notes",)) or rec_desc(g, types=("notes","note","michael_note"))
    if tid == "essence_out_of_sequence": return f"essence '{r.get('Essence')}' — " + rec_desc(g, types=("flag:Essence out of sequence",))
    if tid == "essence_not_a_word": return f"essence written '{PF.ESSENCE_NOT_PLAIN[g]}'"
    if tid == "pindar_index": return f"bip-0039 = {_BIP[g]} ('{PF.WL[_BIP[g]-1] if hasattr(PF,'WL') else _WL[_BIP[g]-1]}')"
    if tid == "essence_color_word": return f"essence '{r.get('Essence')}'"
    if tid in SWARM_EV: return SWARM_EV[tid](g)
    if tid == "color_speedup" and g == 457: return "the collector, 4 October 2026, in chat: 'Yes 457 speeds up' (his 27 September audit: 'color speed up?')"
    kind, _, arg = src.partition(":")
    if kind == "concept": return rec_desc(g, cids=set(arg.split(",")))
    if kind == "ledger": return rec_desc(g, types=set(arg.split(",")))
    return ""

VALUE_TAGS = {  # tags that fan out to one tag per value: tid -> (value getter, label prefix)
    "numclass_gan": (lambda g: NUMCLASS.get(sheet.get(str(g),{}).get("Global number-class (0-512)")), "Piece number is"),
    "numclass_mint": (lambda g: NUMCLASS.get(sheet.get(str(g),{}).get("Global number-class (mint)")), "Mint number is"),
    "hiddentype_meaning": (lambda g: sheet.get(str(g),{}).get("HiddenType number-class"), "Hidden-type letter means"),
    "filename_word": (lambda g: [w for w in (PF.FILENAME_WORDS.get(g) or []) if w not in ("smooth", "ancient", "elder")], "Filename word"),
    "creature": (creature_of, "Visible creature"),
}

PRIMARY = {  # tags recomputed from primary sources rather than the ledger
    "chain_hiddentype": set(PF.HIDDENTYPE), "burnt": set(PF.BURNT), "coburned": {g for g in PF.BURNT if PF.burnt_with(g)},
    "double_mint": set(PF.DOUBLE_MINT), "essence_not_a_word": set(PF.ESSENCE_NOT_PLAIN),
    "name_whitespace": set(PF.NAME_DOUBLE_SPACE), "name_case": set(PF.NAME_CASE), "name_radix": set(PF.NAME_RADIX), "name_leet": set(PF.NAME_LEET),
    "name_species_mismatch": set(PF.NAME_SPECIES_MISMATCH),
    "negative_frames": set(PF.NEGATIVE_FRAMES), "frame_count": set(PF.FRAME_COUNT_ODD), "wrong_speed": set(PF.WRONG_SPEED), "held_frame": set(PF.HELD_FRAME), "byte_identity": set(PF.BYTE_IDENTICAL),
    "hiddentype_misfile": set(PF.HT_MISFILE), "hiddentype_palindrome": set(PF.HT_PALINDROME),
    "essence_color_word": {g for g, r in PF.SHEET.items() if r['Essence'].strip().lower() in {'black','blue','brown','gold','green','olive','orange','pink','silver','yellow',
                 # widened on Michael's ruling, audit 2026-09-16: the other colour words the
                 # BIP-39 list carries. Only 'bronze' (#493) is actually used as an essence.
                 'bronze','copper','coral','cream','ivory','lemon','rose','salmon','sand','snow'}},
}
VALUE_TAGS["frame_count"] = (lambda g: (f"{PF.FRAME_COUNT_ODD[g]} frames (normal {PF.STD_FRAMES[PF.DROP(g)]})" if g in PF.FRAME_COUNT_ODD else None), "Frame count")
VALUE_TAGS["essence_instruction"] = (lambda g: _IW.get(_ess(g)), "Instruction word")
_CALLS = json.load(open(f"{SRC}/lore_catalog_runs/chain_glitch_collector_calls.json"))
_CALLW = {"yes": "on purpose", "unsure": "not sure", "no": "an accident or a habit"}
VALUE_TAGS["collector_call"] = (lambda g: _CALLW.get(_CALLS.get(str(g))), "The collector's call")
VALUE_TAGS["numclass_gan"] = (lambda g: PF.numclass(g), "Piece number is")
VALUE_TAGS["first_transfer"] = (lambda g: PF.transfer_period(g), "First left Pindar's wallet")
VALUE_TAGS["filename_primordial"] = (lambda g: PF.PRIMORDIAL_TRANSLATION.get(g), "Filename writes Primordial as")
VALUE_TAGS["review_group"] = (lambda g: PF.REVIEW_GROUP.get(g), "Review filename group")
VALUE_TAGS["numclass_mint"] = (lambda g: PF.numclass(PF.TI[g]) if g in PF.TI else None, "Mint number is")
VALUE_TAGS["hiddentype_meaning"] = (lambda g: PF.ht_meaning(g), "Hidden-type letter means")
for tid, cat, sub, label, desc, ver, src in S.TAGS:
    kind, _, arg = src.partition(":")
    if kind == "theme": continue   # built after the loop from the specific tags
    if tid in VALUE_TAGS: members = set(range(1, 513))
    elif tid == "deliberate_pair": members = set(pair_of)
    elif tid in PRIMARY: members = PRIMARY[tid]
    elif kind == "concept": members = concept_members(set(arg.split(",")))
    elif kind == "ledger": members = ledger_members(set(arg.split(",")))
    elif kind == "field": members = field_members(arg)
    elif kind == "fn" and arg.startswith("visual_audit:"): members = set(_VA[arg.split(":")[1]])
    else: members = FN[arg]
    if tid == "larger_eyes": members = set(members) | {18, 23, 25, 27, 41, 48, 66, 97, 109}
    if tid == "color_speedup": members = set(members) | {457}   # the collector, 4 October 2026: "Yes 457 speeds up" (his 27 September audit had asked "color speed up?"); added 5 October 2026 (D123)
    if tid in _VA_EXT: members = set(members) | _VA_EXT[tid]
    cid = arg.split(",")[0] if kind == "concept" else None
    for g in sorted(members):
        if tid in VALUE_TAGS:
            getter, prefix = VALUE_TAGS[tid]; v = getter(g)
            if not v: continue
            for vv in (v if isinstance(v, list) else [v]):
                full = f"{cat}:{tid}:{vv}"
                add_tag(full, cat, sub, f"{prefix}: {vv}", desc, ver, cid); assign(g, full, evidence(tid, g, src))
        else:
            full = f"{cat}:{tid}"
            add_tag(full, cat, sub, label, desc, ver, cid); assign(g, full, evidence(tid, g, src))

# ---------------- Same names (D95): one tag per shared name; the site sends a click to /names/ ----------------
_SN = json.load(open(os.path.expanduser('~/Desktop/bitgan_puzzle_work/lore_catalog_runs/same_names/site_data.json')))
for _name, _ids in _SN['views']['corrected'].items():
    _n = len(_ids); _sub = "Shared by exactly two pieces" if _n == 2 else "Shared by three or four" if _n <= 4 else "Shared by five or six" if _n <= 6 else "Shared by seven or more"
    _errs = [g for g in _ids if g in set(_SN['errors'])]
    _tid = "samename:" + _name
    add_tag(_tid, "samename", _sub, _name, f"{_n} pieces share the name '{_name}': " + ", ".join("#" + str(g) for g in _ids) + "." + (" Error piece" + ("s" if len(_errs) > 1 else "") + ": " + ", ".join("#" + str(g) for g in _errs) + "." if _errs else "") + " Click to see them side by side.", "data")
    for g in _ids:
        assign(g, _tid, "also the name of " + ", ".join("#" + str(x) + " '" + PF.SHEET[x]['Essence'] + "'" for x in _ids if x != g))

# ---------------- cross-cutting themes (D74, D79). Built from the specific tags; none of them moves. ----------------
# Each theme = a broad kind of glitch. parts = (part label, [specific tag ids], {piece: reason} for members a tag cannot express).
_I = json.load(open(os.path.expanduser('~/Desktop/bitgan_puzzle_work/lore_catalog_runs/glitch_instances.json')))
def _side(r): return 'chain' if r['odd'].startswith('chain') else 'sheet' if r['odd'] == 'sheet' else None
_DROPPED, _ADDED = {}, {}
for r in _I:
    if r['kind'] != 'delete': continue
    sd = _side(r); wrong, right = (r['chain'], r['sheet']) if sd == 'chain' else (r['sheet'], r['chain']) if sd == 'sheet' else (None, None)
    if wrong is None: continue
    (_DROPPED if len(wrong) < len(right) else _ADDED).setdefault(r['g'], []).append(f"{r['field']}: '{right}' became '{wrong}' on the {sd}")
THEMES = [
 ("out_of_sequence", "Out of sequence", [
    ("Frames", ["visual:frame_splice", "visual:fib_full_reverse", "visual:fib_run_boundary_case"], {}),
    ("Letters", ["chain:corrupt_transpose", "chain:corrupt_reverse"], {}),
    ("Trait fields", ["chain:corrupt_swap"], {}),
    ("Essences", ["wordlist:essence_out_of_sequence", "wordlist:essence_transposed", "wordlist:block_displaced"], {}),
    ("Pieces", ["visual:out_of_order", "file:review_group_out_of_order", "chain:mint_out_of_order"], {}),
    ("Files", ["file:save_out_of_order"], {})]),
 ("dropped", "Something dropped", [
    ("Letters and words", [], {g: "; ".join(v) for g, v in _DROPPED.items()}),
    ("Marks and files", ["file:filename_square_unmarked", "file:primes_missing"], {}),
    ("Wordlist", [], {21: "index 21 'action' is the one word between 1 and 224 that no piece holds; #21 carries 'bean' instead"}),
    ("The glitch itself", ["visual:glitch_name_liar"], {}),
    ("Names", ["wordlist:name_error_names"], {})]),
 ("added", "Something added or doubled", [
    ("Letters and marks", ["wordlist:name_stray", "wordlist:name_whitespace", "chain:twins_rename"], {g: "; ".join(v) for g, v in _ADDED.items()}),
    ("Frames", ["visual:frame_splice", "file:negative_frames"] + sorted(t for t in tags if t.startswith("file:frame_count:")), {}),
    ("Whole pieces", ["chain:double_mint", "file:byte_identity"], {}),
    ("Figures", ["visual:twin", "visual:entangled", "visual:merge_convergence"], {}),
    ("Labels", ["chain:named_on_chain_only"], {})]),
 ("swapped", "Swapped for something else", [
    ("Labels", ["chain:corrupt_substitute", "chain:substitution_art", "wordlist:name_species_mismatch", "chain:hiddentype_misfile", "chain:hiddentype_carried", "chain:hiddentype_chain_error", "chain:corrupt_number", "chain:description_anomaly"], {}),
    ("Art", ["visual:wrong_creature", "visual:label_art_contradiction", "visual:cross_drop_hybrid", "visual:skull_robo_hybrid", "visual:eye_type_mixing", "visual:costume", "visual:color_gradient_off", "visual:theme_breaker"], {})]),
 ("reversed", "Reversed or inverted", [
    ("Letters and names", ["chain:corrupt_reverse", "visual:naga_palindrome"], {}),
    ("Frames and colour", ["visual:fib_full_reverse", "file:negative_frames", "visual:bg_black"], {})]),
 ("recoded", "Written in another code", [
    ("Names", ["wordlist:name_leet", "wordlist:name_odd_numbering", "wordlist:name_radix"], {}),
    ("Essences and letters", ["wordlist:essence_not_a_word", "wordlist:essence_index_decoded"], {})]),
 ("retimed", "Wrong speed or wrong time", [
    ("Playback", ["file:wrong_speed", "file:held_frame", "visual:color_speedup"], {}),
    ("Making and minting", ["visual:out_of_order", "file:save_out_of_order", "chain:mint_out_of_order", "file:solo_session"], {})]),
 ("misspelled", "Misspelled or mis-capitalised", [
    ("Names and traits", ["wordlist:name_typo", "wordlist:trait_value_typo", "wordlist:name_case", "chain:corrupt_case", "chain:corrupt_transpose"], {}),
    ("Filenames", ["file:filename_typo", "file:filename_defect"], {}),
    ("Chain record", ["chain:trait_key_misspelled"], {})]),
 ("hidden", "Something hidden in the art", [
    ("Embedded in the picture", ["visual:painted_mark", "visual:hidden_element"], {}),
    ("Read from the picture", ["visual:white_zero_eyes", "visual:name_describes_visual"], {})]),
 ("colour", "Colour breaks", [
    ("Flashes and morphs", ["visual:color_flash", "visual:sine_wave_flash", "visual:quantum_glitch_prelude"], {}),
    ("Channels and palette", ["visual:channel_banding", "visual:color_gradient_off", "visual:color_speedup"], {})]),
 ("echo", "Echoes another piece", [
    ("Pairs", ["relations:deliberate_pair", "relations:day_night_pair"], {}),
    ("Copies and near copies", ["file:byte_identity", "visual:near_miss_homage", "chain:coburned"], {})]),
 ("natives", "Review Drop Natives", [
    ("New to the Review drop", ["visual:visually_novel"], {})]),
]
THEME_TAG_IDS = set()
for _key, _title, _parts in THEMES:
    _t = next(t for t in S.TAGS if t[0] == f"theme_{_key}")
    _all = {}
    for _pl, _ids, _extra in _parts:
        for _id in _ids: assert _id in tags, _id
        THEME_TAG_IDS.update(_ids)
        _pm = {}
        for (g, tid) in list(ptags):
            if tid in _ids: _pm.setdefault(g, []).append(tags[tid]['label'])
        for g, why in _extra.items(): _pm.setdefault(g, []).append(why)
        _pid = f"theme:{_key}:{_pl}"
        add_tag(_pid, "theme", _t[2], f"{_title}: {_pl.lower()}", f"One part of the '{_title}' theme. It gathers " + ("these specific tags, which stay where they are: " + "; ".join(tags[i]['label'] for i in _ids) + "." if _ids else "facts read from the list of separate sheet-versus-chain glitches.") , "note")
        for g, why in _pm.items():
            assign(g, _pid, "; ".join(why)); _all.setdefault(g, []).append(f"{_pl}: " + "; ".join(why))
    add_tag(f"theme:{_key}_all", "theme", _t[2], _t[3], _t[4], _t[5])
    for g, why in _all.items(): assign(g, f"theme:{_key}_all", " | ".join(why))
    # D81, the collector's rule: a SET of glitches is all data or all visual, never mixed. Two clean roll-ups per kind.
    _VIS_PAIRS = {489, 491, 130, 133, 165, 207}     # scanline pair, flash pair, painted-number pair: known only by eye
    _side = {"data": {}, "visual": {}}
    for _pl, _ids, _extra in _parts:
        for (g, tid) in list(ptags):
            if tid not in _ids: continue
            if tid == "relations:deliberate_pair": sd = "visual" if g in _VIS_PAIRS else "data"
            elif tid == "relations:day_night_pair": sd = "data"
            else: sd = "visual" if tid.startswith("visual:") else "data"
            _side[sd].setdefault(g, []).append(tags[tid]['label'])
        for g, why in _extra.items(): _side["data"].setdefault(g, []).append(why)
    for sd, lab, expl in (("data", "from the data", "Only the members known from the chain, the sheet, the wordlist or the file bytes."), ("visual", "by eye", "Only the members known by watching the animation.")):
        if not _side[sd]: continue
        add_tag(f"theme:{_key}:side:{sd}", "theme", _t[2], f"{_title}: {lab}", expl + " The collector's rule: when glitches are read as a set, the set is all data or all visual, never a mix. Use this tag, not the 'all kinds' tag, to build a selection.", "note")
        for g, why in _side[sd].items(): assign(g, f"theme:{_key}:side:{sd}", "; ".join(dict.fromkeys(why)))
# coverage check: glitch-like tags that sit in no theme yet (printed for the collector, never hidden)
_SKIP = ('chain:first_transfer', 'chain:hiddentype_meaning', 'file:review_group:', 'file:filename_word', 'file:filename_primordial', 'visual:creature:', 'visual:bg_', 'wordlist:pindar_index')
_GLITCHY = [t for t in tags if not t.startswith(_SKIP) and t.split(':')[0] in ('chain', 'file', 'visual', 'wordlist') and tags[t]['status'] in ('data', 'eye', 'candidate') and t not in THEME_TAG_IDS]
json.dump(sorted(_GLITCHY), open(os.path.expanduser('~/Desktop/bitgan_puzzle_work/lore_catalog_runs/tags_outside_themes.json'), 'w'), indent=1)

# tag ids an earlier schema produced and this one replaces under a new id (D68). They hold no pieces once the
# build re-assigns them, so the definitions are cleared here to keep the sidebar free of empty duplicates.
SUPERSEDED_IDS = ["number:block_displaced", "published:essence_color_word", "published:essence_index_decoded",
                  "published:essence_out_of_sequence", "published:essence_transposed", "published:pindar_index",
                  "published:essence_not_a_word"]
SUPERSEDED_IDS = [t for t in SUPERSEDED_IDS if t not in tags]
lines = ["-- AUTO-GENERATED unified taxonomy (build_tags.py, D49 schema). Re-runnable.",
         "DELETE FROM piece_tags WHERE source='auto';",
         "DELETE FROM tags WHERE auto=1;"] + ([
         "DELETE FROM piece_tags WHERE tag_id IN (" + ",".join(q(t) for t in SUPERSEDED_IDS) + ");",
         "DELETE FROM tags WHERE id IN (" + ",".join(q(t) for t in SUPERSEDED_IDS) + ");"] if SUPERSEDED_IDS else [])
for tid, t in tags.items():
    lines.append("INSERT OR REPLACE INTO tags(id,category,subcat,label,description,status,concept_id,auto) VALUES("
        f"{q(tid)},{q(t['category'])},{q(t['subcat'])},{q(t['label'])},{q(t['description'])},"
        f"{q(t['status'])},{q(t['concept_id'])},1);")
for (g, tid), ev in sorted(ptags.items()):
    lines.append(f"INSERT OR IGNORE INTO piece_tags(g,tag_id,source,evidence) VALUES({g},{q(tid)},'auto',{q(ev) if ev else 'NULL'});")
open(OUT, "w").write("\n".join(lines) + "\n")
cat = collections.Counter(t["category"] for t in tags.values())
withev = sum(1 for ev in ptags.values() if ev)
print(f"tags.sql: {len(tags)} tags, {len(ptags)} assignments ({withev} with evidence). by category: {dict(cat)}")