From 56a8f48b09c4773251063eb52cef6cf695944d5d Mon Sep 17 00:00:00 2001 From: WestWaters <100190545+WestWaters@users.noreply.github.com> Date: Wed, 16 Sep 2026 03:21:41 -0700 Subject: [PATCH] load: reach a VL model's layers, not just load it Fixing the loader moved the failure two lines down. pollard-probe loaded Qwen2-VL successfully and then died on `len(model.model.layers)` -- which reads as a different bug and is the same one: a vision-language model keeps its decoder under model.language_model, so every direct `model.model.layers` raises AttributeError on a model that has just loaded fine. Seven tools did this: probe, abliterate, doctor, lowbit, gptq, hf_smooth, palette. All go through pollard_load.text_layers() now, which finds the stack for either family. VL: 28 layers, first block Qwen2VLDecoderLayer, attn projections reachable One of those was a latent crash unrelated to VL: pollard_abliterate imported the helper inside main(), while abliterate() uses it at module scope -- so the tool worked when driven from the CLI and raised NameError when called as a library. Hoisted. A test walks every pollard_*.py and fails on a direct layer reach, the same way the loader test does, because the two halves of this bug are easy to fix separately and leave half-broken. 58/58 pass. --- tests/test_recipes.py | 22 ++++++++++++++++++++++ tools/pollard_abliterate.py | 11 ++++++----- tools/pollard_doctor.py | 4 ++-- tools/pollard_gptq.py | 12 ++++++------ tools/pollard_hf_smooth.py | 4 ++-- tools/pollard_lowbit.py | 4 ++-- tools/pollard_palette.py | 4 ++-- tools/pollard_probe.py | 6 +++--- 8 files changed, 45 insertions(+), 22 deletions(-) diff --git a/tests/test_recipes.py b/tests/test_recipes.py index 248cbe9..f59be1a 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -1696,6 +1696,28 @@ def test_kquant_is_not_forced_onto_a_row_length_that_cannot_hold_it(): +def test_layer_access_goes_through_text_layers(): + """Loading a VL model is half the job; reaching its layers is the other half. + + A vision-language model keeps its decoder under model.language_model, so `model.model.layers` + raises AttributeError even after the model loads fine. Fixing the LOADER alone moved the failure + two lines down -- pollard-probe loaded Qwen2-VL successfully and then died on + `len(model.model.layers)`, which looks like a different bug and is the same one. + """ + tools = pathlib.Path(__file__).resolve().parent.parent / "tools" + offenders = [] + for f in sorted(tools.glob("pollard_*.py")): + for i, line in enumerate(f.read_text(encoding="utf-8").splitlines(), 1): + if "model.model.layers" not in line: + continue + if line.lstrip().startswith(("#", "print(", "sys.exit(")) or '"' in line.split("model.model.layers")[0][-2:]: + continue # a message ABOUT the path, not a use of it + offenders.append(f"{f.name}:{i}") + assert not offenders, ("these reach layers directly and break on a VL model: " + + ", ".join(offenders)) + + + def main(): tests = [v for k, v in sorted(globals().items()) if k.startswith("test_") and callable(v)] fails = 0 diff --git a/tools/pollard_abliterate.py b/tools/pollard_abliterate.py index b046b47..4f585ae 100644 --- a/tools/pollard_abliterate.py +++ b/tools/pollard_abliterate.py @@ -30,6 +30,8 @@ import argparse, os, sys import torch +from pollard_load import load_backbone, text_layers + # tiny BENIGN placeholder sets -- only so --selftest exercises the mechanism. # NOT a refusal set; supply real contrast prompts via --harmful/--harmless. @@ -112,7 +114,7 @@ def abliterate(model, r_hat, dev, strength=-1.0): r = r_hat.to(dev).float() a = float(strength) edited = 0 - layers = model.model.layers + layers = text_layers(model) for blk in layers: for lin in (blk.self_attn.o_proj, blk.mlp.down_proj): W = lin.weight.data.float() # [D, in] @@ -157,7 +159,6 @@ def main(): sys.exit("ERROR: pass --out for the abliterated model.") from transformers import AutoTokenizer - from pollard_load import load_backbone import pollard_workspace as ws trc = ws.resolve_trust_remote_code(a.model, a.trust_remote_code) dev = a.device if (a.device != "mps" or torch.backends.mps.is_available()) else "cpu" @@ -183,11 +184,11 @@ def main(): # measure how much of the direction lives in the writers before/after (sanity). # hidden_states[j] is the OUTPUT of block j-1, so that block's o_proj produced it. - sj = min(max(j - 1, 0), len(model.model.layers) - 1) - o0 = model.model.layers[sj].self_attn.o_proj.weight.data.float() + sj = min(max(j - 1, 0), len(text_layers(model)) - 1) + o0 = text_layers(model)[sj].self_attn.o_proj.weight.data.float() before = (r_hat.to(dev).float() @ o0).norm().item() edited = abliterate(model, r_hat, dev, a.strength) - o1 = model.model.layers[sj].self_attn.o_proj.weight.data.float() + o1 = text_layers(model)[sj].self_attn.o_proj.weight.data.float() after = (r_hat.to(dev).float() @ o1).norm().item() want = "collapse to ~0" if a.strength <= -0.999 else f"scale by {1 + a.strength:.2f}x" verb = "orthogonalized" if a.strength < 0 else "amplified" diff --git a/tools/pollard_doctor.py b/tools/pollard_doctor.py index 6c6947b..f858824 100644 --- a/tools/pollard_doctor.py +++ b/tools/pollard_doctor.py @@ -34,11 +34,11 @@ def scan_outliers(source_dir, device, calib, rows, cols, thresh): """Find massive-activation input channels per layer on the fp16 source (the low-bit break predictor).""" import torch from transformers import AutoTokenizer - from pollard_load import load_backbone + from pollard_load import load_backbone, text_layers tok = AutoTokenizer.from_pretrained(source_dir) model = load_backbone(source_dir, torch.float16, device_map=device) try: - layers = model.model.layers + layers = text_layers(model) except AttributeError: print(" (risk-scan: unsupported arch -- expected model.model.layers)"); return [] amax = {} diff --git a/tools/pollard_gptq.py b/tools/pollard_gptq.py index 142f4be..ff1345a 100644 --- a/tools/pollard_gptq.py +++ b/tools/pollard_gptq.py @@ -270,7 +270,7 @@ def sequential_gptq(model, calib, dev, bits, groupsize, act_order, offload=False offload=True keeps the whole model on CPU and moves ONE block to `dev` at a time -- this is what lets a 7B (15 GB) quantize on a 16 GB GPU: peak VRAM is one block + its Hessians, never the whole model.""" - layers = model.model.layers + layers = text_layers(model) # --- capture the input to block 0 (+ the kwargs each block needs) for every sample. # A forward-PRE-hook avoids replacing the layer (so model-level attribute access like # `.attention_type` still works) and stops the pass right before block 0 runs. @@ -367,7 +367,7 @@ def main(): a = ap.parse_args() from transformers import AutoTokenizer - from pollard_load import load_backbone + from pollard_load import load_backbone, text_layers dev = a.device if (a.device != "mps" or torch.backends.mps.is_available()) else "cpu" mdev = "cpu" if a.offload else dev # where the model itself lives print(f"== pollard-gptq :: {a.model} W{a.bits}g{a.groupsize} dev={dev}" @@ -411,13 +411,13 @@ def hook(mod, inp, out): def run(method): if fp16_state is not None: model.load_state_dict(fp16_state) - lins = linear_layers(model.model.layers) # only the transformer-block linears + lins = linear_layers(text_layers(model)) # only the transformer-block linears t0 = time.time() if method in ("gptq-seq", "gptq-seq-ao"): rec = make_recipe("aggr" if a.recipe == "aggr" else "handmix", a.ablate) if a.recipe != "none" else None sequential_gptq(model, calib, dev, a.bits, a.groupsize, act_order=method.endswith("-ao"), offload=a.offload, qmode=a.qmode, - recipe=rec, nlayers=len(model.model.layers)) + recipe=rec, nlayers=len(text_layers(model))) elif method in ("gptq", "gptq-ao"): Hs = collect_hessians(lins) # {n: (Hessian, token count)} ao = (method == "gptq-ao") @@ -453,9 +453,9 @@ def run(method): SYM = {"int": float(a.bits), "ternary": 1.585, "binary": 1.0} if a.recipe != "none" and method in ("gptq-seq", "gptq-seq-ao"): rec = make_recipe("aggr" if a.recipe == "aggr" else "handmix", a.ablate) - nl = len(model.model.layers) + nl = len(text_layers(model)) qbits = 0; qw = 0 # quantized layer weights - for n, m in model.model.layers.named_modules(): + for n, m in text_layers(model).named_modules(): if isinstance(m, nn.Linear): # recover (layer_idx, subname) from the full module path idx = int(n.split(".")[0]); sub = ".".join(n.split(".")[1:]) diff --git a/tools/pollard_hf_smooth.py b/tools/pollard_hf_smooth.py index 7230215..d0c992f 100644 --- a/tools/pollard_hf_smooth.py +++ b/tools/pollard_hf_smooth.py @@ -57,11 +57,11 @@ def main(): import pollard_workspace as ws trc = ws.resolve_trust_remote_code(a.model, a.trust_remote_code) from transformers import AutoTokenizer - from pollard_load import load_backbone + from pollard_load import load_backbone, text_layers tok = AutoTokenizer.from_pretrained(a.model, trust_remote_code=trc) model = load_backbone(a.model, torch.float16, device_map=a.device, trust_remote_code=trc) try: - layers = model.model.layers + layers = text_layers(model) except AttributeError: sys.exit("ERROR: expected a decoder with model.model.layers (Llama/Qwen-style). " "Add the seam map for this architecture.") diff --git a/tools/pollard_lowbit.py b/tools/pollard_lowbit.py index ad8523e..9e94260 100644 --- a/tools/pollard_lowbit.py +++ b/tools/pollard_lowbit.py @@ -156,7 +156,7 @@ def main(): a = ap.parse_args() from transformers import AutoTokenizer - from pollard_load import load_backbone + from pollard_load import load_backbone, text_layers print(f"== pollard-lowbit :: {a.model} bits={a.bits} keep={a.keep} levels={a.levels}", flush=True) tok = AutoTokenizer.from_pretrained(a.model) model = load_backbone(a.model, torch.float16, a.device, eval_mode=False) @@ -165,7 +165,7 @@ def main(): ppl_fp16 = eval_ppl(model, test); print(f"fp16 PPL: {ppl_fp16:.4f}", flush=True) import copy; state = copy.deepcopy(model.state_dict()) - lins = linears(model.model.layers) + lins = linears(text_layers(model)) print("collecting per-channel importance ...", flush=True) imp = per_channel_importance(model, calib, lins) diff --git a/tools/pollard_palette.py b/tools/pollard_palette.py index cfc9629..930f498 100644 --- a/tools/pollard_palette.py +++ b/tools/pollard_palette.py @@ -112,7 +112,7 @@ def main(): a = ap.parse_args() from transformers import AutoTokenizer - from pollard_load import load_backbone + from pollard_load import load_backbone, text_layers dev = a.device if (a.device != "mps" or torch.backends.mps.is_available()) else "cpu" print(f"== pollard-palette :: {a.model} gs={a.groupsize} dev={dev}", flush=True) tok = AutoTokenizer.from_pretrained(a.model) @@ -122,7 +122,7 @@ def main(): ppl_fp16 = eval_ppl(model, test); print(f"fp16 PPL: {ppl_fp16:.4f}", flush=True) state = copy.deepcopy(model.state_dict()) - lins = linear_layers(model.model.layers) + lins = linear_layers(text_layers(model)) print(f"collecting Hessians for {len(lins)} tensors ...", flush=True) Hs = collect_hessians(model, calib, lins, dev) probe_calib = calib[:a.probe_chunks] diff --git a/tools/pollard_probe.py b/tools/pollard_probe.py index afc13df..3322ae1 100644 --- a/tools/pollard_probe.py +++ b/tools/pollard_probe.py @@ -71,7 +71,7 @@ def _kl_vs(model, chunks, ref_logp, dev): def _linears(model, layer, group): parent, names = GROUP_ATTR[group] - mod = getattr(model.model.layers[layer], parent) + mod = getattr(text_layers(model)[layer], parent) return [getattr(mod, n) for n in names if hasattr(mod, n)] @@ -143,13 +143,13 @@ def main(): a = ap.parse_args() from transformers import AutoTokenizer - from pollard_load import load_backbone + from pollard_load import load_backbone, text_layers dev = a.device if (a.device != "mps" or torch.backends.mps.is_available()) else "cpu" groups = [g.strip() for g in a.groups.split(",") if g.strip()] print(f"== pollard-probe :: {a.model} probe={a.probe_bits}bit dev={dev}", flush=True) tok = AutoTokenizer.from_pretrained(a.model) model = load_backbone(a.model, torch.float16, dev) - layers = len(model.model.layers) + layers = len(text_layers(model)) ch = _chunks(tok, open(a.eval, encoding="utf-8").read(), a.seqlen, a.chunks) if a.stream: