[{"data":1,"prerenderedAt":1011},["ShallowReactive",2],{"navigation":3,"\u002Fnews\u002Finsights\u002Fshallow-safety-alignment-refusal":357,"\u002Fnews\u002Finsights\u002Fshallow-safety-alignment-refusal-surround":713},[4,8,17,21,25,29,33,345,349,353],{"title":5,"path":6,"stem":7},"About","\u002Fabout","about",{"title":9,"path":10,"stem":11,"children":12},"Authentication","\u002Fauth","auth",[13],{"title":14,"path":15,"stem":16},"Email Confirmation","\u002Fauth\u002Fconfirmation","auth\u002Fconfirmation",{"title":18,"path":19,"stem":20},"Industry Briefs","\u002Fcase-studies","case-studies",{"title":22,"path":23,"stem":24},"Contact","\u002Fcontact","contact",{"title":26,"path":27,"stem":28},"Thinkata | Production AI Engineering","\u002F","index",{"title":30,"path":31,"stem":32},"Insights","\u002Finsights","insights",{"title":34,"path":35,"stem":36,"children":37},"News","\u002Fnews","news",[38,41,65],{"title":39,"path":35,"stem":40},"News & Insights","news\u002Findex",{"title":18,"path":42,"stem":43,"children":44},"\u002Fnews\u002Fcase-studies","news\u002Fcase-studies",[45,49,53,57,61],{"title":46,"path":47,"stem":48},"Building Secure and Scalable AI Infrastructure: Integrating with Existing Systems through Modern Cloud Frameworks","\u002Fnews\u002Fcase-studies\u002Fcloud-infrastructure-ai","news\u002Fcase-studies\u002Fcloud-infrastructure-ai",{"title":50,"path":51,"stem":52},"Making Sense of Financial Regulations: How AI Teams Can Tackle Complex Documents","\u002Fnews\u002Fcase-studies\u002Ffinancial-regulations","news\u002Fcase-studies\u002Ffinancial-regulations",{"title":54,"path":55,"stem":56},"AI-Powered Transformations in Healthcare","\u002Fnews\u002Fcase-studies\u002Fhealth-care","news\u002Fcase-studies\u002Fhealth-care",{"title":58,"path":59,"stem":60},"Generative AI in Upstream Natural Gas: Shell's Exploration Initiative","\u002Fnews\u002Fcase-studies\u002Foil-gas","news\u002Fcase-studies\u002Foil-gas",{"title":62,"path":63,"stem":64},"Optimizing Manufacturing with AI-Driven Multi-Agent Systems","\u002Fnews\u002Fcase-studies\u002Fsupply-chain-optimization","news\u002Fcase-studies\u002Fsupply-chain-optimization",{"title":30,"path":66,"stem":67,"children":68},"\u002Fnews\u002Finsights","news\u002Finsights",[69,73,77,81,85,89,93,97,101,105,109,113,117,121,125,129,133,137,141,145,149,153,157,161,165,169,173,177,181,185,189,193,197,201,205,209,213,217,221,225,229,233,237,241,245,249,253,257,261,265,269,273,277,281,285,289,293,297,301,305,309,313,317,321,325,329,333,337,341],{"title":70,"path":71,"stem":72},"Nothing Gets Deleted, Just Blurred","\u002Fnews\u002Finsights\u002Fadaptive-context-memory","news\u002Finsights\u002Fadaptive-context-memory",{"title":74,"path":75,"stem":76},"The Capability-Reliability Split in Agent Systems","\u002Fnews\u002Finsights\u002Fagent-capability-reliability-split","news\u002Finsights\u002Fagent-capability-reliability-split",{"title":78,"path":79,"stem":80},"When Code Gets Cheap, Judgment Gets Expensive","\u002Fnews\u002Finsights\u002Fagentic-coding-engineer-skills","news\u002Finsights\u002Fagentic-coding-engineer-skills",{"title":82,"path":83,"stem":84},"The Rise of AI Agents in Cyberattacks: Latest Research and Threats","\u002Fnews\u002Finsights\u002Fai-agent-cyber-threats","news\u002Finsights\u002Fai-agent-cyber-threats",{"title":86,"path":87,"stem":88},"Winning the Bid Is Not the Same as Knowing the Price","\u002Fnews\u002Finsights\u002Fai-agent-economics","news\u002Finsights\u002Fai-agent-economics",{"title":90,"path":91,"stem":92},"The Smart Enterprise AI Stack: Why Teams of AI Agents Beat Solo Models Consistently","\u002Fnews\u002Finsights\u002Fai-architecture","news\u002Finsights\u002Fai-architecture",{"title":94,"path":95,"stem":96},"When Seeing Everything Becomes the Only Option","\u002Fnews\u002Finsights\u002Fai-comprehensive-observability","news\u002Finsights\u002Fai-comprehensive-observability",{"title":98,"path":99,"stem":100},"The Data Infrastructure AI-Native Systems Can't Ignore","\u002Fnews\u002Finsights\u002Fai-data-layer","news\u002Finsights\u002Fai-data-layer",{"title":102,"path":103,"stem":104},"Enterprise AI Triage Systems: Intelligent Automation for Large-Scale Operations","\u002Fnews\u002Finsights\u002Fai-enterprise-triage","news\u002Finsights\u002Fai-enterprise-triage",{"title":106,"path":107,"stem":108},"When Oversight Becomes Infrastructure","\u002Fnews\u002Finsights\u002Fai-governed-autonomy","news\u002Finsights\u002Fai-governed-autonomy",{"title":110,"path":111,"stem":112},"Designing for Graceful Failure in Compound AI Systems","\u002Fnews\u002Finsights\u002Fai-graceful-failure","news\u002Finsights\u002Fai-graceful-failure",{"title":114,"path":115,"stem":116},"Intelligent Composability: Building AI Systems Like Orchestra, Not Soloists","\u002Fnews\u002Finsights\u002Fai-intelligent-composability","news\u002Finsights\u002Fai-intelligent-composability",{"title":118,"path":119,"stem":120},"Building the Plane While Flying It — Migrating from Monolith to AI-Native Without Stopping","\u002Fnews\u002Finsights\u002Fai-migration-path","news\u002Finsights\u002Fai-migration-path",{"title":122,"path":123,"stem":124},"Stability Through Continuous Adaptation","\u002Fnews\u002Finsights\u002Fai-native-overview","news\u002Finsights\u002Fai-native-overview",{"title":126,"path":127,"stem":128},"Provable Stability: Mathematical Guarantees for Adaptive AI Systems","\u002Fnews\u002Finsights\u002Fai-provable-stability","news\u002Finsights\u002Fai-provable-stability",{"title":130,"path":131,"stem":132},"How Temperature Tuning Makes or Breaks Reinforcement Learning","\u002Fnews\u002Finsights\u002Fai-soft-actor-critic-entropy-collapse","news\u002Finsights\u002Fai-soft-actor-critic-entropy-collapse",{"title":134,"path":135,"stem":136},"Testing What Can't Be Predicted","\u002Fnews\u002Finsights\u002Fai-systems-testing","news\u002Finsights\u002Fai-systems-testing",{"title":138,"path":139,"stem":140},"Delegating the Job Is Not the Same as Delegating the Rules","\u002Fnews\u002Finsights\u002Fauthorization-drift-multi-agent","news\u002Finsights\u002Fauthorization-drift-multi-agent",{"title":142,"path":143,"stem":144},"Closing the Loop: How Human Corrections Can Make AI Systems Smarter Over Time","\u002Fnews\u002Finsights\u002Fclosing-the-loop","news\u002Finsights\u002Fclosing-the-loop",{"title":146,"path":147,"stem":148},"A Panel Can Be Safer Than Any Member of It","\u002Fnews\u002Finsights\u002Fcoalitional-alignment-panel-review","news\u002Finsights\u002Fcoalitional-alignment-panel-review",{"title":150,"path":151,"stem":152},"Multi-Path Reasoning: Collaborative and Competitive Approaches in AI","\u002Fnews\u002Finsights\u002Fcollaborative-competitive-agents","news\u002Finsights\u002Fcollaborative-competitive-agents",{"title":154,"path":155,"stem":156},"Why Challenges Supercharge Smarts for Humans and AI","\u002Fnews\u002Finsights\u002Fcompetition-improves-ai","news\u002Finsights\u002Fcompetition-improves-ai",{"title":158,"path":159,"stem":160},"Context is Infrastructure, Not Instructions","\u002Fnews\u002Finsights\u002Fcontext-is-infrastructure","news\u002Finsights\u002Fcontext-is-infrastructure",{"title":162,"path":163,"stem":164},"Context is the New Code","\u002Fnews\u002Finsights\u002Fcontext-is-new-code","news\u002Finsights\u002Fcontext-is-new-code",{"title":166,"path":167,"stem":168},"Continuous Thought Machines","\u002Fnews\u002Finsights\u002Fcontinuous-thought-machines","news\u002Finsights\u002Fcontinuous-thought-machines",{"title":170,"path":171,"stem":172},"Don't Vibe, Architect","\u002Fnews\u002Finsights\u002Fdont-vibe-architect","news\u002Finsights\u002Fdont-vibe-architect",{"title":174,"path":175,"stem":176},"The Edge of the Underdefined","\u002Fnews\u002Finsights\u002Fedge-of-the-underdefined","news\u002Finsights\u002Fedge-of-the-underdefined",{"title":178,"path":179,"stem":180},"Experts All the Way Down","\u002Fnews\u002Finsights\u002Fexperts-all-the-way","news\u002Finsights\u002Fexperts-all-the-way",{"title":182,"path":183,"stem":184},"A Multi-Tier Safety Architecture for Critical Applications","\u002Fnews\u002Finsights\u002Ffour-tier-architecture","news\u002Finsights\u002Ffour-tier-architecture",{"title":186,"path":187,"stem":188},"Green Dashboard, Unhappy Users","\u002Fnews\u002Finsights\u002Fgreen-dashboard-unhappy-users","news\u002Finsights\u002Fgreen-dashboard-unhappy-users",{"title":190,"path":191,"stem":192},"Hybrid Autoregressive Residual Tokens","\u002Fnews\u002Finsights\u002Fhart-model","news\u002Finsights\u002Fhart-model",{"title":194,"path":195,"stem":196},"Hierarchical Reasoning in Artificial Intelligence","\u002Fnews\u002Finsights\u002Fhierarchical-approaches","news\u002Finsights\u002Fhierarchical-approaches",{"title":198,"path":199,"stem":200},"Taking Turns Instead of Splitting the Difference","\u002Fnews\u002Finsights\u002Finterleaved-distillation-rl","news\u002Finsights\u002Finterleaved-distillation-rl",{"title":202,"path":203,"stem":204},"Latent Diffusion for Language Generation: A Comprehensive Overview","\u002Fnews\u002Finsights\u002Flatent-diffusion-for-language","news\u002Finsights\u002Flatent-diffusion-for-language",{"title":206,"path":207,"stem":208},"Breaking Language Barriers: How AI Can Translate Without Examples","\u002Fnews\u002Finsights\u002Flearning-languages","news\u002Finsights\u002Flearning-languages",{"title":210,"path":211,"stem":212},"The Emergence of AI Deception: How Large Language Models Have Learned to Strategically Mislead Users","\u002Fnews\u002Finsights\u002Fllm-deception","news\u002Finsights\u002Fllm-deception",{"title":214,"path":215,"stem":216},"Grading on a Shared Curve","\u002Fnews\u002Finsights\u002Fllm-judge-correlated-errors","news\u002Finsights\u002Fllm-judge-correlated-errors",{"title":218,"path":219,"stem":220},"Synergizing Specialized Reasoning and General Capabilities in AI","\u002Fnews\u002Finsights\u002Fllm-reasoning-advances","news\u002Finsights\u002Fllm-reasoning-advances",{"title":222,"path":223,"stem":224},"The Expensive Default","\u002Fnews\u002Finsights\u002Fllm-routing-cost-quality","news\u002Finsights\u002Fllm-routing-cost-quality",{"title":226,"path":227,"stem":228},"The AI That Rewrites Itself: MIT's Breakthrough in Self-Adapting Language Models","\u002Fnews\u002Finsights\u002Fllm-seal","news\u002Finsights\u002Fllm-seal",{"title":230,"path":231,"stem":232},"Forgetting on Request","\u002Fnews\u002Finsights\u002Fmachine-unlearning-removal-suppression","news\u002Finsights\u002Fmachine-unlearning-removal-suppression",{"title":234,"path":235,"stem":236},"The Map and the Walk","\u002Fnews\u002Finsights\u002Fmap-and-walk","news\u002Finsights\u002Fmap-and-walk",{"title":238,"path":239,"stem":240},"Metacognitive Reinforcement Learning for Self-Improving AI Systems","\u002Fnews\u002Finsights\u002Fmetacognitive-reinforcement-learning","news\u002Finsights\u002Fmetacognitive-reinforcement-learning",{"title":242,"path":243,"stem":244},"Revolutionary Advancements in Mixture of Experts (MoE) Architectures","\u002Fnews\u002Finsights\u002Fmixture-of-experts","news\u002Finsights\u002Fmixture-of-experts",{"title":246,"path":247,"stem":248},"One Model, Many Customers, and the Leak Nobody Tests For","\u002Fnews\u002Finsights\u002Fmulti-tenant-ai-isolation","news\u002Finsights\u002Fmulti-tenant-ai-isolation",{"title":250,"path":251,"stem":252},"Balancing Neural Plasticity and Stability","\u002Fnews\u002Finsights\u002Fneural-plasticity","news\u002Finsights\u002Fneural-plasticity",{"title":254,"path":255,"stem":256},"Offline RL and the Data Flywheel","\u002Fnews\u002Finsights\u002Foffline-rl-data-flywheel","news\u002Finsights\u002Foffline-rl-data-flywheel",{"title":258,"path":259,"stem":260},"Orchestration Without a Signal From the Room","\u002Fnews\u002Finsights\u002Fplanner-in-the-loop","news\u002Finsights\u002Fplanner-in-the-loop",{"title":262,"path":263,"stem":264},"Second-Guessing Has a Price","\u002Fnews\u002Finsights\u002Freasoning-budget-allocation","news\u002Finsights\u002Freasoning-budget-allocation",{"title":266,"path":267,"stem":268},"Reasoning You Can Check","\u002Fnews\u002Finsights\u002Freasoning-you-can-check","news\u002Finsights\u002Freasoning-you-can-check",{"title":270,"path":271,"stem":272},"When Optimization Optimizes Itself","\u002Fnews\u002Finsights\u002Frecursive-goodhart","news\u002Finsights\u002Frecursive-goodhart",{"title":274,"path":275,"stem":276},"Reward Design as Architecture","\u002Fnews\u002Finsights\u002Freward-design-as-architecture","news\u002Finsights\u002Freward-design-as-architecture",{"title":278,"path":279,"stem":280},"When Success Has No Author: The Temporal Credit Assignment Problem","\u002Fnews\u002Finsights\u002Frl-credit-assignment-problem","news\u002Finsights\u002Frl-credit-assignment-problem",{"title":282,"path":283,"stem":284},"Beyond Entropy Collapse: When Exploration Succeeds but Learning Fails","\u002Fnews\u002Finsights\u002Frl-optimization-gaps","news\u002Finsights\u002Frl-optimization-gaps",{"title":286,"path":287,"stem":288},"The Body Was Supposed to Be the Easy Part","\u002Fnews\u002Finsights\u002Frobot-embodiment-interface","news\u002Finsights\u002Frobot-embodiment-interface",{"title":290,"path":291,"stem":292},"The Path to Practical Confidential Computing for AI Systems","\u002Fnews\u002Finsights\u002Fsecure-ai-architectures","news\u002Finsights\u002Fsecure-ai-architectures",{"title":294,"path":295,"stem":296},"When an AI Edits Its Own Memory","\u002Fnews\u002Finsights\u002Fself-editing-agent-memory","news\u002Finsights\u002Fself-editing-agent-memory",{"title":298,"path":299,"stem":300},"The Guard at the Front Door","\u002Fnews\u002Finsights\u002Fshallow-safety-alignment-refusal","news\u002Finsights\u002Fshallow-safety-alignment-refusal",{"title":302,"path":303,"stem":304},"Guess First, Check Later","\u002Fnews\u002Finsights\u002Fspeculative-execution-pattern","news\u002Finsights\u002Fspeculative-execution-pattern",{"title":306,"path":307,"stem":308},"Spiking Neural Networks for Energy-Efficient AI","\u002Fnews\u002Finsights\u002Fspiking-neural-networks","news\u002Finsights\u002Fspiking-neural-networks",{"title":310,"path":311,"stem":312},"When Replay Is Not an Option: Streaming Q-Learning and SARSA Get a Second Look","\u002Fnews\u002Finsights\u002Fstreaming-q-learning-revival","news\u002Finsights\u002Fstreaming-q-learning-revival",{"title":314,"path":315,"stem":316},"The Turn as the Unit of Quality","\u002Fnews\u002Finsights\u002Fstructured-iteration-quality","news\u002Finsights\u002Fstructured-iteration-quality",{"title":318,"path":319,"stem":320},"AI Speech Translation: Breaking Down Language Barriers","\u002Fnews\u002Finsights\u002Fsts-performance-advances","news\u002Finsights\u002Fsts-performance-advances",{"title":322,"path":323,"stem":324},"Test-Time Training Layers: The Next Evolution in Transformer Architecture","\u002Fnews\u002Finsights\u002Ftest-time-training-layers","news\u002Finsights\u002Ftest-time-training-layers",{"title":326,"path":327,"stem":328},"The Part You Meant to Keep","\u002Fnews\u002Finsights\u002Fthe-part-you-meant-to-keep","news\u002Finsights\u002Fthe-part-you-meant-to-keep",{"title":330,"path":331,"stem":332},"Predicting the Next Second: Tile Coding as a Real-Time Prediction Engine","\u002Fnews\u002Finsights\u002Ftile-coding-real-time-prediction","news\u002Finsights\u002Ftile-coding-real-time-prediction",{"title":334,"path":335,"stem":336},"Breakthrough: Large Language Models Pass the Turing Test","\u002Fnews\u002Finsights\u002Fturing-tests","news\u002Finsights\u002Fturing-tests",{"title":338,"path":339,"stem":340},"Algorithms Used in Autonomous Fighter Jet Flight Research","\u002Fnews\u002Finsights\u002Fvenom-f16-flight-autonomy","news\u002Finsights\u002Fvenom-f16-flight-autonomy",{"title":342,"path":343,"stem":344},"Training in a World That Does Not Exist Yet","\u002Fnews\u002Finsights\u002Fworld-models-as-infrastructure","news\u002Finsights\u002Fworld-models-as-infrastructure",{"title":346,"path":347,"stem":348},"Privacy Policy","\u002Fprivacy","privacy",{"title":350,"path":351,"stem":352},"Research","\u002Fresearch","research",{"title":354,"path":355,"stem":356},"Terms of Service","\u002Fterms","terms",{"id":358,"title":298,"body":359,"date":694,"description":695,"extension":696,"image":697,"meta":698,"navigation":710,"path":299,"seo":711,"stem":300,"__hash__":712},"insights\u002Fnews\u002Finsights\u002Fshallow-safety-alignment-refusal.md",{"type":360,"value":361,"toc":680},"minimark",[362,381,391,395,398,406,410,421,424,427,437,441,451,461,464,468,483,493,515,530,533,537,547,550,554,565,568],[363,364,367,368,367,374],"div",{"className":365},[366],"page-title","\n  ",[369,370,298],"h1",{"className":371,"id":373},[372],"page-title__main","the-guard-at-the-front-door",[375,376,380],"h2",{"className":377,"id":379},[378],"page-title__sub","why-a-language-model-that-refuses-can-usually-still-comply","Why a Language Model That Refuses Can Usually Still Comply",[363,382,384,385],{"style":383},"width: 100%; padding: 2%;","\n    ",[386,387],"img",{"src":388,"alt":389,"style":390},"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1758448721205-8465cebc26af?w=1200&h=750&fit=crop&auto=format","A modern office lobby with a reception desk and elevators behind it","width: 100%; height: auto;",[392,393,394],"p",{},"A model that answers \"I can't help with that\" sounds like a model that can't. Ask a chat assistant how to make a nerve agent or write working ransomware, and the polite refusal reads as a statement about what the model is able to do. Research from the past three years suggests it is usually a statement about what the model has been trained to say first.",[392,396,397],{},"The lobby of an office tower is a useful picture. A guard at the front desk checks badges and turns visitors away, and to anyone standing at the entrance the building looks closed. The offices upstairs are still full. Someone who comes in through the loading dock, or who is already past the desk, finds every floor unguarded. Safety training, the stage where a model learns to decline harmful requests, appears to work a lot like that guard. It is very good at the front door and much weaker everywhere else.",[392,399,400,401,405],{},"An earlier Thinkata piece on ",[402,403,404],"a",{"href":231},"machine unlearning"," found that methods meant to make a model forget mostly make it quieter. Refusal raises the same question from the other side. A model that declines might be unable to help, or it might just be unwilling at that moment. The difference matters a great deal to anyone deciding how much to trust it.",[375,407,409],{"id":408},"a-guard-who-only-watches-the-door","A Guard Who Only Watches the Door",[392,411,412,413,420],{},"A language model writes its reply one token at a time. A token is a word or a piece of a word, and each new one is chosen based on everything written so far. To find where safety training actually changes that process, a team from Princeton and Google DeepMind compared a safety-trained model with the base model it started from ",[414,415,416],"sup",{},[402,417,419],{"href":418},"#source-1","[1]",". The differences were concentrated in the first few tokens of the reply. After that, the two models behaved much alike.",[392,422,423],{},"One experiment in the paper makes the point vividly. The researchers took a base model that had never had any safety training and simply forced its reply to begin with the words \"I cannot.\" That one change took it from giving harmful answers most of the time to almost never, close to the safety-trained version. A large share of what looks like learned judgment, in other words, can be reproduced by getting the opening words right.",[392,425,426],{},"The reverse works too. In a prefilling attack, the attacker writes the first few words of the reply, something like \"Sure, here is how,\" and lets the model continue. Some model providers allow this directly, and anyone running an open model can do it trivially. The more harmful words were placed at the start, the more often the safety-trained model carried on as if it had agreed. The paper received an outstanding paper award at ICLR 2025. It calls the pattern shallow safety alignment, and much of the follow-up work has adopted the term.",[392,428,429,430,436],{},"Why would training settle on the doorway? A 2026 preprint from the University of Southampton offers an explanation rooted in how these models generate text ",[414,431,432],{},[402,433,435],{"href":434},"#source-2","[2]",". Once a reply has started down a path, the model strongly tends to keep going along it, a property the authors call autoregressive consistency. A refusal that opens with \"I'm sorry, but\" almost writes itself from there. Training has little left to correct in the later words, so its changes land on the opening ones. On this account, the shallowness is less an accident of particular datasets than a side effect of next-word prediction itself. The argument rests on a theorem with stated assumptions and on tests of a few open models, and it has not yet been peer reviewed.",[375,438,440],{"id":439},"one-switch-on-the-wall","One Switch on the Wall",[392,442,443,444,450],{},"If refusal mostly lives in the opening words, what decides which opening the model picks? Researchers looking inside thirteen open chat models, the largest with 72 billion parameters, found what looks like a single control ",[414,445,446],{},[402,447,449],{"href":448},"#source-3","[3]",". Deep in a model's internal activations, the running numerical state it carries while processing a prompt, there was one direction associated with refusing. Erasing that direction left the models answering harmful requests they had previously declined. Adding it made them refuse perfectly harmless ones. Their other abilities came through largely intact.",[392,452,453,454,460],{},"A separate study arrives at a compatible picture by removing pieces of a model rather than steering it ",[414,455,456],{},[402,457,459],{"href":458},"#source-4","[4]",". The parts that matter most for safety turned out to be small and separate from the parts that make the model useful. They made up around 3 percent of its parameters, the adjustable numbers a model learns during training. Remove them and safety breaks down while the model's usefulness barely changes.",[392,462,463],{},"In the lobby picture, the guard answers to a single switch on the wall, and the switch is not wired to the offices upstairs. That separation is what makes the arrangement fragile. A capability and the refusal to use it are stored in different places, so it is possible to touch one without disturbing the other.",[375,465,467],{"id":466},"retraining-the-guard","Retraining the Guard",[392,469,470,471,477,478,482],{},"Most people will never edit a model's internal activations. Many more can fine-tune one, which means giving it a short round of extra training on examples of their choosing, a service several model providers sell. In 2023, researchers made a safety-trained model comply with nearly any harmful instruction using just ten carefully written examples ",[414,472,473],{},[402,474,476],{"href":475},"#source-5","[5]",". Fine-tuning on them through a commercial service cost under twenty cents. More troubling for anyone offering fine-tuning as a product, training on ordinary, harmless datasets also eroded safety, less dramatically but noticeably, even when nobody intended it. The shallow alignment paper suggests a reason. Ordinary training on benign data tends to move the opening tokens most, which is exactly where the guard stands ",[414,479,480],{},[402,481,419],{"href":418},".",[392,484,485,486,492],{},"An earlier analysis framed the deeper issue as mismatched generalization ",[414,487,488],{},[402,489,491],{"href":490},"#source-6","[6]",". A model's abilities reach widely, to unfamiliar languages, encoded text, and odd phrasings, because pretraining on vast amounts of text taught them broadly. Safety training covers a much narrower range of examples. Wherever the two diverge, the ability is present and the refusal is not. The authors argued that safety measures would need to be roughly as capable as the model they protect, and that scale alone would not close the gap.",[363,494,384,496,384,500,384,506],{"style":495},"width: 100%; margin: 20px 0;",[386,497],{"src":498,"alt":499,"style":390},"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1760484376470-5e66c195978b?w=1200&auto=format&fit=crop","A long empty hallway with doors on both sides",[501,502,505],"h3",{"style":503,"id":504},"margin: 1rem 0 0.5rem 0;","guards-on-every-floor","Guards on Every Floor",[392,507,509,510,514],{"style":508},"margin: 0;","Every door along a corridor is another place to slip in, and a guard posted only in the lobby never sees most of them. The obvious repair is to train the model to recover partway through a reply, not just at the start. The shallow alignment paper did this with what it calls safety recovery examples. These are replies that begin harmfully and then turn back to a refusal. Training on them made prefilling and several other attacks far less effective ",[414,511,512],{},[402,513,419],{"href":418},". The same paper proposed a training rule that limits how far later fine-tuning can move the first few tokens, guarding the doorway itself.",[392,516,517,518,522,523,529],{},"Follow-up work suggests the corridor is longer than it first appeared. The Southampton group's random insertion attack drops a short harmful passage into the middle of an otherwise firm refusal. Models often pick up the harmful thread and continue it, even after refusing for a long stretch ",[414,519,520],{},[402,521,435],{"href":434},". A second 2026 preprint, from researchers in Seoul, found that injecting a few words at almost any point in a reply could change what the model did next ",[414,524,525],{},[402,526,528],{"href":527},"#source-7","[7]",". The effect was strongest early on, but no point in the reply was fully safe. Both groups trained models on deliberately interrupted replies and reported large drops in attack success.",[392,531,532],{},"The Seoul study adds a complication. Models whose internal state lined up strongly with the refusal direction were not, as a result, any harder to push off course. A model can look internally committed to refusing and still give in. The single switch may be a handy lever for attackers, then, without being a reliable gauge for defenders. The deeper training also made two of the models turn down harmless requests noticeably more often, a reminder that a guard on every floor costs something in friction.",[375,534,536],{"id":535},"locking-the-rooms","Locking the Rooms",[392,538,539,540,546],{},"A different line of defense leaves the refusal alone and acts on the capability. Circuit breakers, introduced in 2024, watch for the internal patterns a model produces when it starts generating harmful content, and scramble them as they appear ",[414,541,542],{},[402,543,545],{"href":544},"#source-8","[8]",". The model loses its train of thought instead of declining politely. Because the method targets what the model is doing rather than how its reply begins, a prefilled \"Sure, here is\" does not get around it in the same way. The authors report strong resistance to attacks the system was never trained against, without much loss in usefulness. In the lobby picture, this is closer to locking the rooms upstairs than to adding guards.",[392,548,549],{},"Locked rooms are still full rooms. The knowledge remains in the model, and someone with access to the weights could in principle retrain the lock away. That leads back to the unlearning question. A model that truly cannot help would have to lack the ability, and current methods struggle to show they have removed anything at all.",[375,551,553],{"id":552},"what-a-refusal-can-honestly-claim","What a Refusal Can Honestly Claim",[392,555,556,557,560,561,564],{},"From a systems perspective, the evidence points toward treating a refusal as a behavior observed under particular conditions, rather than as a fixed property of the model. A claim that a model declines harmful requests is only as strong as the list of access routes it was tested against. A closed model behind an interface that blocks prefilled replies and custom fine-tuning faces far fewer attacks than the same model with those features turned on. An open model, whose weights anyone can download, faces all of them, including direct removal of the refusal direction. Decisions about which features to expose are, in effect, safety decisions. Earlier Thinkata pieces on ",[402,558,559],{"href":107},"enforcement that sits outside the model"," and on ",[402,562,563],{"href":247},"isolation between customers"," make a related case for controls that do not depend on the model's own good behavior.",[392,566,567],{},"The harder question is how anyone could tell a model that will not from a model that cannot. The closest available test is to deliberately try to undo a model's safety training and then check what it can still do. Like the relearning attacks in unlearning research, that only sets a floor, since a more determined attacker might recover more. Specialists in biosecurity or cybersecurity would know better than a systems reading can how much leftover ability actually matters in their fields. The research so far does make two things fairly clear. The guard at the front desk is doing real work, and the building behind the guard is still full.",[363,569,367,573,367,576],{"className":570},[571,572],"references","mt-8",[375,574,575],{"id":571},"References",[577,578,384,584,384,602,384,614,384,626,384,638,384,649,384,660,384,670,367],"ol",{"className":579},[580,581,582,583],"list-decimal","list-inside","space-y-2","mt-4",[585,586,588,589,593,594],"li",{"id":587},"source-1","X. Qi et al., \"Safety Alignment Should Be Made More Than Just a Few Tokens Deep,\" in ",[590,591,592],"em",{},"Proc. International Conference on Learning Representations (ICLR'25)",", 2025, ",[402,595,601],{"href":596,"target":597,"className":598},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2406.05946","_blank",[599,600],"text-blue-600","underline","[Online]",[585,603,605,606,609,610],{"id":604},"source-2","B. Lyu et al., \"When Autoregressive Consistency Hurts Safety Alignment,\" ",[590,607,608],{},"arXiv",", 2026, ",[402,611,601],{"href":612,"target":597,"className":613},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2606.04168",[599,600],[585,615,617,618,621,622],{"id":616},"source-3","A. Arditi et al., \"Refusal in Language Models Is Mediated by a Single Direction,\" in ",[590,619,620],{},"Advances in Neural Information Processing Systems",", vol. 37, 2024, ",[402,623,601],{"href":624,"target":597,"className":625},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2406.11717",[599,600],[585,627,629,630,633,634],{"id":628},"source-4","B. Wei et al., \"Assessing the Brittleness of Safety Alignment via Pruning and Low-Rank Modifications,\" in ",[590,631,632],{},"Proc. 41st International Conference on Machine Learning (ICML'24)",", 2024, ",[402,635,601],{"href":636,"target":597,"className":637},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2402.05162",[599,600],[585,639,641,642,633,645],{"id":640},"source-5","X. Qi et al., \"Fine-tuning Aligned Language Models Compromises Safety, Even When Users Do Not Intend To!,\" in ",[590,643,644],{},"Proc. International Conference on Learning Representations (ICLR'24)",[402,646,601],{"href":647,"target":597,"className":648},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2310.03693",[599,600],[585,650,652,653,655,656],{"id":651},"source-6","A. Wei, N. Haghtalab, and J. Steinhardt, \"Jailbroken: How Does LLM Safety Training Fail?,\" in ",[590,654,620],{},", vol. 36, 2023, ",[402,657,601],{"href":658,"target":597,"className":659},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2307.02483",[599,600],[585,661,663,664,609,666],{"id":662},"source-7","K. Park and T. Kim, \"Inference-Time Vulnerability Beyond Shallow Safety: Alignment Along Generation Trajectories,\" ",[590,665,608],{},[402,667,601],{"href":668,"target":597,"className":669},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2606.04778",[599,600],[585,671,673,674,621,676],{"id":672},"source-8","A. Zou et al., \"Improving Alignment and Robustness with Circuit Breakers,\" in ",[590,675,620],{},[402,677,601],{"href":678,"target":597,"className":679},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2406.04313",[599,600],{"title":681,"searchDepth":682,"depth":682,"links":683},"",2,[684,685,686,687,691,692,693],{"id":379,"depth":682,"text":380},{"id":408,"depth":682,"text":409},{"id":439,"depth":682,"text":440},{"id":466,"depth":682,"text":467,"children":688},[689],{"id":504,"depth":690,"text":505},3,{"id":535,"depth":682,"text":536},{"id":552,"depth":682,"text":553},{"id":571,"depth":682,"text":575},"2026-11-13","A language model that declines a harmful request looks like a model that cannot help with it. Research on prefilled replies, a single internal refusal switch, and cheap fine-tuning suggests that safety training mostly shapes the first few words of a reply and leaves the underlying ability in place, and that telling a refusal from an inability is the harder problem.","md",{"src":388},{"authors":699,"badge":705,"source":707},[700],{"avatar":701,"name":703,"to":704},{"src":702},"\u002Fimg\u002Fmark_avatar.png","Mark Williams","#",{"label":706},"AI Safety",{"name":708,"url":709},"Thinkata Research","https:\u002F\u002Fthinkata.com",true,{"title":298,"description":695},"OwDhgJTHLO5IB5dU6_0UCNNGquc-PX8A4D-8pri9pbY",[714,715],null,{"id":716,"title":330,"body":717,"date":999,"description":1000,"extension":696,"image":1001,"meta":1002,"navigation":710,"path":331,"seo":1009,"stem":332,"__hash__":1010,"_path":331},"insights\u002Fnews\u002Finsights\u002Ftile-coding-real-time-prediction.md",{"type":360,"value":718,"toc":986},[719,732,738,746,754,758,761,768,775,783,787,795,798,816,820,828,836,853,857,864,877,881,888,901],[363,720,367,722,367,727],{"className":721},[366],[369,723,726],{"className":724,"id":725},[372],"predicting-the-next-second","Predicting the Next Second",[375,728,731],{"className":729,"id":730},[378],"tile-coding-as-a-real-time-prediction-engine","Tile Coding as a Real-Time Prediction Engine",[363,733,384,734],{"style":383},[386,735],{"src":736,"alt":737,"style":390},"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1531259960912-746d0c872ee6?w=1200&auto=format&fit=crop","A cyclist in motion blur on a city street",[392,739,740,741,745],{},"A cyclist weaving through traffic is not reacting to the street as it is. The body is already braced for the pothole a second ahead, already leaning into a turn that has not started yet. Psychologists call this constant, small-scale forecasting nexting, and a group of reinforcement learning researchers borrowed the word to describe a robot that learned to do something similar ",[414,742,743],{},[402,744,419],{"href":418},". Reinforcement learning is the branch of machine learning in which software improves through trial and error, and one of its quieter strengths turns out to be learning to predict.",[392,747,748,749,753],{},"What made the robot practical was not a large neural network. It was tile coding, a simple technique with roots in an older control method known as CMAC ",[414,750,751],{},[402,752,435],{"href":434},", and one that most students meet briefly before moving on to deep learning. Looking at where tile coding has actually been put to work suggests it deserves a second look, less as a way to make machines act and more as a way to help them anticipate.",[375,755,757],{"id":756},"overlapping-grids","Overlapping Grids",[392,759,760],{},"Imagine laying a sheet of graph paper over a map. Any location now falls in one square, and everything learned about that square applies to the whole of it. That is a coarse way to describe a place, since two points just either side of a line end up in different squares and share nothing.",[392,762,763,764,482],{},"Now add a second sheet, shifted a little, and a third shifted a little more. A location falls in one square on every sheet. Two nearby locations share most of their squares, two distant ones share none, and the overlap between sheets lets the system tell apart points much finer than any single square. That is tile coding. Each sheet is a tiling, each square a tile, and the set of tiles a reading lands in becomes its description ",[414,765,766],{},[402,767,449],{"href":448},[392,769,770,771,482],{},"The same trick works for any measurement, not just position on a map. A motor's current, a joint's angle or a room's temperature can all be described this way. Richard Sutton's reference software for the technique, called tiles3, is short enough to fit on a single page ",[414,772,773],{},[402,774,435],{"href":434},[392,776,777,778,782],{},"From an engineering point of view, the appeal is predictability. Every reading lights up the same small number of tiles, so each learning step does the same amount of work, and the software caps how much memory the tiles can use ",[414,779,780],{},[402,781,435],{"href":434},". Learning about one situation also leaves unrelated situations untouched, which matters a great deal for a system that has to keep learning while it runs.",[375,784,786],{"id":785},"asking-questions-about-the-future","Asking Questions About the Future",[392,788,789,790,794],{},"The second idea is about what to predict. Reinforcement learning usually estimates how much reward lies ahead. The nexting research kept that machinery but pointed it at ordinary sensor readings instead, so the robot could learn how bright its surroundings were about to get, or how hard a motor would be working over the next few seconds ",[414,791,792],{},[402,793,419],{"href":418},". A single setting controls how far ahead each question looks, from a fraction of a second to several seconds.",[392,796,797],{},"Because every question reads from the same tiles, asking one more costs very little. There is no new model to build, only one more small set of numbers to keep up to date.",[363,799,384,800,384,804,384,808],{"style":495},[386,801],{"src":802,"alt":803,"style":390},"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1678225867994-e7a5b071ebfd?w=1200&auto=format&fit=crop","A small hobby robot with exposed wiring and an ultrasonic sensor",[501,805,807],{"style":503,"id":806},"thousands-of-forecasts-on-a-laptop","Thousands of Forecasts on a Laptop",[392,809,810,811,815],{"style":508},"Every wire on a hobby robot like this one carries a signal that could be forecast, battery level, wheel effort, the distance reported by the sensor up front. The research robot in the nexting study had several dozen such signals. Its learning system made more than two thousand forecasts about them at once, from an ordinary laptop, and still finished each round of updates comfortably inside the robot's tenth-of-a-second rhythm ",[414,812,813],{},[402,814,419],{"href":418},". Most of the learning was done within about half an hour of driving around its pen.",[375,817,819],{"id":818},"what-the-robot-learned","What the Robot Learned",[392,821,822,823,827],{},"Speed alone would not make the result interesting. The researchers also compared the robot's forecasts, learned on the fly, against the best forecasts that could be worked out afterward with all the data in hand, and the live versions held up well ",[414,824,825],{},[402,826,419],{"href":418},". A single set of tiles, built once, was enough to forecast light, motion and motor signals across every time horizon tested. Nobody had to design separate inputs for each kind of signal.",[392,829,830,831,835],{},"One small detail in the paper hints at a practical use. The robot's motors had overheating protection and would stop to cool down every so often ",[414,832,833],{},[402,834,419],{"href":418},". In the experiment that was just a nuisance. For anyone responsible for motors, pumps or fans, a forecast of the next few seconds of effort and heat, learned from nothing but the machine's own sensors, starts to look like an early warning.",[363,837,384,838,384,842,384,846],{"style":495},[386,839],{"src":840,"alt":841,"style":390},"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1581090464777-f3220bbe1b8b?w=1200&h=700&auto=format&fit=crop","A robotic prosthetic hand raised against a teal background",[501,843,845],{"style":503,"id":844},"anticipating-the-next-joint","Anticipating the Next Joint",[392,847,848,849,482],{"style":508},"A powered hand like this one has more joints than its wearer has muscle signals to drive them. Many prosthetic arms controlled by muscle signals solve this with a switch that steps control from elbow to wrist to hand in a fixed order, an approach widely regarded as slow and difficult to use ",[414,850,851],{},[402,852,459],{"href":458},[375,854,856],{"id":855},"predictions-that-help-a-person","Predictions That Help a Person",[392,858,859,860,482],{},"Patrick Pilarski and colleagues at the University of Alberta asked whether the same kind of forecasting could reorder that list as a person used the arm, offering first whichever joint the person was most likely to want next. Their system tile coded the arm's position and other signals, learned its forecasts in real time on a laptop, and on a standard skills test beat even the best fixed order, one identified afterward from the same session, cutting the expected wait between joints by more than 14 percent ",[414,861,862],{},[402,863,459],{"href":458},[392,865,866,867,871,872,876],{},"Later work moved closer to everyday use. Real-time prediction learning of this kind was demonstrated with an amputee using a training robot arm ",[414,868,869],{},[402,870,476],{"href":475},", and a follow-up case series found that adaptive switching reduced both how often and how long people spent switching, for an amputee and for able-bodied participants ",[414,873,874],{},[402,875,491],{"href":490},". These are small studies, and clinicians are far better placed to judge what they mean for people who wear prostheses every day. From a machine learning perspective, the notable part is that the system learned from each person's own signals while the arm was in use, with no separate training phase beforehand.",[375,878,880],{"id":879},"what-this-suggests-for-builders","What This Suggests for Builders",[392,882,883,884,482],{},"For anyone building software for physical equipment, this work points to a pattern that seems underused. A handful of sensor readings, overlapping grids, and a set of simple forecasting questions can give a machine a running sense of what is about to happen, on hardware no bigger than a laptop and at a cost that is known in advance. The tiles3 manual covers most of the practical setup, such as choosing how wide each tile should be and which measurements to combine ",[414,885,886],{},[402,887,435],{"href":434},[392,889,890,891,895,896,900],{},"The catch is that the number of tiles explodes as more measurements are combined. The nexting robot, for instance, tiled most of its sensors on their own or in small pairs rather than all together ",[414,892,893],{},[402,894,419],{"href":418},". Deciding which measurements belong together is where knowledge of the equipment enters the design. For richer, more complex data, one research team working on laser welding first used a neural network to boil the raw data down to a few key features, then handed those to the same kind of lightweight forecasting ",[414,897,898],{},[402,899,528],{"href":527},". Whether that pairing, a learned summary in front of a cheap and predictable forecaster, becomes a common recipe for small devices is still an open question. It looks like a reasonable bet for machines that need to keep learning from their own sensors without ever stopping to retrain.",[363,902,367,904,367,906],{"className":903},[571,572],[375,905,575],{"id":571},[577,907,384,909,384,920,384,931,384,942,384,953,384,964,384,975,367],{"className":908},[580,581,582,583],[585,910,911,912,915,916],{"id":587},"J. Modayil et al., \"Multi-timescale Nexting in a Reinforcement Learning Robot,\" in ",[590,913,914],{},"From Animals to Animats 12 (SAB 2012), Lecture Notes in Computer Science",", vol. 7426, Springer, 2012, pp. 299–309. DOI: ",[402,917,601],{"href":918,"target":597,"className":919},"https:\u002F\u002Fdoi.org\u002F10.1007\u002F978-3-642-33093-3_30",[599,600],[585,921,922,923,926,927],{"id":604},"R. S. Sutton, \"Tile Coding Software, Reference Manual, Version 3.0,\" ",[590,924,925],{},"Reinforcement Learning and Artificial Intelligence Lab, University of Alberta",", accessed Nov. 6, 2026, ",[402,928,601],{"href":929,"target":597,"className":930},"http:\u002F\u002Fincompleteideas.net\u002Ftiles\u002Ftiles3.html",[599,600],[585,932,933,934,937,938],{"id":616},"R. S. Sutton and A. G. Barto, ",[590,935,936],{},"Reinforcement Learning: An Introduction",", 2nd ed., Cambridge, MA, USA, MIT Press, 2018, sec. 9.5.4, p. 217. ",[402,939,601],{"href":940,"target":597,"className":941},"http:\u002F\u002Fincompleteideas.net\u002Fbook\u002FRLbook2018.pdf#page=239",[599,600],[585,943,944,945,948,949],{"id":628},"P. M. Pilarski et al., \"Dynamic switching and real-time machine learning for improved human control of assistive biomedical robots,\" in ",[590,946,947],{},"Proc. 4th IEEE RAS\u002FEMBS International Conference on Biomedical Robotics and Biomechatronics (BioRob'12)",", Rome, Italy, 2012, pp. 296–302. DOI: ",[402,950,601],{"href":951,"target":597,"className":952},"https:\u002F\u002Fdoi.org\u002F10.1109\u002FBioRob.2012.6290309",[599,600],[585,954,955,956,959,960],{"id":640},"P. M. Pilarski et al., \"Adaptive artificial limbs: a real-time approach to prediction and anticipation,\" ",[590,957,958],{},"IEEE Robotics & Automation Magazine",", vol. 20, no. 1, pp. 53–64, 2013. DOI: ",[402,961,601],{"href":962,"target":597,"className":963},"https:\u002F\u002Fdoi.org\u002F10.1109\u002FMRA.2012.2229948",[599,600],[585,965,966,967,970,971],{"id":651},"A. L. Edwards et al., \"Application of real-time machine learning to myoelectric prosthesis control: A case series in adaptive switching,\" ",[590,968,969],{},"Prosthetics and Orthotics International",", vol. 40, no. 5, pp. 573–581, 2016. DOI: ",[402,972,601],{"href":973,"target":597,"className":974},"https:\u002F\u002Fdoi.org\u002F10.1177\u002F0309364615605373",[599,600],[585,976,977,978,981,982],{"id":662},"J. Günther et al., \"Intelligent laser welding through representation, prediction, and control learning: An architecture with deep neural networks and reinforcement learning,\" ",[590,979,980],{},"Mechatronics",", vol. 34, pp. 1–11, 2016. DOI: ",[402,983,601],{"href":984,"target":597,"className":985},"https:\u002F\u002Fdoi.org\u002F10.1016\u002Fj.mechatronics.2015.09.004",[599,600],{"title":681,"searchDepth":682,"depth":682,"links":987},[988,989,990,993,996,997,998],{"id":730,"depth":682,"text":731},{"id":756,"depth":682,"text":757},{"id":785,"depth":682,"text":786,"children":991},[992],{"id":806,"depth":690,"text":807},{"id":818,"depth":682,"text":819,"children":994},[995],{"id":844,"depth":690,"text":845},{"id":855,"depth":682,"text":856},{"id":879,"depth":682,"text":880},{"id":571,"depth":682,"text":575},"2026-11-06","Tile coding is usually taught as a classroom exercise in reinforcement learning. Its most practical use may be prediction, helping a robot or a prosthetic arm learn to anticipate what happens next while it runs, on modest hardware and with no separate training phase.",{"src":736},{"authors":1003,"badge":1006,"source":1008},[1004],{"avatar":1005,"name":703,"to":704},{"src":702},{"label":1007},"Reinforcement Learning",{"name":708,"url":709},{"title":330,"description":1000},"aBKbRiZDaa0itNJxwsVAVsbUsxF-1fiWgFR524wtojw",1794600584634]