[{"data":1,"prerenderedAt":1128},["ShallowReactive",2],{"navigation":3,"\u002Fnews\u002Finsights\u002Fmulti-tenant-ai-isolation":293,"\u002Fnews\u002Finsights\u002Fmulti-tenant-ai-isolation-surround":603},[4,8,17,21,25,29,33,37,281,285,289],{"title":5,"path":6,"stem":7},"About Thinkata Intelligence","\u002Fabout","about",{"title":9,"path":10,"stem":11,"children":12},"Authentication","\u002Fauth","auth",[13],{"title":14,"path":15,"stem":16},"Email Confirmation","\u002Fauth\u002Fconfirmation","auth\u002Fconfirmation",{"title":18,"path":19,"stem":20},"Case Studies","\u002Fcase-studies","case-studies",{"title":22,"path":23,"stem":24},"Contact Us","\u002Fcontact","contact",{"title":26,"path":27,"stem":28},"Thinkata - Advanced AI Engineering & Multi-Agent System Solutions","\u002F","index",{"title":30,"path":31,"stem":32},"Insights","\u002Finsights","insights",{"title":34,"path":35,"stem":36},"Leadership","\u002Fleadership","leadership",{"title":38,"path":39,"stem":40,"children":41},"News","\u002Fnews","news",[42,45,69],{"title":43,"path":39,"stem":44},"News & Insights","news\u002Findex",{"title":18,"path":46,"stem":47,"children":48},"\u002Fnews\u002Fcase-studies","news\u002Fcase-studies",[49,53,57,61,65],{"title":50,"path":51,"stem":52},"Building Secure and Scalable AI Infrastructure: Integrating with Existing Systems through Modern Cloud Frameworks","\u002Fnews\u002Fcase-studies\u002Fcloud-infrastructure-ai","news\u002Fcase-studies\u002Fcloud-infrastructure-ai",{"title":54,"path":55,"stem":56},"Making Sense of Financial Regulations: How AI Teams Can Tackle Complex Documents","\u002Fnews\u002Fcase-studies\u002Ffinancial-regulations","news\u002Fcase-studies\u002Ffinancial-regulations",{"title":58,"path":59,"stem":60},"AI-Powered Transformations in Healthcare","\u002Fnews\u002Fcase-studies\u002Fhealth-care","news\u002Fcase-studies\u002Fhealth-care",{"title":62,"path":63,"stem":64},"Generative AI in Upstream Natural Gas: Shell's Exploration Initiative","\u002Fnews\u002Fcase-studies\u002Foil-gas","news\u002Fcase-studies\u002Foil-gas",{"title":66,"path":67,"stem":68},"Optimizing Manufacturing with AI-Driven Multi-Agent Systems","\u002Fnews\u002Fcase-studies\u002Fsupply-chain-optimization","news\u002Fcase-studies\u002Fsupply-chain-optimization",{"title":30,"path":70,"stem":71,"children":72},"\u002Fnews\u002Finsights","news\u002Finsights",[73,77,81,85,89,93,97,101,105,109,113,117,121,125,129,133,137,141,145,149,153,157,161,165,169,173,177,181,185,189,193,197,201,205,209,213,217,221,225,229,233,237,241,245,249,253,257,261,265,269,273,277],{"title":74,"path":75,"stem":76},"The Capability-Reliability Split in Agent Systems","\u002Fnews\u002Finsights\u002Fagent-capability-reliability-split","news\u002Finsights\u002Fagent-capability-reliability-split",{"title":78,"path":79,"stem":80},"The Rise of AI Agents in Cyberattacks: Latest Research and Threats","\u002Fnews\u002Finsights\u002Fai-agent-cyber-threats","news\u002Finsights\u002Fai-agent-cyber-threats",{"title":82,"path":83,"stem":84},"The Smart Enterprise AI Stack: Why Teams of AI Agents Beat Solo Models Consistently","\u002Fnews\u002Finsights\u002Fai-architecture","news\u002Finsights\u002Fai-architecture",{"title":86,"path":87,"stem":88},"When Seeing Everything Becomes the Only Option","\u002Fnews\u002Finsights\u002Fai-comprehensive-observability","news\u002Finsights\u002Fai-comprehensive-observability",{"title":90,"path":91,"stem":92},"The Data Infrastructure AI-Native Systems Can't Ignore","\u002Fnews\u002Finsights\u002Fai-data-layer","news\u002Finsights\u002Fai-data-layer",{"title":94,"path":95,"stem":96},"Enterprise AI Triage Systems: Intelligent Automation for Large-Scale Operations","\u002Fnews\u002Finsights\u002Fai-enterprise-triage","news\u002Finsights\u002Fai-enterprise-triage",{"title":98,"path":99,"stem":100},"When Oversight Becomes Infrastructure","\u002Fnews\u002Finsights\u002Fai-governed-autonomy","news\u002Finsights\u002Fai-governed-autonomy",{"title":102,"path":103,"stem":104},"Designing for Graceful Failure in Compound AI Systems","\u002Fnews\u002Finsights\u002Fai-graceful-failure","news\u002Finsights\u002Fai-graceful-failure",{"title":106,"path":107,"stem":108},"Intelligent Composability: Building AI Systems Like Orchestra, Not Soloists","\u002Fnews\u002Finsights\u002Fai-intelligent-composability","news\u002Finsights\u002Fai-intelligent-composability",{"title":110,"path":111,"stem":112},"Building the Plane While Flying It — Migrating from Monolith to AI-Native Without Stopping","\u002Fnews\u002Finsights\u002Fai-migration-path","news\u002Finsights\u002Fai-migration-path",{"title":114,"path":115,"stem":116},"Stability Through Continuous Adaptation","\u002Fnews\u002Finsights\u002Fai-native-overview","news\u002Finsights\u002Fai-native-overview",{"title":118,"path":119,"stem":120},"Provable Stability: Mathematical Guarantees for Adaptive AI Systems","\u002Fnews\u002Finsights\u002Fai-provable-stability","news\u002Finsights\u002Fai-provable-stability",{"title":122,"path":123,"stem":124},"How Temperature Tuning Makes or Breaks Reinforcement Learning","\u002Fnews\u002Finsights\u002Fai-soft-actor-critic-entropy-collapse","news\u002Finsights\u002Fai-soft-actor-critic-entropy-collapse",{"title":126,"path":127,"stem":128},"Testing What Can't Be Predicted","\u002Fnews\u002Finsights\u002Fai-systems-testing","news\u002Finsights\u002Fai-systems-testing",{"title":130,"path":131,"stem":132},"Closing the Loop: How Human Corrections Can Make AI Systems Smarter Over Time","\u002Fnews\u002Finsights\u002Fclosing-the-loop","news\u002Finsights\u002Fclosing-the-loop",{"title":134,"path":135,"stem":136},"Multi-Path Reasoning: Collaborative and Competitive Approaches in AI","\u002Fnews\u002Finsights\u002Fcollaborative-competitive-agents","news\u002Finsights\u002Fcollaborative-competitive-agents",{"title":138,"path":139,"stem":140},"Why Challenges Supercharge Smarts for Humans and AI","\u002Fnews\u002Finsights\u002Fcompetition-improves-ai","news\u002Finsights\u002Fcompetition-improves-ai",{"title":142,"path":143,"stem":144},"Context is Infrastructure, Not Instructions","\u002Fnews\u002Finsights\u002Fcontext-is-infrastructure","news\u002Finsights\u002Fcontext-is-infrastructure",{"title":146,"path":147,"stem":148},"Context is the New Code","\u002Fnews\u002Finsights\u002Fcontext-is-new-code","news\u002Finsights\u002Fcontext-is-new-code",{"title":150,"path":151,"stem":152},"Continuous Thought Machines","\u002Fnews\u002Finsights\u002Fcontinuous-thought-machines","news\u002Finsights\u002Fcontinuous-thought-machines",{"title":154,"path":155,"stem":156},"Don't Vibe, Architect","\u002Fnews\u002Finsights\u002Fdont-vibe-architect","news\u002Finsights\u002Fdont-vibe-architect",{"title":158,"path":159,"stem":160},"The Edge of the Underdefined","\u002Fnews\u002Finsights\u002Fedge-of-the-underdefined","news\u002Finsights\u002Fedge-of-the-underdefined",{"title":162,"path":163,"stem":164},"Experts All the Way Down","\u002Fnews\u002Finsights\u002Fexperts-all-the-way","news\u002Finsights\u002Fexperts-all-the-way",{"title":166,"path":167,"stem":168},"A Multi-Tier Safety Architecture for Critical Applications","\u002Fnews\u002Finsights\u002Ffour-tier-architecture","news\u002Finsights\u002Ffour-tier-architecture",{"title":170,"path":171,"stem":172},"Green Dashboard, Unhappy Users","\u002Fnews\u002Finsights\u002Fgreen-dashboard-unhappy-users","news\u002Finsights\u002Fgreen-dashboard-unhappy-users",{"title":174,"path":175,"stem":176},"Hybrid Autoregressive Residual Tokens","\u002Fnews\u002Finsights\u002Fhart-model","news\u002Finsights\u002Fhart-model",{"title":178,"path":179,"stem":180},"Hierarchical Reasoning in Artificial Intelligence","\u002Fnews\u002Finsights\u002Fhierarchical-approaches","news\u002Finsights\u002Fhierarchical-approaches",{"title":182,"path":183,"stem":184},"Latent Diffusion for Language Generation: A Comprehensive Overview","\u002Fnews\u002Finsights\u002Flatent-diffusion-for-language","news\u002Finsights\u002Flatent-diffusion-for-language",{"title":186,"path":187,"stem":188},"Breaking Language Barriers: How AI Can Translate Without Examples","\u002Fnews\u002Finsights\u002Flearning-languages","news\u002Finsights\u002Flearning-languages",{"title":190,"path":191,"stem":192},"The Emergence of AI Deception: How Large Language Models Have Learned to Strategically Mislead Users","\u002Fnews\u002Finsights\u002Fllm-deception","news\u002Finsights\u002Fllm-deception",{"title":194,"path":195,"stem":196},"Grading on a Shared Curve","\u002Fnews\u002Finsights\u002Fllm-judge-correlated-errors","news\u002Finsights\u002Fllm-judge-correlated-errors",{"title":198,"path":199,"stem":200},"Synergizing Specialized Reasoning and General Capabilities in AI","\u002Fnews\u002Finsights\u002Fllm-reasoning-advances","news\u002Finsights\u002Fllm-reasoning-advances",{"title":202,"path":203,"stem":204},"The Expensive Default","\u002Fnews\u002Finsights\u002Fllm-routing-cost-quality","news\u002Finsights\u002Fllm-routing-cost-quality",{"title":206,"path":207,"stem":208},"The AI That Rewrites Itself: MIT's Breakthrough in Self-Adapting Language Models","\u002Fnews\u002Finsights\u002Fllm-seal","news\u002Finsights\u002Fllm-seal",{"title":210,"path":211,"stem":212},"Metacognitive Reinforcement Learning for Self-Improving AI Systems","\u002Fnews\u002Finsights\u002Fmetacognitive-reinforcement-learning","news\u002Finsights\u002Fmetacognitive-reinforcement-learning",{"title":214,"path":215,"stem":216},"Revolutionary Advancements in Mixture of Experts (MoE) Architectures","\u002Fnews\u002Finsights\u002Fmixture-of-experts","news\u002Finsights\u002Fmixture-of-experts",{"title":218,"path":219,"stem":220},"One Model, Many Customers, and the Leak Nobody Tests For","\u002Fnews\u002Finsights\u002Fmulti-tenant-ai-isolation","news\u002Finsights\u002Fmulti-tenant-ai-isolation",{"title":222,"path":223,"stem":224},"Balancing Neural Plasticity and Stability","\u002Fnews\u002Finsights\u002Fneural-plasticity","news\u002Finsights\u002Fneural-plasticity",{"title":226,"path":227,"stem":228},"Offline RL and the Data Flywheel","\u002Fnews\u002Finsights\u002Foffline-rl-data-flywheel","news\u002Finsights\u002Foffline-rl-data-flywheel",{"title":230,"path":231,"stem":232},"Reasoning You Can Check","\u002Fnews\u002Finsights\u002Freasoning-you-can-check","news\u002Finsights\u002Freasoning-you-can-check",{"title":234,"path":235,"stem":236},"When Optimization Optimizes Itself","\u002Fnews\u002Finsights\u002Frecursive-goodhart","news\u002Finsights\u002Frecursive-goodhart",{"title":238,"path":239,"stem":240},"Reward Design as Architecture","\u002Fnews\u002Finsights\u002Freward-design-as-architecture","news\u002Finsights\u002Freward-design-as-architecture",{"title":242,"path":243,"stem":244},"When Success Has No Author: The Temporal Credit Assignment Problem","\u002Fnews\u002Finsights\u002Frl-credit-assignment-problem","news\u002Finsights\u002Frl-credit-assignment-problem",{"title":246,"path":247,"stem":248},"Beyond Entropy Collapse: When Exploration Succeeds but Learning Fails","\u002Fnews\u002Finsights\u002Frl-optimization-gaps","news\u002Finsights\u002Frl-optimization-gaps",{"title":250,"path":251,"stem":252},"The Path to Practical Confidential Computing for AI Systems","\u002Fnews\u002Finsights\u002Fsecure-ai-architectures","news\u002Finsights\u002Fsecure-ai-architectures",{"title":254,"path":255,"stem":256},"Guess First, Check Later","\u002Fnews\u002Finsights\u002Fspeculative-execution-pattern","news\u002Finsights\u002Fspeculative-execution-pattern",{"title":258,"path":259,"stem":260},"Spiking Neural Networks for Energy-Efficient AI","\u002Fnews\u002Finsights\u002Fspiking-neural-networks","news\u002Finsights\u002Fspiking-neural-networks",{"title":262,"path":263,"stem":264},"The Turn as the Unit of Quality","\u002Fnews\u002Finsights\u002Fstructured-iteration-quality","news\u002Finsights\u002Fstructured-iteration-quality",{"title":266,"path":267,"stem":268},"AI Speech Translation: Breaking Down Language Barriers","\u002Fnews\u002Finsights\u002Fsts-performance-advances","news\u002Finsights\u002Fsts-performance-advances",{"title":270,"path":271,"stem":272},"Test-Time Training Layers: The Next Evolution in Transformer Architecture","\u002Fnews\u002Finsights\u002Ftest-time-training-layers","news\u002Finsights\u002Ftest-time-training-layers",{"title":274,"path":275,"stem":276},"Breakthrough: Large Language Models Pass the Turing Test","\u002Fnews\u002Finsights\u002Fturing-tests","news\u002Finsights\u002Fturing-tests",{"title":278,"path":279,"stem":280},"Training in a World That Does Not Exist Yet","\u002Fnews\u002Finsights\u002Fworld-models-as-infrastructure","news\u002Finsights\u002Fworld-models-as-infrastructure",{"title":282,"path":283,"stem":284},"Privacy Policy","\u002Fprivacy","privacy",{"title":286,"path":287,"stem":288},"Research","\u002Fresearch","research",{"title":290,"path":291,"stem":292},"Terms of Service","\u002Fterms","terms",{"id":294,"title":218,"body":295,"date":585,"description":586,"extension":587,"image":588,"meta":589,"navigation":600,"path":219,"seo":601,"stem":220,"__hash__":602},"insights\u002Fnews\u002Finsights\u002Fmulti-tenant-ai-isolation.md",{"type":296,"value":297,"toc":574},"minimark",[298,317,327,331,334,337,341,353,356,366,370,376,379,389,397,401,407,410,427,437,441,449,462,466,469],[299,300,303,304,303,310],"div",{"className":301},[302],"page-title","\n  ",[305,306,218],"h1",{"className":307,"id":309},[308],"page-title__main","one-model-many-customers-and-the-leak-nobody-tests-for",[311,312,316],"h2",{"className":313,"id":315},[314],"page-title__sub","where-cross-tenant-data-leaks-in-shared-ai-systems-and-where-the-boundary-has-to-live","Where cross-tenant data leaks in shared AI systems, and where the boundary has to live",[299,318,320,321],{"style":319},"width: 100%; padding: 2%;","\n    ",[322,323],"img",{"src":324,"alt":325,"style":326},"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1769117617809-cebd713b82bb?w=1200&auto=format&fit=crop","A long hotel hallway lined with identical guest room doors, analogous to a multi-tenant system where every customer looks safely separated until a shared mechanism, like a keycard coded for more than one room, quietly crosses the boundary","width: 100%; height: auto;",[328,329,330],"p",{},"A hotel floor looks like strong isolation. Every room has its own door, its own number, and its own guest, and nobody expects one room's contents to turn up next door. The failure that matters is rarely a door left open. It is a keycard cut for one room that quietly opens the one beside it. That is the shape of the risk in shared AI systems. The separation between customers looks structural until something built for efficiency, a shared model, a shared index, or a shared cache, turns into a keycard that opens more than one room.",[328,332,333],{},"A B2B software company ships an AI feature. It summarizes contracts, answers questions over a customer's documents, or drafts replies from a customer's history. The demo lands, the feature works, usage climbs. One test rarely makes the plan. Can customer A pull customer B's data out of the same system. The feature working and the feature keeping customers apart are two different claims, and only the first one shows up in a product demo.",[328,335,336],{},"The word tenant here means one customer's isolated slice of a shared system, the same way a hotel gives each guest a private room inside one shared building. Most B2B AI products are multi-tenant by design, because running a separate model, database, and server stack for every customer is expensive. The saving comes from sharing, and sharing is exactly where the boundary between tenants can quietly disappear. Three surfaces tend to be shared, the model weights, the retrieval layer, and the serving cache. Each one has a documented way of moving one tenant's data into another tenant's output.",[311,338,340],{"id":339},"the-model-can-memorize-one-tenant-and-recite-it-to-another","The Model Can Memorize One Tenant and Recite It to Another",[328,342,343,344,352],{},"Language models memorize. One experiment extracted hundreds of verbatim sequences from a public model using only ordinary query access, including names, phone numbers, and other personal details, and some of those strings appeared in just a single training document ",[345,346,347],"sup",{},[348,349,351],"a",{"href":350},"#source-1","[1]",". The finding that should worry anyone pooling customer data is that larger models memorized more, not less. A string does not need to be common to be recoverable.",[328,354,355],{},"The multi-tenant version of this risk shows up when a single model is fine-tuned on the combined data of every customer. Fine-tuning means continuing to train a base model on your own examples so it adapts to your domain. Do it on the pooled corpus of all tenants and the weights themselves become a shared surface. A prompt from tenant B can, at least in principle, surface a string that only tenant A ever provided, and no request-time permission check sees it happen, because the leak is baked into the parameters rather than pulled from a database.",[328,357,358,359,365],{},"There is a genuinely interesting counterpoint here. One study found that retrieval-augmented generation (RAG) can reduce a model's tendency to reproduce memorized training data, because the model leans on the freshly retrieved text instead of its own parameters ",[345,360,361],{},[348,362,364],{"href":363},"#source-2","[2]",". That is real, but it moves the risk rather than removing it. The retrieved text is the next shared surface.",[311,367,369],{"id":368},"retrieval-ranks-by-relevance-not-by-permission","Retrieval Ranks by Relevance, Not by Permission",[299,371,320,372],{"style":319},[322,373],{"src":374,"alt":375,"style":326},"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1762965119363-af950b523dca?w=1200&auto=format&fit=crop","Suitcases riding an airport baggage carousel, analogous to a retrieval step that surfaces whatever best matches the request as it comes around, without checking whether the person reaching for it is the rightful owner",[328,377,378],{},"Retrieval systems have one job, to find the most relevant text for a question, and they are good at it. Relevance is not permission. A baggage carousel sorts bags by the flight they arrived on and trusts travelers to claim the right one, which holds up until two bags look alike. A retrieval index sorts by semantic similarity, how close two pieces of text are in meaning, and by default carries no idea of who is allowed to see what. The index behind a shared RAG feature is usually one index for all tenants, kept that way for cost and for retrieval quality. The documents most similar to a question get pulled into the model's context before anything has checked whose documents they are.",[328,380,381,382,388],{},"One research group names this the relevance-authorization gap, and its measurements are stark. In a multitenant test setup, retrieval with no authorization gate leaked cross-tenant data in 98 to 100 percent of probes ",[345,383,384],{},[348,385,387],{"href":386},"#source-3","[3]",". The proposed fix is not a smarter filter placed after retrieval runs. It is a layered architecture that gates the candidate set at ingestion and at retrieval time, before the ranking step ever touches a document the requesting tenant should not see.",[328,390,391,392,396],{},"A follow-up test used a composite structured prompt, one part written to trigger retrieval and one part instructing the model to repeat the context it was given, and pulled 112 exact matches and 208 near matches from 250 prompts against a private email dataset ",[345,393,394],{},[348,395,364],{"href":363},". The common instinct, retrieve everything and then filter the results the user should not see, arrives too late. By the time the filter runs, the restricted text is already sitting in the context window.",[311,398,400],{"id":399},"the-cache-is-a-side-channel","The Cache Is a Side Channel",[299,402,320,403],{"style":319},[322,404],{"src":405,"alt":406,"style":326},"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1701187540994-054295adacc5?w=1200&auto=format&fit=crop","A wall of numbered mailboxes in a building lobby, analogous to a shared mailroom whose delivery speed for a repeated item quietly signals what a neighbor already received",[328,408,409],{},"To avoid recomputing work, serving systems store intermediate state, often the key-value cache that holds a request's attention data, and reuse it when a new request shares a prefix with an earlier one. Shared across tenants, a cache hit comes back faster than a cache miss, the way a mailroom hands over a familiar package quicker than a first-time delivery, and that timing gap is measurable from the outside. A watchful neighbor can learn what someone nearby received without ever seeing it.",[328,411,412,413,419,420,426],{},"One attack, in work now published in IEEE Transactions on Information Forensics and Security, turned that difference into a working exploit. It detected per-token cache hits at 86 percent accuracy, recovered prompt tokens at 92.3 percent, and a peeping neighbor variant identified another user's cached content above 95 percent accuracy after three trials ",[345,414,415],{},[348,416,418],{"href":417},"#source-4","[4]",". In another experiment, presented at the Network and Distributed System Security Symposium, the same idea ran against SGLang, a production serving framework, and reconstructed a stranger's entire prompt with no prior knowledge of its content at a 95 percent success rate, a figure that climbed to 99 percent once the attacker already knew the surrounding template. A case study inside that same paper recovered a health questionnaire's embedded gender, age, weight, and height fields in just 60 requests ",[345,421,422],{},[348,423,425],{"href":424},"#source-5","[5]",". None of this requires special access. The attacker just sends normal requests and measures how long the answers take.",[328,428,429,430,436],{},"This is not confined to research prototypes. An audit of live commercial APIs, presented at ICML 2025, detected global cache sharing across users in seven providers, including OpenAI, which means a fast response can carry information about a stranger's prompt ",[345,431,432],{},[348,433,435],{"href":434},"#source-6","[6]",". The optimization that makes shared inference affordable is the same one that opens the channel.",[311,438,440],{"id":439},"where-the-boundary-has-to-live","Where the Boundary Has to Live",[328,442,443,444,448],{},"The pattern across all three surfaces is the same. The boundary cannot be a filter at the end of the pipeline. It has to be enforced before the shared resource is touched. For retrieval, that means gating the candidate set before the model reads anything, not after, an approach validated earlier against ungated retrieval ",[345,445,446],{},[348,447,387],{"href":386},". For the model, pooling every customer into one fine-tune trades isolation for convenience, and per-tenant routing or per-tenant adapters keep one customer's data out of the surface another customer queries. For the cache, the same tension holds. Per-tenant cache isolation closes the timing channel entirely, but it gives up the throughput gains that shared caching exists to provide, and that tradeoff has to be sized against how sensitive the cached content is rather than defaulted away in either direction. Isolation is not free, and its cost belongs in the design, not in the incident report.",[328,450,451,452,458,459,461],{},"This is the case for carrying tenant context, the identity of the customer a request belongs to, through every layer of the system. Retrieval, model selection, cache, and logging each need to know which tenant they are serving, rather than trusting a single check at the front door. A boundary enforced at each layer is harder to build than a filter bolted onto the output, and it is close to what confidential computing work already argues for at the hardware level, where isolation is treated as a property of the whole path rather than a wrapper around it ",[345,453,454],{},[348,455,457],{"href":456},"#source-7","[7]",". The engineering effort is the difference between a feature that works and a feature that can prove it keeps customers apart. A platform should treat per-tenant isolation, model routing, and tenant context at every layer as the default rather than a retrofit, for exactly the reasons the research keeps surfacing. The related tradeoffs are discussed further in ",[348,460,250],{"href":251},".",[311,463,465],{"id":464},"what-to-actually-test","What to Actually Test",[328,467,468],{},"The useful test is adversarial and a little boring. Sit in tenant B and try to retrieve, extract, or time a path into tenant A's data. Ask the model to repeat the context it was handed. Send prompts that share long prefixes with a known tenant and watch the latency. Probe the retrieval index with a query only another tenant's documents could answer. The evidence across memorization, retrieval, and caching says these leaks are reachable with ordinary access, so the open question for any team shipping B2B AI is not whether the failure mode exists. It is how much isolation the product can afford while still hitting its latency and cost targets. That number is worth knowing before a customer's security review asks for it.",[299,470,303,474,303,477],{"className":471},[472,473],"references","mt-8",[311,475,476],{"id":472},"References",[478,479,320,485,320,503,320,515,320,527,320,539,320,551,320,563,303],"ol",{"className":480},[481,482,483,484],"list-decimal","list-inside","space-y-2","mt-4",[486,487,489,490,494,495],"li",{"id":488},"source-1","N. Carlini et al., \"Extracting Training Data from Large Language Models,\" in ",[491,492,493],"em",{},"Proc. 30th USENIX Security Symposium (USENIX Security '21)",", 2021, pp. 2633–2650. ",[348,496,502],{"href":497,"target":498,"className":499},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2012.07805","_blank",[500,501],"text-blue-600","underline","[Online]",[486,504,506,507,510,511],{"id":505},"source-2","S. Zeng et al., \"The Good and The Bad: Exploring Privacy Issues in Retrieval-Augmented Generation (RAG),\" in ",[491,508,509],{},"Findings of the Association for Computational Linguistics: ACL 2024",", 2024, ",[348,512,502],{"href":513,"target":498,"className":514},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2402.16893",[500,501],[486,516,518,519,522,523],{"id":517},"source-3","F. J. Arceo and V. P. Narsing, \"Securing the Agent: Vendor-Neutral, Multitenant Enterprise Retrieval and Tool Use,\" in ",[491,520,521],{},"Proc. ACM Conf. on AI and Agentic Systems (CAIS '26)",", 2026, ",[348,524,502],{"href":525,"target":498,"className":526},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2605.05287",[500,501],[486,528,530,531,534,535],{"id":529},"source-4","L. Song et al., \"The Early Bird Catches the Leak: Unveiling Timing Side Channels in LLM Serving Systems,\" ",[491,532,533],{},"IEEE Transactions on Information Forensics and Security",", vol. 20, pp. 11431–11446, 2025. DOI: ",[348,536,502],{"href":537,"target":498,"className":538},"https:\u002F\u002Fdoi.org\u002F10.1109\u002FTIFS.2025.3622954",[500,501],[486,540,542,543,546,547],{"id":541},"source-5","G. Wu et al., \"I Know What You Asked: Prompt Leakage via KV-Cache Sharing in Multi-Tenant LLM Serving,\" in ",[491,544,545],{},"Proc. Network and Distributed System Security Symposium (NDSS 2025)",", 2025. DOI: ",[348,548,502],{"href":549,"target":498,"className":550},"https:\u002F\u002Fdoi.org\u002F10.14722\u002Fndss.2025.241772",[500,501],[486,552,554,555,558,559],{"id":553},"source-6","C. Gu et al., \"Auditing Prompt Caching in Language Model APIs,\" in ",[491,556,557],{},"Proc. 42nd International Conference on Machine Learning (ICML 2025)",", PMLR 267, 2025, ",[348,560,502],{"href":561,"target":498,"className":562},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2502.07776",[500,501],[486,564,566,567,546,570],{"id":565},"source-7","S. Zobaed and M. A. Salehi, \"Confidential Computing Across Edge-To-Cloud for Machine Learning: A Survey Study,\" ",[491,568,569],{},"Software: Practice and Experience",[348,571,502],{"href":572,"target":498,"className":573},"https:\u002F\u002Fdoi.org\u002F10.1002\u002Fspe.3398",[500,501],{"title":575,"searchDepth":576,"depth":576,"links":577},"",2,[578,579,580,581,582,583,584],{"id":315,"depth":576,"text":316},{"id":339,"depth":576,"text":340},{"id":368,"depth":576,"text":369},{"id":399,"depth":576,"text":400},{"id":439,"depth":576,"text":440},{"id":464,"depth":576,"text":465},{"id":472,"depth":576,"text":476},"2026-07-07","Your B2B AI feature works. The test that rarely makes the plan is whether customer A can pull customer B's data out of it. A shared model, a shared retrieval index, and a shared cache each have a documented way of letting one tenant's data surface in another tenant's output, and the boundary has to live earlier than most teams put it.","md",{"src":324},{"authors":590,"badge":596,"source":598},[591],{"avatar":592,"name":594,"to":595},{"src":593},"\u002Fimg\u002Fmark_avatar.png","Mark Williams","https:\u002F\u002Fthinkata.com",{"label":597},"Multi-Tenant AI",{"name":599,"url":595},"Thinkata Research",true,{"title":218,"description":586},"v9Jm6DXc7zDxSpyin44HfrKZx3bn6JavTbUPi3SkNLc",[604,797],{"id":605,"title":202,"body":606,"date":785,"description":786,"extension":587,"image":787,"meta":788,"navigation":600,"path":203,"seo":795,"stem":204,"__hash__":796,"_path":203},"insights\u002Fnews\u002Finsights\u002Fllm-routing-cost-quality.md",{"type":296,"value":607,"toc":778},[608,620,626,629,642,646,652,660,668,672,678,686,694,698,706,717],[299,609,303,611,303,615],{"className":610},[302],[305,612,202],{"className":613,"id":614},[308],"the-expensive-default",[311,616,619],{"className":617,"id":618},[314],"llm-routing-promises-to-send-easy-questions-to-cheap-models-and-research-keeps-finding-it-reaches-for-the-expensive-one-instead","LLM routing promises to send easy questions to cheap models, and research keeps finding it reaches for the expensive one instead",[299,621,320,622],{"style":319},[322,623],{"src":624,"alt":625,"style":326},"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1744745439306-f16d00620b38?w=1200&auto=format&fit=crop","A cluster of directional signs on a single post pointing toward many different destinations, standing in for the decision a production AI system faces on every incoming request, which of several available models should handle this one",[328,627,628],{},"Every product built on a large language model eventually runs into a question that has nothing to do with prompts or fine-tuning. A request arrives, and something has to decide which model answers it. Send everything to the most capable model available and the bill grows in proportion to traffic, whether or not the traffic needed that much capability. Send everything to a cheaper model and some fraction of users get worse answers than the product could have given them.",[328,630,631,632,636,637,641],{},"The engineering answer to that tradeoff is a router, a lightweight system placed in front of the language models that reads each incoming request and assigns it to one of them. Early academic work on this problem found that many questions do not need the strongest available model at all. A quality-aware router trained to send only the harder queries to a large model reduced calls to that model by up to 40 percent with no measurable drop in response quality ",[345,633,634],{},[348,635,351],{"href":350},". A follow-up framework refined the idea using human preference data collected from model comparisons, and on one benchmark it matched 95 percent of a strong model's score while sending only 13.4 percent of requests to that model ",[345,638,639],{},[348,640,364],{"href":363},". The premise looked sound. Most requests are easier than the hardest request a product will ever see, and a router that can tell the difference should save money without anyone noticing.",[311,643,645],{"id":644},"reaching-for-the-expensive-model-anyway","Reaching for the Expensive Model Anyway",[299,647,320,648],{"style":319},[322,649],{"src":650,"alt":651,"style":326},"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1740560516658-5a94b0b715ed?w=1200&auto=format&fit=crop","A mercury thermometer planted in sand against a blue sky, its red column climbing toward the top of the scale, standing in for a routing system that keeps escalating toward its most expensive, most capable model even when a lower reading would perform just as well",[328,653,654,655,659],{},"Deployed routers turn out to behave differently from the ones described in the papers that introduced the idea. A 2026 study tracking router behavior as the allowed budget per query increases found something researchers had not previously isolated as a distinct failure. Instead of spreading requests across the available models based on difficulty, the call rate of the single most expensive model climbs steadily and eventually saturates near 100 percent, the reading pushed toward the top of the scale regardless of whether the query in front of it needed that much, even though a hindsight-optimal router on the same benchmark uses that model for fewer than 20 percent of queries under the same budget ",[345,656,657],{},[348,658,387],{"href":386},". The researchers behind that finding named it routing collapse, and traced the cause to something structural rather than a training bug. Most queries have several models bunched close together in quality, and when the top two or three candidates are nearly tied, a routing model's small prediction errors are enough to flip which one looks best. As the budget grows and more models become affordable, that instability consistently pulls the decision toward the strongest, most expensive option, because a router comparing near-ties has no reliable way to settle on the cheaper one instead.",[328,661,662,663,667],{},"The opposite failure shows up just as often. A benchmark spanning more than 400,000 queries across 21 datasets and 33 models found that current routers, even ones sold as commercial products, still fail to land on the one correct model for a meaningful slice of requests. On the subset of test queries where only one to three of the available models could answer correctly, two widely used routing methods hit just 23 to 25 percent accuracy at identifying which one ",[345,664,665],{},[348,666,418],{"href":417},". So a router can spend too much by defaulting to the strongest model when it did not need to, and separately miss the model that actually had the right answer when specificity mattered. Both problems come from the same root cause. Comparing models that perform almost identically on most requests is a harder discrimination task than comparing models with a wide quality gap, and most real traffic falls into the first category.",[311,669,671],{"id":670},"naming-the-target-instead-of-guessing","Naming the Target Instead of Guessing",[299,673,320,674],{"style":319},[322,675],{"src":676,"alt":677,"style":326},"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1629122433131-53e850a3a2ce?w=1200&auto=format&fit=crop","A round target with several arrows clustered around, but not on, the center ring, standing in for the scatter that results when a routing system is tuned through indirect proxies instead of being told exactly where the target sits",[328,679,680,681,685],{},"Most routers ask an operator to set an indirect parameter, a cost threshold or a confidence cutoff, and observe afterward what accuracy that setting happened to produce. The result lands somewhere near the outcome the operator wanted, rarely exactly on it. One router built specifically against this complaint takes the opposite approach, accepting a target accuracy as a direct input rather than something inferred from unrelated knobs ",[345,682,683],{},[348,684,425],{"href":424},". Internally it tracks how far recent decisions have drifted from the requested target and adjusts its own aggressiveness in real time, letting a single trained system serve a whole range of accuracy requirements without retraining for each one.",[328,687,688,689,693],{},"The reported results suggest the direct-target approach closes much of the gap described above. Tested against a baseline that also tried to enforce a minimum accuracy level, the target-based router met its stated floor consistently, where the baseline met it only 22 percent of the time. On one benchmark it reached within 1.3 percent of the best achievable accuracy while cutting cost by as much as 89.8 percent compared to always using the strongest model ",[345,690,691],{},[348,692,425],{"href":424},". The gain is not that the underlying models changed. It is that the system was finally asked to hold a specific, checkable commitment instead of a proxy for one.",[311,695,697],{"id":696},"where-the-discipline-has-to-live","Where the Discipline Has to Live",[328,699,700,701,705],{},"Cost and quality are not the only axis worth tracking, and treating them as the whole problem hides a third variable that shows up the moment routing decisions reach production. Two models can score almost identically on both accuracy and price and still differ enormously in how long a response takes to arrive. One documented pair produced comparable results at comparable cost with response times of 32 seconds and 262 seconds ",[345,702,703],{},[348,704,418],{"href":417},". A router optimizing on two dimensions has no way to notice that gap, and a user waiting eight times longer for an equivalent answer will notice it regardless of what the router's dashboard says about savings.",[328,707,708,709,713,714,716],{},"None of this argues against routing. It argues against treating a cost threshold as a stand-in for the outcome a product actually promises its users. A routing layer is only as trustworthy as the evaluation underneath it, the same argument that applies to any system where one model's output stands in for a judgment about quality ",[345,710,711],{},[348,712,364],{"href":363},", a point explored further in ",[348,715,194],{"href":195},". A platform doing model routing should treat the accuracy floor as the product commitment and the routing table as the implementation detail underneath it, not the other way around. The question worth asking before shipping a router is not how much it saves on average. It is what happens on the request where the cheap model was wrong and nobody was checking.",[299,718,303,720,303,722],{"className":719},[472,473],[311,721,476],{"id":472},[478,723,320,725,320,736,320,747,320,758,320,769,303],{"className":724},[481,482,483,484],[486,726,727,728,731,732],{"id":488},"D. Ding, A. Mallick, C. Wang, R. Sim, S. Mukherjee, V. Rühle, L. V. S. Lakshmanan, and A. H. Awadallah, \"Hybrid LLM: Cost-Efficient and Quality-Aware Query Routing,\" in ",[491,729,730],{},"Proc. Twelfth International Conference on Learning Representations (ICLR 2024)",", 2024. ",[348,733,502],{"href":734,"target":498,"className":735},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2404.14618",[500,501],[486,737,738,739,742,743],{"id":505},"I. Ong, A. Almahairi, V. Wu, W. Chiang, T. Wu, J. E. Gonzalez, M. W. Kadous, and I. Stoica, \"RouteLLM: Learning to Route LLMs with Preference Data,\" in ",[491,740,741],{},"Proc. International Conference on Learning Representations (ICLR 2025)",", 2025. ",[348,744,502],{"href":745,"target":498,"className":746},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2406.18665",[500,501],[486,748,749,750,753,754],{"id":517},"G. Lai and H.-J. Ye, \"When Routing Collapses: On the Degenerate Convergence of LLM Routers,\" ",[491,751,752],{},"arXiv",", 2026. ",[348,755,502],{"href":756,"target":498,"className":757},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2602.03478",[500,501],[486,759,760,761,764,765],{"id":529},"H. Li, Y. Zhang, Z. Guo, C. Wang, S. Tang, Q. Zhang, Y. Chen, B. Qi, P. Ye, L. Bai, Z. Wang, and S. Hu, \"LLMRouterBench: A Massive Benchmark and Unified Framework for LLM Routing,\" in ",[491,762,763],{},"Findings of the Association for Computational Linguistics: ACL 2026",", 2026, pp. 37733–37754. ",[348,766,502],{"href":767,"target":498,"className":768},"https:\u002F\u002Faclanthology.org\u002F2026.findings-acl.1881\u002F",[500,501],[486,770,771,772,753,774],{"id":541},"A. S. Bhatti, V. Vaddina, and D. Birru, \"PROTEUS: SLA-Aware Routing via Lagrangian RL for Multi-LLM Serving Systems,\" ",[491,773,752],{},[348,775,502],{"href":776,"target":498,"className":777},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2601.19402",[500,501],{"title":575,"searchDepth":576,"depth":576,"links":779},[780,781,782,783,784],{"id":618,"depth":576,"text":619},{"id":644,"depth":576,"text":645},{"id":670,"depth":576,"text":671},{"id":696,"depth":576,"text":697},{"id":472,"depth":576,"text":476},"2026-07-14","LLM routing promises to send easy questions to cheap models and hard questions to expensive ones. Recent research keeps finding that the systems built to do this default to the expensive model anyway, and still miss the right pick when it counts.",{"src":624},{"authors":789,"badge":792,"source":794},[790],{"avatar":791,"name":594,"to":595},{"src":593},{"label":793},"AI Infrastructure",{"name":599,"url":595},{"title":202,"description":786},"ThnI8PTU6GLgSiv9QGVXFfqAtl-64CfR7BfnMA4ryxE",{"id":798,"title":230,"body":799,"date":1116,"description":1117,"extension":587,"image":1118,"meta":1119,"navigation":600,"path":231,"seo":1126,"stem":232,"__hash__":1127,"_path":231},"insights\u002Fnews\u002Finsights\u002Freasoning-you-can-check.md",{"type":296,"value":800,"toc":1107},[801,813,819,822,825,829,841,854,862,866,873,879,887,895,899,921,937,941,951,955,967,975],[299,802,303,804,303,808],{"className":803},[302],[305,805,230],{"className":806,"id":807},[308],"reasoning-you-can-check",[311,809,812],{"className":810,"id":811},[314],"coupling-language-models-with-classical-solvers-and-where-the-trust-actually-lives","Coupling Language Models with Classical Solvers, and Where the Trust Actually Lives",[299,814,320,815],{"style":319},[322,816],{"src":817,"alt":818,"style":326},"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1501594907352-04cda38ebc29?w=1200&auto=format&fit=crop","The Golden Gate Bridge stretching across open water to the far shore, analogous to carrying a vague human question across to a machine that can answer it exactly, where the whole crossing depends on how well each end is anchored",[328,820,821],{},"A language model guesses the next word. A solver proves a fact. The first is fluent and often wrong in ways that are hard to spot. The second is rigid and, when it answers at all, answers in a way another program can double-check. The obvious move is to put them together, let the model read the messy human sentence and hand the actual working-out to a solver. A solver is an exact, rule-crunching program, the kind of thing behind a Sudoku helper or an airline scheduler, and it only ever returns an answer that provably follows from the rules it was handed. The reasoning step then comes with a guarantee the model alone cannot give. The interesting part is what that guarantee does and does not cover.",[328,823,824],{},"A bridge captures the shape of it. The far bank is the solver, fixed and dependable, the same crossing every time. The near bank is the language model, fluent and fast and willing to say almost anything. The hard part is the connection at each end, carrying the question over without dropping what it meant, and that is where the surprises hide.",[311,826,828],{"id":827},"the-pattern-and-what-it-buys","The Pattern and What It Buys",[328,830,831,832,836,837,461],{},"The dominant setup is the simplest one. Treat the solver as a tool. The model rewrites a plain-language problem as a precise specification, a solver works it out, and the model turns the result back into ordinary prose. Logic-LM is a clean instance. It sends each problem to whichever engine fits, a logic programming engine, a theorem prover, a constraint solver for puzzles with many interacting rules, or Z3, a widely used engine that also handles arithmetic. Across five reasoning datasets, the approach reported an average gain of 39.2 percent over a model prompted directly and 18.4 percent over chain-of-thought prompting, which is the practice of asking the model to write out its intermediate steps ",[345,833,834],{},[348,835,351],{"href":350},". SatLM makes the same bet through declarative prompting, parsing a question into a satisfiability problem and letting Z3 plan the actual inference, so the solver guarantees the execution is correct as long as the specification is ",[345,838,839],{},[348,840,364],{"href":363},[328,842,843,844,848,849,853],{},"The same shape appears outside pure logic. LINC translates premises and a conclusion into first-order logic, the formal language of \"for all\" and \"there exists\". A theorem prover named Prover9 handles the inference, and a vote across several translations smooths out the noise ",[345,845,846],{},[348,847,387],{"href":386},". Planning fits the mold too. LLM+P rewrites the task in PDDL, a standard planning format, then hands it to a classical planner that searches for a correct or optimal route ",[345,850,851],{},[348,852,418],{"href":417},". On its own the model could not draft even a workable plan. With the planner doing the search, it produced optimal ones. The lesson keeps repeating. The model is a good reader and a poor calculator, and the split plays to both.",[328,855,856,857,861],{},"A practical question follows immediately. What happens when the solver returns unsatisfiable, its way of reporting that no answer can possibly fit the rules, or simply runs out of time. Logic-LM is candid about this. When the generated program cannot be executed, the system falls back to chain-of-thought output, and when even that is unavailable it guesses ",[345,858,859],{},[348,860,351],{"href":350},". The guarantee, in other words, is conditional on getting a runnable, faithful specification in the first place.",[311,863,865],{"id":864},"the-trust-moves-to-the-edges","The Trust Moves to the Edges",[328,867,868,869,461],{},"The solver makes the middle step sound, which mostly relocates the problem. The remaining errors pile up at the two edges, translating in and reporting out. A study of compositional translation found that models struggle to capture the logic hidden in ordinary sentences, the every or some that is implied rather than spelled out, the exception buried in a side clause. Its answer was to parse each sentence into a dependency structure first, translate the pieces in sequence, and then use the satisfiability solver itself to compare candidate translations and throw out the inconsistent ones, topping seven reasoning benchmarks ",[345,870,871],{},[348,872,425],{"href":424},[299,874,320,875],{"style":319},[322,876],{"src":877,"alt":878,"style":326},"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1457369804613-52c61a468e7d?w=1200&auto=format&fit=crop","Several open books fanned out on a surface, analogous to the ordinary human writing a language model has read by the shelfload, in contrast with the strict solver notation it has barely seen and therefore writes unreliably",[328,880,881,882,886],{},"Stacked open books make the imbalance plain. A model has read ordinary human writing by the shelfload, and the strict notation a solver expects almost not at all. Those exact code formats show up rarely in the text a model learns from, so asking it to write them directly produces frequent mistakes, the equivalent of a comma in the wrong place that stops the whole program. NL2Logic responds by splitting the job in two. One step turns the sentence into a tidy outline of the logic that does not depend on any particular solver, and a second, purely mechanical step turns that outline into solver-ready code. The split reached 99 percent correct syntax and, dropped into Logic-LM in place of its original translator, improved downstream accuracy by 31 percent ",[345,883,884],{},[348,885,435],{"href":434},". The pattern worth noticing is that the fix is structural rather than a matter of asking the model more nicely.",[328,888,889,890,894],{},"Cleaning up the input edge still leaves the other one exposed. The reporting edge is subtler and easy to miss. One analysis breaks the pipeline into three parts, formalizing the question, deciding it, and narrating the result, and points out that prior work checked the first two and assumed the third was free. It is not. A solver can return a sound verdict with a certificate, an independently checkable proof of an unsatisfiable instance, and the model can still narrate the wrong conclusion to the user. Under prompt injection the authors flipped a verified answer while the underlying verdict stayed correct, a failure that is invisible to anyone auditing only the solver ",[345,891,892],{},[348,893,457],{"href":456},". Soundness in the engine does not imply soundness in the sentence the user reads.",[311,896,898],{"id":897},"closing-the-loop","Closing the Loop",[328,900,901,902,906,907,913,914,920],{},"If the weak link is translation, the obvious lever is feedback. The solver does not only return an answer, it returns error messages, and those can drive revision. Logic-LM already used a self-refinement step that feeds solver errors back for correction ",[345,903,904],{},[348,905,351],{"href":350},". ChatLogic puts the model at the center as a controller, translating problems into pyDatalog programs and running dedicated semantic and syntax correction modules before execution ",[345,908,909],{},[348,910,912],{"href":911},"#source-8","[8]",". The most pointed result in this direction comes from work pairing models with Answer Set Programming, a style of logic built for rules that hold until something overrides them, the way birds fly unless the bird happens to be a penguin. The system ran the clingo solver in a loop, feeding each round of complaints back for another attempt. Across six benchmarks it needed no task-specific setup and still beat the stricter logic approaches on these exception-laden problems ",[345,915,916],{},[348,917,919],{"href":918},"#source-9","[9]",". The interesting part is what drove those gains. The correction loop did most of the work, with no hand-written domain knowledge required. That speaks to a live question, whether solver feedback can help without fine-tuning, and here the loop carried it alone.",[328,922,923,924,930,931,461],{},"Training is the other route. Thought-Like-Pro has a model generate rules and facts, lets a Prolog engine derive verified reasoning paths to the target, translates only the verified paths into natural language, and fine-tunes the model to imitate them, which improved performance on out-of-distribution reasoning tasks the model had not seen ",[345,925,926],{},[348,927,929],{"href":928},"#source-10","[10]",". The solver here is a labeler that never approves an invalid step. A related concern is expressiveness, since the target language can be too narrow. LoRP, published in a peer-reviewed venue, shows that some basic first-order logic constructs cannot be stated directly in Prolog and supplies a systematic translation from first-order logic into Prolog to widen what the pipeline can represent, reporting stable gains across model architectures ",[345,932,933],{},[348,934,936],{"href":935},"#source-11","[11]",[311,938,940],{"id":939},"proving-code-and-math","Proving Code and Math",[328,942,943,944,950],{},"The strongest version of verifiable reasoning is formal proof, where a machine-checked argument leaves no room for a plausible-sounding mistake. HybridProver targets the Isabelle proof assistant and combines two styles, writing a whole proof at once and building it one small step at a time, with a helper tool called Sledgehammer filling gaps. It reached a 59.4 percent success rate on the miniF2F benchmark of competition mathematics problems, up from a previous best of 56.1 percent for Isabelle ",[345,945,946],{},[348,947,949],{"href":948},"#source-12","[12]",". The same work is honest about scope. Isabelle is often used for system verification rather than mathematics, proofs there depend on extensive project-specific context, and reported numbers on a math benchmark are not the same as proving a real codebase correct. The toolchains exist and are improving, though treating them as turnkey verification for production software would be premature.",[311,952,954],{"id":953},"what-is-settled-and-what-is-not","What Is Settled and What Is Not",[328,956,957,958,962,966],{},"The settled part is that handing the deductive step to a solver removes a real and well-documented class of failure, the model that reasons fluently to a wrong conclusion. The unsettled part is everything around that step. Ambiguous input and underspecified constraints are not fixed by a solver, since the engine faithfully executes whatever specification it is given, including a wrong one. The semantic gap between a probabilistic reader and a deterministic engine is the root problem, and the more durable responses attack it structurally, through compositional parsing, intermediate syntax trees, and verification of the translation itself, rather than through retries and voting alone ",[345,959,960],{},[348,961,425],{"href":424},[345,963,964],{},[348,965,435],{"href":434},". This is also where a parallel research line sits, the differentiable approaches such as logic tensor networks and logical neural networks that fold logic inside the model rather than calling an external engine. Whether those converge with the pipeline approach or remain a separate track is an open question, and the practical reasoning systems shipping today are overwhelmingly pipelines.",[328,968,969,970,974],{},"The upcoming LLM-Solve workshop, set for the 2026 Federated Logic Conference in Lisbon, captures the current mood. It frames the work as a two-way street. Models help non-experts build formal specifications, and solvers supply verification and structure for model-driven agents. As for production beyond research demos, the honest answer is that it is thin. Planning and scheduling look closest, given how directly they map onto classical solvers, and the engineering bill is real, covering translation latency, solver timeout policies, schema upkeep, and monitoring for the stealthy narration failures described earlier ",[345,971,972],{},[348,973,457],{"href":456},". The appeal of the idea is that it offers reasoning you can check. The work that remains is making sure the part you check is the part that matters.",[299,976,303,978,303,980],{"className":977},[472,473],[311,979,476],{"id":472},[478,981,320,983,320,994,320,1005,320,1015,320,1024,320,1033,320,1043,320,1052,320,1064,320,1075,320,1085,320,1096,303],{"className":982},[481,482,483,484],[486,984,985,986,989,990],{"id":488},"L. Pan, A. Albalak, X. Wang, and W. Y. Wang, \"Logic-LM: Empowering Large Language Models with Symbolic Solvers for Faithful Logical Reasoning,\" in ",[491,987,988],{},"Findings of the Association for Computational Linguistics (EMNLP)",", 2023. DOI: ",[348,991,502],{"href":992,"target":498,"className":993},"https:\u002F\u002Fdoi.org\u002F10.18653\u002Fv1\u002F2023.findings-emnlp.248",[500,501],[486,995,996,997,1000,1001],{"id":505},"X. Ye, Q. Chen, I. Dillig, and G. Durrett, \"SatLM: Satisfiability-Aided Language Models Using Declarative Prompting,\" in ",[491,998,999],{},"Advances in Neural Information Processing Systems (NeurIPS)",", 2023, ",[348,1002,502],{"href":1003,"target":498,"className":1004},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2305.09656",[500,501],[486,1006,1007,1008,1000,1011],{"id":517},"T. X. Olausson et al., \"LINC: A Neurosymbolic Approach for Logical Reasoning by Combining Language Models with First-Order Logic Provers,\" in ",[491,1009,1010],{},"Proceedings of the Conference on Empirical Methods in Natural Language Processing (EMNLP)",[348,1012,502],{"href":1013,"target":498,"className":1014},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2310.15164",[500,501],[486,1016,1017,1018,1000,1020],{"id":529},"B. Liu et al., \"LLM+P: Empowering Large Language Models with Optimal Planning Proficiency,\" ",[491,1019,752],{},[348,1021,502],{"href":1022,"target":498,"className":1023},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2304.11477",[500,501],[486,1025,1026,1027,510,1029],{"id":541},"H. Ryu, G. Kim, H. S. Lee, and E. Yang, \"Divide and Translate: Compositional First-Order Logic Translation and Verification for Complex Logical Reasoning,\" ",[491,1028,752],{},[348,1030,502],{"href":1031,"target":498,"className":1032},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2410.08047",[500,501],[486,1034,1035,1036,522,1039],{"id":553},"R. R. Putra, R. S. P. Basuki, Y. Cheng, and P. Gao, \"NL2Logic: AST-Guided Translation of Natural Language into First-Order Logic with Large Language Models,\" in ",[491,1037,1038],{},"Findings of the Association for Computational Linguistics (EACL)",[348,1040,502],{"href":1041,"target":498,"className":1042},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2602.13237",[500,501],[486,1044,1045,1046,522,1048],{"id":565},"Z. Huang and S. Deng, \"Analyzing the Narration Gap in LLM-Solver Loops,\" ",[491,1047,752],{},[348,1049,502],{"href":1050,"target":498,"className":1051},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2606.19588",[500,501],[486,1053,1055,1056,1059,1060],{"id":1054},"source-8","Z. Wang, J. Liu, Q. Bao, H. Rong, and J. Zhang, \"ChatLogic: Integrating Logic Programming with Large Language Models for Multi-Step Reasoning,\" in ",[491,1057,1058],{},"Proceedings of the International Joint Conference on Neural Networks (IJCNN)",", 2024. DOI: ",[348,1061,502],{"href":1062,"target":498,"className":1063},"https:\u002F\u002Fdoi.org\u002F10.1109\u002FIJCNN60899.2024.10650138",[500,501],[486,1065,1067,1068,522,1071],{"id":1066},"source-9","A. Ishay and J. Lee, \"LLMs as ASP Programmers: Self-Correction Enables Task-Agnostic Nonmonotonic Reasoning,\" in ",[491,1069,1070],{},"Findings of the Association for Computational Linguistics (ACL)",[348,1072,502],{"href":1073,"target":498,"className":1074},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2604.27960",[500,501],[486,1076,1078,1079,510,1081],{"id":1077},"source-10","X. Tan, Y. Deng, X. Qiu, W. Xu, C. Qu, W. Chu, Y. Xu, and Y. Qi, \"Thought-Like-Pro: Enhancing Reasoning of Large Language Models through Self-Driven Prolog-based Chain-of-Thought,\" ",[491,1080,752],{},[348,1082,502],{"href":1083,"target":498,"className":1084},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2407.14562",[500,501],[486,1086,1088,1089,546,1092],{"id":1087},"source-11","Z. Di, C. Zhang, H. Lv, L. Cui, and L. Liu, \"LoRP: LLM-based Logical Reasoning via Prolog,\" ",[491,1090,1091],{},"Knowledge-Based Systems",[348,1093,502],{"href":1094,"target":498,"className":1095},"https:\u002F\u002Fdoi.org\u002F10.1016\u002Fj.knosys.2025.114140",[500,501],[486,1097,1099,1100,1102,1103],{"id":1098},"source-12","J. Hu, J. Zhang, Y. Zhao, and T. Ringer, \"HybridProver: Augmenting Theorem Proving with LLM-Driven Proof Synthesis and Refinement,\" ",[491,1101,752],{},", 2025, ",[348,1104,502],{"href":1105,"target":498,"className":1106},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2505.15740",[500,501],{"title":575,"searchDepth":576,"depth":576,"links":1108},[1109,1110,1111,1112,1113,1114,1115],{"id":811,"depth":576,"text":812},{"id":827,"depth":576,"text":828},{"id":864,"depth":576,"text":865},{"id":897,"depth":576,"text":898},{"id":939,"depth":576,"text":940},{"id":953,"depth":576,"text":954},{"id":472,"depth":576,"text":476},"2026-06-26","Pairing a language model with a classical solver, an exact rule-crunching program, promises reasoning that is sound by construction, since the solver decides the hard step and returns an answer that can be checked independently. The catch is that trust does not disappear. It moves to the edges, into the act of turning a messy sentence into a precise spec and the act of reporting the result back, which is where most of the recent research now lives.",{"src":817},{"authors":1120,"badge":1123,"source":1125},[1121],{"avatar":1122,"name":594,"to":595},{"src":593},{"label":1124},"Neurosymbolic AI",{"name":599,"url":595},{"title":230,"description":1117},"nNK32oKCt0uQ2WCxV9FibOImiKDLxL6zwIpyLN0KTeA",1784123265237]