KQL Search

Query Details

Custom Agents - Instruction-override tool result before risky model output

Custom Tool Result Risky Output Chain

Query

let Evidence=materialize(CustomAgentInlineContent(1h));
let ToolResults=Evidence
| where IsTool and ToolResult matches regex @'(?i)(ignore (all|any|the|previous|prior|above).{0,40}instruction|disregard.{0,40}instruction|developer mode|jailbreak|do anything now|reveal your (instructions|prompt))'
| where isnotnull(DurationMs) and DurationMs >= 0
| project _ResourceId, OperationId, AgentKey, TriggerSpanId=SpanId,
    TriggerTime=EventTime + DurationMs * 1ms, TriggerTool=Tool, TriggerToolResult=ToolResult,
    TriggerServerDisplay=ServerDisplay, TriggerTargetHost=TargetHost;
Evidence
| where Operation == 'chat'
| extend SecretOutput=OutputText matches regex @'(-----BEGIN (RSA |EC |OPENSSH )?PRIVATE KEY-----|(?:AKIA|ASIA)[A-Z0-9]{16}|gh[pousr]_[A-Za-z0-9]{36,}|sk-[A-Za-z0-9_-]{20,})', ActiveOutput=OutputText matches regex @'(?i)(<script(?:\s|>)|javascript\s*:|data\s*:\s*text/html|<iframe(?:\s|>)|<object(?:\s|>)|<embed(?:\s|>)|on(?:error|load|click)\s*=|document\.cookie|window\.location\s*=)'
| where SecretOutput or ActiveOutput
| extend RiskyOutputKind=case(SecretOutput and ActiveOutput, 'Secret-shaped and active-content output',
    SecretOutput, 'Secret-shaped output', 'Active-content output')
| join kind=inner ToolResults on _ResourceId, OperationId, AgentKey
| where EventTime between (TriggerTime .. TriggerTime + 10m)
| summarize EventCount=dcount(SpanId), RelatedToolResults=dcount(TriggerSpanId), arg_min(EventTime, *)
    by _ResourceId, OperationId, AgentKey
| extend Signal='Instruction-override tool result precedes risky model output; causation unproven'
| extend TimeGenerated=EventTime, ResourceId=_ResourceId, TraceId=OperationId,
    EventCount=tolong(column_ifexists('EventCount', 1)),
    SuccessfulCalls=tolong(column_ifexists('SuccessfulCalls', 0)),
    FailedCalls=tolong(column_ifexists('FailedCalls', 0))
| project TimeGenerated, ResourceId, Agent, AgentId, AgentKey, AppRoleName,
    TraceId, SpanId, Conversation, Model, Tool, TargetHost, ServerDisplay,
    Signal, EventCount, SuccessfulCalls, FailedCalls,
    TriggerSpanId=tostring(column_ifexists('TriggerSpanId', '')),
    TriggerTime=todatetime(column_ifexists('TriggerTime', datetime(null))),
    TriggerTool=tostring(column_ifexists('TriggerTool', '')),
    TriggerToolResult=tostring(column_ifexists('TriggerToolResult', '')),
    TriggerServerDisplay=tostring(column_ifexists('TriggerServerDisplay', '')),
    TriggerTargetHost=tostring(column_ifexists('TriggerTargetHost', '')),
    RiskyOutputKind=tostring(column_ifexists('RiskyOutputKind', '')),
    RelatedToolResults=tolong(column_ifexists('RelatedToolResults', 0)),
    MatchedIntentPrompt=tostring(column_ifexists('IntentPrompt', '')),
    MatchedIntentSpanId=tostring(column_ifexists('IntentSpanId', ''))
| join kind=leftouter (Evidence
    | project ResourceId=_ResourceId, TraceId=OperationId, SpanId, AgentKey, Prompt, Output, ToolArguments, ToolResult, InputMessages, OutputMessages,
    EvidenceSource, PromptEvidenceOrigin, OutputEvidenceOrigin, PromptEvidenceSpanId, OutputEvidenceSpanId,
    InputParseState, OutputParseState, ToolArgumentsState, ToolResultState)
    on ResourceId, TraceId, SpanId, AgentKey
| extend Prompt=coalesce(MatchedIntentPrompt, Prompt),
    PromptEvidenceOrigin=iff(isnotempty(MatchedIntentPrompt), 'Matched intent span: user role', PromptEvidenceOrigin),
    PromptEvidenceSpanId=coalesce(MatchedIntentSpanId, PromptEvidenceSpanId)
| extend PreviewCharacters=toint(min_of(160, max_of(0, (1500 - string_size(tostring(bag_pack(
    'AgentId', AgentId, 'AgentKey', AgentKey, 'TraceId', TraceId, 'SpanId', SpanId,
    'Conversation', Conversation, 'ResourceId', ResourceId, 'Signal', Signal)))) / 12)))
| extend PromptPreview=substring(Prompt, 0, PreviewCharacters),
    OutputPreview=substring(Output, 0, PreviewCharacters)
| extend EvidencePreviewStatus=case(PreviewCharacters == 0, 'Metadata exhausted preview budget; inspect full query results',
    strlen(Prompt) > PreviewCharacters or strlen(Output) > PreviewCharacters, 'Inline evidence; previews shortened; full text in query results',
    isempty(Prompt) and isempty(Output), 'No prompt/output captured or parsed; not a protection verdict',
    'Inline evidence; full recorded text in query results')
| project TimeGenerated, ResourceId, Agent, AgentId, AgentKey, AppRoleName,
    TraceId, SpanId, Conversation, Model, Tool, TargetHost, ServerDisplay,
    Signal, EventCount, SuccessfulCalls, FailedCalls, Prompt, Output, ToolArguments, ToolResult, InputMessages, OutputMessages,
    EvidenceSource, PromptEvidenceOrigin, OutputEvidenceOrigin, PromptEvidenceSpanId, OutputEvidenceSpanId,
    InputParseState, OutputParseState, ToolArgumentsState, ToolResultState,
    PromptPreview, OutputPreview, EvidencePreviewStatus, TriggerSpanId, TriggerTime, TriggerTool, TriggerToolResult,
    TriggerServerDisplay, TriggerTargetHost, RiskyOutputKind, RelatedToolResults
| order by TimeGenerated desc

Explanation

This query is designed to detect potentially risky behavior in a system involving custom agents. Here's a simplified breakdown of what it does:

  1. Objective: The query aims to identify instances where a tool's result, which includes an instruction-override (like ignoring previous instructions or entering developer mode), is followed by a chat output that contains sensitive information (like secret keys) or active content (like scripts).

  2. Conditions:

    • The tool result must have a non-negative duration.
    • The chat output must occur within ten minutes after the tool result, on the same resource, agent, and trace.

  3. Data Sources: It uses data from Application Insights, specifically "AppRequests" and "AppDependencies".

  4. Frequency and Period: The query runs every hour and looks back over the last 14 days.

  5. Output: If the conditions are met, it returns details about the tool result and the risky output, including timestamps, resource IDs, and other relevant metadata.

  6. Severity and Tactics: The severity of this alert is high, and it falls under the "Collection" tactic, indicating data collection or exfiltration.

  7. Additional Information:

    • The query does not prove causation, meaning just because the tool result and risky output are observed in sequence, it doesn't confirm that the tool result caused the risky output.
    • The query is currently disabled (enabled: false).

  8. Incident Management: If triggered, it creates an incident and groups alerts based on specific criteria.

In essence, this query is a security measure to monitor for potentially dangerous outputs following specific tool interactions, helping to identify and mitigate risks associated with instruction overrides and sensitive data exposure.