KQL Search

Query Details

Custom Agents - Recorded injection candidate before tool call

Custom Prompt Injection Tool Chain

Query

let Evidence=materialize(CustomAgentInlineContent(1h));
let Intent=Evidence
| where UserText matches regex @'(?i)(ignore (all|any|the|previous|prior|above).{0,40}instruction|disregard.{0,40}instruction|developer mode|jailbreak|do anything now|reveal your (instructions|prompt))'
| summarize arg_min(EventTime, UserText, SpanId) by _ResourceId, OperationId, AgentKey
| project _ResourceId, OperationId, AgentKey, IntentTime=EventTime, IntentPrompt=UserText, IntentSpanId=SpanId;
CustomAgentSpans(1h)
| where Classification !in ('Excluded managed platform', 'Unattributed GenAI') and HasLineage
| where IsTool and true
| join kind=inner Intent on _ResourceId, OperationId, AgentKey
| where EventTime between (IntentTime .. IntentTime + 10m)
| summarize EventCount=count(), SuccessfulCalls=countif(Success == true),
    FailedCalls=countif(Success == false), arg_min(EventTime, *)
    by _ResourceId, OperationId, AgentKey
| extend Signal='Recorded user intent before observed tool span; impact unproven'
| extend TimeGenerated=EventTime, ResourceId=_ResourceId, TraceId=OperationId,
    EventCount=tolong(column_ifexists('EventCount', 1)),
    SuccessfulCalls=tolong(column_ifexists('SuccessfulCalls', 0)),
    FailedCalls=tolong(column_ifexists('FailedCalls', 0))
| project TimeGenerated, ResourceId, Agent, AgentId, AgentKey, AppRoleName,
    TraceId, SpanId, Conversation, Model, Tool, TargetHost, ServerDisplay,
    Signal, EventCount, SuccessfulCalls, FailedCalls,
    TriggerSpanId=tostring(column_ifexists('TriggerSpanId', '')),
    TriggerTime=todatetime(column_ifexists('TriggerTime', datetime(null))),
    TriggerTool=tostring(column_ifexists('TriggerTool', '')),
    TriggerToolResult=tostring(column_ifexists('TriggerToolResult', '')),
    TriggerServerDisplay=tostring(column_ifexists('TriggerServerDisplay', '')),
    TriggerTargetHost=tostring(column_ifexists('TriggerTargetHost', '')),
    RiskyOutputKind=tostring(column_ifexists('RiskyOutputKind', '')),
    RelatedToolResults=tolong(column_ifexists('RelatedToolResults', 0)),
    MatchedIntentPrompt=tostring(column_ifexists('IntentPrompt', '')),
    MatchedIntentSpanId=tostring(column_ifexists('IntentSpanId', ''))
| join kind=leftouter (Evidence
    | project ResourceId=_ResourceId, TraceId=OperationId, SpanId, AgentKey, Prompt, Output, ToolArguments, ToolResult, InputMessages, OutputMessages,
    EvidenceSource, PromptEvidenceOrigin, OutputEvidenceOrigin, PromptEvidenceSpanId, OutputEvidenceSpanId,
    InputParseState, OutputParseState, ToolArgumentsState, ToolResultState)
    on ResourceId, TraceId, SpanId, AgentKey
| extend Prompt=coalesce(MatchedIntentPrompt, Prompt),
    PromptEvidenceOrigin=iff(isnotempty(MatchedIntentPrompt), 'Matched intent span: user role', PromptEvidenceOrigin),
    PromptEvidenceSpanId=coalesce(MatchedIntentSpanId, PromptEvidenceSpanId)
| extend PreviewCharacters=toint(min_of(160, max_of(0, (1500 - string_size(tostring(bag_pack(
    'AgentId', AgentId, 'AgentKey', AgentKey, 'TraceId', TraceId, 'SpanId', SpanId,
    'Conversation', Conversation, 'ResourceId', ResourceId, 'Signal', Signal)))) / 12)))
| extend PromptPreview=substring(Prompt, 0, PreviewCharacters),
    OutputPreview=substring(Output, 0, PreviewCharacters)
| extend EvidencePreviewStatus=case(PreviewCharacters == 0, 'Metadata exhausted preview budget; inspect full query results',
    strlen(Prompt) > PreviewCharacters or strlen(Output) > PreviewCharacters, 'Inline evidence; previews shortened; full text in query results',
    isempty(Prompt) and isempty(Output), 'No prompt/output captured or parsed; not a protection verdict',
    'Inline evidence; full recorded text in query results')
| project TimeGenerated, ResourceId, Agent, AgentId, AgentKey, AppRoleName,
    TraceId, SpanId, Conversation, Model, Tool, TargetHost, ServerDisplay,
    Signal, EventCount, SuccessfulCalls, FailedCalls, Prompt, Output, ToolArguments, ToolResult, InputMessages, OutputMessages,
    EvidenceSource, PromptEvidenceOrigin, OutputEvidenceOrigin, PromptEvidenceSpanId, OutputEvidenceSpanId,
    InputParseState, OutputParseState, ToolArgumentsState, ToolResultState,
    PromptPreview, OutputPreview, EvidencePreviewStatus, TriggerSpanId, TriggerTime, TriggerTool, TriggerToolResult,
    TriggerServerDisplay, TriggerTargetHost, RiskyOutputKind, RelatedToolResults
| order by TimeGenerated desc

Explanation

This query is designed to detect potential security incidents involving custom agents in an application. Here's a simplified explanation:

  1. Purpose: The query aims to identify instances where a user might have attempted to override or manipulate the behavior of an application agent by using specific language patterns. It then checks if any tool actions were recorded shortly after these user inputs.

  2. Data Sources: It uses data from Application Insights, specifically looking at application requests and dependencies.

  3. Process:

    • It first gathers evidence of user inputs that match certain suspicious patterns (e.g., "ignore previous instructions", "developer mode", "jailbreak").
    • It then looks for tool actions (spans) that occurred within the same resource, agent, and trace within ten minutes of the suspicious input.
    • The query checks if these tool actions were successful or failed and counts them.

  4. Output:

    • The results include details about the resource, agent, and trace, along with a signal indicating that a user intent was recorded before a tool action was observed.
    • It provides a preview of the user input and output, along with metadata about the evidence.

  5. Severity and Configuration:

    • The severity of this detection is marked as "High".
    • The query runs every hour and looks back over the past 14 days.
    • It is currently disabled but is configured to create incidents if enabled.

  6. Additional Details:

    • The query includes mappings to Azure resources and cloud applications.
    • It has custom details and grouping settings for incidents, ensuring that alerts are grouped by specific entities and details.

Overall, this query is part of a security monitoring setup to detect and alert on potentially risky user interactions with application agents.