Foundry - MCP / plugin tool description changed

Foundry Mcp Tool Description Change

Query

let baselineWindow = 14d;
let recentWindow   = 1h;
let toolDescs =
    AppDependencies
    | where TimeGenerated > ago(baselineWindow)
    | extend
        ToolName = tolower(tostring(Properties["gen_ai.tool.name"])),
        ToolDesc = tostring(coalesce(
                     Properties["gen_ai.tool.description"],
                     Properties["microsoft.agent.tool.description"],
                     Properties["mcp.tool.description"])),
        ToolServer = tolower(tostring(coalesce(
                     Properties["gen_ai.tool.server_url"],
                     Properties["mcp.server.url"],
                     Properties["server_url"],
                     Properties["gen_ai.tool.server_label"],
                     Properties["mcp.server.label"],
                     Properties["server_label"],
                     Properties["gen_ai.tool.server"],
                     Properties["mcp.server.name"],
                     Properties["microsoft.agent.tool.server"], Target, "unknown")))
    | where isnotempty(ToolName) and isnotempty(ToolDesc)
    | extend NormDesc = trim(@"\s+", replace_regex(ToolDesc, @"\s+", " "))
    | extend DescHash = hash_sha256(NormDesc);
let baseline =
    toolDescs
    | where TimeGenerated between (ago(baselineWindow) .. ago(recentWindow))
    | summarize Calls = count(),
                FirstSeen = min(TimeGenerated)
        by ToolServer, ToolName, DescHash;
let stable =
    baseline
    | summarize Calls = sum(Calls), FirstSeen = min(FirstSeen)
        by ToolServer, ToolName
    | where Calls >= 10 and FirstSeen < ago(3d)
    | distinct ToolServer, ToolName;
let recent =
    toolDescs
    | where TimeGenerated > ago(recentWindow)
    | summarize Hits = count(),
                Sample = take_any(substring(ToolDesc, 0, 1024)),
                AnyAgent = take_any(tostring(Properties["gen_ai.agent.name"])),
                AnyConv  = take_any(tostring(Properties["gen_ai.conversation.id"])),
                FirstSeen = min(TimeGenerated)
        by ToolServer, ToolName, DescHash;
recent
| join kind=leftanti baseline on ToolServer, ToolName, DescHash
| join kind=inner stable on ToolServer, ToolName
| where Hits >= 1
| extend AccountName = iff(isempty(AnyAgent), "unknown-agent", AnyAgent)
| project FirstSeen, AccountName, Agent = AnyAgent, ToolServer, ToolName,
          DescHash, NewDescriptionSample = Sample, Hits, ConvId = AnyConv
| order by FirstSeen desc

Explanation

This query is designed to detect changes in the description of tools used by agents, which could indicate a security threat known as "tool description poisoning." Here's a simplified breakdown of what the query does:

Purpose: The query monitors for changes in the natural-language descriptions of tools that agents have been using consistently. If a tool's description suddenly changes, it could be a sign of a supply-chain compromise, where the tool's instructions have been altered without changing the tool's binary code.
Data Source: It uses data from Application Insights, specifically looking at application dependencies.
Time Frame: The query checks for new tool descriptions that have appeared in the last hour but were not present in the previous 14 days.
Criteria for Evaluation:
- Only tools that have been called at least 10 times and were first seen more than 3 days ago are considered. This helps avoid false positives from new tool onboarding.
- It hashes the tool descriptions to detect changes and compares these hashes over time.
Alerting: If a new description is detected for an established tool, an alert is triggered. The alert includes details like the first time the change was seen, the agent involved, the server and tool name, and a sample of the new description.
Severity and Tactics: The alert is considered high severity and is associated with tactics like Initial Access, Defense Evasion, and Persistence, which are common in cyber attacks.
Incident Management: If an alert is triggered, an incident is created. The system groups related alerts by account to manage incidents efficiently.

In essence, this query helps identify potential security threats by monitoring unexpected changes in tool descriptions, which could indicate malicious activity in the software supply chain.