Skip to content

Commit fe7eabd

Browse files
BazookaMusicBazookaMusic
committed
Add run from agents into the user prompt and fix an issue with classifying it as a system prompt injection
1 parent 535adc7 commit fe7eabd

5 files changed

Lines changed: 107 additions & 41 deletions

File tree

‎javascript/ql/src/experimental/semmle/javascript/frameworks/OpenAI.qll‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,23 @@ module AgentSDK {
242242
)
243243
}
244244

245+
/**
246+
* Gets user prompt sinks for run(agent, input).
247+
* Covers string input and user-role array messages.
248+
*/
249+
API::Node getUserPromptNode() {
250+
// run(agent, "string") — string input is the user prompt
251+
result = run().getParameter(1)
252+
or
253+
// run(agent, [{ role: "user", content: ... }])
254+
exists(API::Node msg |
255+
msg = run().getParameter(1).getArrayElement() and
256+
not isSystemOrDevMessage(msg)
257+
|
258+
result = msg.getMember("content")
259+
)
260+
}
261+
245262
/**
246263
* Gets an agent constructor config that visibly lacks input guardrails.
247264
* Covers both native Agent({ inputGuardrails: [...] }) and

‎javascript/ql/src/experimental/semmle/javascript/security/PromptInjection/UserPromptInjectionCustomizations.qll‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,8 @@ module UserPromptInjection {
6363
this = Anthropic::getUserPromptNode().asSink()
6464
or
6565
this = GoogleGenAI::getUserPromptNode().asSink()
66+
or
67+
this = AgentSDK::getUserPromptNode().asSink()
6668
}
6769
}
6870

‎javascript/ql/test/experimental/Security/CWE-1427/SystemPromptInjection/agents_test.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,8 @@ app.get("/agents", async (req, res) => {
6363

6464
// === run() with string input ===
6565

66-
// SHOULD ALERT - string input to run() is used as a prompt
67-
const r1 = await run(agent1, query); // $ Alert[js/prompt-injection]
66+
// SHOULD NOT ALERT - string input to run() is a user prompt, not system prompt
67+
const r1 = await run(agent1, query); // OK - user prompt sink
6868

6969
// === run() with array input: system role ===
7070

‎javascript/ql/test/experimental/Security/CWE-1427/UserPromptInjection/UserPromptInjection.expected‎

Lines changed: 51 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -9,19 +9,23 @@ edges
99
| gemini_user_test.js:8:9:8:17 | userInput | gemini_user_test.js:51:13:51:21 | userInput | provenance | |
1010
| gemini_user_test.js:8:9:8:17 | userInput | gemini_user_test.js:58:13:58:21 | userInput | provenance | |
1111
| gemini_user_test.js:8:21:8:39 | req.query.userInput | gemini_user_test.js:8:9:8:17 | userInput | provenance | |
12-
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:22:12:22:20 | userInput | provenance | |
13-
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:31:18:31:26 | userInput | provenance | |
14-
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:42:18:42:26 | userInput | provenance | |
15-
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:56:19:56:27 | userInput | provenance | |
16-
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:66:13:66:21 | userInput | provenance | |
17-
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:71:13:71:21 | userInput | provenance | |
18-
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:75:13:75:21 | userInput | provenance | |
19-
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:82:13:82:21 | userInput | provenance | |
20-
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:88:13:88:21 | userInput | provenance | |
21-
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:94:14:94:22 | userInput | provenance | |
22-
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:100:12:100:20 | userInput | provenance | |
23-
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:147:12:147:20 | userInput | provenance | |
24-
| openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:14:9:14:17 | userInput | provenance | |
12+
| openai_user_test.js:15:9:15:17 | userInput | openai_user_test.js:23:12:23:20 | userInput | provenance | |
13+
| openai_user_test.js:15:9:15:17 | userInput | openai_user_test.js:32:18:32:26 | userInput | provenance | |
14+
| openai_user_test.js:15:9:15:17 | userInput | openai_user_test.js:43:18:43:26 | userInput | provenance | |
15+
| openai_user_test.js:15:9:15:17 | userInput | openai_user_test.js:57:19:57:27 | userInput | provenance | |
16+
| openai_user_test.js:15:9:15:17 | userInput | openai_user_test.js:67:13:67:21 | userInput | provenance | |
17+
| openai_user_test.js:15:9:15:17 | userInput | openai_user_test.js:72:13:72:21 | userInput | provenance | |
18+
| openai_user_test.js:15:9:15:17 | userInput | openai_user_test.js:76:13:76:21 | userInput | provenance | |
19+
| openai_user_test.js:15:9:15:17 | userInput | openai_user_test.js:83:13:83:21 | userInput | provenance | |
20+
| openai_user_test.js:15:9:15:17 | userInput | openai_user_test.js:89:13:89:21 | userInput | provenance | |
21+
| openai_user_test.js:15:9:15:17 | userInput | openai_user_test.js:95:14:95:22 | userInput | provenance | |
22+
| openai_user_test.js:15:9:15:17 | userInput | openai_user_test.js:101:12:101:20 | userInput | provenance | |
23+
| openai_user_test.js:15:9:15:17 | userInput | openai_user_test.js:148:12:148:20 | userInput | provenance | |
24+
| openai_user_test.js:15:9:15:17 | userInput | openai_user_test.js:192:20:192:28 | userInput | provenance | |
25+
| openai_user_test.js:15:9:15:17 | userInput | openai_user_test.js:196:30:196:38 | userInput | provenance | |
26+
| openai_user_test.js:15:9:15:17 | userInput | openai_user_test.js:201:27:201:35 | userInput | provenance | |
27+
| openai_user_test.js:15:9:15:17 | userInput | openai_user_test.js:205:30:205:38 | userInput | provenance | |
28+
| openai_user_test.js:15:21:15:39 | req.query.userInput | openai_user_test.js:15:9:15:17 | userInput | provenance | |
2529
nodes
2630
| anthropic_user_test.js:8:9:8:17 | userInput | semmle.label | userInput |
2731
| anthropic_user_test.js:8:21:8:39 | req.query.userInput | semmle.label | req.query.userInput |
@@ -35,20 +39,24 @@ nodes
3539
| gemini_user_test.js:44:13:44:21 | userInput | semmle.label | userInput |
3640
| gemini_user_test.js:51:13:51:21 | userInput | semmle.label | userInput |
3741
| gemini_user_test.js:58:13:58:21 | userInput | semmle.label | userInput |
38-
| openai_user_test.js:14:9:14:17 | userInput | semmle.label | userInput |
39-
| openai_user_test.js:14:21:14:39 | req.query.userInput | semmle.label | req.query.userInput |
40-
| openai_user_test.js:22:12:22:20 | userInput | semmle.label | userInput |
41-
| openai_user_test.js:31:18:31:26 | userInput | semmle.label | userInput |
42-
| openai_user_test.js:42:18:42:26 | userInput | semmle.label | userInput |
43-
| openai_user_test.js:56:19:56:27 | userInput | semmle.label | userInput |
44-
| openai_user_test.js:66:13:66:21 | userInput | semmle.label | userInput |
45-
| openai_user_test.js:71:13:71:21 | userInput | semmle.label | userInput |
46-
| openai_user_test.js:75:13:75:21 | userInput | semmle.label | userInput |
47-
| openai_user_test.js:82:13:82:21 | userInput | semmle.label | userInput |
48-
| openai_user_test.js:88:13:88:21 | userInput | semmle.label | userInput |
49-
| openai_user_test.js:94:14:94:22 | userInput | semmle.label | userInput |
50-
| openai_user_test.js:100:12:100:20 | userInput | semmle.label | userInput |
51-
| openai_user_test.js:147:12:147:20 | userInput | semmle.label | userInput |
42+
| openai_user_test.js:15:9:15:17 | userInput | semmle.label | userInput |
43+
| openai_user_test.js:15:21:15:39 | req.query.userInput | semmle.label | req.query.userInput |
44+
| openai_user_test.js:23:12:23:20 | userInput | semmle.label | userInput |
45+
| openai_user_test.js:32:18:32:26 | userInput | semmle.label | userInput |
46+
| openai_user_test.js:43:18:43:26 | userInput | semmle.label | userInput |
47+
| openai_user_test.js:57:19:57:27 | userInput | semmle.label | userInput |
48+
| openai_user_test.js:67:13:67:21 | userInput | semmle.label | userInput |
49+
| openai_user_test.js:72:13:72:21 | userInput | semmle.label | userInput |
50+
| openai_user_test.js:76:13:76:21 | userInput | semmle.label | userInput |
51+
| openai_user_test.js:83:13:83:21 | userInput | semmle.label | userInput |
52+
| openai_user_test.js:89:13:89:21 | userInput | semmle.label | userInput |
53+
| openai_user_test.js:95:14:95:22 | userInput | semmle.label | userInput |
54+
| openai_user_test.js:101:12:101:20 | userInput | semmle.label | userInput |
55+
| openai_user_test.js:148:12:148:20 | userInput | semmle.label | userInput |
56+
| openai_user_test.js:192:20:192:28 | userInput | semmle.label | userInput |
57+
| openai_user_test.js:196:30:196:38 | userInput | semmle.label | userInput |
58+
| openai_user_test.js:201:27:201:35 | userInput | semmle.label | userInput |
59+
| openai_user_test.js:205:30:205:38 | userInput | semmle.label | userInput |
5260
subpaths
5361
#select
5462
| anthropic_user_test.js:18:18:18:26 | userInput | anthropic_user_test.js:8:21:8:39 | req.query.userInput | anthropic_user_test.js:18:18:18:26 | userInput | This prompt construction depends on a $@. | anthropic_user_test.js:8:21:8:39 | req.query.userInput | user-provided value |
@@ -59,15 +67,19 @@ subpaths
5967
| gemini_user_test.js:44:13:44:21 | userInput | gemini_user_test.js:8:21:8:39 | req.query.userInput | gemini_user_test.js:44:13:44:21 | userInput | This prompt construction depends on a $@. | gemini_user_test.js:8:21:8:39 | req.query.userInput | user-provided value |
6068
| gemini_user_test.js:51:13:51:21 | userInput | gemini_user_test.js:8:21:8:39 | req.query.userInput | gemini_user_test.js:51:13:51:21 | userInput | This prompt construction depends on a $@. | gemini_user_test.js:8:21:8:39 | req.query.userInput | user-provided value |
6169
| gemini_user_test.js:58:13:58:21 | userInput | gemini_user_test.js:8:21:8:39 | req.query.userInput | gemini_user_test.js:58:13:58:21 | userInput | This prompt construction depends on a $@. | gemini_user_test.js:8:21:8:39 | req.query.userInput | user-provided value |
62-
| openai_user_test.js:22:12:22:20 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:22:12:22:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
63-
| openai_user_test.js:31:18:31:26 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:31:18:31:26 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
64-
| openai_user_test.js:42:18:42:26 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:42:18:42:26 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
65-
| openai_user_test.js:56:19:56:27 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:56:19:56:27 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
66-
| openai_user_test.js:66:13:66:21 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:66:13:66:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
67-
| openai_user_test.js:71:13:71:21 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:71:13:71:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
68-
| openai_user_test.js:75:13:75:21 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:75:13:75:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
69-
| openai_user_test.js:82:13:82:21 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:82:13:82:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
70-
| openai_user_test.js:88:13:88:21 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:88:13:88:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
71-
| openai_user_test.js:94:14:94:22 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:94:14:94:22 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
72-
| openai_user_test.js:100:12:100:20 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:100:12:100:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
73-
| openai_user_test.js:147:12:147:20 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:147:12:147:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
70+
| openai_user_test.js:23:12:23:20 | userInput | openai_user_test.js:15:21:15:39 | req.query.userInput | openai_user_test.js:23:12:23:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:15:21:15:39 | req.query.userInput | user-provided value |
71+
| openai_user_test.js:32:18:32:26 | userInput | openai_user_test.js:15:21:15:39 | req.query.userInput | openai_user_test.js:32:18:32:26 | userInput | This prompt construction depends on a $@. | openai_user_test.js:15:21:15:39 | req.query.userInput | user-provided value |
72+
| openai_user_test.js:43:18:43:26 | userInput | openai_user_test.js:15:21:15:39 | req.query.userInput | openai_user_test.js:43:18:43:26 | userInput | This prompt construction depends on a $@. | openai_user_test.js:15:21:15:39 | req.query.userInput | user-provided value |
73+
| openai_user_test.js:57:19:57:27 | userInput | openai_user_test.js:15:21:15:39 | req.query.userInput | openai_user_test.js:57:19:57:27 | userInput | This prompt construction depends on a $@. | openai_user_test.js:15:21:15:39 | req.query.userInput | user-provided value |
74+
| openai_user_test.js:67:13:67:21 | userInput | openai_user_test.js:15:21:15:39 | req.query.userInput | openai_user_test.js:67:13:67:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:15:21:15:39 | req.query.userInput | user-provided value |
75+
| openai_user_test.js:72:13:72:21 | userInput | openai_user_test.js:15:21:15:39 | req.query.userInput | openai_user_test.js:72:13:72:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:15:21:15:39 | req.query.userInput | user-provided value |
76+
| openai_user_test.js:76:13:76:21 | userInput | openai_user_test.js:15:21:15:39 | req.query.userInput | openai_user_test.js:76:13:76:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:15:21:15:39 | req.query.userInput | user-provided value |
77+
| openai_user_test.js:83:13:83:21 | userInput | openai_user_test.js:15:21:15:39 | req.query.userInput | openai_user_test.js:83:13:83:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:15:21:15:39 | req.query.userInput | user-provided value |
78+
| openai_user_test.js:89:13:89:21 | userInput | openai_user_test.js:15:21:15:39 | req.query.userInput | openai_user_test.js:89:13:89:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:15:21:15:39 | req.query.userInput | user-provided value |
79+
| openai_user_test.js:95:14:95:22 | userInput | openai_user_test.js:15:21:15:39 | req.query.userInput | openai_user_test.js:95:14:95:22 | userInput | This prompt construction depends on a $@. | openai_user_test.js:15:21:15:39 | req.query.userInput | user-provided value |
80+
| openai_user_test.js:101:12:101:20 | userInput | openai_user_test.js:15:21:15:39 | req.query.userInput | openai_user_test.js:101:12:101:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:15:21:15:39 | req.query.userInput | user-provided value |
81+
| openai_user_test.js:148:12:148:20 | userInput | openai_user_test.js:15:21:15:39 | req.query.userInput | openai_user_test.js:148:12:148:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:15:21:15:39 | req.query.userInput | user-provided value |
82+
| openai_user_test.js:192:20:192:28 | userInput | openai_user_test.js:15:21:15:39 | req.query.userInput | openai_user_test.js:192:20:192:28 | userInput | This prompt construction depends on a $@. | openai_user_test.js:15:21:15:39 | req.query.userInput | user-provided value |
83+
| openai_user_test.js:196:30:196:38 | userInput | openai_user_test.js:15:21:15:39 | req.query.userInput | openai_user_test.js:196:30:196:38 | userInput | This prompt construction depends on a $@. | openai_user_test.js:15:21:15:39 | req.query.userInput | user-provided value |
84+
| openai_user_test.js:201:27:201:35 | userInput | openai_user_test.js:15:21:15:39 | req.query.userInput | openai_user_test.js:201:27:201:35 | userInput | This prompt construction depends on a $@. | openai_user_test.js:15:21:15:39 | req.query.userInput | user-provided value |
85+
| openai_user_test.js:205:30:205:38 | userInput | openai_user_test.js:15:21:15:39 | req.query.userInput | openai_user_test.js:205:30:205:38 | userInput | This prompt construction depends on a $@. | openai_user_test.js:15:21:15:39 | req.query.userInput | user-provided value |

‎javascript/ql/test/experimental/Security/CWE-1427/UserPromptInjection/openai_user_test.js‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ const {
55
GuardrailsOpenAI,
66
GuardrailsAzureOpenAI,
77
} = require("@openai/guardrails");
8+
const { Agent, run, Runner } = require("@openai/agents");
89

910
const app = express();
1011
const client = new OpenAI();
@@ -180,5 +181,39 @@ app.get("/test", async (req, res) => {
180181
],
181182
});
182183

184+
// === Agent SDK: run() user prompt sinks (SHOULD ALERT) ===
185+
186+
const agent = new Agent({
187+
name: "Assistant",
188+
instructions: "You are a helpful assistant",
189+
});
190+
191+
// run() with string input (user prompt)
192+
await run(agent, userInput); // $ Alert[js/user-prompt-injection]
193+
194+
// run() with user-role array message
195+
await run(agent, [
196+
{ role: "user", content: userInput }, // $ Alert[js/user-prompt-injection]
197+
]);
198+
199+
// Runner instance with string input
200+
const runner = new Runner();
201+
await runner.run(agent, userInput); // $ Alert[js/user-prompt-injection]
202+
203+
// Runner instance with user-role array message
204+
await runner.run(agent, [
205+
{ role: "user", content: userInput }, // $ Alert[js/user-prompt-injection]
206+
]);
207+
208+
// === Agent SDK: system/developer role in run() (SHOULD NOT ALERT for user-prompt) ===
209+
210+
await run(agent, [
211+
{ role: "system", content: userInput }, // OK for user-prompt-injection (system prompt sink)
212+
]);
213+
214+
await run(agent, [
215+
{ role: "developer", content: userInput }, // OK for user-prompt-injection (system prompt sink)
216+
]);
217+
183218
res.send("done");
184219
});

0 commit comments

Comments
 (0)