diff --git a/extension/llm/apple/ExecuTorchLLM/__tests__/MultimodalRunnerTest.swift b/extension/llm/apple/ExecuTorchLLM/__tests__/MultimodalRunnerTest.swift
index 1d5af4998d3..a0e6e8c984f 100644
--- a/extension/llm/apple/ExecuTorchLLM/__tests__/MultimodalRunnerTest.swift
+++ b/extension/llm/apple/ExecuTorchLLM/__tests__/MultimodalRunnerTest.swift
@@ -123,6 +123,8 @@ class MultimodalRunnerTest: XCTestCase {
         MultimodalInput(String(format: chatTemplate, userPrompt)),
       ], Config {
         $0.sequenceLength = sequenceLength
+        $0.maximumNewTokens = 128
+        $0.temperature = 0
       }) { token in
         text += token
         if token == "<end_of_turn>" {
@@ -143,6 +145,8 @@ class MultimodalRunnerTest: XCTestCase {
         MultimodalInput(String(format: chatTemplate, userPrompt)),
       ], Config {
         $0.sequenceLength = sequenceLength
+        $0.maximumNewTokens = 128
+        $0.temperature = 0
       }) { token in
         text += token
         if token == "<end_of_turn>" {
@@ -237,7 +241,7 @@ class MultimodalRunnerTest: XCTestCase {
         )),
         MultimodalInput(String(format: chatTemplate, userPrompt)),
       ], Config {
-        $0.maximumNewTokens = 256
+        $0.maximumNewTokens = 128
         $0.temperature = 0
       }) { token in
         text += token