diff --git a/demos/code_local_assistant/README.md b/demos/code_local_assistant/README.md index c55160e602..86f834f1ac 100644 --- a/demos/code_local_assistant/README.md +++ b/demos/code_local_assistant/README.md @@ -16,7 +16,7 @@ With the rise of AI PC capabilities, hosting own Visual Studio code assistant is :sync: OpenVINO/Qwen3-Coder-30B-A3B-Instruct-int4-ov ```bat mkdir c:\models -ovms --model_repository_path c:\models --source_model OpenVINO/Qwen3-Coder-30B-A3B-Instruct-int4-ov --task text_generation --target_device GPU --tool_parser qwen3coder --rest_port 8000 --cache_dir .ovcache --model_name Qwen3-Coder-30B-A3B-Instruct +ovms --model_repository_path c:\models --source_model OpenVINO/Qwen3-Coder-30B-A3B-Instruct-int4-ov --task text_generation --target_device GPU --rest_port 8000 --cache_dir .ovcache --model_name Qwen3-Coder-30B-A3B-Instruct ``` > **Note:** For deployment, the model requires ~16GB disk space and recommended 19GB+ of VRAM on the GPU. @@ -27,7 +27,7 @@ ovms --model_repository_path c:\models --source_model OpenVINO/Qwen3-Coder-30B-A :sync: OpenVINO/gpt-oss-20b-int4-ov ```bat mkdir c:\models -ovms --model_repository_path c:\models --source_model OpenVINO/gpt-oss-20b-int4-ov --task text_generation --target_device GPU --tool_parser gptoss --reasoning_parser gptoss --rest_port 8000 --cache_dir .ovcache --model_name gpt-oss-20b +ovms --model_repository_path c:\models --source_model OpenVINO/gpt-oss-20b-int4-ov --task text_generation --target_device GPU --rest_port 8000 --cache_dir .ovcache --model_name gpt-oss-20b ``` > **Note:** For deployment, the model requires ~12GB disk space and recommended 16GB+ of VRAM on the GPU. ::: @@ -36,7 +36,7 @@ ovms --model_repository_path c:\models --source_model OpenVINO/gpt-oss-20b-int4- :sync: OpenVINO/Qwen3-8B-int4-ov ```bat mkdir c:\models -ovms --model_repository_path c:\models --source_model OpenVINO/Qwen3-8B-int4-ov --task text_generation --target_device GPU --tool_parser hermes3 --reasoning_parser qwen3 --rest_port 8000 --cache_dir .ovcache --model_name Qwen3-8B +ovms --model_repository_path c:\models --source_model OpenVINO/Qwen3-8B-int4-ov --task text_generation --target_device GPU --rest_port 8000 --cache_dir .ovcache --model_name Qwen3-8B ``` > **Note:** For deployment, the model requires ~4GB disk space and recommended 6GB+ of VRAM on the GPU. ::: @@ -44,7 +44,7 @@ ovms --model_repository_path c:\models --source_model OpenVINO/Qwen3-8B-int4-ov :sync: OpenVINO/Qwen3-8B-int4-cw-ov ```bat mkdir c:\models -ovms --model_repository_path c:\models --source_model OpenVINO/Qwen3-8B-int4-cw-ov --task text_generation --target_device NPU --tool_parser hermes3 --rest_port 8000 --max_prompt_len 16384 --plugin_config "{\"NPUW_LLM_PREFILL_ATTENTION_HINT\":\"PYRAMID\"}" --cache_dir .ovcache --model_name Qwen3-8B +ovms --model_repository_path c:\models --source_model OpenVINO/Qwen3-8B-int4-cw-ov --task text_generation --target_device NPU --rest_port 8000 --max_prompt_len 16384 --plugin_config "{\"NPUW_LLM_PREFILL_ATTENTION_HINT\":\"PYRAMID\"}" --cache_dir .ovcache --model_name Qwen3-8B ``` > **Note:** First model initialization might be long. With the compilation cache, sequential model loading will be fast. ::: @@ -67,7 +67,7 @@ ovms --model_repository_path c:\models --source_model OpenVINO/Qwen3-VL-8B-Instr mkdir -p models docker run -d -p 8000:8000 --rm --user $(id -u):$(id -g) -v $(pwd)/models:/models/:rw --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) \ openvino/model_server:weekly \ - --model_repository_path /models --source_model OpenVINO/Qwen3-Coder-30B-A3B-Instruct-int4-ov --task text_generation --target_device GPU --tool_parser qwen3coder --rest_port 8000 --model_name Qwen3-Coder-30B-A3B-Instruct + --model_repository_path /models --source_model OpenVINO/Qwen3-Coder-30B-A3B-Instruct-int4-ov --task text_generation --target_device GPU --rest_port 8000 --model_name Qwen3-Coder-30B-A3B-Instruct ``` > **Note:** For deployment, the model requires ~16GB disk space and recommended 19GB+ of VRAM on the GPU. @@ -80,7 +80,7 @@ docker run -d -p 8000:8000 --rm --user $(id -u):$(id -g) -v $(pwd)/models:/model mkdir -p models docker run -d -p 8000:8000 --rm --user $(id -u):$(id -g) -v $(pwd)/models:/models/:rw --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) \ openvino/model_server:weekly \ - --model_repository_path /models --source_model OpenVINO/gpt-oss-20b-int4-ov --task text_generation --target_device GPU --tool_parser gptoss --reasoning_parser gptoss --rest_port 8000 --model_name gpt-oss-20b + --model_repository_path /models --source_model OpenVINO/gpt-oss-20b-int4-ov --task text_generation --target_device GPU --rest_port 8000 --model_name gpt-oss-20b ``` > **Note:** For deployment, the model requires ~12GB disk space and recommended 16GB+ of VRAM on the GPU. ::: @@ -91,7 +91,7 @@ docker run -d -p 8000:8000 --rm --user $(id -u):$(id -g) -v $(pwd)/models:/model mkdir c:\models docker run -d -p 8000:8000 --rm --user $(id -u):$(id -g) -v $(pwd)/models:/models/:rw --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) \ openvino/model_server:weekly \ - --model_repository_path /models --source_model OpenVINO/Qwen3-8B-int4-ov --task text_generation --target_device GPU --tool_parser hermes3 --reasoning_parser qwen3 --rest_port 8000 --model_name Qwen3-8B + --model_repository_path /models --source_model OpenVINO/Qwen3-8B-int4-ov --task text_generation --target_device GPU --rest_port 8000 --model_name Qwen3-8B ``` > **Note:** For deployment, the model requires ~4GB disk space and recommended 6GB+ of VRAM on the GPU. ::: @@ -101,7 +101,7 @@ docker run -d -p 8000:8000 --rm --user $(id -u):$(id -g) -v $(pwd)/models:/model mkdir -p models docker run -d -p 8000:8000 --rm --user $(id -u):$(id -g) -v $(pwd)/models:/models/:rw --device /dev/accel --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) \ openvino/model_server:weekly \ - --model_repository_path /models --source_model OpenVINO/Qwen3-8B-int4-cw-ov --task text_generation --target_device NPU --tool_parser hermes3 --rest_port 8000 --max_prompt_len 16384 --plugin_config '{"NPUW_LLM_PREFILL_ATTENTION_HINT":"PYRAMID"}' --model_name Qwen3-8B + --model_repository_path /models --source_model OpenVINO/Qwen3-8B-int4-cw-ov --task text_generation --target_device NPU --rest_port 8000 --max_prompt_len 16384 --plugin_config '{"NPUW_LLM_PREFILL_ATTENTION_HINT":"PYRAMID"}' --model_name Qwen3-8B ``` > **Note:** First model initialization might be long. With the compilation cache, sequential model loading will be fast. ::: @@ -124,10 +124,10 @@ Models which are not published in OpenVINO format can be exported and quantized ``` mkdir models -python export_model.py text_generation --source_model unsloth/Devstral-Small-2507 --weight-format int4 --config_file_path models/config_all.json --model_repository_path models --tool_parser devstral --target_device GPU +python export_model.py text_generation --source_model unsloth/Devstral-Small-2507 --weight-format int4 --config_file_path models/config_all.json --model_repository_path models --target_device GPU curl -L -o models/unsloth/Devstral-Small-2507/chat_template.jinja https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/extras/chat_template_examples/chat_template_devstral.jinja -ovms --model_repository_path models --source_model unsloth/Devstral-Small-2507 --task text_generation --target_device GPU --tool_parser devstral --rest_port 8000 --cache_dir .ovcache +ovms --model_repository_path models --source_model unsloth/Devstral-Small-2507 --task text_generation --target_device GPU --rest_port 8000 --cache_dir .ovcache ``` > **Note:** Exporting models is a one time operation but might consume RAM at least of the model size and might take a lot of time depending on the model size. diff --git a/demos/continuous_batching/README.md b/demos/continuous_batching/README.md index dfc6fef039..7144f15ebd 100644 --- a/demos/continuous_batching/README.md +++ b/demos/continuous_batching/README.md @@ -35,7 +35,7 @@ That makes it easy to use and efficient especially on on Intel® Xeon® processo Running this command starts the container with CPU only target device: ```bash mkdir -p ${HOME}/models -docker run -it -p 8000:8000 --rm --user $(id -u):$(id -g) -v ${HOME}/models:/models/:rw openvino/model_server:weekly --model_repository_path /models --source_model OpenVINO/Qwen3-30B-A3B-Instruct-2507-int4-ov --task text_generation --target_device CPU --tool_parser hermes3 --rest_port 8000 --model_name Qwen3-30B-A3B-Instruct-2507-int4-ov +docker run -it -p 8000:8000 --rm --user $(id -u):$(id -g) -v ${HOME}/models:/models/:rw openvino/model_server:weekly --model_repository_path /models --source_model OpenVINO/Qwen3-30B-A3B-Instruct-2507-int4-ov --task text_generation --target_device CPU --rest_port 8000 --model_name Qwen3-30B-A3B-Instruct-2507-int4-ov ``` > **Note:** In case you want to use GPU target device, add extra docker parameters `--device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1)` to `docker run` command. The parameter `--target_device` should be also updated to `GPU`. @@ -46,7 +46,7 @@ to `docker run` command. The parameter `--target_device` should be also updated After ovms is installed according to steps from [baremetal deployment guide](../../docs/deploying_server_baremetal.md), run the following command: ```bat -ovms.exe --model_repository_path c:\models --source_model OpenVINO/Qwen3-30B-A3B-Instruct-2507-int4-ov --task text_generation --target_device GPU --tool_parser hermes3 --rest_port 8000 --model_name Qwen3-30B-A3B-Instruct-2507-int4-ov +ovms.exe --model_repository_path c:\models --source_model OpenVINO/Qwen3-30B-A3B-Instruct-2507-int4-ov --task text_generation --target_device GPU --rest_port 8000 --model_name Qwen3-30B-A3B-Instruct-2507-int4-ov ``` diff --git a/demos/continuous_batching/agentic_ai/README.md b/demos/continuous_batching/agentic_ai/README.md index 1b76928aeb..93fa997d13 100644 --- a/demos/continuous_batching/agentic_ai/README.md +++ b/demos/continuous_batching/agentic_ai/README.md @@ -51,7 +51,7 @@ as mentioned in [deployment guide](../../../docs/deploying_server_baremetal.md), :sync: Qwen3-VL-8B Pull and start OVMS: ```bat -ovms.exe --rest_port 8000 --source_model OpenVINO/Qwen3-VL-8B-Instruct-int4-ov --model_repository_path c:\models --tool_parser hermes3 --target_device GPU --task text_generation --pipeline_type VLM_CB --cache_dir .cache --allowed_media_domains raw.githubusercontent.com +ovms.exe --rest_port 8000 --source_model OpenVINO/Qwen3-VL-8B-Instruct-int4-ov --model_repository_path c:\models --target_device GPU --task text_generation --pipeline_type VLM_CB --cache_dir .cache --allowed_media_domains raw.githubusercontent.com ``` Use MCP server, with additional image of Gdańsk old town. VLM model deduces location and calls `get_weather` tool to summarize the weather conditions in the city. @@ -76,7 +76,7 @@ The current weather in Gdańsk is overcast with a temperature of 8.8°C (feels l :sync: Qwen3-4B Pull and start OVMS: ```bat -ovms.exe --rest_port 8000 --source_model OpenVINO/Qwen3-4B-int4-ov --model_repository_path c:\models --tool_parser hermes3 --target_device GPU --task text_generation --cache_dir .cache +ovms.exe --rest_port 8000 --source_model OpenVINO/Qwen3-4B-int4-ov --model_repository_path c:\models --target_device GPU --task text_generation --cache_dir .cache ``` Use MCP server: @@ -93,7 +93,7 @@ The current weather in Tokyo is overcast with a temperature of 9.4°C (feels lik :sync: Phi-4-mini-instruct Pull and start OVMS: ```bat -ovms.exe --rest_port 8000 --source_model OpenVINO/Phi-4-mini-instruct-int4-ov --model_repository_path c:\models --tool_parser phi4 --target_device GPU --task text_generation --enable_tool_guided_generation true --cache_dir .cache --max_num_batched_tokens 99999 +ovms.exe --rest_port 8000 --source_model OpenVINO/Phi-4-mini-instruct-int4-ov --model_repository_path c:\models --target_device GPU --task text_generation --enable_tool_guided_generation true --cache_dir .cache --max_num_batched_tokens 99999 ``` Use MCP server: @@ -110,7 +110,7 @@ The current weather in Tokyo is Overcast with a temperature of 9.4°C (feels lik :sync: Qwen3-30B-A3B-Instruct-2507 Pull and start OVMS: ```bat -ovms.exe --rest_port 8000 --source_model OpenVINO/Qwen3-30B-A3B-Instruct-2507-int4-ov --model_repository_path c:\models --tool_parser hermes3 --target_device GPU --task text_generation --cache_dir .cache +ovms.exe --rest_port 8000 --source_model OpenVINO/Qwen3-30B-A3B-Instruct-2507-int4-ov --model_repository_path c:\models --target_device GPU --task text_generation --cache_dir .cache ``` Use MCP server: @@ -129,7 +129,7 @@ Vision Language MoE model (35B total / 3B active parameters). Requires OpenVINO Pull and start OVMS: ```bat -ovms.exe --rest_port 8000 --source_model OpenVINO/Qwen3.6-35B-A3B-int4-ov --model_repository_path c:\models --reasoning_parser qwen3 --tool_parser qwen3coder --target_device GPU --task text_generation --cache_dir .cache --allowed_media_domains raw.githubusercontent.com +ovms.exe --rest_port 8000 --source_model OpenVINO/Qwen3.6-35B-A3B-int4-ov --model_repository_path c:\models --target_device GPU --task text_generation --cache_dir .cache --allowed_media_domains raw.githubusercontent.com ``` Use MCP server, with additional image of Gdańsk old town. VLM model deduces location and calls `get_weather` tool to summarize the weather conditions in the city. @@ -149,7 +149,7 @@ python openai_agent.py --query "What is the current weather in location depicted :sync: gpt-oss-20b Pull and start OVMS: ```bat -ovms.exe --rest_port 8000 --source_model OpenVINO/gpt-oss-20b-int4-ov --model_repository_path c:\models --tool_parser gptoss --reasoning_parser gptoss --task text_generation --target_device GPU +ovms.exe --rest_port 8000 --source_model OpenVINO/gpt-oss-20b-int4-ov --model_repository_path c:\models --task text_generation --target_device GPU ``` > **Note:** Continuous batching and paged attention are supported for GPT‑OSS. However, when deployed on GPU, the model may experience reduced accuracy under high‑concurrency workloads. This issue will be resolved in version 2026.1 and in the upcoming weekly release. CPU execution is not affected. @@ -184,7 +184,7 @@ Let me know if you'd like forecast details or anything else! :sync: Qwen3-8B Pull and start OVMS: ```bat -ovms.exe --rest_port 8000 --source_model OpenVINO/Qwen3-8B-int4-cw-ov --model_repository_path c:\models --tool_parser hermes3 --target_device NPU --task text_generation --cache_dir .cache --max_prompt_len 8000 +ovms.exe --rest_port 8000 --source_model OpenVINO/Qwen3-8B-int4-cw-ov --model_repository_path c:\models --target_device NPU --task text_generation --cache_dir .cache --max_prompt_len 8000 ``` Use MCP server: @@ -201,7 +201,7 @@ The current weather in Tokyo is overcast with a temperature of 9.4°C (feels lik :sync: Qwen3-4B Pull and start OVMS: ```bat -ovms.exe --rest_port 8000 --source_model FluidInference/qwen3-4b-int4-ov-npu --model_repository_path c:\models --tool_parser hermes3 --target_device NPU --task text_generation --cache_dir .cache --max_prompt_len 8000 +ovms.exe --rest_port 8000 --source_model FluidInference/qwen3-4b-int4-ov-npu --model_repository_path c:\models --target_device NPU --task text_generation --cache_dir .cache --max_prompt_len 8000 ``` Use MCP server: @@ -227,7 +227,7 @@ Pull and start OVMS: ```bash mkdir -p ${HOME}/models docker run -d --user $(id -u):$(id -g) --rm -p 8000:8000 -v ${HOME}/models:/models openvino/model_server:weekly \ ---rest_port 8000 --model_repository_path /models --source_model OpenVINO/Qwen3-VL-8B-Instruct-int4-ov --tool_parser hermes3 --task text_generation --pipeline_type VLM_CB --allowed_media_domains raw.githubusercontent.com +--rest_port 8000 --model_repository_path /models --source_model OpenVINO/Qwen3-VL-8B-Instruct-int4-ov --task text_generation --pipeline_type VLM_CB --allowed_media_domains raw.githubusercontent.com ``` Use MCP server, with additional image of Gdańsk old town. VLM model deduces location and calls `get_weather` tool to summarize the weather conditions in the city. @@ -254,7 +254,7 @@ Pull and start OVMS: ```bash mkdir -p ${HOME}/models docker run -d --user $(id -u):$(id -g) --rm -p 8000:8000 -v ${HOME}/models:/models openvino/model_server:weekly \ ---rest_port 8000 --model_repository_path /models --source_model OpenVINO/Qwen3-4B-int4-ov --tool_parser hermes3 --task text_generation +--rest_port 8000 --model_repository_path /models --source_model OpenVINO/Qwen3-4B-int4-ov --task text_generation ``` Use MCP server: @@ -273,7 +273,7 @@ Pull and start OVMS: ```bash mkdir -p ${HOME}/models docker run -d --user $(id -u):$(id -g) --rm -p 8000:8000 -v ${HOME}/models:/models openvino/model_server:weekly \ ---rest_port 8000 --model_repository_path /models --source_model OpenVINO/Phi-4-mini-instruct-int4-ov --tool_parser phi4 --task text_generation --max_num_batched_tokens 99999 +--rest_port 8000 --model_repository_path /models --source_model OpenVINO/Phi-4-mini-instruct-int4-ov --task text_generation --max_num_batched_tokens 99999 ``` Use MCP server: @@ -292,7 +292,7 @@ Pull and start OVMS: ```bash mkdir -p ${HOME}/models docker run -d --user $(id -u):$(id -g) --rm -p 8000:8000 -v ${HOME}/models:/models openvino/model_server:weekly \ ---rest_port 8000 --source_model OpenVINO/Qwen3-30B-A3B-Instruct-2507-int4-ov --model_repository_path /models --tool_parser hermes3 --task text_generation +--rest_port 8000 --source_model OpenVINO/Qwen3-30B-A3B-Instruct-2507-int4-ov --model_repository_path /models --task text_generation ``` Use MCP server: @@ -313,7 +313,7 @@ Pull and start OVMS: ```bash mkdir -p ${HOME}/models docker run -d --user $(id -u):$(id -g) --rm -p 8000:8000 -v ${HOME}/models:/models openvino/model_server:weekly \ ---rest_port 8000 --source_model OpenVINO/Qwen3.6-35B-A3B-int4-ov --model_repository_path /models --reasoning_parser qwen3 --tool_parser qwen3coder --task text_generation --allowed_media_domains raw.githubusercontent.com +--rest_port 8000 --source_model OpenVINO/Qwen3.6-35B-A3B-int4-ov --model_repository_path /models --task text_generation --allowed_media_domains raw.githubusercontent.com ``` Use MCP server, with additional image of Gdańsk old town. VLM model deduces location and calls `get_weather` tool to summarize the weather conditions in the city. @@ -336,7 +336,7 @@ Pull and start OVMS: mkdir -p ${HOME}/models docker run -d --user $(id -u):$(id -g) --rm -p 8000:8000 -v ${HOME}/models:/models openvino/model_server:weekly \ --rest_port 8000 --source_model OpenVINO/gpt-oss-20b-int4-ov --model_repository_path /models \ ---tool_parser gptoss --reasoning_parser gptoss --task text_generation +--task text_generation ``` Use MCP server: @@ -376,7 +376,7 @@ Pull and start OVMS: ```bash mkdir -p ${HOME}/models docker run -d --user $(id -u):$(id -g) --rm -p 8000:8000 -v ${HOME}/models:/models --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) openvino/model_server:weekly \ ---rest_port 8000 --model_repository_path /models --source_model OpenVINO/Qwen3-VL-8B-Instruct-int4-ov --tool_parser hermes3 --target_device GPU --task text_generation --pipeline_type VLM_CB --allowed_media_domains raw.githubusercontent.com +--rest_port 8000 --model_repository_path /models --source_model OpenVINO/Qwen3-VL-8B-Instruct-int4-ov --target_device GPU --task text_generation --pipeline_type VLM_CB --allowed_media_domains raw.githubusercontent.com ``` Use MCP server, with additional image of Gdańsk old town. VLM model deduces location and calls `get_weather` tool to summarize the weather conditions in the city. @@ -403,7 +403,7 @@ Pull and start OVMS: ```bash mkdir -p ${HOME}/models docker run -d --user $(id -u):$(id -g) --rm -p 8000:8000 -v ${HOME}/models:/models --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) openvino/model_server:weekly \ ---rest_port 8000 --model_repository_path /models --source_model OpenVINO/Qwen3-4B-int4-ov --tool_parser hermes3 --target_device GPU --task text_generation +--rest_port 8000 --model_repository_path /models --source_model OpenVINO/Qwen3-4B-int4-ov --target_device GPU --task text_generation ``` Use MCP server: @@ -422,7 +422,7 @@ Pull and start OVMS: ```bash mkdir -p ${HOME}/models docker run -d --user $(id -u):$(id -g) --rm -p 8000:8000 -v ${HOME}/models:/models --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) openvino/model_server:weekly \ ---rest_port 8000 --model_repository_path /models --source_model OpenVINO/Phi-4-mini-instruct-int4-ov --tool_parser phi4 --task text_generation --target_device GPU --max_num_batched_tokens 99999 +--rest_port 8000 --model_repository_path /models --source_model OpenVINO/Phi-4-mini-instruct-int4-ov --task text_generation --target_device GPU --max_num_batched_tokens 99999 ``` Use MCP server: @@ -441,7 +441,7 @@ Pull and start OVMS: ```bash mkdir -p ${HOME}/models docker run -d --user $(id -u):$(id -g) --rm -p 8000:8000 -v ${HOME}/models:/models --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) openvino/model_server:weekly \ ---rest_port 8000 --source_model OpenVINO/Qwen3-30B-A3B-Instruct-2507-int4-ov --model_repository_path /models --tool_parser hermes3 --target_device GPU --task text_generation --enable_tool_guided_generation true +--rest_port 8000 --source_model OpenVINO/Qwen3-30B-A3B-Instruct-2507-int4-ov --model_repository_path /models --target_device GPU --task text_generation --enable_tool_guided_generation true ``` Use MCP server: @@ -462,7 +462,7 @@ Pull and start OVMS: ```bash mkdir -p ${HOME}/models docker run -d --user $(id -u):$(id -g) --rm -p 8000:8000 -v ${HOME}/models:/models --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) openvino/model_server:weekly \ ---rest_port 8000 --source_model OpenVINO/Qwen3.6-35B-A3B-int4-ov --model_repository_path /models --reasoning_parser qwen3 --tool_parser qwen3coder --target_device GPU --task text_generation --allowed_media_domains raw.githubusercontent.com +--rest_port 8000 --source_model OpenVINO/Qwen3.6-35B-A3B-int4-ov --model_repository_path /models --target_device GPU --task text_generation --allowed_media_domains raw.githubusercontent.com ``` Use MCP server, with additional image of Gdańsk old town. VLM model deduces location and calls `get_weather` tool to summarize the weather conditions in the city. @@ -485,7 +485,7 @@ Pull and start OVMS: mkdir -p ${HOME}/models docker run -d --user $(id -u):$(id -g) --rm -p 8000:8000 -v ${HOME}/models:/models --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) openvino/model_server:weekly \ --rest_port 8000 --source_model OpenVINO/gpt-oss-20b-int4-ov --model_repository_path /models \ ---tool_parser gptoss --reasoning_parser gptoss --target_device GPU --task text_generation +--target_device GPU --task text_generation ``` > **Note:** Continuous batching and paged attention are supported for GPT‑OSS. However, when deployed on GPU, the model may experience reduced accuracy under high‑concurrency workloads. This issue will be resolved in version 2026.1 and in the upcoming weekly release. CPU execution is not affected. @@ -525,7 +525,7 @@ Pull and start OVMS: ```bash mkdir -p ${HOME}/models docker run -d --user $(id -u):$(id -g) --rm -p 8000:8000 -v ${HOME}/models:/models --device /dev/accel --group-add=$(stat -c "%g" /dev/dri/render* | head -1) openvino/model_server:weekly \ ---rest_port 8000 --model_repository_path /models --source_model OpenVINO/Qwen3-8B-int4-cw-ov --tool_parser hermes3 --target_device NPU --task text_generation --max_prompt_len 8000 +--rest_port 8000 --model_repository_path /models --source_model OpenVINO/Qwen3-8B-int4-cw-ov --target_device NPU --task text_generation --max_prompt_len 8000 ``` Use MCP server: @@ -544,7 +544,7 @@ Pull and start OVMS: ```bash mkdir -p ${HOME}/models docker run -d --user $(id -u):$(id -g) --rm -p 8000:8000 -v ${HOME}/models:/models --device /dev/accel --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) openvino/model_server:weekly \ ---rest_port 8000 --model_repository_path /models --source_model FluidInference/qwen3-4b-int4-ov-npu --tool_parser hermes3 --target_device NPU --task text_generation --max_prompt_len 8000 +--rest_port 8000 --model_repository_path /models --source_model FluidInference/qwen3-4b-int4-ov-npu --target_device NPU --task text_generation --max_prompt_len 8000 ``` Use MCP server: @@ -569,7 +569,7 @@ Pull and start OVMS: ```bash mkdir -p ${HOME}/models docker run -d --user $(id -u):$(id -g) --rm -p 8000:8000 -v ${HOME}/models:/models openvino/model_server:weekly \ ---rest_port 8000 --model_repository_path /models --source_model OpenVINO/Qwen3-8B-int4-ov --tool_parser hermes3 --task text_generation +--rest_port 8000 --model_repository_path /models --source_model OpenVINO/Qwen3-8B-int4-ov --task text_generation ``` You can try also similar implementation based on llama_index library working the same way like openai-agent: @@ -615,12 +615,12 @@ Run `export_model.py` script to download and quantize the model: > **Note:** The users in China need to set environment variable HF_ENDPOINT="https://hf-mirror.com" or "https://www.modelscope.cn/models" before running the export script to connect to the HF Hub. ```text -python export_model.py text_generation --source_model meta-llama/Llama-3.2-3B-Instruct --weight-format int4 --config_file_path models/config.json --model_repository_path models --tool_parser llama3 +python export_model.py text_generation --source_model meta-llama/Llama-3.2-3B-Instruct --weight-format int4 --config_file_path models/config.json --model_repository_path models curl -L -o models/meta-llama/Llama-3.2-3B-Instruct/chat_template.jinja https://raw.githubusercontent.com/vllm-project/vllm/refs/tags/v0.9.0/examples/tool_chat_template_llama3.2_json.jinja ``` > **Note:** To use these models on NPU, set `--weight-format` to either **int4** or **nf4**. When specifying `--extra_quantization_params`, ensure that `ratio` is set to **1.0** and `group_size` is set to **-1** or **128**. For example: ```text -python export_model.py text_generation --source_model meta-llama/Llama-3.2-3B-Instruct --weight-format nf4 --config_file_path models/config.json --model_repository_path models --tool_parser llama3 --extra_quantization_params "--library transformers --sym group_size -1" +python export_model.py text_generation --source_model meta-llama/Llama-3.2-3B-Instruct --weight-format nf4 --config_file_path models/config.json --model_repository_path models --extra_quantization_params "--library transformers --sym group_size -1" ``` For more details, see [OpenVINO GenAI on NPU](https://docs.openvino.ai/nightly/openvino-workflow-generative/inference-with-genai/inference-with-genai-on-npu.html). \ No newline at end of file diff --git a/demos/continuous_batching/long_context/README.md b/demos/continuous_batching/long_context/README.md index ff0af163d1..9215f6bf40 100644 --- a/demos/continuous_batching/long_context/README.md +++ b/demos/continuous_batching/long_context/README.md @@ -44,12 +44,12 @@ mkdir models ::: {tab-item} GPU :sync: GPU ```bash -docker run --user $(id -u):$(id -g) -d --rm -v $(pwd)/models:/models:rw -p 8000:8000 --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) openvino/model_server:latest-gpu --rest_port 8000 --model_repository_path /models --source_model OpenVINO/gpt-oss-20b-int4-ov --tool_parser gptoss --reasoning_parser gptoss --task text_generation --kv_cache_precision u4 --target_device GPU --cache_size 5 --max_num_batched_tokens 4096 +docker run --user $(id -u):$(id -g) -d --rm -v $(pwd)/models:/models:rw -p 8000:8000 --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) openvino/model_server:latest-gpu --rest_port 8000 --model_repository_path /models --source_model OpenVINO/gpt-oss-20b-int4-ov --task text_generation --kv_cache_precision u4 --target_device GPU --cache_size 5 --max_num_batched_tokens 4096 ``` ::: :::{tab-item} NPU ```bash -docker run --user $(id -u):$(id -g) -d --rm -v $(pwd)/models:/models:rw -p 8000:8000 --device /dev/accel --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) openvino/model_server:latest-gpu --rest_port 8000 --model_repository_path /models --source_model OpenVINO/Qwen3-8B-int4-cw-ov --max_prompt_len 16000 --tool_parser hermes3 --task text_generation --target_device NPU +docker run --user $(id -u):$(id -g) -d --rm -v $(pwd)/models:/models:rw -p 8000:8000 --device /dev/accel --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) openvino/model_server:latest-gpu --rest_port 8000 --model_repository_path /models --source_model OpenVINO/Qwen3-8B-int4-cw-ov --max_prompt_len 16000 --task text_generation --target_device NPU ``` ::: :::: diff --git a/demos/integration_with_OpenWebUI/README.md b/demos/integration_with_OpenWebUI/README.md index f077c98de9..e89c121421 100644 --- a/demos/integration_with_OpenWebUI/README.md +++ b/demos/integration_with_OpenWebUI/README.md @@ -31,7 +31,7 @@ This demo can be followed without changes on Panther Lake host with 64GB RAM and :sync: Windows ```bat mkdir models -ovms.exe --pull --source_model OpenVINO/gpt-oss-20b-int4-ov --model_repository_path models --tool_parser gptoss --reasoning_parser gptoss --task text_generation --target_device GPU +ovms.exe --pull --source_model OpenVINO/gpt-oss-20b-int4-ov --model_repository_path models --task text_generation --target_device GPU ovms.exe --add_to_config --config_path models\config.json --model_path OpenVINO\gpt-oss-20b-int4-ov --model_name ovms-model ovms.exe --rest_port 8000 --config_path models\config.json --allowed_media_domains raw.githubusercontent.com ``` @@ -40,7 +40,7 @@ ovms.exe --rest_port 8000 --config_path models\config.json --allowed_media_domai :sync: Linux ```bash mkdir models -docker run --rm -u $(id -u):$(id -g) -v $PWD/models:/models --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) openvino/model_server:weekly --pull --source_model OpenVINO/gpt-oss-20b-int4-ov --model_repository_path /models --task text_generation --tool_parser gptoss --reasoning_parser gptoss --target_device GPU +docker run --rm -u $(id -u):$(id -g) -v $PWD/models:/models --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) openvino/model_server:weekly --pull --source_model OpenVINO/gpt-oss-20b-int4-ov --model_repository_path /models --task text_generation --target_device GPU docker run --rm -u $(id -u):$(id -g) -v $PWD/models:/models openvino/model_server:weekly --add_to_config --config_path /models/config.json --model_path OpenVINO/gpt-oss-20b-int4-ov --model_name ovms-model docker run -d -u $(id -u):$(id -g) -v $PWD/models:/models -p 8000:8000 --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) openvino/model_server:weekly --rest_port 8000 --config_path /models/config.json --allowed_media_domains raw.githubusercontent.com ``` diff --git a/demos/llm_npu/README.md b/demos/llm_npu/README.md index f2d60c2660..dafb890d73 100644 --- a/demos/llm_npu/README.md +++ b/demos/llm_npu/README.md @@ -26,7 +26,7 @@ Multiple [OpenVINO models optimized for NPU](https://huggingface.co/collections/ :sync: Linux ```bash mkdir -p models -docker run -d --rm -u $(id -u):$(id -g) -v $(pwd)/models:/models:rw openvino/model_server:latest-gpu --pull --source_model OpenVINO/Qwen3-8B-int4-cw-ov --model_repository_path /models --target_device NPU --task text_generation --tool_parser hermes3 --cache_dir /models/.ov_cache --enable_prefix_caching true --max_prompt_len 2000 +docker run -d --rm -u $(id -u):$(id -g) -v $(pwd)/models:/models:rw openvino/model_server:latest-gpu --pull --source_model OpenVINO/Qwen3-8B-int4-cw-ov --model_repository_path /models --target_device NPU --task text_generation --cache_dir /models/.ov_cache --enable_prefix_caching true --max_prompt_len 2000 docker run -d --rm -u $(id -u):$(id -g) -v $(pwd)/models:/models:rw openvino/model_server:latest-gpu --add_to_config --config_path /models/config.json --model_name OpenVINO/Qwen3-8B-int4-cw-ov --model_path /models/OpenVINO/Qwen3-8B-int4-cw-ov ``` ::: @@ -34,7 +34,7 @@ docker run -d --rm -u $(id -u):$(id -g) -v $(pwd)/models:/models:rw openvino/mod :sync: Windows ```bat mkdir models -ovms.exe --pull --source_model OpenVINO/Qwen3-8B-int4-cw-ov --model_repository_path models --target_device NPU --task text_generation --tool_parser hermes3 --cache_dir .ov_cache --enable_prefix_caching true --max_prompt_len 2000 +ovms.exe --pull --source_model OpenVINO/Qwen3-8B-int4-cw-ov --model_repository_path models --target_device NPU --task text_generation --cache_dir .ov_cache --enable_prefix_caching true --max_prompt_len 2000 ovms.exe --add_to_config --config_path models\config.json --model_name OpenVINO/Qwen3-8B-int4-cw-ov --model_path OpenVINO\Qwen3-8B-int4-cw-ov ``` ::: diff --git a/docs/starting_server.md b/docs/starting_server.md index c52bcd1652..3be4cf4ec8 100644 --- a/docs/starting_server.md +++ b/docs/starting_server.md @@ -106,7 +106,7 @@ hf download OpenVINO/Qwen3-0.6B-int4-ov --local-dir Qwen3-0.6B-int4-ov ```bash docker run -d --rm -v ${PWD}/Qwen3-0.6B-int4-ov:/model -p 8000:8000 openvino/model_server:latest \ ---model_path /model/ --model_name qwen3-0.6 --rest_port 8000 --task text_generation --target_device CPU --tool_parser hermes3 --reasoning_parser qwen3 +--model_path /model/ --model_name qwen3-0.6 --rest_port 8000 --task text_generation --target_device CPU ``` - Expose the container ports to **open ports** on your host or virtual machine. @@ -115,7 +115,7 @@ docker run -d --rm -v ${PWD}/Qwen3-0.6B-int4-ov:/model -p 8000:8000 openvino/mod :::{tab-item} On Baremetal Host :sync: baremetal ```text -ovms --model_path Qwen3-0.6B-int4-ov --model_name qwen3-0.6 --rest_port 8000 --task text_generation --target_device CPU --tool_parser hermes3 --reasoning_parser qwen3 +ovms --model_path Qwen3-0.6B-int4-ov --model_name qwen3-0.6 --rest_port 8000 --task text_generation --target_device CPU ``` ::: :::: diff --git a/extras/llama_swap/README.md b/extras/llama_swap/README.md index 796a789dd2..7bff58e31f 100644 --- a/extras/llama_swap/README.md +++ b/extras/llama_swap/README.md @@ -16,9 +16,9 @@ While this tool was implemented for llama-cpp project, it can be easily enabled ```bat ovms --pull --task embeddings --model_name OpenVINO/Qwen3-Embedding-0.6B-int8-ov --target_device GPU --cache_dir .ov_cache --pooling LAST -ovms --pull --task text_generation --model_name OpenVINO/Qwen3-4B-int4-ov --target_device GPU --cache_dir .ov_cache --tool_parser hermes3 +ovms --pull --task text_generation --model_name OpenVINO/Qwen3-4B-int4-ov --target_device GPU --cache_dir .ov_cache ovms --pull --task text_generation --model_name OpenVINO/InternVL2-2B-int4-ov --target_device GPU --cache_dir .ov_cache -ovms --pull --task text_generation --model_name OpenVINO/Mistral-7B-Instruct-v0.3-int4-ov --target_device GPU --cache_dir .ov_cache --tool_parser mistral +ovms --pull --task text_generation --model_name OpenVINO/Mistral-7B-Instruct-v0.3-int4-ov --target_device GPU --cache_dir .ov_cache ``` ## Configure config.yaml for llama_swap