diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 04fe81cbea..561b732604 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -315,6 +315,12 @@ RUN rm -f /usr/lib64/cmake/OpenSSL/OpenSSLConfig.cmake # hadolint ignore=SC2046 RUN bazel build --jobs=$JOBS ${debug_bazel_flags} ${minitrace_flags} //src:ovms $(if [ "$OPTIMIZE_BUILDING_TESTS" == "1" ] ; then echo -n //src:ovms_test; fi) +# espeak-ng is built as a separate step, independent of the OVMS binary. +# Set ESPEAK=0 to skip the espeak build. +ARG ESPEAK=1 +# hadolint ignore=DL3059 +RUN if [ "$ESPEAK" == "1" ]; then bazel build --jobs=$JOBS ${debug_bazel_flags} //third_party:espeak_ng //third_party:espeak_ng_data; fi + # Tests execution COPY ci/check_coverage.bat /ovms/ ARG CHECK_COVERAGE=0 diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index ed38a3cb02..59a7fbdf23 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -331,6 +331,12 @@ ARG OPTIMIZE_BUILDING_TESTS=0 # hadolint ignore=SC2046 RUN if [ "$FUZZER_BUILD" == "0" ]; then bazel build --jobs=$JOBS ${debug_bazel_flags} ${minitrace_flags} //src:ovms $(if [ "${OPTIMIZE_BUILDING_TESTS}" == "1" ] ; then echo -n //src:ovms_test; fi); fi; +# espeak-ng is built as a separate step, independent of the OVMS binary. +# Set ESPEAK=0 to skip the espeak build. +ARG ESPEAK=1 +# hadolint ignore=DL3059 +RUN if [ "$ESPEAK" == "1" ]; then bazel build --jobs=$JOBS ${debug_bazel_flags} //third_party:espeak_ng //third_party:espeak_ng_data; fi + ARG RUN_TESTS=0 RUN if [ "$RUN_TESTS" == "1" ] ; then mkdir -p demos/common/export_models/ && mv export_model.py demos/common/export_models/ && ./prepare_llm_models.sh /ovms/src/test/llm_testing docker && ./run_unit_tests.sh ; fi diff --git a/Makefile b/Makefile index 654875576c..ff800383ca 100644 --- a/Makefile +++ b/Makefile @@ -148,13 +148,8 @@ else ifeq ($(findstring redhat,$(BASE_OS)),redhat) else $(error BASE_OS must be either ubuntu or redhat) endif -ifeq ($(ESPEAK),1) - ESPEAK_PARAMS = " --//:espeak=on" -else - ESPEAK_PARAMS = " --//:espeak=off" -endif -CAPI_FLAGS = "--strip=$(STRIP)"$(BAZEL_DEBUG_BUILD_FLAGS)" --config=mp_off_py_off"$(OV_TRACING_PARAMS)$(TARGET_DISTRO_PARAMS)$(ESPEAK_PARAMS) -BAZEL_DEBUG_FLAGS="--strip=$(STRIP)"$(BAZEL_DEBUG_BUILD_FLAGS)$(DISABLE_PARAMS)$(FUZZER_BUILD_PARAMS)$(OV_TRACING_PARAMS)$(TARGET_DISTRO_PARAMS)$(ESPEAK_PARAMS)$(REPO_ENV) +CAPI_FLAGS = "--strip=$(STRIP)"$(BAZEL_DEBUG_BUILD_FLAGS)" --config=mp_off_py_off"$(OV_TRACING_PARAMS)$(TARGET_DISTRO_PARAMS) +BAZEL_DEBUG_FLAGS="--strip=$(STRIP)"$(BAZEL_DEBUG_BUILD_FLAGS)$(DISABLE_PARAMS)$(FUZZER_BUILD_PARAMS)$(OV_TRACING_PARAMS)$(TARGET_DISTRO_PARAMS)$(REPO_ENV) # Option to Override release image. # Release image OS *must have* glibc version >= glibc version on BASE_OS: @@ -249,7 +244,8 @@ BUILD_ARGS = --build-arg http_proxy=$(HTTP_PROXY)\ --build-arg JOBS=$(JOBS)\ --build-arg CAPI_FLAGS=$(CAPI_FLAGS)\ --build-arg VERBOSE_LOGS=$(VERBOSE_LOGS)\ - --build-arg KONFLUX=$(KONFLUX) + --build-arg KONFLUX=$(KONFLUX)\ + --build-arg ESPEAK=$(ESPEAK) .PHONY: default docker_build \ diff --git a/common_settings.bzl b/common_settings.bzl index 2bc23a3430..a60243df16 100644 --- a/common_settings.bzl +++ b/common_settings.bzl @@ -20,7 +20,7 @@ load("@bazel_skylib//lib:selects.bzl", "selects") load("@mediapipe//mediapipe/framework:more_selects.bzl", "more_selects") load("@bazel_skylib//rules:common_settings.bzl", "string_flag") -load("//:distro.bzl", "distro_flag", "espeak_flag") +load("//:distro.bzl", "distro_flag") # cc_library rule wrapper that will accept the same arguments but if user will not provide # copts, linkopts, local_defines it will set them to the defaults @@ -58,7 +58,6 @@ def ovms_cc_library(**kwargs): def create_config_settings(): distro_flag() - espeak_flag() native.config_setting( name = "disable_mediapipe", define_values = { diff --git a/create_package.sh b/create_package.sh index 69812d7302..6220a93ec9 100755 --- a/create_package.sh +++ b/create_package.sh @@ -36,6 +36,7 @@ ESPEAK_DATA_SRC=$(find /ovms/bazel-out/k8-*/bin/external/espeak_ng -type d -name if [ -n "$ESPEAK_DATA_SRC" ] && [ -d "$ESPEAK_DATA_SRC" ] ; then mkdir -p /ovms_release/share cp -rL "$ESPEAK_DATA_SRC" /ovms_release/share/ ; + chmod -R u+w /ovms_release/share/espeak-ng-data 2>/dev/null || true ; fi # Resolve the packaged eSpeak shared object dynamically so version bumps # do not require touching this script. diff --git a/demos/audio/README.md b/demos/audio/README.md index 8b13a657b1..2956aa9c92 100644 --- a/demos/audio/README.md +++ b/demos/audio/README.md @@ -8,116 +8,80 @@ Check supported [Speech Recognition Models](https://openvinotoolkit.github.io/op ## Prerequisites -**OVMS version 2025.4** This demo require version 2025.4 or nightly release. - -**Model preparation**: Python 3.10 or higher with pip +**OVMS version 2026.3** This demo require version 2026.3 or nightly release. **Model Server deployment**: Installed Docker Engine or OVMS binary package according to the [baremetal deployment guide](../../docs/deploying_server_baremetal.md) **Client**: curl or Python for using OpenAI client package ## Speech generation -### Prepare speaker embeddings -When generating speech you can use default speaker voice or you can prepare your own speaker embedding file. Here you can see how to do it with downloaded file from online repository, but you can try with your own speech recording as well: -```console -pip install -r https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/audio/requirements.txt -mkdir audio_samples -curl --create-dirs "https://www.voiptroubleshooter.com/open_speech/american/OSR_us_000_0032_8k.wav" -o audio_samples/audio.wav -curl --create-dirs https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/audio/create_speaker_embedding.py -o models/speakers/create_speaker_embedding.py -python models/speakers/create_speaker_embedding.py audio_samples/audio.wav models/speakers/voice1.bin -``` - +Kokoro is the primary example in this demo, but SpeechT5 remains supported for existing deployments. ### Model preparation -Supported models should use the topology of [microsoft/speecht5_tts](https://huggingface.co/microsoft/speecht5_tts) which needs to be converted to IR format before using in OVMS. - -Specific OVMS pull mode example for models requiring conversion is described in the [Ovms pull mode](../../docs/pull_hf_models.md#pulling-models-outside-openvino-organization) - -Or you can use the python export_model.py script described below. - -Here, the original Text to Speech model will be converted to IR format and optionally quantized. -That ensures faster initialization time, better performance and lower memory consumption. -Execution parameters will be defined inside the `graph.pbtxt` file. - -Download export script, install it's dependencies and create directory for the models: -```console -curl https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/common/export_models/export_model.py -o export_model.py -pip install -r https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/common/export_models/requirements.txt -mkdir models -``` - -Run `export_model.py` script to download and quantize the model: - -> **Note:** The users in China need to set environment variable HF_ENDPOINT="https://hf-mirror.com" before running the export script to connect to the HF Hub. -> **Note:** Exporting `microsoft/speecht5_tts` model requires Python 3.10 - -**CPU** -```console -python export_model.py text2speech --source_model microsoft/speecht5_tts --weight-format fp16 --model_name microsoft/speecht5_tts --config_file_path models/config.json --model_repository_path models --overwrite_models --vocoder microsoft/speecht5_hifigan --speaker_name voice1 --speaker_path models/speakers/voice1.bin -``` +This demo uses a pre-exported OpenVINO IR model [luis-castillo/Kokoro-82M-OpenVINO-FP16-OVMS](https://huggingface.co/luis-castillo/Kokoro-82M-OpenVINO-FP16-OVMS) available on HuggingFace. +The model can be pulled directly by OVMS without any conversion step. -> **Note:** Change the `--weight-format` to quantize the model to `int8` precision to reduce memory consumption and improve performance. -> **Note:** `speaker_name` and `speaker_path` may be omitted if the default model voice is sufficient +> **Note:** The users in China need to set environment variable HF_ENDPOINT="https://hf-mirror.com" before starting OVMS to connect to the HF Hub. +> **Note:** Kokoro voices are loaded from the `voices/` directory in the model repository and can be selected per request with the `voice` field. -The default configuration should work in most cases but the parameters can be tuned via `export_model.py` script arguments. Run the script with `--help` argument to check available parameters and see the [T2s calculator documentation](../../docs/speech_generation/reference.md) to learn more about configuration options and limitations. +See the [T2s calculator documentation](../../docs/speech_generation/reference.md) to learn more about configuration options and limitations. ### Deployment +:::{dropdown} **Deploying with Docker** + **CPU** -Running this command starts the container with CPU only target device: +Running this command starts the container with CPU target device: ```bash -mkdir -p models -docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 -v $(pwd)/models:/models:rw openvino/model_server:latest --rest_port 8000 --model_path /models/microsoft/speecht5_tts --model_name microsoft/speecht5_tts +mkdir models +docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 -v $(pwd)/models:/models:rw openvino/model_server:latest --rest_port 8000 --source_model luis-castillo/Kokoro-82M-OpenVINO-FP16-OVMS --model_repository_path /models --model_name Kokoro-82M-OpenVINO-FP16-OVMS --target_device CPU --task text2speech ``` -**Deploying on Bare Metal** +**GPU** -```bat +In case you want to use GPU device, add extra docker parameters `--device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1)` and use the GPU image: +```bash mkdir models -ovms --rest_port 8000 --model_path models/microsoft/speecht5_tts --model_name microsoft/speecht5_tts +docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -v $(pwd)/models:/models:rw openvino/model_server:latest-gpu --rest_port 8000 --source_model luis-castillo/Kokoro-82M-OpenVINO-FP16-OVMS --model_repository_path /models --model_name Kokoro-82M-OpenVINO-FP16-OVMS --target_device GPU --task text2speech ``` -### Request Generation - -:::{dropdown} **Unary call with curl with default voice** - +**NPU** +In case you want to use NPU device, add extra docker parameters `--device /dev/accel --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1)` and use the GPU image: ```bash -curl http://localhost:8000/v3/audio/speech -H "Content-Type: application/json" -d "{\"model\": \"microsoft/speecht5_tts\", \"input\": \"The quick brown fox jumped over the lazy dog\"}" -o speech.wav +mkdir models +docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 --device /dev/accel --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -v $(pwd)/models:/models:rw openvino/model_server:latest-gpu --rest_port 8000 --source_model luis-castillo/Kokoro-82M-OpenVINO-FP16-OVMS --model_repository_path /models --model_name Kokoro-82M-OpenVINO-FP16-OVMS --target_device NPU --task text2speech ``` ::: -:::{dropdown} **Unary call with OpenAI python library with default voice** - -```python -from pathlib import Path -from openai import OpenAI - -prompt = "The quick brown fox jumped over the lazy dog" -filename = "speech.wav" -url="http://localhost:8000/v3" - - -speech_file_path = Path(__file__).parent / "speech.wav" -client = OpenAI(base_url=url, api_key="not_used") +:::{dropdown} **Deploying on Bare Metal** -with client.audio.speech.with_streaming_response.create( - model="microsoft/speecht5_tts", - voice=None, - input=prompt -) as response: - response.stream_to_file(speech_file_path) +**CPU** +```bat +mkdir c:\models +ovms --rest_port 8000 --source_model luis-castillo/Kokoro-82M-OpenVINO-FP16-OVMS --model_repository_path c:\models --model_name Kokoro-82M-OpenVINO-FP16-OVMS --target_device CPU +``` +**GPU** +```bat +mkdir c:\models +ovms --rest_port 8000 --source_model luis-castillo/Kokoro-82M-OpenVINO-FP16-OVMS --model_repository_path c:\models --model_name Kokoro-82M-OpenVINO-FP16-OVMS --target_device GPU +``` -print("Generation finished") +**NPU** +```bat +mkdir c:\models +ovms --rest_port 8000 --source_model luis-castillo/Kokoro-82M-OpenVINO-FP16-OVMS --model_repository_path c:\models --model_name Kokoro-82M-OpenVINO-FP16-OVMS --target_device NPU ``` ::: +### Request Generation + :::{dropdown} **Unary call with curl** ```bash -curl http://localhost:8000/v3/audio/speech -H "Content-Type: application/json" -d "{\"model\": \"microsoft/speecht5_tts\", \"voice\":\"voice1\", \"input\": \"The quick brown fox jumped over the lazy dog\"}" -o speech.wav +curl http://localhost:8000/v3/audio/speech -H "Content-Type: application/json" -d "{\"model\": \"Kokoro-82M-OpenVINO-FP16-OVMS\", \"voice\": \"af_alloy\", \"input\": \"The quick brown fox jumped over the lazy dog\"}" -o speech.wav ``` ::: @@ -136,8 +100,8 @@ speech_file_path = Path(__file__).parent / "speech.wav" client = OpenAI(base_url=url, api_key="not_used") with client.audio.speech.with_streaming_response.create( - model="microsoft/speecht5_tts", - voice="voice1", + model="Kokoro-82M-OpenVINO-FP16-OVMS", + voice="af_alloy", input=prompt ) as response: response.stream_to_file(speech_file_path) @@ -152,20 +116,25 @@ Play speech.wav file to check generated speech. ## Benchmarking speech generation An asynchronous benchmarking client can be used to access the model server performance with various load conditions. Below are execution examples captured on Intel(R) Core(TM) Ultra 7 258V. +> **Note:** `RTFx` (Real-Time Factor, inverted) is calculated as `generated_audio_duration / generation_time`. +> Values greater than `1.0x` mean faster-than-real-time generation, while values below `1.0x` mean slower-than-real-time. + ```console -pip install -r https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/benchmark/v3/requirements.txt -curl https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/benchmark/v3/benchmark.py -o benchmark.py -python benchmark.py --api_url http://localhost:8000/v3/audio/speech --model microsoft/speecht5_tts --batch_size 1 --limit 100 --request_rate inf --backend text2speech --dataset edinburghcstr/ami --hf-subset ihm --tokenizer openai/whisper-large-v3-turbo --trust-remote-code True -Number of documents: 100 -100%|████████████████████████████████████████████████████████████████████████████████| 100/100 [01:58<00:00, 1.19s/it] -Tokens: 1802 -Success rate: 100.0%. (100/100) -Throughput - Tokens per second: 15.2 -Mean latency: 63653.98 ms -Median latency: 66736.83 ms -Average document length: 18.02 tokens +pip install -r https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/mkulakow/kokoro_improvements/demos/benchmark/v3/requirements.txt +curl https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/mkulakow/kokoro_improvements/demos/benchmark/v3/benchmark.py -o benchmark.py +python benchmark.py --api_url http://localhost:8000/v3/audio/speech --model Kokoro-82M-OpenVINO-FP16-OVMS --batch_size 1 --limit 100 --request_rate inf --backend text2speech --dataset edinburghcstr/ami --hf-subset ihm --voice af_alloy +Number of documents: 1000 +100%|█████████████████████████████████████████████████████████████████████████████████| 1000/1000 [15:53<00:00, 1.05it/s] +Success rate: 100.0%. (1000/1000) +Mean latency: 742.48 ms +Median latency: 662.10 ms +Mean RTFx: 5.091x +Median RTFx: 5.173x ``` +> **Note:** `RTFx` (Real-Time Factor, inverted) is calculated as `generated_audio_duration / generation_time`. +> Values greater than `1.0x` mean faster-than-real-time generation, while values below `1.0x` mean slower-than-real-time. + ## Transcription ### Model preparation Many variances of Whisper models can be deployed in a single command by using pre-configured models from [OpenVINO HuggingFace organization](https://huggingface.co/collections/OpenVINO/speech-to-text) and used both for translations and transcriptions endpoints. @@ -235,7 +204,7 @@ ovms --rest_port 8000 --model_path /models/openai/whisper-large-v3-turbo --model The default configuration should work in most cases but the parameters can be tuned via `export_model.py` script arguments. Run the script with `--help` argument to check available parameters and see the [s2t calculator documentation](../../docs/speech_recognition/reference.md) to learn more about configuration options and limitations. ### Request Generation -Transcript file that was previously generated with audio/speech endpoint. +Transcribe the speech.wav file generated in the [Speech generation](#speech-generation) section. > **Note:** Streaming responses are supported for `audio/transcriptions`. `audio/translations` does not support streaming. @@ -360,33 +329,13 @@ Average document length: 10.948 tokens ``` ## Translation -To test translations endpoint we first need to prepare audio file with speech in language other than English, e.g. Spanish. To generate such sample we will use finetuned version of microsoft/speecht5_tts model. - -**Deploying with Docker** +To test the translations endpoint we first need to prepare an audio file with speech in a language other than English, e.g. Spanish. To generate such a sample, follow the [Speech generation](#speech-generation) section to deploy Kokoro and then run: -```bash -mkdir -p models - -python export_model.py text2speech --source_model Sandiago21/speecht5_finetuned_facebook_voxpopuli_spanish --weight-format fp16 --model_name speecht5_tts_spanish --config_file_path models/config.json --model_repository_path models --overwrite_models --vocoder microsoft/speecht5_hifigan - -docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 -v $(pwd)/models:/models:rw openvino/model_server:latest --rest_port 8000 --model_path /models/speecht5_tts_spanish --model_name speecht5_tts_spanish - -curl http://localhost:8000/v3/audio/speech -H "Content-Type: application/json" -d "{\"model\": \"speecht5_tts_spanish\", \"input\": \"Madrid es la capital de España\"}" -o speech_spanish.wav -``` - -**Deploying on Bare Metal** - -```bat -mkdir models - -python export_model.py text2speech --source_model Sandiago21/speecht5_finetuned_facebook_voxpopuli_spanish --weight-format fp16 --model_name speecht5_tts_spanish --config_file_path models/config.json --model_repository_path models --overwrite_models --vocoder microsoft/speecht5_hifigan - -ovms --rest_port 8000 --model_path models/speecht5_tts_spanish --model_name speecht5_tts_spanish - -curl http://localhost:8000/v3/audio/speech -H "Content-Type: application/json" -d "{\"model\": \"speecht5_tts_spanish\", \"input\": \"Madrid es la capital de España\"}" -o speech_spanish.wav +```console +curl http://localhost:8000/v3/audio/speech -H "Content-Type: application/json" -d "{\"model\": \"Kokoro-82M-OpenVINO-FP16-OVMS\", \"voice\": \"em_alex\", \"input\": \"Madrid es la capital de España\"}" -o speech_spanish.wav ``` -### Model preparation +### Deployment Whisper models can be deployed in a single command by using pre-configured models from [OpenVINO HuggingFace organization](https://huggingface.co/collections/OpenVINO/speech-to-text) and used both for translations and transcriptions endpoints. Here is an example of OpenVINO/whisper-large-v3-fp16-ov deployment: @@ -427,7 +376,7 @@ ovms --rest_port 8000 --source_model OpenVINO/whisper-large-v3-fp16-ov --model_r ::: ### Request Generation -Transcript and translate file that was previously generated with audio/speech endpoint. +Translate the speech_spanish.wav file generated above. :::{dropdown} **Unary call with cURL** diff --git a/demos/benchmark/v3/benchmark.py b/demos/benchmark/v3/benchmark.py index 88c9aa8f18..f08052e3f7 100644 --- a/demos/benchmark/v3/benchmark.py +++ b/demos/benchmark/v3/benchmark.py @@ -52,6 +52,7 @@ parser.add_argument('--model', required=False, default='Alibaba-NLP/gte-large-en-v1.5', help='HF model name', dest='model') parser.add_argument('--request_rate', required=False, default='inf', help='Average amount of requests per seconds in random distribution', dest='request_rate') parser.add_argument('--batch_size', required=False, type=int, default=16, help='Number of strings in every requests', dest='batch_size') +parser.add_argument('--voice', required=False, help='Voice name for text2speech backend', dest='voice') parser.add_argument('--backend', required=False, default='ovms-embeddings', choices=['ovms-embeddings','tei-embed','infinity-embeddings','ovms_rerank','text2speech','speech2text', 'translations'], help='Backend serving API type', dest='backend') parser.add_argument('--limit', required=False, type=int, default=1000, help='Number of documents to use in testing', dest='limit') parser.add_argument('--split', required=False, default='train', help='Dataset split', dest='split') @@ -104,6 +105,7 @@ class RequestFuncOutput: success: bool = False latency: float = 0.0 tokens_len: int = 0 + audio_duration: float = 0.0 error: str = "" text: str = "" @@ -128,10 +130,13 @@ async def async_request_text2speech( "model": request_func_input.model, "input": request_func_input.documents[0], } + if args["voice"] is not None: + payload["voice"] = args["voice"] headers = application_json_headers output = RequestFuncOutput() st = time.perf_counter() + audio_bytes = bytearray() try: async with session.post(url=api_url, json=payload, headers=headers) as response: @@ -139,13 +144,13 @@ async def async_request_text2speech( async for chunk_bytes in response.content: if not chunk_bytes: continue - # uncomment for response debugging - # chunk_bytes = chunk_bytes.decode("utf-8") - # data = json.loads(chunk_bytes) - # TBD: saving response to file - timestamp = time.perf_counter() - output.success = True - output.latency = timestamp - st + audio_bytes.extend(chunk_bytes) + output.success = True + output.latency = time.perf_counter() - st + try: + output.audio_duration = soundfile.info(io.BytesIO(audio_bytes)).duration + except Exception: + output.error = "Could not decode WAV response to compute audio duration" else: output.error = response.reason or "" output.success = False @@ -387,6 +392,7 @@ async def limited_request_func(request_func_input, pbar): "latencies": [output.latency for output in outputs], "successes": [output.success for output in outputs], "token_count": [output.tokens_len for output in outputs], + "audio_durations": [output.audio_duration for output in outputs], } return result @@ -428,18 +434,47 @@ async def limited_request_func(request_func_input, pbar): args["api_url"] = default_api_url benchmark_results = asyncio.run(benchmark(docs=docs, model=args["model"], api_url=args["api_url"], request_rate=float(args["request_rate"]), backend_function=backend_function)) -if args["backend"] == "speech2text" or args["backend"] == "translations": + +success_count = sum(benchmark_results['successes']) +total_count = len(benchmark_results['successes']) +success_rate = (success_count / total_count * 100.0) if total_count else 0.0 + +if args["backend"] == "text2speech": + rtfx_values = [] + for latency, audio_duration, success in zip( + benchmark_results['latencies'], + benchmark_results['audio_durations'], + benchmark_results['successes'], + ): + if success and latency > 0 and audio_duration > 0: + # RTFx = generated audio seconds per one second of processing time. + rtfx_values.append(audio_duration / latency) + + print(f"Success rate: {success_rate}%. ({success_count}/{total_count})") + print(f"Mean latency: {np.mean(benchmark_results['latencies'])*1000:.2f} ms") + print(f"Median latency: {np.median(benchmark_results['latencies'])*1000:.2f} ms") + + if len(rtfx_values) > 0: + print(f"Mean RTFx: {np.mean(rtfx_values):.3f}x") + print(f"Median RTFx: {np.median(rtfx_values):.3f}x") + else: + print("Mean RTFx: n/a") + print("Median RTFx: n/a") +elif args["backend"] == "speech2text" or args["backend"] == "translations": num_tokens = sum(benchmark_results['token_count']) + #print(benchmark_results) + print("Tokens:",num_tokens) + print(f"Success rate: {success_rate}%. ({success_count}/{total_count})") + print(f"Throughput - Tokens per second: {num_tokens / benchmark_results['duration']:^,.1f}") + print(f"Mean latency: {np.mean(benchmark_results['latencies'])*1000:.2f} ms") + print(f"Median latency: {np.median(benchmark_results['latencies'])*1000:.2f} ms") + print(f"Average document length: {num_tokens / len(docs)} tokens") else: num_tokens = count_tokens(docs=docs,model=args["model"]) -#print(benchmark_results) -print("Tokens:",num_tokens) -print(f"Success rate: {sum(benchmark_results['successes'])/len(benchmark_results['successes'])*100}%. ({sum(benchmark_results['successes'])}/{len(benchmark_results['successes'])})") -print(f"Throughput - Tokens per second: {num_tokens / benchmark_results['duration']:^,.1f}") -print(f"Mean latency: {np.mean(benchmark_results['latencies'])*1000:.2f} ms") -print(f"Median latency: {np.median(benchmark_results['latencies'])*1000:.2f} ms") -# add printing 10 percentiles of latency to better understand latency distribution -percentiles = [10, 25, 50, 75, 90, 95, 99] -for p in percentiles: - print(f"{p}th percentile latency: {np.percentile(benchmark_results['latencies'], p)*1000:.2f} ms") -print(f"Average document length: {num_tokens / len(docs)} tokens") + #print(benchmark_results) + print("Tokens:",num_tokens) + print(f"Success rate: {success_rate}%. ({success_count}/{total_count})") + print(f"Throughput - Tokens per second: {num_tokens / benchmark_results['duration']:^,.1f}") + print(f"Mean latency: {np.mean(benchmark_results['latencies'])*1000:.2f} ms") + print(f"Median latency: {np.median(benchmark_results['latencies'])*1000:.2f} ms") + print(f"Average document length: {num_tokens / len(docs)} tokens") diff --git a/demos/common/export_models/export_model.py b/demos/common/export_models/export_model.py index 137804cd20..fd3e226d5c 100644 --- a/demos/common/export_models/export_model.py +++ b/demos/common/export_models/export_model.py @@ -92,7 +92,7 @@ def add_common_arguments(parser): parser_text2speech = subparsers.add_parser('text2speech', help='export model for text2speech endpoint') add_common_arguments(parser_text2speech) parser_text2speech.add_argument('--num_streams', default=0, type=int, help='The number of parallel execution streams to use for the models in the pipeline.', dest='num_streams') -parser_text2speech.add_argument('--model_type', default='speecht5', choices=['speecht5', 'kokoro'], help='Type of the source TTS model. speecht5 uses optimum-cli; kokoro uses a dedicated PyTorch->OpenVINO conversion path.', dest='model_type') +parser_text2speech.add_argument('--model_type', default='kokoro', choices=['speecht5', 'kokoro'], help='Type of the source TTS model. speecht5 uses optimum-cli; kokoro uses a dedicated PyTorch->OpenVINO conversion path.', dest='model_type') parser_text2speech.add_argument('--vocoder', type=str, help='The vocoder model to use for speecht5. For example microsoft/speecht5_hifigan. Ignored for kokoro.', dest='vocoder') parser_text2speech.add_argument('--speaker_name', type=str, help='Name of the speaker (speecht5 only; for kokoro all voices from the HF repo are exported).', dest='speaker_name') parser_text2speech.add_argument('--speaker_path', type=str, help='Path to the speaker.bin file (speecht5 only; for kokoro all voices from the HF repo are exported).', dest='speaker_path') @@ -503,15 +503,6 @@ def export_embeddings_model_ov(model_repository_path, source_model, model_name, print("Created graph {}".format(os.path.join(model_repository_path, model_name, 'graph.pbtxt'))) add_servable_to_config(config_file_path, model_name, os.path.relpath(os.path.join(model_repository_path, model_name), os.path.dirname(config_file_path))) -def _list_kokoro_voices(destination_path): - """optimum-cli's Kokoro exporter writes per-voice speaker embeddings to - /voices/.bin. Return the sorted list of voice names.""" - voices_dir = os.path.join(destination_path, "voices") - if not os.path.isdir(voices_dir): - print("Warning: no voices/ directory found under", destination_path) - return [] - return sorted(Path(p).stem for p in Path(voices_dir).glob("*.bin")) - def export_text2speech_model(model_repository_path, source_model, model_name, precision, task_parameters, config_file_path): destination_path = os.path.join(model_repository_path, model_name) print("Exporting text2speech model to ",destination_path) @@ -519,16 +510,12 @@ def export_text2speech_model(model_repository_path, source_model, model_name, pr if model_type == 'kokoro': # optimum-intel registers Kokoro under library_name=kokoro / task=text-to-audio. - # The kokoro exporter also dumps each speaker embedding to voices/.bin. if not os.path.isfile(os.path.join(destination_path, 'openvino_model.xml')) or args['overwrite_models']: optimum_command = "optimum-cli export openvino --model {} --task text-to-audio --weight-format {} {} --trust-remote-code {}".format( source_model, precision, task_parameters['extra_quantization_params'], destination_path) print('Running command:', optimum_command) if os.system(optimum_command): raise ValueError("Failed to export kokoro model", source_model) - voice_names = _list_kokoro_voices(destination_path) - # Render the graph with every available voice (path is relative to graph.pbtxt). - task_parameters['voices'] = [{'name': n, 'path': f'./voices/{n}.bin'} for n in voice_names] else: if not os.path.isdir(destination_path) or args['overwrite_models']: if not task_parameters.get('vocoder'): diff --git a/distro.bzl b/distro.bzl index c01ad7bb56..a5f43dfd36 100644 --- a/distro.bzl +++ b/distro.bzl @@ -43,28 +43,4 @@ def distro_flag(): negate = ":ubuntu_build", ) -# Controls whether espeak-ng is built from source (via Bazel) and bundled -# into the OVMS release. When "off", no espeak-ng artifacts are produced -# and the runtime will not have phonemization fallback available. -def espeak_flag(): - string_flag( - name = "espeak", - values = ["on", "off"], - build_setting_default = "on", - ) - native.config_setting( - name = "espeak_on", - flag_values = { - "espeak": "on", - }, - ) - native.config_setting( - name = "espeak_off", - flag_values = { - "espeak": "off", - }, - ) - more_selects.config_setting_negation( - name = "not_espeak_on", - negate = ":espeak_on", - ) + diff --git a/docs/build_from_source.md b/docs/build_from_source.md index 1b3c9f8579..159792787b 100644 --- a/docs/build_from_source.md +++ b/docs/build_from_source.md @@ -156,6 +156,32 @@ dist/ubuntu22 └── ovms.tar.gz.sha256 ``` +### Optional `espeak-ng` for Speech Generation + +`espeak-ng` is optional in OVMS builds. It is used by Kokoro speech generation for: + +- non-English grapheme-to-phoneme conversion, +- fallback phonemization of some out-of-vocabulary (OOV) English words. + +If you do not need Kokoro non-English support and can accept reduced English OOV fallback behavior, you can: + +- skip building `espeak-ng` during image/package build: + +```bash +make targz_package ESPEAK=0 +``` + +- or remove it from an already prepared package by deleting: + - `libespeak-ng.so*` from the package `lib` directory + - `share/espeak-ng-data/` + +Consequences of removing or disabling `espeak-ng`: + +- Speech generation endpoint still works for Kokoro English usage. +- Non-English Kokoro text normalization/phonemization paths that depend on `espeak-ng` are not available. +- English OOV words no longer use `espeak-ng` fallback, so pronunciation quality/coverage for uncommon words may degrade. +- In practice, treat Kokoro as English-focused (for example `en-us` and `en-gb`) when `espeak-ng` is not present. + --- Read more details about building and testing changes in [developer guide](./developer_guide.md). diff --git a/prepare_llm_models.sh b/prepare_llm_models.sh index 1ec7a61ba7..e7936f00cf 100755 --- a/prepare_llm_models.sh +++ b/prepare_llm_models.sh @@ -27,7 +27,7 @@ LEGACY_MODEL_FILE="1/model.bin" EMBEDDING_MODEL="thenlper/gte-small" RERANK_MODEL="BAAI/bge-reranker-base" VLM_MODEL="OpenVINO/InternVL2-1B-int4-ov" -TTS_MODEL="microsoft/speecht5_tts" +TTS_MODEL="hexgrad/Kokoro-82M" STT_MODEL="openai/whisper-tiny" # Models for tools testing. Only tokenizers are downloaded. @@ -47,7 +47,7 @@ echo "Downloading LLM testing models to directory $1" export PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu https://storage.openvinotoolkit.org/simple/wheels/nightly" if [ "$2" = "docker" ]; then export PATH=$PATH:/opt/intel/openvino/python/bin - python3 -m pip install "optimum-intel"@git+https://github.com/huggingface/optimum-intel.git nncf sentence_transformers==5.3.0 sentencepiece requests protobuf==7.35.0 + python3 -m pip install "optimum-intel"@git+https://github.com/huggingface/optimum-intel.git nncf sentence_transformers==5.3.0 sentencepiece requests protobuf==7.35.0 kokoro else python3 -m venv .venv . .venv/bin/activate @@ -82,13 +82,13 @@ if [ ! -f "$1/$FACEBOOK_MODEL/chat_template.jinja" ]; then cp src/test/llm/dummy_facebook_template.jinja "$1/$FACEBOOK_MODEL/chat_template.jinja" fi -if [ -f "$1/$TTS_MODEL/$TOKENIZER_FILE" ]; then - echo "Model file $1/$TTS_MODEL/$TOKENIZER_FILE exists. Skipping downloading models." +if [ -f "$1/$TTS_MODEL/openvino_model.xml" ]; then + echo "Model file $1/$TTS_MODEL/openvino_model.xml exists. Skipping downloading models." else - python3 demos/common/export_models/export_model.py text2speech --source_model "$TTS_MODEL" --weight-format int4 --model_repository_path $1 --vocoder microsoft/speecht5_hifigan + python3 demos/common/export_models/export_model.py text2speech --source_model "$TTS_MODEL" --model_type kokoro --weight-format int8 --model_repository_path $1 fi -if [ ! -f "$1/$TTS_MODEL/$TOKENIZER_FILE" ]; then - echo "[ERROR] Model file $1/$TTS_MODEL/$TOKENIZER_FILE does not exist." +if [ ! -f "$1/$TTS_MODEL/openvino_model.xml" ]; then + echo "[ERROR] Model file $1/$TTS_MODEL/openvino_model.xml does not exist." exit 1 fi diff --git a/src/BUILD b/src/BUILD index 4e16a82413..0e6b6273f4 100644 --- a/src/BUILD +++ b/src/BUILD @@ -2204,12 +2204,6 @@ cc_binary( "//src/python/binding:pyovms.so", ], "//:disable_python": [] - }) + select({ - "//:espeak_on": [ - "//third_party:espeak_ng", - "//third_party:espeak_ng_data", - ], - "//:espeak_off": [], }), # linkstatic = False, # Use for dynamic linking when necessary ) diff --git a/src/audio/text_to_speech/t2s_servable.cpp b/src/audio/text_to_speech/t2s_servable.cpp index 501c867e5e..8d31aa685a 100644 --- a/src/audio/text_to_speech/t2s_servable.cpp +++ b/src/audio/text_to_speech/t2s_servable.cpp @@ -15,10 +15,13 @@ //***************************************************************************** #include +#include #include #include #include #include +#include +#include #include "openvino/genai/speech_generation/text2speech_pipeline.hpp" #include "src/audio/text_to_speech/t2s_calculator.pb.h" @@ -41,6 +44,35 @@ static size_t getShapeElementsCount(const ov::Shape& shape) { return elementsCount; } +static std::vector getVoiceEmbeddingPaths(const std::filesystem::path& voicesDir) { + std::vector voicePaths; + std::error_code errorCode; + std::filesystem::directory_iterator directoryIt(voicesDir, errorCode); + if (errorCode) { + throw std::runtime_error("Failed to open voices directory '" + voicesDir.string() + "': " + errorCode.message()); + } + + const std::filesystem::directory_iterator directoryEnd; + for (; directoryIt != directoryEnd; directoryIt.increment(errorCode)) { + if (errorCode) { + throw std::runtime_error("Failed to iterate voices directory '" + voicesDir.string() + "': " + errorCode.message()); + } + const auto& entry = *directoryIt; + const bool isRegularFile = entry.is_regular_file(errorCode); + if (errorCode) { + throw std::runtime_error("Failed to inspect entry '" + entry.path().string() + "' in voices directory '" + voicesDir.string() + "': " + errorCode.message()); + } + if (!isRegularFile) { + continue; + } + if (entry.path().extension() == ".bin") { + voicePaths.emplace_back(entry.path()); + } + } + std::sort(voicePaths.begin(), voicePaths.end()); + return voicePaths; +} + static ov::Tensor readSpeakerEmbedding(const std::filesystem::path& filePath, const ov::Shape& expectedShape) { std::ifstream input(filePath, std::ios::binary); if (input.fail()) { @@ -102,14 +134,28 @@ TtsServable::TtsServable(const std::string& modelDir, const std::string& targetD } ttsPipeline = std::make_shared(parsedModelsPath.string(), targetDevice, config); const ov::Shape speakerEmbeddingShape = ttsPipeline->get_speaker_embedding_shape(); - for (const auto& voice : graphVoices) { - std::filesystem::path voicePath(voice.path()); - if (voicePath.is_relative()) { - voicePath = graphDir / voicePath; + if (!graphVoices.empty()) { + for (const auto& voice : graphVoices) { + std::filesystem::path voicePath(voice.path()); + if (voicePath.is_relative()) { + voicePath = graphDir / voicePath; + } + if (!std::filesystem::exists(voicePath)) + throw std::runtime_error{"Requested voice speaker embeddings file does not exist: " + voicePath.string()}; + voices[voice.name()] = readSpeakerEmbedding(voicePath, speakerEmbeddingShape); } - if (!std::filesystem::exists(voicePath)) - throw std::runtime_error{"Requested voice speaker embeddings file does not exist: " + voicePath.string()}; - voices[voice.name()] = readSpeakerEmbedding(voicePath, speakerEmbeddingShape); + return; + } + + const std::filesystem::path voicesDir = parsedModelsPath / "voices"; + std::error_code ec; + if (!std::filesystem::is_directory(voicesDir, ec)) { + SPDLOG_DEBUG("No voices configured in graph and voices directory not found: {}", voicesDir.string()); + return; + } + + for (const auto& voicePath : getVoiceEmbeddingPaths(voicesDir)) { + voices[voicePath.stem().string()] = readSpeakerEmbedding(voicePath, speakerEmbeddingShape); } } } // namespace ovms diff --git a/src/graph_export/graph_export.cpp b/src/graph_export/graph_export.cpp index 5bc7c03413..c007fdf4f6 100644 --- a/src/graph_export/graph_export.cpp +++ b/src/graph_export/graph_export.cpp @@ -339,23 +339,6 @@ static Status createTextToSpeechGraphTemplate(const std::string& directoryPath, SPDLOG_TRACE("modelsPath: {}, directoryPath: {}, ggufFilename: {}", modelsPath, directoryPath, ggufFilename.value_or("std::nullopt")); GET_PLUGIN_CONFIG_OPT_OR_FAIL_AND_RETURN(exportSettings); - // Enumerate kokoro speaker embeddings dumped by optimum-cli to /voices/*.bin. - std::vector voiceNames; - if (exportSettings.modelType == "kokoro") { - std::filesystem::path voicesDir = std::filesystem::path(directoryPath) / "voices"; - std::error_code ec; - if (std::filesystem::is_directory(voicesDir, ec)) { - for (const auto& entry : std::filesystem::directory_iterator(voicesDir, ec)) { - if (entry.is_regular_file(ec) && !ec && entry.path().extension() == ".bin") { - voiceNames.push_back(entry.path().stem().string()); - } - } - std::sort(voiceNames.begin(), voiceNames.end()); - } else { - SPDLOG_WARN("Kokoro voices directory not found at {}", voicesDir.string()); - } - } - // clang-format off oss << R"( input_stream: "HTTP_REQUEST_PAYLOAD:input" @@ -377,19 +360,6 @@ node { oss << R"(plugin_config: ')" << pluginConfigOpt.value() << R"(' )"; } - if (!voiceNames.empty()) { - oss << R"(voices: [)"; - for (size_t i = 0; i < voiceNames.size(); ++i) { - oss << R"( - { name: ")" << voiceNames[i] << R"(", path: "./voices/)" << voiceNames[i] << R"(.bin" })"; - if (i + 1 < voiceNames.size()) { - oss << ","; - } - } - oss << R"( - ] - )"; - } oss << R"(} } })"; diff --git a/src/test/audio/graph_tts.pbtxt b/src/test/audio/graph_tts.pbtxt index 021c525225..71c35b7d9c 100644 --- a/src/test/audio/graph_tts.pbtxt +++ b/src/test/audio/graph_tts.pbtxt @@ -24,15 +24,9 @@ node { output_stream: "HTTP_RESPONSE_PAYLOAD:output" node_options: { [type.googleapis.com / mediapipe.T2sCalculatorOptions]: { - models_path: "/ovms/src/test/llm_testing/microsoft/speecht5_tts" + models_path: "/ovms/src/test/llm_testing/hexgrad/Kokoro-82M" plugin_config: '{"NUM_STREAMS": "1" }', target_device: "CPU" - voices: [ - { - name: "speaker1", - path: "/ovms/src/test/audio/speaker.bin", - } - ] } } } \ No newline at end of file diff --git a/src/test/audio/text2speech_test.cpp b/src/test/audio/text2speech_test.cpp index 208e706cc8..a6632d6f7e 100644 --- a/src/test/audio/text2speech_test.cpp +++ b/src/test/audio/text2speech_test.cpp @@ -58,7 +58,8 @@ TEST_F(Text2SpeechHttpTest, simplePositive) { { "model": ")" + modelName + R"(", - "input": "The quick brown fox jumped over the lazy dog." + "input": "The quick brown fox jumped over the lazy dog.", + "voice": "af_alloy" } )"; ASSERT_EQ( @@ -74,7 +75,8 @@ TEST_F(Text2SpeechHttpTest, emptyInput) { { "model": ")" + modelName + R"(", - "input": "" + "input": "", + "voice": "af_alloy" } )"; ASSERT_EQ( @@ -103,7 +105,7 @@ TEST_F(Text2SpeechHttpTest, positiveWithVoice) { "model": ")" + modelName + R"(", "input": "The quick brown fox jumped over the lazy dog.", - "voice": "speaker1" + "voice": "af_alloy" } )"; ASSERT_EQ( @@ -143,7 +145,7 @@ TEST_F(Text2SpeechConfigTest, NodeNameMissing) { output_stream: "HTTP_RESPONSE_PAYLOAD:output" node_options: { [type.googleapis.com / mediapipe.T2sCalculatorOptions]: { - models_path: "/ovms/src/test/llm_testing/microsoft/speecht5_tts" + models_path: "/ovms/src/test/llm_testing/hexgrad/Kokoro-82M" target_device: "CPU" } } @@ -169,7 +171,7 @@ TEST_F(Text2SpeechConfigTest, SidePacketMissing) { output_stream: "HTTP_RESPONSE_PAYLOAD:output" node_options: { [type.googleapis.com / mediapipe.T2sCalculatorOptions]: { - models_path: "/ovms/src/test/llm_testing/microsoft/speecht5_tts" + models_path: "/ovms/src/test/llm_testing/hexgrad/Kokoro-82M" target_device: "CPU" } } @@ -222,7 +224,7 @@ TEST_F(Text2SpeechConfigTest, InvalidPluginConfig) { output_stream: "HTTP_RESPONSE_PAYLOAD:output" node_options: { [type.googleapis.com / mediapipe.T2sCalculatorOptions]: { - models_path: "/ovms/src/test/llm_testing/microsoft/speecht5_tts" + models_path: "/ovms/src/test/llm_testing/hexgrad/Kokoro-82M" plugin_config: 'INVALID', target_device: "CPU" } @@ -236,6 +238,34 @@ TEST_F(Text2SpeechConfigTest, InvalidPluginConfig) { ASSERT_EQ(mediapipeDummy.validate(manager), StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID); } +TEST_F(Text2SpeechConfigTest, MissingVoicesInGraphUsesModelVoicesDir) { + ConstructorEnabledModelManager manager; + std::string testPbtxt = R"( + input_stream: "HTTP_REQUEST_PAYLOAD:input" + output_stream: "HTTP_RESPONSE_PAYLOAD:output" + + node { + name: "ttsNode1" + input_side_packet: "TTS_NODE_RESOURCES:t2s_servable" + calculator: "T2sCalculator" + input_stream: "HTTP_REQUEST_PAYLOAD:input" + output_stream: "HTTP_RESPONSE_PAYLOAD:output" + node_options: { + [type.googleapis.com / mediapipe.T2sCalculatorOptions]: { + models_path: "/ovms/src/test/llm_testing/hexgrad/Kokoro-82M" + plugin_config: '{"NUM_STREAMS": "1" }', + target_device: "CPU" + } + } + } + )"; + + ovms::MediapipeGraphConfig mgc{"mediaDummy", "", ""}; + DummyMediapipeGraphDefinition mediapipeDummy("mediaDummy", mgc, testPbtxt, nullptr); + mediapipeDummy.inputConfig = testPbtxt; + ASSERT_EQ(mediapipeDummy.validate(manager), StatusCode::OK); +} + TEST_F(Text2SpeechConfigTest, NonExistingVoicePath) { ConstructorEnabledModelManager manager; std::string testPbtxt = R"( @@ -250,7 +280,7 @@ TEST_F(Text2SpeechConfigTest, NonExistingVoicePath) { output_stream: "HTTP_RESPONSE_PAYLOAD:output" node_options: { [type.googleapis.com / mediapipe.T2sCalculatorOptions]: { - models_path: "/ovms/src/test/llm_testing/microsoft/speecht5_tts" + models_path: "/ovms/src/test/llm_testing/hexgrad/Kokoro-82M" plugin_config: '{"NUM_STREAMS": "1" }', target_device: "CPU" voices: [ @@ -284,7 +314,7 @@ TEST_F(Text2SpeechConfigTest, VoiceMissingPath) { output_stream: "HTTP_RESPONSE_PAYLOAD:output" node_options: { [type.googleapis.com / mediapipe.T2sCalculatorOptions]: { - models_path: "/ovms/src/test/llm_testing/microsoft/speecht5_tts" + models_path: "/ovms/src/test/llm_testing/hexgrad/Kokoro-82M" plugin_config: '{"NUM_STREAMS": "1" }', target_device: "CPU" voices: [ @@ -317,7 +347,7 @@ TEST_F(Text2SpeechConfigTest, VoiceInvalidFile) { output_stream: "HTTP_RESPONSE_PAYLOAD:output" node_options: { [type.googleapis.com / mediapipe.T2sCalculatorOptions]: { - models_path: "/ovms/src/test/llm_testing/microsoft/speecht5_tts" + models_path: "/ovms/src/test/llm_testing/hexgrad/Kokoro-82M" plugin_config: '{"NUM_STREAMS": "1" }', target_device: "CPU" voices: [ diff --git a/src/test/graph_export_test.cpp b/src/test/graph_export_test.cpp index ed1f4203f0..1b0136ebf7 100644 --- a/src/test/graph_export_test.cpp +++ b/src/test/graph_export_test.cpp @@ -431,10 +431,6 @@ node { [type.googleapis.com / mediapipe.T2sCalculatorOptions]: { models_path: "./" target_device: "CPU" - voices: [ - { name: "af_alloy", path: "./voices/af_alloy.bin" }, - { name: "am_adam", path: "./voices/am_adam.bin" } - ] } } } @@ -822,12 +818,6 @@ TEST_F(GraphCreationTest, textToSpeechPositiveDefault) { } TEST_F(GraphCreationTest, textToSpeechPositiveKokoro) { - // Pre-create the voices/ directory that optimum-cli would have populated for kokoro. - std::filesystem::path voicesDir = std::filesystem::path(this->directoryPath) / "voices"; - std::filesystem::create_directories(voicesDir); - { std::ofstream f(voicesDir / "af_alloy.bin"); } - { std::ofstream f(voicesDir / "am_adam.bin"); } - ovms::HFSettingsImpl hfSettings; hfSettings.task = ovms::TEXT_TO_SPEECH_GRAPH; hfSettings.exportSettings.modelName = "myModel"; diff --git a/third_party/BUILD b/third_party/BUILD index bce804d7a1..2622baa0a9 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -61,27 +61,16 @@ alias( ) # espeak-ng built from source via Bazel (rules_foreign_cc cmake). -# Selected on/off via the //:espeak build flag. When disabled this resolves -# to an empty cc_library so dependents can unconditionally list it. -cc_library( - name = "espeak_ng_empty", - visibility = ["//visibility:public"], -) - +# Built as a separate step in the Dockerfile (ARG ESPEAK=1/0), not as +# a dependency of the OVMS binary. alias( name = "espeak_ng", - actual = select({ - "//:espeak_on": "@espeak_ng//:espeak_ng", - "//:espeak_off": ":espeak_ng_empty", - }), + actual = "@espeak_ng//:espeak_ng", visibility = ["//visibility:public"], ) alias( name = "espeak_ng_data", - actual = select({ - "//:espeak_on": "@espeak_ng//:espeak_ng_data", - "//:espeak_off": ":espeak_ng_empty", - }), + actual = "@espeak_ng//:espeak_ng_data", visibility = ["//visibility:public"], ) \ No newline at end of file diff --git a/windows_build.bat b/windows_build.bat index eac54243e7..4c3f100075 100644 --- a/windows_build.bat +++ b/windows_build.bat @@ -49,18 +49,10 @@ IF "%~4"=="--integrity" ( set "buildWithIntegrity=" ) -:: Allow disabling espeak-ng (built from source via Bazel) by setting -:: ESPEAK=0 before invoking this script. Defaults to on. -IF "%ESPEAK%"=="0" ( - set "espeakArg=--//:espeak=off" -) ELSE ( - set "espeakArg=--//:espeak=on" -) - set "bazelStartupCmd=--output_user_root=!BAZEL_SHORT_PATH!" set "openvino_dir=!BAZEL_SHORT_PATH!/openvino/runtime/cmake" -set "buildCommand=bazel %bazelStartupCmd% build %buildWithIntegrity% %bazelBuildArgs% %espeakArg% --action_env OpenVINO_DIR=%openvino_dir% --jobs=%NUMBER_OF_PROCESSORS% --verbose_failures %buildTargets% 2>&1 | tee win_build.log" +set "buildCommand=bazel %bazelStartupCmd% build %buildWithIntegrity% %bazelBuildArgs% --action_env OpenVINO_DIR=%openvino_dir% --jobs=%NUMBER_OF_PROCESSORS% --verbose_failures %buildTargets% 2>&1 | tee win_build.log" set "setOvmsVersionCmd=python windows_set_ovms_version.py" :: Setting PATH environment variable based on default windows node settings: Added ovms_windows specific python settings and c:/opt and removed unused Nvidia and OCL specific tools. diff --git a/windows_prepare_llm_models.bat b/windows_prepare_llm_models.bat index c49fdc1f0f..e2de5ab059 100644 --- a/windows_prepare_llm_models.bat +++ b/windows_prepare_llm_models.bat @@ -33,7 +33,7 @@ set "RERANK_MODEL=BAAI/bge-reranker-base" set "TEXT_GENERATION_MODEL=HuggingFaceTB/SmolLM2-360M-Instruct" set "FACEBOOK_MODEL=facebook/opt-125m" set "VLM_MODEL=OpenVINO/InternVL2-1B-int4-ov" -set "TTS_MODEL=microsoft/speecht5_tts" +set "TTS_MODEL=hexgrad/Kokoro-82M" set "STT_MODEL=openai/whisper-tiny" :: Models for tools testing. Only tokenizers are downloaded. @@ -63,7 +63,7 @@ if not exist "%~1" mkdir "%~1" :: Export models -call :download_export_model_tts "%TTS_MODEL%" "text2speech" "--weight-format int4" "%~1" +call :download_export_model_tts "%TTS_MODEL%" "text2speech" "--model_type kokoro --weight-format int8" "%~1" call :download_export_model "%STT_MODEL%" "speech2text" "--weight-format int4" "%~1" call :download_openvino "%VLM_MODEL%" "%~1" OpenGVLab/InternVL2-1B call :download_export_model "%TEXT_GENERATION_MODEL%" "text_generation" "--weight-format int8" "%~1" @@ -97,11 +97,11 @@ set "model_type=%~2" set "export_args=%~3" set "repository=%~4" -if not exist "%repository%\%model%\openvino_tokenizer.bin" ( +if not exist "%repository%\%model%\openvino_model.xml" ( echo Downloading %model_type% model to %repository%\%model% directory. python demos\common\export_models\export_model.py %model_type% --source_model "%model%" %export_args% --model_repository_path %repository% ) else ( - echo Models file %repository%\%model%\openvino_tokenizer.bin exists. Skipping downloading models. + echo Models file %repository%\%model%\openvino_model.xml exists. Skipping downloading models. ) exit /b 0 @@ -111,11 +111,11 @@ set "model_type=%~2" set "export_args=%~3" set "repository=%~4" -if not exist "%repository%\%model%\openvino_tokenizer.bin" ( +if not exist "%repository%\%model%\openvino_model.xml" ( echo Downloading %model_type% model to %repository%\%model% directory. - python demos\common\export_models\export_model.py %model_type% --source_model "%model%" %export_args% --vocoder microsoft/speecht5_hifigan --model_repository_path %repository% + python demos\common\export_models\export_model.py %model_type% --source_model "%model%" %export_args% --model_repository_path %repository% ) else ( - echo Models file %repository%\%model%\openvino_tokenizer.bin exists. Skipping downloading models. + echo Models file %repository%\%model%\openvino_model.xml exists. Skipping downloading models. ) exit /b 0