diff --git a/.bazelrc b/.bazelrc index f4cd26bb00..fb3745fc90 100644 --- a/.bazelrc +++ b/.bazelrc @@ -56,7 +56,6 @@ build --define=MEDIAPIPE_DISABLE_GPU=1 build --define=MEDIAPIPE_DISABLE=0 build --define=CLOUD_DISABLE=0 build --define=PYTHON_DISABLE=0 -build --define=USE_DROGON=1 # 0=legacy net_http, available only on linux # Sets the default Apple platform to macOS. build --apple_platform_type=macos diff --git a/BUILD.bazel b/BUILD.bazel index aebd22244f..685ec389c9 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -33,15 +33,12 @@ create_config_settings() cc_library( name = "ovms_dependencies", deps = [ - "@tensorflow_serving//tensorflow_serving/apis:prediction_service_cc_proto", - "@tensorflow_serving//tensorflow_serving/apis:model_service_cc_proto", "@minitrace//:trace", "@com_github_grpc_grpc//:grpc++", "@com_github_tencent_rapidjson//:rapidjson", "@org_tensorflow//tensorflow/core:framework", "@com_github_gabime_spdlog//:spdlog", "@com_github_jarro2783_cxxopts//:cxxopts", - "@tensorflow_serving//tensorflow_serving/util:json_tensor", "//third_party:openvino", "//third_party:opencv", "//third_party:curl", diff --git a/WORKSPACE b/WORKSPACE index 31290a54af..c331e27718 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -158,21 +158,6 @@ cc_library( ) -# Used for gRPC API protos only -# Tensorflow serving -git_repository( - name = "tensorflow_serving", - remote = "https://github.com/tensorflow/serving.git", - tag = "2.18.0", - patch_args = ["-p1"], - patches = ["net_http.patch", "listen.patch", "partial_2.18.patch"] - # ^^^^^^^^^^^^ - # make bind address configurable - # ^^^^^^^^^^^^ - # allow all http methods ^^^^^^^^^ - # implements partial responses -) - ########################################################### Mediapipe http_archive( name = "com_google_protobuf", @@ -434,8 +419,6 @@ http_archive( repo_mapping = {"@curl" : "@curl"} ) -load("@tensorflow_serving//tensorflow_serving:workspace.bzl", "tf_serving_workspace") - # Initialize TensorFlow's external dependencies. load("@org_tensorflow//tensorflow:workspace3.bzl", "workspace") workspace() @@ -473,7 +456,6 @@ workspace() load("@org_tensorflow//tensorflow:workspace0.bzl", "workspace") workspace() -tf_serving_workspace() #moved past TF # required after update to mp 0.10.18 load( "@org_tensorflow//third_party/gpus/cuda/hermetic:cuda_configure.bzl", diff --git a/ci/build_test_OnCommit.groovy b/ci/build_test_OnCommit.groovy index 863f074895..0435645689 100644 --- a/ci/build_test_OnCommit.groovy +++ b/ci/build_test_OnCommit.groovy @@ -343,7 +343,7 @@ pipeline { def test_doc_files_str = test_doc_files_linux.split('\n').join(' or ') sh "make create-venv && rm -f tests/functional && ln -s ${pwd}/../tests/functional tests/functional" def cmd_venv_activate = ". .venv/bin/activate" - def cmd_export = "export TT_OVMS_C_REPO_PATH=../ && export TT_RUN_REGRESSION_TESTS=True && export TT_REGRESSION_WEEKLY_TESTS=True && export TT_TARGET_DEVICE=CPU,GPU,NPU && export TT_ENABLE_UAT_TESTS=True && export TT_ENABLE_SMOKE_TESTS=False && export TT_OVMS_C_REPO_PATH=${ovms_c_repo_path} && export TT_LOGGING_LEVEL_OVMS=DEBUG && export TT_WAIT_FOR_MESSAGES_TIMEOUT=1500" + def cmd_export = "export TT_OVMS_C_REPO_PATH=../ && export TT_RUN_REGRESSION_TESTS=True && export TT_REGRESSION_WEEKLY_TESTS=True && export TT_TARGET_DEVICE=CPU,GPU,NPU && export TT_ENABLE_UAT_TESTS=True && export TT_ENABLE_SMOKE_TESTS=False && export TT_OVMS_C_REPO_PATH=${ovms_c_repo_path} && export TT_LOGGING_LEVEL_OVMS=DEBUG && export TT_WAIT_FOR_MESSAGES_TIMEOUT=1500 && export CORE_BRANCH=${env.CHANGE_BRANCH ?: 'main'}" def cmd_pytest = "pytest tests/non_functional/documentation -k '${test_doc_files_str}' -n 0 --dist loadgroup" def cmd = "" if ( image_build_needed == "true" ) { @@ -408,7 +408,7 @@ pipeline { def ovms_c_repo_path = bat(returnStdout: true, script: 'cd .. && cd').trim().split('\n').last().trim() def cmd_link_ovms = "(if exist ${current_path}\\tests\\functional rmdir ${current_path}\\tests\\functional) && mklink /D ${current_path}\\tests\\functional ${ovms_c_repo_path}\\tests\\functional" def cmd_requirements = "(if not exist .venv virtualenv .venv --python=python3.12) && call .venv\\Scripts\\activate.bat && pip install -r requirements.txt" - def cmd_export = "set \"TT_OVMS_C_REPO_PATH=../\" && set \"TT_LOGGING_LEVEL_OVMS=DEBUG\" && set \"TT_RUN_REGRESSION_TESTS=True\" && set \"TT_REGRESSION_WEEKLY_TESTS=True\" && set \"TT_TARGET_DEVICE=CPU,GPU,NPU\" && set \"TT_BASE_OS=windows\" && set \"TT_OVMS_TYPE=BINARY\" && set \"TT_ENABLE_UAT_TESTS=True\" && set \"TT_ENABLE_SMOKE_TESTS=False\" && set \"TT_DISABLE_DMESG_LOG_MONITOR=True\" && set \"TT_OVMS_C_REPO_PATH=${ovms_c_repo_path}\" && set \"TT_WAIT_FOR_MESSAGES_TIMEOUT=1500\" && set \"PYTHONUTF8=1\" && set \"PYTHONIOENCODING=utf-8\"" + def cmd_export = "set \"TT_OVMS_C_REPO_PATH=../\" && set \"TT_LOGGING_LEVEL_OVMS=DEBUG\" && set \"TT_RUN_REGRESSION_TESTS=True\" && set \"TT_REGRESSION_WEEKLY_TESTS=True\" && set \"TT_TARGET_DEVICE=CPU,GPU,NPU\" && set \"TT_BASE_OS=windows\" && set \"TT_OVMS_TYPE=BINARY\" && set \"TT_ENABLE_UAT_TESTS=True\" && set \"TT_ENABLE_SMOKE_TESTS=False\" && set \"TT_DISABLE_DMESG_LOG_MONITOR=True\" && set \"TT_OVMS_C_REPO_PATH=${ovms_c_repo_path}\" && set \"TT_WAIT_FOR_MESSAGES_TIMEOUT=1500\" && set \"PYTHONUTF8=1\" && set \"PYTHONIOENCODING=utf-8\" && set \"CORE_BRANCH=${env.CHANGE_BRANCH ?: 'main'}\"" def cmd_pytest = "pytest tests/non_functional/documentation -k \"${test_doc_files_str}\" -n 0 --dist loadgroup --basetemp=\"C:\\tmp\\pytest-${BRANCH_NAME}-${BUILD_NUMBER}\"" def cmd = "" if ( win_image_build_needed == "true" ) { diff --git a/client/python/README.md b/client/python/README.md index e41ee148a9..43130c29cd 100644 --- a/client/python/README.md +++ b/client/python/README.md @@ -2,14 +2,5 @@ OpenVINO Model Server exposes network interface that client applications can interact with. -There's also [tensorflow-serving-api](https://pypi.org/project/tensorflow-serving-api/) package that can be used to send requests to OpenVINO Model Server. - -> **Note**: `tensorflow-serving-api` comes with `tensorflow` dependency which makes the package heavy. - -There are sample scripts for `tensorflow-serving-api`: -- [tensorflow-serving-api samples](tensorflow-serving-api/samples) - -Additionally, starting with the 2022.2 release, OpenVINO Model Server also supports [KServe API](https://github.com/kserve/kserve/blob/master/docs/predict-api/v2/required_api.md) calls. You can try it out with the: - - [KServe samples](kserve-api/samples) - -> **Note**: 2025.3 is the last release that includes ovmsclient and it is not supported anymore. If you still need it, use [release 2025.3](https://github.com/openvinotoolkit/model_server/tree/releases/2025/3/client/python/ovmsclient) \ No newline at end of file +OpenVINO Model Server supports [KServe API](https://github.com/kserve/kserve/blob/master/docs/predict-api/v2/required_api.md) calls. You can try it out with the: + - [KServe samples](kserve-api/samples) \ No newline at end of file diff --git a/client/python/tensorflow-serving-api/samples/README.md b/client/python/tensorflow-serving-api/samples/README.md deleted file mode 100644 index b3e78cc71e..0000000000 --- a/client/python/tensorflow-serving-api/samples/README.md +++ /dev/null @@ -1,609 +0,0 @@ -# Samples based on tensorflow-serving-api package - -This document contains examples to run *GetModelStatus*, *GetModelMetadata*, *Predict* functions over gRPC API and REST API. -Samples are based on [tensorflow-serving-api](https://pypi.org/project/tensorflow-serving-api/) package. - -It covers following topics: -* [gRPC API Client Examples](#grpc-api-client-examples) -* [REST API Client Examples](#rest-api-client-examples) - -## Requirement - -**Note**: Provided examples and their dependencies are updated and validated for Python 3.7+ version. For older versions of Python, dependencies versions adjustment might be required. - -Clone the repository and enter directory: - -```bash -git clone https://github.com/openvinotoolkit/model_server.git -cd model_server/client/python/tensorflow-serving-api/samples -``` - -Install client dependencies: -```bash -pip3 install -r requirements.txt -``` - -In the examples listed below, OVMS can be started using a command: -```bash -wget -N https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/resnet50-binary-0001/FP32-INT1/resnet50-binary-0001.{xml,bin} -P models/resnet50/1 -docker run -d -u $(id -u) -v $(pwd)/models:/models -p 8000:8000 -p 9000:9000 openvino/model_server:latest --model_name resnet --model_path /models/resnet50 --port 9000 --rest_port 8000 -``` - -## gRPC API Client Examples - -### Model Status API - -- Command - -```bash -python grpc_get_model_status.py --help -usage: grpc_get_model_status.py [-h] [--grpc_address GRPC_ADDRESS] - [--grpc_port GRPC_PORT] - [--model_name MODEL_NAME] - [--model_version MODEL_VERSION] - -``` - -- Arguments - -| Argument | Description | -| :--- | :---- | -| -h, --help | Show help message and exit | -| --grpc_address GRPC_ADDRESS | Specify url to grpc service. Default: localhost | -| --grpc_port GRPC_PORT | Specify port to grpc service. Default: 9000 | -| --model_name MODEL_NAME | Model name to query. Default: resnet | -| --model_version MODEL_VERSION | Model version to query. Lists all versions if not specified | - - -- Usage Example - -```bash -python grpc_get_model_status.py --grpc_port 9000 --model_name resnet - -Getting model status for model: resnet - -Model version: 1 -State AVAILABLE -Error code: 0 -Error message: OK -``` - - -### Model Metadata API - - -- Command - -```bash -python grpc_get_model_metadata.py --help -usage: grpc_get_model_metadata.py [-h] [--grpc_address GRPC_ADDRESS] - [--grpc_port GRPC_PORT] - [--model_name MODEL_NAME] - [--model_version MODEL_VERSION] - -``` - -- Arguments - -| Argument | Description | -| :--- | :---- | -| -h, --help | Show this help message and exit | -| --grpc_address GRPC_ADDRESS | Specify url to grpc service. Default:localhost | -| --grpc_port GRPC_PORT | Specify port to grpc service. Default: 9000 | -| --model_name MODEL_NAME | Define model name, must be same as is in service. Default: resnet | -| --model_version MODEL_VERSION | Define model version - must be numerical | - - -- Usage Example - -```bash -python grpc_get_model_metadata.py --grpc_port 9000 --model_name resnet --model_version 1 - -Getting model metadata for model: resnet -Inputs metadata: - Input name: 0; shape: [1, 3, 224, 224]; dtype: DT_FLOAT -Outputs metadata: - Output name: 1463; shape: [1, 1000]; dtype: DT_FLOAT -``` - -### Predict API - -#### **Submitting gRPC requests based on a dataset from numpy files:** - -- Command - -```bash -python grpc_predict_resnet.py --help -usage: grpc_predict_resnet.py [-h] --images_numpy_path IMAGES_NUMPY_PATH - [--labels_numpy_path LABELS_NUMPY_PATH] - [--grpc_address GRPC_ADDRESS] - [--grpc_port GRPC_PORT] - [--input_name INPUT_NAME] - [--output_name OUTPUT_NAME] - [--transpose_input {False,True}] - [--transpose_method {nchw2nhwc,nhwc2nchw}] - [--iterations ITERATIONS] - [--batchsize BATCHSIZE] - [--model_name MODEL_NAME] - [--pipeline_name PIPELINE_NAME] - [--dag-batch-size-auto] [--tls] - [--server_cert SERVER_CERT] - [--client_cert CLIENT_CERT] - [--client_key CLIENT_KEY] -``` - -- Arguments - -| Argument | Description | -| :--- | :---- | -| -h,--help | Show help message and exit | -| --images_numpy_path | Numpy in shape [n,w,h,c] or [n,c,h,w] | -| --labels_numpy_path | Numpy in shape [n,1] - can be used to check model accuracy | -| --grpc_address GRPC_ADDRESS | Specify url to grpc service. Default:localhost | -| --grpc_port GRPC_PORT | Specify port to grpc service. Default: 9000 | -| --input_name | Specify input tensor name. Default: input | -| --output_name | Specify output name. Default: resnet_v1_50/predictions/Reshape_1 | -| --transpose_input {False,True}| Set to False to skip NHWC>NCHW or NCHW>NHWC input transposing. Default: True| -| --transpose_method {nchw2nhwc,nhwc2nchw} | How the input transposition should be executed: nhwc2nchw or nhwc2nchw. Default nhwc2nchw| -| --iterations | Number of requests iterations, as default use number of images in numpy memmap. Default: 0 (consume all frames)| -| --batchsize | Number of images in a single request. Default: 1 | -| --model_name | Define model name, must be same as is in service. Default: resnet| -| --pipeline_name | Define pipeline name, must be same as is in service | -| --dag-batch-size-auto | Add demultiplexer dimension at front | -| --tls | enables TLS communication with gRPC endpoint | -| --server_cert SERVER_CERT | Path to the server certificate, used only with TLS communication | -| --client_cert CLIENT_CERT | Path to the client certificate, used only with TLS communication | -| --client_key CLIENT_KEY | Path to the client key, used only with TLS communication | - - -- Usage example - -```bash -python grpc_predict_resnet.py --grpc_port 9000 --images_numpy_path ../../imgs.npy --input_name 0 --output_name 1463 --transpose_input False --labels_numpy_path ../../lbs.npy - -Image data range: 0.0 : 255.0 -Start processing: - Model name: resnet - Iterations: 10 - Images numpy path: ../../imgs.npy - Numpy file shape: (10, 3, 224, 224) - -Iteration 1; Processing time: 29.75 ms; speed 33.62 fps -imagenet top results in a single batch: - 0 airliner 404 ; Correct match. -Iteration 2; Processing time: 27.06 ms; speed 36.96 fps -imagenet top results in a single batch: - 0 Arctic fox, white fox, Alopex lagopus 279 ; Correct match. -Iteration 3; Processing time: 26.81 ms; speed 37.30 fps -imagenet top results in a single batch: - 0 bee 309 ; Correct match. -Iteration 4; Processing time: 27.25 ms; speed 36.70 fps -imagenet top results in a single batch: - 0 golden retriever 207 ; Correct match. -Iteration 5; Processing time: 25.64 ms; speed 39.00 fps -imagenet top results in a single batch: - 0 gorilla, Gorilla gorilla 366 ; Correct match. -Iteration 6; Processing time: 34.41 ms; speed 29.06 fps -imagenet top results in a single batch: - 0 magnetic compass 635 ; Correct match. -Iteration 7; Processing time: 23.39 ms; speed 42.75 fps -imagenet top results in a single batch: - 0 peacock 84 ; Correct match. -Iteration 8; Processing time: 23.82 ms; speed 41.98 fps -imagenet top results in a single batch: - 0 pelican 144 ; Correct match. -Iteration 9; Processing time: 25.66 ms; speed 38.97 fps -imagenet top results in a single batch: - 0 snail 113 ; Correct match. -Iteration 10; Processing time: 22.44 ms; speed 44.57 fps -imagenet top results in a single batch: - 0 zebra 340 ; Correct match. - -processing time for all iterations -average time: 26.10 ms; average speed: 38.31 fps -median time: 25.50 ms; median speed: 39.22 fps -max time: 34.00 ms; min speed: 29.41 fps -min time: 22.00 ms; max speed: 45.45 fps -time percentile 90: 29.50 ms; speed percentile 90: 33.90 fps -time percentile 50: 25.50 ms; speed percentile 50: 39.22 fps -time standard deviation: 3.33 -time variance: 11.09 -Classification accuracy: 100.00 -``` - -#### **Submitting gRPC requests with data in binary format:** - -Using binary inputs feature requires loading model with layout set to `--layout NHWC:NCHW`: -```bash -wget -N https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/resnet50-binary-0001/FP32-INT1/resnet50-binary-0001.{xml,bin} -P models/resnet50/1 -docker run -d -u $(id -u) -v $(pwd)/models:/models -p 8000:8000 -p 9000:9000 openvino/model_server:latest --model_name resnet --model_path /models/resnet50 --port 9000 --rest_port 8000 --layout NHWC:NCHW -``` -```bash -python grpc_predict_binary_resnet.py --help -usage: grpc_predict_binary_resnet.py [-h] [--images_list IMAGES_LIST] - [--grpc_address GRPC_ADDRESS] - [--grpc_port GRPC_PORT] - [--input_name INPUT_NAME] - [--output_name OUTPUT_NAME] - [--model_name MODEL_NAME] - [--batchsize BATCHSIZE] -``` - -- Arguments - -| Argument | Description | -| :--- | :---- | -| -h, --help | show this help message and exit -| --images_list IMAGES_LIST | path to a file with a list of labeled images | -| --grpc_address GRPC_ADDRESS | Specify url to grpc service. default:localhost | -| --grpc_port GRPC_PORT | Specify port to grpc service. default: 9000 | -| --input_name INPUT_NAME | Specify input tensor name. default: image_bytes | -| --output_name OUTPUT_NAME | Specify output name. default: probabilities | -| --model_name MODEL_NAME | Define model name, must be same as is in service default: resnet | -| --batchsize BATCHSIZE | Number of images in a single request. default: 1 | - - -- Usage example -```bash -python grpc_predict_binary_resnet.py --grpc_address localhost --model_name resnet --input_name 0 --output_name 1463 --grpc_port 9000 --images ../../resnet_input_images.txt - -Start processing: - Model name: resnet - Images list file: ../../resnet_input_images.txt -Batch: 0; Processing time: 31.59 ms; speed 31.65 fps - 1 airliner 404 ; Correct match. -Batch: 1; Processing time: 30.08 ms; speed 33.24 fps - 2 Arctic fox, white fox, Alopex lagopus 279 ; Correct match. -Batch: 2; Processing time: 28.80 ms; speed 34.72 fps - 3 bee 309 ; Correct match. -Batch: 3; Processing time: 29.39 ms; speed 34.03 fps - 4 golden retriever 207 ; Correct match. -Batch: 4; Processing time: 29.91 ms; speed 33.43 fps - 5 gorilla, Gorilla gorilla 366 ; Correct match. -Batch: 5; Processing time: 28.52 ms; speed 35.07 fps - 6 magnetic compass 635 ; Correct match. -Batch: 6; Processing time: 28.69 ms; speed 34.85 fps - 7 peacock 84 ; Correct match. -Batch: 7; Processing time: 26.02 ms; speed 38.43 fps - 8 pelican 144 ; Correct match. -Batch: 8; Processing time: 24.28 ms; speed 41.19 fps - 9 snail 113 ; Correct match. -Batch: 9; Processing time: 29.93 ms; speed 33.41 fps - 10 zebra 340 ; Correct match. -Overall accuracy= 100.0 % -Average latency= 28.2 ms -``` - - -## REST API Client Examples - -Access to Google Cloud Storage might require proper configuration of https_proxy in the docker engine or in the docker container. -In the examples listed below, OVMS can be started using a command: -```bash -wget -N https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/resnet50-binary-0001/FP32-INT1/resnet50-binary-0001.{xml,bin} -P models/resnet50/1 -docker run -d -u $(id -u) -v $(pwd)/models:/models -p 8000:8000 -p 9000:9000 openvino/model_server:latest --model_name resnet --model_path /models/resnet50 --port 9000 --rest_port 8000 -``` - -### Model Status API -- Command -```bash -python rest_get_model_status.py --help -usage: rest_get_model_status.py [-h] [--rest_url REST_URL] - [--rest_port REST_PORT] - [--model_name MODEL_NAME] - [--model_version MODEL_VERSION] - [--client_cert CLIENT_CERT] - [--client_key CLIENT_KEY] - [--ignore_server_verification] - [--server_cert SERVER_CERT] -``` -- Arguments - -| Argument | Description | -| :--- | :---- | -| -h, --help | Show help message and exit| -| --rest_url REST_URL | Specify url to REST API service. Default:http://localhost| -| --rest_port REST_PORT | Specify port to REST API service. Default: 8000| -| --model_name MODEL_NAME| Model name to query, must be same as is in service. Default : resnet| -| --model_version MODEL_VERSION | Model version to query - must be numerical. List all version if omitted | -| --client_cert CLIENT_CERT | Specify mTLS client certificate file. Default: None.| -| --client_key CLIENT_KEY | Specify mTLS client key file. Default: None. | -| --ignore_server_verification | Skip TLS host verification. Do not use in production. Default: False. | -| --server_cert SERVER_CERT | Path to a custom directory containing trusted CA certificates, server certificate, or a CA_BUNDLE file. Default: None, will use default system CA cert store. | - -- Usage Example -```bash -python rest_get_model_status.py --rest_port 8000 --model_version 1 -{ - "model_version_status": [ - { - "version": "1", - "state": "AVAILABLE", - "status": { - "error_code": "OK", - "error_message": "OK" - } - } - ] -} -``` - -### Model Metadata API -- Command -```bash -python rest_get_model_metadata.py --help -usage: rest_get_model_metadata.py [-h] [--rest_url REST_URL] - [--rest_port REST_PORT] - [--model_name MODEL_NAME] - [--model_version MODEL_VERSION] - [--client_cert CLIENT_CERT] - [--client_key CLIENT_KEY] - [--ignore_server_verification] - [--server_cert SERVER_CERT] -``` -- Arguments - -| Argument | Description | -| :--- | :---- | -| -h, --help | show this help message and exit | -| --rest_url REST_URL | Specify url to REST API service. default: http://localhost | -| --rest_port REST_PORT | Specify port to REST API service. default: 8000 | -| --model_name MODEL_NAME | Define model name, must be same as is in service. default: resnet | -| --model_version MODEL_VERSION | Define model version - must be numerical | -| --client_cert CLIENT_CERT | Specify mTLS client certificate file. Default: None. | -| --client_key CLIENT_KEY | Specify mTLS client key file. Default: None. | -| --ignore_server_verification | Skip TLS host verification. Do not use in production. Default: False. | -| --server_cert SERVER_CERT | Path to a custom directory containing trusted CA certificates, server certificate, or a CA_BUNDLE file. | - -- Usage Example -```bash -python rest_get_model_metadata.py --rest_port 8000 -{ - "modelSpec": { - "name": "resnet", - "signatureName": "", - "version": "1" - }, - "metadata": { - "signature_def": { - "@type": "type.googleapis.com/tensorflow.serving.SignatureDefMap", - "signatureDef": { - "serving_default": { - "inputs": { - "0": { - "dtype": "DT_FLOAT", - "tensorShape": { - "dim": [ - { - "size": "1", - "name": "" - }, - { - "size": "3", - "name": "" - }, - { - "size": "224", - "name": "" - }, - { - "size": "224", - "name": "" - } - ], - "unknownRank": false - }, - "name": "0" - } - }, - "outputs": { - "1463": { - "dtype": "DT_FLOAT", - "tensorShape": { - "dim": [ - { - "size": "1", - "name": "" - }, - { - "size": "1000", - "name": "" - } - ], - "unknownRank": false - }, - "name": "1463" - } - }, - "methodName": "", - "defaults": {} - } - } - } - } -} -``` - -### Predict API - -- Command : -```bash -python rest_predict_resnet.py --help -usage: rest_predict_resnet.py [-h] --images_numpy_path IMAGES_NUMPY_PATH - [--labels_numpy_path LABELS_NUMPY_PATH] - [--rest_url REST_URL] [--rest_port REST_PORT] - [--input_name INPUT_NAME] - [--output_name OUTPUT_NAME] - [--transpose_input {False,True}] - [--transpose_method {nchw2nhwc,nhwc2nchw}] - [--iterations ITERATIONS] - [--batchsize BATCHSIZE] - [--model_name MODEL_NAME] - [--request_format {row_noname,row_name,column_noname,column_name}] - [--model_version MODEL_VERSION] - [--client_cert CLIENT_CERT] - [--client_key CLIENT_KEY] - [--ignore_server_verification] - [--server_cert SERVER_CERT] -``` - -- Arguments : - -| Argument | Description | -| :--- | :---- | -| -h, --help | Show help message and exit | -| --images_numpy_path IMAGES_NUMPY_PATH | Numpy in shape [n,w,h,c] or [n,c,h,w] | -| --labels_numpy_path LABELS_NUMPY_PATH| Numpy in shape [n,1] - can be used to check model accuracy | -| --rest_url REST_URL| Specify url to REST API service. Default: http://localhost | -| --rest_port REST_PORT| Specify port to REST API service. Default: 8000 | -| --input_name INPUT_NAME| Specify input tensor name. Default: input | -| --output_name OUTPUT_NAME| Specify output name. Default: resnet_v1_50/predictions/Reshape_1 | -| --transpose_input {False,True}| Set to False to skip NHWC>NCHW or NCHW>NHWC input transposing. Default: True| -| --transpose_method {nchw2nhwc,nhwc2nchw} | How the input transposition should be executed: nhwc2nchw or nhwc2nchw | -| --iterations ITERATIONS| Number of requests iterations, as default use number of images in numpy memmap. Default: 0 (consume all frames)| -| --batchsize BATCHSIZE| Number of images in a single request. Default: 1 | -| --model_name MODEL_NAME| Define model name, must be same as is in service. Default: resnet| -| --request_format {row_noname,row_name,column_noname,column_name}| Request format according to TF Serving API:row_noname,row_name,column_noname,column_name| -| --model_version MODEL_VERSION| Model version to be used. Default: LATEST | -| --client_cert CLIENT_CERT | Specify mTLS client certificate file. Default: None | -| --client_key CLIENT_KEY | Specify mTLS client key file. Default: None | -| --ignore_server_verification | Skip TLS host verification. Do not use in production. Default: False | -| --server_cert SERVER_CERT | Path to a custom directory containing trusted CA certificates, server certificate, or a CA_BUNDLE file. Default: None, will use default system CA cert store | - -- Usage Example -```bash -python rest_predict_resnet.py --images_numpy_path ../../imgs.npy --labels_numpy_path ../../lbs.npy --input_name 0 --output_name 1463 --rest_port 8000 --transpose_input False - -Image data range: 0 : 255 -Start processing: - Model name: resnet - Iterations: 10 - Images numpy path: ../../imgs.npy - Images in shape: (10, 3, 224, 224) - -output shape: (1, 1000) -Iteration 1; Processing time: 54.98 ms; speed 18.19 fps -imagenet top results in a single batch: - 0 airliner 404 ; Correct match. -output shape: (1, 1000) -Iteration 2; Processing time: 46.54 ms; speed 21.49 fps -imagenet top results in a single batch: - 0 Arctic fox, white fox, Alopex lagopus 279 ; Correct match. -output shape: (1, 1000) -Iteration 3; Processing time: 50.70 ms; speed 19.73 fps -imagenet top results in a single batch: - 0 bee 309 ; Correct match. -output shape: (1, 1000) -Iteration 4; Processing time: 46.89 ms; speed 21.33 fps -imagenet top results in a single batch: - 0 golden retriever 207 ; Correct match. -output shape: (1, 1000) -Iteration 5; Processing time: 45.78 ms; speed 21.84 fps -imagenet top results in a single batch: - 0 gorilla, Gorilla gorilla 366 ; Correct match. -output shape: (1, 1000) -Iteration 6; Processing time: 48.72 ms; speed 20.53 fps -imagenet top results in a single batch: - 0 magnetic compass 635 ; Correct match. -output shape: (1, 1000) -Iteration 7; Processing time: 45.20 ms; speed 22.12 fps -imagenet top results in a single batch: - 0 peacock 84 ; Correct match. -output shape: (1, 1000) -Iteration 8; Processing time: 45.50 ms; speed 21.98 fps -imagenet top results in a single batch: - 0 pelican 144 ; Correct match. -output shape: (1, 1000) -Iteration 9; Processing time: 45.30 ms; speed 22.08 fps -imagenet top results in a single batch: - 0 snail 113 ; Correct match. -output shape: (1, 1000) -Iteration 10; Processing time: 44.19 ms; speed 22.63 fps -imagenet top results in a single batch: - 0 zebra 340 ; Correct match. - -processing time for all iterations -average time: 46.80 ms; average speed: 21.37 fps -median time: 45.50 ms; median speed: 21.98 fps -max time: 54.00 ms; min speed: 18.52 fps -min time: 44.00 ms; max speed: 22.73 fps -time percentile 90: 50.40 ms; speed percentile 90: 19.84 fps -time percentile 50: 45.50 ms; speed percentile 50: 21.98 fps -time standard deviation: 2.93 -time variance: 8.56 -Classification accuracy: 100.00 -``` -#### **Submitting REST requests with data in binary format:** - -Using binary inputs feature requires loading model with layout set to `--layout NHWC:NCHW`: -```bash -wget -N https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/resnet50-binary-0001/FP32-INT1/resnet50-binary-0001.{xml,bin} -P models/resnet50/1 -docker run -d -u $(id -u) -v $(pwd)/models:/models -p 8000:8000 -p 9000:9000 openvino/model_server:latest --model_name resnet --model_path /models/resnet50 --port 9000 --rest_port 8000 --layout NHWC:NCHW -``` - -```bash -python rest_predict_binary_resnet.py --help -usage: rest_predict_binary_resnet.py [-h] [--images_list IMAGES_LIST] - [--rest_url REST_URL] - [--input_name INPUT_NAME] - [--output_name OUTPUT_NAME] - [--model_name MODEL_NAME] - [--request_format {row_noname,row_name,column_noname,column_name}] - [--batchsize BATCHSIZE] -``` - -- Arguments - - -| Argument | Description | -| :--- | :---- | -| -h, --help | show this help message and exit | -| --images_list IMAGES_LIST | path to a file with a list of labeled images | -| --rest_url REST_URL | Specify url to REST API service. default: http://localhost:8000 | -| --input_name INPUT_NAME | Specify input tensor name. default: image_bytes | -| --output_name OUTPUT_NAME | Specify output name. default: probabilities | -| --model_name MODEL_NAME | Define model name, must be same as is in service default: resnet | -| --request_format {row_noname,row_name,column_noname,column_name} | Request format according to TF Serving API: row_noname,row_name,column_noname,column_name | -| --batchsize BATCHSIZE | Number of images in a single request. default: 1 | - - -- Usage example -```bash -python rest_predict_binary_resnet.py --rest_url http://localhost:8000 --model_name resnet --input_name 0 --output_name 1463 --images ../../resnet_input_images.txt - -Start processing: - Model name: resnet - Images list file: ../../resnet_input_images.txt -Batch: 0; Processing time: 21.47 ms; speed 46.57 fps -output shape: (1, 1000) - 1 airliner 404 ; Correct match. -Batch: 1; Processing time: 22.43 ms; speed 44.58 fps -output shape: (1, 1000) - 2 Arctic fox, white fox, Alopex lagopus 279 ; Correct match. -Batch: 2; Processing time: 24.57 ms; speed 40.71 fps -output shape: (1, 1000) - 3 bee 309 ; Correct match. -Batch: 3; Processing time: 30.92 ms; speed 32.34 fps -output shape: (1, 1000) - 4 golden retriever 207 ; Correct match. -Batch: 4; Processing time: 30.70 ms; speed 32.58 fps -output shape: (1, 1000) - 5 gorilla, Gorilla gorilla 366 ; Correct match. -Batch: 5; Processing time: 30.26 ms; speed 33.05 fps -output shape: (1, 1000) - 6 magnetic compass 635 ; Correct match. -Batch: 6; Processing time: 31.26 ms; speed 31.99 fps -output shape: (1, 1000) - 7 peacock 84 ; Correct match. -Batch: 7; Processing time: 29.17 ms; speed 34.28 fps -output shape: (1, 1000) - 8 pelican 144 ; Correct match. -Batch: 8; Processing time: 25.83 ms; speed 38.71 fps -output shape: (1, 1000) - 9 snail 113 ; Correct match. -Batch: 9; Processing time: 32.50 ms; speed 30.77 fps -output shape: (1, 1000) - 10 zebra 340 ; Correct match. -Overall accuracy= 100.0 % -Average latency= 27.4 ms -``` diff --git a/client/python/tensorflow-serving-api/samples/grpc_get_model_metadata.py b/client/python/tensorflow-serving-api/samples/grpc_get_model_metadata.py deleted file mode 100644 index d1300a25f1..0000000000 --- a/client/python/tensorflow-serving-api/samples/grpc_get_model_metadata.py +++ /dev/null @@ -1,113 +0,0 @@ -# -# Copyright (c) 2018 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import argparse -from tensorflow_serving.apis import get_model_metadata_pb2 -from tensorflow_serving.apis import prediction_service_pb2_grpc -import grpc - -def model_metadata_response(response): - signature_def = response.metadata['signature_def'] - signature_map = get_model_metadata_pb2.SignatureDefMap() - signature_map.ParseFromString(signature_def.value) - serving_default = signature_map.ListFields()[0][1]['serving_default'] - serving_inputs = serving_default.inputs - input_blobs_keys = {key: {} for key in serving_inputs.keys()} - tensor_shape = {key: serving_inputs[key].tensor_shape - for key in serving_inputs.keys()} - for input_blob in input_blobs_keys: - inputs_shape = [d.size for d in tensor_shape[input_blob].dim] - tensor_dtype = serving_inputs[input_blob].dtype - input_blobs_keys[input_blob].update({'shape': inputs_shape}) - input_blobs_keys[input_blob].update({'dtype': tensor_dtype}) - - serving_outputs = serving_default.outputs - output_blobs_keys = {key: {} for key in serving_outputs.keys()} - tensor_shape = {key: serving_outputs[key].tensor_shape - for key in serving_outputs.keys()} - for output_blob in output_blobs_keys: - outputs_shape = [d.size for d in tensor_shape[output_blob].dim] - tensor_dtype = serving_outputs[output_blob].dtype - output_blobs_keys[output_blob].update({'shape': outputs_shape}) - output_blobs_keys[output_blob].update({'dtype': tensor_dtype}) - - return input_blobs_keys, output_blobs_keys - -dtype_name = [ 'DT_INVALID', - 'DT_FLOAT', - 'DT_DOUBLE', - 'DT_INT32', - 'DT_UINT8', - 'DT_INT16', - 'DT_INT8', - 'DT_STRING', - 'DT_COMPLEX64', - 'DT_INT64', - 'DT_BOOL', - 'DT_QINT8', - 'DT_QUINT8', - 'DT_QINT32', - 'DT_BFLOAT16', - 'DT_QINT16', - 'DT_QUINT16', - 'DT_UINT16', - 'DT_COMPLEX128', - 'DT_HALF', - 'DT_RESOURCE', - 'DT_VARIANT', - 'DT_UINT32', - 'DT_UINT64'] -# https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor.proto - -def main(): - parser = argparse.ArgumentParser(description='Get information about served models') - parser.add_argument('--grpc_address',required=False, default='localhost', help='Specify url to grpc service. default:localhost') - parser.add_argument('--grpc_port',required=False, default=9000, help='Specify port to grpc service. default: 9000') - parser.add_argument('--model_name', default='resnet', help='Define model name, must be same as is in service. default: resnet', - dest='model_name') - parser.add_argument('--model_version', default=None, type=int, help='Define model version - must be numerical', - dest='model_version') - args = vars(parser.parse_args()) - - channel = grpc.insecure_channel("{}:{}".format(args['grpc_address'],args['grpc_port'])) - - stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) - - print('Getting model metadata for model:',args.get('model_name')) - - metadata_field = "signature_def" - request = get_model_metadata_pb2.GetModelMetadataRequest() - request.model_spec.name = args.get('model_name') - if args.get('model_version') is not None: - request.model_spec.version.value = args.get('model_version') - request.metadata_field.append(metadata_field) - - result = stub.GetModelMetadata(request, 10.0) # result includes a dictionary with all model outputs - input_metadata, output_metadata = model_metadata_response( - response=result) - print('Inputs metadata:') - for i in input_metadata: - print("\tInput name: {}; shape: {}; dtype: {}" - .format(i, input_metadata[i]['shape'], - dtype_name[input_metadata[i]['dtype']])) - print('Outputs metadata:') - for i in output_metadata: - print("\tOutput name: {}; shape: {}; dtype: {}" - .format(i, output_metadata[i]['shape'], - dtype_name[output_metadata[i]['dtype']])) - -if __name__ == "__main__": - main() diff --git a/client/python/tensorflow-serving-api/samples/grpc_get_model_status.py b/client/python/tensorflow-serving-api/samples/grpc_get_model_status.py deleted file mode 100644 index 2eaa4c9214..0000000000 --- a/client/python/tensorflow-serving-api/samples/grpc_get_model_status.py +++ /dev/null @@ -1,65 +0,0 @@ -# -# Copyright (c) 2019 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import argparse -from tensorflow_serving.apis import get_model_status_pb2 -from tensorflow_serving.apis import model_service_pb2_grpc -import grpc - -def print_status_response(response): - version_status = response.model_version_status - for i in version_status: - print("\nModel version: {}".format(i.version)) - print("State",state_names[i.state]) - print("Error code: ",i.status.error_code) - print("Error message: ",i.status.error_message) - - return - -state_names = { - 0: "UNKNOWN", - 10: "START", - 20: "LOADING", - 30: "AVAILABLE", - 40: "UNLOADING", - 50: "END" -} -# https://github.com/tensorflow/serving/blob/master/tensorflow_serving/apis/get_model_status.proto - -parser = argparse.ArgumentParser(description='Get information about the status of served models over gRPC interface') -parser.add_argument('--grpc_address',required=False, default='localhost', help='Specify url to grpc service. default:localhost') -parser.add_argument('--grpc_port',required=False, default=9000, help='Specify port to grpc service. default: 9000') -parser.add_argument('--model_name', default='resnet', help='Model name to query. default: resnet', - dest='model_name') -parser.add_argument('--model_version', type=int, help='Model version to query. Lists all versions if omitted', - dest='model_version') -args = vars(parser.parse_args()) - -channel = grpc.insecure_channel("{}:{}".format(args['grpc_address'],args['grpc_port'])) - -stub = model_service_pb2_grpc.ModelServiceStub(channel) - -print('Getting model status for model:',args.get('model_name')) - -request = get_model_status_pb2.GetModelStatusRequest() -request.model_spec.name = args.get('model_name') -if args.get('model_version') is not None: - request.model_spec.version.value = args.get('model_version') - - -result = stub.GetModelStatus(request, 10.0) # result includes a dictionary with all model outputs - -print_status_response(response=result) diff --git a/client/python/tensorflow-serving-api/samples/grpc_predict_binary_resnet.py b/client/python/tensorflow-serving-api/samples/grpc_predict_binary_resnet.py deleted file mode 100644 index 4149a21d20..0000000000 --- a/client/python/tensorflow-serving-api/samples/grpc_predict_binary_resnet.py +++ /dev/null @@ -1,108 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import sys -sys.path.append("../../../../demos/common/python") - -import grpc -import numpy as np -import classes -from tensorflow import make_tensor_proto, make_ndarray, make_tensor_proto -import datetime -import argparse -from tensorflow_serving.apis import predict_pb2 -from tensorflow_serving.apis import prediction_service_pb2_grpc - -parser = argparse.ArgumentParser(description='Do requests to ie_serving and tf_serving using images in binary format') -parser.add_argument('--images_list', required=False, default='input_images.txt', help='path to a file with a list of labeled images') -parser.add_argument('--grpc_address',required=False, default='localhost', help='Specify url to grpc service. default:localhost') -parser.add_argument('--grpc_port',required=False, default=9000, help='Specify port to grpc service. default: 9000') -parser.add_argument('--input_name',required=False, default='image_bytes', help='Specify input tensor name. default: image_bytes') -parser.add_argument('--output_name',required=False, default='probabilities', help='Specify output name. default: probabilities') -parser.add_argument('--model_name', default='resnet', help='Define model name, must be same as is in service. default: resnet', - dest='model_name') -# If input numpy file has too few frames according to the value of iterations and the batch size, it will be -# duplicated to match requested number of frames -parser.add_argument('--batchsize', default=1, help='Number of images in a single request. default: 1', - dest='batchsize') -args = vars(parser.parse_args()) - -channel = grpc.insecure_channel("{}:{}".format(args['grpc_address'],args['grpc_port'])) -stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) -input_images = args.get('images_list') -with open(input_images) as f: - lines = f.readlines() -batch_size = int(args.get('batchsize')) -while batch_size > len(lines): - lines += lines -print('Start processing:') -print('\tModel name: {}'.format(args.get('model_name'))) -print('\tImages list file: {}'.format(args.get('images_list'))) - -count = 0 -matched = 0 -processing_times = np.zeros((0),int) - -batch_i = 0 -image_data = [] -labels = [] -for line in lines: - batch_i += 1 - path, label = line.strip().split(" ") - with open(path, 'rb') as f: - image_data.append(f.read()) - labels.append(label) - if batch_i < batch_size: - continue - request = predict_pb2.PredictRequest() - request.model_spec.name = args.get('model_name') - request.inputs[args['input_name']].CopyFrom(make_tensor_proto(image_data, shape=[len(image_data)])) - start_time = datetime.datetime.now() - result = stub.Predict(request, 10.0) # result includes a dictionary with all model outputs - end_time = datetime.datetime.now() - if args['output_name'] not in result.outputs: - print("Invalid output name", args['output_name']) - print("Available outputs:") - for Y in result.outputs: - print(Y) - exit(1) - duration = (end_time - start_time).total_seconds() * 1000 - processing_times = np.append(processing_times,np.array([int(duration)])) - output = make_ndarray(result.outputs[args['output_name']]) - nu = np.array(output) - # for object classification models show imagenet class - print('Batch: {}; Processing time: {:.2f} ms; speed {:.2f} fps'.format(count // batch_size, round(duration, 2), round(1000 / duration, 2))) - for i in range(nu.shape[0]): - offset = 0 - if nu.shape[1] == 1001: - offset = 1 - ma = np.argmax(nu[i]) - offset - mark_message = "" - if int(labels[i]) == ma: - matched += 1 - mark_message = "; Correct match." - else: - mark_message = "; Incorrect match. Should be {} {}".format(label, classes.imagenet_classes[int(label)]) - count += 1 - print("\t", count, classes.imagenet_classes[ma], ma, mark_message) - image_data = [] - labels = [] - batch_i = 0 - -latency = np.average(processing_times) -accuracy = matched / count - -print("Overall accuracy=",accuracy*100,"%") -print("Average latency=",latency,"ms") diff --git a/client/python/tensorflow-serving-api/samples/grpc_predict_resnet.py b/client/python/tensorflow-serving-api/samples/grpc_predict_resnet.py deleted file mode 100644 index d5ff6fa310..0000000000 --- a/client/python/tensorflow-serving-api/samples/grpc_predict_resnet.py +++ /dev/null @@ -1,178 +0,0 @@ -# -# Copyright (c) 2018-2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import sys -sys.path.append("../../../../demos/common/python") - -import grpc -import numpy as np -from tensorflow import make_tensor_proto, make_ndarray -import classes -import datetime -import argparse -from tensorflow_serving.apis import predict_pb2 -from tensorflow_serving.apis import prediction_service_pb2_grpc -from client_utils import print_statistics, prepare_certs - - -parser = argparse.ArgumentParser(description='Sends requests via TFS gRPC API using images in numpy format. ' - 'It displays performance statistics and optionally the model accuracy') -parser.add_argument('--images_numpy_path', required=True, help='numpy in shape [n,w,h,c] or [n,c,h,w]') -parser.add_argument('--labels_numpy_path', required=False, help='numpy in shape [n,1] - can be used to check model accuracy') -parser.add_argument('--grpc_address',required=False, default='localhost', help='Specify url to grpc service. default:localhost') -parser.add_argument('--grpc_port',required=False, default=9000, help='Specify port to grpc service. default: 9000') -parser.add_argument('--input_name',required=False, default='input', help='Specify input tensor name. default: input') -parser.add_argument('--output_name',required=False, default='resnet_v1_50/predictions/Reshape_1', - help='Specify output name. default: resnet_v1_50/predictions/Reshape_1') -parser.add_argument('--transpose_input', choices=["False", "True"], default="True", - help='Set to False to skip NHWC>NCHW or NCHW>NHWC input transposing. default: True', - dest="transpose_input") -parser.add_argument('--transpose_method', choices=["nchw2nhwc","nhwc2nchw"], default="nhwc2nchw", - help="How the input transposition should be executed: nhwc2nchw or nchw2nhwc", - dest="transpose_method") -parser.add_argument('--iterations', default=0, - help='Number of requests iterations, as default use number of images in numpy memmap. default: 0 (consume all frames)', - dest='iterations', type=int) -# If input numpy file has too few frames according to the value of iterations and the batch size, it will be -# duplicated to match requested number of frames -parser.add_argument('--batchsize', default=1, - help='Number of images in a single request. default: 1', - dest='batchsize') -parser.add_argument('--model_name', default='resnet', help='Define model name, must be same as is in service. default: resnet', - dest='model_name') -parser.add_argument('--pipeline_name', default='', help='Define pipeline name, must be same as is in service', - dest='pipeline_name') -parser.add_argument('--dag-batch-size-auto', default=False, action='store_true', help='add demultiplexer dimension at front', dest='dag-batch-size-auto') -parser.add_argument('--tls', default=False, action='store_true', help='use TLS communication with gRPC endpoint') -parser.add_argument('--server_cert', required=False, help='Path to server certificate') -parser.add_argument('--client_cert', required=False, help='Path to client certificate') -parser.add_argument('--client_key', required=False, help='Path to client key') - -args = vars(parser.parse_args()) - -address = "{}:{}".format(args['grpc_address'],args['grpc_port']) - -if args.get('tls'): - server_ca_cert, client_key, client_cert = prepare_certs(server_cert=args['server_cert'], - client_key=args['client_key'], - client_ca=args['client_cert']) - creds = grpc.ssl_channel_credentials(root_certificates=server_ca_cert, - private_key=client_key, certificate_chain=client_cert) - channel = grpc.secure_channel(address, creds) -else: - channel = grpc.insecure_channel(address) - -stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) - -processing_times = np.zeros((0),int) - -# optional preprocessing depending on the model -imgs = np.load(args['images_numpy_path'], mmap_mode='r', allow_pickle=False) -imgs = imgs - np.min(imgs) # Normalization 0-255 -imgs = imgs / np.ptp(imgs) * 255 # Normalization 0-255 -#imgs = imgs[:,:,:,::-1] # RGB to BGR -print('Image data range:', np.amin(imgs), ':', np.amax(imgs)) -# optional preprocessing depending on the model - -if args.get('labels_numpy_path') is not None: - lbs = np.load(args['labels_numpy_path'], mmap_mode='r', allow_pickle=False) - matched_count = 0 - total_executed = 0 -batch_size = int(args.get('batchsize')) - - -while batch_size >= imgs.shape[0]: - imgs = np.append(imgs, imgs, axis=0) - if args.get('labels_numpy_path') is not None: - lbs = np.append(lbs, lbs, axis=0) - -iterations = int((imgs.shape[0]//batch_size) if not (args.get('iterations') or args.get('iterations') != 0) else args.get('iterations')) - -print('Start processing:') -print('\tModel name: {}'.format(args.get('pipeline_name') if bool(args.get('pipeline_name')) else args.get('model_name'))) -print('\tIterations: {}'.format(iterations)) -print('\tImages numpy path: {}'.format(args.get('images_numpy_path'))) -if args.get('transpose_input') == "True": - if args.get('transpose_method') == "nhwc2nchw": - imgs = imgs.transpose((0,3,1,2)) - if args.get('transpose_method') == "nchw2nhwc": - imgs = imgs.transpose((0,2,3,1)) -print('\tNumpy file shape: {}\n'.format(imgs.shape)) - -iteration = 0 -is_pipeline_request = bool(args.get('pipeline_name')) - -while iteration <= iterations: - for x in range(0, imgs.shape[0] - batch_size + 1, batch_size): - iteration += 1 - if iteration > iterations: break - request = predict_pb2.PredictRequest() - request.model_spec.name = args.get('pipeline_name') if is_pipeline_request else args.get('model_name') - img = imgs[x:(x + batch_size)] - if args.get('labels_numpy_path') is not None: - lb = lbs[x:(x + batch_size)] - if args.get('dag-batch-size-auto'): - newShape = img.shape[0:1] + (1,) + img.shape[1:] - request.inputs[args['input_name']].CopyFrom(make_tensor_proto(img, shape=newShape)) - else: - request.inputs[args['input_name']].CopyFrom(make_tensor_proto(img, shape=(img.shape))) - start_time = datetime.datetime.now() - result = stub.Predict(request, 10.0) # result includes a dictionary with all model outputs - end_time = datetime.datetime.now() - if args['output_name'] not in result.outputs: - print("Invalid output name", args['output_name']) - print("Available outputs:") - for Y in result.outputs: - print(Y) - exit(1) - duration = (end_time - start_time).total_seconds() * 1000 - processing_times = np.append(processing_times,np.array([int(duration)])) - output = make_ndarray(result.outputs[args['output_name']]) - - nu = np.array(output) - # for object classification models show imagenet class - print('Iteration {}; Processing time: {:.2f} ms; speed {:.2f} fps'.format(iteration,round(np.average(duration), 2), - round(1000 * batch_size / np.average(duration), 2) - )) - # Comment out this section for non imagenet datasets - print("imagenet top results in a single batch:") - for i in range(nu.shape[0]): - if is_pipeline_request: - # shape (1,) - print("response shape", output.shape) - ma = nu[0] - 1 # indexes needs to be shifted left due to 1x1001 shape - else: - # shape (1,1000) - single_result = nu[[i],...] - offset = 0 - if nu.shape[1] == 1001: - offset = 1 - ma = np.argmax(single_result) - offset - mark_message = "" - if args.get('labels_numpy_path') is not None: - total_executed += 1 - if ma == lb[i]: - matched_count += 1 - mark_message = "; Correct match." - else: - mark_message = "; Incorrect match. Should be {} {}".format(lb[i], classes.imagenet_classes[lb[i]] ) - print("\t",i, classes.imagenet_classes[ma],ma, mark_message) - # Comment out this section for non imagenet datasets - -print_statistics(processing_times, batch_size) - -if args.get('labels_numpy_path') is not None: - print('Classification accuracy: {:.2f}'.format(100*matched_count/total_executed)) diff --git a/client/python/tensorflow-serving-api/samples/requirements.txt b/client/python/tensorflow-serving-api/samples/requirements.txt deleted file mode 100644 index 35c518d783..0000000000 --- a/client/python/tensorflow-serving-api/samples/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -tensorflow-serving-api>=2.16.1 -tensorflow>=2.16.1 -protobuf<=5.29.6 diff --git a/client/python/tensorflow-serving-api/samples/rest_get_model_metadata.py b/client/python/tensorflow-serving-api/samples/rest_get_model_metadata.py deleted file mode 100644 index 1d947ff6b6..0000000000 --- a/client/python/tensorflow-serving-api/samples/rest_get_model_metadata.py +++ /dev/null @@ -1,62 +0,0 @@ -# -# Copyright (c) 2019 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -import argparse -import json -import requests - - -parser = argparse.ArgumentParser(description='Get information about served models') -parser.add_argument('--rest_url', required=False, default='http://localhost', help='Specify url to REST API service. default: http://localhost') -parser.add_argument('--rest_port', required=False, default=8000, help='Specify port to REST API service. default: 8000') -parser.add_argument('--model_name', default='resnet', help='Define model name, must be same as is in service. default: resnet', - dest='model_name') -parser.add_argument('--model_version', default=None, type=int, help='Define model version - must be numerical', - dest='model_version') -parser.add_argument('--client_cert', required=False, default=None, help='Specify mTLS client certificate file. Default: None.') -parser.add_argument('--client_key', required=False, default=None, help='Specify mTLS client key file. Default: None.') -parser.add_argument('--ignore_server_verification', required=False, action='store_true', help='Skip TLS host verification. Do not use in production. Default: False.') -parser.add_argument('--server_cert', required=False, default=None, help='Path to a custom directory containing trusted CA certificates, server certificate, or a CA_BUNDLE file. Default: None, will use default system CA cert store.') - -args = vars(parser.parse_args()) - -certs = None -verify_server = None -if args.get('client_cert') is not None or args.get('client_key') is not None: - if args.get('client_cert') is not None and args.get('client_key') is not None and args.get('rest_url').startswith("https"): - certs = (args.get('client_cert'), args.get('client_key')) - if args.get('server_cert') is not None: - verify_server = args.get('server_cert') - if args.get('ignore_server_verification') is True: - verify_server = False - else: - print("Error: in order to use mTLS, you need to provide both --client_cert and --client_key. In addition, your --rest_url flag has to begin with 'https://'.") - exit(1) - -version = "" -if args.get('model_version') is not None: - version = "/versions/{}".format(args.get('model_version')) -result = requests.get("{}:{}/v1/models/{}{}/metadata".format(args['rest_url'], args['rest_port'], args['model_name'], version), cert=certs, verify=verify_server, timeout=5) - -try: - result_dic = json.loads(result.text) -except ValueError: - print("The server response is not json format: {}",format(result.text)) - exit(1) - -print(result.text) - diff --git a/client/python/tensorflow-serving-api/samples/rest_get_model_status.py b/client/python/tensorflow-serving-api/samples/rest_get_model_status.py deleted file mode 100644 index 9f2030347a..0000000000 --- a/client/python/tensorflow-serving-api/samples/rest_get_model_status.py +++ /dev/null @@ -1,61 +0,0 @@ -# -# Copyright (c) 2019 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -import argparse -import json -import requests - - -parser = argparse.ArgumentParser(description="Get served model's status for RESTful API interface") -parser.add_argument('--rest_url', required=False, default='http://localhost', help='Specify url to REST API service. default: http://localhost') -parser.add_argument('--rest_port', required=False, default=8000, help='Specify port to REST API service. default: 8000') -parser.add_argument('--model_name', default='resnet', help='Model name to query, must be the same as in the service. default: resnet', - dest='model_name') -parser.add_argument('--model_version', default=None, type=int, help='Model version to query - must be numerical. List all versions if omitted', - dest='model_version') -parser.add_argument('--client_cert', required=False, default=None, help='Specify mTLS client certificate file. Default: None.') -parser.add_argument('--client_key', required=False, default=None, help='Specify mTLS client key file. Default: None.') -parser.add_argument('--ignore_server_verification', required=False, action='store_true', help='Skip TLS host verification. Do not use in production. Default: False.') -parser.add_argument('--server_cert', required=False, default=None, help='Path to a custom directory containing trusted CA certificates, server certificate, or a CA_BUNDLE file. Default: None, will use default system CA cert store.') - -args = vars(parser.parse_args()) - -certs = None -verify_server = None -if args.get('client_cert') is not None or args.get('client_key') is not None: - if args.get('client_cert') is not None and args.get('client_key') is not None and args.get('rest_url').startswith("https"): - certs = (args.get('client_cert'), args.get('client_key')) - if args.get('server_cert') is not None: - verify_server = args.get('server_cert') - if args.get('ignore_server_verification') is True: - verify_server = False - else: - print("Error: in order to use mTLS, you need to provide both --client_cert and --client_key. In addition, your --rest_url flag has to begin with 'https://'.") - exit(1) - -version = "" -if args.get('model_version') is not None: - version = "/versions/{}".format(args.get('model_version')) -result = requests.get("{}:{}/v1/models/{}{}".format(args['rest_url'], args['rest_port'], args['model_name'], version), cert=certs, verify=verify_server, timeout=5) - -try: - result_dic = json.loads(result.text) -except ValueError: - print("The server response is not json format: {}",format(result.text)) - exit(1) - -print(result.text) diff --git a/client/python/tensorflow-serving-api/samples/rest_predict_binary_resnet.py b/client/python/tensorflow-serving-api/samples/rest_predict_binary_resnet.py deleted file mode 100644 index 23e508e351..0000000000 --- a/client/python/tensorflow-serving-api/samples/rest_predict_binary_resnet.py +++ /dev/null @@ -1,168 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import sys -sys.path.append("../../../../demos/common/python") - -import requests -import numpy as np -import base64 -import json -import classes -import datetime -import argparse - -def create_request(image_data, request_format): - signature = "serving_default" - instances = [] - if request_format == "row_name": - for image in image_data: - jpeg_bytes = base64.b64encode(image).decode('utf-8') - instances.append({args['input_name']: {"b64": jpeg_bytes}}) - else: - for image in image_data: - jpeg_bytes = base64.b64encode(image).decode('utf-8') - instances.append({"b64": jpeg_bytes}) - if request_format == "row_name": - data_obj = {"signature_name": signature, "instances": instances} - elif request_format == "row_noname": - data_obj = {"signature_name": signature, "instances": instances} - elif request_format == "column_name": - data_obj = {"signature_name": signature, 'inputs': {args['input_name']: instances}} - elif request_format == "column_noname": - data_obj = {"signature_name": signature, 'inputs': instances} - else: - print("invalid request format defined") - exit(1) - data_json = json.dumps(data_obj) - return data_json - -parser = argparse.ArgumentParser(description='Sends requests via TensorFlow Serving RESTful API using images in binary format. ' - 'It displays performance statistics and optionally the model accuracy') -parser.add_argument('--images_list', required=False, default='input_images.txt', - help='path to a file with a list of labeled images') -parser.add_argument('--rest_url', required=False, default='http://localhost:8000', - help='Specify url to REST API service. default: http://localhost:8000') -parser.add_argument('--input_name', required=False, default='image_bytes', - help='Specify input tensor name. default: image_bytes') -parser.add_argument('--output_name', required=False, default='probabilities', - help='Specify output name. default: probabilities') -parser.add_argument('--model_name', default='resnet', help='Define model name, must be same as is in service. default: resnet', - dest='model_name') -parser.add_argument('--request_format', default='row_noname', help='Request format according to TF Serving API: row_noname,row_name,column_noname,column_name', - choices=["row_noname", "row_name", "column_noname", "column_name"], dest='request_format') -# If input numpy file has too few frames according to the value of iterations and the batch size, it will be -# duplicated to match requested number of frames -parser.add_argument('--batchsize', default=1, help='Number of images in a single request. default: 1', - dest='batchsize') -args = vars(parser.parse_args()) - -address = "{}/v1/models/{}:predict".format( - args['rest_url'], args['model_name']) -input_images = args.get('images_list') -with open(input_images) as f: - lines = f.readlines() -batch_size = int(args.get('batchsize')) -while batch_size > len(lines): - lines += lines -print('Start processing:') -print('\tModel name: {}'.format(args.get('model_name'))) -print('\tImages list file: {}'.format(args.get('images_list'))) - -count = 0 -matched = 0 -processing_times = np.zeros((0), int) - -batch_i = 0 -image_data = [] -labels = [] -session = requests.Session() -for line in lines: - batch_i += 1 - path, label = line.strip().split(" ") - with open(path, 'rb') as f: - image_data.append(f.read()) - labels.append(label) - if batch_i < batch_size: - continue - # Compose a JSON Predict request (send JPEG image in base64). - predict_request = create_request(image_data, args['request_format']) - start_time = datetime.datetime.now() - result = session.post(address, data=predict_request) - end_time = datetime.datetime.now() - try: - result_dict = json.loads(result.text) - except ValueError: - print("The server response is not json format: {}",format(result.text)) - exit(1) - if "error" in result_dict: - print('Server returned error: {}'.format(result_dict)) - exit(1) - - if "outputs" in result_dict: # is column format - keyname = "outputs" - if type(result_dict[keyname]) is dict: - if args['output_name'] not in result_dict[keyname]: - print("Invalid output name", args['output_name']) - print("Available outputs:") - for Y in result_dict[keyname]: - print(Y) - exit(1) - output = result_dict[keyname][args['output_name']] - else: - output = result_dict[keyname] - elif "predictions" in result_dict: # is row format - keyname = "predictions" - if type(result_dict[keyname][0]) is dict: # are multiple outputs - output = [] - for row in result_dict[keyname]: # iterate over all results in the batch - output.append(row[args['output_name']]) - else: - output = result_dict[keyname] - else: - print("Missing required response in {}".format(result_dict)) - exit(1) - duration = (end_time - start_time).total_seconds() * 1000 - processing_times = np.append(processing_times, np.array([int(duration)])) - # for object classification models show imagenet class - print('Batch: {}; Processing time: {:.2f} ms; speed {:.2f} fps'.format( - count // batch_size, round(duration, 2), round(1000 / duration, 2))) - - nu = np.array(output) # numpy array with inference results - print("output shape: {}".format(nu.shape)) - for i in range(nu.shape[0]): - single_result = nu[[i], ...] - offset = 0 - if nu.shape[1] == 1001: - offset = 1 - ma = np.argmax(single_result) - offset - mark_message = "" - if int(labels[i]) == ma: - matched += 1 - mark_message = "; Correct match." - else: - mark_message = "; Incorrect match. Should be {} {}".format( - label, classes.imagenet_classes[int(label)]) - count += 1 - print("\t", count, classes.imagenet_classes[ma], ma, mark_message) - image_data = [] - labels = [] - batch_i = 0 - -latency = np.average(processing_times) -accuracy = matched / count - -print("Overall accuracy=", accuracy*100, "%") -print("Average latency=", latency, "ms") diff --git a/client/python/tensorflow-serving-api/samples/rest_predict_resnet.py b/client/python/tensorflow-serving-api/samples/rest_predict_resnet.py deleted file mode 100644 index 37ee1c86be..0000000000 --- a/client/python/tensorflow-serving-api/samples/rest_predict_resnet.py +++ /dev/null @@ -1,215 +0,0 @@ -# -# Copyright (c) 2019 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import sys -sys.path.append("../../../../demos/common/python") - -import numpy as np -import classes -import datetime -import argparse -import json -import requests -from client_utils import print_statistics - - -def create_request(img, request_format): - signature = "serving_default" - if request_format == "row_name": - instances = [] - for i in range(0, img.shape[0], 1): - instances.append({args['input_name']: img[i].tolist()}) - data_obj = {"signature_name": signature, "instances": instances} - elif request_format == "row_noname": - data_obj = {"signature_name": signature, 'instances': img.tolist()} - elif request_format == "column_name": - data_obj = {"signature_name": signature, - 'inputs': {args['input_name']: img.tolist()}} - elif request_format == "column_noname": - data_obj = {"signature_name": signature, 'inputs': img.tolist()} - else: - print("invalid request format defined") - exit(1) - data_json = json.dumps(data_obj) - return data_json - - -parser = argparse.ArgumentParser(description='Sends requests via TensorFlow Serving RESTfull API using images in numpy format. ' - 'It displays performance statistics and optionally the model accuracy') -parser.add_argument('--images_numpy_path', required=True, help='numpy in shape [n,w,h,c] or [n,c,h,w]') -parser.add_argument('--labels_numpy_path', required=False, help='numpy in shape [n,1] - can be used to check model accuracy') -parser.add_argument('--rest_url', required=False, default='http://localhost', help='Specify url to REST API service. Default: http://localhost') -parser.add_argument('--rest_port', required=False, default=8000, help='Specify port to REST API service. Default: 8000') -parser.add_argument('--input_name', required=False, default='input', help='Specify input tensor name. Default: input') -parser.add_argument('--output_name', required=False, default='resnet_v1_50/predictions/Reshape_1', - help='Specify output name. Default: resnet_v1_50/predictions/Reshape_1') -parser.add_argument('--transpose_input', choices=["False", "True"], default="True", - help='Set to False to skip NHWC>NCHW or NCHW>NHWC input transposing. Default: True', - dest="transpose_input") -parser.add_argument('--transpose_method', choices=["nchw2nhwc", "nhwc2nchw"], default="nhwc2nchw", - help="How the input transposition should be executed: nhwc2nchw or nhwc2nchw", - dest="transpose_method") -parser.add_argument('--iterations', default=0, - help='Number of requests iterations, as default use number of images in numpy memmap. Default: 0 (consume all frames)', - dest='iterations', type=int) -# If input numpy file has too few frames according to the value of iterations and the batch size, it will be -# duplicated to match requested number of frames -parser.add_argument('--batchsize', default=1, - help='Number of images in a single request. Default: 1', - dest='batchsize') -parser.add_argument('--model_name', default='resnet', help='Define model name, must be same as is in service. Default: resnet', - dest='model_name') -parser.add_argument('--request_format', default='row_noname', help='Request format according to TF Serving API: row_noname,row_name,column_noname,column_name', - choices=["row_noname", "row_name", "column_noname", "column_name"], dest='request_format') -parser.add_argument('--model_version', help='Model version to be used. Default: LATEST', - type=int, dest='model_version') -parser.add_argument('--client_cert', required=False, default=None, help='Specify mTLS client certificate file. Default: None.') -parser.add_argument('--client_key', required=False, default=None, help='Specify mTLS client key file. Default: None.') -parser.add_argument('--ignore_server_verification', required=False, action='store_true', help='Skip TLS host verification. Do not use in production. Default: False.') -parser.add_argument('--server_cert', required=False, default=None, help='Path to a custom directory containing trusted CA certificates, server certificate, or a CA_BUNDLE file. Default: None, will use default system CA cert store.') - -args = vars(parser.parse_args()) - -certs = None -verify_server = None -if args.get('client_cert') is not None or args.get('client_key') is not None: - if args.get('client_cert') is not None and args.get('client_key') is not None and args.get('rest_url').startswith("https"): - certs = (args.get('client_cert'), args.get('client_key')) - if args.get('server_cert') is not None: - verify_server = args.get('server_cert') - if args.get('ignore_server_verification') is True: - verify_server = False - else: - print("Error: in order to use mTLS, you need to provide both --client_cert and --client_key. In addition, your --rest_url flag has to begin with 'https://'.") - exit(1) - -processing_times = np.zeros((0), int) - -# optional preprocessing depending on the model -imgs = np.load(args['images_numpy_path'], mmap_mode='r', allow_pickle=False) -imgs = imgs - np.min(imgs) # Normalization 0-255 -imgs = imgs / np.ptp(imgs) * 255 # Normalization 0-255 -# imgs = imgs[:,:,:,::-1] # RGB to BGR -imgs = imgs.astype(np.uint8) -print('Image data range:', np.amin(imgs), ':', np.amax(imgs)) -# optional preprocessing depending on the model - -if args.get('labels_numpy_path') is not None: - lbs = np.load(args['labels_numpy_path'], mmap_mode='r', allow_pickle=False) - matched_count = 0 - total_executed = 0 -batch_size = int(args.get('batchsize')) - - -while batch_size >= imgs.shape[0]: - imgs = np.append(imgs, imgs, axis=0) - if args.get('labels_numpy_path') is not None: - lbs = np.append(lbs, lbs, axis=0) - -iterations = int((imgs.shape[0]//batch_size) if not (args.get('iterations') or args.get('iterations') != 0) else args.get('iterations')) - -print('Start processing:') -print('\tModel name: {}'.format(args.get('model_name'))) -print('\tIterations: {}'.format(iterations)) -print('\tImages numpy path: {}'.format(args.get('images_numpy_path'))) - -if args.get('transpose_input') == "True": - if args.get('transpose_method') == "nhwc2nchw": - imgs = imgs.transpose((0, 3, 1, 2)) - if args.get('transpose_method') == "nchw2nhwc": - imgs = imgs.transpose((0, 2, 3, 1)) -print('\tImages in shape: {}\n'.format(imgs.shape)) - -iteration = 0 - -session = requests.Session() -while iteration <= iterations: - for x in range(0, imgs.shape[0] - batch_size + 1, batch_size): - iteration += 1 - if iteration > iterations: - break - - img = imgs[x:(x + batch_size)] - if args.get('labels_numpy_path') is not None: - lb = lbs[x:(x + batch_size)] - data_json = create_request(img, args.get('request_format')) - version = "" - if args.get('model_version') is not None: - version = "/versions/{}".format(args.get('model_version')) - start_time = datetime.datetime.now() - result = session.post("{}:{}/v1/models/{}{}:predict".format(args['rest_url'], args['rest_port'], args['model_name'], version), data=data_json, cert=certs, verify=verify_server) - end_time = datetime.datetime.now() - try: - result_dict = json.loads(result.text) - except ValueError: - print("The server response is not json format: {}",format(result.text)) - exit(1) - if "error" in result_dict: - print('Server returned error: {}'.format(result_dict)) - exit(1) - - if "outputs" in result_dict: # is column format - keyname = "outputs" - if type(result_dict[keyname]) is dict: - if args['output_name'] not in result_dict[keyname]: - print("Invalid output name", args['output_name']) - print("Available outputs:") - for Y in result_dict[keyname]: - print(Y) - exit(1) - output = result_dict[keyname][args['output_name']] - else: - output = result_dict[keyname] - elif "predictions" in result_dict: # is row format - keyname = "predictions" - if type(result_dict[keyname][0]) is dict: # are multiple outputs - output = [] - for row in result_dict[keyname]: # iterate over all results in the batch - output.append(row[args['output_name']]) - else: - output = result_dict[keyname] - else: - print("Missing required response in {}".format(result_dict)) - exit(1) - - duration = (end_time - start_time).total_seconds() * 1000 - processing_times = np.append(processing_times, np.array([int(duration)])) - # print(output) - nu = np.array(output) # numpy array with inference results - print("output shape: {}".format(nu.shape)) - - # for object classification models show imagenet class - print('Iteration {}; Processing time: {:.2f} ms; speed {:.2f} fps'.format(iteration, round(np.average(duration), 2), - round(1000 * batch_size / np.average(duration), 2))) - # Comment out this section for non imagenet datasets - print("imagenet top results in a single batch:") - for i in range(nu.shape[0]): - single_result = nu[[i], ...] - ma = np.argmax(single_result) - mark_message = "" - if args.get('labels_numpy_path') is not None: - total_executed += 1 - if ma == lb[i]: - matched_count += 1 - mark_message = "; Correct match." - else: - mark_message = "; Incorrect match. Should be {} {}".format(lb[i], classes.imagenet_classes[lb[i]]) - print("\t", i, classes.imagenet_classes[ma], ma, mark_message) - # Comment out this section for non imagenet datasets - -print_statistics(processing_times, batch_size) - -if args.get('labels_numpy_path') is not None: - print('Classification accuracy: {:.2f}'.format(100*matched_count/total_executed)) diff --git a/common_settings.bzl b/common_settings.bzl index 2bc23a3430..8818ec83fe 100644 --- a/common_settings.bzl +++ b/common_settings.bzl @@ -70,17 +70,6 @@ def create_config_settings(): name = "not_disable_mediapipe", negate = ":disable_mediapipe", ) - native.config_setting( - name = "enable_drogon", - define_values = { - "USE_DROGON": "1", - }, - visibility = ["//visibility:public"], - ) - more_selects.config_setting_negation( - name = "enable_net_http", - negate = ":enable_drogon", - ) native.config_setting( name = "disable_cloud", define_values = { @@ -243,10 +232,6 @@ COMMON_STATIC_LIBS_LINKOPTS = select({ "/LTCG", ], }) -COPTS_DROGON = select({ - "//conditions:default": ["-DUSE_DROGON=0"], - "//:enable_drogon" : ["-DUSE_DROGON=1"], -}) DEFINES_PYTHON = select({ "//conditions:default": ["PYTHON_DISABLE=1"], "//:not_disable_python" : ["PYTHON_DISABLE=0"], diff --git a/demos/bert_question_answering/python/bert_question_answering.py b/demos/bert_question_answering/python/bert_question_answering.py index 4bb294e043..dd6444ae18 100644 --- a/demos/bert_question_answering/python/bert_question_answering.py +++ b/demos/bert_question_answering/python/bert_question_answering.py @@ -24,7 +24,7 @@ from tokens_bert import text_to_tokens, load_vocab_file, Token from html_reader import get_paragraphs import numpy as np -import ovmsclient +import tritonclient.grpc as grpcclient class ConcatenatedParagraph(): def __init__(self, text="", tokens=[]): @@ -99,7 +99,7 @@ def main(): args = build_argparser().parse_args() # create grpc connection - client = ovmsclient.make_grpc_client("{}:{}".format(args.grpc_address,args.grpc_port)) + client = grpcclient.InferenceServerClient(url="{}:{}".format(args.grpc_address,args.grpc_port)) if args.colors: COLOR_RED = "\033[91m" @@ -180,7 +180,16 @@ def main(): inputs[input_names[3]] = np.arange(input_ids_length, dtype=np.int64)[None,:] t_start = time.perf_counter() - res = client.predict(inputs, args.model_name, timeout=10.0) + infer_inputs = [] + for inp_name in inputs: + t = inputs[inp_name] + infer_input = grpcclient.InferInput(inp_name, t.shape, "INT64") + infer_input.set_data_from_numpy(t) + infer_inputs.append(infer_input) + result = client.infer(args.model_name, infer_inputs) + res = {} + for out_name in output_names: + res[out_name] = result.as_numpy(out_name) t_end = time.perf_counter() t_count += 1 diff --git a/demos/bert_question_answering/python/requirements.txt b/demos/bert_question_answering/python/requirements.txt index 0deafd1de3..44cb2c6a07 100644 --- a/demos/bert_question_answering/python/requirements.txt +++ b/demos/bert_question_answering/python/requirements.txt @@ -1,2 +1,2 @@ -ovmsclient validators +tritonclient[grpc] diff --git a/demos/bert_question_answering/python/utils.py b/demos/bert_question_answering/python/utils.py index e86aae8ada..573e823cc5 100644 --- a/demos/bert_question_answering/python/utils.py +++ b/demos/bert_question_answering/python/utils.py @@ -17,11 +17,8 @@ from tokens_bert import text_to_tokens, load_vocab_file from html_reader import get_paragraphs -import grpc import numpy as np -from tensorflow import make_tensor_proto, make_ndarray -from tensorflow_serving.apis import predict_pb2 -from tensorflow_serving.apis import prediction_service_pb2_grpc +import tritonclient.grpc as grpcclient # return entire sentence as start-end positions for a given answer (within the sentence). def find_sentence_range(context, s, e): @@ -40,8 +37,7 @@ def find_sentence_range(context, s, e): def predict_bert(question): # create grpc connection - channel = grpc.insecure_channel("{}:{}".format("localhost", 9000)) - stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) + client = grpcclient.InferenceServerClient(url="{}:{}".format("localhost", 9000)) # load vocabulary file for model vocab = load_vocab_file("vocab.txt") @@ -103,18 +99,18 @@ def predict_bert(question): #print("inputs:",inputs) - # create grpc prediction request - request = predict_pb2.PredictRequest() - request.model_spec.name = "bert" + # create kserve grpc prediction request + infer_inputs = [] for inp_name in inputs: - request.inputs[inp_name].CopyFrom(make_tensor_proto(inputs[inp_name], shape=(inputs[inp_name].shape))) + t = inputs[inp_name] + infer_input = grpcclient.InferInput(inp_name, t.shape, "INT32") + infer_input.set_data_from_numpy(t) + infer_inputs.append(infer_input) - result = stub.Predict(request, 10.0) # result includes a dictionary with all model outputs - #print("\nresult:", result) + result = client.infer("bert", infer_inputs) res = {} for out_name in output_names: - # print("out_name:",out_name) - res[out_name] = make_ndarray(result.outputs[out_name]) + res[out_name] = result.as_numpy(out_name) # get start-end scores for context def get_score(name): diff --git a/demos/classification_using_paddlepaddle_model/python/classification_using_paddlepaddle_model.py b/demos/classification_using_paddlepaddle_model/python/classification_using_paddlepaddle_model.py index 678d42d87e..666633c9b0 100644 --- a/demos/classification_using_paddlepaddle_model/python/classification_using_paddlepaddle_model.py +++ b/demos/classification_using_paddlepaddle_model/python/classification_using_paddlepaddle_model.py @@ -21,7 +21,7 @@ import numpy as np import argparse -from ovmsclient import make_grpc_client +import tritonclient.grpc as grpcclient def image_preprocess_mobilenetv3(img_path): @@ -57,10 +57,14 @@ def build_parser(): modelname = 'mobilenet' test_image = image_preprocess_mobilenetv3(args.image_input_path) - client = make_grpc_client(f"{args.grpc_address}:{args.grpc_port}") - input_key = next(iter(client.get_model_metadata(model_name=modelname)['inputs'])) + client = grpcclient.InferenceServerClient(url=f"{args.grpc_address}:{args.grpc_port}") + metadata = client.get_model_metadata(modelname) + input_key = metadata.inputs[0].name - classification_output = client.predict({ input_key: test_image }, modelname) + infer_input = grpcclient.InferInput(input_key, test_image.shape, "FP32") + infer_input.set_data_from_numpy(test_image) + result = client.infer(modelname, [infer_input]) + classification_output = result.as_numpy(metadata.outputs[0].name) #filter and print the top 5 results top_k(classification_output) diff --git a/demos/classification_using_paddlepaddle_model/python/requirements.txt b/demos/classification_using_paddlepaddle_model/python/requirements.txt index 0f86a44cd0..23ba65aea3 100644 --- a/demos/classification_using_paddlepaddle_model/python/requirements.txt +++ b/demos/classification_using_paddlepaddle_model/python/requirements.txt @@ -1,3 +1,3 @@ -ovmsclient +tritonclient[grpc] opencv-python==4.8.1.78 numpy<2.0.0 \ No newline at end of file diff --git a/demos/common/python/requirements.txt b/demos/common/python/requirements.txt index 9dfebf25db..427c7cae26 100644 --- a/demos/common/python/requirements.txt +++ b/demos/common/python/requirements.txt @@ -1,5 +1,4 @@ futures==3.1.1 opencv-python>=4.8.1.78 -tensorflow-serving-api==2.16.1 -tensorflow==2.16.1 +tritonclient[grpc] numpy<2.0.0 diff --git a/demos/face_blur/python/Makefile b/demos/face_blur/python/Makefile deleted file mode 100644 index 5d2545b1bf..0000000000 --- a/demos/face_blur/python/Makefile +++ /dev/null @@ -1,37 +0,0 @@ -# -# Copyright (c) 2022 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -.PHONY: setup_repository - -default: setup_repository - -BASE_OS?=ubuntu - -setup_repository: -# Download face detection model - mkdir -p workspace/face-detection-retail-0004/1 - curl https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/face-detection-retail-0004/FP32/face-detection-retail-0004.xml -o workspace/face-detection-retail-0004/1/face-detection-retail-0004.xml - curl https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/face-detection-retail-0004/FP32/face-detection-retail-0004.bin -o workspace/face-detection-retail-0004/1/face-detection-retail-0004.bin -# Build face_blur custom node - cd ../../../src/custom_nodes && \ - make BASE_OS=${BASE_OS} NODES=face_blur && \ - mkdir -p ../../demos/face_blur/python/workspace/lib/ && \ - mv lib/${BASE_OS}/libcustom_node_face_blur.so ../../demos/face_blur/python/workspace/lib/libcustom_node_face_blur.so -# Copy configuration file to workspace directory - cp config.json workspace/. - -clean: - @rm -rf workspace diff --git a/demos/face_blur/python/README.md b/demos/face_blur/python/README.md deleted file mode 100644 index 10d73bae2e..0000000000 --- a/demos/face_blur/python/README.md +++ /dev/null @@ -1,75 +0,0 @@ -# Face Blur Pipeline Demo with OVMS {#ovms_demo_face_blur_pipeline} - -This document demonstrates how to create pipelines using object detection models from OpenVINO Model Zoo in order to blur the image. As an example, we will use [face-detection-retail-0004](https://github.com/openvinotoolkit/open_model_zoo/blob/2022.1.0/models/intel/face-detection-retail-0004/README.md) to detect multiple faces on the image. Then, for each detected face we will blur it using [face_blur](https://github.com/openvinotoolkit/model_server/blob/main/src/custom_nodes/face_blur) example custom node. - -## Pipeline Configuration Graph - -Below is depicted graph implementing face blur pipeline execution. - -![Face Blur Pipeline Graph](face_blur_pipeline.svg) - -It include the following Nodes: -- Model `face-detection-retail-0004` - deep learning model which takes user image as input. Its output contain information about faces coordinates and confidence levels. -- Custom node `face_blur` - it includes C++ implementation of image blurring. By analysing the output it produces image blurred in spots detected by object detection model based on the configurable score level threshold. Custom node also resizes it to the target resolution. All operations on the images employ OpenCV libraries which are preinstalled in the OVMS. Learn more about the [face_blur custom node](https://github.com/openvinotoolkit/model_server/blob/main/src/custom_nodes/face_blur). -- Response - image blurred in spots detected by object detection model. - -## Prepare workspace to run the demo - -To successfully deploy face blur pipeline you need to have a workspace that contains: -- [face-detection-retail-0004](https://github.com/openvinotoolkit/open_model_zoo/blob/2022.1.0/models/intel/face-detection-retail-0004/README.md) -- Custom node for image blurring -- Configuration file - -Clone the repository and enter face_blur directory -```bash -git clone https://github.com/openvinotoolkit/model_server.git -cd model_server/demos/face_blur/python -``` - -You can prepare the workspace that contains all the above by just running - -```bash -make -``` - -### Final directory structure - -Once the `make` procedure is finished, you should have `workspace` directory ready with the following content. -```bash -workspace -├── config.json -├── face-detection-retail-0004 -│   └── 1 -│   ├── face-detection-retail-0004.bin -│   └── face-detection-retail-0004.xml -└── lib - └── libcustom_node_face_blur.so -``` - -## Deploying OVMS - -Deploy OVMS with face blur pipeline using the following command: - -```bash -docker run -p 9000:9000 -d -v ${PWD}/workspace:/workspace openvino/model_server --config_path /workspace/config.json --port 9000 -``` - -## Requesting the Service - -Install python dependencies: -```bash -pip3 install -r requirements.txt -``` - -Now you can create a directory for blurred image and run the client: -```bash -mkdir results -``` - -```bash -python face_blur.py --grpc_port 9000 --image_input_path ../../common/static/images/people/people1.jpeg --blurred_image_save_path ./results --image_width 600 --image_height 400 --image_layout NHWC -``` - -Exemplary result of running the demo: - -![Face Blur Image](face_blur_image.jpg) diff --git a/demos/face_blur/python/config.json b/demos/face_blur/python/config.json deleted file mode 100644 index 45894b590c..0000000000 --- a/demos/face_blur/python/config.json +++ /dev/null @@ -1,91 +0,0 @@ -{ - "model_config_list": [ - { - "config": { - "name": "face-detection", - "base_path": "/workspace/face-detection-retail-0004", - "shape": "(1,400,600,3)", - "layout": "NHWC:NCHW" - } - } - ], - "custom_node_library_config_list": [ - { - "name": "face_blur_lib", - "base_path": "/workspace/lib/libcustom_node_face_blur.so" - } - ], - "pipeline_config_list": [ - { - "name": "face_blur_pipeline", - "inputs": [ - "image" - ], - "nodes": [ - { - "name": "face_detection_node", - "model_name": "face-detection", - "type": "DL model", - "inputs": [ - { - "data": { - "node_name": "request", - "data_item": "image" - } - } - ], - "outputs": [ - { - "data_item": "detection_out", - "alias": "detection" - } - ] - }, - { - "name": "face_blur_node", - "library_name": "face_blur_lib", - "type": "custom", - "params": { - "original_image_width": "600", - "original_image_height": "400", - "target_image_width": "300", - "target_image_height": "200", - "original_image_layout": "NHWC", - "target_image_layout": "NHWC", - "confidence_threshold": "0.7", - "debug": "true", - "gaussian_blur_kernel_size": "51" - }, - "inputs": [ - { - "image": { - "node_name": "request", - "data_item": "image" - } - }, - { - "detection": { - "node_name": "face_detection_node", - "data_item": "detection" - } - } - ], - "outputs": [ - { - "data_item": "image", - "alias": "blurred_image" - } - ] - } - ], - "outputs": [ - { - "image": { - "node_name": "face_blur_node", - "data_item": "blurred_image" - } - } - ] - } - ] -} diff --git a/demos/face_blur/python/face_blur.py b/demos/face_blur/python/face_blur.py deleted file mode 100644 index 9fb15a55e7..0000000000 --- a/demos/face_blur/python/face_blur.py +++ /dev/null @@ -1,76 +0,0 @@ -# -# Copyright (c) 2022 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import cv2 -import os -import numpy as np -import argparse -import ovmsclient - -parser = argparse.ArgumentParser(description='Client for OCR pipeline') -parser.add_argument('--grpc_address', required=False, default='localhost', help='Specify url to grpc service. default:localhost') -parser.add_argument('--grpc_port', required=False, default=9000, help='Specify port to grpc service. default: 9178') -parser.add_argument('--pipeline_name', required=False, default='face_blur_pipeline', help='Pipeline name to request. default: face_blur_pipeline') -parser.add_argument('--image_input_name', required=False, default='image', help='Pipeline input name for input with image. default: image') -parser.add_argument('--image_input_path', required=True, help='Input image path.') -parser.add_argument('--image_width', required=False, default=600, help='Original image width. default: 600') -parser.add_argument('--image_height', required=False, default=400, help='Original image height. default: 400') -parser.add_argument('--image_layout', required=False, default='NHWC', choices=['NCHW', 'NHWC', 'BINARY'], help='Pipeline input image layout. default: NCHW') -parser.add_argument('--blurred_image_save_path', required=False, default='', help='Path to save blurred image') - -args = vars(parser.parse_args()) - -def prepare_input_in_nchw_format(name, path, height, width): - img = cv2.imread(path).astype(np.float32) # BGR color format, shape HWC - img = cv2.resize(img, (width, height)) - img = img.transpose(2,0,1).reshape(1,3,height,width) - return {name: img} - -def prepare_input_in_nhwc_format(name, path, height, width): - img = cv2.imread(path).astype(np.float32) # BGR color format, shape HWC - img = cv2.resize(img, (width, height)) - img = img.reshape(1,height,width,3) - return {name: img} - -def prepare_input_in_binary_format(name, path): - with open(path, 'rb') as f: - data = f.read() - return {name: data} - -def save_image(image, location): - image = image[0] - if len(image.shape) == 3 and image.shape[0] == 3: # NCHW - image = image.transpose(1,2,0) - cv2.imwrite(os.path.join(location, 'image.jpg'), image) - -address = "{}:{}".format(args['grpc_address'],args['grpc_port']) - -client = ovmsclient.make_grpc_client(address) - -if args['image_layout'] == 'NCHW': - input = prepare_input_in_nchw_format(args['image_input_name'], args['image_input_path'], int(args['image_height']), int(args['image_width'])) -elif args['image_layout'] == 'NHWC': - input = prepare_input_in_nhwc_format(args['image_input_name'], args['image_input_path'], int(args['image_height']), int(args['image_width'])) -else: - input = prepare_input_in_binary_format(args['image_input_name'], args['image_input_path']) - -try: - response = client.predict(input, args['pipeline_name']) -except Exception as err: - raise err - -print(f'Saving image to {args["blurred_image_save_path"]}') -save_image(response, args['blurred_image_save_path']) diff --git a/demos/face_blur/python/face_blur_image.jpg b/demos/face_blur/python/face_blur_image.jpg deleted file mode 100644 index d062b0f6ff..0000000000 Binary files a/demos/face_blur/python/face_blur_image.jpg and /dev/null differ diff --git a/demos/face_blur/python/face_blur_pipeline.svg b/demos/face_blur/python/face_blur_pipeline.svg deleted file mode 100644 index 326542defe..0000000000 --- a/demos/face_blur/python/face_blur_pipeline.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Request
inputs: image
Request...
Model: face-detection-retail-0004
inputs: image
outputs: detection_out
Model: face-detection-retail-00...
Custom Node: face_blur
inputs: image, detection
outputs: image
Custom Node: face_blur...
Response
outputs: image
Response...Text is not SVG - cannot display diff --git a/demos/face_blur/python/requirements.txt b/demos/face_blur/python/requirements.txt deleted file mode 100644 index ff19d1b01b..0000000000 --- a/demos/face_blur/python/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -ovmsclient -opencv-python>=4.8.1.78 -numpy<2.0.0 \ No newline at end of file diff --git a/demos/face_detection/python/README.md b/demos/face_detection/python/README.md index 96496170dd..b51ac388da 100644 --- a/demos/face_detection/python/README.md +++ b/demos/face_detection/python/README.md @@ -19,7 +19,7 @@ The example relies on the model [face-detection-retail-0004](https://github.com/ Clone the repository and enter face_detection directory ```console git clone https://github.com/openvinotoolkit/model_server.git -cd model_server/demos/face_detection/python +cd model_server/demos/face_detection/python/ ``` Prepare environment: @@ -37,7 +37,7 @@ usage: face_detection.py [-h] [--input_images_dir INPUT_IMAGES_DIR] [--server_cert SERVER_CERT] [--client_cert CLIENT_CERT] [--client_key CLIENT_KEY] -Demo for face detection requests via TFS gRPC API analyses input images and +Demo for face detection requests via gRPC API analyses input images and saves images with bounding boxes drawn around detected faces. It relies on face_detection model... diff --git a/demos/face_detection/python/face_detection.py b/demos/face_detection/python/face_detection.py index d3b0a549fc..d2080fe10c 100644 --- a/demos/face_detection/python/face_detection.py +++ b/demos/face_detection/python/face_detection.py @@ -20,12 +20,9 @@ import argparse import cv2 import datetime -import grpc import numpy as np import os -from tensorflow import make_tensor_proto, make_ndarray -from tensorflow_serving.apis import predict_pb2 -from tensorflow_serving.apis import prediction_service_pb2_grpc +import tritonclient.grpc as grpcclient from client_utils import print_statistics, prepare_certs @@ -36,7 +33,7 @@ def load_image(file_path): # change shape to NCHW return img -parser = argparse.ArgumentParser(description='Demo for face detection requests via TFS gRPC API analyses input images ' +parser = argparse.ArgumentParser(description='Demo for face detection requests via gRPC API analyses input images ' 'and saves images with bounding boxes drawn around detected faces. ' 'It relies on face_detection model...') @@ -56,18 +53,16 @@ def load_image(file_path): address = "{}:{}".format(args['grpc_address'],args['grpc_port']) -channel = None if args.get('tls'): server_ca_cert, client_key, client_cert = prepare_certs(server_cert=args['server_cert'], client_key=args['client_key'], client_ca=args['client_cert']) - creds = grpc.ssl_channel_credentials(root_certificates=server_ca_cert, - private_key=client_key, certificate_chain=client_cert) - channel = grpc.secure_channel(address, creds) + client = grpcclient.InferenceServerClient(url=address, ssl=True, + root_certificates=server_ca_cert, + private_key=client_key, + certificate_chain=client_cert) else: - channel = grpc.insecure_channel(address) - -stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) + client = grpcclient.InferenceServerClient(url=address) files = os.listdir(args['input_images_dir']) batch_size = args['batch_size'] @@ -87,18 +82,17 @@ def load_image(file_path): for x in range(0, imgs.shape[0] - batch_size + 1, batch_size): iteration += 1 - request = predict_pb2.PredictRequest() - request.model_spec.name = args['model_name'] img = imgs[x:(x + batch_size)] print("\nRequest shape", img.shape) - request.inputs["data"].CopyFrom(make_tensor_proto(img, shape=(img.shape))) + infer_input = grpcclient.InferInput("data", img.shape, "FP32") + infer_input.set_data_from_numpy(img) start_time = datetime.datetime.now() - result = stub.Predict(request, 10.0) # result includes a dictionary with all model outputs + result = client.infer(args['model_name'], [infer_input]) end_time = datetime.datetime.now() duration = (end_time - start_time).total_seconds() * 1000 processing_times = np.append(processing_times,np.array([int(duration)])) - output = make_ndarray(result.outputs["detection_out"]) + output = result.as_numpy("detection_out") print("Response shape", output.shape) for y in range(0,img.shape[0]): # iterate over responses from all images in the batch img_out = img[y,:,:,:] diff --git a/demos/horizontal_text_detection/python/Makefile b/demos/horizontal_text_detection/python/Makefile deleted file mode 100644 index e93828d042..0000000000 --- a/demos/horizontal_text_detection/python/Makefile +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright (c) 2022 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -.PHONY: setup_repository - -default: setup_repository - -BUILD_CUSTOM_NODE=false -BASE_OS=ubuntu - -setup_repository: -# Download horizontal text detection model - mkdir -p workspace/horizontal-text-detection-0001/1 - curl https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/horizontal-text-detection-0001/FP32/horizontal-text-detection-0001.bin -o workspace/horizontal-text-detection-0001/1/horizontal-text-detection-0001.bin - curl https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/horizontal-text-detection-0001/FP32/horizontal-text-detection-0001.xml -o workspace/horizontal-text-detection-0001/1/horizontal-text-detection-0001.xml -# Download text recognition model - mkdir -p workspace/text-recognition-0012/1 - curl https://storage.openvinotoolkit.org/repositories/open_model_zoo/2023.0/models_bin/1/text-recognition-0012/FP32/text-recognition-0012.bin -o workspace/text-recognition-0012/1/text-recognition-0012.bin - curl https://storage.openvinotoolkit.org/repositories/open_model_zoo/2023.0/models_bin/1/text-recognition-0012/FP32/text-recognition-0012.xml -o workspace/text-recognition-0012/1/text-recognition-0012.xml -ifeq ($(BUILD_CUSTOM_NODE),true) -# Build custom node - cd ../../../src/custom_nodes && \ - make BASE_OS=$(BASE_OS) NODES=horizontal_ocr && \ - mkdir -p ../../demos/horizontal_text_detection/python/workspace/lib && \ - mv lib/$(BASE_OS)/libcustom_node_horizontal_ocr.so ../../demos/horizontal_text_detection/python/workspace/lib/libcustom_node_horizontal_ocr.so -endif -# Copy configuration file to workspace directory - cp config.json workspace/. -ifeq ($(BUILD_CUSTOM_NODE),false) - sed -i 's:\/workspace\/lib\/libcustom_node_horizontal_ocr.so:\/ovms\/lib\/custom_nodes\/libcustom_node_horizontal_ocr.so:g' workspace/config.json -endif -clean: - @rm -rf workspace \ No newline at end of file diff --git a/demos/horizontal_text_detection/python/README.md b/demos/horizontal_text_detection/python/README.md deleted file mode 100644 index 810ae48bfd..0000000000 --- a/demos/horizontal_text_detection/python/README.md +++ /dev/null @@ -1,267 +0,0 @@ -# Horizontal Text Detection in Real-Time {#ovms_demo_horizontal_text_detection} -This demo presents a use case with a client written in python which captures camera frames and performs text spotting analysis via gRPC requests to OVMS. The client visualizes the results as a boxes depicted on the original image frames using OpenCV in real-time. -The client can work efficiently also over slow internet connection with long latency thanks to image data compression and parallel execution for multiple frames. - -![horizontal text detection](horizontal-text-detection.gif) - -### Download horizontal text detection model from OpenVINO Model Zoo - -```bash -curl -L --create-dirs https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/horizontal-text-detection-0001/FP32/horizontal-text-detection-0001.bin -o horizontal-text-detection-0001/1/horizontal-text-detection-0001.bin https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/horizontal-text-detection-0001/FP32/horizontal-text-detection-0001.xml -o horizontal-text-detection-0001/1/horizontal-text-detection-0001.xml -``` - -```bash -tree horizontal-text-detection-0001 -horizontal-text-detection-0001 -└── 1 - ├── horizontal-text-detection-0001.bin - └── horizontal-text-detection-0001.xml -``` - -### Start the OVMS container: -```bash -chmod -R 755 horizontal-text-detection-0001 -docker run -d -u $(id -u):$(id -g) -v $(pwd)/horizontal-text-detection-0001:/model -p 9000:9000 openvino/model_server:latest \ ---model_path /model --model_name text --port 9000 --layout NHWC:NCHW -``` - -### Run the client - -Clone the repository and enter horizontal_text_detection directory -```bash -git clone https://github.com/openvinotoolkit/model_server.git -cd model_server/demos/horizontal_text_detection/python -``` - -Install required packages: -```bash -pip3 install -r requirements.txt -``` - -Start the client -```bash -python horizontal_text_detection.py --grpc_address localhost --grpc_port 9000 -``` -You can also change the camera ID: -```bash -python horizontal_text_detection.py --grpc_address localhost --grpc_port 9000 --video_source 0 -``` -Or choose to work with video file as well: -```bash -python horizontal_text_detection.py --grpc_address localhost --grpc_port 9000 --video_source ~/video.mp4 -``` -Example output: -```bash -Initializing requesting thread index: 0 -Initializing requesting thread index: 1 -Initializing requesting thread index: 2 -Initializing requesting thread index: 3 -Launching requesting thread index: 0 -Launching requesting thread index: 1 -Launching requesting thread index: 2 -Launching requesting thread index: 3 -ThreadID: 0; Current FPS: 31.25; Average FPS: 25.64; Average latency: 140.98ms -ThreadID: 1; Current FPS: 31.23; Average FPS: 25.67; Average latency: 136.36ms -ThreadID: 2; Current FPS: 29.41; Average FPS: 25.70; Average latency: 130.88ms -ThreadID: 3; Current FPS: 30.30; Average FPS: 25.73; Average latency: 135.65ms -... -``` - -> **NOTE**: Video source is cropped to 704x704 resolution to match model input size. - -## Recognize Detected Text with OCR Pipeline -Optical Character Recognition (OCR) pipeline based on [horizontal text detection](https://github.com/openvinotoolkit/open_model_zoo/blob/releases/2023/0/models/intel/horizontal-text-detection-0001/README.md) model, [text recognition](https://github.com/openvinotoolkit/open_model_zoo/tree/2023.0.0/models/intel/text-recognition-0012) -combined with a custom node implementation can be used with the same python script used before. OCR pipeline provides location of detected text boxes on the image and additionally recognized text for each box. - -![horizontal text detection using OCR pipeline](horizontal-text-detection-ocr.gif) - -### Prepare workspace to run the demo - -To successfully deploy OCR pipeline you need to have a workspace that contains: -- [horizontal text detection](https://github.com/openvinotoolkit/open_model_zoo/blob/releases/2022/1/models/intel/horizontal-text-detection-0001/README.md) and [text recognition](https://github.com/openvinotoolkit/open_model_zoo/tree/2023.0.0/models/intel/text-recognition-0012) models -- Custom node for image processing -- Configuration file - -Clone the repository and enter horizontal_text_detection directory -```bash -git clone https://github.com/openvinotoolkit/model_server.git -cd model_server/demos/horizontal_text_detection/python -``` - -You can prepare the workspace that contains all the above by just running `make` command. -Since custom node used in this demo is included in OpenVINO Model Server image you can either use the custom node from the image, or build one. - -If you just want to quickly run this demo and use already compiled custom node, run: - -```bash -make -``` - -#### Directory structure (without custom node) - -Once the `make` procedure is finished, you should have `workspace` directory ready with the following content. - -```bash -workspace/ -├── config.json -├── horizontal-text-detection-0001 -│   └── 1 -│   ├── horizontal-text-detection-0001.bin -│   └── horizontal-text-detection-0001.xml -└── text-recognition-0012 - └── 1 - ├── text-recognition-0012.bin - └── text-recognition-0012.xml - -``` - -If you modified the custom node or for some other reason, you want to have it compiled and then attached to the container, run: - -```bash -make BUILD_CUSTOM_NODE=true BASE_OS=ubuntu -``` - -#### Directory structure (with custom node) - -Once the `make` procedure is finished, you should have `workspace` directory ready with the following content. - -```bash -workspace/ -├── config.json -├── horizontal-text-detection-0001 -│   └── 1 -│   ├── horizontal-text-detection-0001.bin -│   └── horizontal-text-detection-0001.xml -├── lib -│   └── libcustom_node_horizontal_ocr.so -└── text-recognition-0012 - └── 1 - ├── text-recognition-0012.bin - └── text-recognition-0012.xml - -``` -## Deploying OVMS - -Deploy OVMS with faces analysis pipeline using the following command: - -```bash -docker run -p 9000:9000 -d -v ${PWD}/workspace:/workspace openvino/model_server --config_path /workspace/config.json --port 9000 -``` - -### Sending Request to the Server - -Install python dependencies: -```bash -pip3 install -r requirements.txt -``` - -Start the client -```bash -python horizontal_text_detection.py --grpc_address localhost --grpc_port 9000 --use_case ocr -``` -You can also change the camera ID: -```bash -python horizontal_text_detection.py --grpc_address localhost --grpc_port 9000 --use_case ocr --video_source 0 -``` -Or choose to work with video file as well: -```bash -python horizontal_text_detection.py --grpc_address localhost --grpc_port 9000 --use_case ocr --video_source ~/video.mp4 -``` -Example output: -```bash -Initializing requesting thread index: 0 -Initializing requesting thread index: 1 -Initializing requesting thread index: 2 -Initializing requesting thread index: 3 -Launching requesting thread index: 0 -Launching requesting thread index: 1 -Launching requesting thread index: 2 -Launching requesting thread index: 3 -ThreadID: 0; Current FPS: 31.25; Average FPS: 25.64; Average latency: 140.98ms -ThreadID: 1; Current FPS: 31.23; Average FPS: 25.67; Average latency: 136.36ms -ThreadID: 2; Current FPS: 29.41; Average FPS: 25.70; Average latency: 130.88ms -ThreadID: 3; Current FPS: 30.30; Average FPS: 25.73; Average latency: 135.65ms -... -``` - -## RTSP Client - -Build docker image containing rtsp client along with its dependencies -The rtsp client app needs to have access to RTSP stream to read from and write to. - -Example rtsp server [mediamtx](https://github.com/bluenviron/mediamtx) - -```bash -docker run --rm -d -p 8080:8554 -e RTSP_PROTOCOLS=tcp bluenviron/mediamtx:latest -``` - -Then write to the server using ffmpeg, example using video or camera - -```bash -ffmpeg -stream_loop -1 -i ./video.mp4 -f rtsp -rtsp_transport tcp rtsp://localhost:8080/channel1 -``` - - -``` -ffmpeg -f dshow -i video="HP HD Camera" -f rtsp -rtsp_transport tcp rtsp://localhost:8080/channel1 -``` - -Build the docker image with the python client for video stream reading an remote analysis: -```bash -docker build ../../common/stream_client/ -t rtsp_client -``` - -### Start the client - -- Command - -```bash -docker run -v $(pwd):/workspace rtsp_client --help -usage: client.py [-h] [--grpc_address GRPC_ADDRESS] - [--input_stream INPUT_STREAM] [--output_stream OUTPUT_STREAM] - [--model_name MODEL_NAME] [--width WIDTH] [--height HEIGHT] - [--input_name INPUT_NAME] [--verbose] [--benchmark] - [--limit_stream_duration LIMIT_STREAM_DURATION] - [--limit_frames LIMIT_FRAMES] - -options: - -h, --help show this help message and exit - --grpc_address GRPC_ADDRESS - Specify url to grpc service - --input_stream INPUT_STREAM - Url of input rtsp stream - --output_stream OUTPUT_STREAM - Url of output rtsp stream - --model_name MODEL_NAME - Name of the model - --width WIDTH Width of model's input image - --height HEIGHT Height of model's input image - --input_name INPUT_NAME - Name of the model's input - --verbose Should client dump debug information - --benchmark Should client collect processing times - --limit_stream_duration LIMIT_STREAM_DURATION - Limit how long client should run - --limit_frames LIMIT_FRAMES - Limit how many frames should be processed -``` - -- Usage example - -```bash -docker run --network="host" -v $(pwd):/workspace rtsp_client --grpc_address localhost:9000 --input_stream 'rtsp://localhost:8080/channel1' --output_stream 'rtsp://localhost:8080/channel2' -``` - -Then read rtsp stream using ffplay - -```bash -ffplay -pixel_format yuv420p -video_size 704x704 -rtsp_transport tcp rtsp://localhost:8080/channel2 -``` - - -One might as well use prerecorded video and schedule it for inference. -Replace horizontal_text.mp4 with your video file. - -```bash -docker run --network="host" -v $(pwd):/workspace rtsp_client --grpc_address localhost:9000 --input_stream 'workspace/horizontal_text.mp4' --output_stream 'workspace/output.mp4' -``` diff --git a/demos/horizontal_text_detection/python/client.py b/demos/horizontal_text_detection/python/client.py deleted file mode 100755 index b56720e5f2..0000000000 --- a/demos/horizontal_text_detection/python/client.py +++ /dev/null @@ -1,96 +0,0 @@ -# -# Copyright (c) 2023 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import cv2 -import argparse -import sys -sys.path.append('/') -from stream_client import StreamClient - -parser = argparse.ArgumentParser() -parser.add_argument('--grpc_address', required=False, default='localhost:9000', help='Specify url to grpc service') -parser.add_argument('--input_stream', required=False, default="rtsp://localhost:8080/channel1", type=str, help='Url of input rtsp stream') -parser.add_argument('--output_stream', required=False, default="rtsp://localhost:8080/channel2", type=str, help='Url of output rtsp stream') -parser.add_argument('--model_name', required=False, default="detect_text_images", type=str, help='Name of the model') -parser.add_argument('--width', required=False, default=704, type=int, help='Width of model\'s input image') -parser.add_argument('--height', required=False, default=704, type=int, help='Height of model\'s input image') -parser.add_argument('--input_name', required=False, default="image", type=str, help='Name of the model\'s input') -parser.add_argument('--verbose', required=False, default=False, help='Should client dump debug information', action='store_true') -parser.add_argument('--benchmark', required=False, default=False, help='Should client collect processing times', action='store_true') -parser.add_argument('--limit_stream_duration', required=False, default=0, type=int, help='Limit how long client should run') -parser.add_argument('--limit_frames', required=False, default=0, type=int, help='Limit how many frames should be processed') -args = parser.parse_args() - -WIDTH = args.width -HEIGHT = args.height - - - -def decode(text): - word = '' - last_character = None - for character in text: - if character == last_character: - continue - elif character == '#': - last_character = None - else: - last_character = character - word += character - return word - -def get_text(output): - alphabet = '0123456789abcdefghijklmnopqrstuvwxyz#' - preds = output.argmax(2) - word = '' - for pred in preds: - word += alphabet[pred[0]] - return decode(word) - -def preprocess(frame): - if frame.shape[0] > HEIGHT and frame.shape[1] > WIDTH: - frame = cv2.resize(frame, (HEIGHT, WIDTH), interpolation=cv2.INTER_AREA) - else: - frame = cv2.resize(frame, (HEIGHT, WIDTH), interpolation=cv2.INTER_LINEAR) - return frame - -def postprocess(frame, result): - if result is not None: - texts = result.as_numpy("texts") - text_coordinates = result.as_numpy('text_coordinates') - for i in range(len(texts)): - text = get_text(texts[i]) - coords = text_coordinates[i][0] - x_min = coords[0] - y_min = coords[1] - x_max = coords[0] + coords[2] - y_max = coords[1] + coords[3] - frame = cv2.rectangle(frame, (x_min,y_min), (x_max,y_max), (0,0,255), 1) - cv2.putText(frame, text, (x_min, y_min-5), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,255), 2) - if args.verbose: - print(text) - print((x_min,y_min), (x_max,y_max)) - return frame - -if args.output_stream[:4] == "rtsp": - backend = StreamClient.OutputBackends.ffmpeg - exact = False -else: - backend = StreamClient.OutputBackends.cv2 - exact = True -client = StreamClient(postprocess_callback = postprocess, preprocess_callback=preprocess, output_backend=backend, source=args.input_stream, sink=args.output_stream, exact=exact, benchmark=args.benchmark) -client.start(ovms_address=args.grpc_address, input_name=args.input_name, model_name=args.model_name, limit_stream_duration = args.limit_stream_duration, limit_frames = args.limit_frames) - diff --git a/demos/horizontal_text_detection/python/config.json b/demos/horizontal_text_detection/python/config.json deleted file mode 100644 index c953c1ba66..0000000000 --- a/demos/horizontal_text_detection/python/config.json +++ /dev/null @@ -1,102 +0,0 @@ -{ - "model_config_list": [ - { - "config": { - "name": "text-detection", - "base_path": "/workspace/horizontal-text-detection-0001", - "layout": "NHWC:NCHW", - "plugin_config": { - "PERFORMANCE_HINT": "LATENCY" - } - } - }, - { - "config": { - "name": "text-recognition", - "base_path": "/workspace/text-recognition-0012" - } - } - ], - "custom_node_library_config_list": [ - { - "name": "ocr_image_extractor", - "base_path": "/workspace/lib/libcustom_node_horizontal_ocr.so" - } - ], - "pipeline_config_list": [ - { - "name": "detect_text_images", - "inputs": ["image"], - "nodes": [ - { - "name": "text_detection_node", - "model_name": "text-detection", - "type": "DL model", - "inputs": [ - {"image": {"node_name": "request", - "data_item": "image"}} - ], - "outputs": [ - {"data_item": "boxes", - "alias": "boxes"} - ] - }, - { - "name": "extract_node", - "library_name": "ocr_image_extractor", - "type": "custom", - "demultiply_count": 0, - "params": { - "original_image_width": "704", - "original_image_height": "704", - "original_image_layout": "NHWC", - "target_image_width": "120", - "target_image_height": "32", - "target_image_layout": "NHWC", - "convert_to_gray_scale": "true", - "confidence_threshold": "0.3", - "max_output_batch": "100", - "debug": "true" - }, - "inputs": [ - {"image": {"node_name": "request", - "data_item": "image"}}, - {"boxes": {"node_name": "text_detection_node", - "data_item": "boxes"}} - ], - "outputs": [ - {"data_item": "text_images", - "alias": "text_images"}, - {"data_item": "text_coordinates", - "alias": "text_coordinates"}, - {"data_item": "confidence_levels", - "alias": "confidence_levels"} - ] - }, - { - "name": "text_recognition_node", - "model_name": "text-recognition", - "type": "DL model", - "inputs": [ - {"Placeholder": {"node_name": "extract_node", - "data_item": "text_images"}} - ], - "outputs": [ - {"data_item": "shadow/LSTMLayers/transpose_time_major", - "alias": "texts"} - ] - } - ], - "outputs": [ - {"text_images": {"node_name": "extract_node", - "data_item": "text_images"}}, - {"text_coordinates": {"node_name": "extract_node", - "data_item": "text_coordinates"}}, - {"confidence_levels": {"node_name": "extract_node", - "data_item": "confidence_levels"}}, - {"texts": {"node_name": "text_recognition_node", - "data_item": "texts"}} - ] - } - ] -} \ No newline at end of file diff --git a/demos/horizontal_text_detection/python/horizontal-text-detection-ocr.gif b/demos/horizontal_text_detection/python/horizontal-text-detection-ocr.gif deleted file mode 100644 index 23e04982a1..0000000000 Binary files a/demos/horizontal_text_detection/python/horizontal-text-detection-ocr.gif and /dev/null differ diff --git a/demos/horizontal_text_detection/python/horizontal-text-detection.gif b/demos/horizontal_text_detection/python/horizontal-text-detection.gif deleted file mode 100755 index a8f41089c6..0000000000 Binary files a/demos/horizontal_text_detection/python/horizontal-text-detection.gif and /dev/null differ diff --git a/demos/horizontal_text_detection/python/horizontal_text_detection.py b/demos/horizontal_text_detection/python/horizontal_text_detection.py deleted file mode 100644 index 141fdd5c2c..0000000000 --- a/demos/horizontal_text_detection/python/horizontal_text_detection.py +++ /dev/null @@ -1,250 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -import cv2 -import grpc -import io -import threading -import time -import sys -import argparse - -from tensorflow import make_tensor_proto, make_ndarray -from tensorflow_serving.apis import predict_pb2 -from tensorflow_serving.apis import prediction_service_pb2_grpc - - -parser = argparse.ArgumentParser() -parser.add_argument('--grpc_address', required=False, default='localhost', help='Specify url to grpc service') -parser.add_argument('--grpc_port', required=False, default=8080, type=int, help='Specify port to grpc service') -parser.add_argument('--num_threads', required=False, default=4, type=int, help='Number of threads for parallel service requesting') -parser.add_argument('--video_source', required=False, default="0", type=str, help='Camera ID number or path to a video file') -parser.add_argument('--use_case', required=False, default="text_spotting", type=str, choices=['text_spotting', 'ocr'], help='Which servable to request and how to postprocess') -args = parser.parse_args() - -address = "{}:{}".format(args.grpc_address, args.grpc_port) -channel = grpc.insecure_channel(address) -stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) - -try: - source = int(args.video_source) -except ValueError: - source = args.video_source - -cap = cv2.VideoCapture(source) -force_exit = False - - -def decode(text): - word = '' - last_character = None - for character in text: - if character == last_character: - continue - elif character == '#': - last_character = None - else: - last_character = character - word += character - return word - -def get_text(output): - alphabet = '#1234567890abcdefghijklmnopqrstuvwxyz' - preds = output.argmax(2) - word = '' - for i in range(preds.shape[0]): - word += alphabet[preds[i,0]] - return decode(word) - -def draw_boxes_spotting(frame, result): - output = make_ndarray(result.outputs['boxes']) - for i in range(0, output.shape[0]): # there is returned a dynamic list of boxes - detection = output[i,:] - if detection[4] > 0.3: - x_min = int(detection[0]) - y_min = int(detection[1]) - x_max = int(detection[2]) - y_max = int(detection[3]) - frame = cv2.rectangle(frame,(x_min,y_min),(x_max,y_max),(0,0,255),1) - return frame - -def draw_boxes_ocr(frame, result): - texts = make_ndarray(result.outputs['texts']) - text_coordinates = make_ndarray(result.outputs['text_coordinates']) - for i in range(len(texts)): - text = get_text(texts[i]) - x_min = text_coordinates[i][0][0] - y_min = text_coordinates[i][0][1] - x_max = text_coordinates[i][0][0] + text_coordinates[i][0][2] - y_max = text_coordinates[i][0][1] + text_coordinates[i][0][3] - frame = cv2.rectangle(frame, (x_min,y_min), (x_max,y_max), (0,0,255), 1) - cv2.putText(frame, text, (x_min, y_min-5), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,255), 2) - return frame - - -class RequestingThread(threading.Thread): - def __init__(self, index): - print(f"Initializing requesting thread index: {index}") - super().__init__() - self.index = index - self.input_frame = None - self.output_frame = None - self.predict_durations = [] - self.input_ready_event = threading.Event() - self.output_ready_event = threading.Event() - - def is_initialized(self): - return not (self.input_frame is None and self.output_frame is None) - - def wait_for_input(self): - self.input_ready_event.wait() - self.input_ready_event.clear() - - def wait_for_result(self): - self.output_ready_event.wait() - self.output_ready_event.clear() - - def notify_input_ready(self): - self.input_ready_event.set() - - def notify_output_ready(self): - self.output_ready_event.set() - - def set_input(self, frame): - self.input_frame = frame - self.notify_input_ready() - - def get_output(self): - return self.output_frame - - def get_average_latency(self): - return np.average(np.array(self.predict_durations)) - - def run(self): - print(f"Launching requesting thread index: {self.index}") - global force_exit - while (True): - self.wait_for_input() - if force_exit: - print("Detected exit signal...") - break - is_success, buffer = cv2.imencode(".jpg", self.input_frame) - if not is_success: - print("Encoding a frame as JPG failed") - continue - request = predict_pb2.PredictRequest() - request.model_spec.name = request.model_spec.name = 'detect_text_images' if args.use_case == 'ocr' else 'text' - request.inputs['image'].CopyFrom(make_tensor_proto([buffer.tobytes()], shape=[1])) - - predict_start_time = time.time() - result = None - try: - result = stub.Predict(request, 10.0) - except grpc.RpcError as err: - print("Encountered gRPC error") - if err.code() == grpc.StatusCode.ABORTED: - pass - else: - raise err - predict_duration = time.time() - predict_start_time - predict_duration *= 1000 - self.predict_durations.append(predict_duration) - - if result is None: - self.output_frame = np.array(self.input_frame, copy=True) - else: - if args.use_case == 'ocr': - self.output_frame = draw_boxes_ocr(self.input_frame, result) - else: - self.output_frame = draw_boxes_spotting(self.input_frame, result) - self.notify_output_ready() - print(f"Stopping requesting thread index: {self.index}") - - -threads = [RequestingThread(i) for i in range(args.num_threads)] - -for thread in threads: - thread.start() - -def finish(): - global force_exit - force_exit = True - for thread in threads: - thread.notify_input_ready() - thread.join() - -def grab_frame(cap): - WIDTH = 704 - HEIGHT = 704 - success, frame = cap.read() - if not success: - print("[WARNING] No Input frame") - finish() - return None - - # crop and resize if original image is too small or too big for the model - if frame.shape[0] > HEIGHT and frame.shape[1] > WIDTH: - frame = frame[0:HEIGHT, 0:WIDTH] - else: - res = min(frame.shape[0], frame.shape[1]) - frame = frame[0:res, 0:res] - frame = cv2.resize(frame, (HEIGHT, WIDTH), interpolation=cv2.INTER_AREA) - return frame - - -i = 0 -frames_processed = 0 -last_display_time = time.time() -app_start_time = time.time() - -if grab_frame(cap) is None: - print("[ERROR] Check camera input...") - force_exit = True - -while not force_exit: - if not threads[i].is_initialized(): - threads[i].set_input(grab_frame(cap)) - i = (i + 1) % args.num_threads - continue - - threads[i].wait_for_result() - avg_latency_for_thread = threads[i].get_average_latency() - frame_to_display = threads[i].get_output() - threads[i].set_input(grab_frame(cap)) - - cv2.imshow('frame', frame_to_display) - now = time.time() - time_since_last_display = now - last_display_time - last_display_time = now - - frames_processed += 1 - - current_fps = 1 / (time_since_last_display if time_since_last_display > 0 else 1) - avg_fps = 1 / ((now - app_start_time) / frames_processed) - - print(f"ThreadID: {i:3}; Current FPS: {current_fps:8.2f}; Average FPS: {avg_fps:8.2f}; Average latency: {avg_latency_for_thread:8.2f}ms") - if cv2.waitKey(1) & 0xFF == ord('q'): - finish() - break - - i = (i + 1) % args.num_threads - -finish() -# When everything done, release the capture -cap.release() -cv2.destroyAllWindows() -print(f"Finished.") diff --git a/demos/horizontal_text_detection/python/requirements.txt b/demos/horizontal_text_detection/python/requirements.txt deleted file mode 100644 index fa28f0f955..0000000000 --- a/demos/horizontal_text_detection/python/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --r ../../common/python/requirements.txt \ No newline at end of file diff --git a/demos/image_classification/go/Dockerfile b/demos/image_classification/go/Dockerfile deleted file mode 100644 index 07a25b3f5f..0000000000 --- a/demos/image_classification/go/Dockerfile +++ /dev/null @@ -1,51 +0,0 @@ - - -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM golang:1.24.0 -ARG http_proxy - -RUN echo "Acquire::http::Proxy \"$http_proxy\";" > /etc/apt/apt.conf.d/proxy.conf -RUN apt-get update && \ - apt-get -y install git unzip build-essential autoconf libtool protobuf-compiler libdc1394-dev libprotobuf-dev - -RUN apt-get update && apt-get install -y sudo - -RUN rm -rf /var/lib/apt/lists/* -RUN git clone https://github.com/hybridgroup/gocv.git && \ - cd gocv && \ - make install - -RUN mkdir /app -COPY . /app -WORKDIR /app - -RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.1 -RUN go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.4.0 - -# Compile API -RUN protoc -I apis/ apis/tensorflow_serving/apis/*.proto --go_out="./" --go-grpc_out="./" -RUN protoc -I apis/ apis/tensorflow/core/framework/*.proto --go_out="./" --go-grpc_out="./" - -# Move compiled protos under GOROOT -RUN mv tensorflow /usr/local/go/src/ -RUN mv tensorflow_serving /usr/local/go/src/ - -RUN go mod tidy -RUN go build . - -ENTRYPOINT ["/app/ovmsclient"] diff --git a/demos/image_classification/go/README.md b/demos/image_classification/go/README.md deleted file mode 100644 index 0615d092e6..0000000000 --- a/demos/image_classification/go/README.md +++ /dev/null @@ -1,74 +0,0 @@ -# Image Classification Demo (Go) {#ovms_demo_image_classification_go} - -This client demonstrates how to interact with OpenVINO Model Server prediction endpoints from a Go application. The example shows end-to-end workflow for running classification on JPEG/PNG images using a ResNet50 model. To simplify the environment setup, the demo is run inside a Docker container. - -Clone the repository and enter directory: - -```bash -git clone https://github.com/openvinotoolkit/model_server.git -cd model_server/demos/image_classification/go -``` - -## Get the model - -To run end to end flow and get correct results, please download `resnet-50` model and set preprocessing layers in OVMS. Commands are presented below. - -For example: -```bash -curl --fail -L --create-dirs https://github.com/onnx/models/raw/main/validated/vision/classification/resnet/model/resnet50-caffe2-v1-9.onnx -o models/resnet/1/resnet50-caffe2-v1-9.onnx -chmod -R 755 models -``` - -## Build Go client docker image - -Before building the image let's copy single zebra image, here so it's included in the docker build context. This way client container will already have a sample to run prediction on: - -```bash -cp ../../common/static/images/zebra.jpeg . -``` -Then build the docker image and tag it `ovmsclient`: -```bash -docker build . -t ovmsclient -``` - -## Start OpenVINO Model Server with ResNet model - -Before running the client launch OVMS with prepared ResNet model. You can do that with a command similar to: - -```bash -docker run -d --rm -p 9000:9000 -v ${PWD}/models:/models openvino/model_server:latest --model_name resnet --model_path /models/resnet --port 9000 --layout NHWC:NCHW --mean "[123.675,116.28,103.53]" --scale "[58.395,57.12,57.375]" --shape "(1,224,224,3)" --color_format BGR:RGB --precision uint8:fp32 -``` - -**Note** Layout for downloaded resnet model is changed to NHWC. It ensures that the model will accept binary input generated by the client. See [binary inputs](../../../docs/binary_input.md) doc if you want to learn more about this feature. - -## Run prediction with Go client - -In order to run prediction on the model served by the OVMS using Go client run the following command: - -```bash -docker run --net=host --rm ovmsclient --serving-address localhost:9000 zebra.jpeg -``` -Exemplary output: -``` -2026/02/23 14:22:11 Request sent successfully -Predicted class: zebra -Classification confidence: 99.837669% -``` - -Command explained: -- `--net=host` option is required so the container with the client can access container with the model server via host network (localhost), -- `--serving-address` parameter defines the address of the model server gRPC endpoint, -- the last part in the command is a path to the image that will be send to OVMS for prediction. The image must be accessible from the inside of the container (could be mounted). Single zebra picture - `zebra.jpeg` - has been embedded in the docker image to simplify the example, so above command would work out of the box. If you wish to use other image you need to provide it to the container and change the path. - -You can also choose if the image should be sent as binary input (raw JPG or PNG bytes) or should be converted on the client side to the data array accepted by the model. -To send raw bytes just add `--binary-input` flag like this: - -```bash -docker run --net=host --rm ovmsclient --serving-address localhost:9000 --binary-input zebra.jpeg -``` -Exemplary output: -``` -2026/02/24 06:44:53 Request sent successfully -Predicted class: zebra -Classification confidence: 99.857803% -``` diff --git a/demos/image_classification/go/apis/tensorflow/core/framework/resource_handle.proto b/demos/image_classification/go/apis/tensorflow/core/framework/resource_handle.proto deleted file mode 100644 index 1f6183719e..0000000000 --- a/demos/image_classification/go/apis/tensorflow/core/framework/resource_handle.proto +++ /dev/null @@ -1,45 +0,0 @@ -syntax = "proto3"; - -package tensorflow; - -import "tensorflow/core/framework/tensor_shape.proto"; -import "tensorflow/core/framework/types.proto"; - -option cc_enable_arenas = true; -option java_outer_classname = "ResourceHandle"; -option java_multiple_files = true; -option java_package = "org.tensorflow.framework"; -option go_package = "tensorflow/core/framework"; - -// Protocol buffer representing a handle to a tensorflow resource. Handles are -// not valid across executions, but can be serialized back and forth from within -// a single run. -message ResourceHandleProto { - // Unique name for the device containing the resource. - string device = 1; - - // Container in which this resource is placed. - string container = 2; - - // Unique name of this resource. - string name = 3; - - // Hash code for the type of the resource. Is only valid in the same device - // and in the same execution. - uint64 hash_code = 4; - - // For debug-only, the name of the type pointed to by this handle, if - // available. - string maybe_type_name = 5; - - // Protocol buffer representing a pair of (data type, tensor shape). - message DtypeAndShape { - DataType dtype = 1; - TensorShapeProto shape = 2; - } - - // Data types and shapes for the underlying resource. - repeated DtypeAndShape dtypes_and_shapes = 6; - - reserved 7; -} diff --git a/demos/image_classification/go/apis/tensorflow/core/framework/tensor.proto b/demos/image_classification/go/apis/tensorflow/core/framework/tensor.proto deleted file mode 100644 index 84e386ebe8..0000000000 --- a/demos/image_classification/go/apis/tensorflow/core/framework/tensor.proto +++ /dev/null @@ -1,96 +0,0 @@ -syntax = "proto3"; - -package tensorflow; - -import "tensorflow/core/framework/resource_handle.proto"; -import "tensorflow/core/framework/tensor_shape.proto"; -import "tensorflow/core/framework/types.proto"; - -option cc_enable_arenas = true; -option java_outer_classname = "TensorProtos"; -option java_multiple_files = true; -option java_package = "org.tensorflow.framework"; -option go_package = "tensorflow/core/framework"; - -// Protocol buffer representing a tensor. -message TensorProto { - DataType dtype = 1; - - // Shape of the tensor. TODO(touts): sort out the 0-rank issues. - TensorShapeProto tensor_shape = 2; - - // Only one of the representations below is set, one of "tensor_contents" and - // the "xxx_val" attributes. We are not using oneof because as oneofs cannot - // contain repeated fields it would require another extra set of messages. - - // Version number. - // - // In version 0, if the "repeated xxx" representations contain only one - // element, that element is repeated to fill the shape. This makes it easy - // to represent a constant Tensor with a single value. - int32 version_number = 3; - - // Serialized raw tensor content from either Tensor::AsProtoTensorContent or - // memcpy in tensorflow::grpc::EncodeTensorToByteBuffer. This representation - // can be used for all tensor types. The purpose of this representation is to - // reduce serialization overhead during RPC call by avoiding serialization of - // many repeated small items. - bytes tensor_content = 4; - - // Type specific representations that make it easy to create tensor protos in - // all languages. Only the representation corresponding to "dtype" can - // be set. The values hold the flattened representation of the tensor in - // row major order. - - // DT_HALF, DT_BFLOAT16. Note that since protobuf has no int16 type, we'll - // have some pointless zero padding for each value here. - repeated int32 half_val = 13 [packed = true]; - - // DT_FLOAT. - repeated float float_val = 5 [packed = true]; - - // DT_DOUBLE. - repeated double double_val = 6 [packed = true]; - - // DT_INT32, DT_INT16, DT_INT8, DT_UINT8. - repeated int32 int_val = 7 [packed = true]; - - // DT_STRING - repeated bytes string_val = 8; - - // DT_COMPLEX64. scomplex_val(2*i) and scomplex_val(2*i+1) are real - // and imaginary parts of i-th single precision complex. - repeated float scomplex_val = 9 [packed = true]; - - // DT_INT64 - repeated int64 int64_val = 10 [packed = true]; - - // DT_BOOL - repeated bool bool_val = 11 [packed = true]; - - // DT_COMPLEX128. dcomplex_val(2*i) and dcomplex_val(2*i+1) are real - // and imaginary parts of i-th double precision complex. - repeated double dcomplex_val = 12 [packed = true]; - - // DT_RESOURCE - repeated ResourceHandleProto resource_handle_val = 14; - - // DT_VARIANT - repeated VariantTensorDataProto variant_val = 15; - - // DT_UINT32 - repeated uint32 uint32_val = 16 [packed = true]; - - // DT_UINT64 - repeated uint64 uint64_val = 17 [packed = true]; -} - -// Protocol buffer representing the serialization format of DT_VARIANT tensors. -message VariantTensorDataProto { - // Name of the type of objects being serialized. - string type_name = 1; - // Portions of the object that are not Tensors. - bytes metadata = 2; - // Tensors contained within objects being serialized. - repeated TensorProto tensors = 3; -} diff --git a/demos/image_classification/go/apis/tensorflow/core/framework/tensor_shape.proto b/demos/image_classification/go/apis/tensorflow/core/framework/tensor_shape.proto deleted file mode 100644 index ac475c4bc1..0000000000 --- a/demos/image_classification/go/apis/tensorflow/core/framework/tensor_shape.proto +++ /dev/null @@ -1,46 +0,0 @@ -// Protocol buffer representing the shape of tensors. - -syntax = "proto3"; -option cc_enable_arenas = true; -option java_outer_classname = "TensorShapeProtos"; -option java_multiple_files = true; -option java_package = "org.tensorflow.framework"; -option go_package = "tensorflow/core/framework"; - -package tensorflow; - -// Dimensions of a tensor. -message TensorShapeProto { - // One dimension of the tensor. - message Dim { - // Size of the tensor in that dimension. - // This value must be >= -1, but values of -1 are reserved for "unknown" - // shapes (values of -1 mean "unknown" dimension). Certain wrappers - // that work with TensorShapeProto may fail at runtime when deserializing - // a TensorShapeProto containing a dim value of -1. - int64 size = 1; - - // Optional name of the tensor dimension. - string name = 2; - }; - - // Dimensions of the tensor, such as {"input", 30}, {"output", 40} - // for a 30 x 40 2D tensor. If an entry has size -1, this - // corresponds to a dimension of unknown size. The names are - // optional. - // - // The order of entries in "dim" matters: It indicates the layout of the - // values in the tensor in-memory representation. - // - // The first entry in "dim" is the outermost dimension used to layout the - // values, the last entry is the innermost dimension. This matches the - // in-memory layout of RowMajor Eigen tensors. - // - // If "dim.size()" > 0, "unknown_rank" must be false. - repeated Dim dim = 2; - - // If true, the number of dimensions in the shape is unknown. - // - // If true, "dim.size()" must be 0. - bool unknown_rank = 3; -}; diff --git a/demos/image_classification/go/apis/tensorflow/core/framework/types.proto b/demos/image_classification/go/apis/tensorflow/core/framework/types.proto deleted file mode 100644 index 409ac3de1a..0000000000 --- a/demos/image_classification/go/apis/tensorflow/core/framework/types.proto +++ /dev/null @@ -1,89 +0,0 @@ -syntax = "proto3"; - -package tensorflow; -option cc_enable_arenas = true; -option java_outer_classname = "TypesProtos"; -option java_multiple_files = true; -option java_package = "org.tensorflow.framework"; -option go_package = "tensorflow/core/framework"; - -// (== suppress_warning documentation-presence ==) -// LINT.IfChange -enum DataType { - // Not a legal value for DataType. Used to indicate a DataType field - // has not been set. - DT_INVALID = 0; - - // Data types that all computation devices are expected to be - // capable to support. - DT_FLOAT = 1; - DT_DOUBLE = 2; - DT_INT32 = 3; - DT_UINT8 = 4; - DT_INT16 = 5; - DT_INT8 = 6; - DT_STRING = 7; - DT_COMPLEX64 = 8; // Single-precision complex - DT_INT64 = 9; - DT_BOOL = 10; - DT_QINT8 = 11; // Quantized int8 - DT_QUINT8 = 12; // Quantized uint8 - DT_QINT32 = 13; // Quantized int32 - DT_BFLOAT16 = 14; // Float32 truncated to 16 bits. Only for cast ops. - DT_QINT16 = 15; // Quantized int16 - DT_QUINT16 = 16; // Quantized uint16 - DT_UINT16 = 17; - DT_COMPLEX128 = 18; // Double-precision complex - DT_HALF = 19; - DT_RESOURCE = 20; - DT_VARIANT = 21; // Arbitrary C++ data types - DT_UINT32 = 22; - DT_UINT64 = 23; - - // Do not use! These are only for parameters. Every enum above - // should have a corresponding value below (verified by types_test). - DT_FLOAT_REF = 101; - DT_DOUBLE_REF = 102; - DT_INT32_REF = 103; - DT_UINT8_REF = 104; - DT_INT16_REF = 105; - DT_INT8_REF = 106; - DT_STRING_REF = 107; - DT_COMPLEX64_REF = 108; - DT_INT64_REF = 109; - DT_BOOL_REF = 110; - DT_QINT8_REF = 111; - DT_QUINT8_REF = 112; - DT_QINT32_REF = 113; - DT_BFLOAT16_REF = 114; - DT_QINT16_REF = 115; - DT_QUINT16_REF = 116; - DT_UINT16_REF = 117; - DT_COMPLEX128_REF = 118; - DT_HALF_REF = 119; - DT_RESOURCE_REF = 120; - DT_VARIANT_REF = 121; - DT_UINT32_REF = 122; - DT_UINT64_REF = 123; -} -// LINT.ThenChange( -// https://www.tensorflow.org/code/tensorflow/c/tf_datatype.h, -// https://www.tensorflow.org/code/tensorflow/go/tensor.go, -// https://www.tensorflow.org/code/tensorflow/core/framework/tensor.cc, -// https://www.tensorflow.org/code/tensorflow/core/framework/types.h, -// https://www.tensorflow.org/code/tensorflow/core/framework/types.cc, -// https://www.tensorflow.org/code/tensorflow/python/framework/dtypes.py, -// https://www.tensorflow.org/code/tensorflow/python/framework/function.py) - -// For identifying the underlying type of a variant. For variants, the types -// listed here are a subset of the types in the variant type registry, -// corresponding to commonly used variants which must occasionally be -// special-cased. -enum SpecializedType { - // Invalid/unknown specialized type. - ST_INVALID = 0; - // "tensorflow::TensorList" in the variant type registry. - ST_TENSOR_LIST = 1; - // "tensorflow::data::Optional" in the variant type registry. - ST_OPTIONAL = 2; -} diff --git a/demos/image_classification/go/apis/tensorflow_serving/apis/model.proto b/demos/image_classification/go/apis/tensorflow_serving/apis/model.proto deleted file mode 100644 index 3e5ad46689..0000000000 --- a/demos/image_classification/go/apis/tensorflow_serving/apis/model.proto +++ /dev/null @@ -1,34 +0,0 @@ -syntax = "proto3"; - -package tensorflow.serving; -option cc_enable_arenas = true; -option go_package = "./tensorflow_serving"; - -import "google/protobuf/wrappers.proto"; - -// Metadata for an inference request such as the model name and version. -message ModelSpec { - // Required servable name. - string name = 1; - - // Optional choice of which version of the model to use. - // - // Recommended to be left unset in the common case. Should be specified only - // when there is a strong version consistency requirement. - // - // When left unspecified, the system will serve the best available version. - // This is typically the latest version, though during version transitions, - // notably when serving on a fleet of instances, may be either the previous or - // new version. - oneof version_choice { - // Use this specific version number. - google.protobuf.Int64Value version = 2; - - // Use the version associated with the given label. - string version_label = 4; - } - - // A named signature to evaluate. If unspecified, the default signature will - // be used. - string signature_name = 3; -} diff --git a/demos/image_classification/go/apis/tensorflow_serving/apis/predict.proto b/demos/image_classification/go/apis/tensorflow_serving/apis/predict.proto deleted file mode 100644 index 0897e86c81..0000000000 --- a/demos/image_classification/go/apis/tensorflow_serving/apis/predict.proto +++ /dev/null @@ -1,41 +0,0 @@ -syntax = "proto3"; - -package tensorflow.serving; -option cc_enable_arenas = true; -option go_package = "./tensorflow_serving"; - -import "tensorflow/core/framework/tensor.proto"; -import "tensorflow_serving/apis/model.proto"; - -// PredictRequest specifies which TensorFlow model to run, as well as -// how inputs are mapped to tensors and how outputs are filtered before -// returning to user. -message PredictRequest { - // Model Specification. If version is not specified, will use the latest - // (numerical) version. - ModelSpec model_spec = 1; - - // Input tensors. - // Names of input tensor are alias names. The mapping from aliases to real - // input tensor names is stored in the SavedModel export as a prediction - // SignatureDef under the 'inputs' field. - map inputs = 2; - - // Output filter. - // Names specified are alias names. The mapping from aliases to real output - // tensor names is stored in the SavedModel export as a prediction - // SignatureDef under the 'outputs' field. - // Only tensors specified here will be run/fetched and returned, with the - // exception that when none is specified, all tensors specified in the - // named signature will be run/fetched and returned. - repeated string output_filter = 3; -} - -// Response for PredictRequest on successful run. -message PredictResponse { - // Effective Model Specification used to process PredictRequest. - ModelSpec model_spec = 2; - - // Output tensors. - map outputs = 1; -} diff --git a/demos/image_classification/go/apis/tensorflow_serving/apis/prediction_service.proto b/demos/image_classification/go/apis/tensorflow_serving/apis/prediction_service.proto deleted file mode 100644 index 896b892ff0..0000000000 --- a/demos/image_classification/go/apis/tensorflow_serving/apis/prediction_service.proto +++ /dev/null @@ -1,15 +0,0 @@ -syntax = "proto3"; - -package tensorflow.serving; -option cc_enable_arenas = true; -option go_package = "./tensorflow_serving"; - -import "tensorflow_serving/apis/predict.proto"; - -// open source marker; do not remove -// PredictionService provides access to machine-learned models loaded by -// model_servers. -service PredictionService { - // Predict -- provides access to loaded TensorFlow model. - rpc Predict(PredictRequest) returns (PredictResponse); -} diff --git a/demos/image_classification/go/go.mod b/demos/image_classification/go/go.mod deleted file mode 100644 index a14d0548aa..0000000000 --- a/demos/image_classification/go/go.mod +++ /dev/null @@ -1,34 +0,0 @@ -// -// Copyright (c) 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -module ovmsclient - -go 1.24.0 - -require ( - github.com/golang/protobuf v1.5.4 - github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 - gocv.io/x/gocv v0.35.0 - google.golang.org/grpc v1.79.3 -) - -require ( - golang.org/x/net v0.48.0 // indirect - golang.org/x/sys v0.39.0 // indirect - golang.org/x/text v0.32.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect - google.golang.org/protobuf v1.36.10 // indirect -) diff --git a/demos/image_classification/go/go.sum b/demos/image_classification/go/go.sum deleted file mode 100644 index 8f007a3ffd..0000000000 --- a/demos/image_classification/go/go.sum +++ /dev/null @@ -1,44 +0,0 @@ -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= -github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/hybridgroup/mjpeg v0.0.0-20140228234708-4680f319790e/go.mod h1:eagM805MRKrioHYuU7iKLUyFPVKqVV6um5DAvCkUtXs= -github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= -github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= -github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= -go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= -go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= -go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= -go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= -go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= -gocv.io/x/gocv v0.35.0 h1:Qaxb5KdVyy8Spl4S4K0SMZ6CVmKtbfoSGQAxRD3FZlw= -gocv.io/x/gocv v0.35.0/go.mod h1:oc6FvfYqfBp99p+yOEzs9tbYF9gOrAQSeL/dyIPefJU= -golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= -golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= -golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= -golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= -golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= -google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= -google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= -google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= -google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= diff --git a/demos/image_classification/go/labels.go b/demos/image_classification/go/labels.go deleted file mode 100644 index 853d89c8c8..0000000000 --- a/demos/image_classification/go/labels.go +++ /dev/null @@ -1,1035 +0,0 @@ -/* -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/ - -package main - -import ( - "errors" -) - -// DecodeResnetLabel takes an index and returns the label as a string -func DecodeResnetLabel(index int) (label string, err error) { - if index > len(labels) || index < 0 { - err = errors.New("index out of bounds") - return - } - - label = labels[index] - return -} - -// Labels is an array of RESNET labels -var labels = [...]string{ - "tench, Tinca tinca", - "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", - "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", - "stingray", - "cock", - "hen", - "ostrich, Struthio camelus", - "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", - "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", - "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", - "bulbul", - "jay", - "magpie", - "chickadee", - "water ouzel, dipper", - "kite", - "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", - "great grey owl, great gray owl, Strix nebulosa", - "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", - "eft", - "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", - "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", - "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", - "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", - "terrapin", - "box turtle, box tortoise", - "banded gecko", - "common iguana, iguana, Iguana iguana", - "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", - "agama", - "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", - "Gila monster, Heloderma suspectum", - "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", - "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", - "American alligator, Alligator mississipiensis", - "triceratops", - "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", - "hognose snake, puff adder, sand viper", - "green snake, grass snake", - "king snake, kingsnake", - "garter snake, grass snake", - "water snake", - "vine snake", - "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", - "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", - "green mamba", - "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", - "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", - "harvestman, daddy longlegs, Phalangium opilio", - "scorpion", - "black and gold garden spider, Argiope aurantia", - "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", - "black widow, Latrodectus mactans", - "tarantula", - "wolf spider, hunting spider", - "tick", - "centipede", - "black grouse", - "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", - "prairie chicken, prairie grouse, prairie fowl", - "peacock", - "quail", - "partridge", - "African grey, African gray, Psittacus erithacus", - "macaw", - "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", - "coucal", - "bee eater", - "hornbill", - "hummingbird", - "jacamar", - "toucan", - "drake", - "red-breasted merganser, Mergus serrator", - "goose", - "black swan, Cygnus atratus", - "tusker", - "echidna, spiny anteater, anteater", - "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", - "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", - "jellyfish", - "sea anemone, anemone", - "brain coral", - "flatworm, platyhelminth", - "nematode, nematode worm, roundworm", - "conch", - "snail", - "slug", - "sea slug, nudibranch", - "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", - "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", - "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", - "hermit crab", - "isopod", - "white stork, Ciconia ciconia", - "black stork, Ciconia nigra", - "spoonbill", - "flamingo", - "little blue heron, Egretta caerulea", - "American egret, great white heron, Egretta albus", - "bittern", - "crane", - "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", - "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", - "ruddy turnstone, Arenaria interpres", - "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", - "dowitcher", - "oystercatcher, oyster catcher", - "pelican", - "king penguin, Aptenodytes patagonica", - "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", - "sea lion", - "Chihuahua", - "Japanese spaniel", - "Maltese dog, Maltese terrier, Maltese", - "Pekinese, Pekingese, Peke", - "Shih-Tzu", - "Blenheim spaniel", - "papillon", - "toy terrier", - "Rhodesian ridgeback", - "Afghan hound, Afghan", - "basset, basset hound", - "beagle", - "bloodhound, sleuthhound", - "bluetick", - "black-and-tan coonhound", - "Walker hound, Walker foxhound", - "English foxhound", - "redbone", - "borzoi, Russian wolfhound", - "Irish wolfhound", - "Italian greyhound", - "whippet", - "Ibizan hound, Ibizan Podenco", - "Norwegian elkhound, elkhound", - "otterhound, otter hound", - "Saluki, gazelle hound", - "Scottish deerhound, deerhound", - "Weimaraner", - "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", - "Border terrier", - "Kerry blue terrier", - "Irish terrier", - "Norfolk terrier", - "Norwich terrier", - "Yorkshire terrier", - "wire-haired fox terrier", - "Lakeland terrier", - "Sealyham terrier, Sealyham", - "Airedale, Airedale terrier", - "cairn, cairn terrier", - "Australian terrier", - "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", - "miniature schnauzer", - "giant schnauzer", - "standard schnauzer", - "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", - "silky terrier, Sydney silky", - "soft-coated wheaten terrier", - "West Highland white terrier", - "Lhasa, Lhasa apso", - "flat-coated retriever", - "curly-coated retriever", - "golden retriever", - "Labrador retriever", - "Chesapeake Bay retriever", - "German short-haired pointer", - "vizsla, Hungarian pointer", - "English setter", - "Irish setter, red setter", - "Gordon setter", - "Brittany spaniel", - "clumber, clumber spaniel", - "English springer, English springer spaniel", - "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", - "Sussex spaniel", - "Irish water spaniel", - "kuvasz", - "schipperke", - "groenendael", - "malinois", - "briard", - "kelpie", - "komondor", - "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", - "collie", - "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", - "Rottweiler", - "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", - "miniature pinscher", - "Greater Swiss Mountain dog", - "Bernese mountain dog", - "Appenzeller", - "EntleBucher", - "boxer", - "bull mastiff", - "Tibetan mastiff", - "French bulldog", - "Great Dane", - "Saint Bernard, St Bernard", - "Eskimo dog, husky", - "malamute, malemute, Alaskan malamute", - "Siberian husky", - "dalmatian, coach dog, carriage dog", - "affenpinscher, monkey pinscher, monkey dog", - "basenji", - "pug, pug-dog", - "Leonberg", - "Newfoundland, Newfoundland dog", - "Great Pyrenees", - "Samoyed, Samoyede", - "Pomeranian", - "chow, chow chow", - "keeshond", - "Brabancon griffon", - "Pembroke, Pembroke Welsh corgi", - "Cardigan, Cardigan Welsh corgi", - "toy poodle", - "miniature poodle", - "standard poodle", - "Mexican hairless", - "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", - "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", - "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", - "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", - "red fox, Vulpes vulpes", - "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", - "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", - "tiger cat", - "Persian cat", - "Siamese cat, Siamese", - "Egyptian cat", - "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", - "leopard, Panthera pardus", - "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", - "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", - "cheetah, chetah, Acinonyx jubatus", - "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", - "mongoose", - "meerkat, mierkat", - "tiger beetle", - "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", - "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", - "dung beetle", - "rhinoceros beetle", - "weevil", - "fly", - "bee", - "ant, emmet, pismire", - "grasshopper, hopper", - "cricket", - "walking stick, walkingstick, stick insect", - "cockroach, roach", - "mantis, mantid", - "cicada, cicala", - "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devils darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", - "admiral", - "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", - "sulphur butterfly, sulfur butterfly", - "lycaenid, lycaenid butterfly", - "starfish, sea star", - "sea urchin", - "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", - "hare", - "Angora, Angora rabbit", - "hamster", - "porcupine, hedgehog", - "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", - "beaver", - "guinea pig, Cavia cobaya", - "sorrel", - "zebra", - "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", - "warthog", - "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", - "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", - "ram, tup", - "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", - "hartebeest", - "impala, Aepyceros melampus", - "gazelle", - "Arabian camel, dromedary, Camelus dromedarius", - "llama", - "weasel", - "mink", - "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", - "otter", - "skunk, polecat, wood pussy", - "badger", - "armadillo", - "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", - "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", - "gibbon, Hylobates lar", - "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", - "patas, hussar monkey, Erythrocebus patas", - "baboon", - "macaque", - "langur", - "colobus, colobus monkey", - "proboscis monkey, Nasalis larvatus", - "marmoset", - "capuchin, ringtail, Cebus capucinus", - "howler monkey, howler", - "titi, titi monkey", - "spider monkey, Ateles geoffroyi", - "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", - "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", - "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", - "eel", - "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", - "anemone fish", - "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", - "lionfish", - "puffer, pufferfish, blowfish, globefish", - "abacus", - "abaya", - "academic gown, academic robe, judges robe", - "accordion, piano accordion, squeeze box", - "acoustic guitar", - "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", - "airship, dirigible", - "altar", - "ambulance", - "amphibian, amphibious vehicle", - "analog clock", - "apiary, bee house", - "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", - "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", - "balance beam, beam", - "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", - "Band Aid", - "banjo", - "bannister, banister, balustrade, balusters, handrail", - "barbell", - "barber chair", - "barbershop", - "barn", - "barometer", - "barrel, cask", - "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", - "basketball", - "bassinet", - "bassoon", - "bathing cap, swimming cap", - "bath towel", - "bathtub, bathing tub, bath, tub", - "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", - "beaker", - "bearskin, busby, shako", - "beer bottle", - "beer glass", - "bell cote, bell cot", - "bib", - "bicycle-built-for-two, tandem bicycle, tandem", - "bikini, two-piece", - "binder, ring-binder", - "binoculars, field glasses, opera glasses", - "birdhouse", - "boathouse", - "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", - "bonnet, poke bonnet", - "bookcase", - "bookshop, bookstore, bookstall", - "bottlecap", - "bow", - "bow tie, bow-tie, bowtie", - "brass, memorial tablet, plaque", - "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", - "breastplate, aegis, egis", - "broom", - "bucket, pail", - "buckle", - "bulletproof vest", - "bullet train, bullet", - "butcher shop, meat market", - "cab, hack, taxi, taxicab", - "caldron, cauldron", - "candle, taper, wax light", - "cannon", - "canoe", - "can opener, tin opener", - "cardigan", - "car mirror", - "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenters kit, tool kit", - "carton", - "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", - "cassette player", - "castle", - "catamaran", - "CD player", - "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", - "chainlink fence", - "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", - "chest", - "chiffonier, commode", - "chime, bell, gong", - "china cabinet, china closet", - "Christmas stocking", - "church, church building", - "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", - "cliff dwelling", - "cloak", - "clog, geta, patten, sabot", - "cocktail shaker", - "coffee mug", - "coffeepot", - "coil, spiral, volute, whorl, helix", - "combination lock", - "computer keyboard, keypad", - "confectionery, confectionary, candy store", - "container ship, containership, container vessel", - "convertible", - "corkscrew, bottle screw", - "cornet, horn, trumpet, trump", - "cowboy boot", - "cowboy hat, ten-gallon hat", - "cradle", - "crane", - "crash helmet", - "crate", - "crib, cot", - "Crock Pot", - "croquet ball", - "crutch", - "cuirass", - "dam, dike, dyke", - "desk", - "desktop computer", - "dial telephone, dial phone", - "diaper, nappy, napkin", - "digital clock", - "digital watch", - "dining table, board", - "dishrag, dishcloth", - "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", - "dock, dockage, docking facility", - "dogsled, dog sled, dog sleigh", - "dome", - "doormat, welcome mat", - "drilling platform, offshore rig", - "drum, membranophone, tympan", - "drumstick", - "dumbbell", - "Dutch oven", - "electric fan, blower", - "electric guitar", - "electric locomotive", - "entertainment center", - "envelope", - "espresso maker", - "face powder", - "feather boa, boa", - "file, file cabinet, filing cabinet", - "fireboat", - "fire engine, fire truck", - "fire screen, fireguard", - "flagpole, flagstaff", - "flute, transverse flute", - "folding chair", - "football helmet", - "forklift", - "fountain", - "fountain pen", - "four-poster", - "freight car", - "French horn, horn", - "frying pan, frypan, skillet", - "fur coat", - "garbage truck, dustcart", - "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", - "goblet", - "go-kart", - "golf ball", - "golfcart, golf cart", - "gondola", - "gong, tam-tam", - "gown", - "grand piano, grand", - "greenhouse, nursery, glasshouse", - "grille, radiator grille", - "grocery store, grocery, food market, market", - "guillotine", - "hair slide", - "hair spray", - "half track", - "hammer", - "hamper", - "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", - "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", - "harmonica, mouth organ, harp, mouth harp", - "harp", - "harvester, reaper", - "hatchet", - "holster", - "home theater, home theatre", - "honeycomb", - "hook, claw", - "hoopskirt, crinoline", - "horizontal bar, high bar", - "horse cart, horse-cart", - "hourglass", - "iPod", - "iron, smoothing iron", - "jack-o-lantern", - "jean, blue jean, denim", - "jeep, landrover", - "jersey, T-shirt, tee shirt", - "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", - "joystick", - "kimono", - "knee pad", - "knot", - "lab coat, laboratory coat", - "ladle", - "lampshade, lamp shade", - "laptop, laptop computer", - "lawn mower, mower", - "lens cap, lens cover", - "letter opener, paper knife, paperknife", - "library", - "lifeboat", - "lighter, light, igniter, ignitor", - "limousine, limo", - "liner, ocean liner", - "lipstick, lip rouge", - "Loafer", - "lotion", - "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jewelers loupe", - "lumbermill, sawmill", - "magnetic compass", - "mailbag, postbag", - "mailbox, letter box", - "maillot", - "maillot, tank suit", - "manhole cover", - "maraca", - "marimba, xylophone", - "mask", - "matchstick", - "maypole", - "maze, labyrinth", - "measuring cup", - "medicine chest, medicine cabinet", - "megalith, megalithic structure", - "microphone, mike", - "microwave, microwave oven", - "military uniform", - "milk can", - "minibus", - "miniskirt, mini", - "minivan", - "missile", - "mitten", - "mixing bowl", - "mobile home, manufactured home", - "Model T", - "modem", - "monastery", - "monitor", - "moped", - "mortar", - "mortarboard", - "mosque", - "mosquito net", - "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", - "mountain tent", - "mouse, computer mouse", - "mousetrap", - "moving van", - "muzzle", - "nail", - "neck brace", - "necklace", - "nipple", - "notebook, notebook computer", - "obelisk", - "oboe, hautboy, hautbois", - "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", - "oil filter", - "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", - "overskirt", - "oxcart", - "oxygen mask", - "packet", - "paddle, boat paddle", - "paddlewheel, paddle wheel", - "padlock", - "paintbrush", - "pajama, pyjama, pjs, jammies", - "palace", - "panpipe, pandean pipe, syrinx", - "paper towel", - "parachute, chute", - "parallel bars, bars", - "park bench", - "parking meter", - "passenger car, coach, carriage", - "patio, terrace", - "pay-phone, pay-station", - "pedestal, plinth, footstall", - "pencil box, pencil case", - "pencil sharpener", - "perfume, essence", - "Petri dish", - "photocopier", - "pick, plectrum, plectron", - "pickelhaube", - "picket fence, paling", - "pickup, pickup truck", - "pier", - "piggy bank, penny bank", - "pill bottle", - "pillow", - "ping-pong ball", - "pinwheel", - "pirate, pirate ship", - "pitcher, ewer", - "plane, carpenters plane, woodworking plane", - "planetarium", - "plastic bag", - "plate rack", - "plow, plough", - "plunger, plumbers helper", - "Polaroid camera, Polaroid Land camera", - "pole", - "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", - "pool table, billiard table, snooker table", - "pop bottle, soda bottle", - "pot, flowerpot", - "potters wheel", - "power drill", - "prayer rug, prayer mat", - "printer", - "prison, prison house", - "projectile, missile", - "projector", - "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", - "purse", - "quill, quill pen", - "quilt, comforter, comfort, puff", - "racer, race car, racing car", - "racket, racquet", - "radiator", - "radio, wireless", - "radio telescope, radio reflector", - "rain barrel", - "recreational vehicle, RV, R.V.", - "reel", - "reflex camera", - "refrigerator, icebox", - "remote control, remote", - "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", - "rifle", - "rocking chair, rocker", - "rotisserie", - "rubber eraser, rubber, pencil eraser", - "rugby ball", - "rule, ruler", - "running shoe", - "safe", - "safety pin", - "saltshaker, salt shaker", - "sandal", - "sarong", - "sax, saxophone", - "scabbard", - "scale, weighing machine", - "school bus", - "schooner", - "scoreboard", - "screen, CRT screen", - "screw", - "screwdriver", - "seat belt, seatbelt", - "sewing machine", - "shield, buckler", - "shoe shop, shoe-shop, shoe store", - "shoji", - "shopping basket", - "shopping cart", - "shovel", - "shower cap", - "shower curtain", - "ski", - "ski mask", - "sleeping bag", - "slide rule, slipstick", - "sliding door", - "slot, one-armed bandit", - "snorkel", - "snowmobile", - "snowplow, snowplough", - "soap dispenser", - "soccer ball", - "sock", - "solar dish, solar collector, solar furnace", - "sombrero", - "soup bowl", - "space bar", - "space heater", - "space shuttle", - "spatula", - "speedboat", - "spider web, spiders web", - "spindle", - "sports car, sport car", - "spotlight, spot", - "stage", - "steam locomotive", - "steel arch bridge", - "steel drum", - "stethoscope", - "stole", - "stone wall", - "stopwatch, stop watch", - "stove", - "strainer", - "streetcar, tram, tramcar, trolley, trolley car", - "stretcher", - "studio couch, day bed", - "stupa, tope", - "submarine, pigboat, sub, U-boat", - "suit, suit of clothes", - "sundial", - "sunglass", - "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", - "suspension bridge", - "swab, swob, mop", - "sweatshirt", - "swimming trunks, bathing trunks", - "swing", - "switch, electric switch, electrical switch", - "syringe", - "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", - "teapot", - "teddy, teddy bear", - "television, television system", - "tennis ball", - "thatch, thatched roof", - "theater curtain, theatre curtain", - "thimble", - "thresher, thrasher, threshing machine", - "throne", - "tile roof", - "toaster", - "tobacco shop, tobacconist shop, tobacconist", - "toilet seat", - "torch", - "totem pole", - "tow truck, tow car, wrecker", - "toyshop", - "tractor", - "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", - "trench coat", - "tricycle, trike, velocipede", - "trimaran", - "tripod", - "triumphal arch", - "trolleybus, trolley coach, trackless trolley", - "trombone", - "tub, vat", - "turnstile", - "typewriter keyboard", - "umbrella", - "unicycle, monocycle", - "upright, upright piano", - "vacuum, vacuum cleaner", - "vase", - "vault", - "velvet", - "vending machine", - "vestment", - "viaduct", - "violin, fiddle", - "volleyball", - "waffle iron", - "wall clock", - "wallet, billfold, notecase, pocketbook", - "wardrobe, closet, press", - "warplane, military plane", - "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", - "water bottle", - "water jug", - "water tower", - "whiskey jug", - "whistle", - "wig", - "window screen", - "window shade", - "Windsor tie", - "wine bottle", - "wing", - "wok", - "wooden spoon", - "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", - "wreck", - "yawl", - "yurt", - "web site, website, internet site, site", - "comic book", - "crossword puzzle, crossword", - "street sign", - "traffic light, traffic signal, stoplight", - "book jacket, dust cover, dust jacket, dust wrapper", - "menu", - "plate", - "guacamole", - "consomme", - "hot pot, hotpot", - "trifle", - "ice cream, icecream", - "ice lolly, lolly, lollipop, popsicle", - "French loaf", - "bagel, beigel", - "pretzel", - "cheeseburger", - "hotdog, hot dog, red hot", - "mashed potato", - "head cabbage", - "broccoli", - "cauliflower", - "zucchini, courgette", - "spaghetti squash", - "acorn squash", - "butternut squash", - "cucumber, cuke", - "artichoke, globe artichoke", - "bell pepper", - "cardoon", - "mushroom", - "Granny Smith", - "strawberry", - "orange", - "lemon", - "fig", - "pineapple, ananas", - "banana", - "jackfruit, jak, jack", - "custard apple", - "pomegranate", - "hay", - "carbonara", - "chocolate sauce, chocolate syrup", - "dough", - "meat loaf, meatloaf", - "pizza, pizza pie", - "potpie", - "burrito", - "red wine", - "espresso", - "cup", - "eggnog", - "alp", - "bubble", - "cliff, drop, drop-off", - "coral reef", - "geyser", - "lakeside, lakeshore", - "promontory, headland, head, foreland", - "sandbar, sand bar", - "seashore, coast, seacoast, sea-coast", - "valley, vale", - "volcano", - "ballplayer, baseball player", - "groom, bridegroom", - "scuba diver", - "rapeseed", - "daisy", - "yellow ladys slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", - "acorn", - "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", - "coral fungus", - "agaric", - "gyromitra", - "stinkhorn, carrion fungus", - "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", - "ear, spike, capitulum", - "toilet tissue, toilet paper, bathroom tissue"} diff --git a/demos/image_classification/go/resnet_predict.go b/demos/image_classification/go/resnet_predict.go deleted file mode 100644 index e5d41bbe93..0000000000 --- a/demos/image_classification/go/resnet_predict.go +++ /dev/null @@ -1,266 +0,0 @@ -/* -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/ - -package main - -import ( - "context" - "flag" - "fmt" - "image" - _ "image/jpeg" - "io/ioutil" - "log" - "os" - "path/filepath" - framework "tensorflow/core/framework" - pb "tensorflow_serving" - - google_protobuf "github.com/golang/protobuf/ptypes/wrappers" - "github.com/nfnt/resize" - "gocv.io/x/gocv" - "google.golang.org/grpc" -) - -func run_binary_input(servingAddress string, imgPath string) { - // Read the image in binary form - imgBytes, err := ioutil.ReadFile(imgPath) - if err != nil { - log.Fatalln(err) - } - - // Target model specification - const MODEL_NAME string = "resnet" - const INPUT_NAME string = "gpu_0/data_0" - const OUTPUT_NAME string = "gpu_0/softmax_1" - - // Create Predict Request to OVMS - predictRequest := &pb.PredictRequest{ - ModelSpec: &pb.ModelSpec{ - Name: MODEL_NAME, - SignatureName: "serving_default", - VersionChoice: &pb.ModelSpec_Version{ - Version: &google_protobuf.Int64Value{ - Value: int64(0), - }, - }, - }, - Inputs: map[string]*framework.TensorProto{ - INPUT_NAME: &framework.TensorProto{ - Dtype: framework.DataType_DT_STRING, - TensorShape: &framework.TensorShapeProto{ - Dim: []*framework.TensorShapeProto_Dim{ - &framework.TensorShapeProto_Dim{ - Size: int64(1), - }, - }, - }, - StringVal: [][]byte{imgBytes}, - }, - }, - } - - // Setup connection with the model server via gRPC - conn, err := grpc.Dial(servingAddress, grpc.WithInsecure()) - if err != nil { - log.Fatalf("Cannot connect to the grpc server: %v\n", err) - } - defer conn.Close() - - // Create client instance to prediction service - client := pb.NewPredictionServiceClient(conn) - - // Send predict request and receive response - predictResponse, err := client.Predict(context.Background(), predictRequest) - if err != nil { - log.Fatalln(err) - } - - log.Println("Request sent successfully") - - // Read prediction results - responseProto, ok := predictResponse.Outputs[OUTPUT_NAME] - if !ok { - log.Fatalf("Expected output: %s does not exist in the response", OUTPUT_NAME) - } - responseContent := responseProto.GetTensorContent() - - // Get details about output shape - outputShape := responseProto.GetTensorShape() - dim := outputShape.GetDim() - classesNum := dim[1].GetSize() - - // Convert bytes to matrix - outMat, err := gocv.NewMatFromBytes(1, int(classesNum), gocv.MatTypeCV32FC1, responseContent) - outMat = outMat.Reshape(1, 1) - - // Find maximum value along with its index in the output - _, maxVal, _, maxLoc := gocv.MinMaxLoc(outMat) - - // Get label of the class with the highest confidence - var label string - if classesNum == 1000 { - label = labels[maxLoc.X] - } else if classesNum == 1001 { - label = labels[maxLoc.X-1] - } else { - fmt.Printf("Unexpected class number in the output") - return - } - - fmt.Printf("Predicted class: %s\nClassification confidence: %f%%\n", label, maxVal*100) -} - -func run_with_conversion(servingAddress string, imgPath string) { - file, err := os.Open(imgPath) - if err != nil { - log.Fatal(err) - os.Exit(1) - } - defer file.Close() - - // Decode file to get Image type - decodedImg, _, err := image.Decode(file) - if err != nil { - log.Fatal(err) - os.Exit(1) - } - - // Resize image to match ResNet input - resizedImg := resize.Resize(224, 224, decodedImg, resize.Lanczos3) - - // Convert image to gocv.Mat type (HWC layout) - imgMat, err := gocv.ImageToMatRGB(resizedImg) - if err != nil { - log.Fatal(err) - os.Exit(1) - } - - // Keep uint8 precision and convert Mat to []bytes - imgBytes := imgMat.ToBytes() - - // Target model specification - const MODEL_NAME string = "resnet" - const INPUT_NAME string = "gpu_0/data_0" - const OUTPUT_NAME string = "gpu_0/softmax_1" - - // Create Predict Request to OVMS - predictRequest := &pb.PredictRequest{ - ModelSpec: &pb.ModelSpec{ - Name: MODEL_NAME, - SignatureName: "serving_default", - VersionChoice: &pb.ModelSpec_Version{ - Version: &google_protobuf.Int64Value{ - Value: int64(0), - }, - }, - }, - Inputs: map[string]*framework.TensorProto{ - INPUT_NAME: &framework.TensorProto{ - Dtype: framework.DataType_DT_UINT8, - TensorShape: &framework.TensorShapeProto{ - Dim: []*framework.TensorShapeProto_Dim{ - &framework.TensorShapeProto_Dim{ - Size: int64(1), - }, - &framework.TensorShapeProto_Dim{ - Size: int64(224), - }, - &framework.TensorShapeProto_Dim{ - Size: int64(224), - }, - &framework.TensorShapeProto_Dim{ - Size: int64(3), - }, - }, - }, - TensorContent: imgBytes, - }, - }, - } - - conn, err := grpc.Dial(servingAddress, grpc.WithInsecure()) - if err != nil { - log.Fatalf("Cannot connect to the grpc server: %v\n", err) - } - defer conn.Close() - - // Create client instance to prediction service - client := pb.NewPredictionServiceClient(conn) - - // Send predict request and receive response - predictResponse, err := client.Predict(context.Background(), predictRequest) - if err != nil { - log.Fatalln(err) - } - - log.Println("Request sent successfully") - - // Read prediction results - responseProto, ok := predictResponse.Outputs[OUTPUT_NAME] - if !ok { - log.Fatalf("Expected output: %s does not exist in the response", OUTPUT_NAME) - } - responseContent := responseProto.GetTensorContent() - - // Get details about output shape - outputShape := responseProto.GetTensorShape() - dim := outputShape.GetDim() - classesNum := dim[1].GetSize() - - // Convert bytes to matrix - outMat, err := gocv.NewMatFromBytes(1, int(classesNum), gocv.MatTypeCV32FC1, responseContent) - outMat = outMat.Reshape(1, 1) - - // Find maximum value along with its index in the output - _, maxVal, _, maxLoc := gocv.MinMaxLoc(outMat) - - // Get label of the class with the highest confidence - var label string - if classesNum == 1000 { - label = labels[maxLoc.X] - } else if classesNum == 1001 { - label = labels[maxLoc.X-1] - } else { - fmt.Printf("Unexpected class number in the output") - return - } - - fmt.Printf("Predicted class: %s\nClassification confidence: %f%%\n", label, maxVal*100) -} - -func main() { - servingAddress := flag.String("serving-address", "localhost:8500", "The tensorflow serving address") - binaryInput := flag.Bool("binary-input", false, "Send JPG/PNG raw bytes") - flag.Parse() - - if flag.NArg() > 2 { - fmt.Println("Usage: " + os.Args[0] + " --serving-address localhost:8500 path/to/img") - os.Exit(1) - } - - imgPath, err := filepath.Abs(flag.Arg(0)) - if err != nil { - log.Fatalln(err) - os.Exit(1) - } - - if *binaryInput { - run_binary_input(*servingAddress, imgPath) - } else { - run_with_conversion(*servingAddress, imgPath) - } -} diff --git a/demos/image_classification/python/image_classification.py b/demos/image_classification/python/image_classification.py index b191f59945..ce60b17d98 100644 --- a/demos/image_classification/python/image_classification.py +++ b/demos/image_classification/python/image_classification.py @@ -17,14 +17,11 @@ import sys sys.path.append("../../common/python") -import grpc import numpy as np import classes -from tensorflow import make_tensor_proto, make_ndarray, make_tensor_proto import datetime import argparse -from tensorflow_serving.apis import predict_pb2 -from tensorflow_serving.apis import prediction_service_pb2_grpc +import tritonclient.grpc as grpcclient import cv2 def crop_resize(img,cropx,cropy): @@ -58,7 +55,7 @@ def getJpeg(path, size, rgb_image=0): print(path, img.shape, "; data range:",np.amin(img),":",np.amax(img)) return img -parser = argparse.ArgumentParser(description='Do requests to ie_serving and tf_serving using images in numpy format') +parser = argparse.ArgumentParser(description='Do requests to OpenVINO Model Server using images in numpy format') parser.add_argument('--images_list', required=False, default='input_images.txt', help='path to a file with a list of labeled images') parser.add_argument('--grpc_address',required=False, default='localhost', help='Specify url to grpc service. default:localhost') parser.add_argument('--grpc_port',required=False, default=9000, help='Specify port to grpc service. default: 9000') @@ -70,8 +67,7 @@ def getJpeg(path, size, rgb_image=0): parser.add_argument('--rgb_image',required=False, default=0, type=int, help='Convert BGR channels to RGB channels in the input image') args = vars(parser.parse_args()) -channel = grpc.insecure_channel("{}:{}".format(args['grpc_address'],args['grpc_port'])) -stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) +client = grpcclient.InferenceServerClient(url="{}:{}".format(args['grpc_address'],args['grpc_port'])) input_images = args.get('images_list') size = args.get('size') with open(input_images) as f: @@ -91,21 +87,20 @@ def getJpeg(path, size, rgb_image=0): path, label = line.strip().split(" ") img = getJpeg(path, size, rgb_image) - request = predict_pb2.PredictRequest() - request.model_spec.name = args.get('model_name') - request.inputs[args['input_name']].CopyFrom(make_tensor_proto(img, shape=(img.shape))) + infer_input = grpcclient.InferInput(args['input_name'], img.shape, "FP32") + infer_input.set_data_from_numpy(img) start_time = datetime.datetime.now() - result = stub.Predict(request, 10.0) # result includes a dictionary with all model outputs + result = client.infer(args.get('model_name'), [infer_input]) end_time = datetime.datetime.now() - if args['output_name'] not in result.outputs: + if args['output_name'] not in [output.name for output in result.get_output()]: print("Invalid output name", args['output_name']) print("Available outputs:") - for Y in result.outputs: - print(Y) + for out in result.get_output(): + print(out.name) exit(1) duration = (end_time - start_time).total_seconds() * 1000 processing_times = np.append(processing_times,np.array([int(duration)])) - output = make_ndarray(result.outputs[args['output_name']]) + output = result.as_numpy(args['output_name']) nu = np.array(output) # for object classification models show imagenet class print('Processing time: {:.2f} ms; speed {:.2f} fps'.format(round(duration, 2), round(1000 / duration, 2))) diff --git a/demos/image_classification_using_tf_model/python/image_classification_using_tf_model.py b/demos/image_classification_using_tf_model/python/image_classification_using_tf_model.py index 9a96d560f5..489b661be4 100644 --- a/demos/image_classification_using_tf_model/python/image_classification_using_tf_model.py +++ b/demos/image_classification_using_tf_model/python/image_classification_using_tf_model.py @@ -14,7 +14,7 @@ # limitations under the License. # -from ovmsclient import make_grpc_client +import tritonclient.grpc as grpcclient import cv2 import numpy as np import argparse @@ -33,13 +33,15 @@ def load_img(path): scale = [127.5,127.5,127.5] img_f = (img_f - np.array(mean, dtype=np.float32))/np.array(scale, dtype=np.float32) img_f = img_f.reshape(1, img_f.shape[0], img_f.shape[1], 3) - return {"input:0": img_f} + return img_f def build_parser(): parser = argparse.ArgumentParser(description='Client for OCR pipeline') parser.add_argument('--grpc_address', required=False, default='localhost', help='Specify url to grpc service. default:localhost') parser.add_argument('--grpc_port', required=False, default=9000, help='Specify port to grpc service. default: 9000') parser.add_argument('--image_input_path', required=True, help='Image input path') + parser.add_argument('--input_name', required=False, default='', help='Input tensor name. default: auto-detected from model metadata') + parser.add_argument('--output_name', required=False, default='', help='Output tensor name. default: auto-detected from model metadata') return parser if __name__ == "__main__": @@ -47,9 +49,22 @@ def build_parser(): img_path = args['image_input_path'] address = "{}:{}".format(args['grpc_address'],args['grpc_port']) - input = load_img(img_path) - client = make_grpc_client(address) - classification_output = client.predict(input, "resnet") + client = grpcclient.InferenceServerClient(url=address) + + input_name = args['input_name'] + output_name = args['output_name'] + if not input_name or not output_name: + metadata = client.get_model_metadata("resnet") + if not input_name: + input_name = metadata.inputs[0].name + if not output_name: + output_name = metadata.outputs[0].name + + img = load_img(img_path) + infer_input = grpcclient.InferInput(input_name, img.shape, "FP32") + infer_input.set_data_from_numpy(img) + result = client.infer("resnet", [infer_input]) + classification_output = result.as_numpy(output_name) print("Image classified as " + classes.imagenet_classes[classification_output.argmax() - 1]) diff --git a/demos/image_classification_using_tf_model/python/requirements.txt b/demos/image_classification_using_tf_model/python/requirements.txt index ff19d1b01b..aad18a7b0c 100644 --- a/demos/image_classification_using_tf_model/python/requirements.txt +++ b/demos/image_classification_using_tf_model/python/requirements.txt @@ -1,3 +1,3 @@ -ovmsclient +tritonclient[grpc] opencv-python>=4.8.1.78 numpy<2.0.0 \ No newline at end of file diff --git a/demos/image_classification_with_string_output/README.md b/demos/image_classification_with_string_output/README.md index 03a33a4ead..89cb1f214e 100644 --- a/demos/image_classification_with_string_output/README.md +++ b/demos/image_classification_with_string_output/README.md @@ -41,8 +41,7 @@ Use example client to send requests containing images via KServ REST API: ```console python image_classification_with_string_output.py --http_port 8000 ``` -Request may be sent also using other APIs (KServ GRPC, TFS). In this sections you can find short code samples how to do this: -- [TensorFlow Serving API](../../docs/clients_tfs.md) +Request may be sent also using other APIs (KServe GRPC). In this section you can find short code samples how to do this: - [KServe API](../../docs/clients_kfs.md) diff --git a/demos/model_ensemble/python/.gitignore b/demos/model_ensemble/python/.gitignore deleted file mode 100644 index 55c32418e3..0000000000 --- a/demos/model_ensemble/python/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -workspace -models -.tf_env diff --git a/demos/model_ensemble/python/Makefile b/demos/model_ensemble/python/Makefile deleted file mode 100644 index 73b85320fd..0000000000 --- a/demos/model_ensemble/python/Makefile +++ /dev/null @@ -1,47 +0,0 @@ -# -# Copyright (c) 2022 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -.PHONY: client_preprocessing, server_preprocessing - -default: prepare_repository - -THIS_DIR := $(dir $(abspath $(firstword $(MAKEFILE_LIST)))) - -venv: - virtualenv -p python3 .tf_env - . .tf_env/bin/activate - pip3 install tensorflow==2.3.1 - -download_models: venv - mkdir -p workspace/models/argmax - docker run -u $(id -u):$(id -g) -e http_proxy=${http_proxy} -e https_proxy=${https_proxy} -e no_proxy=${no_proxy} -v $(THIS_DIR)/workspace/models:/models:rw openvino/ubuntu18_dev:2024.6.0 omz_downloader --name googlenet-v2-tf --output_dir /models - docker run -u $(id -u):$(id -g) -e http_proxy=${http_proxy} -e https_proxy=${https_proxy} -e no_proxy=${no_proxy} -v $(THIS_DIR)/workspace/models:/models:rw openvino/ubuntu18_dev:2024.6.0 omz_downloader --name resnet-50-tf --output_dir /models - python3 ../../../tests/models/argmax_sum.py --input_size 1001 --export_dir $(THIS_DIR)/workspace/models/argmax - -convert_models: download_models - docker run -u $(id -u):$(id -g) -v $(THIS_DIR)/workspace/models:/models:rw openvino/ubuntu18_dev:2024.6.0 omz_converter --name googlenet-v2-tf --download_dir /models --output_dir /models --precisions FP32 - docker run -u $(id -u):$(id -g) -v $(THIS_DIR)/workspace/models:/models:rw openvino/ubuntu18_dev:2024.6.0 omz_converter --name resnet-50-tf --download_dir /models --output_dir /models --precisions FP32 - docker run -u $(id -u):$(id -g) -v $(THIS_DIR)/workspace/models:/models:rw openvino/ubuntu18_dev:2024.6.0 mo --input input1,input2 --input_shape '[1,1001],[1,1001]' --saved_model_dir /models/argmax/ --output_dir /models/argmax/1 - -prepare_repository: convert_models - mkdir -p models/googlenet-v2-tf/1 - mkdir -p models/resnet-50-tf/1 - mkdir -p models/argmax/1 - cp workspace/models/public/resnet-50-tf/FP32/* models/resnet-50-tf/1/ - cp workspace/models/public/googlenet-v2-tf/FP32/* models/googlenet-v2-tf/1/ - cp workspace/models/argmax/1/* models/argmax/1/ - cp config.json models/ - tree models diff --git a/demos/model_ensemble/python/README.md b/demos/model_ensemble/python/README.md deleted file mode 100644 index 6a7e768b67..0000000000 --- a/demos/model_ensemble/python/README.md +++ /dev/null @@ -1,240 +0,0 @@ -# Model Ensemble Pipeline Demo {#ovms_docs_demo_ensemble} - -This guide shows how to implement a model ensemble using the [DAG Scheduler](../../../docs/dag_scheduler.md). - -- Let's consider you develop an application to perform image classification. There are many different models that can be used for this task. The goal is to combine results from inferences executed on two different models and calculate argmax to pick the most probable classification label. -- For this task, select two models: [googlenet-v2](https://github.com/openvinotoolkit/open_model_zoo/blob/master/models/public/googlenet-v2-tf/README.md) and [resnet-50](https://github.com/openvinotoolkit/open_model_zoo/blob/master/models/public/resnet-50-tf/README.md). Additionally, create own model **argmax** to combine and select top result. The aim is to perform this task on the server side with no intermediate results passed over the network. The server should take care of feeding inputs/outputs in subsequent models. Both - googlenet and resnet predictions should run in parallel. -- Diagram for this pipeline would look like this: - -![diagram](model_ensemble_diagram.svg) - -## Step 1: Prepare the repository - -Clone the repository and enter model_ensemble directory -```bash -git clone https://github.com/openvinotoolkit/model_server.git -cd model_server/demos/model_ensemble/python -``` - -Repository preparation is simplified with `make` script, just run `make` in this repository. -```bash -make -``` - -The steps in `Makefile` are: - -1. Download and use the models from [open model zoo](https://github.com/openvinotoolkit/open_model_zoo). -2. Use [python script](https://github.com/openvinotoolkit/model_server/blob/main/tests/models/argmax_sum.py) located in this repository. Since it uses tensorflow to create models in _saved model_ format, hence tensorflow pip package is required. -3. Prepare argmax model with `(1, 1001)` input shapes to match output of the googlenet and resnet output shapes. The generated model will sum inputs and calculate the index with the highest value. The model output will indicate the most likely predicted class from the ImageNet* dataset. -4. Convert models to IR format and [prepare models repository](../../../docs/models_repository.md). - -```bash -... -models -├── argmax -│ └── 1 -│ ├── saved_model.bin -│ ├── saved_model.mapping -│ └── saved_model.xml -├── config.json -├── googlenet-v2-tf -│ └── 1 -│ ├── googlenet-v2-tf.bin -│ ├── googlenet-v2-tf.mapping -│ └── googlenet-v2-tf.xml -└── resnet-50-tf - └── 1 - ├── resnet-50-tf.bin - ├── resnet-50-tf.mapping - └── resnet-50-tf.xml - -6 directories, 10 files -``` - -## Step 2: Define required models and pipeline -Pipelines need to be defined in the configuration file to use them. The same configuration file is used to define served models and served pipelines. - -Use the [config.json located here](https://github.com/openvinotoolkit/model_server/blob/main/demos/model_ensemble/python/config.json), the content is as follows: -```bash -cat config.json -{ - "model_config_list": [ - { - "config": { - "name": "googlenet", - "base_path": "/models/googlenet-v2-tf" - } - }, - { - "config": { - "name": "resnet", - "base_path": "/models/resnet-50-tf" - } - }, - { - "config": { - "name": "argmax", - "base_path": "/models/argmax" - } - } - ], - "pipeline_config_list": [ - { - "name": "image_classification_pipeline", - "inputs": ["image"], - "nodes": [ - { - "name": "googlenet_node", - "model_name": "googlenet", - "type": "DL model", - "inputs": [ - {"input": {"node_name": "request", - "data_item": "image"}} - ], - "outputs": [ - {"data_item": "InceptionV2/Predictions/Softmax", - "alias": "probability"} - ] - }, - { - "name": "resnet_node", - "model_name": "resnet", - "type": "DL model", - "inputs": [ - {"map/TensorArrayStack/TensorArrayGatherV3": {"node_name": "request", - "data_item": "image"}} - ], - "outputs": [ - {"data_item": "softmax_tensor", - "alias": "probability"} - ] - }, - { - "name": "argmax_node", - "model_name": "argmax", - "type": "DL model", - "inputs": [ - {"input1": {"node_name": "googlenet_node", - "data_item": "probability"}}, - {"input2": {"node_name": "resnet_node", - "data_item": "probability"}} - ], - "outputs": [ - {"data_item": "argmax:0", - "alias": "most_probable_label"} - ] - } - ], - "outputs": [ - {"label": {"node_name": "argmax_node", - "data_item": "most_probable_label"}} - ] - } - ] -} -``` -In the `model_config_list` section, three models are defined as usual. We can refer to them by name in the pipeline definition but we can also request single inference on them separately. The same inference gRPC and REST API is used to request models and pipelines. OpenVINO™ Model Server will first try to search for a model with the requested name. If not found, it will try to find pipeline. - - -## Step 3: Start the Model Server - -Run command to start the Model Server -```bash -docker run --rm -v $(pwd)/models/:/models:ro -p 9100:9100 -p 8100:8100 openvino/model_server:latest --config_path /models/config.json --port 9100 --rest_port 8100 --log_level DEBUG -``` - -## Step 4: Requesting the service - -Input images can be sent to the service requesting resource name `image_classification_pipeline`. There is an example client doing that: - -Check accuracy of the pipeline by running the client in another terminal: -```bash -cd ../../../client/python/tensorflow-serving-api/samples -virtualenv .venv -. .venv/bin/activate && pip3 install -r requirements.txt -python grpc_predict_resnet.py --pipeline_name image_classification_pipeline --images_numpy_path ../../imgs.npy \ - --labels_numpy_path ../../lbs.npy --grpc_port 9100 --input_name image --output_name label --transpose_input True --transpose_method nchw2nhwc --iterations 10 -Image data range: 0.0 : 255.0 -Start processing: - Model name: image_classification_pipeline - Iterations: 10 - Images numpy path: ../../imgs.npy - Numpy file shape: (10, 224, 224, 3) - -Iteration 1; Processing time: 33.51 ms; speed 29.85 fps -imagenet top results in a single batch: -response shape (1,) - 0 airliner 404 ; Correct match. -Iteration 2; Processing time: 42.52 ms; speed 23.52 fps -imagenet top results in a single batch: -response shape (1,) - 0 Arctic fox, white fox, Alopex lagopus 279 ; Correct match. -Iteration 3; Processing time: 34.42 ms; speed 29.05 fps -imagenet top results in a single batch: -response shape (1,) - 0 bee 309 ; Correct match. -Iteration 4; Processing time: 32.34 ms; speed 30.92 fps -imagenet top results in a single batch: -response shape (1,) - 0 golden retriever 207 ; Correct match. -Iteration 5; Processing time: 35.92 ms; speed 27.84 fps -imagenet top results in a single batch: -response shape (1,) - 0 gorilla, Gorilla gorilla 366 ; Correct match. -Iteration 6; Processing time: 33.63 ms; speed 29.74 fps -imagenet top results in a single batch: -response shape (1,) - 0 magnetic compass 635 ; Correct match. -Iteration 7; Processing time: 37.22 ms; speed 26.86 fps -imagenet top results in a single batch: -response shape (1,) - 0 peacock 84 ; Correct match. -Iteration 8; Processing time: 35.84 ms; speed 27.90 fps -imagenet top results in a single batch: -response shape (1,) - 0 pelican 144 ; Correct match. -Iteration 9; Processing time: 33.69 ms; speed 29.68 fps -imagenet top results in a single batch: -response shape (1,) - 0 snail 113 ; Correct match. -Iteration 10; Processing time: 46.54 ms; speed 21.49 fps -imagenet top results in a single batch: -response shape (1,) - 0 zebra 340 ; Correct match. - -processing time for all iterations -average time: 36.00 ms; average speed: 27.78 fps -median time: 34.50 ms; median speed: 28.99 fps -max time: 46.00 ms; min speed: 21.74 fps -min time: 32.00 ms; max speed: 31.25 fps -time percentile 90: 42.40 ms; speed percentile 90: 23.58 fps -time percentile 50: 34.50 ms; speed percentile 50: 28.99 fps -time standard deviation: 4.31 -time variance: 18.60 -Classification accuracy: 100.00 -``` - -## Step 5: Analyze pipeline execution in server logs - -By analyzing debug logs and timestamps it is seen that GoogleNet and ResNet model inferences were started in parallel. Just after all inputs became ready - argmax node has started its job. -```bash -docker logs -[2022-02-28 11:30:20.159][485][serving][debug][prediction_service.cpp:69] Processing gRPC request for model: image_classification_pipeline; version: 0 -[2022-02-28 11:30:20.159][485][serving][debug][prediction_service.cpp:80] Requested model: image_classification_pipeline does not exist. Searching for pipeline with that name... -[2022-02-28 11:30:20.160][485][dag_executor][debug][pipeline.cpp:83] Started execution of pipeline: image_classification_pipeline -[2022-02-28 11:30:20.160][485][serving][debug][modelmanager.cpp:1280] Requesting model: resnet; version: 0. -[2022-02-28 11:30:20.160][485][serving][debug][modelmanager.cpp:1280] Requesting model: googlenet; version: 0. -[2022-02-28 11:30:20.194][485][serving][debug][modelmanager.cpp:1280] Requesting model: argmax; version: 0. -``` - -## Step 6: Requesting pipeline metadata - -We can use the same gRPC/REST example client as we use for requesting model metadata. The only difference is we specify pipeline name instead of the model name. -```bash -python grpc_get_model_metadata.py --grpc_port 9100 --model_name image_classification_pipeline -Getting model metadata for model: image_classification_pipeline -Inputs metadata: - Input name: image; shape: [1, 224, 224, 3]; dtype: DT_FLOAT -Outputs metadata: - Output name: label; shape: [1]; dtype: DT_INT64 -``` diff --git a/demos/model_ensemble/python/config.json b/demos/model_ensemble/python/config.json deleted file mode 100644 index 33ab8d4de5..0000000000 --- a/demos/model_ensemble/python/config.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "model_config_list": [ - { - "config": { - "name": "googlenet", - "base_path": "/models/googlenet-v2-tf" - } - }, - { - "config": { - "name": "resnet", - "base_path": "/models/resnet-50-tf" - } - }, - { - "config": { - "name": "argmax", - "base_path": "/models/argmax" - } - } - ], - "pipeline_config_list": [ - { - "name": "image_classification_pipeline", - "inputs": ["image"], - "nodes": [ - { - "name": "googlenet_node", - "model_name": "googlenet", - "type": "DL model", - "inputs": [ - {"input": {"node_name": "request", - "data_item": "image"}} - ], - "outputs": [ - {"data_item": "InceptionV2/Predictions/Softmax", - "alias": "probability"} - ] - }, - { - "name": "resnet_node", - "model_name": "resnet", - "type": "DL model", - "inputs": [ - {"map/TensorArrayStack/TensorArrayGatherV3": {"node_name": "request", - "data_item": "image"}} - ], - "outputs": [ - {"data_item": "softmax_tensor", - "alias": "probability"} - ] - }, - { - "name": "argmax_node", - "model_name": "argmax", - "type": "DL model", - "inputs": [ - {"input1": {"node_name": "googlenet_node", - "data_item": "probability"}}, - {"input2": {"node_name": "resnet_node", - "data_item": "probability"}} - ], - "outputs": [ - {"data_item": "argmax:0", - "alias": "most_probable_label"} - ] - } - ], - "outputs": [ - {"label": {"node_name": "argmax_node", - "data_item": "most_probable_label"}} - ] - } - ] -} diff --git a/demos/model_ensemble/python/model_ensemble_diagram.svg b/demos/model_ensemble/python/model_ensemble_diagram.svg deleted file mode 100644 index 5b5c32c919..0000000000 --- a/demos/model_ensemble/python/model_ensemble_diagram.svg +++ /dev/null @@ -1,3 +0,0 @@ - - -
name: request
inputs: image
outputs: image
name: request...
name: response
inputs: label
outputs: label
name: response...
name: googlenet_node
model: googlenet
inputs: input
outputs: InceptionV2/Predictions/Softmax (probability)
name: googlenet_node...
name: resnet_node
model: resnet
inputs: map/TensorArrayStack/TensorArrayGatherV3
outputs: softmax_tensor (probability)
name: resnet_node...
name: argmax_node
model: argmax
inputs: input1, input2
outputs: argmax (most_probable_label)
name: argmax_node...
(1, 3, 224, 224), FP32, NCHW
image → map/TensorArrayStack/TensorArrayGatherV3
(1, 3, 224, 224), FP32, NCHW...
(1, 3, 224, 224), FP32, NCHW
image → input
(1, 3, 224, 224), FP32, NCHW...
(1, 1001), FP32, NC
probability → input1
(1, 1001), FP32, NC...
(1, 1001), FP32, NC
probability → input2
(1, 1001), FP32, NC...
(1), FP32, N
most_probable_label → label
(1), FP32, N...
gRPC/REST request
inputs: image
gRPC/REST request...
gRPC/REST response
outputs: label
gRPC/REST response...Viewer does not support full SVG 1.1 \ No newline at end of file diff --git a/demos/multi_faces_analysis_pipeline/python/.gitignore b/demos/multi_faces_analysis_pipeline/python/.gitignore deleted file mode 100644 index f1981605f0..0000000000 --- a/demos/multi_faces_analysis_pipeline/python/.gitignore +++ /dev/null @@ -1 +0,0 @@ -workspace \ No newline at end of file diff --git a/demos/multi_faces_analysis_pipeline/python/Makefile b/demos/multi_faces_analysis_pipeline/python/Makefile deleted file mode 100644 index 54f43eaf50..0000000000 --- a/demos/multi_faces_analysis_pipeline/python/Makefile +++ /dev/null @@ -1,50 +0,0 @@ -# -# Copyright (c) 2022 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -.PHONY: setup_repository - -default: setup_repository - -BUILD_CUSTOM_NODE=false -BASE_OS=ubuntu - -setup_repository: -# Download face detection model - mkdir -p workspace/face-detection-retail-0004/1 - curl https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/face-detection-retail-0004/FP32/face-detection-retail-0004.xml -o workspace/face-detection-retail-0004/1/face-detection-retail-0004.xml - curl https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/face-detection-retail-0004/FP32/face-detection-retail-0004.bin -o workspace/face-detection-retail-0004/1/face-detection-retail-0004.bin -# Download age gender recognition model - mkdir -p workspace/age-gender-recognition-retail-0013/1 - curl https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013.xml -o workspace/age-gender-recognition-retail-0013/1/age-gender-recognition-retail-0013.xml - curl https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013.bin -o workspace/age-gender-recognition-retail-0013/1/age-gender-recognition-retail-0013.bin -# Download emotion recognition model - mkdir -p workspace/emotion-recognition-retail-0003/1 - curl https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/emotions-recognition-retail-0003/FP32/emotions-recognition-retail-0003.xml -o workspace/emotion-recognition-retail-0003/1/emotions-recognition-retail-0003.xml - curl https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/emotions-recognition-retail-0003/FP32/emotions-recognition-retail-0003.bin -o workspace/emotion-recognition-retail-0003/1/emotions-recognition-retail-0003.bin -ifeq ($(BUILD_CUSTOM_NODE),true) -# Build custom node - cd ../../../src/custom_nodes && \ - make BASE_OS=$(BASE_OS) NODES=model_zoo_intel_object_detection && \ - mkdir -p ../../demos/multi_faces_analysis_pipeline/python/workspace/lib && \ - mv lib/$(BASE_OS)/libcustom_node_model_zoo_intel_object_detection.so ../../demos/multi_faces_analysis_pipeline/python/workspace/lib/libcustom_node_model_zoo_intel_object_detection.so -endif -# Copy configuration file to workspace directory - cp config.json workspace/. -ifeq ($(BUILD_CUSTOM_NODE),false) - sed -i 's:\/workspace\/lib\/libcustom_node_model_zoo_intel_object_detection.so:\/ovms\/lib\/custom_nodes\/libcustom_node_model_zoo_intel_object_detection.so:g' workspace/config.json -endif -clean: - @rm -rf workspace \ No newline at end of file diff --git a/demos/multi_faces_analysis_pipeline/python/README.md b/demos/multi_faces_analysis_pipeline/python/README.md deleted file mode 100644 index e6148ef29f..0000000000 --- a/demos/multi_faces_analysis_pipeline/python/README.md +++ /dev/null @@ -1,153 +0,0 @@ -# Multi Faces Analysis Pipeline Demo {#ovms_demo_multi_faces_analysis_pipeline} - - -This document demonstrates how to create complex pipelines using object detection and object recognition models from OpenVINO Model Zoo. As an example, we will use [face-detection-retail-0004](https://github.com/openvinotoolkit/open_model_zoo/blob/2022.1.0/models/intel/face-detection-retail-0004/README.md) to detect multiple faces on the image. Then, for each detected face we will crop it using [model_zoo_intel_object_detection](https://github.com/openvinotoolkit/model_server/tree/main/src/custom_nodes/model_zoo_intel_object_detection) example custom node. Finally, each image face image will be forwarded to [age-gender-recognition-retail-0013](https://github.com/openvinotoolkit/open_model_zoo/blob/2022.1.0/models/intel/age-gender-recognition-retail-0013/README.md) and [emotion-recognition-retail-0003](https://github.com/openvinotoolkit/open_model_zoo/blob/2022.1.0/models/intel/emotions-recognition-retail-0003/README.md) models. - -![Multi Faces Analysis Graph](multi_faces_analysis.png) - -Using such a pipeline, a single request to OVMS can perform a complex set of operations to determine all faces and their properties. - -### See also - -For simpler use case with single face analysis see [single_face_analysis_pipeline](../../single_face_analysis_pipeline/python/README.md) demo. - -## Pipeline Configuration Graph - -Below is depicted graph implementing faces analysis pipeline execution. - -![Multi Faces Analysis Pipeline Graph](multi_faces_analysis_graph.svg) - -It includes the following Nodes: -- Model `face-detection` - deep learning model which takes user image as input. Its outputs contain information about face coordinates and confidence levels. -- Custom node `model_zoo_intel_object_detection` - it includes C++ implementation of common object detection models results processing. By analysing the output it produces cropped face images based on the configurable score level threshold. Custom node also resizes them to the target resolution and combines into a single output of a dynamic batch size. The output batch size is determined by the number of detected -boxes according to the configured criteria. All operations on the images employ OpenCV libraries which are preinstalled in the OVMS. Learn more about the [model_zoo_intel_object_detection custom node](https://github.com/openvinotoolkit/model_server/tree/main/src/custom_nodes/model_zoo_intel_object_detection). -- demultiplexer - outputs from the custom node model_zoo_intel_object_detection have variable batch size. In order to match it with the sequential recognition models, data is split into individual images with each batch size equal to 1. -Such smaller requests can be submitted for inference in parallel to the next Model Nodes. Learn more about the [demultiplexing](../../../docs/demultiplexing.md). -- Model `age-gender-recognition` - this model recognizes age and gender on given face image -- Model `emotion-recognition` - this model outputs emotion probability for emotions: neutral, happy, sad, surprised and angry -- Response - the output of the whole pipeline combines the recognized face images with their metadata: coordinates, age, gender, emotions and detection confidence level. - -## Prepare workspace to run the demo - -To successfully deploy face analysis pipeline you need to have a workspace that contains: -- [face-detection-retail-0004](https://github.com/openvinotoolkit/open_model_zoo/blob/2022.1.0/models/intel/face-detection-retail-0004/README.md), -[age-gender-recognition-retail-0013](https://github.com/openvinotoolkit/open_model_zoo/blob/2022.1.0/models/intel/age-gender-recognition-retail-0013/README.md) and -[emotion-recognition-retail-0003](https://github.com/openvinotoolkit/open_model_zoo/blob/2022.1.0/models/intel/emotions-recognition-retail-0003/README.md) models -- Custom node for image processing -- Configuration file - -Clone the repository and enter multi_faces_analysis_pipeline directory -```bash -git clone https://github.com/openvinotoolkit/model_server.git -cd model_server/demos/multi_faces_analysis_pipeline/python -``` - -You can prepare the workspace that contains all the above by just running `make` command. -Since custom node used in this demo is included in OpenVINO Model Server image you can either use the custom node from the image, or build one. - -If you just want to quickly run this demo and use already compiled custom node, run: - -```bash -make -``` - -#### Directory structure (without custom node) - -Once the `make` procedure is finished, you should have `workspace` directory ready with the following content. - -```bash -workspace -├── age-gender-recognition-retail-0013 -│ └── 1 -│ ├── age-gender-recognition-retail-0013.bin -│ └── age-gender-recognition-retail-0013.xml -├── config.json -├── emotion-recognition-retail-0003 -│ └── 1 -│ ├── emotions-recognition-retail-0003.bin -│ └── emotions-recognition-retail-0003.xml -└── face-detection-retail-0004 - └── 1 - ├── face-detection-retail-0004.bin - └── face-detection-retail-0004.xml - -``` - -If you modified the custom node or for some other reason, you want to have it compiled and then attached to the container, run: - -```bash -make BUILD_CUSTOM_NODE=true BASE_OS=ubuntu -``` - -#### Directory structure (with custom node) - -Once the `make` procedure is finished, you should have `workspace` directory ready with the following content. -```bash -workspace -├── age-gender-recognition-retail-0013 -│ └── 1 -│ ├── age-gender-recognition-retail-0013.bin -│ └── age-gender-recognition-retail-0013.xml -├── config.json -├── emotion-recognition-retail-0003 -│ └── 1 -│ ├── emotions-recognition-retail-0003.bin -│ └── emotions-recognition-retail-0003.xml -├── face-detection-retail-0004 -│ └── 1 -│ ├── face-detection-retail-0004.bin -│ └── face-detection-retail-0004.xml -└── lib - └── libcustom_node_model_zoo_intel_object_detection.so -``` - -## Deploying OVMS - -Deploy OVMS with faces analysis pipeline using the following command: - -```bash -docker run -p 9000:9000 -d -v ${PWD}/workspace:/workspace openvino/model_server --config_path /workspace/config.json --port 9000 -``` - -## Requesting the Service - -Exemplary client [multi_faces_analysis_pipeline.py](https://github.com/openvinotoolkit/model_server/blob/main/demos/multi_faces_analysis_pipeline/python/multi_faces_analysis_pipeline.py) can be used to request pipeline deployed in previous step. - -```bash -pip3 install -r requirements.txt -``` - -Now you can create a directory for text images and run the client: -```bash -mkdir results -``` -```bash -python multi_faces_analysis_pipeline.py --pipeline_name find_face_images --grpc_port 9000 --image_input_path ../../common/static/images/people/people1.jpeg --face_images_output_name face_images --face_images_save_path ./results --image_width 600 --image_height 400 --input_image_layout NHWC -Output: name[emotions] - numpy => shape[(10, 1, 5, 1, 1)] data[float32] -Output: name[ages] - numpy => shape[(10, 1, 1, 1, 1)] data[float32] -Output: name[confidence_levels] - numpy => shape[(10, 1, 1)] data[float32] -Output: name[genders] - numpy => shape[(10, 1, 2, 1, 1)] data[float32] -Output: name[face_images] - numpy => shape[(10, 1, 64, 64, 3)] data[float32] -Output: name[face_coordinates] - numpy => shape[(10, 1, 4)] data[float32] - -Found 10 faces: -Age: 36 ; Gender: male ; Emotion: happy ; Original image coordinate: [0.5524015 0.30247137 0.5912314 0.39167264] -Age: 24 ; Gender: male ; Emotion: neutral ; Original image coordinate: [0.33651015 0.6238419 0.38452235 0.7109271 ] -Age: 22 ; Gender: male ; Emotion: happy ; Original image coordinate: [0.2273316 0.34603435 0.26951137 0.44243896] -Age: 46 ; Gender: male ; Emotion: happy ; Original image coordinate: [0.44218776 0.29372302 0.48234263 0.37782392] -Age: 41 ; Gender: female ; Emotion: happy ; Original image coordinate: [0.51707435 0.5602793 0.56729674 0.65029866] -Age: 26 ; Gender: male ; Emotion: happy ; Original image coordinate: [0.798531 0.35693872 0.8411773 0.42878723] -Age: 26 ; Gender: female ; Emotion: happy ; Original image coordinate: [0.6506455 0.5917437 0.69963944 0.6924231 ] -Age: 30 ; Gender: male ; Emotion: happy ; Original image coordinate: [0.67255247 0.3240037 0.72031224 0.4141158 ] -Age: 22 ; Gender: female ; Emotion: happy ; Original image coordinate: [0.13879135 0.36219013 0.18784639 0.4526276 ] -Age: 22 ; Gender: female ; Emotion: happy ; Original image coordinate: [0.33016038 0.37727404 0.36954236 0.44583446] -``` - -With additional parameter `--face_images_save_path` the client script saves all detected text images to jpeg files into directory path to confirm -if the image was analyzed correctly. diff --git a/demos/multi_faces_analysis_pipeline/python/config.json b/demos/multi_faces_analysis_pipeline/python/config.json deleted file mode 100644 index 09aa78a61d..0000000000 --- a/demos/multi_faces_analysis_pipeline/python/config.json +++ /dev/null @@ -1,124 +0,0 @@ -{ - "model_config_list": [ - {"config": { - "name": "face_detection", - "base_path": "/workspace/face-detection-retail-0004/", - "shape": "(1,400,600,3)", - "layout": "NHWC:NCHW"}}, - {"config": { - "name": "age_gender_recognition", - "base_path": "/workspace/age-gender-recognition-retail-0013/", - "shape": "(1,64,64,3)", - "layout": "NHWC:NCHW"}}, - {"config": { - "name": "emotion_recognition", - "base_path": "/workspace/emotion-recognition-retail-0003/", - "shape": "(1,64,64,3)", - "layout": "NHWC:NCHW"}} - ], - "custom_node_library_config_list": [ - {"name": "object_detection_image_extractor", - "base_path": "/workspace/lib/libcustom_node_model_zoo_intel_object_detection.so"} - ], - "pipeline_config_list": [ - { - "name": "find_face_images", - "inputs": [ - "image" - ], - "nodes": [ - { - "name": "face_detection_node", - "model_name": "face_detection", - "type": "DL model", - "inputs": [ - {"data": { - "node_name": "request", - "data_item": "image"}}], - "outputs": [ - {"data_item": "detection_out", - "alias": "detection"}] - }, - { - "name": "extract_node", - "library_name": "object_detection_image_extractor", - "type": "custom", - "demultiply_count": 0, - "params": { - "original_image_width": "600", - "original_image_height": "400", - "target_image_width": "64", - "target_image_height": "64", - "original_image_layout": "NHWC", - "target_image_layout": "NHWC", - "convert_to_gray_scale": "false", - "max_output_batch": "100", - "confidence_threshold": "0.7", - "debug": "true", - "buffer_queue_size": "24" - }, - "inputs": [ - {"image": { - "node_name": "request", - "data_item": "image"}}, - {"detection": { - "node_name": "face_detection_node", - "data_item": "detection"}}], - "outputs": [ - {"data_item": "images", - "alias": "face_images"}, - {"data_item": "coordinates", - "alias": "face_coordinates"}, - {"data_item": "confidences", - "alias": "confidence_levels"}] - }, - { - "name": "age_gender_recognition_node", - "model_name": "age_gender_recognition", - "type": "DL model", - "inputs": [ - {"data": { - "node_name": "extract_node", - "data_item": "face_images"}}], - "outputs": [ - {"data_item": "age_conv3", - "alias": "age"}, - {"data_item": "prob", - "alias": "gender"}] - }, - { - "name": "emotion_recognition_node", - "model_name": "emotion_recognition", - "type": "DL model", - "inputs": [ - {"data": { - "node_name": "extract_node", - "data_item": "face_images"}}], - "outputs": [ - {"data_item": "prob_emotion", - "alias": "emotion"}] - } - ], - "outputs": [ - {"face_images": { - "node_name": "extract_node", - "data_item": "face_images"}}, - {"face_coordinates": { - "node_name": "extract_node", - "data_item": "face_coordinates"}}, - {"confidence_levels": { - "node_name": "extract_node", - "data_item": "confidence_levels"}}, - {"ages": { - "node_name": "age_gender_recognition_node", - "data_item": "age"}}, - {"genders": { - "node_name": "age_gender_recognition_node", - "data_item": "gender"}}, - {"emotions": { - "node_name": "emotion_recognition_node", - "data_item": "emotion"}} - ] - } - ] -} diff --git a/demos/multi_faces_analysis_pipeline/python/multi_faces_analysis.png b/demos/multi_faces_analysis_pipeline/python/multi_faces_analysis.png deleted file mode 100755 index 68b8b708b8..0000000000 Binary files a/demos/multi_faces_analysis_pipeline/python/multi_faces_analysis.png and /dev/null differ diff --git a/demos/multi_faces_analysis_pipeline/python/multi_faces_analysis_graph.svg b/demos/multi_faces_analysis_pipeline/python/multi_faces_analysis_graph.svg deleted file mode 100755 index a667329673..0000000000 --- a/demos/multi_faces_analysis_pipeline/python/multi_faces_analysis_graph.svg +++ /dev/null @@ -1,3 +0,0 @@ - - -
Request
inputs: image
Request...
Model: face-detection
inputs: image
outputs: detection_out
Model: face-detection...
Custom Node: model_zoo_intel_object_detection
inputs: image, detection_out
outputs: face_images, face coordinates, confidence_levels
Custom Node: model_zoo_intel_object_detection...
Demultiplexer
Demultiplexer
Model: age-gender-recognition
inputs: face_image
outputs: age, gender
Model: age-gender-recognition...
Model: emotion-recognition
inputs: face_image
outputs: emotion
Model: emotion-recognition...
Response
outputs: face_images, face_coordinates, confidence_levels, ages, genders, emotion
Response...
Gatherer
GathererViewer does not support full SVG 1.1 \ No newline at end of file diff --git a/demos/multi_faces_analysis_pipeline/python/multi_faces_analysis_pipeline.py b/demos/multi_faces_analysis_pipeline/python/multi_faces_analysis_pipeline.py deleted file mode 100644 index 6dc9d8fc56..0000000000 --- a/demos/multi_faces_analysis_pipeline/python/multi_faces_analysis_pipeline.py +++ /dev/null @@ -1,160 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import grpc -import cv2 -import os -import numpy as np -from tensorflow import make_tensor_proto, make_ndarray -import argparse -from tensorflow_serving.apis import predict_pb2 -from tensorflow_serving.apis import prediction_service_pb2_grpc -parser = argparse.ArgumentParser(description='Client for detailed multi faces analysis pipeline') -parser.add_argument('--grpc_address', required=False, default='localhost', help='Specify url to grpc service. default:localhost') -parser.add_argument('--grpc_port', required=False, default=9178, help='Specify port to grpc service. default: 9178') -parser.add_argument('--pipeline_name', required=False, default='ocr', help='Pipeline name to request. default: ocr') -parser.add_argument('--image_input_name', required=False, default='image', help='Pipeline input name for input with image with faces. default: image') -parser.add_argument('--image_input_path', required=True, help='Input image path.') -parser.add_argument('--face_images_output_name', required=False, default='face_images', help='Pipeline output name for cropped images with faces. default: face_images') -parser.add_argument('--face_images_save_path', required=False, default='', help='If specified, face images will be saved to disk.') -parser.add_argument('--image_width', required=False, default=600, help='Pipeline input image width. default: 600') -parser.add_argument('--image_height', required=False, default=400, help='Pipeline input image height. default: 400') -parser.add_argument('--input_image_layout', required=False, default='NHWC', choices=['NCHW', 'NHWC', 'BINARY'], help='Pipeline input image layout. default: NHWC') - -args = vars(parser.parse_args()) - -def prepare_img_input_in_nchw_format(request, name, path, resize_to_shape): - img = cv2.imread(path).astype(np.float32) # BGR color format, shape HWC - img = cv2.resize(img, (resize_to_shape[1], resize_to_shape[0])) - target_shape = (img.shape[0], img.shape[1]) - img = img.transpose(2,0,1).reshape(1,3,target_shape[0],target_shape[1]) - request.inputs[name].CopyFrom(make_tensor_proto(img, shape=img.shape)) - -def prepare_img_input_in_nhwc_format(request, name, path, resize_to_shape): - img = cv2.imread(path).astype(np.float32) # BGR color format, shape HWC - img = cv2.resize(img, (resize_to_shape[1], resize_to_shape[0])) - target_shape = (img.shape[0], img.shape[1]) - img = img.reshape(1,target_shape[0],target_shape[1],3) - request.inputs[name].CopyFrom(make_tensor_proto(img, shape=img.shape)) - -def prepare_img_input_in_binary_format(request, name, path): - with open(path, 'rb') as f: - data = f.read() - request.inputs[name].CopyFrom(make_tensor_proto(data, shape=[1])) - -def save_face_images_as_jpgs(output_nd, name, location): - for i in range(output_nd.shape[0]): - out = output_nd[i][0] - if len(out.shape) == 3 and out.shape[0] == 3: # NCHW - out = out.transpose(1,2,0) - cv2.imwrite(os.path.join(location, name + '_' + str(i) + '.jpg'), out) - -def update_people_ages(output_nd, people): - for i in range(output_nd.shape[0]): - age = int(output_nd[i,0,0,0,0] * 100) - if len(people) < i + 1: - people.append({'age': age}) - else: - people[i].update({'age': age}) - return people - -def update_people_genders(output_nd, people): - for i in range(output_nd.shape[0]): - gender = 'male' if output_nd[i,0,0,0,0] < output_nd[i,0,1,0,0] else 'female' - if len(people) < i + 1: - people.append({'gender': gender}) - else: - people[i].update({'gender': gender}) - return people - -def update_people_emotions(output_nd, people): - emotion_names = { - 0: 'neutral', - 1: 'happy', - 2: 'sad', - 3: 'surprised', - 4: 'angry' - } - for i in range(output_nd.shape[0]): - emotion_id = np.argmax(output_nd[i,0,:,0,0]) - emotion = emotion_names[emotion_id] - if len(people) < i + 1: - people.append({'emotion': emotion}) - else: - people[i].update({'emotion': emotion}) - return people - -def update_people_coordinate(output_nd, people): - for i in range(output_nd.shape[0]): - if len(people) < i + 1: - people.append({'coordinate': output_nd[i,0,:]}) - else: - people[i].update({'coordinate': output_nd[i,0,:]}) - return people - - -address = "{}:{}".format(args['grpc_address'],args['grpc_port']) -MAX_MESSAGE_LENGTH = 1024 * 1024 * 1024 -channel = grpc.insecure_channel(address, - options=[ - ('grpc.max_send_message_length', MAX_MESSAGE_LENGTH), - ('grpc.max_receive_message_length', MAX_MESSAGE_LENGTH), - ]) - -stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) -request = predict_pb2.PredictRequest() -request.model_spec.name = args['pipeline_name'] - -if args['input_image_layout'] == 'NCHW': - prepare_img_input_in_nchw_format(request, args['image_input_name'], args['image_input_path'], (int(args['image_height']), int(args['image_width']))) -elif args['input_image_layout'] == 'NHWC': - prepare_img_input_in_nhwc_format(request, args['image_input_name'], args['image_input_path'], (int(args['image_height']), int(args['image_width']))) -else: - prepare_img_input_in_binary_format(request, args['image_input_name'], args['image_input_path']) - -try: - response = stub.Predict(request, 30.0) -except grpc.RpcError as err: - if err.code() == grpc.StatusCode.ABORTED: - print('No face has been found in the image') - exit(1) - else: - raise err - -people = [] - -for name in response.outputs: - print(f"Output: name[{name}]") - tensor_proto = response.outputs[name] - output_nd = make_ndarray(tensor_proto) - print(f" numpy => shape[{output_nd.shape}] data[{output_nd.dtype}]") - - if name == args['face_images_output_name'] and len(args['face_images_save_path']) > 0: - save_face_images_as_jpgs(output_nd, name, args['face_images_save_path']) - - if name == 'ages': - people = update_people_ages(output_nd, people) - if name == 'genders': - people = update_people_genders(output_nd, people) - if name == 'emotions': - people = update_people_emotions(output_nd, people) - if name == 'face_coordinates': - people = update_people_coordinate(output_nd, people) - - -print('\nFound', len(people), 'faces:') -for person in people: - print('Age:', person['age'], '; Gender:', person['gender'], '; Emotion:', person['emotion'], '; Original image coordinate:', person['coordinate']) diff --git a/demos/multi_faces_analysis_pipeline/python/requirements.txt b/demos/multi_faces_analysis_pipeline/python/requirements.txt deleted file mode 100644 index fa28f0f955..0000000000 --- a/demos/multi_faces_analysis_pipeline/python/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --r ../../common/python/requirements.txt \ No newline at end of file diff --git a/demos/object_detection/python/object_detection.py b/demos/object_detection/python/object_detection.py index eaec4878d1..445f6a9627 100644 --- a/demos/object_detection/python/object_detection.py +++ b/demos/object_detection/python/object_detection.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from ovmsclient import make_grpc_client +import tritonclient.grpc as grpcclient import cv2 import numpy as np import argparse @@ -29,7 +29,7 @@ help='Model name to query. default: faster_rcnn') parser.add_argument('--input_name', default='input_tensor', help='Input name to query. default: input_tensor') -parser.add_argument('--model_version', default=0, type=int, +parser.add_argument('--model_version', default="0", type=str, help='Model version to query. default: latest available') parser.add_argument('--labels', default="coco_91cl.txt", type=str, help='Path to COCO dataset labels with human readable class names') @@ -39,14 +39,16 @@ image = cv2.imread(filename=str(args.image)) image = cv2.cvtColor(image, code=cv2.COLOR_BGR2RGB) resized_image = cv2.resize(src=image, dsize=(255, 255)) -network_input_image = np.expand_dims(resized_image, 0) +network_input_image = np.expand_dims(resized_image, 0).astype(np.uint8) -client = make_grpc_client(args.service_url) -inputs = { - args.input_name: network_input_image -} +client = grpcclient.InferenceServerClient(url=args.service_url) +infer_input = grpcclient.InferInput(args.input_name, network_input_image.shape, "UINT8") +infer_input.set_data_from_numpy(network_input_image) +result = client.infer(args.model_name, [infer_input], model_version=args.model_version) -response = client.predict(inputs, args.model_name, args.model_version) +response = {} +for output in result.get_response().outputs: + response[output.name] = result.as_numpy(output.name) def add_detection_box(box, image, label): """ diff --git a/demos/object_detection/python/requirements.txt b/demos/object_detection/python/requirements.txt index 4ea3efbad5..4e6ab2d092 100644 --- a/demos/object_detection/python/requirements.txt +++ b/demos/object_detection/python/requirements.txt @@ -1,3 +1,3 @@ -ovmsclient +tritonclient[grpc] opencv-python-headless==4.8.1.78 numpy<2.0.0 \ No newline at end of file diff --git a/demos/optical_character_recognition/python/README.md b/demos/optical_character_recognition/python/README.md deleted file mode 100644 index fb410488ef..0000000000 --- a/demos/optical_character_recognition/python/README.md +++ /dev/null @@ -1,212 +0,0 @@ -# Optical Character Recognition with Directed Acyclic Graph {#ovms_demo_optical_character_recognition} - -This document demonstrates how to create and use an Optical Character Recognition (OCR) pipeline based on [east-resnet50](https://github.com/argman/EAST) text detection model, -[text-recognition](https://github.com/openvinotoolkit/open_model_zoo/tree/2023.0.0/models/intel/text-recognition-0012) combined with a custom node implementation. - -Using such a pipeline, a single request to OVMS can perform a complex set of operations with a response containing -recognized characters for all detected text boxes. - -## OCR Graph - -Below is depicted the graph implementing complete OCR pipelines. - -![OCR graph](east_ocr.png) - -It includes the following nodes: - -- Model east-resnet50 - inference execution which takes the user image as input. It returns two outputs including information about all detected boxes, their location and scores. -- Custom node east_ocr - it includes C++ implementation of east-resnet50 model results processing. It analyses the detected boxes coordinates, filters the results -based on the configurable score level threshold and and applies non-max suppression algorithm to remove overlapping boxes. Finally the custom node east-ocr crops all detected boxes -from the original image, resize them to the target resolution and combines into a single output of a dynamic batch size. The output batch size is determined by the number of detected -boxes according to the configured criteria. All operations on the images employ OpenCV libraries which are preinstalled in the OVMS. Learn more about the [east_ocr custom node](https://github.com/openvinotoolkit/model_server/tree/main/src/custom_nodes/east_ocr) -- demultiplexer - output from the Custom node east_ocr have variable batch size. In order to match it with the sequential text detection model, the data is split into individual images with batch size 1 each. -Such smaller requests can be submitted for inference in parallel to the next Model Node. Learn more about the [demultiplexing](../../../docs/demultiplexing.md) -- Model text-recognition - this model recognizes characters included in the input image. -- Response - the output of the whole pipeline combines the recognized `image_texts` with their metadata. -The metadata are the `text_coordinates` and the `confidence_level` outputs. - -## Preparing the Models - -### East-resnet50 model - -The original pretrained model for east-resnet50 topology is stored on https://github.com/argman/EAST in TensorFlow checkpoint format. - -Clone GitHub repository: -```bash -git clone https://github.com/argman/EAST -cd EAST -``` -Download and unzip the file east_icdar2015_resnet_v1_50_rbox.zip as instructed in the Readme.md file to EAST folder with the GitHub repository. -```bash -unzip ./east_icdar2015_resnet_v1_50_rbox.zip -``` -Inside the EAST folder add a file `freeze_east_model.py` by executing the following `echo` command from the command line: -```bash -echo "from tensorflow.python.framework import graph_util -import tensorflow as tf -import model - -def export_model(input_checkpoint, output_graph): - with tf.get_default_graph().as_default(): - input_images = tf.placeholder(tf.float32, shape=[None, None, None, 3], name='input_images') - global_step = tf.get_variable('global_step', [], initializer=tf.constant_initializer(0), trainable=False) - f_score, f_geometry = model.model(input_images, is_training=False) - graph = tf.get_default_graph() - input_graph_def = graph.as_graph_def() - - init_op = tf.global_variables_initializer() - saver = tf.train.Saver() - with tf.Session() as sess: - sess.run(init_op) - saver.restore(sess, input_checkpoint) - output_graph_def = graph_util.convert_variables_to_constants(sess=sess, input_graph_def=input_graph_def, output_node_names=['feature_fusion/concat_3','feature_fusion/Conv_7/Sigmoid']) - with tf.gfile.GFile(output_graph, 'wb') as f: - f.write(output_graph_def.SerializeToString()) - -export_model('./east_icdar2015_resnet_v1_50_rbox/model.ckpt-49491','./model.pb')" >> freeze_east_model.py -cd .. -``` -Freeze the model in checkpoint format and save it in proto buffer format in `model.pb`: - -```bash -docker run -u $(id -u):$(id -g) -v ${PWD}/EAST/:/EAST:rw -w /EAST tensorflow/tensorflow:1.15.5 python3 freeze_east_model.py -``` - -Convert the TensorFlow frozen model to Intermediate Representation format using the model_optimizer tool: -```bash -docker run -u $(id -u):$(id -g) -v ${PWD}/EAST/:/EAST:rw openvino/ubuntu20_dev:2022.1.0 mo \ ---framework=tf --input_shape=[1,1024,1920,3] --input=input_images --output=feature_fusion/Conv_7/Sigmoid,feature_fusion/concat_3 \ ---input_model /EAST/model.pb --output_dir /EAST/IR/1/ -``` -It will create model files in `${PWD}/EAST/IR/1/` folder. -```bash -EAST/IR/1/ -├── model.bin -├── model.mapping -└── model.xml -``` -Converted east-resnet50 model will have the following interface: -- Input name: `input_images` ; shape: `[1 1024 1920 3]` ; precision: `FP32` ; layout: `N...` -- Output name: `feature_fusion/Conv_7/Sigmoid` ; shape: `[1 256 480 1]` ; precision: `FP32` ; layout: `N...` -- Output name: `feature_fusion/concat_3` ; shape: `[1 256 480 5]` ; precision: `FP32`; layout: `N...` - -### Text-recognition model -Download [text-recognition](https://github.com/openvinotoolkit/open_model_zoo/tree/2023.0.0/models/intel/text-recognition-0012) model and store it in `${PWD}/text-recognition/1` folder. -```bash -curl -L --create-dirs https://storage.openvinotoolkit.org/repositories/open_model_zoo/2023.0/models_bin/1/text-recognition-0012/FP32/text-recognition-0012.bin -o text-recognition/1/model.bin https://storage.openvinotoolkit.org/repositories/open_model_zoo/2023.0/models_bin/1/text-recognition-0012/FP32/text-recognition-0012.xml -o text-recognition/1/model.xml -chmod -R 755 text-recognition -``` - -text-recognition model will have the following interface: -- Input name: `imgs`; shape: `[1 1 32 128]` ; precision: `FP32`, layout: `N...` -- Output name: `logits` ; shape: `[16 1 37]` ; precision: `FP32` - -## Building the Custom Node "east_ocr" Library - -Custom nodes are loaded into OVMS as dynamic library implementing OVMS API from [custom_node_interface.h](https://github.com/openvinotoolkit/model_server/blob/main/src/custom_node_interface.h). -It can use OpenCV libraries included in OVMS or it could use other third party components. - -The custom node east_ocr can be built inside a docker container via the following procedure: -- go to the directory with custom node examples [src/custom_node](https://github.com/openvinotoolkit/model_server/blob/main/src/custom_nodes) -- run `make` command: - -```bash -git clone https://github.com/openvinotoolkit/model_server.git -cd model_server/src/custom_nodes -# replace to 'redhat` if using UBI base image -export BASE_OS=ubuntu -make NODES=east_ocr BASE_OS=${BASE_OS} -cd ../../../ -``` - -This command will export the compiled library in `./lib` folder. -Copy this `lib` folder to the same location with `text-recognition` and `east_icdar2015_resnet_v1_50_rbox`. - -```bash -mkdir -p OCR/east_fp32 OCR/lib -cp -R model_server/src/custom_nodes/lib/${BASE_OS}/libcustom_node_east_ocr.so OCR/lib/ -cp -R text-recognition OCR/text-recognition -cp -R EAST/IR/1 OCR/east_fp32/1 -``` - -## OVMS Configuration File - -The configuration file for running the OCR demo is stored in [config.json](https://github.com/openvinotoolkit/model_server/blob/main/demos/optical_character_recognition/python/config.json) -Copy this file along with the model files and the custom node library like presented below: -```bash -cp model_server/demos/optical_character_recognition/python/config.json OCR -``` -```bash -OCR -├── config.json -├── east_fp32 -│   └── 1 -│   ├── model.bin -│   └── model.xml -├── lib -│   └── libcustom_node_east_ocr.so -└── text-recognition - └── 1 - ├── model.bin - └── model.xml -``` - -**NOTE:** east_fp32 model created before 2022.1 requires additional parameters in config.json: -- `layout: {"input_images": "NHWC:NCHW", "feature_fusion/Conv_7/Sigmoid": "NHWC:NCHW", "feature_fusion/concat_3": "NHWC:NCHW"}` - -## Deploying OVMS - -Deploy OVMS with OCR demo pipeline using the following command: - -```bash -docker run -p 9000:9000 -d -v ${PWD}/OCR:/OCR openvino/model_server --config_path /OCR/config.json --port 9000 -``` - -## Requesting the Service - -Enter optical_character_recognition directory -```bash -cd model_server/demos/optical_character_recognition/python -``` - -Install python dependencies: -```bash -pip3 install -r requirements.txt -``` - -Now you can create a directory for text images and run the client: -```bash -mkdir results -``` -```bash -python optical_character_recognition.py --grpc_port 9000 --image_input_path demo_images/input.jpg --pipeline_name detect_text_images --text_images_save_path ./results/ --image_layout NHWC -Output: name[confidence_levels] - numpy => shape[(9, 1, 1)] data[float32] -Output: name[texts] - numpy => shape[(9, 16, 1, 37)] data[float32] -performance -gdansk -server -model -openvino -pipeline -2021 -intel -rotation -Output: name[text_images] - numpy => shape[(9, 1, 32, 128, 1)] data[float32] -Output: name[text_coordinates] - numpy => shape[(9, 1, 4)] data[int32] -``` - -With additional parameter `--text_images_save_path` the client script saves all detected text images to jpeg files into directory path to confirm -if the image was analyzed correctly. - -Below is the exemplary input image. -![image](input.jpg) - -The custom node generates the following text images retrieved from the original input to CRNN model: -![image](crnn_table.png) - -## Accuracy -Please note that it is possible to swap the models included in DAG with your own to adjust pipeline accuracy for various scenarios and datasets. diff --git a/demos/optical_character_recognition/python/config.json b/demos/optical_character_recognition/python/config.json deleted file mode 100644 index a80d5876bc..0000000000 --- a/demos/optical_character_recognition/python/config.json +++ /dev/null @@ -1,109 +0,0 @@ -{ - "model_config_list": [ - { - "config": { - "name": "east", - "base_path": "/OCR/east_fp32", - "plugin_config": { - "PERFORMANCE_HINT": "LATENCY" - } - } - }, - { - "config": { - "name": "text-recognition", - "base_path": "/OCR/text-recognition" - } - } - ], - "custom_node_library_config_list": [ - { - "name": "east_image_extractor", - "base_path": "/OCR/lib/libcustom_node_east_ocr.so" - } - ], - "pipeline_config_list": [ - { - "name": "detect_text_images", - "inputs": ["image"], - "nodes": [ - { - "name": "east_node", - "model_name": "east", - "type": "DL model", - "inputs": [ - {"input_images": {"node_name": "request", - "data_item": "image"}} - ], - "outputs": [ - {"data_item": "feature_fusion/Conv_7/Sigmoid", - "alias": "scores"}, - {"data_item": "feature_fusion/concat_3", - "alias": "geometry"} - ] - }, - { - "name": "extract_node", - "library_name": "east_image_extractor", - "type": "custom", - "demultiply_count": 0, - "params": { - "original_image_width": "1920", - "original_image_height": "1024", - "original_image_layout": "NHWC", - "target_image_width": "120", - "target_image_height": "32", - "target_image_layout": "NHWC", - "convert_to_gray_scale": "true", - "confidence_threshold": "0.9", - "overlap_threshold": "0.1", - "max_output_batch": "100", - "box_width_adjustment": "0.1", - "box_height_adjustment": "0.0", - "rotation_angle_threshold": "8", - "debug": "true" - }, - "inputs": [ - {"image": {"node_name": "request", - "data_item": "image"}}, - {"scores": {"node_name": "east_node", - "data_item": "scores"}}, - {"geometry": {"node_name": "east_node", - "data_item": "geometry"}} - ], - "outputs": [ - {"data_item": "text_images", - "alias": "text_images"}, - {"data_item": "text_coordinates", - "alias": "text_coordinates"}, - {"data_item": "confidence_levels", - "alias": "confidence_levels"} - ] - }, - { - "name": "text-recognition_node", - "model_name": "text-recognition", - "type": "DL model", - "inputs": [ - {"Placeholder": {"node_name": "extract_node", - "data_item": "text_images"}} - ], - "outputs": [ - {"data_item": "shadow/LSTMLayers/transpose_time_major", - "alias": "texts"} - ] - } - ], - "outputs": [ - {"text_images": {"node_name": "extract_node", - "data_item": "text_images"}}, - {"text_coordinates": {"node_name": "extract_node", - "data_item": "text_coordinates"}}, - {"confidence_levels": {"node_name": "extract_node", - "data_item": "confidence_levels"}}, - {"texts": {"node_name": "text-recognition_node", - "data_item": "texts"}} - ] - } - ] -} diff --git a/demos/optical_character_recognition/python/crnn_table.png b/demos/optical_character_recognition/python/crnn_table.png deleted file mode 100644 index cfbbe42693..0000000000 Binary files a/demos/optical_character_recognition/python/crnn_table.png and /dev/null differ diff --git a/demos/optical_character_recognition/python/demo_images/input.jpg b/demos/optical_character_recognition/python/demo_images/input.jpg deleted file mode 100755 index e4caaab7e8..0000000000 Binary files a/demos/optical_character_recognition/python/demo_images/input.jpg and /dev/null differ diff --git a/demos/optical_character_recognition/python/demo_images/text_images_0.jpg b/demos/optical_character_recognition/python/demo_images/text_images_0.jpg deleted file mode 100644 index c480db828d..0000000000 Binary files a/demos/optical_character_recognition/python/demo_images/text_images_0.jpg and /dev/null differ diff --git a/demos/optical_character_recognition/python/demo_images/text_images_1.jpg b/demos/optical_character_recognition/python/demo_images/text_images_1.jpg deleted file mode 100644 index e16ee46845..0000000000 Binary files a/demos/optical_character_recognition/python/demo_images/text_images_1.jpg and /dev/null differ diff --git a/demos/optical_character_recognition/python/demo_images/text_images_2.jpg b/demos/optical_character_recognition/python/demo_images/text_images_2.jpg deleted file mode 100644 index 4bfe5c28f5..0000000000 Binary files a/demos/optical_character_recognition/python/demo_images/text_images_2.jpg and /dev/null differ diff --git a/demos/optical_character_recognition/python/demo_images/text_images_3.jpg b/demos/optical_character_recognition/python/demo_images/text_images_3.jpg deleted file mode 100644 index 6038229dd4..0000000000 Binary files a/demos/optical_character_recognition/python/demo_images/text_images_3.jpg and /dev/null differ diff --git a/demos/optical_character_recognition/python/demo_images/text_images_4.jpg b/demos/optical_character_recognition/python/demo_images/text_images_4.jpg deleted file mode 100644 index 1eb45ad826..0000000000 Binary files a/demos/optical_character_recognition/python/demo_images/text_images_4.jpg and /dev/null differ diff --git a/demos/optical_character_recognition/python/demo_images/text_images_5.jpg b/demos/optical_character_recognition/python/demo_images/text_images_5.jpg deleted file mode 100644 index 9c36bfeda2..0000000000 Binary files a/demos/optical_character_recognition/python/demo_images/text_images_5.jpg and /dev/null differ diff --git a/demos/optical_character_recognition/python/demo_images/text_images_6.jpg b/demos/optical_character_recognition/python/demo_images/text_images_6.jpg deleted file mode 100644 index 40e54b5cef..0000000000 Binary files a/demos/optical_character_recognition/python/demo_images/text_images_6.jpg and /dev/null differ diff --git a/demos/optical_character_recognition/python/demo_images/text_images_7.jpg b/demos/optical_character_recognition/python/demo_images/text_images_7.jpg deleted file mode 100644 index e726e85120..0000000000 Binary files a/demos/optical_character_recognition/python/demo_images/text_images_7.jpg and /dev/null differ diff --git a/demos/optical_character_recognition/python/demo_images/text_images_8.jpg b/demos/optical_character_recognition/python/demo_images/text_images_8.jpg deleted file mode 100644 index d212dd3451..0000000000 Binary files a/demos/optical_character_recognition/python/demo_images/text_images_8.jpg and /dev/null differ diff --git a/demos/optical_character_recognition/python/east_ocr.png b/demos/optical_character_recognition/python/east_ocr.png deleted file mode 100644 index e3a63a1e52..0000000000 Binary files a/demos/optical_character_recognition/python/east_ocr.png and /dev/null differ diff --git a/demos/optical_character_recognition/python/input.jpg b/demos/optical_character_recognition/python/input.jpg deleted file mode 100644 index f3f6962a25..0000000000 Binary files a/demos/optical_character_recognition/python/input.jpg and /dev/null differ diff --git a/demos/optical_character_recognition/python/optical_character_recognition.py b/demos/optical_character_recognition/python/optical_character_recognition.py deleted file mode 100644 index b6e3eddec6..0000000000 --- a/demos/optical_character_recognition/python/optical_character_recognition.py +++ /dev/null @@ -1,127 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import grpc -import cv2 -import os -import numpy as np -from tensorflow import make_tensor_proto, make_ndarray -import argparse -from tensorflow_serving.apis import predict_pb2 -from tensorflow_serving.apis import prediction_service_pb2_grpc - -parser = argparse.ArgumentParser(description='Client for OCR pipeline') -parser.add_argument('--grpc_address', required=False, default='localhost', help='Specify url to grpc service. default:localhost') -parser.add_argument('--grpc_port', required=False, default=9178, help='Specify port to grpc service. default: 9178') -parser.add_argument('--pipeline_name', required=False, default='ocr', help='Pipeline name to request. default: ocr') -parser.add_argument('--image_input_name', required=False, default='image', help='Pipeline input name for input with image. default: image') -parser.add_argument('--image_input_path', required=True, help='Input image path.') -parser.add_argument('--texts_output_name', required=False, default='texts', help='Pipeline output name for output with recognized texts. default: texts') -parser.add_argument('--text_images_output_name', required=False, default='text_images', help='Pipeline output name for cropped images with text. default: text_images') -parser.add_argument('--text_images_save_path', required=False, default='', help='If specified, images will be saved to disk.') -parser.add_argument('--image_width', required=False, default=1920, help='Original image width. default: 1920') -parser.add_argument('--image_height', required=False, default=1024, help='Original image height. default: 1024') -parser.add_argument('--image_layout', required=False, default='NCHW', choices=['NCHW', 'NHWC', 'BINARY'], help='Pipeline input image layout. default: NCHW') - -args = vars(parser.parse_args()) - -def prepare_img_input_in_nchw_format(request, name, path, resize_to_shape): - img = cv2.imread(path).astype(np.float32) # BGR color format, shape HWC - img = cv2.resize(img, (resize_to_shape[1], resize_to_shape[0])) - target_shape = (img.shape[0], img.shape[1]) - img = img.transpose(2,0,1).reshape(1,3,target_shape[0],target_shape[1]) - request.inputs[name].CopyFrom(make_tensor_proto(img, shape=img.shape)) - -def prepare_img_input_in_nhwc_format(request, name, path, resize_to_shape): - img = cv2.imread(path).astype(np.float32) # BGR color format, shape HWC - img = cv2.resize(img, (resize_to_shape[1], resize_to_shape[0])) - target_shape = (img.shape[0], img.shape[1]) - img = img.reshape(1,target_shape[0],target_shape[1],3) - request.inputs[name].CopyFrom(make_tensor_proto(img, shape=img.shape)) - -def prepare_img_input_in_binary_format(request, name, path): - with open(path, 'rb') as f: - data = f.read() - request.inputs[name].CopyFrom(make_tensor_proto(data, shape=[1])) - -def save_text_images_as_jpgs(output_nd, name, location): - for i in range(output_nd.shape[0]): - out = output_nd[i][0] - if len(out.shape) == 3 and out.shape[0] == 3: # NCHW - out = out.transpose(1,2,0) - cv2.imwrite(os.path.join(location, name + '_' + str(i) + '.jpg'), out) - -def decode(text): - word = '' - last_character = None - for character in text: - if character == last_character: - continue - elif character == '#': - last_character = None - else: - last_character = character - word += character - return word - -def text_recognition_output_to_text(output_nd): - for i in range(output_nd.shape[0]): - data = output_nd[i] - alphabet = '0123456789abcdefghijklmnopqrstuvwxyz#' - preds = data.argmax(2) - word = '' - for i in range(preds.shape[0]): - word += alphabet[preds[i,0]] - print(decode(word)) - - -address = "{}:{}".format(args['grpc_address'],args['grpc_port']) -MAX_MESSAGE_LENGTH = 1024 * 1024 * 1024 -channel = grpc.insecure_channel(address, - options=[ - ('grpc.max_send_message_length', MAX_MESSAGE_LENGTH), - ('grpc.max_receive_message_length', MAX_MESSAGE_LENGTH), - ]) - -stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) -request = predict_pb2.PredictRequest() -request.model_spec.name = args['pipeline_name'] - -if args['image_layout'] == 'NCHW': - prepare_img_input_in_nchw_format(request, args['image_input_name'], args['image_input_path'], (int(args['image_height']), int(args['image_width']))) -elif args['image_layout'] == 'NHWC': - prepare_img_input_in_nhwc_format(request, args['image_input_name'], args['image_input_path'], (int(args['image_height']), int(args['image_width']))) -else: - prepare_img_input_in_binary_format(request, args['image_input_name'], args['image_input_path']) - -try: - response = stub.Predict(request, 30.0) -except grpc.RpcError as err: - if err.code() == grpc.StatusCode.ABORTED: - print('No text has been found in the image') - exit(1) - else: - raise err - -for name in response.outputs: - print(f"Output: name[{name}]") - tensor_proto = response.outputs[name] - output_nd = make_ndarray(tensor_proto) - print(f" numpy => shape[{output_nd.shape}] data[{output_nd.dtype}]") - if name == args['text_images_output_name'] and len(args['text_images_save_path']) > 0: - save_text_images_as_jpgs(output_nd, name, args['text_images_save_path']) - if name == args['texts_output_name']: - text_recognition_output_to_text(output_nd) \ No newline at end of file diff --git a/demos/optical_character_recognition/python/requirements.txt b/demos/optical_character_recognition/python/requirements.txt deleted file mode 100644 index fa28f0f955..0000000000 --- a/demos/optical_character_recognition/python/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --r ../../common/python/requirements.txt \ No newline at end of file diff --git a/demos/person_vehicle_bike_detection/python/person_vehicle_bike_detection.py b/demos/person_vehicle_bike_detection/python/person_vehicle_bike_detection.py index fce4671adc..5dc06a9ea8 100644 --- a/demos/person_vehicle_bike_detection/python/person_vehicle_bike_detection.py +++ b/demos/person_vehicle_bike_detection/python/person_vehicle_bike_detection.py @@ -27,17 +27,15 @@ from __future__ import print_function from argparse import ArgumentParser, SUPPRESS -from tensorflow_serving.apis import predict_pb2 -from tensorflow_serving.apis import prediction_service_pb2_grpc from time import time, sleep import sys import os import cv2 -import grpc import threading import logging as log -from tensorflow import make_tensor_proto, make_ndarray +import numpy as np +import tritonclient.grpc as grpcclient # global data (shared between threads & main) CLASSES = ["None", "Pedestrian", "Vehicle", "Bike", "Other"] @@ -140,12 +138,7 @@ def thread_function(thr_id, network_name, input_layer, output_layer, input_dimen cam_start_time = time() # ovms connection - channel = grpc.insecure_channel("{}:{}".format(ip, port)) - stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) - - request = predict_pb2.PredictRequest() - # Note: Pls maintain the same name while launching ovms docker container - request.model_spec.name = network_name + client = grpcclient.InferenceServerClient(url="{}:{}".format(ip, port)) global exit_ok while exit_ok == False: @@ -179,20 +172,18 @@ def thread_function(thr_id, network_name, input_layer, output_layer, input_dimen image = image.astype('float32') inf_time = time() - # send the input as protobuf - request.inputs[input_layer].CopyFrom( - make_tensor_proto(image, shape=None)) + infer_input = grpcclient.InferInput(input_layer, image.shape, "FP32") + infer_input.set_data_from_numpy(image) try: - result = stub.Predict(request, 10.0) + result = client.infer(network_name, [infer_input]) except Exception as e: log.error('Caught exception {}'.format(e)) cam.release() return duration = time() - inf_time - # decode the received output as protobuf - res = make_ndarray(result.outputs[output_layer]) + res = result.as_numpy(output_layer) if not res.any(): log.error('Thr{}: Predictions came back with wrong output layer name'.format(thr_id)) diff --git a/demos/single_face_analysis_pipeline/python/README.md b/demos/single_face_analysis_pipeline/python/README.md deleted file mode 100644 index 30f31b4272..0000000000 --- a/demos/single_face_analysis_pipeline/python/README.md +++ /dev/null @@ -1,95 +0,0 @@ -# Single Face Analysis Pipeline Demo {#ovms_demo_single_face_analysis_pipeline} - -This document presents a models ensemble as an example of [DAG Scheduler](../../../docs/dag_scheduler.md) implementation. -It describes how to combine several models to perform multiple inference operations with a single prediction call. -When you need to execute several predictions on the same data, you can create a pipeline, which combines the results from several models. - -![diagram](single_face_analysis_pipeline.png) - -## Prerequisites - -**Model preparation**: Python 3.9 or higher with pip - -**Model Server deployment**: Installed Docker Engine or OVMS binary package according to the [baremetal deployment guide](../../../docs/deploying_server_baremetal.md) - -## Prepare models - -In this example the following models are used: - -[age-gender-recognition-retail-0013](https://github.com/openvinotoolkit/open_model_zoo/blob/2022.1.0/models/intel/age-gender-recognition-retail-0013/README.md) - -[emotions-recognition-retail-0003](https://github.com/openvinotoolkit/open_model_zoo/blob/2022.1.0/models/intel/emotions-recognition-retail-0003/README.md) - -Clone the repository and enter single_face_analysis_pipeline directory -```console -git clone https://github.com/openvinotoolkit/model_server.git -cd model_server/demos/single_face_analysis_pipeline/python -``` - -You can prepare the workspace that contains all the above by just running - - -You can prepare the workspace that contains all the above by running - -```console - curl --create-dirs https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013.xml -o workspace/age-gender-recognition-retail-0013/1/age-gender-recognition-retail-0013.xml - curl --create-dirs https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013.bin -o workspace/age-gender-recognition-retail-0013/1/age-gender-recognition-retail-0013.bin - curl --create-dirs https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/emotions-recognition-retail-0003/FP32/emotions-recognition-retail-0003.xml -o workspace/emotions-recognition-retail-0003/1/emotions-recognition-retail-0003.xml - curl --create-dirs https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/emotions-recognition-retail-0003/FP32/emotions-recognition-retail-0003.bin -o workspace/emotions-recognition-retail-0003/1/emotions-recognition-retail-0003.bin -``` - -### Final directory structure - -You should have `workspace` directory ready with the following content. -```console -workspace/ -├── age-gender-recognition-retail-0013 -│   └── 1 -│   ├── age-gender-recognition-retail-0013.bin -│   └── age-gender-recognition-retail-0013.xml -└── emotions-recognition-retail-0003 - └── 1 - ├── emotions-recognition-retail-0003.bin - └── emotions-recognition-retail-0003.xml -``` - -## Server Deployment -:::{dropdown} **Deploying with Docker** -```bash -cp config.json workspace/. -chmod -R 755 workspace -docker run -p 9000:9000 -d -v ${PWD}/workspace:/workspace openvino/model_server --config_path /workspace/config.json --port 9000 -``` -::: -:::{dropdown} **Deploying on Bare Metal** -Assuming you have unpacked model server package, make sure to: - -- **On Windows**: run `setupvars` script -- **On Linux**: set `LD_LIBRARY_PATH` and `PATH` environment variables - -as mentioned in [deployment guide](../../../docs/deploying_server_baremetal.md), in every new shell that will start OpenVINO Model Server. -```bat -copy config.json workspace -ovms --config_path workspace/config.json --port 9001 -``` -::: -## Requesting the Service - -Exemplary client [single_face_analysis_pipeline.py](https://github.com/openvinotoolkit/model_server/blob/main/demos/single_face_analysis_pipeline/python/single_face_analysis_pipeline.py) can be used to request pipeline deployed in previous step. - -```console -pip3 install -r requirements.txt -``` - -Now you can create directory for text images and run the client: - -```console -python single_face_analysis_pipeline.py --image_path ../../common/static/images/faces/face1.jpg --grpc_port 9000 -Age results: [[[21.099792]]] -Gender results: Female: 0.9483401 ; Male: 0.051659837 -Emotion results: Natural: 0.02335789 ; Happy: 0.9449672 ; Sad: 0.001236845 ; Surprise: 0.028111042 ; Angry: 0.0023269346 -``` - -### Next step - -For more advanced use case with extracting and analysing multiple faces on the same image see [multi_faces_analysis_pipeline](../../multi_faces_analysis_pipeline/python/README.md) demo. \ No newline at end of file diff --git a/demos/single_face_analysis_pipeline/python/config.json b/demos/single_face_analysis_pipeline/python/config.json deleted file mode 100644 index ba60014681..0000000000 --- a/demos/single_face_analysis_pipeline/python/config.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "model_config_list": [ - { - "config": { - "name": "emotions-recognition", - "base_path": "emotions-recognition-retail-0003" - } - }, - { - "config": { - "name": "age-gender-recognition", - "base_path": "age-gender-recognition-retail-0013", - "shape": "(1, 3, 64, 64)" - } - } - ], - "pipeline_config_list": [ - { - "name": "single_face_analysis", - "inputs": ["image"], - "nodes": [ - { - "name": "emotions", - "model_name": "emotions-recognition", - "type": "DL model", - "inputs": [ - {"data": {"node_name": "request", - "data_item": "image"}} - ], - "outputs": [ - {"data_item": "prob_emotion", - "alias": "emotion"} - ] - }, - { - "name": "age-gender", - "model_name": "age-gender-recognition", - "type": "DL model", - "inputs": [ - {"data": {"node_name": "request", - "data_item": "image"}} - ], - "outputs": [ - {"data_item": "age_conv3", - "alias": "age"}, - {"data_item": "prob", - "alias": "gender"} - ] - } - ], - "outputs": [ - {"age": {"node_name": "age-gender", - "data_item": "age"}}, - {"gender": {"node_name": "age-gender", - "data_item": "gender"}}, - {"emotion": {"node_name": "emotions", - "data_item": "emotion"}} - ] - } - ] -} \ No newline at end of file diff --git a/demos/single_face_analysis_pipeline/python/requirements.txt b/demos/single_face_analysis_pipeline/python/requirements.txt deleted file mode 100644 index fa28f0f955..0000000000 --- a/demos/single_face_analysis_pipeline/python/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --r ../../common/python/requirements.txt \ No newline at end of file diff --git a/demos/single_face_analysis_pipeline/python/single_face_analysis_pipeline.png b/demos/single_face_analysis_pipeline/python/single_face_analysis_pipeline.png deleted file mode 100644 index 7b0e32934f..0000000000 Binary files a/demos/single_face_analysis_pipeline/python/single_face_analysis_pipeline.png and /dev/null differ diff --git a/demos/single_face_analysis_pipeline/python/single_face_analysis_pipeline.py b/demos/single_face_analysis_pipeline/python/single_face_analysis_pipeline.py deleted file mode 100644 index b8f1761f2c..0000000000 --- a/demos/single_face_analysis_pipeline/python/single_face_analysis_pipeline.py +++ /dev/null @@ -1,55 +0,0 @@ -# -# Copyright (c) 2022 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import cv2 -from tensorflow import make_tensor_proto, make_ndarray -import grpc -from tensorflow_serving.apis import predict_pb2 -from tensorflow_serving.apis import prediction_service_pb2_grpc -import argparse - -parser = argparse.ArgumentParser(description='Client for single face analysis pipeline') -parser.add_argument('--grpc_address', required=False, default='localhost', help='Specify url to grpc service. default:localhost') -parser.add_argument('--grpc_port', required=False, default=9000, help='Specify port to grpc service. default: 9000') -parser.add_argument('--pipeline_name', required=False, default='single_face_analysis', help='Pipeline name to request. default: single_face_analysis') -parser.add_argument('--image_path', required=True, help='Path to the file with the input image') - -args = vars(parser.parse_args()) - -def getJpeg(path, size): - - img = cv2.imread(path, cv2.IMREAD_COLOR) - # retrieved array has BGR format and 0-255 normalization - img = cv2.resize(img, (size, size)) - img = img.astype('float32') - img = img.transpose(2,0,1).reshape(1,3,size,size) - return img - -my_image = getJpeg(args["image_path"],64) - -channel = grpc.insecure_channel("{}:{}".format(args["grpc_address"], args["grpc_port"])) -stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) -request = predict_pb2.PredictRequest() -request.model_spec.name = args["pipeline_name"] -request.inputs["image"].CopyFrom(make_tensor_proto(my_image, shape=(my_image.shape))) -result = stub.Predict(request, 10.0) -age_results = make_ndarray(result.outputs["age"]) -gender_results = make_ndarray(result.outputs["gender"]) -emotion_results = make_ndarray(result.outputs["emotion"]) - -print("Age results:", age_results[0]*100) -print("Gender results: Female:", gender_results[0,0,0,0], "; Male:", gender_results[0,1,0,0] ) -print("Emotion results: Natural:", emotion_results[0,0,0,0], "; Happy:", emotion_results[0,1,0,0], "; Sad:", emotion_results[0,2,0,0], "; Surprise:", emotion_results[0,3,0,0], "; Angry:", emotion_results[0,4,0,0] ) \ No newline at end of file diff --git a/demos/universal-sentence-encoder/README.md b/demos/universal-sentence-encoder/README.md index ad36dcbcb0..391b17d3f3 100644 --- a/demos/universal-sentence-encoder/README.md +++ b/demos/universal-sentence-encoder/README.md @@ -62,13 +62,15 @@ ovms --model_name usem --model_path universal-sentence-encoder-multilingual/ --p ## Send string data as inference request -OpenVINO Model Server can accept the input in a form of strings. Below is a code snipped based on `tensorflow_serving_api` python library: +OpenVINO Model Server can accept the input in a form of strings. Below is a code snippet based on `tritonclient` python library (KServe gRPC API): ```python -data = np.array(["string1", "string1", "string_n"]) -predict_request = predict_pb2.PredictRequest() -predict_request.model_spec.name = "my_model" -predict_request.inputs["input_name"].CopyFrom(make_tensor_proto(data)) -predict_response = prediction_service_stub.Predict(predict_request, 10.0) +import tritonclient.grpc as grpcclient + +client = grpcclient.InferenceServerClient(url="localhost:9000") +input_data = np.array(["string1", "string2"], dtype=np.object_) +infer_input = grpcclient.InferInput("inputs", [len(input_data)], "BYTES") +infer_input.set_data_from_numpy(input_data) +result = client.infer("my_model", [infer_input]) ``` Clone the repo: @@ -90,35 +92,10 @@ Output subset [-0.00552395 0.00599533 -0.01480555 0.01098945 -0.09355522 -0.08 ``` -The same can be achieved using REST API interface and even a simple `curl` command: +The same can be achieved using REST API interface and even a simple `curl` command (KServe format): ```bash -curl -X POST http://localhost:8000/v1/models/usem:predict \ +curl -X POST http://localhost:8000/v2/models/usem/infer \ -H 'Content-Type: application/json' \ --d '{"instances": ["dog", "Puppies are nice.", "I enjoy taking long walks along the beach with my dog."]}' +-d '{"inputs": [{"name": "inputs", "shape": [3], "datatype": "BYTES", "data": ["dog", "Puppies are nice.", "I enjoy taking long walks along the beach with my dog."]}]}' ``` - - -## Compare results with TFS - -The same client code can be used to send the requests to TensorFlow Serving component. There is full compatibility in the API. - -Start TFS container: -```bash -docker run -it -p 8500:8500 -p 9500:9500 -v $(pwd)/universal-sentence-encoder-multilingual:/models/usem -e MODEL_NAME=usem tensorflow/serving --port=9500 --rest_api_port=8500 -``` - -Run the client -```bash -python model_server/demos/universal-sentence-encoder/send_strings.py --grpc_port 9500 --input_name inputs --output_name outputs --string "I enjoy taking long walks along the beach with my dog." - -processing time 12.167000000000002 ms. -Output shape (1, 512) -Output subset [-0.00552387 0.00599531 -0.0148055 0.01098951 -0.09355522 -0.08445048 - -0.02802679 -0.05219323 -0.06759984 0.03127313 -0.03223493 -0.01282088 - 0.06131843 0.02626882 -0.00983502 0.00298053 0.00141208 0.03229369 - 0.06957125 0.01543701] - -``` - -> NOTE: Do not use this model with `--cache_dir`, the model does not support caching. diff --git a/demos/universal-sentence-encoder/requirements.txt b/demos/universal-sentence-encoder/requirements.txt index 03c70c239e..4d1f6f5c9b 100644 --- a/demos/universal-sentence-encoder/requirements.txt +++ b/demos/universal-sentence-encoder/requirements.txt @@ -1,2 +1,2 @@ -tensorflow-serving-api==2.18.1 -tensorflow==2.18.1 \ No newline at end of file +tritonclient[grpc] +numpy \ No newline at end of file diff --git a/demos/universal-sentence-encoder/send_strings.py b/demos/universal-sentence-encoder/send_strings.py index 5f4de71da3..68ed3bd311 100644 --- a/demos/universal-sentence-encoder/send_strings.py +++ b/demos/universal-sentence-encoder/send_strings.py @@ -14,37 +14,32 @@ # limitations under the License. # -import grpc import numpy as np -from tensorflow import make_tensor_proto, make_ndarray, make_tensor_proto import datetime import argparse -from tensorflow_serving.apis import predict_pb2 -from tensorflow_serving.apis import prediction_service_pb2_grpc +import tritonclient.grpc as grpcclient -parser = argparse.ArgumentParser(description='Do requests to ie_serving and tf_serving using images in string format') -parser.add_argument('--grpc_address',required=False, default='localhost', help='Specify url to grpc service. default:localhost') -parser.add_argument('--grpc_port',required=False, default=9000, help='Specify port to grpc service. default: 9000') -parser.add_argument('--input_name',required=False, default='inputs', help='Specify input tensor name. default: inputs') -parser.add_argument('--output_name',required=False, default='outputs', help='Specify output name. default: outputs') +parser = argparse.ArgumentParser(description='Do requests to OpenVINO Model Server using strings in KServe gRPC format') +parser.add_argument('--grpc_address', required=False, default='localhost', help='Specify url to grpc service. default:localhost') +parser.add_argument('--grpc_port', required=False, default=9000, help='Specify port to grpc service. default: 9000') +parser.add_argument('--input_name', required=False, default='inputs', help='Specify input tensor name. default: inputs') +parser.add_argument('--output_name', required=False, default='outputs', help='Specify output name. default: outputs') parser.add_argument('--model_name', default='usem', help='Define model name, must be same as is in service. default: usem') -parser.add_argument('--string',required=True, default='', help='String to query.') +parser.add_argument('--string', required=True, default='', help='String to query.') args = vars(parser.parse_args()) -channel = grpc.insecure_channel("{}:{}".format(args['grpc_address'],args['grpc_port'])) -stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) +client = grpcclient.InferenceServerClient(url="{}:{}".format(args['grpc_address'], args['grpc_port'])) +input_data = np.array([args['string']], dtype=np.object_) +infer_input = grpcclient.InferInput(args['input_name'], [len(input_data)], "BYTES") +infer_input.set_data_from_numpy(input_data) -data = np.array([args['string']]) -predict_request = predict_pb2.PredictRequest() -predict_request.model_spec.name = args['model_name'] -predict_request.inputs[args['input_name']].CopyFrom(make_tensor_proto(data)) start_time = datetime.datetime.now() -predict_response = stub.Predict(predict_request, 10.0) +result = client.infer(args['model_name'], [infer_input]) end_time = datetime.datetime.now() duration = (end_time - start_time).total_seconds() * 1000 print("processing time", duration, "ms.") -output = make_ndarray(predict_response.outputs[args['output_name']]) +output = result.as_numpy(args['output_name']) print("Output shape", output.shape) print("Output subset", output[0, :20]) diff --git a/demos/using_onnx_model/python/README.md b/demos/using_onnx_model/python/README.md index c78aa1b63c..842caf2fed 100644 --- a/demos/using_onnx_model/python/README.md +++ b/demos/using_onnx_model/python/README.md @@ -46,15 +46,4 @@ Running inference with image: ../../common/static/images/bee.jpeg Class with highest score: 309 Detected class name: bee ``` - -The client can be also run with flag `--send_tensor` which reads encoded input image and sends it with uint8 precision. -```bash -python onnx_model_demo.py --service_url localhost:9001 --send_tensor -``` -Output: -``` -Running inference with image: ../../common/static/images/bee.jpeg -Class with highest score: 309 -Detected class name: bee -``` > **Note:** While adding preprocessing to the model input, shape needs to be set as static. \ No newline at end of file diff --git a/demos/using_onnx_model/python/onnx_model_demo.py b/demos/using_onnx_model/python/onnx_model_demo.py index 341e6a3d97..9295b0bb34 100644 --- a/demos/using_onnx_model/python/onnx_model_demo.py +++ b/demos/using_onnx_model/python/onnx_model_demo.py @@ -20,7 +20,7 @@ import argparse import numpy as np from PIL import Image -from ovmsclient import make_grpc_client +import tritonclient.grpc as grpcclient import classes @@ -35,21 +35,20 @@ def load_image(path): parser = argparse.ArgumentParser(description='Run prediction on ONNX ResNet50 Model') parser.add_argument('--image_path', required=False, default='../../common/static/images/bee.jpeg', help='Path to a file with a JPEG image') parser.add_argument('--service_url',required=False, default='localhost:9001', help='Specify url to grpc service. default:localhost:9001') -parser.add_argument('--send_tensor', action="store_true", required=False, help='Send image after loading it with Pillow') +parser.add_argument('--output_name', required=False, default='gpu_0/softmax_1', help='Output tensor name. default: gpu_0/softmax_1') args = vars(parser.parse_args()) -print(f"Running inference with image: {args['image_path']}") -if args["send_tensor"]: - img = load_image(args["image_path"]) -else: - with open(args["image_path"], "rb") as f: - img = f.read() - input_name = "gpu_0/data_0" -client = make_grpc_client(args["service_url"]) -output = client.predict({input_name: img}, "resnet") +print(f"Running inference with image: {args['image_path']}") +img = load_image(args["image_path"]) + +client = grpcclient.InferenceServerClient(url=args["service_url"]) +infer_input = grpcclient.InferInput(input_name, img.shape, "UINT8") +infer_input.set_data_from_numpy(img) +result = client.infer("resnet", [infer_input]) +output = result.as_numpy(args['output_name']) -max = np.argmax(output) -print(f"Class with highest score: {max}") -print(f"Detected class name: {classes.imagenet_classes[max]}") +max_idx = np.argmax(output) +print(f"Class with highest score: {max_idx}") +print(f"Detected class name: {classes.imagenet_classes[max_idx]}") diff --git a/demos/using_onnx_model/python/requirements.txt b/demos/using_onnx_model/python/requirements.txt index ab578cf53a..52c6d25e40 100644 --- a/demos/using_onnx_model/python/requirements.txt +++ b/demos/using_onnx_model/python/requirements.txt @@ -1,3 +1,3 @@ -ovmsclient +tritonclient[grpc] numpy<2.0.0 pillow==12.2.0 \ No newline at end of file diff --git a/demos/vehicle_analysis_pipeline/python/.gitignore b/demos/vehicle_analysis_pipeline/python/.gitignore deleted file mode 100644 index f1981605f0..0000000000 --- a/demos/vehicle_analysis_pipeline/python/.gitignore +++ /dev/null @@ -1 +0,0 @@ -workspace \ No newline at end of file diff --git a/demos/vehicle_analysis_pipeline/python/Makefile b/demos/vehicle_analysis_pipeline/python/Makefile deleted file mode 100644 index 8783ecdb19..0000000000 --- a/demos/vehicle_analysis_pipeline/python/Makefile +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright (c) 2022 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -.PHONY: setup_repository - -default: setup_repository - -BUILD_CUSTOM_NODE=false -BASE_OS=ubuntu - -setup_repository: -# Download vehicle detection model - mkdir -p workspace/vehicle-detection-0202/1 - curl https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/vehicle-detection-0202/FP32/vehicle-detection-0202.xml -o workspace/vehicle-detection-0202/1/vehicle-detection-0202.xml - curl https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/vehicle-detection-0202/FP32/vehicle-detection-0202.bin -o workspace/vehicle-detection-0202/1/vehicle-detection-0202.bin -# Download vehicle attributes recognition model - mkdir -p workspace/vehicle-attributes-recognition-barrier-0042/1 - curl https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/vehicle-attributes-recognition-barrier-0042/FP32/vehicle-attributes-recognition-barrier-0042.xml -o workspace/vehicle-attributes-recognition-barrier-0042/1/vehicle-attributes-recognition-barrier-0042.xml - curl https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/vehicle-attributes-recognition-barrier-0042/FP32/vehicle-attributes-recognition-barrier-0042.bin -o workspace/vehicle-attributes-recognition-barrier-0042/1/vehicle-attributes-recognition-barrier-0042.bin -ifeq ($(BUILD_CUSTOM_NODE),true) -# Build custom node - cd ../../../src/custom_nodes && \ - make BASE_OS=$(BASE_OS) NODES=model_zoo_intel_object_detection && \ - mkdir -p ../../demos/vehicle_analysis_pipeline/python/workspace/lib && \ - mv lib/$(BASE_OS)/libcustom_node_model_zoo_intel_object_detection.so ../../demos/vehicle_analysis_pipeline/python/workspace/lib/libcustom_node_model_zoo_intel_object_detection.so -endif -# Copy configuration file to workspace directory - cp config.json workspace/. -ifeq ($(BUILD_CUSTOM_NODE),false) - sed -i 's:\/workspace\/lib\/libcustom_node_model_zoo_intel_object_detection.so:\/ovms\/lib\/custom_nodes\/libcustom_node_model_zoo_intel_object_detection.so:g' workspace/config.json -endif -clean: - @rm -rf workspace \ No newline at end of file diff --git a/demos/vehicle_analysis_pipeline/python/README.md b/demos/vehicle_analysis_pipeline/python/README.md deleted file mode 100644 index eb19a9b04d..0000000000 --- a/demos/vehicle_analysis_pipeline/python/README.md +++ /dev/null @@ -1,160 +0,0 @@ -# Vehicle Analysis Pipeline Demo {#ovms_demo_vehicle_analysis_pipeline} -This document demonstrates how to create complex pipelines using object detection and object recognition models from OpenVINO Model Zoo. As an example, we will use [vehicle-detection-0202](https://github.com/openvinotoolkit/open_model_zoo/blob/2022.1.0/models/intel/vehicle-detection-0202/README.md) to detect multiple vehicles on the image. Then, for each detected vehicle we will crop it using [model_zoo_intel_object_detection](https://github.com/openvinotoolkit/model_server/tree/main/src/custom_nodes/model_zoo_intel_object_detection) example custom node. Finally, each vehicle image will be forwarded to [vehicle-attributes-recognition-barrier-0042](https://github.com/openvinotoolkit/open_model_zoo/blob/2022.1.0/models/intel/vehicle-attributes-recognition-barrier-0042/README.md) model. - -![Vehicles analysis visualization](vehicles_analysis.png) - -Using such pipeline, a single request to OVMS can perform a complex set of operations to determine all vehicles and their properties. - -## Pipeline Configuration Graph - -Below is depicted graph implementing vehicles analysis pipeline execution. - -![Vehicles Analysis Pipeline Graph](vehicles_analysis_graph.svg) - -It includes the following Nodes: -- Model `vehicle_detection` - deep learning model which takes user image as input. Its outputs contain information about vehicle coordinates and confidence levels. -- Custom node `model_zoo_intel_object_detection` - it includes C++ implementation of common object detection models results processing. By analysing the output it produces cropped vehicle images based on the configurable score level threshold. Custom node also resizes them to the target resolution and combines into a single output of a dynamic batch size. The output batch size is determined by the number of detected -boxes according to the configured criteria. All operations on the images employ OpenCV libraries which are preinstalled in the OVMS. Learn more about the [model_zoo_intel_object_detection custom node](https://github.com/openvinotoolkit/model_server/tree/main/src/custom_nodes/model_zoo_intel_object_detection). -- demultiplexer - outputs from the custom node model_zoo_intel_object_detection have variable batch size. In order to match it with the sequential recognition models, data is split into individual images with each batch size equal to 1. -Such smaller requests can be submitted for inference in parallel to the next Model Nodes. Learn more about the [demultiplexing](../../../docs/demultiplexing.md). -- Model `vehicle_attributes_recognition` - this model recognizes type and color for given vehicle image -- Response - the output of the whole pipeline combines the recognized vehicle images with their metadata: coordinates, type, color, and detection confidence level. - -## Prepare workspace to run the demo - -To successfully deploy face analysis pipeline you need to have a workspace that contains: -- Deep learning models for inference -- Custom node for image processing -- Configuration file - -Clone the repository and enter vehicle_analysis_pipeline directory -```bash -git clone https://github.com/openvinotoolkit/model_server.git -cd model_server/demos/vehicle_analysis_pipeline/python -``` - -You can prepare the workspace that contains all the above by just running `make` command. -Since custom node used in this demo is included in OpenVINO Model Server image you can either use the custom node from the image, or build one. - -If you just want to quickly run this demo and use already compiled custom node, run: - -```bash -make -``` - -#### Directory structure (without custom node) - -Once the `make` procedure is finished, you should have `workspace` directory ready with the following content. - -```bash -workspace -├── config.json -├── vehicle-attributes-recognition-barrier-0042 -│ └── 1 -│ ├── vehicle-attributes-recognition-barrier-0042.bin -│ └── vehicle-attributes-recognition-barrier-0042.xml -└── vehicle-detection-0202 - └── 1 - ├── vehicle-detection-0202.bin - └── vehicle-detection-0202.xml - -``` - -If you modified the custom node or for some other reason, you want to have it compiled and then attached to the container, run: - -```bash -make BUILD_CUSTOM_NODE=true BASE_OS=ubuntu -``` - -#### Directory structure (with custom node) - -Once the `make` procedure is finished, you should have `workspace` directory ready with the following content. -```bash -workspace -├── config.json -├── lib -│ └── libcustom_node_model_zoo_intel_object_detection.so -├── vehicle-attributes-recognition-barrier-0042 -│ └── 1 -│ ├── vehicle-attributes-recognition-barrier-0042.bin -│ └── vehicle-attributes-recognition-barrier-0042.xml -└── vehicle-detection-0202 - └── 1 - ├── vehicle-detection-0202.bin - └── vehicle-detection-0202.xml -``` - -## Deploying OVMS - -Deploy OVMS with vehicles analysis pipeline using the following command: - -```bash -docker run -p 9000:9000 -d -v ${PWD}/workspace:/workspace openvino/model_server --config_path /workspace/config.json --port 9000 -``` - -## Requesting the Service - -Install python dependencies: -```bash -pip3 install -r requirements.txt -``` - -Now you can create a directory for text images and run the client: -```bash -mkdir results -``` -```bash -python vehicles_analysis_pipeline.py --pipeline_name multiple_vehicle_recognition --grpc_port 9000 --image_input_path ../../common/static/images/cars/road1.jpg --vehicle_images_output_name vehicle_images --vehicle_images_save_path ./results --image_width 512 --image_height 512 --input_image_layout NHWC -Output: name[types] - numpy => shape[(37, 1, 4)] data[float32] -Output: name[vehicle_coordinates] - numpy => shape[(37, 1, 4)] data[float32] -Output: name[colors] - numpy => shape[(37, 1, 7)] data[float32] -Output: name[confidence_levels] - numpy => shape[(37, 1, 1)] data[float32] -Output: name[vehicle_images] - numpy => shape[(37, 1, 72, 72, 3)] data[float32] - -Found 37 vehicles: -0 Type: van Color: gray -1 Type: car Color: gray -2 Type: car Color: black -3 Type: car Color: white -4 Type: car Color: black -5 Type: truck Color: red -6 Type: truck Color: gray -7 Type: car Color: white -8 Type: car Color: blue -9 Type: car Color: red -10 Type: car Color: yellow -11 Type: car Color: gray -12 Type: van Color: gray -13 Type: car Color: gray -14 Type: car Color: red -15 Type: truck Color: gray -16 Type: truck Color: red -17 Type: car Color: red -18 Type: car Color: gray -19 Type: car Color: black -20 Type: truck Color: gray -21 Type: car Color: red -22 Type: truck Color: blue -23 Type: truck Color: red -24 Type: car Color: gray -25 Type: truck Color: red -26 Type: car Color: black -27 Type: truck Color: gray -28 Type: truck Color: blue -29 Type: truck Color: gray -30 Type: car Color: gray -31 Type: car Color: white -32 Type: car Color: yellow -33 Type: car Color: red -34 Type: truck Color: gray -35 Type: truck Color: gray -36 Type: car Color: red -``` - -With additional parameter `--vehicle_images_save_path`, the client script saves all detected vehicle images to jpeg files into directory path to confirm -if the image was analyzed correctly. diff --git a/demos/vehicle_analysis_pipeline/python/config.json b/demos/vehicle_analysis_pipeline/python/config.json deleted file mode 100644 index bc607af6c5..0000000000 --- a/demos/vehicle_analysis_pipeline/python/config.json +++ /dev/null @@ -1,102 +0,0 @@ -{ - "model_config_list": [ - {"config": { - "name": "vehicle_detection", - "base_path": "/workspace/vehicle-detection-0202/", - "layout": "NHWC:NCHW"}}, - {"config": { - "name": "vehicle_attributes_recognition", - "base_path": "/workspace/vehicle-attributes-recognition-barrier-0042/", - "layout": "NHWC:NCHW"}} - ], - "custom_node_library_config_list": [ - {"name": "object_detection_image_extractor", - "base_path": "/workspace/lib/libcustom_node_model_zoo_intel_object_detection.so"} - ], - "pipeline_config_list": [ - { - "name": "multiple_vehicle_recognition", - "inputs": [ - "image" - ], - "nodes": [ - { - "name": "vehicle_detection_node", - "model_name": "vehicle_detection", - "type": "DL model", - "inputs": [ - {"image": { - "node_name": "request", - "data_item": "image"}}], - "outputs": [ - {"data_item": "detection_out", - "alias": "detection"}] - }, - { - "name": "extract_node", - "library_name": "object_detection_image_extractor", - "type": "custom", - "demultiply_count": 0, - "params": { - "original_image_width": "512", - "original_image_height": "512", - "target_image_width": "72", - "target_image_height": "72", - "original_image_layout": "NHWC", - "target_image_layout": "NHWC", - "convert_to_gray_scale": "false", - "max_output_batch": "100", - "confidence_threshold": "0.7", - "debug": "true", - "buffer_queue_size": "24" - }, - "inputs": [ - {"image": { - "node_name": "request", - "data_item": "image"}}, - {"detection": { - "node_name": "vehicle_detection_node", - "data_item": "detection"}}], - "outputs": [ - {"data_item": "images", - "alias": "vehicle_images"}, - {"data_item": "coordinates", - "alias": "vehicle_coordinates"}, - {"data_item": "confidences", - "alias": "confidence_levels"}] - }, - { - "name": "vehicle_attributes_recognition_node", - "model_name": "vehicle_attributes_recognition", - "type": "DL model", - "inputs": [ - {"input": { - "node_name": "extract_node", - "data_item": "vehicle_images"}}], - "outputs": [ - {"data_item": "color", - "alias": "color"}, - {"data_item": "type", - "alias": "type"}] - } - ], - "outputs": [ - {"vehicle_images": { - "node_name": "extract_node", - "data_item": "vehicle_images"}}, - {"vehicle_coordinates": { - "node_name": "extract_node", - "data_item": "vehicle_coordinates"}}, - {"confidence_levels": { - "node_name": "extract_node", - "data_item": "confidence_levels"}}, - {"colors": { - "node_name": "vehicle_attributes_recognition_node", - "data_item": "color"}}, - {"types": { - "node_name": "vehicle_attributes_recognition_node", - "data_item": "type"}} - ] - } - ] -} diff --git a/demos/vehicle_analysis_pipeline/python/requirements.txt b/demos/vehicle_analysis_pipeline/python/requirements.txt deleted file mode 100644 index fa28f0f955..0000000000 --- a/demos/vehicle_analysis_pipeline/python/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --r ../../common/python/requirements.txt \ No newline at end of file diff --git a/demos/vehicle_analysis_pipeline/python/vehicles_analysis.png b/demos/vehicle_analysis_pipeline/python/vehicles_analysis.png deleted file mode 100755 index 68b37ba3c1..0000000000 Binary files a/demos/vehicle_analysis_pipeline/python/vehicles_analysis.png and /dev/null differ diff --git a/demos/vehicle_analysis_pipeline/python/vehicles_analysis_graph.svg b/demos/vehicle_analysis_pipeline/python/vehicles_analysis_graph.svg deleted file mode 100755 index 773e045c0c..0000000000 --- a/demos/vehicle_analysis_pipeline/python/vehicles_analysis_graph.svg +++ /dev/null @@ -1,3 +0,0 @@ - - -
Request
inputs: image
Request...
Model: vehicle-detection
inputs: image
outputs: detection_out
Model: vehicle-detection...
Custom Node: model_zoo_intel_object_detection
inputs: image, detection_out
outputs: vehicle_images, vehicle_coordinates, confidence_levels
Custom Node: model_zoo_intel_object_detection...
Demultiplexer
Demultiplexer
Model: vehicle-attributes-recognition
inputs: image
outputs: color, type
Model: vehicle-attributes-recognition...
Response
outputs:
vehicle_images, vehicle_coordinates, confidence_levels,
colors, types
Response...
Gatherer
GathererViewer does not support full SVG 1.1 \ No newline at end of file diff --git a/demos/vehicle_analysis_pipeline/python/vehicles_analysis_pipeline.py b/demos/vehicle_analysis_pipeline/python/vehicles_analysis_pipeline.py deleted file mode 100644 index e748ed1fdf..0000000000 --- a/demos/vehicle_analysis_pipeline/python/vehicles_analysis_pipeline.py +++ /dev/null @@ -1,146 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import grpc -import cv2 -import os -import numpy as np -from tensorflow import make_tensor_proto, make_ndarray -import argparse -from tensorflow_serving.apis import predict_pb2 -from tensorflow_serving.apis import prediction_service_pb2_grpc -parser = argparse.ArgumentParser(description='Client for multiple vehicles analysis pipeline') -parser.add_argument('--grpc_address', required=False, default='localhost', help='Specify url to grpc service. default:localhost') -parser.add_argument('--grpc_port', required=False, default=9178, help='Specify port to grpc service. default: 9178') -parser.add_argument('--pipeline_name', required=False, default='multiple_vehicle_recognition', help='Pipeline name to request. default: multiple_vehicle_recognition') -parser.add_argument('--image_input_name', required=False, default='image', help='Pipeline input name for input with image with vehicles. default: image') -parser.add_argument('--image_input_path', required=True, help='Input image path.') -parser.add_argument('--vehicle_images_output_name', required=False, default='vehicle_images', help='Pipeline output name for cropped images with vehicles. default: vehicle_images') -parser.add_argument('--vehicle_images_save_path', required=False, default='', help='If specified, vehicle images will be saved to disk.') -parser.add_argument('--image_width', required=False, default=600, help='Pipeline input image width. default: 600') -parser.add_argument('--image_height', required=False, default=400, help='Pipeline input image height. default: 400') -parser.add_argument('--input_image_layout', required=False, default='NHWC', choices=['NCHW', 'NHWC', 'BINARY'], help='Pipeline input image layout. default: NHWC') - -args = vars(parser.parse_args()) - -def prepare_img_input_in_nchw_format(request, name, path, resize_to_shape): - img = cv2.imread(path).astype(np.float32) # BGR color format, shape HWC - img = cv2.resize(img, (resize_to_shape[1], resize_to_shape[0])) - target_shape = (img.shape[0], img.shape[1]) - img = img.transpose(2,0,1).reshape(1,3,target_shape[0],target_shape[1]) - request.inputs[name].CopyFrom(make_tensor_proto(img, shape=img.shape)) - -def prepare_img_input_in_nhwc_format(request, name, path, resize_to_shape): - img = cv2.imread(path).astype(np.float32) # BGR color format, shape HWC - img = cv2.resize(img, (resize_to_shape[1], resize_to_shape[0])) - target_shape = (img.shape[0], img.shape[1]) - img = img.reshape(1,target_shape[0],target_shape[1],3) - request.inputs[name].CopyFrom(make_tensor_proto(img, shape=img.shape)) - -def prepare_img_input_in_binary_format(request, name, path): - with open(path, 'rb') as f: - data = f.read() - request.inputs[name].CopyFrom(make_tensor_proto(data, shape=[1])) - -def save_vehicle_images_as_jpgs(output_nd, name, location): - for i in range(output_nd.shape[0]): - out = output_nd[i][0] - if len(out.shape) == 3 and out.shape[0] == 3: - out = out.transpose(1,2,0) - cv2.imwrite(os.path.join(location, name + '_' + str(i) + '.jpg'), out) - -def update_vehicle_types(output_nd, vehicles): - for i in range(output_nd.shape[0]): - detection = np.argmax(output_nd[i,0]) - detection_str = { - 0: 'car', - 1: 'van', - 2: 'truck', - 3: 'bus' - }[detection] - if len(vehicles) < i + 1: - vehicles.append({'type': detection_str}) - else: - vehicles[i].update({'type': detection_str}) - return vehicles - -def update_vehicle_colors(output_nd, vehicles): - for i in range(output_nd.shape[0]): - detection = np.argmax(output_nd[i,0]) - detection_str = { - 0: 'white', - 1: 'gray', - 2: 'yellow', - 3: 'red', - 4: 'green', - 5: 'blue', - 6: 'black' - }[detection] - if len(vehicles) < i + 1: - vehicles.append({'color': detection_str}) - else: - vehicles[i].update({'color': detection_str}) - return vehicles - -address = "{}:{}".format(args['grpc_address'],args['grpc_port']) -MAX_MESSAGE_LENGTH = 1024 * 1024 * 1024 -channel = grpc.insecure_channel(address, - options=[ - ('grpc.max_send_message_length', MAX_MESSAGE_LENGTH), - ('grpc.max_receive_message_length', MAX_MESSAGE_LENGTH), - ]) - -stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) -request = predict_pb2.PredictRequest() -request.model_spec.name = args['pipeline_name'] - -if args['input_image_layout'] == 'NCHW': - prepare_img_input_in_nchw_format(request, args['image_input_name'], args['image_input_path'], (int(args['image_height']), int(args['image_width']))) -elif args['input_image_layout'] == 'NHWC': - prepare_img_input_in_nhwc_format(request, args['image_input_name'], args['image_input_path'], (int(args['image_height']), int(args['image_width']))) -else: - prepare_img_input_in_binary_format(request, args['image_input_name'], args['image_input_path']) - -try: - response = stub.Predict(request, 30.0) -except grpc.RpcError as err: - if err.code() == grpc.StatusCode.ABORTED: - print('No vehicle has been found in the image') - exit(1) - else: - raise err - -vehicles = [] - -for name in response.outputs: - print(f"Output: name[{name}]") - tensor_proto = response.outputs[name] - output_nd = make_ndarray(tensor_proto) - print(f" numpy => shape[{output_nd.shape}] data[{output_nd.dtype}]") - - if name == args['vehicle_images_output_name'] and len(args['vehicle_images_save_path']) > 0: - save_vehicle_images_as_jpgs(output_nd, name, args['vehicle_images_save_path']) - - if name == 'types': - vehicles = update_vehicle_types(output_nd, vehicles) - if name == 'colors': - vehicles = update_vehicle_colors(output_nd, vehicles) - -print('\nFound', len(vehicles), 'vehicles:') -i = 0 -for vehicle in vehicles: - print(i, 'Type:', vehicle['type'], 'Color:', vehicle['color']) - i += 1 diff --git a/docs/binary_input.md b/docs/binary_input.md index c4aa732984..38491e777f 100644 --- a/docs/binary_input.md +++ b/docs/binary_input.md @@ -7,7 +7,6 @@ hidden: --- ovms_docs_binary_input_layout_and_shape -ovms_docs_binary_input_tfs ovms_docs_binary_input_kfs ovms_docs_demo_tensorflow_conversion ``` @@ -19,11 +18,10 @@ For images, to reduce data size and lower bandwidth usage you can send them in b While OpenVINO models don't have the ability to process images directly in their binary format, the model server can accept them and convert automatically from JPEG/PNG to OpenVINO friendly format using built-in [OpenCV](https://opencv.org/) library. To take advantage of this feature, there are two requirements: 1. Model input, that receives binary encoded image, must have a proper shape and layout. Learn more about this requirement in [input shape and layout considerations](./binary_input_layout_and_shape.md) document. - 2. Inference request sent to the server must have certain properties. These properties are different depending on the API (KServe or TensorFlow Serving) and interface (gRPC or REST). Choose the API you are using and learn more: - - [TensorFlow Serving API](./binary_input_tfs.md) + 2. Inference request sent to the server must have certain properties. These properties are different depending on the interface (gRPC or REST). Learn more: - [KServe API](./binary_input_kfs.md) -It's worth noting that with KServe API, you can also send raw data with or without image encoding via REST API. This makes KServe REST API more performant choice comparing to json format in TFS API. The guide linked above explains how to work with both regular data in binary format as well as JPEG/PNG encoded images. +With KServe API, you can also send raw data with or without image encoding via REST API. The guide linked above explains how to work with both regular data in binary format as well as JPEG/PNG encoded images. **MediaPipe Graphs**: diff --git a/docs/binary_input_tfs.md b/docs/binary_input_tfs.md deleted file mode 100644 index dc08cea395..0000000000 --- a/docs/binary_input_tfs.md +++ /dev/null @@ -1,37 +0,0 @@ -# Predict on Binary Inputs via TensorFlow Serving API {#ovms_docs_binary_input_tfs} - -## GRPC - -TensorFlow Serving API allows sending the model input data in a variety of formats inside the [TensorProto](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor.proto) objects. -Array data is passed inside the `tensor_content` field, which represents the input data buffer. - -When the data is sent in the `string_val` field to the model or pipeline that have 4 (or 5 in case of [demultiplexing](demultiplexing.md)) shape dimensions, such input is interpreted as a binary encoded image. - -Note, that while the model metadata reports the inputs shape with layout NHWC, the binary data must be sent with -shape: [N] with dtype: DT_STRING. Where N represents number of images converted to string bytes. - -When sending data in the array format, all bytes are in the same sequence in `tensor_content` field and when loaded on the server side, the shape gives information on how to interpret them. For binary encoded data, bytes for each image in the batch are put in a separate sequence in the `string_val` field. The only information given by the `tensor_shape` field is the amount of images in the batch. On the server side, the bytes in each element of the `string_val` field are loaded, resized to match model input shape and converted to the OpenVINO-friendly array format by OpenCV. - -## HTTP - -TensorFlow Serving API also allows sending encoded images via HTTP interface to the model or pipeline that have 4 (or 5 in case of [demultiplexing](demultiplexing.md)) shape dimensions. The binary data needs to be Base64 encoded and put into `inputs` or `instances` field as a map in form: - -``` -: {"b64":} -``` - -On the server side, the Base64 encoded data is decoded to raw binary and loaded using OpenCV which then converts it to OpenVINO-friendly data format for inference. - -## API Reference -- [TensorFlow Serving gRPC API Reference Guide](./model_server_grpc_api_tfs.md) -- [TensorFlow Serving REST API Reference Guide](./model_server_rest_api_tfs.md) - - -## Recommendations: - -Sending the data in binary format can significantly simplify the client code and it's preprocessing load. With the REST API -client, only curl and base64 tool or the requests python package is needed. In case the original input data is jpeg or png -encoded, there is no preprocessing needed to send the request. - -Binary data can significantly reduce the network utilization. In many cases it allows reducing the latency and achieve -very high throughput even with slower network bandwidth. diff --git a/docs/clients_genai.md b/docs/clients_genai.md index eeaa2aff74..bc6dbb55af 100644 --- a/docs/clients_genai.md +++ b/docs/clients_genai.md @@ -16,7 +16,7 @@ Speech to text API Text to speech API ``` ## Introduction -Besides TensorFlow Serving API (`/v1`) and KServe API (`/v2`) frontends, the model server supports a range of endpoints for generative use cases (`v3`). They are extendible using MediaPipe graphs. +Besides KServe API (`/v2`) frontend, the model server supports a range of endpoints for generative use cases (`v3`). They are extendible using MediaPipe graphs. Currently supported endpoints are: OpenAI compatible endpoints: diff --git a/docs/clients_tfs.md b/docs/clients_tfs.md deleted file mode 100644 index b288420fcd..0000000000 --- a/docs/clients_tfs.md +++ /dev/null @@ -1,221 +0,0 @@ -# TensorFlow Serving API {#ovms_docs_clients_tfs} - -```{toctree} ---- -maxdepth: 1 -hidden: ---- - -gRPC API -RESTful API -Examples -``` - -## Python Client - -When creating a Python-based client application, you can use [tensorflow-serving-api](https://pypi.org/project/tensorflow-serving-api/) package that can be used with OpenVINO Model Server. - -### Install the Package - -::::{tab-set} -:::{tab-item} tensorflow-serving-api -:sync: tensorflow-serving-api -```{code} sh -pip3 install tensorflow-serving-api -``` -::: -:::: - -### Request Model Status - -::::{tab-set} -:::{tab-item} tensorflow-serving-api -:sync: tensorflow-serving-api -```{code} python -import grpc -from tensorflow_serving.apis import model_service_pb2_grpc, get_model_status_pb2 -from tensorflow_serving.apis.get_model_status_pb2 import ModelVersionStatus - -channel = grpc.insecure_channel("localhost:9000") -model_service_stub = model_service_pb2_grpc.ModelServiceStub(channel) - -status_request = get_model_status_pb2.GetModelStatusRequest() -status_request.model_spec.name = "my_model" -status_response = model_service_stub.GetModelStatus(status_request, 10.0) - -model_status = {} -model_version_status = status_response.model_version_status -for model_version in model_version_status: - model_status[model_version.version] = dict([ - ('state', ModelVersionStatus.State.Name(model_version.state)), - ('error_code', model_version.status.error_code), - ('error_message', model_version.status.error_message), - ]) -``` -::: -:::{tab-item} curl -:sync: curl -```{code} sh -curl http://localhost:8000/v1/models/my_model -``` -::: -:::: - -### Request Model Metadata - -::::{tab-set} -:::{tab-item} tensorflow-serving-api -:sync: tensorflow-serving-api -```{code} python -import grpc -from tensorflow_serving.apis import prediction_service_pb2_grpc, get_model_metadata_pb2 -from tensorflow.core.framework.types_pb2 import DataType - -channel = grpc.insecure_channel("localhost:9000") -prediction_service_stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) - -metadata_request = get_model_metadata_pb2.GetModelMetadataRequest() -metadata_request.model_spec.name = "my_model" -metadata_response = prediction_service_stub.GetModelMetadata(metadata_request, 10.0) - -model_metadata = {} - -signature_def = metadata_response.metadata['signature_def'] -signature_map = get_model_metadata_pb2.SignatureDefMap() -signature_map.ParseFromString(signature_def.value) -model_signature = signature_map.ListFields()[0][1]['serving_default'] - -inputs_metadata = {} -for input_name, input_info in model_signature.inputs.items(): - input_shape = [d.size for d in input_info.tensor_shape.dim] - inputs_metadata[input_name] = dict([ - ("shape", input_shape), - ("dtype", DataType.Name(input_info.dtype)) - ]) - -outputs_metadata = {} -for output_name, output_info in model_signature.outputs.items(): - output_shape = [d.size for d in output_info.tensor_shape.dim] - outputs_metadata[output_name] = dict([ - ("shape", output_shape), - ("dtype", DataType.Name(output_info.dtype)) - ]) - -version = metadata_response.model_spec.version.value -model_metadata = dict([ - ("model_version", version), - ("inputs", inputs_metadata), - ("outputs", outputs_metadata) -]) -``` -::: -:::{tab-item} curl -:sync: curl -```{code} sh -curl http://localhost:8000/v1/models/my_model/metadata -``` -::: -:::: - -### Request Prediction on a Binary Input - -::::{tab-set} -:::{tab-item} tensorflow-serving-api -:sync: tensorflow-serving-api -```{code} python -import grpc -from tensorflow_serving.apis import prediction_service_pb2_grpc, predict_pb2 -from tensorflow import make_tensor_proto, make_ndarray - -channel = grpc.insecure_channel("localhost:9000") -prediction_service_stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) - -with open("img.jpeg", "rb") as f: - data = f.read() -predict_request = predict_pb2.PredictRequest() -predict_request.model_spec.name = "my_model" -predict_request.inputs["input_name"].CopyFrom(make_tensor_proto(data)) -predict_response = prediction_service_stub.Predict(predict_request, 10.0) -results = make_ndarray(predict_response.outputs["output_name"]) -``` -::: -:::{tab-item} curl -:sync: curl -```{code} sh -curl -X POST http://localhost:8000/v1/models/my_model:predict --H 'Content-Type: application/json' --d '{"instances": [{"input_name": {"b64":"YXdlc29tZSBpbWFnZSBieXRlcw=="}}]}' -``` -::: -:::: - -### Request Prediction on a Numpy Array - -::::{tab-set} -:::{tab-item} tensorflow-serving-api -:sync: tensorflow-serving-api -```{code} python -import grpc -from tensorflow_serving.apis import prediction_service_pb2_grpc, predict_pb2 -from tensorflow import make_tensor_proto - -channel = grpc.insecure_channel("localhost:9000") -prediction_service_stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) - -data = np.array([1.0, 2.0, ..., 1000.0]) -predict_request = predict_pb2.PredictRequest() -predict_request.model_spec.name = "my_model" -predict_request.inputs["input_name"].CopyFrom(make_tensor_proto(data)) -predict_response = prediction_service_stub.Predict(predict_request, 10.0) -results = make_ndarray(predict_response.outputs["output_name"]) -``` -::: -:::{tab-item} curl -:sync: curl -```{code} sh -curl -X POST http://localhost:8000/v1/models/my_model:predict --H 'Content-Type: application/json' --d '{"instances": [{"input_name": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}]}' -``` -::: -:::: - -### Request Prediction on a string - -::::{tab-set} -:::{tab-item} tensorflow-serving-api -:sync: tensorflow-serving-api -```{code} python -import grpc -from tensorflow_serving.apis import prediction_service_pb2_grpc, predict_pb2 -from tensorflow import make_tensor_proto - -channel = grpc.insecure_channel("localhost:9000") -prediction_service_stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) - -data = [""] -predict_request = predict_pb2.PredictRequest() -predict_request.model_spec.name = "my_model" -predict_request.inputs["input_name"].CopyFrom(make_tensor_proto(data)) -predict_response = prediction_service_stub.Predict(predict_request, 1) -results = predict_response.outputs["output_name"] -``` -::: -:::{tab-item} curl -:sync: curl -```{code} sh -curl -X POST http://localhost:8000/v1/models/my_model:predict --H 'Content-Type: application/json' --d '{"instances": [{"input_name": ""}]}' -``` -::: -:::: - - -## C++ and Go Clients - -Creating a client application in C++ or [Go](https://go.dev/) follows the same principles as Python, but using them adds some complexity. There is no package or library available for them with convenient functions to interact with OpenVINO Model Server. - -To successfully set up communication with the model server, you need to implement the logic to communicate with endpoints specified in the [API](writing_app.md). For gRPC, download and compile protos, then link and use them in your application according to the [gRPC API specification](model_server_grpc_api_tfs.md). For REST, prepare your data and pack it into the appropriate JSON structure according to the [REST API specification](model_server_rest_api_tfs.md). - -See our [Go demo](../demos/image_classification/go/README.md) to learn how to build a sample Go-based client application in a Docker container and get predictions via the gRPC API. diff --git a/docs/dag_scheduler.md b/docs/dag_scheduler.md index 14a888268a..89f5255bac 100644 --- a/docs/dag_scheduler.md +++ b/docs/dag_scheduler.md @@ -195,12 +195,12 @@ Pipelines can use the same API as the models. There are exactly the same calls f the predictions. The request format must match the pipeline definition inputs. -The pipeline configuration can be queried using [gRPC GetModelMetadata](model_server_grpc_api_tfs.md) calls and -[REST Metadata](model_server_rest_api_tfs.md). +The pipeline configuration can be queried using [gRPC ModelMetadata](model_server_grpc_api_kfs.md) calls and +[REST Metadata](model_server_rest_api_kfs.md). It returns the definition of the pipelines inputs and outputs. -Similarly, pipelines can be queried for their state using the calls [GetModelStatus](model_server_grpc_api_tfs.md) -and [REST Model Status](model_server_rest_api_tfs.md) +Similarly, pipelines can be queried for their state using [KServe Model Ready](model_server_grpc_api_kfs.md) +and [REST Model Ready](model_server_rest_api_kfs.md) The only difference in using the pipelines and individual models is in version management. In all calls to the pipelines, the version parameter is ignored. Pipelines are not versioned. Though, they can reference a particular version of the models in the graph. diff --git a/docs/deploying_server_docker.md b/docs/deploying_server_docker.md index e2c51d4d79..b66befb26f 100644 --- a/docs/deploying_server_docker.md +++ b/docs/deploying_server_docker.md @@ -47,10 +47,10 @@ wget https://raw.githubusercontent.com/openvinotoolkit/model_server/main/demos/c wget https://raw.githubusercontent.com/openvinotoolkit/model_server/main/demos/common/python/classes.py ``` -##### 2.3 Install the Python-based ovmsclient package +##### 2.3 Install the Python-based client package ```bash -pip3 install ovmsclient +pip3 install tritonclient[grpc] numpy ``` @@ -60,14 +60,24 @@ pip3 install ovmsclient ```bash echo 'import numpy as np from classes import imagenet_classes -from ovmsclient import make_grpc_client +import tritonclient.grpc as grpcclient -client = make_grpc_client("localhost:9000") +client = grpcclient.InferenceServerClient(url="localhost:9000") +# Get model metadata to discover input/output names +metadata = client.get_model_metadata("resnet") +input_name = metadata.inputs[0].name +output_name = metadata.outputs[0].name + +# Load image as binary with open("zebra.jpeg", "rb") as f: - img = f.read() + img_bytes = f.read() -output = client.predict({"0": img}, "resnet") +# Run inference +infer_input = grpcclient.InferInput(input_name, [1], "BYTES") +infer_input.set_data_from_numpy(np.array([img_bytes], dtype=np.object_)) +result = client.infer("resnet", [infer_input]) +output = result.as_numpy(output_name) result_index = np.argmax(output[0]) print(imagenet_classes[result_index])' >> predict.py diff --git a/docs/dynamic_bs_auto_reload.md b/docs/dynamic_bs_auto_reload.md deleted file mode 100644 index 67bd917291..0000000000 --- a/docs/dynamic_bs_auto_reload.md +++ /dev/null @@ -1,141 +0,0 @@ -# Dynamic Batch Size with Automatic Model Reloading{#ovms_docs_dynamic_bs_auto_reload} - -*NOTE*: This feature is deprecated. Use [model dynamic shapes instead](dynamic_shape_dynamic_model.md). - -## Introduction -This guide shows how to configure a model to accept input data with different batch sizes. In this example, it is done by reloading the model with a new batch size each time a request is received with a batch size different than what is currently set. - -Enabling dynamic batch size via model reload is as simple as setting the `batch_size` parameter to `auto`. To configure and use the dynamic batch size, take advantage of: - -- An example client in Python [grpc_predict_resnet.py](https://github.com/openvinotoolkit/model_server/blob/main/client/python/tensorflow-serving-api/samples/grpc_predict_resnet.py) that can be used to request inference with the desired batch size. - -- A sample [resnet](https://github.com/openvinotoolkit/open_model_zoo/blob/2022.1.0/models/intel/resnet50-binary-0001/README.md) model. - - When using the resnet model with `grpc_predict_resnet.py`, the script processes the output from the server and displays the inference results using the previously prepared file containing labels. Inside this file, each image has an assigned number, which indicates the correct classification result. - -## Steps -Clone OpenVINO™ Model Server GitHub repository and enter `model_server` directory. -```bash -git clone https://github.com/openvinotoolkit/model_server.git -cd model_server -``` -#### Download the Pretrained Model -Download the model files and store them in the `models` directory -```bash -mkdir -p models/resnet/1 -curl https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/resnet50-binary-0001/FP32-INT1/resnet50-binary-0001.bin https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/resnet50-binary-0001/FP32-INT1/resnet50-binary-0001.xml -o models/resnet/1/resnet50-binary-0001.bin -o models/resnet/1/resnet50-binary-0001.xml -``` - -#### Pull the Latest Model Server Image -Pull the latest version of OpenVINO™ Model Server from Docker Hub : -```bash -docker pull openvino/model_server:latest -``` - -#### Start the Model Server Container with Downloaded Model and Dynamic Batch Size -Start the server container with the image pulled in the previous step and mount the `models` directory : -```bash -docker run --rm -d -v $(pwd)/models:/models -p 9000:9000 openvino/model_server:latest --model_name resnet --model_path /models/resnet --batch_size auto --port 9000 -``` - -#### Run the Client -```bash -cd client/python/tensorflow-serving-api/samples -virtualenv .venv -. .venv/bin/activate -pip install -r requirements.txt - -python grpc_predict_resnet.py --grpc_port 9000 --images_numpy_path ../../imgs.npy --labels_numpy_path ../../lbs.npy --input_name 0 --output_name 1463 --model_name resnet --transpose_input False --batchsize 1 > b1.txt && python grpc_predict_resnet.py --grpc_port 9000 --images_numpy_path ../../imgs.npy --labels_numpy_path ../../lbs.npy --input_name 0 --output_name 1463 --model_name resnet --transpose_input False --batchsize 8 > b8.txt; -``` -*NOTE*: Results of running the client will be available in .txt files in the current directory. - -#### Script Output -Output with `batchsize 1` stored in `b1.txt`: -```bash -cat b1.txt -Image data range: 0.0 : 255.0 -Start processing: - Model name: resnet - Iterations: 10 - Images numpy path: ../../imgs.npy - Numpy file shape: (10, 3, 224, 224) - -Iteration 1; Processing time: 21.16 ms; speed 47.25 fps -imagenet top results in a single batch: - 0 airliner 404 ; Correct match. -Iteration 2; Processing time: 8.08 ms; speed 123.79 fps -imagenet top results in a single batch: - 0 Arctic fox, white fox, Alopex lagopus 279 ; Correct match. -Iteration 3; Processing time: 104.76 ms; speed 9.55 fps -imagenet top results in a single batch: - 0 bee 309 ; Correct match. -Iteration 4; Processing time: 8.86 ms; speed 112.83 fps -imagenet top results in a single batch: - 0 golden retriever 207 ; Correct match. -Iteration 5; Processing time: 19.05 ms; speed 52.48 fps -imagenet top results in a single batch: - 0 gorilla, Gorilla gorilla 366 ; Correct match. -Iteration 6; Processing time: 9.31 ms; speed 107.47 fps -imagenet top results in a single batch: - 0 magnetic compass 635 ; Correct match. -Iteration 7; Processing time: 7.10 ms; speed 140.81 fps -imagenet top results in a single batch: - 0 peacock 84 ; Correct match. -Iteration 8; Processing time: 6.83 ms; speed 146.50 fps -imagenet top results in a single batch: - 0 pelican 144 ; Correct match. -Iteration 9; Processing time: 6.74 ms; speed 148.26 fps -imagenet top results in a single batch: - 0 snail 113 ; Correct match. -Iteration 10; Processing time: 7.08 ms; speed 141.26 fps -imagenet top results in a single batch: - 0 zebra 340 ; Correct match. - -processing time for all iterations -average time: 19.50 ms; average speed: 51.28 fps -median time: 8.00 ms; median speed: 125.00 fps -max time: 104.00 ms; min speed: 9.62 fps -min time: 6.00 ms; max speed: 166.67 fps -time percentile 90: 29.30 ms; speed percentile 90: 34.13 fps -time percentile 50: 8.00 ms; speed percentile 50: 125.00 fps -time standard deviation: 28.63 -time variance: 819.45 -Classification accuracy: 100.00 - -``` -Output with `batchsize 8` stored in `b8.txt`: -```bash -cat b8.txt -Image data range: 0.0 : 255.0 -Start processing: - Model name: resnet - Iterations: 1 - Images numpy path: ../../imgs.npy - Numpy file shape: (10, 3, 224, 224) - -Iteration 1; Processing time: 121.12 ms; speed 66.05 fps -imagenet top results in a single batch: - 0 airliner 404 ; Correct match. - 1 Arctic fox, white fox, Alopex lagopus 279 ; Correct match. - 2 bee 309 ; Correct match. - 3 golden retriever 207 ; Correct match. - 4 gorilla, Gorilla gorilla 366 ; Correct match. - 5 magnetic compass 635 ; Correct match. - 6 peacock 84 ; Correct match. - 7 pelican 144 ; Correct match. - -processing time for all iterations -average time: 121.00 ms; average speed: 66.12 fps -median time: 121.00 ms; median speed: 66.12 fps -max time: 121.00 ms; min speed: 66.12 fps -min time: 121.00 ms; max speed: 66.12 fps -time percentile 90: 121.00 ms; speed percentile 90: 66.12 fps -time percentile 50: 121.00 ms; speed percentile 50: 66.12 fps -time standard deviation: 0.00 -time variance: 0.00 -Classification accuracy: 100.00 - -``` -Each iteration presents the results of each inference request and details for each image in the batch. - -> Note that reloading the model takes time and during the reload new requests get queued up. Therefore, frequent model reloading may negatively affect overall performance. diff --git a/docs/dynamic_bs_demultiplexer.md b/docs/dynamic_bs_demultiplexer.md deleted file mode 100644 index d00a9754f5..0000000000 --- a/docs/dynamic_bs_demultiplexer.md +++ /dev/null @@ -1,219 +0,0 @@ -# Dynamic batch size with OpenVINO™ Model Server Demultiplexer {#ovms_docs_dynamic_bs_demultiplexer} - -## Introduction -This document shows how to configure a DAG Scheduler pipeline to send predict requests with an arbitrary batch size without model reloading. - -With OpenVINO™ Model Server Demultiplexing infer request sent from client application can have various batch sizes and changing batch size does not require model reload. - -More information about this feature can be found in [dynamic batch size in demultiplexing](./demultiplexing.md) - -> **NOTE**: Only one dynamic demultiplexer (`demultiply_count` with value `-1`) can exist in the pipeline. - -- Example client in python [grpc_predict_resnet.py](https://github.com/openvinotoolkit/model_server/blob/main/client/python/tensorflow-serving-api/samples/grpc_predict_resnet.py) can be used to request the pipeline. Use `--dag-batch-size-auto` flag to add an additional dimension to the input shape which is required for demultiplexing feature. - -- The example uses model [resnet](https://github.com/openvinotoolkit/open_model_zoo/blob/2022.1.0/models/intel/resnet50-binary-0001/README.md). - -- While using resnet model with grpc_predict_resnet.py the script processes the output from the server and displays the inference results using the previously prepared file with labels. Inside this file, each image has an assigned number, which indicates the correct recognition answer. - -## Steps -Clone OpenVINO™ Model Server GitHub repository and enter `model_server` directory. -```bash -git clone https://github.com/openvinotoolkit/model_server.git -cd model_server -``` -#### Download the pretrained model -Download model files and store them in the `models` directory -```bash -mkdir -p models/resnet/1 -curl https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/resnet50-binary-0001/FP32-INT1/resnet50-binary-0001.bin https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/resnet50-binary-0001/FP32-INT1/resnet50-binary-0001.xml -o models/resnet/1/resnet50-binary-0001.bin -o models/resnet/1/resnet50-binary-0001.xml - -chmod -R 755 ./models -``` - -#### Pull the latest OVMS image from dockerhub -Pull the latest version of OpenVINO™ Model Server from Dockerhub : -```bash -docker pull openvino/model_server:latest -``` - -#### OVMS configuration file -Go to the `models` directory: -```bash -cd models -``` - -Create a new file named `config.json` there: -```bash -echo '{ - "model_config_list": [ - { - "config": { - "name": "resnet", - "base_path": "/models/resnet", - "plugin_config": { - "NUM_STREAMS" : "1" - } - } - } - ], - "pipeline_config_list": [ - { - "name": "resnet50DAG", - "inputs": [ - "0" - ], - "demultiply_count" : -1, - "nodes": [ - { - "name": "resnetNode", - "model_name": "resnet", - "type": "DL model", - "inputs": [ - { - "0": { - "node_name": "request", - "data_item": "0" - } - } - ], - "outputs": [ - { - "data_item": "1463", - "alias": "1463" - } - ] - } - ], - "outputs": [ - {"1463": { - "node_name": "resnetNode", - "data_item": "1463"}} - ] - } - ] -}' >> config.json -``` - -#### Start ovms docker container with downloaded model -Start ovms container with image pulled in previous step and mount `models` directory : -```bash -docker run --rm -d -v $(pwd):/models -p 9000:9000 openvino/model_server:latest --config_path /models/config.json --port 9000 -``` - -#### Checking metadata -```bash -cd ../client/python/tensorflow-serving-api/samples -virtualenv .venv -. .venv/bin/activate -pip install -r requirements.txt - -python grpc_get_model_metadata.py --grpc_port 9000 --model_name resnet50DAG -``` - -```bash -... -Getting model metadata for model: resnet50DAG -Inputs metadata: - Input name: 0; shape: [-1, 1, 3, 224, 224]; dtype: DT_FLOAT -Outputs metadata: - Output name: 1463; shape: [-1, 1, 1000]; dtype: DT_FLOAT -``` - -> **NOTE**: While using the dynamic batching feature both input and output shape has an additional dimension, which represents split batch size. Setting batch size parameter to `--batchsize 8` would set input shape to `[8,1,3,244,244]` and output shape to `[8,1,1000]`. - -#### Run the client -```bash -python grpc_predict_resnet.py --grpc_port 9000 --images_numpy_path ../../imgs.npy --labels_numpy_path ../../lbs.npy --input_name 0 --output_name 1463 --model_name resnet50DAG --dag-batch-size-auto --transpose_input False --batchsize 1 > b1.txt && python grpc_predict_resnet.py --grpc_port 9000 --images_numpy_path ../../imgs.npy --labels_numpy_path ../../lbs.npy --input_name 0 --output_name 1463 --model_name resnet50DAG --dag-batch-size-auto --transpose_input False --batchsize 8 > b8.txt; -``` -*Note:* Results of running the client will be available in .txt files in the current directory. - -#### Output of the script -Output with `batchsize 1` stored in `b1.txt`: -```bash -cat b1.txt -Image data range: 0.0 : 255.0 -Start processing: - Model name: resnet50DAG - Iterations: 10 - Images numpy path: ../../imgs.npy - Numpy file shape: (10, 3, 224, 224) - -Iteration 1; Processing time: 21.16 ms; speed 47.25 fps -imagenet top results in a single batch: - 0 airliner 404 ; Correct match. -Iteration 2; Processing time: 8.08 ms; speed 123.79 fps -imagenet top results in a single batch: - 0 Arctic fox, white fox, Alopex lagopus 279 ; Correct match. -Iteration 3; Processing time: 104.76 ms; speed 9.55 fps -imagenet top results in a single batch: - 0 bee 309 ; Correct match. -Iteration 4; Processing time: 8.86 ms; speed 112.83 fps -imagenet top results in a single batch: - 0 golden retriever 207 ; Correct match. -Iteration 5; Processing time: 19.05 ms; speed 52.48 fps -imagenet top results in a single batch: - 0 gorilla, Gorilla gorilla 366 ; Correct match. -Iteration 6; Processing time: 9.31 ms; speed 107.47 fps -imagenet top results in a single batch: - 0 magnetic compass 635 ; Correct match. -Iteration 7; Processing time: 7.10 ms; speed 140.81 fps -imagenet top results in a single batch: - 0 peacock 84 ; Correct match. -Iteration 8; Processing time: 6.83 ms; speed 146.50 fps -imagenet top results in a single batch: - 0 pelican 144 ; Correct match. -Iteration 9; Processing time: 6.74 ms; speed 148.26 fps -imagenet top results in a single batch: - 0 snail 113 ; Correct match. -Iteration 10; Processing time: 7.08 ms; speed 141.26 fps -imagenet top results in a single batch: - 0 zebra 340 ; Correct match. - -processing time for all iterations -average time: 19.50 ms; average speed: 51.28 fps -median time: 8.00 ms; median speed: 125.00 fps -max time: 104.00 ms; min speed: 9.62 fps -min time: 6.00 ms; max speed: 166.67 fps -time percentile 90: 29.30 ms; speed percentile 90: 34.13 fps -time percentile 50: 8.00 ms; speed percentile 50: 125.00 fps -time standard deviation: 28.63 -time variance: 819.45 -Classification accuracy: 100.00 - -``` -Output with `batchsize 8` stored in `b8.txt`: -```bash -cat b8.txt -Image data range: 0.0 : 255.0 -Start processing: - Model name: resnet50DAG - Iterations: 1 - Images numpy path: ../../imgs.npy - Numpy file shape: (10, 3, 224, 224) - -Iteration 1; Processing time: 121.12 ms; speed 66.05 fps -imagenet top results in a single batch: - 0 airliner 404 ; Correct match. - 1 Arctic fox, white fox, Alopex lagopus 279 ; Correct match. - 2 bee 309 ; Correct match. - 3 golden retriever 207 ; Correct match. - 4 gorilla, Gorilla gorilla 366 ; Correct match. - 5 magnetic compass 635 ; Correct match. - 6 peacock 84 ; Correct match. - 7 pelican 144 ; Correct match. - -processing time for all iterations -average time: 121.00 ms; average speed: 66.12 fps -median time: 121.00 ms; median speed: 66.12 fps -max time: 121.00 ms; min speed: 66.12 fps -min time: 121.00 ms; max speed: 66.12 fps -time percentile 90: 121.00 ms; speed percentile 90: 66.12 fps -time percentile 50: 121.00 ms; speed percentile 50: 66.12 fps -time standard deviation: 0.00 -time variance: 0.00 -Classification accuracy: 100.00 - -``` -Each iteration presents the results of each infer request and details for each image in batch. - -With this feature, we were able to successfully run client simultaneously with different batch size parameters without performance impact from the model reloading. \ No newline at end of file diff --git a/docs/dynamic_input.md b/docs/dynamic_input.md index b4191601d5..18725d6207 100644 --- a/docs/dynamic_input.md +++ b/docs/dynamic_input.md @@ -8,8 +8,6 @@ hidden: ovms_docs_dynamic_shape_dynamic_model ovms_docs_dynamic_shape_binary_inputs -ovms_docs_dynamic_bs_auto_reload -ovms_docs_dynamic_shape_auto_reload ovms_docs_dynamic_bs_demultiplexer ovms_docs_dynamic_shape_custom_node ``` diff --git a/docs/dynamic_shape_auto_reload.md b/docs/dynamic_shape_auto_reload.md deleted file mode 100644 index dd2e256b59..0000000000 --- a/docs/dynamic_shape_auto_reload.md +++ /dev/null @@ -1,58 +0,0 @@ -# Dynamic Shape with Automatic Model Reloading{#ovms_docs_dynamic_shape_auto_reload} - -*NOTE*: This feature is deprecated. Use [model dynamic shapes instead](dynamic_shape_dynamic_model.md). - -## Introduction -This guide explains how to configure a model to accept input data in different shapes. In this example, it is done by reloading the model with a new shape each time it receives the request with a shape different than the one which is currently set. - -Enable dynamic shape via model reloading by setting the `shape` parameter to `auto`. To configure and use the dynamic batch size, take advantage of: - -- Example client in Python [face_detection.py](https://github.com/openvinotoolkit/model_server/blob/main/demos/face_detection/python/face_detection.py) that can be used to request inference with the desired input shape. - -- An example [face_detection_retail_0004](https://github.com/openvinotoolkit/open_model_zoo/blob/releases/2021/4/models/intel/face-detection-retail-0004/README.md) model. - -When using the `face_detection_retail_0004` model with the `face_detection.py` script, images are reloaded and resized to the desired width and height. Then, the output is processed from the server, and the inference results are displayed with bounding boxes drawn around the predicted faces. - -## Steps -Clone OpenVINO™ Model Server GitHub repository and enter `model_server` directory. -```bash -git clone https://github.com/openvinotoolkit/model_server.git -cd model_server -``` -#### Download the Pretrained Model -Download the model files and store them in the `models` directory -```bash -mkdir -p models/face_detection/1 -curl https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/3/face-detection-retail-0004/FP32/face-detection-retail-0004.bin https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/3/face-detection-retail-0004/FP32/face-detection-retail-0004.xml -o models/face_detection/1/face-detection-retail-0004.bin -o models/face_detection/1/face-detection-retail-0004.xml -``` - -#### Pull the Latest Model Server Image -Pull the latest version of OpenVINO™ Model Server from Docker Hub: -```bash -docker pull openvino/model_server:latest -``` - -#### Start the Model Server Container with the Model and Dynamic Batch Size -Start the container using the image pulled in the previous step and mount the `models` directory: -```bash -docker run --rm -d -v $(pwd)/models:/models -p 9000:9000 openvino/model_server:latest --model_name face-detection --model_path /models/face_detection --shape auto --port 9000 -``` - -#### Run the Client -```bash -cd demos/face_detection/python -virtualenv .venv -. .venv/bin/activate -pip install -r ../../common/python/requirements.txt -mkdir results_500x500 - -python face_detection.py --grpc_port 9000 --width 500 --height 500 --input_images_dir ../../common/static/images/people --output_dir results_500x500 - -mkdir results_600x400 - -python face_detection.py --grpc_port 9000 --width 600 --height 400 --input_images_dir ../../common/static/images/people --output_dir results_600x400 -``` -The results from running the client will be saved in the directory specified by `--output_dir` - - ->**NOTE**: reloading the model takes time and during each reload new requests are queued. Frequent model reloading may negatively affect overall performance. diff --git a/docs/dynamic_shape_binary_inputs.md b/docs/dynamic_shape_binary_inputs.md index 9bf8dcda81..acd333bf6f 100644 --- a/docs/dynamic_shape_binary_inputs.md +++ b/docs/dynamic_shape_binary_inputs.md @@ -28,10 +28,10 @@ Start the container with the image pulled in the previous step and mount the `mo docker run --rm -d -v $(pwd)/models:/models -p 9000:9000 openvino/model_server:latest --model_name resnet --model_path /models/resnet --layout NHWC:NCHW --port 9000 ``` -### Download ovmsclient Package +### Download Client Package ```bash -pip3 install ovmsclient +pip3 install tritonclient[grpc] numpy ``` ### Download a Sample Image and Label Mappings @@ -45,16 +45,23 @@ wget https://raw.githubusercontent.com/openvinotoolkit/model_server/main/demos/c ```bash echo ' -import numpy as np -from classes import imagenet_classes -from ovmsclient import make_grpc_client -client = make_grpc_client("localhost:9000") - -with open("zebra.jpeg", "rb") as f: - img = f.read() - -output = client.predict(inputs={ "0": img}, model_name= "resnet") -result_index = np.argmax(output[0]) +import numpy as np +from classes import imagenet_classes +import tritonclient.grpc as grpcclient + +client = grpcclient.InferenceServerClient(url="localhost:9000") +metadata = client.get_model_metadata("resnet") +input_name = metadata.inputs[0].name +output_name = metadata.outputs[0].name + +with open("zebra.jpeg", "rb") as f: + img_bytes = f.read() + +infer_input = grpcclient.InferInput(input_name, [1], "BYTES") +infer_input.set_data_from_numpy(np.array([img_bytes], dtype=np.object_)) +result = client.infer("resnet", [infer_input]) +output = result.as_numpy(output_name) +result_index = np.argmax(output[0]) print(imagenet_classes[result_index])' >> predict.py python predict.py diff --git a/docs/home.md b/docs/home.md index 8b791b9d82..207273b7ae 100644 --- a/docs/home.md +++ b/docs/home.md @@ -30,7 +30,7 @@ Model Server hosts models and makes them accessible to software components over ## Serving with OpenVINO Model Server -OpenVINO™ Model Server (OVMS) is a high-performance system for serving models. Implemented in C++ for scalability and optimized for deployment on Intel architectures. It uses the same API as [OpenAI](./genai.md), [Cohere](./model_server_rest_api_rerank.md), [KServe](./model_server_grpc_api_kfs.md) and [TensorFlow Serving](./model_server_rest_api_tfs.md) and while applying OpenVINO for inference execution. Inference service is provided via gRPC or REST API, making deploying new algorithms and AI experiments easy. +OpenVINO™ Model Server (OVMS) is a high-performance system for serving models. Implemented in C++ for scalability and optimized for deployment on Intel architectures. It uses the same API as [OpenAI](./genai.md), [Cohere](./model_server_rest_api_rerank.md) and [KServe](./model_server_grpc_api_kfs.md) while applying OpenVINO for inference execution. Inference service is provided via gRPC or REST API, making deploying new algorithms and AI experiments easy. Check how to write the client applications using [generative endpoints](./clients_genai.md). diff --git a/docs/model_server_grpc_api_tfs.md b/docs/model_server_grpc_api_tfs.md deleted file mode 100644 index 73c0a91ffc..0000000000 --- a/docs/model_server_grpc_api_tfs.md +++ /dev/null @@ -1,52 +0,0 @@ -# TensorFlow Serving compatible gRPC API {#ovms_docs_grpc_api_tfs} - -## Introduction -This document gives information about OpenVINO™ Model Server gRPC API compatible with TensorFlow Serving. It is documented in the proto buffer files in [tensorflow_serving_api](https://github.com/tensorflow/serving/tree/r2.9/tensorflow_serving/apis). -Using the gRPC interface is recommended for optimal performance due to its faster implementation of input data deserialization. gRPC achieves lower latency, especially with larger input messages like images. - -This document covers following API's endpoints coming from Tensorflow Serving gRPC API: -* [Model Status API](#model-status-api) -* [Model Metadata API ](#model-metadata-api) -* [Predict API](#predict-api) - -> **NOTE**: The implementations for *Predict*, *GetModelMetadata* and *GetModelStatus* function calls are currently available. -These are the most generic function calls and should address most of the usage scenarios. - -## Model Status API - -Gets information about the status of served models including Model Version - - [Get Model Status proto](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/apis/get_model_status.proto) defines three message definitions used while calling Status endpoint: *GetModelStatusRequest*, *ModelVersionStatus*, *GetModelStatusResponse* that are used to report all exposed versions including their state in their lifecycle. - - Read more about [Get Model Status API usage](https://github.com/openvinotoolkit/model_server/blob/main/client/python/tensorflow-serving-api/samples/README.md#model-status-api). - - -## Model Metadata API - -Gets information about the served models. A function called GetModelMetadata accepts model spec information as input and returns Signature Definition content in a format similar to TensorFlow Serving. - -[Get Model Metadata proto](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/apis/get_model_metadata.proto) has three message definitions: *SignatureDefMap*, *GetModelMetadataRequest*, *GetModelMetadataResponse*. - -Read more about [Get Model Metadata API usage](https://github.com/openvinotoolkit/model_server/blob/main/client/python/tensorflow-serving-api/samples/README.md#model-metadata-api). - - -## Predict API - -Endpoint for running an inference with loaded models or [DAGs](./dag_scheduler.md). - -[Predict proto](https://github.com/tensorflow/serving/blob/r2.9/tensorflow_serving/apis/predict.proto) has two message definitions: *PredictRequest* and *PredictResponse*. - * *PredictRequest* specifies information about the model spec, a map of input data serialized via -[TensorProto](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor.proto) to a string format. - * *PredictResponse* includes a map of outputs serialized by -[TensorProto](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor.proto) and information about the used model spec. - -Read more about [Predict API usage](https://github.com/openvinotoolkit/model_server/blob/main/client/python/tensorflow-serving-api/samples/README.md#predict-api) - -Also, using `string_val` field it is possible to send binary encoded images that would be preprocessed by OVMS using opencv and converted to OpenVINO-friendly format. For more information check [how binary data is handled in OpenVINO Model Server](./binary_input_tfs.md) - -## See Also - -- [Example client code](https://github.com/openvinotoolkit/model_server/blob/main/client/python/tensorflow-serving-api/samples/README.md) shows how to use GRPC API and REST API. -- [TensorFlow Serving](https://github.com/tensorflow/serving) -- [gRPC](https://grpc.io/) - diff --git a/docs/model_server_rest_api_tfs.md b/docs/model_server_rest_api_tfs.md deleted file mode 100644 index 315256acb8..0000000000 --- a/docs/model_server_rest_api_tfs.md +++ /dev/null @@ -1,380 +0,0 @@ -# TensorFlow Serving compatible RESTful API {#ovms_docs_rest_api_tfs} - -## Introduction -In addition to [gRPC APIs](./model_server_grpc_api_tfs.md) OpenVINO™ model server also supports RESTful APIs which follows the documentation from [tensorflow serving REST API](https://www.tensorflow.org/tfx/serving/api_rest). Both row and column formats of the request are implemented in these APIs. REST API is recommended when the primary goal is in reducing the number of client-side python dependencies and simpler application code. - -> **Note** : Only numerical data type is supported. - -This document covers the following API: -* [Model Status API](#model-status-api) -* [Model MetaData API](#model-metadata-api) -* [Predict API](#predict-api) -* [Config Reload API](#config-reload-api) -* [Config Status API](#config-status-api) - - -> **NOTE** : The implementations for Predict, GetModelMetadata and GetModelStatus function calls are currently available. These are the most generic function calls and should address most of the usage scenarios. - -## Model Status API -**Description** - -Get information about the status of served models. - -**URL** - -``` -GET http://${REST_URL}:${REST_PORT}/v1/models/${MODEL_NAME}/versions/${MODEL_VERSION} -``` - -> **Note** : Including /versions/${MODEL_VERSION} is optional. If omitted status for all versions is returned in the response. - -**Response format** - -If successful, returns a JSON of following format : -```JSON -{ - "model_version_status":[ - { - "version": |, - "state": |, - "status": { - "error_code": |, - "error_message": | - } - } - ] -} -``` - -**Usage Example** -``` -$ curl http://localhost:8001/v1/models/person-detection/versions/1 -``` - -```JSON -{ - 'model_version_status':[ - { - 'version': '1', - 'state': 'AVAILABLE', - 'status': { - 'error_code': 'OK', - 'error_message': '' - } - } - ] -} -``` -Read more about [Get Model Status API usage](https://github.com/openvinotoolkit/model_server/blob/main/client/python/tensorflow-serving-api/samples/README.md#model-status-api-1) - -## Model Metadata API -**Description** - -Get the metadata of a model in the model server. - -**URL** -``` -GET http://${REST_URL}:${REST_PORT}/v1/models/${MODEL_NAME}/versions/${MODEL_VERSION}/metadata -``` - -> **Note** :Including ${MODEL_VERSION} is optional. If omitted the model metadata for the latest version is returned in the response. - -**Response format** - -If successful, returns a JSON representation of [GetModelMetadataResponse](https://github.com/tensorflow/serving/blob/5369880e9143aa00d586ee536c12b04e945a977c/tensorflow_serving/apis/get_model_metadata.proto#L23) protobuf. - -**Usage example** -```JSON -$ curl http://localhost:8001/v1/models/person-detection/versions/1/metadata -{ - "modelSpec": { - "name": "person-detection", - "version": "1" - }, - "metadata": { - "signature_def": { - "@type": "type.googleapis.com/tensorflow.serving.SignatureDefMap", - "signatureDef": { - "serving_default": { - "inputs": { - "data": { - "name": "data_2:0", - "dtype": "DT_FLOAT", - "tensorShape": { - "dim": [ - { - "size": "1" - }, - { - "size": "3" - }, - { - "size": "400" - }, - { - "size": "600" - } - ] - } - } - }, - "outputs": { - "detection_out": { - "name": "detection_out_2:0", - "dtype": "DT_FLOAT", - "tensorShape": { - "dim": [ - { - "size": "1" - }, - { - "size": "1" - }, - { - "size": "200" - }, - { - "size": "7" - } - ] - } - } - }, - "methodName": "tensorflow/serving/predict" - } - } - } - } -} -``` - -Read more about [Get Model Metadata API usage](https://github.com/openvinotoolkit/model_server/blob/main/client/python/tensorflow-serving-api/samples/README.md#model-metadata-api-1) - -## Predict API -**Description** - -Endpoint for running an inference with loaded models or [DAGs](./dag_scheduler.md). - -**URL** -``` -POST http://${REST_URL}:${REST_PORT}/v1/models/${MODEL_NAME}/versions/${MODEL_VERSION}:predict -``` - -**Request Header** -```JSON -{ - // (Optional) Serving signature to use. - // If unspecified default serving signature is used. - "signature_name": , - - // Input Tensors in row ("instances") or columnar ("inputs") format. - // A request can have either of them but NOT both. - "instances": |<(nested)list>| - "inputs": |<(nested)list>| -} -``` - -Read [How to specify input tensors in row format](https://www.tensorflow.org/tfx/serving/api_rest#specifying_input_tensors_in_row_format) and [How to specify input tensors in column format](https://www.tensorflow.org/tfx/serving/api_rest#specifying_input_tensors_in_column_format) for more details. - -**Response** - -A request in [row format](https://www.tensorflow.org/tfx/serving/api_rest#specifying_input_tensors_in_row_format) has response formatted as follows : - -```JSON -{ - "predictions": |<(nested)list>| -} -``` -A request in [column format](https://www.tensorflow.org/tfx/serving/api_rest#specifying_input_tensors_in_column_format) has response formatted as follows : - -```JSON -{ - "outputs": |<(nested)list>| -} -``` - -Besides numerical values, it is possible to pass JPEG / PNG encoded images as binary inputs. They must be Base64 encoded in passed in `b64` key like below: - -```JSON -{ - "instances": [ - { - "image": { "b64": "aW1hZ2UgYnl0ZXM=" }, - }, - { - "image": { "b64": "YXdlc29tZSBpbWFnZSBieXRlcw==" }, - } - ] -} -``` - -On the server side, the binary encoded data is loaded using OpenCV which then converts it to OpenVINO-friendly data format for inference. - -Check [how binary data is handled in OpenVINO Model Server](./binary_input.md) for more information. - -Read more about [Predict API usage](https://github.com/openvinotoolkit/model_server/blob/main/client/python/tensorflow-serving-api/samples/README.md#predict-api-1) - -## Config Reload API -**Description** - -Sends requests via RESTful API to trigger config reloading and gets models and [DAGs](./dag_scheduler.md) statuses as a response. This endpoint can be used with disabled automatic config reload to ensure configuration changes are applied in a specific time and also to get confirmation about reload operation status. Typically this option is to be used when OVMS is started with a parameter `--file_system_poll_wait_seconds 0`. -Reload operation does not pass new configuration to OVMS server. The configuration file changes need to be applied by the OVMS administrator. The REST API call just initiate applying the configuration file which is already present. - -**URL** -``` -POST http://${REST_URL}:${REST_PORT}/v1/config/reload -``` - -**Flow** - -Flow after receiving the request: -1) If config file was changed - reload config. -2) If any model version directory was changed or a new version was added - reload this model. -3) If any model that is part of a DAG was changed or a new version was added - reload this pipeline. -4) In case there are no errors in the reload operation, the response includes the status of all models and DAGs, otherwise the error message is returned. - -**Request** - -To trigger reload, an HTTP POST request with the empty body should be sent on the given URL. Example `curl` command: - -``` -curl --request POST http://${REST_URL}:${REST_PORT}/v1/config/reload -``` - -**Response** - -In case of config reload success, the response contains JSON with aggregation of getModelStatus responses for all models and DAGs after reload is finished, along with operation status: -```JSON -{ -"": -{ - "model_version_status": [ - { - "version": |, - "state": |, - "status": - { - "error_code": |, - "error_message": | - } - }, - ... - ] -}, -... -} -``` - -In case of any failure during execution: - -```JSON -{ - "error": | -} -``` -When an operation succeeds HTTP response status code is - - `201` when config(config file or model version) was reloaded - - `200` when reload was not required, already applied or OVMS was started in single model mode - -When an operation fails another status code is returned. -Possible messages returned on error: - -- obtaining config file change time failed (file is not existing or cannot be accessed): -```JSON -{ - "error": "Config file not found or cannot open." -} -``` -- config file was changed and config reloading failed (file content is not a valid JSON, any of model or DAG config is incorrect): -```JSON -{ - "error": "Reloading config file failed. Check server logs for more info." -} -``` - -- config file was not changed and model versions reloading failed (model directory was removed): -```JSON -{ - "error": "Reloading models versions failed. Check server logs for more info." -} -``` - -- retrieving the status of one of the models failed: -```JSON -{ - "error": "Retrieving all model statuses failed. Check server logs for more info." -} -``` - -- converting model status responses to JSON failed: -```JSON -{ - "error": "Serializing model statuses to JSON failed. Check server logs for more info." -} -``` - -Even if one of models reload failed other may be working properly. To check state of loaded models use [Config Status API](#config-status-api). To detect exact cause of errors described above analyzing sever logs may be necessary. - -## Config Status API -**Description** - -Sends requests via RESTful API to get a response that contains an aggregation of getModelStatus responses for all models and [DAGs](./dag_scheduler.md). - -**URL** -``` -GET http://${REST_URL}:${REST_PORT}/v1/config -``` -**Request** -To trigger this API HTTP GET request should be sent on a given URL. Example `curl` command: - -``` -curl --request GET http://${REST_URL}:${REST_PORT}/v1/config -``` - -**Response** -In case of success, the response contains JSON with aggregation of getModelStatus responses for all models and DAGs, along with operation status: - -```JSON -{ -"": -{ - "model_version_status": [ - { - "version": |, - "state": |, - "status": - { - "error_code": |, - "error_message": | - } - }, - ... - ] -}, -... -} -``` - -In case of any failure during execution: - -```JSON -{ - "error": | -} -``` -When operation succeeded HTTP response status code is 200, otherwise, another code is returned. -Possible messages returned on error: - -- retrieving the status of one of the models failed: -```JSON -{ - "error": "Retrieving all model statuses failed. Check server logs for more info." -} -``` - -- converting model status responses to JSON failed: -```JSON -{ - "error": "Serializing model statuses to json failed. Check server logs for more info." -} -``` diff --git a/docs/online_config_changes.md b/docs/online_config_changes.md index 11dabd7b03..c359febb1a 100644 --- a/docs/online_config_changes.md +++ b/docs/online_config_changes.md @@ -5,7 +5,7 @@ OpenVINO Model Server monitors changes to the configuration file and applies req 1. Automatically, with an interval defined by the parameter `--file_system_poll_wait_seconds`. (introduced in version 2021.1) -2. On demand, using the [Config Reload API](./model_server_rest_api_tfs.md). (introduced in version 2021.3) +2. On demand, using the Config Reload REST API endpoint `POST /v1/config/reload`. (introduced in version 2021.3) Configuration reload triggers the following operations: @@ -22,5 +22,5 @@ Model Server behavior in case of errors during configuration reloading: - if a new `config.json` is not compliant with JSON schema, no changes are applied to the served models. - if the new model, [DAG](./dag_scheduler.md), [MediaPipe Graph](./mediapipe.md) or [custom loader](./custom_model_loader.md) has an invalid configuration, it will be ignored until the next configuration reload. Configurations may be invalid due to incorrect paths (leading to non-existent directories), forbidden values in the config, invalid [DAG](./dag_scheduler.md) structure (e.g. cycle found in a graph), invalid [MediaPipe Graph](./mediapipe.md) (e.g. using not accessible calculator), etc. - an error occurs when a model, [DAG](./dag_scheduler.md), [MediaPipe Graph](./mediapipe.md) or [custom loader](./custom_model_loader.md) is reloading but does not prevent the reload of the remaining updated models. -- errors from configuration reload are triggered internally and saved in the logs. If [Config Reload API](./model_server_rest_api_tfs.md) is used, the response will also contain an error message. +- errors from configuration reload are triggered internally and saved in the logs. If the Config Reload API is used, the response will also contain an error message. diff --git a/docs/ovms_quickstart.md b/docs/ovms_quickstart.md index dcc54e98f4..a8cc295b3c 100644 --- a/docs/ovms_quickstart.md +++ b/docs/ovms_quickstart.md @@ -133,4 +133,4 @@ In our case, it will be a modified input image with bounding boxes indicating de or [PaddlePaddle model demo](../demos/classification_using_paddlepaddle_model/python/README.md). Congratulations, you have completed the QuickStart guide. Try other Model Server [demos](../demos/README.md) or explore more [features](features.md) to create your application. -Check also how to write the client code using [TFS API](./clients_tfs.md) and [KServe API](./clients_kfs.md) +Check also how to write the client code using [KServe API](./clients_kfs.md) diff --git a/docs/performance_tuning.md b/docs/performance_tuning.md index 74a890eb1b..d919340ae9 100644 --- a/docs/performance_tuning.md +++ b/docs/performance_tuning.md @@ -101,7 +101,7 @@ By default, OpenVINO Model Server will enable CPU threads pinning for better per While using REST API, you can adjust the data format to optimize the communication and deserialization from json format. Here are some tips to effectively use REST interface when working with OpenVINO Model Server: -- use [binary data format](binary_input.md) when possible(for TFS API binary data format is support only for JPEG/PNG inputs, for KFS API there are no such limitations ) - binary data representation is smaller in terms of request size and easier to process on the server side. +- use [binary data format](binary_input.md) when possible - binary data representation is smaller in terms of request size and easier to process on the server side. - when working with images, consider sending JPEG/PNG directly - compressed data will greatly reduce the traffic and speed up the communication. - with JPEG/PNG it is the most efficient to send the images with the resolution of the configured model. It will avoid image resizing on the server to fit the model. - if you decide to send data inside JSON object, try to adjust the numerical data type to reduce the message size i.e. reduce the numbers precisions in the json message with a command similar to `np.round(imgs.astype(np.float),decimals=2)`. diff --git a/docs/text_handling.md b/docs/text_handling.md index 750132a480..fccc39959b 100644 --- a/docs/text_handling.md +++ b/docs/text_handling.md @@ -36,4 +36,4 @@ Such data in a tensor format can be passed to the custom node to perform the pre Similarly, a custom node can perform string detokenization and return a string to the model server client. -The client API snippets with string data format are included in [KServe API](./clients_kfs.md) and [TFS API](./clients_tfs.md). +The client API snippets with string data format are included in [KServe API](./clients_kfs.md). diff --git a/docs/tf_model_binary_input.md b/docs/tf_model_binary_input.md index ba2beabf23..f308f0fd52 100644 --- a/docs/tf_model_binary_input.md +++ b/docs/tf_model_binary_input.md @@ -60,35 +60,43 @@ docker run -d -p 9000:9000 -v ${PWD}/resnet_v2/models:/models openvino/model_ser ```bash git clone https://github.com/openvinotoolkit/model_server.git -cd model_server/client/python/tensorflow-serving-api/samples -virtualenv .venv +cd model_server/client/python/kserve-api/samples +python3 -m venv .venv . .venv/bin/activate pip install -r requirements.txt -python grpc_predict_binary_resnet.py --grpc_address localhost --model_name resnet --input_name input_tensor --output_name softmax_tensor:0 --grpc_port 9000 --images ../../resnet_input_images.txt +python grpc_infer_binary_resnet.py --grpc_address localhost --model_name resnet --input_name input_tensor --output_name softmax_tensor:0 --grpc_port 9000 --images ../../resnet_input_images.txt Start processing: Model name: resnet - Images list file: ../../resnet_input_images.txt -Batch: 0; Processing time: 17.62 ms; speed 56.76 fps - 1 airliner 404 ; Correct match. -Batch: 1; Processing time: 13.20 ms; speed 75.75 fps - 2 Arctic fox, white fox, Alopex lagopus 279 ; Correct match. -Batch: 2; Processing time: 11.60 ms; speed 86.19 fps - 3 bee 309 ; Correct match. -Batch: 3; Processing time: 10.48 ms; speed 95.37 fps - 4 golden retriever 207 ; Correct match. -Batch: 4; Processing time: 9.73 ms; speed 102.76 fps - 5 gorilla, Gorilla gorilla 366 ; Correct match. -Batch: 5; Processing time: 9.71 ms; speed 103.04 fps - 6 magnetic compass 635 ; Correct match. -Batch: 6; Processing time: 9.84 ms; speed 101.64 fps - 7 peacock 84 ; Correct match. -Batch: 7; Processing time: 9.60 ms; speed 104.16 fps - 8 pelican 144 ; Correct match. -Batch: 8; Processing time: 9.86 ms; speed 101.42 fps - 9 snail 113 ; Correct match. -Batch: 9; Processing time: 10.68 ms; speed 93.68 fps - 10 zebra 340 ; Correct match. -Overall accuracy= 100.0 % -Average latency= 10.6 ms +Iteration 0; Processing time: 14.48 ms; speed 69.05 fps +imagenet top results in a single batch: + 0 airliner 404 ; Correct match. +Iteration 1; Processing time: 13.59 ms; speed 73.58 fps +imagenet top results in a single batch: + 0 Arctic fox, white fox, Alopex lagopus 279 ; Correct match. +Iteration 2; Processing time: 12.07 ms; speed 82.84 fps +imagenet top results in a single batch: + 0 bee 309 ; Correct match. +Iteration 3; Processing time: 11.68 ms; speed 85.62 fps +imagenet top results in a single batch: + 0 golden retriever 207 ; Correct match. +Iteration 4; Processing time: 11.97 ms; speed 83.56 fps +imagenet top results in a single batch: + 0 gorilla, Gorilla gorilla 366 ; Correct match. +Iteration 5; Processing time: 11.82 ms; speed 84.60 fps +imagenet top results in a single batch: + 0 magnetic compass 635 ; Correct match. +Iteration 6; Processing time: 12.55 ms; speed 79.71 fps +imagenet top results in a single batch: + 0 peacock 84 ; Correct match. +Iteration 7; Processing time: 12.14 ms; speed 82.36 fps +imagenet top results in a single batch: + 0 pelican 144 ; Correct match. +Iteration 8; Processing time: 11.99 ms; speed 83.38 fps +imagenet top results in a single batch: + 0 snail 113 ; Correct match. +Iteration 9; Processing time: 13.55 ms; speed 73.81 fps +imagenet top results in a single batch: + 0 zebra 340 ; Correct match. +Classification accuracy: 100.00 ``` \ No newline at end of file diff --git a/docs/writing_app.md b/docs/writing_app.md index 344b4eff56..5554d1142a 100644 --- a/docs/writing_app.md +++ b/docs/writing_app.md @@ -7,7 +7,6 @@ hidden: --- Generative AI Use Cases -TensorFlow Serving API KServe API OpenVINO Model Server C-API ``` @@ -15,20 +14,17 @@ OpenVINO Model Server C-API OpenVINO™ Model Server supports multiple APIs, for easy integration with systems using one of them for inference. The APIs are: -* one compatible with TensorFlow Serving, * KServe API for inference * OpenAI API for text generation. * OpenAI API for embeddings * Cohere API for reranking -Both TFS and KServe APIs work on gRPC and REST interfaces. +KServe API works on both gRPC and REST interfaces. The REST API endpoints for generative use cases support both streamed and unary responses. Check the following articles to learn more about the supported APIs: -- [TensorFlow Serving gRPC API](./model_server_grpc_api_tfs.md) - [KServe gRPC API](./model_server_grpc_api_kfs.md) -- [TensorFlow Serving REST API](./model_server_rest_api_tfs.md) - [KServe REST API](./model_server_rest_api_kfs.md) - [OpenAI chat completions API](./model_server_rest_api_chat.md) - [OpenAI completions API](./model_server_rest_api_completions.md) @@ -39,6 +35,5 @@ Check the following articles to learn more about the supported APIs: - [OpenAI speech generation API](./model_server_rest_api_text_to_speech.md) In this section you can find short code samples to interact with OpenVINO Model Server endpoints via: -- [TensorFlow Serving API](./clients_tfs.md) - [KServe API](./clients_kfs.md) - [Generative AI clients](./clients_genai.md) diff --git a/external/listen.patch b/external/listen.patch deleted file mode 100644 index e2bc57aed9..0000000000 --- a/external/listen.patch +++ /dev/null @@ -1,49 +0,0 @@ -diff -uraN a/tensorflow_serving/util/net_http/server/internal/evhttp_server.cc b/tensorflow_serving/util/net_http/server/internal/evhttp_server.cc ---- a/tensorflow_serving/util/net_http/server/internal/evhttp_server.cc 2020-10-22 08:44:39.000000000 +0000 -+++ b/tensorflow_serving/util/net_http/server/internal/evhttp_server.cc 2020-10-23 09:54:03.911295393 +0000 -@@ -215,13 +215,14 @@ - } - - const int port = server_options_->ports().front(); -+ const std::string address = server_options_->address(); - - // "::" => in6addr_any - ev_uint16_t ev_port = static_cast(port); -- ev_listener_ = evhttp_bind_socket_with_handle(ev_http_, "::", ev_port); -+ ev_listener_ = evhttp_bind_socket_with_handle(ev_http_, address.c_str(), ev_port); - if (ev_listener_ == nullptr) { - // in case ipv6 is not supported, fallback to inaddr_any -- ev_listener_ = evhttp_bind_socket_with_handle(ev_http_, nullptr, ev_port); -+ ev_listener_ = evhttp_bind_socket_with_handle(ev_http_, address.c_str(), ev_port); - if (ev_listener_ == nullptr) { - NET_LOG(ERROR, "Couldn't bind to port %d", port); - return false; -diff -uraN a/tensorflow_serving/util/net_http/server/public/httpserver_interface.h b/tensorflow_serving/util/net_http/server/public/httpserver_interface.h ---- a/tensorflow_serving/util/net_http/server/public/httpserver_interface.h 2020-10-22 08:44:39.000000000 +0000 -+++ b/tensorflow_serving/util/net_http/server/public/httpserver_interface.h 2020-10-23 10:25:10.170275251 +0000 -@@ -61,6 +61,17 @@ - ports_.emplace_back(port); - } - -+ void SetAddress(const std::string& address) { -+ address_ = address; -+ } -+ -+ std::string address() { -+ if(address_ == "") { -+ return "0.0.0.0"; -+ } -+ return address_; -+ } -+ - // The default executor for running I/O event polling. - // This is a mandatory option. - void SetExecutor(std::unique_ptr executor) { -@@ -74,6 +85,7 @@ - private: - std::vector ports_; - std::unique_ptr executor_; -+ std::string address_; - }; - - // Options to specify when registering a handler (given a uri pattern). diff --git a/external/net_http.patch b/external/net_http.patch deleted file mode 100644 index 72f7821e46..0000000000 --- a/external/net_http.patch +++ /dev/null @@ -1,52 +0,0 @@ -diff --git a/tensorflow_serving/util/BUILD b/tensorflow_serving/util/BUILD -index 4802a79f..4f676b8b 100644 ---- a/tensorflow_serving/util/BUILD -+++ b/tensorflow_serving/util/BUILD -@@ -4,7 +4,7 @@ load("//tensorflow_serving:serving.bzl", "serving_proto_library") - - package( - default_visibility = [ -- "//tensorflow_serving:internal", -+ "//visibility:public", - ], - features = ["-layering_check"], - ) -diff --git a/tensorflow_serving/util/net_http/server/internal/evhttp_server.cc b/tensorflow_serving/util/net_http/server/internal/evhttp_server.cc -index 36c925a8..78e0eb66 100644 ---- a/tensorflow_serving/util/net_http/server/internal/evhttp_server.cc -+++ b/tensorflow_serving/util/net_http/server/internal/evhttp_server.cc -@@ -105,13 +105,18 @@ bool EvHTTPServer::Initialize() { - return false; - } - -+ std::size_t maxBodySize = 1024 * 1024 * 1024; -+ evhttp_set_max_body_size(ev_http_, maxBodySize); -+ std::size_t maxHeadersSize = 8 * 1024; -+ evhttp_set_max_headers_size(ev_http_, maxHeadersSize); -+ - // By default libevents only allow GET, POST, HEAD, PUT, DELETE request - // we have to manually turn OPTIONS and PATCH flag on documentation: - // (http://www.wangafu.net/~nickm/libevent-2.0/doxygen/html/http_8h.html) - evhttp_set_allowed_methods( - ev_http_, EVHTTP_REQ_GET | EVHTTP_REQ_POST | EVHTTP_REQ_HEAD | - EVHTTP_REQ_PUT | EVHTTP_REQ_DELETE | EVHTTP_REQ_OPTIONS | -- EVHTTP_REQ_PATCH); -+ EVHTTP_REQ_CONNECT | EVHTTP_REQ_TRACE | EVHTTP_REQ_PATCH); - evhttp_set_gencb(ev_http_, &DispatchEvRequestFn, this); - - return true; -diff --git a/tensorflow_serving/util/net_http/server/public/BUILD b/tensorflow_serving/util/net_http/server/public/BUILD -index 1953a10d..f0f6efdc 100644 ---- a/tensorflow_serving/util/net_http/server/public/BUILD -+++ b/tensorflow_serving/util/net_http/server/public/BUILD -@@ -1,9 +1,7 @@ - # Description: a lightweight http server and related utils to support Web clients - - package(default_visibility = [ -- "//tensorflow_serving:internal", -- "//tensorflow_serving/util/net_http:__subpackages__", -- "//tensorflow_serving/util/net_http/public:http_server_clients", -+ "//visibility:public", - ]) - - licenses(["notice"]) diff --git a/external/partial_2.18.patch b/external/partial_2.18.patch deleted file mode 100644 index d1b0e6d10a..0000000000 --- a/external/partial_2.18.patch +++ /dev/null @@ -1,201 +0,0 @@ -diff --git a/tensorflow_serving/util/net_http/server/internal/evhttp_request.cc b/tensorflow_serving/util/net_http/server/internal/evhttp_request.cc -index 75bf2f38..4a0771e2 100644 ---- a/tensorflow_serving/util/net_http/server/internal/evhttp_request.cc -+++ b/tensorflow_serving/util/net_http/server/internal/evhttp_request.cc -@@ -121,18 +121,44 @@ bool ParsedEvRequest::decode() { - return true; - } - -+static void connection_close_callback(struct evhttp_connection *conn, void *arg) { -+ EvHTTPRequest* req = (EvHTTPRequest*)arg; -+ req->ExecuteDisconnectionCallback(); -+} -+ - EvHTTPRequest::EvHTTPRequest(std::unique_ptr request, - ServerSupport* server) - : server_(server), - parsed_request_(std::move(request)), -- output_buf(nullptr) {} -+ output_buf(nullptr) { -+ struct evhttp_connection *conn = evhttp_request_get_connection(parsed_request_->request); -+ evhttp_connection_set_closecb(conn, connection_close_callback, (void*)this); -+} - - EvHTTPRequest::~EvHTTPRequest() { -+ struct evhttp_connection *conn = evhttp_request_get_connection(parsed_request_->request); -+ if (conn != NULL) { -+ evhttp_connection_set_closecb(conn, NULL, NULL); -+ } -+ - if (output_buf != nullptr) { - evbuffer_free(output_buf); - } - } - -+void EvHTTPRequest::RegisterDisconnectionCallback(std::function callback) { -+ std::unique_lock lk(this->disconnection_mx_); -+ this->disconnected_callback_ = std::move(callback); -+} -+ -+void EvHTTPRequest::ExecuteDisconnectionCallback() { -+ std::unique_lock lk(this->disconnection_mx_); -+ this->is_disconnected_ = true; -+ if (this->disconnected_callback_) -+ this->disconnected_callback_(); -+} -+ -+ - absl::string_view EvHTTPRequest::uri_path() const { - return parsed_request_->path_and_query; - } -@@ -341,12 +367,18 @@ void EvHTTPRequest::AppendResponseHeader(absl::string_view header, - } - } - --void EvHTTPRequest::PartialReplyWithStatus(HTTPStatusCode status) { -- NET_LOG(FATAL, "PartialReplyWithStatus not implemented."); -+void EvHTTPRequest::PartialReplyWithStatus(std::string data, HTTPStatusCode status) { -+ bool result = -+ server_->EventLoopSchedule([this, data = std::move(data), status]() mutable { EvPartialSendReply(std::move(data), status); }); -+ -+ if (!result) { -+ NET_LOG(ERROR, "Failed to EventLoopSchedule PartialReply()"); -+ Abort(); -+ } - } - --void EvHTTPRequest::PartialReply() { -- NET_LOG(FATAL, "PartialReplyWithStatus not implemented."); -+void EvHTTPRequest::PartialReply(std::string data) { -+ return PartialReplyWithStatus(std::move(data), HTTPStatusCode::OK); - } - - ServerRequestInterface::CallbackStatus -@@ -374,6 +406,33 @@ void EvHTTPRequest::EvSendReply(HTTPStatusCode status) { - delete this; - } - -+void EvHTTPRequest::EvPartialSendReply(std::string data, HTTPStatusCode status) { -+ std::unique_lock lk(this->disconnection_mx_); -+ if (this->is_disconnected_) -+ return; -+ if (!this->is_reply_started_) { -+ evhttp_send_reply_start(parsed_request_->request, static_cast(status), "reply start"); -+ this->is_reply_started_ = true; -+ } -+ evbuffer_add(output_buf, data.data(), static_cast(data.size())); -+ evhttp_send_reply_chunk(parsed_request_->request, output_buf); -+} -+ -+void EvHTTPRequest::EvPartialReplyEnd() { -+ std::unique_lock lk(this->disconnection_mx_); -+ if (!this->is_disconnected_) { -+ if (!this->is_reply_started_) { -+ // Start before we end can end the reply -+ evhttp_send_reply_start(parsed_request_->request, HTTP_OK, "no messages"); -+ } -+ -+ evhttp_send_reply_end(parsed_request_->request); -+ } -+ -+ server_->DecOps(); -+ delete this; -+} -+ - void EvHTTPRequest::Reply() { ReplyWithStatus(HTTPStatusCode::OK); } - - // Treats this as 500 for now and let libevent decide what to do -@@ -384,6 +443,15 @@ void EvHTTPRequest::Abort() { - delete this; - } - -+void EvHTTPRequest::PartialReplyEnd() { -+ bool result = -+ server_->EventLoopSchedule([this]() { EvPartialReplyEnd(); }); -+ -+ if (!result) { -+ NET_LOG(ERROR, "Failed to EventLoopSchedule PartialReplyEnd()"); -+ Abort(); -+ } -+} - } // namespace net_http - } // namespace serving - } // namespace tensorflow -diff --git a/tensorflow_serving/util/net_http/server/internal/evhttp_request.h b/tensorflow_serving/util/net_http/server/internal/evhttp_request.h -index 2f8e601d..54dc17b0 100644 ---- a/tensorflow_serving/util/net_http/server/internal/evhttp_request.h -+++ b/tensorflow_serving/util/net_http/server/internal/evhttp_request.h -@@ -19,7 +19,9 @@ limitations under the License. - #define TENSORFLOW_SERVING_UTIL_NET_HTTP_SERVER_INTERNAL_EVHTTP_REQUEST_H_ - - #include -+#include - #include -+#include - #include - - #include "tensorflow_serving/util/net_http/server/internal/server_support.h" -@@ -93,8 +95,13 @@ class EvHTTPRequest final : public ServerRequestInterface { - void AppendResponseHeader(absl::string_view header, - absl::string_view value) override; - -- void PartialReplyWithStatus(HTTPStatusCode status) override; -- void PartialReply() override; -+ void PartialReplyWithStatus(std::string data, HTTPStatusCode status) override; -+ void PartialReply(std::string data) override; -+ void PartialReplyEnd() override; -+ -+ bool IsDisconnected() const override { return this->is_disconnected_; } -+ void RegisterDisconnectionCallback(std::function callback) override; -+ void ExecuteDisconnectionCallback(); - - CallbackStatus PartialReplyWithFlushCallback( - std::function callback) override; -@@ -114,6 +121,8 @@ class EvHTTPRequest final : public ServerRequestInterface { - - private: - void EvSendReply(HTTPStatusCode status); -+ void EvPartialSendReply(std::string data, HTTPStatusCode status); -+ void EvPartialReplyEnd(); - - // Returns true if the data needs be uncompressed - bool NeedUncompressGzipContent(); -@@ -133,6 +142,11 @@ class EvHTTPRequest final : public ServerRequestInterface { - std::unique_ptr parsed_request_; - - evbuffer* output_buf; // owned by this -+ bool is_reply_started_{false}; -+ bool is_disconnected_{false}; -+ -+ std::function disconnected_callback_; -+ std::mutex disconnection_mx_; - }; - - } // namespace net_http -diff --git a/tensorflow_serving/util/net_http/server/public/server_request_interface.h b/tensorflow_serving/util/net_http/server/public/server_request_interface.h -index e5f4b05f..47554412 100644 ---- a/tensorflow_serving/util/net_http/server/public/server_request_interface.h -+++ b/tensorflow_serving/util/net_http/server/public/server_request_interface.h -@@ -143,8 +143,9 @@ class ServerRequestInterface { - // Trying to modify headers or specifying a status after the first - // PartialReply() is called is considered a programming error and - // the underlying behavior is undefined. -- virtual void PartialReplyWithStatus(HTTPStatusCode status) = 0; -- virtual void PartialReply() = 0; -+ virtual void PartialReplyWithStatus(std::string data, HTTPStatusCode status) = 0; -+ virtual void PartialReply(std::string data) = 0; -+ virtual void PartialReplyEnd() = 0; - - // Similar to PartialReply() but with an on_flush callback which will be - // invoked when the response data has been completely flushed by the -@@ -182,6 +183,10 @@ class ServerRequestInterface { - // by the server runtime. - virtual void Abort() = 0; - -+ // Helpers for handling disconnection states -+ virtual bool IsDisconnected() const = 0; -+ virtual void RegisterDisconnectionCallback(std::function callback) = 0; -+ - protected: - ServerRequestInterface() = default; - diff --git a/src/BUILD b/src/BUILD index 2b0549620b..b542a3cb86 100644 --- a/src/BUILD +++ b/src/BUILD @@ -16,7 +16,7 @@ load("@bazel_skylib//lib:selects.bzl", "selects") load("@mediapipe//mediapipe/framework:more_selects.bzl", "more_selects") load("//:common_settings.bzl", - "COMMON_STATIC_TEST_COPTS", "COMMON_STATIC_LIBS_COPTS", "COMMON_STATIC_LIBS_LINKOPTS", "COMMON_FUZZER_COPTS", "COMMON_FUZZER_LINKOPTS", "COMMON_LOCAL_DEFINES", "COPTS_DROGON", + "COMMON_STATIC_TEST_COPTS", "COMMON_STATIC_LIBS_COPTS", "COMMON_STATIC_LIBS_LINKOPTS", "COMMON_FUZZER_COPTS", "COMMON_FUZZER_LINKOPTS", "COMMON_LOCAL_DEFINES", "create_config_settings", "PYBIND_DEPS", "ovms_cc_library") COPTS_OV_TRACE = select({ @@ -32,7 +32,7 @@ COPTS_CLOUD = select({ "//:not_disable_cloud" : ["-DCLOUD_DISABLE=0"], }) -COPTS_TESTS = COMMON_STATIC_TEST_COPTS + COPTS_DROGON + COPTS_CLOUD +COPTS_TESTS = COMMON_STATIC_TEST_COPTS + COPTS_CLOUD config_setting( @@ -68,7 +68,6 @@ cc_shared_library( dynamic_deps = [], static_deps = [ #"//src/filesystem:libovmsfilesystemfactory", // TODO missing symbols - "@tensorflow_serving//:__subpackages__", "@minitrace//:__subpackages__", "@com_github_gabime_spdlog//:__subpackages__", "@fmtlib//:__subpackages__", @@ -355,7 +354,6 @@ ovms_cc_library( "ovms_exit_codes", ], visibility = ["//visibility:public",], - additional_copts = COPTS_DROGON, ) ovms_cc_library( name = "libovms_cliparser", @@ -517,7 +515,7 @@ ovms_cc_library( }) + [ "//src/mediapipe_internal:libovms_mediapipe", "predict_request_validation_utils", - "tfs_utils", + "tensorflow_type_utils", "libovms_kfs_utils", "libovms_kfs_grpc_inference_service_h", "//src/kfserving_api:kfserving_api_cpp", @@ -627,9 +625,6 @@ ovms_cc_library( deps = [ "@com_github_tencent_rapidjson//:rapidjson", "@com_google_absl//absl/strings", - "@tensorflow_serving//tensorflow_serving/apis:prediction_service_cc_proto", - "@tensorflow_serving//tensorflow_serving/util:json_tensor", - "@org_tensorflow//tensorflow/core:framework", "//src/kfserving_api:kfserving_api_cpp", "//src/utils:rapidjson_utils", "libovms_kfs_grpc_inference_service_h", @@ -639,7 +634,6 @@ ovms_cc_library( "libovmsprofiler", "libovmsstatus", "libovmstimer", - "tfs_utils", ], visibility = ["//visibility:public"], ) @@ -663,6 +657,27 @@ ovms_cc_library( visibility = ["//visibility:public"], ) +ovms_cc_library( + name = "config_status_utils", + hdrs = ["config_status_utils.hpp"], + srcs = ["config_status_utils.cpp"], + deps = [ + "model", + "modelinstance", + "model_metric_reporter", + "libovms_model_instance_provider", + "libovms_servable_name_checker", + "libovms_single_version_servable_definition", + "libovms_execution_context", + "libovmsmodelversionstatus", + "libovmsstatus", + "//src/dags:pipelinedefinitionstatus", + "//src/metrics:libovmsmetrics", + "//src/port:rapidjson_stringbuffer", + "//src/port:rapidjson_writer", + ], +) + ovms_cc_library( name = "model", hdrs = ["model.hpp"], @@ -706,8 +721,6 @@ ovms_cc_library( ], srcs = [ "capi_frontend/capi.cpp", - "get_model_metadata_impl.cpp", - "get_model_metadata_impl.hpp", "http_server.hpp", "http_server.cpp", "http_rest_api_handler.hpp", @@ -717,10 +730,6 @@ ovms_cc_library( "grpcservermodule.cpp", "grpcservermodule.hpp", "kfs_frontend/kfs_grpc_inference_service.cpp", - "model_service.hpp", - "model_service.cpp", - "prediction_service.cpp", - "prediction_service.hpp", "server.cpp", "server.hpp", ], @@ -755,20 +764,14 @@ ovms_cc_library( "//src/rerank:rerankcalculator_ov", "//src/llm:llmcalculator", ], - }) + select({ - "//:enable_drogon": [ - "libdrogon_http_server", - "libmultipart_parser_drogon_impl", - ], - "//conditions:default" : [ - "libnet_http_server", - # Add net-http implementation here if we ever want one - ], }) + [ + "libdrogon_http_server", + "libmultipart_parser_drogon_impl", "cpp_headers", "libovms_shutdown_state", "ovms_header", "capi_backend_impl", + "config_status_utils", "libovms_capi_servable_metadata", "libmodelconfigjsonparser", "modelconfig", @@ -800,7 +803,6 @@ ovms_cc_library( "//src/dags:pipeline_factory", "//src/dags:dag_capi", "//src/dags:dag_kfs", - "//src/dags:dag_tfs", "//src/dags:custom_node_library_manager", "//src/dags:node", "//src/dags:node_library", @@ -844,7 +846,6 @@ ovms_cc_library( "libovmsprecision_configuration", "libovms_tensorinfo", "libovmstimer", - "tfs_utils", "libovms_version", "libovms_version_impl", "serialization_common", @@ -869,7 +870,6 @@ ovms_cc_library( "//src/servables_config_manager_module:servablesconfigmanagermodule", "predict_request_validation_utils", # to be removed when capi has its own lib and added there @atobisze "kfs_backend_impl", - "tfs_backend_impl", "anonymous_input_name", "libovms_servable_name_checker", "//src/metrics:libovms_metric_provider", @@ -891,7 +891,7 @@ ovms_cc_library( ], }), visibility = ["//visibility:public",], - additional_copts = COPTS_DROGON + COPTS_OV_TRACE, + additional_copts = COPTS_OV_TRACE, additional_linkopts = select({ "//conditions:default": ["-lOpenCL"], # TODO make as direct dependency "//src:windows" : ["/DEFAULTLIB:Rpcrt4.lib"],}), @@ -1266,21 +1266,14 @@ cc_import( ) ovms_cc_library( - name = "tfs_utils", - hdrs = ["tfs_frontend/tfs_utils.hpp",], - srcs = [ - "tfs_frontend/tfs_utils.cpp",], + + name = "tensorflow_type_utils", + hdrs = ["tensorflow_type_utils.hpp"], + srcs = ["tensorflow_type_utils.cpp"], deps = [ - "@tensorflow_serving//tensorflow_serving/apis:prediction_service_cc_proto", - "@tensorflow_serving//tensorflow_serving/apis:model_service_cc_proto", - "@org_tensorflow//tensorflow/core:framework", # Eigen Tensor - "libovmslogging", - "libovmsprofiler", + "@org_tensorflow//tensorflow/core:framework", + "libovmsprecision", "libovmsstatus", - "opencv_dep", - "tensor_conversion_common", - "inference_request_common", - "predict_request_validation_utils_impl", ], visibility = ["//visibility:public"], ) @@ -1297,7 +1290,6 @@ ovms_cc_library( "//src/port:rapidjson_stringbuffer", ], visibility = ["//visibility:public"], - additional_copts = COPTS_DROGON, ) @@ -1491,21 +1483,6 @@ ovms_cc_library( ], visibility = ["//visibility:public"], ) -ovms_cc_library( - name = "tfs_request_utils", - hdrs = ["tfs_frontend/tfs_request_utils.hpp",], - srcs = ["tfs_frontend/tfs_request_utils.cpp",], - deps = [ - "libovmslogging", - "tfs_utils", - "libovmsstatus", - "libovmsshape", - "extractchoice", - "requesttensorextractor", - ], - visibility = ["//visibility:public"], - alwayslink = 1, -) ovms_cc_library( name = "kfs_backend_impl", srcs = [ @@ -1530,30 +1507,6 @@ ovms_cc_library( visibility = ["//visibility:public"], alwayslink = 1, ) -ovms_cc_library( - name = "tfs_backend_impl", - srcs = [ - "tfs_frontend/tfs_backend_impl.cpp", - "tfs_frontend/tensor_conversion.cpp", - ], - deps = [ - "tensor_conversion_after", - "modelinstance_h", - "tfs_utils", - "tfs_request_utils", - "requestprocessor", - "libovmstimer", - "tfs_serialization", - "inference_executor", - "tfs_deserialization", - "deserialization_main", - "executingstreamidguard", - "predict_request_validation_utils", - "tfs_validation", - ], - visibility = ["//visibility:public"], - alwayslink = 1, -) ovms_cc_library( name = "kfs_validation", hdrs = ["kfs_frontend/validation.hpp"], @@ -1569,21 +1522,6 @@ ovms_cc_library( ], visibility = ["//visibility:public",], ) -ovms_cc_library( - name = "tfs_validation", - hdrs = [], - srcs = ["tfs_frontend/validation.cpp"], - deps = ["predict_request_validation_utils", - "libovmsprofiler", - "libovmslogging", - "libovmsstatus", - "libovmsshape", - "tfs_utils", - "libovms_tensorinfo", - "libovmsprecision", - ], - visibility = ["//visibility:public",], -) ovms_cc_library( name = "kfs_serialization", hdrs = ["kfs_frontend/serialization.hpp",], @@ -1600,24 +1538,6 @@ ovms_cc_library( ], visibility = ["//visibility:public"], ) -ovms_cc_library( - name = "tfs_serialization", - hdrs = ["tfs_frontend/serialization.hpp",], - srcs = ["tfs_frontend/serialization.cpp",], - deps = [ - "//third_party:openvino", - "libovmslogging", - "libovmsstatus", - "tfs_utils", - "libovmsprofiler", - "serialization_common", - "libovms_tensorinfo", - "tensor_conversion", - "tensor_conversion_common", - "inference_request_common", - ], - visibility = ["//visibility:public"], -) ovms_cc_library( name = "kfs_deserialization", hdrs = ["kfs_frontend/deserialization.hpp",], @@ -1636,26 +1556,6 @@ ovms_cc_library( ], visibility = ["//visibility:public"], ) -ovms_cc_library( - name = "tfs_deserialization", - hdrs = ["tfs_frontend/deserialization.hpp",], - srcs = ["tfs_frontend/deserialization.cpp",], - deps = [ - "//third_party:openvino", - "libovmslogging", - "deserialization_common", - "libovms_itensorfactory", - "tfs_utils", - "libovmsprofiler", - "libovmsstatus", - "libovms_tensorinfo", - "requesttensorextractor", - "tensor_conversion", - "tensor_conversion_common", - "inference_request_common", - ], - visibility = ["//visibility:public"], -) ovms_cc_library( name = "capiinferenceparameter", hdrs = ["capi_frontend/inferenceparameter.hpp",], @@ -2102,26 +2002,6 @@ ovms_cc_library( visibility = ["//visibility:public",], ) -# HTTP Server implementation using net_http of tensorflow -# To use other library simply create new target and implementation of libhttp_async_writer_interface -ovms_cc_library( - name = "libnet_http_server", - srcs = [ - "net_http_async_writer_impl.hpp", - "net_http_async_writer_impl.cpp", - ], - deps = [ - "libhttp_async_writer_interface", - "@tensorflow_serving//tensorflow_serving/util/net_http/server/public:http_server", - "@tensorflow_serving//tensorflow_serving/util/net_http/server/public:http_server_api", - "@tensorflow_serving//tensorflow_serving/util:threadpool_executor", - "libovmslogging", - "libovmsstatus", - "libhttp_status_code" - ], - visibility = ["//visibility:public",], -) - ovms_cc_library( # TODO split further name = "cpp_headers", hdrs = [ @@ -2227,16 +2107,8 @@ cc_test( "test/demultiplexer_node_test.cpp", "test/deserialization_tests.cpp", "test/ensemble_config_change_stress.cpp", - "test/ensemble_flow_custom_node_tests.cpp", - "test/ensemble_mapping_config_tests.cpp", "test/ensemble_metadata_test.cpp", - "test/ensemble_tests.cpp", - "test/gather_node_test.cpp", - "test/get_model_metadata_response_test.cpp", "test/standalone_http_server_test.cpp", - "test/get_model_metadata_signature_test.cpp", - "test/get_model_metadata_validation_test.cpp", - "test/get_pipeline_metadata_response_test.cpp", "test/http_rest_api_handler_test.cpp", "test/kfs_metadata_test.cpp", "test/kfs_rest_test.cpp", @@ -2247,7 +2119,6 @@ cc_test( "test/metrics_flow_test.cpp", "test/mockmodelinstancechangingstates.hpp", "test/model_cache_test.cpp", - "test/model_service_test.cpp", "test/model_test.cpp", "test/model_version_policy_test.cpp", "test/modelconfig_test.cpp", @@ -2261,7 +2132,6 @@ cc_test( "test/ovmsconfig_test.cpp", "test/pipelinedefinitionstatus_test.cpp", "test/predict_validation_test.cpp", - "test/prediction_service_test.cpp", "test/rest_utils_test.cpp", "test/schema_test.cpp", "test/serialization_tests.cpp", @@ -2274,10 +2144,6 @@ cc_test( "test/tensorinfo_test.cpp", "test/tensorutils_test.cpp", "test/test_http_utils.hpp", - "test/tfs_rest_parser_binary_inputs_test.cpp", - "test/tfs_rest_parser_column_test.cpp", - "test/tfs_rest_parser_nonamed_test.cpp", - "test/tfs_rest_parser_row_test.cpp", "test/threadsafequeue_test.cpp", "test/unit_tests.cpp", "test/localfilesystem_test.cpp", @@ -2327,13 +2193,9 @@ cc_test( "test/llm/llmtemplate_test.cpp", ], "//:disable_python" : [], - }) + select({ - "//:enable_drogon": [ - "test/multi_part_parser_drogon_test.cpp", - ], - "//conditions:default" : [ - ], - }), + }) + [ + "test/multi_part_parser_drogon_test.cpp", + ], data = [ "test/add_two_inputs_model/1/add.xml", "test/add_two_inputs_model/1/add.bin", diff --git a/src/capi_frontend/capi.cpp b/src/capi_frontend/capi.cpp index 48b7b4101a..5ff98510d2 100644 --- a/src/capi_frontend/capi.cpp +++ b/src/capi_frontend/capi.cpp @@ -33,7 +33,6 @@ #include "../servable_definition_unload_guard.hpp" #include "../execution_context.hpp" #include "../version.hpp" -#include "../model_service.hpp" #include "../modelinstance.hpp" #include "capi_request_utils.hpp" // TODO @atobisze must be before executor #include "capi_utils.hpp" @@ -44,7 +43,6 @@ #include "../modelmanager.hpp" #include "../module_names.hpp" #include "../ovms.h" // NOLINT -#include "../prediction_service.hpp" #include "../profiler.hpp" #include "../dags/pipelinedefinitionstatus.hpp" #include "../servable_definition.hpp" diff --git a/src/config_status_utils.cpp b/src/config_status_utils.cpp new file mode 100644 index 0000000000..9972e6d4cd --- /dev/null +++ b/src/config_status_utils.cpp @@ -0,0 +1,138 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "config_status_utils.hpp" + +#include +#include +#include + +#include "dags/pipelinedefinitionstatus.hpp" +#include "execution_context.hpp" +#include "metrics/metric.hpp" +#include "model.hpp" +#include "model_metric_reporter.hpp" +#include "modelinstance.hpp" +#include "model_instance_provider.hpp" +#include "modelversionstatus.hpp" +#include "servable_name_checker.hpp" +#include "src/port/rapidjson_stringbuffer.hpp" +#include "src/port/rapidjson_writer.hpp" +#include "single_version_servable_definition.hpp" +#include "status.hpp" + +namespace ovms { + +Status getAllModelsStatuses(ModelsStatuses& modelsStatuses, ModelInstanceProvider& modelProvider, ServableNameChecker& servableChecker, ExecutionContext context) { + ModelsStatuses tmp; + + const auto servableNames = servableChecker.getServableDefinitionNames(); + for (const auto& servableName : servableNames) { + std::vector versions; + + auto model = modelProvider.findModelByName(servableName); + if (model) { + auto modelVersionsInstances = model->getModelVersionsMapCopy(); + for (const auto& [modelVersion, modelInstance] : modelVersionsInstances) { + const auto& status = modelInstance.getStatus(); + ModelVersionStatusDetails details{ + modelVersion, + status.getState(), + status.getErrorCode(), + status.getErrorMsg()}; + versions.push_back(std::move(details)); + } + } else { + auto* definition = servableChecker.findServableDefinition(servableName); + if (!definition) { + continue; + } + auto* svsd = dynamic_cast(definition); + if (!svsd) { + continue; + } + auto [state, errorCode] = svsd->getStatus().convertToModelStatus(); + ModelVersionStatusDetails details{ + svsd->getVersion(), + state, + errorCode, + ModelVersionStatusErrorCodeToString(errorCode)}; + versions.push_back(std::move(details)); + } + + tmp[servableName] = std::move(versions); + } + + modelsStatuses.merge(tmp); + return StatusCode::OK; +} + +// Returns the JSON-escaped content of a string value (without surrounding quotes). +static std::string jsonEscapeStringValue(const std::string& s) { + rapidjson::StringBuffer sb; + rapidjson::Writer w(sb); + w.String(s.c_str(), static_cast(s.size())); + // w.String() writes "value" including surrounding quotes — strip them. + return std::string(sb.GetString() + 1, sb.GetSize() - 2); +} + +Status serializeModelsStatuses2Json(const ModelsStatuses& modelsStatuses, std::string& output) { + if (modelsStatuses.empty()) { + output = "{}"; + return StatusCode::OK; + } + + std::string outputTmp; + outputTmp += "{\n"; + + bool firstModel = true; + for (const auto& [modelName, versions] : modelsStatuses) { + if (!firstModel) { + outputTmp += ",\n"; + } + firstModel = false; + + outputTmp += "\"" + jsonEscapeStringValue(modelName) + "\" : \n{\n \"model_version_status\": ["; + + if (versions.empty()) { + outputTmp += "]\n}"; + } else { + outputTmp += "\n"; + bool firstVersion = true; + for (const auto& v : versions) { + if (!firstVersion) { + outputTmp += ",\n"; + } + firstVersion = false; + + outputTmp += " {\n"; + outputTmp += " \"version\": \"" + std::to_string(v.version) + "\",\n"; + outputTmp += " \"state\": \"" + ModelVersionStateToString(v.state) + "\",\n"; + outputTmp += " \"status\": {\n"; + outputTmp += " \"error_code\": \"" + ModelVersionStatusErrorCodeToString(v.errorCode) + "\",\n"; + outputTmp += " \"error_message\": \"" + jsonEscapeStringValue(v.errorMessage) + "\"\n"; + outputTmp += " }\n"; + outputTmp += " }"; + } + outputTmp += "\n ]\n}"; + } + } + + outputTmp += "\n}"; + output = std::move(outputTmp); + return StatusCode::OK; +} + +} // namespace ovms diff --git a/src/config_status_utils.hpp b/src/config_status_utils.hpp new file mode 100644 index 0000000000..cfc68f125d --- /dev/null +++ b/src/config_status_utils.hpp @@ -0,0 +1,43 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include + +#include "execution_context.hpp" +#include "modelversion.hpp" +#include "modelversionstatus.hpp" + +namespace ovms { +class ModelInstanceProvider; +class ServableNameChecker; +class Status; + +struct ModelVersionStatusDetails { + model_version_t version; + ModelVersionState state; + ModelVersionStatusErrorCode errorCode; + std::string errorMessage; +}; + +using ModelsStatuses = std::map>; + +Status getAllModelsStatuses(ModelsStatuses& modelsStatuses, ModelInstanceProvider& modelProvider, ServableNameChecker& servableChecker, ExecutionContext context); +Status serializeModelsStatuses2Json(const ModelsStatuses& modelsStatuses, std::string& output); + +} // namespace ovms diff --git a/src/dags/dag_tfs.cpp b/src/dags/dag_tfs.cpp deleted file mode 100644 index 6a0aaf6804..0000000000 --- a/src/dags/dag_tfs.cpp +++ /dev/null @@ -1,61 +0,0 @@ -//***************************************************************************** -// Copyright 2026 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -// TFS frontend-specific headers — must precede _impl.hpp includes -// so that two-phase lookup resolves dependent names at instantiation. -// tfs_utils.hpp must precede deserialization.hpp (getBinaryInput/getBinaryInputsSize). -#include "src/tfs_frontend/tfs_utils.hpp" -#include "src/tfs_frontend/tfs_request_utils.hpp" -#include "src/tfs_frontend/deserialization.hpp" -#include "src/tfs_frontend/serialization.hpp" - -#pragma warning(push) -#pragma warning(disable : 4624 6001 6385 6386 6326 6011 4457 6308 6387 6246) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#pragma GCC diagnostic ignored "-Wunused-but-set-variable" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop -#pragma warning(pop) - -#include "entry_node_impl.hpp" -#include "exit_node_impl.hpp" -#include "exitnodesession_impl.hpp" -#include "pipelinedefinition_create_impl.hpp" -#include "pipeline_factory_create_impl.hpp" - -namespace ovms { - -template class EntryNode; - -template class ExitNode; - -template class ExitNodeSession; - -template Status PipelineDefinition::create( - std::unique_ptr& pipeline, - const tensorflow::serving::PredictRequest* request, - tensorflow::serving::PredictResponse* response, - ModelInstanceProvider& modelInstanceProvider); - -template Status PipelineFactory::create( - std::unique_ptr& pipeline, - const std::string& name, - const tensorflow::serving::PredictRequest* request, - tensorflow::serving::PredictResponse* response, - ModelInstanceProvider& modelInstanceProvider) const; - -} // namespace ovms diff --git a/src/dags/nodesessionmetadata.cpp b/src/dags/nodesessionmetadata.cpp index d2d9e91bd4..718dedc984 100644 --- a/src/dags/nodesessionmetadata.cpp +++ b/src/dags/nodesessionmetadata.cpp @@ -26,7 +26,7 @@ namespace ovms { NodeSessionMetadata::NodeSessionMetadata() : - context({ExecutionContext::Interface::GRPC, ExecutionContext::Method::Predict}) {} + context({ExecutionContext::Interface::GRPC, ExecutionContext::Method::ModelInfer}) {} NodeSessionMetadata::NodeSessionMetadata(ExecutionContext context) : context(context) {} diff --git a/src/dags/pipeline.cpp b/src/dags/pipeline.cpp index 02daf727d5..8cc1be6ab3 100644 --- a/src/dags/pipeline.cpp +++ b/src/dags/pipeline.cpp @@ -89,7 +89,7 @@ Status Pipeline::execute(ExecutionContext context) { OVMS_PROFILE_FUNCTION(); SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Started execution of pipeline: {}", getName()); - if (context.method != ExecutionContext::Method::Predict && context.method != ExecutionContext::Method::ModelInfer) { + if (context.method != ExecutionContext::Method::ModelInfer) { SPDLOG_LOGGER_ERROR(dag_executor_logger, "Executing pipeline: {} wrong context", getName()); return StatusCode::INTERNAL_ERROR; } diff --git a/src/execution_context.hpp b/src/execution_context.hpp index d272e9befa..6aaf321007 100644 --- a/src/execution_context.hpp +++ b/src/execution_context.hpp @@ -24,11 +24,6 @@ struct ExecutionContext { REST, }; enum class Method : uint8_t { - // TensorflowServing - Predict, - GetModelMetadata, - GetModelStatus, - // Model Control API ConfigReload, ConfigStatus, diff --git a/src/get_model_metadata_impl.cpp b/src/get_model_metadata_impl.cpp deleted file mode 100644 index ae4202649c..0000000000 --- a/src/get_model_metadata_impl.cpp +++ /dev/null @@ -1,214 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "get_model_metadata_impl.hpp" - -#include - -#include "dags/pipeline_factory.hpp" -#include "dags/pipelinedefinition.hpp" -#include "servable_definition_unload_guard.hpp" -#include "execution_context.hpp" -#include "model.hpp" -#include "modelinstance.hpp" -#include "modelinstanceunloadguard.hpp" -#include "modelmanager.hpp" -#include "servablemanagermodule.hpp" -#include "server.hpp" -#include "status.hpp" -#include "tfs_frontend/tfs_utils.hpp" - -using google::protobuf::util::JsonPrintOptions; -using google::protobuf::util::MessageToJsonString; - -namespace ovms { -GetModelMetadataImpl::GetModelMetadataImpl(ovms::Server& ovmsServer) : - modelManager(dynamic_cast(ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME))->getServableManager()) { - if (nullptr == ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME)) { - const char* message = "Tried to create model metadata impl without servable manager module"; - SPDLOG_ERROR(message); - throw std::logic_error(message); - } -} - -Status GetModelMetadataImpl::getModelStatus( - const tensorflow::serving::GetModelMetadataRequest* request, - tensorflow::serving::GetModelMetadataResponse* response, - ExecutionContext context) const { - auto status = validate(request); - if (!status.ok()) { - return status; - } - return getModelStatus(request, response, modelManager, context); -} - -Status GetModelMetadataImpl::getModelStatus( - const tensorflow::serving::GetModelMetadataRequest* request, - tensorflow::serving::GetModelMetadataResponse* response, - ModelManager& manager, - ExecutionContext context) { - const auto& name = request->model_spec().name(); - model_version_t version = request->model_spec().has_version() ? request->model_spec().version().value() : 0; - - auto model = manager.findModelByName(name); - if (model == nullptr) { - SPDLOG_DEBUG("GetModelMetadata: Model {} is missing, trying to find pipeline with such name", name); - auto pipelineDefinition = manager.getPipelineFactory().findDefinitionByName(name); - if (!pipelineDefinition) { - return StatusCode::MODEL_NAME_MISSING; - } - auto status = buildResponse(*pipelineDefinition, response, manager); - INCREMENT_IF_ENABLED(pipelineDefinition->getMetricReporter().getGetModelMetadataRequestMetric(context, status.ok())); - return status; - } - - std::shared_ptr instance = nullptr; - if (version != 0) { - SPDLOG_DEBUG("GetModelMetadata requested model: name {}; version {}", name, version); - instance = model->getModelInstanceByVersion(version); - if (instance == nullptr) { - SPDLOG_DEBUG("GetModelMetadata requested model {}; version {} is missing", name, version); - return StatusCode::MODEL_VERSION_MISSING; - } - } else { - SPDLOG_DEBUG("GetModelMetadata requested model: name {}; default version", name); - instance = model->getDefaultModelInstance(); - if (instance == nullptr) { - SPDLOG_DEBUG("GetModelMetadata requested model {}; default version is missing", name); - return StatusCode::MODEL_VERSION_MISSING; - } - } - - auto status = buildResponse(instance, response); - INCREMENT_IF_ENABLED(instance->getMetricReporter().getGetModelMetadataRequestMetric(context, status.ok())); - return status; -} - -Status GetModelMetadataImpl::validate( - const tensorflow::serving::GetModelMetadataRequest* request) { - - if (!request->has_model_spec()) { - return StatusCode::MODEL_SPEC_MISSING; - } - - if (request->metadata_field_size() != 1) { - return StatusCode::INVALID_SIGNATURE_DEF; - } - - const auto& signature = request->metadata_field().at(0); - - if (signature != "signature_def") { - return StatusCode::INVALID_SIGNATURE_DEF; - } - - return StatusCode::OK; -} - -void GetModelMetadataImpl::convert( - const tensor_map_t& from, - proto_signature_map_t* to) { - for (const auto& [name, tensor] : from) { - auto& input = (*to)[name]; - - input.set_dtype(getPrecisionAsDataType(tensor->getPrecision())); - - // Since this method is used for models and pipelines we cannot rely on tensor getMappedName(). - // In both cases we can rely on tensor_map key values as final names. - *input.mutable_name() = name; - *input.mutable_tensor_shape() = tensorflow::TensorShapeProto(); - - for (const auto& dim : tensor->getShape()) { - if (dim.isStatic()) { - input.mutable_tensor_shape()->add_dim()->set_size(dim.getStaticValue()); - } else { - input.mutable_tensor_shape()->add_dim()->set_size(DYNAMIC_DIMENSION); - } - } - } -} - -Status GetModelMetadataImpl::buildResponse( - std::shared_ptr instance, - tensorflow::serving::GetModelMetadataResponse* response) { - - std::unique_ptr unloadGuard; - - // 0 meaning immediately return unload guard if possible, otherwise do not wait for available state - auto status = instance->waitForLoaded(0, unloadGuard); - if (!status.ok()) { - return status; - } - - response->Clear(); - response->mutable_model_spec()->set_name(instance->getName()); - response->mutable_model_spec()->mutable_version()->set_value(instance->getVersion()); - - tensorflow::serving::SignatureDefMap def; - convert(instance->getInputsInfo(), ((*def.mutable_signature_def())["serving_default"]).mutable_inputs()); - convert(instance->getOutputsInfo(), ((*def.mutable_signature_def())["serving_default"]).mutable_outputs()); - - (*response->mutable_metadata())["signature_def"].PackFrom(def); - return StatusCode::OK; -} - -Status GetModelMetadataImpl::buildResponse( - PipelineDefinition& pipelineDefinition, - tensorflow::serving::GetModelMetadataResponse* response, - const ModelManager& manager) { - - // 0 meaning immediately return unload guard if possible, otherwise do not wait for available state - std::unique_ptr unloadGuard; - auto status = pipelineDefinition.waitForLoaded(unloadGuard, 0); - if (!status.ok()) { - return status; - } - - const tensor_map_t& inputs = pipelineDefinition.getInputsInfo(); - const tensor_map_t& outputs = pipelineDefinition.getOutputsInfo(); - - response->Clear(); - response->mutable_model_spec()->set_name(pipelineDefinition.getName()); - response->mutable_model_spec()->mutable_version()->set_value(1); - - tensorflow::serving::SignatureDefMap def; - convert(inputs, ((*def.mutable_signature_def())["serving_default"]).mutable_inputs()); - convert(outputs, ((*def.mutable_signature_def())["serving_default"]).mutable_outputs()); - - (*response->mutable_metadata())["signature_def"].PackFrom(def); - return StatusCode::OK; -} - -Status GetModelMetadataImpl::createGrpcRequest(const std::string& model_name, std::optional model_version, tensorflow::serving::GetModelMetadataRequest* request) { - request->mutable_model_spec()->set_name(model_name); - if (model_version.has_value()) { - request->mutable_model_spec()->mutable_version()->set_value(model_version.value()); - } - request->mutable_metadata_field()->Add("signature_def"); - return StatusCode::OK; -} - -Status GetModelMetadataImpl::serializeResponse2Json(const tensorflow::serving::GetModelMetadataResponse* response, std::string* output) { - JsonPrintOptions opts; - opts.add_whitespace = true; - opts.always_print_primitive_fields = true; - const auto& status = MessageToJsonString(*response, output, opts); - if (!status.ok()) { - SPDLOG_ERROR("Failed to convert proto to json. Error: ", status.ToString()); - return StatusCode::JSON_SERIALIZATION_ERROR; - } - return StatusCode::OK; -} - -} // namespace ovms diff --git a/src/get_model_metadata_impl.hpp b/src/get_model_metadata_impl.hpp deleted file mode 100644 index 60bab928f7..0000000000 --- a/src/get_model_metadata_impl.hpp +++ /dev/null @@ -1,78 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include - -#pragma warning(push) -#pragma warning(disable : 4624 6001 6385 6386 6326 6011 4457 6308 6387 6246) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#pragma GCC diagnostic ignored "-Wunused-but-set-variable" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop -#pragma warning(pop) - -#include "tensorinfo.hpp" - -namespace ovms { - -using proto_signature_map_t = google::protobuf::Map; - -struct ExecutionContext; -class ModelInstance; -class ModelManager; -class PipelineDefinition; -class Server; -class Status; - -class GetModelMetadataImpl { - ModelManager& modelManager; - -public: - GetModelMetadataImpl(ovms::Server& ovmsServer); - static Status validate( - const tensorflow::serving::GetModelMetadataRequest* request); - - static void convert( - const tensor_map_t& from, - proto_signature_map_t* to); - - static Status buildResponse( - std::shared_ptr instance, - tensorflow::serving::GetModelMetadataResponse* response); - static Status buildResponse( - PipelineDefinition& pipelineDefinition, - tensorflow::serving::GetModelMetadataResponse* response, - const ModelManager& manager); - - Status getModelStatus( - const tensorflow::serving::GetModelMetadataRequest* request, - tensorflow::serving::GetModelMetadataResponse* response, ExecutionContext context) const; - - static Status getModelStatus( - const tensorflow::serving::GetModelMetadataRequest* request, - tensorflow::serving::GetModelMetadataResponse* response, - ModelManager& manager, - ExecutionContext context); - static Status createGrpcRequest(const std::string& model_name, std::optional model_version, tensorflow::serving::GetModelMetadataRequest* request); - static Status serializeResponse2Json(const tensorflow::serving::GetModelMetadataResponse* response, std::string* output); -}; - -} // namespace ovms diff --git a/src/grpcservermodule.cpp b/src/grpcservermodule.cpp index 4296e49489..a0e1c4d657 100644 --- a/src/grpcservermodule.cpp +++ b/src/grpcservermodule.cpp @@ -33,10 +33,8 @@ #include "config.hpp" #include "kfs_frontend/kfs_grpc_inference_service.hpp" #include "logging.hpp" -#include "model_service.hpp" #include "modelmanager.hpp" #include "network_utils.hpp" -#include "prediction_service.hpp" #include "servablemanagermodule.hpp" #include "server.hpp" #include "stringutils.hpp" @@ -93,8 +91,6 @@ GRPCServerModule::~GRPCServerModule() { GRPCServerModule::GRPCServerModule(Server& server) : server(server), - tfsPredictService(this->server), - tfsModelService(this->server), kfsGrpcInferenceService(this->server) {} static std::string host_with_port(const std::string& host, int port) { @@ -137,8 +133,6 @@ Status GRPCServerModule::start(const ovms::Config& config) { SPDLOG_INFO("Binding gRPC server to address: {}", hostWithPort); builder.AddListeningPort(hostWithPort, grpc::InsecureServerCredentials()); } - builder.RegisterService(&tfsPredictService); - builder.RegisterService(&tfsModelService); builder.RegisterService(&kfsGrpcInferenceService); for (auto& [name, value] : channel_arguments) { // gRPC accept arguments of two types, int and string. We will attempt to @@ -211,9 +205,6 @@ void GRPCServerModule::shutdown() { SPDLOG_INFO("{} shutdown", GRPC_SERVER_MODULE_NAME); } -const GetModelMetadataImpl& GRPCServerModule::getTFSModelMetadataImpl() const { - return this->tfsPredictService.getTFSModelMetadataImpl(); -} KFSInferenceServiceImpl& GRPCServerModule::getKFSGrpcImpl() const { return this->kfsGrpcInferenceService; } diff --git a/src/grpcservermodule.hpp b/src/grpcservermodule.hpp index e3589a5813..8d3af7d2a5 100644 --- a/src/grpcservermodule.hpp +++ b/src/grpcservermodule.hpp @@ -21,9 +21,7 @@ #include #include "kfs_frontend/kfs_grpc_inference_service.hpp" -#include "model_service.hpp" #include "module.hpp" -#include "prediction_service.hpp" namespace ovms { class Config; @@ -31,8 +29,6 @@ class Server; class GRPCServerModule : public Module { Server& server; - PredictionServiceImpl tfsPredictService; - ModelServiceImpl tfsModelService; mutable KFSInferenceServiceImpl kfsGrpcInferenceService; std::vector> servers; @@ -42,7 +38,6 @@ class GRPCServerModule : public Module { Status start(const ovms::Config& config) override; void shutdown() override; - const GetModelMetadataImpl& getTFSModelMetadataImpl() const; KFSInferenceServiceImpl& getKFSGrpcImpl() const; }; } // namespace ovms diff --git a/src/http_rest_api_handler.cpp b/src/http_rest_api_handler.cpp index 82911119b0..9a144caf5e 100644 --- a/src/http_rest_api_handler.cpp +++ b/src/http_rest_api_handler.cpp @@ -43,7 +43,7 @@ #include "servable_definition_unload_guard.hpp" #include "execution_context.hpp" #include "filesystem/filesystem.hpp" -#include "get_model_metadata_impl.hpp" +#include "config_status_utils.hpp" #include "grpcservermodule.hpp" #include "kfs_frontend/kfs_grpc_inference_service.hpp" #include "kfs_frontend/kfs_utils.hpp" @@ -51,7 +51,6 @@ #include "metrics/metric_module.hpp" #include "metrics/metric_registry.hpp" #include "model_metric_reporter.hpp" -#include "model_service.hpp" #include "modelinstance.hpp" #include "modelinstanceunloadguard.hpp" #include "modelmanager.hpp" @@ -74,17 +73,11 @@ #include "mediapipe_internal/mediapipegraphexecutor.hpp" #endif -#include "tfs_frontend/tfs_utils.hpp" -#include "tfs_frontend/tfs_request_utils.hpp" -#include "tfs_frontend/deserialization.hpp" #include "kfs_frontend/kfs_request_utils.hpp" #include "predict_request_validation_utils.hpp" #include "deserialization_main.hpp" #include "inference_executor.hpp" -using tensorflow::serving::PredictRequest; -using tensorflow::serving::PredictResponse; - using rapidjson::Document; using rapidjson::SizeType; using rapidjson::Value; @@ -100,10 +93,6 @@ const std::string DEFAULT_VERSION = "DEFAULT"; namespace ovms { -const std::string HttpRestApiHandler::predictionRegexExp = - R"((.?)\/v1\/models\/([^\/:]+)(?:(?:\/versions\/(\d+))|(?:\/labels\/(\w+)))?:(classify|regress|predict))"; -const std::string HttpRestApiHandler::modelstatusRegexExp = - R"((.?)\/v1\/models(?:\/([^\/:]+))?(?:(?:\/versions\/(\d+))|(?:\/labels\/(\w+)))?(?:\/(metadata))?)"; const std::string HttpRestApiHandler::configReloadRegexExp = R"((.?)\/v1\/config\/reload)"; const std::string HttpRestApiHandler::configStatusRegexExp = R"((.?)\/v1\/config)"; @@ -131,8 +120,6 @@ const std::string HttpRestApiHandler::metricsRegexExp = R"((.?)\/metrics(\?(.*)) HttpRestApiHandler::HttpRestApiHandler(ovms::Server& ovmsServer, int timeout_in_ms, const std::string& apiKey) : apiKey(apiKey), - predictionRegex(predictionRegexExp), - modelstatusRegex(modelstatusRegexExp), configReloadRegex(configReloadRegexExp), configStatusRegex(configStatusRegexExp), kfs_modelreadyRegex(kfs_modelreadyRegexExp), @@ -149,7 +136,6 @@ HttpRestApiHandler::HttpRestApiHandler(ovms::Server& ovmsServer, int timeout_in_ ovmsServer(ovmsServer), kfsGrpcImpl(dynamic_cast(this->ovmsServer.getModule(GRPC_SERVER_MODULE_NAME))->getKFSGrpcImpl()), - grpcGetModelMetadataImpl(dynamic_cast(this->ovmsServer.getModule(GRPC_SERVER_MODULE_NAME))->getTFSModelMetadataImpl()), modelManager(dynamic_cast(this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME))->getServableManager()) { if (nullptr == this->ovmsServer.getModule(GRPC_SERVER_MODULE_NAME)) throw std::logic_error("Tried to create http rest api handler without grpc server module"); @@ -177,24 +163,6 @@ void HttpRestApiHandler::registerHandler(RequestType type, HandlerCallbackFn f) } void HttpRestApiHandler::registerAll() { - registerHandler(Predict, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter, std::shared_ptr multiPartParser) -> Status { - if (request_components.processing_method == "predict") { - return processPredictRequest(request_components.model_name, request_components.model_version, - request_components.model_version_label, request_body, &response); - } else { - SPDLOG_DEBUG("Requested REST resource not found"); - return StatusCode::REST_NOT_FOUND; - } - }); - - registerHandler(GetModelMetadata, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter, std::shared_ptr multiPartParser) { - return processModelMetadataRequest(request_components.model_name, request_components.model_version, - request_components.model_version_label, &response); - }); - registerHandler(GetModelStatus, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter, std::shared_ptr multiPartParser) { - return processModelStatusRequest(request_components.model_name, request_components.model_version, - request_components.model_version_label, &response); - }); registerHandler(ConfigReload, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter, std::shared_ptr multiPartParser) -> Status { return processConfigReloadRequest(response, this->modelManager); }); @@ -991,24 +959,6 @@ Status HttpRestApiHandler::parseRequestComponents(HttpRequestComponents& request } if (http_method == "POST") { - if (std::regex_match(request_path, sm, predictionRegex)) { - requestComponents.type = Predict; - requestComponents.model_name = urlDecode(sm[2]); - - std::string model_version_str = sm[3]; - auto status = parseModelVersion(model_version_str, requestComponents.model_version); - if (!status.ok()) - return status; - - std::string model_version_label_str = sm[4]; - if (!model_version_label_str.empty()) { - requestComponents.model_version_label = model_version_label_str; - } - - requestComponents.processing_method = sm[5]; - - return StatusCode::OK; - } if (std::regex_match(request_path, sm, kfs_inferRegex, std::regex_constants::match_any)) { requestComponents.type = KFS_Infer; requestComponents.model_name = urlDecode(sm[1]); @@ -1034,8 +984,7 @@ Status HttpRestApiHandler::parseRequestComponents(HttpRequestComponents& request requestComponents.type = ConfigReload; return StatusCode::OK; } - return (std::regex_match(request_path, sm, modelstatusRegex) || - std::regex_match(request_path, sm, kfs_serverliveRegex) || + return (std::regex_match(request_path, sm, kfs_serverliveRegex) || std::regex_match(request_path, sm, configStatusRegex) || std::regex_match(request_path, sm, kfs_serverreadyRegex) || std::regex_match(request_path, sm, kfs_servermetadataRegex) || @@ -1048,26 +997,6 @@ Status HttpRestApiHandler::parseRequestComponents(HttpRequestComponents& request : StatusCode::REST_INVALID_URL; } else if (http_method == "GET") { - if (std::regex_match(request_path, sm, modelstatusRegex)) { - requestComponents.model_name = urlDecode(sm[2]); - std::string model_version_str = sm[3]; - auto status = parseModelVersion(model_version_str, requestComponents.model_version); - if (!status.ok()) - return status; - - std::string model_version_label_str = sm[4]; - if (!model_version_label_str.empty()) { - requestComponents.model_version_label = model_version_label_str; - } - - requestComponents.model_subresource = sm[5]; - if (!requestComponents.model_subresource.empty() && requestComponents.model_subresource == "metadata") { - requestComponents.type = GetModelMetadata; - } else { - requestComponents.type = GetModelStatus; - } - return StatusCode::OK; - } if (std::regex_match(request_path, sm, configStatusRegex)) { requestComponents.type = ConfigStatus; return StatusCode::OK; @@ -1102,8 +1031,6 @@ Status HttpRestApiHandler::parseRequestComponents(HttpRequestComponents& request requestComponents.type = KFS_GetModelReady; return StatusCode::OK; } - if (std::regex_match(request_path, sm, predictionRegex)) - return StatusCode::REST_UNSUPPORTED_METHOD; if (std::regex_match(request_path, sm, metricsRegex)) { std::string params = sm[3]; if (!params.empty()) { @@ -1121,8 +1048,7 @@ Status HttpRestApiHandler::parseRequestComponents(HttpRequestComponents& request requestComponents.type = V3_RetrieveModel; return StatusCode::OK; } - return (std::regex_match(request_path, sm, predictionRegex) || - std::regex_match(request_path, sm, kfs_inferRegex, std::regex_constants::match_any) || + return (std::regex_match(request_path, sm, kfs_inferRegex, std::regex_constants::match_any) || std::regex_match(request_path, sm, configReloadRegex)) ? StatusCode::REST_UNSUPPORTED_METHOD : StatusCode::REST_INVALID_URL; @@ -1158,225 +1084,6 @@ Status HttpRestApiHandler::processRequest( return dispatchToProcessor(request_path, request_body, response, requestComponents, responseComponents, std::move(serverReaderWriter), std::move(multiPartParser)); } -Status HttpRestApiHandler::processPredictRequest( - const std::string& modelName, - const std::optional& modelVersion, - const std::optional& modelVersionLabel, - const std::string& request, - std::string* response) { - // model_version_label currently is not in use - - Timer timer; - timer.start(TOTAL); - using std::chrono::microseconds; - - std::string modelVersionLog = modelVersion.has_value() ? std::to_string(modelVersion.value()) : DEFAULT_VERSION; - SPDLOG_DEBUG("Processing REST request for model: {}; version: {}", - modelName, modelVersionLog); - Order requestOrder = Order::UNKNOWN; - tensorflow::serving::PredictResponse responseProto; - Status status; - - ServableMetricReporter* reporterOut = nullptr; - if (this->modelManager.modelExists(modelName)) { - SPDLOG_DEBUG("Found model with name: {}. Searching for requested version...", modelName); - status = processSingleModelRequest(modelName, modelVersion, request, requestOrder, responseProto, reporterOut); - } else if (this->modelManager.servableExists(modelName, ServableQueryType::Pipeline)) { - SPDLOG_DEBUG("Found pipeline with name: {}", modelName); - status = processPipelineRequest(modelName, request, requestOrder, responseProto, reporterOut); - } else { - SPDLOG_DEBUG("Model or pipeline matching request parameters not found - name: {}, version: {}", modelName, modelVersionLog); - status = StatusCode::MODEL_NAME_MISSING; - } - if (!status.ok()) - return status; - status = makeJsonFromPredictResponse(responseProto, response, requestOrder); - if (!status.ok()) - return status; - - timer.stop(TOTAL); - double requestTime = timer.elapsed(TOTAL); - SPDLOG_DEBUG("Total REST request processing time: {} ms", requestTime / 1000); - if (!reporterOut) { - return StatusCode::OK; - // There is no request time metric for MediaPipe endpoints - } - OBSERVE_IF_ENABLED(reporterOut->requestTimeRest, requestTime); - return StatusCode::OK; -} - -Status HttpRestApiHandler::processSingleModelRequest(const std::string& modelName, - const std::optional& modelVersion, - const std::string& request, - Order& requestOrder, - tensorflow::serving::PredictResponse& responseProto, - ServableMetricReporter*& reporterOut) { - - std::shared_ptr modelInstance; - std::unique_ptr modelInstanceUnloadGuard; - auto status = this->modelManager.getModelInstance( - modelName, - modelVersion.value_or(0), - modelInstance, - modelInstanceUnloadGuard); - - if (!status.ok()) { - if (modelInstance) { - INCREMENT_IF_ENABLED(modelInstance->getMetricReporter().requestFailRestPredict); - } - std::string modelVersionLog = modelVersion.has_value() ? std::to_string(modelVersion.value()) : DEFAULT_VERSION; - SPDLOG_DEBUG("Requested model instance - name: {}, version: {} - does not exist.", modelName, modelVersionLog); - return status; - } - reporterOut = &modelInstance->getMetricReporter(); - Timer timer; - timer.start(TOTAL); - TFSRestParser requestParser(modelInstance->getInputsInfo()); - status = requestParser.parse(request.c_str()); - if (!status.ok()) { - INCREMENT_IF_ENABLED(modelInstance->getMetricReporter().requestFailRestPredict); - return status; - } - requestOrder = requestParser.getOrder(); - timer.stop(TOTAL); - SPDLOG_DEBUG("JSON request parsing time: {} ms", timer.elapsed(TOTAL) / 1000); - - tensorflow::serving::PredictRequest& requestProto = requestParser.getProto(); - requestProto.mutable_model_spec()->set_name(modelName); - if (modelVersion.has_value()) { - requestProto.mutable_model_spec()->mutable_version()->set_value(modelVersion.value()); - } - status = infer(*modelInstance, &requestProto, &responseProto, modelInstanceUnloadGuard); - INCREMENT_IF_ENABLED(modelInstance->getMetricReporter().getInferRequestMetric(ExecutionContext{ExecutionContext::Interface::REST, ExecutionContext::Method::Predict}, status.ok())); - return status; -} - -Status HttpRestApiHandler::getReporter(const HttpRequestComponents& components, ovms::ServableMetricReporter*& reporter) { - std::shared_ptr modelInstance; - std::unique_ptr pipelinePtr; - std::unique_ptr modelInstanceUnloadGuard; - auto status = this->modelManager.getModelInstance(components.model_name, components.model_version.value_or(0), modelInstance, modelInstanceUnloadGuard); - if (status == StatusCode::MODEL_NAME_MISSING) { - auto pipelineDefinition = this->modelManager.getPipelineFactory().findDefinitionByName(components.model_name); - if (!pipelineDefinition) { - return StatusCode::MODEL_MISSING; - } - reporter = &pipelineDefinition->getMetricReporter(); - } else if (status.ok()) { - reporter = &modelInstance->getMetricReporter(); - } else { - return StatusCode::MODEL_MISSING; - } - return StatusCode::OK; -} - -Status HttpRestApiHandler::getPipelineInputsAndReporter(const std::string& modelName, ovms::tensor_map_t& inputs, ovms::ServableMetricReporter*& reporter) { - auto pipelineDefinition = this->modelManager.getPipelineFactory().findDefinitionByName(modelName); - if (!pipelineDefinition) { - return StatusCode::MODEL_MISSING; - } - std::unique_ptr unloadGuard; - Status status = pipelineDefinition->waitForLoaded(unloadGuard); - if (!status.ok()) { - return status; - } - reporter = &pipelineDefinition->getMetricReporter(); - inputs = pipelineDefinition->getInputsInfo(); - return StatusCode::OK; -} - -Status HttpRestApiHandler::processPipelineRequest(const std::string& modelName, - const std::string& request, - Order& requestOrder, - tensorflow::serving::PredictResponse& responseProto, - ServableMetricReporter*& reporterOut) { - ExecutionContext executionContext{ExecutionContext::Interface::REST, ExecutionContext::Method::Predict}; - std::unique_ptr pipelinePtr; - - Timer timer; - timer.start(TOTAL); - ovms::tensor_map_t inputs; - auto status = getPipelineInputsAndReporter(modelName, inputs, reporterOut); - if (!status.ok()) { - if (reporterOut) { - INCREMENT_IF_ENABLED(reporterOut->getInferRequestMetric(executionContext, false)); - } - return status; - } - - TFSRestParser requestParser(inputs); - status = requestParser.parse(request.c_str()); - if (!status.ok()) { - INCREMENT_IF_ENABLED(reporterOut->getInferRequestMetric(executionContext, false)); - return status; - } - requestOrder = requestParser.getOrder(); - timer.stop(TOTAL); - SPDLOG_DEBUG("JSON request parsing time: {} ms", timer.elapsed(TOTAL) / 1000); - - tensorflow::serving::PredictRequest& requestProto = requestParser.getProto(); - requestProto.mutable_model_spec()->set_name(modelName); - status = this->modelManager.getPipelineFactory().create(pipelinePtr, modelName, &requestProto, &responseProto, this->modelManager); - if (!status.ok()) { - INCREMENT_IF_ENABLED(reporterOut->getInferRequestMetric(executionContext, false)); - return status; - } - status = pipelinePtr->execute(executionContext); - INCREMENT_IF_ENABLED(pipelinePtr->getMetricReporter().getInferRequestMetric(executionContext, status.ok())); - return status; -} - -Status HttpRestApiHandler::processModelMetadataRequest( - const std::string_view model_name, - const std::optional& model_version, - const std::optional& model_version_label, - std::string* response) { - // model_version_label currently is not in use - tensorflow::serving::GetModelMetadataRequest grpc_request; - tensorflow::serving::GetModelMetadataResponse grpc_response; - Status status; - std::string modelName(model_name); - status = grpcGetModelMetadataImpl.createGrpcRequest(modelName, model_version, &grpc_request); - if (!status.ok()) { - return status; - } - status = grpcGetModelMetadataImpl.getModelStatus(&grpc_request, &grpc_response, ExecutionContext(ExecutionContext::Interface::REST, ExecutionContext::Method::GetModelMetadata)); - if (!status.ok()) { - return status; - } - status = grpcGetModelMetadataImpl.serializeResponse2Json(&grpc_response, response); - if (!status.ok()) { - return status; - } - return StatusCode::OK; -} - -Status HttpRestApiHandler::processModelStatusRequest( - const std::string_view model_name, - const std::optional& model_version, - const std::optional& model_version_label, - std::string* response) { - // model_version_label currently is not in use - SPDLOG_DEBUG("Processing model status request"); - tensorflow::serving::GetModelStatusRequest grpc_request; - tensorflow::serving::GetModelStatusResponse grpc_response; - Status status; - std::string modelName(model_name); - status = GetModelStatusImpl::createGrpcRequest(modelName, model_version, &grpc_request); - if (!status.ok()) { - return status; - } - status = GetModelStatusImpl::getModelStatus(&grpc_request, &grpc_response, this->modelManager, ExecutionContext(ExecutionContext::Interface::REST, ExecutionContext::Method::GetModelStatus)); - if (!status.ok()) { - return status; - } - status = GetModelStatusImpl::serializeResponse2Json(&grpc_response, response); - if (!status.ok()) { - return status; - } - return StatusCode::OK; -} - inline static std::string createErrorJsonWithMessage(std::string message) { return "{\n\t\"error\": \"" + message + "\"\n}"; } @@ -1421,14 +1128,14 @@ Status HttpRestApiHandler::processConfigReloadRequest(std::string& response, Mod reloadNeeded = true; } - std::map modelsStatuses; - status = GetModelStatusImpl::getAllModelsStatuses(modelsStatuses, manager, ExecutionContext(ExecutionContext::Interface::REST, ExecutionContext::Method::ConfigReload)); + ModelsStatuses modelsStatuses; + status = getAllModelsStatuses(modelsStatuses, manager, manager, ExecutionContext(ExecutionContext::Interface::REST, ExecutionContext::Method::ConfigReload)); if (!status.ok()) { response = createErrorJsonWithMessage("Retrieving all model statuses failed. Check server logs for more info."); return status; } - status = GetModelStatusImpl::serializeModelsStatuses2Json(modelsStatuses, response); + status = serializeModelsStatuses2Json(modelsStatuses, response); if (!status.ok()) { response = createErrorJsonWithMessage("Serializing model statuses to json failed. Check server logs for more info."); return status; @@ -1445,14 +1152,14 @@ Status HttpRestApiHandler::processConfigStatusRequest(std::string& response, Mod SPDLOG_DEBUG("Processing config status request started."); Status status; - std::map modelsStatuses; - status = GetModelStatusImpl::getAllModelsStatuses(modelsStatuses, manager, ExecutionContext(ExecutionContext::Interface::REST, ExecutionContext::Method::ConfigStatus)); + ModelsStatuses modelsStatuses; + status = getAllModelsStatuses(modelsStatuses, manager, manager, ExecutionContext(ExecutionContext::Interface::REST, ExecutionContext::Method::ConfigStatus)); if (!status.ok()) { response = createErrorJsonWithMessage("Retrieving all model statuses failed."); return status; } - status = GetModelStatusImpl::serializeModelsStatuses2Json(modelsStatuses, response); + status = serializeModelsStatuses2Json(modelsStatuses, response); if (!status.ok()) { response = createErrorJsonWithMessage("Serializing model statuses to json failed."); return status; @@ -1488,4 +1195,42 @@ std::string urlDecode(const std::string& encoded) { return decoded.str(); } +Status HttpRestApiHandler::getReporter(const HttpRequestComponents& components, ovms::ServableMetricReporter*& reporter) { + std::shared_ptr modelInstance; + std::unique_ptr modelInstanceUnloadGuard; + auto status = this->modelManager.getModelInstance( + components.model_name, + components.model_version.value_or(0), + modelInstance, + modelInstanceUnloadGuard); + if (status == StatusCode::MODEL_NAME_MISSING) { + auto pipelineDefinition = this->modelManager.getPipelineFactory().findDefinitionByName(components.model_name); + if (!pipelineDefinition) { + return StatusCode::MODEL_MISSING; + } + reporter = &pipelineDefinition->getMetricReporter(); + return StatusCode::OK; + } + if (!status.ok()) { + return StatusCode::MODEL_MISSING; + } + reporter = &modelInstance->getMetricReporter(); + return StatusCode::OK; +} + +Status HttpRestApiHandler::getPipelineInputsAndReporter(const std::string& modelName, ovms::tensor_map_t& inputs, ovms::ServableMetricReporter*& reporter) { + auto pipelineDefinition = this->modelManager.getPipelineFactory().findDefinitionByName(modelName); + if (!pipelineDefinition) { + return StatusCode::MODEL_MISSING; + } + std::unique_ptr unloadGuard; + Status status = pipelineDefinition->waitForLoaded(unloadGuard); + if (!status.ok()) { + return status; + } + reporter = &pipelineDefinition->getMetricReporter(); + inputs = pipelineDefinition->getInputsInfo(); + return StatusCode::OK; +} + } // namespace ovms diff --git a/src/http_rest_api_handler.hpp b/src/http_rest_api_handler.hpp index 65cd9b8890..613500b7e1 100644 --- a/src/http_rest_api_handler.hpp +++ b/src/http_rest_api_handler.hpp @@ -25,30 +25,20 @@ #include #include -#pragma warning(push) -#pragma warning(disable : 6001 4324 6326 4457 6308 6387 6246) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop -#pragma warning(pop) - #include "http_async_writer_interface.hpp" #include "multi_part_parser.hpp" #include "rest_parser.hpp" #include "status.hpp" +#include "tensorinfo_fwd.hpp" namespace ovms { class ServableMetricReporter; class KFSInferenceServiceImpl; -class GetModelMetadataImpl; class Server; class ModelManager; -enum RequestType { Predict, - GetModelStatus, - GetModelMetadata, - ConfigReload, +// note since removal of TFS, V3 endpoints (from OpenAI) are will be also accepted as V1 +enum RequestType { ConfigReload, ConfigStatus, KFS_GetModelReady, KFS_Infer, @@ -97,8 +87,6 @@ std::string urlDecode(const std::string& encoded); class HttpRestApiHandler { public: - static const std::string predictionRegexExp; - static const std::string modelstatusRegexExp; static const std::string configReloadRegexExp; static const std::string configStatusRegexExp; @@ -164,70 +152,6 @@ class HttpRestApiHandler { std::shared_ptr writer, std::shared_ptr multiPartParser); - /** - * @brief Process predict request - * - * @param modelName - * @param modelVersion - * @param modelVersionLabel - * @param request - * @param response - * - * @return StatusCode - */ - Status processPredictRequest( - const std::string& modelName, - const std::optional& modelVersion, - const std::optional& modelVersionLabel, - const std::string& request, - std::string* response); - - Status processSingleModelRequest( - const std::string& modelName, - const std::optional& modelVersion, - const std::string& request, - Order& requestOrder, - tensorflow::serving::PredictResponse& responseProto, - ServableMetricReporter*& reporterOut); - - Status processPipelineRequest( - const std::string& modelName, - const std::string& request, - Order& requestOrder, - tensorflow::serving::PredictResponse& responseProto, - ServableMetricReporter*& reporterOut); - - /** - * @brief Process Model Metadata request - * - * @param model_name - * @param model_version - * @param model_version_label - * @param response - * - * @return StatusCode - */ - Status processModelMetadataRequest( - const std::string_view model_name, - const std::optional& model_version, - const std::optional& model_version_label, - std::string* response); - - /** - * @brief Process Model Status request - * - * @param model_name - * @param model_version - * @param model_version_label - * @param response - * @return StatusCode - */ - Status processModelStatusRequest( - const std::string_view model_name, - const std::optional& model_version, - const std::optional& model_version_label, - std::string* response); - Status processConfigReloadRequest(std::string& response, ModelManager& manager); void convertShapeType(rapidjson::Value& scope, rapidjson::Document& doc); @@ -251,8 +175,6 @@ class HttpRestApiHandler { const std::string apiKey; private: - const std::regex predictionRegex; - const std::regex modelstatusRegex; const std::regex configReloadRegex; const std::regex configStatusRegex; @@ -275,7 +197,6 @@ class HttpRestApiHandler { ovms::Server& ovmsServer; ovms::KFSInferenceServiceImpl& kfsGrpcImpl; - const GetModelMetadataImpl& grpcGetModelMetadataImpl; ovms::ModelManager& modelManager; Status getReporter(const HttpRequestComponents& components, ovms::ServableMetricReporter*& reporter); diff --git a/src/kfs_frontend/kfs_graph_executor_impl.cpp b/src/kfs_frontend/kfs_graph_executor_impl.cpp index 1b519a2636..5c197db6f8 100644 --- a/src/kfs_frontend/kfs_graph_executor_impl.cpp +++ b/src/kfs_frontend/kfs_graph_executor_impl.cpp @@ -29,7 +29,7 @@ #include "../mediapipe_internal/mediapipegraphdefinition.hpp" #include "../predict_request_validation_utils.hpp" #include "../status.hpp" -#include "../tfs_frontend/tfs_utils.hpp" +#include "../tensorflow_type_utils.hpp" #pragma warning(push) #pragma warning(disable : 6385 6386 6326 6011 6294 6201 4309 4005 4456 6246) diff --git a/src/model_metric_reporter.cpp b/src/model_metric_reporter.cpp index 93d40140cc..1e60689957 100644 --- a/src/model_metric_reporter.cpp +++ b/src/model_metric_reporter.cpp @@ -58,47 +58,6 @@ ServableMetricReporter::ServableMetricReporter(const MetricConfig* metricConfig, THROW_IF_NULL(family, "cannot create family"); if (metricConfig->isFamilyEnabled(familyName)) { - // TFS - this->requestSuccessGrpcPredict = family->addMetric({{"name", modelName}, - {"version", std::to_string(modelVersion)}, - {"api", "TensorFlowServing"}, - {"method", "Predict"}, - {"interface", "gRPC"}}); - THROW_IF_NULL(this->requestSuccessGrpcPredict, "cannot create metric"); - - this->requestSuccessGrpcGetModelMetadata = family->addMetric({{"name", modelName}, - {"version", std::to_string(modelVersion)}, - {"api", "TensorFlowServing"}, - {"method", "GetModelMetadata"}, - {"interface", "gRPC"}}); - THROW_IF_NULL(this->requestSuccessGrpcGetModelMetadata, "cannot create metric"); - - this->requestSuccessGrpcGetModelStatus = family->addMetric({{"name", modelName}, - {"api", "TensorFlowServing"}, - {"method", "GetModelStatus"}, - {"interface", "gRPC"}}); - THROW_IF_NULL(this->requestSuccessGrpcGetModelStatus, "cannot create metric"); - - this->requestSuccessRestPredict = family->addMetric({{"name", modelName}, - {"version", std::to_string(modelVersion)}, - {"api", "TensorFlowServing"}, - {"method", "Predict"}, - {"interface", "REST"}}); - THROW_IF_NULL(this->requestSuccessRestPredict, "cannot create metric"); - - this->requestSuccessRestGetModelMetadata = family->addMetric({{"name", modelName}, - {"version", std::to_string(modelVersion)}, - {"api", "TensorFlowServing"}, - {"method", "GetModelMetadata"}, - {"interface", "REST"}}); - THROW_IF_NULL(this->requestSuccessRestGetModelMetadata, "cannot create metric"); - - this->requestSuccessRestGetModelStatus = family->addMetric({{"name", modelName}, - {"api", "TensorFlowServing"}, - {"method", "GetModelStatus"}, - {"interface", "REST"}}); - THROW_IF_NULL(this->requestSuccessRestGetModelStatus, "cannot create metric"); - // KFS this->requestSuccessGrpcModelInfer = family->addMetric({{"name", modelName}, {"version", std::to_string(modelVersion)}, @@ -147,47 +106,6 @@ ServableMetricReporter::ServableMetricReporter(const MetricConfig* metricConfig, THROW_IF_NULL(family, "cannot create family"); if (metricConfig->isFamilyEnabled(familyName)) { - // TFS - this->requestFailGrpcPredict = family->addMetric({{"name", modelName}, - {"version", std::to_string(modelVersion)}, - {"api", "TensorFlowServing"}, - {"method", "Predict"}, - {"interface", "gRPC"}}); - THROW_IF_NULL(this->requestFailGrpcPredict, "cannot create metric"); - - this->requestFailGrpcGetModelMetadata = family->addMetric({{"name", modelName}, - {"version", std::to_string(modelVersion)}, - {"api", "TensorFlowServing"}, - {"method", "GetModelMetadata"}, - {"interface", "gRPC"}}); - THROW_IF_NULL(this->requestFailGrpcGetModelMetadata, "cannot create metric"); - - this->requestFailGrpcGetModelStatus = family->addMetric({{"name", modelName}, - {"api", "TensorFlowServing"}, - {"method", "GetModelStatus"}, - {"interface", "gRPC"}}); - THROW_IF_NULL(this->requestFailGrpcGetModelStatus, "cannot create metric"); - - this->requestFailRestPredict = family->addMetric({{"name", modelName}, - {"version", std::to_string(modelVersion)}, - {"api", "TensorFlowServing"}, - {"method", "Predict"}, - {"interface", "REST"}}); - THROW_IF_NULL(this->requestFailRestPredict, "cannot create metric"); - - this->requestFailRestGetModelMetadata = family->addMetric({{"name", modelName}, - {"version", std::to_string(modelVersion)}, - {"api", "TensorFlowServing"}, - {"method", "GetModelMetadata"}, - {"interface", "REST"}}); - THROW_IF_NULL(this->requestFailRestGetModelMetadata, "cannot create metric"); - - this->requestFailRestGetModelStatus = family->addMetric({{"name", modelName}, - {"api", "TensorFlowServing"}, - {"method", "GetModelStatus"}, - {"interface", "REST"}}); - THROW_IF_NULL(this->requestFailRestGetModelStatus, "cannot create metric"); - // KFS this->requestFailGrpcModelInfer = family->addMetric({{"name", modelName}, {"version", std::to_string(modelVersion)}, diff --git a/src/model_metric_reporter.hpp b/src/model_metric_reporter.hpp index e300334742..4ca4cbbedc 100644 --- a/src/model_metric_reporter.hpp +++ b/src/model_metric_reporter.hpp @@ -34,7 +34,6 @@ class StatusMetricReporter { virtual ~StatusMetricReporter() = default; virtual std::unique_ptr& getModelReadyMetric(const ExecutionContext& context, bool success = true) = 0; virtual std::unique_ptr& getModelMetadataMetric(const ExecutionContext& context, bool success = true) = 0; - virtual std::unique_ptr& getGetModelStatusRequestSuccessMetric(const ExecutionContext& context) = 0; }; class ServableMetricReporter : public StatusMetricReporter { @@ -47,23 +46,6 @@ class ServableMetricReporter : public StatusMetricReporter { ServableMetricReporter(const MetricConfig* metricConfig, MetricRegistry* registry, const std::string& modelName, model_version_t modelVersion); virtual ~ServableMetricReporter(); - // TFS - std::unique_ptr requestSuccessGrpcPredict; - std::unique_ptr requestSuccessGrpcGetModelMetadata; - std::unique_ptr requestSuccessGrpcGetModelStatus; - - std::unique_ptr requestSuccessRestPredict; - std::unique_ptr requestSuccessRestGetModelMetadata; - std::unique_ptr requestSuccessRestGetModelStatus; - - std::unique_ptr requestFailGrpcPredict; - std::unique_ptr requestFailGrpcGetModelMetadata; - std::unique_ptr requestFailGrpcGetModelStatus; - - std::unique_ptr requestFailRestPredict; - std::unique_ptr requestFailRestGetModelMetadata; - std::unique_ptr requestFailRestGetModelStatus; - // KFS std::unique_ptr requestSuccessGrpcModelInfer; std::unique_ptr requestSuccessGrpcModelMetadata; @@ -81,54 +63,16 @@ class ServableMetricReporter : public StatusMetricReporter { std::unique_ptr requestFailRestModelMetadata; std::unique_ptr requestFailRestModelReady; - std::unique_ptr requestTimeGrpc; - std::unique_ptr requestTimeRest; - - inline std::unique_ptr& getGetModelStatusRequestSuccessMetric(const ExecutionContext& context) override { - if (context.method != ExecutionContext::Method::GetModelStatus) { - static std::unique_ptr empty = nullptr; - return empty; // In case something calls it from ConfigReload/ConfigStatus methods - } + inline std::unique_ptr& getInferRequestMetric(const ExecutionContext& context, bool success = true) { if (context.interface == ExecutionContext::Interface::GRPC) { - return this->requestSuccessGrpcGetModelStatus; + return success ? this->requestSuccessGrpcModelInfer : this->requestFailGrpcModelInfer; } else { - return this->requestSuccessRestGetModelStatus; + return success ? this->requestSuccessRestModelInfer : this->requestFailRestModelInfer; } } - inline std::unique_ptr& getGetModelMetadataRequestMetric(const ExecutionContext& context, bool success) { - if (success) { - if (context.interface == ExecutionContext::Interface::GRPC) { - return this->requestSuccessGrpcGetModelMetadata; - } else { - return this->requestSuccessRestGetModelMetadata; - } - } else { - if (context.interface == ExecutionContext::Interface::GRPC) { - return this->requestFailGrpcGetModelMetadata; - } else { - return this->requestFailRestGetModelMetadata; - } - } - } - - inline std::unique_ptr& getInferRequestMetric(const ExecutionContext& context, bool success = true) { - if (context.method == ExecutionContext::Method::Predict) { - if (context.interface == ExecutionContext::Interface::GRPC) { - return success ? this->requestSuccessGrpcPredict : this->requestFailGrpcPredict; - } else { - return success ? this->requestSuccessRestPredict : this->requestFailRestPredict; - } - } else if (context.method == ExecutionContext::Method::ModelInfer) { - if (context.interface == ExecutionContext::Interface::GRPC) { - return success ? this->requestSuccessGrpcModelInfer : this->requestFailGrpcModelInfer; - } else { - return success ? this->requestSuccessRestModelInfer : this->requestFailRestModelInfer; - } - } else { - throw std::logic_error("wrong context method for inference"); - } - } + std::unique_ptr requestTimeGrpc; + std::unique_ptr requestTimeRest; inline std::unique_ptr& getModelMetadataMetric(const ExecutionContext& context, bool success = true) override { if (context.interface == ExecutionContext::Interface::GRPC) { @@ -323,11 +267,6 @@ class MediapipeServableMetricReporter : public StatusMetricReporter { } } - inline std::unique_ptr& getGetModelStatusRequestSuccessMetric(const ExecutionContext& context) override { - static std::unique_ptr empty{nullptr}; - return empty; - } - MediapipeServableMetricReporter(const MetricConfig* metricConfig, MetricRegistry* registry, const std::string& graphName); }; diff --git a/src/model_service.cpp b/src/model_service.cpp deleted file mode 100644 index 06cd882c20..0000000000 --- a/src/model_service.cpp +++ /dev/null @@ -1,223 +0,0 @@ -//***************************************************************************** -// Copyright 2020-2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** - -#include "model_service.hpp" - -#include -#include -#include -#include -#include - -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#include "tensorflow_serving/apis/get_model_status.pb.h" -#include "tensorflow_serving/apis/model_service.grpc.pb.h" -#include "tensorflow_serving/apis/model_service.pb.h" -#pragma GCC diagnostic pop - -#include "dags/pipelinedefinitionstatus.hpp" -#include "execution_context.hpp" -#include "grpc_utils.hpp" -#include "model.hpp" -#include "modelinstance.hpp" -#include "modelmanager.hpp" -#include "servable_definition.hpp" -#include "servablemanagermodule.hpp" -#include "server.hpp" -#include "single_version_servable_definition.hpp" -#include "status.hpp" - -using google::protobuf::util::JsonPrintOptions; -using google::protobuf::util::MessageToJsonString; - -namespace ovms { - -void addStatusToResponse(tensorflow::serving::GetModelStatusResponse* response, model_version_t version, const ModelVersionStatus& model_version_status) { - SPDLOG_DEBUG("add_status_to_response version={} status={}", version, model_version_status.getStateString()); - auto status_to_fill = response->add_model_version_status(); - status_to_fill->set_state(static_cast(static_cast(model_version_status.getState()))); - status_to_fill->set_version(version); - status_to_fill->clear_status(); - status_to_fill->mutable_status()->set_error_code(static_cast(static_cast(model_version_status.getErrorCode()))); - status_to_fill->mutable_status()->set_error_message(model_version_status.getErrorMsg()); -} - -void addStatusToResponse(tensorflow::serving::GetModelStatusResponse* response, const model_version_t version, const PipelineDefinitionStatus& pipeline_status) { - auto [state, error_code] = pipeline_status.convertToModelStatus(); - SPDLOG_DEBUG("add_status_to_response state={} error_code={}", state, error_code); - auto status_to_fill = response->add_model_version_status(); - status_to_fill->set_state(static_cast(static_cast(state))); - status_to_fill->set_version(version); - status_to_fill->clear_status(); - status_to_fill->mutable_status()->set_error_code(static_cast(static_cast(error_code))); - status_to_fill->mutable_status()->set_error_message(ModelVersionStatusErrorCodeToString(error_code)); -} - -::grpc::Status ModelServiceImpl::GetModelStatus( - ::grpc::ServerContext* context, const tensorflow::serving::GetModelStatusRequest* request, - tensorflow::serving::GetModelStatusResponse* response) { - return grpc(GetModelStatusImpl::getModelStatus(request, response, modelManager, ExecutionContext(ExecutionContext::Interface::GRPC, ExecutionContext::Method::GetModelStatus))); -} - -Status GetModelStatusImpl::createGrpcRequest(const std::string& model_name, const std::optional model_version, tensorflow::serving::GetModelStatusRequest* request) { - request->mutable_model_spec()->set_name(model_name); - if (model_version.has_value()) { - request->mutable_model_spec()->mutable_version()->set_value(model_version.value()); - } - return StatusCode::OK; -} - -Status GetModelStatusImpl::serializeResponse2Json(const tensorflow::serving::GetModelStatusResponse* response, std::string* output) { - JsonPrintOptions opts; - opts.add_whitespace = true; - opts.always_print_primitive_fields = true; - const auto& status = MessageToJsonString(*response, output, opts); - if (!status.ok()) { - SPDLOG_ERROR("Failed to convert proto to json. Error: ", status.ToString()); - return StatusCode::JSON_SERIALIZATION_ERROR; - } - return StatusCode::OK; -} - -Status GetModelStatusImpl::getModelStatus( - const tensorflow::serving::GetModelStatusRequest* request, - tensorflow::serving::GetModelStatusResponse* response, - ModelManager& manager, - ExecutionContext context) { - SPDLOG_DEBUG("model_service: request: {}", request->DebugString()); - - bool has_requested_version = request->model_spec().has_version(); - auto requested_version = request->model_spec().version().value(); - std::string requested_model_name = request->model_spec().name(); - auto model_ptr = manager.findModelByName(requested_model_name); - if (!model_ptr) { - SPDLOG_DEBUG("GetModelStatus: Model {} is missing, trying to find definition with such name", requested_model_name); - auto* definition = manager.findServableDefinition(requested_model_name); - if (!definition) { - return StatusCode::MODEL_NAME_MISSING; - } - auto* svsd = dynamic_cast(definition); - if (!svsd) { - return StatusCode::MODEL_NAME_MISSING; - } - INCREMENT_IF_ENABLED(svsd->getMetricReporter().getGetModelStatusRequestSuccessMetric(context)); - addStatusToResponse(response, svsd->getVersion(), svsd->getStatus()); - SPDLOG_DEBUG("model_service: response: {}", response->DebugString()); - SPDLOG_DEBUG("MODEL_STATUS created a response for {} - {}", requested_model_name, requested_version); - return StatusCode::OK; - } - - SPDLOG_DEBUG("requested model: {}, has_version: {} (version: {})", requested_model_name, has_requested_version, requested_version); - if (has_requested_version && requested_version != 0) { - // return details only for a specific version of requested model; NOT_FOUND otherwise. If requested_version == 0, default is returned. - std::shared_ptr modelInstance = model_ptr->getModelInstanceByVersion(requested_version); - if (!modelInstance) { - SPDLOG_DEBUG("requested model {} in version {} was not found.", requested_model_name, requested_version); - return StatusCode::MODEL_VERSION_MISSING; - } - INCREMENT_IF_ENABLED(modelInstance->getMetricReporter().getGetModelStatusRequestSuccessMetric(context)); - const auto& status = modelInstance->getStatus(); - SPDLOG_DEBUG("adding model {} - {} :: {} to response", requested_model_name, requested_version, status.getStateString()); - addStatusToResponse(response, requested_version, status); - } else { - // return status details of all versions of a requested model. - auto modelVersionsInstances = model_ptr->getModelVersionsMapCopy(); - bool reported = false; - for (const auto& [modelVersion, modelInstance] : modelVersionsInstances) { - // GetModelStatus is tracked once for all versions (no version label) - this is why we only report once in a loop (each model instance metric refer to the same metric) - if (!reported) { - INCREMENT_IF_ENABLED(modelInstance.getMetricReporter().getGetModelStatusRequestSuccessMetric(context)); - reported = true; - } - const auto& status = modelInstance.getStatus(); - SPDLOG_DEBUG("adding model {} - {} :: {} to response", requested_model_name, modelVersion, status.getStateString()); - addStatusToResponse(response, modelVersion, status); - } - } - SPDLOG_DEBUG("model_service: response: {}", response->DebugString()); - SPDLOG_DEBUG("MODEL_STATUS created a response for {} - {}", requested_model_name, requested_version); - return StatusCode::OK; -} - -Status GetModelStatusImpl::getAllModelsStatuses(std::map& modelsStatuses, ModelManager& manager, ExecutionContext context) { - std::map modelsStatusesTmp; - - const auto servableNames = manager.getServableDefinitionNames(); - for (const auto& servableName : servableNames) { - std::optional noValueModelVersion; - tensorflow::serving::GetModelStatusRequest request; - GetModelStatusImpl::createGrpcRequest(servableName, noValueModelVersion, &request); - tensorflow::serving::GetModelStatusResponse response; - auto status = GetModelStatusImpl::getModelStatus(&request, &response, manager, context); - if (status != StatusCode::OK) { - continue; - } - modelsStatusesTmp.insert({servableName, response}); - } - - modelsStatuses.merge(modelsStatusesTmp); - return StatusCode::OK; -} - -Status GetModelStatusImpl::serializeModelsStatuses2Json(const std::map& modelsStatuses, std::string& output) { - std::string outputTmp; - if (modelsStatuses.begin() == modelsStatuses.end()) { - output = "{}"; - return StatusCode::OK; - } - - outputTmp += "{\n"; - for (auto modelStatus = modelsStatuses.begin(); modelStatus != modelsStatuses.end(); modelStatus++) { - outputTmp += ("\"" + modelStatus->first + "\" : \n"); - std::string responseStr; - auto status = GetModelStatusImpl::serializeResponse2Json(&modelStatus->second, &responseStr); - if (status != StatusCode::OK) { - return status; - } - responseStr.pop_back(); - outputTmp += responseStr; - if (std::next(modelStatus) != modelsStatuses.end()) { - outputTmp += (",\n"); - } - } - outputTmp += "\n}"; - output = std::move(outputTmp); - - return StatusCode::OK; -} - -ModelServiceImpl::ModelServiceImpl(ovms::Server& ovmsServer) : - modelManager(dynamic_cast(ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME))->getServableManager()) { - if (nullptr == ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME)) { - const char* message = "Tried to create model service impl without servable manager module"; - SPDLOG_ERROR(message); - throw std::logic_error(message); - } -} - -::grpc::Status ModelServiceImpl::HandleReloadConfigRequest( - ::grpc::ServerContext* context, const tensorflow::serving::ReloadConfigRequest* request, - tensorflow::serving::ReloadConfigResponse* response) { - SPDLOG_INFO("Requested HandleReloadConfigRequest - but this service is reloading config automatically by itself, therefore this operation has no *EXTRA* affect."); - return grpc::Status::OK; // we're reloading config all the time; for a total client compatibility, this means returning success here. -} - -} // namespace ovms diff --git a/src/model_service.hpp b/src/model_service.hpp deleted file mode 100644 index 9aec5cbf27..0000000000 --- a/src/model_service.hpp +++ /dev/null @@ -1,70 +0,0 @@ -//***************************************************************************** -// Copyright 2020-2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include - -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#include "tensorflow_serving/apis/get_model_status.pb.h" -#include "tensorflow_serving/apis/model_service.grpc.pb.h" -#include "tensorflow_serving/apis/model_service.pb.h" -#pragma GCC diagnostic pop - -#include "modelversion.hpp" - -namespace ovms { -struct ExecutionContext; -class ModelManager; -class ModelVersionStatus; -class Server; -class Status; - -void addStatusToResponse(tensorflow::serving::GetModelStatusResponse* response, model_version_t version, const ModelVersionStatus& model_version_status); - -class ModelServiceImpl final : public tensorflow::serving::ModelService::Service { - ovms::ModelManager& modelManager; - -public: - ModelServiceImpl(ovms::Server& ovmsServer); - ::grpc::Status GetModelStatus(::grpc::ServerContext* context, - const tensorflow::serving::GetModelStatusRequest* request, - tensorflow::serving::GetModelStatusResponse* response) override; - ::grpc::Status HandleReloadConfigRequest(::grpc::ServerContext* context, - const tensorflow::serving::ReloadConfigRequest* request, - tensorflow::serving::ReloadConfigResponse* response) override; -}; - -class GetModelStatusImpl { - ovms::Server& ovmsServer; - -public: - GetModelStatusImpl(ovms::Server& ovmsServer); - static Status getModelStatus(const tensorflow::serving::GetModelStatusRequest* request, tensorflow::serving::GetModelStatusResponse* response, ModelManager& manager, ExecutionContext context); - static Status createGrpcRequest(const std::string& model_name, const std::optional model_version, tensorflow::serving::GetModelStatusRequest* request); - static Status serializeResponse2Json(const tensorflow::serving::GetModelStatusResponse* response, std::string* output); - - static Status getAllModelsStatuses(std::map& models_versions, ModelManager& manager, ExecutionContext context); - static Status serializeModelsStatuses2Json(const std::map& models_versions, std::string& output); -}; - -} // namespace ovms diff --git a/src/modelmanager.hpp b/src/modelmanager.hpp index 22b80b5251..21f8dbad0e 100644 --- a/src/modelmanager.hpp +++ b/src/modelmanager.hpp @@ -477,9 +477,9 @@ class ModelManager : public ServableNameChecker, public MetricProvider, public M bool servableExists(const std::string& name, ServableQueryType check = ServableQueryType::All) const override; bool aliasesConflict(const std::vector& aliases, const std::string& ownGraphName) const override; - ServableDefinition* findServableDefinition(const std::string& name) const; + ServableDefinition* findServableDefinition(const std::string& name) const override; - std::vector getServableDefinitionNames() const; + std::vector getServableDefinitionNames() const override; MetricRegistry* getMetricRegistry() const override { return this->metricRegistry; } }; diff --git a/src/net_http_async_writer_impl.cpp b/src/net_http_async_writer_impl.cpp deleted file mode 100644 index 6f410f9adc..0000000000 --- a/src/net_http_async_writer_impl.cpp +++ /dev/null @@ -1,52 +0,0 @@ -//***************************************************************************** -// Copyright 2024 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "net_http_async_writer_impl.hpp" - -#include -#include -#include - -#include "http_status_code.hpp" - -namespace ovms { - -// Used by V3 handler -void NetHttpAsyncWriterImpl::OverwriteResponseHeader(const std::string& key, const std::string& value) { - this->req->OverwriteResponseHeader(key, value); -} -void NetHttpAsyncWriterImpl::PartialReplyWithStatus(std::string message, HTTPStatusCode status) { - this->req->PartialReplyWithStatus(message, tensorflow::serving::net_http::HTTPStatusCode(int(status))); -} -void NetHttpAsyncWriterImpl::PartialReplyBegin(std::function callback) { - callback(); // net_http can simply run the callback sequentially -} -void NetHttpAsyncWriterImpl::PartialReplyEnd() { - this->req->PartialReplyEnd(); -} -// Used by graph executor impl -void NetHttpAsyncWriterImpl::PartialReply(std::string message) { - this->req->PartialReply(std::move(message)); -} -// Used by calculator via HttpClientConnection -bool NetHttpAsyncWriterImpl::IsDisconnected() const { - return this->req->IsDisconnected(); -} - -void NetHttpAsyncWriterImpl::RegisterDisconnectionCallback(std::function callback) { - this->req->RegisterDisconnectionCallback(std::move(callback)); -} - -} // namespace ovms diff --git a/src/net_http_async_writer_impl.hpp b/src/net_http_async_writer_impl.hpp deleted file mode 100644 index 727b8cf90a..0000000000 --- a/src/net_http_async_writer_impl.hpp +++ /dev/null @@ -1,58 +0,0 @@ -//***************************************************************************** -// Copyright 2024 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include - -#include "http_async_writer_interface.hpp" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#pragma GCC diagnostic ignored "-Wunused-but-set-variable" -#include "tensorflow_serving/util/net_http/public/response_code_enum.h" -#include "tensorflow_serving/util/net_http/server/public/httpserver.h" -#include "tensorflow_serving/util/net_http/server/public/server_request_interface.h" -#include "tensorflow_serving/util/threadpool_executor.h" -#pragma GCC diagnostic pop - -#include "http_status_code.hpp" - -namespace ovms { - -class NetHttpAsyncWriterImpl : public HttpAsyncWriter { - tensorflow::serving::net_http::ServerRequestInterface* req; - -public: - NetHttpAsyncWriterImpl( - tensorflow::serving::net_http::ServerRequestInterface* req) : - req(req) {} - - // Used by V3 handler - void OverwriteResponseHeader(const std::string& key, const std::string& value) override; - void PartialReplyWithStatus(std::string message, HTTPStatusCode status) override; - void PartialReplyBegin(std::function callback) override; - void PartialReplyEnd() override; - - // Used by graph executor impl - void PartialReply(std::string message) override; - - // Used by calculator via HttpClientConnection - bool IsDisconnected() const override; - void RegisterDisconnectionCallback(std::function callback) override; -}; - -} // namespace ovms diff --git a/src/predict_request_validation_utils.hpp b/src/predict_request_validation_utils.hpp index 2e9bfcb4ed..99d6bb86d6 100644 --- a/src/predict_request_validation_utils.hpp +++ b/src/predict_request_validation_utils.hpp @@ -31,9 +31,6 @@ namespace inference { class ModelInferRequest; } -namespace tensorflow::serving { -class PredictRequest; -} namespace ovms { @@ -289,7 +286,7 @@ Status RequestValidator Status RequestValidator::validate() { - if ((std::is_same::value || std::is_same::value) && choice == ValidationChoice::OUTPUT) { + if (std::is_same::value && choice == ValidationChoice::OUTPUT) { return StatusCode::NOT_IMPLEMENTED; } Status finalStatus = StatusCode::OK; diff --git a/src/prediction_service.cpp b/src/prediction_service.cpp deleted file mode 100644 index e211ab1ccc..0000000000 --- a/src/prediction_service.cpp +++ /dev/null @@ -1,165 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "prediction_service.hpp" - -#include -#include -#include -#include - -#pragma warning(push) -#pragma warning(disable : 4624 6001 4324 6385 6386 6326 6011 4457 6308 6387 6246) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow/core/framework/tensor.h" -#pragma GCC diagnostic pop -#pragma warning(pop) - -#include "tfs_frontend/tfs_utils.hpp" -#include "tfs_frontend/deserialization.hpp" -#include "tfs_frontend/tfs_request_utils.hpp" - -#include "dags/pipeline.hpp" -#include "dags/pipeline_factory.hpp" -#include "execution_context.hpp" -#include "get_model_metadata_impl.hpp" -#include "grpc_utils.hpp" -#include "deserialization_main.hpp" -#include "inference_executor.hpp" -#include "logging.hpp" -#include "modelinstance.hpp" -#include "modelinstanceunloadguard.hpp" -#include "modelmanager.hpp" -#include "ovinferrequestsqueue.hpp" -#include "profiler.hpp" -#include "servablemanagermodule.hpp" -#include "server.hpp" -#include "status.hpp" -#include "timer.hpp" - -using grpc::ServerContext; - -using tensorflow::TensorProto; - -using tensorflow::serving::PredictionService; -using tensorflow::serving::PredictRequest; -using tensorflow::serving::PredictResponse; - -namespace { -enum : unsigned int { - TOTAL, - TIMER_END -}; -} - -namespace ovms { - -PredictionServiceImpl::PredictionServiceImpl(ovms::Server& ovmsServer) : - ovmsServer(ovmsServer), - getModelMetadataImpl(ovmsServer), - modelManager(dynamic_cast(this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME))->getServableManager()) { - if (nullptr == this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME)) { - const char* message = "Tried to create prediction service impl without servable manager module"; - SPDLOG_ERROR(message); - throw std::logic_error(message); - } -} - -Status PredictionServiceImpl::getModelInstance(const PredictRequest* request, - std::shared_ptr& modelInstance, - std::unique_ptr& modelInstanceUnloadGuardPtr) { - OVMS_PROFILE_FUNCTION(); - return this->modelManager.getModelInstance(request->model_spec().name(), request->model_spec().version().value(), modelInstance, modelInstanceUnloadGuardPtr); -} - -Status PredictionServiceImpl::getPipeline(const PredictRequest* request, - PredictResponse* response, - std::unique_ptr& pipelinePtr) { - OVMS_PROFILE_FUNCTION(); - return this->modelManager.getPipelineFactory().create(pipelinePtr, request->model_spec().name(), request, response, this->modelManager); -} - -grpc::Status ovms::PredictionServiceImpl::Predict( - ServerContext* context, - const PredictRequest* request, - PredictResponse* response) { - OVMS_PROFILE_FUNCTION(); - Timer timer; - timer.start(TOTAL); - using std::chrono::microseconds; - SPDLOG_DEBUG("Processing gRPC request for model: {}; version: {}", - request->model_spec().name(), - request->model_spec().version().value()); - - std::shared_ptr modelInstance; - std::unique_ptr pipelinePtr; - - std::unique_ptr modelInstanceUnloadGuard; - auto status = getModelInstance(request, modelInstance, modelInstanceUnloadGuard); - - if (status == StatusCode::MODEL_NAME_MISSING) { - SPDLOG_DEBUG("Requested model: {} does not exist. Searching for pipeline with that name...", request->model_spec().name()); - status = getPipeline(request, response, pipelinePtr); - } - - if (!status.ok()) { - if (modelInstance) { - INCREMENT_IF_ENABLED(modelInstance->getMetricReporter().requestFailGrpcPredict); - } - SPDLOG_DEBUG("Getting modelInstance or pipeline failed. {}", status.string()); - return grpc(status); - } - - ExecutionContext executionContext{ - ExecutionContext::Interface::GRPC, - ExecutionContext::Method::Predict}; - - if (pipelinePtr) { - status = pipelinePtr->execute(executionContext); - INCREMENT_IF_ENABLED(pipelinePtr->getMetricReporter().getInferRequestMetric(executionContext, status.ok())); - } else { - status = infer(*modelInstance, request, response, modelInstanceUnloadGuard); - INCREMENT_IF_ENABLED(modelInstance->getMetricReporter().getInferRequestMetric(executionContext, status.ok())); - } - - if (!status.ok()) { - return grpc(status); - } - - timer.stop(TOTAL); - double requestTotal = timer.elapsed(TOTAL); - if (pipelinePtr) { - OBSERVE_IF_ENABLED(pipelinePtr->getMetricReporter().requestTimeGrpc, requestTotal); - } else { - OBSERVE_IF_ENABLED(modelInstance->getMetricReporter().requestTimeGrpc, requestTotal); - } - SPDLOG_DEBUG("Total gRPC request processing time: {} ms", requestTotal / 1000); - return grpc::Status::OK; -} - -grpc::Status PredictionServiceImpl::GetModelMetadata( - grpc::ServerContext* context, - const tensorflow::serving::GetModelMetadataRequest* request, - tensorflow::serving::GetModelMetadataResponse* response) { - OVMS_PROFILE_FUNCTION(); - return grpc(getModelMetadataImpl.getModelStatus(request, response, ExecutionContext(ExecutionContext::Interface::GRPC, ExecutionContext::Method::GetModelMetadata))); -} - -const GetModelMetadataImpl& PredictionServiceImpl::getTFSModelMetadataImpl() const { - return this->getModelMetadataImpl; -} - -} // namespace ovms diff --git a/src/prediction_service.hpp b/src/prediction_service.hpp deleted file mode 100644 index 89b844bbcd..0000000000 --- a/src/prediction_service.hpp +++ /dev/null @@ -1,65 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include - -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop - -#include "get_model_metadata_impl.hpp" - -namespace ovms { -class ModelInstance; -class ModelInstanceUnloadGuard; -class ModelManager; -class Pipeline; -class Server; -class Status; - -class PredictionServiceImpl final : public tensorflow::serving::PredictionService::Service { - ovms::Server& ovmsServer; - GetModelMetadataImpl getModelMetadataImpl; - ModelManager& modelManager; - -public: - PredictionServiceImpl(ovms::Server& ovmsServer); - grpc::Status Predict( - grpc::ServerContext* context, - const tensorflow::serving::PredictRequest* request, - tensorflow::serving::PredictResponse* response) override; - - grpc::Status GetModelMetadata( - grpc::ServerContext* context, - const tensorflow::serving::GetModelMetadataRequest* request, - tensorflow::serving::GetModelMetadataResponse* response) override; - - const GetModelMetadataImpl& getTFSModelMetadataImpl() const; - -protected: - Status getModelInstance(const tensorflow::serving::PredictRequest* request, - std::shared_ptr& modelInstance, - std::unique_ptr& modelInstanceUnloadGuardPtr); - Status getPipeline(const tensorflow::serving::PredictRequest* request, - tensorflow::serving::PredictResponse* response, - std::unique_ptr& pipelinePtr); -}; - -} // namespace ovms diff --git a/src/rest_parser.cpp b/src/rest_parser.cpp index c3e51ae47a..c020cfc495 100644 --- a/src/rest_parser.cpp +++ b/src/rest_parser.cpp @@ -15,528 +15,16 @@ //***************************************************************************** #include "rest_parser.hpp" -#include #include -#include "precision.hpp" -#include "rest_utils.hpp" +#include "logging.hpp" #include "status.hpp" -#include "tfs_frontend/tfs_utils.hpp" #include "utils/rapidjson_utils.hpp" namespace ovms { static constexpr int MAX_NESTING_DEPTH = 100; -TFSRestParser::TFSRestParser(const tensor_map_t& tensors) { - for (const auto& kv : tensors) { - const auto& name = kv.first; - const auto& tensor = kv.second; - tensorPrecisionMap[name] = tensor->getPrecision(); - auto& input = (*requestProto.mutable_inputs())[name]; - input.set_dtype(getPrecisionAsDataType(tensor->getPrecision())); - - auto fold = [](size_t a, const Dimension& b) { - if (b.isDynamic()) { - if (b.isAny()) { - return static_cast(0); - } else { - return static_cast(b.getMaxValue()); - } - } else { - return a * static_cast(b.getStaticValue()); - } - }; - input.mutable_tensor_content()->reserve(std::accumulate( - tensor->getShape().cbegin(), - tensor->getShape().cend(), - 1, - fold) * - DataTypeSize(getPrecisionAsDataType(tensor->getPrecision()))); - } -} - -void TFSRestParser::removeUnusedInputs() { - auto& inputs = (*requestProto.mutable_inputs()); - auto it = inputs.begin(); - while (it != inputs.end()) { - if (!inputsFoundInRequest.count(it->first)) { - SPDLOG_DEBUG("Removing {} input from proto since it's not included in the request", it->first); - it = inputs.erase(it); - } else { - it++; - } - } -} - -bool TFSRestParser::parseSequenceIdInput(rapidjson::Value& doc, tensorflow::TensorProto& proto, const std::string& tensorName) { - proto.set_dtype(tensorflow::DataType::DT_UINT64); - for (auto& value : doc.GetArray()) { - if (value.IsUint64()) - proto.add_uint64_val(value.GetUint64()); - else - return false; - } - return true; -} - -bool TFSRestParser::parseSequenceControlInput(rapidjson::Value& doc, tensorflow::TensorProto& proto, const std::string& tensorName) { - proto.set_dtype(tensorflow::DataType::DT_UINT32); - for (auto& value : doc.GetArray()) { - if (value.IsUint()) - proto.add_uint32_val(value.GetUint()); - else - return false; - } - return true; -} - -bool TFSRestParser::parseSpecialInput(rapidjson::Value& doc, tensorflow::TensorProto& proto, const std::string& tensorName) { - // Special tensors are given in 1 dimensional array - if (doc.GetArray()[0].IsArray()) - return false; - - if (tensorName == "sequence_id") - return parseSequenceIdInput(doc, proto, tensorName); - else if (tensorName == "sequence_control_input") - return parseSequenceControlInput(doc, proto, tensorName); - - return false; -} - -static bool isBinary(const rapidjson::Value& value) { - if (!value.IsObject()) { - return false; - } - - if (!(value.HasMember("b64") && ((value.MemberEnd() - value.MemberBegin()) == 1))) { - return false; - } - - if (!value["b64"].IsString()) { - return false; - } - - return true; -} - -bool TFSRestParser::parseArray(rapidjson::Value& doc, int dim, tensorflow::TensorProto& proto, const std::string& tensorName) { - if (isBinary(doc)) { - if (!addValue(proto, doc)) { - return false; - } - return true; - } - if (doc.IsString() && tensorPrecisionMap[tensorName] == ovms::Precision::U8 && (proto.dtype() == tensorflow::DataType::DT_UINT8 || proto.dtype() == tensorflow::DataType::DT_STRING)) { - if (!addValue(proto, doc)) { - return false; - } - return true; - } - if (!doc.IsArray()) { - return false; - } - if (!setDimOrValidate(proto, dim, doc.GetArray().Size())) { - return false; - } - if (tensorName == "sequence_id" || tensorName == "sequence_control_input") { - if (!parseSpecialInput(doc, proto, tensorName)) { - return false; - } - return true; - } - if (doc.GetArray().Size() == 0) { - return true; - } - if (doc.GetArray()[0].IsArray()) { - for (auto& itr : doc.GetArray()) { - if (!parseArray(itr, dim + 1, proto, tensorName)) { - return false; - } - } - return true; - } else { - if (!setDTypeIfNotSet(doc.GetArray()[0], proto, tensorName)) { - return false; - } - for (auto& value : doc.GetArray()) { - if (!addValue(proto, value)) { - return false; - } - } - return true; - } - return false; -} - -bool TFSRestParser::parseInstance(rapidjson::Value& doc) { - if (doc.GetObject().MemberCount() == 0) { - return false; - } - for (auto& itr : doc.GetObject()) { - std::string tensorName = itr.name.GetString(); - inputsFoundInRequest.insert(tensorName); - auto& proto = (*requestProto.mutable_inputs())[tensorName]; - increaseBatchSize(proto); - if (itr.value.IsNumber() || itr.value.IsString()) { - // Expecting vector of scalars. - // If there are non-scalar elements (arrays, objects) in the batch already, - // it means we have incorrect json. - - // More than one dimension indicates there's an array in the batch. - if (proto.tensor_shape().dim_size() > 1) { - return false; - } - if (!addValue(proto, itr.value)) { - return false; - } - } else if (isBinary(itr.value)) { - // Expecting vector of binary objects. - // If there are non-binary elements (scalars, arrays) in the batch already, - // it means we have incorrect json. - - // More than one dimension indicates there's an array in the batch. - if (proto.tensor_shape().dim_size() > 1) { - return false; - } - if (!parseArray(itr.value, 1, proto, tensorName)) { - return false; - } - } else { - // Expecting array. - // If there are non-array elements (scalars, objects) in the batch already, - // it means we have incorrect json. - - // Only one dimension and multiple elements indicates there's a scalar or object in the batch. - if (proto.tensor_shape().dim(0).size() > 1 && - proto.tensor_shape().dim_size() == 1) { - return false; - } - if (!parseArray(itr.value, 1, proto, tensorName)) { - return false; - } - } - } - return true; -} - -bool TFSRestParser::isBatchSizeEqualForAllInputs() const { - int64_t size = 0; - for (const auto& kv : requestProto.inputs()) { - if (size == 0) { - size = kv.second.tensor_shape().dim(0).size(); - } else if (kv.second.tensor_shape().dim(0).size() != size) { - return false; - } - } - return true; -} - -Status TFSRestParser::parseRowFormat(rapidjson::Value& node) { - order = Order::ROW; - if (!node.IsArray()) { - return StatusCode::REST_INSTANCES_NOT_AN_ARRAY; - } - if (node.GetArray().Size() == 0) { - return StatusCode::REST_NO_INSTANCES_FOUND; - } - if (node.GetArray()[0].IsObject() && !isBinary(node.GetArray()[0])) { - // named format - for (auto& instance : node.GetArray()) { - if (!instance.IsObject()) { - return StatusCode::REST_NAMED_INSTANCE_NOT_AN_OBJECT; - } - - if (!this->parseInstance(instance)) { - return StatusCode::REST_COULD_NOT_PARSE_INSTANCE; - } - } - } else if (node.GetArray()[0].IsArray() || node.GetArray()[0].IsNumber() || isBinary(node.GetArray()[0]) || node.GetArray()[0].IsString()) { - // no named format - if (requestProto.inputs_size() == 0) { - return StatusCode::REST_INPUT_NOT_PREALLOCATED; - } else if (requestProto.inputs_size() != 1) { - return StatusCode::INVALID_INPUT_FORMAT; - } - auto inputsIterator = requestProto.mutable_inputs()->begin(); - if (inputsIterator == requestProto.mutable_inputs()->end()) { - const std::string details = "Failed to parse row formatted request."; - SPDLOG_ERROR("Internal error occurred: {}", details); - return Status(StatusCode::INTERNAL_ERROR, details); - } - if (!parseArray(node, 0, inputsIterator->second, inputsIterator->first)) { - return StatusCode::REST_COULD_NOT_PARSE_INSTANCE; - } else { - format = Format::NONAMED; - return StatusCode::OK; - } - } else { - return StatusCode::REST_INSTANCES_NOT_NAMED_OR_NONAMED; - } - removeUnusedInputs(); - if (!isBatchSizeEqualForAllInputs()) { - return StatusCode::REST_INSTANCES_BATCH_SIZE_DIFFER; - } - format = Format::NAMED; - return StatusCode::OK; -} - -template -static bool addToTensorContent(tensorflow::TensorProto& proto, T value) { - if (sizeof(T) != DataTypeSize(proto.dtype())) { - return false; - } - proto.mutable_tensor_content()->append(reinterpret_cast(&value), sizeof(T)); - return true; -} - -template -static bool addToTensorContent(tensorflow::TensorProto& proto, const rapidjson::Value& value) { - if (value.IsDouble()) { - return addToTensorContent(proto, static_cast(value.GetDouble())); - } - if (value.IsInt64()) { - return addToTensorContent(proto, static_cast(value.GetInt64())); - } - if (value.IsUint64()) { - return addToTensorContent(proto, static_cast(value.GetUint64())); - } - if (value.IsInt()) { - return addToTensorContent(proto, static_cast(value.GetInt())); - } - if (value.IsUint()) { - return addToTensorContent(proto, static_cast(value.GetUint())); - } - - return false; -} - -static bool addToHalfVal(tensorflow::TensorProto& proto, const rapidjson::Value& value) { - if (value.IsDouble()) { - proto.add_half_val(value.GetDouble()); - return true; - } - if (value.IsInt64()) { - proto.add_half_val(value.GetInt64()); - return true; - } - if (value.IsUint64()) { - proto.add_half_val(value.GetUint64()); - return true; - } - if (value.IsInt()) { - proto.add_half_val(value.GetInt()); - return true; - } - if (value.IsUint()) { - proto.add_half_val(value.GetUint()); - return true; - } - - return false; -} - -static bool addToIntVal(tensorflow::TensorProto& proto, const rapidjson::Value& value) { - if (value.IsDouble()) { - proto.add_int_val(value.GetDouble()); - return true; - } - if (value.IsInt64()) { - proto.add_int_val(value.GetInt64()); - return true; - } - if (value.IsUint64()) { - proto.add_int_val(value.GetUint64()); - return true; - } - if (value.IsInt()) { - proto.add_int_val(value.GetInt()); - return true; - } - if (value.IsUint()) { - proto.add_int_val(value.GetUint()); - return true; - } - - return false; -} - -static bool addNumber(tensorflow::TensorProto& proto, const rapidjson::Value& value) { - switch (proto.dtype()) { - case tensorflow::DataType::DT_FLOAT: - return addToTensorContent(proto, value); - case tensorflow::DataType::DT_INT32: - return addToTensorContent(proto, value); - case tensorflow::DataType::DT_INT8: - return addToTensorContent(proto, value); - case tensorflow::DataType::DT_UINT8: - return addToTensorContent(proto, value); - case tensorflow::DataType::DT_DOUBLE: - return addToTensorContent(proto, value); - case tensorflow::DataType::DT_HALF: - return addToHalfVal(proto, value); - case tensorflow::DataType::DT_INT16: - return addToTensorContent(proto, value); - case tensorflow::DataType::DT_UINT16: - return addToIntVal(proto, value); - case tensorflow::DataType::DT_INT64: - return addToTensorContent(proto, value); - case tensorflow::DataType::DT_UINT32: - return addToTensorContent(proto, value); - case tensorflow::DataType::DT_UINT64: - return addToTensorContent(proto, value); - default: - return false; - } -} - -Status TFSRestParser::parseColumnFormat(rapidjson::Value& node) { - order = Order::COLUMN; - // no named scalar - if (node.IsNumber() && requestProto.inputs_size() == 1) { - addNumber(requestProto.mutable_inputs()->begin()->second, node); - format = Format::NONAMED; - return StatusCode::OK; - } - // no named format - if (node.IsArray()) { - if (requestProto.inputs_size() == 0) { - return StatusCode::REST_INPUT_NOT_PREALLOCATED; - } else if (requestProto.inputs_size() != 1) { - return StatusCode::INVALID_INPUT_FORMAT; - } - auto inputsIterator = requestProto.mutable_inputs()->begin(); - if (inputsIterator == requestProto.mutable_inputs()->end()) { - const std::string details = "Failed to parse column formatted request."; - SPDLOG_ERROR("Internal error occurred: {}", details); - return Status(StatusCode::INTERNAL_ERROR, details); - } - if (!parseArray(node, 0, inputsIterator->second, inputsIterator->first)) { - return StatusCode::REST_COULD_NOT_PARSE_INPUT; - } - format = Format::NONAMED; - return StatusCode::OK; - } - // named format - if (!node.IsObject()) { - return StatusCode::REST_INPUTS_NOT_AN_OBJECT; - } - if (node.GetObject().MemberCount() == 0) { - return StatusCode::REST_NO_INPUTS_FOUND; - } - for (auto& kv : node.GetObject()) { - std::string tensorName = kv.name.GetString(); - inputsFoundInRequest.insert(tensorName); - auto& proto = (*requestProto.mutable_inputs())[tensorName]; - // scalar - if (kv.value.IsNumber()) { - setDTypeIfNotSet(kv.value, proto, tensorName); - addNumber(proto, kv.value); - } else if (!parseArray(kv.value, 0, proto, tensorName)) { - return StatusCode::REST_COULD_NOT_PARSE_INPUT; - } - } - removeUnusedInputs(); - format = Format::NAMED; - return StatusCode::OK; -} - -Status TFSRestParser::parse(const char* json) { - rapidjson::Document doc; - auto status = parseJsonWithDepthLimit(doc, json, MAX_NESTING_DEPTH); - if (!status.ok()) { - SPDLOG_DEBUG("Request is not a valid JSON: {}", status.string()); - return status; - } - if (!doc.IsObject()) { - return StatusCode::REST_BODY_IS_NOT_AN_OBJECT; - } - auto instancesItr = doc.FindMember("instances"); - auto inputsItr = doc.FindMember("inputs"); - if (instancesItr != doc.MemberEnd() && inputsItr != doc.MemberEnd()) { - return StatusCode::REST_PREDICT_UNKNOWN_ORDER; - } - if (instancesItr != doc.MemberEnd()) { - return parseRowFormat(instancesItr->value); - } - if (inputsItr != doc.MemberEnd()) { - return parseColumnFormat(inputsItr->value); - } - return StatusCode::REST_PREDICT_UNKNOWN_ORDER; -} - -void TFSRestParser::increaseBatchSize(tensorflow::TensorProto& proto) { - if (proto.tensor_shape().dim_size() < 1) { - proto.mutable_tensor_shape()->add_dim()->set_size(0); - } - proto.mutable_tensor_shape()->mutable_dim(0)->set_size(proto.tensor_shape().dim(0).size() + 1); -} - -bool TFSRestParser::setDimOrValidate(tensorflow::TensorProto& proto, int dim, int size) { - if (proto.tensor_shape().dim_size() > dim) { - return proto.tensor_shape().dim(dim).size() == size; - } else { - while (proto.tensor_shape().dim_size() <= dim) { - proto.mutable_tensor_shape()->add_dim()->set_size(0); - } - proto.mutable_tensor_shape()->mutable_dim(dim)->set_size(size); - return true; - } -} - -static bool getB64FromValue(const rapidjson::Value& value, std::string& b64Val) { - if (!isBinary(value)) { - return false; - } - - b64Val = value["b64"].GetString(); - return true; -} - -bool TFSRestParser::addValue(tensorflow::TensorProto& proto, const rapidjson::Value& value) { - if (isBinary(value)) { - std::string b64Val; - if (!getB64FromValue(value, b64Val)) - return false; - std::string decodedBytes; - if (decodeBase64(b64Val, decodedBytes) == StatusCode::OK) { - proto.add_string_val(decodedBytes.c_str(), decodedBytes.length()); - proto.set_dtype(tensorflow::DataType::DT_STRING); - return true; - } else { - return false; - } - } - if (value.IsString() && (proto.dtype() == tensorflow::DataType::DT_UINT8 || proto.dtype() == tensorflow::DataType::DT_STRING)) { - proto.add_string_val(value.GetString(), strlen(value.GetString())); - proto.set_dtype(tensorflow::DataType::DT_STRING); - return true; - } - - if (!value.IsNumber()) { - return false; - } - - return addNumber(proto, value); -} - -// This is still required for parsing inputs which are not present in model/DAG. -// Such inputs are then removed from proto at the end of parsing phase. -bool TFSRestParser::setDTypeIfNotSet(const rapidjson::Value& value, tensorflow::TensorProto& proto, const std::string& tensorName) { - if (tensorPrecisionMap.count(tensorName)) - return true; - - if (value.IsInt()) - tensorPrecisionMap[tensorName] = ovms::Precision::I32; - else if (value.IsDouble()) - tensorPrecisionMap[tensorName] = ovms::Precision::FP32; - else - return false; - - proto.set_dtype(getPrecisionAsDataType(tensorPrecisionMap[tensorName])); - return true; -} - Status KFSRestParser::parseId(rapidjson::Value& node) { if (!node.IsString()) { return StatusCode::REST_COULD_NOT_PARSE_INPUT; @@ -630,6 +118,9 @@ Status KFSRestParser::parseOutputs(rapidjson::Value& node) { } Status KFSRestParser::parseData(rapidjson::Value& node, ::KFSRequest::InferInputTensor& input) { + if (!node.IsArray()) { + return StatusCode::REST_COULD_NOT_PARSE_INPUT; + } if (input.datatype() == "FP32") { HANDLE_VALUE(mutable_fp32_contents, GetFloat, IsNumber) } else if (input.datatype() == "INT64") { diff --git a/src/rest_parser.hpp b/src/rest_parser.hpp index d729eff985..e5f5ccf82f 100644 --- a/src/rest_parser.hpp +++ b/src/rest_parser.hpp @@ -15,215 +15,20 @@ //***************************************************************************** #pragma once -#include #include -#include -#pragma warning(push) -#pragma warning(disable : 6313) -#include -#pragma warning(pop) -#include + +#include "src/port/rapidjson_document.hpp" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" - #include "kfs_frontend/kfs_grpc_inference_service.hpp" #pragma GCC diagnostic pop -#include "tensorinfo.hpp" - namespace ovms { class Status; -/** - * @brief Request order types - */ -enum class Order { - UNKNOWN, - ROW, - COLUMN -}; - -/** - * @brief Request format types - */ -enum class Format { - UNKNOWN, - NAMED, - NONAMED -}; - class RestParser {}; -/** - * @brief This class encapsulates http request body string parsing to request proto. - */ -class TFSRestParser : RestParser { - /** - * @brief Request order - */ - Order order = Order::UNKNOWN; - - /** - * @brief Request format - */ - Format format = Format::UNKNOWN; - - /** - * @brief Request proto - */ - tensorflow::serving::PredictRequest requestProto; - - std::set inputsFoundInRequest; - - /** - * @brief Request content precision - */ - std::unordered_map tensorPrecisionMap; - - void removeUnusedInputs(); - - /** - * @brief Increases batch size (0th-dimension) of tensor - */ - static void increaseBatchSize(tensorflow::TensorProto& proto); - - /** - * @brief Sets specific dimension to given size - * - * @return returns false if dimension already existed and did not match requested size, true otherwise - */ - static bool setDimOrValidate(tensorflow::TensorProto& proto, int dim, int size); - - /** - * Parses and adds rapidjson value to tensor proto depending on underlying tensor data type - */ - static bool addValue(tensorflow::TensorProto& proto, const rapidjson::Value& value); - - bool parseSequenceIdInput(rapidjson::Value& doc, tensorflow::TensorProto& proto, const std::string& tensorName); - bool parseSequenceControlInput(rapidjson::Value& doc, tensorflow::TensorProto& proto, const std::string& tensorName); - bool parseSpecialInput(rapidjson::Value& doc, tensorflow::TensorProto& proto, const std::string& tensorName); - - /** - * @brief Parses rapidjson Node for arrays or numeric values on certain level of nesting. - * - * @param doc rapidjson Node - * @param dim level of nesting - * @param input destination for parsing numeric values - * - * @return false if processing failed, true when succeeded - * - * Rapid json node expected to be passed in following structure: - * [ - * [...], - * [...], - * ... - * ] - */ - bool parseArray(rapidjson::Value& doc, int dim, tensorflow::TensorProto& proto, const std::string& tensorName); - - /** - * @brief Parses rapidjson Node for inputs in a string(name)=>array(data) format - * - * @param doc rapidjson Node - * - * @return false if processing failed, true when succeeded - * - * Rapid json node expected to be passed in following structure: - * { - * "input1": [[...], [...], ...], - * "input2": [[...], [...], ...], - * ... - * } - */ - bool parseInstance(rapidjson::Value& doc); - - /** - * @brief Checks whether all inputs have equal batch size, 0th-dimension - * - * @return true or false - */ - bool isBatchSizeEqualForAllInputs() const; - - /** - * @brief Parses row format: list of objects, each object corresponding to one batch with one or multiple inputs. - * When no named format is detected, instance is treated as array of single input batches with no name. - * - * @param node rapidjson Node - * - * @return Status indicating if processing succeeded, error code otherwise - * - * Rapid json node expected to be passed in following structure: - * [{inputs...}, {inputs...}, {inputs...}, ...] - * or: - * [no named input data batches...] - */ - Status parseRowFormat(rapidjson::Value& node); - - /** - * @brief Parses column format: object of input:batches key value pairs. - * When no named format is detected, instance is treated as array of single input batches with no name. - * - * @param node rapidjson Node - * - * @return Status indicating if processing succeeded, error code otherwise - * - * Rapid json node expected to be passed in following structure: - * {"inputA": [...], "inputB": [...], ...} - * or: - * [no named input data batches...] - */ - Status parseColumnFormat(rapidjson::Value& node); - -public: - bool setDTypeIfNotSet(const rapidjson::Value& value, tensorflow::TensorProto& proto, const std::string& tensorName); - /** - * @brief Constructor for preallocating memory for inputs beforehand. Size is calculated from tensor shape required by backend. - * - * @param tensors Tensor map with model input parameters - */ - TFSRestParser(const tensor_map_t& tensors); - - /** - * @brief Gets parsed request proto - * - * @return proto - */ - tensorflow::serving::PredictRequest& getProto() { return requestProto; } - - /** - * @brief Gets request order - */ - Order getOrder() const { - return order; - } - - /** - * @brief Gets request format - */ - Format getFormat() const { - return format; - } - - /** - * @brief Parses http request body string - * - * @param json request string - * - * @return Status indicating error code or success - * - * JSON expected to be passed in following structure: - * { - * "signature_name": "serving_default", - * "instances": [ - * {...}, {...}, {...}, ... - * ] - * } - */ - Status parse(const char* json); -}; - class KFSRestParser : RestParser { ::KFSRequest requestProto; Status parseId(rapidjson::Value& node); diff --git a/src/rest_utils.cpp b/src/rest_utils.cpp index 8d65fe8ee5..b3fb2768b2 100644 --- a/src/rest_utils.cpp +++ b/src/rest_utils.cpp @@ -30,196 +30,31 @@ #pragma warning(push) #pragma warning(disable : 6001 4324 6385 6386 6011 4457 6308 6387 6246) #include "absl/strings/escaping.h" -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#pragma GCC diagnostic ignored "-Wunused-but-set-variable" -#include "tensorflow_serving/util/json_tensor.h" -#pragma GCC diagnostic pop #pragma warning(pop) #include "kfs_frontend/kfs_utils.hpp" #include "precision.hpp" #include "src/kfserving_api/grpc_predict_v2.grpc.pb.h" #include "status.hpp" -#include "tfs_frontend/tfs_utils.hpp" #include "timer.hpp" -using tensorflow::DataType; -using tensorflow::DataTypeSize; -using tensorflow::serving::JsonPredictRequestFormat; -using tensorflow::serving::MakeJsonFromTensors; -using tensorflow::serving::PredictResponse; - namespace { enum : unsigned int { CONVERT, - MAKE_JSON_FROM_TENSORS, TIMER_END }; } namespace ovms { -static Status checkValField(const size_t& fieldSize, const size_t& expectedElementsNumber) { - if (fieldSize != expectedElementsNumber) { +static Status checkValField(size_t valFieldSize, size_t expectedElementsNumber) { + if (valFieldSize != expectedElementsNumber) { std::stringstream ss; - ss << "Expected val field elements number: " << expectedElementsNumber << "; actual: " << fieldSize; + ss << "Expected val field elements number: " << expectedElementsNumber << "; actual: " << valFieldSize; return Status(StatusCode::REST_SERIALIZE_VAL_FIELD_INVALID_SIZE, ss.str()); } return StatusCode::OK; } -Status makeJsonFromPredictResponse( - PredictResponse& response_proto, - std::string* response_json, - Order order) { - if (order == Order::UNKNOWN) { - return StatusCode::REST_PREDICT_UNKNOWN_ORDER; - } - - Timer timer; - using std::chrono::microseconds; - - timer.start(CONVERT); - - for (auto& kv : *response_proto.mutable_outputs()) { - auto& tensor = kv.second; - - size_t dataTypeSize = DataTypeSize(tensor.dtype()); - size_t expectedContentSize = dataTypeSize; - for (int i = 0; i < tensor.tensor_shape().dim_size(); i++) { - expectedContentSize *= tensor.tensor_shape().dim(i).size(); - } - size_t expectedElementsNumber = dataTypeSize > 0 ? expectedContentSize / dataTypeSize : 0; - bool seekDataInValField = false; - - if (tensor.tensor_content().size() == 0) - seekDataInValField = true; - else if (tensor.tensor_content().size() != expectedContentSize) - return StatusCode::REST_SERIALIZE_TENSOR_CONTENT_INVALID_SIZE; - - switch (tensor.dtype()) { - case DataType::DT_FLOAT: - if (seekDataInValField) { - auto status = checkValField(tensor.float_val_size(), expectedElementsNumber); - if (!status.ok()) - return status; - } else { - for (size_t i = 0; i < tensor.tensor_content().size(); i += sizeof(float)) - tensor.add_float_val(*reinterpret_cast(tensor.mutable_tensor_content()->data() + i)); - } - break; - case DataType::DT_INT32: - if (seekDataInValField) { - auto status = checkValField(tensor.int_val_size(), expectedElementsNumber); - if (!status.ok()) - return status; - } else { - for (size_t i = 0; i < tensor.tensor_content().size(); i += sizeof(int32_t)) - tensor.add_int_val(*reinterpret_cast(tensor.mutable_tensor_content()->data() + i)); - } - break; - case DataType::DT_INT8: - if (seekDataInValField) { - auto status = checkValField(tensor.int_val_size(), expectedElementsNumber); - if (!status.ok()) - return status; - } else { - for (size_t i = 0; i < tensor.tensor_content().size(); i += sizeof(int8_t)) - tensor.add_int_val(*reinterpret_cast(tensor.mutable_tensor_content()->data() + i)); - } - break; - case DataType::DT_UINT8: - if (seekDataInValField) { - auto status = checkValField(tensor.int_val_size(), expectedElementsNumber); - if (!status.ok()) - return status; - } else { - for (size_t i = 0; i < tensor.tensor_content().size(); i += sizeof(uint8_t)) - tensor.add_int_val(*reinterpret_cast(tensor.mutable_tensor_content()->data() + i)); - } - break; - case DataType::DT_DOUBLE: - if (seekDataInValField) { - auto status = checkValField(tensor.double_val_size(), expectedElementsNumber); - if (!status.ok()) - return status; - } else { - for (size_t i = 0; i < tensor.tensor_content().size(); i += sizeof(double)) - tensor.add_double_val(*reinterpret_cast(tensor.mutable_tensor_content()->data() + i)); - } - break; - case DataType::DT_INT16: - if (seekDataInValField) { - auto status = checkValField(tensor.int_val_size(), expectedElementsNumber); - if (!status.ok()) - return status; - } else { - for (size_t i = 0; i < tensor.tensor_content().size(); i += sizeof(int16_t)) - tensor.add_int_val(*reinterpret_cast(tensor.mutable_tensor_content()->data() + i)); - } - break; - case DataType::DT_INT64: - if (seekDataInValField) { - auto status = checkValField(tensor.int64_val_size(), expectedElementsNumber); - if (!status.ok()) - return status; - } else { - for (size_t i = 0; i < tensor.tensor_content().size(); i += sizeof(int64_t)) - tensor.add_int64_val(*reinterpret_cast(tensor.mutable_tensor_content()->data() + i)); - } - break; - case DataType::DT_UINT32: - if (seekDataInValField) { - auto status = checkValField(tensor.uint32_val_size(), expectedElementsNumber); - if (!status.ok()) - return status; - } else { - for (size_t i = 0; i < tensor.tensor_content().size(); i += sizeof(uint32_t)) - tensor.add_uint32_val(*reinterpret_cast(tensor.mutable_tensor_content()->data() + i)); - } - break; - case DataType::DT_UINT64: - if (seekDataInValField) { - auto status = checkValField(tensor.uint64_val_size(), expectedElementsNumber); - if (!status.ok()) - return status; - } else { - for (size_t i = 0; i < tensor.tensor_content().size(); i += sizeof(uint64_t)) - tensor.add_uint64_val(*reinterpret_cast(tensor.mutable_tensor_content()->data() + i)); - } - break; - case DataType::DT_STRING: - if (seekDataInValField) { - auto status = checkValField(tensor.string_val_size(), tensor.tensor_shape().dim(0).size()); - if (!status.ok()) - return status; - } - break; - default: - return StatusCode::REST_UNSUPPORTED_PRECISION; - } - } - - timer.stop(CONVERT); - timer.start(MAKE_JSON_FROM_TENSORS); - - const auto& tf_status = MakeJsonFromTensors( - response_proto.outputs(), - order == Order::ROW ? JsonPredictRequestFormat::kRow : JsonPredictRequestFormat::kColumnar, - response_json); - - timer.stop(MAKE_JSON_FROM_TENSORS); - SPDLOG_DEBUG("tensor_content to *_val container conversion: {:.3f} ms", timer.elapsed(CONVERT) / 1000); - SPDLOG_DEBUG("MakeJsonFromTensors call: {:.3f} ms", timer.elapsed(MAKE_JSON_FROM_TENSORS) / 1000); - - if (!tf_status.ok()) { - SPDLOG_ERROR("Creating json from tensors failed: {}", tf_status.message()); - return StatusCode::REST_PROTO_TO_STRING_ERROR; - } - - return StatusCode::OK; -} - static Status parseResponseParameters(const ::KFSResponse& response_proto, rapidjson::PrettyWriter& writer) { if (response_proto.parameters_size() > 0) { writer.Key("parameters"); diff --git a/src/rest_utils.hpp b/src/rest_utils.hpp index 4d2fae048e..ec63e12c65 100644 --- a/src/rest_utils.hpp +++ b/src/rest_utils.hpp @@ -19,20 +19,10 @@ #include #include -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop - -#include "rest_parser.hpp" +#include "kfs_frontend/kfs_grpc_inference_service.hpp" namespace ovms { class Status; -Status makeJsonFromPredictResponse( - tensorflow::serving::PredictResponse& response_proto, - std::string* response_json, - Order order); - Status makeJsonFromPredictResponse( const ::KFSResponse& response_proto, std::string* response_json, diff --git a/src/servable_name_checker.hpp b/src/servable_name_checker.hpp index 2b8e9baf86..11af2ad186 100644 --- a/src/servable_name_checker.hpp +++ b/src/servable_name_checker.hpp @@ -21,6 +21,8 @@ namespace ovms { +class ServableDefinition; + enum class ServableQueryType : uint8_t { Model = 1 << 0, Pipeline = 1 << 1, @@ -43,6 +45,8 @@ class ServableNameChecker { virtual ~ServableNameChecker() = default; virtual bool servableExists(const std::string& name, ServableQueryType check = ServableQueryType::All) const = 0; virtual bool aliasesConflict(const std::vector& aliases, const std::string& ownGraphName) const = 0; + virtual ServableDefinition* findServableDefinition(const std::string& name) const = 0; + virtual std::vector getServableDefinitionNames() const = 0; }; } // namespace ovms diff --git a/src/server.cpp b/src/server.cpp index d182d6feb5..49419f9239 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -60,10 +60,8 @@ #include "kfs_frontend/kfs_grpc_inference_service.hpp" #include "logging.hpp" #include "metrics/metric_module.hpp" -#include "model_service.hpp" #include "modelmanager.hpp" #include "ovms_exit_codes.hpp" -#include "prediction_service.hpp" #include "profiler.hpp" #include "profilermodule.hpp" #include "pull_module/hf_pull_model_module.hpp" diff --git a/src/tensorflow_type_utils.cpp b/src/tensorflow_type_utils.cpp new file mode 100644 index 0000000000..eca0683104 --- /dev/null +++ b/src/tensorflow_type_utils.cpp @@ -0,0 +1,67 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "tensorflow_type_utils.hpp" + +#include + +#include "precision.hpp" + +namespace ovms { + +Precision TFSPrecisionToOvmsPrecision(const TFSDataType& datatype) { + static std::unordered_map precisionMap{ + {TFSDataType::DT_FLOAT, Precision::FP32}, + {TFSDataType::DT_DOUBLE, Precision::FP64}, + {TFSDataType::DT_HALF, Precision::FP16}, + {TFSDataType::DT_INT64, Precision::I64}, + {TFSDataType::DT_INT32, Precision::I32}, + {TFSDataType::DT_INT16, Precision::I16}, + {TFSDataType::DT_INT8, Precision::I8}, + {TFSDataType::DT_UINT64, Precision::U64}, + {TFSDataType::DT_UINT16, Precision::U16}, + {TFSDataType::DT_UINT8, Precision::U8}, + {TFSDataType::DT_STRING, Precision::STRING}, + {TFSDataType::DT_BOOL, Precision::BOOL}}; + auto it = precisionMap.find(datatype); + if (it == precisionMap.end()) { + return Precision::UNDEFINED; + } + return it->second; +} + +TFSDataType getPrecisionAsDataType(Precision precision) { + static std::unordered_map precisionMap{ + {Precision::FP64, TFSDataType::DT_DOUBLE}, + {Precision::FP32, TFSDataType::DT_FLOAT}, + {Precision::FP16, TFSDataType::DT_HALF}, + {Precision::I16, TFSDataType::DT_INT16}, + {Precision::U8, TFSDataType::DT_UINT8}, + {Precision::I8, TFSDataType::DT_INT8}, + {Precision::U16, TFSDataType::DT_UINT16}, + {Precision::I32, TFSDataType::DT_INT32}, + {Precision::I64, TFSDataType::DT_INT64}, + {Precision::U32, TFSDataType::DT_UINT32}, + {Precision::U64, TFSDataType::DT_UINT64}, + {Precision::BOOL, TFSDataType::DT_BOOL}, + {Precision::STRING, TFSDataType::DT_STRING}}; + auto it = precisionMap.find(precision); + if (it == precisionMap.end()) { + return TFSDataType::DT_INVALID; + } + return it->second; +} + +} // namespace ovms diff --git a/src/tfs_frontend/tensor_conversion.cpp b/src/tensorflow_type_utils.hpp similarity index 50% rename from src/tfs_frontend/tensor_conversion.cpp rename to src/tensorflow_type_utils.hpp index 2336bde57b..fbcbb9ec46 100644 --- a/src/tfs_frontend/tensor_conversion.cpp +++ b/src/tensorflow_type_utils.hpp @@ -1,5 +1,5 @@ //***************************************************************************** -// Copyright 2025 Intel Corporation +// Copyright 2026 Intel Corporation // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,39 +13,24 @@ // See the License for the specific language governing permissions and // limitations under the License. //***************************************************************************** -#include "tfs_utils.hpp" -#include "../tensor_conversion.hpp" -#include "../tensor_conversion_common.hpp" +#pragma once +#pragma warning(push) +#pragma warning(disable : 4624 6001 6385 6386 6326 6011 4457 6308 6387 6246) #pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" #pragma GCC diagnostic ignored "-Wunused-but-set-variable" +#pragma GCC diagnostic ignored "-Wall" #include "tensorflow/core/framework/tensor.h" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" #pragma GCC diagnostic pop +#pragma warning(pop) -#include -#include -#include -#include -#include - -#include +#include "precision.hpp" -#include "../deps/opencv.hpp" -#include "../logging.hpp" -#include "../profiler.hpp" -#include "../status.hpp" +using TFSDataType = tensorflow::DataType; // TODO @atobiszei since we dont have TFS now we can rename to TFDataType? namespace ovms { -Status convertStringRequestFromBufferToOVTensor2D(const tensorflow::TensorProto& src, ov::Tensor& tensor, const std::string* buffer) { - SPDLOG_ERROR("String conversion is not implemented for TFS-API"); - return StatusCode::NOT_IMPLEMENTED; -} -} // namespace ovms -// we need to see declarations before -#include "../tensor_conversion_after.hpp" -namespace ovms { -template Status convertStringRequestToOVTensor2D(const TFSInputTensorType& src, ov::Tensor& tensor, const std::string* buffer); +Precision TFSPrecisionToOvmsPrecision(const TFSDataType& datatype); +TFSDataType getPrecisionAsDataType(Precision precision); + } // namespace ovms diff --git a/src/test/c_api_stress_tests.cpp b/src/test/c_api_stress_tests.cpp index 4b134cd8d3..c247ddc96c 100644 --- a/src/test/c_api_stress_tests.cpp +++ b/src/test/c_api_stress_tests.cpp @@ -26,10 +26,8 @@ #include "../dags/pipeline.hpp" #include "../dags/pipeline_factory.hpp" #include "../dags/pipelinedefinition.hpp" -#include "../get_model_metadata_impl.hpp" #include "src/filesystem/localfilesystem.hpp" #include "../logging.hpp" -#include "../model_service.hpp" #include "../modelconfig.hpp" #include "../modelinstance.hpp" #include "../prediction_service_utils.hpp" @@ -42,8 +40,116 @@ #include "test_utils.hpp" using namespace ovms; -using namespace tensorflow; -using namespace tensorflow::serving; + +static const char* stressTestPipelineOneDummyConfigSpecificVersionUsed = R"( +{ + "model_config_list": [ + { + "config": { + "name": "dummy", + "base_path": "/ovms/src/test/dummy", + "target_device": "CPU", + "model_version_policy": {"latest": {"num_versions":1}}, + "nireq": 100, + "shape": {"b": "(1,10) "} + } + } + ], + "pipeline_config_list": [ + { + "name": "pipeline1Dummy", + "inputs": ["custom_dummy_input"], + "nodes": [ + { + "name": "dummyNode", + "model_name": "dummy", + "version": 1, + "type": "DL model", + "inputs": [ + {"b": {"node_name": "request", + "data_item": "custom_dummy_input"}} + ], + "outputs": [ + {"data_item": "a", + "alias": "new_dummy_output"} + ] + } + ], + "outputs": [ + {"custom_dummy_output": {"node_name": "dummyNode", + "data_item": "new_dummy_output"} + } + ] + } + ] +})"; + +static const char* stressPipelineCustomNodeAddOneThenDummy = R"( +{ + "custom_node_library_config_list": [ + { + "name": "lib_add_one", + "base_path": "/ovms/bazel-bin/src/libcustom_node_add_one.so" + } + ], + "model_config_list": [ + { + "config": { + "name": "dummy", + "base_path": "/ovms/src/test/dummy", + "target_device": "CPU", + "model_version_policy": {"all": {}}, + "nireq": 20, + "shape": {"b": "(1,10) "} + } + } + ], + "pipeline_config_list": [ + { + "name": "pipeline1Dummy", + "inputs": ["custom_dummy_input"], + "nodes": [ + { + "name": "custom_node", + "library_name": "lib_add_one", + "type": "custom", + "params": { + "output_queue_size": "20", + "info_queue_size": "20", + "add_number": "1", + "sub_number": "0" + }, + "inputs": [ + {"input_numbers": {"node_name": "request", + "data_item": "custom_dummy_input"}} + ], + "outputs": [ + {"data_item": "output_numbers", + "alias": "custom_node_output"} + ] + }, + { + "name": "dummyNode", + "model_name": "dummy", + "type": "DL model", + "inputs": [ + {"b": {"node_name": "custom_node", + "data_item": "custom_node_output"}} + ], + "outputs": [ + {"data_item": "a", + "alias": "dummy_output"} + ] + } + ], + "outputs": [ + {"custom_dummy_output": {"node_name": "dummyNode", + "data_item": "dummy_output"} + } + ] + } + ] +})"; using testing::_; using testing::Return; diff --git a/src/test/custom_loader_test.cpp b/src/test/custom_loader_test.cpp index 9fe0148699..4907928f08 100644 --- a/src/test/custom_loader_test.cpp +++ b/src/test/custom_loader_test.cpp @@ -1,1465 +1,179 @@ -//***************************************************************************** -// Copyright 2020-2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "../executingstreamidguard.hpp" -#include "../get_model_metadata_impl.hpp" -#include "src/filesystem/localfilesystem.hpp" -#include "../model.hpp" -#include "../model_service.hpp" -#include "../modelinstance.hpp" -#include "../modelinstanceunloadguard.hpp" -#include "../modelmanager.hpp" -#include "../modelversionstatus.hpp" -#include "../prediction_service_utils.hpp" -#include "../schema.hpp" -#include "../sequence_processing_spec.hpp" -#include "mockmodelinstancechangingstates.hpp" -#include "test_utils.hpp" -#include "light_test_utils.hpp" -#include "platform_utils.hpp" - -using testing::_; -using testing::ContainerEq; -using testing::Each; -using testing::Eq; -using ::testing::NiceMock; -using testing::Return; -using testing::ReturnRef; -using testing::UnorderedElementsAre; - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wnarrowing" - -using namespace ovms; - -/* ------------------------------------------------- -AFTER SAMPLE CUSTOM LOADER REMOVAL BELOW CONFIGURATIONS ARE NOT USED -REMOVE THIS ENTIRE FILE ONCE THE FEATURE IS REMOVED -------------------------------------------------- - -namespace { - -// Custom Loader Config Keys -#define ENABLE_FORCE_BLACKLIST_CHECK "ENABLE_FORCE_BLACKLIST_CHECK" - -// config_model_with_customloader -const char* custom_loader_config_model = R"({ - "custom_loader_config_list":[ - { - "config":{ - "loader_name":"sample-loader", - "library_path": "/ovms/bazel-bin/src/libsampleloader.so" - } - } - ], - "model_config_list":[ - { - "config":{ - "name":"dummy", - "base_path": "/tmp/test_cl_models/model1", - "nireq": 1, - "custom_loader_options": {"loader_name": "sample-loader", "model_file": "dummy.xml", "bin_file": "dummy.bin"} - } - } - ] - })"; - -// config_model_with_customloader -const char* custom_loader_config_model_relative_paths = R"({ - "custom_loader_config_list":[ - { - "config":{ - "loader_name":"sample-loader", - "library_path": "libsampleloader.so" - } - } - ], - "model_config_list":[ - { - "config":{ - "name":"dummy", - "base_path": "test_cl_models/model1", - "nireq": 1, - "custom_loader_options": {"loader_name": "sample-loader", "model_file": "dummy.xml", "bin_file": "dummy.bin"} - } - } - ] - })"; - -// config_no_model_with_customloader -const char* custom_loader_config_model_deleted = R"({ - "custom_loader_config_list":[ - { - "config":{ - "loader_name":"sample-loader", - "library_path": "/ovms/bazel-bin/src/libsampleloader.so" - } - } - ], - "model_config_list":[] - })"; - -// config_2_models_with_customloader -const char* custom_loader_config_model_new = R"({ - "custom_loader_config_list":[ - { - "config":{ - "loader_name":"sample-loader", - "library_path": "/ovms/bazel-bin/src/libsampleloader.so" - } - } - ], - "model_config_list":[ - { - "config":{ - "name":"dummy", - "base_path": "/tmp/test_cl_models/model1", - "nireq": 1, - "custom_loader_options": {"loader_name": "sample-loader", "model_file": "dummy.xml", "bin_file": "dummy.bin"} - } - }, - { - "config":{ - "name":"dummy-new", - "base_path": "/tmp/test_cl_models/model2", - "nireq": 1, - "custom_loader_options": {"loader_name": "sample-loader", "model_file": "dummy.xml", "bin_file": "dummy.bin"} - } - } - ] - })"; - -// config_model_without_customloader_options -const char* custom_loader_config_model_customloader_options_removed = R"({ - "custom_loader_config_list":[ - { - "config":{ - "loader_name":"sample-loader", - "library_path": "/ovms/bazel-bin/src/libsampleloader.so" - } - } - ], - "model_config_list":[ - { - "config":{ - "name":"dummy", - "base_path": "/tmp/test_cl_models/model1", - "nireq": 1 - } - } - ] - })"; - -const char* config_model_with_customloader_options_unknown_loadername = R"({ - "custom_loader_config_list":[ - { - "config":{ - "loader_name":"sample-loader", - "library_path": "/ovms/bazel-bin/src/libsampleloader.so" - } - } - ], - "model_config_list":[ - { - "config":{ - "name":"dummy", - "base_path": "/tmp/test_cl_models/model1", - "nireq": 1, - "custom_loader_options": {"loader_name": "unknown", "model_file": "dummy.xml", "bin_file": "dummy.bin"} - } - } - ] - })"; - -// config_model_with_customloader -const char* custom_loader_config_model_multiple = R"({ - "custom_loader_config_list":[ - { - "config":{ - "loader_name":"sample-loader-a", - "library_path": "/ovms/bazel-bin/src/libsampleloader.so" - } - }, - { - "config":{ - "loader_name":"sample-loader-b", - "library_path": "/ovms/bazel-bin/src/libsampleloader.so" - } - }, - { - "config":{ - "loader_name":"sample-loader-c", - "library_path": "/ovms/bazel-bin/src/libsampleloader.so" - } - } - ], - "model_config_list":[ - { - "config":{ - "name":"dummy-a", - "base_path": "/tmp/test_cl_models/model1", - "nireq": 1, - "custom_loader_options": {"loader_name": "sample-loader-a", "model_file": "dummy.xml", "bin_file": "dummy.bin"} - } - }, - { - "config":{ - "name":"dummy-b", - "base_path": "/tmp/test_cl_models/model1", - "nireq": 1, - "custom_loader_options": {"loader_name": "sample-loader-b", "model_file": "dummy.xml", "bin_file": "dummy.bin"} - } - }, - { - "config":{ - "name":"dummy-c", - "base_path": "/tmp/test_cl_models/model1", - "nireq": 1, - "custom_loader_options": {"loader_name": "sample-loader-c", "model_file": "dummy.xml", "bin_file": "dummy.bin"} - } - } - ] - })"; - -const char* custom_loader_config_model_blacklist = R"({ - "custom_loader_config_list":[ - { - "config":{ - "loader_name":"sample-loader", - "library_path": "/ovms/bazel-bin/src/libsampleloader.so", - "loader_config_file": "sample-loader-config" - } - } - ], - "model_config_list":[ - { - "config":{ - "name":"dummy", - "base_path": "/tmp/test_cl_models/model1", - "nireq": 1, - "custom_loader_options": {"loader_name": "sample-loader", "model_file": "dummy.xml", "bin_file": "dummy.bin", "enable_file": "dummy.status"} - } - } - ] - })"; - -const char* empty_config = R"({ - "custom_loader_config_list":[], - "model_config_list":[] - })"; - -const char* expected_json_available = R"({ - "model_version_status": [ - { - "version": "1", - "state": "AVAILABLE", - "status": { - "error_code": "OK", - "error_message": "OK" - } - } - ] -} -)"; - -const char* expected_json_end = R"({ - "model_version_status": [ - { - "version": "1", - "state": "END", - "status": { - "error_code": "OK", - "error_message": "OK" - } - } - ] -} -)"; - -const char* expected_json_loading_error = R"({ - "model_version_status": [ - { - "version": "1", - "state": "LOADING", - "status": { - "error_code": "UNKNOWN", - "error_message": "UNKNOWN" - } - } - ] -} -)"; - -} // namespace - -*/ - -class TestCustomLoader : public ::testing::Test { -public: - void SetUp() { - const ::testing::TestInfo* const test_info = - ::testing::UnitTest::GetInstance()->current_test_info(); - - cl_models_path = getGenericFullPathForTmp("/tmp/" + std::string(test_info->name())); - cl_model_1_path = cl_models_path + "/model1/"; - cl_model_2_path = cl_models_path + "/model2/"; - - const std::string FIRST_MODEL_NAME = "dummy"; - const std::string SECOND_MODEL_NAME = "dummy_new"; - - std::filesystem::remove_all(cl_models_path); - std::filesystem::create_directories(cl_model_1_path); - } - void TearDown() { - // Create config file with an empty config & reload - const char* empty_config = R"({ - "custom_loader_config_list":[], - "model_config_list":[] - })"; - std::string configStr = empty_config; - std::string fileToReload = cl_models_path + "/cl_config.json"; - createConfigFileWithContent(configStr, fileToReload); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - // Clean up temporary destination - std::filesystem::remove_all(cl_models_path); - } - - /** - * @brief This function should mimic most closely predict request to check for thread safety - */ - void performPredict(const std::string modelName, - const ovms::model_version_t modelVersion, - const tensorflow::serving::PredictRequest& request, - std::unique_ptr> waitBeforeGettingModelInstance = nullptr, - std::unique_ptr> waitBeforePerformInference = nullptr); - - void deserialize(const std::vector& input, ov::InferRequest& inferRequest, std::shared_ptr modelInstance) { - try { - ov::Tensor tensor( - modelInstance->getInputsInfo().at(DUMMY_MODEL_INPUT_NAME)->getOvPrecision(), - modelInstance->getInputsInfo().at(DUMMY_MODEL_INPUT_NAME)->getShape().createPartialShape().get_shape(), - const_cast(reinterpret_cast(input.data()))); - inferRequest.set_tensor(DUMMY_MODEL_INPUT_NAME, tensor); - } catch (...) { - ASSERT_TRUE(false) << "exception during deserialize"; - } - } - - void serializeAndCheck(int outputSize, ov::InferRequest& inferRequest) { - std::vector output(outputSize); - ASSERT_THAT(output, Each(Eq(0.))); - auto tensorOutput = inferRequest.get_tensor(DUMMY_MODEL_OUTPUT_NAME); - ASSERT_EQ(tensorOutput.get_byte_size(), outputSize * sizeof(float)); - std::memcpy(output.data(), tensorOutput.data(), outputSize * sizeof(float)); - EXPECT_THAT(output, Each(Eq(2.))); - } - - ovms::Status performInferenceWithRequest(const tensorflow::serving::PredictRequest& request, tensorflow::serving::PredictResponse& response) { - std::shared_ptr model; - std::unique_ptr unload_guard; - auto status = manager.getModelInstance("dummy", 0, model, unload_guard); - if (!status.ok()) { - return status; - } - - response.Clear(); - return model->infer(&request, &response, unload_guard); - } - -public: - ConstructorEnabledModelManager manager; - - ~TestCustomLoader() { - std::cout << "Destructor of TestCustomLoader()" << std::endl; - } - - std::string cl_models_path; - std::string cl_model_1_path; - std::string cl_model_2_path; -}; - -class MockModelInstance : public ovms::ModelInstance { -public: - MockModelInstance(ov::Core& ieCore) : - ModelInstance("UNUSED_NAME", 42, ieCore) {} - const ovms::Status mockValidate(const tensorflow::serving::PredictRequest* request) { - return validate(request); - } -}; - -void TestCustomLoader::performPredict(const std::string modelName, - const ovms::model_version_t modelVersion, - const tensorflow::serving::PredictRequest& request, - std::unique_ptr> waitBeforeGettingModelInstance, - std::unique_ptr> waitBeforePerformInference) { - // only validation is skipped - std::shared_ptr modelInstance; - std::unique_ptr modelInstanceUnloadGuard; - - auto& tensorProto = request.inputs().find("b")->second; - size_t batchSize = tensorProto.tensor_shape().dim(0).size(); - size_t inputSize = 1; - for (int i = 0; i < tensorProto.tensor_shape().dim_size(); i++) { - inputSize *= tensorProto.tensor_shape().dim(i).size(); - } - - if (waitBeforeGettingModelInstance) { - std::cout << "Waiting before getModelInstance. Batch size: " << batchSize << std::endl; - waitBeforeGettingModelInstance->get(); - } - ASSERT_EQ(manager.getModelInstance(modelName, modelVersion, modelInstance, modelInstanceUnloadGuard), ovms::StatusCode::OK); - - if (waitBeforePerformInference) { - std::cout << "Waiting before performInfernce." << std::endl; - waitBeforePerformInference->get(); - } - ovms::Status validationStatus = (std::static_pointer_cast(modelInstance))->mockValidate(&request); - std::cout << validationStatus.string() << std::endl; - ASSERT_TRUE(validationStatus == ovms::StatusCode::OK || - validationStatus == ovms::StatusCode::RESHAPE_REQUIRED || - validationStatus == ovms::StatusCode::BATCHSIZE_CHANGE_REQUIRED); - auto bsPositionIndex = 0; - auto requestBatchSize = ovms::getRequestBatchSize(&request, bsPositionIndex); - auto requestShapes = ovms::getRequestShapes(&request); - ASSERT_EQ(modelInstance->reloadModelIfRequired(validationStatus, requestBatchSize, requestShapes, modelInstanceUnloadGuard), ovms::StatusCode::OK); - - ovms::ExecutingStreamIdGuard executingStreamIdGuard(modelInstance->getInferRequestsQueue(), modelInstance->getMetricReporter()); - ov::InferRequest& inferRequest = executingStreamIdGuard.getInferRequest(); - std::vector input(inputSize); - std::generate(input.begin(), input.end(), []() { return 1.; }); - ASSERT_THAT(input, Each(Eq(1.))); - deserialize(input, inferRequest, modelInstance); - auto status = modelInstance->performInference(inferRequest); - ASSERT_EQ(status, ovms::StatusCode::OK); - size_t outputSize = batchSize * DUMMY_MODEL_OUTPUT_SIZE; - serializeAndCheck(outputSize, inferRequest); -} - -// Schema Validation - -TEST_F(TestCustomLoader, CustomLoaderConfigMatchingSchema) { - const char* customloaderConfigMatchingSchema = R"( - { - "custom_loader_config_list":[ - { - "config":{ - "loader_name":"dummy-loader", - "library_path": "/tmp/loader/dummyloader", - "loader_config_file": "dummyloader-config" - } - } - ], - "model_config_list":[ - { - "config":{ - "name":"dummy-loader-model", - "base_path": "/tmp/models/dummy1", - "custom_loader_options": {"loader_name": "dummy-loader"} - } - } - ] - } - )"; - - rapidjson::Document customloaderConfigMatchingSchemaParsed; - customloaderConfigMatchingSchemaParsed.Parse(customloaderConfigMatchingSchema); - auto result = ovms::validateJsonAgainstSchema(customloaderConfigMatchingSchemaParsed, ovms::MODELS_CONFIG_SCHEMA.c_str()); - EXPECT_EQ(result, ovms::StatusCode::OK); -} - -TEST_F(TestCustomLoader, CustomLoaderConfigMissingLoaderName) { - const char* customloaderConfigMissingLoaderName = R"( - { - "custom_loader_config_list":[ - { - "config":{ - "library_path": "dummyloader", - "loader_config_file": "dummyloader-config" - } - } - ], - "model_config_list": [] - } - )"; - - rapidjson::Document customloaderConfigMissingLoaderNameParsed; - customloaderConfigMissingLoaderNameParsed.Parse(customloaderConfigMissingLoaderName); - auto result = ovms::validateJsonAgainstSchema(customloaderConfigMissingLoaderNameParsed, ovms::MODELS_CONFIG_SCHEMA.c_str()); - EXPECT_EQ(result, ovms::StatusCode::JSON_INVALID); -} - -TEST_F(TestCustomLoader, CustomLoaderConfigMissingLibraryPath) { - const char* customloaderConfigMissingLibraryPath = R"( - { - "custom_loader_config_list":[ - { - "config":{ - "loader_name":"dummy-loader", - "loader_config_file": "dummyloader-config" - } - } - ], - "model_config_list": [] - } - )"; - - rapidjson::Document customloaderConfigMissingLibraryPathParsed; - customloaderConfigMissingLibraryPathParsed.Parse(customloaderConfigMissingLibraryPath); - auto result = ovms::validateJsonAgainstSchema(customloaderConfigMissingLibraryPathParsed, ovms::MODELS_CONFIG_SCHEMA.c_str()); - EXPECT_EQ(result, ovms::StatusCode::JSON_INVALID); -} - -TEST_F(TestCustomLoader, CustomLoaderConfigMissingLoaderConfig) { - const char* customloaderConfigMissingLoaderConfig = R"( - { - "custom_loader_config_list":[ - { - "config":{ - "loader_name":"dummy-loader", - "library_path": "dummyloader" - } - } - ], - "model_config_list": [] - } - )"; - - rapidjson::Document customloaderConfigMissingLoaderConfigParsed; - customloaderConfigMissingLoaderConfigParsed.Parse(customloaderConfigMissingLoaderConfig); - auto result = ovms::validateJsonAgainstSchema(customloaderConfigMissingLoaderConfigParsed, ovms::MODELS_CONFIG_SCHEMA.c_str()); - EXPECT_EQ(result, ovms::StatusCode::OK); -} - -TEST_F(TestCustomLoader, CustomLoaderConfigInvalidCustomLoaderConfig) { - const char* customloaderConfigInvalidCustomLoaderConfig = R"( - { - "model_config_list":[ - { - "config":{ - "name":"dummy-loader-model", - "base_path": "/tmp/models/dummy1", - "custom_loader_options_invalid": {"loader_name": "dummy-loader"} - } - } - ] - } - )"; - - rapidjson::Document customloaderConfigInvalidCustomLoaderConfigParsed; - customloaderConfigInvalidCustomLoaderConfigParsed.Parse(customloaderConfigInvalidCustomLoaderConfig); - auto result = ovms::validateJsonAgainstSchema(customloaderConfigInvalidCustomLoaderConfigParsed, ovms::MODELS_CONFIG_SCHEMA.c_str()); - EXPECT_EQ(result, ovms::StatusCode::JSON_INVALID); -} - -TEST_F(TestCustomLoader, CustomLoaderConfigMissingLoaderNameInCustomLoaderOptions) { - const char* customloaderConfigMissingLoaderNameInCustomLoaderOptions = R"( - { - "model_config_list":[ - { - "config":{ - "name":"dummy-loader-model", - "base_path": "/tmp/models/dummy1", - "custom_loader_options": {"a": "SS"} - } - } - ] - } - )"; - - rapidjson::Document customloaderConfigMissingLoaderNameInCustomLoaderOptionsParsed; - customloaderConfigMissingLoaderNameInCustomLoaderOptionsParsed.Parse(customloaderConfigMissingLoaderNameInCustomLoaderOptions); - auto result = ovms::validateJsonAgainstSchema(customloaderConfigMissingLoaderNameInCustomLoaderOptionsParsed, ovms::MODELS_CONFIG_SCHEMA.c_str()); - EXPECT_EQ(result, ovms::StatusCode::JSON_INVALID); -} - -TEST_F(TestCustomLoader, CustomLoaderConfigMultiplePropertiesInCustomLoaderOptions) { - const char* customloaderConfigMultiplePropertiesInCustomLoaderOptions = R"( - { - "model_config_list":[ - { - "config":{ - "name":"dummy-loader-model", - "base_path": "/tmp/models/dummy1", - "custom_loader_options": {"loader_name": "dummy-loader", "1": "a", "2": "b", "3": "c", "4":"d", "5":"e", "6":"f"} - } - } - ] - } - )"; - - rapidjson::Document customloaderConfigMultiplePropertiesInCustomLoaderOptionsParsed; - customloaderConfigMultiplePropertiesInCustomLoaderOptionsParsed.Parse(customloaderConfigMultiplePropertiesInCustomLoaderOptions); - auto result = ovms::validateJsonAgainstSchema(customloaderConfigMultiplePropertiesInCustomLoaderOptionsParsed, ovms::MODELS_CONFIG_SCHEMA.c_str()); - EXPECT_EQ(result, ovms::StatusCode::OK); -} - -// Functional Validation -/* ------------------------------------------------- -AFTER SAMPLE CUSTOM LOADER REMOVAL BELOW TESTS ARE NOT VALID -REMOVE THIS ENTIRE FILE ONCE THE FEATURE IS REMOVED -------------------------------------------------- -TEST_F(TestCustomLoader, CustomLoaderPrediction) { -#ifdef _WIN32 - GTEST_SKIP() << "Test disabled on windows"; -#endif - // Copy dummy model to temporary destination - std::filesystem::copy(getGenericFullPathForSrcTest("/ovms/src/test/dummy"), cl_model_1_path, std::filesystem::copy_options::recursive); - - // Replace model path in the config string - std::string configStr = custom_loader_config_model; - configStr.replace(configStr.find("/tmp/test_cl_models"), std::string("/tmp/test_cl_models").size(), cl_models_path); - - // Create config file - std::string fileToReload = cl_models_path + "/cl_config.json"; - createConfigFileWithContent(configStr, fileToReload); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - tensorflow::serving::PredictRequest request; - preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{1, 10}, ovms::Precision::FP32}}}); - performPredict("dummy", 1, request); -} - -TEST_F(TestCustomLoader, CustomLoaderPredictionRelativePath) { -#ifdef _WIN32 - GTEST_SKIP() << "Test disabled on windows"; -#endif - // Copy dummy model to temporary destination - std::filesystem::copy(getGenericFullPathForSrcTest("/ovms/src/test/dummy"), cl_model_1_path, std::filesystem::copy_options::recursive); - std::filesystem::copy(getGenericFullPathForSrcTest("/ovms/bazel-bin/src/libsampleloader.so"), cl_models_path, std::filesystem::copy_options::recursive); - - // Replace model path in the config string - std::string configStr = custom_loader_config_model_relative_paths; - configStr.replace(configStr.find("test_cl_models"), std::string("test_cl_models").size(), cl_models_path); - - // Create config file - std::string fileToReload = cl_models_path + "/cl_config.json"; - createConfigFileWithContent(configStr, fileToReload); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - tensorflow::serving::PredictRequest request; - preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{1, 10}, ovms::Precision::FP32}}}); - performPredict("dummy", 1, request); -} - -TEST_F(TestCustomLoader, CustomLoaderGetStatus) { -#ifdef _WIN32 - GTEST_SKIP() << "Test disabled on windows"; -#endif - // Copy dummy model to temporary destination - std::filesystem::copy(getGenericFullPathForSrcTest("/ovms/src/test/dummy"), cl_model_1_path, std::filesystem::copy_options::recursive); - - // Replace model path in the config string - std::string configStr = custom_loader_config_model; - configStr.replace(configStr.find("/tmp/test_cl_models"), std::string("/tmp/test_cl_models").size(), cl_models_path); - - // Create config file - std::string fileToReload = cl_models_path + "/cl_config.json"; - createConfigFileWithContent(configStr, fileToReload); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - tensorflow::serving::GetModelStatusRequest req; - tensorflow::serving::GetModelStatusResponse res; - - auto model_spec = req.mutable_model_spec(); - model_spec->Clear(); - model_spec->set_name("dummy"); - model_spec->mutable_version()->set_value(1); - ASSERT_EQ(GetModelStatusImpl::getModelStatus(&req, &res, manager, DEFAULT_TEST_CONTEXT), StatusCode::OK); - - const tensorflow::serving::GetModelStatusResponse response_const = res; - std::string json_output; - Status error_status = GetModelStatusImpl::serializeResponse2Json(&response_const, &json_output); - ASSERT_EQ(error_status, StatusCode::OK); - EXPECT_EQ(json_output, expected_json_available); -} - -TEST_F(TestCustomLoader, CustomLoaderPredictDeletePredict) { -#ifdef _WIN32 - GTEST_SKIP() << "Test disabled on windows"; -#endif - // Copy dummy model to temporary destination - std::filesystem::copy(getGenericFullPathForSrcTest("/ovms/src/test/dummy"), cl_model_1_path, std::filesystem::copy_options::recursive); - - // Replace model path in the config string - std::string configStr = custom_loader_config_model; - configStr.replace(configStr.find("/tmp/test_cl_models"), std::string("/tmp/test_cl_models").size(), cl_models_path); - - // Create config file - std::string fileToReload = cl_models_path + "/cl_config.json"; - createConfigFileWithContent(configStr, fileToReload); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - tensorflow::serving::PredictRequest request; - preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{1, 10}, ovms::Precision::FP32}}}); - tensorflow::serving::PredictResponse response; - ASSERT_EQ(performInferenceWithRequest(request, response), ovms::StatusCode::OK); - - // Re-create config file - createConfigFileWithContent(custom_loader_config_model_deleted, fileToReload); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - ASSERT_EQ(performInferenceWithRequest(request, response), ovms::StatusCode::MODEL_VERSION_MISSING); -} - -TEST_F(TestCustomLoader, CustomLoaderPredictNewVersionPredict) { -#ifdef _WIN32 - GTEST_SKIP() << "Test disabled on windows"; -#endif - // Copy dummy model to temporary destination - std::filesystem::copy(getGenericFullPathForSrcTest("/ovms/src/test/dummy"), cl_model_1_path, std::filesystem::copy_options::recursive); - - // Replace model path in the config string - std::string configStr = custom_loader_config_model; - configStr.replace(configStr.find("/tmp/test_cl_models"), std::string("/tmp/test_cl_models").size(), cl_models_path); - - // Create config file - std::string fileToReload = cl_models_path + "/cl_config.json"; - createConfigFileWithContent(configStr, fileToReload); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - tensorflow::serving::PredictRequest request; - preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{1, 10}, ovms::Precision::FP32}}}); - performPredict("dummy", 1, request); - - // Copy version 1 to version 2 - std::filesystem::create_directories(cl_model_1_path + "2"); - std::filesystem::copy(cl_model_1_path + "1", cl_model_1_path + "2", std::filesystem::copy_options::recursive); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{1, 10}, ovms::Precision::FP32}}}); - performPredict("dummy", 2, request); -} - -TEST_F(TestCustomLoader, CustomLoaderPredictNewModelPredict) { -#ifdef _WIN32 - GTEST_SKIP() << "Test disabled on windows"; -#endif - // Copy dummy model to temporary destination - std::filesystem::copy(getGenericFullPathForSrcTest("/ovms/src/test/dummy"), cl_model_1_path, std::filesystem::copy_options::recursive); - - // Replace model path in the config string - std::string configStr = custom_loader_config_model; - configStr.replace(configStr.find("/tmp/test_cl_models"), std::string("/tmp/test_cl_models").size(), cl_models_path); - - // Create config file - std::string fileToReload = cl_models_path + "/cl_config.json"; - createConfigFileWithContent(configStr, fileToReload); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - tensorflow::serving::PredictRequest request; - preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{1, 10}, ovms::Precision::FP32}}}); - performPredict("dummy", 1, request); - - // Copy model1 to model2 - std::filesystem::copy(getGenericFullPathForSrcTest("/ovms/src/test/dummy"), cl_model_2_path, std::filesystem::copy_options::recursive); - - // Replace model path in the config string - configStr = custom_loader_config_model_new; - configStr.replace(configStr.find("/tmp/test_cl_models"), std::string("/tmp/test_cl_models").size(), cl_models_path); - configStr.replace(configStr.find("/tmp/test_cl_models"), std::string("/tmp/test_cl_models").size(), cl_models_path); - - // Re-create config file - createConfigFileWithContent(configStr, fileToReload); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{1, 10}, ovms::Precision::FP32}}}); - performPredict("dummy", 1, request); - performPredict("dummy-new", 1, request); -} - -TEST_F(TestCustomLoader, CustomLoaderPredictRemoveCustomLoaderOptionsPredict) { -#ifdef _WIN32 - GTEST_SKIP() << "Test disabled on windows"; -#endif - // Copy dummy model to temporary destination - std::filesystem::copy(getGenericFullPathForSrcTest("/ovms/src/test/dummy"), cl_model_1_path, std::filesystem::copy_options::recursive); - - // Replace model path in the config string - std::string configStr = custom_loader_config_model; - configStr.replace(configStr.find("/tmp/test_cl_models"), std::string("/tmp/test_cl_models").size(), cl_models_path); - - // Create config file - std::string fileToReload = cl_models_path + "/cl_config.json"; - createConfigFileWithContent(configStr, fileToReload); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - tensorflow::serving::PredictRequest request; - preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{1, 10}, ovms::Precision::FP32}}}); - performPredict("dummy", 1, request); - - // Replace model path in the config string - configStr = custom_loader_config_model_customloader_options_removed; - configStr.replace(configStr.find("/tmp/test_cl_models"), std::string("/tmp/test_cl_models").size(), cl_models_path); - - // Re-create config file - createConfigFileWithContent(configStr, fileToReload); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - performPredict("dummy", 1, request); -} - -TEST_F(TestCustomLoader, PredictNormalModelAddCustomLoaderOptionsPredict) { -#ifdef _WIN32 - GTEST_SKIP() << "Test disabled on windows"; -#endif - // Copy dummy model to temporary destination - std::filesystem::copy(getGenericFullPathForSrcTest("/ovms/src/test/dummy"), cl_model_1_path, std::filesystem::copy_options::recursive); - - // Replace model path in the config string - std::string configStr = custom_loader_config_model_customloader_options_removed; - configStr.replace(configStr.find("/tmp/test_cl_models"), std::string("/tmp/test_cl_models").size(), cl_models_path); - - // Create config file - std::string fileToReload = cl_models_path + "/cl_config.json"; - createConfigFileWithContent(configStr, fileToReload); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - tensorflow::serving::PredictRequest request; - preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{1, 10}, ovms::Precision::FP32}}}); - performPredict("dummy", 1, request); - - // Replace model path in the config string - configStr = custom_loader_config_model; - configStr.replace(configStr.find("/tmp/test_cl_models"), std::string("/tmp/test_cl_models").size(), cl_models_path); - - // Create config file - fileToReload = cl_models_path + "/cl_config.json"; - createConfigFileWithContent(configStr, fileToReload); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - performPredict("dummy", 1, request); -} - -TEST_F(TestCustomLoader, CustomLoaderOptionWithUnknownLibrary) { -#ifdef _WIN32 - GTEST_SKIP() << "Test disabled on windows"; -#endif - // Copy dummy model to temporary destination - std::filesystem::copy(getGenericFullPathForSrcTest("/ovms/src/test/dummy"), cl_model_1_path, std::filesystem::copy_options::recursive); - - // Replace model path in the config string - std::string configStr = config_model_with_customloader_options_unknown_loadername; - configStr.replace(configStr.find("/tmp/test_cl_models"), std::string("/tmp/test_cl_models").size(), cl_models_path); - - // Create config file - std::string fileToReload = cl_models_path + "/cl_config.json"; - createConfigFileWithContent(configStr, fileToReload); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - tensorflow::serving::PredictRequest request; - preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{1, 10}, ovms::Precision::FP32}}}); - tensorflow::serving::PredictResponse response; - ASSERT_EQ(performInferenceWithRequest(request, response), ovms::StatusCode::MODEL_VERSION_MISSING); -} - -TEST_F(TestCustomLoader, CustomLoaderWithMissingModelFiles) { -#ifdef _WIN32 - GTEST_SKIP() << "Test disabled on windows"; -#endif - // Replace model path in the config string - std::string configStr = custom_loader_config_model; - configStr.replace(configStr.find("/tmp/test_cl_models"), std::string("/tmp/test_cl_models").size(), cl_models_path); - - // Create config file - std::string fileToReload = cl_models_path + "/cl_config.json"; - createConfigFileWithContent(configStr, fileToReload); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - tensorflow::serving::PredictRequest request; - preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{1, 10}, ovms::Precision::FP32}}}); - tensorflow::serving::PredictResponse response; - ASSERT_EQ(performInferenceWithRequest(request, response), ovms::StatusCode::MODEL_VERSION_MISSING); -} - -TEST_F(TestCustomLoader, CustomLoaderGetStatusDeleteModelGetStatus) { -#ifdef _WIN32 - GTEST_SKIP() << "Test disabled on windows"; -#endif - // Copy dummy model to temporary destination - std::filesystem::copy(getGenericFullPathForSrcTest("/ovms/src/test/dummy"), cl_model_1_path, std::filesystem::copy_options::recursive); - - // Replace model path in the config string - std::string configStr = custom_loader_config_model; - configStr.replace(configStr.find("/tmp/test_cl_models"), std::string("/tmp/test_cl_models").size(), cl_models_path); - - // Create config file - std::string fileToReload = cl_models_path + "/cl_config.json"; - createConfigFileWithContent(configStr, fileToReload); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - tensorflow::serving::GetModelStatusRequest req; - tensorflow::serving::GetModelStatusResponse res; - - auto model_spec = req.mutable_model_spec(); - model_spec->Clear(); - model_spec->set_name("dummy"); - model_spec->mutable_version()->set_value(1); - ASSERT_EQ(GetModelStatusImpl::getModelStatus(&req, &res, manager, DEFAULT_TEST_CONTEXT), StatusCode::OK); - - const tensorflow::serving::GetModelStatusResponse response_const = res; - std::string json_output; - Status error_status = GetModelStatusImpl::serializeResponse2Json(&response_const, &json_output); - ASSERT_EQ(error_status, StatusCode::OK); - EXPECT_EQ(json_output, expected_json_available); - - // Re-create config file - createConfigFileWithContent(custom_loader_config_model_deleted, fileToReload); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - tensorflow::serving::GetModelStatusRequest reqx; - tensorflow::serving::GetModelStatusResponse resx; - - auto model_specx = reqx.mutable_model_spec(); - model_specx->Clear(); - model_specx->set_name("dummy"); - model_specx->mutable_version()->set_value(1); - - ASSERT_EQ(GetModelStatusImpl::getModelStatus(&reqx, &resx, manager, DEFAULT_TEST_CONTEXT), StatusCode::OK); - - const tensorflow::serving::GetModelStatusResponse response_constx = resx; - json_output = ""; - error_status = GetModelStatusImpl::serializeResponse2Json(&response_constx, &json_output); - ASSERT_EQ(error_status, StatusCode::OK); - EXPECT_EQ(json_output, expected_json_end); -} - -TEST_F(TestCustomLoader, CustomLoaderPredictionUsingManyCustomLoaders) { -#ifdef _WIN32 - GTEST_SKIP() << "Test disabled on windows"; -#endif - // Copy dummy model to temporary destination - std::filesystem::copy(getGenericFullPathForSrcTest("/ovms/src/test/dummy"), cl_model_1_path, std::filesystem::copy_options::recursive); - - // Replace model path in the config string - std::string configStr = custom_loader_config_model_multiple; - configStr.replace(configStr.find("/tmp/test_cl_models"), std::string("/tmp/test_cl_models").size(), cl_models_path); - configStr.replace(configStr.find("/tmp/test_cl_models"), std::string("/tmp/test_cl_models").size(), cl_models_path); - configStr.replace(configStr.find("/tmp/test_cl_models"), std::string("/tmp/test_cl_models").size(), cl_models_path); - - // Create config file - std::string fileToReload = cl_models_path + "/cl_config.json"; - createConfigFileWithContent(configStr, fileToReload); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - tensorflow::serving::PredictRequest request; - preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{1, 10}, ovms::Precision::FP32}}}); - - performPredict("dummy-a", 1, request); - performPredict("dummy-b", 1, request); - performPredict("dummy-c", 1, request); -} - -TEST_F(TestCustomLoader, CustomLoaderGetMetaData) { -#ifdef _WIN32 - GTEST_SKIP() << "Test disabled on windows"; -#endif - const char* expected_json = R"({ - "modelSpec": { - "name": "dummy", - "signatureName": "", - "version": "1" - }, - "metadata": { - "signature_def": { - "@type": "type.googleapis.com/tensorflow.serving.SignatureDefMap", - "signatureDef": { - "serving_default": { - "inputs": { - "b": { - "dtype": "DT_FLOAT", - "tensorShape": { - "dim": [ - { - "size": "1", - "name": "" - }, - { - "size": "10", - "name": "" - } - ], - "unknownRank": false - }, - "name": "b" - } - }, - "outputs": { - "a": { - "dtype": "DT_FLOAT", - "tensorShape": { - "dim": [ - { - "size": "1", - "name": "" - }, - { - "size": "10", - "name": "" - } - ], - "unknownRank": false - }, - "name": "a" - } - }, - "methodName": "", - "defaults": {} - } - } - } - } -} -)"; - - // Copy dummy model to temporary destination - std::filesystem::copy(getGenericFullPathForSrcTest("/ovms/src/test/dummy"), cl_model_1_path, std::filesystem::copy_options::recursive); - - // Replace model path in the config string - std::string configStr = custom_loader_config_model; - configStr.replace(configStr.find("/tmp/test_cl_models"), std::string("/tmp/test_cl_models").size(), cl_models_path); - - // Create config file - std::string fileToReload = cl_models_path + "/cl_config.json"; - createConfigFileWithContent(configStr, fileToReload); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - std::shared_ptr model; - std::unique_ptr unload_guard; - ASSERT_EQ(manager.getModelInstance("dummy", 1, model, unload_guard), ovms::StatusCode::OK); - - tensorflow::serving::GetModelMetadataResponse response; - ovms::GetModelMetadataImpl::buildResponse(model, &response); - - std::string json_output = ""; - ovms::GetModelMetadataImpl::serializeResponse2Json(&response, &json_output); - - EXPECT_TRUE(response.has_model_spec()); - EXPECT_EQ(response.model_spec().name(), "dummy"); - - tensorflow::serving::SignatureDefMap def; - response.metadata().at("signature_def").UnpackTo(&def); - - const auto& inputs = ((*def.mutable_signature_def())["serving_default"]).inputs(); - const auto& outputs = ((*def.mutable_signature_def())["serving_default"]).outputs(); - - EXPECT_EQ(inputs.size(), 1); - EXPECT_EQ(outputs.size(), 1); - EXPECT_EQ(json_output, expected_json); -} - -TEST_F(TestCustomLoader, CustomLoaderMultipleLoaderWithSameLoaderName) { -#ifdef _WIN32 - GTEST_SKIP() << "Test disabled on windows"; -#endif - const char* custom_loader_config_model_xx = R"({ - "custom_loader_config_list":[ - { - "config":{ - "loader_name":"sample-loader", - "library_path": "/ovms/bazel-bin/src/libsampleloader.so" - } - }, - { - "config":{ - "loader_name":"sample-loader", - "library_path": "/ovms/bazel-bin/src/libsampleloader.so" - } - } - ], - "model_config_list":[ - { - "config":{ - "name":"dummy", - "base_path": "/tmp/test_cl_models/model1", - "nireq": 1, - "custom_loader_options": {"loader_name": "sample-loader", "model_file": "dummy.xml", "bin_file": "dummy.bin"} - } - } - ] - })"; - - // Copy dummy model to temporary destination - std::filesystem::copy(getGenericFullPathForSrcTest("/ovms/src/test/dummy"), cl_model_1_path, std::filesystem::copy_options::recursive); - - // Replace model path in the config string - std::string configStr = custom_loader_config_model_xx; - configStr.replace(configStr.find("/tmp/test_cl_models"), std::string("/tmp/test_cl_models").size(), cl_models_path); - - // Create config file - std::string fileToReload = cl_models_path + "/cl_config.json"; - createConfigFileWithContent(configStr, fileToReload); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - tensorflow::serving::PredictRequest request; - preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{1, 10}, ovms::Precision::FP32}}}); - performPredict("dummy", 1, request); -} - -TEST_F(TestCustomLoader, CustomLoaderBlackListingModel) { -#ifdef _WIN32 - GTEST_SKIP() << "Test disabled on windows"; -#endif - // Copy dummy model to temporary destination - std::filesystem::copy(getGenericFullPathForSrcTest("/ovms/src/test/dummy"), cl_model_1_path, std::filesystem::copy_options::recursive); - - // Create Sample Custom Loader Config - std::string cl_config_file_path = cl_models_path; - std::string cl_config_str = ENABLE_FORCE_BLACKLIST_CHECK; - std::string cl_config_file = cl_config_file_path + "/customloader_config"; - createConfigFileWithContent(cl_config_str, cl_config_file); - - // Replace model path in the config string - std::string configStr = custom_loader_config_model_blacklist; - configStr.replace(configStr.find("/tmp/test_cl_models"), std::string("/tmp/test_cl_models").size(), cl_models_path); - configStr.replace(configStr.find("sample-loader-config"), std::string("sample-loader-config").size(), cl_config_file); - - // Create config file - std::string fileToReload = cl_models_path + "/cl_config.json"; - createConfigFileWithContent(configStr, fileToReload); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - tensorflow::serving::GetModelStatusRequest req; - tensorflow::serving::GetModelStatusResponse res; - - auto model_spec = req.mutable_model_spec(); - model_spec->Clear(); - model_spec->set_name("dummy"); - model_spec->mutable_version()->set_value(1); - ASSERT_EQ(GetModelStatusImpl::getModelStatus(&req, &res, manager, DEFAULT_TEST_CONTEXT), StatusCode::OK); - - tensorflow::serving::GetModelStatusResponse response_const = res; - std::string json_output; - Status error_status = GetModelStatusImpl::serializeResponse2Json(&response_const, &json_output); - ASSERT_EQ(error_status, StatusCode::OK); - EXPECT_EQ(json_output, expected_json_available); - - // copy status file - std::string status_file_path = cl_model_1_path + "1"; - std::string status_str = "DISABLED"; - std::string status_file = status_file_path + "/dummy.status"; - createConfigFileWithContent(status_str, status_file); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - tensorflow::serving::GetModelStatusRequest reqx; - tensorflow::serving::GetModelStatusResponse resx; - - auto model_specx = reqx.mutable_model_spec(); - model_specx->Clear(); - model_specx->set_name("dummy"); - model_specx->mutable_version()->set_value(1); - - ASSERT_EQ(GetModelStatusImpl::getModelStatus(&reqx, &resx, manager, DEFAULT_TEST_CONTEXT), StatusCode::OK); - - const tensorflow::serving::GetModelStatusResponse response_constx = resx; - json_output = ""; - error_status = GetModelStatusImpl::serializeResponse2Json(&response_constx, &json_output); - ASSERT_EQ(error_status, StatusCode::OK); - EXPECT_EQ(json_output, expected_json_end); -} - -TEST_F(TestCustomLoader, CustomLoaderBlackListingRevoke) { -#ifdef _WIN32 - GTEST_SKIP() << "Test disabled on windows"; -#endif - // Copy dummy model to temporary destination - std::filesystem::copy(getGenericFullPathForSrcTest("/ovms/src/test/dummy"), cl_model_1_path, std::filesystem::copy_options::recursive); - - // Create Sample Custom Loader Config - std::string cl_config_file_path = cl_models_path; - std::string cl_config_str = ENABLE_FORCE_BLACKLIST_CHECK; - std::string cl_config_file = cl_config_file_path + "/customloader_config"; - createConfigFileWithContent(cl_config_str, cl_config_file); - - // Replace model path in the config string - std::string configStr = custom_loader_config_model_blacklist; - configStr.replace(configStr.find("/tmp/test_cl_models"), std::string("/tmp/test_cl_models").size(), cl_models_path); - configStr.replace(configStr.find("sample-loader-config"), std::string("sample-loader-config").size(), cl_config_file); - - // Create config file - std::string fileToReload = cl_models_path + "/cl_config.json"; - createConfigFileWithContent(configStr, fileToReload); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - tensorflow::serving::GetModelStatusRequest req; - tensorflow::serving::GetModelStatusResponse res; - - auto model_spec = req.mutable_model_spec(); - model_spec->Clear(); - model_spec->set_name("dummy"); - model_spec->mutable_version()->set_value(1); - ASSERT_EQ(GetModelStatusImpl::getModelStatus(&req, &res, manager, DEFAULT_TEST_CONTEXT), StatusCode::OK); - - const tensorflow::serving::GetModelStatusResponse response_const = res; - std::string json_output; - Status error_status = GetModelStatusImpl::serializeResponse2Json(&response_const, &json_output); - ASSERT_EQ(error_status, StatusCode::OK); - EXPECT_EQ(json_output, expected_json_available); - - // copy status file - std::string status_file_path = cl_model_1_path + "1"; - std::string status_str = "DISABLED"; - std::string status_file = status_file_path + "/dummy.status"; - createConfigFileWithContent(status_str, status_file); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - tensorflow::serving::GetModelStatusRequest req1; - tensorflow::serving::GetModelStatusResponse res1; - - auto model_spec1 = req1.mutable_model_spec(); - model_spec1->Clear(); - model_spec1->set_name("dummy"); - model_spec1->mutable_version()->set_value(1); - ASSERT_EQ(GetModelStatusImpl::getModelStatus(&req1, &res1, manager, DEFAULT_TEST_CONTEXT), StatusCode::OK); - - const tensorflow::serving::GetModelStatusResponse response_const1 = res1; - json_output = ""; - error_status = GetModelStatusImpl::serializeResponse2Json(&response_const1, &json_output); - ASSERT_EQ(error_status, StatusCode::OK); - EXPECT_EQ(json_output, expected_json_end); - - // Remove status file - std::filesystem::remove(status_file); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - tensorflow::serving::GetModelStatusRequest req2; - tensorflow::serving::GetModelStatusResponse res2; - - auto model_spec2 = req2.mutable_model_spec(); - model_spec2->Clear(); - model_spec2->set_name("dummy"); - model_spec2->mutable_version()->set_value(1); - ASSERT_EQ(GetModelStatusImpl::getModelStatus(&req2, &res2, manager, DEFAULT_TEST_CONTEXT), StatusCode::OK); - - const tensorflow::serving::GetModelStatusResponse response_const2 = res2; - json_output = ""; - error_status = GetModelStatusImpl::serializeResponse2Json(&response_const2, &json_output); - ASSERT_EQ(error_status, StatusCode::OK); - EXPECT_EQ(json_output, expected_json_available); -} - -TEST_F(TestCustomLoader, CustomLoaderBlackListModelReloadError) { -#ifdef _WIN32 - GTEST_SKIP() << "Test disabled on windows"; -#endif - // Copy dummy model to temporary destination - std::filesystem::copy(getGenericFullPathForSrcTest("/ovms/src/test/dummy"), cl_model_1_path, std::filesystem::copy_options::recursive); - - // Create Sample Custom Loader Config - std::string cl_config_file_path = cl_models_path; - std::string cl_config_str = ENABLE_FORCE_BLACKLIST_CHECK; - std::string cl_config_file = cl_config_file_path + "/customloader_config"; - createConfigFileWithContent(cl_config_str, cl_config_file); - - // Replace model path in the config string - std::string configStr = custom_loader_config_model_blacklist; - configStr.replace(configStr.find("/tmp/test_cl_models"), std::string("/tmp/test_cl_models").size(), cl_models_path); - configStr.replace(configStr.find("sample-loader-config"), std::string("sample-loader-config").size(), cl_config_file); - - // Create config file - std::string fileToReload = cl_models_path + "/cl_config.json"; - createConfigFileWithContent(configStr, fileToReload); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - tensorflow::serving::GetModelStatusRequest req; - tensorflow::serving::GetModelStatusResponse res; - - auto model_spec = req.mutable_model_spec(); - model_spec->Clear(); - model_spec->set_name("dummy"); - model_spec->mutable_version()->set_value(1); - ASSERT_EQ(GetModelStatusImpl::getModelStatus(&req, &res, manager, DEFAULT_TEST_CONTEXT), StatusCode::OK); - - const tensorflow::serving::GetModelStatusResponse response_const = res; - std::string json_output; - Status error_status = GetModelStatusImpl::serializeResponse2Json(&response_const, &json_output); - ASSERT_EQ(error_status, StatusCode::OK); - EXPECT_EQ(json_output, expected_json_available); - - // copy status file - std::string status_file_path = cl_model_1_path + "1"; - std::string status_str = "DISABLED"; - std::string status_file = status_file_path + "/dummy.status"; - createConfigFileWithContent(status_str, status_file); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - tensorflow::serving::GetModelStatusRequest req1; - tensorflow::serving::GetModelStatusResponse res1; - - auto model_spec1 = req1.mutable_model_spec(); - model_spec1->Clear(); - model_spec1->set_name("dummy"); - model_spec1->mutable_version()->set_value(1); - ASSERT_EQ(GetModelStatusImpl::getModelStatus(&req1, &res1, manager, DEFAULT_TEST_CONTEXT), StatusCode::OK); - - const tensorflow::serving::GetModelStatusResponse response_const1 = res1; - json_output = ""; - error_status = GetModelStatusImpl::serializeResponse2Json(&response_const1, &json_output); - ASSERT_EQ(error_status, StatusCode::OK); - EXPECT_EQ(json_output, expected_json_end); - - // Remove status file & the Dummy.bin file - std::filesystem::remove(status_file); - std::string bin_file = status_file_path + "/dummy.bin"; - std::filesystem::remove(bin_file); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::FILE_INVALID); - - tensorflow::serving::GetModelStatusRequest req2; - tensorflow::serving::GetModelStatusResponse res2; - - auto model_spec2 = req2.mutable_model_spec(); - model_spec2->Clear(); - model_spec2->set_name("dummy"); - model_spec2->mutable_version()->set_value(1); - ASSERT_EQ(GetModelStatusImpl::getModelStatus(&req2, &res2, manager, DEFAULT_TEST_CONTEXT), StatusCode::OK); - - const tensorflow::serving::GetModelStatusResponse response_const2 = res2; - json_output = ""; - error_status = GetModelStatusImpl::serializeResponse2Json(&response_const2, &json_output); - ASSERT_EQ(error_status, StatusCode::OK); - EXPECT_EQ(json_output, expected_json_loading_error); - - // Copy back the model files & try reload - std::filesystem::copy(getGenericFullPathForSrcTest("/ovms/src/test/dummy"), cl_model_1_path, std::filesystem::copy_options::recursive | std::filesystem::copy_options::overwrite_existing); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - tensorflow::serving::GetModelStatusRequest req3; - tensorflow::serving::GetModelStatusResponse res3; - - auto model_spec3 = req3.mutable_model_spec(); - model_spec3->Clear(); - model_spec3->set_name("dummy"); - model_spec3->mutable_version()->set_value(1); - ASSERT_EQ(GetModelStatusImpl::getModelStatus(&req3, &res3, manager, DEFAULT_TEST_CONTEXT), StatusCode::OK); - - const tensorflow::serving::GetModelStatusResponse response_const3 = res3; - json_output = ""; - error_status = GetModelStatusImpl::serializeResponse2Json(&response_const3, &json_output); - ASSERT_EQ(error_status, StatusCode::OK); - EXPECT_EQ(json_output, expected_json_available); -} - -TEST_F(TestCustomLoader, CustomLoaderLoadBlackListedModel) { -#ifdef _WIN32 - GTEST_SKIP() << "Test disabled on windows"; -#endif - // Copy dummy model to temporary destination - std::filesystem::copy(getGenericFullPathForSrcTest("/ovms/src/test/dummy"), cl_model_1_path, std::filesystem::copy_options::recursive); - - // Create Sample Custom Loader Config - std::string cl_config_file_path = cl_models_path; - std::string cl_config_str = ENABLE_FORCE_BLACKLIST_CHECK; - std::string cl_config_file = cl_config_file_path + "/customloader_config"; - createConfigFileWithContent(cl_config_str, cl_config_file); - - // Replace model path in the config string - std::string configStr = custom_loader_config_model_blacklist; - configStr.replace(configStr.find("/tmp/test_cl_models"), std::string("/tmp/test_cl_models").size(), cl_models_path); - configStr.replace(configStr.find("sample-loader-config"), std::string("sample-loader-config").size(), cl_config_file); - - // Create config file - std::string fileToReload = cl_models_path + "/cl_config.json"; - createConfigFileWithContent(configStr, fileToReload); - - // Create status file - std::string status_file_path = cl_model_1_path + "1"; - std::string status_str = "DISABLED"; - std::string status_file = status_file_path + "/dummy.status"; - createConfigFileWithContent(status_str, status_file); - ovms::Status status1 = manager.loadConfig(fileToReload); - ASSERT_TRUE(status1 == ovms::StatusCode::INTERNAL_ERROR); - - tensorflow::serving::GetModelStatusRequest req1; - tensorflow::serving::GetModelStatusResponse res1; - - auto model_spec1 = req1.mutable_model_spec(); - model_spec1->Clear(); - model_spec1->set_name("dummy"); - model_spec1->mutable_version()->set_value(1); - ASSERT_EQ(GetModelStatusImpl::getModelStatus(&req1, &res1, manager, DEFAULT_TEST_CONTEXT), StatusCode::OK); - - const tensorflow::serving::GetModelStatusResponse response_const1 = res1; - std::string json_output1; - Status error_status1 = GetModelStatusImpl::serializeResponse2Json(&response_const1, &json_output1); - ASSERT_EQ(error_status1, StatusCode::OK); - EXPECT_EQ(json_output1, expected_json_loading_error); - - // remove enable_file from config file - std::string status_config = ", \"enable_file\": \"dummy.status\""; - configStr.replace(configStr.find(status_config), std::string(status_config).size(), ""); - createConfigFileWithContent(configStr, fileToReload); - - ovms::Status status2 = manager.loadConfig(fileToReload); - ASSERT_TRUE(status2 == ovms::StatusCode::OK); - - tensorflow::serving::GetModelStatusRequest req2; - tensorflow::serving::GetModelStatusResponse res2; - - auto model_spec2 = req2.mutable_model_spec(); - model_spec2->Clear(); - model_spec2->set_name("dummy"); - model_spec2->mutable_version()->set_value(1); - ASSERT_EQ(GetModelStatusImpl::getModelStatus(&req2, &res2, manager, DEFAULT_TEST_CONTEXT), StatusCode::OK); - - const tensorflow::serving::GetModelStatusResponse response_const2 = res2; - std::string json_output2; - Status error_status2 = GetModelStatusImpl::serializeResponse2Json(&response_const2, &json_output2); - ASSERT_EQ(error_status2, StatusCode::OK); - EXPECT_EQ(json_output2, expected_json_available); -} -*/ - -#pragma GCC diagnostic pop +//***************************************************************************** +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include + +#include "../schema.hpp" + +using namespace ovms; + +class TestCustomLoader : public ::testing::Test {}; + +TEST_F(TestCustomLoader, CustomLoaderConfigMatchingSchema) { + const char* customloaderConfigMatchingSchema = R"( + { + "custom_loader_config_list":[ + { + "config":{ + "loader_name":"dummy-loader", + "library_path": "/tmp/loader/dummyloader", + "loader_config_file": "dummyloader-config" + } + } + ], + "model_config_list":[ + { + "config":{ + "name":"dummy-loader-model", + "base_path": "/tmp/models/dummy1", + "custom_loader_options": {"loader_name": "dummy-loader"} + } + } + ] + } + )"; + + rapidjson::Document customloaderConfigMatchingSchemaParsed; + customloaderConfigMatchingSchemaParsed.Parse(customloaderConfigMatchingSchema); + auto result = ovms::validateJsonAgainstSchema(customloaderConfigMatchingSchemaParsed, ovms::MODELS_CONFIG_SCHEMA.c_str()); + EXPECT_EQ(result, ovms::StatusCode::OK); +} + +TEST_F(TestCustomLoader, CustomLoaderConfigMissingLoaderName) { + const char* customloaderConfigMissingLoaderName = R"( + { + "custom_loader_config_list":[ + { + "config":{ + "library_path": "dummyloader", + "loader_config_file": "dummyloader-config" + } + } + ], + "model_config_list": [] + } + )"; + + rapidjson::Document customloaderConfigMissingLoaderNameParsed; + customloaderConfigMissingLoaderNameParsed.Parse(customloaderConfigMissingLoaderName); + auto result = ovms::validateJsonAgainstSchema(customloaderConfigMissingLoaderNameParsed, ovms::MODELS_CONFIG_SCHEMA.c_str()); + EXPECT_EQ(result, ovms::StatusCode::JSON_INVALID); +} + +TEST_F(TestCustomLoader, CustomLoaderConfigMissingLibraryPath) { + const char* customloaderConfigMissingLibraryPath = R"( + { + "custom_loader_config_list":[ + { + "config":{ + "loader_name":"dummy-loader", + "loader_config_file": "dummyloader-config" + } + } + ], + "model_config_list": [] + } + )"; + + rapidjson::Document customloaderConfigMissingLibraryPathParsed; + customloaderConfigMissingLibraryPathParsed.Parse(customloaderConfigMissingLibraryPath); + auto result = ovms::validateJsonAgainstSchema(customloaderConfigMissingLibraryPathParsed, ovms::MODELS_CONFIG_SCHEMA.c_str()); + EXPECT_EQ(result, ovms::StatusCode::JSON_INVALID); +} + +TEST_F(TestCustomLoader, CustomLoaderConfigMissingLoaderConfig) { + const char* customloaderConfigMissingLoaderConfig = R"( + { + "custom_loader_config_list":[ + { + "config":{ + "loader_name":"dummy-loader", + "library_path": "dummyloader" + } + } + ], + "model_config_list": [] + } + )"; + + rapidjson::Document customloaderConfigMissingLoaderConfigParsed; + customloaderConfigMissingLoaderConfigParsed.Parse(customloaderConfigMissingLoaderConfig); + auto result = ovms::validateJsonAgainstSchema(customloaderConfigMissingLoaderConfigParsed, ovms::MODELS_CONFIG_SCHEMA.c_str()); + EXPECT_EQ(result, ovms::StatusCode::OK); +} + +TEST_F(TestCustomLoader, CustomLoaderConfigInvalidCustomLoaderConfig) { + const char* customloaderConfigInvalidCustomLoaderConfig = R"( + { + "model_config_list":[ + { + "config":{ + "name":"dummy-loader-model", + "base_path": "/tmp/models/dummy1", + "custom_loader_options_invalid": {"loader_name": "dummy-loader"} + } + } + ] + } + )"; + + rapidjson::Document customloaderConfigInvalidCustomLoaderConfigParsed; + customloaderConfigInvalidCustomLoaderConfigParsed.Parse(customloaderConfigInvalidCustomLoaderConfig); + auto result = ovms::validateJsonAgainstSchema(customloaderConfigInvalidCustomLoaderConfigParsed, ovms::MODELS_CONFIG_SCHEMA.c_str()); + EXPECT_EQ(result, ovms::StatusCode::JSON_INVALID); +} + +TEST_F(TestCustomLoader, CustomLoaderConfigMissingLoaderNameInCustomLoaderOptions) { + const char* customloaderConfigMissingLoaderNameInCustomLoaderOptions = R"( + { + "model_config_list":[ + { + "config":{ + "name":"dummy-loader-model", + "base_path": "/tmp/models/dummy1", + "custom_loader_options": {"a": "SS"} + } + } + ] + } + )"; + + rapidjson::Document customloaderConfigMissingLoaderNameInCustomLoaderOptionsParsed; + customloaderConfigMissingLoaderNameInCustomLoaderOptionsParsed.Parse(customloaderConfigMissingLoaderNameInCustomLoaderOptions); + auto result = ovms::validateJsonAgainstSchema(customloaderConfigMissingLoaderNameInCustomLoaderOptionsParsed, ovms::MODELS_CONFIG_SCHEMA.c_str()); + EXPECT_EQ(result, ovms::StatusCode::JSON_INVALID); +} + +TEST_F(TestCustomLoader, CustomLoaderConfigMultiplePropertiesInCustomLoaderOptions) { + const char* customloaderConfigMultiplePropertiesInCustomLoaderOptions = R"( + { + "model_config_list":[ + { + "config":{ + "name":"dummy-loader-model", + "base_path": "/tmp/models/dummy1", + "custom_loader_options": {"loader_name": "dummy-loader", "1": "a", "2": "b", "3": "c", "4":"d", "5":"e", "6":"f"} + } + } + ] + } + )"; + + rapidjson::Document customloaderConfigMultiplePropertiesInCustomLoaderOptionsParsed; + customloaderConfigMultiplePropertiesInCustomLoaderOptionsParsed.Parse(customloaderConfigMultiplePropertiesInCustomLoaderOptions); + auto result = ovms::validateJsonAgainstSchema(customloaderConfigMultiplePropertiesInCustomLoaderOptionsParsed, ovms::MODELS_CONFIG_SCHEMA.c_str()); + EXPECT_EQ(result, ovms::StatusCode::OK); +} diff --git a/src/test/deserialization_tests.cpp b/src/test/deserialization_tests.cpp index 1983aa872d..f59c2a4e91 100644 --- a/src/test/deserialization_tests.cpp +++ b/src/test/deserialization_tests.cpp @@ -25,27 +25,15 @@ #include -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop - #include "../capi_frontend/buffer.hpp" #include "../capi_frontend/capi_utils.hpp" -#include "../tfs_frontend/tfs_utils.hpp" #include "../kfs_frontend/kfs_utils.hpp" #include "../capi_frontend/deserialization.hpp" -#include "../tfs_frontend/deserialization.hpp" #include "../kfs_frontend/deserialization.hpp" #include "../deserialization_main.hpp" #include "../regularovtensorfactory.hpp" #include "test_utils.hpp" -using TFTensorProto = tensorflow::TensorProto; - -using TFPredictRequest = tensorflow::serving::PredictRequest; -using TFPredictResponse = tensorflow::serving::PredictResponse; - using namespace ovms; using testing::_; @@ -66,34 +54,6 @@ std::vector> cartesianProduct(const std::vector return result; } -class TensorflowGRPCPredict : public ::testing::TestWithParam { -protected: - void SetUp() override { - auto precision = ovms::Precision::FP32; - tensorMap[tensorName] = std::make_shared( - tensorName, - precision, - shape_t{1, DUMMY_MODEL_INPUT_SIZE}, - Layout{"NC"}); - SetUpTensorProto(getPrecisionAsDataType(precision)); - } - void SetUpTensorProto(tensorflow::DataType dataType) { - tensorProto.set_dtype(dataType); - auto tensorShape = tensorProto.mutable_tensor_shape(); - tensorShape->Clear(); - tensorShape->add_dim()->set_size(1); - tensorShape->add_dim()->set_size(DUMMY_MODEL_INPUT_SIZE); - *(tensorProto.mutable_tensor_content()) = std::string(1 * sizeof(float) * DUMMY_MODEL_INPUT_SIZE, '1'); - } - TFTensorProto tensorProto; - const char* tensorName = DUMMY_MODEL_INPUT_NAME; - ovms::tensor_map_t tensorMap; - bool isPipeline = false; -}; - -class DeserializeTFTensorProto : public TensorflowGRPCPredict {}; -class DeserializeTFTensorProtoNegative : public TensorflowGRPCPredict {}; - class CAPIPredict : public ::testing::TestWithParam { protected: void SetUp() override { @@ -173,72 +133,6 @@ TEST_F(CAPIPredictRequest, ShouldSuccessForSupportedPrecision) { class DeserializeCAPITensor : public CAPIPredict {}; class DeserializeCAPITensorProtoNegative : public CAPIPredict {}; -GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(GRPCPredictRequest); - -class GRPCPredictRequest : public TensorflowGRPCPredict { -public: - void SetUp() { - TensorflowGRPCPredict::SetUp(); - (*request.mutable_inputs())[tensorName] = tensorProto; - } - void TearDown() { - request.mutable_inputs()->clear(); - } - -public: - TFPredictRequest request; -}; - -class GRPCPredictRequestNegative : public GRPCPredictRequest {}; - -TEST_P(GRPCPredictRequestNegative, ShouldReturnDeserializationErrorForPrecision) { - ovms::Precision testedPrecision = GetParam(); - tensorMap[tensorName] = createTensorInfoCopyWithPrecision(tensorMap[tensorName], testedPrecision); - ov::InferRequest inferRequest; - InputSink inputSink(inferRequest); - std::unordered_map> factories; - factories.emplace(OVMS_BUFFERTYPE_CPU, std::make_shared()); - auto status = deserializePredictRequest(request, tensorMap, {}, inputSink, isPipeline, factories); - EXPECT_EQ(status, ovms::StatusCode::OV_UNSUPPORTED_DESERIALIZATION_PRECISION) - << "Unsupported OVMS precision:" - << toString(testedPrecision) - << " should return error"; -} - -TEST_P(GRPCPredictRequestNegative, ShouldReturnDeserializationErrorForSetTensorException) { - ovms::Precision testedPrecision = GetParam(); - tensorMap[tensorName] = createTensorInfoCopyWithPrecision(tensorMap[tensorName], testedPrecision); - ov::InferRequest inferRequest; - InputSink inputSink(inferRequest); - std::unordered_map> factories; - factories.emplace(OVMS_BUFFERTYPE_CPU, std::make_shared()); - auto status = deserializePredictRequest(request, tensorMap, {}, inputSink, isPipeline, factories); - EXPECT_EQ(status, ovms::StatusCode::OV_UNSUPPORTED_DESERIALIZATION_PRECISION) << status.string(); -} - -TEST_F(GRPCPredictRequest, ShouldSuccessForSupportedPrecision) { - ov::Core ieCore; - std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().u8string() + "/src/test/dummy/1/dummy.xml"); - ov::CompiledModel compiledModel = ieCore.compile_model(model, "CPU"); - ov::InferRequest inferRequest = compiledModel.create_infer_request(); - InputSink inputSink(inferRequest); - std::unordered_map> factories; - factories.emplace(OVMS_BUFFERTYPE_CPU, std::make_shared()); - auto status = deserializePredictRequest(request, tensorMap, {}, inputSink, isPipeline, factories); - EXPECT_TRUE(status.ok()); -} - -TEST_P(DeserializeTFTensorProtoNegative, ShouldReturnNullptrForPrecision) { - ovms::Precision testedPrecision = GetParam(); - tensorMap[tensorName] = createTensorInfoCopyWithPrecision(tensorMap[tensorName], testedPrecision); - std::unordered_map> factories; - factories.emplace(OVMS_BUFFERTYPE_CPU, std::make_shared()); - ov::Tensor tensor = ConcreteTensorProtoDeserializator::deserializeTensor(tensorProto, tensorMap[tensorName], factories, nullptr); - EXPECT_FALSE((bool)tensor) << "Unsupported OVMS precision:" - << toString(testedPrecision) - << " should return nullptr"; -} - TEST_P(DeserializeCAPITensorProtoNegative, ShouldReturnNullptrForPrecision) { ovms::Precision testedPrecision = GetParam(); tensorMap[tensorName] = createTensorInfoCopyWithPrecision(tensorMap[tensorName], testedPrecision); @@ -250,18 +144,6 @@ TEST_P(DeserializeCAPITensorProtoNegative, ShouldReturnNullptrForPrecision) { << " should return nullptr"; } -TEST_P(DeserializeTFTensorProto, ShouldReturnValidTensor) { - ovms::Precision testedPrecision = GetParam(); - SetUpTensorProto(getPrecisionAsDataType(testedPrecision)); - tensorMap[tensorName] = createTensorInfoCopyWithPrecision(tensorMap[tensorName], testedPrecision); - std::unordered_map> factories; - factories.emplace(OVMS_BUFFERTYPE_CPU, std::make_shared()); - ov::Tensor tensor = ConcreteTensorProtoDeserializator::deserializeTensor(tensorProto, tensorMap[tensorName], factories, nullptr); - EXPECT_TRUE((bool)tensor) << "Supported OVMS precision:" - << toString(testedPrecision) - << " should return valid tensor ptr"; - EXPECT_EQ(ovElementTypeToOvmsPrecision(tensor.get_element_type()), testedPrecision); -} TEST_P(DeserializeCAPITensor, ShouldReturnValidTensor) { ovms::Precision testedPrecision = GetParam(); SetUpTensorProto(getPrecisionAsOVMSDataType(testedPrecision)); @@ -516,29 +398,6 @@ INSTANTIATE_TEST_SUITE_P( [](const ::testing::TestParamInfo& info) { return toString(info.param); }); -INSTANTIATE_TEST_SUITE_P( - TestDeserialize, - GRPCPredictRequestNegative, - ::testing::ValuesIn(UNSUPPORTED_INPUT_PRECISIONS), - [](const ::testing::TestParamInfo& info) { - return toString(info.param); - }); -INSTANTIATE_TEST_SUITE_P( - TestDeserialize, - GRPCPredictRequest, - ::testing::ValuesIn(SUPPORTED_INPUT_PRECISIONS), - [](const ::testing::TestParamInfo& info) { - return toString(info.param); - }); - -INSTANTIATE_TEST_SUITE_P( - Test, - DeserializeTFTensorProtoNegative, - ::testing::ValuesIn(UNSUPPORTED_INPUT_PRECISIONS), - [](const ::testing::TestParamInfo& info) { - return toString(info.param); - }); - INSTANTIATE_TEST_SUITE_P( Test, DeserializeCAPITensorProtoNegative, @@ -547,13 +406,6 @@ INSTANTIATE_TEST_SUITE_P( return toString(info.param); }); -INSTANTIATE_TEST_SUITE_P( - Test, - DeserializeTFTensorProto, - ::testing::ValuesIn(SUPPORTED_INPUT_PRECISIONS), - [](const ::testing::TestParamInfo& info) { - return toString(info.param); - }); INSTANTIATE_TEST_SUITE_P( Test, DeserializeCAPITensor, diff --git a/src/test/ensemble_config_change_stress.cpp b/src/test/ensemble_config_change_stress.cpp index ff86bcc3b5..2a7e2e5123 100644 --- a/src/test/ensemble_config_change_stress.cpp +++ b/src/test/ensemble_config_change_stress.cpp @@ -22,111 +22,18 @@ #include "../dags/pipeline.hpp" #include "../dags/pipeline_factory.hpp" #include "../dags/pipelinedefinition.hpp" -#include "../get_model_metadata_impl.hpp" -#if (MEDIAPIPE_DISABLE == 0) #include "../kfs_frontend/kfs_graph_executor_impl.hpp" -#endif #include "../kfs_frontend/kfs_utils.hpp" #include "src/filesystem/localfilesystem.hpp" #include "../logging.hpp" -#include "../model_service.hpp" -#include "../modelconfig.hpp" -#include "../modelinstance.hpp" -#include "../prediction_service_utils.hpp" #include "../servablemanagermodule.hpp" #include "../server.hpp" #include "../status.hpp" #include "../stringutils.hpp" -#include "../tfs_frontend/tfs_utils.hpp" -#include "c_api_test_utils.hpp" #include "stress_test_utils.hpp" #include "test_models.hpp" #include "test_utils.hpp" -static const char* stressPipelineCustomNodeDifferentOperationsThenDummyThenChooseMaximumConfig = R"( -{ - "custom_node_library_config_list": [ - { - "name": "lib_perform_different_operations", - "base_path": "/ovms/bazel-bin/src/lib_node_perform_different_operations.so" - }, - { - "name": "lib_choose_maximum", - "base_path": "/ovms/bazel-bin/src/lib_node_choose_maximum.so" - } - ], - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 100 - } - } - ], - "pipeline_config_list": [ - { - "name": "pipeline1Dummy", - "inputs": ["custom_dummy_input", "pipeline_factors"], - "nodes": [ - { - "name": "custom_node", - "library_name": "lib_perform_different_operations", - "type": "custom", - "demultiply_count": 4, - "inputs": [ - {"input_numbers": {"node_name": "request", - "data_item": "custom_dummy_input"}}, - {"op_factors": {"node_name": "request", - "data_item": "pipeline_factors"}} - ], - "outputs": [ - {"data_item": "different_ops_results", - "alias": "custom_node_output"} - ] - }, - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "custom_node", - "data_item": "custom_node_output"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "dummy_output"} - ] - }, - { - "name": "choose_max", - "library_name": "lib_choose_maximum", - "type": "custom", - "gather_from_node": "custom_node", - "params": { - "selection_criteria": "MAXIMUM_MINIMUM" - }, - "inputs": [ - {"input_tensors": {"node_name": "dummyNode", - "data_item": "dummy_output"}} - ], - "outputs": [ - {"data_item": "maximum_tensor", - "alias": "maximum_tensor_alias"} - ] - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "choose_max", - "data_item": "maximum_tensor_alias"} - } - ] - } - ] -})"; - #if (MEDIAPIPE_DISABLE == 0) template <> void mediaexec(std::unique_ptr& executorPtr, ovms::ModelManager& manager, KFSRequest& request, KFSResponse& response, ovms::Status& status) { @@ -134,7 +41,7 @@ void mediaexec(std::unique_ptr& &response, ovms::ExecutionContext( ovms::ExecutionContext::Interface::GRPC, - ovms::ExecutionContext::Method::Predict)); + ovms::ExecutionContext::Method::ModelInfer)); } template <> @@ -152,462 +59,6 @@ class StressPipelineConfigChanges : public ConfigChangeStressTest { } }; -class StressModelConfigChanges : public StressPipelineConfigChanges { - const std::string modelName = "dummy"; - const std::string modelInputName = "b"; - const std::string modelOutputName = "a"; - -public: - std::string getServableName() override { - return modelName; - } - void SetUp() override { - SetUpCAPIServerInstance(initialClearConfig); - } -}; - -TEST_F(StressPipelineConfigChanges, AddNewVersionDuringPredictLoad) { - bool performWholeConfigReload = false; // we just need to have all model versions rechecked - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {}; - performStressTest( - &ConfigChangeStressTest::triggerPredictInALoop, - &ConfigChangeStressTest::defaultVersionAdd, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} -TEST_F(StressPipelineConfigChanges, KFSAddNewVersionDuringPredictLoad) { - bool performWholeConfigReload = false; // we just need to have all model versions rechecked - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {}; - performStressTest( - // XYZ &ConfigChangeStressTest::triggerPredictInALoop, - &ConfigChangeStressTest::triggerPredictInALoop, - &ConfigChangeStressTest::defaultVersionAdd, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} -// Disabled because we cannot start http server multiple times https://github.com/drogonframework/drogon/issues/2210 - -TEST_F(ConfigChangeStressTest, DISABLED_GetMetricsDuringLoad) { - bool performWholeConfigReload = false; // we just need to have all model versions rechecked - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {}; - performStressTest( - &ConfigChangeStressTest::triggerPredictInALoop, - &ConfigChangeStressTest::testCurrentRequestsMetric, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} -TEST_F(StressPipelineConfigChanges, RemoveDefaultVersionDuringPredictLoad) { - std::set requiredLoadResults = {StatusCode::OK, - StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET, // we hit when all config changes finish to propagate - StatusCode::MODEL_VERSION_MISSING}; // there is no default version since all are either not loaded properly or retired - std::set allowedLoadResults = { - StatusCode::MODEL_VERSION_NOT_LOADED_ANYMORE, // we hit default version which is unloaded already but default is not changed yet - }; - // we need whole config reload since there is no other way to dispose - // all model versions different than removing model from config - bool performWholeConfigReload = true; - performStressTest( - &ConfigChangeStressTest::triggerPredictInALoop, - &ConfigChangeStressTest::defaultVersionRemove, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} -TEST_F(StressPipelineConfigChanges, ChangeToShapeAutoDuringPredictLoad) { - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; - performStressTest( - &ConfigChangeStressTest::triggerPredictInALoop, - &ConfigChangeStressTest::changeToAutoShape, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} -TEST_F(StressPipelineConfigChanges, RemovePipelineDefinitionDuringPredictLoad) { - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK, - StatusCode::PIPELINE_DEFINITION_NOT_LOADED_ANYMORE}; // we expect to stop creating pipelines - std::set allowedLoadResults = {}; - performStressTest( - &ConfigChangeStressTest::triggerPredictInALoop, - &ConfigChangeStressTest::removePipelineDefinition, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} -TEST_F(StressPipelineConfigChanges, ChangedPipelineConnectionNameDuringPredictLoad) { - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; - performStressTest( - &ConfigChangeStressTest::triggerPredictInALoop, - &ConfigChangeStressTest::changeConnectionName, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} -TEST_F(StressPipelineConfigChanges, AddedNewPipelineDuringPredictLoad) { - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {}; - performStressTest( - &ConfigChangeStressTest::triggerPredictInALoop, - &ConfigChangeStressTest::addNewPipeline, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} -TEST_F(StressPipelineConfigChanges, RetireSpecificVersionUsedDuringPredictLoad) { - // we declare specific version used (1) and latest model version policy with count=1 - // then we add version 2 causing previous default to be retired - SetUpConfig(stressTestPipelineOneDummyConfigSpecificVersionUsed); - bool performWholeConfigReload = false; - std::set requiredLoadResults = {StatusCode::OK, // we expect full continuity of operation - StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET, // we hit when all config changes finish to propagate - StatusCode::MODEL_VERSION_NOT_LOADED_ANYMORE}; // version is retired but pipeline not invalidated yet - std::set allowedLoadResults = {}; - performStressTest( - &ConfigChangeStressTest::triggerPredictInALoop, - &ConfigChangeStressTest::retireSpecificVersionUsed, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} -TEST_F(StressPipelineConfigChanges, AddNewVersionDuringGetMetadataLoad) { - bool performWholeConfigReload = false; // we just need to have all model versions rechecked - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {}; - performStressTest( - &ConfigChangeStressTest::triggerGetPipelineMetadataInALoop, - &ConfigChangeStressTest::defaultVersionAdd, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} -TEST_F(StressPipelineConfigChanges, RemoveDefaultVersionDuringGetMetadataLoad) { - std::set requiredLoadResults = {StatusCode::OK, - StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; // we hit when all config changes finish to propagate - std::set allowedLoadResults = {}; - // we need whole config reload since there is no other way to dispose - // all model versions different than removing model from config - bool performWholeConfigReload = true; - performStressTest( - &ConfigChangeStressTest::triggerGetPipelineMetadataInALoop, - &ConfigChangeStressTest::defaultVersionRemove, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} -TEST_F(StressPipelineConfigChanges, ChangeToShapeAutoDuringGetMetadataLoad) { - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; - performStressTest( - &ConfigChangeStressTest::triggerGetPipelineMetadataInALoop, - &ConfigChangeStressTest::changeToAutoShape, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} -TEST_F(StressPipelineConfigChanges, RemovePipelineDefinitionDuringGetMetadataLoad) { - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK, - StatusCode::PIPELINE_DEFINITION_NOT_LOADED_ANYMORE}; // when pipeline is retired - std::set allowedLoadResults = {}; - performStressTest( - &ConfigChangeStressTest::triggerGetPipelineMetadataInALoop, - &ConfigChangeStressTest::removePipelineDefinition, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} -TEST_F(StressPipelineConfigChanges, ChangedPipelineConnectionNameDuringGetMetadataLoad) { - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; - performStressTest( - &ConfigChangeStressTest::triggerGetPipelineMetadataInALoop, - &ConfigChangeStressTest::changeConnectionName, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} -TEST_F(StressPipelineConfigChanges, AddedNewPipelineDuringGetMetadataLoad) { - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {}; - performStressTest( - &ConfigChangeStressTest::triggerGetPipelineMetadataInALoop, - &ConfigChangeStressTest::addNewPipeline, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} -TEST_F(StressPipelineConfigChanges, RetireSpecificVersionUsedDuringGetMetadataLoad) { - // we declare specific version used (1) and latest model version policy with count=1 - // then we add version 2 causing previous default to be retired - SetUpConfig(stressTestPipelineOneDummyConfigSpecificVersionUsed); - bool performWholeConfigReload = false; - std::set requiredLoadResults = {StatusCode::OK, // we expect full continuity of operation - StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; // we hit when all config changes finish to propagate - std::set allowedLoadResults = {}; - performStressTest( - &ConfigChangeStressTest::triggerGetPipelineMetadataInALoop, - &ConfigChangeStressTest::retireSpecificVersionUsed, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} - -class StressPipelineCustomNodesWithPreallocatedBuffersConfigChanges : public StressPipelineConfigChanges { -public: - void checkPipelineResponse(const std::string& pipelineOutputName, - tensorflow::serving::PredictRequest& request, - tensorflow::serving::PredictResponse& response) override { - std::vector result(requestData.begin(), requestData.end()); - std::transform(result.begin(), result.end(), result.begin(), [this](float f) -> float { return f + 1 - 0; }); - checkDummyResponse(pipelineOutputName, result, request, response, 1); - } -}; - -TEST_F(StressPipelineCustomNodesWithPreallocatedBuffersConfigChanges, RemoveCustomLibraryDuringPredictLoad) { - SetUpConfig(stressPipelineCustomNodeAddOneThenDummy); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK, // we expect full continuity of operation - StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; // we hit when all config changes finish to propagate - std::set allowedLoadResults = {}; - performStressTest( - &ConfigChangeStressTest::triggerPredictInALoop, - &ConfigChangeStressTest::removePreallocatedCustomLibraryUsed, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} - -TEST_F(StressPipelineCustomNodesWithPreallocatedBuffersConfigChanges, RenameCustomLibraryDuringPredictLoad) { - SetUpConfig(stressPipelineCustomNodeAddOneThenDummy); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; // might hit reload phase - performStressTest( - &ConfigChangeStressTest::triggerPredictInALoop, - &ConfigChangeStressTest::renamePreallocatedCustomLibraryUsed, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} - -TEST_F(StressPipelineCustomNodesWithPreallocatedBuffersConfigChanges, ChangeParamCustomLibraryDuringPredictLoad) { - SetUpConfig(stressPipelineCustomNodeAddOneThenDummy); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; // might hit reload phase - performStressTest( - &ConfigChangeStressTest::triggerPredictInALoop, - &ConfigChangeStressTest::changeParamPreallocatedCustomLibraryUsed, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} - -TEST_F(StressPipelineCustomNodesWithPreallocatedBuffersConfigChanges, ReduceQueueSizeCustomLibraryDuringPredictLoad) { - SetUpConfig(stressPipelineCustomNodeAddOneThenDummy); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; // might hit reload phase - performStressTest( - &ConfigChangeStressTest::triggerPredictInALoop, - &ConfigChangeStressTest::reduceQueueSizePreallocatedCustomLibraryUsed, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} - -TEST_F(StressPipelineCustomNodesWithPreallocatedBuffersConfigChanges, IncreaseQueueSizeCustomLibraryDuringPredictLoad) { - SetUpConfig(stressPipelineCustomNodeAddOneThenDummy); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; // might hit reload phase - performStressTest( - &ConfigChangeStressTest::triggerPredictInALoop, - &ConfigChangeStressTest::increaseQueueSizePreallocatedCustomLibraryUsed, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} - -TEST_F(StressPipelineCustomNodesWithPreallocatedBuffersConfigChanges, RemoveCustomLibraryDuringGetMetadataLoad) { - SetUpConfig(stressPipelineCustomNodeAddOneThenDummy); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK, // we expect full continuity of operation - StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; // we hit when all config changes finish to propagate - std::set allowedLoadResults = {}; - performStressTest( - &ConfigChangeStressTest::triggerGetPipelineMetadataInALoop, - &ConfigChangeStressTest::removePreallocatedCustomLibraryUsed, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} - -TEST_F(StressPipelineCustomNodesWithPreallocatedBuffersConfigChanges, RenameCustomLibraryDuringGetMetadataLoad) { - SetUpConfig(stressPipelineCustomNodeAddOneThenDummy); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; // might hit reload phase - performStressTest( - &ConfigChangeStressTest::triggerGetPipelineMetadataInALoop, - &ConfigChangeStressTest::renamePreallocatedCustomLibraryUsed, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} - -TEST_F(StressPipelineCustomNodesWithPreallocatedBuffersConfigChanges, ChangeParamCustomLibraryDuringGetMetadataLoad) { - SetUpConfig(stressPipelineCustomNodeAddOneThenDummy); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; // might hit reload phase - performStressTest( - &ConfigChangeStressTest::triggerGetPipelineMetadataInALoop, - &ConfigChangeStressTest::changeParamPreallocatedCustomLibraryUsed, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} - -TEST_F(StressPipelineCustomNodesWithPreallocatedBuffersConfigChanges, ReduceQueueSizeCustomLibraryDuringGetMetadataLoad) { - SetUpConfig(stressPipelineCustomNodeAddOneThenDummy); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; // might hit reload phase - performStressTest( - &ConfigChangeStressTest::triggerGetPipelineMetadataInALoop, - &ConfigChangeStressTest::reduceQueueSizePreallocatedCustomLibraryUsed, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} - -TEST_F(StressPipelineCustomNodesWithPreallocatedBuffersConfigChanges, IncreaseQueueSizeCustomLibraryDuringGetMetadataLoad) { - SetUpConfig(stressPipelineCustomNodeAddOneThenDummy); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; // might hit reload phase - performStressTest( - &ConfigChangeStressTest::triggerGetPipelineMetadataInALoop, - &ConfigChangeStressTest::increaseQueueSizePreallocatedCustomLibraryUsed, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} - -class StressPipelineCustomNodesConfigChanges : public StressPipelineConfigChanges { - const int64_t differentOpsFactorsInputSize = 4; - const std::vector factorsData{1., 3, 2, 2}; - const std::string pipelineFactorsInputName{"pipeline_factors"}; - -public: - tensorflow::serving::PredictRequest preparePipelinePredictRequest(tensorflow::serving::PredictRequest) override { - tensorflow::serving::PredictRequest request; - preparePredictRequest(request, getExpectedInputsInfo()); - auto& input = (*request.mutable_inputs())[pipelineInputName]; - input.mutable_tensor_content()->assign((char*)requestData.data(), requestData.size() * sizeof(float)); - auto& factors = (*request.mutable_inputs())[pipelineFactorsInputName]; - factors.mutable_tensor_content()->assign((char*)factorsData.data(), factorsData.size() * sizeof(float)); - return request; - } - inputs_info_t getExpectedInputsInfo() override { - return {{pipelineInputName, - std::tuple{{1, DUMMY_MODEL_INPUT_SIZE}, ovms::Precision::FP32}}, - {pipelineFactorsInputName, - std::tuple{{1, differentOpsFactorsInputSize}, ovms::Precision::FP32}}}; - } - void checkPipelineResponse(const std::string& pipelineOutputName, - tensorflow::serving::PredictRequest& request, - tensorflow::serving::PredictResponse& response) override { - // we need to imitate -> different ops then dummy then max - std::vector result(requestData.begin(), requestData.end()); - std::transform(result.begin(), result.end(), result.begin(), [this](float f) -> float { return f * factorsData[2]; }); - checkDummyResponse(pipelineOutputName, result, request, response, 1); - } -}; - -TEST_F(StressPipelineCustomNodesConfigChanges, RemoveCustomLibraryDuringPredictLoad) { - SetUpConfig(stressPipelineCustomNodeDifferentOperationsThenDummyThenChooseMaximumConfig); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK, // we expect full continuity of operation - StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; // we hit when all config changes finish to propagate - std::set allowedLoadResults = {}; - performStressTest( - &ConfigChangeStressTest::triggerPredictInALoop, - &ConfigChangeStressTest::removeCustomLibraryUsed, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} -TEST_F(StressPipelineCustomNodesConfigChanges, ChangeCustomLibraryParamDuringPredictLoad) { - // we change used PARAM during load. This change does not effect results, but should be enough to verify - // correctness of this operation - no segfaults etc. - SetUpConfig(stressPipelineCustomNodeDifferentOperationsThenDummyThenChooseMaximumConfig); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {}; - performStressTest( - &ConfigChangeStressTest::triggerPredictInALoop, - &ConfigChangeStressTest::changeCustomLibraryParam, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} -TEST_F(StressPipelineCustomNodesConfigChanges, RemoveCustomLibraryDuringGetMetadataLoad) { - SetUpConfig(stressPipelineCustomNodeDifferentOperationsThenDummyThenChooseMaximumConfig); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK, // we expect full continuity of operation - StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; // we hit when all config changes finish to propagate - std::set allowedLoadResults = {}; - performStressTest( - &ConfigChangeStressTest::triggerGetPipelineMetadataInALoop, - &ConfigChangeStressTest::removeCustomLibraryUsed, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} -TEST_F(StressPipelineCustomNodesConfigChanges, ChangeCustomLibraryParamDuringGetMetadataLoad) { - SetUpConfig(stressPipelineCustomNodeDifferentOperationsThenDummyThenChooseMaximumConfig); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation most of the time - std::set allowedLoadResults = {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; // might hit reload phase - performStressTest( - &ConfigChangeStressTest::triggerGetPipelineMetadataInALoop, - &ConfigChangeStressTest::changeCustomLibraryParam, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} -TEST_F(StressModelConfigChanges, AddModelDuringGetModelStatusLoad) { - bool performWholeConfigReload = true; // we just need to have all model versions rechecked - std::set requiredLoadResults = { - StatusCode::MODEL_NAME_MISSING, // until first model is loaded - StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = { - StatusCode::MODEL_VERSION_MISSING // this should be hit if test is stressing enough, sporadically does not happen - }; - performStressTest( - &ConfigChangeStressTest::triggerGetPipelineStatusInALoop, - &ConfigChangeStressTest::addFirstModel, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} - #if (MEDIAPIPE_DISABLE == 0) class StressMediapipeChanges : public StressPipelineConfigChanges { const std::string modelName = PIPELINE_1_DUMMY_NAME; @@ -695,69 +146,6 @@ TEST_F(StressMediapipeChanges, ReloadMediapipeGraphDuringPredictLoad) { allowedLoadResults); } -TEST_F(StressMediapipeChanges, AddGraphDuringStatusLoad) { - // we add another definition during load - SetUpConfig(basicMediapipeConfig); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {}; - performStressTest( - &ConfigChangeStressTest::triggerGetPipelineStatusInALoop, - &ConfigChangeStressTest::addNewMediapipeGraph, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} -TEST_F(StressMediapipeChanges, RemoveGraphDuringStatusLoad) { - SetUpConfig(basicMediapipeConfig); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {}; - performStressTest( - &ConfigChangeStressTest::triggerGetPipelineStatusInALoop, - &ConfigChangeStressTest::removeMediapipeGraph, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} -TEST_F(StressMediapipeChanges, RemoveModelDuringStatusLoad) { - SetUpConfig(basicMediapipeConfig); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {}; - performStressTest( - &ConfigChangeStressTest::triggerGetPipelineStatusInALoop, - &ConfigChangeStressTest::removeMediapipeGraphUsedModel, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} -TEST_F(StressMediapipeChanges, ReloadModelDuringStatusLoad) { - // we change nireq during load - SetUpConfig(basicMediapipeConfig); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {}; - performStressTest( - &ConfigChangeStressTest::triggerGetPipelineStatusInALoop, - &ConfigChangeStressTest::reloadMediapipeGraphUsedModel, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} -TEST_F(StressMediapipeChanges, ReloadMediapipeGraphDuringStatusLoad) { - // we change nireq during load - SetUpConfig(basicMediapipeConfig); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {}; - performStressTest( - &ConfigChangeStressTest::triggerGetPipelineStatusInALoop, - &ConfigChangeStressTest::reloadMediapipeGraph, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} TEST_F(StressMediapipeChanges, AddGraphDuringMetadataLoad) { // we add another definition during load SetUpConfig(basicMediapipeConfig); diff --git a/src/test/ensemble_mapping_config_tests.cpp b/src/test/ensemble_mapping_config_tests.cpp deleted file mode 100644 index 134aa85630..0000000000 --- a/src/test/ensemble_mapping_config_tests.cpp +++ /dev/null @@ -1,445 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include - -#include -#include - -#include "../dags/entry_node.hpp" -#include "../dags/exit_node.hpp" -#include "../dags/pipeline.hpp" -#include "../dags/pipeline_factory.hpp" -#include "../dags/pipelinedefinition.hpp" -#include "../execution_context.hpp" -#include "../get_model_metadata_impl.hpp" -#include "../kfs_frontend/kfs_grpc_inference_service.hpp" -#include "../model.hpp" -#include "../model_metric_reporter.hpp" -#include "../modelconfig.hpp" -#include "../modelmanager.hpp" - -#include "constructor_enabled_model_manager.hpp" -#include "test_models_configs.hpp" -#include "test_utils.hpp" -#include "light_test_utils.hpp" -#include "platform_utils.hpp" -#include "test_with_temp_dir.hpp" - -using namespace ovms; - -using ::testing::ElementsAre; - -/* - * Scenario with pipelines with underlying DL models loaded with mapping_config.json. - * This set of tests ensures model input/output mapping is respected by pipeline execution. - */ - -class PipelineWithInputOutputNameMappedModel : public TestWithTempDir { -protected: - void SetUp() override { - TestWithTempDir::SetUp(); - - configPath = directoryPath + "/config.json"; - modelPath = directoryPath + "/dummy"; - mappingConfigPath = modelPath + "/1/mapping_config.json"; - - std::filesystem::copy(getGenericFullPathForSrcTest("/ovms/src/test/dummy"), modelPath, std::filesystem::copy_options::recursive); - } - - std::string configPath; - std::string modelPath; - std::string mappingConfigPath; - - ConstructorEnabledModelManager managerWithDummyModel; -}; - -TEST_F(PipelineWithInputOutputNameMappedModel, SuccessfullyReferToMappedNamesAndExecute) { - // Create mapping config for model - createConfigFileWithContent(R"({ - "inputs": {"b": "input_tensor"}, - "outputs": {"a": "output_tensor"} - })", - mappingConfigPath); - - // Load models - auto modelConfig = DUMMY_MODEL_CONFIG; - adjustConfigToAllowModelFileRemovalWhenLoaded(modelConfig); - modelConfig.setBasePath(getGenericFullPathForSrcTest(modelPath)); - ASSERT_EQ(managerWithDummyModel.reloadModelWithVersions(modelConfig), StatusCode::OK_RELOADED); - - // Create pipeline definition - PipelineFactory factory; - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{"vector", "vector"}}}, - {NodeKind::DL, "dummyA", "dummy", std::nullopt, {{"output_tensor", "output_tensor"}}}, - {NodeKind::DL, "dummyB", "dummy", std::nullopt, {{"output_tensor", "output_tensor"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["dummyA"] = { - {ENTRY_NODE_NAME, {{"vector", "input_tensor"}}}}; - connections["dummyB"] = { - {"dummyA", {{"output_tensor", "input_tensor"}}}}; - connections[EXIT_NODE_NAME] = { - {"dummyB", {{"output_tensor", "response_tensor_name"}}}}; - - // Ensure definition created without errors - ASSERT_EQ(factory.createDefinition("pipeline", info, connections, managerWithDummyModel, managerWithDummyModel, managerWithDummyModel), StatusCode::OK); - - // Prepare request - std::unique_ptr pipeline; - tensorflow::serving::PredictRequest request; - tensorflow::serving::PredictResponse response; - auto& input_proto = (*request.mutable_inputs())["vector"]; - input_proto.set_dtype(tensorflow::DataType::DT_FLOAT); - input_proto.mutable_tensor_shape()->add_dim()->set_size(1); - input_proto.mutable_tensor_shape()->add_dim()->set_size(DUMMY_MODEL_INPUT_SIZE); - - std::vector input_data(DUMMY_MODEL_INPUT_SIZE); - std::iota(input_data.begin(), input_data.end(), 1); // 1, 2, 3, ..., 10 - input_proto.mutable_tensor_content()->assign((char*)input_data.data(), sizeof(float) * DUMMY_MODEL_INPUT_SIZE); - - // Execute pipeline - factory.create(pipeline, "pipeline", &request, &response, managerWithDummyModel); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - - // Compare response - ASSERT_EQ(response.outputs_size(), 1); - ASSERT_EQ(response.outputs().count("response_tensor_name"), 1); - const auto& output_proto = response.outputs().at("response_tensor_name"); - ASSERT_EQ(output_proto.dtype(), tensorflow::DataType::DT_FLOAT); - ASSERT_THAT(asVector(output_proto.tensor_shape()), ElementsAre(1, 10)); - - std::vector output_data(DUMMY_MODEL_INPUT_SIZE); - std::transform(input_data.begin(), input_data.end(), output_data.begin(), [](float& v) { return v + 2.0; }); // 3, 4, ... 12 - ASSERT_EQ(asVector(output_proto.tensor_content()), output_data); -} - -TEST_F(PipelineWithInputOutputNameMappedModel, ReferingToOriginalInputNameFailsCreation) { - // Create mapping config for model - createConfigFileWithContent(R"({ - "inputs": {"b": "input_tensor"}, - "outputs": {"a": "output_tensor"} - })", - mappingConfigPath); - - // Load models - auto modelConfig = DUMMY_MODEL_CONFIG; - adjustConfigToAllowModelFileRemovalWhenLoaded(modelConfig); - modelConfig.setBasePath(getGenericFullPathForSrcTest(modelPath)); - ASSERT_EQ(managerWithDummyModel.reloadModelWithVersions(modelConfig), StatusCode::OK_RELOADED); - - // Create pipeline definition - PipelineFactory factory; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{"vector", "vector"}}}, - {NodeKind::DL, "dummyA", "dummy", std::nullopt, {{"output_tensor", "output_tensor"}}}, - {NodeKind::DL, "dummyB", "dummy", std::nullopt, {{"output_tensor", "output_tensor"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["dummyA"] = { - {ENTRY_NODE_NAME, {{"vector", "b"}}}}; - connections["dummyB"] = { - {"dummyA", {{"output_tensor", "b"}}}}; - connections[EXIT_NODE_NAME] = { - {"dummyB", {{"output_tensor", "response_tensor_name"}}}}; - - EXPECT_EQ(factory.createDefinition("pipeline", info, connections, managerWithDummyModel, managerWithDummyModel, managerWithDummyModel), StatusCode::PIPELINE_CONNECTION_TO_MISSING_MODEL_INPUT); -} - -TEST_F(PipelineWithInputOutputNameMappedModel, ReferingToOriginalOutputNameFailsCreation) { - // Create mapping config for model - createConfigFileWithContent(R"({ - "inputs": {"b": "input_tensor"}, - "outputs": {"a": "output_tensor"} - })", - mappingConfigPath); - - // Load models - auto modelConfig = DUMMY_MODEL_CONFIG; - adjustConfigToAllowModelFileRemovalWhenLoaded(modelConfig); - modelConfig.setBasePath(getGenericFullPathForSrcTest(modelPath)); - ASSERT_EQ(managerWithDummyModel.reloadModelWithVersions(modelConfig), StatusCode::OK_RELOADED); - - // Create pipeline definition - PipelineFactory factory; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{"vector", "vector"}}}, - {NodeKind::DL, "dummyA", "dummy", std::nullopt, {{"output_tensor", "a"}}}, - {NodeKind::DL, "dummyB", "dummy", std::nullopt, {{"output_tensor", "a"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["dummyA"] = { - {ENTRY_NODE_NAME, {{"vector", "input_tensor"}}}}; - connections["dummyB"] = { - {"dummyA", {{"output_tensor", "input_tensor"}}}}; - connections[EXIT_NODE_NAME] = { - {"dummyB", {{"output_tensor", "response_tensor_name"}}}}; - - EXPECT_EQ(factory.createDefinition("pipeline", info, connections, managerWithDummyModel, managerWithDummyModel, managerWithDummyModel), StatusCode::PIPELINE_NODE_REFERING_TO_MISSING_MODEL_OUTPUT); -} - -TEST_F(PipelineWithInputOutputNameMappedModel, SuccessfullyReferToMappedNamesAndGetMetadata) { - // Create mapping config for model - createConfigFileWithContent(R"({ - "inputs": {"b": "input_tensor"}, - "outputs": {"a": "output_tensor"} - })", - mappingConfigPath); - - // Load models - auto modelConfig = DUMMY_MODEL_CONFIG; - adjustConfigToAllowModelFileRemovalWhenLoaded(modelConfig); - modelConfig.setBasePath(getGenericFullPathForSrcTest(modelPath)); - ASSERT_EQ(managerWithDummyModel.reloadModelWithVersions(modelConfig), StatusCode::OK_RELOADED); - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{"vector", "vector"}}}, - {NodeKind::DL, "dummyA", "dummy", std::nullopt, {{"output_tensor", "output_tensor"}}}, - {NodeKind::DL, "dummyB", "dummy", std::nullopt, {{"output_tensor", "output_tensor"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["dummyA"] = { - {ENTRY_NODE_NAME, {{"vector", "input_tensor"}}}}; - connections["dummyB"] = { - {"dummyA", {{"output_tensor", "input_tensor"}}}}; - connections[EXIT_NODE_NAME] = { - {"dummyB", {{"output_tensor", "response_tensor_name"}}}}; - - auto def = std::make_unique( - "my_new_pipeline", info, connections); - - ASSERT_EQ(def->validate(managerWithDummyModel, managerWithDummyModel, managerWithDummyModel), StatusCode::OK); - - auto inputs = def->getInputsInfo(); - auto outputs = def->getOutputsInfo(); - ASSERT_EQ(inputs.size(), 1); - ASSERT_EQ(outputs.size(), 1); - ASSERT_NE(inputs.find("vector"), inputs.end()); - ASSERT_NE(outputs.find("response_tensor_name"), outputs.end()); - - const auto& vector = inputs.at("vector"); - EXPECT_EQ(vector->getShape(), ovms::Shape({1, DUMMY_MODEL_INPUT_SIZE})); - EXPECT_EQ(vector->getPrecision(), ovms::Precision::FP32); - - const auto& response_tensor_name = outputs.at("response_tensor_name"); - EXPECT_EQ(response_tensor_name->getShape(), ovms::Shape({1, DUMMY_MODEL_OUTPUT_SIZE})); - EXPECT_EQ(response_tensor_name->getPrecision(), ovms::Precision::FP32); -} - -TEST_F(PipelineWithInputOutputNameMappedModel, SuccessfullyReloadPipelineAfterAddingModelMapping) { - // Load models - auto modelConfig = DUMMY_MODEL_CONFIG; - adjustConfigToAllowModelFileRemovalWhenLoaded(modelConfig); - modelConfig.setBasePath(getGenericFullPathForSrcTest(modelPath)); - ASSERT_EQ(managerWithDummyModel.reloadModelWithVersions(modelConfig), StatusCode::OK_RELOADED); - - // Create pipeline definition - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{"vector", "vector"}}}, - {NodeKind::DL, "dummyA", "dummy", std::nullopt, {{"output_tensor", "output_tensor"}}}, - {NodeKind::DL, "dummyB", "dummy", std::nullopt, {{"output_tensor", "output_tensor"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - pipeline_connections_t connections; - - connections["dummyA"] = { - {ENTRY_NODE_NAME, {{"vector", "input_tensor"}}}}; - connections["dummyB"] = { - {"dummyA", {{"output_tensor", "input_tensor"}}}}; - connections[EXIT_NODE_NAME] = { - {"dummyB", {{"output_tensor", "response_tensor_name"}}}}; - - // Validation fails since mapping is expected - PipelineDefinition pd("UNUSED_NAME", info, connections); - auto status = pd.validate(managerWithDummyModel, managerWithDummyModel, managerWithDummyModel); - EXPECT_TRUE(status.getCode() == ovms::StatusCode::PIPELINE_CONNECTION_TO_MISSING_MODEL_INPUT) - << status.string(); - - // Create mapping config for model - createConfigFileWithContent(R"({ - "inputs": {"b": "input_tensor"}, - "outputs": {"a": "output_tensor"} - })", - mappingConfigPath); - - // reload pipeline definition after adding mapping - // we need to trigger reload - just adding model mapping won't do the trick - // hence setting nireq - modelConfig.setNireq(modelConfig.getNireq() + 1); - status = managerWithDummyModel.reloadModelWithVersions(modelConfig); - ASSERT_TRUE(status.ok()) << status.string(); - status = pd.reload(managerWithDummyModel, managerWithDummyModel, managerWithDummyModel, std::move(info), std::move(connections)); - ASSERT_TRUE(status.ok()) << status.string(); - - // Prepare request - std::unique_ptr pipeline; - tensorflow::serving::PredictRequest request; - tensorflow::serving::PredictResponse response; - auto& input_proto = (*request.mutable_inputs())["vector"]; - input_proto.set_dtype(tensorflow::DataType::DT_FLOAT); - input_proto.mutable_tensor_shape()->add_dim()->set_size(1); - input_proto.mutable_tensor_shape()->add_dim()->set_size(DUMMY_MODEL_INPUT_SIZE); - - std::vector input_data(DUMMY_MODEL_INPUT_SIZE); - std::iota(input_data.begin(), input_data.end(), 1); // 1, 2, 3, ..., 10 - input_proto.mutable_tensor_content()->assign((char*)input_data.data(), sizeof(float) * DUMMY_MODEL_INPUT_SIZE); - - // Execute pipeline - pd.create(pipeline, &request, &response, managerWithDummyModel); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - - // Compare response - ASSERT_EQ(response.outputs_size(), 1); - ASSERT_EQ(response.outputs().count("response_tensor_name"), 1); - const auto& output_proto = response.outputs().at("response_tensor_name"); - ASSERT_EQ(output_proto.dtype(), tensorflow::DataType::DT_FLOAT); - ASSERT_THAT(asVector(output_proto.tensor_shape()), ElementsAre(1, 10)); - - std::vector output_data(DUMMY_MODEL_INPUT_SIZE); - std::transform(input_data.begin(), input_data.end(), output_data.begin(), [](float& v) { return v + 2.0; }); // 3, 4, ... 12 - ASSERT_EQ(asVector(output_proto.tensor_content()), output_data); -} - -TEST_F(PipelineWithInputOutputNameMappedModel, ReloadPipelineAfterRemovalOfModelMappingWillFail) { - // Create mapping config for model - createConfigFileWithContent(R"({ - "inputs": {"b": "input_tensor"}, - "outputs": {"a": "output_tensor"} - })", - mappingConfigPath); - // Load models - auto modelConfig = DUMMY_MODEL_CONFIG; - adjustConfigToAllowModelFileRemovalWhenLoaded(modelConfig); - modelConfig.setBasePath(getGenericFullPathForSrcTest(modelPath)); - ASSERT_EQ(managerWithDummyModel.reloadModelWithVersions(modelConfig), StatusCode::OK_RELOADED); - - // Create pipeline definition - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{"vector", "vector"}}}, - {NodeKind::DL, "dummyA", "dummy", std::nullopt, {{"output_tensor", "output_tensor"}}}, - {NodeKind::DL, "dummyB", "dummy", std::nullopt, {{"output_tensor", "output_tensor"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - pipeline_connections_t connections; - - connections["dummyA"] = { - {ENTRY_NODE_NAME, {{"vector", "input_tensor"}}}}; - connections["dummyB"] = { - {"dummyA", {{"output_tensor", "input_tensor"}}}}; - connections[EXIT_NODE_NAME] = { - {"dummyB", {{"output_tensor", "response_tensor_name"}}}}; - - // Ensure definition created without errors - PipelineDefinition pd("UNUSED_NAME", info, connections); - auto status = pd.validate(managerWithDummyModel, managerWithDummyModel, managerWithDummyModel); - ASSERT_TRUE(status.ok()) << status.string(); - - // reload pipeline definition after removing mapping - // we need to trigger reload - just adding model mapping won't do the trick - // hence setting nireq - ASSERT_TRUE(std::filesystem::remove(mappingConfigPath)); - modelConfig.setNireq(modelConfig.getNireq() + 1); - status = managerWithDummyModel.reloadModelWithVersions(modelConfig); - ASSERT_TRUE(status.ok()) << status.string(); - status = pd.reload(managerWithDummyModel, managerWithDummyModel, managerWithDummyModel, std::move(info), std::move(connections)); - EXPECT_TRUE(status.getCode() == ovms::StatusCode::PIPELINE_CONNECTION_TO_MISSING_MODEL_INPUT) - << status.string(); -} - -class ModelWithInputOutputNameMappedModel : public PipelineWithInputOutputNameMappedModel { -public: - KFSModelMetadataResponse kfsResponse; - tensorflow::serving::GetModelMetadataResponse tfsResponse; - std::shared_ptr instance; -}; - -TEST_F(ModelWithInputOutputNameMappedModel, GetModelMetadataOnKFSEndpoint) { - // Create mapping config for model - createConfigFileWithContent(R"({ - "inputs": {"b": "input_tensor"}, - "outputs": {"a": "output_tensor"} - })", - mappingConfigPath); - - // Load models - auto modelConfig = DUMMY_MODEL_CONFIG; - adjustConfigToAllowModelFileRemovalWhenLoaded(modelConfig); - modelConfig.setBasePath(getGenericFullPathForSrcTest(modelPath)); - ASSERT_EQ(managerWithDummyModel.reloadModelWithVersions(modelConfig), StatusCode::OK_RELOADED); - - auto model = managerWithDummyModel.findModelByName("dummy"); - ASSERT_NE(model, nullptr); - instance = model->getDefaultModelInstance(); - ASSERT_NE(instance, nullptr); - KFSModelExtraMetadata extraMetadata; - ASSERT_EQ(ovms::KFSInferenceServiceImpl::buildResponse(*model, *instance, &kfsResponse, extraMetadata), ovms::StatusCode::OK); - - EXPECT_EQ(kfsResponse.inputs_size(), 1); - auto input = kfsResponse.inputs().at(0); - EXPECT_EQ(input.name(), "input_tensor"); - - EXPECT_EQ(kfsResponse.outputs_size(), 1); - auto output = kfsResponse.outputs().at(0); - EXPECT_EQ(output.name(), "output_tensor"); -} - -TEST_F(ModelWithInputOutputNameMappedModel, GetModelMetadataOnTfsEndpoint) { - // Create mapping config for model - createConfigFileWithContent(R"({ - "inputs": {"b": "input_tensor"}, - "outputs": {"a": "output_tensor"} - })", - mappingConfigPath); - - // Load models - auto modelConfig = DUMMY_MODEL_CONFIG; - adjustConfigToAllowModelFileRemovalWhenLoaded(modelConfig); - modelConfig.setBasePath(getGenericFullPathForSrcTest(modelPath)); - ASSERT_EQ(managerWithDummyModel.reloadModelWithVersions(modelConfig), StatusCode::OK_RELOADED); - - auto model = managerWithDummyModel.findModelByName("dummy"); - ASSERT_NE(model, nullptr); - instance = model->getDefaultModelInstance(); - ASSERT_NE(instance, nullptr); - - ASSERT_EQ(ovms::GetModelMetadataImpl::buildResponse(instance, &tfsResponse), ovms::StatusCode::OK); - - tensorflow::serving::SignatureDefMap def; - tfsResponse.metadata().at("signature_def").UnpackTo(&def); - - const auto& inputs = ((*def.mutable_signature_def())["serving_default"]).inputs(); - const auto& outputs = ((*def.mutable_signature_def())["serving_default"]).outputs(); - - EXPECT_EQ(inputs.size(), 1); - EXPECT_EQ(inputs.at("input_tensor").name(), "input_tensor"); - - EXPECT_EQ(outputs.size(), 1); - EXPECT_EQ(outputs.at("output_tensor").name(), "output_tensor"); -} diff --git a/src/test/get_mediapipe_graph_metadata_response_test.cpp b/src/test/get_mediapipe_graph_metadata_response_test.cpp index 1452afcc0b..ab4c3debd5 100644 --- a/src/test/get_mediapipe_graph_metadata_response_test.cpp +++ b/src/test/get_mediapipe_graph_metadata_response_test.cpp @@ -27,12 +27,10 @@ #include "../mediapipe_internal/mediapipegraphexecutor.hpp" #endif #include "../executingstreamidguard.hpp" -#include "../get_model_metadata_impl.hpp" #include "../grpcservermodule.hpp" #include "../kfs_frontend/kfs_grpc_inference_service.hpp" #include "src/filesystem/localfilesystem.hpp" #include "../model.hpp" -#include "../model_service.hpp" #include "../modelinstance.hpp" #include "../modelinstanceunloadguard.hpp" #include "../modelmanager.hpp" @@ -207,33 +205,6 @@ class TestImplGetModelStatus : public ::testing::Test { } }; -TEST_F(TestImplGetModelStatus, NegativeTfsGetModelStatus) { - // Create config file with an empty config & reload - std::string configStr = dummy_config; - configStr = configStr.replace(configStr.find("/tmp/test_cl_models"), std::string("/tmp/test_cl_models").size(), cl_models_path); - std::string fileToReload = cl_models_path + "/cl_config.json"; - createConfigFileWithContent(configStr, fileToReload); - ASSERT_EQ(manager.loadConfig(fileToReload), ovms::StatusCode::OK); - - tensorflow::serving::GetModelMetadataRequest req; - tensorflow::serving::GetModelMetadataResponse res; - - auto model_spec = req.mutable_model_spec(); - model_spec->Clear(); - model_spec->set_name("dummy2"); - model_spec->mutable_version()->set_value(2); - ASSERT_EQ(GetModelMetadataImpl::getModelStatus(&req, &res, manager, DEFAULT_TEST_CONTEXT), StatusCode::MODEL_NAME_MISSING); - - model_spec->Clear(); - model_spec->set_name("dummy"); - model_spec->mutable_version()->set_value(2); - ASSERT_EQ(GetModelMetadataImpl::getModelStatus(&req, &res, manager, DEFAULT_TEST_CONTEXT), StatusCode::MODEL_VERSION_MISSING); - - model_spec->Clear(); - model_spec->set_name("dummy"); - ASSERT_EQ(GetModelMetadataImpl::getModelStatus(&req, &res, manager, DEFAULT_TEST_CONTEXT), StatusCode::MODEL_VERSION_MISSING); -} - class ServerShutdownGuard { ovms::Server& ovmsServer; @@ -285,20 +256,20 @@ TEST_F(TestImplGetModelStatus, NegativeKfsGetModelStatus) { const ovms::Module* grpcModule = server.getModule(ovms::GRPC_SERVER_MODULE_NAME); KFSInferenceServiceImpl& impl = dynamic_cast(grpcModule)->getKFSGrpcImpl(); - ASSERT_EQ(impl.ModelMetadataImpl(nullptr, &req, &res, ovms::ExecutionContext(ovms::ExecutionContext::Interface::GRPC, ovms::ExecutionContext::Method::GetModelMetadata), extraMetadata), StatusCode::MODEL_NAME_MISSING); + ASSERT_EQ(impl.ModelMetadataImpl(nullptr, &req, &res, ovms::ExecutionContext(ovms::ExecutionContext::Interface::GRPC, ovms::ExecutionContext::Method::ModelMetadata), extraMetadata), StatusCode::MODEL_NAME_MISSING); req.Clear(); req.set_name("dummy"); req.set_version("2"); - ASSERT_EQ(impl.ModelMetadataImpl(nullptr, &req, &res, ovms::ExecutionContext(ovms::ExecutionContext::Interface::GRPC, ovms::ExecutionContext::Method::GetModelMetadata), extraMetadata), StatusCode::MODEL_VERSION_MISSING); + ASSERT_EQ(impl.ModelMetadataImpl(nullptr, &req, &res, ovms::ExecutionContext(ovms::ExecutionContext::Interface::GRPC, ovms::ExecutionContext::Method::ModelMetadata), extraMetadata), StatusCode::MODEL_VERSION_MISSING); req.Clear(); req.set_name("dummy"); - ASSERT_EQ(impl.ModelMetadataImpl(nullptr, &req, &res, ovms::ExecutionContext(ovms::ExecutionContext::Interface::GRPC, ovms::ExecutionContext::Method::GetModelMetadata), extraMetadata), StatusCode::MODEL_VERSION_MISSING); + ASSERT_EQ(impl.ModelMetadataImpl(nullptr, &req, &res, ovms::ExecutionContext(ovms::ExecutionContext::Interface::GRPC, ovms::ExecutionContext::Method::ModelMetadata), extraMetadata), StatusCode::MODEL_VERSION_MISSING); req.Clear(); req.set_name("dummy"); req.set_version("$$"); - ASSERT_EQ(impl.ModelMetadataImpl(nullptr, &req, &res, ovms::ExecutionContext(ovms::ExecutionContext::Interface::GRPC, ovms::ExecutionContext::Method::GetModelMetadata), extraMetadata), StatusCode::MODEL_VERSION_INVALID_FORMAT); + ASSERT_EQ(impl.ModelMetadataImpl(nullptr, &req, &res, ovms::ExecutionContext(ovms::ExecutionContext::Interface::GRPC, ovms::ExecutionContext::Method::ModelMetadata), extraMetadata), StatusCode::MODEL_VERSION_INVALID_FORMAT); #ifdef _WIN32 // Unload model to allow folder delete on Windows diff --git a/src/test/get_model_metadata_response_test.cpp b/src/test/get_model_metadata_response_test.cpp deleted file mode 100644 index 9b9dca66c1..0000000000 --- a/src/test/get_model_metadata_response_test.cpp +++ /dev/null @@ -1,416 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** - -#include - -#include -#include - -#include -#include - -#include "../get_model_metadata_impl.hpp" -#include "../modelmanager.hpp" -#include "../status.hpp" -#include "mockmodelinstancechangingstates.hpp" -#include "test_models_configs.hpp" - -using ::testing::NiceMock; -using ::testing::Return; -using ::testing::ReturnRef; -using namespace rapidjson; - -class GetModelMetadataResponse : public ::testing::Test { - class MockModelInstance : public MockModelInstanceChangingStates { - public: - MockModelInstance(ov::Core& ieCore) : - MockModelInstanceChangingStates("UNUSED_NAME", UNUSED_MODEL_VERSION, ieCore) { - status = ovms::ModelVersionStatus("UNUSED_NAME", UNUSED_MODEL_VERSION, ovms::ModelVersionState::AVAILABLE); - } - - // Keeps the model in loading state forever - ovms::Status loadModel(const ovms::ModelConfig& config) override { - status.setLoading(); - return ovms::StatusCode::OK; - } - - MOCK_METHOD(const ovms::tensor_map_t&, getInputsInfo, (), (const, override)); - MOCK_METHOD(const ovms::tensor_map_t&, getOutputsInfo, (), (const, override)); - MOCK_METHOD(const std::string&, getName, (), (const, override)); - MOCK_METHOD(ovms::model_version_t, getVersion, (), (const, override)); - }; - -protected: - struct Info { - ovms::Precision precision; - ovms::shape_t shape; - }; - - using tensor_desc_map_t = std::unordered_map; - - tensor_desc_map_t inputTensors; - tensor_desc_map_t outputTensors; - ovms::tensor_map_t servableInputs; - ovms::tensor_map_t servableOutputs; - - std::string modelName = "resnet"; - ovms::model_version_t modelVersion = 23; - - std::shared_ptr> instance; - tensorflow::serving::GetModelMetadataResponse response; - std::unique_ptr ieCore; - - virtual void prepare() { - instance = std::make_shared>(*ieCore); - - inputTensors = tensor_desc_map_t({ - {"Input_FP32_1_3_224_224", { - ovms::Precision::FP32, - {1, 3, 224, 224}, - }}, - {"Input_U8_1_3_62_62", { - ovms::Precision::U8, - {1, 3, 62, 62}, - }}, - {"Input_I64_Scalar", { - ovms::Precision::I64, - {}, - }}, - }); - - outputTensors = tensor_desc_map_t({{"Output_I32_1_2000", { - ovms::Precision::I32, - {1, 2000}, - }}, - {"Output_FP32_2_20_3", { - ovms::Precision::FP32, - {2, 20, 3}, - }}, - {"Output_I64_Scalar", { - ovms::Precision::I64, - {}, - }}}); - - auto prepare = [](const tensor_desc_map_t& desc, - ovms::tensor_map_t& tensors) { - for (const auto& pair : desc) { - tensors[pair.first] = std::make_shared( - pair.first, - pair.second.precision, - pair.second.shape); - } - }; - - prepare(inputTensors, servableInputs); - prepare(outputTensors, servableOutputs); - - ON_CALL(*instance, getInputsInfo()) - .WillByDefault(ReturnRef(servableInputs)); - ON_CALL(*instance, getOutputsInfo()) - .WillByDefault(ReturnRef(servableOutputs)); - ON_CALL(*instance, getName()) - .WillByDefault(ReturnRef(modelName)); - ON_CALL(*instance, getVersion()) - .WillByDefault(Return(modelVersion)); - } - - void SetUp() override { - ieCore = std::make_unique(); - this->prepare(); - } - void TearDown() override { - ieCore.reset(); - } -}; - -class GetModelMetadataResponseBuild : public GetModelMetadataResponse { -protected: - void prepare() override { - GetModelMetadataResponse::prepare(); - ASSERT_EQ(ovms::GetModelMetadataImpl::buildResponse(instance, &response), ovms::StatusCode::OK); - } -}; - -TEST_F(GetModelMetadataResponseBuild, StringValidRespone) { - servableInputs["SingleInput"] = std::make_shared("SingleInput", ovms::Precision::STRING, ovms::Shape()); - servableOutputs["SingleOutput"] = std::make_shared("SingleOutput", ovms::Precision::STRING, ovms::Shape()); - EXPECT_EQ(ovms::GetModelMetadataImpl::buildResponse(instance, &response), ovms::StatusCode::OK); - - tensorflow::serving::SignatureDefMap def; - response.metadata().at("signature_def").UnpackTo(&def); - - const auto& inputs = ((*def.mutable_signature_def())["serving_default"]).inputs(); - const auto& outputs = ((*def.mutable_signature_def())["serving_default"]).outputs(); - EXPECT_EQ(inputs.at("SingleInput").dtype(), tensorflow::DT_STRING); - EXPECT_EQ(outputs.at("SingleOutput").dtype(), tensorflow::DT_STRING); -} - -TEST_F(GetModelMetadataResponseBuild, HasModelSpec) { - EXPECT_TRUE(response.has_model_spec()); -} - -TEST_F(GetModelMetadataResponseBuild, HasCorrectName) { - EXPECT_EQ(response.model_spec().name(), "resnet"); -} - -TEST_F(GetModelMetadataResponseBuild, HasVersion) { - EXPECT_TRUE(response.model_spec().has_version()); -} - -TEST_F(GetModelMetadataResponseBuild, HasCorrectVersion) { - EXPECT_EQ(response.model_spec().version().value(), 23); -} - -TEST_F(GetModelMetadataResponseBuild, HasOneMetadataInfo) { - EXPECT_EQ(response.metadata_size(), 1); -} - -TEST_F(GetModelMetadataResponseBuild, HasCorrectMetadataSignatureName) { - EXPECT_NE( - response.metadata().find("signature_def"), - response.metadata().end()); -} - -TEST_F(GetModelMetadataResponseBuild, HasOneSignatureDef) { - tensorflow::serving::SignatureDefMap def; - response.metadata().at("signature_def").UnpackTo(&def); - - EXPECT_EQ(def.signature_def_size(), 1); -} - -TEST_F(GetModelMetadataResponseBuild, HasCorrectSignatureDefName) { - tensorflow::serving::SignatureDefMap def; - response.metadata().at("signature_def").UnpackTo(&def); - - EXPECT_NE( - def.signature_def().find("serving_default"), - def.signature_def().end()); -} - -TEST_F(GetModelMetadataResponseBuild, HasCorrectTensorNames) { - tensorflow::serving::SignatureDefMap def; - response.metadata().at("signature_def").UnpackTo(&def); - - const auto& inputs = ((*def.mutable_signature_def())["serving_default"]).inputs(); - const auto& outputs = ((*def.mutable_signature_def())["serving_default"]).outputs(); - - EXPECT_EQ(inputs.size(), 3); - EXPECT_EQ(outputs.size(), 3); - - EXPECT_EQ( - inputs.at("Input_FP32_1_3_224_224").name(), - "Input_FP32_1_3_224_224"); - EXPECT_EQ( - inputs.at("Input_U8_1_3_62_62").name(), - "Input_U8_1_3_62_62"); - EXPECT_EQ( - inputs.at("Input_I64_Scalar").name(), - "Input_I64_Scalar"); - EXPECT_EQ( - outputs.at("Output_I32_1_2000").name(), - "Output_I32_1_2000"); - EXPECT_EQ( - outputs.at("Output_FP32_2_20_3").name(), - "Output_FP32_2_20_3"); - EXPECT_EQ( - outputs.at("Output_I64_Scalar").name(), - "Output_I64_Scalar"); -} - -TEST_F(GetModelMetadataResponseBuild, HasCorrectPrecision) { - tensorflow::serving::SignatureDefMap def; - response.metadata().at("signature_def").UnpackTo(&def); - - const auto& inputs = ((*def.mutable_signature_def())["serving_default"]).inputs(); - const auto& outputs = ((*def.mutable_signature_def())["serving_default"]).outputs(); - - EXPECT_EQ( - inputs.at("Input_FP32_1_3_224_224").dtype(), - tensorflow::DT_FLOAT); - EXPECT_EQ( - inputs.at("Input_U8_1_3_62_62").dtype(), - tensorflow::DT_UINT8); - EXPECT_EQ( - inputs.at("Input_I64_Scalar").dtype(), - tensorflow::DT_INT64); - EXPECT_EQ( - outputs.at("Output_I32_1_2000").dtype(), - tensorflow::DT_INT32); - EXPECT_EQ( - outputs.at("Output_FP32_2_20_3").dtype(), - tensorflow::DT_FLOAT); - EXPECT_EQ( - outputs.at("Output_I64_Scalar").dtype(), - tensorflow::DT_INT64); -} - -TEST_F(GetModelMetadataResponseBuild, HasCorrectShape) { - tensorflow::serving::SignatureDefMap def; - response.metadata().at("signature_def").UnpackTo(&def); - - const auto& inputs = ((*def.mutable_signature_def())["serving_default"]).inputs(); - const auto& outputs = ((*def.mutable_signature_def())["serving_default"]).outputs(); - - EXPECT_TRUE(isShapeTheSame( - inputs.at("Input_FP32_1_3_224_224").tensor_shape(), - {1, 3, 224, 224})); - EXPECT_TRUE(isShapeTheSame( - inputs.at("Input_U8_1_3_62_62").tensor_shape(), - {1, 3, 62, 62})); - EXPECT_TRUE(isShapeTheSame( - inputs.at("Input_I64_Scalar").tensor_shape(), - {})); - EXPECT_TRUE(isShapeTheSame( - outputs.at("Output_I32_1_2000").tensor_shape(), - {1, 2000})); - EXPECT_TRUE(isShapeTheSame( - outputs.at("Output_FP32_2_20_3").tensor_shape(), - {2, 20, 3})); - EXPECT_TRUE(isShapeTheSame( - outputs.at("Output_I64_Scalar").tensor_shape(), - {})); -} - -TEST_F(GetModelMetadataResponse, ModelVersionNotLoadedAnymore) { - instance->retireModel(); - EXPECT_EQ(ovms::GetModelMetadataImpl::buildResponse(instance, &response), ovms::StatusCode::MODEL_VERSION_NOT_LOADED_ANYMORE); -} - -TEST_F(GetModelMetadataResponse, ModelVersionNotLoadedYet) { - instance->loadModel(DUMMY_MODEL_CONFIG); - EXPECT_EQ(ovms::GetModelMetadataImpl::buildResponse(instance, &response), ovms::StatusCode::MODEL_VERSION_NOT_LOADED_YET); -} - -TEST_F(GetModelMetadataResponseBuild, serialize2Json) { - std::string json_output; - const tensorflow::serving::GetModelMetadataResponse* response_p = &response; - ovms::Status error_status = ovms::GetModelMetadataImpl::serializeResponse2Json(response_p, &json_output); - const char* json_array = json_output.c_str(); - Document received_doc; - received_doc.Parse(json_array); - EXPECT_TRUE(received_doc.IsObject()); - EXPECT_TRUE(received_doc.HasMember("modelSpec")); - EXPECT_TRUE(received_doc.HasMember("metadata")); -} - -class GetModelMetadataScalarResponseBuild : public GetModelMetadataResponseBuild { -protected: - void prepare() override { - GetModelMetadataResponse::prepare(); - inputTensors = tensor_desc_map_t({{"Input_Scalar", { - ovms::Precision::FP32, - {}, - }}}); - - outputTensors = tensor_desc_map_t({{"Output_Scalar", { - ovms::Precision::I32, - {}, - }}}); - - auto prepare = [](const tensor_desc_map_t& desc, - ovms::tensor_map_t& tensors) { - tensors.clear(); - for (const auto& pair : desc) { - tensors[pair.first] = std::make_shared( - pair.first, - pair.second.precision, - pair.second.shape); - } - }; - - prepare(inputTensors, servableInputs); - prepare(outputTensors, servableOutputs); - ASSERT_EQ(ovms::GetModelMetadataImpl::buildResponse(instance, &response), ovms::StatusCode::OK); - } -}; - -TEST_F(GetModelMetadataScalarResponseBuild, serializeScalarToJson) { - std::string json_output; - const tensorflow::serving::GetModelMetadataResponse* response_p = &response; - ovms::Status error_status = ovms::GetModelMetadataImpl::serializeResponse2Json(response_p, &json_output); - EXPECT_EQ(error_status, ovms::StatusCode::OK); - EXPECT_EQ(json_output, R"({ - "modelSpec": { - "name": "resnet", - "signatureName": "", - "version": "23" - }, - "metadata": { - "signature_def": { - "@type": "type.googleapis.com/tensorflow.serving.SignatureDefMap", - "signatureDef": { - "serving_default": { - "inputs": { - "Input_Scalar": { - "dtype": "DT_FLOAT", - "tensorShape": { - "dim": [], - "unknownRank": false - }, - "name": "Input_Scalar" - } - }, - "outputs": { - "Output_Scalar": { - "dtype": "DT_INT32", - "tensorShape": { - "dim": [], - "unknownRank": false - }, - "name": "Output_Scalar" - } - }, - "methodName": "", - "defaults": {} - } - } - } - } -} -)"); -} - -TEST(RESTGetModelMetadataResponse, createGrpcRequestVersionSet) { - std::string model_name = "dummy"; - std::optional model_version = 1; - tensorflow::serving::GetModelMetadataRequest request_grpc; - tensorflow::serving::GetModelMetadataRequest* request_p = &request_grpc; - ovms::Status status = ovms::GetModelMetadataImpl::createGrpcRequest(model_name, model_version, request_p); - bool has_requested_version = request_p->model_spec().has_version(); - auto requested_version = request_p->model_spec().version().value(); - std::string metadata_field = request_p->metadata_field(0); - std::string requested_model_name = request_p->model_spec().name(); - ASSERT_EQ(status, ovms::StatusCode::OK); - EXPECT_EQ(has_requested_version, true); - EXPECT_EQ(requested_version, 1); - EXPECT_EQ(requested_model_name, "dummy"); - EXPECT_EQ(metadata_field, "signature_def"); -} - -TEST(RESTGetModelMetadataResponse, createGrpcRequestNoVersion) { - std::string model_name = "dummy"; - std::optional model_version; - tensorflow::serving::GetModelMetadataRequest request_grpc; - tensorflow::serving::GetModelMetadataRequest* request_p = &request_grpc; - ovms::Status status = ovms::GetModelMetadataImpl::createGrpcRequest(model_name, model_version, request_p); - bool has_requested_version = request_p->model_spec().has_version(); - std::string metadata_field = request_p->metadata_field(0); - std::string requested_model_name = request_p->model_spec().name(); - ASSERT_EQ(status, ovms::StatusCode::OK); - EXPECT_EQ(has_requested_version, false); - EXPECT_EQ(requested_model_name, "dummy"); - EXPECT_EQ(metadata_field, "signature_def"); -} diff --git a/src/test/get_model_metadata_signature_test.cpp b/src/test/get_model_metadata_signature_test.cpp deleted file mode 100644 index 22a77ca346..0000000000 --- a/src/test/get_model_metadata_signature_test.cpp +++ /dev/null @@ -1,103 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** - -#include - -#include - -#include "../get_model_metadata_impl.hpp" -#include "test_utils.hpp" - -class GetModelMetadataSignature : public ::testing::Test { - struct Info { - ovms::Precision precision; - ovms::shape_t shape; - }; - - using tensor_desc_map_t = std::unordered_map; - - tensor_desc_map_t tensors; - -protected: - ovms::tensor_map_t inputs; - ovms::proto_signature_map_t signature; - - void SetUp() override { - tensors = tensor_desc_map_t{ - {"Input_FP32_1_3_224_224", { - ovms::Precision::FP32, - {1, 3, 224, 224}, - }}, - {"Input_I64_1_6_128_128_16", { - ovms::Precision::I64, - {1, 6, 128, 128, 16}, - }}, - {"Input_U8_Scalar", { - ovms::Precision::U8, - {}, - }}, - }; - - for (const auto& pair : tensors) { - inputs[pair.first] = std::make_shared( - pair.first, - pair.second.precision, - pair.second.shape); - } - } -}; - -TEST_F(GetModelMetadataSignature, ConvertCorrectNumberOfInputs) { - ovms::GetModelMetadataImpl::convert(inputs, &signature); - EXPECT_EQ(signature.size(), 3); -} - -TEST_F(GetModelMetadataSignature, ConvertInputsExist) { - ovms::GetModelMetadataImpl::convert(inputs, &signature); - EXPECT_NE(signature.find("Input_FP32_1_3_224_224"), signature.end()); - EXPECT_NE(signature.find("Input_I64_1_6_128_128_16"), signature.end()); - EXPECT_NE(signature.find("Input_U8_Scalar"), signature.end()); -} - -TEST_F(GetModelMetadataSignature, ConvertCorrectInputNames) { - ovms::GetModelMetadataImpl::convert(inputs, &signature); - EXPECT_EQ(signature["Input_FP32_1_3_224_224"].name(), "Input_FP32_1_3_224_224"); - EXPECT_EQ(signature["Input_I64_1_6_128_128_16"].name(), "Input_I64_1_6_128_128_16"); - EXPECT_EQ(signature["Input_U8_Scalar"].name(), "Input_U8_Scalar"); -} - -TEST_F(GetModelMetadataSignature, ConvertCorrectPrecision) { - ovms::GetModelMetadataImpl::convert(inputs, &signature); - EXPECT_EQ(signature["Input_FP32_1_3_224_224"].dtype(), tensorflow::DT_FLOAT); - EXPECT_EQ(signature["Input_I64_1_6_128_128_16"].dtype(), tensorflow::DT_INT64); - EXPECT_EQ(signature["Input_U8_Scalar"].dtype(), tensorflow::DT_UINT8); -} - -TEST_F(GetModelMetadataSignature, ConvertCorrectTensorShape) { - ovms::GetModelMetadataImpl::convert(inputs, &signature); - - EXPECT_TRUE(isShapeTheSame( - signature["Input_FP32_1_3_224_224"].tensor_shape(), - {1, 3, 224, 224})); - - EXPECT_TRUE(isShapeTheSame( - signature["Input_I64_1_6_128_128_16"].tensor_shape(), - {1, 6, 128, 128, 16})); - - EXPECT_TRUE(isShapeTheSame( - signature["Input_U8_Scalar"].tensor_shape(), - {})); -} diff --git a/src/test/get_model_metadata_validation_test.cpp b/src/test/get_model_metadata_validation_test.cpp deleted file mode 100644 index 1a4512d701..0000000000 --- a/src/test/get_model_metadata_validation_test.cpp +++ /dev/null @@ -1,65 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** - -#include - -#include "../get_model_metadata_impl.hpp" -#include "../status.hpp" - -class GetModelMetadataValidation : public ::testing::Test { -protected: - tensorflow::serving::GetModelMetadataRequest request; - - void SetUp() override { - request.Clear(); - request.mutable_model_spec()->set_name("ResNet50"); - request.mutable_metadata_field()->Add("signature_def"); - } -}; - -TEST_F(GetModelMetadataValidation, ValidRequestWithNoVersionSpecified) { - auto status = ovms::GetModelMetadataImpl::validate(&request); - EXPECT_TRUE(status.ok()); -} - -TEST_F(GetModelMetadataValidation, ValidRequestWithVersionSpecified) { - request.mutable_model_spec()->mutable_version()->set_value(170); - auto status = ovms::GetModelMetadataImpl::validate(&request); - EXPECT_TRUE(status.ok()); -} - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-result" - -TEST_F(GetModelMetadataValidation, RequestMissingModelSpec) { - request.release_model_spec(); - auto status = ovms::GetModelMetadataImpl::validate(&request); - EXPECT_EQ(status, ovms::StatusCode::MODEL_SPEC_MISSING); -} - -#pragma GCC diagnostic pop - -TEST_F(GetModelMetadataValidation, RequestMissingMetadataField) { - request.mutable_metadata_field()->RemoveLast(); - auto status = ovms::GetModelMetadataImpl::validate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_SIGNATURE_DEF); -} - -TEST_F(GetModelMetadataValidation, RequestMetadataInvalidSignatureName) { - request.mutable_metadata_field()->at(0) = "wrong_signature_name"; - auto status = ovms::GetModelMetadataImpl::validate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_SIGNATURE_DEF); -} diff --git a/src/test/get_pipeline_metadata_response_test.cpp b/src/test/get_pipeline_metadata_response_test.cpp deleted file mode 100644 index ce65a1f55a..0000000000 --- a/src/test/get_pipeline_metadata_response_test.cpp +++ /dev/null @@ -1,296 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** - -#include -#include -#include - -#include -#include -#include - -#include "../dags/pipelinedefinition.hpp" -#include "../get_model_metadata_impl.hpp" -#include "../model_metric_reporter.hpp" - -#include "constructor_enabled_model_manager.hpp" -#include "test_utils.hpp" - -using namespace ovms; -using namespace rapidjson; - -class GetPipelineMetadataResponse : public ::testing::Test { -protected: - class MockPipelineDefinitionGetInputsOutputsInfo : public PipelineDefinition { - Status status = StatusCode::OK; - - public: - MockPipelineDefinitionGetInputsOutputsInfo() : - PipelineDefinition("pipeline_name", {}, {}) { - PipelineDefinition::status.handle(ValidationPassedEvent()); - } - - void mockMetadata(const tensor_map_t& inputsInfo, const tensor_map_t& outputsInfo) { - this->inputsInfo = inputsInfo; - this->outputsInfo = outputsInfo; - } - - void mockStatus(Status status) { - this->status = status; - } - PipelineDefinitionStatus& getPipelineDefinitionStatus() { - return PipelineDefinition::status; - } - }; - - MockPipelineDefinitionGetInputsOutputsInfo pipelineDefinition; - tensorflow::serving::GetModelMetadataResponse response; - ConstructorEnabledModelManager manager; - - virtual void prepare() { - pipelineDefinition.mockMetadata({{"Input_FP32_1_3_224_224", - std::make_shared("Input_FP32_1_3_224_224", ovms::Precision::FP32, ovms::Shape{1, 3, 224, 224})}, - {"Input_U8_1_3_62_62", - std::make_shared("Input_U8_1_3_62_62", ovms::Precision::U8, ovms::Shape{1, 3, 62, 62})}, - {"Input_Unspecified", - TensorInfo::getUnspecifiedTensorInfo()}}, - {{"Output_I32_1_2000", - std::make_shared("Output_I32_1_2000", ovms::Precision::I32, ovms::Shape{1, 2000})}, - {"Output_FP32_2_20_3", - std::make_shared("Output_FP32_2_20_3", ovms::Precision::FP32, ovms::Shape{2, 20, 3})}, - {"Output_Unspecified", - TensorInfo::getUnspecifiedTensorInfo()}}); - } - - void SetUp() override { - this->prepare(); - } -}; - -class GetPipelineMetadataResponseBuild : public GetPipelineMetadataResponse { -protected: - void prepare() override { - GetPipelineMetadataResponse::prepare(); - ASSERT_EQ(ovms::GetModelMetadataImpl::buildResponse(pipelineDefinition, &response, manager), ovms::StatusCode::OK); - } -}; - -TEST_F(GetPipelineMetadataResponseBuild, HasModelSpec) { - EXPECT_TRUE(response.has_model_spec()); -} - -TEST_F(GetPipelineMetadataResponseBuild, HasCorrectName) { - EXPECT_EQ(response.model_spec().name(), "pipeline_name"); -} - -TEST_F(GetPipelineMetadataResponseBuild, HasVersion) { - EXPECT_TRUE(response.model_spec().has_version()); -} - -TEST_F(GetPipelineMetadataResponseBuild, HasCorrectVersion) { - EXPECT_EQ(response.model_spec().version().value(), 1); -} - -TEST_F(GetPipelineMetadataResponseBuild, HasOneMetadataInfo) { - EXPECT_EQ(response.metadata_size(), 1); -} - -TEST_F(GetPipelineMetadataResponseBuild, HasCorrectMetadataSignatureName) { - EXPECT_NE( - response.metadata().find("signature_def"), - response.metadata().end()); -} - -TEST_F(GetPipelineMetadataResponseBuild, HasOneSignatureDef) { - tensorflow::serving::SignatureDefMap def; - response.metadata().at("signature_def").UnpackTo(&def); - - EXPECT_EQ(def.signature_def_size(), 1); -} - -TEST_F(GetPipelineMetadataResponseBuild, HasCorrectSignatureDefName) { - tensorflow::serving::SignatureDefMap def; - response.metadata().at("signature_def").UnpackTo(&def); - - EXPECT_NE( - def.signature_def().find("serving_default"), - def.signature_def().end()); -} - -TEST_F(GetPipelineMetadataResponseBuild, HasCorrectTensorNames) { - tensorflow::serving::SignatureDefMap def; - response.metadata().at("signature_def").UnpackTo(&def); - - const auto& inputs = ((*def.mutable_signature_def())["serving_default"]).inputs(); - const auto& outputs = ((*def.mutable_signature_def())["serving_default"]).outputs(); - - EXPECT_EQ(inputs.size(), 3); - EXPECT_EQ(outputs.size(), 3); - - EXPECT_EQ( - inputs.at("Input_FP32_1_3_224_224").name(), - "Input_FP32_1_3_224_224"); - EXPECT_EQ( - inputs.at("Input_U8_1_3_62_62").name(), - "Input_U8_1_3_62_62"); - EXPECT_EQ( - inputs.at("Input_Unspecified").name(), - "Input_Unspecified"); - EXPECT_EQ( - outputs.at("Output_I32_1_2000").name(), - "Output_I32_1_2000"); - EXPECT_EQ( - outputs.at("Output_FP32_2_20_3").name(), - "Output_FP32_2_20_3"); - EXPECT_EQ( - outputs.at("Output_Unspecified").name(), - "Output_Unspecified"); -} - -TEST_F(GetPipelineMetadataResponseBuild, HasCorrectPrecision) { - tensorflow::serving::SignatureDefMap def; - response.metadata().at("signature_def").UnpackTo(&def); - - const auto& inputs = ((*def.mutable_signature_def())["serving_default"]).inputs(); - const auto& outputs = ((*def.mutable_signature_def())["serving_default"]).outputs(); - - EXPECT_EQ( - inputs.at("Input_FP32_1_3_224_224").dtype(), - tensorflow::DT_FLOAT); - EXPECT_EQ( - inputs.at("Input_U8_1_3_62_62").dtype(), - tensorflow::DT_UINT8); - EXPECT_EQ( - inputs.at("Input_Unspecified").dtype(), - tensorflow::DT_INVALID); - EXPECT_EQ( - outputs.at("Output_I32_1_2000").dtype(), - tensorflow::DT_INT32); - EXPECT_EQ( - outputs.at("Output_FP32_2_20_3").dtype(), - tensorflow::DT_FLOAT); - EXPECT_EQ( - outputs.at("Output_Unspecified").dtype(), - tensorflow::DT_INVALID); -} - -TEST_F(GetPipelineMetadataResponseBuild, HasCorrectShape) { - tensorflow::serving::SignatureDefMap def; - response.metadata().at("signature_def").UnpackTo(&def); - - const auto& inputs = ((*def.mutable_signature_def())["serving_default"]).inputs(); - const auto& outputs = ((*def.mutable_signature_def())["serving_default"]).outputs(); - - EXPECT_TRUE(isShapeTheSame( - inputs.at("Input_FP32_1_3_224_224").tensor_shape(), - {1, 3, 224, 224})); - EXPECT_TRUE(isShapeTheSame( - inputs.at("Input_U8_1_3_62_62").tensor_shape(), - {1, 3, 62, 62})); - EXPECT_TRUE(isShapeTheSame( - inputs.at("Input_Unspecified").tensor_shape(), - {})); - EXPECT_TRUE(isShapeTheSame( - outputs.at("Output_I32_1_2000").tensor_shape(), - {1, 2000})); - EXPECT_TRUE(isShapeTheSame( - outputs.at("Output_FP32_2_20_3").tensor_shape(), - {2, 20, 3})); - EXPECT_TRUE(isShapeTheSame( - outputs.at("Output_Unspecified").tensor_shape(), - {})); -} - -TEST_F(GetPipelineMetadataResponse, ModelVersionNotLoadedAnymoreButPipelineNotReloadedYet) { - pipelineDefinition.mockStatus(StatusCode::MODEL_VERSION_NOT_LOADED_ANYMORE); - EXPECT_EQ(ovms::GetModelMetadataImpl::buildResponse(pipelineDefinition, &response, manager), ovms::StatusCode::OK); -} - -TEST_F(GetPipelineMetadataResponse, ModelVersionNotLoadedYet) { - pipelineDefinition.mockStatus(StatusCode::MODEL_VERSION_NOT_LOADED_YET); - EXPECT_EQ(ovms::GetModelMetadataImpl::buildResponse(pipelineDefinition, &response, manager), ovms::StatusCode::OK); -} - -TEST_F(GetPipelineMetadataResponse, PipelineNotLoadedAnymore) { - this->pipelineDefinition.getPipelineDefinitionStatus().handle(RetireEvent()); - auto status = ovms::GetModelMetadataImpl::buildResponse(pipelineDefinition, &response, manager); - ASSERT_EQ(status, ovms::StatusCode::PIPELINE_DEFINITION_NOT_LOADED_ANYMORE) << status.string(); -} - -TEST_F(GetPipelineMetadataResponse, PipelineNotLoadedYet) { - this->pipelineDefinition.getPipelineDefinitionStatus().handle(UsedModelChangedEvent()); - this->pipelineDefinition.getPipelineDefinitionStatus().handle(ValidationFailedEvent()); - auto status = ovms::GetModelMetadataImpl::buildResponse(pipelineDefinition, &response, manager); - ASSERT_EQ(status, ovms::StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET) << status.string(); - this->pipelineDefinition.getPipelineDefinitionStatus().handle(UsedModelChangedEvent()); - status = ovms::GetModelMetadataImpl::buildResponse(pipelineDefinition, &response, manager); - ASSERT_EQ(ovms::GetModelMetadataImpl::buildResponse(pipelineDefinition, &response, manager), ovms::StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET); -} - -TEST_F(GetPipelineMetadataResponse, PipelineAvailableOrAvailableRequiringRevalidation) { - this->pipelineDefinition.getPipelineDefinitionStatus().handle(UsedModelChangedEvent()); - EXPECT_EQ(ovms::GetModelMetadataImpl::buildResponse(pipelineDefinition, &response, manager), ovms::StatusCode::OK); -} - -TEST_F(GetPipelineMetadataResponseBuild, serialize2Json) { - std::string json_output; - const tensorflow::serving::GetModelMetadataResponse* response_p = &response; - ovms::Status error_status = ovms::GetModelMetadataImpl::serializeResponse2Json(response_p, &json_output); - const char* json_array = json_output.c_str(); - Document received_doc; - received_doc.Parse(json_array); - EXPECT_TRUE(received_doc.IsObject()); - EXPECT_TRUE(received_doc.HasMember("modelSpec")); - EXPECT_TRUE(received_doc.HasMember("metadata")); -} - -class GetPipelineMetadataResponseBuildWithDynamicAndRangeShapes : public GetPipelineMetadataResponseBuild { -protected: - void prepare() override { - GetPipelineMetadataResponse::prepare(); - pipelineDefinition.mockMetadata( - {{"Input_FP32_1_-1_224_224", - std::make_shared("Input_FP32_1_-1_224_224", ovms::Precision::FP32, ovms::Shape{1, ovms::Dimension::any(), 224, 224})}, - {"Input_U8_1_3_62:92_62:92", - std::make_shared("Input_U8_1_3_62:92_62:92", ovms::Precision::U8, ovms::Shape{1, 3, {62, 92}, ovms::Dimension::any()})}}, - {{"Output_I32_1_-1", - std::make_shared("Output_I32_1_-1", ovms::Precision::I32, ovms::Shape{1, ovms::Dimension::any()})}, - {"Output_FP32_1_224:294_224:294_3", - std::make_shared("Output_FP32_1_224:294_224:294_3", ovms::Precision::FP32, ovms::Shape{1, {224, 294}, {224, 294}, 3})}}); - ASSERT_EQ(ovms::GetModelMetadataImpl::buildResponse(pipelineDefinition, &response, manager), ovms::StatusCode::OK); - } -}; - -TEST_F(GetPipelineMetadataResponseBuildWithDynamicAndRangeShapes, HandleDynamicAndRangeShapes) { - tensorflow::serving::SignatureDefMap def; - response.metadata().at("signature_def").UnpackTo(&def); - - const auto& inputs = ((*def.mutable_signature_def())["serving_default"]).inputs(); - const auto& outputs = ((*def.mutable_signature_def())["serving_default"]).outputs(); - - EXPECT_TRUE(isShapeTheSame( - inputs.at("Input_FP32_1_-1_224_224").tensor_shape(), - {1, -1, 224, 224})); - EXPECT_TRUE(isShapeTheSame( - inputs.at("Input_U8_1_3_62:92_62:92").tensor_shape(), - {1, 3, -1, -1})); - EXPECT_TRUE(isShapeTheSame( - outputs.at("Output_I32_1_-1").tensor_shape(), - {1, -1})); - EXPECT_TRUE(isShapeTheSame( - outputs.at("Output_FP32_1_224:294_224:294_3").tensor_shape(), - {1, -1, -1, 3})); -} diff --git a/src/test/kfs_rest_test.cpp b/src/test/kfs_rest_test.cpp index 7f4e271988..ea58b0dd7b 100644 --- a/src/test/kfs_rest_test.cpp +++ b/src/test/kfs_rest_test.cpp @@ -408,14 +408,14 @@ TEST_F(HttpRestApiHandlerTest, MetricsParameters) { } TEST_F(HttpRestApiHandlerTest, GetModelMetadataWithLongVersion) { - std::string request = "/v1/models/dummy/versions/72487667423532349025128558057"; + std::string request = "/v2/models/dummy/versions/72487667423532349025128558057"; ovms::HttpRequestComponents comp; ASSERT_EQ(handler->parseRequestComponents(comp, "GET", request), StatusCode::MODEL_VERSION_MISSING); } TEST_F(HttpRestApiHandlerTest, GetModelMetadataWithEscapedPath) { - std::string request = "/v1/models/..iO!.0?E*/versions/1/metadata"; + std::string request = "/v2/models/..iO!.0?E*/versions/1"; ovms::HttpRequestComponents comp; ASSERT_EQ(handler->parseRequestComponents(comp, "GET", request), StatusCode::OK); diff --git a/src/test/mediapipe_framework_test.cpp b/src/test/mediapipe_framework_test.cpp index 8b05f194d3..9b1613529b 100644 --- a/src/test/mediapipe_framework_test.cpp +++ b/src/test/mediapipe_framework_test.cpp @@ -40,13 +40,14 @@ #include "mediapipe/framework/port/parse_text_proto.h" #include "src/metrics/metric_config.hpp" #include "src/metrics/metric_module.hpp" -#include "../model_service.hpp" #include "../precision.hpp" #include "../servablemanagermodule.hpp" #include "../server.hpp" #include "../shape.hpp" #include "../stringutils.hpp" -#include "../tfs_frontend/tfs_utils.hpp" +#include "src/tensorflow_type_utils.hpp" +#include "src/timer.hpp" +#include "constructor_enabled_model_manager.hpp" #include "c_api_test_utils.hpp" #include "environment.hpp" #include "test_utils.hpp" diff --git a/src/test/mediapipeflow_test.cpp b/src/test/mediapipeflow_test.cpp index 459a0d5115..a0f7fe5e69 100644 --- a/src/test/mediapipeflow_test.cpp +++ b/src/test/mediapipeflow_test.cpp @@ -52,14 +52,13 @@ #include "src/metrics/metric_config.hpp" #include "src/metrics/metric_module.hpp" #include "../model.hpp" -#include "../model_service.hpp" #include "../ovms_exit_codes.hpp" #include "../precision.hpp" #include "../servablemanagermodule.hpp" #include "../server.hpp" #include "../shape.hpp" #include "../stringutils.hpp" -#include "../tfs_frontend/tfs_utils.hpp" +#include "src/tensorflow_type_utils.hpp" #include "constructor_enabled_model_manager.hpp" #include "c_api_test_utils.hpp" #include "mediapipe/framework/formats/image_frame.h" diff --git a/src/test/metric_config_test.cpp b/src/test/metric_config_test.cpp index 23184bdf73..450530fb6d 100644 --- a/src/test/metric_config_test.cpp +++ b/src/test/metric_config_test.cpp @@ -494,30 +494,30 @@ class ModelMetricReporterTest : public ::testing::Test {}; TEST_F(ModelMetricReporterTest, MetricReporterConstructorTest) { MetricRegistry registry; ModelMetricReporter reporter1(nullptr, nullptr, "example_pipeline_name", 1); - ASSERT_EQ(reporter1.requestFailGrpcGetModelMetadata, nullptr); + ASSERT_EQ(reporter1.requestFailGrpcModelMetadata, nullptr); ModelMetricReporter reporter2(nullptr, ®istry, "example_pipeline_name", 1); auto metrics = registry.collect(); ASSERT_EQ(metrics, ""); - ASSERT_EQ(reporter2.requestFailGrpcGetModelMetadata, nullptr); + ASSERT_EQ(reporter2.requestFailGrpcModelMetadata, nullptr); MetricConfig metricConfig; ModelMetricReporter reporter3(&metricConfig, ®istry, "example_pipeline_name", 1); metrics = registry.collect(); ASSERT_EQ(metrics, ""); - ASSERT_EQ(reporter3.requestFailGrpcGetModelMetadata, nullptr); + ASSERT_EQ(reporter3.requestFailGrpcModelMetadata, nullptr); metricConfig.setDefaultMetricsTo(true); ModelMetricReporter reporter4(&metricConfig, ®istry, "example_pipeline_name", 1); metrics = registry.collect(); ASSERT_EQ(metrics, ""); - ASSERT_EQ(reporter4.requestFailGrpcGetModelMetadata, nullptr); + ASSERT_EQ(reporter4.requestFailGrpcModelMetadata, nullptr); metricConfig.metricsEnabled = true; ModelMetricReporter reporter5(&metricConfig, ®istry, "example_pipeline_name", 1); metrics = registry.collect(); ASSERT_NE(metrics, ""); - ASSERT_TRUE(reporter5.requestFailGrpcGetModelMetadata != nullptr); + ASSERT_TRUE(reporter5.requestFailGrpcModelMetadata != nullptr); } class MetricsCli : public ::testing::Test { diff --git a/src/test/metrics_flow_test.cpp b/src/test/metrics_flow_test.cpp index 61349c8b64..43b42c15aa 100644 --- a/src/test/metrics_flow_test.cpp +++ b/src/test/metrics_flow_test.cpp @@ -27,14 +27,11 @@ #include #include "../config.hpp" -#include "../get_model_metadata_impl.hpp" #include "../http_rest_api_handler.hpp" #include "../kfs_frontend/kfs_grpc_inference_service.hpp" #include "src/metrics/metric_config.hpp" #include "src/metrics/metric_module.hpp" -#include "../model_service.hpp" #include "../precision.hpp" -#include "../prediction_service.hpp" #include "../servablemanagermodule.hpp" #include "../server.hpp" #include "../shape.hpp" @@ -54,32 +51,16 @@ using testing::Not; // This checks for counter to be present with exact value and other remaining metrics of the family to be 0. static void checkRequestsCounter(const std::string& collectedMetricData, const std::string& metricName, const std::string& endpointName, std::optional endpointVersion, const std::string& interfaceName, const std::string& method, const std::string& api, int value) { for (std::string _interface : std::set{"gRPC", "REST"}) { - for (std::string _api : std::set{"TensorFlowServing", "KServe"}) { - if (_api == "KServe") { - for (std::string _method : std::set{"ModelInfer", "ModelMetadata", "ModelReady"}) { - std::stringstream ss; - ss << metricName << "{api=\"" << _api << "\",interface=\"" << _interface << "\",method=\"" << _method << "\",name=\"" << endpointName << "\""; - if (_method != "ModelReady") { - ss << ",version=\"" << endpointVersion.value() << "\""; - } - ss << "}"; - int expectedValue = interfaceName == _interface && method == _method && api == _api ? value : 0; - ss << " " << expectedValue << "\n"; - ASSERT_THAT(collectedMetricData, HasSubstr(ss.str())); - } - } else { - for (std::string _method : std::set{"Predict", "GetModelMetadata", "GetModelStatus"}) { - std::stringstream ss; - ss << metricName << "{api=\"" << _api << "\",interface=\"" << _interface << "\",method=\"" << _method << "\",name=\"" << endpointName << "\""; - if (_method != "GetModelStatus") { - ss << ",version=\"" << endpointVersion.value() << "\""; - } - ss << "}"; - int expectedValue = interfaceName == _interface && method == _method && api == _api ? value : 0; - ss << " " << expectedValue << "\n"; - ASSERT_THAT(collectedMetricData, HasSubstr(ss.str())); - } + for (std::string _method : std::set{"ModelInfer", "ModelMetadata", "ModelReady"}) { + std::stringstream ss; + ss << metricName << "{api=\"KServe\",interface=\"" << _interface << "\",method=\"" << _method << "\",name=\"" << endpointName << "\""; + if (_method != "ModelReady") { + ss << ",version=\"" << endpointVersion.value() << "\""; } + ss << "}"; + int expectedValue = interfaceName == _interface && method == _method && api == "KServe" ? value : 0; + ss << " " << expectedValue << "\n"; + ASSERT_THAT(collectedMetricData, HasSubstr(ss.str())); } } } @@ -215,123 +196,6 @@ class MetricFlowTest : public TestWithTempDir { } }; -TEST_F(MetricFlowTest, GrpcPredict) { - PredictionServiceImpl impl(server); - tensorflow::serving::PredictRequest request; - tensorflow::serving::PredictResponse response; - - // Successful single model calls - for (int i = 0; i < numberOfSuccessRequests; i++) { - request.Clear(); - response.Clear(); - request.mutable_model_spec()->mutable_name()->assign(modelName); - inputs_info_t inputsMeta{{DUMMY_MODEL_INPUT_NAME, {DUMMY_MODEL_SHAPE, correctPrecision}}}; - preparePredictRequest(request, inputsMeta); - ASSERT_EQ(impl.Predict(nullptr, &request, &response).error_code(), grpc::StatusCode::OK); - } - // Failed single model calls - for (int i = 0; i < numberOfFailedRequests; i++) { - request.Clear(); - response.Clear(); - request.mutable_model_spec()->mutable_name()->assign(modelName); - inputs_info_t inputsMeta{{DUMMY_MODEL_INPUT_NAME, {DUMMY_MODEL_SHAPE, wrongPrecision}}}; - preparePredictRequest(request, inputsMeta); - ASSERT_EQ(impl.Predict(nullptr, &request, &response).error_code(), grpc::StatusCode::INVALID_ARGUMENT); - } - - // Successful DAG calls - for (int i = 0; i < numberOfSuccessRequests; i++) { - request.Clear(); - response.Clear(); - request.mutable_model_spec()->mutable_name()->assign(dagName); - inputs_info_t inputsMeta{{DUMMY_MODEL_INPUT_NAME, {ovms::signed_shape_t{dynamicBatch, 1, DUMMY_MODEL_INPUT_SIZE}, correctPrecision}}}; - preparePredictRequest(request, inputsMeta); - ASSERT_EQ(impl.Predict(nullptr, &request, &response).error_code(), grpc::StatusCode::OK); - } - - // Failed DAG calls - for (int i = 0; i < numberOfFailedRequests; i++) { - request.Clear(); - response.Clear(); - request.mutable_model_spec()->mutable_name()->assign(dagName); - inputs_info_t inputsMeta{{DUMMY_MODEL_INPUT_NAME, {ovms::signed_shape_t{dynamicBatch, 1, DUMMY_MODEL_INPUT_SIZE}, wrongPrecision}}}; - preparePredictRequest(request, inputsMeta); - ASSERT_EQ(impl.Predict(nullptr, &request, &response).error_code(), grpc::StatusCode::INVALID_ARGUMENT); - } - - // ovms_requests_success - checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, modelName, 1, "gRPC", "Predict", "TensorFlowServing", dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests); // ran by demultiplexer + real request - checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, dagName, 1, "gRPC", "Predict", "TensorFlowServing", numberOfSuccessRequests); // ran by real request - - checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_FAIL, modelName, 1, "gRPC", "Predict", "TensorFlowServing", numberOfFailedRequests); // ran by real request - checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_FAIL, dagName, 1, "gRPC", "Predict", "TensorFlowServing", numberOfFailedRequests); // ran by real request - - EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_REQUEST_TIME + std::string{"_count{interface=\"gRPC\",name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(numberOfSuccessRequests))); - EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_REQUEST_TIME + std::string{"_count{interface=\"gRPC\",name=\""} + dagName + std::string{"\",version=\"1\"} "} + std::to_string(numberOfSuccessRequests))); - EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_REQUEST_TIME + std::string{"_count{interface=\"REST\",name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(0))); - EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_REQUEST_TIME + std::string{"_count{interface=\"REST\",name=\""} + dagName + std::string{"\",version=\"1\"} "} + std::to_string(0))); - - EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_INFERENCE_TIME + std::string{"_count{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests))); - EXPECT_THAT(server.collect(), Not(HasSubstr(METRIC_NAME_INFERENCE_TIME + std::string{"_count{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); - - EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_WAIT_FOR_INFER_REQ_TIME + std::string{"_count{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests))); - EXPECT_THAT(server.collect(), Not(HasSubstr(METRIC_NAME_WAIT_FOR_INFER_REQ_TIME + std::string{"_count{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); - - EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_STREAMS + std::string{"{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(4))); - EXPECT_THAT(server.collect(), Not(HasSubstr(METRIC_NAME_STREAMS + std::string{"{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); - - EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_INFER_REQ_QUEUE_SIZE + std::string{"{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(2))); - EXPECT_THAT(server.collect(), Not(HasSubstr(METRIC_NAME_INFER_REQ_QUEUE_SIZE + std::string{"{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); -} - -TEST_F(MetricFlowTest, GrpcGetModelMetadata) { - PredictionServiceImpl impl(server); - tensorflow::serving::GetModelMetadataRequest request; - tensorflow::serving::GetModelMetadataResponse response; - - for (int i = 0; i < numberOfSuccessRequests; i++) { - request.Clear(); - response.Clear(); - request.mutable_model_spec()->mutable_name()->assign(modelName); - request.add_metadata_field("signature_def"); - ASSERT_EQ(impl.GetModelMetadata(nullptr, &request, &response).error_code(), grpc::StatusCode::OK); - } - - for (int i = 0; i < numberOfSuccessRequests; i++) { - request.Clear(); - response.Clear(); - request.mutable_model_spec()->mutable_name()->assign(dagName); - request.add_metadata_field("signature_def"); - ASSERT_EQ(impl.GetModelMetadata(nullptr, &request, &response).error_code(), grpc::StatusCode::OK); - } - - checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, modelName, 1, "gRPC", "GetModelMetadata", "TensorFlowServing", numberOfSuccessRequests); // ran by real request - checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, dagName, 1, "gRPC", "GetModelMetadata", "TensorFlowServing", numberOfSuccessRequests); // ran by real request -} - -TEST_F(MetricFlowTest, GrpcGetModelStatus) { - ModelServiceImpl impl(server); - tensorflow::serving::GetModelStatusRequest request; - tensorflow::serving::GetModelStatusResponse response; - - for (int i = 0; i < numberOfSuccessRequests; i++) { - request.Clear(); - response.Clear(); - request.mutable_model_spec()->mutable_name()->assign(modelName); - ASSERT_EQ(impl.GetModelStatus(nullptr, &request, &response).error_code(), grpc::StatusCode::OK); - } - - for (int i = 0; i < numberOfSuccessRequests; i++) { - request.Clear(); - response.Clear(); - request.mutable_model_spec()->mutable_name()->assign(dagName); - ASSERT_EQ(impl.GetModelStatus(nullptr, &request, &response).error_code(), grpc::StatusCode::OK); - } - - checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, modelName, 1, "gRPC", "GetModelStatus", "TensorFlowServing", numberOfSuccessRequests); // ran by real request - checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, dagName, 1, "gRPC", "GetModelStatus", "TensorFlowServing", numberOfSuccessRequests); // ran by real request -} - TEST_F(MetricFlowTest, GrpcModelInfer) { KFSInferenceServiceImpl impl(server); ::KFSRequest request; @@ -575,91 +439,6 @@ TEST_F(MetricFlowTest, GrpcModelReady) { #endif } -TEST_F(MetricFlowTest, RestPredict) { - HttpRestApiHandler handler(server, 0); - - for (int i = 0; i < numberOfSuccessRequests; i++) { - std::string request = R"({"signature_name": "serving_default", "instances": [[1,2,3,4,5,6,7,8,9,10]]})"; - std::string response; - ASSERT_EQ(handler.processPredictRequest(modelName, modelVersion, modelVersionLabel, request, &response), ovms::StatusCode::OK); - } - - for (int i = 0; i < numberOfFailedRequests; i++) { - std::string request = R"({"signature_name": "serving_default", "instances": [[1,2,3,4,5,6,7,8,9]]})"; - std::string response; - ASSERT_EQ(handler.processPredictRequest(modelName, modelVersion, modelVersionLabel, request, &response), ovms::StatusCode::INVALID_SHAPE); - } - - for (int i = 0; i < numberOfSuccessRequests; i++) { - std::string request = R"({"signature_name": "serving_default", "instances": [[[1,2,3,4,5,6,7,8,9,10]],[[1,2,3,4,5,6,7,8,9,10]],[[1,2,3,4,5,6,7,8,9,10]]]})"; - std::string response; - ASSERT_EQ(handler.processPredictRequest(dagName, modelVersion, modelVersionLabel, request, &response), ovms::StatusCode::OK); - } - - for (int i = 0; i < numberOfFailedRequests; i++) { - std::string request = R"({"signature_name": "serving_default", "instances": [[[1,2,3,4,5,6,7,8,9,10]],[[1,2,3,4,5,6,7,8,9,10]],[[1,2,3,4,5,6,7,8,9]]]})"; - std::string response; - ASSERT_EQ(handler.processPredictRequest(dagName, modelVersion, modelVersionLabel, request, &response), ovms::StatusCode::REST_COULD_NOT_PARSE_INSTANCE); - } - - checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, modelName, 1, "REST", "Predict", "TensorFlowServing", dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests); // ran by demultiplexer + real request - checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, dagName, 1, "REST", "Predict", "TensorFlowServing", numberOfSuccessRequests); // ran by real request - - checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_FAIL, modelName, 1, "REST", "Predict", "TensorFlowServing", numberOfFailedRequests); // ran by real request - checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_FAIL, dagName, 1, "REST", "Predict", "TensorFlowServing", numberOfFailedRequests); // ran by real request - - EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_REQUEST_TIME + std::string{"_count{interface=\"gRPC\",name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(0))); - EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_REQUEST_TIME + std::string{"_count{interface=\"gRPC\",name=\""} + dagName + std::string{"\",version=\"1\"} "} + std::to_string(0))); - EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_REQUEST_TIME + std::string{"_count{interface=\"REST\",name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(numberOfSuccessRequests))); - EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_REQUEST_TIME + std::string{"_count{interface=\"REST\",name=\""} + dagName + std::string{"\",version=\"1\"} "} + std::to_string(numberOfSuccessRequests))); - - EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_INFERENCE_TIME + std::string{"_count{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests))); - EXPECT_THAT(server.collect(), Not(HasSubstr(METRIC_NAME_INFERENCE_TIME + std::string{"_count{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); - - EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_WAIT_FOR_INFER_REQ_TIME + std::string{"_count{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests))); - EXPECT_THAT(server.collect(), Not(HasSubstr(METRIC_NAME_WAIT_FOR_INFER_REQ_TIME + std::string{"_count{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); - - EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_STREAMS + std::string{"{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(4))); - EXPECT_THAT(server.collect(), Not(HasSubstr(METRIC_NAME_STREAMS + std::string{"{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); - - EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_INFER_REQ_QUEUE_SIZE + std::string{"{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(2))); - EXPECT_THAT(server.collect(), Not(HasSubstr(METRIC_NAME_INFER_REQ_QUEUE_SIZE + std::string{"{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); -} - -TEST_F(MetricFlowTest, RestGetModelMetadata) { - HttpRestApiHandler handler(server, 0); - - for (int i = 0; i < numberOfSuccessRequests; i++) { - std::string response; - ASSERT_EQ(handler.processModelMetadataRequest(modelName, modelVersion, modelVersionLabel, &response), ovms::StatusCode::OK); - } - - for (int i = 0; i < numberOfSuccessRequests; i++) { - std::string response; - ASSERT_EQ(handler.processModelMetadataRequest(dagName, modelVersion, modelVersionLabel, &response), ovms::StatusCode::OK); - } - - checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, modelName, 1, "REST", "GetModelMetadata", "TensorFlowServing", numberOfSuccessRequests); // ran by real request - checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, dagName, 1, "REST", "GetModelMetadata", "TensorFlowServing", numberOfSuccessRequests); // ran by real request -} - -TEST_F(MetricFlowTest, RestGetModelStatus) { - HttpRestApiHandler handler(server, 0); - - for (int i = 0; i < numberOfSuccessRequests; i++) { - std::string response; - ASSERT_EQ(handler.processModelStatusRequest(modelName, modelVersion, modelVersionLabel, &response), ovms::StatusCode::OK); - } - - for (int i = 0; i < numberOfSuccessRequests; i++) { - std::string response; - ASSERT_EQ(handler.processModelStatusRequest(dagName, modelVersion, modelVersionLabel, &response), ovms::StatusCode::OK); - } - - checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, modelName, 1, "REST", "GetModelStatus", "TensorFlowServing", numberOfSuccessRequests); // ran by real request - checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, dagName, 1, "REST", "GetModelStatus", "TensorFlowServing", numberOfSuccessRequests); // ran by real request -} - TEST_F(MetricFlowTest, RestModelInfer) { HttpRestApiHandler handler(server, 0); HttpRequestComponents components; diff --git a/src/test/model_service_test.cpp b/src/test/model_service_test.cpp deleted file mode 100644 index 7fbb5d0e8c..0000000000 --- a/src/test/model_service_test.cpp +++ /dev/null @@ -1,576 +0,0 @@ -//***************************************************************************** -// Copyright 2020-2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** - -#include -#include -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#include "tensorflow_serving/apis/get_model_status.pb.h" -#include "tensorflow_serving/apis/model_service.grpc.pb.h" -#include "tensorflow_serving/apis/model_service.pb.h" -#pragma GCC diagnostic pop - -#include "../dags/pipelinedefinition.hpp" -#include "../execution_context.hpp" -#include "../grpc_utils.hpp" -#include "../model_service.hpp" -#include "../modelinstanceunloadguard.hpp" -#include "../model_version_policy.hpp" -#include "../modelmanager.hpp" -#include "../modelversionstatus.hpp" -#include "../server.hpp" -#include "gtest/gtest.h" -#include "platform_utils.hpp" -#include "test_utils.hpp" -#include "constructor_enabled_model_manager.hpp" -#include "test_models_configs.hpp" -#include "light_test_utils.hpp" - -using namespace ovms; - -using TFSGetModelStatusRequest = tensorflow::serving::GetModelStatusRequest; -using TFSGetModelStatusResponse = tensorflow::serving::GetModelStatusResponse; -using TFSGetModelStatusInterface = std::pair; -using KFSGetModelStatusInterface = std::pair; - -template -class ModelServiceTest : public ::testing::Test { -public: - ConstructorEnabledModelManager manager; - RequestType modelStatusRequest; - ResponseType modelStatusResponse; - void SetUp() { - auto config = DUMMY_MODEL_CONFIG; - ASSERT_EQ(this->manager.reloadModelWithVersions(config), StatusCode::OK_RELOADED); - this->modelStatusRequest.Clear(); - this->modelStatusResponse.Clear(); - } -}; - -using MyTypes = ::testing::Types< - TFSGetModelStatusInterface, - KFSGetModelStatusInterface>; - -TYPED_TEST_SUITE(ModelServiceTest, MyTypes); - -static void executeModelStatus(const TFSGetModelStatusRequest& modelStatusRequest, TFSGetModelStatusResponse& modelStatusResponse, ModelManager& manager, ExecutionContext context, ovms::StatusCode statusCode = StatusCode::OK) { - modelStatusResponse.Clear(); - ASSERT_EQ(GetModelStatusImpl::getModelStatus(&modelStatusRequest, &modelStatusResponse, manager, context), statusCode); -} - -static void setModelStatusRequest(TFSGetModelStatusRequest& modelStatusRequest, const std::string& name, int version) { - modelStatusRequest.Clear(); - auto model_spec = modelStatusRequest.mutable_model_spec(); - model_spec->Clear(); - model_spec->set_name(name); - if (version) { - model_spec->mutable_version()->set_value(version); - } -} - -static void verifyModelStatusResponse(const TFSGetModelStatusResponse& modelStatusResponse, const std::vector& versions = {1}) { - ASSERT_EQ(modelStatusResponse.model_version_status_size(), versions.size()); - for (size_t i = 0; i < versions.size(); i++) { - auto& model_version_status = modelStatusResponse.model_version_status()[i]; - ASSERT_EQ(model_version_status.state(), tensorflow::serving::ModelVersionStatus_State_AVAILABLE); - ASSERT_EQ(model_version_status.version(), versions[i]); - ASSERT_EQ(model_version_status.has_status(), true); - ASSERT_EQ(model_version_status.status().error_code(), tensorflow::error::OK); - ASSERT_EQ(model_version_status.status().error_message(), "OK"); - } -} - -static void verifyModelStatusResponse(const KFSGetModelStatusResponse& modelStatusResponse, const std::vector& versions = {1}) { - ASSERT_TRUE(modelStatusResponse.ready()); -} - -static void executeModelStatus(const KFSGetModelStatusRequest& modelStatusRequest, KFSGetModelStatusResponse& modelStatusResponse, ModelManager& manager, ExecutionContext context, ovms::StatusCode statusCode = StatusCode::OK) { - modelStatusResponse.Clear(); - ASSERT_EQ(KFSInferenceServiceImpl::getModelReady(&modelStatusRequest, &modelStatusResponse, manager, context), statusCode); -} - -static void setModelStatusRequest(KFSGetModelStatusRequest& modelStatusRequest, const std::string& name, int version) { - modelStatusRequest.Clear(); - modelStatusRequest.set_name(name); - if (version) - modelStatusRequest.set_version(std::to_string(version)); -} - -TYPED_TEST(ModelServiceTest, empty_request) { - executeModelStatus(this->modelStatusRequest, this->modelStatusResponse, this->manager, DEFAULT_TEST_CONTEXT, StatusCode::MODEL_NAME_MISSING); -} - -TYPED_TEST(ModelServiceTest, single_version_model) { - const std::string name = "dummy"; - auto version = 1; // existing version - setModelStatusRequest(this->modelStatusRequest, name, version); - executeModelStatus(this->modelStatusRequest, this->modelStatusResponse, this->manager, DEFAULT_TEST_CONTEXT); - verifyModelStatusResponse(this->modelStatusResponse); -} - -static const char* pipelineOneDummyConfig = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "dummyPipeline", - "inputs": ["custom_dummy_input"], - "nodes": [ - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "custom_dummy_input"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ] - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode", - "data_item": "new_dummy_output"} - } - ] - } - ] -})"; - -TYPED_TEST(ModelServiceTest, pipeline) { - std::string fileToReload = getGenericFullPathForTmp("/tmp/ovms_single_version_pipeline.json"); - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineOneDummyConfig), fileToReload); - ASSERT_EQ(this->manager.startFromFile(fileToReload), StatusCode::OK); - - const std::string name = "dummyPipeline"; - - // existing version - int version = 1; - setModelStatusRequest(this->modelStatusRequest, name, version); - executeModelStatus(this->modelStatusRequest, this->modelStatusResponse, this->manager, DEFAULT_TEST_CONTEXT); - verifyModelStatusResponse(this->modelStatusResponse); - - // No version specified - with 0 version value is not set in helper function - version = 0; - setModelStatusRequest(this->modelStatusRequest, name, version); - executeModelStatus(this->modelStatusRequest, this->modelStatusResponse, this->manager, DEFAULT_TEST_CONTEXT); - verifyModelStatusResponse(this->modelStatusResponse); - - // Any version - version = 5; - setModelStatusRequest(this->modelStatusRequest, name, version); - executeModelStatus(this->modelStatusRequest, this->modelStatusResponse, this->manager, DEFAULT_TEST_CONTEXT); - verifyModelStatusResponse(this->modelStatusResponse); -} - -#if (MEDIAPIPE_DISABLE == 0) -TYPED_TEST(ModelServiceTest, MediapipeGraph) { - std::string fileToReload = getGenericFullPathForSrcTest("/ovms/src/test/mediapipe/config_mediapipe_dummy_adapter_full.json"); - ASSERT_EQ(this->manager.startFromFile(fileToReload), StatusCode::OK); - - const std::string name = "mediaDummyADAPTFULL"; - - // existing version - int version = 1; - setModelStatusRequest(this->modelStatusRequest, name, version); - executeModelStatus(this->modelStatusRequest, this->modelStatusResponse, this->manager, DEFAULT_TEST_CONTEXT); - verifyModelStatusResponse(this->modelStatusResponse); - - // No version specified - with 0 version value is not set in helper function - version = 0; - setModelStatusRequest(this->modelStatusRequest, name, version); - executeModelStatus(this->modelStatusRequest, this->modelStatusResponse, this->manager, DEFAULT_TEST_CONTEXT); - verifyModelStatusResponse(this->modelStatusResponse); - - // Any version - version = 5; - setModelStatusRequest(this->modelStatusRequest, name, version); - executeModelStatus(this->modelStatusRequest, this->modelStatusResponse, this->manager, DEFAULT_TEST_CONTEXT); - verifyModelStatusResponse(this->modelStatusResponse); -} -#endif - -TYPED_TEST(ModelServiceTest, non_existing_model) { - const std::string name = "non_existing_model"; - int version = 0; - setModelStatusRequest(this->modelStatusRequest, name, version); - executeModelStatus(this->modelStatusRequest, this->modelStatusResponse, this->manager, DEFAULT_TEST_CONTEXT, StatusCode::MODEL_NAME_MISSING); -} - -TYPED_TEST(ModelServiceTest, non_existing_version) { - const std::string name = "dummy"; - int version = 989464; - setModelStatusRequest(this->modelStatusRequest, name, version); - executeModelStatus(this->modelStatusRequest, this->modelStatusResponse, this->manager, DEFAULT_TEST_CONTEXT, StatusCode::MODEL_VERSION_MISSING); -} - -TYPED_TEST(ModelServiceTest, negative_version) { - const std::string name = "dummy"; - int version = -1; - setModelStatusRequest(this->modelStatusRequest, name, version); - executeModelStatus(this->modelStatusRequest, this->modelStatusResponse, this->manager, DEFAULT_TEST_CONTEXT, StatusCode::MODEL_VERSION_MISSING); -} - -TEST(RestModelStatus, CreateGrpcRequestVersionSet) { - std::string model_name = "dummy"; - const std::optional model_version = 1; - tensorflow::serving::GetModelStatusRequest request_grpc; - tensorflow::serving::GetModelStatusRequest* request_p = &request_grpc; - Status status = GetModelStatusImpl::createGrpcRequest(model_name, model_version, request_p); - bool has_requested_version = request_p->model_spec().has_version(); - auto requested_version = request_p->model_spec().version().value(); - std::string requested_model_name = request_p->model_spec().name(); - ASSERT_EQ(status, StatusCode::OK); - EXPECT_EQ(has_requested_version, true); - EXPECT_EQ(requested_version, 1); - EXPECT_EQ(requested_model_name, "dummy"); -} - -TEST(RestModelStatus, CreateGrpcRequestNoVersion) { - std::string model_name = "dummy1"; - const std::optional model_version; - tensorflow::serving::GetModelStatusRequest request_grpc; - tensorflow::serving::GetModelStatusRequest* request_p = &request_grpc; - Status status = GetModelStatusImpl::createGrpcRequest(model_name, model_version, request_p); - bool has_requested_version = request_p->model_spec().has_version(); - std::string requested_model_name = request_p->model_spec().name(); - ASSERT_EQ(status, StatusCode::OK); - EXPECT_EQ(has_requested_version, false); - EXPECT_EQ(requested_model_name, "dummy1"); -} - -TEST(RestModelStatus, serialize2Json) { - const char* expected_json = R"({ - "model_version_status": [ - { - "version": "2", - "state": "START", - "status": { - "error_code": "OK", - "error_message": "OK" - } - } - ] -} -)"; - tensorflow::serving::GetModelStatusResponse response; - model_version_t requested_version = 2; - const std::string& model_name = "dummy"; - ModelVersionStatus status = ModelVersionStatus(model_name, requested_version, ModelVersionState::START); - addStatusToResponse(&response, requested_version, status); - const tensorflow::serving::GetModelStatusResponse response_const = response; - std::string json_output; - Status error_status = GetModelStatusImpl::serializeResponse2Json(&response_const, &json_output); - ASSERT_EQ(error_status, StatusCode::OK); - EXPECT_EQ(json_output, expected_json); -} - -const ovms::ModelConfig DUMMY_MODEL_WITH_ONLY_NAME_CONFIG{ - "dummy", -}; - -// Some tests are specific for TFS because you can ask for more versions than one in one request -class ModelServiceDummyWith2Versions : public ::testing::Test { -protected: - ConstructorEnabledModelManager manager; - - void SetUp() override { - const ::testing::TestInfo* const test_info = - ::testing::UnitTest::GetInstance()->current_test_info(); - - const std::string directoryName = std::string(test_info->test_suite_name()); - directoryPath = getGenericFullPathForTmp("/tmp/" + directoryName); - modelPath = directoryPath + "/dummy"; - - // Copy dummy model to temporary destination - std::filesystem::remove_all(directoryPath); - std::filesystem::create_directories(modelPath + "/1/"); - std::filesystem::create_directories(modelPath + "/2/"); - std::filesystem::copy(getGenericFullPathForSrcTest("/ovms/src/test/dummy/1"), modelPath + "/1", std::filesystem::copy_options::recursive); - std::filesystem::copy(getGenericFullPathForSrcTest("/ovms/src/test/dummy/1"), modelPath + "/2", std::filesystem::copy_options::recursive); - } - - void TearDown() override { -#ifdef _WIN32 - // Unload model to allow folder delete on Windows - std::shared_ptr modelInstance1; - std::unique_ptr modelInstanceUnloadGuard; - manager.getModelInstance("dummy", 1, modelInstance1, modelInstanceUnloadGuard); - // Release guard - modelInstanceUnloadGuard.reset(); - // Unload model - modelInstance1->retireModel(); - - manager.getModelInstance("dummy", 2, modelInstance1, modelInstanceUnloadGuard); - // Release guard - modelInstanceUnloadGuard.reset(); - // Unload model - modelInstance1->retireModel(); -#endif - // Clean up temporary destination - std::filesystem::remove_all(directoryPath); - } - - std::string directoryPath; - std::string modelPath; -}; - -TEST_F(ModelServiceDummyWith2Versions, all_versions) { - tensorflow::serving::GetModelStatusRequest modelStatusRequest; - tensorflow::serving::GetModelStatusResponse modelStatusResponse; - auto config = DUMMY_MODEL_CONFIG; - config.setBasePath(getGenericFullPathForSrcTest(modelPath)); - config.setModelVersionPolicy(std::make_shared()); - ASSERT_EQ(manager.reloadModelWithVersions(config), StatusCode::OK_RELOADED); - - // no version specified - const std::string name = "dummy"; - int version = 0; - setModelStatusRequest(modelStatusRequest, name, version); - executeModelStatus(modelStatusRequest, modelStatusResponse, this->manager, DEFAULT_TEST_CONTEXT); - verifyModelStatusResponse(modelStatusResponse, {1, 2}); -} - -TEST_F(ModelServiceDummyWith2Versions, getAllModelsStatuses_one_model_two_versions) { - auto config = DUMMY_MODEL_WITH_ONLY_NAME_CONFIG; - this->manager.reloadModelWithVersions(config); - std::map modelsStatuses; - GetModelStatusImpl::getAllModelsStatuses(modelsStatuses, this->manager, DEFAULT_TEST_CONTEXT); - EXPECT_EQ(modelsStatuses.size(), 1); - EXPECT_EQ(modelsStatuses.begin()->second.model_version_status_size(), 0); - - config = DUMMY_MODEL_CONFIG; - config.setBasePath(getGenericFullPathForSrcTest(modelPath)); - config.setModelVersionPolicy(std::make_shared()); - this->manager.reloadModelWithVersions(config); - std::map modelsStatusesAfterReload; - GetModelStatusImpl::getAllModelsStatuses(modelsStatusesAfterReload, this->manager, DEFAULT_TEST_CONTEXT); - - ASSERT_EQ(modelsStatusesAfterReload.size(), 1); - verifyModelStatusResponse(modelsStatusesAfterReload.begin()->second, {1, 2}); -} - -// Some tests are specific for TFS because you can ask for more versions than one in one request -using TFSModelServiceTest = ModelServiceTest; - -TEST_F(TFSModelServiceTest, getAllModelsStatuses_two_models_with_one_versions) { - std::map modelsStatuses; - GetModelStatusImpl::getAllModelsStatuses(modelsStatuses, this->manager, DEFAULT_TEST_CONTEXT); - verifyModelStatusResponse(modelsStatuses.begin()->second); - - auto config = SUM_MODEL_CONFIG; - this->manager.reloadModelWithVersions(config); - std::map modelsStatusesAfterReload; - GetModelStatusImpl::getAllModelsStatuses(modelsStatusesAfterReload, this->manager, DEFAULT_TEST_CONTEXT); - ASSERT_EQ(modelsStatusesAfterReload.size(), 2); - auto dummyModelStatus = modelsStatusesAfterReload.find("dummy"); - auto sumModelStatus = modelsStatusesAfterReload.find("sum"); - ASSERT_NE(dummyModelStatus, modelsStatusesAfterReload.end()); - ASSERT_NE(sumModelStatus, modelsStatusesAfterReload.end()); - verifyModelStatusResponse(dummyModelStatus->second); - verifyModelStatusResponse(sumModelStatus->second); -} - -TEST_F(TFSModelServiceTest, config_reload) { - std::string port = "9000"; - randomizeAndEnsureFree(port); - char* argv[] = { - (char*)"OpenVINO Model Server", - (char*)"--model_name", - (char*)"dummy", - (char*)"--model_path", - (char*)getGenericFullPathForSrcTest("/ovms/src/test/dummy").c_str(), - (char*)"--log_level", - (char*)"DEBUG", - (char*)"--port", - (char*)port.c_str(), - nullptr}; - ovms::Server& server = ovms::Server::instance(); - std::thread t([&argv, &server]() { - ASSERT_EQ(EXIT_SUCCESS, server.start(9, argv)); - }); - EnsureServerStartedWithTimeout(server, 5); - ModelServiceImpl s(server); - tensorflow::serving::ReloadConfigRequest modelStatusRequest; - tensorflow::serving::ReloadConfigResponse modelStatusResponse; - - spdlog::info("req={} this->res={}", this->modelStatusRequest.DebugString(), this->modelStatusResponse.DebugString()); - ::grpc::Status ret = s.HandleReloadConfigRequest(nullptr, &modelStatusRequest, &modelStatusResponse); - spdlog::info("returned grpc status: ok={} code={} msg='{}'", ret.ok(), ret.error_code(), ret.error_details()); - EXPECT_EQ(ret.ok(), true); - server.setShutdownRequest(1); - t.join(); - server.setShutdownRequest(0); -} - -TEST_F(TFSModelServiceTest, getAllModelsStatuses_one_model_one_version) { - ConstructorEnabledModelManager manager; // intentionally uses separate manager as we don't want any unloaded models - auto config = DUMMY_MODEL_WITH_ONLY_NAME_CONFIG; - manager.reloadModelWithVersions(config); - std::map modelsStatuses; - GetModelStatusImpl::getAllModelsStatuses(modelsStatuses, manager, DEFAULT_TEST_CONTEXT); - EXPECT_EQ(modelsStatuses.size(), 1); - EXPECT_EQ(modelsStatuses.begin()->second.model_version_status_size(), 0); - - config = DUMMY_MODEL_CONFIG; - manager.reloadModelWithVersions(config); - std::map modelsStatusesAfterReload; - GetModelStatusImpl::getAllModelsStatuses(modelsStatusesAfterReload, manager, DEFAULT_TEST_CONTEXT); - - ASSERT_EQ(modelsStatusesAfterReload.size(), 1); - verifyModelStatusResponse(modelsStatusesAfterReload.begin()->second); -} - -TEST_F(TFSModelServiceTest, serializeModelsStatuses2Json_with_one_response) { - const char* expectedJson = R"({ -"dummy" : -{ - "model_version_status": [ - { - "version": "2", - "state": "START", - "status": { - "error_code": "OK", - "error_message": "OK" - } - } - ] -} -})"; - tensorflow::serving::GetModelStatusResponse modelStatusResponse; - model_version_t requestedVersion = 2; - const std::string& model_name = "dummy"; - ModelVersionStatus modelStatus = ModelVersionStatus(model_name, requestedVersion, ModelVersionState::START); - addStatusToResponse(&modelStatusResponse, requestedVersion, modelStatus); - std::map modelsStatuses; - modelsStatuses.insert(std::pair("dummy", modelStatusResponse)); - std::string jsonOutput; - Status status = GetModelStatusImpl::serializeModelsStatuses2Json(modelsStatuses, jsonOutput); - ASSERT_EQ(status, StatusCode::OK); - EXPECT_EQ(jsonOutput, expectedJson); -} - -TEST_F(TFSModelServiceTest, serializeModelsStatuses2Json_with_two_responses) { - const char* expectedJson = R"({ -"dummy1" : -{ - "model_version_status": [ - { - "version": "2", - "state": "START", - "status": { - "error_code": "OK", - "error_message": "OK" - } - } - ] -}, -"dummy2" : -{ - "model_version_status": [ - { - "version": "3", - "state": "LOADING", - "status": { - "error_code": "OK", - "error_message": "OK" - } - } - ] -} -})"; - tensorflow::serving::GetModelStatusResponse firstResponse; - model_version_t requestedVersion = 2; - const std::string& modelName1 = "dummy1"; - ModelVersionStatus modelStatus = ModelVersionStatus(modelName1, requestedVersion, ModelVersionState::START); - addStatusToResponse(&firstResponse, requestedVersion, modelStatus); - - tensorflow::serving::GetModelStatusResponse secondResponse; - requestedVersion = 3; - const std::string& modelName2 = "dummy2"; - modelStatus = ModelVersionStatus(modelName2, requestedVersion, ModelVersionState::LOADING); - addStatusToResponse(&secondResponse, requestedVersion, modelStatus); - - std::map modelsStatuses; - modelsStatuses.insert(std::pair("dummy1", firstResponse)); - modelsStatuses.insert(std::pair("dummy2", secondResponse)); - - std::string jsonOutput; - Status status = GetModelStatusImpl::serializeModelsStatuses2Json(modelsStatuses, jsonOutput); - ASSERT_EQ(status, StatusCode::OK); - EXPECT_EQ(jsonOutput, expectedJson); -} - -TEST_F(TFSModelServiceTest, serializeModelsStatuses2Json_one_response_with_two_versions) { - const char* expectedJson = R"({ -"dummy" : -{ - "model_version_status": [ - { - "version": "2", - "state": "START", - "status": { - "error_code": "OK", - "error_message": "OK" - } - }, - { - "version": "3", - "state": "LOADING", - "status": { - "error_code": "OK", - "error_message": "OK" - } - } - ] -} -})"; - tensorflow::serving::GetModelStatusResponse response; - model_version_t requestedVersion = 2; - const std::string& modelName = "dummy"; - ModelVersionStatus modelStatus = ModelVersionStatus(modelName, requestedVersion, ModelVersionState::START); - addStatusToResponse(&response, requestedVersion, modelStatus); - - requestedVersion = 3; - modelStatus = ModelVersionStatus(modelName, requestedVersion, ModelVersionState::LOADING); - addStatusToResponse(&response, requestedVersion, modelStatus); - - std::map modelsStatuses; - modelsStatuses.insert(std::pair("dummy", response)); - - std::string jsonOutput; - Status status = GetModelStatusImpl::serializeModelsStatuses2Json(modelsStatuses, jsonOutput); - ASSERT_EQ(status, StatusCode::OK); - EXPECT_EQ(jsonOutput, expectedJson); -} diff --git a/src/test/modelinstance_test.cpp b/src/test/modelinstance_test.cpp index 7ccc2ddd85..1425377b86 100644 --- a/src/test/modelinstance_test.cpp +++ b/src/test/modelinstance_test.cpp @@ -14,8 +14,10 @@ // limitations under the License. //***************************************************************************** #include +#include #include #include +#include #include #include @@ -23,7 +25,6 @@ #include #include -#include "../get_model_metadata_impl.hpp" #include "../modelinstance.hpp" #include "../modelinstanceunloadguard.hpp" #include "gpuenvironment.hpp" @@ -130,43 +131,27 @@ TEST_F(TestUnloadModel, NoNameOutput) { } TEST_F(TestUnloadModel, UnloadWaitsUntilMetadataResponseIsBuilt) { - static std::thread thread; - static std::shared_ptr instance; - - class MockModelInstanceTriggeringUnload : public ovms::ModelInstance { - public: - MockModelInstanceTriggeringUnload(ov::Core& ieCore) : - ModelInstance("UNUSED_NAME", UNUSED_MODEL_VERSION, ieCore) {} - // This is to trigger model unloading in separate thread during GetModelMetadataImpl::buildResponse call. - const ovms::tensor_map_t& getInputsInfo() const override { - thread = std::thread([]() { - instance->retireModel(); - }); - // We need to wait for thread to start and trigger model unloading - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - return ovms::ModelInstance::getInputsInfo(); - } - }; - instance = std::make_shared(*ieCore); - ovms::Status status = instance->loadModel(DUMMY_MODEL_CONFIG); - ASSERT_EQ(status, ovms::StatusCode::OK); - ASSERT_EQ(ovms::ModelVersionState::AVAILABLE, instance->getStatus().getState()); - tensorflow::serving::GetModelMetadataResponse response; - EXPECT_EQ(ovms::GetModelMetadataImpl::buildResponse(instance, &response), ovms::StatusCode::OK); - thread.join(); - EXPECT_EQ(ovms::ModelVersionState::END, instance->getStatus().getState()); - - // We expect unload to wait for response building by checking if packed data is correct. - // If unloadModel didn't wait for building to complete we would have empty input/output map. - tensorflow::serving::SignatureDefMap def; - response.metadata().at("signature_def").UnpackTo(&def); - const auto& inputs = ((*def.mutable_signature_def())["serving_default"]).inputs(); - const auto& outputs = ((*def.mutable_signature_def())["serving_default"]).outputs(); - EXPECT_EQ(inputs.size(), 1); - EXPECT_EQ(outputs.size(), 1); - EXPECT_EQ(inputs.begin()->second.name(), DUMMY_MODEL_INPUT_NAME); - EXPECT_EQ(outputs.begin()->second.name(), DUMMY_MODEL_OUTPUT_NAME); - instance.reset(); + ovms::ModelInstance modelInstance("UNUSED_NAME", UNUSED_MODEL_VERSION, *ieCore); + ASSERT_EQ(modelInstance.loadModel(DUMMY_MODEL_CONFIG), ovms::StatusCode::OK); + ASSERT_EQ(ovms::ModelVersionState::AVAILABLE, modelInstance.getStatus().getState()); + + // Acquire unload guard — same mechanism KFS buildResponse uses internally + std::unique_ptr unloadGuard; + ASSERT_EQ(modelInstance.waitForLoaded(0, unloadGuard), ovms::StatusCode::OK); + ASSERT_NE(unloadGuard, nullptr); + + // retireModel must block while guard is held + std::thread retireThread([&modelInstance]() { + modelInstance.retireModel(); + }); + + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + EXPECT_EQ(ovms::ModelVersionState::UNLOADING, modelInstance.getStatus().getState()); + + // Release guard — retire should now complete + unloadGuard.reset(); + retireThread.join(); + EXPECT_EQ(ovms::ModelVersionState::END, modelInstance.getStatus().getState()); } TEST_F(TestUnloadModel, CheckIfCanUnload) { diff --git a/src/test/modelversionstatus_test.cpp b/src/test/modelversionstatus_test.cpp index 9f8eba20d8..bee3bc7889 100644 --- a/src/test/modelversionstatus_test.cpp +++ b/src/test/modelversionstatus_test.cpp @@ -17,44 +17,26 @@ #include #include -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#include "tensorflow_serving/apis/get_model_status.pb.h" -#include "tensorflow_serving/apis/model_service.grpc.pb.h" -#include "tensorflow_serving/apis/model_service.pb.h" -#pragma GCC diagnostic pop - #include "../modelversionstatus.hpp" #include "gtest/gtest.h" using namespace ovms; -TEST(ModelVersionStatus, casting_to_protobuf_enum) { +TEST(ModelVersionStatus, state_transitions) { ModelVersionStatus mvs("SampleModelName", 15); - tensorflow::serving::ModelVersionStatus_State tf_state = static_cast(static_cast(mvs.getState())); EXPECT_EQ(mvs.getState(), ovms::ModelVersionState::START); - EXPECT_EQ(tf_state, tensorflow::serving::ModelVersionStatus_State_START); mvs.setLoading(ModelVersionStatusErrorCode::OK); - tf_state = static_cast(static_cast(mvs.getState())); EXPECT_EQ(mvs.getState(), ovms::ModelVersionState::LOADING); - EXPECT_EQ(tf_state, tensorflow::serving::ModelVersionStatus_State_LOADING); mvs.setAvailable(ModelVersionStatusErrorCode::OK); - tf_state = static_cast(static_cast(mvs.getState())); EXPECT_EQ(mvs.getState(), ovms::ModelVersionState::AVAILABLE); - EXPECT_EQ(tf_state, tensorflow::serving::ModelVersionStatus_State_AVAILABLE); mvs.setUnloading(ModelVersionStatusErrorCode::OK); - tf_state = static_cast(static_cast(mvs.getState())); EXPECT_EQ(mvs.getState(), ovms::ModelVersionState::UNLOADING); - EXPECT_EQ(tf_state, tensorflow::serving::ModelVersionStatus_State_UNLOADING); mvs.setEnd(ModelVersionStatusErrorCode::OK); - tf_state = static_cast(static_cast(mvs.getState())); EXPECT_EQ(mvs.getState(), ovms::ModelVersionState::END); - EXPECT_EQ(tf_state, tensorflow::serving::ModelVersionStatus_State_END); } TEST(ModelVersionStatus, simple_test_flow) { diff --git a/src/test/pipelinedefinitionstatus_test.cpp b/src/test/pipelinedefinitionstatus_test.cpp index 578ad38295..8c3d722f6c 100644 --- a/src/test/pipelinedefinitionstatus_test.cpp +++ b/src/test/pipelinedefinitionstatus_test.cpp @@ -24,8 +24,6 @@ #include "test_utils.hpp" using namespace ovms; -using namespace tensorflow; -using namespace tensorflow::serving; using testing::_; using testing::Return; diff --git a/src/test/predict_validation_test.cpp b/src/test/predict_validation_test.cpp index c8fcb1574d..74d6cc873b 100644 --- a/src/test/predict_validation_test.cpp +++ b/src/test/predict_validation_test.cpp @@ -23,7 +23,6 @@ #include #include "../kfs_frontend/kfs_grpc_inference_service.hpp" -#include "../tfs_frontend/tfs_utils.hpp" #include "../modelconfig.hpp" #include "../modelinstance.hpp" #include "../predict_request_validation_utils.hpp" @@ -33,864 +32,6 @@ using ::testing::NiceMock; using ::testing::Return; using ::testing::ReturnRef; -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-variable" - -class TfsPredictValidation : public ::testing::Test { -protected: - std::unique_ptr ieCore; - std::unique_ptr> instance; - tensorflow::serving::PredictRequest request; - ovms::ModelConfig modelConfig{"model_name", "model_path"}; - ovms::tensor_map_t servableInputs; - ovms::tensor_map_t servableOutputs = ovms::tensor_map_t({{"SOME_OUTPUT", std::make_shared("OUTPUT", ovms::Precision::FP32, ovms::shape_t{1, 1, 1, 1}, ovms::Layout{"NHWC"})}}); // not used for TFS - - void SetUp() override { - ieCore = std::make_unique(); - instance = std::make_unique>(*ieCore); - - servableInputs = ovms::tensor_map_t({ - {"Input_FP32_1_224_224_3_NHWC", - std::make_shared("Input_FP32_1_3_224_224_NHWC", ovms::Precision::FP32, ovms::shape_t{1, 224, 224, 3}, ovms::Layout{"NHWC"})}, - {"Input_U8_1_3_62_62_NCHW", - std::make_shared("Input_U8_1_3_62_62_NCHW", ovms::Precision::U8, ovms::shape_t{1, 3, 62, 62}, ovms::Layout{"NCHW"})}, - {"Input_I64_1_6_128_128_16_NCDHW", - std::make_shared("Input_I64_1_6_128_128_16_NCDHW", ovms::Precision::I64, ovms::shape_t{1, 6, 128, 128, 16}, ovms::Layout{"NCDHW"})}, - {"Input_U16_1_2_8_4_NCHW", - std::make_shared("Input_U16_1_2_8_4_NCHW", ovms::Precision::U16, ovms::shape_t{1, 2, 8, 4}, ovms::Layout{"NCHW"})}, - }); - - ON_CALL(*instance, getInputsInfo()).WillByDefault(ReturnRef(servableInputs)); - ON_CALL(*instance, getOutputsInfo()).WillByDefault(ReturnRef(servableOutputs)); - ON_CALL(*instance, getBatchSize()).WillByDefault(Return(1)); - ON_CALL(*instance, getModelConfig()).WillByDefault(ReturnRef(modelConfig)); - - preparePredictRequest(request, - { - {"Input_FP32_1_224_224_3_NHWC", - std::tuple{{1, 224, 224, 3}, ovms::Precision::FP32}}, - {"Input_U8_1_3_62_62_NCHW", - std::tuple{{1, 3, 62, 62}, ovms::Precision::U8}}, - {"Input_I64_1_6_128_128_16_NCDHW", - std::tuple{{1, 6, 128, 128, 16}, ovms::Precision::I64}}, - }); - - // U16 uses int_val instead of tensor_content so it needs separate test - auto& inputD = (*request.mutable_inputs())["Input_U16_1_2_8_4_NCHW"]; - inputD.set_dtype(tensorflow::DataType::DT_UINT16); - inputD.mutable_int_val()->Resize(1 * 2 * 8 * 4, 1); - inputD.mutable_tensor_shape()->add_dim()->set_size(1); - inputD.mutable_tensor_shape()->add_dim()->set_size(2); - inputD.mutable_tensor_shape()->add_dim()->set_size(8); - inputD.mutable_tensor_shape()->add_dim()->set_size(4); - } - - static void prepareTensorContent(tensorflow::TensorProto& proto) { - if (proto.tensor_shape().dim_size() == 0) { - *proto.mutable_tensor_content() = ""; - return; - } - size_t numberOfElements = 1; - for (int i = 0; i < proto.tensor_shape().dim_size(); i++) { - numberOfElements *= proto.tensor_shape().dim(i).size(); - } - *proto.mutable_tensor_content() = std::string(numberOfElements * tensorflow::DataTypeSize(proto.dtype()), '1'); - } -}; - -TEST_F(TfsPredictValidation, ValidRequest) { - auto status = instance->mockValidate(&request); - EXPECT_TRUE(status.ok()); -} - -TEST_F(TfsPredictValidation, RequestWithScalar) { - servableInputs = ovms::tensor_map_t({{"Input_FP32_Scalar", - std::make_shared("Input_FP32_Scalar", ovms::Precision::FP32, ovms::shape_t{}, ovms::Layout{"..."})}}); - preparePredictRequest(request, - {{"Input_FP32_Scalar", - std::tuple{std::vector{}, ovms::Precision::FP32}}}); - auto status = instance->mockValidate(&request); - EXPECT_TRUE(status.ok()); -} - -// Requesting 0 batch via TFS API -// Mocked endpoints tested: dynamic batch (-1), range (0-100) and static 0. -TEST_F(TfsPredictValidation, RequestWithZeroBatch) { - std::vector shapes{ - ovms::Shape{ovms::Dimension::any(), 400, 99}, // dynamic - ovms::Shape{ovms::Dimension{0, 100}, 400, 99}, // range - ovms::Shape{0, 400, 99} // static - }; - - ovms::signed_shape_t actualRequestShape{0, 400, 99}; - - for (const auto& shape : shapes) { - servableInputs = ovms::tensor_map_t({{"Input", - std::make_shared("Input", ovms::Precision::FP32, shape, ovms::Layout{"N..."})}}); - preparePredictRequest(request, - {{"Input", - std::tuple{actualRequestShape, ovms::Precision::FP32}}}); - auto status = instance->mockValidate(&request); - EXPECT_TRUE(status.ok()); - } -} - -// Requesting 0 dimension in position other than batch via TFS API -// Mocked endpoints tested: dynamic shape (-1), range (0-100) and static 0. -TEST_F(TfsPredictValidation, RequestWithZeroDim) { - std::vector shapes{ - ovms::Shape{200, ovms::Dimension::any(), 99}, // dynamic - ovms::Shape{200, ovms::Dimension{0, 100}, 99}, // range - ovms::Shape{200, 0, 99} // static - }; - - ovms::signed_shape_t actualRequestShape{200, 0, 99}; - - for (const auto& shape : shapes) { - servableInputs = ovms::tensor_map_t({{"Input", - std::make_shared("Input", ovms::Precision::FP32, shape, ovms::Layout{"N..."})}}); - preparePredictRequest(request, - {{"Input", - std::tuple{actualRequestShape, ovms::Precision::FP32}}}); - auto status = instance->mockValidate(&request); - EXPECT_TRUE(status.ok()); - } -} - -TEST_F(TfsPredictValidation, RequestNotEnoughInputs) { - request.mutable_inputs()->erase("Input_U8_1_3_62_62_NCHW"); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_NO_OF_INPUTS); -} - -TEST_F(TfsPredictValidation, RequestTooManyInputs) { - auto& inputD = (*request.mutable_inputs())["input_d"]; - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_NO_OF_INPUTS); -} - -TEST_F(TfsPredictValidation, RequestWrongInputName) { - auto input = (*request.mutable_inputs())["Input_I64_1_6_128_128_16_NCDHW"]; - request.mutable_inputs()->erase("Input_I64_1_6_128_128_16_NCDHW"); - (*request.mutable_inputs())["Some_Input"] = input; - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_MISSING_INPUT); -} - -TEST_F(TfsPredictValidation, RequestTooManyShapeDimensions) { - auto& input = (*request.mutable_inputs())["Input_FP32_1_224_224_3_NHWC"]; - input.mutable_tensor_shape()->add_dim()->set_size(16); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS); -} - -TEST_F(TfsPredictValidation, RequestNotEnoughShapeDimensions) { - auto& input = (*request.mutable_inputs())["Input_FP32_1_224_224_3_NHWC"]; - input.mutable_tensor_shape()->clear_dim(); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS); -} - -TEST_F(TfsPredictValidation, RequestWrongBatchSize) { - auto& input = (*request.mutable_inputs())["Input_U8_1_3_62_62_NCHW"]; - input.mutable_tensor_shape()->mutable_dim(0)->set_size(10); // dim(0) is batch size - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_BATCH_SIZE); -} - -TEST_F(TfsPredictValidation, RequestWrongBatchSizeAuto) { - modelConfig.setBatchingParams("auto"); - auto& input = (*request.mutable_inputs())["Input_U8_1_3_62_62_NCHW"]; - input.mutable_tensor_shape()->mutable_dim(0)->set_size(10); // dim(0) is batch size - prepareTensorContent(input); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::BATCHSIZE_CHANGE_REQUIRED); -} - -TEST_F(TfsPredictValidation, ValidRequestBinaryInputs) { - modelConfig.setBatchingParams("auto"); - std::string inputName = "Binary_Input"; - tensorflow::serving::PredictRequest binaryInputRequest; - - auto& input = (*binaryInputRequest.mutable_inputs())[inputName]; - input.set_dtype(tensorflow::DataType::DT_STRING); - const int requestBatchSize = 1; - for (int i = 0; i < requestBatchSize; i++) { - input.add_string_val("val"); - } - input.mutable_tensor_shape()->add_dim()->set_size(requestBatchSize); - - servableInputs.clear(); - ovms::shape_t shape = {1, 3, 224, 224}; - servableInputs[inputName] = std::make_shared( - inputName, - ovms::Precision::FP32, - shape, - ovms::Layout{"NHWC"}); - - auto status = instance->mockValidate(&binaryInputRequest); - EXPECT_TRUE(status.ok()); -} - -TEST_F(TfsPredictValidation, Batch0RequestBinaryInputs) { - modelConfig.setBatchingParams("0"); - std::string inputName = "Binary_Input"; - tensorflow::serving::PredictRequest binaryInputRequest; - - auto& input = (*binaryInputRequest.mutable_inputs())[inputName]; - input.set_dtype(tensorflow::DataType::DT_STRING); - const int requestBatchSize = 0; - input.mutable_tensor_shape()->add_dim()->set_size(requestBatchSize); - - servableInputs.clear(); - ovms::shape_t shape = {0, 3, 224, 224}; - servableInputs[inputName] = std::make_shared( - inputName, - ovms::Precision::FP32, - shape, - ovms::Layout{"NHWC"}); - - auto status = instance->mockValidate(&binaryInputRequest); - EXPECT_TRUE(status.ok()); -} - -TEST_F(TfsPredictValidation, RequestWrongBatchSizeBinaryInputs) { - std::string inputName = "Binary_Input"; - tensorflow::serving::PredictRequest binaryInputRequest; - - auto& input = (*binaryInputRequest.mutable_inputs())[inputName]; - input.set_dtype(tensorflow::DataType::DT_STRING); - const int requestBatchSize = 2; - for (int i = 0; i < requestBatchSize; i++) { - input.add_string_val("val"); - } - input.mutable_tensor_shape()->add_dim()->set_size(requestBatchSize); - - servableInputs.clear(); - ovms::shape_t shape = {1, 3, 224, 224}; - servableInputs[inputName] = std::make_shared( - inputName, - ovms::Precision::FP32, - shape, - ovms::Layout{"NHWC"}); - - auto status = instance->mockValidate(&binaryInputRequest); - EXPECT_EQ(status, ovms::StatusCode::INVALID_BATCH_SIZE); -} - -TEST_F(TfsPredictValidation, BinaryRequestEndpointScalar) { - std::string inputName = "Binary_Input"; - tensorflow::serving::PredictRequest binaryInputRequest; - - auto& input = (*binaryInputRequest.mutable_inputs())[inputName]; - input.set_dtype(tensorflow::DataType::DT_STRING); - const int requestBatchSize = 2; - for (int i = 0; i < requestBatchSize; i++) { - input.add_string_val("val"); - } - input.mutable_tensor_shape()->add_dim()->set_size(requestBatchSize); - - servableInputs.clear(); - ovms::shape_t shape = {}; - servableInputs[inputName] = std::make_shared( - inputName, - ovms::Precision::FP32, - shape, - ovms::Layout{"..."}); - - auto status = instance->mockValidate(&binaryInputRequest); - EXPECT_EQ(status, ovms::StatusCode::NOT_IMPLEMENTED); -} - -TEST_F(TfsPredictValidation, RequestWrongBatchSizeAutoBinaryInputs) { - modelConfig.setBatchingParams("auto"); - std::string inputName = "Binary_Input"; - tensorflow::serving::PredictRequest binaryInputRequest; - - auto& input = (*binaryInputRequest.mutable_inputs())[inputName]; - input.set_dtype(tensorflow::DataType::DT_STRING); - const int requestBatchSize = 2; - for (int i = 0; i < requestBatchSize; i++) { - input.add_string_val("val"); - } - input.mutable_tensor_shape()->add_dim()->set_size(requestBatchSize); - - servableInputs.clear(); - ovms::shape_t shape = {1, 3, 224, 224}; - servableInputs[inputName] = std::make_shared( - inputName, - ovms::Precision::FP32, - shape, - ovms::Layout{"NHWC"}); - - auto status = instance->mockValidate(&binaryInputRequest); - EXPECT_EQ(status, ovms::StatusCode::BATCHSIZE_CHANGE_REQUIRED); -} - -TEST_F(TfsPredictValidation, RequestWithScalarBatchSizeAuto) { - modelConfig.setBatchingParams("auto"); - - // First is incorrect, second is correct, but endpoint is abnormal anyway (scalar with batch size auto) - preparePredictRequest(request, {{"im_data", {{3, 3, 800, 1344}, ovms::Precision::FP32}}, - {"im_info", {std::vector{}, ovms::Precision::FP32}}}); - - servableInputs.clear(); - servableInputs = ovms::tensor_map_t{ - {"im_data", std::make_shared("im_data", ovms::Precision::FP32, ovms::shape_t{1, 3, 800, 1344}, ovms::Layout{"NCHW"})}, - {"im_info", std::make_shared("im_info", ovms::Precision::FP32, ovms::shape_t{}, ovms::Layout{"..."})}, - }; - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INTERNAL_ERROR); - - // First and second is correct, but endpoint is abnormal (scalar with batch size auto) - preparePredictRequest(request, {{"im_data", {{1, 3, 800, 1344}, ovms::Precision::FP32}}, - {"im_info", {std::vector{}, ovms::Precision::FP32}}}); - - status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INTERNAL_ERROR); -} - -TEST_F(TfsPredictValidation, RequestWrongAndCorrectBatchSizeAuto) { - modelConfig.setBatchingParams("auto"); - - // First is incorrect, second is correct - preparePredictRequest(request, {{"im_data", {{3, 3, 800, 1344}, ovms::Precision::FP32}}, - {"im_info", {{1, 3}, ovms::Precision::FP32}}}); - - servableInputs.clear(); - servableInputs = ovms::tensor_map_t{ - {"im_data", std::make_shared("im_data", ovms::Precision::FP32, ovms::shape_t{1, 3, 800, 1344}, ovms::Layout{"NCHW"})}, - {"im_info", std::make_shared("im_info", ovms::Precision::FP32, ovms::shape_t{1, 3}, ovms::Layout{"NC"})}, - }; - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::BATCHSIZE_CHANGE_REQUIRED); - - preparePredictRequest(request, {{"im_data", {{1, 3, 800, 1344}, ovms::Precision::FP32}}, - {"im_info", {{3, 3}, ovms::Precision::FP32}}}); - - status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::BATCHSIZE_CHANGE_REQUIRED); -} - -// Requesting 0 batch via TFS API -// Mocked endpoints tested: static shape with option batch=auto -// Expect force model batch change status. -TEST_F(TfsPredictValidation, RequestZeroDimBatchAuto) { - modelConfig.setBatchingParams("auto"); - - preparePredictRequest(request, {{"im_data", {{0, 3, 800, 1344}, ovms::Precision::FP32}}}); - - servableInputs.clear(); - servableInputs = ovms::tensor_map_t{ - {"im_data", std::make_shared("im_data", ovms::Precision::FP32, ovms::shape_t{1, 3, 800, 1344}, ovms::Layout{"NCHW"})}, - }; - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::BATCHSIZE_CHANGE_REQUIRED); -} - -TEST_F(TfsPredictValidation, RequestWrongAndCorrectShapeAuto) { - modelConfig.parseShapeParameter("auto"); - preparePredictRequest(request, {{"im_data", {{1, 3, 900, 1344}, ovms::Precision::FP32}}, - {"im_info", {{1, 3}, ovms::Precision::FP32}}}); - - // First is incorrect, second is correct - servableInputs.clear(); - servableInputs = ovms::tensor_map_t{ - {"im_data", std::make_shared("im_data", ovms::Precision::FP32, ovms::shape_t{1, 3, 800, 1344}, ovms::Layout{"NCHW"})}, - {"im_info", std::make_shared("im_info", ovms::Precision::FP32, ovms::shape_t{1, 3}, ovms::Layout{"NC"})}, - }; - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::RESHAPE_REQUIRED); - - // First is correct, second is incorrect - preparePredictRequest(request, {{"im_data", {{1, 3, 800, 1344}, ovms::Precision::FP32}}, - {"im_info", {{1, 6}, ovms::Precision::FP32}}}); - - status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::RESHAPE_REQUIRED); -} - -// Requesting with shape containing 0-value dim via TFS API -// Mocked endpoints tested: static shape with option shape=auto -// Expect force model reshape status. -TEST_F(TfsPredictValidation, RequestZeroDimShapeAuto) { - modelConfig.parseShapeParameter("auto"); - preparePredictRequest(request, {{"im_data", {{1, 3, 0, 1344}, ovms::Precision::FP32}}}); - - // First is incorrect, second is correct - servableInputs.clear(); - servableInputs = ovms::tensor_map_t{ - {"im_data", std::make_shared("im_data", ovms::Precision::FP32, ovms::shape_t{1, 3, 800, 1344}, ovms::Layout{"NCHW"})}, - }; - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::RESHAPE_REQUIRED); -} - -TEST_F(TfsPredictValidation, RequestWithScalarShapeAuto) { - modelConfig.parseShapeParameter("auto"); - - // First is incorrect, second is correct, expect reshape request due to shape=auto - preparePredictRequest(request, {{"im_data", {{1, 3, 801, 1344}, ovms::Precision::FP32}}, - {"im_info", {std::vector{}, ovms::Precision::FP32}}}); - - servableInputs.clear(); - servableInputs = ovms::tensor_map_t{ - {"im_data", std::make_shared("im_data", ovms::Precision::FP32, ovms::shape_t{1, 3, 800, 1344}, ovms::Layout{"NCHW"})}, - {"im_info", std::make_shared("im_info", ovms::Precision::FP32, ovms::shape_t{}, ovms::Layout{"..."})}, - }; - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::RESHAPE_REQUIRED) << status.string(); - - // First and second is correct, expect no further reshaping due to shape=auto - preparePredictRequest(request, {{"im_data", {{1, 3, 800, 1344}, ovms::Precision::FP32}}, - {"im_info", {std::vector{}, ovms::Precision::FP32}}}); - - status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::OK) << status.string(); -} - -TEST_F(TfsPredictValidation, RequestValidBatchSizeAuto) { - modelConfig.setBatchingParams("auto"); - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::OK); -} - -TEST_F(TfsPredictValidation, RequestWrongShapeValues) { - auto& input = (*request.mutable_inputs())["Input_U8_1_3_62_62_NCHW"]; - input.mutable_tensor_shape()->mutable_dim(0)->set_size(1); - input.mutable_tensor_shape()->mutable_dim(1)->set_size(4); - input.mutable_tensor_shape()->mutable_dim(2)->set_size(63); - input.mutable_tensor_shape()->mutable_dim(3)->set_size(63); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_SHAPE); -} - -TEST_F(TfsPredictValidation, RequestWrongShapeValuesTwoInputsOneWrong) { // one input fails validation, request denied - modelConfig.parseShapeParameter("{\"Input_U8_1_3_62_62_NCHW\": \"auto\"}"); - auto& input = (*request.mutable_inputs())["Input_U8_1_3_62_62_NCHW"]; - input.mutable_tensor_shape()->mutable_dim(0)->set_size(1); - input.mutable_tensor_shape()->mutable_dim(1)->set_size(4); - input.mutable_tensor_shape()->mutable_dim(2)->set_size(63); - input.mutable_tensor_shape()->mutable_dim(3)->set_size(63); - - auto& input2 = (*request.mutable_inputs())["Input_U16_1_2_8_4_NCHW"]; - input2.mutable_tensor_shape()->mutable_dim(0)->set_size(2); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_BATCH_SIZE); -} - -TEST_F(TfsPredictValidation, RequestWrongShapeValuesAuto) { - modelConfig.parseShapeParameter("{\"Input_U8_1_3_62_62_NCHW\": \"auto\"}"); - auto& input = (*request.mutable_inputs())["Input_U8_1_3_62_62_NCHW"]; - input.mutable_tensor_shape()->mutable_dim(0)->set_size(1); - input.mutable_tensor_shape()->mutable_dim(1)->set_size(4); - input.mutable_tensor_shape()->mutable_dim(2)->set_size(63); - input.mutable_tensor_shape()->mutable_dim(3)->set_size(63); - prepareTensorContent(input); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::RESHAPE_REQUIRED); -} - -TEST_F(TfsPredictValidation, RequestWrongShapeValuesAutoTwoInputs) { - modelConfig.parseShapeParameter("{\"Input_U8_1_3_62_62_NCHW\": \"auto\", \"Input_U16_1_2_8_4_NCHW\": \"auto\"}"); - - auto& input = (*request.mutable_inputs())["Input_U8_1_3_62_62_NCHW"]; - input.mutable_tensor_shape()->mutable_dim(0)->set_size(1); - input.mutable_tensor_shape()->mutable_dim(1)->set_size(4); - input.mutable_tensor_shape()->mutable_dim(2)->set_size(63); - input.mutable_tensor_shape()->mutable_dim(3)->set_size(63); - prepareTensorContent(input); - - auto& input2 = (*request.mutable_inputs())["Input_U16_1_2_8_4_NCHW"]; - input.mutable_tensor_shape()->mutable_dim(0)->set_size(1); - input.mutable_tensor_shape()->mutable_dim(1)->set_size(2); - input.mutable_tensor_shape()->mutable_dim(2)->set_size(16); - input.mutable_tensor_shape()->mutable_dim(3)->set_size(8); - prepareTensorContent(input); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::RESHAPE_REQUIRED); -} - -TEST_F(TfsPredictValidation, RequestWrongShapeValuesAutoNoNamedInput) { - modelConfig.parseShapeParameter("auto"); - - auto& input = (*request.mutable_inputs())["Input_U8_1_3_62_62_NCHW"]; - input.mutable_tensor_shape()->mutable_dim(0)->set_size(1); - input.mutable_tensor_shape()->mutable_dim(1)->set_size(4); - input.mutable_tensor_shape()->mutable_dim(2)->set_size(63); - input.mutable_tensor_shape()->mutable_dim(3)->set_size(63); - prepareTensorContent(input); - - auto& input2 = (*request.mutable_inputs())["Input_U16_1_2_8_4_NCHW"]; - input.mutable_tensor_shape()->mutable_dim(0)->set_size(1); - input.mutable_tensor_shape()->mutable_dim(1)->set_size(2); - input.mutable_tensor_shape()->mutable_dim(2)->set_size(16); - input.mutable_tensor_shape()->mutable_dim(3)->set_size(8); - prepareTensorContent(input); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::RESHAPE_REQUIRED); -} - -TEST_F(TfsPredictValidation, RequestWrongShapeValuesAutoFirstDim) { - modelConfig.parseShapeParameter("{\"Input_U8_1_3_62_62_NCHW\": \"auto\"}"); - auto& input = (*request.mutable_inputs())["Input_U8_1_3_62_62_NCHW"]; - input.mutable_tensor_shape()->mutable_dim(0)->set_size(2); - input.mutable_tensor_shape()->mutable_dim(1)->set_size(3); - input.mutable_tensor_shape()->mutable_dim(2)->set_size(62); - input.mutable_tensor_shape()->mutable_dim(3)->set_size(62); - prepareTensorContent(input); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::RESHAPE_REQUIRED); -} - -TEST_F(TfsPredictValidation, RequestValidShapeValuesTwoInputsFixed) { - modelConfig.parseShapeParameter("{\"Input_U8_1_3_62_62_NCHW\": \"(1,3,62,62)\", \"Input_U16_1_2_8_4_NCHW\": \"(1,2,8,4)\"}"); - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::OK); -} - -TEST_F(TfsPredictValidation, RequestWrongShapeValuesFixed) { - modelConfig.parseShapeParameter("{\"Input_U8_1_3_62_62_NCHW\": \"(1,3,62,62)\"}"); - - auto& input = (*request.mutable_inputs())["Input_U8_1_3_62_62_NCHW"]; - input.mutable_tensor_shape()->mutable_dim(0)->set_size(1); - input.mutable_tensor_shape()->mutable_dim(1)->set_size(4); - input.mutable_tensor_shape()->mutable_dim(2)->set_size(63); - input.mutable_tensor_shape()->mutable_dim(3)->set_size(63); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_SHAPE); -} -TEST_F(TfsPredictValidation, RequestWrongShapeValuesFixedFirstDim) { - modelConfig.parseShapeParameter("{\"Input_U8_1_3_62_62_NCHW\": \"(1,3,62,62)\"}"); - - auto& input = (*request.mutable_inputs())["Input_U8_1_3_62_62_NCHW"]; - input.mutable_tensor_shape()->mutable_dim(0)->set_size(2); - input.mutable_tensor_shape()->mutable_dim(1)->set_size(3); - input.mutable_tensor_shape()->mutable_dim(2)->set_size(62); - input.mutable_tensor_shape()->mutable_dim(3)->set_size(62); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_BATCH_SIZE); -} - -TEST_F(TfsPredictValidation, RequestIncorrectContentSize) { - auto& input = (*request.mutable_inputs())["Input_I64_1_6_128_128_16_NCDHW"]; - *input.mutable_tensor_content() = std::string(1 * 6, '1'); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_CONTENT_SIZE); -} - -TEST_F(TfsPredictValidation, RequestIncorrectContentSizeForScalarEndpoint) { - servableInputs = ovms::tensor_map_t({{"Input_FP32_Scalar", - std::make_shared("Input_FP32_Scalar", ovms::Precision::FP32, ovms::shape_t{}, ovms::Layout{"..."})}}); - preparePredictRequest(request, - {{"Input_FP32_Scalar", - std::tuple{std::vector{}, ovms::Precision::FP32}}}); - - auto& input = (*request.mutable_inputs())["Input_FP32_Scalar"]; - *input.mutable_tensor_content() = std::string(sizeof(float) + 1, '1'); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_CONTENT_SIZE); -} - -TEST_F(TfsPredictValidation, RequestIncorrectContentSizeBatchAuto) { - modelConfig.setBatchingParams("auto"); - auto& input = (*request.mutable_inputs())["Input_I64_1_6_128_128_16_NCDHW"]; - input.mutable_tensor_shape()->mutable_dim(0)->set_size(3); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_CONTENT_SIZE); -} - -TEST_F(TfsPredictValidation, RequestIncorrectContentSizeShapeAuto) { - modelConfig.parseShapeParameter("auto"); - auto& input = (*request.mutable_inputs())["Input_I64_1_6_128_128_16_NCDHW"]; - input.mutable_tensor_shape()->mutable_dim(1)->set_size(8); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_CONTENT_SIZE); -} - -TEST_F(TfsPredictValidation, RequestIncorrectValueCount) { - auto& input = (*request.mutable_inputs())["Input_U16_1_2_8_4_NCHW"]; - input.mutable_int_val()->Clear(); - input.mutable_int_val()->Resize(2, 1); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_VALUE_COUNT); -} - -TEST_F(TfsPredictValidation, RequestIncorrectValueCountBatchAuto) { - modelConfig.setBatchingParams("auto"); - auto& input = (*request.mutable_inputs())["Input_U16_1_2_8_4_NCHW"]; - input.mutable_tensor_shape()->mutable_dim(0)->set_size(3); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_VALUE_COUNT); -} - -TEST_F(TfsPredictValidation, RequestIncorrectValueCountShapeAuto) { - modelConfig.parseShapeParameter("auto"); - auto& input = (*request.mutable_inputs())["Input_U16_1_2_8_4_NCHW"]; - input.mutable_tensor_shape()->mutable_dim(2)->set_size(10); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_VALUE_COUNT); -} - -TEST_F(TfsPredictValidation, RequestWrongPrecision) { - auto& input = (*request.mutable_inputs())["Input_FP32_1_224_224_3_NHWC"]; - input.set_dtype(tensorflow::DataType::DT_UINT8); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_PRECISION); -} - -TEST_F(TfsPredictValidation, RequestNegativeValueInShape) { - auto& input = (*request.mutable_inputs())["Input_FP32_1_224_224_3_NHWC"]; - input.mutable_tensor_shape()->mutable_dim(1)->set_size(-4); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_SHAPE); -} - -class TfsPredictValidationArbitraryBatchPosition : public TfsPredictValidation { -protected: - void SetUp() override { - TfsPredictValidation::SetUp(); - - servableInputs = ovms::tensor_map_t({ - {"Input_FP32_224_224_3_1_HWCN", - std::make_shared("Input_FP32_224_224_3_1_HWCN", ovms::Precision::FP32, ovms::shape_t{224, 224, 3, 1}, ovms::Layout{"HWCN"})}, - {"Input_U8_3_1_128_CNH", - std::make_shared("Input_U8_3_1_128_CNH", ovms::Precision::U8, ovms::shape_t{3, 1, 128}, ovms::Layout{"CNH"})}, - }); - - preparePredictRequest(request, - { - {"Input_FP32_224_224_3_1_HWCN", - std::tuple{{224, 224, 3, 1}, ovms::Precision::FP32}}, - {"Input_U8_3_1_128_CNH", - std::tuple{{3, 1, 128}, ovms::Precision::U8}}, - }); - } -}; - -TEST_F(TfsPredictValidationArbitraryBatchPosition, Valid) { - auto status = instance->mockValidate(&request); - EXPECT_TRUE(status.ok()); -} - -TEST_F(TfsPredictValidationArbitraryBatchPosition, RequestWrongBatchSize) { - auto& input = (*request.mutable_inputs())["Input_FP32_224_224_3_1_HWCN"]; - - // Edit fourth dimension (N), expect validator to report wrong batch size instead of wrong shape. - input.mutable_tensor_shape()->mutable_dim(3)->set_size(10); - prepareTensorContent(input); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_BATCH_SIZE); -} - -TEST_F(TfsPredictValidationArbitraryBatchPosition, RequestWrongBatchSizeAuto) { - modelConfig.setBatchingParams("auto"); - - auto& input = (*request.mutable_inputs())["Input_FP32_224_224_3_1_HWCN"]; - - // Edit fourth dimension (N), expect validator to report batch size change request instead of reshape request. - input.mutable_tensor_shape()->mutable_dim(3)->set_size(10); - prepareTensorContent(input); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::BATCHSIZE_CHANGE_REQUIRED); -} - -TEST_F(TfsPredictValidationArbitraryBatchPosition, RequestWrongShapeValues) { - auto& input = (*request.mutable_inputs())["Input_FP32_224_224_3_1_HWCN"]; - - // Edit first dimension (H), expect validator to report wrong shape instead of wrong batch size. - input.mutable_tensor_shape()->mutable_dim(0)->set_size(10); - prepareTensorContent(input); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_SHAPE); -} - -TEST_F(TfsPredictValidationArbitraryBatchPosition, RequestWrongShapeValuesAuto) { - modelConfig.parseShapeParameter("auto"); - - auto& input = (*request.mutable_inputs())["Input_FP32_224_224_3_1_HWCN"]; - - // Edit first dimension (H), expect validator to report reshape request instead of requesting batch size change. - input.mutable_tensor_shape()->mutable_dim(0)->set_size(10); - prepareTensorContent(input); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::RESHAPE_REQUIRED); -} - -class TfsPredictValidationDynamicModel : public TfsPredictValidation { -protected: - void SetUp() override { - TfsPredictValidation::SetUp(); - - servableInputs = ovms::tensor_map_t({{"Input_FP32_any_224:512_224:512_3_NHWC", - std::make_shared("Input_FP32_any_224:512_224:512_3_NHWC", ovms::Precision::FP32, ovms::Shape{ovms::Dimension::any(), {224, 512}, {224, 512}, 3}, ovms::Layout{"NHWC"})}, - {"Input_U8_100:200_any_CN", - std::make_shared("Input_U8_100:200_any_CN", ovms::Precision::U8, ovms::Shape{{100, 200}, ovms::Dimension::any()}, ovms::Layout{"CN"})}}); - - ON_CALL(*instance, getBatchSize()).WillByDefault(Return(ovms::Dimension::any())); - - const ovms::dimension_value_t requestBatchSize = 16; - preparePredictRequest(request, - { - {"Input_FP32_any_224:512_224:512_3_NHWC", - std::tuple{{requestBatchSize, 300, 320, 3}, ovms::Precision::FP32}}, - {"Input_U8_100:200_any_CN", - std::tuple{{101, requestBatchSize}, ovms::Precision::U8}}, - }); - } -}; - -TEST_F(TfsPredictValidationDynamicModel, ValidRequest) { - auto status = instance->mockValidate(&request); - EXPECT_TRUE(status.ok()); -} - -TEST_F(TfsPredictValidationDynamicModel, RequestBatchNotInRangeFirstPosition) { - auto& input = (*request.mutable_inputs())["Input_FP32_any_224:512_224:512_3_NHWC"]; - input.mutable_tensor_shape()->mutable_dim(1)->set_size(98); // Should be in 1-5 range - - servableInputs["Input_FP32_any_224:512_224:512_3_NHWC"] = std::make_shared("Input_FP32_any_224:512_224:512_3_NHWC", ovms::Precision::FP32, ovms::Shape{{1, 5}, {224, 512}, {224, 512}, 3}, ovms::Layout{"NHWC"}); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_BATCH_SIZE); -} - -TEST_F(TfsPredictValidationDynamicModel, RequestDimensionNotInRangeFirstPosition) { - auto& input = (*request.mutable_inputs())["Input_U8_100:200_any_CN"]; - input.mutable_tensor_shape()->mutable_dim(0)->set_size(98); // Should be in 100-200 range - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_SHAPE); -} - -TEST_F(TfsPredictValidationDynamicModel, RequestBatchNotInRangeSecondPosition) { - auto& input = (*request.mutable_inputs())["Input_U8_100:200_any_CN"]; - input.mutable_tensor_shape()->mutable_dim(1)->set_size(98); // Should be in 1-5 range - - servableInputs["Input_U8_100:200_any_CN"] = std::make_shared("Input_U8_100:200_any_CN", ovms::Precision::U8, ovms::Shape{{100, 200}, {1, 5}}, ovms::Layout{"CN"}); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_BATCH_SIZE); -} - -TEST_F(TfsPredictValidationDynamicModel, RequestDimensionNotInRangeSecondPosition) { - auto& input = (*request.mutable_inputs())["Input_FP32_any_224:512_224:512_3_NHWC"]; - input.mutable_tensor_shape()->mutable_dim(1)->set_size(223); // Should be in 224-512 range - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_SHAPE); -} - -TEST_F(TfsPredictValidationDynamicModel, RequestDimensionInRangeWrongTensorContent) { - auto& input = (*request.mutable_inputs())["Input_U8_100:200_any_CN"]; - - size_t numberOfElements = 1; - for (int i = 0; i < input.tensor_shape().dim_size(); i++) { - numberOfElements *= input.tensor_shape().dim(i).size(); - } - numberOfElements -= 1; - *input.mutable_tensor_content() = std::string(numberOfElements * tensorflow::DataTypeSize(input.dtype()), '1'); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_CONTENT_SIZE); -} - -// Regression test: tensor buffer size validation must use overflow-safe arithmetic. -// Both dimensions are near INT64_MAX/2; their product (~2^124) greatly exceeds SIZE_MAX. -// The server must detect the overflow and reject the request with INVALID_CONTENT_SIZE. -TEST_F(TfsPredictValidationDynamicModel, RequestContentSizeIntegerOverflowRejected) { - // Each dim ~2^62; product ~2^124 overflows size_t. - constexpr int64_t kOverflowDim0 = std::numeric_limits::max() / 2; - constexpr int64_t kOverflowDim1 = std::numeric_limits::max() / 2; - // What naive unchecked arithmetic produces (wraps around SIZE_MAX). - constexpr size_t kWrappedExpectedSize = - static_cast(kOverflowDim0) * static_cast(kOverflowDim1) * sizeof(float); - - servableInputs = ovms::tensor_map_t({{"overflow_input", - std::make_shared("overflow_input", ovms::Precision::FP32, - ovms::Shape{ovms::Dimension::any(), ovms::Dimension::any()}, ovms::Layout{"NC"})}}); - ON_CALL(*instance, getBatchSize()).WillByDefault(Return(ovms::Dimension::any())); - - request.Clear(); - auto& input = (*request.mutable_inputs())["overflow_input"]; - input.set_dtype(tensorflow::DataType::DT_FLOAT); - auto* shape = input.mutable_tensor_shape(); - shape->add_dim()->set_size(kOverflowDim0); - shape->add_dim()->set_size(kOverflowDim1); - *input.mutable_tensor_content() = std::string(kWrappedExpectedSize, 'A'); - - auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_CONTENT_SIZE) << status.string(); -} - -class TfsPredictValidationPrecision : public ::testing::TestWithParam { -protected: - void SetUp() override { - auto precision = ovms::Precision::FP32; - mockedInputsInfo[tensorName] = std::make_shared(tensorName, precision, ovms::shape_t{1, DUMMY_MODEL_INPUT_SIZE}, ovms::Layout{"NC"}); - } - tensorflow::serving::PredictRequest request; - const char* tensorName = DUMMY_MODEL_INPUT_NAME; - ovms::tensor_map_t mockedInputsInfo; - ovms::tensor_map_t mockedOutputsInfo; -}; - -TEST_P(TfsPredictValidationPrecision, ValidPrecisions) { - ovms::Precision testedPrecision = GetParam(); - mockedInputsInfo[tensorName] = createTensorInfoCopyWithPrecision(mockedInputsInfo[tensorName], testedPrecision); - preparePredictRequest(request, - { - {tensorName, - std::tuple{{1, DUMMY_MODEL_INPUT_SIZE}, testedPrecision}}, - }); - auto status = ovms::request_validation_utils::validate(request, mockedInputsInfo, this->mockedOutputsInfo, "dummy", ovms::model_version_t{1}); - EXPECT_EQ(status, ovms::StatusCode::OK) << "Precision validation failed:" - << toString(testedPrecision) - << " should pass validation"; -} - -INSTANTIATE_TEST_SUITE_P( - Test, - TfsPredictValidationPrecision, - ::testing::ValuesIn(SUPPORTED_INPUT_PRECISIONS), - [](const ::testing::TestParamInfo& info) { - return toString(info.param); - }); - class KFSPredictValidation : public ::testing::Test { protected: std::unique_ptr ieCore; @@ -1905,14 +1046,6 @@ static void prepareInferStringInputWithTwoDimensionShapeTensor(::KFSRequest& req tensor->add_shape(1); } -static void prepareInferStringInputWithTwoDimensionShapeTensor(tensorflow::serving::PredictRequest& request, const std::string& name) { - request.mutable_inputs()->clear(); - auto& input = (*request.mutable_inputs())[name]; - input.set_dtype(tensorflow::DataType::DT_STRING); - input.mutable_tensor_shape()->add_dim()->set_size(1); - input.mutable_tensor_shape()->add_dim()->set_size(1); -} - static void prepareInferStringInputWithNegativeShape(::KFSRequest& request, const std::string& name) { KFSTensorInputProto* tensor = request.add_inputs(); tensor->set_name(name); @@ -1921,13 +1054,6 @@ static void prepareInferStringInputWithNegativeShape(::KFSRequest& request, cons tensor->add_shape(-5); } -static void prepareInferStringInputWithNegativeShape(tensorflow::serving::PredictRequest& request, const std::string& name) { - request.mutable_inputs()->clear(); - auto& input = (*request.mutable_inputs())[name]; - input.set_dtype(tensorflow::DataType::DT_STRING); - input.mutable_tensor_shape()->add_dim()->set_size(-5); -} - static void prepareInferStringInputWithZeroDimShape(::KFSRequest& request, const std::string& name) { KFSTensorInputProto* tensor = request.add_inputs(); tensor->set_name(name); @@ -1936,13 +1062,6 @@ static void prepareInferStringInputWithZeroDimShape(::KFSRequest& request, const tensor->add_shape(0); } -static void prepareInferStringInputWithZeroDimShape(tensorflow::serving::PredictRequest& request, const std::string& name) { - request.mutable_inputs()->clear(); - auto& input = (*request.mutable_inputs())[name]; - input.set_dtype(tensorflow::DataType::DT_STRING); - input.mutable_tensor_shape()->add_dim()->set_size(0); -} - template class PredictValidationString2DTest : public ::testing::Test { protected: @@ -1956,7 +1075,7 @@ class PredictValidationString2DTest : public ::testing::Test { } }; -using MyTypes = ::testing::Types; +using MyTypes = ::testing::Types<::KFSRequest>; TYPED_TEST_SUITE(PredictValidationString2DTest, MyTypes); TYPED_TEST(PredictValidationString2DTest, positive) { @@ -1974,8 +1093,6 @@ TYPED_TEST(PredictValidationString2DTest, positive) { } TYPED_TEST(PredictValidationString2DTest, positive_data_in_buffer) { - if (typeid(TypeParam) == typeid(TFSRequestType)) - GTEST_SKIP() << "String inputs in buffer not supported for TFS api"; // bs=1 std::vector inputStrings = {"String_123"}; prepareInferStringRequest(this->request, this->tensorName, inputStrings, false); @@ -2006,8 +1123,6 @@ TYPED_TEST(PredictValidationString2DTest, negative_over_1gb_after_expansion) { } TYPED_TEST(PredictValidationString2DTest, negative_no_string_in_buffer) { - if (typeid(TypeParam) == typeid(TFSRequestType)) - GTEST_SKIP() << "String inputs in buffer not supported for TFS api"; std::vector inputStrings = {}; prepareInferStringRequest(this->request, this->tensorName, inputStrings, false); auto status = ovms::request_validation_utils::validate(this->request, this->mockedInputsInfo, this->mockedOutputsInfo, "dummy", ovms::model_version_t{1}); @@ -2072,7 +1187,7 @@ class PredictValidationString1DTest : public ::testing::Test { } }; -using MyTypes = ::testing::Types; +using MyTypes = ::testing::Types<::KFSRequest>; TYPED_TEST_SUITE(PredictValidationString1DTest, MyTypes); TYPED_TEST(PredictValidationString1DTest, positive) { diff --git a/src/test/prediction_service_test.cpp b/src/test/prediction_service_test.cpp deleted file mode 100644 index 8e36f48187..0000000000 --- a/src/test/prediction_service_test.cpp +++ /dev/null @@ -1,2156 +0,0 @@ -//***************************************************************************** -// Copyright 2020-2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "../capi_frontend/capi_utils.hpp" -#include "../tfs_frontend/tfs_utils.hpp" -#include "../kfs_frontend/kfs_utils.hpp" -#include "../capi_frontend/deserialization.hpp" -#include "../tfs_frontend/deserialization.hpp" -#include "../kfs_frontend/deserialization.hpp" -#include "../deserialization_main.hpp" - -#include "../capi_frontend/serialization.hpp" -#include "../kfs_frontend/serialization.hpp" -#include "../tfs_frontend/serialization.hpp" - -#include "kfs_frontend/kfs_request_utils.hpp" -#include "tfs_frontend/tfs_request_utils.hpp" -#include "../inference_executor.hpp" - -#include "../capi_frontend/buffer.hpp" -#include "../capi_frontend/inferenceparameter.hpp" -#include "../capi_frontend/inferencerequest.hpp" -#include "../capi_frontend/inferenceresponse.hpp" -#include "../capi_frontend/inferencetensor.hpp" -#include "../executingstreamidguard.hpp" -#include "../modelinstance.hpp" -#include "../modelinstanceunloadguard.hpp" -#include "../modelversion.hpp" -#include "../regularovtensorfactory.hpp" -#include "constructor_enabled_model_manager.hpp" -#include "test_models_configs.hpp" -#include "test_utils.hpp" -#include "light_test_utils.hpp" -#include "platform_utils.hpp" -#include "test_with_temp_dir.hpp" - -using testing::Each; -using testing::ElementsAre; -using testing::Eq; - -using ovms::Buffer; -using ovms::InferenceResponse; -using ovms::InferenceTensor; -using ovms::StatusCode; - -const int SLEEP_TIME_AFTER_THREAD_STARTED_MS = 1; - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wnarrowing" -static void serializeAndCheck(int outputSize, ov::InferRequest& inferRequest, const std::string& outputName, const ovms::tensor_map_t& outputsInfo) { - std::vector output(outputSize); - tensorflow::serving::PredictResponse response; - ovms::OutputGetter outputGetter(inferRequest); - auto status = serializePredictResponse(outputGetter, UNUSED_SERVABLE_NAME, UNUSED_MODEL_VERSION, outputsInfo, &response, ovms::getTensorInfoName); - ASSERT_EQ(status, ovms::StatusCode::OK) << status.string(); - ASSERT_EQ(response.outputs().count(outputName), 1) << "Did not find:" << outputName; - std::memcpy(output.data(), (float*)response.outputs().at(outputName).tensor_content().data(), outputSize * sizeof(float)); - EXPECT_THAT(output, Each(Eq(1.))); -} - -static ovms::Status getOutput(const KFSResponse& response, const std::string& name, KFSOutputTensorIteratorType& it, size_t& bufferId) { - it = response.outputs().begin(); - bufferId = 0; - while (it != response.outputs().end()) { - if (it->name() == name) { - break; - } - ++it; - ++bufferId; - } - if (it != response.outputs().end()) { - return StatusCode::OK; - } - return StatusCode::INVALID_MISSING_INPUT; -} - -static ovms::Status getOutput(const TFSResponseType& response, const std::string& name, TFSOutputTensorIteratorType& it, size_t& bufferId) { - it = response.outputs().find(name); - if (it != response.outputs().end()) { - return StatusCode::OK; - } - return StatusCode::INVALID_MISSING_INPUT; -} - -using inputs_info_elem_t = std::pair>; -static size_t calculateByteSize(const inputs_info_elem_t& e) { - auto& [inputName, shapeDatatypeTuple] = e; - auto& [shape, precision] = shapeDatatypeTuple; - size_t shapeProduct = std::accumulate(shape.begin(), shape.end(), 1, std::multiplies()); - return shapeProduct * ovms::DataTypeToByteSize(ovms::getPrecisionAsOVMSDataType(precision)); -} -template -class Preparer { - std::vector>> dataKeeper; - -public: - void preparePredictRequest(RequestType& request, inputs_info_t requestInputs) { - ::preparePredictRequest(request, requestInputs); - } -}; -template <> -void Preparer::preparePredictRequest(ovms::InferenceRequest& request, inputs_info_t requestInputs) { - auto inputWithGreatestRequirements = std::max_element(requestInputs.begin(), requestInputs.end(), [](const inputs_info_elem_t& a, const inputs_info_elem_t& b) { - return calculateByteSize(a) < calculateByteSize(b); - }); - size_t byteSizeToPreserve = calculateByteSize(*inputWithGreatestRequirements); - auto& currentData = dataKeeper.emplace_back(std::make_unique>(byteSizeToPreserve)); - memset(reinterpret_cast(const_cast(currentData->data())), '1', byteSizeToPreserve); - ::preparePredictRequest(request, requestInputs, *currentData); -} - -template -class TestPredict : public ::testing::Test { -public: - void SetUp() { - ovms::ModelConfig config = DUMMY_MODEL_CONFIG; - const int initialBatchSize = 1; - config.setBatchSize(initialBatchSize); - config.setNireq(2); - } - /** - * @brief This function should mimic most closely predict request to check for thread safety - */ - void performPredict(const std::string modelName, - const ovms::model_version_t modelVersion, - const RequestType& request, - std::unique_ptr> waitBeforeGettingModelInstance = nullptr, - std::unique_ptr> waitBeforePerformInference = nullptr); - - void testConcurrentPredicts(const int initialBatchSize, const uint32_t waitingBeforePerformInferenceCount, const uint32_t waitingBeforeGettingModelCount) { - ASSERT_GE(20, waitingBeforePerformInferenceCount); - config.setNireq(20); - ASSERT_EQ(manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - std::vector> releaseWaitBeforeGettingModelInstance(waitingBeforeGettingModelCount); - std::vector> releaseWaitBeforePerformInference(waitingBeforePerformInferenceCount); - std::vector> threadsWaitingBeforeGettingModelInstanceStarted(waitingBeforeGettingModelCount); - std::vector> threadsWaitingBeforePerformInferenceStarted(waitingBeforePerformInferenceCount); - std::promise thread1Started, thread2Started; - - std::vector predictsWaitingBeforeGettingModelInstance; - std::vector predictsWaitingBeforeInference; - for (auto i = 0u; i < waitingBeforeGettingModelCount; ++i) { - predictsWaitingBeforeGettingModelInstance.emplace_back( - std::thread( - [this, initialBatchSize, &releaseWaitBeforeGettingModelInstance, &threadsWaitingBeforeGettingModelInstanceStarted, i]() { - RequestType request; - Preparer preparer; - preparer.preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{(initialBatchSize + (i % 3)), 10}, ovms::Precision::FP32}}}); - threadsWaitingBeforeGettingModelInstanceStarted[i].set_value(); - performPredict(config.getName(), config.getVersion(), request, - std::make_unique>(releaseWaitBeforeGettingModelInstance[i].get_future())); - })); - } - for (auto i = 0u; i < waitingBeforePerformInferenceCount; ++i) { - predictsWaitingBeforeInference.emplace_back( - std::thread( - [this, initialBatchSize, &releaseWaitBeforePerformInference, &threadsWaitingBeforePerformInferenceStarted, i]() { - RequestType request; - Preparer preparer; - preparer.preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{initialBatchSize, 10}, ovms::Precision::FP32}}}); - threadsWaitingBeforePerformInferenceStarted[i].set_value(); - performPredict(config.getName(), config.getVersion(), request, nullptr, - std::make_unique>(releaseWaitBeforePerformInference[i].get_future())); - })); - } - // sleep to allow all threads to initialize - for (auto& p : threadsWaitingBeforeGettingModelInstanceStarted) { - p.get_future().get(); - } - for (auto& p : threadsWaitingBeforePerformInferenceStarted) { - p.get_future().get(); - } - std::this_thread::sleep_for(std::chrono::milliseconds(SLEEP_TIME_AFTER_THREAD_STARTED_MS)); - for (auto& promise : releaseWaitBeforeGettingModelInstance) { - promise.set_value(); - } - for (auto& promise : releaseWaitBeforePerformInference) { - promise.set_value(); - } - for (auto& thread : predictsWaitingBeforeGettingModelInstance) { - thread.join(); - } - for (auto& thread : predictsWaitingBeforeInference) { - thread.join(); - } - } - - void testConcurrentBsChanges(const int initialBatchSize, const uint32_t numberOfThreads) { - ASSERT_GE(20, numberOfThreads); - config.setNireq(20); - ASSERT_EQ(manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - std::vector> releaseWaitBeforeGettingModelInstance(numberOfThreads); - std::vector> threadsWaitingBeforeGettingModelInstanceStarted(numberOfThreads); - std::vector predictThreads; - for (auto i = 0u; i < numberOfThreads; ++i) { - predictThreads.emplace_back( - std::thread( - [this, initialBatchSize, &releaseWaitBeforeGettingModelInstance, &threadsWaitingBeforeGettingModelInstanceStarted, i]() { - RequestType request; - Preparer preparer; - preparer.preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{(initialBatchSize + i), 10}, ovms::Precision::FP32}}}); - threadsWaitingBeforeGettingModelInstanceStarted[i].set_value(); - performPredict(config.getName(), config.getVersion(), request, - std::make_unique>(releaseWaitBeforeGettingModelInstance[i].get_future())); - })); - } - // sleep to allow all threads to initialize - for (auto& p : threadsWaitingBeforeGettingModelInstanceStarted) { - p.get_future().get(); - } - std::this_thread::sleep_for(std::chrono::milliseconds(SLEEP_TIME_AFTER_THREAD_STARTED_MS)); - for (auto& promise : releaseWaitBeforeGettingModelInstance) { - promise.set_value(); - } - - for (auto& thread : predictThreads) { - thread.join(); - } - } - - void checkOutputShape(const ResponseType& response, const ovms::signed_shape_t& shape, const std::string& outputName = "a"); - - static void checkOutputValuesString(const TFSResponseType& response, const std::vector& expectedValues, const std::string& outputName = PASSTHROUGH_STRING_MODEL_OUTPUT_NAME, bool checkRaw = true) { - ASSERT_EQ(response.outputs().size(), 1); - ASSERT_EQ(response.outputs().count(outputName), 1); - const auto& proto = response.outputs().at(outputName); - ASSERT_EQ(proto.string_val().size(), expectedValues.size()); - for (size_t i = 0; i < expectedValues.size(); i++) { - ASSERT_EQ(proto.string_val(i), expectedValues[i]); - } - } - - static void checkOutputValuesString(const KFSResponse& response, const std::vector& expectedValues, const std::string& outputName = PASSTHROUGH_STRING_MODEL_OUTPUT_NAME, bool checkRaw = true) { - KFSOutputTensorIteratorType it; - size_t bufferId; - auto status = getOutput(response, outputName, it, bufferId); - ASSERT_TRUE(status.ok()) << "Couldn't find output:" << outputName; - auto& responseOutput = *it; - ASSERT_EQ(responseOutput.datatype(), "BYTES"); - ASSERT_EQ(responseOutput.shape().size(), 1); - ASSERT_EQ(responseOutput.shape()[0], expectedValues.size()); - if (checkRaw) { - const std::string& data = response.raw_output_contents(bufferId); - size_t offset = 0; - for (size_t i = 0; i < expectedValues.size(); i++) { - ASSERT_GE(data.size(), offset + 4); - uint32_t batchLength = *((uint32_t*)(data.data() + offset)); - ASSERT_EQ(batchLength, expectedValues[i].size()); - offset += 4; - ASSERT_GE(data.size(), offset + batchLength); - ASSERT_EQ(std::string(data.data() + offset, batchLength), expectedValues[i]); - offset += batchLength; - } - ASSERT_EQ(offset, data.size()); - } else { - ASSERT_EQ(0, response.raw_output_contents().size()); - ASSERT_EQ(responseOutput.contents().bytes_contents().size(), expectedValues.size()); - for (size_t i = 0; i < expectedValues.size(); i++) { - ASSERT_EQ(responseOutput.contents().bytes_contents()[i], expectedValues[i]); - } - } - } - - static void checkOutputValuesString(const ovms::InferenceResponse& res, const std::vector& expectedValues, const std::string& outputName = PASSTHROUGH_STRING_MODEL_OUTPUT_NAME, bool checkRaw = true) { - FAIL() << "not supported"; - } - - static void checkOutputValuesU8(const TFSResponseType& response, const std::vector& expectedValues, const std::string& outputName = INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME, bool checkRaw = true) { - ASSERT_EQ(response.outputs().count(outputName), 1); - const auto& output_tensor = response.outputs().at(outputName); - uint8_t* buffer = reinterpret_cast(const_cast(output_tensor.tensor_content().data())); - std::vector actualValues(buffer, buffer + output_tensor.tensor_content().size() / sizeof(uint8_t)); - ASSERT_EQ(actualValues.size(), expectedValues.size()); - ASSERT_EQ(0, std::memcmp(actualValues.data(), expectedValues.data(), expectedValues.size() * sizeof(uint8_t))) - << readableError(expectedValues.data(), actualValues.data(), expectedValues.size() * sizeof(uint8_t)); - } - static void checkOutputValues(const TFSResponseType& response, const std::vector& expectedValues, const std::string& outputName = INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME) { - ASSERT_EQ(response.outputs().count(outputName), 1); - const auto& output_tensor = response.outputs().at(outputName); - float* buffer = reinterpret_cast(const_cast(output_tensor.tensor_content().data())); - std::vector actualValues(buffer, buffer + output_tensor.tensor_content().size() / sizeof(float)); - ASSERT_EQ(0, std::memcmp(actualValues.data(), expectedValues.data(), expectedValues.size() * sizeof(float))) - << readableError(expectedValues.data(), actualValues.data(), expectedValues.size() * sizeof(float)); - } - static void checkOutputValuesU8(const ovms::InferenceResponse& res, const std::vector& expectedValues, const std::string& outputName = INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME, bool checkRaw = true) { - FAIL() << "not supported"; - } - static void checkOutputValues(const ovms::InferenceResponse& res, const std::vector& expectedValues, const std::string& outputName = INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME) { - InferenceResponse& response = const_cast(res); - size_t outputCount = response.getOutputCount(); - ASSERT_GE(1, outputCount); - size_t outputId = 0; - while (outputId < outputCount) { - const std::string* cppName; - InferenceTensor* tensor; - auto status = response.getOutput(outputId, &cppName, &tensor); - ASSERT_EQ(status, StatusCode::OK) << status.string(); - ASSERT_NE(nullptr, tensor); - ASSERT_NE(nullptr, cppName); - if (outputName == *cppName) { - const Buffer* buffer = tensor->getBuffer(); - ASSERT_NE(nullptr, buffer); - ASSERT_EQ(expectedValues.size() * sizeof(float), buffer->getByteSize()); - float* bufferRaw = reinterpret_cast(const_cast(buffer->data())); - ASSERT_EQ(0, std::memcmp(bufferRaw, expectedValues.data(), expectedValues.size() * sizeof(float))) - << readableError(expectedValues.data(), bufferRaw, expectedValues.size() * sizeof(float)); - return; - } - ++outputId; - } - ASSERT_TRUE(false) << "did not found output with name: " << outputName; - } - static void checkOutputValues(const KFSResponse& response, const std::vector& expectedValues, const std::string& outputName = INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME) { - KFSOutputTensorIteratorType it; - size_t bufferId; - auto status = getOutput(response, outputName, it, bufferId); - ASSERT_TRUE(status.ok()) << "Couldn't find output:" << outputName; - if (response.raw_output_contents().size() > 0) { - float* buffer = reinterpret_cast(const_cast(response.raw_output_contents(bufferId).data())); - ASSERT_EQ(0, std::memcmp(buffer, expectedValues.data(), expectedValues.size() * sizeof(float))) - << readableError(expectedValues.data(), buffer, expectedValues.size() * sizeof(float)); - } else { - auto& responseOutput = *it; - if (responseOutput.datatype() == "FP32") { - for (size_t i = 0; i < expectedValues.size(); i++) { - ASSERT_EQ(responseOutput.contents().fp32_contents()[i], expectedValues[i]); - } - } else if (responseOutput.datatype() == "BYTES") { - ASSERT_EQ(0, std::memcmp(&responseOutput.contents().bytes_contents(), expectedValues.data(), expectedValues.size() * sizeof(float))); - } - } - } - static void checkOutputValuesU8(const KFSResponse& response, const std::vector& expectedValues, const std::string& outputName = INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME, bool checkRaw = true) { - KFSOutputTensorIteratorType it; - size_t bufferId; - auto status = getOutput(response, outputName, it, bufferId); - ASSERT_TRUE(status.ok()) << "Couldn't find output:" << outputName; - if (checkRaw) { - ASSERT_GT(response.raw_output_contents().size(), 0); - uint8_t* buffer = reinterpret_cast(const_cast(response.raw_output_contents(bufferId).data())); - ASSERT_EQ(response.raw_output_contents(bufferId).size(), expectedValues.size()); - ASSERT_EQ(0, std::memcmp(buffer, expectedValues.data(), expectedValues.size() * sizeof(uint8_t))) - << readableError(expectedValues.data(), buffer, expectedValues.size() * sizeof(uint8_t)); - } else { - auto& responseOutput = *it; - ASSERT_EQ(responseOutput.datatype(), "UINT8") << "other precision testing not supported"; - ASSERT_EQ(expectedValues.size(), responseOutput.contents().uint_contents().size()); - for (size_t i = 0; i < expectedValues.size(); i++) { - ASSERT_EQ(expectedValues[i], responseOutput.contents().uint_contents(i)) - << "Wrong value at index " << i << ", expected: " << expectedValues[i] << " actual: " << responseOutput.contents().uint_contents(i); - } - } - } - - ovms::Status performInferenceWithRequest(const RequestType& request, ResponseType& response, const std::string& servableName = "dummy") { - std::shared_ptr model; - std::unique_ptr unload_guard; - auto status = manager.getModelInstance(servableName, 0, model, unload_guard); - if (!status.ok()) { - return status; - } - response.Clear(); - return ovms::infer(*model, &request, &response, unload_guard); - } - - ovms::Status performInferenceWithShape(ResponseType& response, const ovms::signed_shape_t& shape = {1, 10}, const ovms::Precision precision = ovms::Precision::FP32) { - RequestType request; - Preparer preparer; - preparer.preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, std::tuple{shape, precision}}}); - return performInferenceWithRequest(request, response); - } - - ovms::Status performInferenceWithBatchSize(ResponseType& response, int batchSize = 1, const ovms::Precision precision = ovms::Precision::FP32, const size_t batchSizePosition = 0) { - ovms::signed_shape_t shape = {1, 10}; - shape[batchSizePosition] = batchSize; - RequestType request; - Preparer preparer; - preparer.preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, std::tuple{shape, precision}}}); - return performInferenceWithRequest(request, response); - } - - ovms::Status performInferenceWithImageInput(ResponseType& response, const ovms::signed_shape_t& shape, const std::vector& data = {}, const std::string& servableName = "increment_1x3x4x5", int batchSize = 1, const ovms::Precision precision = ovms::Precision::FP32) { - RequestType request; - Preparer preparer; - if (data.size()) { - preparePredictRequest(request, - {{INCREMENT_1x3x4x5_MODEL_INPUT_NAME, std::tuple{shape, precision}}}, data); - } else { - preparer.preparePredictRequest(request, - {{INCREMENT_1x3x4x5_MODEL_INPUT_NAME, std::tuple{shape, precision}}}); - } - return performInferenceWithRequest(request, response, servableName); - } - - ovms::Status performInferenceWithBinaryImageInput(ResponseType& response, const std::string& inputName, const std::string& servableName = "increment_1x3x4x5", int batchSize = 1) { - RequestType request; - prepareBinaryPredictRequest(request, inputName, batchSize); - return performInferenceWithRequest(request, response, servableName); - } - -public: - ConstructorEnabledModelManager manager; - ovms::ModelConfig config = DUMMY_MODEL_CONFIG; - ~TestPredict() { - std::cout << "Destructor of TestPredict()" << std::endl; - } -}; - -template <> -void TestPredict::checkOutputShape(const TFSResponseType& response, const ovms::signed_shape_t& shape, const std::string& outputName) { - ASSERT_EQ(response.outputs().count(outputName), 1); - const auto& output_tensor = response.outputs().at(outputName); - ASSERT_EQ(output_tensor.tensor_shape().dim_size(), shape.size()); - for (unsigned int i = 0; i < shape.size(); i++) { - EXPECT_EQ(output_tensor.tensor_shape().dim(i).size(), shape[i]); - } -} - -template <> -void TestPredict::checkOutputShape(const ovms::InferenceResponse& cresponse, const ovms::signed_shape_t& shape, const std::string& outputName) { - size_t outputCount = cresponse.getOutputCount(); - EXPECT_GE(1, outputCount); - size_t outputId = 0; - while (outputId < outputCount) { - const std::string* cppName; - InferenceTensor* tensor; - InferenceResponse& response = const_cast(cresponse); - auto status = response.getOutput(outputId, &cppName, &tensor); - EXPECT_EQ(status, StatusCode::OK) << status.string(); - EXPECT_NE(nullptr, tensor); - EXPECT_NE(nullptr, cppName); - if (outputName == *cppName) { - auto resultShape = tensor->getShape(); - EXPECT_EQ(shape.size(), resultShape.size()); - for (size_t i = 0; i < shape.size(); ++i) { - EXPECT_EQ(resultShape[i], shape[i]); - } - } - ++outputId; - } - return; -} - -template <> -void TestPredict::checkOutputShape(const KFSResponse& response, const ovms::signed_shape_t& shape, const std::string& outputName) { - auto it = response.outputs().begin(); - size_t bufferId; - auto status = getOutput(response, outputName, it, bufferId); - ASSERT_EQ(status, StatusCode::OK); - ASSERT_EQ(it->shape().size(), shape.size()); - for (unsigned int i = 0; i < shape.size(); i++) { - EXPECT_EQ(it->shape()[i], shape[i]); - } -} - -class MockModelInstance : public ovms::ModelInstance { -public: - MockModelInstance(ov::Core& ieCore) : - ModelInstance(UNUSED_SERVABLE_NAME, UNUSED_MODEL_VERSION, ieCore) {} - template - const ovms::Status mockValidate(const RequestType* request) { - return ovms::request_validation_utils::validate(*request, - this->getInputsInfo(), - this->getOutputsInfo(), - this->getName(), - this->getVersion(), - this->getOptionalInputNames(), - this->getModelConfig().getBatchingMode(), - this->getModelConfig().getShapes()); - } -}; -const size_t DUMMY_DIM_POS = 1; -template -static size_t extractDummyOutputSize(const RequestType& request); - -template <> -size_t extractDummyOutputSize(const TFSPredictRequest& request) { - auto it = request.inputs().begin(); - EXPECT_NE(it, request.inputs().end()); - auto shape = it->second.tensor_shape(); - return shape.dim(DUMMY_DIM_POS).size(); -} -template <> -size_t extractDummyOutputSize(const KFSRequest& request) { - auto it = request.inputs().begin(); - EXPECT_NE(it, request.inputs().end()); - auto shape = it->shape(); - return shape[DUMMY_DIM_POS]; -} -template <> -size_t extractDummyOutputSize(const ovms::InferenceRequest& request) { - return request.getRequestShapes().begin()->second[DUMMY_DIM_POS]; -} - -template -static void performPrediction(const std::string modelName, - const ovms::model_version_t modelVersion, - const RequestType& request, - std::unique_ptr> waitBeforeGettingModelInstance, - std::unique_ptr> waitBeforePerformInference, - ovms::ModelManager& manager, - const std::string& inputName, - const std::string& outputName) { - // only validation is skipped - std::shared_ptr modelInstance; - std::unique_ptr modelInstanceUnloadGuard; - - auto bsPositionIndex = 0; - auto requestBatchSizeOpt = ovms::getRequestBatchSize(&request, bsPositionIndex); - ASSERT_TRUE(requestBatchSizeOpt); - ASSERT_TRUE(requestBatchSizeOpt.value().isStatic()); - auto requestBatchSize = requestBatchSizeOpt.value().getStaticValue(); - - if (waitBeforeGettingModelInstance) { - std::cout << "Waiting before getModelInstance. Batch size: " << requestBatchSize << std::endl; - waitBeforeGettingModelInstance->get(); - } - ASSERT_EQ(manager.getModelInstance(modelName, modelVersion, modelInstance, modelInstanceUnloadGuard), ovms::StatusCode::OK); - - if (waitBeforePerformInference) { - std::cout << "Waiting before performInfernce." << std::endl; - waitBeforePerformInference->get(); - } - ovms::Status validationStatus = (std::static_pointer_cast(modelInstance))->mockValidate(&request); - ASSERT_TRUE(validationStatus == ovms::StatusCode::OK || - validationStatus == ovms::StatusCode::RESHAPE_REQUIRED || - validationStatus == ovms::StatusCode::BATCHSIZE_CHANGE_REQUIRED); - auto requestShapes = ovms::getRequestShapes(&request); - ASSERT_EQ(modelInstance->reloadModelIfRequired(validationStatus, requestBatchSize, requestShapes, modelInstanceUnloadGuard), ovms::StatusCode::OK); - - ovms::ExecutingStreamIdGuard executingStreamIdGuard(modelInstance->getInferRequestsQueue(), modelInstance->getMetricReporter()); - ov::InferRequest& inferRequest = executingStreamIdGuard.getInferRequest(); - ovms::InputSink inputSink(inferRequest); - bool isPipeline = false; - - std::unordered_map> factories; - factories.emplace(OVMS_BUFFERTYPE_CPU, std::make_shared()); - auto status = ovms::deserializePredictRequest>(request, modelInstance->getInputsInfo(), modelInstance->getOutputsInfo(), inputSink, isPipeline, factories); - ASSERT_EQ(status, ovms::StatusCode::OK); - status = modelInstance->performInference(inferRequest); - ASSERT_EQ(status, ovms::StatusCode::OK); - size_t outputSize = requestBatchSize * extractDummyOutputSize(request); - serializeAndCheck(outputSize, inferRequest, outputName, modelInstance->getOutputsInfo()); -} -template -void TestPredict::performPredict(const std::string modelName, - const ovms::model_version_t modelVersion, - const RequestType& request, - std::unique_ptr> waitBeforeGettingModelInstance, - std::unique_ptr> waitBeforePerformInference) { - performPrediction(modelName, - modelVersion, - request, - std::move(waitBeforeGettingModelInstance), - std::move(waitBeforePerformInference), - this->manager, - DUMMY_MODEL_INPUT_NAME, - DUMMY_MODEL_OUTPUT_NAME); -} - -using MyTypes = ::testing::Types; -TYPED_TEST_SUITE(TestPredict, MyTypes); - -TYPED_TEST(TestPredict, SuccesfullOnDummyModel) { - typename TypeParam::first_type request; - Preparer preparer; - preparer.preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{1, 10}, ovms::Precision::FP32}}}); - ovms::ModelConfig config = DUMMY_MODEL_CONFIG; - config.setBatchSize(1); - - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - this->performPredict(config.getName(), config.getVersion(), request); -} - -static const char* oneDummyWithMappedInputConfig = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"latest": {"num_versions":1}}, - "nireq": 10, - "shape": {"input_tensor": "(1,10) "} - } - } - ] -})"; - -static const char* oneDummyWithMappedInputSpecificAutoShapeConfig = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"latest": {"num_versions":1}}, - "nireq": 10, - "shape": {"input_tensor": "auto"} - } - } - ] -})"; -static const char* oneDummyWithMappedInputAnonymousAutoShapeConfig = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"latest": {"num_versions":1}}, - "nireq": 10, - "shape": "auto" - } - } - ] -})"; -template -class TestPredictWithMapping : public TestWithTempDir { -public: - std::string ovmsConfig; - std::string modelPath; - std::string configFilePath; - std::string mappingConfigPath; - const std::string dummyModelInputMapping = "input_tensor"; - const std::string dummyModelOutputMapping = "output_tensor"; - - void SetUpConfig(const std::string& configContent) { - ovmsConfig = configContent; - const std::string modelPathToReplace{"/ovms/src/test/dummy"}; - auto it = ovmsConfig.find(modelPathToReplace); - if (it != std::string::npos) { - ovmsConfig.replace(ovmsConfig.find(modelPathToReplace), modelPathToReplace.size(), modelPath); - } - configFilePath = directoryPath + "/ovms_config.json"; - } - void SetUp() { - TestWithTempDir::SetUp(); - } - void SetUp(const std::string& configContent) { - modelPath = directoryPath + "/dummy/"; - mappingConfigPath = modelPath + "1/mapping_config.json"; - SetUpConfig(configContent); - std::filesystem::copy(getGenericFullPathForSrcTest("/ovms/src/test/dummy"), modelPath, std::filesystem::copy_options::recursive); - createConfigFileWithContent(ovmsConfig, configFilePath); - createConfigFileWithContent(R"({ - "inputs": {"b":"input_tensor"}, - "outputs": {"a": "output_tensor"} - })", - mappingConfigPath); - } -}; - -TYPED_TEST_SUITE(TestPredictWithMapping, MyTypes); - -TYPED_TEST(TestPredictWithMapping, SuccesfullOnDummyModelWithMapping) { - Preparer preparer; - typename TypeParam::first_type request; - preparer.preparePredictRequest(request, - {{this->dummyModelInputMapping, - std::tuple{{1, 10}, ovms::Precision::FP32}}}); - this->SetUp(oneDummyWithMappedInputConfig); - ovms::ModelConfig config = DUMMY_MODEL_CONFIG; - ConstructorEnabledModelManager manager; - auto status = manager.loadConfig(this->configFilePath); - ASSERT_EQ(status, ovms::StatusCode::OK) << status.string(); - performPrediction(config.getName(), config.getVersion(), request, nullptr, nullptr, manager, this->dummyModelInputMapping, this->dummyModelOutputMapping); -} - -TYPED_TEST(TestPredictWithMapping, SuccesfullOnPassthrough_2D_U8ModelWithMapping) { - if (typeid(typename TypeParam::first_type) == typeid(ovms::InferenceRequest)) - GTEST_SKIP() << "String inputs not supported for C-API"; - this->modelPath = this->directoryPath + "/passthrough/"; - this->mappingConfigPath = this->modelPath + "1/mapping_config.json"; - std::filesystem::copy(getGenericFullPathForSrcTest("/ovms/src/test/passthrough"), this->modelPath, std::filesystem::copy_options::recursive); - this->ovmsConfig = R"( -{ - "model_config_list": [ - { - "config": { - "name": "passhtrough_u8", - "base_path": "/ovms/src/test/passthrough" - } - } - ] -})"; - const std::string modelPathToReplace{"/ovms/src/test/passthrough"}; - auto it = this->ovmsConfig.find(modelPathToReplace); - ASSERT_NE(it, std::string::npos); - this->ovmsConfig.replace(this->ovmsConfig.find(modelPathToReplace), modelPathToReplace.size(), this->modelPath); - this->configFilePath = this->directoryPath + "/ovms_config.json"; - createConfigFileWithContent(this->ovmsConfig, this->configFilePath); - createConfigFileWithContent(R"({ - "outputs": {"copy:0": "copy:0_string"} - })", - this->mappingConfigPath); - ConstructorEnabledModelManager manager; - auto status = manager.loadConfig(this->configFilePath); - ASSERT_EQ(status, ovms::StatusCode::OK) << status.string(); - typename TypeParam::first_type request; - prepareInferStringRequest(request, PASSTHROUGH_MODEL_INPUT_NAME, {"String_123", "", "zebra"}); - std::shared_ptr modelInstance; - std::unique_ptr modelInstanceUnloadGuard; - ASSERT_EQ(manager.getModelInstance("passhtrough_u8", 1, modelInstance, modelInstanceUnloadGuard), ovms::StatusCode::OK); - typename TypeParam::second_type response; - ASSERT_EQ(ovms::infer(*modelInstance, &request, &response, modelInstanceUnloadGuard), ovms::StatusCode::OK); - assertStringResponse(response, {"String_123", "", "zebra"}, "copy:0_string"); -} - -TYPED_TEST(TestPredictWithMapping, SuccesfullOnDummyModelWithMappingSpecificShapeAuto) { - Preparer preparer; - typename TypeParam::first_type request; - preparer.preparePredictRequest(request, - {{this->dummyModelInputMapping, - std::tuple{{1, 5}, ovms::Precision::FP32}}}); - this->SetUp(oneDummyWithMappedInputSpecificAutoShapeConfig); - ovms::ModelConfig config = DUMMY_MODEL_CONFIG; - auto status = config.parseShapeParameter("auto"); - ConstructorEnabledModelManager manager; - status = manager.loadConfig(this->configFilePath); - ASSERT_EQ(status, ovms::StatusCode::OK) << status.string(); - performPrediction(config.getName(), config.getVersion(), request, nullptr, nullptr, manager, this->dummyModelInputMapping, this->dummyModelOutputMapping); -} -TYPED_TEST(TestPredictWithMapping, SuccesfullOnDummyModelWithMappingAnonymousShapeAuto) { - Preparer preparer; - typename TypeParam::first_type request; - preparer.preparePredictRequest(request, - {{this->dummyModelInputMapping, - std::tuple{{1, 5}, ovms::Precision::FP32}}}); - this->SetUp(oneDummyWithMappedInputAnonymousAutoShapeConfig); - ovms::ModelConfig config = DUMMY_MODEL_CONFIG; - auto status = config.parseShapeParameter("auto"); - ConstructorEnabledModelManager manager; - status = manager.loadConfig(this->configFilePath); - ASSERT_EQ(status, ovms::StatusCode::OK) << status.string(); - performPrediction(config.getName(), config.getVersion(), request, nullptr, nullptr, manager, this->dummyModelInputMapping, this->dummyModelOutputMapping); -} - -TYPED_TEST(TestPredict, SuccesfullReloadFromAlreadyLoadedWithNewBatchSize) { - Preparer preparer; - typename TypeParam::first_type request; - preparer.preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{1, 10}, ovms::Precision::FP32}}}); - ovms::ModelConfig config = DUMMY_MODEL_CONFIG; - const auto initialBatchSize = config.getBatchSize(); - config.setBatchSize(initialBatchSize); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - this->performPredict(config.getName(), config.getVersion(), request); -} - -TYPED_TEST(TestPredict, SuccesfullReloadWhen1InferenceInProgress) { - // FIRST LOAD MODEL WITH BS=1 - Preparer preparer; - typename TypeParam::first_type requestBs1; - preparer.preparePredictRequest(requestBs1, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{1, 10}, ovms::Precision::FP32}}}); - typename TypeParam::first_type requestBs2; - preparer.preparePredictRequest(requestBs2, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{2, 10}, ovms::Precision::FP32}}}); - - this->config.setBatchingParams("auto"); - this->config.setNireq(2); - ASSERT_EQ(this->manager.reloadModelWithVersions(this->config), ovms::StatusCode::OK_RELOADED); - - std::promise releaseWaitBeforePerformInferenceBs1, releaseWaitBeforeGetModelInstanceBs2; - std::promise thread1Started, thread2Started; - std::thread t1( - [this, &requestBs1, &releaseWaitBeforePerformInferenceBs1, &thread1Started]() { - thread1Started.set_value(); - this->performPredict(this->config.getName(), this->config.getVersion(), requestBs1, nullptr, - std::make_unique>(releaseWaitBeforePerformInferenceBs1.get_future())); - }); - std::thread t2( - [this, &requestBs2, &releaseWaitBeforeGetModelInstanceBs2, &thread2Started]() { - thread2Started.set_value(); - this->performPredict(this->config.getName(), this->config.getVersion(), requestBs2, - std::make_unique>(releaseWaitBeforeGetModelInstanceBs2.get_future()), - nullptr); - }); - thread1Started.get_future().get(); - thread2Started.get_future().get(); - std::this_thread::sleep_for(std::chrono::milliseconds(SLEEP_TIME_AFTER_THREAD_STARTED_MS)); - releaseWaitBeforePerformInferenceBs1.set_value(); - releaseWaitBeforeGetModelInstanceBs2.set_value(); - t1.join(); - t2.join(); -} - -TYPED_TEST(TestPredict, SuccesfullReloadWhen1InferenceAboutToStart) { - // FIRST LOAD MODEL WITH BS=1 - Preparer preparer; - typename TypeParam::first_type requestBs2; - preparer.preparePredictRequest(requestBs2, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{2, 10}, ovms::Precision::FP32}}}); - typename TypeParam::first_type requestBs1; - preparer.preparePredictRequest(requestBs1, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{1, 10}, ovms::Precision::FP32}}}); - - this->config.setBatchingParams("auto"); - this->config.setNireq(2); - ASSERT_EQ(this->manager.reloadModelWithVersions(this->config), ovms::StatusCode::OK_RELOADED); - - std::promise releaseWaitBeforeGetModelInstanceBs1, releaseWaitBeforePerformInferenceBs2; - std::promise thread1Started, thread2Started; - std::thread t1( - [this, &requestBs1, &releaseWaitBeforeGetModelInstanceBs1, &thread1Started]() { - thread1Started.set_value(); - this->performPredict(this->config.getName(), this->config.getVersion(), requestBs1, - std::make_unique>(releaseWaitBeforeGetModelInstanceBs1.get_future()), - nullptr); - }); - std::thread t2( - [this, &requestBs2, &releaseWaitBeforePerformInferenceBs2, &thread2Started]() { - thread2Started.set_value(); - this->performPredict(this->config.getName(), this->config.getVersion(), requestBs2, nullptr, - std::make_unique>(releaseWaitBeforePerformInferenceBs2.get_future())); - }); - thread1Started.get_future().get(); - thread2Started.get_future().get(); - std::this_thread::sleep_for(std::chrono::milliseconds(SLEEP_TIME_AFTER_THREAD_STARTED_MS)); - releaseWaitBeforePerformInferenceBs2.set_value(); - releaseWaitBeforeGetModelInstanceBs1.set_value(); - t1.join(); - t2.join(); -} - -TYPED_TEST(TestPredict, SuccesfullReloadWhenSeveralInferRequestJustBeforeGettingModelInstance) { - const int initialBatchSize = 1; - this->config.setBatchingParams("auto"); - - const uint32_t waitingBeforePerformInferenceCount = 0; - const uint32_t waitingBeforeGettingModelCount = 9; - this->testConcurrentPredicts(initialBatchSize, waitingBeforePerformInferenceCount, waitingBeforeGettingModelCount); -} - -TYPED_TEST(TestPredict, SuccesfullReloadWhenSeveralInferRequestJustBeforeInference) { - const int initialBatchSize = 1; - this->config.setBatchingParams("auto"); - - const uint32_t waitingBeforePerformInferenceCount = 9; - const uint32_t waitingBeforeGettingModelCount = 0; - this->testConcurrentPredicts(initialBatchSize, waitingBeforePerformInferenceCount, waitingBeforeGettingModelCount); -} - -TYPED_TEST(TestPredict, SuccesfullReloadWhenSeveralInferRequestAtDifferentStages) { - const int initialBatchSize = 1; - this->config.setBatchingParams("auto"); - - const uint32_t waitingBeforePerformInferenceCount = 9; - const uint32_t waitingBeforeGettingModelCount = 9; - this->testConcurrentPredicts(initialBatchSize, waitingBeforePerformInferenceCount, waitingBeforeGettingModelCount); -} - -TYPED_TEST(TestPredict, SuccesfullReloadForMultipleThreadsDifferentBS) { - const int initialBatchSize = 2; - this->config.setBatchingParams("auto"); - - const uint32_t numberOfThreads = 5; - this->testConcurrentBsChanges(initialBatchSize, numberOfThreads); -} -TYPED_TEST(TestPredict, SuccesfullReshapeViaRequestOnDummyModel) { - // Prepare model this->manager with dynamic shaped dummy model, originally loaded with 1x10 shape - ovms::ModelConfig config = DUMMY_MODEL_CONFIG; - config.setBatchingParams(""); - config.parseShapeParameter("auto"); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - // Prepare request with 1x5 shape, expect reshape - Preparer preparer; - typename TypeParam::first_type request; - preparer.preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{1, 5}, ovms::Precision::FP32}}}); - - typename TypeParam::second_type response; - - // Do the inference - auto status = this->performInferenceWithRequest(request, response, "dummy"); - ASSERT_EQ(status, StatusCode::OK) << status.string(); - // Expect reshape to 1x5 - this->checkOutputShape(response, {1, 5}, DUMMY_MODEL_OUTPUT_NAME); -} - -TYPED_TEST(TestPredict, SuccesfullInferenceOnModelWithScalar) { - ovms::ModelConfig config = SCALAR_MODEL_CONFIG; - config.setBatchingParams(""); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - // Prepare request with empty shape - Preparer preparer; - typename TypeParam::first_type request; - preparer.preparePredictRequest(request, - {{SCALAR_MODEL_INPUT_NAME, - std::tuple{std::vector{}, ovms::Precision::FP32}}}); - - typename TypeParam::second_type response; - - // Do the inference - auto status = this->performInferenceWithRequest(request, response, "scalar"); - ASSERT_EQ(status, StatusCode::OK) << status.string(); - this->checkOutputShape(response, {}, SCALAR_MODEL_OUTPUT_NAME); -} - -TYPED_TEST(TestPredict, Succesfull0DimInferenceOnModelWithDynamicBatch) { - ovms::ModelConfig config = DUMMY_MODEL_CONFIG; - config.setBatchingParams(""); - config.parseShapeParameter("(-1,2)"); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - // Prepare request with empty shape - Preparer preparer; - typename TypeParam::first_type request; - preparer.preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{0, 2}, ovms::Precision::FP32}}}); - - typename TypeParam::second_type response; - - // Do the inference - auto status = this->performInferenceWithRequest(request, response, "dummy"); - ASSERT_EQ(status, StatusCode::OK) << status.string(); - this->checkOutputShape(response, {0, 2}, DUMMY_MODEL_OUTPUT_NAME); -} - -TYPED_TEST(TestPredict, Succesfull0DimInferenceOnModelWithDynamicDim) { - ovms::ModelConfig config = DUMMY_MODEL_CONFIG; - config.setBatchingParams(""); - config.parseShapeParameter("(1,-1)"); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - // Prepare request with empty shape - Preparer preparer; - typename TypeParam::first_type request; - preparer.preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{1, 0}, ovms::Precision::FP32}}}); - - typename TypeParam::second_type response; - - // Do the inference - auto status = this->performInferenceWithRequest(request, response, "dummy"); - ASSERT_EQ(status, StatusCode::OK) << status.string(); - this->checkOutputShape(response, {1, 0}, DUMMY_MODEL_OUTPUT_NAME); -} - -// TODO: Re-enable positive check when models with static 0 dimension become available in OpenVINO -TYPED_TEST(TestPredict, DISABLED_Succesfull0DimInferenceOnModelWithStaticZeroDim) { - ovms::ModelConfig config = DUMMY_MODEL_CONFIG; - config.setBatchingParams(""); - config.parseShapeParameter("(1,0)"); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - // Prepare request with empty shape - Preparer preparer; - typename TypeParam::first_type request; - preparer.preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{1, 0}, ovms::Precision::FP32}}}); - - typename TypeParam::second_type response; - - // Do the inference - auto status = this->performInferenceWithRequest(request, response, "dummy"); - ASSERT_EQ(status, StatusCode::OK) << status.string(); - this->checkOutputShape(response, {1, 0}, DUMMY_MODEL_OUTPUT_NAME); -} - -TYPED_TEST(TestPredict, Succesfull0DimInferenceOnBatchAutoModel) { - ovms::ModelConfig config = DUMMY_MODEL_CONFIG; - config.setBatchingParams("auto"); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - // Prepare request with empty shape - Preparer preparer; - typename TypeParam::first_type request; - preparer.preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{0, 10}, ovms::Precision::FP32}}}); - - typename TypeParam::second_type response; - - // Do the inference - auto status = this->performInferenceWithRequest(request, response, "dummy"); - ASSERT_EQ(status, StatusCode::OK) << status.string(); - this->checkOutputShape(response, {0, 10}, DUMMY_MODEL_OUTPUT_NAME); - - // Prepare non 0-dim request, test recovery - preparer.preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{1, 10}, ovms::Precision::FP32}}}); - status = this->performInferenceWithRequest(request, response, "dummy"); - ASSERT_EQ(status, StatusCode::OK) << status.string(); - this->checkOutputShape(response, {1, 10}, DUMMY_MODEL_OUTPUT_NAME); -} - -TYPED_TEST(TestPredict, Succesfull0DimInferenceOnShapeAutoModel) { - ovms::ModelConfig config = DUMMY_MODEL_CONFIG; - config.setBatchingParams(""); - config.parseShapeParameter("auto"); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - // Prepare request with empty shape - Preparer preparer; - typename TypeParam::first_type request; - preparer.preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{1, 0}, ovms::Precision::FP32}}}); - - typename TypeParam::second_type response; - - // Do the inference - auto status = this->performInferenceWithRequest(request, response, "dummy"); - ASSERT_EQ(status, StatusCode::OK) << status.string(); - this->checkOutputShape(response, {1, 0}, DUMMY_MODEL_OUTPUT_NAME); - - // Prepare non 0-dim request, test recovery - preparer.preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{1, 10}, ovms::Precision::FP32}}}); - status = this->performInferenceWithRequest(request, response, "dummy"); - ASSERT_EQ(status, StatusCode::OK) << status.string(); - this->checkOutputShape(response, {1, 10}, DUMMY_MODEL_OUTPUT_NAME); -} - -TYPED_TEST(TestPredict, NegativeInferenceOnModelWithScalarBatchAuto) { - ovms::ModelConfig config = SCALAR_MODEL_CONFIG; - config.setBatchingParams("auto"); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::MODEL_WITH_SCALAR_AUTO_UNSUPPORTED); -} - -TYPED_TEST(TestPredict, NegativeInferenceOnModelWithScalarShapeAuto) { - ovms::ModelConfig config = SCALAR_MODEL_CONFIG; - config.setBatchingParams(""); - config.parseShapeParameter("auto"); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - // Prepare request with 1-dim shape - Preparer preparer; - typename TypeParam::first_type request; - preparer.preparePredictRequest(request, - {{SCALAR_MODEL_INPUT_NAME, - std::tuple{{1}, ovms::Precision::FP32}}}); - - typename TypeParam::second_type response; - - // Do the inference, expect wrong number of dimensions - auto status = this->performInferenceWithRequest(request, response, "scalar"); - ASSERT_EQ(status, StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS) << status.string(); -} - -/* - * Scenario - perform inferences with different shapes and model reload via this->config.json change - * - * 1. Load model with shape=auto, initial internal shape (1,10) - * 2. Do the inference with (1,12) shape - expect status OK and result (1,12) - * 3. Reshape model to fixed=(1,11) with this->config.json change - * 4. Do the inference with (1,12) shape - expect status INVALID_SHAPE - * 5. Do the inference with (1,11) shape - expect status OK and result (1,11) - * 6. Reshape model back to shape=auto, initial internal shape (1,10) - * 7. Do the inference with (1,12) shape - expect status OK and result (1,12) - * - */ -TYPED_TEST(TestPredict, ReshapeViaRequestAndConfigChange) { - using namespace ovms; - - // Prepare model with shape=auto (initially (1,10) shape) - ModelConfig config = DUMMY_MODEL_CONFIG; - config.setBatchingParams(""); - config.parseShapeParameter("auto"); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - typename TypeParam::second_type response; - - // Perform reshape to (1,12) using request - ASSERT_EQ(this->performInferenceWithShape(response, {1, 12}), ovms::StatusCode::OK); - this->checkOutputShape(response, {1, 12}); - - // Reshape with model reload to Fixed=(1,11) - config.setBatchingParams(""); - config.parseShapeParameter("(1,11)"); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - // Cannot do the inference with (1,12) - ASSERT_EQ(this->performInferenceWithShape(response, {1, 12}), ovms::StatusCode::INVALID_SHAPE); - - // Successful inference with (1,11) - ASSERT_EQ(this->performInferenceWithShape(response, {1, 11}), ovms::StatusCode::OK); - this->checkOutputShape(response, {1, 11}); - - // Reshape back to AUTO, internal shape is (1,10) - config.setBatchingParams(""); - config.parseShapeParameter("auto"); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - // Perform reshape to (1,12) using request - ASSERT_EQ(this->performInferenceWithShape(response, {1, 12}), ovms::StatusCode::OK); - this->checkOutputShape(response, {1, 12}); -} -/* - * Scenario - perform inferences with different batch size and model reload via this->config.json change - * - * 1. Load model with bs=auto, initial internal shape (1,10) - * 2. Do the inference with (3,10) shape - expect status OK and result (3,10) - * 3. Change model batch size to fixed=4 with this->config.json change - * 4. Do the inference with (3,10) shape - expect status INVALID_BATCH_SIZE - * 5. Do the inference with (4,10) shape - expect status OK and result (4,10) - * 6. Reshape model back to batchsize=auto, initial internal shape (1,10) - * 7. Do the inference with (3,10) shape - expect status OK and result (3,10) - */ -TYPED_TEST(TestPredict, ChangeBatchSizeViaRequestAndConfigChange) { - using namespace ovms; - // Prepare model with shape=auto (initially (1,10) shape) - ModelConfig config = DUMMY_MODEL_CONFIG; - this->config.setBatchingParams("auto"); - ASSERT_EQ(this->manager.reloadModelWithVersions(this->config), ovms::StatusCode::OK_RELOADED); - - typename TypeParam::second_type response; - - // Perform batch size change to 3 using request - ASSERT_EQ(this->performInferenceWithBatchSize(response, 3), ovms::StatusCode::OK); - this->checkOutputShape(response, {3, 10}); - - // Change batch size with model reload to Fixed=4 - this->config.setBatchingParams("4"); - ASSERT_EQ(this->manager.reloadModelWithVersions(this->config), ovms::StatusCode::OK_RELOADED); - - // Cannot do the inference with (3,10) - ASSERT_EQ(this->performInferenceWithBatchSize(response, 3), ovms::StatusCode::INVALID_BATCH_SIZE); - - // Successful inference with (4,10) - ASSERT_EQ(this->performInferenceWithBatchSize(response, 4), ovms::StatusCode::OK); - this->checkOutputShape(response, {4, 10}); - - // Reshape back to AUTO, internal shape is (1,10) - this->config.setBatchingParams("auto"); - ASSERT_EQ(this->manager.reloadModelWithVersions(this->config), ovms::StatusCode::OK_RELOADED); - - // Perform batch change to 3 using request - ASSERT_EQ(this->performInferenceWithBatchSize(response, 3), ovms::StatusCode::OK); - this->checkOutputShape(response, {3, 10}); -} - -/* - * Scenario - perform inference with NHWC input layout changed via this->config.json. - * - * 1. Load model with layout=nhwc:nchw, initial internal layout: nchw, initial shape=(1,3,4,5) - * 2. Do the inference with (1,4,5,3) shape - expect status OK and result (1,3,4,5) - * 3. Do the inference with (1,3,4,5) shape - expect INVALID_SHAPE - * 4. Remove layout setting - * 5. Do the inference with (1,3,4,5) shape - expect status OK and result (1,3,4,5) - * 6. Do the inference with (1,4,5,3) shape - expect INVALID_SHAPE - * 7. Adding layout setting to nchw - * 8. Do the inference with (1,3,4,5) shape - expect status OK and result (1,3,4,5) - * 9. Do the inference with (1,4,5,3) shape - expect INVALID_SHAPE - */ -TYPED_TEST(TestPredict, PerformInferenceChangeModelInputLayout) { - using namespace ovms; - - // Prepare model with changed layout to nhwc (internal layout=nchw) - ModelConfig config = INCREMENT_1x3x4x5_MODEL_CONFIG; - config.setBatchingParams(""); - ASSERT_EQ(config.parseLayoutParameter("nhwc:nchw"), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - typename TypeParam::second_type response; - - // Perform inference with NHWC layout, ensure status OK and correct results - auto status = this->performInferenceWithImageInput(response, {1, 4, 5, 3}); - ASSERT_EQ(status, ovms::StatusCode::OK) << status.string(); - this->checkOutputShape(response, {1, 3, 4, 5}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - // Perform inference with NCHW layout, ensure error - ASSERT_EQ(this->performInferenceWithImageInput(response, {1, 3, 4, 5}), ovms::StatusCode::INVALID_SHAPE); - - // Reload model with layout setting removed, model is back to NCHW - ASSERT_EQ(config.parseLayoutParameter(""), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - // Perform inference with NCHW layout, ensure status OK and correct results - ASSERT_EQ(this->performInferenceWithImageInput(response, {1, 3, 4, 5}), ovms::StatusCode::OK); - this->checkOutputShape(response, {1, 3, 4, 5}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - - // Perform inference with NHWC layout, ensure error - ASSERT_EQ(this->performInferenceWithImageInput(response, {1, 4, 5, 3}), ovms::StatusCode::INVALID_SHAPE); - - // Prepare model with layout changed back to nchw - ASSERT_EQ(config.parseLayoutParameter("nchw"), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - // Perform inference with NCHW layout, ensure OK - ASSERT_EQ(this->performInferenceWithImageInput(response, {1, 3, 4, 5}), ovms::StatusCode::OK); - this->checkOutputShape(response, {1, 3, 4, 5}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - - // Perform inference with NHWC layout, ensure error - ASSERT_EQ(this->performInferenceWithImageInput(response, {1, 4, 5, 3}), ovms::StatusCode::INVALID_SHAPE); -} -/* - * Scenario - perform inference with NHWC input layout changed and shape changed via this->config.json. - * - * 1. Load model with layout=nchw:nhwc and shape=(1,1,2,3), initial internal layout: nchw, initial shape=(1,3,4,5) - * 2. Do the inference with (1,1,2,3) shape - expect status OK and result (1,3,1,2) - * 3. Do the inference with (1,3,1,2) shape - expect INVALID_SHAPE - * 4. Remove layout setting - * 5. Do the inference with (1,1,2,3) shape - expect status OK and result (1,3,1,2) - * 6. Do the inference with (1,3,1,2) shape - expect INVALID_SHAPE - * 7. Adding layout setting to nchw - * 8. Do the inference with (1,3,1,2) shape - expect status OK and result (1,3,1,2) - * 9. Do the inference with (1,1,2,3) shape - expect INVALID_SHAPE - */ -TYPED_TEST(TestPredict, PerformInferenceChangeModelInputLayoutAndShape) { - using namespace ovms; - - // Prepare model with changed layout to nhwc (internal layout=nchw) - ModelConfig config = INCREMENT_1x3x4x5_MODEL_CONFIG; - config.setBatchingParams(""); - ASSERT_EQ(config.parseShapeParameter("(1,1,2,3)"), ovms::StatusCode::OK); - ASSERT_EQ(config.parseLayoutParameter("nhwc:nchw"), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - typename TypeParam::second_type response; - - // Perform inference with NHWC layout, ensure status OK and correct results - ASSERT_EQ(this->performInferenceWithImageInput(response, {1, 1, 2, 3}, {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}), ovms::StatusCode::OK); - this->checkOutputShape(response, {1, 3, 1, 2}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - this->checkOutputValues(response, {2.0, 5.0, 3.0, 6.0, 4.0, 7.0}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - - // Perform inference with NCHW layout, ensure error - ASSERT_EQ(this->performInferenceWithImageInput(response, {1, 3, 1, 2}, {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}), ovms::StatusCode::INVALID_SHAPE); - - // Reload model with layout setting removed, model is back to NCHW - ASSERT_EQ(config.parseShapeParameter("(1,3,1,2)"), ovms::StatusCode::OK); - ASSERT_EQ(config.parseLayoutParameter(""), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - // Perform inference with NCHW layout, ensure status OK and correct results - ASSERT_EQ(this->performInferenceWithImageInput(response, {1, 3, 1, 2}, {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}), ovms::StatusCode::OK); - this->checkOutputShape(response, {1, 3, 1, 2}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - this->checkOutputValues(response, {2.0, 3.0, 4.0, 5.0, 6.0, 7.0}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - - // Perform inference with NHWC layout, ensure error - ASSERT_EQ(this->performInferenceWithImageInput(response, {1, 1, 2, 3}, {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}), ovms::StatusCode::INVALID_SHAPE); - - // Prepare model with layout changed back to nchw - ASSERT_EQ(config.parseShapeParameter("(1,3,1,2)"), ovms::StatusCode::OK); - ASSERT_EQ(config.parseLayoutParameter("nchw"), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - // Perform inference with NCHW layout, ensure OK - ASSERT_EQ(this->performInferenceWithImageInput(response, {1, 3, 1, 2}, {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}), ovms::StatusCode::OK); - this->checkOutputShape(response, {1, 3, 1, 2}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - this->checkOutputValues(response, {2.0, 3.0, 4.0, 5.0, 6.0, 7.0}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - - // Perform inference with NHWC layout, ensure error - ASSERT_EQ(this->performInferenceWithImageInput(response, {1, 1, 2, 3}, {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}), ovms::StatusCode::INVALID_SHAPE); -} - -/** - * Scenario - change output layout of model and perform inference. - * - * 1. Load model with output layout=nhwc:nchw, initial internal layout: nchw - * 2. Do the inference with (1,3,4,5) shape - expect status OK and result in NHWC layout - * 3. Remove layout setting - * 4. Do the inference with (1,3,4,5) shape - expect status OK and result in NCHW layout - * 5. Roll back layout setting to internal nchw - * 6. Do the inference with (1,3,4,5) shape - expect status OK and result in NCHW layout - */ -TYPED_TEST(TestPredict, PerformInferenceChangeModelOutputLayout) { - using namespace ovms; - - // Prepare model with changed output layout to nhwc (internal layout=nchw) - ModelConfig config = INCREMENT_1x3x4x5_MODEL_CONFIG; - config.setBatchingParams(""); - ASSERT_EQ(config.parseLayoutParameter(std::string("{\"") + INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME + std::string("\":\"nhwc:nchw\"}")), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - typename TypeParam::second_type response; - - // Perform inference with NCHW layout, ensure status OK and results in NHWC order - ASSERT_EQ(this->performInferenceWithImageInput(response, {1, 3, 4, 5}), ovms::StatusCode::OK); - this->checkOutputShape(response, {1, 4, 5, 3}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - - // Reload model with layout setting removed - ASSERT_EQ(config.parseLayoutParameter(""), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - // Perform inference with NCHW layout, ensure status OK and results still in NHWC order - ASSERT_EQ(this->performInferenceWithImageInput(response, {1, 3, 4, 5}), ovms::StatusCode::OK); - this->checkOutputShape(response, {1, 3, 4, 5}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - - // Change output layout back to original nchw. - ASSERT_EQ(config.parseLayoutParameter(std::string("{\"") + INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME + std::string("\":\"nchw\"}")), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - ASSERT_EQ(this->performInferenceWithImageInput(response, {1, 3, 4, 5}), ovms::StatusCode::OK); - this->checkOutputShape(response, {1, 3, 4, 5}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); -} - -/* - * Scenario - change output layout of model, modify shape and perform inference. Check results if in correct order. - * - * 1. Load model with output layout=nhwc:nchw, shape (1,1,2,3) initial internal layout: nchw - * 2. Do the inference with (1,3,4,5) shape - expect status OK and result in NHWC layout - * 3. Remove layout setting - * 4. Do the inference with (1,3,4,5) shape - expect status OK and result in NCHW layout - * 5. Roll back layout setting to internal nchw - * 6. Do the inference with (1,3,4,5) shape - expect status OK and result in NCHW layout - */ -TYPED_TEST(TestPredict, PerformInferenceChangeModelOutputLayoutAndShape) { - using namespace ovms; - - // Prepare model with changed output layout to nhwc (internal layout=nchw) - ModelConfig config = INCREMENT_1x3x4x5_MODEL_CONFIG; - config.setBatchingParams(""); - ASSERT_EQ(config.parseShapeParameter("(1,3,1,2)"), ovms::StatusCode::OK); - ASSERT_EQ(config.parseLayoutParameter(std::string("{\"") + INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME + std::string("\":\"nhwc:nchw\"}")), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - typename TypeParam::second_type response; - - // Perform inference with NCHW layout, ensure status OK and results in NHWC order - ASSERT_EQ(this->performInferenceWithImageInput(response, {1, 3, 1, 2}, {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}), ovms::StatusCode::OK); - this->checkOutputShape(response, {1, 1, 2, 3}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - this->checkOutputValues(response, {2.0, 4.0, 6.0, 3.0, 5.0, 7.0}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - // Reload model with layout setting removed - ASSERT_EQ(config.parseLayoutParameter(""), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - // Perform inference with NCHW layout, ensure status OK and results still in NHWC order - ASSERT_EQ(this->performInferenceWithImageInput(response, {1, 3, 1, 2}, {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}), ovms::StatusCode::OK); - this->checkOutputShape(response, {1, 3, 1, 2}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - this->checkOutputValues(response, {2.0, 3.0, 4.0, 5.0, 6.0, 7.0}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - - // Change output layout back to original nchw. - ASSERT_EQ(config.parseLayoutParameter(std::string("{\"") + INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME + std::string("\":\"nchw\"}")), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - ASSERT_EQ(this->performInferenceWithImageInput(response, {1, 3, 1, 2}, {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}), ovms::StatusCode::OK); - this->checkOutputShape(response, {1, 3, 1, 2}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - this->checkOutputValues(response, {2.0, 3.0, 4.0, 5.0, 6.0, 7.0}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); -} - -/* Scenario - change input layout and changing batch size at runtime. Expect shape dimension order to stay the same. - * - * 1. Load model with output layout=nhwc:nchw, native unchanged shape (1,4,5,3) initial internal layout: nchw - * 2. Do the inference with (1,4,5,3) shape - expect status OK and result in NCHW layout - * 3. Change batch size setting to 10 - * 4. Do the inference with (10,4,5,3) shape - expect status OK and result in NCHW layout - * 5. Change batch size setting to 15 - * 6. Do the inference with (15,4,5,3) shape - expect status OK and result in NCHW layout - * */ -TYPED_TEST(TestPredict, PerformInferenceChangeModelLayoutAndKeepChangingBatchSize) { - using namespace ovms; - - // Prepare model with changed output layout to nhwc (internal layout=nchw) - ModelConfig config = INCREMENT_1x3x4x5_MODEL_CONFIG; - config.setBatchingParams(""); - ASSERT_EQ(config.parseLayoutParameter("nhwc:nchw"), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - typename TypeParam::second_type response; - - // Perform inference with NHWC layout, ensure status OK and results in NHWC order - ASSERT_EQ(this->performInferenceWithImageInput(response, {1, 4, 5, 3}), ovms::StatusCode::OK); - this->checkOutputShape(response, {1, 3, 4, 5}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - - // Reload model with batch size changed - config.setBatchingParams("10"); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - // Perform inference with NHWC layout batch=10, ensure status OK and results still in NCHW order - ASSERT_EQ(this->performInferenceWithImageInput(response, {10, 4, 5, 3}), ovms::StatusCode::OK); - this->checkOutputShape(response, {10, 3, 4, 5}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - - // Change bs to 15 - config.setBatchingParams("15"); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - // Perform inference with NHWC layout batch=15, ensure status OK and input/output layout has not changed - ASSERT_EQ(this->performInferenceWithImageInput(response, {15, 4, 5, 3}), ovms::StatusCode::OK); - this->checkOutputShape(response, {15, 3, 4, 5}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); -} - -TYPED_TEST(TestPredict, ErrorWhenLayoutSetForMissingTensor) { - ovms::ModelConfig config = INCREMENT_1x3x4x5_MODEL_CONFIG; - ASSERT_EQ(config.parseLayoutParameter("{\"invalid_tensor_name\":\"nhwc\"}"), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::CONFIG_LAYOUT_IS_NOT_IN_MODEL); -} - -TYPED_TEST(TestPredict, NetworkNotLoadedWhenLayoutAndDimsInconsistent) { - // Dummy has 2 dimensions: (1,10), changing layout to NHWC should fail - ovms::ModelConfig config = DUMMY_MODEL_CONFIG; - ASSERT_EQ(config.parseLayoutParameter("nhwc:nchw"), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::MODEL_NOT_LOADED); -} - -/* Scenario - change input layout of model and perform inference with binary input. Check results. - * - * 1. Load model with input layout=nhwc, initial internal layout: nchw - * 2. Do the inference with single binary image tensor - expect status OK and result in NCHW layout - * 3. Set layout setting to internal nchw - * 4. Do the inference with single binary image tensor - expect status UNSUPPORTED_LAYOUT - * 5. Set back layout setting to nhwc - * 6. Do the inference with single binary image tensor - expect status OK and result in NCHW layout - * */ -TYPED_TEST(TestPredict, PerformInferenceWithBinaryInputChangeModelInputLayout) { - if (typeid(typename TypeParam::first_type) == typeid(ovms::InferenceRequest)) - GTEST_SKIP() << "Binary inputs not implemented for C-API yet"; - - using namespace ovms; - - // Prepare model with changed layout to nhwc (internal layout=nchw) - ModelConfig config = INCREMENT_1x3x4x5_MODEL_CONFIG; - config.setBatchingParams(""); - ASSERT_EQ(config.parseShapeParameter("(1,1,2,3)"), ovms::StatusCode::OK); - ASSERT_EQ(config.parseLayoutParameter("nhwc:nchw"), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - typename TypeParam::second_type response; - - // Perform inference with binary input, ensure status OK and correct results - ASSERT_EQ(this->performInferenceWithBinaryImageInput(response, INCREMENT_1x3x4x5_MODEL_INPUT_NAME), ovms::StatusCode::OK); - this->checkOutputShape(response, {1, 3, 1, 2}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - this->checkOutputValues(response, {37.0, 37.0, 28.0, 28.0, 238.0, 238.0}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - - // Reload model with layout setting removed - ASSERT_EQ(config.parseLayoutParameter("nchw"), ovms::StatusCode::OK); - ASSERT_EQ(config.parseShapeParameter("(1,3,1,2)"), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - // Perform inference with binary input, ensure validation rejects the request due to NCHW setting - ASSERT_EQ(this->performInferenceWithBinaryImageInput(response, INCREMENT_1x3x4x5_MODEL_INPUT_NAME), ovms::StatusCode::UNSUPPORTED_LAYOUT); - - // Switch back to nhwc - ASSERT_EQ(config.parseLayoutParameter("nhwc:nchw"), ovms::StatusCode::OK); - ASSERT_EQ(config.parseShapeParameter("(1,1,2,3)"), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - // Perform inference with binary input, ensure status OK after switching layout and correct results - ASSERT_EQ(this->performInferenceWithBinaryImageInput(response, INCREMENT_1x3x4x5_MODEL_INPUT_NAME), ovms::StatusCode::OK); - this->checkOutputShape(response, {1, 3, 1, 2}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - this->checkOutputValues(response, {37.0, 37.0, 28.0, 28.0, 238.0, 238.0}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); -} - -/* Scenario - perform inference with binary input with width exceeding shape range when model shape is dynamic. Check results. - * - * 1. Load model with dynamic shape and input layout=nhwc, initial internal layout: nchw - * 2. Do the inference with single binary image tensor with width exceeding shape range - expect status OK and reshaped output tensor - */ -TYPED_TEST(TestPredict, PerformInferenceWithBinaryInputAndShapeDynamic) { - if (typeid(typename TypeParam::first_type) == typeid(ovms::InferenceRequest)) - GTEST_SKIP() << "Binary inputs not implemented for C-API yet"; - using namespace ovms; - - // Prepare model with changed layout to nhwc (internal layout=nchw) - ModelConfig config = INCREMENT_1x3x4x5_MODEL_CONFIG; - config.setBatchingParams(""); - // binary input shape is [1,1,1,3] so it should be resized to the nearest border which is in this case [1,1,2,3] - ASSERT_EQ(config.parseShapeParameter("(1,1,2:5,3)"), ovms::StatusCode::OK); - ASSERT_EQ(config.parseLayoutParameter("nhwc:nchw"), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - typename TypeParam::second_type response; - - // Perform inference with binary input, ensure status OK and correct results - ASSERT_EQ(this->performInferenceWithBinaryImageInput(response, INCREMENT_1x3x4x5_MODEL_INPUT_NAME, "increment_1x3x4x5"), ovms::StatusCode::OK); - this->checkOutputShape(response, {1, 3, 1, 2}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - this->checkOutputValues(response, {37.0, 37.0, 28.0, 28.0, 238.0, 238.0}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); -} - -/* Scenario - perform inference with binary input of batch 0 when model shape is dynamic. Check results. - * - * 1. Load model with dynamic shape and input layout=nhwc, initial internal layout: nchw - * 2. Do the inference with 0 binary image tensors with width exceeding shape range - expect status OK and reshaped output tensor - */ -TYPED_TEST(TestPredict, PerformInferenceWithZeroBinaryInputsAndShapeDynamic) { - if (typeid(typename TypeParam::first_type) == typeid(ovms::InferenceRequest)) - GTEST_SKIP() << "Binary inputs not implemented for C-API yet"; - using namespace ovms; - - // Prepare model with changed layout to nhwc (internal layout=nchw) - ModelConfig config = INCREMENT_1x3x4x5_MODEL_CONFIG; - config.setBatchingParams(""); - // binary input shape is [0] so it should not proceed to inference anyway - ASSERT_EQ(config.parseShapeParameter("(-1,1,2:5,3)"), ovms::StatusCode::OK); - ASSERT_EQ(config.parseLayoutParameter("nhwc:nchw"), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - typename TypeParam::second_type response; - - // Perform inference with 0 binary inputs, ensure status INVALID_BATCH_SIZE - const int batchSize = 0; - ASSERT_EQ(this->performInferenceWithBinaryImageInput(response, INCREMENT_1x3x4x5_MODEL_INPUT_NAME, "increment_1x3x4x5", batchSize), ovms::StatusCode::INVALID_BATCH_SIZE); -} - -/* - * Scenario - send binary input request to model accepting auto batch size. - * - * 1. Load model with input layout=nhwc, batch_size=auto, initial internal layout: nchw, batch_size=1 - * 2. Do the inference with batch=5 binary image tensor - expect status OK and result in NCHW layout - */ -TYPED_TEST(TestPredict, PerformInferenceWithBinaryInputBatchSizeAuto) { - if (typeid(typename TypeParam::first_type) == typeid(ovms::InferenceRequest)) - GTEST_SKIP() << "Binary inputs not implemented for C-API yet"; - using namespace ovms; - - // Prepare model with changed layout to nhwc (internal layout=nchw) - ModelConfig config = INCREMENT_1x3x4x5_MODEL_CONFIG; - config.setBatchingParams("auto"); - ASSERT_EQ(config.parseShapeParameter("(1,1,2,3)"), ovms::StatusCode::OK); - ASSERT_EQ(config.parseLayoutParameter("nhwc:nchw"), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - typename TypeParam::second_type response; - - const int batchSize = 5; - // Perform inference with binary input, ensure status OK and correct results - ASSERT_EQ(this->performInferenceWithBinaryImageInput(response, INCREMENT_1x3x4x5_MODEL_INPUT_NAME, "increment_1x3x4x5", batchSize), ovms::StatusCode::OK); - this->checkOutputShape(response, {5, 3, 1, 2}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - this->checkOutputValues(response, {37.0, 37.0, 28.0, 28.0, 238.0, 238.0, 37.0, 37.0, 28.0, 28.0, 238.0, 238.0, 37.0, 37.0, 28.0, 28.0, 238.0, 238.0, 37.0, 37.0, 28.0, 28.0, 238.0, 238.0, 37.0, 37.0, 28.0, 28.0, 238.0, 238.0}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); -} - -/* Scenario - send binary input request with no shape set. - * - * 1. Load model with input layout=nhwc, batch_size=auto, initial internal layout: nchw, batch_size=1 - * 2. Do the inference with binary image tensor with no shape set - expect status INVALID_NO_OF_SHAPE_DIMENSIONS - */ -TYPED_TEST(TestPredict, PerformInferenceWithBinaryInputNoInputShape) { - if (typeid(typename TypeParam::first_type) == typeid(ovms::InferenceRequest)) - GTEST_SKIP() << "Binary inputs not implemented for C-API yet"; - using namespace ovms; - - // Prepare model with changed layout to nhwc (internal layout=nchw) - ModelConfig config = INCREMENT_1x3x4x5_MODEL_CONFIG; - config.setBatchingParams("auto"); - ASSERT_EQ(config.parseShapeParameter("(1,1,2,3)"), ovms::StatusCode::OK); - ASSERT_EQ(config.parseLayoutParameter("nhwc"), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - typename TypeParam::first_type request; - typename TypeParam::second_type response; - prepareBinaryPredictRequestNoShape(request, INCREMENT_1x3x4x5_MODEL_INPUT_NAME, 1); - - // Perform inference with binary input, ensure status INVALID_NO_OF_SHAPE_DIMENSIONS - ASSERT_EQ(this->performInferenceWithRequest(request, response, "increment_1x3x4x5"), ovms::StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS); -} - -/* - * Scenario - perform inference with with batch size set to auto and batch size not matching on position other than first - * - * 1. Load model with bs=auto, layout=b=>cn,a=>cn initial internal shape (1,10) - * 2. Do the inference with (1,30) shape - expect status OK and result (1,30) - * 3. Change model batch size to fixed=4 with config.json change - * 4. Do the inference with (1,30) shape - expect status INVALID_BATCH_SIZE - * 5. Do the inference with (1,4) shape - expect status OK and result (1,4) - * 6. Reshape model back to batchsize=auto, initial internal shape (1,10) - * 7. Do the inference with (1,30) shape - expect status OK and result (1,30) - * 8. Do the inference with (30,10) shape - expect status INVALID_SHAPE - */ -TYPED_TEST(TestPredict, ChangeBatchSizeViaRequestAndConfigChangeArbitraryPosition) { - using namespace ovms; - size_t batchSizePosition = 1; // [0:C, 1:N] - - // Prepare model with bs=auto, layout=b=>cn,a=>cn (initially (1,10) shape) - ModelConfig config = DUMMY_MODEL_CONFIG; - config.setBatchingParams("auto"); - ASSERT_EQ(config.parseLayoutParameter("{\"b\":\"cn\",\"a\":\"cn\"}"), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - typename TypeParam::second_type response; - - // Perform batch size change to 30 using request - ASSERT_EQ(this->performInferenceWithBatchSize(response, 30, ovms::Precision::FP32, batchSizePosition), ovms::StatusCode::OK); - this->checkOutputShape(response, {1, 30}); - - // Change batch size with model reload to Fixed=4 - config.setBatchingParams("4"); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - // Cannot do the inference with (1,30) - ASSERT_EQ(this->performInferenceWithBatchSize(response, 30, ovms::Precision::FP32, batchSizePosition), ovms::StatusCode::INVALID_BATCH_SIZE); - - // Successful inference with (1,4) - ASSERT_EQ(this->performInferenceWithBatchSize(response, 4, ovms::Precision::FP32, batchSizePosition), ovms::StatusCode::OK); - this->checkOutputShape(response, {1, 4}); - - // Reshape back to AUTO, internal shape is (1,10) - config.setBatchingParams("auto"); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - // Perform batch change to 30 using request - ASSERT_EQ(this->performInferenceWithBatchSize(response, 30, ovms::Precision::FP32, batchSizePosition), ovms::StatusCode::OK); - this->checkOutputShape(response, {1, 30}); - - // Ensure cannot change batch size with first dimension - batchSizePosition = 0; - ASSERT_EQ(this->performInferenceWithBatchSize(response, 30, ovms::Precision::FP32, batchSizePosition), ovms::StatusCode::INVALID_SHAPE); -} - -/* Scenario - inference with different shapes with dynamic dummy, both dimensions reshaped to any. - * No model reload performed between requests. - * - * 1. Load model with input shape (-1, -1) - * 2. Do the inference with (3, 2) shape, expect correct output shape - * 3. Do the inference with (1, 4) shape, expect correct output shape - */ -TYPED_TEST(TestPredict, PerformInferenceDummyAllDimensionsAny) { - using namespace ovms; - - ModelConfig config = DUMMY_MODEL_CONFIG; - config.setBatchingParams(""); - ASSERT_EQ(config.parseShapeParameter("(-1,-1)"), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - typename TypeParam::second_type response; - - // Do the inference with (3, 2) - ASSERT_EQ(this->performInferenceWithShape(response, {3, 2}), ovms::StatusCode::OK); - this->checkOutputShape(response, {3, 2}, DUMMY_MODEL_OUTPUT_NAME); - - // Do the inference with (1, 4) - ASSERT_EQ(this->performInferenceWithShape(response, {1, 4}), ovms::StatusCode::OK); - this->checkOutputShape(response, {1, 4}, DUMMY_MODEL_OUTPUT_NAME); -} - -/* Scenario - inference with different batch sizes for dynamic batch dummy. - * No model reload performed between requests. - * - * 1. Load model with input shape (-1, 10) - * 2. Do the X inferences with (x, 10) shape, expect correct output shapes. x=[1, 3, 5, 7, 11, 17, 21, 57, 99] - */ -TYPED_TEST(TestPredict, PerformInferenceDummyBatchSizeAny) { - using namespace ovms; - - ModelConfig config = DUMMY_MODEL_CONFIG; - config.setBatchingParams("-1"); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - typename TypeParam::second_type response; - - for (int32_t i : {1, 3, 5, 7, 11, 17, 21, 57, 99}) { - ASSERT_EQ(this->performInferenceWithShape(response, {i, 10}), ovms::StatusCode::OK); - this->checkOutputShape(response, {i, 10}, DUMMY_MODEL_OUTPUT_NAME); - } -} - -/* Scenario - inference with dummy precision fp32. - * - * 1. Load model with input shape (-1, 10) - * 2. Do the inferences with (3, 10) shape, expect correct output shapes and precision - */ - -static ovms::Precision getPrecisionFromResponse(ovms::InferenceResponse& response, const std::string& name) { - size_t outputCount = response.getOutputCount(); - EXPECT_GE(1, outputCount); - size_t outputId = 0; - while (outputId < outputCount) { - const std::string* cppName; - InferenceTensor* tensor; - auto status = response.getOutput(outputId, &cppName, &tensor); - EXPECT_EQ(status, StatusCode::OK) << status.string(); - EXPECT_NE(nullptr, tensor); - EXPECT_NE(nullptr, cppName); - if (name == *cppName) { - return ovms::getOVMSDataTypeAsPrecision(tensor->getDataType()); - } - ++outputId; - } - return ovms::getOVMSDataTypeAsPrecision(OVMS_DATATYPE_UNDEFINED); -} - -static ovms::Precision getPrecisionFromResponse(KFSResponse& response, const std::string& name) { - KFSOutputTensorIteratorType it; - size_t bufferId; - auto status = getOutput(response, name, it, bufferId); - EXPECT_TRUE(status.ok()); - return ovms::KFSPrecisionToOvmsPrecision(it->datatype()); -} - -static ovms::Precision getPrecisionFromResponse(TFSResponseType& response, const std::string& name) { - TFSOutputTensorIteratorType it; - size_t bufferId; - auto status = getOutput(response, name, it, bufferId); - EXPECT_TRUE(status.ok()); - if (!status.ok()) - return ovms::Precision::UNDEFINED; - return ovms::TFSPrecisionToOvmsPrecision(it->second.dtype()); -} -TYPED_TEST(TestPredict, PerformInferenceDummyFp64) { - using namespace ovms; - - ModelConfig config = DUMMY_FP64_MODEL_CONFIG; - config.setBatchingParams("3"); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - typename TypeParam::first_type request; - typename TypeParam::second_type response; - - Preparer preparer; - preparer.preparePredictRequest(request, {{"input:0", std::tuple{{3, 10}, ovms::Precision::FP64}}}); - ASSERT_EQ(this->performInferenceWithRequest(request, response, "dummy_fp64"), ovms::StatusCode::OK); - this->checkOutputShape(response, {3, 10}, "output:0"); - ASSERT_EQ(getPrecisionFromResponse(response, "output:0"), ovms::Precision::FP64); -} - -/* Scenario - inference with different shapes with dynamic dummy, both dimensions reshaped to range. - * No model reload performed between requests. - * - * 1. Load model with input shape (2:4, 1:5) - * 2. Do the inference with (1, 1) shape, expect not in range - * 3. Do the inference with (2, 1) shape, expect success and correct output shape - * 4. Do the inference with (3, 2) shape, expect success and correct output shape - * 5. Do the inference with (3, 5) shape, expect success and correct output shape - * 6. Do the inference with (3, 6) shape, expect not in range - * 7. Do the inference with (5, 5) shape, expect not in range - */ -TYPED_TEST(TestPredict, PerformInferenceDummyAllDimensionsHaveRange) { - using namespace ovms; - - ModelConfig config = DUMMY_MODEL_CONFIG; - config.setBatchingParams(""); - ASSERT_EQ(config.parseShapeParameter("(2:4,1:5)"), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - typename TypeParam::second_type response; - - ASSERT_EQ(this->performInferenceWithShape(response, {1, 1}), ovms::StatusCode::INVALID_BATCH_SIZE); - - ASSERT_EQ(this->performInferenceWithShape(response, {2, 1}), ovms::StatusCode::OK); - this->checkOutputShape(response, {2, 1}, DUMMY_MODEL_OUTPUT_NAME); - - ASSERT_EQ(this->performInferenceWithShape(response, {3, 2}), ovms::StatusCode::OK); - this->checkOutputShape(response, {3, 2}, DUMMY_MODEL_OUTPUT_NAME); - - ASSERT_EQ(this->performInferenceWithShape(response, {3, 5}), ovms::StatusCode::OK); - this->checkOutputShape(response, {3, 5}, DUMMY_MODEL_OUTPUT_NAME); - - ASSERT_EQ(this->performInferenceWithShape(response, {3, 6}), ovms::StatusCode::INVALID_SHAPE); - ASSERT_EQ(this->performInferenceWithShape(response, {5, 5}), ovms::StatusCode::INVALID_BATCH_SIZE); -} - -/* Scenario - send binary input request to model accepting dynamic batch size. - * - * 1. Load model with input layout=nhwc, batch_size=-1, resolution 1x2, initial internal layout: nchw, batch_size=1 - * 2. Do the inference with batch=5 binary image tensor 1x1 - expect status INVALID_SHAPE, because if any dimension is dynamic, we perform no resize operation. - */ -TYPED_TEST(TestPredict, PerformInferenceWithBinaryInputBatchSizeAnyResolutionNotMatching) { - if (typeid(typename TypeParam::first_type) == typeid(ovms::InferenceRequest)) - GTEST_SKIP() << "Binary inputs not implemented for C-API yet"; - using namespace ovms; - - // Prepare model with changed layout to nhwc (internal layout=nchw) - ModelConfig config = INCREMENT_1x3x4x5_MODEL_CONFIG; - config.setBatchingParams(""); - ASSERT_EQ(config.parseShapeParameter("(-1,1,2,3)"), ovms::StatusCode::OK); - ASSERT_EQ(config.parseLayoutParameter("nhwc:nchw"), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - typename TypeParam::second_type response; - - const int batchSize = 5; - // Perform inference with binary input 1x1, expect status BINARY_IMAGES_RESOLUTION_MISMATCH, because if any dimension is dynamic, we perform no resize operation. - ASSERT_EQ(this->performInferenceWithBinaryImageInput(response, INCREMENT_1x3x4x5_MODEL_INPUT_NAME, "increment_1x3x4x5", batchSize), ovms::StatusCode::INVALID_SHAPE); -} - -/* Scenario - send binary input request to model accepting dynamic batch size. - * - * 1. Load model with input layout=nhwc, batch_size=-1, resolution 1x1, initial internal layout: nchw, batch_size=1 - * 2. Do the inference with batch=5 binary image tensor 1x1 - expect status OK, and correct results. - */ -TYPED_TEST(TestPredict, PerformInferenceWithBinaryInputBatchSizeAnyResolutionMatching) { - if (typeid(typename TypeParam::first_type) == typeid(ovms::InferenceRequest)) - GTEST_SKIP() << "Binary inputs not implemented for C-API yet"; - using namespace ovms; - - // Prepare model with changed layout to nhwc (internal layout=nchw) - ModelConfig config = INCREMENT_1x3x4x5_MODEL_CONFIG; - config.setBatchingParams(""); - ASSERT_EQ(config.parseShapeParameter("(-1,1,1,3)"), ovms::StatusCode::OK); - ASSERT_EQ(config.parseLayoutParameter("nhwc:nchw"), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - typename TypeParam::second_type response; - - const int batchSize = 5; - // Perform inference with binary input, ensure status OK and correct results - ASSERT_EQ(this->performInferenceWithBinaryImageInput(response, INCREMENT_1x3x4x5_MODEL_INPUT_NAME, "increment_1x3x4x5", batchSize), ovms::StatusCode::OK); - this->checkOutputShape(response, {5, 3, 1, 1}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - this->checkOutputValues(response, {37.0, 28.0, 238.0, 37.0, 28.0, 238.0, 37.0, 28.0, 238.0, 37.0, 28.0, 238.0, 37.0, 28.0, 238.0}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); -} - -/* Scenario - send binary input request to model accepting dynamic resolution. - * - * 1. Load model with input layout=nhwc, shape 1,-1,-1,3, initial internal layout: nchw, batch_size=1 - * 2. Do the inference with resolution 1x1 binary image tensor - expect status OK and result in NCHW layout - */ -TYPED_TEST(TestPredict, PerformInferenceWithBinaryInputResolutionAny) { - if (typeid(typename TypeParam::first_type) == typeid(ovms::InferenceRequest)) - GTEST_SKIP() << "Binary inputs not implemented for C-API yet"; - using namespace ovms; - - // Prepare model with changed layout to nhwc (internal layout=nchw) - ModelConfig config = INCREMENT_1x3x4x5_MODEL_CONFIG; - config.setBatchingParams(""); - ASSERT_EQ(config.parseShapeParameter("(1,-1,-1,3)"), ovms::StatusCode::OK); - ASSERT_EQ(config.parseLayoutParameter("nhwc:nchw"), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - typename TypeParam::second_type response; - - // Perform inference with binary input, ensure status OK and correct results - ASSERT_EQ(this->performInferenceWithBinaryImageInput(response, INCREMENT_1x3x4x5_MODEL_INPUT_NAME, "increment_1x3x4x5"), ovms::StatusCode::OK); - this->checkOutputShape(response, {1, 3, 1, 1}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - this->checkOutputValues(response, {37.0, 28.0, 238.0}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); -} - -/* Scenario - send binary input request to model accepting range of resolution. - * - * 1. Load model with input layout=nhwc, shape 1,1:2,1:2,3, initial internal layout: nchw, batch_size=1 - * 2. Do the inference with resolution 4x4 binary image tensor - expect status OK and reshaped to 2x2 - * 3. Do the inference with resolution 1x1 binary image tensor - expect status OK and result in NCHW layout - */ -TYPED_TEST(TestPredict, PerformInferenceWithBinaryInputResolutionRange) { - if (typeid(typename TypeParam::first_type) == typeid(ovms::InferenceRequest)) - GTEST_SKIP() << "Binary inputs not implemented for C-API yet"; - using namespace ovms; - - // Prepare model with changed layout to nhwc (internal layout=nchw) - ModelConfig config = INCREMENT_1x3x4x5_MODEL_CONFIG; - config.setBatchingParams(""); - ASSERT_EQ(config.parseShapeParameter("(1,1:2,1:2,3)"), ovms::StatusCode::OK); - ASSERT_EQ(config.parseLayoutParameter("nhwc:nchw"), ovms::StatusCode::OK); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - typename TypeParam::second_type response; - typename TypeParam::first_type request; - prepareBinary4x4PredictRequest(request, INCREMENT_1x3x4x5_MODEL_INPUT_NAME); - - ASSERT_EQ( - this->performInferenceWithRequest( - request, response, "increment_1x3x4x5"), - ovms::StatusCode::OK); - this->checkOutputShape(response, {1, 3, 2, 2}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - - response.Clear(); - - // Perform inference with binary input, ensure status OK and correct results - ASSERT_EQ(this->performInferenceWithBinaryImageInput(response, INCREMENT_1x3x4x5_MODEL_INPUT_NAME, "increment_1x3x4x5"), ovms::StatusCode::OK); - this->checkOutputShape(response, {1, 3, 1, 1}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - this->checkOutputValues(response, {37.0, 28.0, 238.0}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); -} - -TYPED_TEST(TestPredict, InferenceWithNegativeShape) { - typename TypeParam::first_type request; - std::vector data{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - int64_t negativeBatch = -5; - preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{negativeBatch, 10}, ovms::Precision::FP32}}}, - data); - ovms::ModelConfig config = DUMMY_MODEL_CONFIG; - config.setBatchSize(1); - - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - std::shared_ptr modelInstance; - std::unique_ptr modelInstanceUnloadGuard; - ASSERT_EQ(this->manager.getModelInstance(config.getName(), config.getVersion(), modelInstance, modelInstanceUnloadGuard), ovms::StatusCode::OK); - typename TypeParam::second_type response; - ASSERT_NE(ovms::infer(*modelInstance, &request, &response, modelInstanceUnloadGuard), ovms::StatusCode::OK); -} - -TYPED_TEST(TestPredict, InferenceWithNegativeShapeDynamicParameter) { - typename TypeParam::first_type request; - std::vector data{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - int64_t negativeBatch = -5; - preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{negativeBatch, 10}, ovms::Precision::FP32}}}, - data); - ovms::ModelConfig config = DUMMY_MODEL_CONFIG; - config.setBatchingParams("auto"); - - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - std::shared_ptr modelInstance; - std::unique_ptr modelInstanceUnloadGuard; - ASSERT_EQ(this->manager.getModelInstance(config.getName(), config.getVersion(), modelInstance, modelInstanceUnloadGuard), ovms::StatusCode::OK); - typename TypeParam::second_type response; - ASSERT_NE(ovms::infer(*modelInstance, &request, &response, modelInstanceUnloadGuard), ovms::StatusCode::OK); -} - -TYPED_TEST(TestPredict, InferenceWithStringInputs_positive_2D) { - if (typeid(typename TypeParam::first_type) == typeid(ovms::InferenceRequest)) - GTEST_SKIP() << "String inputs not supported for C-API"; - typename TypeParam::first_type request; - std::vector inputStrings = {"String_123", "String"}; - prepareInferStringRequest(request, PASSTHROUGH_MODEL_INPUT_NAME, inputStrings); - ovms::ModelConfig config = PASSTHROUGH_MODEL_CONFIG; - config.setBatchingParams("auto"); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - std::shared_ptr modelInstance; - std::unique_ptr modelInstanceUnloadGuard; - ASSERT_EQ(this->manager.getModelInstance(config.getName(), config.getVersion(), modelInstance, modelInstanceUnloadGuard), ovms::StatusCode::OK); - typename TypeParam::second_type response; - ASSERT_EQ(ovms::infer(*modelInstance, &request, &response, modelInstanceUnloadGuard), ovms::StatusCode::OK); - this->checkOutputShape(response, {2, 11}, PASSTHROUGH_MODEL_OUTPUT_NAME); - std::vector expectedData = { - 'S', 't', 'r', 'i', 'n', 'g', '_', '1', '2', '3', 0, - 'S', 't', 'r', 'i', 'n', 'g', 0, 0, 0, 0, 0}; - bool checkRaw = true; - this->checkOutputValuesU8(response, expectedData, PASSTHROUGH_MODEL_OUTPUT_NAME, checkRaw); -} - -TYPED_TEST(TestPredict, InferenceWithStringInputs_positive_batch0_2D) { - if (typeid(typename TypeParam::first_type) == typeid(ovms::InferenceRequest)) - GTEST_SKIP() << "String inputs not supported for C-API"; - typename TypeParam::first_type request; - std::vector inputStrings = {}; - prepareInferStringRequest(request, PASSTHROUGH_MODEL_INPUT_NAME, inputStrings); - ovms::ModelConfig config = PASSTHROUGH_MODEL_CONFIG; - config.setBatchingParams(""); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - std::shared_ptr modelInstance; - std::unique_ptr modelInstanceUnloadGuard; - ASSERT_EQ(this->manager.getModelInstance(config.getName(), config.getVersion(), modelInstance, modelInstanceUnloadGuard), ovms::StatusCode::OK); - typename TypeParam::second_type response; - auto status = ovms::infer(*modelInstance, &request, &response, modelInstanceUnloadGuard); - ASSERT_EQ(status, ovms::StatusCode::INVALID_BATCH_SIZE) << status.string(); -} - -TYPED_TEST(TestPredict, InferenceWithStringInputs_positive_2D_data_in_buffer) { - if (typeid(typename TypeParam::first_type) == typeid(ovms::InferenceRequest) || typeid(typename TypeParam::first_type) == typeid(TFSRequestType)) - GTEST_SKIP() << "String inputs in buffer not supported for C-API and TFS api"; - typename TypeParam::first_type request; - std::vector inputStrings = {"String_123", "String"}; - prepareInferStringRequest(request, PASSTHROUGH_MODEL_INPUT_NAME, inputStrings, false); - ovms::ModelConfig config = PASSTHROUGH_MODEL_CONFIG; - config.setBatchingParams("auto"); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - std::shared_ptr modelInstance; - std::unique_ptr modelInstanceUnloadGuard; - ASSERT_EQ(this->manager.getModelInstance(config.getName(), config.getVersion(), modelInstance, modelInstanceUnloadGuard), ovms::StatusCode::OK); - typename TypeParam::second_type response; - ASSERT_EQ(ovms::infer(*modelInstance, &request, &response, modelInstanceUnloadGuard), ovms::StatusCode::OK); - this->checkOutputShape(response, {2, 11}, PASSTHROUGH_MODEL_OUTPUT_NAME); - std::vector expectedData = { - 'S', 't', 'r', 'i', 'n', 'g', '_', '1', '2', '3', 0, - 'S', 't', 'r', 'i', 'n', 'g', 0, 0, 0, 0, 0}; - bool checkRaw = true; - this->checkOutputValuesU8(response, expectedData, PASSTHROUGH_MODEL_OUTPUT_NAME, checkRaw); -} - -// Legacy, supported via Native OV String since 2024.0 -TYPED_TEST(TestPredict, InferenceWithStringInputs_positive_1D) { - if (typeid(typename TypeParam::first_type) == typeid(ovms::InferenceRequest)) - GTEST_SKIP() << "String inputs not supported for C-API"; - typename TypeParam::first_type request; - std::vector inputStrings = {"ala", "", "ma", "kota"}; - prepareInferStringRequest(request, PASSTHROUGH_MODEL_INPUT_NAME, inputStrings); - ovms::ModelConfig config = PASSTHROUGH_MODEL_CONFIG; - config.setBatchingParams(""); - config.parseShapeParameter("(-1)"); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - std::shared_ptr modelInstance; - std::unique_ptr modelInstanceUnloadGuard; - ASSERT_EQ(this->manager.getModelInstance(config.getName(), config.getVersion(), modelInstance, modelInstanceUnloadGuard), ovms::StatusCode::OK); - typename TypeParam::second_type response; - ASSERT_EQ(ovms::infer(*modelInstance, &request, &response, modelInstanceUnloadGuard), ovms::StatusCode::NOT_IMPLEMENTED); -} - -TYPED_TEST(TestPredict, InferenceWithStringInputs_positive_NativeString) { - if (typeid(typename TypeParam::first_type) == typeid(ovms::InferenceRequest)) - GTEST_SKIP() << "String inputs not supported for C-API"; - typename TypeParam::first_type request; - std::vector inputStrings = {"ala", "", "ma", "kota"}; - bool putBufferInInputTensorContent = true; - prepareInferStringRequest(request, PASSTHROUGH_STRING_MODEL_INPUT_NAME, inputStrings, putBufferInInputTensorContent); - ovms::ModelConfig config = NATIVE_STRING_MODEL_CONFIG; - config.setBatchingParams(""); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - std::shared_ptr modelInstance; - std::unique_ptr modelInstanceUnloadGuard; - ASSERT_EQ(this->manager.getModelInstance(config.getName(), config.getVersion(), modelInstance, modelInstanceUnloadGuard), ovms::StatusCode::OK); - typename TypeParam::second_type response; - ASSERT_EQ(ovms::infer(*modelInstance, &request, &response, modelInstanceUnloadGuard), ovms::StatusCode::OK); - bool checkRaw = true; - this->checkOutputValuesString(response, inputStrings, PASSTHROUGH_STRING_MODEL_OUTPUT_NAME, checkRaw); -} - -// Legacy, supported via Native OV String since 2024.0 -TYPED_TEST(TestPredict, InferenceWithStringInputs_positive_batch0_1D) { - if (typeid(typename TypeParam::first_type) == typeid(ovms::InferenceRequest)) - GTEST_SKIP() << "String inputs not supported for C-API"; - typename TypeParam::first_type request; - std::vector inputStrings = {}; - prepareInferStringRequest(request, PASSTHROUGH_MODEL_INPUT_NAME, inputStrings); - ovms::ModelConfig config = PASSTHROUGH_MODEL_CONFIG; - config.setBatchingParams(""); - config.parseShapeParameter("(-1)"); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - std::shared_ptr modelInstance; - std::unique_ptr modelInstanceUnloadGuard; - ASSERT_EQ(this->manager.getModelInstance(config.getName(), config.getVersion(), modelInstance, modelInstanceUnloadGuard), ovms::StatusCode::OK); - typename TypeParam::second_type response; - ASSERT_EQ(ovms::infer(*modelInstance, &request, &response, modelInstanceUnloadGuard), ovms::StatusCode::NOT_IMPLEMENTED); -} - -TYPED_TEST(TestPredict, InferenceWithStringInputs_positive_batch0_NativeString) { - if (typeid(typename TypeParam::first_type) == typeid(ovms::InferenceRequest)) - GTEST_SKIP() << "String inputs not supported for C-API"; - typename TypeParam::first_type request; - std::vector inputStrings = {}; - prepareInferStringRequest(request, PASSTHROUGH_STRING_MODEL_INPUT_NAME, inputStrings); - ovms::ModelConfig config = NATIVE_STRING_MODEL_CONFIG; - config.setBatchingParams(""); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - std::shared_ptr modelInstance; - std::unique_ptr modelInstanceUnloadGuard; - ASSERT_EQ(this->manager.getModelInstance(config.getName(), config.getVersion(), modelInstance, modelInstanceUnloadGuard), ovms::StatusCode::OK); - typename TypeParam::second_type response; - ASSERT_EQ(ovms::infer(*modelInstance, &request, &response, modelInstanceUnloadGuard), ovms::StatusCode::OK); - bool checkRaw = true; - this->checkOutputValuesString(response, inputStrings, PASSTHROUGH_STRING_MODEL_OUTPUT_NAME, checkRaw); -} - -// Legacy, supported via Native OV String since 2024.0 -TYPED_TEST(TestPredict, InferenceWithStringInputs_positive_1D_data_in_buffer) { - if (typeid(typename TypeParam::first_type) == typeid(ovms::InferenceRequest) || typeid(typename TypeParam::first_type) == typeid(TFSRequestType)) - GTEST_SKIP() << "String inputs in buffer not supported for C-API and TFS api"; - typename TypeParam::first_type request; - std::vector inputStrings = {"ala", "", "ma", "kota"}; - prepareInferStringRequest(request, PASSTHROUGH_MODEL_INPUT_NAME, inputStrings, false); - ovms::ModelConfig config = PASSTHROUGH_MODEL_CONFIG; - config.setBatchingParams(""); - config.parseShapeParameter("(-1)"); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - std::shared_ptr modelInstance; - std::unique_ptr modelInstanceUnloadGuard; - ASSERT_EQ(this->manager.getModelInstance(config.getName(), config.getVersion(), modelInstance, modelInstanceUnloadGuard), ovms::StatusCode::OK); - typename TypeParam::second_type response; - ASSERT_EQ(ovms::infer(*modelInstance, &request, &response, modelInstanceUnloadGuard), ovms::StatusCode::NOT_IMPLEMENTED); -} - -TYPED_TEST(TestPredict, InferenceWithStringInputs_positive_NativeString_data_in_buffer) { - if (typeid(typename TypeParam::first_type) == typeid(ovms::InferenceRequest) || typeid(typename TypeParam::first_type) == typeid(TFSRequestType)) - GTEST_SKIP() << "String inputs in buffer not supported for C-API and TFS api"; - typename TypeParam::first_type request; - std::vector inputStrings = {"ala", "", "ma", "kota"}; - bool putBufferInInputTensorContent = false; - prepareInferStringRequest(request, PASSTHROUGH_STRING_MODEL_INPUT_NAME, inputStrings, putBufferInInputTensorContent); - ovms::ModelConfig config = NATIVE_STRING_MODEL_CONFIG; - config.setBatchingParams(""); - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - std::shared_ptr modelInstance; - std::unique_ptr modelInstanceUnloadGuard; - ASSERT_EQ(this->manager.getModelInstance(config.getName(), config.getVersion(), modelInstance, modelInstanceUnloadGuard), ovms::StatusCode::OK); - typename TypeParam::second_type response; - ASSERT_EQ(ovms::infer(*modelInstance, &request, &response, modelInstanceUnloadGuard), ovms::StatusCode::OK); - bool checkRaw = true; - this->checkOutputValuesString(response, inputStrings, PASSTHROUGH_STRING_MODEL_OUTPUT_NAME, checkRaw); -} - -class TestPredictKFS : public TestPredict {}; - -TEST_F(TestPredictKFS, RequestDataInFp32ContentResponseInRaw) { - KFSRequest request; - std::vector data{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - bool putBufferInInputTensorContent = true; // put in fp32_content - preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{1, 10}, ovms::Precision::FP32}}}, - data, - putBufferInInputTensorContent); - ovms::ModelConfig config = DUMMY_MODEL_CONFIG; - - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - std::shared_ptr modelInstance; - std::unique_ptr modelInstanceUnloadGuard; - ASSERT_EQ(this->manager.getModelInstance(config.getName(), config.getVersion(), modelInstance, modelInstanceUnloadGuard), ovms::StatusCode::OK); - KFSResponse response; - ASSERT_EQ(ovms::infer(*modelInstance, &request, &response, modelInstanceUnloadGuard), ovms::StatusCode::OK); - ASSERT_EQ(response.outputs_size(), 1); - ASSERT_FALSE(response.outputs(0).has_contents()); - ASSERT_GT(response.raw_output_contents_size(), 0); -} - -TEST_F(TestPredictKFS, RequestDataInRawResponseInRaw) { - KFSRequest request; - std::vector data{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - bool putBufferInInputTensorContent = false; // put in raw - preparePredictRequest(request, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{1, 10}, ovms::Precision::FP32}}}, - data, - putBufferInInputTensorContent); - ovms::ModelConfig config = DUMMY_MODEL_CONFIG; - - ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - std::shared_ptr modelInstance; - std::unique_ptr modelInstanceUnloadGuard; - ASSERT_EQ(this->manager.getModelInstance(config.getName(), config.getVersion(), modelInstance, modelInstanceUnloadGuard), ovms::StatusCode::OK); - KFSResponse response; - ASSERT_EQ(ovms::infer(*modelInstance, &request, &response, modelInstanceUnloadGuard), ovms::StatusCode::OK); - ASSERT_EQ(response.outputs_size(), 1); - ASSERT_FALSE(response.outputs(0).has_contents()); - ASSERT_GT(response.raw_output_contents_size(), 0); -} - -#pragma GCC diagnostic pop diff --git a/src/test/pythonnode_test.cpp b/src/test/pythonnode_test.cpp index 3f7495650c..955cc65499 100644 --- a/src/test/pythonnode_test.cpp +++ b/src/test/pythonnode_test.cpp @@ -38,7 +38,6 @@ #include "../mediapipe_internal/mediapipegraphexecutor.hpp" #include "src/metrics/metric_config.hpp" #include "src/metrics/metric_module.hpp" -#include "../model_service.hpp" #include "../precision.hpp" #include "../python/pythoninterpretermodule.hpp" #include "../python/pythonnoderesources.hpp" @@ -46,7 +45,7 @@ #include "../server.hpp" #include "../shape.hpp" #include "../stringutils.hpp" -#include "../tfs_frontend/tfs_utils.hpp" +#include "src/tensorflow_type_utils.hpp" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" #include "mediapipe/framework/calculator_graph.h" @@ -76,7 +75,7 @@ std::unique_ptr serverThread; class PythonFlowTest : public ::testing::Test { protected: - ovms::ExecutionContext defaultExecutionContext{ovms::ExecutionContext::Interface::GRPC, ovms::ExecutionContext::Method::Predict}; + ovms::ExecutionContext defaultExecutionContext{ovms::ExecutionContext::Interface::GRPC, ovms::ExecutionContext::Method::ModelInfer}; std::unique_ptr reporter; public: diff --git a/src/test/rest_utils_test.cpp b/src/test/rest_utils_test.cpp index 6d6df46be5..fe10425da9 100644 --- a/src/test/rest_utils_test.cpp +++ b/src/test/rest_utils_test.cpp @@ -39,694 +39,6 @@ TEST_F(Base64DecodeTest, WrongLength) { std::string decodedBytes; EXPECT_EQ(decodeBase64(bytes, decodedBytes), StatusCode::REST_BASE64_DECODE_ERROR); } - -class TFSMakeJsonFromPredictResponseRawTest : public ::testing::TestWithParam { -protected: - TFSResponseType proto; - std::string json; - TFSOutputTensorType *output1, *output2; - - void SetUp() override { - output1 = &((*proto.mutable_outputs())["output1"]); - output2 = &((*proto.mutable_outputs())["output2"]); - - output1->set_dtype(tensorflow::DataType::DT_FLOAT); - output2->set_dtype(tensorflow::DataType::DT_INT8); - - float data1[8] = {5.0f, 10.0f, -3.0f, 2.5f, - 9.0f, 55.5f, -0.5f, -1.5f}; - output1->mutable_tensor_content()->assign(reinterpret_cast(data1), 8 * sizeof(float)); - output1->mutable_tensor_shape()->add_dim()->set_size(2); - output1->mutable_tensor_shape()->add_dim()->set_size(1); - output1->mutable_tensor_shape()->add_dim()->set_size(4); - - int8_t data2[10] = {5, 2, 3, 8, -2, - -100, 0, 125, 4, -1}; - output2->mutable_tensor_content()->assign(reinterpret_cast(data2), 10 * sizeof(int8_t)); - output2->mutable_tensor_shape()->add_dim()->set_size(2); - output2->mutable_tensor_shape()->add_dim()->set_size(5); - } -}; - -class TFSMakeJsonFromPredictResponseStringTest : public ::testing::Test { -protected: - TFSResponseType proto; - std::string json; - TFSOutputTensorType* output1; - - void SetUp() override { - output1 = &((*proto.mutable_outputs())["output1_string"]); - output1->set_dtype(tensorflow::DataType::DT_STRING); - } -}; - -TEST_F(TFSMakeJsonFromPredictResponseStringTest, PositiveRow) { - output1->add_string_val("Hello"); - output1->mutable_tensor_shape()->add_dim()->set_size(1); - std::string expected_json = "{\n \"predictions\": [\"Hello\"\n ]\n}"; - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, Order::ROW), StatusCode::OK); - EXPECT_EQ(json, expected_json); -} - -TEST_F(TFSMakeJsonFromPredictResponseStringTest, PositiveRowBatchSize2) { - output1->add_string_val("Hello"); - output1->add_string_val("World"); - output1->mutable_tensor_shape()->add_dim()->set_size(2); - std::string expected_json = "{\n \"predictions\": [\"Hello\", \"World\"\n ]\n}"; - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, Order::ROW), StatusCode::OK); - EXPECT_EQ(json, expected_json); -} - -TEST_F(TFSMakeJsonFromPredictResponseStringTest, PositiveColumn) { - output1->add_string_val("Hello"); - output1->mutable_tensor_shape()->add_dim()->set_size(1); - std::string expected_json = "{\n \"outputs\": [\n \"Hello\"\n ]\n}"; - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, Order::COLUMN), StatusCode::OK); - EXPECT_EQ(json, expected_json); -} - -TEST_F(TFSMakeJsonFromPredictResponseStringTest, ScalarColumn) { - float data = 5.1f; - *output1->mutable_tensor_content() = std::string((char*)(&data), ((char*)(&data)) + sizeof(float)); - output1->set_dtype(tensorflow::DataType::DT_FLOAT); - std::string expected_json = "{\n \"outputs\": 5.1\n}"; - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, Order::COLUMN), StatusCode::OK); - EXPECT_EQ(json, expected_json); -} - -TEST_F(TFSMakeJsonFromPredictResponseStringTest, PositiveColumnBatchSize2) { - output1->add_string_val("Hello"); - output1->add_string_val("World"); - output1->mutable_tensor_shape()->add_dim()->set_size(2); - std::string expected_json = "{\n \"outputs\": [\n \"Hello\",\n \"World\"\n ]\n}"; - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, Order::COLUMN), StatusCode::OK); - EXPECT_EQ(json, expected_json); -} - -TEST_F(TFSMakeJsonFromPredictResponseRawTest, CannotConvertUnknownOrder) { - EXPECT_EQ(makeJsonFromPredictResponse(proto, &json, Order::UNKNOWN), StatusCode::REST_PREDICT_UNKNOWN_ORDER); -} - -TEST_F(TFSMakeJsonFromPredictResponseRawTest, CannotConvertInvalidPrecision) { - output1->set_dtype(tensorflow::DataType::DT_INVALID); - output1->mutable_tensor_content()->clear(); - EXPECT_EQ(makeJsonFromPredictResponse(proto, &json, Order::COLUMN), StatusCode::REST_UNSUPPORTED_PRECISION); -} - -const char* rawPositiveFirstOrderResponseRow = R"({ - "predictions": [ - { - "output1": [[5.0, 10.0, -3.0, 2.5]], - "output2": [5, 2, 3, 8, -2] - }, - { - "output1": [[9.0, 55.5, -0.5, -1.5]], - "output2": [-100, 0, 125, 4, -1] - } - ] -})"; - -const char* rawPositiveFirstOrderResponseColumn = R"({ - "outputs": { - "output1": [ - [ - [ - 5.0, - 10.0, - -3.0, - 2.5 - ] - ], - [ - [ - 9.0, - 55.5, - -0.5, - -1.5 - ] - ] - ], - "output2": [ - [ - 5, - 2, - 3, - 8, - -2 - ], - [ - -100, - 0, - 125, - 4, - -1 - ] - ] - } -})"; - -static std::string getJsonResponseDependsOnOrder(ovms::Order order, const char* rowOrderResponse, const char* columnOrderResponse) { - switch (order) { - case Order::ROW: - return rowOrderResponse; - case Order::COLUMN: - return columnOrderResponse; - default: - return ""; - } -} - -const char* rawPositiveSecondOrderResponseRow = R"({ - "predictions": [ - { - "output2": [5, 2, 3, 8, -2], - "output1": [[5.0, 10.0, -3.0, 2.5]] - }, - { - "output2": [-100, 0, 125, 4, -1], - "output1": [[9.0, 55.5, -0.5, -1.5]] - } - ] -})"; - -const char* rawPositiveSecondOrderResponseColumn = R"({ - "outputs": { - "output2": [ - [ - 5, - 2, - 3, - 8, - -2 - ], - [ - -100, - 0, - 125, - 4, - -1 - ] - ], - "output1": [ - [ - [ - 5.0, - 10.0, - -3.0, - 2.5 - ] - ], - [ - [ - 9.0, - 55.5, - -0.5, - -1.5 - ] - ] - ] - } -})"; - -TEST_P(TFSMakeJsonFromPredictResponseRawTest, PositiveNamed) { - auto order = GetParam(); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, order), StatusCode::OK); - bool is_in_first_order = (json == getJsonResponseDependsOnOrder(order, rawPositiveFirstOrderResponseRow, rawPositiveFirstOrderResponseColumn)); - - bool is_in_second_order = (json == getJsonResponseDependsOnOrder(order, rawPositiveSecondOrderResponseRow, rawPositiveSecondOrderResponseColumn)); - - EXPECT_TRUE(is_in_first_order || is_in_second_order); -} - -const char* rawPositiveNonameResponseRow = R"({ - "predictions": [[[5.0, 10.0, -3.0, 2.5]], [[9.0, 55.5, -0.5, -1.5]] - ] -})"; - -const char* rawPositiveNonameResponseColumn = R"({ - "outputs": [ - [ - [ - 5.0, - 10.0, - -3.0, - 2.5 - ] - ], - [ - [ - 9.0, - 55.5, - -0.5, - -1.5 - ] - ] - ] -})"; - -TEST_P(TFSMakeJsonFromPredictResponseRawTest, Positive_Noname) { - auto order = GetParam(); - proto.mutable_outputs()->erase("output2"); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, order), StatusCode::OK); - EXPECT_EQ(json, getJsonResponseDependsOnOrder(order, rawPositiveNonameResponseRow, rawPositiveNonameResponseColumn)); -} - -std::vector SupportedOrders = {Order::ROW, Order::COLUMN}; - -static std::string toString(ovms::Order order) { - switch (order) { - case Order::ROW: - return "ROW"; - case Order::COLUMN: - return "COLUMN"; - default: - return ""; - } -} - -TEST_P(TFSMakeJsonFromPredictResponseRawTest, TensorZeroDimPositive) { - auto order = GetParam(); - output1->mutable_tensor_content()->clear(); - output1->mutable_tensor_shape()->add_dim()->set_size(1); - output1->mutable_tensor_shape()->add_dim()->set_size(0); - EXPECT_EQ(makeJsonFromPredictResponse(proto, &json, order), StatusCode::OK); -} - -TEST_P(TFSMakeJsonFromPredictResponseRawTest, EmptyScalarTensorContentError) { - auto order = GetParam(); - output1->mutable_tensor_content()->clear(); - EXPECT_EQ(makeJsonFromPredictResponse(proto, &json, order), StatusCode::REST_SERIALIZE_VAL_FIELD_INVALID_SIZE); -} - -TEST_P(TFSMakeJsonFromPredictResponseRawTest, InvalidTensorContentSizeError) { - auto order = GetParam(); - output1->mutable_tensor_content()->assign("\xFF\xFF\x55\x55", 4); - EXPECT_EQ(makeJsonFromPredictResponse(proto, &json, order), StatusCode::REST_SERIALIZE_TENSOR_CONTENT_INVALID_SIZE); -} - -TEST_P(TFSMakeJsonFromPredictResponseRawTest, ErrorWhenNoOutputs) { - auto order = GetParam(); - proto.mutable_outputs()->clear(); - EXPECT_EQ(makeJsonFromPredictResponse(proto, &json, order), StatusCode::REST_PROTO_TO_STRING_ERROR); -} - -INSTANTIATE_TEST_SUITE_P( - TestGrpcRestResponseConversion, - TFSMakeJsonFromPredictResponseRawTest, - ::testing::ValuesIn(SupportedOrders), - [](const ::testing::TestParamInfo& info) { - return toString(info.param); - }); - -class TFSMakeJsonFromPredictResponsePrecisionTest : public ::testing::TestWithParam { -protected: - TFSResponseType proto; - std::string json; - TFSOutputTensorType* output; - - void SetUp() override { - output = &((*proto.mutable_outputs())["output"]); - output->mutable_tensor_shape()->add_dim()->set_size(1); - output->mutable_tensor_shape()->add_dim()->set_size(1); - } -}; - -const char* noDataResponseRow = R"({ - "predictions": [[] - ] -})"; - -const char* noDataResponseColumn = R"({ - "outputs": [ - [] - ] -})"; - -TEST_P(TFSMakeJsonFromPredictResponsePrecisionTest, NoData) { - auto order = GetParam(); - output->mutable_tensor_shape()->mutable_dim(1)->set_size(0); - output->set_dtype(tensorflow::DataType::DT_FLOAT); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, order), StatusCode::OK); - EXPECT_EQ(json, getJsonResponseDependsOnOrder(order, noDataResponseRow, noDataResponseColumn)); -} - -const char* floatResponseRow = R"({ - "predictions": [[92.5] - ] -})"; - -const char* floatResponseColumn = R"({ - "outputs": [ - [ - 92.5 - ] - ] -})"; - -TEST_P(TFSMakeJsonFromPredictResponsePrecisionTest, Float) { - auto order = GetParam(); - float data = 92.5f; - output->set_dtype(tensorflow::DataType::DT_FLOAT); - output->mutable_tensor_content()->assign(reinterpret_cast(&data), sizeof(float)); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, order), StatusCode::OK); - EXPECT_EQ(json, getJsonResponseDependsOnOrder(order, floatResponseRow, floatResponseColumn)); -} - -const char* doubleResponseRow = R"({ - "predictions": [[15.99] - ] -})"; - -const char* doubleResponseColumn = R"({ - "outputs": [ - [ - 15.99 - ] - ] -})"; - -TEST_P(TFSMakeJsonFromPredictResponsePrecisionTest, Double) { - auto order = GetParam(); - double data = 15.99; - output->set_dtype(tensorflow::DataType::DT_DOUBLE); - output->mutable_tensor_content()->assign(reinterpret_cast(&data), sizeof(double)); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, GetParam()), StatusCode::OK); - EXPECT_EQ(json, getJsonResponseDependsOnOrder(order, doubleResponseRow, doubleResponseColumn)); -} - -const char* int32ResponseRow = R"({ - "predictions": [[-82] - ] -})"; - -const char* int32ResponseColumn = R"({ - "outputs": [ - [ - -82 - ] - ] -})"; - -TEST_P(TFSMakeJsonFromPredictResponsePrecisionTest, Int32) { - auto order = GetParam(); - int32_t data = -82; - output->set_dtype(tensorflow::DataType::DT_INT32); - output->mutable_tensor_content()->assign(reinterpret_cast(&data), sizeof(int32_t)); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, order), StatusCode::OK); - EXPECT_EQ(json, getJsonResponseDependsOnOrder(order, int32ResponseRow, int32ResponseColumn)); -} - -const char* int16ResponseRow = R"({ - "predictions": [[-945] - ] -})"; - -const char* int16ResponseColumn = R"({ - "outputs": [ - [ - -945 - ] - ] -})"; - -TEST_P(TFSMakeJsonFromPredictResponsePrecisionTest, Int16) { - auto order = GetParam(); - int16_t data = -945; - output->set_dtype(tensorflow::DataType::DT_INT16); - output->mutable_tensor_content()->assign(reinterpret_cast(&data), sizeof(int16_t)); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, order), StatusCode::OK); - EXPECT_EQ(json, getJsonResponseDependsOnOrder(order, int16ResponseRow, int16ResponseColumn)); -} - -const char* int8ResponseRow = R"({ - "predictions": [[-53] - ] -})"; - -const char* int8ResponseColumn = R"({ - "outputs": [ - [ - -53 - ] - ] -})"; - -TEST_P(TFSMakeJsonFromPredictResponsePrecisionTest, Int8) { - auto order = GetParam(); - int8_t data = -53; - output->set_dtype(tensorflow::DataType::DT_INT8); - output->mutable_tensor_content()->assign(reinterpret_cast(&data), sizeof(int8_t)); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, order), StatusCode::OK); - EXPECT_EQ(json, getJsonResponseDependsOnOrder(order, int8ResponseRow, int8ResponseColumn)); -} - -const char* uint8ResponseRow = R"({ - "predictions": [[250] - ] -})"; - -const char* uint8ResponseColumn = R"({ - "outputs": [ - [ - 250 - ] - ] -})"; - -TEST_P(TFSMakeJsonFromPredictResponsePrecisionTest, Uint8) { - auto order = GetParam(); - uint8_t data = 250; - output->set_dtype(tensorflow::DataType::DT_UINT8); - output->mutable_tensor_content()->assign(reinterpret_cast(&data), sizeof(uint8_t)); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, order), StatusCode::OK); - EXPECT_EQ(json, getJsonResponseDependsOnOrder(order, uint8ResponseRow, uint8ResponseColumn)); -} - -const char* int64ResponseRow = R"({ - "predictions": [[-658324] - ] -})"; - -const char* int64ResponseColumn = R"({ - "outputs": [ - [ - -658324 - ] - ] -})"; - -TEST_P(TFSMakeJsonFromPredictResponsePrecisionTest, Int64) { - auto order = GetParam(); - int64_t data = -658324; - output->set_dtype(tensorflow::DataType::DT_INT64); - output->mutable_tensor_content()->assign(reinterpret_cast(&data), sizeof(int64_t)); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, order), StatusCode::OK); - EXPECT_EQ(json, getJsonResponseDependsOnOrder(order, int64ResponseRow, int64ResponseColumn)); -} - -const char* uint32ResponseRow = R"({ - "predictions": [[1245353] - ] -})"; - -const char* uint32ResponseColumn = R"({ - "outputs": [ - [ - 1245353 - ] - ] -})"; - -TEST_P(TFSMakeJsonFromPredictResponsePrecisionTest, Uint32) { - auto order = GetParam(); - uint32_t data = 1245353; - output->set_dtype(tensorflow::DataType::DT_UINT32); - output->mutable_tensor_content()->assign(reinterpret_cast(&data), sizeof(uint32_t)); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, order), StatusCode::OK); - EXPECT_EQ(json, getJsonResponseDependsOnOrder(order, uint32ResponseRow, uint32ResponseColumn)); -} - -const char* uint64ResponseRow = R"({ - "predictions": [[63456412] - ] -})"; - -const char* uint64ResponseColumn = R"({ - "outputs": [ - [ - 63456412 - ] - ] -})"; - -TEST_P(TFSMakeJsonFromPredictResponsePrecisionTest, Uint64) { - auto order = GetParam(); - uint64_t data = 63456412; - output->set_dtype(tensorflow::DataType::DT_UINT64); - output->mutable_tensor_content()->assign(reinterpret_cast(&data), sizeof(uint64_t)); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, order), StatusCode::OK); - EXPECT_EQ(json, getJsonResponseDependsOnOrder(order, uint64ResponseRow, uint64ResponseColumn)); -} - -INSTANTIATE_TEST_SUITE_P( - TestGrpcRestResponseConversion, - TFSMakeJsonFromPredictResponsePrecisionTest, - ::testing::ValuesIn(SupportedOrders), - [](const ::testing::TestParamInfo& info) { - return toString(info.param); - }); - -class TFSMakeJsonFromPredictResponseValTest : public ::testing::TestWithParam { -protected: - TFSResponseType proto; - std::string json; - TFSOutputTensorType *tensor_content_output, *single_uint64_val, *two_uint32_vals; - - void SetUp() override { - tensor_content_output = &((*proto.mutable_outputs())["tensor_content_output"]); - single_uint64_val = &((*proto.mutable_outputs())["single_uint64_val"]); - two_uint32_vals = &((*proto.mutable_outputs())["two_uint32_vals"]); - - tensor_content_output->set_dtype(tensorflow::DataType::DT_FLOAT); - single_uint64_val->set_dtype(tensorflow::DataType::DT_UINT64); - two_uint32_vals->set_dtype(tensorflow::DataType::DT_UINT32); - - float data[8] = {5.0f, 10.0f, -3.0f, 2.5f, - 9.0f, 55.5f, -0.5f, -1.5f}; - tensor_content_output->mutable_tensor_content()->assign(reinterpret_cast(data), 8 * sizeof(float)); - tensor_content_output->mutable_tensor_shape()->add_dim()->set_size(2); - tensor_content_output->mutable_tensor_shape()->add_dim()->set_size(1); - tensor_content_output->mutable_tensor_shape()->add_dim()->set_size(4); - - single_uint64_val->mutable_tensor_shape()->add_dim()->set_size(1); - single_uint64_val->add_uint64_val(5000000000); - - two_uint32_vals->mutable_tensor_shape()->add_dim()->set_size(2); - two_uint32_vals->add_uint32_val(4000000000); - two_uint32_vals->add_uint32_val(1); - } -}; - -TEST_F(TFSMakeJsonFromPredictResponseValTest, MakeJsonFromPredictResponse_ColumnOrder_ContainSingleUint64Val) { - proto.mutable_outputs()->erase("two_uint32_vals"); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, Order::COLUMN), StatusCode::OK); - - bool is_in_first_order = json == R"({ - "outputs": { - "tensor_content_output": [ - [ - [ - 5.0, - 10.0, - -3.0, - 2.5 - ] - ], - [ - [ - 9.0, - 55.5, - -0.5, - -1.5 - ] - ] - ], - "single_uint64_val": [ - 5000000000 - ] - } -})"; - - bool is_in_second_order = json == R"({ - "outputs": { - "single_uint64_val": [ - 5000000000 - ], - "tensor_content_output": [ - [ - [ - 5.0, - 10.0, - -3.0, - 2.5 - ] - ], - [ - [ - 9.0, - 55.5, - -0.5, - -1.5 - ] - ] - ] - } -})"; - - EXPECT_TRUE(is_in_first_order || is_in_second_order); -} - -TEST_F(TFSMakeJsonFromPredictResponseValTest, MakeJsonFromPredictResponse_ColumnOrder_ContainTwoUint32Vals) { - proto.mutable_outputs()->erase("single_uint64_val"); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, Order::COLUMN), StatusCode::OK); - - bool is_in_first_order = json == R"({ - "outputs": { - "tensor_content_output": [ - [ - [ - 5.0, - 10.0, - -3.0, - 2.5 - ] - ], - [ - [ - 9.0, - 55.5, - -0.5, - -1.5 - ] - ] - ], - "two_uint32_vals": [ - 4000000000, - 1 - ] - } -})"; - - bool is_in_second_order = json == R"({ - "outputs": { - "two_uint32_vals": [ - 4000000000, - 1 - ], - "tensor_content_output": [ - [ - [ - 5.0, - 10.0, - -3.0, - 2.5 - ] - ], - [ - [ - 9.0, - 55.5, - -0.5, - -1.5 - ] - ] - ] - } -})"; - - EXPECT_TRUE(is_in_first_order || is_in_second_order); -} - class KFSMakeJsonFromPredictResponseRawTest : public ::testing::Test { protected: KFSResponse proto; diff --git a/src/test/serialization_tests.cpp b/src/test/serialization_tests.cpp index 13356316a7..ce7654502c 100644 --- a/src/test/serialization_tests.cpp +++ b/src/test/serialization_tests.cpp @@ -26,25 +26,13 @@ #include -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop - #include "../capi_frontend/buffer.hpp" #include "../capi_frontend/inferenceresponse.hpp" #include "../capi_frontend/inferencetensor.hpp" -#include "../tfs_frontend/serialization.hpp" #include "../kfs_frontend/serialization.hpp" #include "../capi_frontend/serialization.hpp" -#include "../tfs_frontend/tfs_utils.hpp" #include "test_utils.hpp" -using TFTensorProto = tensorflow::TensorProto; - -using TFPredictRequest = tensorflow::serving::PredictRequest; -using TFPredictResponse = tensorflow::serving::PredictResponse; - using namespace ovms; using testing::_; @@ -53,7 +41,6 @@ using testing::NiceMock; using testing::Throw; KFSRequest* KFSRequestNULL{nullptr}; -TFSPredictRequest* TFSRequestNULL{nullptr}; ovms::InferenceRequest* CAPIRequestNULL{nullptr}; const std::vector SUPPORTED_OUTPUT_PRECISIONS{ @@ -177,146 +164,6 @@ const std::string UNUSED_NAME{"UNUSED_NAME"}; const model_version_t UNUSED_VERSION{0}; } // namespace -class TensorflowGRPCPredict : public ::testing::TestWithParam { -protected: - void SetUp() override { - ovms::Precision precision = ovms::Precision::FP32; - - tensorMap[tensorName] = std::make_shared( - tensorName, - precision, - shape_t{1, 3, 1, 1}, - Layout{"NHWC"}); - SetUpTensorProto(getPrecisionAsDataType(precision)); - } - - void SetUpTensorProto(tensorflow::DataType dataType) { - tensorProto.set_dtype(dataType); - auto tensorShape = tensorProto.mutable_tensor_shape(); - tensorShape->Clear(); - tensorShape->add_dim()->set_size(1); - tensorShape->add_dim()->set_size(3); - tensorShape->add_dim()->set_size(1); - tensorShape->add_dim()->set_size(1); - *(tensorProto.mutable_tensor_content()) = std::string(1 * 3 * 1 * 1, '1'); - } - TFTensorProto tensorProto; - const char* tensorName = "Input_PRECISION_1_3_1_1_NHWC"; - ovms::tensor_map_t tensorMap; -}; - -class SerializeTFTensorProto : public TensorflowGRPCPredict { -public: - std::tuple< - std::shared_ptr, - ov::Tensor> - getInputs(ovms::Precision precision) { - std::shared_ptr servableOutput = - std::make_shared( - std::string("2_values_C_layout"), - precision, - ovms::Shape{2}, - Layout{"C"}); - ov::Tensor mockTensor = ov::Tensor( - ovmsPrecisionToIE2Precision(precision), ov::Shape{2}); - return std::make_tuple(servableOutput, mockTensor); - } -}; - -TEST(SerializeTFTensorProtoSingle, NegativeMismatchBetweenTensorInfoAndTensorPrecision) { - ovms::Precision tensorInfoPrecision = ovms::Precision::FP32; - shape_t tensorInfoShape{1, 3, 224, 224}; - auto layout = Layout{"NCHW"}; - const std::string name = "NOT_IMPORTANT"; - auto tensorInfo = std::make_shared(name, tensorInfoPrecision, tensorInfoShape, layout); - ov::Tensor tensor(ov::element::i32, tensorInfoShape); - TFTensorProto responseOutput; - auto status = serializeTensorToTensorProto(responseOutput, - tensorInfo, - tensor); - EXPECT_EQ(status.getCode(), ovms::StatusCode::INTERNAL_ERROR); -} - -TEST(SerializeTFTensorProtoSingle, NegativeMismatchBetweenTensorInfoAndTensorShape) { - ovms::Precision tensorInfoPrecision = ovms::Precision::FP32; - shape_t tensorInfoShape{1, 3, 224, 224}; - shape_t tensorShape{1, 3, 225, 225}; - auto layout = Layout{"NCHW"}; - const std::string name = "NOT_IMPORTANT"; - auto tensorInfo = std::make_shared(name, tensorInfoPrecision, tensorInfoShape, layout); - ov::Tensor tensor(tensorInfo->getOvPrecision(), tensorShape); - TFTensorProto responseOutput; - auto status = serializeTensorToTensorProto(responseOutput, - tensorInfo, - tensor); - EXPECT_EQ(status.getCode(), ovms::StatusCode::INTERNAL_ERROR); -} - -TEST_P(SerializeTFTensorProto, SerializeTensorProtoShouldSucceedForPrecision) { - ovms::Precision testedPrecision = GetParam(); - auto inputs = getInputs(testedPrecision); - TFTensorProto responseOutput; - ov::Tensor mockTensor = std::get<1>(inputs); - auto status = serializeTensorToTensorProto(responseOutput, - std::get<0>(inputs), - mockTensor); - EXPECT_TRUE(status.ok()) - << "Supported OV serialization precision" - << toString(testedPrecision) - << "should succeed"; -} - -class SerializeTFTensorProtoNegative : public SerializeTFTensorProto {}; - -TEST_P(SerializeTFTensorProtoNegative, SerializeTensorProtoShouldFailForPrecision) { - ovms::Precision testedPrecision = GetParam(); - auto inputs = getInputs(testedPrecision); - TFTensorProto responseOutput; - auto status = serializeTensorToTensorProto(responseOutput, - std::get<0>(inputs), - std::get<1>(inputs)); - EXPECT_EQ(status, ovms::StatusCode::OV_UNSUPPORTED_SERIALIZATION_PRECISION) - << "Unsupported OV serialization precision" - << toString(testedPrecision) - << "should fail"; -} - -TEST(SerializeTFGRPCPredictResponse, ShouldSuccessForSupportedPrecision) { - TFPredictResponse response; - ov::Core ieCore; - std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().u8string() + "/src/test/dummy/1/dummy.xml"); - ov::CompiledModel compiledModel = ieCore.compile_model(model, "CPU"); - ov::InferRequest inferRequest = compiledModel.create_infer_request(); - ovms::tensor_map_t tenMap; - std::shared_ptr tensorInfo = std::make_shared( - DUMMY_MODEL_OUTPUT_NAME, - ovms::Precision::FP32, - ovms::Shape{1, 10}, - Layout{"NC"}); - tenMap[DUMMY_MODEL_OUTPUT_NAME] = tensorInfo; - ov::Tensor tensor(tensorInfo->getOvPrecision(), ov::Shape{1, 10}); - inferRequest.set_tensor(DUMMY_MODEL_OUTPUT_NAME, tensor); - OutputGetter outputGetter(inferRequest); - auto status = serializePredictResponse(outputGetter, UNUSED_NAME, UNUSED_VERSION, tenMap, &response, getTensorInfoName); - EXPECT_TRUE(status.ok()); -} - -INSTANTIATE_TEST_SUITE_P( - Test, - SerializeTFTensorProto, - ::testing::ValuesIn(SUPPORTED_OUTPUT_PRECISIONS), - [](const ::testing::TestParamInfo& info) { - return toString(info.param); - }); - -INSTANTIATE_TEST_SUITE_P( - Test, - SerializeTFTensorProtoNegative, - ::testing::ValuesIn(UNSUPPORTED_OUTPUT_PRECISIONS), - [](const ::testing::TestParamInfo& info) { - return toString(info.param); - }); - class KFServingGRPCPredict : public ::testing::TestWithParam { protected: void SetUp() override { @@ -673,18 +520,6 @@ Status OutputGetter::get(const std::string& name, ov::Ten namespace ovms { template <> -Status serializePredictResponse( - OutputGetter& outputGetter, - const std::string& servableName, - model_version_t servableVersion, - const tensor_map_t& outputMap, - const tensorflow::serving::PredictRequest* request, - tensorflow::serving::PredictResponse* response, - outputNameChooser_t outputNameChooser, - bool useSharedOutputContent) { - return serializePredictResponse(outputGetter, servableName, servableVersion, outputMap, response, outputNameChooser, useSharedOutputContent); -} -template <> Status serializePredictResponse( OutputGetter& outputGetter, const std::string& servableName, @@ -697,17 +532,6 @@ Status serializePredictResponse( return serializePredictResponse(outputGetter, servableName, servableVersion, outputMap, response, outputNameChooser, useSharedOutputContent); } } // namespace ovms -template ovms::Status ovms::serializePredictResponse( - ovms::OutputGetter&, - const std::string&, - model_version_t servableVersion, - const tensor_map_t& outputMap, - const TFSPredictRequest* request, - tensorflow::serving::PredictResponse*, - outputNameChooser_t outputNameChooser, - bool); - template ovms::Status ovms::serializePredictResponse( ovms::OutputGetter&, @@ -821,7 +645,7 @@ class SerializeString : public ::testing::Test { RequestType request; }; -using MyTypes = ::testing::Types; +using MyTypes = ::testing::Types; TYPED_TEST_SUITE(SerializeString, MyTypes); // Serialization to string due to suffix _string in mapping diff --git a/src/test/stress_test_utils.hpp b/src/test/stress_test_utils.hpp index a5a7caffdd..41bd4aaf47 100644 --- a/src/test/stress_test_utils.hpp +++ b/src/test/stress_test_utils.hpp @@ -38,12 +38,10 @@ #include "../dags/pipeline.hpp" #include "../dags/pipeline_factory.hpp" #include "../dags/pipelinedefinition.hpp" -#include "../get_model_metadata_impl.hpp" #include "../kfs_frontend/kfs_utils.hpp" #include "src/metrics/metric_config.hpp" #include "src/filesystem/localfilesystem.hpp" #include "../logging.hpp" -#include "../model_service.hpp" #include "../modelconfig.hpp" #include "../modelinstance.hpp" #include "../prediction_service_utils.hpp" @@ -52,7 +50,7 @@ #include "../status.hpp" #include "../stringutils.hpp" #include "src/timer.hpp" -#include "../tfs_frontend/tfs_utils.hpp" +#include "../tensorflow_type_utils.hpp" #include "c_api_test_utils.hpp" #include "test_utils.hpp" #include "light_test_utils.hpp" @@ -63,8 +61,6 @@ #endif using namespace ovms; -using namespace tensorflow; -using namespace tensorflow::serving; using testing::_; using testing::Return; @@ -346,49 +342,6 @@ static const char* stressTestPipelineOneDummyConfigAddNewPipeline = R"( ] ] })"; -static const char* stressTestPipelineOneDummyConfigSpecificVersionUsed = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"latest": {"num_versions":1}}, - "nireq": 100, - "shape": {"b": "(1,10) "} - } - } - ], - "pipeline_config_list": [ - { - "name": "pipeline1Dummy", - "inputs": ["custom_dummy_input"], - "nodes": [ - { - "name": "dummyNode", - "model_name": "dummy", - "version": 1, - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "custom_dummy_input"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ] - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode", - "data_item": "new_dummy_output"} - } - ] - } - ] -})"; - static const char* stressPipelineCustomNodeDifferentOperationsThenDummyThenChooseMaximumRemovedLibraryConfig = R"( { "custom_node_library_config_list": [ @@ -553,73 +506,6 @@ static const char* stressPipelineCustomNodeDifferentOperationsThenDummyThenChoos ] })"; -static const char* stressPipelineCustomNodeAddOneThenDummy = R"( -{ - "custom_node_library_config_list": [ - { - "name": "lib_add_one", - "base_path": "/ovms/bazel-bin/src/libcustom_node_add_one.so" - } - ], - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 20, - "shape": {"b": "(1,10) "} - } - } - ], - "pipeline_config_list": [ - { - "name": "pipeline1Dummy", - "inputs": ["custom_dummy_input"], - "nodes": [ - { - "name": "custom_node", - "library_name": "lib_add_one", - "type": "custom", - "params": { - "output_queue_size": "20", - "info_queue_size": "20", - "add_number": "1", - "sub_number": "0" - }, - "inputs": [ - {"input_numbers": {"node_name": "request", - "data_item": "custom_dummy_input"}} - ], - "outputs": [ - {"data_item": "output_numbers", - "alias": "custom_node_output"} - ] - }, - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "custom_node", - "data_item": "custom_node_output"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "dummy_output"} - ] - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode", - "data_item": "dummy_output"} - } - ] - } - ] -})"; - static const char* stressPipelineCustomNodeAddOneThenDummyRemovedLibraryConfig = R"( { "custom_node_library_config_list": [], @@ -1580,85 +1466,6 @@ class ConfigChangeStressTest : public TestWithTempDir { SPDLOG_INFO("{} end", __FUNCTION__); } - bool isMetadataResponseCorrect(tensorflow::serving::GetModelMetadataResponse& response, SERVABLE_TYPE servableType) { - tensorflow::serving::SignatureDefMap def; - EXPECT_EQ(response.model_spec().name(), pipelineName); - EXPECT_TRUE(response.model_spec().has_version()); - EXPECT_EQ(response.model_spec().version().value(), 1); - EXPECT_EQ(response.metadata_size(), 1); - EXPECT_NE( - response.metadata().find("signature_def"), - response.metadata().end()); - response.metadata().at("signature_def").UnpackTo(&def); - response.metadata().at("signature_def").UnpackTo(&def); - const auto& inputs = ((*def.mutable_signature_def())["serving_default"]).inputs(); - const auto& outputs = ((*def.mutable_signature_def())["serving_default"]).outputs(); - auto expectedInputs = getExpectedInputsInfo(); - std::cout << "Expected inputs: " << expectedInputs.size() << std::endl; - bool inputsSizeCorrect{inputs.size() == expectedInputs.size()}; - EXPECT_TRUE(inputsSizeCorrect) << "Expected: " << expectedInputs.size() << " actual: " << inputs.size(); - bool outputsSizeCorrect{outputs.size() == 1}; - EXPECT_TRUE(outputsSizeCorrect) << "Expected: " << 1 << " actual: " << outputs.size(); - if (!inputsSizeCorrect || !outputsSizeCorrect) { - return false; - } - for (auto& [expectedInputName, shapeTypeTuple] : expectedInputs) { - bool inputNameExist = inputs.find(expectedInputName.c_str()) != inputs.end(); - EXPECT_TRUE(inputNameExist); - if (!inputNameExist) { - return false; - } - bool inputNameCorrect{inputs.at(pipelineInputName.c_str()).name() == pipelineInputName}; - EXPECT_TRUE(inputNameCorrect); - if (!inputNameCorrect) { - return false; - } - bool inputTypeCorrect{inputs.at(pipelineInputName.c_str()).dtype() == tensorflow::DT_FLOAT}; - EXPECT_TRUE(inputTypeCorrect); - if (!inputTypeCorrect) { - return false; - } - bool inputShapeCorrect = false; - std::vector expectedShape; - if (servableType == SERVABLE_TYPE::DAG) { - expectedShape = std::vector{1, 10}; - } else if (servableType == SERVABLE_TYPE::MEDIAPIPE) { - expectedShape = std::vector{}; - } else { - EXPECT_TRUE(false) << "Unsupported checks"; - } - inputShapeCorrect = isShapeTheSame( - inputs.at(pipelineInputName.c_str()).tensor_shape(), - std::move(expectedShape)); - EXPECT_TRUE(inputShapeCorrect); - if (!inputShapeCorrect) { - return false; - } - } - bool outputNameExist{outputs.find(pipelineOutputName.c_str()) != outputs.end()}; - EXPECT_TRUE(outputNameExist); - if (!outputNameExist) { - return false; - } - bool outputNameCorrect{outputs.at(pipelineOutputName.c_str()).name() == pipelineOutputName}; - EXPECT_TRUE(outputNameCorrect); - if (!outputNameCorrect) { - return false; - } - bool outputTypeCorrect{outputs.at(pipelineOutputName.c_str()).dtype() == tensorflow::DT_FLOAT}; - EXPECT_TRUE(outputTypeCorrect); - if (!outputTypeCorrect) { - return false; - } - bool outputShapeCorrect{isShapeTheSame( - outputs.at(pipelineOutputName.c_str()).tensor_shape(), - {1, 10})}; - EXPECT_TRUE(outputShapeCorrect); - if (!outputShapeCorrect) { - return false; - } - return true; - } #if (MEDIAPIPE_DISABLE == 0) void isKFSMetadataResponseCorrect(KFSModelMetadataResponse& response, SERVABLE_TYPE servableType) { EXPECT_EQ(response.name(), pipelineName); @@ -1674,52 +1481,6 @@ class ConfigChangeStressTest : public TestWithTempDir { EXPECT_EQ(response.outputs()[0].datatype(), "INVALID"); EXPECT_TRUE(isShapeTheSame(response.outputs()[0].shape(), std::move(std::vector{}))); } -#endif - template < - typename RequestType = tensorflow::serving::GetModelMetadataRequest, - typename ResponseType = tensorflow::serving::GetModelMetadataResponse, - typename ServableType = ovms::Pipeline> - void triggerGetPipelineMetadataInALoop( - std::future& startSignal, - std::future& stopSignal, - const std::set& requiredLoadResults, - const std::set& allowedLoadResults, - std::unordered_map>& createPipelineRetCodesCounters) { - tensorflow::serving::GetModelMetadataRequest request; - startSignal.get(); - // stressIterationsCounter is additional safety measure - auto stressIterationsCounter = stressIterationsLimit; - while (stressIterationsCounter-- > 0) { - auto futureWaitResult = stopSignal.wait_for(std::chrono::milliseconds(0)); - if (futureWaitResult == std::future_status::ready) { - SPDLOG_INFO("Got stop signal. Ending Load"); - break; - } - auto status = ovms::GetModelMetadataImpl::createGrpcRequest(pipelineName, 1, &request); - tensorflow::serving::GetModelMetadataResponse response; - status = ovms::GetModelMetadataImpl::getModelStatus(&request, &response, *(this->manager), ovms::ExecutionContext(ovms::ExecutionContext::Interface::GRPC, ovms::ExecutionContext::Method::GetModelMetadata)); - createPipelineRetCodesCounters[status.getCode()]++; - EXPECT_TRUE((requiredLoadResults.find(status.getCode()) != requiredLoadResults.end()) || - (allowedLoadResults.find(status.getCode()) != allowedLoadResults.end())) - << status.string() << "\n"; - if (!status.ok()) { - continue; - } - // Check response if correct - SERVABLE_TYPE servableType = SERVABLE_TYPE::DAG; -#if (MEDIAPIPE_DISABLE == 0) - if (typeid(ServableType) == typeid(MediapipeGraphExecutor)) { - servableType = SERVABLE_TYPE::MEDIAPIPE; - } -#endif - EXPECT_TRUE(isMetadataResponseCorrect(response, servableType)); - if (::testing::Test::HasFailure()) { - SPDLOG_INFO("Earlier fail detected. Stopping execution"); - break; - } - } - } -#if (MEDIAPIPE_DISABLE == 0) template < typename RequestType = KFSModelMetadataRequest, typename ResponseType = KFSModelMetadataResponse, @@ -1745,7 +1506,7 @@ class ConfigChangeStressTest : public TestWithTempDir { KFSModelExtraMetadata extraMetadata; ovms::Server& server = ovms::Server::instance(); KFSInferenceServiceImpl impl(server); - auto status = impl.ModelMetadataImpl(nullptr, &request, &response, ovms::ExecutionContext(ovms::ExecutionContext::Interface::GRPC, ovms::ExecutionContext::Method::GetModelMetadata), extraMetadata); + auto status = impl.ModelMetadataImpl(nullptr, &request, &response, ovms::ExecutionContext(ovms::ExecutionContext::Interface::GRPC, ovms::ExecutionContext::Method::ModelMetadata), extraMetadata); createPipelineRetCodesCounters[status.getCode()]++; EXPECT_TRUE((requiredLoadResults.find(status.getCode()) != requiredLoadResults.end()) || (allowedLoadResults.find(status.getCode()) != allowedLoadResults.end())) @@ -1766,59 +1527,12 @@ class ConfigChangeStressTest : public TestWithTempDir { } } #endif - void triggerGetPipelineStatusInALoop( - std::future& startSignal, - std::future& stopSignal, - const std::set& requiredLoadResults, - const std::set& allowedLoadResults, - std::unordered_map>& createPipelineRetCodesCounters) { - tensorflow::serving::GetModelStatusRequest request; - startSignal.get(); - // stressIterationsCounter is additional safety measure - // for getModelStatus requests it must be much higher since the response time is much lower - // as in contrast to predict/metadata requests - auto stressIterationsCounter = stressIterationsLimit * 100000; - while (stressIterationsCounter-- > 0) { - auto futureWaitResult = stopSignal.wait_for(std::chrono::milliseconds(0)); - if (futureWaitResult == std::future_status::ready) { - SPDLOG_INFO("Got stop signal. Ending Load"); - break; - } - auto status = ovms::GetModelStatusImpl::createGrpcRequest(getServableName(), 1, &request); - tensorflow::serving::GetModelStatusResponse response; - status = ovms::GetModelStatusImpl::getModelStatus(&request, &response, *(this->manager), ovms::ExecutionContext(ovms::ExecutionContext::Interface::GRPC, ovms::ExecutionContext::Method::GetModelStatus)); - createPipelineRetCodesCounters[status.getCode()]++; - EXPECT_TRUE((requiredLoadResults.find(status.getCode()) != requiredLoadResults.end()) || - (allowedLoadResults.find(status.getCode()) != allowedLoadResults.end())) - << status.string() << "\n"; - if (!status.ok()) { - continue; - } - if (::testing::Test::HasFailure()) { - SPDLOG_INFO("Earlier fail detected. Stopping execution"); - break; - } - } - } - - virtual tensorflow::serving::PredictRequest preparePipelinePredictRequest(tensorflow::serving::PredictRequest) { - tensorflow::serving::PredictRequest request; - preparePredictRequest(request, getExpectedInputsInfo()); - auto& input = (*request.mutable_inputs())[pipelineInputName]; - input.mutable_tensor_content()->assign((char*)requestData.data(), requestData.size() * sizeof(float)); - return request; - } virtual KFSRequest preparePipelinePredictRequest(KFSRequest&) { KFSRequest request; preparePredictRequest(request, getExpectedInputsInfo(), requestData); request.set_model_name(PIPELINE_1_DUMMY_NAME); return request; } - virtual void checkPipelineResponse(const std::string& pipelineOutputName, - TFSPredictRequest& request, - TFSPredictResponse& response) { - checkDummyResponse(pipelineOutputName, requestData, request, response, 1); - } virtual void checkPipelineResponse(const std::string& pipelineOutputName, KFSRequest& request, KFSResponse& response) { @@ -1883,7 +1597,7 @@ class ConfigChangeStressTest : public TestWithTempDir { if (typeid(ServableType) == typeid(ovms::Pipeline)) { executePipelineStatus = pipelinePtr->execute(ovms::ExecutionContext( ovms::ExecutionContext::Interface::GRPC, - ovms::ExecutionContext::Method::Predict)); + ovms::ExecutionContext::Method::ModelInfer)); #if (MEDIAPIPE_DISABLE == 0) } else if (typeid(ServableType) == typeid(ovms::MediapipeGraphExecutor)) { mediaexec(executorPtr, *(this->manager), request, response, executePipelineStatus); diff --git a/src/test/tensor_conversion_test.cpp b/src/test/tensor_conversion_test.cpp index 950fd6aed1..d0581daff9 100644 --- a/src/test/tensor_conversion_test.cpp +++ b/src/test/tensor_conversion_test.cpp @@ -20,10 +20,8 @@ #include #include "../capi_frontend/capi_utils.hpp" -#include "../tfs_frontend/tfs_utils.hpp" #include "../kfs_frontend/kfs_utils.hpp" #include "../capi_frontend/deserialization.hpp" -#include "../tfs_frontend/deserialization.hpp" #include "../kfs_frontend/deserialization.hpp" #include "../tensor_conversion.hpp" #include "opencv2/opencv.hpp" @@ -40,14 +38,6 @@ class NativeFileInputConversionTest : public ::testing::Test { void SetUp() override { prepareBinaryTensor(requestTensor); } - - void prepareBinaryTensor(tensorflow::TensorProto& tensor, std::unique_ptr& image_bytes, const size_t filesize, const size_t batchSize = 1) { - for (size_t i = 0; i < batchSize; i++) { - tensor.add_string_val(image_bytes.get(), filesize); - } - tensor.mutable_tensor_shape()->add_dim()->set_size(batchSize); - tensor.set_dtype(tensorflow::DataType::DT_STRING); - } void prepareBinaryTensor(::KFSRequest::InferInputTensor& tensor, std::unique_ptr& image_bytes, const size_t filesize, const size_t batchSize = 1) { for (size_t i = 0; i < batchSize; i++) { tensor.mutable_contents()->add_bytes_contents(image_bytes.get(), filesize); @@ -55,13 +45,6 @@ class NativeFileInputConversionTest : public ::testing::Test { tensor.mutable_shape()->Add(batchSize); tensor.set_datatype("BYTES"); } - void prepareBinaryTensor(tensorflow::TensorProto& tensor) { - size_t filesize; - std::unique_ptr image_bytes; - - readRgbJpg(filesize, image_bytes); - prepareBinaryTensor(tensor, image_bytes, filesize); - } void prepareBinaryTensor(::KFSRequest::InferInputTensor& tensor) { size_t filesize; std::unique_ptr image_bytes; @@ -70,17 +53,13 @@ class NativeFileInputConversionTest : public ::testing::Test { prepareBinaryTensor(tensor, image_bytes, filesize); } - void prepareBinaryTensor(tensorflow::TensorProto& tensor, std::string input) { - tensor.set_dtype(tensorflow::DataType::DT_STRING); - tensor.add_string_val(input); - } void prepareBinaryTensor(::KFSRequest::InferInputTensor& tensor, std::string input) { tensor.mutable_contents()->add_bytes_contents(input); tensor.set_datatype("BYTES"); } }; -using MyTypes = ::testing::Types; +using MyTypes = ::testing::Types<::KFSRequest::InferInputTensor>; TYPED_TEST_SUITE(NativeFileInputConversionTest, MyTypes); TYPED_TEST(NativeFileInputConversionTest, tensorWithNonMatchingBatchsize) { @@ -110,10 +89,7 @@ TYPED_TEST(NativeFileInputConversionTest, tensorWithEmptyTensor) { ov::Tensor tensor; auto tensorInfo = std::make_shared("", ovms::Precision::U8, ovms::Shape{1, 1, 1, 3}, Layout{"NHWC"}); - if (std::is_same::value) - EXPECT_EQ(convertNativeFileFormatRequestTensorToOVTensor(requestTensorEmptyInput, tensor, *tensorInfo, nullptr), ovms::StatusCode::STRING_VAL_EMPTY); - else - EXPECT_EQ(convertNativeFileFormatRequestTensorToOVTensor(requestTensorEmptyInput, tensor, *tensorInfo, nullptr), StatusCode::BYTES_CONTENTS_EMPTY); + EXPECT_EQ(convertNativeFileFormatRequestTensorToOVTensor(requestTensorEmptyInput, tensor, *tensorInfo, nullptr), StatusCode::BYTES_CONTENTS_EMPTY); } TYPED_TEST(NativeFileInputConversionTest, tensorWithNonSupportedLayout) { @@ -451,116 +427,41 @@ TYPED_TEST(NativeFileInputConversionTest, positive_range_resolution_matching_in_ } } -class NativeFileInputConversionTFSPrecisionTest : public ::testing::TestWithParam { +class NativeFileInputConversionKFSPrecisionTest : public ::testing::TestWithParam { protected: void SetUp() override { readRgbJpg(filesize, image_bytes); - stringVal.set_dtype(tensorflow::DataType::DT_STRING); - stringVal.add_string_val(image_bytes.get(), filesize); + inferTensorContent.mutable_contents()->add_bytes_contents(image_bytes.get(), filesize); } size_t filesize; std::unique_ptr image_bytes; - tensorflow::TensorProto stringVal; + ::KFSRequest::InferInputTensor inferTensorContent; }; -class NativeFileInputConversionTFSValidPrecisionTest : public NativeFileInputConversionTFSPrecisionTest {}; -class NativeFileInputConversionTFSInvalidPrecisionTest : public NativeFileInputConversionTFSPrecisionTest {}; - -TEST_P(NativeFileInputConversionTFSValidPrecisionTest, Valid) { - ovms::Precision testedPrecision = GetParam(); - - auto tensorInfo = std::make_shared("", - testedPrecision, - ovms::Shape{1, 1, 1, 3}, - Layout{"NHWC"}); - - ov::Tensor tensor; - ASSERT_EQ(convertNativeFileFormatRequestTensorToOVTensor(stringVal, tensor, *tensorInfo, nullptr), ovms::StatusCode::OK); - ASSERT_EQ(tensor.get_shape(), (ov::Shape{1, 1, 1, 3})); - ASSERT_EQ(tensor.get_size(), 3); - ASSERT_EQ(tensor.get_element_type(), ovmsPrecisionToIE2Precision(testedPrecision)); -} - -TEST_P(NativeFileInputConversionTFSInvalidPrecisionTest, Invalid) { - ovms::Precision testedPrecision = GetParam(); - - auto tensorInfo = std::make_shared("", - testedPrecision, - ovms::Shape{1, 1, 1, 3}, - Layout{"NHWC"}); - - ov::Tensor tensor; - ASSERT_EQ(convertNativeFileFormatRequestTensorToOVTensor(stringVal, tensor, *tensorInfo, nullptr), ovms::StatusCode::INVALID_PRECISION); -} +class NativeFileInputConversionKFSValidPrecisionTest : public NativeFileInputConversionKFSPrecisionTest {}; +class NativeFileInputConversionKFSInvalidPrecisionTest : public NativeFileInputConversionKFSPrecisionTest {}; const std::vector BINARY_SUPPORTED_INPUT_PRECISIONS{ - // ovms::Precision::UNSPECIFIED, - // ovms::Precision::MIXED, ovms::Precision::FP64, ovms::Precision::FP32, ovms::Precision::FP16, - // InferenceEngine::Precision::Q78, ovms::Precision::I16, ovms::Precision::U8, ovms::Precision::I8, ovms::Precision::U16, ovms::Precision::I32, - // ovms::Precision::I64, - // ovms::Precision::BIN, - // ovms::Precision::BOOL - // ovms::Precision::CUSTOM) }; -INSTANTIATE_TEST_SUITE_P( - Test, - NativeFileInputConversionTFSValidPrecisionTest, - ::testing::ValuesIn(BINARY_SUPPORTED_INPUT_PRECISIONS), - [](const ::testing::TestParamInfo& info) { - return toString(info.param); - }); - static const std::vector BINARY_UNSUPPORTED_INPUT_PRECISIONS{ ovms::Precision::UNDEFINED, ovms::Precision::MIXED, - // ovms::Precision::FP64, - // ovms::Precision::FP32, - // ovms::Precision::FP16, ovms::Precision::Q78, - // ovms::Precision::I16, - // ovms::Precision::U8, - // ovms::Precision::I8, - // ovms::Precision::U16, - // ovms::Precision::I32, ovms::Precision::I64, ovms::Precision::BIN, - ovms::Precision::BOOL - // ovms::Precision::CUSTOM) -}; - -INSTANTIATE_TEST_SUITE_P( - Test, - NativeFileInputConversionTFSInvalidPrecisionTest, - ::testing::ValuesIn(BINARY_UNSUPPORTED_INPUT_PRECISIONS), - [](const ::testing::TestParamInfo& info) { - return toString(info.param); - }); - -class NativeFileInputConversionKFSPrecisionTest : public ::testing::TestWithParam { -protected: - void SetUp() override { - readRgbJpg(filesize, image_bytes); - inferTensorContent.mutable_contents()->add_bytes_contents(image_bytes.get(), filesize); - } - - size_t filesize; - std::unique_ptr image_bytes; - ::KFSRequest::InferInputTensor inferTensorContent; + ovms::Precision::BOOL, }; -class NativeFileInputConversionKFSValidPrecisionTest : public NativeFileInputConversionKFSPrecisionTest {}; -class NativeFileInputConversionKFSInvalidPrecisionTest : public NativeFileInputConversionKFSPrecisionTest {}; - TEST_P(NativeFileInputConversionKFSValidPrecisionTest, Valid) { ovms::Precision testedPrecision = GetParam(); @@ -593,7 +494,7 @@ INSTANTIATE_TEST_SUITE_P( Test, NativeFileInputConversionKFSValidPrecisionTest, ::testing::ValuesIn(BINARY_SUPPORTED_INPUT_PRECISIONS), - [](const ::testing::TestParamInfo& info) { + [](const ::testing::TestParamInfo& info) { return toString(info.param); }); @@ -601,7 +502,7 @@ INSTANTIATE_TEST_SUITE_P( Test, NativeFileInputConversionKFSInvalidPrecisionTest, ::testing::ValuesIn(BINARY_UNSUPPORTED_INPUT_PRECISIONS), - [](const ::testing::TestParamInfo& info) { + [](const ::testing::TestParamInfo& info) { return toString(info.param); }); @@ -699,22 +600,16 @@ class StringInputsConversionTest : public ::testing::Test { TensorType requestTensor; void SetUp() override {} - void prepareStringTensor(tensorflow::TensorProto& tensor, std::vector inputStrings) { - prepareInferStringTensor(tensor, "UNUSED", inputStrings, true, nullptr); - } void prepareStringTensor(::KFSRequest::InferInputTensor& tensor, std::vector inputStrings) { prepareInferStringTensor(tensor, "UNUSED", inputStrings, true, nullptr); } - void prepareStringTensorWithRawInputContents(tensorflow::TensorProto& tensor, std::vector inputStrings, std::string& buffer) { - SPDLOG_ERROR("RawInputContents not supported for TFS API"); - } void prepareStringTensorWithRawInputContents(::KFSRequest::InferInputTensor& tensor, std::vector inputStrings, std::string& buffer) { prepareInferStringTensor(tensor, "UNUSED", inputStrings, false, &buffer); } }; -using MyTypes = ::testing::Types; +using MyTypes = ::testing::Types<::KFSRequest::InferInputTensor>; TYPED_TEST_SUITE(StringInputsConversionTest, MyTypes); TYPED_TEST(StringInputsConversionTest, positive) { @@ -726,8 +621,6 @@ TYPED_TEST(StringInputsConversionTest, positive) { } TYPED_TEST(StringInputsConversionTest, rawInputContents_positive) { - if (typeid(TypeParam) == typeid(TFSInputTensorType)) - GTEST_SKIP() << "String inputs in buffer not supported for TFS api"; std::vector expectedStrings = {"String_123"}; std::string rawInputContents; this->prepareStringTensorWithRawInputContents(this->requestTensor, expectedStrings, rawInputContents); @@ -745,8 +638,6 @@ TYPED_TEST(StringInputsConversionTest, positive_batch_size_2) { } TYPED_TEST(StringInputsConversionTest, rawInputContents_positive_batch_size_2) { - if (typeid(TypeParam) == typeid(TFSInputTensorType)) - GTEST_SKIP() << "String inputs in buffer not supported for TFS api"; std::vector expectedStrings = {"String_123", "zebra"}; std::string rawInputContents; this->prepareStringTensorWithRawInputContents(this->requestTensor, expectedStrings, rawInputContents); @@ -764,8 +655,6 @@ TYPED_TEST(StringInputsConversionTest, positive_batch_size_3_one_string_empty) { } TYPED_TEST(StringInputsConversionTest, rawInputContents_positive_batch_size_3_one_string_empty) { - if (typeid(TypeParam) == typeid(TFSInputTensorType)) - GTEST_SKIP() << "String inputs in buffer not supported for TFS api"; std::vector expectedStrings = {"String_123", "zebra", ""}; std::string rawInputContents; this->prepareStringTensorWithRawInputContents(this->requestTensor, expectedStrings, rawInputContents); @@ -785,8 +674,6 @@ TYPED_TEST(StringInputsConversionTest, positive_empty_inputs) { TYPED_TEST(StringInputsConversionTest, rawInputContents_positive_empty_inputs) { // This case can't happen because request validation dont allow empty strings - if (typeid(TypeParam) == typeid(TFSInputTensorType)) - GTEST_SKIP() << "String inputs in buffer not supported for TFS api"; std::vector expectedStrings = {}; std::string rawInputContents; this->prepareStringTensorWithRawInputContents(this->requestTensor, expectedStrings, rawInputContents); @@ -810,8 +697,6 @@ TYPED_TEST(StringInputsConversionTest, native_ov_string) { } TYPED_TEST(StringInputsConversionTest, rawInputContents_native_ov_string) { - if (typeid(TypeParam) == typeid(TFSInputTensorType)) - GTEST_SKIP() << "String inputs in buffer not supported for TFS api"; std::vector expectedStrings = {"ala", "", "ma", "kota"}; std::string rawInputContents; this->prepareStringTensorWithRawInputContents(this->requestTensor, expectedStrings, rawInputContents); @@ -833,7 +718,7 @@ class StringOutputsConversionTest : public ::testing::Test { void SetUp() override {} }; -using OutputTensorProtos = ::testing::Types; +using OutputTensorProtos = ::testing::Types; TYPED_TEST_SUITE(StringOutputsConversionTest, OutputTensorProtos); TYPED_TEST(StringOutputsConversionTest, positive) { @@ -848,3 +733,39 @@ TYPED_TEST(StringOutputsConversionTest, positive) { } } // namespace + +TEST(StringInputsConversionKFSTest, amplification_probe_empty_strings) { + // Demonstrates memory amplification: N empty strings in proto occupy ~2 bytes each on the wire, + // but ov::Tensor(string, {N}) allocates N*sizeof(std::string) objects on the heap. + // Conversion itself has no element-count limit; the guard lives in request validation. + // + // Projected worst-case (1 GB proto of empty strings, ~500M entries): + // proto size ~1 GB + // tensor obj ~500M * 32 B = ~16 GB <-- amplification + const int N = 1'000'000; + ::KFSRequest::InferInputTensor requestTensor; + requestTensor.set_datatype("BYTES"); + // No shape - falls back to 1D {N} + for (int i = 0; i < N; i++) { + requestTensor.mutable_contents()->add_bytes_contents(""); + } + size_t protoBytes = requestTensor.ByteSizeLong(); + size_t stringObjBytes = static_cast(N) * sizeof(std::string); + + ov::Tensor tensor; + // Conversion has NO element-count limit - this succeeds even for large N. + // The limit is enforced upstream by validateAgainstMaxNativeStringElementCount. + auto status = convertStringRequestToOVTensor(requestTensor, tensor, nullptr); + ASSERT_EQ(status, ovms::StatusCode::OK); + ASSERT_EQ(tensor.get_shape()[0], static_cast(N)); + + size_t limitElements = (1ULL << 30) / sizeof(std::string); + std::cout << "\n[amplification_probe]\n" + << " N entries : " << N << "\n" + << " Proto size : " << protoBytes / 1024 << " KB\n" + << " sizeof(string) : " << sizeof(std::string) << " bytes\n" + << " String obj alloc : " << stringObjBytes / (1024 * 1024) << " MB\n" + << " Amplification : " << stringObjBytes / (protoBytes + 1) << "x\n" + << " Validation limit : " << limitElements << " elements (~" + << limitElements * sizeof(std::string) / (1024 * 1024) << " MB object alloc)\n"; +} diff --git a/src/test/test_models.hpp b/src/test/test_models.hpp index c5f686d2ac..c6ff0ee054 100644 --- a/src/test/test_models.hpp +++ b/src/test/test_models.hpp @@ -18,6 +18,7 @@ #include #include "platform_utils.hpp" +#include "src/shape.hpp" const std::string dummy_model_location = getGenericFullPathForSrcTest(std::filesystem::current_path().u8string() + "/src/test/dummy", false); const std::string dummy_fp64_model_location = getGenericFullPathForSrcTest(std::filesystem::current_path().u8string() + "/src/test/dummy_fp64", false); diff --git a/src/test/test_utils.cpp b/src/test/test_utils.cpp index 64a704da0d..89ab7569fc 100644 --- a/src/test/test_utils.cpp +++ b/src/test/test_utils.cpp @@ -23,20 +23,16 @@ #include #include #include +#include #include #include "../capi_frontend/capi_utils.hpp" #include "../capi_frontend/inferenceparameter.hpp" #include "../kfs_frontend/kfs_utils.hpp" #include "../network_utils.hpp" -#include "../prediction_service_utils.hpp" #include "../servablemanagermodule.hpp" #include "../server.hpp" #include "../tensorinfo.hpp" -#include "../tfs_frontend/tfs_utils.hpp" - -using tensorflow::serving::PredictRequest; -using tensorflow::serving::PredictResponse; using ovms::TensorInfo; @@ -101,59 +97,6 @@ void printTensor(const ov::Tensor& tensor) { std::cout << "[ERROR] Unsupported data type: " << elementType << std::endl; } -void preparePredictRequest(tensorflow::serving::PredictRequest& request, inputs_info_t requestInputs, const std::vector& data) { - request.mutable_inputs()->clear(); - for (auto const& it : requestInputs) { - auto& name = it.first; - auto [shape, precision] = it.second; - - auto& input = (*request.mutable_inputs())[name]; - auto datatype = getPrecisionAsDataType(precision); - input.set_dtype(datatype); - size_t numberOfElements = 1; - for (auto const& dim : shape) { - input.mutable_tensor_shape()->add_dim()->set_size(dim); - numberOfElements *= dim; - } - switch (datatype) { - case tensorflow::DataType::DT_HALF: { - if (data.size() == 0) { - for (size_t i = 0; i < numberOfElements; i++) { - input.add_half_val('1'); - } - } else { - for (size_t i = 0; i < data.size(); i++) { - input.add_half_val(data[i]); - } - } - break; - } - case tensorflow::DataType::DT_UINT16: { - if (data.size() == 0) { - for (size_t i = 0; i < numberOfElements; i++) { - input.add_int_val('1'); - } - } else { - for (size_t i = 0; i < data.size(); i++) { - input.add_int_val(data[i]); - } - } - break; - } - default: { - if (data.size() == 0) { - *input.mutable_tensor_content() = std::string(numberOfElements * tensorflow::DataTypeSize(datatype), '1'); - } else { - std::string content; - content.resize(data.size() * tensorflow::DataTypeSize(datatype)); - std::memcpy(content.data(), data.data(), content.size()); - *input.mutable_tensor_content() = content; - } - } - } - } -} - void waitForOVMSConfigReload(ovms::ModelManager& manager) { // This is effectively multiplying by 5 to have at least 1 config reload in between // two test steps, but we check if config files changed to exit earlier if changes are already applied @@ -213,37 +156,6 @@ std::string readableSetError(std::unordered_set actual, std::unorde return ss.str(); } -void checkDummyResponse(const std::string outputName, - const std::vector& requestData, - PredictRequest& request, PredictResponse& response, int seriesLength, int batchSize, const std::string& servableName, size_t expectedOutputsCount) { - ASSERT_EQ(response.outputs().count(outputName), 1) << "Did not find:" << outputName; - const auto& output_proto = response.outputs().at(outputName); - - ASSERT_EQ(output_proto.tensor_content().size(), batchSize * DUMMY_MODEL_OUTPUT_SIZE * sizeof(float)); - ASSERT_EQ(output_proto.tensor_shape().dim_size(), 2); - ASSERT_EQ(output_proto.tensor_shape().dim(0).size(), batchSize); - ASSERT_EQ(output_proto.tensor_shape().dim(1).size(), DUMMY_MODEL_OUTPUT_SIZE); - - std::vector responseData = requestData; - std::for_each(responseData.begin(), responseData.end(), [seriesLength](float& v) { v += 1.0 * seriesLength; }); - - float* actual_output = (float*)output_proto.tensor_content().data(); - float* expected_output = responseData.data(); - const int dataLengthToCheck = DUMMY_MODEL_OUTPUT_SIZE * batchSize * sizeof(float); - checkBuffers(actual_output, expected_output, dataLengthToCheck); -} - -void checkScalarResponse(const std::string outputName, - float inputScalar, PredictResponse& response, const std::string& servableName) { - ASSERT_EQ(response.outputs().count(outputName), 1) << "Did not find:" << outputName; - const auto& output_proto = response.outputs().at(outputName); - - ASSERT_EQ(output_proto.tensor_shape().dim_size(), 0); - - ASSERT_EQ(output_proto.tensor_content().size(), sizeof(float)); - ASSERT_EQ(*((float*)output_proto.tensor_content().data()), inputScalar); -} - void checkScalarResponse(const std::string outputName, float inputScalar, ::KFSResponse& response, const std::string& servableName) { ASSERT_EQ(response.model_name(), servableName); @@ -259,21 +171,6 @@ void checkScalarResponse(const std::string outputName, ASSERT_EQ(*((float*)content->data()), inputScalar); } -void checkStringResponse(const std::string outputName, - const std::vector& inputStrings, PredictResponse& response, const std::string& servableName) { - ASSERT_EQ(response.outputs().count(outputName), 1) << "Did not find:" << outputName; - const auto& output_proto = response.outputs().at(outputName); - - ASSERT_EQ(output_proto.tensor_shape().dim_size(), 1); - ASSERT_EQ(output_proto.tensor_shape().dim(0).size(), inputStrings.size()); - ASSERT_EQ(output_proto.dtype(), tensorflow::DT_STRING); - - ASSERT_EQ(output_proto.string_val_size(), inputStrings.size()); - for (size_t i = 0; i < inputStrings.size(); i++) { - ASSERT_EQ(output_proto.string_val(i), inputStrings[i]); - } -} - void checkStringResponse(const std::string outputName, const std::vector& inputStrings, ::KFSResponse& response, const std::string& servableName) { ASSERT_EQ(response.model_name(), servableName); @@ -326,18 +223,6 @@ void checkAddResponse(const std::string outputName, checkBuffers(actual_output, expected_output, dataLengthToCheck); } -void checkIncrement4DimShape(const std::string outputName, - PredictResponse& response, - const std::vector& expectedShape) { - ASSERT_EQ(response.outputs().count(outputName), 1) << "Did not find:" << outputName; - const auto& output_proto = response.outputs().at(outputName); - - ASSERT_EQ(output_proto.tensor_shape().dim_size(), expectedShape.size()); - for (size_t i = 0; i < expectedShape.size(); i++) { - ASSERT_EQ(output_proto.tensor_shape().dim(i).size(), expectedShape[i]); - } -} - void RemoveReadonlyFileAttributeFromDir(std::string& directoryPath) { for (const std::filesystem::directory_entry& dir_entry : std::filesystem::recursive_directory_iterator(directoryPath)) { std::filesystem::permissions(dir_entry, std::filesystem::perms::owner_read | std::filesystem::perms::owner_write | std::filesystem::perms::owner_exec | std::filesystem::perms::group_read | std::filesystem::perms::group_write | std::filesystem::perms::others_read, std::filesystem::perm_options::add); @@ -351,32 +236,6 @@ void SetReadonlyFileAttributeFromDir(std::string& directoryPath) { } } -bool isShapeTheSame(const tensorflow::TensorShapeProto& actual, const std::vector&& expected) { - bool same = true; - if (static_cast(actual.dim_size()) != expected.size()) { - SPDLOG_ERROR("Unexpected dim_size. Got: {}, Expect: {}", actual.dim_size(), expected.size()); - return false; - } - for (int i = 0; i < actual.dim_size(); i++) { - if (actual.dim(i).size() != expected[i]) { - SPDLOG_ERROR("Unexpected dim[{}]. Got: {}, Expect: {}", i, actual.dim(i).size(), expected[i]); - same = false; - } - } - if (same == false) { - std::stringstream ss; - for (int i = 0; i < actual.dim_size(); i++) { - ss << "dim[" - << i - << "] got:" - << actual.dim(i).size() - << " expect:" << expected[i]; - } - SPDLOG_ERROR("Shape mismatch: {}", ss.str()); - } - return same; -} - bool isShapeTheSame(const KFSShapeType& actual, const std::vector&& expected) { bool same = true; int a_size = actual.size(); @@ -479,20 +338,6 @@ void prepareInferStringRequest(::KFSRequest& request, const std::string& name, c prepareInferStringTensor(*tensor, name, data, putBufferInInputTensorContent, content); } -void prepareInferStringTensor(tensorflow::TensorProto& tensor, const std::string& name, const std::vector& data, bool putBufferInInputTensorContent, std::string* content) { - tensor.set_dtype(tensorflow::DataType::DT_STRING); - tensor.mutable_tensor_shape()->add_dim()->set_size(data.size()); - for (auto inputData : data) { - tensor.add_string_val(inputData); - } -} - -void prepareInferStringRequest(tensorflow::serving::PredictRequest& request, const std::string& name, const std::vector& data, bool putBufferInInputTensorContent) { - request.mutable_inputs()->clear(); - auto& input = (*request.mutable_inputs())[name]; - prepareInferStringTensor(input, name, data, putBufferInInputTensorContent, nullptr); -} - void assertOutputTensorMatchExpectations(const ov::Tensor& tensor, std::vector expectedStrings) { size_t maxStringLength = 0; for (const auto& input : expectedStrings) { @@ -518,12 +363,6 @@ void assertOutputTensorMatchExpectations(const ov::Tensor& tensor, std::vector& expectedStrings) { - ASSERT_EQ(proto.string_val_size(), expectedStrings.size()); - for (size_t i = 0; i < expectedStrings.size(); i++) { - ASSERT_EQ(proto.string_val(i), expectedStrings[i]); - } -} void assertStringOutputProto(const KFSTensorOutputProto& proto, const std::vector& expectedStrings) { ASSERT_EQ(proto.contents().bytes_contents_size(), expectedStrings.size()); for (size_t i = 0; i < expectedStrings.size(); i++) { @@ -534,13 +373,6 @@ void assertStringOutputProto(const ovms::InferenceTensor& proto, const std::vect FAIL() << "not implemented"; } -void assertStringResponse(const tensorflow::serving::PredictResponse& proto, const std::vector& expectedStrings, const std::string& outputName) { - ASSERT_EQ(proto.outputs().count(outputName), 1); - ASSERT_EQ(proto.outputs().at(outputName).dtype(), tensorflow::DataType::DT_STRING); - ASSERT_EQ(proto.outputs().at(outputName).tensor_shape().dim_size(), 1); - ASSERT_EQ(proto.outputs().at(outputName).tensor_shape().dim(0).size(), expectedStrings.size()); - assertStringOutputProto(proto.outputs().at(outputName), expectedStrings); -} void assertStringResponse(const ::KFSResponse& proto, const std::vector& expectedStrings, const std::string& outputName) { ASSERT_EQ(proto.outputs_size(), 1); ASSERT_EQ(proto.outputs(0).name(), outputName); @@ -561,19 +393,6 @@ void assertStringResponse(const ovms::InferenceResponse& proto, const std::vecto FAIL() << "not implemented"; } -void prepareBinaryPredictRequest(tensorflow::serving::PredictRequest& request, const std::string& inputName, const int batchSize) { - auto& tensor = (*request.mutable_inputs())[inputName]; - size_t filesize = 0; - std::unique_ptr image_bytes = nullptr; - readRgbJpg(filesize, image_bytes); - - for (int i = 0; i < batchSize; i++) { - tensor.add_string_val(image_bytes.get(), filesize); - } - tensor.set_dtype(tensorflow::DataType::DT_STRING); - tensor.mutable_tensor_shape()->add_dim()->set_size(batchSize); -} - void prepareBinaryPredictRequest(::KFSRequest& request, const std::string& inputName, const int batchSize) { request.add_inputs(); auto tensor = request.mutable_inputs()->Mutable(0); @@ -589,18 +408,6 @@ void prepareBinaryPredictRequest(::KFSRequest& request, const std::string& input tensor->mutable_shape()->Add(batchSize); } -void prepareBinaryPredictRequestNoShape(tensorflow::serving::PredictRequest& request, const std::string& inputName, const int batchSize) { - auto& tensor = (*request.mutable_inputs())[inputName]; - size_t filesize = 0; - std::unique_ptr image_bytes = nullptr; - readRgbJpg(filesize, image_bytes); - - for (int i = 0; i < batchSize; i++) { - tensor.add_string_val(image_bytes.get(), filesize); - } - tensor.set_dtype(tensorflow::DataType::DT_STRING); -} - void prepareBinaryPredictRequestNoShape(::KFSRequest& request, const std::string& inputName, const int batchSize) { request.add_inputs(); auto tensor = request.mutable_inputs()->Mutable(0); @@ -615,19 +422,6 @@ void prepareBinaryPredictRequestNoShape(::KFSRequest& request, const std::string tensor->set_datatype("BYTES"); } -void prepareBinary4x4PredictRequest(tensorflow::serving::PredictRequest& request, const std::string& inputName, const int batchSize) { - auto& tensor = (*request.mutable_inputs())[inputName]; - size_t filesize = 0; - std::unique_ptr image_bytes = nullptr; - read4x4RgbJpg(filesize, image_bytes); - - for (int i = 0; i < batchSize; i++) { - tensor.add_string_val(image_bytes.get(), filesize); - } - tensor.set_dtype(tensorflow::DataType::DT_STRING); - tensor.mutable_tensor_shape()->add_dim()->set_size(batchSize); -} - void prepareBinary4x4PredictRequest(::KFSRequest& request, const std::string& inputName, const int batchSize) { request.add_inputs(); auto tensor = request.mutable_inputs()->Mutable(0); diff --git a/src/test/test_utils.hpp b/src/test/test_utils.hpp index 2510ca2b60..c0d8c768ea 100644 --- a/src/test/test_utils.hpp +++ b/src/test/test_utils.hpp @@ -33,10 +33,6 @@ #include #include -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop #include "../capi_frontend/inferencerequest.hpp" #include "../capi_frontend/inferenceresponse.hpp" #include "../config.hpp" @@ -67,16 +63,8 @@ using inputs_info_t = std::map::const_iterator; -using TFSOutputTensorIteratorType = google::protobuf::Map::const_iterator; -using TFSInterface = std::pair; using KFSInterface = std::pair; using CAPIInterface = std::pair; @@ -88,8 +76,6 @@ ovms::tensor_map_t prepareTensors( const std::unordered_map&& tensors, ovms::Precision precision = ovms::Precision::FP32); -void preparePredictRequest(tensorflow::serving::PredictRequest& request, inputs_info_t requestInputs, const std::vector& data = std::vector{}); - KFSTensorInputProto* findKFSInferInputTensor(::KFSRequest& request, const std::string& name); std::string* findKFSInferInputTensorContentInRawInputs(::KFSRequest& request, const std::string& name); @@ -280,23 +266,18 @@ void preparePredictRequest(ovms::InferenceRequest& request, inputs_info_t reques uint32_t decrementBufferSize = 0, OVMS_BufferType bufferType = OVMS_BUFFERTYPE_CPU, std::optional deviceId = std::nullopt); void prepareInferStringTensor(::KFSRequest::InferInputTensor& tensor, const std::string& name, const std::vector& data, bool putBufferInInputTensorContent, std::string* content); -void prepareInferStringTensor(tensorflow::TensorProto& tensor, const std::string& name, const std::vector& data, bool putBufferInInputTensorContent, std::string* content); void prepareInferStringTensor(ovms::InferenceTensor& tensor, const std::string& name, const std::vector& data, bool putBufferInInputTensorContent, std::string* content); void prepareInferStringRequest(::KFSRequest& request, const std::string& name, const std::vector& data, bool putBufferInInputTensorContent = true); -void prepareInferStringRequest(tensorflow::serving::PredictRequest& request, const std::string& name, const std::vector& data, bool putBufferInInputTensorContent = true); void prepareInferStringRequest(ovms::InferenceRequest& request, const std::string& name, const std::vector& data, bool putBufferInInputTensorContent = true); // CAPI binary not supported void assertOutputTensorMatchExpectations(const ov::Tensor& tensor, std::vector expectedStrings); -void prepareBinaryPredictRequest(tensorflow::serving::PredictRequest& request, const std::string& inputName, const int batchSize); void prepareBinaryPredictRequest(::KFSRequest& request, const std::string& inputName, const int batchSize); void prepareBinaryPredictRequest(ovms::InferenceRequest& request, const std::string& inputName, const int batchSize); // CAPI binary not supported -void prepareBinaryPredictRequestNoShape(tensorflow::serving::PredictRequest& request, const std::string& inputName, const int batchSize); void prepareBinaryPredictRequestNoShape(::KFSRequest& request, const std::string& inputName, const int batchSize); void prepareBinaryPredictRequestNoShape(ovms::InferenceRequest& request, const std::string& inputName, const int batchSize); // CAPI binary not supported -void prepareBinary4x4PredictRequest(tensorflow::serving::PredictRequest& request, const std::string& inputName, const int batchSize = 1); void prepareBinary4x4PredictRequest(::KFSRequest& request, const std::string& inputName, const int batchSize = 1); void prepareBinary4x4PredictRequest(ovms::InferenceRequest& request, const std::string& inputName, const int batchSize = 1); // CAPI binary not supported @@ -317,10 +298,6 @@ std::string readableError(const T* expected_output, const T* actual_output, cons std::string readableSetError(std::unordered_set expected, std::unordered_set actual); -void checkDummyResponse(const std::string outputName, - const std::vector& requestData, - tensorflow::serving::PredictRequest& request, tensorflow::serving::PredictResponse& response, int seriesLength, int batchSize = 1, const std::string& servableName = "", size_t expectedOutputsCount = 1); - static std::string vectorTypeToKfsString(const std::type_info& vectorType) { // {Precision::BF16, "BF16"}, // {Precision::FP16, "FP16"}, @@ -396,23 +373,15 @@ void checkDummyResponse(const std::string outputName, << readableError(expected_output, actual_output, dataLengthToCheck / sizeof(T)); } -void checkScalarResponse(const std::string outputName, - float inputScalar, tensorflow::serving::PredictResponse& response, const std::string& servableName = ""); - void checkScalarResponse(const std::string outputName, float inputScalar, ::KFSResponse& response, const std::string& servableName = ""); -void checkStringResponse(const std::string outputName, - const std::vector& inputStrings, tensorflow::serving::PredictResponse& response, const std::string& servableName = ""); - void checkStringResponse(const std::string outputName, const std::vector& inputStrings, ::KFSResponse& response, const std::string& servableName = ""); -void assertStringOutputProto(const tensorflow::TensorProto& proto, const std::vector& expectedStrings); void assertStringOutputProto(const KFSTensorOutputProto& proto, const std::vector& expectedStrings); void assertStringOutputProto(const ovms::InferenceTensor& proto, const std::vector& expectedStrings); -void assertStringResponse(const tensorflow::serving::PredictResponse& proto, const std::vector& expectedStrings, const std::string& outputName); void assertStringResponse(const ::KFSResponse& proto, const std::vector& expectedStrings, const std::string& outputName); void assertStringResponse(const ovms::InferenceResponse& proto, const std::vector& expectedStrings, const std::string& outputName); @@ -421,30 +390,6 @@ void checkAddResponse(const std::string outputName, const std::vector& requestData2, ::KFSRequest& request, const ::KFSResponse& response, int seriesLength, int batchSize, const std::string& servableName); -template -void checkIncrement4DimResponse(const std::string outputName, - const std::vector& expectedData, - tensorflow::serving::PredictResponse& response, - const std::vector& expectedShape, - bool checkRaw = true) { - ASSERT_EQ(response.outputs().count(outputName), 1) << "Did not find:" << outputName; - const auto& output_proto = response.outputs().at(outputName); - - auto elementsCount = std::accumulate(expectedShape.begin(), expectedShape.end(), 1, std::multiplies()); - - ASSERT_EQ(output_proto.tensor_content().size(), elementsCount * sizeof(T)); - ASSERT_EQ(output_proto.tensor_shape().dim_size(), expectedShape.size()); - for (size_t i = 0; i < expectedShape.size(); i++) { - ASSERT_EQ(output_proto.tensor_shape().dim(i).size(), expectedShape[i]); - } - - T* actual_output = (T*)output_proto.tensor_content().data(); - T* expected_output = (T*)expectedData.data(); - const int dataLengthToCheck = elementsCount * sizeof(T); - EXPECT_EQ(0, std::memcmp(actual_output, expected_output, dataLengthToCheck)) - << readableError(expected_output, actual_output, dataLengthToCheck / sizeof(T)); -} - template void checkIncrement4DimResponse(const std::string outputName, const std::vector& expectedData, @@ -477,23 +422,6 @@ void checkIncrement4DimResponse(const std::string outputName, } } -void checkIncrement4DimShape(const std::string outputName, - tensorflow::serving::PredictResponse& response, - const std::vector& expectedShape); - -static std::vector asVector(const tensorflow::TensorShapeProto& proto) { - std::vector shape; - for (int i = 0; i < proto.dim_size(); i++) { - shape.push_back(proto.dim(i).size()); - } - return shape; -} - -static std::vector asVector(google::protobuf::RepeatedField* container) { - std::vector result(container->size(), 0); - std::memcpy(result.data(), container->mutable_data(), result.size() * sizeof(google::protobuf::int32)); - return result; -} #pragma GCC diagnostic pop template @@ -515,9 +443,6 @@ class MockedMetadataModelIns : public ovms::ModelInstance { MOCK_METHOD(const ovms::tensor_map_t&, getOutputsInfo, (), (const, override)); MOCK_METHOD(std::optional, getBatchSize, (), (const, override)); MOCK_METHOD(const ovms::ModelConfig&, getModelConfig, (), (const, override)); - const ovms::Status mockValidate(const tensorflow::serving::PredictRequest* request) { - return validate(request); - } const ovms::Status mockValidate(const ::KFSRequest* request) { return validate(request); } @@ -557,7 +482,6 @@ static ovms::NodeLibrary createLibraryMock() { T::release}; } -bool isShapeTheSame(const tensorflow::TensorShapeProto&, const std::vector&&); bool isShapeTheSame(const KFSShapeType&, const std::vector&&); void readRgbJpg(size_t& filesize, std::unique_ptr& image_bytes); diff --git a/src/test/text2image_test.cpp b/src/test/text2image_test.cpp index f7dbe0fbcb..b34b7e495d 100644 --- a/src/test/text2image_test.cpp +++ b/src/test/text2image_test.cpp @@ -33,6 +33,7 @@ #include "src/image_gen/imagegenutils.hpp" #include "src/image_gen/imagegen_init.hpp" #include "src/image_conversion.hpp" +#include "src/logging.hpp" #include "src/status.hpp" using ovms::prepareImageGenPipelineArgs; diff --git a/src/test/tfs_rest_parser_binary_inputs_test.cpp b/src/test/tfs_rest_parser_binary_inputs_test.cpp deleted file mode 100644 index 874bd9d6b6..0000000000 --- a/src/test/tfs_rest_parser_binary_inputs_test.cpp +++ /dev/null @@ -1,230 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include -#include - -#include -#include - -#include "../rest_parser.hpp" -#pragma warning(push) -#pragma warning(disable : 6001) -#include "absl/strings/escaping.h" -#pragma warning(pop) -#include "test_utils.hpp" - -using namespace ovms; - -using ::testing::ElementsAre; - -class TFSRestParserBinaryInputs : public ::testing::Test { -protected: - void SetUp() override { - readRgbJpg(filesize, image_bytes); - std::string_view bytes(image_bytes.get(), filesize); - - absl::Base64Escape(bytes, &b64encoded); - } - - size_t filesize; - std::string b64encoded; - std::unique_ptr image_bytes; -}; - -TEST_F(TFSRestParserBinaryInputs, ColumnName) { - std::string request = R"({"signature_name":"","inputs":{"k":[{"b64":")" + b64encoded + R"("}]}})"; - - TFSRestParser parser(prepareTensors({{"k", {1, 1}}})); - ASSERT_EQ(parser.parse(request.c_str()), StatusCode::OK); - ASSERT_EQ(parser.getProto().inputs_size(), 1); - ASSERT_EQ(parser.getProto().inputs().count("k"), 1); - ASSERT_EQ(parser.getProto().inputs().find("k")->second.string_val_size(), 1); - EXPECT_EQ(std::memcmp(parser.getProto().inputs().find("k")->second.string_val(0).c_str(), image_bytes.get(), filesize), 0); -} - -TEST_F(TFSRestParserBinaryInputs, BatchSize2) { - std::string request = R"({"signature_name":"","instances":[{"k":[{"b64":")" + b64encoded + R"("}]},{"i":[{"b64":")" + b64encoded + R"("}]}]})"; - - TFSRestParser parser(TFSRestParser(prepareTensors({{"i", {1, 1}}, {"k", {1, 1}}}))); - ASSERT_EQ(parser.parse(request.c_str()), StatusCode::OK); - ASSERT_EQ(parser.getProto().inputs_size(), 2); - ASSERT_EQ(parser.getProto().inputs().count("k"), 1); - ASSERT_EQ(parser.getProto().inputs().count("i"), 1); - ASSERT_EQ(parser.getProto().inputs().find("k")->second.string_val_size(), 1); - ASSERT_EQ(parser.getProto().inputs().find("i")->second.string_val_size(), 1); - EXPECT_EQ(std::memcmp(parser.getProto().inputs().find("k")->second.string_val(0).c_str(), image_bytes.get(), filesize), 0); - EXPECT_EQ(std::memcmp(parser.getProto().inputs().find("i")->second.string_val(0).c_str(), image_bytes.get(), filesize), 0); -} - -TEST_F(TFSRestParserBinaryInputs, RowStringMixedPrecision) { - std::string request = R"({"signature_name":"","instances":[{"i": "abcd"}, {"i": 1234}]})"; - - TFSRestParser parser(prepareTensors({{"i", {-1, -1}}}, ovms::Precision::U8)); - ASSERT_EQ(parser.parse(request.c_str()), StatusCode::REST_COULD_NOT_PARSE_INSTANCE); -} - -TEST_F(TFSRestParserBinaryInputs, ColumnStringMixedPrecision) { - std::string request = R"({"signature_name":"","inputs":{"i":["abcd", "efg", 52.1, "xyz"]}})"; - - TFSRestParser parser(prepareTensors({{"i", {-1, -1}}}, ovms ::Precision::U8)); - ASSERT_EQ(parser.parse(request.c_str()), StatusCode::REST_COULD_NOT_PARSE_INPUT); -} - -TEST_F(TFSRestParserBinaryInputs, ColumnStringMixedPrecision2) { - std::string request = R"({"signature_name":"","inputs":{"i":[[2,3,4],[5,"abcd",7]]}})"; - - TFSRestParser parser(prepareTensors({{"i", {-1, -1}}}, ovms ::Precision::U8)); - ASSERT_EQ(parser.parse(request.c_str()), StatusCode::REST_COULD_NOT_PARSE_INPUT); -} - -TEST_F(TFSRestParserBinaryInputs, RowString) { - std::string request = R"({"signature_name":"","instances":[{"i":"abcd"}]})"; - - TFSRestParser parser(prepareTensors({{"i", {-1, -1}}}, ovms ::Precision::U8)); - ASSERT_EQ(parser.parse(request.c_str()), StatusCode::OK); - ASSERT_EQ(parser.getProto().inputs_size(), 1); - ASSERT_EQ(parser.getProto().inputs().count("i"), 1); - ASSERT_EQ(parser.getProto().inputs().find("i")->second.string_val_size(), 1); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(1)); - ASSERT_EQ(parser.getProto().inputs().at("i").dtype(), tensorflow::DataType::DT_STRING); - EXPECT_EQ(strcmp(parser.getProto().inputs().find("i")->second.string_val(0).c_str(), "abcd"), 0); -} - -TEST_F(TFSRestParserBinaryInputs, RowStringInvalidPrecision) { - std::string request = R"({"signature_name":"","instances":[{"i":"abcd"}]})"; - - TFSRestParser parser(prepareTensors({{"i", {-1, -1}}}, ovms ::Precision::FP32)); - ASSERT_EQ(parser.parse(request.c_str()), StatusCode::REST_COULD_NOT_PARSE_INSTANCE); -} - -TEST_F(TFSRestParserBinaryInputs, RowStringInvalidShape) { - std::string request = R"({"signature_name":"","instances":[{"i":"abcd"}]})"; - - TFSRestParser parser(prepareTensors({{"i", {-1, -1, -1}}}, ovms ::Precision::U8)); - ASSERT_EQ(parser.parse(request.c_str()), StatusCode::OK); - ASSERT_EQ(parser.getProto().inputs_size(), 1); - ASSERT_EQ(parser.getProto().inputs().count("i"), 1); - ASSERT_EQ(parser.getProto().inputs().find("i")->second.string_val_size(), 1); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(1)); - ASSERT_EQ(parser.getProto().inputs().at("i").dtype(), tensorflow::DataType::DT_STRING); - EXPECT_EQ(strcmp(parser.getProto().inputs().find("i")->second.string_val(0).c_str(), "abcd"), 0); -} - -TEST_F(TFSRestParserBinaryInputs, RowStringStaticShape) { - std::string request = R"({"signature_name":"","instances":[{"i":"abcd"}]})"; - - TFSRestParser parser(prepareTensors({{"i", {1, 4}}}, ovms ::Precision::U8)); - ASSERT_EQ(parser.parse(request.c_str()), StatusCode::OK); - ASSERT_EQ(parser.getProto().inputs_size(), 1); - ASSERT_EQ(parser.getProto().inputs().count("i"), 1); - ASSERT_EQ(parser.getProto().inputs().find("i")->second.string_val_size(), 1); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(1)); - ASSERT_EQ(parser.getProto().inputs().at("i").dtype(), tensorflow::DataType::DT_STRING); - EXPECT_EQ(strcmp(parser.getProto().inputs().find("i")->second.string_val(0).c_str(), "abcd"), 0); -} - -TEST_F(TFSRestParserBinaryInputs, ColumnString) { - std::string request = R"({"signature_name":"","inputs":{"i":["abcd"]}})"; - - TFSRestParser parser(prepareTensors({{"i", {-1, -1}}}, ovms ::Precision::U8)); - ASSERT_EQ(parser.parse(request.c_str()), StatusCode::OK); - ASSERT_EQ(parser.getProto().inputs_size(), 1); - ASSERT_EQ(parser.getProto().inputs().count("i"), 1); - ASSERT_EQ(parser.getProto().inputs().find("i")->second.string_val_size(), 1); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(1)); - ASSERT_EQ(parser.getProto().inputs().at("i").dtype(), tensorflow::DataType::DT_STRING); - EXPECT_EQ(strcmp(parser.getProto().inputs().find("i")->second.string_val(0).c_str(), "abcd"), 0); -} - -TEST_F(TFSRestParserBinaryInputs, ColumnStringUnnamed) { - std::string request = R"({"signature_name":"","inputs":["abcd"]})"; - - TFSRestParser parser(prepareTensors({{"i", {-1, -1}}}, ovms ::Precision::U8)); - ASSERT_EQ(parser.parse(request.c_str()), StatusCode::OK); - ASSERT_EQ(parser.getProto().inputs_size(), 1); - ASSERT_EQ(parser.getProto().inputs().count("i"), 1); - ASSERT_EQ(parser.getProto().inputs().find("i")->second.string_val_size(), 1); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(1)); - ASSERT_EQ(parser.getProto().inputs().at("i").dtype(), tensorflow::DataType::DT_STRING); - EXPECT_EQ(strcmp(parser.getProto().inputs().find("i")->second.string_val(0).c_str(), "abcd"), 0); -} - -TEST_F(TFSRestParserBinaryInputs, RowStringUnnamed) { - std::string request = R"({"signature_name":"","instances":["abcd"]})"; - - TFSRestParser parser(prepareTensors({{"i", {-1, -1}}}, ovms ::Precision::U8)); - ASSERT_EQ(parser.parse(request.c_str()), StatusCode::OK); - ASSERT_EQ(parser.getProto().inputs_size(), 1); - ASSERT_EQ(parser.getProto().inputs().count("i"), 1); - ASSERT_EQ(parser.getProto().inputs().find("i")->second.string_val_size(), 1); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(1)); - ASSERT_EQ(parser.getProto().inputs().at("i").dtype(), tensorflow::DataType::DT_STRING); - EXPECT_EQ(strcmp(parser.getProto().inputs().find("i")->second.string_val(0).c_str(), "abcd"), 0); -} - -TEST_F(TFSRestParserBinaryInputs, RowStringBatchSize2) { - std::string request = R"({"signature_name":"","instances":[{"i":"abcd"}, {"i":"efgh"}]})"; - - TFSRestParser parser(prepareTensors({{"i", {-1, -1}}}, ovms ::Precision::U8)); - ASSERT_EQ(parser.parse(request.c_str()), StatusCode::OK); - ASSERT_EQ(parser.getProto().inputs_size(), 1); - ASSERT_EQ(parser.getProto().inputs().count("i"), 1); - ASSERT_EQ(parser.getProto().inputs().find("i")->second.string_val_size(), 2); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(2)); - ASSERT_EQ(parser.getProto().inputs().at("i").dtype(), tensorflow::DataType::DT_STRING); - EXPECT_EQ(strcmp(parser.getProto().inputs().find("i")->second.string_val(0).c_str(), "abcd"), 0); - EXPECT_EQ(strcmp(parser.getProto().inputs().find("i")->second.string_val(1).c_str(), "efgh"), 0); -} - -TEST_F(TFSRestParserBinaryInputs, RowName) { - std::string request = R"({"signature_name":"","instances":[{"k":[{"b64":")" + b64encoded + R"("}]}]})"; - - TFSRestParser parser(prepareTensors({{"k", {1, 1}}})); - ASSERT_EQ(parser.parse(request.c_str()), StatusCode::OK); - ASSERT_EQ(parser.getProto().inputs_size(), 1); - ASSERT_EQ(parser.getProto().inputs().count("k"), 1); - ASSERT_EQ(parser.getProto().inputs().find("k")->second.string_val_size(), 1); - EXPECT_EQ(std::memcmp(parser.getProto().inputs().find("k")->second.string_val(0).c_str(), image_bytes.get(), filesize), 0); -} - -TEST_F(TFSRestParserBinaryInputs, InvalidObject) { - std::string request = R"({"signature_name":"","inputs":{"k":[{"b64":")" + b64encoded + R"(", "AdditionalField":"someValue"}]}})"; - - TFSRestParser parser(prepareTensors({}, ovms::Precision::FP16)); - ASSERT_EQ(parser.parse(request.c_str()), StatusCode::REST_COULD_NOT_PARSE_INPUT); -} - -TEST_F(TFSRestParserBinaryInputs, ColumnNoNamed) { - std::string request = R"({"signature_name":"","inputs":[{"b64":")" + b64encoded + R"("}]})"; - - TFSRestParser parser(prepareTensors({{"k", {1, 1}}})); - ASSERT_EQ(parser.parse(request.c_str()), StatusCode::OK); - ASSERT_EQ(parser.getProto().inputs_size(), 1); - ASSERT_EQ(parser.getProto().inputs().count("k"), 1); - ASSERT_EQ(parser.getProto().inputs().find("k")->second.string_val_size(), 1); - EXPECT_EQ(std::memcmp(parser.getProto().inputs().find("k")->second.string_val(0).c_str(), image_bytes.get(), filesize), 0); -} - -TEST_F(TFSRestParserBinaryInputs, RowNoNamed) { - std::string request = R"({"signature_name":"","instances":[[{"b64":")" + b64encoded + R"("}]]})"; - - TFSRestParser parser(prepareTensors({{"k", {1, 1}}})); - ASSERT_EQ(parser.parse(request.c_str()), StatusCode::OK); - ASSERT_EQ(parser.getProto().inputs_size(), 1); - ASSERT_EQ(parser.getProto().inputs().count("k"), 1); - ASSERT_EQ(parser.getProto().inputs().find("k")->second.string_val_size(), 1); - EXPECT_EQ(std::memcmp(parser.getProto().inputs().find("k")->second.string_val(0).c_str(), image_bytes.get(), filesize), 0); -} diff --git a/src/test/tfs_rest_parser_column_test.cpp b/src/test/tfs_rest_parser_column_test.cpp deleted file mode 100644 index 7d6b3db83e..0000000000 --- a/src/test/tfs_rest_parser_column_test.cpp +++ /dev/null @@ -1,791 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include -#include - -#pragma warning(push) -#pragma warning(disable : 4624) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#pragma GCC diagnostic ignored "-Wunused-but-set-variable" -#include "tensorflow/core/framework/tensor.h" -#pragma GCC diagnostic pop -#pragma warning(pop) - -#include "../rest_parser.hpp" -#include "test_utils.hpp" - -using namespace ovms; - -using namespace testing; -using ::testing::ElementsAre; - -using tensorflow::DataTypeSize; - -const char* predictRequestColumnNamedJson = R"({ - "inputs": { - "inputA": [ - [ - [[1.0, 2.0], - [3.0, 4.0], - [5.0, 6.0]], - [[7.0, 8.0], - [9.0, 10.0], - [11.0, 12.0]] - ], - [ - [[101.0, 102.0], - [103.0, 104.0], - [105.0, 106.0]], - [[107.0, 108.0], - [109.0, 110.0], - [111.0, 112.0]] - ] - ], - "inputB": [ - [ - [1.0, 2.0, 3.0], - [4.0, 5.0, 6.0] - ], - [ - [11.0, 12.0, 13.0], - [14.0, 15.0, 16.0] - ] - ], - "inputC": [ - {"b64": "ORw0"}, - {"b64": "ORw0"} - ] - }, - "signature_name": "serving_default" -})"; - -TEST(TFSRestParserColumn, ParseValid2Inputs) { - TFSRestParser parser(prepareTensors({{"inputA", {2, 2, 3, 2}}, - {"inputB", {2, 2, 3}}, {"inputC", {2}}})); - - auto status = parser.parse(predictRequestColumnNamedJson); - - ASSERT_EQ(status, StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::COLUMN); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - ASSERT_EQ(parser.getProto().inputs_size(), 3); - ASSERT_EQ(parser.getProto().inputs().count("inputA"), 1); - ASSERT_EQ(parser.getProto().inputs().count("inputB"), 1); - ASSERT_EQ(parser.getProto().inputs().count("inputC"), 1); - const auto& inputA = parser.getProto().inputs().at("inputA"); - const auto& inputB = parser.getProto().inputs().at("inputB"); - const auto& inputC = parser.getProto().inputs().at("inputC"); - EXPECT_EQ(inputA.dtype(), tensorflow::DataType::DT_FLOAT); - EXPECT_EQ(inputB.dtype(), tensorflow::DataType::DT_FLOAT); - EXPECT_EQ(inputC.dtype(), tensorflow::DataType::DT_STRING); - EXPECT_THAT(asVector(inputA.tensor_shape()), ElementsAre(2, 2, 3, 2)); - EXPECT_THAT(asVector(inputB.tensor_shape()), ElementsAre(2, 2, 3)); - EXPECT_THAT(asVector(inputC.tensor_shape()), ElementsAre(2)); - ASSERT_EQ(inputA.tensor_content().size(), 2 * 2 * 3 * 2 * DataTypeSize(tensorflow::DataType::DT_FLOAT)); - ASSERT_EQ(inputB.tensor_content().size(), 2 * 2 * 3 * DataTypeSize(tensorflow::DataType::DT_FLOAT)); - ASSERT_EQ(inputC.string_val().size(), 2); - EXPECT_THAT(asVector(inputA.tensor_content()), ElementsAre( - 1.0, 2.0, - 3.0, 4.0, - 5.0, 6.0, - //------- - 7.0, 8.0, - 9.0, 10.0, - 11.0, 12.0, - //========= - 101.0, 102.0, - 103.0, 104.0, - 105.0, 106.0, - //--------- - 107.0, 108.0, - 109.0, 110.0, - 111.0, 112.0)); - EXPECT_THAT(asVector(inputB.tensor_content()), ElementsAre( - 1.0, 2.0, 3.0, - 4.0, 5.0, 6.0, - //============ - 11.0, 12, 13.0, - 14.0, 15.0, 16.0)); - - char expectedBinary[] = {57, 28, 52}; - EXPECT_EQ(inputC.string_val()[0], std::string(expectedBinary, expectedBinary + 3)); - EXPECT_EQ(inputC.string_val()[1], std::string(expectedBinary, expectedBinary + 3)); -} - -TEST(TFSRestParserColumn, ValidShape_1d_vector_1elem) { - TFSRestParser parser(prepareTensors({{"i", {1}}})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{ - "i":[155.0] - }})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::COLUMN); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(1)); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(155.0)); -} - -TEST(TFSRestParserColumn, ValidShape_1x1) { - TFSRestParser parser(prepareTensors({{"i", {1, 1}}})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{ - "i":[[155.0]] - }})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::COLUMN); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(1, 1)); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(155.0)); -} - -TEST(TFSRestParserColumn, ValidShape_1x2) { - TFSRestParser parser(prepareTensors({{"i", {1, 2}}})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{ - "i":[[155.0, 56.0]] - }})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::COLUMN); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(1, 2)); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(155.0, 56.0)); -} - -TEST(TFSRestParserColumn, ValidShape_0) { - TFSRestParser parser(prepareTensors({{"i", {0}}}, ovms::Precision::FP32)); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{ - "i":[] - }})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::COLUMN); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - EXPECT_EQ(parser.getProto().inputs().at("i").dtype(), tensorflow::DataType::DT_FLOAT); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(0)); - EXPECT_EQ(parser.getProto().inputs().at("i").tensor_content().size(), 0); -} - -TEST(TFSRestParserColumn, ValidShape_2x1) { - TFSRestParser parser(prepareTensors({{"i", {2, 1}}})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{ - "i":[[155.0],[513.0]] - }})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::COLUMN); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(2, 1)); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(155.0, 513.0)); -} - -TEST(TFSRestParserColumn, ValidShape_2x2) { - TFSRestParser parser(prepareTensors({{"i", {2, 2}}})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{ - "i":[[155.0, 9.0], [513.0, -5.0]] - }})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::COLUMN); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(2, 2)); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(155.0, 9.0, 513.0, -5.0)); -} - -TEST(TFSRestParserColumn, ValidShape_2x0) { - TFSRestParser parser(prepareTensors({{"i", {2, 0}}}, ovms::Precision::I64)); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{ - "i":[[],[]] - }})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::COLUMN); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - EXPECT_EQ(parser.getProto().inputs().at("i").dtype(), tensorflow::DataType::DT_INT64); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(2, 0)); - EXPECT_EQ(parser.getProto().inputs().at("i").tensor_content().size(), 0); -} - -TEST(TFSRestParserColumn, ValidShape_2x1x3) { - TFSRestParser parser(prepareTensors({{"i", {2, 1, 3}}})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{ - "i": [ - [[5.0,9.0,2.0]], - [[-5.0,-2.0,-10.0]] - ] - }})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::COLUMN); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(2, 1, 3)); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(5.0, 9.0, 2.0, -5.0, -2.0, -10.0)); -} - -TEST(TFSRestParserColumn, ValidShape_2x3x1) { - TFSRestParser parser(prepareTensors({{"i", {2, 3, 1}}})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{ - "i": [ - [[5.0], [9.0], [1.0]], - [[-1.0], [-9.0], [25.0]] - ] - }})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::COLUMN); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(2, 3, 1)); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(5.0, 9.0, 1.0, -1.0, -9.0, 25.0)); -} - -TEST(TFSRestParserColumn, ValidShape_2x1x2x1) { - TFSRestParser parser(prepareTensors({{"i", {2, 1, 2, 1}}})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{ - "i": [ - [[[5.0], [2.0]]], - [[[6.0], [18.0]]] - ] - }})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::COLUMN); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(2, 1, 2, 1)); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(5.0, 2.0, 6.0, 18.0)); -} - -TEST(TFSRestParserColumn, ValidShape_2x1x3x1x5) { - TFSRestParser parser(prepareTensors({{"i", {2, 1, 3, 1, 5}}})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{ - "i": [ - [[[[1.9, 2.9, 3.9, 4.9, 5.9]], - [[1.9, 2.9, 3.9, 4.9, 5.9]], - [[1.9, 2.9, 3.9, 4.9, 5.9]]]], - [[[[1.9, 2.9, 3.9, 4.9, 5.9]], - [[1.9, 2.9, 3.9, 4.9, 5.9]], - [[1.9, 2.9, 3.9, 4.9, 5.9]]]] - ] - }})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::COLUMN); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(2, 1, 3, 1, 5)); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre( - 1.9, 2.9, 3.9, 4.9, 5.9, - 1.9, 2.9, 3.9, 4.9, 5.9, - 1.9, 2.9, 3.9, 4.9, 5.9, - 1.9, 2.9, 3.9, 4.9, 5.9, - 1.9, 2.9, 3.9, 4.9, 5.9, - 1.9, 2.9, 3.9, 4.9, 5.9)); -} - -TEST(TFSRestParserColumn, ValidShape_2x1x3x1x0) { - TFSRestParser parser(prepareTensors({{"i", {2, 1, 3, 1, 0}}}, ovms::Precision::FP32)); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{ - "i": [ - [[[[ ]], - [[ ]], - [[ ]]]], - [[[[ ]], - [[ ]], - [[ ]]]] - ] - }})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::COLUMN); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(2, 1, 3, 1, 0)); - EXPECT_EQ(parser.getProto().inputs().at("i").dtype(), tensorflow::DataType::DT_FLOAT); - EXPECT_EQ(parser.getProto().inputs().at("i").tensor_content().size(), 0); -} - -TEST(TFSRestParserColumn, ValidScalar) { - TFSRestParser parser(prepareTensors({{"i", {}}})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{ - "i":155.0 - }})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::COLUMN); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - ASSERT_EQ(parser.getProto().inputs().count("i"), 1); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre()); - EXPECT_EQ(parser.getProto().inputs().at("i").dtype(), tensorflow::DT_FLOAT); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(155.0)); -} - -TEST(TFSRestParserColumn, ValidScalarNoMetadataInt32) { - TFSRestParser parser(prepareTensors({})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{ - "i":155 - }})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::COLUMN); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - ASSERT_EQ(parser.getProto().inputs().count("i"), 1); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre()); - EXPECT_EQ(parser.getProto().inputs().at("i").dtype(), tensorflow::DT_INT32); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(155)); -} - -TEST(TFSRestParserColumn, ValidScalarNoMetadataFloat) { - TFSRestParser parser(prepareTensors({})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{ - "i":155.2 - }})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::COLUMN); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - ASSERT_EQ(parser.getProto().inputs().count("i"), 1); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre()); - EXPECT_EQ(parser.getProto().inputs().at("i").dtype(), tensorflow::DT_FLOAT); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(155.2)); -} - -TEST(TFSRestParserColumn, AllowsDifferent0thDimension) { - TFSRestParser parser(prepareTensors({{"i", {2, 1, 2, 2}}, - {"j", {1, 1, 2, 2}}})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{ - "i": [ - [[[5.0, 2.0], [10.0, 7.0]]], - [[[5.0, 2.0], [10.0, 7.0]]] - ], - "j": [ - [[[5.0, 2.0], [10.0, 7.0]]] - ] - }})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::COLUMN); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(2, 1, 2, 2)); - EXPECT_THAT(asVector(parser.getProto().inputs().at("j").tensor_shape()), ElementsAre(1, 1, 2, 2)); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(5.0, 2.0, 10.0, 7.0, 5.0, 2.0, 10.0, 7.0)); - EXPECT_THAT(asVector(parser.getProto().inputs().at("j").tensor_content()), ElementsAre(5.0, 2.0, 10.0, 7.0)); -} - -TEST(TFSRestParserColumn, ParseUint8) { - std::vector parsers{TFSRestParser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::U8))}; - for (TFSRestParser& parser : parsers) { - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[[[0,5,15,255]]]}})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(0, 5, 15, 255)); - } - - parsers = std::vector{TFSRestParser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::U8))}; - for (TFSRestParser& parser : parsers) { - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[[[0.0,5.0,15.0,255.0]]]}})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(0, 5, 15, 255)); - } -} - -TEST(TFSRestParserColumn, ParseInt8) { - std::vector parsers{TFSRestParser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::I8))}; - for (TFSRestParser& parser : parsers) { - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[[[0,-5,127,-128]]]}})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(0, -5, 127, -128)); - } - - parsers = std::vector{TFSRestParser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::I8))}; - for (TFSRestParser& parser : parsers) { - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[[[0.0,-5.0,127.0,-128.0]]]}})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(0, -5, 127, -128)); - } -} - -TEST(TFSRestParserColumn, ParseUint16) { - std::vector parsers{TFSRestParser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::U16))}; - for (TFSRestParser& parser : parsers) { - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[[[0,5,128,65535]]]}})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().mutable_inputs()->at("i").mutable_int_val()), ElementsAre(0, 5, 128, 65535)); - } - - parsers = std::vector{TFSRestParser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::U16))}; - for (TFSRestParser& parser : parsers) { - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[[[0.0,5.0,128.0,65535.0]]]}})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().mutable_inputs()->at("i").mutable_int_val()), ElementsAre(0, 5, 128, 65535)); - } -} - -TEST(TFSRestParserColumn, ParseInt16) { - std::vector parsers{TFSRestParser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::I16))}; - for (TFSRestParser& parser : parsers) { - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[[[0,-5,32768,-32767]]]}})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(0, -5, 32768, -32767)); - } - - parsers = std::vector{TFSRestParser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::I16))}; - for (TFSRestParser& parser : parsers) { - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[[[0.0,-5.0,32768.0,-32767.0]]]}})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(0, -5, 32768, -32767)); - } -} - -TEST(TFSRestParserColumn, ParseInt32) { - TFSRestParser parser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::I32)); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[[[0,-5,2147483648,-2147483647]]]}})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(0, -5, 2147483648, -2147483647)); - - parser = TFSRestParser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::I32)); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[[[0,-5,2147483648,-2147483647]]]}})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(0, -5, 2147483648, -2147483647)); -} - -TEST(TFSRestParserColumn, ParseUint64) { - std::vector parsers{TFSRestParser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::U64))}; - for (TFSRestParser& parser : parsers) { - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[[[0,5,128,18446744073709551615]]]}})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(0, 5, 128, 18446744073709551615U)); - } - - parsers = std::vector{TFSRestParser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::U64))}; - for (TFSRestParser& parser : parsers) { - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[[[0.0,5.0,128.0,555222.0]]]}})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(0, 5, 128, 555222)); // Can't looselessly cast large double to int64 - } -} - -TEST(TFSRestParserColumn, ParseInt64) { - TFSRestParser parser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::I64)); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[[[0,-5,5522,-9223372036854775807]]]}})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(0, -5, 5522, -9223372036854775807)); - - parser = TFSRestParser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::I64)); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[[[0.0,-5.0,5522.0,-55333.0]]]}})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(0, -5, 5522, -55333)); // Can't looselessly cast double to int64 -} - -TEST(TFSRestParserColumn, ParseFloat) { - TFSRestParser parser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::FP32)); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[[[-5.0, 0.0, -4.0, 155234.0]]]}})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(-5.0, 0.0, -4.0, 155234.0)); - - parser = TFSRestParser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::FP32)); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[[[-5.12, 0.4344, -4.521, 155234.221]]]}})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(-5.12, 0.4344, -4.521, 155234.221)); -} - -TEST(TFSRestParserColumn, ParseHalf) { - std::vector parsers{TFSRestParser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::FP16))}; - for (TFSRestParser& parser : parsers) { - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[[[-5, 0, -4, 155234]]]}})"), StatusCode::OK); - } - - parsers = std::vector{TFSRestParser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::FP16))}; - for (TFSRestParser& parser : parsers) { - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[[[-5.1222, 0.434422, -4.52122, 155234.22122]]]}})"), StatusCode::OK); - } -} - -TEST(TFSRestParserColumn, InputsNotAnObject) { - TFSRestParser parser(prepareTensors({}, ovms::Precision::FP16)); - - EXPECT_EQ(parser.parse(R"({"signature_name":"","inputs":"string"})"), StatusCode::REST_INPUTS_NOT_AN_OBJECT); - EXPECT_EQ(parser.parse(R"({"signature_name":"","inputs":5})"), StatusCode::REST_INPUTS_NOT_AN_OBJECT); -} - -TEST(TFSRestParserColumn, NoInputsFound) { - TFSRestParser parser(prepareTensors({}, ovms::Precision::FP16)); - - EXPECT_EQ(parser.parse(R"({"signature_name":"","inputs":{}})"), StatusCode::REST_NO_INPUTS_FOUND); -} - -TEST(TFSRestParserColumn, CannotParseInput) { - TFSRestParser parser(prepareTensors({{"i", {2, 1}}})); - - EXPECT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":null}})"), StatusCode::REST_COULD_NOT_PARSE_INPUT); - EXPECT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[1,null]}})"), StatusCode::REST_COULD_NOT_PARSE_INPUT); - EXPECT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[[1,2],[3,"str"]]}})"), StatusCode::REST_COULD_NOT_PARSE_INPUT); -} - -TEST(TFSRestParserColumn, InputNotNdArray_1) { - TFSRestParser parser(prepareTensors({{"i", {1, 2, 3, 2}}})); - - // [1, 4, 5] size is 3 instead of 2 to be valid - EXPECT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[ - [[[1, 2], - [1, 3], - [1, 4, 5]], - [[5, 8], - [9, 3], - [1, 4]]] - ]}})"), - StatusCode::REST_COULD_NOT_PARSE_INPUT); -} - -TEST(TFSRestParserColumn, InputNotNdArray_2) { - TFSRestParser parser(prepareTensors({{"i", {1, 2, 3, 3}}})); - - EXPECT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[ - [[[1, 2, [8]], - [1, 3, [3]], - [1, 4, [5]]], - [[5, 8, [-1]], - [9, 3, [-5]], - [1, 4, [-4]]]] - ]}})"), - StatusCode::REST_COULD_NOT_PARSE_INPUT); -} - -TEST(TFSRestParserColumn, InputNotNdArray_3) { - TFSRestParser parser(prepareTensors({{"i", {1, 4, 3, 2}}})); - - EXPECT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[ - [[[1, 2], - [1, 3], - [1, 4]], - - [[1, 2]], - - [[5, 8], - [9, 3], - [1, 4]], - - [[5, 8]]] - ]}})"), - StatusCode::REST_COULD_NOT_PARSE_INPUT); -} - -TEST(TFSRestParserColumn, InputNotNdArray_4) { - TFSRestParser parser(prepareTensors({{"i", {1, 2, 3, 2}}})); - - // [5, 6] is not a number but array - EXPECT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[ - [[[1, 2], - [1, 3], - [1, 4, [5, 6]]], - [[5, 8], - [9, 3], - [1, 4]]] - ]}})"), - StatusCode::REST_COULD_NOT_PARSE_INPUT); -} - -TEST(TFSRestParserColumn, InputNotNdArray_5) { - TFSRestParser parser(prepareTensors({{"i", {1, 2, 3, 2}}})); - - // [1] is of wrong shape - EXPECT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[ - [[[1], - [1, 2], - [1, 3], - [1, 4]], - [[5, 8], - [9, 3], - [1, 4]]] - ]}})"), - StatusCode::REST_COULD_NOT_PARSE_INPUT); -} - -TEST(TFSRestParserColumn, InputNotNdArray_6) { - TFSRestParser parser(prepareTensors({{"i", {1, 2, 2, 2}}})); - - // [1, 1] missing - 2x2, 2x3 - EXPECT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[ - [[[1, 2], - [1, 3]], - [[5, 8], - [9, 3], - [1, 4]]] - ]}})"), - StatusCode::REST_COULD_NOT_PARSE_INPUT); -} - -TEST(TFSRestParserColumn, InputNotNdArray_7) { - TFSRestParser parser(prepareTensors({{"i", {1, 2, 3, 2}}})); - - // [1, 5] numbers are on wrong level - EXPECT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[ - [[1, 5], - [[1, 1], - [1, 2], - [1, 3]], - [[5, 8], - [9, 3], - [1, 4]]] - ]}})"), - StatusCode::REST_COULD_NOT_PARSE_INPUT); -} - -TEST(TFSRestParserColumn, InputNotNdArray_8) { - TFSRestParser parser(prepareTensors({{"i", {1, 2, 3, 2}}})); - - // [1, 2], [9, 3] numbers are on wrong level - EXPECT_EQ(parser.parse(R"({"signature_name":"","inputs":{"i":[ - [[[1, 1], - [[1, 2]], - [1, 3]], - [[5, 8], - [[9, 3]], - [1, 4]]] - ]}})"), - StatusCode::REST_COULD_NOT_PARSE_INPUT); -} - -TEST(TFSRestParserColumn, InstancesShapeDiffer_1) { - TFSRestParser parser(prepareTensors({{"i", {2, 2, 3, 2}}})); - - // 2x3x2 vs 2x2x2 - EXPECT_EQ(parser.parse(R"({"signature_name":"","inputs":{ - "i": [ - [ - [[1, 1], - [1, 2], - [1, 3]], - [[5, 8], - [9, 3], - [1, 4]] - ], - [ - [[1, 1], - [1, 2]], - [[5, 8], - [9, 3]] - ] - ] - }})"), - StatusCode::REST_COULD_NOT_PARSE_INPUT); -} - -TEST(TFSRestParserColumn, InstancesShapeDiffer_2) { - TFSRestParser parser(prepareTensors({{"i", {2, 2, 3, 2}}})); - - // 2x3x2 vs 2x3x3 - EXPECT_EQ(parser.parse(R"({"signature_name":"","inputs":{ - "i": [ - [ - [[1, 1], - [1, 2], - [1, 3]], - [[5, 8], - [9, 3], - [1, 4]] - ], - [ - [[1, 1, 3], - [1, 2, 2], - [1, 3, 9]], - [[5, 8, 8], - [9, 3, 3], - [1, 4, 10]] - ] - ] - }})"), - StatusCode::REST_COULD_NOT_PARSE_INPUT); -} - -TEST(TFSRestParserColumn, InstancesShapeDiffer_3) { - TFSRestParser parser(prepareTensors({{"i", {2, 2, 3, 2}}})); - - // 2x3x2 vs 1x2x3x2 - EXPECT_EQ(parser.parse(R"({"signature_name":"","inputs":{ - "i": [ - [ - [[1, 1], - [1, 2], - [1, 3]], - [[5, 8], - [9, 3], - [1, 4]] - ], - [[ - [[1, 1], - [1, 2], - [1, 3]], - [[5, 8], - [9, 3], - [1, 4]] - ]] - ] - }})"), - StatusCode::REST_COULD_NOT_PARSE_INPUT); -} - -TEST(TFSRestParserColumn, RemoveUnnecessaryInputs) { - TFSRestParser parser(prepareTensors({{"i", {1, 1}}, {"j", {1, 1}}, {"k", {1, 1}}, {"l", {1, 1}}, {"m", {}}}, ovms::Precision::FP16)); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{ - "k":[[155.0]], "l": [[1.0]] - }})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::COLUMN); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - ASSERT_EQ(parser.getProto().inputs().count("i"), 0); // missing in request, expect missing after conversion - ASSERT_EQ(parser.getProto().inputs().count("j"), 0); // missing in request, expect missing after conversion - ASSERT_EQ(parser.getProto().inputs().count("k"), 1); // exists in request, expect exists after conversion - ASSERT_EQ(parser.getProto().inputs().count("l"), 1); // exists in request, expect exists after conversion - ASSERT_EQ(parser.getProto().inputs().count("m"), 0); // missing in request, expect missing after conversion - ASSERT_EQ(parser.getProto().inputs().size(), 2); -} - -TEST(TFSRestParserColumn, RemoveUnnecessaryInputs_ExpectedScalarInRequest) { - TFSRestParser parser(prepareTensors({{"i", {1, 1}}, {"j", {1, 1}}, {"k", {1, 1}}, {"l", {1, 1}}, {"m", {}}}, ovms::Precision::FP16)); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{ - "k":[[155.0]], "l": [[1.0]], "m": 3 - }})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::COLUMN); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - ASSERT_EQ(parser.getProto().inputs().count("i"), 0); // missing in request, expect missing after conversion - ASSERT_EQ(parser.getProto().inputs().count("j"), 0); // missing in request, expect missing after conversion - ASSERT_EQ(parser.getProto().inputs().count("k"), 1); // exists in request and endpoint metadata, expect exists after conversion - ASSERT_EQ(parser.getProto().inputs().count("l"), 1); // exists in request and endpoint metadata, expect exists after conversion - ASSERT_EQ(parser.getProto().inputs().count("m"), 1); // exists in request and endpoint metadata, expect exists after conversion - ASSERT_EQ(parser.getProto().inputs().size(), 3); -} - -TEST(TFSRestParserColumn, RemoveUnnecessaryInputs_UnexpectedScalarInRequest) { - TFSRestParser parser(prepareTensors({{"i", {1, 1}}, {"j", {1, 1}}, /*{"k", {1, 1}},*/ {"l", {1, 1}} /*,{"m", {}}*/}, ovms::Precision::FP16)); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":{ - "k":[[155.0]], "l": [[1.0]], "m": 4 - }})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::COLUMN); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - ASSERT_EQ(parser.getProto().inputs().count("i"), 0); // missing in request, expect missing after conversion - ASSERT_EQ(parser.getProto().inputs().count("j"), 0); // missing in request, expect missing after conversion - ASSERT_EQ(parser.getProto().inputs().count("k"), 1); // missing in endpoint metadata but exists in request, expect exists after conversion - ASSERT_EQ(parser.getProto().inputs().count("l"), 1); // exists in request and endpoint metadata, expect exists after conversion - ASSERT_EQ(parser.getProto().inputs().count("m"), 1); // missing in endpoint metadata but exists in request, expect exists after conversion - ASSERT_EQ(parser.getProto().inputs().size(), 3); -} - -static std::string makeNestedArrayJson(int depth) { - return std::string(depth, '[') + "0" + std::string(depth, ']'); -} - -TEST(TFSRestParserColumn, NestingDepthExceeded_ColumnNamedInputs) { - TFSRestParser parser(prepareTensors({{"i", {1}}})); - std::string request = R"({"signature_name":"","inputs":{"i":)" + makeNestedArrayJson(200) + "}}"; - EXPECT_EQ(parser.parse(request.c_str()), StatusCode::JSON_NESTING_DEPTH_EXCEEDED); -} - -TEST(TFSRestParserColumn, NestingDepthExceeded_ColumnNoNamedInputs) { - TFSRestParser parser(prepareTensors({{"i", {1}}})); - std::string request = R"({"signature_name":"","inputs":)" + makeNestedArrayJson(200) + "}"; - EXPECT_EQ(parser.parse(request.c_str()), StatusCode::JSON_NESTING_DEPTH_EXCEEDED); -} diff --git a/src/test/tfs_rest_parser_nonamed_test.cpp b/src/test/tfs_rest_parser_nonamed_test.cpp deleted file mode 100644 index cb149a089a..0000000000 --- a/src/test/tfs_rest_parser_nonamed_test.cpp +++ /dev/null @@ -1,285 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include -#include - -#include "../rest_parser.hpp" -#pragma warning(push) -#pragma warning(disable : 6001) -#include "absl/strings/escaping.h" -#pragma warning(pop) -#include "test_utils.hpp" - -using namespace ovms; - -using namespace testing; -using ::testing::ElementsAre; - -TEST(TFSRestParserNoNamed, RowOrder_2x1x3x1x5) { - TFSRestParser parser(prepareTensors({{"my_input", {2, 1, 3, 1, 5}}})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[ - [ - [ - [[1, 2, 3, 4, 5]], - [[1, 2, 3, 4, 5]], - [[1, 2, 3, 4, 5]] - ] - ], - [ - [ - [[1, 2, 3, 4, 5]], - [[1, 2, 3, 4, 5]], - [[1, 2, 3, 4, 5]] - ] - ] - ]})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::ROW); - EXPECT_EQ(parser.getFormat(), Format::NONAMED); - ASSERT_EQ(parser.getProto().inputs().count("my_input"), 1); - const auto& my_input = parser.getProto().inputs().at("my_input"); - EXPECT_THAT(asVector(my_input.tensor_shape()), ElementsAre(2, 1, 3, 1, 5)); - EXPECT_THAT(asVector(my_input.tensor_content()), ElementsAre( - 1, 2, 3, 4, 5, - 1, 2, 3, 4, 5, - 1, 2, 3, 4, 5, - 1, 2, 3, 4, 5, - 1, 2, 3, 4, 5, - 1, 2, 3, 4, 5)); -} - -TEST(TFSRestParserNoNamed, RowOrderBinary_2) { - TFSRestParser parser(prepareTensors({{"my_input", {2}}})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[ - {"b64": "ORw0"}, - {"b64": "ORw0"} - ]})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::ROW); - EXPECT_EQ(parser.getFormat(), Format::NONAMED); - ASSERT_EQ(parser.getProto().inputs().count("my_input"), 1); - const auto& my_input = parser.getProto().inputs().at("my_input"); - EXPECT_THAT(asVector(my_input.tensor_shape()), ElementsAre(2)); - EXPECT_EQ(my_input.dtype(), tensorflow::DataType::DT_STRING); - - char expectedBinary[] = {57, 28, 52}; - EXPECT_EQ(my_input.string_val()[0], std::string(expectedBinary, expectedBinary + 3)); - EXPECT_EQ(my_input.string_val()[1], std::string(expectedBinary, expectedBinary + 3)); -} - -TEST(TFSRestParserNoNamed, RowOrder_2x1x3x1x0) { - TFSRestParser parser(prepareTensors({{"my_input", {2, 1, 3, 1, 5}}}, ovms::Precision::I32)); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[ - [ - [ - [[ ]], - [[ ]], - [[ ]] - ] - ], - [ - [ - [[ ]], - [[ ]], - [[ ]] - ] - ] - ]})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::ROW); - EXPECT_EQ(parser.getFormat(), Format::NONAMED); - ASSERT_EQ(parser.getProto().inputs().count("my_input"), 1); - EXPECT_EQ(parser.getProto().inputs().at("my_input").dtype(), tensorflow::DT_INT32); - const auto& my_input = parser.getProto().inputs().at("my_input"); - EXPECT_THAT(asVector(my_input.tensor_shape()), ElementsAre(2, 1, 3, 1, 0)); - EXPECT_EQ(my_input.tensor_content().size(), 0); -} - -TEST(TFSRestParserNoNamed, Parse2InputsRow) { - TFSRestParser parser(prepareTensors({{"first", {2}}, {"second", {3}}})); - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[ - [ - [ - [[1, 2, 3, 4, 5]], - [[1, 2, 3, 4, 5]], - [[1, 2, 3, 4, 5]] - ] - ], - [ - [ - [[1, 2, 3, 4, 5]], - [[1, 2, 3, 4, 5]], - [[1, 2, 3, 4, 5]] - ] - ] - ]})"), - StatusCode::INVALID_INPUT_FORMAT); -} - -TEST(TFSRestParserNoNamed, Parse0InputsRow) { - TFSRestParser parser(prepareTensors({})); - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[1]})"), StatusCode::REST_INPUT_NOT_PREALLOCATED); -} -TEST(TFSRestParserNoNamed, Parse0InputsColumn) { - TFSRestParser parser(prepareTensors({})); - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":[1]})"), StatusCode::REST_INPUT_NOT_PREALLOCATED); -} -TEST(TFSRestParserNoNamed, Parse2InputsColumn) { - TFSRestParser parser(prepareTensors({{"first", {2}}, {"second", {3}}})); - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":[ - [ - [ - [[1, 2, 3, 4, 5]], - [[1, 2, 3, 4, 5]], - [[1, 2, 3, 4, 5]] - ] - ], - [ - [ - [[1, 2, 3, 4, 5]], - [[1, 2, 3, 4, 5]], - [[1, 2, 3, 4, 5]] - ] - ] - ]})"), - StatusCode::INVALID_INPUT_FORMAT); -} -TEST(TFSRestParserNoNamed, RowOrder_5) { - TFSRestParser parser(prepareTensors({{"my_input", {5}}})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[1,2,3,4,5]})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::ROW); - EXPECT_EQ(parser.getFormat(), Format::NONAMED); - ASSERT_EQ(parser.getProto().inputs().count("my_input"), 1); - const auto& my_input = parser.getProto().inputs().at("my_input"); - EXPECT_THAT(asVector(my_input.tensor_shape()), ElementsAre(5)); - EXPECT_THAT(asVector(my_input.tensor_content()), ElementsAre(1, 2, 3, 4, 5)); -} - -TEST(TFSRestParserNoNamed, ColumnOrder_2x1x3x1x5) { - TFSRestParser parser(prepareTensors({{"my_input", {2, 1, 3, 1, 5}}})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":[ - [ - [ - [[1, 2, 3, 4, 5]], - [[1, 2, 3, 4, 5]], - [[1, 2, 3, 4, 5]] - ] - ], - [ - [ - [[1, 2, 3, 4, 5]], - [[1, 2, 3, 4, 5]], - [[1, 2, 3, 4, 5]] - ] - ] - ]})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::COLUMN); - EXPECT_EQ(parser.getFormat(), Format::NONAMED); - ASSERT_EQ(parser.getProto().inputs().count("my_input"), 1); - const auto& my_input = parser.getProto().inputs().at("my_input"); - EXPECT_THAT(asVector(my_input.tensor_shape()), ElementsAre(2, 1, 3, 1, 5)); - EXPECT_THAT(asVector(my_input.tensor_content()), ElementsAre( - 1, 2, 3, 4, 5, - 1, 2, 3, 4, 5, - 1, 2, 3, 4, 5, - 1, 2, 3, 4, 5, - 1, 2, 3, 4, 5, - 1, 2, 3, 4, 5)); -} - -TEST(TFSRestParserNoNamed, ColumnOrderBinary_2) { - TFSRestParser parser(prepareTensors({{"my_input", {2}}})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":[ - {"b64": "ORw0"}, - {"b64": "ORw0"} - ]})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::COLUMN); - EXPECT_EQ(parser.getFormat(), Format::NONAMED); - ASSERT_EQ(parser.getProto().inputs().count("my_input"), 1); - const auto& my_input = parser.getProto().inputs().at("my_input"); - EXPECT_THAT(asVector(my_input.tensor_shape()), ElementsAre(2)); - EXPECT_EQ(my_input.dtype(), tensorflow::DataType::DT_STRING); - - char expectedBinary[] = {57, 28, 52}; - EXPECT_EQ(my_input.string_val()[0], std::string(expectedBinary, expectedBinary + 3)); - EXPECT_EQ(my_input.string_val()[1], std::string(expectedBinary, expectedBinary + 3)); -} - -TEST(TFSRestParserNoNamed, ColumnOrder_2x1x3x1x0) { - TFSRestParser parser(prepareTensors({{"my_input", {2, 1, 3, 1, 5}}}, ovms::Precision::FP32)); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":[ - [ - [ - [[ ]], - [[ ]], - [[ ]] - ] - ], - [ - [ - [[ ]], - [[ ]], - [[ ]] - ] - ] - ]})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::COLUMN); - EXPECT_EQ(parser.getFormat(), Format::NONAMED); - ASSERT_EQ(parser.getProto().inputs().count("my_input"), 1); - const auto& my_input = parser.getProto().inputs().at("my_input"); - EXPECT_EQ(my_input.dtype(), tensorflow::DT_FLOAT); - EXPECT_THAT(asVector(my_input.tensor_shape()), ElementsAre(2, 1, 3, 1, 0)); - EXPECT_EQ(my_input.tensor_content().size(), 0); -} - -TEST(TFSRestParserNoNamed, ColumnOrder_1d_5elements) { - TFSRestParser parser(prepareTensors({{"my_input", {5}}})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":[1,2,3,4,5]})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::COLUMN); - EXPECT_EQ(parser.getFormat(), Format::NONAMED); - ASSERT_EQ(parser.getProto().inputs().count("my_input"), 1); - const auto& my_input = parser.getProto().inputs().at("my_input"); - EXPECT_THAT(my_input.dtype(), tensorflow::DT_FLOAT); - EXPECT_THAT(asVector(my_input.tensor_shape()), ElementsAre(5)); - EXPECT_THAT(asVector(my_input.tensor_content()), ElementsAre(1, 2, 3, 4, 5)); -} - -TEST(TFSRestParserNoNamed, ColumnOrder_Scalar) { - TFSRestParser parser(prepareTensors({{"my_input", {}}})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","inputs":5})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::COLUMN); - EXPECT_EQ(parser.getFormat(), Format::NONAMED); - ASSERT_EQ(parser.getProto().inputs().count("my_input"), 1); - const auto& my_input = parser.getProto().inputs().at("my_input"); - EXPECT_THAT(my_input.dtype(), tensorflow::DT_FLOAT); - EXPECT_THAT(asVector(my_input.tensor_shape()), ElementsAre()); - EXPECT_THAT(asVector(my_input.tensor_content()), ElementsAre(5)); -} diff --git a/src/test/tfs_rest_parser_row_test.cpp b/src/test/tfs_rest_parser_row_test.cpp deleted file mode 100644 index 8119d21ef3..0000000000 --- a/src/test/tfs_rest_parser_row_test.cpp +++ /dev/null @@ -1,757 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include -#include - -#pragma warning(push) -#pragma warning(disable : 4624 6001) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#pragma GCC diagnostic ignored "-Wunused-but-set-variable" -#include "tensorflow/core/framework/tensor.h" - -#include "absl/strings/escaping.h" -#pragma GCC diagnostic pop -#pragma warning(pop) - -#include "../rest_parser.hpp" -#include "test_utils.hpp" - -using namespace ovms; - -using namespace testing; -using ::testing::ElementsAre; - -using tensorflow::DataTypeSize; - -const char* predictRequestRowNamedJson = R"({ - "instances": [ - { - "inputA": [ - [[1.0, 2.0], - [3.0, 4.0], - [5.0, 6.0]], - [[7.0, 8.0], - [9.0, 10.0], - [11.0, 12.0]] - ], - "inputB": [ - [1.0, 2.0, 3.0], - [4.0, 5.0, 6.0] - ], - "inputC": {"b64": "ORw0"} - }, - { - "inputA": [ - [[101.0, 102.0], - [103.0, 104.0], - [105.0, 106.0]], - [[107.0, 108.0], - [109.0, 110.0], - [111.0, 112.0]] - ], - "inputB": [ - [11.0, 12.0, 13.0], - [14.0, 15.0, 16.0] - ], - "inputC": {"b64": "ORw0"} - } - ], - "signature_name": "serving_default" -})"; - -TEST(TFSRestParserRow, ParseValid2Inputs) { - TFSRestParser parser(prepareTensors({{"inputA", {2, 2, 3, 2}}, - {"inputB", {2, 2, 3}}, {"inputC", {2}}})); - - auto status = parser.parse(predictRequestRowNamedJson); - - ASSERT_EQ(status, StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::ROW); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - ASSERT_EQ(parser.getProto().inputs_size(), 3); - ASSERT_EQ(parser.getProto().inputs().count("inputA"), 1); - ASSERT_EQ(parser.getProto().inputs().count("inputB"), 1); - ASSERT_EQ(parser.getProto().inputs().count("inputC"), 1); - const auto& inputA = parser.getProto().inputs().at("inputA"); - const auto& inputB = parser.getProto().inputs().at("inputB"); - const auto& inputC = parser.getProto().inputs().at("inputC"); - EXPECT_EQ(inputA.dtype(), tensorflow::DataType::DT_FLOAT); - EXPECT_EQ(inputB.dtype(), tensorflow::DataType::DT_FLOAT); - EXPECT_EQ(inputC.dtype(), tensorflow::DataType::DT_STRING); - EXPECT_THAT(asVector(inputA.tensor_shape()), ElementsAre(2, 2, 3, 2)); - EXPECT_THAT(asVector(inputB.tensor_shape()), ElementsAre(2, 2, 3)); - EXPECT_THAT(asVector(inputC.tensor_shape()), ElementsAre(2)); - ASSERT_EQ(inputA.tensor_content().size(), 2 * 2 * 3 * 2 * DataTypeSize(tensorflow::DataType::DT_FLOAT)); - ASSERT_EQ(inputB.tensor_content().size(), 2 * 2 * 3 * DataTypeSize(tensorflow::DataType::DT_FLOAT)); - ASSERT_EQ(inputC.string_val().size(), 2); - EXPECT_THAT(asVector(inputA.tensor_content()), ElementsAre( - 1.0, 2.0, - 3.0, 4.0, - 5.0, 6.0, - //------- - 7.0, 8.0, - 9.0, 10.0, - 11.0, 12.0, - //========= - 101.0, 102.0, - 103.0, 104.0, - 105.0, 106.0, - //--------- - 107.0, 108.0, - 109.0, 110.0, - 111.0, 112.0)); - EXPECT_THAT(asVector(inputB.tensor_content()), ElementsAre( - 1.0, 2.0, 3.0, - 4.0, 5.0, 6.0, - //============ - 11.0, 12, 13.0, - 14.0, 15.0, 16.0)); - - char expectedBinary[] = {57, 28, 52}; - EXPECT_EQ(inputC.string_val()[0], std::string(expectedBinary, expectedBinary + 3)); - EXPECT_EQ(inputC.string_val()[1], std::string(expectedBinary, expectedBinary + 3)); -} - -TEST(TFSRestParserRow, InvalidShape_1D) { - TFSRestParser parser(prepareTensors({{"i", {2}}})); - - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":[ - {"i":12.0}, {"i":[[13.0]]} - ]})"), - StatusCode::REST_COULD_NOT_PARSE_INSTANCE); - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":[ - {"i":[[12.0]]}, {"i":13.0} - ]})"), - StatusCode::REST_COULD_NOT_PARSE_INSTANCE); - - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":[ - {"i":13.0}, {"i": {"b64": "Wi8VP4wNABwBTCexN6wAAAABJRU5ErkJggg=="}} - ]})"), - StatusCode::REST_COULD_NOT_PARSE_INSTANCE); - - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":[ - {"i": {"b64": "Wi8VP4wNABwBTCexN6wAAAABJRU5ErkJggg=="}}, {"i":13.0} - ]})"), - StatusCode::REST_COULD_NOT_PARSE_INSTANCE); -} - -TEST(TFSRestParserRow, ValidShape_2) { - TFSRestParser parser(prepareTensors({{"i", {2}}})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[ - {"i":12.0}, {"i":13.0} - ]})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::ROW); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(2)); - // Precision? - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(12.0, 13.0)); -} - -TEST(TFSRestParserRow, ValidShape_1x1) { - TFSRestParser parser(prepareTensors({{"i", {1, 1}}})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[ - {"i":[155.0]} - ]})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::ROW); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(1, 1)); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(155.0)); -} - -TEST(TFSRestParserRow, ValidShape_1x2) { - TFSRestParser parser(prepareTensors({{"i", {1, 2}}})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[ - {"i":[155.0, 56.0]} - ]})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::ROW); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(1, 2)); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(155.0, 56.0)); -} - -TEST(TFSRestParserRow, ValidShape_2x0) { - TFSRestParser parser(prepareTensors({{"i", {2, 0}}}, ovms::Precision::I32)); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[ - {"i":[ ]}, {"i":[ ]} - ]})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::ROW); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(2, 0)); - EXPECT_EQ(parser.getProto().inputs().at("i").tensor_content().size(), 0); - EXPECT_EQ(parser.getProto().inputs().at("i").dtype(), tensorflow::DataType::DT_INT32); -} - -TEST(TFSRestParserRow, ValidShape_2x1) { - TFSRestParser parser(prepareTensors({{"i", {2, 1}}})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[ - {"i":[155.0]}, {"i":[513.0]} - ]})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::ROW); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(2, 1)); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(155.0, 513.0)); -} - -TEST(TFSRestParserRow, ValidShape_2x2) { - TFSRestParser parser(prepareTensors({{"i", {2, 2}}})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[ - {"i":[155.0, 9.0]}, {"i":[513.0, -5.0]} - ]})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::ROW); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(2, 2)); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(155.0, 9.0, 513.0, -5.0)); -} - -TEST(TFSRestParserRow, ValidShape_2x1x3) { - TFSRestParser parser(prepareTensors({{"i", {2, 1, 3}}})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[ - {"i":[ - [5.0, 9.0, 2.0] - ]}, - {"i":[ - [-5.0, -2.0, -10.0] - ]} - ]})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::ROW); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(2, 1, 3)); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(5.0, 9.0, 2.0, -5.0, -2.0, -10.0)); -} - -TEST(TFSRestParserRow, ValidShape_2x3x1) { - TFSRestParser parser(prepareTensors({{"i", {2, 3, 1}}})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[ - {"i":[ - [5.9], - [9.9], - [1.9] - ]}, - {"i":[ - [-1.9], - [-9.9], - [25.9] - ]} - ]})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::ROW); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(2, 3, 1)); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(5.9, 9.9, 1.9, -1.9, -9.9, 25.9)); -} - -TEST(TFSRestParserRow, ValidShape_2x1x2x1) { - TFSRestParser parser(prepareTensors({{"i", {2, 1, 2, 1}}})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[ - {"i":[ - [ - [5.0], - [2.0] - ] - ]}, - {"i":[ - [ - [6.0], - [18.0] - ] - ]} - ]})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::ROW); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(2, 1, 2, 1)); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(5.0, 2.0, 6.0, 18.0)); -} - -TEST(TFSRestParserRow, ValidShape_2x1x3x1x5) { - TFSRestParser parser(prepareTensors({{"i", {2, 1, 3, 1, 5}}})); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[ - {"i":[ - [ - [[1.0, 2.0, 3.0, 4.0, 5.0]], - [[1.0, 2.0, 3.0, 4.0, 5.0]], - [[1.0, 2.0, 3.0, 4.0, 5.0]] - ] - ]}, - {"i":[ - [ - [[1.9, 2.9, 3.9, 4.9, 5.9]], - [[1.9, 2.9, 3.9, 4.9, 5.9]], - [[1.9, 2.9, 3.9, 4.9, 5.9]] - ] - ]} - ]})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::ROW); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(2, 1, 3, 1, 5)); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre( - 1.0, 2.0, 3.0, 4.0, 5.0, - 1.0, 2.0, 3.0, 4.0, 5.0, - 1.0, 2.0, 3.0, 4.0, 5.0, - 1.9, 2.9, 3.9, 4.9, 5.9, - 1.9, 2.9, 3.9, 4.9, 5.9, - 1.9, 2.9, 3.9, 4.9, 5.9)); -} - -TEST(TFSRestParserRow, ValidShape_2x1x3x1x0) { - TFSRestParser parser(prepareTensors({{"i", {2, 1, 3, 1, 0}}}, ovms::Precision::FP32)); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[ - {"i":[ - [ - [[ ]], - [[ ]], - [[ ]] - ] - ]}, - {"i":[ - [ - [[ ]], - [[ ]], - [[ ]] - ] - ]} - ]})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::ROW); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_shape()), ElementsAre(2, 1, 3, 1, 0)); - EXPECT_EQ(parser.getProto().inputs().at("i").tensor_content().size(), 0); - EXPECT_EQ(parser.getProto().inputs().at("i").dtype(), tensorflow::DataType::DT_FLOAT); -} - -TEST(TFSRestParserRow, MissingInputInBatch) { - TFSRestParser parser(prepareTensors({{"i", {2, 1, 2, 2}}, - {"j", {1, 1, 2, 2}}})); - - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":[ - { - "i":[[[5, 2], [10, 7]]], - "j":[[[5, 2], [10, 7]]] - }, - { - "i":[[[5, 2], [10, 7]]] - } -]})"), - StatusCode::REST_INSTANCES_BATCH_SIZE_DIFFER); -} - -TEST(TFSRestParserRow, ParseUint8) { - TFSRestParser parser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::U8)); - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[[0,5,15,255]]}]})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(0, 5, 15, 255)); - parser = TFSRestParser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::U8)); - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[[0.0,5.0,15.0,255.0]]}]})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(0, 5, 15, 255)); -} - -TEST(TFSRestParserRow, ParseInt8) { - TFSRestParser parser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::I8)); - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[[0,-5,127,-128]]}]})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(0, -5, 127, -128)); - parser = TFSRestParser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::I8)); - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[[0.0,-5.0,127.0,-128.0]]}]})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(0, -5, 127, -128)); -} - -TEST(TFSRestParserRow, ParseUint16) { - TFSRestParser parser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::U16)); - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[[0,5,128,65535]]}]})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().mutable_inputs()->at("i").mutable_int_val()), ElementsAre(0, 5, 128, 65535)); - parser = TFSRestParser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::U16)); - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[[0.0,5.0,128.0,65535.0]]}]})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().mutable_inputs()->at("i").mutable_int_val()), ElementsAre(0, 5, 128, 65535)); -} - -TEST(TFSRestParserRow, ParseInt16) { - TFSRestParser parser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::I16)); - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[[0,-5,32768,-32767]]}]})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(0, -5, 32768, -32767)); - parser = TFSRestParser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::I16)); - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[[0.0,-5.0,32768.0,-32767.0]]}]})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(0, -5, 32768, -32767)); -} - -TEST(TFSRestParserRow, ParseInt32) { - TFSRestParser parser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::I32)); - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[[0,-5,2147483648,-2147483647]]}]})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(0, -5, 2147483648, -2147483647)); - parser = TFSRestParser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::I32)); - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[[0.0,-5.0,2147483648.0,-2147483647.0]]}]})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(0, -5, 2147483648, -2147483647)); -} - -TEST(TFSRestParserRow, ParseUint64) { - TFSRestParser parser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::U64)); - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[[0,5,128,18446744073709551615]]}]})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(0, 5, 128, 18446744073709551615UL)); - parser = TFSRestParser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::U64)); - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[[0.0,5.0,128.0,555222.0]]}]})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(0, 5, 128, 555222)); // Can't looselessly cast large double to int64 -} - -TEST(TFSRestParserRow, ParseInt64) { - TFSRestParser parser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::I64)); - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[[0,-5,5522,-9223372036854775807]]}]})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(0, -5, 5522, -9223372036854775807)); - parser = TFSRestParser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::I64)); - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[[0.0,-5.0,5522.0,-55333.0]]}]})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(0, -5, 5522, -55333)); // Can't looselessly cast double to int64 -} - -TEST(TFSRestParserRow, ParseFloat) { - TFSRestParser parser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::FP32)); - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[[-5, 0, -4, 155234]]}]})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(-5, 0, -4, 155234)); - parser = TFSRestParser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::FP32)); - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[[-5.12, 0.4344, -4.521, 155234.221]]}]})"), StatusCode::OK); - EXPECT_THAT(asVector(parser.getProto().inputs().at("i").tensor_content()), ElementsAre(-5.12, 0.4344, -4.521, 155234.221)); -} - -TEST(TFSRestParserRow, ParseHalf) { - TFSRestParser parser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::FP16)); - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[[-5, 0, -4, 155234]]}]})"), StatusCode::OK); - parser = TFSRestParser(prepareTensors({{"i", {1, 1, 4}}}, ovms::Precision::FP16)); - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[[-5.1222, 0.434422, -4.52122, 155234.22122]]}]})"), StatusCode::OK); -} - -static void checkParseStatusAndMessage(TFSRestParser& parser, const char* json, StatusCode expectedStatus, const char* expectedMessage) { - auto status = parser.parse(json); - EXPECT_EQ(status.getCode(), expectedStatus); - EXPECT_EQ(status.string(), expectedMessage); -} - -TEST(TFSRestParserRow, InvalidJson) { - TFSRestParser parser(prepareTensors({{"i", {1, 3, 2}}})); - - checkParseStatusAndMessage(parser, "", StatusCode::JSON_INVALID, "The file is not valid json - Error: The document is empty. Offset: 0"); - checkParseStatusAndMessage(parser, "{{}", StatusCode::JSON_INVALID, "The file is not valid json - Error: Missing a name for object member. Offset: 1"); - checkParseStatusAndMessage(parser, R"({"signature_name:"","instances":[{"i":[1]}]})", - StatusCode::JSON_INVALID, "The file is not valid json - Error: Missing a colon after a name of object member. Offset: 18"); - checkParseStatusAndMessage(parser, R"({"signature_name":"","instances":[{i":[1]}]})", // missing " - StatusCode::JSON_INVALID, "The file is not valid json - Error: Missing a name for object member. Offset: 35"); - checkParseStatusAndMessage(parser, R"({"signature_name":"","instances":[{"i":[1}]})", // missing ] - StatusCode::JSON_INVALID, "The file is not valid json - Error: Missing a comma or ']' after an array element. Offset: 41"); - checkParseStatusAndMessage(parser, R"({"signature_name":"","instances":[{"i":[1]}])", // missing } - StatusCode::JSON_INVALID, "The file is not valid json - Error: Missing a comma or '}' after an object member. Offset: 44"); - checkParseStatusAndMessage(parser, R"(["signature_name":"","instances":[{"i":[1]}]})", // missing { - StatusCode::JSON_INVALID, "The file is not valid json - Error: Missing a comma or ']' after an array element. Offset: 17"); - checkParseStatusAndMessage(parser, R"({"signature_name":"","instances":{[{"i":[1]}]})", // too many { - StatusCode::JSON_INVALID, "The file is not valid json - Error: Missing a name for object member. Offset: 34"); - checkParseStatusAndMessage(parser, R"({"signature_name":"","instances":[{"i":[[1.0,5.0],[3.0,0.0] [9.0,5.0]]}]})", // missing , - StatusCode::JSON_INVALID, "The file is not valid json - Error: Missing a comma or ']' after an array element. Offset: 60"); -} - -TEST(TFSRestParserRow, BodyNotAnObject) { - TFSRestParser parser(prepareTensors({}, ovms::Precision::FP16)); - - EXPECT_EQ(parser.parse("[]"), StatusCode::REST_BODY_IS_NOT_AN_OBJECT); - EXPECT_EQ(parser.parse("\"string\""), StatusCode::REST_BODY_IS_NOT_AN_OBJECT); - EXPECT_EQ(parser.parse("1"), StatusCode::REST_BODY_IS_NOT_AN_OBJECT); - EXPECT_EQ(parser.parse("null"), StatusCode::REST_BODY_IS_NOT_AN_OBJECT); -} - -TEST(TFSRestParserRow, CouldNotDetectOrder) { - TFSRestParser parser(prepareTensors({}, ovms::Precision::FP16)); - - EXPECT_EQ(parser.parse(R"({"signature_name":""})"), StatusCode::REST_PREDICT_UNKNOWN_ORDER); - EXPECT_EQ(parser.parse(R"({"signature_name":"","bad":[{"i":[1]}]})"), StatusCode::REST_PREDICT_UNKNOWN_ORDER); - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[1]}],"inputs":{"i":[[1]]}})"), StatusCode::REST_PREDICT_UNKNOWN_ORDER); -} - -TEST(TFSRestParserRow, InstancesNotAnArray) { - TFSRestParser parser(prepareTensors({}, ovms::Precision::FP16)); - - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":{}})"), StatusCode::REST_INSTANCES_NOT_AN_ARRAY); - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":"string"})"), StatusCode::REST_INSTANCES_NOT_AN_ARRAY); - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":5})"), StatusCode::REST_INSTANCES_NOT_AN_ARRAY); -} - -TEST(TFSRestParserRow, NamedInstanceNotAnObject) { - TFSRestParser parser(prepareTensors({{"i", {2, 1}}})); - - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[5]},2,3]})"), StatusCode::REST_NAMED_INSTANCE_NOT_AN_OBJECT); - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[5]},null]})"), StatusCode::REST_NAMED_INSTANCE_NOT_AN_OBJECT); -} - -TEST(TFSRestParserRow, CouldNotDetectNamedOrNoNamed) { - TFSRestParser parser(prepareTensors({}, ovms::Precision::FP16)); - - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":[null, null]})"), StatusCode::REST_INSTANCES_NOT_NAMED_OR_NONAMED); -} - -TEST(TFSRestParserRow, NoInstancesFound) { - TFSRestParser parser(prepareTensors({}, ovms::Precision::FP16)); - - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":[]})"), StatusCode::REST_NO_INSTANCES_FOUND); -} - -TEST(TFSRestParserRow, CannotParseInstance) { - TFSRestParser parser(prepareTensors({{"i", {1, 2}}})); - - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":[{}]})"), StatusCode::REST_COULD_NOT_PARSE_INSTANCE); - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":null}]})"), StatusCode::REST_COULD_NOT_PARSE_INSTANCE); - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[1,null]}]})"), StatusCode::REST_COULD_NOT_PARSE_INSTANCE); - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[[1,2],[3,"str"]]}]})"), StatusCode::REST_COULD_NOT_PARSE_INSTANCE); -} - -TEST(TFSRestParserRow, InputNotNdArray_1) { - TFSRestParser parser(prepareTensors({{"i", {1, 2, 3, 2}}})); - - // [1, 4, 5] size is 3 instead of 2 to be valid - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[ - [[1, 2], - [1, 3], - [1, 4, 5]], - [[5, 8], - [9, 3], - [1, 4]] - ]}]})"), - StatusCode::REST_COULD_NOT_PARSE_INSTANCE); -} - -TEST(TFSRestParserRow, InputNotNdArray_2) { - TFSRestParser parser(prepareTensors({{"i", {1, 2, 3, 3}}})); - - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[ - [[1, 2, [8]], - [1, 3, [3]], - [1, 4, [5]]], - [[5, 8, [-1]], - [9, 3, [-5]], - [1, 4, [-4]]] - ]}]})"), - StatusCode::REST_COULD_NOT_PARSE_INSTANCE); -} - -TEST(TFSRestParserRow, InputNotNdArray_3) { - TFSRestParser parser(prepareTensors({{"i", {1, 4, 3, 2}}})); - - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[ - [[1, 2], - [1, 3], - [1, 4]], - - [[1, 2]], - - [[5, 8], - [9, 3], - [1, 4]], - - [[5, 8]] - ]}]})"), - StatusCode::REST_COULD_NOT_PARSE_INSTANCE); -} - -TEST(TFSRestParserRow, InputNotNdArray_4) { - TFSRestParser parser(prepareTensors({{"i", {1, 2, 3, 2}}})); - - // [5, 6] is not a number but array - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[ - [[1, 2], - [1, 3], - [1, 4, [5, 6]]], - [[5, 8], - [9, 3], - [1, 4]] - ]}]})"), - StatusCode::REST_COULD_NOT_PARSE_INSTANCE); -} - -TEST(TFSRestParserRow, InputNotNdArray_5) { - TFSRestParser parser(prepareTensors({{"i", {1, 2, 3, 2}}})); - - // [1] is of wrong shape - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[ - [[1], - [1, 2], - [1, 3], - [1, 4]], - [[5, 8], - [9, 3], - [1, 4]] - ]}]})"), - StatusCode::REST_COULD_NOT_PARSE_INSTANCE); -} - -TEST(TFSRestParserRow, InputNotNdArray_6) { - TFSRestParser parser(prepareTensors({{"i", {1, 2, 2, 2}}})); - - // [1, 1] missing - 2x2, 2x3 - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[ - [[1, 2], - [1, 3]], - [[5, 8], - [9, 3], - [1, 4]] - ]}]})"), - StatusCode::REST_COULD_NOT_PARSE_INSTANCE); -} - -TEST(TFSRestParserRow, InputNotNdArray_7) { - TFSRestParser parser(prepareTensors({{"i", {1, 2, 3, 2}}})); - - // [1, 5] numbers are on wrong level - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[ - [1, 5], - [[1, 1], - [1, 2], - [1, 3]], - [[5, 8], - [9, 3], - [1, 4]] - ]}]})"), - StatusCode::REST_COULD_NOT_PARSE_INSTANCE); -} - -TEST(TFSRestParserRow, InputNotNdArray_8) { - TFSRestParser parser(prepareTensors({{"i", {1, 2, 3, 2}}})); - - // [1, 2], [9, 3] numbers are on wrong level - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":[{"i":[ - [[1, 1], - [[1, 2]], - [1, 3]], - [[5, 8], - [[9, 3]], - [1, 4]] - ]}]})"), - StatusCode::REST_COULD_NOT_PARSE_INSTANCE); -} - -TEST(TFSRestParserRow, InstancesShapeDiffer_1) { - TFSRestParser parser(prepareTensors({{"i", {2, 2, 3, 2}}})); - - // 2x3x2 vs 2x2x2 - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":[ - {"i":[ - [[1, 1], - [1, 2], - [1, 3]], - [[5, 8], - [9, 3], - [1, 4]] - ]}, - {"i":[ - [[1, 1], - [1, 2]], - [[5, 8], - [9, 3]] - ]} - ]})"), - StatusCode::REST_COULD_NOT_PARSE_INSTANCE); -} - -TEST(TFSRestParserRow, InstancesShapeDiffer_2) { - TFSRestParser parser(prepareTensors({{"i", {2, 2, 3, 2}}})); - - // 2x3x2 vs 2x3x3 - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":[ - {"i":[ - [[1, 1], - [1, 2], - [1, 3]], - [[5, 8], - [9, 3], - [1, 4]] - ]}, - {"i":[ - [[1, 1, 3], - [1, 2, 2], - [1, 3, 9]], - [[5, 8, 8], - [9, 3, 3], - [1, 4, 10]] - ]} - ]})"), - StatusCode::REST_COULD_NOT_PARSE_INSTANCE); -} - -TEST(TFSRestParserRow, InstancesShapeDiffer_3) { - TFSRestParser parser(prepareTensors({{"i", {2, 2, 3, 2}}})); - - // 2x3x2 vs 1x2x3x2 - EXPECT_EQ(parser.parse(R"({"signature_name":"","instances":[ - {"i":[ - [[1, 1], - [1, 2], - [1, 3]], - [[5, 8], - [9, 3], - [1, 4]] - ]}, - {"i":[[ - [[1, 1], - [1, 2], - [1, 3]], - [[5, 8], - [9, 3], - [1, 4]] - ]]} - ]})"), - StatusCode::REST_COULD_NOT_PARSE_INSTANCE); -} - -TEST(TFSRestParserRow, RemoveUnnecessaryInputs) { - TFSRestParser parser(prepareTensors({{"i", {1, 1}}, {"j", {1, 1}}, {"k", {1, 1}}, {"l", {1, 1}}}, ovms::Precision::FP16)); - - ASSERT_EQ(parser.parse(R"({"signature_name":"","instances":[ - {"k":[155.0]}, {"l":[1.0]} - ]})"), - StatusCode::OK); - EXPECT_EQ(parser.getOrder(), Order::ROW); - EXPECT_EQ(parser.getFormat(), Format::NAMED); - ASSERT_EQ(parser.getProto().inputs().size(), 2); - ASSERT_EQ(parser.getProto().inputs().count("i"), 0); - ASSERT_EQ(parser.getProto().inputs().count("j"), 0); - ASSERT_EQ(parser.getProto().inputs().count("k"), 1); - ASSERT_EQ(parser.getProto().inputs().count("l"), 1); -} - -static std::string makeNestedArrayJson(int depth) { - return std::string(depth, '[') + "0" + std::string(depth, ']'); -} - -TEST(TFSRestParserRow, NestingDepthExceeded_RowNamedInstances) { - TFSRestParser parser(prepareTensors({{"i", {1}}})); - std::string request = R"({"signature_name":"","instances":[{"i":)" + makeNestedArrayJson(200) + "}]}"; - EXPECT_EQ(parser.parse(request.c_str()), StatusCode::JSON_NESTING_DEPTH_EXCEEDED); -} - -TEST(TFSRestParserRow, NestingDepthExceeded_RowNoNamedInstances) { - TFSRestParser parser(prepareTensors({{"i", {1}}})); - std::string request = R"({"signature_name":"","instances":)" + makeNestedArrayJson(200) + "}"; - EXPECT_EQ(parser.parse(request.c_str()), StatusCode::JSON_NESTING_DEPTH_EXCEEDED); -} diff --git a/src/tfs_frontend/deserialization.cpp b/src/tfs_frontend/deserialization.cpp deleted file mode 100644 index f16340c294..0000000000 --- a/src/tfs_frontend/deserialization.cpp +++ /dev/null @@ -1,39 +0,0 @@ -//***************************************************************************** -// Copyright 2024 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "deserialization.hpp" - -#include "../itensorfactory.hpp" -#include "../logging.hpp" - -namespace ovms { -ov::Tensor makeTensor(const tensorflow::TensorProto& requestInput, - const std::shared_ptr& tensorInfo) { - OVMS_PROFILE_FUNCTION(); - OV_LOGGER("ov::Shape()"); - ov::Shape shape; - for (int i = 0; i < requestInput.tensor_shape().dim_size(); i++) { - OV_LOGGER("ov::Shape::push_back({})", requestInput.tensor_shape().dim(i).size()); - shape.push_back(requestInput.tensor_shape().dim(i).size()); - } - ov::element::Type_t precision = tensorInfo->getOvPrecision(); - if (!requestInput.tensor_content().size()) { - OV_LOGGER("ov::Tensor({}, shape)", toString(ovms::ovElementTypeToOvmsPrecision(precision))); - return ov::Tensor(precision, shape); - } - OV_LOGGER("ov::Tensor({}, shape, data)", toString(ovms::ovElementTypeToOvmsPrecision(precision))); - return ov::Tensor(precision, shape, const_cast(reinterpret_cast(requestInput.tensor_content().data()))); -} -} // namespace ovms diff --git a/src/tfs_frontend/deserialization.hpp b/src/tfs_frontend/deserialization.hpp deleted file mode 100644 index 8b08d8bbb2..0000000000 --- a/src/tfs_frontend/deserialization.hpp +++ /dev/null @@ -1,136 +0,0 @@ -//***************************************************************************** -// Copyright 2024 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include - -#include -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-but-set-variable" -#pragma GCC diagnostic ignored "-Wall" - -#pragma warning(push) -#pragma warning(disable : 4457 4624 6001 6011 6201 6246 6269 6294 6326 6308 6385 6386 6387) -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma warning(pop) -#pragma GCC diagnostic pop - -#include "../deserialization_common.hpp" -#include "../logging.hpp" -#include "../profiler.hpp" -#include "../status.hpp" -#include "../tensor_conversion.hpp" -#include "../tensorinfo.hpp" -#include "tfs_utils.hpp" - -namespace ovms { -ov::Tensor makeTensor(const tensorflow::TensorProto& requestInput, - const std::shared_ptr& tensorInfo); - -template <> -struct RequestTraits<::TFSPredictRequest> { - using TensorType = ::tensorflow::TensorProto; -}; - -template <> -class ConcreteTensorProtoDeserializator { -public: - static ov::Tensor deserializeTensor( - const tensorflow::TensorProto& requestInput, - const std::shared_ptr& tensorInfo, - const std::unordered_map>& factories, - const std::string* bufferLocation) { - OVMS_PROFILE_FUNCTION(); - switch (tensorInfo->getPrecision()) { - case ovms::Precision::FP32: - case ovms::Precision::U32: - case ovms::Precision::I32: - case ovms::Precision::FP64: - case ovms::Precision::I64: - case ovms::Precision::U8: - case ovms::Precision::I16: - case ovms::Precision::I8: { - return makeTensor(requestInput, tensorInfo); - } -#pragma warning(push) -#pragma warning(disable : 4244 4267) - case ovms::Precision::FP16: { - OV_LOGGER("ov::Shape()"); - ov::Shape shape; - for (std::int64_t i = 0; i < requestInput.tensor_shape().dim_size(); i++) { - OV_LOGGER("ov::Shape::push_back({})", requestInput.tensor_shape().dim(i).size()); - shape.push_back(requestInput.tensor_shape().dim(i).size()); - } - ov::Tensor tensor(ov::element::f16, shape); - // Needs conversion due to zero padding for each value: - // https://github.com/tensorflow/tensorflow/blob/v2.2.0/tensorflow/core/framework/tensor.proto#L55 - uint16_t* ptr = (uint16_t*)tensor.data(); - auto size = static_cast(requestInput.half_val_size()); - for (size_t i = 0; i < size; i++) { - ptr[i] = requestInput.half_val(i); - } - return tensor; - } - case ovms::Precision::U16: { - ov::Shape shape; - for (std::int64_t i = 0; i < requestInput.tensor_shape().dim_size(); i++) { - shape.push_back(requestInput.tensor_shape().dim(i).size()); - } - ov::Tensor tensor(ov::element::u16, shape); - // Needs conversion due to zero padding for each value: - // https://github.com/tensorflow/tensorflow/blob/v2.2.0/tensorflow/core/framework/tensor.proto#L55 - uint16_t* ptr = (uint16_t*)tensor.data(); - auto size = static_cast(requestInput.int_val_size()); - for (size_t i = 0; i < size; i++) { - ptr[i] = requestInput.int_val(i); - } - return tensor; - } -#pragma warning(pop) - case ovms::Precision::U64: - default: - return ov::Tensor(); - } - } -}; -#pragma warning(push) -#pragma warning(disable : 4505) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-function" -template <> // TODO separate for different choice -std::tuple::TensorType*, const std::string*> getRequestTensorPtr(const ::TFSPredictRequest& request, const std::string& name, ExtractChoice extractChoice) { - if (ExtractChoice::EXTRACT_OUTPUT == extractChoice) { - SPDLOG_TRACE("Defining output in TFS is not implemented"); - return std::make_tuple(Status(StatusCode::NOT_IMPLEMENTED, "Failed to deserialize output in request"), nullptr, nullptr); - } - auto requestInputItr = request.inputs().find(name); - if (requestInputItr == request.inputs().end()) { - SPDLOG_DEBUG("Failed to deserialize request. Validation of request failed"); - return std::make_tuple(Status(StatusCode::INTERNAL_ERROR, "Failed to deserialize input in request"), nullptr, nullptr); - } - return std::make_tuple(Status(StatusCode::OK), &requestInputItr->second, nullptr); -} -#pragma GCC diagnostic pop -#pragma warning(pop) - -} // namespace ovms diff --git a/src/tfs_frontend/serialization.cpp b/src/tfs_frontend/serialization.cpp deleted file mode 100644 index f40a467c74..0000000000 --- a/src/tfs_frontend/serialization.cpp +++ /dev/null @@ -1,150 +0,0 @@ -//***************************************************************************** -// Copyright 2024 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "serialization.hpp" -#include "../serialization_common.hpp" - -#include "../logging.hpp" -#include "../precision.hpp" -#include "../status.hpp" -#include "tfs_utils.hpp" // @atobisze must be before lower -#include "../tensor_conversion.hpp" - -namespace ovms { - -static Status serializePrecision( - tensorflow::TensorProto& responseOutput, - const std::shared_ptr& servableOutput, - ov::Tensor& tensor) { - OVMS_PROFILE_FUNCTION(); - if (servableOutput->getOvPrecision() != tensor.get_element_type()) { - SPDLOG_ERROR("Failed to serialize tensor: {}. There is difference in precision expected:{} vs actual:{}", - servableOutput->getName(), - TensorInfo::getPrecisionAsString(servableOutput->getPrecision()), - tensor.get_element_type().get_type_name()); - return StatusCode::INTERNAL_ERROR; - } - switch (servableOutput->getPrecision()) { - case ovms::Precision::FP32: - case ovms::Precision::I32: - case ovms::Precision::FP64: - case ovms::Precision::I8: - case ovms::Precision::U8: - case ovms::Precision::I16: // 2 byte padding [v1, v0, 0, 0, u1, u0, 0, 0, ...] - case ovms::Precision::U16: - case ovms::Precision::FP16: - case ovms::Precision::I64: - case ovms::Precision::STRING: - responseOutput.set_dtype(getPrecisionAsDataType(servableOutput->getPrecision())); - break; - - case ovms::Precision::Q78: - case ovms::Precision::BIN: - case ovms::Precision::BOOL: - case ovms::Precision::MIXED: - case ovms::Precision::CUSTOM: - default: { - Status status = StatusCode::OV_UNSUPPORTED_SERIALIZATION_PRECISION; - SPDLOG_ERROR(status.string()); - return status; - } - } - return StatusCode::OK; -} - -static Status serializeShape( - tensorflow::TensorProto& responseOutput, - const std::shared_ptr& servableOutput, - ov::Tensor& tensor) { - OVMS_PROFILE_FUNCTION(); - responseOutput.mutable_tensor_shape()->Clear(); - auto& effectiveNetworkOutputShape = servableOutput->getShape(); - ov::Shape actualTensorShape = tensor.get_shape(); - if (effectiveNetworkOutputShape.size() != actualTensorShape.size()) { - SPDLOG_ERROR("Failed to serialize tensor: {}. There is difference in number of dimensions expected:{} vs actual:{}", - servableOutput->getName(), effectiveNetworkOutputShape.size(), actualTensorShape.size()); - return StatusCode::INTERNAL_ERROR; - } - for (size_t i = 0; i < effectiveNetworkOutputShape.size(); ++i) { - dimension_value_t dim = actualTensorShape[i]; - if (!effectiveNetworkOutputShape[i].match(dim)) { - SPDLOG_ERROR("Failed to serialize tensor: {}. There is difference in dimension:{} expected:{} vs actual:{}", - servableOutput->getName(), i, effectiveNetworkOutputShape[i].toString(), dim); - return StatusCode::INTERNAL_ERROR; - } - responseOutput.mutable_tensor_shape()->add_dim()->set_size(dim); - } - return StatusCode::OK; -} - -static void serializeOvTensorStringToTfProtoContent(tensorflow::TensorProto& proto, ov::Tensor& tensor) { - OVMS_PROFILE_FUNCTION(); - std::string* strings = tensor.data(); - for (size_t i = 0; i < tensor.get_shape()[0]; i++) { - proto.add_string_val(strings[i]); - } -} -Status serializeTensorToTensorProto( - tensorflow::TensorProto& responseOutput, - const std::shared_ptr& servableOutput, - ov::Tensor& tensor) { - OVMS_PROFILE_FUNCTION(); - if (servableOutput->getPostProcessingHint() == TensorInfo::ProcessingHint::STRING_2D_U8) { - return convertOVTensor2DToStringResponse(tensor, responseOutput); - } - auto status = serializePrecision(responseOutput, servableOutput, tensor); - if (!status.ok()) { - return status; - } - status = serializeShape(responseOutput, servableOutput, tensor); - if (!status.ok()) { - return status; - } - if (servableOutput->getPostProcessingHint() == TensorInfo::ProcessingHint::STRING_NATIVE) { - serializeOvTensorStringToTfProtoContent(responseOutput, tensor); - } else { - serializeContent(responseOutput.mutable_tensor_content(), tensor); - } - return StatusCode::OK; -} -template <> -tensorflow::TensorProto& ProtoGetter::createOutput(const std::string& name) { - OVMS_PROFILE_FUNCTION(); - return (*protoStorage->mutable_outputs())[name]; -} - -template <> -Status serializePredictResponse( - OutputGetter& outputGetter, - const std::string& servableName, - model_version_t servableVersion, - const tensor_map_t& outputMap, - const tensorflow::serving::PredictRequest* request, - tensorflow::serving::PredictResponse* response, - outputNameChooser_t outputNameChooser, - bool useSharedOutputContent) { // does not apply for TFS frontend - return serializePredictResponse(outputGetter, servableName, servableVersion, outputMap, response, outputNameChooser, useSharedOutputContent); -} -template Status serializePredictResponse( - OutputGetter& outputGetter, - const std::string& servableName, - model_version_t servableVersion, - const tensor_map_t& outputMap, - const TFSPredictRequest* request, - TFSPredictResponse* response, - outputNameChooser_t outputNameChooser, - bool useSharedOutputContent); - -} // namespace ovms diff --git a/src/tfs_frontend/serialization.hpp b/src/tfs_frontend/serialization.hpp deleted file mode 100644 index 3eca140279..0000000000 --- a/src/tfs_frontend/serialization.hpp +++ /dev/null @@ -1,82 +0,0 @@ -//***************************************************************************** -// Copyright 2024 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include - -#include -#include - -#include "tfs_utils.hpp" -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#pragma GCC diagnostic ignored "-Wunused-but-set-variable" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop - -#include "../profiler.hpp" -#include "../status.hpp" -#include "../serialization_common.hpp" -#include "../tensorinfo.hpp" - -namespace ovms { - -Status serializeTensorToTensorProto( - tensorflow::TensorProto& responseOutput, - const std::shared_ptr& servableOutput, - ov::Tensor& tensor); - -template -Status serializePredictResponse( - OutputGetter& outputGetter, - const std::string& servableName, - model_version_t servableVersion, - const tensor_map_t& outputMap, - tensorflow::serving::PredictResponse* response, - outputNameChooser_t outputNameChooser, - bool useSharedOutputContent = true) { // does not apply for TFS frontend - OVMS_PROFILE_FUNCTION(); - Status status; - ProtoGetter protoGetter(response); - for (const auto& [outputName, outputInfo] : outputMap) { - ov::Tensor tensor; - status = outputGetter.get(outputNameChooser(outputName, *outputInfo), tensor); - if (!status.ok()) { - return status; - } - auto& tensorProto = protoGetter.createOutput(outputInfo->getMappedName()); - status = serializeTensorToTensorProto(tensorProto, outputInfo, tensor); - if (!status.ok()) { - return status; - } - } - return status; -} - -template <> -Status serializePredictResponse( - OutputGetter& outputGetter, - const std::string& servableName, - model_version_t servableVersion, - const tensor_map_t& outputMap, - const tensorflow::serving::PredictRequest* request, - tensorflow::serving::PredictResponse* response, - outputNameChooser_t outputNameChooser, - bool useSharedOutputContent); - -} // namespace ovms diff --git a/src/tfs_frontend/tfs_backend_impl.cpp b/src/tfs_frontend/tfs_backend_impl.cpp deleted file mode 100644 index b4c4af681b..0000000000 --- a/src/tfs_frontend/tfs_backend_impl.cpp +++ /dev/null @@ -1,47 +0,0 @@ -//***************************************************************************** -// Copyright 2024 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include -#include -#include -#include -#include - -#include "tfs_request_utils.hpp" // TODO @atobisze order matters here -#include "tfs_utils.hpp" -#include "../status.hpp" -#include "../modelinstance.hpp" -#include "../deserialization_main.hpp" -#include "../inference_executor.hpp" -#include "../ovms.h" // NOLINT -#include "serialization.hpp" -#include "../serialization_common.hpp" -#include "deserialization.hpp" -#include "../deserialization_common.hpp" -#include "../requesttensorextractor.hpp" - -namespace ovms { -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-function" -template <> -OVMS_InferenceRequestCompletionCallback_t getCallback(const TFSPredictRequest* request) { - return nullptr; -} -#pragma GCC diagnostic pop -template Status infer(ModelInstance& instance, const TFSPredictRequest*, TFSPredictResponse*, std::unique_ptr&); -template class RequestTensorExtractor; -} // namespace ovms -template class ovms::RequestTensorExtractor; -template class ovms::RequestTensorExtractor; diff --git a/src/tfs_frontend/tfs_request_utils.cpp b/src/tfs_frontend/tfs_request_utils.cpp deleted file mode 100644 index 81ed03c61a..0000000000 --- a/src/tfs_frontend/tfs_request_utils.cpp +++ /dev/null @@ -1,90 +0,0 @@ -//***************************************************************************** -// Copyright 2024 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "tfs_request_utils.hpp" -#include -#include -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop -#include "../extractchoice.hpp" -#include "../requesttensorextractor.hpp" -#include "../logging.hpp" -#include "../shape.hpp" -#include "../status.hpp" - -namespace ovms { -template class RequestTensorExtractor; -template class RequestTensorExtractor; - -// Assuming the request is already validated, therefore no need to check for negative values or zeros -std::optional getRequestBatchSize(const tensorflow::serving::PredictRequest* request, const size_t batchSizeIndex) { - auto requestInputItr = request->inputs().begin(); - if (requestInputItr == request->inputs().end()) { - SPDLOG_DEBUG("Failed to get batch size of a request. Validation of request failed"); - return std::nullopt; - } - auto& requestInput = requestInputItr->second; // assuming same batch size for all inputs - - if (requestInput.tensor_shape().dim_size() < 0) { - SPDLOG_DEBUG("Failed to get batch size of a request. Input shape size cannot be a negative number. Validation of request failed"); - return std::nullopt; - } - - if (static_cast(requestInput.tensor_shape().dim_size()) < batchSizeIndex + 1) { - SPDLOG_DEBUG("Failed to get batch size of a request. Batch size index out of shape range. Validation of request failed"); - return std::nullopt; - } - return Dimension(requestInput.tensor_shape().dim(batchSizeIndex).size()); -} -// Assuming the request is already validated, therefore no need to check for negative values or zeros -std::map getRequestShapes(const tensorflow::serving::PredictRequest* request) { - std::map requestShapes; - for (auto& it : request->inputs()) { - shape_t requestShape; - std::string name = it.first; - auto& requestInput = it.second; - for (int i = 0; i < requestInput.tensor_shape().dim_size(); i++) { - requestShape.push_back(requestInput.tensor_shape().dim(i).size()); - } - requestShapes[name] = std::move(requestShape); - } - return requestShapes; -} -bool useSharedOutputContentFn(const tensorflow::serving::PredictRequest* request) { - // does not apply for TFS frontend - return false; -} - -Status RequestTensorExtractor::extract(const tensorflow::serving::PredictRequest& request, const std::string& name, const tensorflow::TensorProto** tensor, size_t* bufferId) { - if (bufferId == nullptr) { - return StatusCode::INTERNAL_ERROR; - } - auto it = request.inputs().find(name); - if (it == request.inputs().end()) { - return StatusCode::NONEXISTENT_TENSOR; - } - *tensor = &it->second; - return StatusCode::OK; -} -template <> -class RequestTensorExtractor; -template <> -class RequestTensorExtractor; -} // namespace ovms diff --git a/src/tfs_frontend/tfs_request_utils.hpp b/src/tfs_frontend/tfs_request_utils.hpp deleted file mode 100644 index 8a8d8f3607..0000000000 --- a/src/tfs_frontend/tfs_request_utils.hpp +++ /dev/null @@ -1,57 +0,0 @@ -//***************************************************************************** -// Copyright 2024 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once -#include -#include -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop -#include "../extractchoice.hpp" -#include "../requesttensorextractor.hpp" -#include "../profiler.hpp" -#include "../shape.hpp" -#include "../status.hpp" - -namespace ovms { -class InferenceRequest; - -std::optional getRequestBatchSize(const tensorflow::serving::PredictRequest* request, const size_t batchSizeIndex); -std::map getRequestShapes(const tensorflow::serving::PredictRequest* request); - -template <> -class RequestTensorExtractor { -public: - static Status extract(const tensorflow::serving::PredictRequest& request, const std::string& name, const tensorflow::TensorProto** tensor, size_t* bufferId) { - return StatusCode::NOT_IMPLEMENTED; - } -}; - -template <> -class RequestTensorExtractor { -public: - static Status extract(const tensorflow::serving::PredictRequest& request, const std::string& name, const tensorflow::TensorProto** tensor, size_t* bufferId); -}; -/** - * This is specific check required for passing KFS API related info - * which informs how response should be formatted. Therefore return value should not have an impact for - * any other frontend. - */ -bool useSharedOutputContentFn(const tensorflow::serving::PredictRequest* request); -} // namespace ovms diff --git a/src/tfs_frontend/tfs_utils.cpp b/src/tfs_frontend/tfs_utils.cpp deleted file mode 100644 index 16f67fda84..0000000000 --- a/src/tfs_frontend/tfs_utils.cpp +++ /dev/null @@ -1,212 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "tfs_utils.hpp" - -#include -#include -#include -#include -#include -#include - -#include "../logging.hpp" -#include "../profiler.hpp" -#include "../status.hpp" - -namespace ovms { -TFSDataType getPrecisionAsDataType(Precision precision) { - static std::unordered_map precisionMap{ - {Precision::FP32, TFSDataType::DT_FLOAT}, - {Precision::FP64, TFSDataType::DT_DOUBLE}, - {Precision::FP16, TFSDataType::DT_HALF}, - {Precision::I64, TFSDataType::DT_INT64}, - {Precision::I32, TFSDataType::DT_INT32}, - {Precision::I16, TFSDataType::DT_INT16}, - {Precision::I8, TFSDataType::DT_INT8}, - {Precision::U32, TFSDataType::DT_UINT32}, - {Precision::U64, TFSDataType::DT_UINT64}, - {Precision::U16, TFSDataType::DT_UINT16}, - {Precision::U8, TFSDataType::DT_UINT8}, - {Precision::STRING, TFSDataType::DT_STRING}, - // {Precision::MIXED, TFSDataType::DT_INVALID}, - // {Precision::Q78, TFSDataType::DT_INVALID}, - // {Precision::BIN, TFSDataType::DT_INVALID}, - {Precision::BOOL, TFSDataType::DT_BOOL} - // {Precision::CUSTOM, TFSDataType::DT_INVALID} - }; - auto it = precisionMap.find(precision); - if (it == precisionMap.end()) { - return TFSDataType::DT_INVALID; - } - return it->second; -} - -std::string getDataTypeAsString(TFSDataType dataType) { - switch (dataType) { - case TFSDataType::DT_FLOAT: - return "FP32"; - case TFSDataType::DT_DOUBLE: - return "FP64"; - case TFSDataType::DT_INT32: - return "I32"; - case TFSDataType::DT_INT8: - return "I8"; - case TFSDataType::DT_UINT8: - return "U8"; - case TFSDataType::DT_HALF: - return "FP16"; - case TFSDataType::DT_INT16: - return "I16"; - case TFSDataType::DT_UINT16: - return "U16"; - case TFSDataType::DT_UINT64: - return "U64"; - case TFSDataType::DT_INT64: - return "I64"; - case TFSDataType::DT_BOOL: - return "BOOL"; - case TFSDataType::DT_STRING: - return "STRING"; - default: - return "INVALID"; - } -} - -std::string tensorShapeToString(const tensorflow::TensorShapeProto& tensorShape) { - std::ostringstream oss; - oss << "("; - int i = 0; - if (tensorShape.dim_size() > 0) { - for (; i < tensorShape.dim_size() - 1; i++) { - oss << tensorShape.dim(i).size() << ","; - } - oss << tensorShape.dim(i).size(); - } - oss << ")"; - - return oss.str(); -} -Precision TFSPrecisionToOvmsPrecision(const TFSDataType& datatype) { - static std::unordered_map precisionMap{ - {TFSDataType::DT_FLOAT, Precision::FP32}, - {TFSDataType::DT_DOUBLE, Precision::FP64}, - {TFSDataType::DT_HALF, Precision::FP16}, - {TFSDataType::DT_INT64, Precision::I64}, - {TFSDataType::DT_INT32, Precision::I32}, - {TFSDataType::DT_INT16, Precision::I16}, - {TFSDataType::DT_INT8, Precision::I8}, - {TFSDataType::DT_UINT64, Precision::U64}, - {TFSDataType::DT_UINT16, Precision::U16}, - {TFSDataType::DT_UINT8, Precision::U8}, - {TFSDataType::DT_STRING, Precision::STRING}, - {TFSDataType::DT_BOOL, Precision::BOOL}}; - auto it = precisionMap.find(datatype); - if (it == precisionMap.end()) { - return Precision::UNDEFINED; - } - return it->second; -} - -Status prepareConsolidatedTensorImpl(TFSPredictResponse* response, const std::string& name, ov::element::Type_t precision, const ov::Shape& shape, char*& bufferOut, size_t size) { - OVMS_PROFILE_FUNCTION(); - tensorflow::TensorProto tensorProto; - auto [it, isInserted] = response->mutable_outputs()->insert(google::protobuf::MapPair(name, std::move(tensorProto))); - if (!isInserted) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Failed to prepare consolidated tensor, tensor with name {} already prepared", name); - return StatusCode::INTERNAL_ERROR; - } - it->second.mutable_tensor_content()->resize(size); - bufferOut = it->second.mutable_tensor_content()->data(); - return StatusCode::OK; -} -const std::string& getRequestServableName(const TFSPredictRequest& request) { - return request.model_spec().name(); -} -Status isNativeFileFormatUsed(const TFSPredictRequest& request, const std::string& name, bool& nativeFileFormatUsed) { - auto it = request.inputs().find(name); - if (it == request.inputs().end()) { - SPDLOG_DEBUG("Error during checking binary input; input: {} does not exist in request for: {}", name, getRequestServableName(request)); - return StatusCode::INTERNAL_ERROR; - } - nativeFileFormatUsed = isNativeFileFormatUsed(it->second); - return StatusCode::OK; -} - -bool isNativeFileFormatUsed(const TFSInputTensorType& proto) { - return proto.dtype() == TFSDataType::DT_STRING; - // return request.string_val_size() > 0; -} - -bool requiresPreProcessing(const TFSInputTensorType& proto) { - return proto.dtype() == tensorflow::DataType::DT_STRING; -} - -std::string& createOrGetString(TFSInputTensorType& proto, int index) { - while (proto.string_val_size() <= index) { - proto.add_string_val(); - } - return *proto.mutable_string_val(index); -} - -void setBatchSize(TFSInputTensorType& proto, int64_t batch) { - if (proto.tensor_shape().dim_size() == 0) { - proto.mutable_tensor_shape()->add_dim(); - } - proto.mutable_tensor_shape()->mutable_dim(0)->set_size(batch); -} -void setStringPrecision(TFSInputTensorType& proto) { - proto.set_dtype(TFSDataType::DT_STRING); -} -const std::string& getBinaryInput(const tensorflow::TensorProto& tensor, size_t i) { - return tensor.string_val(i); -} -int getBinaryInputsSize(const tensorflow::TensorProto& tensor) { - return tensor.string_val_size(); -} -Status validateTensor(const TensorInfo& tensorInfo, - const tensorflow::TensorProto& src, - const std::string* buffer) { - OVMS_PROFILE_FUNCTION(); - auto status = tensor_conversion::validateLayout(tensorInfo); - if (!status.ok()) { - return status; - } - // 4 for default pipelines, 5 for pipelines with demultiplication at entry - bool isShapeLengthValid = tensorInfo.getShape().size() == 4 || - (tensorInfo.isInfluencedByDemultiplexer() && tensorInfo.getShape().size() == 5); - if (!isShapeLengthValid) { - return StatusCode::INVALID_SHAPE; - } - - if (tensor_conversion::checkBatchSizeMismatch(tensorInfo, src.string_val_size())) { - SPDLOG_DEBUG("Input: {} request batch size is incorrect. Expected: {} Actual: {}", - tensorInfo.getMappedName(), - tensorInfo.getBatchSize().has_value() ? tensorInfo.getBatchSize().value().toString() : std::string{"none"}, - src.string_val_size()); - return StatusCode::INVALID_BATCH_SIZE; - } - - for (int i = 0; i < src.string_val_size(); i++) { - if (src.string_val(i).size() <= 0) { - return StatusCode::STRING_VAL_EMPTY; - } - } - return StatusCode::OK; -} -Status convertBinaryExtensionStringFromBufferToNativeOVTensor(const tensorflow::TensorProto& src, ov::Tensor& tensor, const std::string* buffer) { - return StatusCode::NOT_IMPLEMENTED; -} -} // namespace ovms diff --git a/src/tfs_frontend/tfs_utils.hpp b/src/tfs_frontend/tfs_utils.hpp deleted file mode 100644 index 559d41f155..0000000000 --- a/src/tfs_frontend/tfs_utils.hpp +++ /dev/null @@ -1,60 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include - -#pragma warning(push) -#pragma warning(disable : 4624 6001 6385 6386 6326 6011 4457 6308 6387 6246) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-but-set-variable" -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop -#pragma warning(pop) - -#include "../precision.hpp" -#include "../tensor_conversion_common.hpp" - -using TFSDataType = tensorflow::DataType; -using TFSPredictRequest = tensorflow::serving::PredictRequest; -using TFSPredictResponse = tensorflow::serving::PredictResponse; -using TFSInputTensorType = tensorflow::TensorProto; - -namespace ovms { -class Status; - -Precision TFSPrecisionToOvmsPrecision(const TFSDataType& s); -TFSDataType getPrecisionAsDataType(Precision precision); -std::string getDataTypeAsString(TFSDataType dataType); - -std::string tensorShapeToString(const tensorflow::TensorShapeProto& tensorShape); -Status prepareConsolidatedTensorImpl(TFSPredictResponse* response, const std::string& name, ov::element::Type_t precision, const ov::Shape& shape, char*& bufferOut, size_t size); -const std::string& getRequestServableName(const TFSPredictRequest& request); -Status isNativeFileFormatUsed(const TFSPredictRequest& request, const std::string& name, bool& isNativeFileFormatUsed); -bool isNativeFileFormatUsed(const TFSInputTensorType& request); -bool requiresPreProcessing(const TFSInputTensorType& proto); -std::string& createOrGetString(TFSInputTensorType& proto, int index); -void setBatchSize(TFSInputTensorType& proto, int64_t batch); -void setStringPrecision(TFSInputTensorType& proto); -const std::string& getBinaryInput(const tensorflow::TensorProto& tensor, size_t i); -int getBinaryInputsSize(const tensorflow::TensorProto& tensor); -Status validateTensor(const TensorInfo& tensorInfo, - const tensorflow::TensorProto& src, - const std::string* buffer); -Status convertBinaryExtensionStringFromBufferToNativeOVTensor(const tensorflow::TensorProto& src, ov::Tensor& tensor, const std::string* buffer); -} // namespace ovms diff --git a/src/tfs_frontend/validation.cpp b/src/tfs_frontend/validation.cpp deleted file mode 100644 index 1826d6a112..0000000000 --- a/src/tfs_frontend/validation.cpp +++ /dev/null @@ -1,269 +0,0 @@ -//***************************************************************************** -// Copyright 2024 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop -#include -#include -#include -#include -#include -#include - -#include "tfs_utils.hpp" -#include "../precision.hpp" -#include "../logging.hpp" -#include "../predict_request_validation_utils.hpp" -#include "../profiler.hpp" -#include "../tensorinfo.hpp" -#include "../status.hpp" - -namespace ovms { -namespace request_validation_utils { -using TFSRequestType = tensorflow::serving::PredictRequest; -using TFSInputTensorType = tensorflow::TensorProto; -using TFSInputTensorIteratorType = google::protobuf::Map::const_iterator; -using TFSShapeType = tensorflow::TensorShapeProto; - -template <> -dimension_value_t RequestShapeInfo::getDim(size_t i) { - return tensor.tensor_shape().dim(i).size(); -} -template <> -size_t RequestShapeInfo::getShapeSize() { - return tensor.tensor_shape().dim_size(); -} -template <> -const TFSShapeType& RequestShapeInfo::getShape() { - return tensor.tensor_shape(); -} -template <> -Status RequestValidator::validateRequestCoherency() const { - return StatusCode::OK; -} - -template <> -const std::string RequestValidator::getCurrentlyValidatedTensorName() const { - return *currentlyValidatedName; -} -template <> -const TFSInputTensorType& RequestValidator::getTensorFromIt(const TFSInputTensorIteratorType& it) const { - return it->second; -} -template <> -Status RequestValidator::validateNumberOfTensors() const { - size_t expectedNumberOfInputs = inputsInfo.size(); - for (auto& optionalAllowedInputName : optionalAllowedInputNames) { - if (request.inputs().count(optionalAllowedInputName)) - expectedNumberOfInputs++; - } - if (request.inputs_size() > 0 && expectedNumberOfInputs == static_cast(request.inputs_size())) { - return StatusCode::OK; - } - std::stringstream ss; - ss << "Expected: " << expectedNumberOfInputs << "; Actual: " << request.inputs_size(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid number of inputs - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_NO_OF_INPUTS, details); -} -template <> -Status RequestValidator::validateNumberOfBinaryInputShapeDimensions(const TFSInputTensorType& proto) const { - RequestShapeInfo rsi(proto); - if (rsi.getShapeSize() != 1) { - std::stringstream ss; - ss << "Expected number of binary input shape dimensions: 1; Actual: " << rsi.getShapeSize() << "; input name: " << getCurrentlyValidatedTensorName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid number of shape dimensions - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, details); - } - return StatusCode::OK; -} -template <> -Status RequestValidator::checkBinaryBatchSizeMismatch(const TFSInputTensorType& proto, const std::optional& servableBatchSize, Status& finalStatus, Mode batchingMode, Mode shapeMode, int32_t inputBatchSize) const { - if (!servableBatchSize.has_value()) { - std::stringstream ss; - ss << "Batch not present in input name: " << getCurrentlyValidatedTensorName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid batch size - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_BATCH_SIZE, details); - } - RequestShapeInfo rsi(proto); - if (inputBatchSize < 0) { - std::stringstream ss; - ss << "Batch size must be positive; input name: " << getCurrentlyValidatedTensorName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid batch size - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_BATCH_SIZE, details); - } - if (rsi.getDim(0) != inputBatchSize) { - SPDLOG_DEBUG("[servable name: {} version: {}] Batch size in request {} does not match actual {}", servableName, servableVersion, rsi.getDim(0), inputBatchSize); - return StatusCode::INVALID_BATCH_SIZE; - } - if (servableBatchSize.value().match(rsi.getDim(0))) { - return StatusCode::OK; - } - if (batchingMode == AUTO) { - finalStatus = StatusCode::BATCHSIZE_CHANGE_REQUIRED; - return StatusCode::OK; - } else if (shapeMode != AUTO) { - std::stringstream ss; - ss << "Expected: " << servableBatchSize.value().toString() << "; Actual: " << proto.string_val_size() << "; input name: " << getCurrentlyValidatedTensorName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid batch size - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_BATCH_SIZE, details); - } - return StatusCode::OK; -} -template <> -size_t getStringInputWidth(const tensorflow::TensorProto& src) { - size_t maxStringLength = 0; - for (const auto& str : src.string_val()) { - maxStringLength = std::max(maxStringLength, str.size()); - } - return maxStringLength + 1; -} -template <> -int64_t getStringBatchSize(const tensorflow::TensorProto& src) { - return src.string_val_size(); -} -template <> -Status RequestValidator::validateTensorContent(const TFSInputTensorType& proto, ovms::Precision expectedPrecision, size_t bufferId) const { - /* - int8 data in request.tensor_content - uint8 data in request.tensor_content - int16 data in request.tensor_content - uint16 request.tensor_content is empty, data located in request.int_val - int32 data in request.tensor_content - uint32 data in request.tensor_content - int64 data in request.tensor_content - uint64 data in request.tensor_content - float16 request.tensor_content is empty, data located in request.half_val - float32 data in request.tensor_content - double data in request.tensor_content - - _TENSOR_CONTENT_TYPES - https://github.com/tensorflow/tensorflow/blob/903a6399aab19b549fefd0ead836af644f3d00f8/tensorflow/python/framework/tensor_util.py#L237 -*/ - - // For POD types - size_t expectedValueCount = 0; - if (!request_validation_utils::computeExpectedElementCountReturnFalseIfOverflow( - proto.tensor_shape().dim().begin(), proto.tensor_shape().dim().end(), - [](const auto& d) { return d.size(); }, expectedValueCount)) { - std::stringstream ss; - ss << "Shape dimensions are too large; input name: " << getCurrentlyValidatedTensorName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid content size of tensor proto - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_CONTENT_SIZE, details); - } - - // Network expects tensor content size or value count - if (proto.dtype() == tensorflow::DataType::DT_STRING) { - if (proto.string_val_size() < 0 || proto.string_val_size() != proto.tensor_shape().dim(0).size()) { - std::stringstream ss; - ss << "Expected: " << proto.tensor_shape().dim(0).size() << "; Actual: " << proto.string_val_size() << "; input name: " << getCurrentlyValidatedTensorName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid number of values in tensor proto string container - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_VALUE_COUNT, details); - } - } else if (proto.dtype() == tensorflow::DataType::DT_UINT16) { - if (proto.int_val_size() < 0 || - expectedValueCount != static_cast(proto.int_val_size())) { - std::stringstream ss; - ss << "Expected: " << expectedValueCount << "; Actual: " << proto.int_val_size() << "; input name: " << getCurrentlyValidatedTensorName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid number of values in tensor proto container - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_VALUE_COUNT, details); - } - } else if (proto.dtype() == tensorflow::DataType::DT_HALF) { - if (proto.half_val_size() < 0 || - expectedValueCount != static_cast(proto.half_val_size())) { - std::stringstream ss; - ss << "Expected: " << expectedValueCount << "; Actual: " << proto.half_val_size() << "; input name: " << getCurrentlyValidatedTensorName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid number of values in tensor proto container - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_VALUE_COUNT, details); - } - } else { - size_t expectedContentSize = 0; - const size_t elementByteSize = ov::element::Type(ovmsPrecisionToIE2Precision(expectedPrecision)).size(); - if (!request_validation_utils::computeExpectedBufferSizeReturnFalseIfOverflow(expectedValueCount, elementByteSize, expectedContentSize)) { - std::stringstream ss; - ss << "Shape dimensions overflow size_t; input name: " << getCurrentlyValidatedTensorName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid content size of tensor proto - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_CONTENT_SIZE, details); - } - if (expectedContentSize != proto.tensor_content().size()) { - std::stringstream ss; - ss << "Expected: " << expectedContentSize << " bytes; Actual: " << proto.tensor_content().size() << " bytes; input name: " << getCurrentlyValidatedTensorName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid content size of tensor proto - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_CONTENT_SIZE, details); - } - } - return StatusCode::OK; -} -template <> -Status RequestValidator::validateNumberOfShapeDimensions(const ovms::TensorInfo& tensorInfo, const TFSInputTensorType& proto) const { - // Network and request must have the same number of shape dimensions - const auto& shape = tensorInfo.getShape(); - if (proto.tensor_shape().dim_size() < 0 || - shape.size() != static_cast(proto.tensor_shape().dim_size())) { - std::stringstream ss; - ss << "Expected: " << shape.toString() - << "; Actual: " << tensorShapeToString(proto.tensor_shape()) - << "; input name: " << getCurrentlyValidatedTensorName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid number of shape dimensions - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, details); - } - return StatusCode::OK; -} -template <> -Status RequestValidator::validatePrecision(const ovms::TensorInfo& tensorInfo, const TFSInputTensorType& proto) const { - if (proto.dtype() != getPrecisionAsDataType(tensorInfo.getPrecision())) { - std::stringstream ss; - ss << "Expected: " << tensorInfo.getPrecisionAsString() - << "; Actual: " << getDataTypeAsString(proto.dtype()) - << "; input name: " << getCurrentlyValidatedTensorName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid precision - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_PRECISION, details); - } - return StatusCode::OK; -} -template <> -bool dataInRawInputContents(const TFSRequestType& request) { - return false; -} - -template <> -const std::string* getRawInputContents(const TFSRequestType& request, size_t bufferId) { - SPDLOG_DEBUG("Raw input contents is not supported for TFS API"); - throw std::runtime_error("Raw input contents used in TFSflow."); - return nullptr; -} - -template <> -Status validate(const TFSRequestType& request, const tensor_map_t& inputsInfo, const tensor_map_t& outputsInfo, const std::string& servableName, const model_version_t servableVersion, const std::set& optionalAllowedInputNames, const Mode batchingMode, const shapes_info_map_t& shapeInfo) { - OVMS_PROFILE_FUNCTION(); - return RequestValidator(request, inputsInfo, outputsInfo, servableName, servableVersion, optionalAllowedInputNames, batchingMode, shapeInfo).validate(); -} -} // namespace request_validation_utils -} // namespace ovms