Skip to content

Commit bd18612

Browse files
ChSonnabendalibuild
ChSonnabend
and
alibuild
authored
[NN CF]: Updates and bug-fixes (#15571)
* Adding support for int8 inputs * Reverting int8 changes as int8 computations are taken care of in the ONNX graph due to requirement of additional scaling * Bug-fix * Please consider the following formatting changes * Changing memory layout usage * Adjusting launch bounds * Adding back some helpful comments * Please consider the following formatting changes --------- Co-authored-by: ALICE Action Bot <alibuild@cern.ch>
1 parent 3b04d6e commit bd18612

11 files changed

Lines changed: 157 additions & 129 deletions

File tree

‎Common/ML/src/OrtInterface.cxx‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,9 @@ void OrtModel::initEnvironment()
140140

141141
void OrtModel::initSessionFromBuffer(const char* buffer, size_t bufferSize)
142142
{
143+
if (mAllocateDeviceMemory) {
144+
memoryOnDevice(mDeviceId);
145+
}
143146
mPImplOrt->sessionOptions.AddConfigEntry("session.load_model_format", "ONNX");
144147
mPImplOrt->sessionOptions.AddConfigEntry("session.use_ort_model_bytes_directly", "1");
145148

‎GPU/GPUTracking/Base/cuda/GPUReconstructionCUDA.cu‎

Lines changed: 28 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -631,34 +631,47 @@ void GPUReconstructionCUDA::loadKernelModules(bool perKernel)
631631
} \
632632
}
633633

634-
void GPUReconstructionCUDA::SetONNXGPUStream(Ort::SessionOptions& session_options, int32_t stream, int32_t* deviceId)
634+
void GPUReconstructionCUDA::SetONNXGPUStream(Ort::SessionOptions& sessionOptions, int32_t stream, int32_t* deviceId)
635635
{
636636
GPUChkErr(cudaGetDevice(deviceId));
637+
637638
#if !defined(__HIPCC__) && defined(ORT_CUDA_BUILD)
638639
const OrtApi* api = OrtGetApiBase()->GetApi(ORT_API_VERSION);
639-
OrtCUDAProviderOptionsV2* cuda_options = nullptr;
640-
ORTCHK(api->CreateCUDAProviderOptions(&cuda_options));
641640

641+
#ifdef ORT_TENSORRT_BUILD
642+
OrtTensorRTProviderOptionsV2* trtOptions = nullptr;
643+
ORTCHK(api->CreateTensorRTProviderOptions(&trtOptions));
644+
645+
const std::string device = std::to_string(*deviceId);
646+
const char* keys[] = {"device_id", "trt_int8_enable"};
647+
const char* values[] = {device.c_str(), "1"};
648+
649+
ORTCHK(api->UpdateTensorRTProviderOptions(trtOptions, keys, values, sizeof(keys) / sizeof(keys[0])));
650+
ORTCHK(api->UpdateTensorRTProviderOptionsWithValue(trtOptions, "user_compute_stream", mInternals->Streams[stream]));
651+
ORTCHK(api->SessionOptionsAppendExecutionProvider_TensorRT_V2(sessionOptions, trtOptions)); // Register TensorRT first: it consequently has higher priority.
652+
api->ReleaseTensorRTProviderOptions(trtOptions);
653+
#endif
654+
655+
// CUDA is the fallback for nodes unsupported by TensorRT.
656+
OrtCUDAProviderOptionsV2* cudaOptions = nullptr;
657+
ORTCHK(api->CreateCUDAProviderOptions(&cudaOptions));
642658
// std::vector<const char*> keys{"device_id", "gpu_mem_limit", "arena_extend_strategy", "cudnn_conv_algo_search", "do_copy_in_default_stream", "cudnn_conv_use_max_workspace", "cudnn_conv1d_pad_to_nc1d"};
643659
// std::vector<const char*> values{"0", "2147483648", "kSameAsRequested", "DEFAULT", "1", "1", "1"};
644660
// UpdateCUDAProviderOptions(cuda_options, keys.data(), values.data(), keys.size());
661+
ORTCHK(api->UpdateCUDAProviderOptionsWithValue(cudaOptions, "user_compute_stream", mInternals->Streams[stream]));
662+
ORTCHK(api->SessionOptionsAppendExecutionProvider_CUDA_V2(sessionOptions, cudaOptions));
663+
api->ReleaseCUDAProviderOptions(cudaOptions);
645664

646-
// this implicitly sets "has_user_compute_stream"
647-
ORTCHK(api->UpdateCUDAProviderOptionsWithValue(cuda_options, "user_compute_stream", mInternals->Streams[stream]));
648-
ORTCHK(api->SessionOptionsAppendExecutionProvider_CUDA_V2(session_options, cuda_options));
649-
650-
// Finally, don't forget to release the provider options
651-
api->ReleaseCUDAProviderOptions(cuda_options);
652665
#elif defined(ORT_ROCM_BUILD)
653666
// const auto& api = Ort::GetApi();
654667
// api.GetCurrentGpuDeviceId(deviceId);
655-
OrtROCMProviderOptions rocm_options;
656-
rocm_options.has_user_compute_stream = 1; // Indicate that we are passing a user stream
657-
rocm_options.arena_extend_strategy = 0; // kNextPowerOfTwo = 0, kSameAsRequested = 1 -> https://github.com/search?q=repo%3Amicrosoft%2Fonnxruntime%20kSameAsRequested&type=code
668+
OrtROCMProviderOptions rocmOptions;
669+
rocmOptions.has_user_compute_stream = 1; // Indicate that we are passing a user stream
670+
rocmOptions.arena_extend_strategy = 0; // kNextPowerOfTwo = 0, kSameAsRequested = 1 -> https://github.com/search?q=repo%3Amicrosoft%2Fonnxruntime%20kSameAsRequested&type=code
658671
// rocm_options.gpu_mem_limit = 1073741824; // 0 means no limit
659-
rocm_options.user_compute_stream = mInternals->Streams[stream];
660-
session_options.AppendExecutionProvider_ROCM(rocm_options);
661-
#endif // ORT_ROCM_BUILD
672+
rocmOptions.user_compute_stream = mInternals->Streams[stream];
673+
sessionOptions.AppendExecutionProvider_ROCM(rocmOptions);
674+
#endif
662675
}
663676

664677
#ifndef __HIPCC__ // CUDA

‎GPU/GPUTracking/Definitions/GPUSettingsList.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -300,6 +300,7 @@ AddOption(nnCCDBClassificationLayerType, std::string, "FC", "", 0, "Distinguishe
300300
AddOption(nnCCDBRegressionLayerType, std::string, "FC", "", 0, "Distinguishes between network with different layer types. Options: FC, CNN")
301301
AddOption(nnCCDBBeamType, std::string, "pp", "", 0, "Distinguishes between networks trained for different beam types. Options: pp, pPb, PbPb")
302302
AddOption(nnCCDBInteractionRate, std::string, "500", "", 0, "Distinguishes between networks for different interaction rates [kHz].")
303+
AddOption(nnCCDBExtraMetadata, std::string, "", "", 0, "Extra metadata to distinguish between networks, e.g. for different internal datatypes, etc.")
303304
AddHelp("help", 'h')
304305
EndConfig()
305306

‎GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ GPUMemClean16,"[""GPUCA_THREAD_COUNT_DEFAULT"", 1]",,,,,,,,,,,,,,,
7676
GPUitoa,"[""GPUCA_THREAD_COUNT_DEFAULT"", 1]",,,,,,,,,,,,,,,
7777
GPUTPCCFNoiseSuppression_noiseSuppression,"""GPUCA_LB_GPUTPCCFNoiseSuppression""",,,,,,,,,,,,,,,448
7878
GPUTPCCFNoiseSuppression_updatePeaks,"""GPUCA_LB_GPUTPCCFNoiseSuppression""",,,,,,,,,,,,,,,448
79-
GPUTPCNNClusterizerKernels_runCfClusterizer,"""GPUCA_LB_GPUTPCNNClusterizerKernels""",,,,,,,,,,,,,,,
79+
GPUTPCNNClusterizerKernels_runCfClusterizer,"""GPUCA_LB_GPUTPCCFClusterizer""",,,,,,,,,,,,,,,
8080
GPUTPCNNClusterizerKernels_fillInputNNCPU,"""GPUCA_LB_GPUTPCNNClusterizerKernels""",,,,,,,,,,,,,,,
8181
GPUTPCNNClusterizerKernels_fillInputNNGPU,1024,,,,,,,,,,,,,,,
8282
GPUTPCNNClusterizerKernels_determineClass1Labels,"""GPUCA_LB_GPUTPCNNClusterizerKernels""",,,,,,,,,,,,,,,

‎GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx‎

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1269,15 +1269,15 @@ int32_t GPUChainTracking::RunTPCClusterizer(bool synchronizeOutput)
12691269
if(GetProcessingSettings().debugLevel >= 1 && (doGPU || lane < 4)) { nnTimers[3*lane]->Start(); }
12701270
if (clustererNNShadow.mNnInferenceInputDType == 0) {
12711271
if (clustererNNShadow.mNnInferenceOutputDType == 0) {
1272-
(nnApplication.mModelClass).inference(clustererNNShadow.mInputData_16, iSize, clustererNNShadow.mModelProbabilities_16);
1273-
} else if (clustererNNShadow.mNnInferenceOutputDType == 1) {
1274-
(nnApplication.mModelClass).inference(clustererNNShadow.mInputData_16, iSize, clustererNNShadow.mModelProbabilities_32);
1272+
(nnApplication.mModelClass).inference(clustererNNShadow.mInputData_32, iSize, clustererNNShadow.mModelProbabilities_32);
1273+
} else {
1274+
(nnApplication.mModelClass).inference(clustererNNShadow.mInputData_32, iSize, clustererNNShadow.mModelProbabilities_16);
12751275
}
12761276
} else if (clustererNNShadow.mNnInferenceInputDType == 1) {
12771277
if (clustererNNShadow.mNnInferenceOutputDType == 0) {
1278-
(nnApplication.mModelClass).inference(clustererNNShadow.mInputData_32, iSize, clustererNNShadow.mModelProbabilities_16);
1279-
} else if (clustererNNShadow.mNnInferenceOutputDType == 1) {
1280-
(nnApplication.mModelClass).inference(clustererNNShadow.mInputData_32, iSize, clustererNNShadow.mModelProbabilities_32);
1278+
(nnApplication.mModelClass).inference(clustererNNShadow.mInputData_16, iSize, clustererNNShadow.mModelProbabilities_32);
1279+
} else {
1280+
(nnApplication.mModelClass).inference(clustererNNShadow.mInputData_16, iSize, clustererNNShadow.mModelProbabilities_16);
12811281
}
12821282
}
12831283
if(GetProcessingSettings().debugLevel >= 1 && (doGPU || lane < 4)) { nnTimers[3*lane]->Stop(); } // doGPU || lane<4 -> only for GPU or first 4 CPU lanes (to limit number of concurrent timers). At least gives some statistics for CPU time...
@@ -1289,31 +1289,31 @@ int32_t GPUChainTracking::RunTPCClusterizer(bool synchronizeOutput)
12891289
if(GetProcessingSettings().debugLevel >= 1 && (doGPU || lane < 4)) { nnTimers[3*lane + 1]->Start(); }
12901290
if (clustererNNShadow.mNnInferenceInputDType == 0) {
12911291
if (clustererNNShadow.mNnInferenceOutputDType == 0) {
1292-
(nnApplication.mModelReg1).inference(clustererNNShadow.mInputData_16, iSize, clustererNNShadow.mOutputDataReg1_16);
1293-
} else if (clustererNNShadow.mNnInferenceOutputDType == 1) {
1294-
(nnApplication.mModelReg1).inference(clustererNNShadow.mInputData_16, iSize, clustererNNShadow.mOutputDataReg1_32);
1292+
(nnApplication.mModelReg1).inference(clustererNNShadow.mInputData_32, iSize, clustererNNShadow.mOutputDataReg1_32);
1293+
} else {
1294+
(nnApplication.mModelReg1).inference(clustererNNShadow.mInputData_32, iSize, clustererNNShadow.mOutputDataReg1_16);
12951295
}
1296-
} else if (clustererNNShadow.mNnInferenceInputDType == 1) {
1296+
} else {
12971297
if (clustererNNShadow.mNnInferenceOutputDType == 0) {
1298-
(nnApplication.mModelReg1).inference(clustererNNShadow.mInputData_32, iSize, clustererNNShadow.mOutputDataReg1_16);
1299-
} else if (clustererNNShadow.mNnInferenceOutputDType == 1) {
1300-
(nnApplication.mModelReg1).inference(clustererNNShadow.mInputData_32, iSize, clustererNNShadow.mOutputDataReg1_32);
1298+
(nnApplication.mModelReg1).inference(clustererNNShadow.mInputData_16, iSize, clustererNNShadow.mOutputDataReg1_32);
1299+
} else {
1300+
(nnApplication.mModelReg1).inference(clustererNNShadow.mInputData_16, iSize, clustererNNShadow.mOutputDataReg1_16);
13011301
}
13021302
}
13031303
if(GetProcessingSettings().debugLevel >= 1 && (doGPU || lane < 4)) { nnTimers[3*lane + 1]->Stop(); }
13041304
if (nnApplication.mModelClass.getNumOutputNodes()[0][1] > 1 && nnApplication.mModelReg2.isInitialized()) {
13051305
if(GetProcessingSettings().debugLevel >= 1 && (doGPU || lane < 4)) { nnTimers[3*lane + 2]->Start(); }
13061306
if (clustererNNShadow.mNnInferenceInputDType == 0) {
13071307
if (clustererNNShadow.mNnInferenceOutputDType == 0) {
1308-
(nnApplication.mModelReg2).inference(clustererNNShadow.mInputData_16, iSize, clustererNNShadow.mOutputDataReg2_16);
1309-
} else if (clustererNNShadow.mNnInferenceOutputDType == 1) {
1310-
(nnApplication.mModelReg2).inference(clustererNNShadow.mInputData_16, iSize, clustererNNShadow.mOutputDataReg2_32);
1308+
(nnApplication.mModelReg2).inference(clustererNNShadow.mInputData_32, iSize, clustererNNShadow.mOutputDataReg2_32);
1309+
} else {
1310+
(nnApplication.mModelReg2).inference(clustererNNShadow.mInputData_32, iSize, clustererNNShadow.mOutputDataReg2_16);
13111311
}
13121312
} else if (clustererNNShadow.mNnInferenceInputDType == 1) {
13131313
if (clustererNNShadow.mNnInferenceOutputDType == 0) {
1314-
(nnApplication.mModelReg2).inference(clustererNNShadow.mInputData_32, iSize, clustererNNShadow.mOutputDataReg2_16);
1315-
} else if (clustererNNShadow.mNnInferenceOutputDType == 1) {
1316-
(nnApplication.mModelReg2).inference(clustererNNShadow.mInputData_32, iSize, clustererNNShadow.mOutputDataReg2_32);
1314+
(nnApplication.mModelReg2).inference(clustererNNShadow.mInputData_16, iSize, clustererNNShadow.mOutputDataReg2_32);
1315+
} else {
1316+
(nnApplication.mModelReg2).inference(clustererNNShadow.mInputData_16, iSize, clustererNNShadow.mOutputDataReg2_16);
13171317
}
13181318
}
13191319
if(GetProcessingSettings().debugLevel >= 1 && (doGPU || lane < 4)) { nnTimers[3*lane + 2]->Stop(); }

0 commit comments

Comments
 (0)