GPU coder linking error using cmake on the Jetson nano

Hi, I followed the reference tutorial of the MATLAB:

https://ww2.mathworks.cn/help/gpucoder/examples/deployment-classification-webcam-images-on-NVIDIA-Jetson-TX2.html

to deploy the generated GPU code about the deep learning network on my Jetson nano Target

I want to use the cmake to build the program in the target directly because we installed a new-version opencv . I got the linking error as follows:

CMakeFiles/main.dir/main_generated_DeepLearningNetwork.cu.o: In function `__sti____cudaRegisterAll()':
/tmp/tmpxft_00006eb6_00000000-5_DeepLearningNetwork.cudafe1.stub.c:14: undefined reference to `__cudaRegisterLinkedBinary_54_tmpxft_00006eb6_00000000_6_DeepLearningNetwork_cpp1_ii_3c015049'
CMakeFiles/main.dir/main_generated_MWCNNLayerImpl.cu.o: In function `__sti____cudaRegisterAll()':
/tmp/tmpxft_00006ee8_00000000-5_MWCNNLayerImpl.cudafe1.stub.c:2: undefined reference to `__cudaRegisterLinkedBinary_49_tmpxft_00006ee8_00000000_6_MWCNNLayerImpl_cpp1_ii_74f5c547'
CMakeFiles/main.dir/main_generated_MWConvLayerImpl.cu.o: In function `__sti____cudaRegisterAll()':
/tmp/tmpxft_00006f22_00000000-5_MWConvLayerImpl.cudafe1.stub.c:14: undefined reference to `__cudaRegisterLinkedBinary_50_tmpxft_00006f22_00000000_6_MWConvLayerImpl_cpp1_ii_050be33a'
CMakeFiles/main.dir/main_generated_MWCudaDimUtility.cu.o: In function `__sti____cudaRegisterAll()':
/tmp/tmpxft_00006f57_00000000-5_MWCudaDimUtility.cudafe1.stub.c:14: undefined reference to `__cudaRegisterLinkedBinary_51_tmpxft_00006f57_00000000_6_MWCudaDimUtility_cpp1_ii_dd421695'
CMakeFiles/main.dir/main_generated_MWElementwiseAffineLayerImpl.cu.o: In function `__sti____cudaRegisterAll()':
/tmp/tmpxft_00006f87_00000000-5_MWElementwiseAffineLayerImpl.cudafe1.stub.c:14: undefined reference to `__cudaRegisterLinkedBinary_63_tmpxft_00006f87_00000000_6_MWElementwiseAffineLayerImpl_cpp1_ii_af034ff7'
CMakeFiles/main.dir/main_generated_MWElementwiseAffineLayerImplKernel.cu.o: In function `__sti____cudaRegisterAll()':
/tmp/tmpxft_00006fb7_00000000-5_MWElementwiseAffineLayerImplKernel.cudafe1.stub.c:2: undefined reference to `__cudaRegisterLinkedBinary_69_tmpxft_00006fb7_00000000_6_MWElementwiseAffineLayerImplKernel_cpp1_ii_3f82fa1c'
CMakeFiles/main.dir/main_generated_MWFusedConvReLULayerImpl.cu.o: In function `__sti____cudaRegisterAll()':
/tmp/tmpxft_00006fe6_00000000-5_MWFusedConvReLULayerImpl.cudafe1.stub.c:14: undefined reference to `__cudaRegisterLinkedBinary_59_tmpxft_00006fe6_00000000_6_MWFusedConvReLULayerImpl_cpp1_ii_5beca28b'
CMakeFiles/main.dir/main_generated_MWTargetNetworkImpl.cu.o: In function `__sti____cudaRegisterAll()':
/tmp/tmpxft_0000701a_00000000-5_MWTargetNetworkImpl.cudafe1.stub.c:14: undefined reference to `__cudaRegisterLinkedBinary_54_tmpxft_0000701a_00000000_6_MWTargetNetworkImpl_cpp1_ii_d94959f5'
CMakeFiles/main.dir/main_generated_main_resnet50.cu.o: In function `__sti____cudaRegisterAll()':
/tmp/tmpxft_0000704b_00000000-5_main_resnet50.cudafe1.stub.c:14: undefined reference to `__cudaRegisterLinkedBinary_48_tmpxft_0000704b_00000000_6_main_resnet50_cpp1_ii_642e5a7f'
CMakeFiles/main.dir/main_generated_predict.cu.o: In function `__sti____cudaRegisterAll()':
/tmp/tmpxft_000070b2_00000000-5_predict.cudafe1.stub.c:2: undefined reference to `__cudaRegisterLinkedBinary_42_tmpxft_000070b2_00000000_6_predict_cpp1_ii_d9cb4df2'
CMakeFiles/main.dir/main_generated_resnet50_wrapper.cu.o: In function `__sti____cudaRegisterAll()':
/tmp/tmpxft_000070e5_00000000-5_resnet50_wrapper.cudafe1.stub.c:14: undefined reference to `__cudaRegisterLinkedBinary_51_tmpxft_000070e5_00000000_6_resnet50_wrapper_cpp1_ii_47a35ee0'
CMakeFiles/main.dir/main_generated_resnet50_wrapper_data.cu.o: In function `__sti____cudaRegisterAll()':
/tmp/tmpxft_00007114_00000000-5_resnet50_wrapper_data.cudafe1.stub.c:14: undefined reference to `__cudaRegisterLinkedBinary_56_tmpxft_00007114_00000000_6_resnet50_wrapper_data_cpp1_ii_212c154c'
CMakeFiles/main.dir/main_generated_resnet50_wrapper_initialize.cu.o: In function `__sti____cudaRegisterAll()':
/tmp/tmpxft_00007147_00000000-5_resnet50_wrapper_initialize.cudafe1.stub.c:14: undefined reference to `__cudaRegisterLinkedBinary_62_tmpxft_00007147_00000000_6_resnet50_wrapper_initialize_cpp1_ii_82f5913d'
CMakeFiles/main.dir/main_generated_resnet50_wrapper_rtwutil.cu.o: In function `__sti____cudaRegisterAll()':
/tmp/tmpxft_0000717e_00000000-5_resnet50_wrapper_rtwutil.cudafe1.stub.c:14: undefined reference to `__cudaRegisterLinkedBinary_59_tmpxft_0000717e_00000000_6_resnet50_wrapper_rtwutil_cpp1_ii_aa3d490c'
CMakeFiles/main.dir/main_generated_resnet50_wrapper_terminate.cu.o: In function `__sti____cudaRegisterAll()':
/tmp/tmpxft_00006e86_00000000-5_resnet50_wrapper_terminate.cudafe1.stub.c:14: undefined reference to `__cudaRegisterLinkedBinary_61_tmpxft_00006e86_00000000_6_resnet50_wrapper_terminate_cpp1_ii_5d082191'
collect2: error: ld returned 1 exit status
CMakeFiles/main.dir/build.make:4187: recipe for target 'main' failed
make[2]: *** [main] Error 1
CMakeFiles/Makefile2:75: recipe for target 'CMakeFiles/main.dir/all' failed
make[1]: *** [CMakeFiles/main.dir/all] Error 2
Makefile:83: recipe for target 'all' failed
make: *** [all] Error 2

CMake Error at main_generated_predict.cu.o.cmake:280 (message):
  Error generating file
  /home/zqax/remoteBuildDir333/MATLAB_ws/R2020a/D/MATLAB/nvidia_gettingstarted/codegen/exe/resnet50_wrapper/CMakeFiles/main.dir/././main_generated_predict.cu.o
make[2]: *** [CMakeFiles/main.dir/main_generated_predict.cu.o] Error 1
CMakeFiles/Makefile2:75: recipe for target 'CMakeFiles/main.dir/all' failed
make[1]: *** [CMakeFiles/main.dir/all] Error 2
Makefile:83: recipe for target 'all' failed
make: *** [all] Error 2

Here is my cmakelist.txt, I put all files in the same folder:

# cmake needs this line
cmake_minimum_required(VERSION 3.1)

# Enable C++11
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_STANDARD_REQUIRED TRUE)


project(Test)

set(CUDA_USE_STATIC_CUDA_RUNTIME ON)

set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR})


find_package(CUDA REQUIRED)
message(STATUS "CUDA library status:")
message(STATUS "    version: ${CUDA_VERSION}")
message(STATUS "    include path: ${CUDA_INCLUDE_DIRS}")
message(STATUS "    libraries: ${CUDA_LIBRARIES}")
set(CUDA_NVCC_FLAGS -O3;--relocatable-device-code=true; -gencode arch=compute_53,code=sm_53; -x cu) # nvcc flags
include_directories(${CUDA_INCLUDE_DIRS})
include_directories(/usr/local/cuda/samples/common/inc)
include_directories(/usr/lib/aarch64-linux-gnu)
include_directories(/usr/local/cuda/lib64)

find_package(OpenCV REQUIRED)
message(STATUS "OpenCV library status:")
message(STATUS "    config: ${OpenCV_DIR}")
message(STATUS "    version: ${OpenCV_VERSION}")
message(STATUS "    libraries: ${OpenCV_LIBS}")
message(STATUS "    include path: ${OpenCV_INCLUDE_DIRS}")

AUX_SOURCE_DIRECTORY(. SRC_LIST)
MESSAGE(STATUS "The files need to be built: ${SRC_LIST} \n" )

#cuda_add_library(lib ${SRC_LIST})  
CUDA_ADD_EXECUTABLE(main ${SRC_LIST})
target_link_libraries(main ${OpenCV_LIBS} ${CUDA_LIBRARIES} cudnn cublas)

Thanks!

Yuhao

[Yuhao Sun]

Another question is how to add the cudnn or tensorRT into cmake files?

Answer 1

[Chao Luo]

See the answer in this link.

https://www.mathworks.com/matlabcentral/answers/520208-using-cmake-i-got-a-ploblem-compile-made-by-gpu-coder-static-library-a#answer_427970

[Yuhao Sun]

It seems like the coder in MATLAB for Jetson didn't support to build the dynamic library.

NVIDIA Jetson supports LIB (C/C++ Static Library) or EXE (C/C++ Executable) builds only. Set

code generation configuration to LIB or EXE.

On the other hand, I don't know how to configure the cmake to link the libraries using nvcc.

Answer 2

[Ramakrishna Mandalapu]

Hi Yuhao,

If you want to use the custom OpenCV installed in the different location on the target then you can use the "coder.updateBuildInfo" API to link that.

For example if the OpenCV is installed at "/home/ubuntu/opencv_for_tegra/install" then you can use the above API to use the OpenCV in this folder like below. The below lines of code should be added in the design function at the begining (e.g resnet50_wrapper).

function out = resnet50_wrapper(im) %#codegen
% Wrapper function to call ResNet50 predict function.

%   Copyright 2019 The MathWorks, Inc.   

flags = '-lopencv_dnn -lopencv_ml -lopencv_objdetect -lopencv_shape -lopencv_stitching -lopencv_superres -lopencv_videostab -lopencv_calib3d -lopencv_features2d -lopencv_highgui -lopencv_videoio -lopencv_imgcodecs -lopencv_video -lopencv_photo -lopencv_imgproc -lopencv_flann -lopencv_core';
includePath = '/home/ubuntu/opencv_for_tegra/install/include';
opencv_link_flags = ['-L/home/ubuntu/opencv_for_tegra/install/lib ' flags];
coder.updateBuildInfo('addLinkFlags',opencv_link_flags);
coder.updateBuildInfo('addIncludePaths',includePath);

%
% Some code
%
%
end

In the above line,

includePath - specifies the location of the includes of the custom opencv

opencv_link_flags - Specifies the location of the shared libs and the required libs to use.

Modify the above custom OpenCV locations according to your need and try generating the code (using codegen) then it will generate a makefile which contains the above OpenCV information for building the code.

On the board, you need to export the OpenCV shared libraries path (like below) to successfully run the executable.

export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/ubuntu/opencv_for_tegra/install/lib

If you use the above workflow then you don't need use the CMake for building. The codegen will take care of generating the appropriate Makefile and building the code on the target. You just need to specify which DeepLearning library to use (cuDNN or TensorRT) in the coder config.

Thanks,

Ramakrishna

[Yuhao Sun]

it still doesn't work. The building logs shows the adding library path is wrong as below:

nvcc -rdc=true -Xcudafe "--diag_suppress=unsigned_compare_with_zero" -c -Xcompiler -MMD,-MP -O2 -arch sm_35 -DMW_CUDA_ARCH=350 -D__MW_TARGET_USE_HARDWARE_RESOURCES_H__ -DARM_PROJECT -D_USE_TARGET_UDP_ -D_RUNONTARGETHARDWARE_BUILD_ -DSTACK_SIZE=200000 -DMODEL=resnet50_wrapper -DMW_DL_DATA_PATH="/home/zqax/remoteBuildDir555/MATLAB_ws/R2020a/D/MATLAB/nvidia_gettingstarted/codegen/exe/resnet50_wrapper" -DMW_SCHED_OTHER=1 -DARM_PROJECT -D_USE_TARGET_UDP_ -D_RUNONTARGETHARDWARE_BUILD_ -DSTACK_SIZE=200000 -DMODEL=resnet50_wrapper -I/home/zqax/remoteBuildDir555/MATLAB_ws/R2020a/D/MATLAB/nvidia_gettingstarted/codegen/exe/resnet50_wrapper -I/home/zqax/remoteBuildDir555/MATLAB_ws/R2020a/D/MATLAB/nvidia_gettingstarted -I/home/zqax/remoteBuildDir555/MATLAB_ws/R2020a/usr/local/include/opencv4 -I/home/zqax/remoteBuildDir555/MATLAB_ws/R2020a/C/ProgramData/MATLAB/SupportPackages/R2020a/toolbox/target/supportpackages/nvidia/include -I/home/zqax/remoteBuildDir555/MATLAB_ws/R2020a/D/Program_Files/polyspace/R2020a/extern/include -o main_resnet50.o /home/zqax/remoteBuildDir555/MATLAB_ws/R2020a/D/MATLAB/nvidia_gettingstarted/main_resnet50.cu

resnet50_wrapper_rtw.mk:398: recipe for target 'main_resnet50.o' failed

make: Leaving directory '/home/zqax/remoteBuildDir555/MATLAB_ws/R2020a/D/MATLAB/nvidia_gettingstarted/codegen/exe/resnet50_wrapper'

Why the added path still contains the local path of the generated files in the target? How to remove that?

[Yuhao Sun]

I created a symbolic link for the library path in the local path and finally build the program successfully. But it is so werid for adding extra library. By the way, I got the error when the building program running:

./resnet50_wrapper.elf

[ WARN:0] global /home/nvidia/host/build_opencv/nv_opencv/modules/videoio/src/cap_gstreamer.cpp (1757) handleMessage OpenCV | GStreamer warning: Embedded video playback halted; module v4l2src0 reported: Cannot identify device '/dev/video1'.

[ WARN:0] global /home/nvidia/host/build_opencv/nv_opencv/modules/videoio/src/cap_gstreamer.cpp (886) open OpenCV | GStreamer warning: unable to start pipeline

[ WARN:0] global /home/nvidia/host/build_opencv/nv_opencv/modules/videoio/src/cap_gstreamer.cpp (480) isPipelinePlaying OpenCV | GStreamer warning: GStreamer: pipeline have not been created

VIDEOIO ERROR: V4L: can't open camera by index 1

can't open camera

It seems like the new_version opencv cannot drive the Gstramer. OK, I give it up. I will use the given opencv version in the JetPack.

[Ramakrishna Mandalapu]

Hi Yuhao,

Please check webcam availability on the target, in the main file provided with the example we use camera device 1 (/dev/video1). The camera may be mising on the board.

$ ls /dev/video*

default camera (/dev/video0) can't be used since it uses different interface.

-Ramakrishna