Merge pull request opencv#21036 from fengyuentau:timvx_backend_support

dnn: TIM-VX NPU backend support * Add TimVX NPU backend for DNN module. * use official branch from tim-vx repo; fix detecting viv sdk Co-authored-by: fytao <[email protected]>
MengqingCao · Mar 31, 2022 · 7b582b7 · 7b582b7
1 parent 9390c56
commit 7b582b7
Show file tree

Hide file tree

Showing 37 changed files with 2,982 additions and 30 deletions.
diff --git a/3rdparty/libtim-vx/tim-vx.cmake b/3rdparty/libtim-vx/tim-vx.cmake
@@ -0,0 +1,73 @@
+set(TIMVX_COMMIT_HASH "1d9c7ab941b3d8d9c4d28d80058402725731e3d6")
+set(OCV_TIMVX_DIR "${OpenCV_BINARY_DIR}/3rdparty/libtim-vx")
+set(OCV_TIMVX_SOURCE_PATH "${OCV_TIMVX_DIR}/TIM-VX-${TIMVX_COMMIT_HASH}")
+
+# Download TIM-VX source code
+if(EXISTS "${OCV_TIMVX_SOURCE_PATH}")
+    message(STATUS "TIM-VX: Use cache of TIM-VX source code at ${OCV_TIMVX_SOURCE_PATH}")
+    set(TIMVX_FOUND ON)
+else()
+    set(OCV_TIMVX_FILENAME "${TIMVX_COMMIT_HASH}.zip")
+    set(OCV_TIMVX_URL "https://github.com/VeriSilicon/TIM-VX/archive/")
+    set(timvx_zip_md5sum 92619cc4498014ac7a09834d5e33ebd5)
+
+    ocv_download(FILENAME ${OCV_TIMVX_FILENAME}
+                 HASH ${timvx_zip_md5sum}
+                 URL "${OCV_TIMVX_URL}"
+                 DESTINATION_DIR "${OCV_TIMVX_DIR}"
+                 ID "TIM-VX"
+                 STATUS res
+                 UNPACK RELATIVE_URL)
+    if(res)
+        set(TIMVX_FOUND ON)
+        message(STATUS "TIM-VX: Source code downloaded at ${OCV_TIMVX_SOURCE_PATH}.") 
+    else()
+        set(TIMVX_FOUND OFF)
+        message(STATUS "TIM-VX: Failed to download source code from github. Turning off TIMVX_FOUND")
+        return()
+    endif()
+endif()
+
+# set VIVANTE SDK especially for x86_64 which comes along with TIM-VX source code
+if(CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64)
+    set(VIVANTE_SDK_DIR "${OCV_TIMVX_SOURCE_PATH}/prebuilt-sdk/x86_64_linux")
+    message(STATUS "TIM-VX: Build from source using prebuilt x86_64 VIVANTE SDK.")
+endif()
+
+# Verify if requested VIVANTE SDK libraries are all found
+find_vivante_sdk_libs(missing ${VIVANTE_SDK_DIR})
+if(missing)
+    message(STATUS "TIM-VX: Failed to find ${missing} in ${VIVANTE_SDK_DIR}/lib. Turning off TIMVX_VIV_FOUND")
+    set(TIMVX_VIV_FOUND OFF)
+else()
+    message(STATUS "TIM-VX: dependent VIVANTE SDK libraries are found at ${VIVANTE_SDK_DIR}/lib.")
+    set(TIMVX_VIV_FOUND ON)
+endif()
+
+if(TIMVX_VIV_FOUND)
+    # vars used by TIM-VX CMake scripts
+    set(EXTERNAL_VIV_SDK "${VIVANTE_SDK_DIR}" CACHE INTERNAL "" FORCE)
+    set(VIV_SDK_DRIVER_PREFIX "lib" CACHE INTERNAL "" FORCE)
+endif()
+
+if(TIMVX_FOUND AND TIMVX_VIV_FOUND)
+    set(BUILD_TIMVX ON)
+else()
+    return()
+endif()
+
+if(BUILD_TIMVX)
+    set(HAVE_TIMVX 1)
+
+    ocv_warnings_disable(CMAKE_C_FLAGS -Wunused-parameter -Wstrict-prototypes -Wundef -Wsign-compare -Wmissing-prototypes -Wmissing-declarations -Wstrict-aliasing -Wunused-but-set-variable -Wmaybe-uninitialized -Wshadow -Wsuggest-override -Wswitch)
+    ocv_warnings_disable(CMAKE_CXX_FLAGS -Wunused-parameter -Wstrict-prototypes -Wundef -Wsign-compare -Wunused-but-set-variable -Wshadow -Wsuggest-override -Wmissing-declarations -Wswitch)
+
+    set(TIMVX_INC_DIR "${OCV_TIMVX_SOURCE_PATH}/include" CACHE INTERNAL "TIM-VX include directory")
+    if(EXISTS "${OCV_TIMVX_SOURCE_PATH}/CMakeLists.txt")
+        add_subdirectory("${OCV_TIMVX_SOURCE_PATH}" "${OCV_TIMVX_DIR}/build")
+    else()
+        message(WARNING "TIM-VX: Missing 'CMakeLists.txt' in the source code: ${OCV_TIMVX_SOURCE_PATH}")
+    endif()
+    ocv_install_target(tim-vx EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev)
+    set(TIMVX_LIB "tim-vx")
+endif()
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -453,6 +453,9 @@ OCV_OPTION(WITH_TENGINE "Include Arm Inference Tengine support" OFF
 OCV_OPTION(WITH_ONNX "Include Microsoft ONNX Runtime support" OFF
   VISIBLE_IF TRUE
   VERIFY HAVE_ONNX)
+OCV_OPTION(WITH_TIMVX "Include Tim-VX support" OFF
+  VISIBLE_IF TRUE
+  VERIFY HAVE_TIMVX)
 
 # OpenCV build components
 # ===================================================
@@ -733,6 +736,9 @@ include(cmake/OpenCVFindProtobuf.cmake)
 if(WITH_TENGINE)
   include(cmake/OpenCVFindTengine.cmake)
 endif()
+if(WITH_TIMVX)
+  include(cmake/OpenCVFindTIMVX.cmake)
+endif()
 
 # ----------------------------------------------------------------------------
 #  Detect other 3rd-party libraries/tools
@@ -1645,6 +1651,16 @@ if(WITH_WEBNN OR HAVE_WEBNN)
   endif()
 endif()
 
+if(WITH_TIMVX)
+  status("")
+  status("  Tim-VX:"     HAVE_TIMVX THEN "YES" ELSE "NO")
+  if(HAVE_TIMVX)
+    status("    Include path"  TIMVX_INCLUDE_DIR THEN "${TIMVX_INCLUDE_DIR}" ELSE "NO")
+    status("    Link libraries:" TIMVX_LIBRARY THEN "${TIMVX_LIBRARY}" ELSE "NO")
+    status("    VIVANTE SDK path" VIVANTE_SDK_DIR THEN "${VIVANTE_SDK_DIR}" ELSE "NO")
+  endif()
+endif()
+
 if(WITH_OPENCL OR HAVE_OPENCL)
   ocv_build_features_string(opencl_features
     IF HAVE_OPENCL_SVM THEN "SVM"

diff --git a/cmake/OpenCVFindTIMVX.cmake b/cmake/OpenCVFindTIMVX.cmake
@@ -0,0 +1,69 @@
+set(TIMVX_INSTALL_DIR "" CACHE PATH "Path to libtim-vx installation")
+set(VIVANTE_SDK_DIR "" CACHE PATH "Path to VIVANTE SDK needed by TIM-VX.")
+set(VIVANTE_SDK_LIB_CANDIDATES "OpenVX;VSC;GAL;ArchModelSw;NNArchPerf" CACHE STRING "VIVANTE SDK library candidates")
+
+# Ensure VIVANTE SDK library candidates are present in given search path
+function(find_vivante_sdk_libs _viv_notfound _viv_search_path)
+    foreach(one ${VIVANTE_SDK_LIB_CANDIDATES})
+        #NO_DEFAULT_PATH is used to ensure VIVANTE SDK libs are from one only source
+        find_library(VIV_${one}_LIB ${one} PATHS "${_viv_search_path}/lib" NO_DEFAULT_PATH)
+        if(NOT VIV_${one}_LIB)
+            list(APPEND _viv_notfound_list ${one})
+        endif()
+    endforeach()
+    set(${_viv_notfound} ${_viv_notfound_list} PARENT_SCOPE)
+endfunction()
+# Default value for VIVANTE_SDK_DIR: /usr
+if(NOT VIVANTE_SDK_DIR)
+    set(VIVANTE_SDK_DIR "/usr")
+endif()
+# Environment variable VIVANTE_SDK_DIR overrides the one in this script
+if(DEFINED ENV{VIVANTE_SDK_DIR})
+    set(VIVANTE_SDK_DIR $ENV{VIVANTE_SDK_DIR})
+    message(STATUS "TIM-VX: Load VIVANTE_SDK_DIR from system environment: ${VIVANTE_SDK_DIR}")
+endif()
+
+
+# Compile with pre-installed TIM-VX; Or compile together with TIM-VX from source
+if(TIMVX_INSTALL_DIR AND NOT BUILD_TIMVX)
+    message(STATUS "TIM-VX: Use binaries at ${TIMVX_INSTALL_DIR}")
+    set(BUILD_TIMVX OFF)
+
+    set(TIMVX_INC_DIR "${TIMVX_INSTALL_DIR}/include" CACHE INTERNAL "TIM-VX include directory")
+    find_library(TIMVX_LIB "tim-vx" PATHS "${TIMVX_INSTALL_DIR}/lib")
+    if(TIMVX_LIB)
+        set(TIMVX_FOUND ON)
+    else()
+        set(TIMVX_FOUND OFF)
+    endif()
+
+    # Verify if requested VIVANTE SDK libraries are all found
+    find_vivante_sdk_libs(missing ${VIVANTE_SDK_DIR})
+    if(missing)
+        message(STATUS "TIM-VX: Failed to find ${missing} in ${VIVANTE_SDK_DIR}/lib. Turning off TIMVX_VIV_FOUND")
+        set(TIMVX_VIV_FOUND OFF)
+    else()
+        message(STATUS "TIM-VX: dependent VIVANTE SDK libraries are found at ${VIVANTE_SDK_DIR}/lib.")
+        set(TIMVX_VIV_FOUND ON)
+    endif()
+else()
+    message(STATUS "TIM-VX: Build from source")
+    include("${OpenCV_SOURCE_DIR}/3rdparty/libtim-vx/tim-vx.cmake")
+endif()
+
+if(TIMVX_FOUND AND TIMVX_VIV_FOUND)
+    set(HAVE_TIMVX 1)
+
+    message(STATUS "TIM-VX: Found TIM-VX includes: ${TIMVX_INC_DIR}")
+    message(STATUS "TIM-VX: Found TIM-VX library: ${TIMVX_LIB}")
+    set(TIMVX_LIBRARY   ${TIMVX_LIB})
+    set(TIMVX_INCLUDE_DIR   ${TIMVX_INC_DIR})
+
+    message(STATUS "TIM-VX: Found VIVANTE SDK libraries: ${VIVANTE_SDK_DIR}/lib")
+    link_directories(${VIVANTE_SDK_DIR}/lib)
+endif()
+
+MARK_AS_ADVANCED(
+	TIMVX_INC_DIR
+	TIMVX_LIB
+)
diff --git a/modules/dnn/CMakeLists.txt b/modules/dnn/CMakeLists.txt
@@ -23,6 +23,10 @@ if(WITH_WEBNN AND HAVE_WEBNN)
   add_definitions(-DHAVE_WEBNN=1)
 endif()
 
+if(HAVE_TIMVX)
+  add_definitions(-DHAVE_TIMVX=1)
+endif()
+
 ocv_option(OPENCV_DNN_CUDA "Build with CUDA support"
     HAVE_CUDA
     AND HAVE_CUBLAS
@@ -146,6 +150,11 @@ if(HAVE_TENGINE)
 	list(APPEND libs -Wl,--whole-archive ${TENGINE_LIBRARIES} -Wl,--no-whole-archive)
 endif()
 
+if(HAVE_TIMVX)
+    list(APPEND include_dirs ${TIMVX_INCLUDE_DIR})
+    list(APPEND libs -Wl,--whole-archive ${TIMVX_LIBRARY} -Wl,--no-whole-archive)
+endif()
+
 set(webnn_srcs "")
 if(NOT EMSCRIPTEN)
   if(HAVE_WEBNN)

diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp
@@ -262,7 +262,7 @@ CV__DNN_INLINE_NS_BEGIN
     {
     public:
         int input_zp, output_zp;
-        float output_sc;
+        float input_sc, output_sc;
         static Ptr<BaseConvolutionLayer> create(const LayerParams& params);
     };
 
@@ -322,6 +322,7 @@ CV__DNN_INLINE_NS_BEGIN
     {
     public:
         int input_zp, output_zp;
+        float input_sc, output_sc;
         static Ptr<PoolingLayerInt8> create(const LayerParams& params);
     };
 
@@ -365,7 +366,8 @@ CV__DNN_INLINE_NS_BEGIN
     class CV_EXPORTS InnerProductLayerInt8 : public InnerProductLayer
     {
     public:
-        int output_zp;
+        int input_zp, output_zp;
+        float input_sc, output_sc;
         static Ptr<InnerProductLayerInt8> create(const LayerParams& params);
     };
 

diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp
@@ -75,6 +75,7 @@ CV__DNN_INLINE_NS_BEGIN
         DNN_BACKEND_VKCOM,
         DNN_BACKEND_CUDA,
         DNN_BACKEND_WEBNN,
+        DNN_BACKEND_TIMVX,
 #ifdef __OPENCV_BUILD
         DNN_BACKEND_INFERENCE_ENGINE_NGRAPH = 1000000,     // internal - use DNN_BACKEND_INFERENCE_ENGINE + setInferenceEngineBackendType()
         DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019,      // internal - use DNN_BACKEND_INFERENCE_ENGINE + setInferenceEngineBackendType()
@@ -95,7 +96,8 @@ CV__DNN_INLINE_NS_BEGIN
         DNN_TARGET_FPGA,  //!< FPGA device with CPU fallbacks using Inference Engine's Heterogeneous plugin.
         DNN_TARGET_CUDA,
         DNN_TARGET_CUDA_FP16,
-        DNN_TARGET_HDDL
+        DNN_TARGET_HDDL,
+        DNN_TARGET_NPU,
     };
 
     CV_EXPORTS std::vector< std::pair<Backend, Target> > getAvailableBackends();
@@ -321,6 +323,19 @@ CV__DNN_INLINE_NS_BEGIN
             const std::vector<Ptr<BackendWrapper>>& outputs
         );
 
+        /**
+         * @brief Returns a TimVX backend node
+         *
+         * @param   timVxInfo  void pointer to CSLContext object
+         * @param   inputsWrapper   layer inputs
+         * @param   outputsWrapper  layer outputs
+         * @param   isLast if the node is the last one of the TimVX Graph.
+         */
+        virtual Ptr<BackendNode> initTimVX(void* timVxInfo,
+                                           const std::vector<Ptr<BackendWrapper> > &inputsWrapper,
+                                           const std::vector<Ptr<BackendWrapper> > &outputsWrapper,
+                                           bool isLast);
+
        /**
         * @brief Automatic Halide scheduling based on layer hyper-parameters.
         * @param[in] node Backend node with Halide functions.