yuanlehome · lizexu123 · Nov 3, 2023 · Nov 3, 2023 · Nov 4, 2023 · Nov 4, 2023
diff --git a/.clang-tidy b/.clang-tidy
@@ -140,7 +140,7 @@ clang-analyzer-optin.portability.UnixAPI,
 clang-analyzer-security.insecureAPI.vfork,
 -clang-analyzer-unix.API,
 -clang-analyzer-unix.DynamicMemoryModeling,
--clang-analyzer-unix.Malloc,
+clang-analyzer-unix.Malloc,
 -clang-analyzer-unix.MallocSizeof,
 -clang-analyzer-unix.MismatchedDeallocator,
 clang-analyzer-unix.Vfork,
@@ -158,7 +158,7 @@ cppcoreguidelines-explicit-virtual-functions,
 cppcoreguidelines-init-variables,
 cppcoreguidelines-narrowing-conversions,
 cppcoreguidelines-no-malloc,
--cppcoreguidelines-pro-type-const-cast,
+cppcoreguidelines-pro-type-const-cast,
 -cppcoreguidelines-pro-type-member-init,
 -cppcoreguidelines-slicing,
 -hicpp-avoid-goto,

diff --git a/.flake8 b/.flake8
@@ -26,8 +26,5 @@ per-file-ignores =
     # These files need tabs for testing.
     test/dygraph_to_static/test_error.py:E101,W191
 
-    # temp ignore base directory
-    python/paddle/base/*:
-        E712,
-        E266,
-        E714
+    # Ignore compare with True in sot unittest
+    test/sot/test_dup_top.py:E712
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -56,17 +56,13 @@ repos:
     hooks:
     -   id: black
         files: (.*\.(py|pyi|bzl)|BUILD|.*\.BUILD|WORKSPACE)$
--   repo: https://github.com/pycqa/isort
-    rev: 5.11.5
-    hooks:
-    -   id: isort
 -   repo: https://github.com/PyCQA/flake8
     rev: 5.0.4
     hooks:
     -   id: flake8
         args: ["--config=.flake8"]
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.0.289
+    rev: v0.1.5
     hooks:
     -   id: ruff
         args: [--fix, --exit-non-zero-on-fix, --no-cache]
@@ -160,3 +156,13 @@ repos:
     hooks:
     -   id: cmakelint
         args: [--config=./tools/codestyle/.cmakelintrc]
+# Others
+-   repo: local
+    hooks:
+    -   id: sort-txt-file
+        name: sort-txt-file
+        description: Sorts each line string in a text file
+        entry: python ./tools/codestyle/sort_txt_file.py
+        language: python
+        files: test/white_list/pir_op_test_white_list
+        args: []
diff --git a/cmake/cinn.cmake b/cmake/cinn.cmake
@@ -164,8 +164,8 @@ cinn_cc_library(
 add_dependencies(cinnapi GEN_LLVM_RUNTIME_IR_HEADER ZLIB::ZLIB)
 add_dependencies(cinnapi GEN_LLVM_RUNTIME_IR_HEADER ${core_deps})
 if(NOT CINN_ONLY)
-  target_link_libraries(cinnapi pd_op_dialect phi)
-  add_dependencies(cinnapi pd_op_dialect phi)
+  target_link_libraries(cinnapi op_dialect_vjp phi)
+  add_dependencies(cinnapi op_dialect_vjp phi)
 endif()
 
 target_link_libraries(cinnapi ${PYTHON_LIBRARIES})
@@ -222,8 +222,8 @@ function(gen_cinncore LINKTYPE)
   add_dependencies(${CINNCORE_TARGET} GEN_LLVM_RUNTIME_IR_HEADER ZLIB::ZLIB)
   add_dependencies(${CINNCORE_TARGET} GEN_LLVM_RUNTIME_IR_HEADER ${core_deps})
   if(NOT CINN_ONLY)
-    target_link_libraries(${CINNCORE_TARGET} pd_op_dialect phi)
-    add_dependencies(${CINNCORE_TARGET} pd_op_dialect phi)
+    target_link_libraries(${CINNCORE_TARGET} op_dialect_vjp phi)
+    add_dependencies(${CINNCORE_TARGET} op_dialect_vjp phi)
   endif()
 
   add_dependencies(${CINNCORE_TARGET} pybind)

diff --git a/cmake/cinn/external/absl.cmake b/cmake/cinn/external/absl.cmake
@@ -5,7 +5,7 @@ set(ABSL_INSTALL_DIR ${THIRD_PARTY_PATH}/install/absl)
 set(ABSL_PREFIX_DIR ${THIRD_PARTY_PATH}/absl)
 set(ABSL_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
 
-set(ABSL_REPOSITORY "https://github.com/abseil/abseil-cpp.git")
+set(ABSL_REPOSITORY "${GIT_URL}/abseil/abseil-cpp.git")
 set(ABSL_TAG "20210324.2")
 
 set(OPTIONAL_ARGS

diff --git a/cmake/cinn/external/jitify.cmake b/cmake/cinn/external/jitify.cmake
@@ -7,7 +7,7 @@ include(ExternalProject)
 
 # clone jitify to Paddle/third_party
 set(JITIFY_SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/jitify)
-set(JITIFY_URL https://github.com/NVIDIA/jitify.git)
+set(JITIFY_URL ${GIT_URL}/NVIDIA/jitify.git)
 set(JITIFY_TAG 57de649139c866eb83acacfe50c92ad7c6278776)
 
 ExternalProject_Add(

diff --git a/cmake/external/brpc.cmake b/cmake/external/brpc.cmake
@@ -40,7 +40,7 @@ include_directories(${BRPC_INCLUDE_DIR})
 
 # clone brpc to Paddle/third_party
 set(BRPC_SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/brpc)
-set(BRPC_URL https://github.com/apache/brpc.git)
+set(BRPC_URL ${GIT_URL}/apache/brpc.git)
 set(BRPC_TAG 1.4.0)
 
 # Reference https://stackoverflow.com/questions/45414507/pass-a-list-of-prefix-paths-to-externalproject-add-in-cmake-args

diff --git a/cmake/external/cudnn-frontend.cmake b/cmake/external/cudnn-frontend.cmake
@@ -34,7 +34,7 @@ if((NOT DEFINED CUDNN_FRONTEND_NAME) OR (NOT DEFINED CUDNN_FRONTEND_URL))
       "cudnn-frontend"
       CACHE STRING "" FORCE)
   set(CUDNN_FRONTEND_URL
-      "https://github.com/NVIDIA/cudnn-frontend/archive/refs/tags/${CUDNN_FRONTEND_VER}.tar.gz"
+      "${GIT_URL}/NVIDIA/cudnn-frontend/archive/refs/tags/${CUDNN_FRONTEND_VER}.tar.gz"
       CACHE STRING "" FORCE)
 endif()
 set(CUDNN_FRONTEND_CACHE_FILENAME "${CUDNN_FRONTEND_VER}.tar.gz")

diff --git a/cmake/external/dirent.cmake b/cmake/external/dirent.cmake
@@ -25,7 +25,7 @@ if((NOT DEFINED DIRENT_NAME) OR (NOT DEFINED DIRENT_URL))
       "dirent"
       CACHE STRING "" FORCE)
   set(DIRENT_URL
-      "https://github.com/tronkko/dirent/archive/refs/tags/1.23.2.tar.gz"
+      "${GIT_URL}/tronkko/dirent/archive/refs/tags/1.23.2.tar.gz"
       CACHE STRING "" FORCE)
   set(DIRENT_CACHE_FILENAME "1.23.2.tar.gz")
 endif()

diff --git a/cmake/external/jemalloc.cmake b/cmake/external/jemalloc.cmake
@@ -5,9 +5,7 @@ set(JEMALLOC_DOWNLOAD_DIR
 set(JEMALLOC_PROJECT "extern_jemalloc")
 set(JEMALLOC_BUILD ${THIRD_PARTY_PATH}/jemalloc/src/extern_jemalloc)
 set(JEMALLOC_PREFIX_DIR ${THIRD_PARTY_PATH}/jemalloc)
-set(JEMALLOC_URL
-    https://github.com/jemalloc/jemalloc/releases/download/5.1.0/jemalloc-5.1.0.tar.bz2
-)
+set(JEMALLOC_URL https://paddle-ci.gz.bcebos.com/jemalloc-5.1.0.tar.bz2)
 set(JEMALLOC_INSTALL ${THIRD_PARTY_PATH}/install/jemalloc)
 set(JEMALLOC_INCLUDE_DIR ${JEMALLOC_INSTALL}/include)
 

diff --git a/cmake/external/libxsmm.cmake b/cmake/external/libxsmm.cmake
@@ -31,9 +31,8 @@ set(LIBXSMMNOBLAS_LIB "${LIBXSMM_LIBRARY_DIR}/libxsmmnoblas.a")
 file(GLOB LIBXSMM_SOURCE_FILE_LIST ${LIBXSMM_SOURCE_DIR})
 list(LENGTH LIBXSMM_SOURCE_FILE_LIST RES_LEN)
 if(RES_LEN EQUAL 0)
-  execute_process(
-    COMMAND ${GIT_EXECUTABLE} clone -b ${LIBXSMM_TAG}
-            "https://github.com/hfp/libxsmm.git" ${LIBXSMM_SOURCE_DIR})
+  execute_process(COMMAND ${GIT_EXECUTABLE} clone -b ${LIBXSMM_TAG}
+                          "${GIT_URL}/hfp/libxsmm.git" ${LIBXSMM_SOURCE_DIR})
 else()
   # check git tag
   execute_process(

diff --git a/cmake/external/onnxruntime.cmake b/cmake/external/onnxruntime.cmake
@@ -44,19 +44,19 @@ set(ONNXRUNTIME_DOWNLOAD_DIR
 
 if(WIN32)
   set(ONNXRUNTIME_URL
-      "https://github.com/microsoft/onnxruntime/releases/download/v${ONNXRUNTIME_VERSION}/onnxruntime-win-x64-${ONNXRUNTIME_VERSION}.zip"
+      "${GIT_URL}/microsoft/onnxruntime/releases/download/v${ONNXRUNTIME_VERSION}/onnxruntime-win-x64-${ONNXRUNTIME_VERSION}.zip"
   )
   set(ONNXRUNTIME_URL_MD5 f21d6bd1feef15935a5f4e1007797593)
   set(ONNXRUNTIME_CACHE_EXTENSION "zip")
 elseif(APPLE)
   set(ONNXRUNTIME_URL
-      "https://github.com/microsoft/onnxruntime/releases/download/v${ONNXRUNTIME_VERSION}/onnxruntime-osx-x86_64-${ONNXRUNTIME_VERSION}.tgz"
+      "${GIT_URL}/microsoft/onnxruntime/releases/download/v${ONNXRUNTIME_VERSION}/onnxruntime-osx-x86_64-${ONNXRUNTIME_VERSION}.tgz"
   )
   set(ONNXRUNTIME_URL_MD5 6a6f6b7df97587da59976042f475d3f4)
   set(ONNXRUNTIME_CACHE_EXTENSION "tgz")
 else()
   set(ONNXRUNTIME_URL
-      "https://github.com/microsoft/onnxruntime/releases/download/v${ONNXRUNTIME_VERSION}/onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}.tgz"
+      "${GIT_URL}/microsoft/onnxruntime/releases/download/v${ONNXRUNTIME_VERSION}/onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}.tgz"
   )
   set(ONNXRUNTIME_URL_MD5 ce3f2376854b3da4b483d6989666995a)
   set(ONNXRUNTIME_CACHE_EXTENSION "tgz")

diff --git a/cmake/external/openblas.cmake b/cmake/external/openblas.cmake
@@ -19,12 +19,16 @@ set(CBLAS_INSTALL_DIR ${THIRD_PARTY_PATH}/install/openblas)
 set(CBLAS_SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/openblas)
 set(CBLAS_TAG v0.3.7)
 
-# OpenBLAS support Raptor Lake from v0.3.22
-if(UNIX
-   AND NOT APPLE
-   AND NOT WITH_ROCM
+# Why use v0.3.18?  The IDG business line encountered a random openblas error,
+# which can be resolved after upgrading openblas.
+# And why compile when gcc>8.2? Please refer to
+# https://github.com/spack/spack/issues/19932#issuecomment-733452619
+# v0.3.18 only support gcc>=8.3 or gcc>=7.4
+if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+   AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 8.2
    AND NOT WITH_XPU)
-  set(CBLAS_TAG v0.3.23)
+  # We only compile with openblas 0.3.18 when gcc >= 8.3
+  set(CBLAS_TAG v0.3.18)
 endif()
 
 if(APPLE AND WITH_ARM)
@@ -42,9 +46,8 @@ endif()
 file(GLOB CBLAS_SOURCE_FILE_LIST ${CBLAS_SOURCE_DIR})
 list(LENGTH CBLAS_SOURCE_FILE_LIST RES_LEN)
 if(RES_LEN EQUAL 0)
-  execute_process(
-    COMMAND ${GIT_EXECUTABLE} clone -b ${CBLAS_TAG}
-            "https://github.com/xianyi/OpenBLAS.git" ${CBLAS_SOURCE_DIR})
+  execute_process(COMMAND ${GIT_EXECUTABLE} clone -b ${CBLAS_TAG}
+                          "${GIT_URL}/xianyi/OpenBLAS.git" ${CBLAS_SOURCE_DIR})
 else()
   # check git tag
   execute_process(

diff --git a/cmake/external/paddle2onnx.cmake b/cmake/external/paddle2onnx.cmake
@@ -71,19 +71,19 @@ endif()
 
 if(WIN32)
   set(PADDLE2ONNX_URL
-      "https://github.com/PaddlePaddle/Paddle2ONNX/releases/download/v${PADDLE2ONNX_VERSION}/paddle2onnx-win-x64-${PADDLE2ONNX_VERSION}.zip"
+      "${GIT_URL}/PaddlePaddle/Paddle2ONNX/releases/download/v${PADDLE2ONNX_VERSION}/paddle2onnx-win-x64-${PADDLE2ONNX_VERSION}.zip"
   )
   set(PADDLE2ONNX_URL_MD5 "122b864cb57338191a7e9ef5f607c4ba")
   set(PADDLE2ONNX_CACHE_EXTENSION "zip")
 elseif(APPLE)
   set(PADDLE2ONNX_URL
-      "https://github.com/PaddlePaddle/Paddle2ONNX/releases/download/v${PADDLE2ONNX_VERSION}/paddle2onnx-osx-x86_64-${PADDLE2ONNX_VERSION}.tgz"
+      "${GIT_URL}/PaddlePaddle/Paddle2ONNX/releases/download/v${PADDLE2ONNX_VERSION}/paddle2onnx-osx-x86_64-${PADDLE2ONNX_VERSION}.tgz"
   )
   set(PADDLE2ONNX_URL_MD5 "32a4381ff8441b69d58ef0fd6fd919eb")
   set(PADDLE2ONNX_CACHE_EXTENSION "tgz")
 else()
   set(PADDLE2ONNX_URL
-      "https://github.com/PaddlePaddle/Paddle2ONNX/releases/download/v${PADDLE2ONNX_VERSION}/paddle2onnx-linux-x64-${PADDLE2ONNX_VERSION}.tgz"
+      "${GIT_URL}/PaddlePaddle/Paddle2ONNX/releases/download/v${PADDLE2ONNX_VERSION}/paddle2onnx-linux-x64-${PADDLE2ONNX_VERSION}.tgz"
   )
   set(PADDLE2ONNX_URL_MD5 "3fbb074987ba241327797f76514e937f")
   set(PADDLE2ONNX_CACHE_EXTENSION "tgz")

diff --git a/cmake/external/protobuf.cmake b/cmake/external/protobuf.cmake
@@ -244,7 +244,7 @@ function(build_protobuf TARGET_NAME BUILD_FOR_HOST)
     set(PROTOBUF_TAG 01a05a53f40ca2ac5f0af10c6cc0810bee39b792)
   else()
     if(WITH_PSLIB)
-      set(PROTOBUF_REPOSITORY "https://github.com/google/protobuf.git")
+      set(PROTOBUF_REPOSITORY "${GIT_URL}/google/protobuf.git")
       set(PROTOBUF_TAG "9f75c5aa851cd877fb0d93ccc31b8567a6706546")
     else()
       set(PROTOBUF_REPOSITORY ${GIT_URL}/protocolbuffers/protobuf.git)

diff --git a/cmake/external/xpu.cmake b/cmake/external/xpu.cmake
@@ -24,9 +24,9 @@ set(XPU_XFT_LIB_NAME "libxft.so")
 set(XPU_XPTI_LIB_NAME "libxpti.so")
 
 if(NOT DEFINED XPU_BASE_DATE)
-  set(XPU_BASE_DATE "20230926")
+  set(XPU_BASE_DATE "20231103")
 endif()
-set(XPU_XCCL_BASE_VERSION "1.0.53.6")
+set(XPU_XCCL_BASE_VERSION "1.1.6.1")
 if(NOT DEFINED XPU_XFT_BASE_VERSION)
   set(XPU_XFT_BASE_VERSION "20230602")
 endif()

diff --git a/cmake/generic.cmake b/cmake/generic.cmake
@@ -499,12 +499,15 @@ function(cc_test_run TARGET_NAME)
       NAME ${TARGET_NAME}
       COMMAND ${cc_test_COMMAND} ${cc_test_ARGS}
       WORKING_DIRECTORY ${cc_test_DIR})
-    set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT
-                                              FLAGS_cpu_deterministic=true)
-    set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT
-                                              FLAGS_init_allocated_mem=true)
-    set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT
-                                              FLAGS_cudnn_deterministic=true)
+    set_property(
+      TEST ${TARGET_NAME}
+      PROPERTY
+        ENVIRONMENT
+        FLAGS_cpu_deterministic=true
+        FLAGS_init_allocated_mem=true
+        FLAGS_cudnn_deterministic=true
+        LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${PADDLE_BINARY_DIR}/python/paddle/libs:${PADDLE_BINARY_DIR}/python/paddle/base
+    )
     # No unit test should exceed 2 minutes.
     if(WIN32)
       set_tests_properties(${TARGET_NAME} PROPERTIES TIMEOUT 150)
@@ -726,6 +729,7 @@ function(nv_test TARGET_NAME)
     # 2. cuda_add_executable does not support ccache.
     # Reference: https://cmake.org/cmake/help/v3.10/module/FindCUDA.html
     add_executable(${TARGET_NAME} ${nv_test_SRCS})
+    target_compile_definitions(${TARGET_NAME} PUBLIC STATIC_PADDLE)
     get_property(os_dependency_modules GLOBAL PROPERTY OS_DEPENDENCY_MODULES)
     target_link_libraries(${TARGET_NAME} ${nv_test_DEPS}
                           ${os_dependency_modules} paddle_gtest_main phi)

diff --git a/cmake/hip.cmake b/cmake/hip.cmake
@@ -118,6 +118,11 @@ list(APPEND HIP_CXX_FLAGS -Wno-unused-value)
 list(APPEND HIP_CXX_FLAGS -Wno-braced-scalar-init)
 list(APPEND HIP_CXX_FLAGS -Wno-return-type)
 list(APPEND HIP_CXX_FLAGS -Wno-pragma-once-outside-header)
+list(APPEND HIP_CXX_FLAGS -Wno-deprecated-builtins)
+list(APPEND HIP_CXX_FLAGS -Wno-switch)
+list(APPEND HIP_CXX_FLAGS -Wno-literal-conversion)
+list(APPEND HIP_CXX_FLAGS -Wno-constant-conversion)
+list(APPEND HIP_CXX_FLAGS -Wno-defaulted-function-deleted)
 
 if(WITH_CINN)
   list(APPEND HIP_CXX_FLAGS -std=c++14)

diff --git a/cmake/operators.cmake b/cmake/operators.cmake
@@ -684,6 +684,9 @@ function(prune_pybind_h)
   list(APPEND op_list "load_combine")
   list(APPEND op_list "tensorrt_engine")
 
+  # TODO(ming1753): conditional_block_infer is temporarily reserved here to avoid link errors in functions of standalone_executor
+  list(APPEND op_list "conditional_block_infer")
+
   # add fused_op in op_list
   list(APPEND op_list "fc")
   list(APPEND op_list "conv2d_fusion")

diff --git a/paddle/cinn/CMakeLists.txt b/paddle/cinn/CMakeLists.txt
@@ -2,6 +2,7 @@ if(WITH_TESTING)
   cinn_cc_library(cinn_gtest_main SRCS gtest_main.cc DEPS gtest ${flags_dep})
 endif()
 
+add_subdirectory(adt)
 add_subdirectory(api)
 add_subdirectory(ast_gen_ius)
 add_subdirectory(auto_schedule)