Integrate paddle backend in devel branch

.pre-commit-config.yaml

@@ -51,19 +51,19 @@ repos:
     hooks:
       - id: blacken-docs
   # C++
-  - repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v18.1.8
-    hooks:
-      - id: clang-format
-        exclude: ^source/3rdparty|source/lib/src/gpu/cudart/.+\.inc
+  # - repo: https://github.com/pre-commit/mirrors-clang-format
+  #   rev: v18.1.8
+  #   hooks:
+  #     - id: clang-format
+  #       exclude: ^source/3rdparty|source/lib/src/gpu/cudart/.+\.inc
   # markdown, yaml, CSS, javascript
-  - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v4.0.0-alpha.8
-    hooks:
-      - id: prettier
-        types_or: [markdown, yaml, css]
-        # workflow files cannot be modified by pre-commit.ci
-        exclude: ^(source/3rdparty|\.github/workflows|\.clang-format)
+  # - repo: https://github.com/pre-commit/mirrors-prettier
+  #   rev: v4.0.0-alpha.8
+  #   hooks:
+  #     - id: prettier
+  #       types_or: [markdown, yaml, css]
+  #       # workflow files cannot be modified by pre-commit.ci
+  #       exclude: ^(source/3rdparty|\.github/workflows|\.clang-format)
   # Shell
   - repo: https://github.com/scop/pre-commit-shfmt
     rev: v3.9.0-1
@@ -75,25 +75,25 @@ repos:
     hooks:
       - id: cmake-format
       #- id: cmake-lint
-  - repo: https://github.com/njzjz/mirrors-bibtex-tidy
-    rev: v1.13.0
-    hooks:
-      - id: bibtex-tidy
-        args:
-          - --curly
-          - --numeric
-          - --align=13
-          - --blank-lines
-          # disable sort: the order of keys and fields has explict meanings
-          #- --sort=key
-          - --duplicates=key,doi,citation,abstract
-          - --merge=combine
-          #- --sort-fields
-          #- --strip-comments
-          - --trailing-commas
-          - --encode-urls
-          - --remove-empty-fields
-          - --wrap=80
+  # - repo: https://github.com/njzjz/mirrors-bibtex-tidy
+  #   rev: v1.13.0
+  #   hooks:
+  #     - id: bibtex-tidy
+  #       args:
+  #         - --curly
+  #         - --numeric
+  #         - --align=13
+  #         - --blank-lines
+  #         # disable sort: the order of keys and fields has explict meanings
+  #         #- --sort=key
+  #         - --duplicates=key,doi,citation,abstract
+  #         - --merge=combine
+  #         #- --sort-fields
+  #         #- --strip-comments
+  #         - --trailing-commas
+  #         - --encode-urls
+  #         - --remove-empty-fields
+  #         - --wrap=80
   # license header
   - repo: https://github.com/Lucas-C/pre-commit-hooks
     rev: v1.5.5

backend/dp_backend.py

@@ -7,6 +7,9 @@
 
 from scikit_build_core import build as _orig
 
+from .find_paddle import (
+    find_paddle,
+)
 from .find_pytorch import (
     find_pytorch,
 )
@@ -47,6 +50,7 @@ def get_requires_for_build_wheel(
         _orig.get_requires_for_build_wheel(config_settings)
         + find_tensorflow()[1]
         + find_pytorch()[1]
+        + find_paddle()[1]
     )
 
 
@@ -57,4 +61,5 @@ def get_requires_for_build_editable(
         _orig.get_requires_for_build_editable(config_settings)
         + find_tensorflow()[1]
         + find_pytorch()[1]
+        + find_paddle()[1]
     )

backend/find_paddle.py

@@ -0,0 +1,145 @@
+# SPDX-License-Identifier: LGPL-3.0-or-later
+import importlib
+import os
+import site
+from functools import (
+    lru_cache,
+)
+from importlib.machinery import (
+    FileFinder,
+)
+from importlib.util import (
+    find_spec,
+)
+from pathlib import (
+    Path,
+)
+from sysconfig import (
+    get_path,
+)
+from typing import (
+    List,
+    Optional,
+    Tuple,
+    Union,
+)
+
+from packaging.version import (
+    Version,
+)
+
+
+@lru_cache
+def find_paddle() -> Tuple[Optional[str], List[str]]:
+    """Find PaddlePadle library.
+
+    Tries to find PaddlePadle in the order of:
+
+    1. Environment variable `PADDLE_ROOT` if set
+    2. The current Python environment.
+    3. user site packages directory if enabled
+    4. system site packages directory (purelib)
+
+    Considering the default PaddlePadle package still uses old CXX11 ABI, we
+    cannot install it automatically.
+
+    Returns
+    -------
+    str, optional
+        PaddlePadle library path if found.
+    list of str
+        TensorFlow requirement if not found. Empty if found.
+    """
+    if os.environ.get("DP_ENABLE_PADDLE", "0") == "0":
+        return None, []
+    requires = []
+    pd_spec = None
+
+    if (pd_spec is None or not pd_spec) and os.environ.get("PADDLE_ROOT") is not None:
+        site_packages = Path(os.environ.get("PADDLE_ROOT")).parent.absolute()
+        pd_spec = FileFinder(str(site_packages)).find_spec("paddle")
+
+    # get paddle spec
+    # note: isolated build will not work for backend
+    if pd_spec is None or not pd_spec:
+        pd_spec = find_spec("paddle")
+
+    if not pd_spec and site.ENABLE_USER_SITE:
+        # first search TF from user site-packages before global site-packages
+        site_packages = site.getusersitepackages()
+        if site_packages:
+            pd_spec = FileFinder(site_packages).find_spec("paddle")
+
+    if not pd_spec:
+        # purelib gets site-packages path
+        site_packages = get_path("purelib")
+        if site_packages:
+            pd_spec = FileFinder(site_packages).find_spec("paddle")
+
+    # get install dir from spec
+    try:
+        pd_install_dir = pd_spec.submodule_search_locations[0]  # type: ignore
+        # AttributeError if ft_spec is None
+        # TypeError if submodule_search_locations are None
+        # IndexError if submodule_search_locations is an empty list
+    except (AttributeError, TypeError, IndexError):
+        pd_install_dir = None
+        requires.extend(get_pd_requirement()["paddle"])
+    return pd_install_dir, requires
+
+
+@lru_cache
+def get_pd_requirement(pd_version: str = "") -> dict:
+    """Get PaddlePadle requirement when Paddle is not installed.
+
+    If pd_version is not given and the environment variable `PADDLE_VERSION` is set, use it as the requirement.
+
+    Parameters
+    ----------
+    pd_version : str, optional
+        Paddle version
+
+    Returns
+    -------
+    dict
+        PaddlePadle requirement.
+    """
+    if pd_version is None:
+        return {"paddle": []}
+    if pd_version == "":
+        pd_version = os.environ.get("PADDLE_VERSION", "")
+
+    return {
+        "paddle": [
+            # uv has different local version behaviors, i.e. `==2.3.1` cannot match `==2.3.1+cpu`
+            # https://github.com/astral-sh/uv/blob/main/PIP_COMPATIBILITY.md#local-version-identifiers
+            # luckily, .* (prefix matching) defined in PEP 440 can match any local version
+            # https://peps.python.org/pep-0440/#version-matching
+            f"paddle=={Version(pd_version).base_version}.*"
+            if pd_version != ""
+            else "paddle>=3.0.0",
+        ],
+    }
+
+
+@lru_cache
+def get_pd_version(pd_path: Optional[Union[str, Path]]) -> str:
+    """Get Paddle version from a Paddle Python library path.
+
+    Parameters
+    ----------
+    pd_path : str or Path
+        pd Python library path
+
+    Returns
+    -------
+    str
+        version
+    """
+    if pd_path is None or pd_path == "":
+        return ""
+    version_file = Path(pd_path) / "version.py"
+    spec = importlib.util.spec_from_file_location("paddle.version", version_file)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module.__version__

backend/read_env.py

@@ -13,6 +13,9 @@
     Version,
 )
 
+from .find_paddle import (
+    find_paddle,
+)
 from .find_pytorch import (
     find_pytorch,
     get_pt_version,
@@ -120,6 +123,19 @@ def get_argument_from_env() -> Tuple[str, list, list, dict, str, str]:
         cmake_args.append("-DENABLE_PYTORCH=OFF")
         pt_version = None
 
+    if os.environ.get("DP_ENABLE_PADDLE", "0") == "1":
+        pd_install_dir, _ = find_paddle()
+        pt_version = get_pt_version(pd_install_dir)
+        cmake_args.extend(
+            [
+                "-DENABLE_PADDLE=ON",
+                f"-DCMAKE_PREFIX_PATH={pd_install_dir}",
+            ]
+        )
+    else:
+        cmake_args.append("-DENABLE_PADDLE=OFF")
+        pt_version = None
+
     cmake_args = [
         "-DBUILD_PY_IF:BOOL=TRUE",
         *cmake_args,