Merge branch 'microsoft:main' into main

.config/1espt/PipelineAutobaseliningConfig.yml

@@ -0,0 +1,77 @@
+## DO NOT MODIFY THIS FILE MANUALLY. This is part of auto-baselining from 1ES Pipeline Templates. Go to [https://aka.ms/1espt-autobaselining] for more details.
+
+pipelines:
+  1624:
+    retail:
+      source:
+        credscan:
+          lastModifiedDate: 2024-10-25
+        policheck:
+          lastModifiedDate: 2024-10-25
+        eslint:
+          lastModifiedDate: 2024-10-25
+        psscriptanalyzer:
+          lastModifiedDate: 2024-10-25
+        armory:
+          lastModifiedDate: 2024-10-25
+    usedNonDefaultBranch: true
+  1299:
+    retail:
+      source:
+        credscan:
+          lastModifiedDate: 2024-10-25
+        eslint:
+          lastModifiedDate: 2024-10-25
+        psscriptanalyzer:
+          lastModifiedDate: 2024-10-25
+        armory:
+          lastModifiedDate: 2024-10-25
+        policheck:
+          lastModifiedDate: 2024-10-29
+      binary:
+        credscan:
+          lastModifiedDate: 2024-10-25
+        binskim:
+          lastModifiedDate: 2024-10-25
+        spotbugs:
+          lastModifiedDate: 2024-10-25
+  1625:
+    retail:
+      source:
+        credscan:
+          lastModifiedDate: 2024-11-05
+        policheck:
+          lastModifiedDate: 2024-11-05
+        eslint:
+          lastModifiedDate: 2024-11-05
+        psscriptanalyzer:
+          lastModifiedDate: 2024-11-05
+        armory:
+          lastModifiedDate: 2024-11-05
+      binary:
+        credscan:
+          lastModifiedDate: 2024-11-13
+        binskim:
+          lastModifiedDate: 2024-11-13
+        spotbugs:
+          lastModifiedDate: 2024-11-13
+  1626:
+    retail:
+      source:
+        credscan:
+          lastModifiedDate: 2024-11-13
+        policheck:
+          lastModifiedDate: 2024-11-13
+        eslint:
+          lastModifiedDate: 2024-11-13
+        psscriptanalyzer:
+          lastModifiedDate: 2024-11-13
+        armory:
+          lastModifiedDate: 2024-11-13
+      binary:
+        credscan:
+          lastModifiedDate: 2024-11-13
+        binskim:
+          lastModifiedDate: 2024-11-13
+        spotbugs:
+          lastModifiedDate: 2024-11-13

.config/guardian/.gdnbaselines

@@ -0,0 +1,43 @@
+{
+  "properties": {
+    "helpUri": "https://eng.ms/docs/microsoft-security/security/azure-security/cloudai-security-fundamentals-engineering/security-integration/guardian-wiki/microsoft-guardian/general/baselines"
+  },
+  "version": "1.0.0",
+  "baselines": {
+    "default": {
+      "name": "default",
+      "createdDate": "2024-11-13 00:40:35Z",
+      "lastUpdatedDate": "2024-11-13 00:40:35Z"
+    }
+  },
+  "results": {
+    "48f03e2797fc40ecea50f878a0268947c7e13db1b2fa51aa3981246844fc4c68": {
+      "signature": "48f03e2797fc40ecea50f878a0268947c7e13db1b2fa51aa3981246844fc4c68",
+      "alternativeSignatures": [],
+      "target": "ScanTelemetry_20241113003616898.json",
+      "line": 1,
+      "memberOf": [
+        "default"
+      ],
+      "tool": "credscan",
+      "ruleId": "CSCAN-AZURE0130",
+      "createdDate": "2024-11-13 00:40:35Z",
+      "expirationDate": "2025-05-02 01:29:47Z",
+      "justification": "This error is baselined with an expiration date of 180 days from 2024-11-13 01:29:47Z"
+    },
+    "9cb6eddb3f3e886ad06cae65f5886412ff0c5fb0b96d4e943e4efa237be617b1": {
+      "signature": "9cb6eddb3f3e886ad06cae65f5886412ff0c5fb0b96d4e943e4efa237be617b1",
+      "alternativeSignatures": [],
+      "target": "ScanTelemetry_20241113111547065.json",
+      "line": 1,
+      "memberOf": [
+        "default"
+      ],
+      "tool": "credscan",
+      "ruleId": "CSCAN-AZURE0130",
+      "createdDate": "2024-11-13 11:20:17Z",
+      "expirationDate": "2025-05-02 11:55:15Z",
+      "justification": "This error is baselined with an expiration date of 180 days from 2024-11-13 11:55:15Z"
+    }
+  }
+}

.github/codeql/codeql-config.yml

@@ -0,0 +1,7 @@
+name: "CodeQL config"
+queries: 
+  - uses: security-extended
+  - uses: security-and-quality
+paths-ignore:
+  - tests
+  - build

.github/workflows/cffconvert.yml

@@ -8,7 +8,7 @@ on:
 jobs:
   validate:
     name: "validate"
-    runs-on: ubuntu-latest
+    runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-Ubuntu2204-AMD-CPU"]
     steps:
       - name: Check out a copy of the repository
         uses: actions/checkout@v4

.github/workflows/codeql.yml

@@ -15,10 +15,14 @@ on:
   schedule:
     - cron: '41 13 * * 0'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   analyze:
     name: Analyze
-    runs-on: ubuntu-latest
+    runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-Ubuntu2204-AMD-CPU"]
     permissions:
       actions: read
       contents: read
@@ -55,6 +59,11 @@ jobs:
         java-version: '11'
         distribution: 'microsoft'
 
+    - if: ${{ matrix.language == 'javascript' }}
+      uses: actions/setup-node@v4
+      with:
+        node-version: 20
+
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - if: ${{ matrix.language != 'cpp' }}

.github/workflows/gradle-wrapper-validation.yml

@@ -8,7 +8,7 @@ on: [push, pull_request]
 jobs:
   validation:
     name: "Validation"
-    runs-on: ubuntu-latest
+    runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-Ubuntu2204-AMD-CPU"]
     steps:
       - uses: actions/checkout@v4
       - uses: gradle/actions/wrapper-validation@v4

.github/workflows/labeler.yml

@@ -8,7 +8,7 @@ permissions:
 
 jobs:
   triage:
-    runs-on: ubuntu-latest
+    runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-Ubuntu2204-AMD-CPU"]
     steps:
     - uses: github/[email protected]
       with:

.github/workflows/lint.yml

@@ -36,23 +36,29 @@ jobs:
   lint-python-format:
     # Required workflow
     name: Python format
-    runs-on: ubuntu-latest
+    runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-Ubuntu2204-AMD-CPU"]
+    permissions:
+      contents: read
+      security-events: write
     steps:
       - uses: actions/checkout@v4
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          # Version range or exact version of Python to use, using SemVer's version range syntax. Reads from .python-version if unset.
+          # Use the version configured in target-version of [tool.black] section in pyproject.toml.
           python-version: "3.10"
       - name: Setup Rust
         uses: actions-rs/toolchain@v1
         with:
           toolchain: stable
           components: rustfmt
+      - name: Update PATH
+        run: |
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
       - name: Install dependencies
         run: |
-          python -m pip install -r requirements-dev.txt
-          python -m pip install lintrunner lintrunner-adapters
+          set -e -x
+          python -m pip install --user -r requirements-dev.txt
           lintrunner init
       - name: Run lintrunner on all files
         run: |
@@ -81,8 +87,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@master
+      - name: Update PATH
+        run: |
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+
       - name: Install ninja
-        run: python -m pip install --upgrade ninja
+        run: python -m pip install --user --upgrade ninja
       - name: Generate compile_commands.json
         run: |
           python tools/ci_build/build.py \
@@ -114,9 +124,12 @@ jobs:
 
   lint-js:
     name: Lint JavaScript
-    runs-on: ubuntu-latest
+    runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-Ubuntu2204-AMD-CPU"]
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
       - uses: reviewdog/action-eslint@v1
         with:
           reporter: github-pr-check

.github/workflows/linux_training.yml

@@ -0,0 +1,55 @@
+name: orttraining-linux-ci-pipeline
+on:
+  push:
+    branches:
+      - main
+      - rel-*
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  orttraining-linux-ci-pipeline:
+    runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-Ubuntu2204-AMD-CPU"]
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+    steps:
+      - uses: actions/checkout@v4
+      - run: |
+         python3 -m pip install --user -r tools/ci_build/github/linux/python/requirements.txt
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          config-file: ./.github/codeql/codeql-config.yml
+          languages: 'cpp'
+      - run: |
+         set -e -x
+         rm -rf build
+         python3 tools/ci_build/build.py --build_dir build --config Release --enable_training --skip_submodule_sync --parallel --update --build
+         
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+          category: "/language:cpp"
+          output: sarif-results
+          upload: failure-only
+
+      - name: filter-sarif
+        uses: advanced-security/filter-sarif@v1
+        with:
+          patterns: |
+            +**/*.cc
+            +**/*.h
+            -tests/**/*.*
+            -build/**/*.*
+          input: sarif-results/cpp.sarif
+          output: sarif-results/cpp.sarif
+
+      - name: Upload SARIF
+        uses: github/codeql-action/upload-sarif@v3
+        with:
+          sarif_file: sarif-results/cpp.sarif

.github/workflows/mac.yml

@@ -105,7 +105,7 @@ jobs:
           --use_xnnpack \
           --use_binskim_compliant_compile_flags \
           --ios \
-          --apple_deploy_target=13.0 \
+          --apple_deploy_target=15.1 \
           --apple_sysroot=iphonesimulator \
           --osx_arch=${{ matrix.target_arch }}
 
@@ -117,17 +117,21 @@ jobs:
       with:
         python-version: ${{ env.python_version }}
 
+    - name: "Run Homebrew"
+      run: brew install autoconf automake autoconf-archive
+
     - name: "Run vcpkg(x64-osx)"
       uses: lukka/run-vcpkg@v11
       with:
         vcpkgDirectory: "${{ runner.temp }}/vcpkg"
-        vcpkgGitCommitId: "1de2026f28ead93ff1773e6e680387643e914ea1" # 2024.07.12
-        runVcpkgInstall: true
+        vcpkgGitCommitId: "b322364f06308bdd24823f9d8f03fe0cc86fd46f" # 2024.12.16
+        runVcpkgInstall: true # vcpkg install --x-manifest-root cmake --x-install-root .build --overlay-triplets cmake/vcpkg-triplets/default
         vcpkgJsonGlob: "cmake/vcpkg.json"
         vcpkgConfigurationJsonGlob: "cmake/vcpkg-configuration.json"
       env:
         VCPKG_INSTALLED_DIR: "${{ github.workspace }}/.build"
         VCPKG_DEFAULT_TRIPLET: "x64-osx"
+        VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
         # VCPKG_BINARY_SOURCES: "default" # https://learn.microsoft.com/en-us/vcpkg/reference/binarycaching
 
     - name: "Run compile_schema.py"
@@ -163,19 +167,22 @@ jobs:
           --cmake_extra_defines "VCPKG_TARGET_TRIPLET=x64-osx" \
           --cmake_extra_defines "VCPKG_INSTALLED_DIR:PATH=${{ github.workspace }}/.build" \
           --cmake_extra_defines "VCPKG_INSTALL_OPTIONS=--x-feature=tests"
+      env:
+        VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
       shell: bash
 
     - name: "Run vcpkg(arm64-osx)"
       uses: lukka/run-vcpkg@v11
       with:
         vcpkgDirectory: "${{ runner.temp }}/vcpkg"
-        vcpkgGitCommitId: "1de2026f28ead93ff1773e6e680387643e914ea1" # 2024.07.12
-        runVcpkgInstall: true
+        doNotUpdateVcpkg: true
+        runVcpkgInstall: true # vcpkg install --x-manifest-root cmake --x-install-root .build --overlay-triplets cmake/vcpkg-triplets/default
         vcpkgJsonGlob: "cmake/vcpkg.json"
         vcpkgConfigurationJsonGlob: "cmake/vcpkg-configuration.json"
       env:
         VCPKG_INSTALLED_DIR: "${{ github.workspace }}/.build"
         VCPKG_DEFAULT_TRIPLET: "arm64-osx"
+        VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
         # VCPKG_BINARY_SOURCES: "default" # https://learn.microsoft.com/en-us/vcpkg/reference/binarycaching
 
     - name: "Run build.py(arm64-osx)"
@@ -193,6 +200,8 @@ jobs:
           --cmake_extra_defines "VCPKG_TARGET_TRIPLET=arm64-osx" \
           --cmake_extra_defines "VCPKG_INSTALLED_DIR:PATH=${{ github.workspace }}/.build" \
           --cmake_extra_defines "VCPKG_INSTALL_OPTIONS=--x-feature=tests"
+      env:
+        VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
       shell: bash
 
   Objective-C-StaticAnalysis: