Snyk scans Logstash container vulnerabilities. (#15117)

* Snyk scans Logstash container vulnerabilities. * Exclude integ test and tools when Snyk scanning. * Remote repo url fix for main branch. * Update .buildkite/scripts/snyk/report.sh Simplify the logic to retrieve the version from `versions.yml` Co-authored-by: kaisecheng <[email protected]> * Add backstage definition for Snyk Report pipeline. --------- Co-authored-by: kaisecheng <[email protected]>
elastic · Jul 12, 2023 · 07b6635 · 07b6635
1 parent c2bbed8
commit 07b6635
Show file tree

Hide file tree

Showing 3 changed files with 107 additions and 15 deletions.
diff --git a/.buildkite/scripts/snyk/report.sh b/.buildkite/scripts/snyk/report.sh
@@ -18,8 +18,6 @@ resolve_latest_branches() {
     IFS='.'
     read -a versions <<< "$SNAPSHOT_VERSION"
     version=${versions[0]}.${versions[1]}
-    version="${version%\"}"
-    version="${version#\"}"
     TARGET_BRANCHES+=("$version")
   done
 }
@@ -56,15 +54,15 @@ report() {
   echo "Reporting to Snyk..."
   cd logstash
   REMOTE_REPO_URL=$1
-  if [ "$REMOTE_REPO_URL" != "$MAIN_BRANCH" ]; then
+  if [ "$REMOTE_REPO_URL" != "main" ]; then
     MAJOR_VERSION=$(echo "$REMOTE_REPO_URL"| cut -d'.' -f 1)
     REMOTE_REPO_URL="$MAJOR_VERSION".latest
     echo "Using '$REMOTE_REPO_URL' remote repo url."
   fi
 
   # adding git commit hash to Snyk tag to improve visibility
   GIT_HEAD=$(git rev-parse --short HEAD 2> /dev/null)
-  ./snyk monitor --all-projects --org=logstash --remote-repo-url="$REMOTE_REPO_URL" --target-reference="$REMOTE_REPO_URL" --detection-depth=6 --exclude=requirements.txt --project-tags=branch="$TARGET_BRANCH",git_head="$GIT_HEAD" && true
+  ./snyk monitor --all-projects --org=logstash --remote-repo-url="$REMOTE_REPO_URL" --target-reference="$REMOTE_REPO_URL" --detection-depth=6 --exclude=qa,tools,devtools,requirements.txt --project-tags=branch="$TARGET_BRANCH",git_head="$GIT_HEAD" && true
   cd ..
 }
 
@@ -79,4 +77,67 @@ do
   echo "Using $TARGET_BRANCH branch."
   build_logstash "$TARGET_BRANCH"
   report "$TARGET_BRANCH"
+done
+
+report_docker_image() {
+  image=$1
+  project_name=$2
+  platform=$3
+  echo "Reporting $image to Snyk started..."
+  docker pull "$image"
+  if [[ $platform != null ]]; then
+    ./snyk container monitor "$image" --org=logstash --platform="$platform" --project-name="$project_name" --project-tags=version="$version" && true
+  else
+    ./snyk container monitor "$image" --org=logstash --project-name="$project_name" --project-tags=version="$version" && true
+  fi
+}
+
+report_docker_images() {
+  version=$1
+  echo "Version value: $version"
+
+  image=$REPOSITORY_BASE_URL"logstash:$version-SNAPSHOT"
+  snyk_project_name="logstash-$version-SNAPSHOT"
+  report_docker_image "$image" "$snyk_project_name"
+
+  image=$REPOSITORY_BASE_URL"logstash-oss:$version-SNAPSHOT"
+  snyk_project_name="logstash-oss-$version-SNAPSHOT"
+  report_docker_image "$image" "$snyk_project_name"
+
+  image=$REPOSITORY_BASE_URL"logstash:$version-SNAPSHOT-arm64"
+  snyk_project_name="logstash-$version-SNAPSHOT-arm64"
+  report_docker_image "$image" "$snyk_project_name" "linux/arm64"
+
+  image=$REPOSITORY_BASE_URL"logstash:$version-SNAPSHOT-amd64"
+  snyk_project_name="logstash-$version-SNAPSHOT-amd64"
+  report_docker_image "$image" "$snyk_project_name" "linux/amd64"
+
+  image=$REPOSITORY_BASE_URL"logstash-oss:$version-SNAPSHOT-arm64"
+  snyk_project_name="logstash-oss-$version-SNAPSHOT-arm64"
+  report_docker_image "$image" "$snyk_project_name" "linux/arm64"
+
+  image=$REPOSITORY_BASE_URL"logstash-oss:$version-SNAPSHOT-amd64"
+  snyk_project_name="logstash-oss-$version-SNAPSHOT-amd64"
+  report_docker_image "$image" "$snyk_project_name" "linux/amd64"
+}
+
+resolve_version_and_report_docker_images() {
+  cd logstash
+  git reset --hard HEAD # reset if any generated files appeared
+  git checkout "$1"
+
+  # parse version (ex: 8.8.2 from 8.8 branch, or 8.9.0 from main branch)
+  versions_file="$PWD/versions.yml"
+  version=$(awk '/logstash:/ { print $2 }' "$versions_file")
+  report_docker_images "$version"
+  cd ..
+}
+
+REPOSITORY_BASE_URL="docker.elastic.co/logstash/"
+
+# resolve docker artifact and report
+for TARGET_BRANCH in "${TARGET_BRANCHES[@]}"
+do
+  echo "Using $TARGET_BRANCH branch for docker images."
+  resolve_version_and_report_docker_images "$TARGET_BRANCH"
 done
diff --git a/.buildkite/scripts/snyk/resolve_stack_version.sh b/.buildkite/scripts/snyk/resolve_stack_version.sh
@@ -10,16 +10,10 @@ VERSION_URL="https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash
 
 echo "Fetching versions from $VERSION_URL"
 VERSIONS=$(curl --silent $VERSION_URL)
-SNAPSHOTS=$(echo $VERSIONS | jq '.snapshots' | jq 'keys | .[]')
-IFS=$'\n' read -d "\034" -r -a SNAPSHOT_KEYS <<<"${SNAPSHOTS}\034"
+SNAPSHOT_KEYS=$(echo "$VERSIONS" | jq -r '.snapshots | .[]')
 
 SNAPSHOT_VERSIONS=()
-for KEY in "${SNAPSHOT_KEYS[@]}"
-do
-  # remove starting and trailing double quotes
-  KEY="${KEY%\"}"
-  KEY="${KEY#\"}"
-  SNAPSHOT_VERSION=$(echo $VERSIONS | jq '.snapshots."'"$KEY"'"')
-  echo "Resolved snapshot version: $SNAPSHOT_VERSION"
-  SNAPSHOT_VERSIONS+=("$SNAPSHOT_VERSION")
-done
+while IFS= read -r line; do
+  SNAPSHOT_VERSIONS+=("$line")
+  echo "Resolved snapshot version: $line"
+done <<< "$SNAPSHOT_KEYS"
diff --git a/catalog-info.yaml b/catalog-info.yaml
@@ -48,3 +48,40 @@ spec:
           branch: core_serverless_test
           cronline: "@daily"
           message: "Run the serverless integration test every day."
+
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: logstash-snyk-report
+  description: 'The logstash-snyk-report pipeline.'
+spec:
+  type: buildkite-pipeline
+  owner: group:ingest-fp
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: logstash-snyk-report-ci
+      description: ':logstash: The logstash-snyk-report :pipeline:'
+    spec:
+      repository: elastic/logstash
+      pipeline_file: ".buildkite/snyk_report_pipeline.yml"
+      provider_settings:
+        trigger_mode: none # don't trigger jobs
+      env:
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+        SLACK_NOTIFICATIONS_CHANNEL: '#logstash-build'
+        SLACK_NOTIFICATIONS_ON_SUCCESS: 'false'
+      teams:
+        ingest-fp:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY
+      schedules:
+        Daily Snyk scan:
+          branch: main
+          cronline: "@daily"
+          message: "Run the Logstash Snyk report every day."