Skip to content

Commit

Permalink
feat: support snyk code test (#176)
Browse files Browse the repository at this point in the history 
* feat: support snyk code test

modified cli calls to support snyk code test
modified report html iframe display

* feat: support snyk code test

targeted snyk code test --json execution to piped file
  • Loading branch information
@gwnlng
gwnlng authored Nov 8, 2023
1 parent 634face commit e8d0ad7
Show file tree
Hide file tree
Showing 8 changed files with 202 additions and 43 deletions.
80 changes: 68 additions & 12 deletions snykTask/src/__tests__/test-task-args.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
which I don't want to do. */

import { TaskArgs } from '../task-args';
import { Severity } from '../task-lib';
import { Severity, TestType } from '../task-lib';

function defaultTaskArgs(): TaskArgs {
return new TaskArgs({
Expand All @@ -44,6 +44,23 @@ test('ensure no problems if both targetFile and docker-file-path are both not se
expect(fileArg).toBe('');
});

test('ensure no problems if testType set to code and both targetFile and dockerImageName are both set', () => {
const args = defaultTaskArgs();
args.testType = TestType.CODE;
args.dockerImageName = 'some-docker-image';
args.targetFile = 'some-target-file';
args.dockerfilePath = null as any;

const fileArg = args.getFileParameter();
console.log(`fileArg: ${fileArg}`);
if (fileArg == null) {
console.log('fileArg is null');
}

expect(args.testType).toBe('code');
expect(fileArg).toBe('');
});

test("if dockerImageName is specified and (dockerfilePath is not specified but targetFile is and does not contain 'Dockerfile') then return empty string", () => {
const args = defaultTaskArgs();
args.dockerImageName = 'some-docker-image';
Expand Down Expand Up @@ -126,17 +143,27 @@ describe('TaskArgs.setMonitorWhen', () => {

args.setMonitorWhen('always');
expect(args.monitorWhen).toBe('always');

args.testType = TestType.CODE;
args.setMonitorWhen('always');
expect(args.monitorWhen).toBe('never');
});
});

describe('TaskArgs.validate', () => {
const args = defaultTaskArgs();
const validSeverityThresholds = [
Severity.CRITICAL,
Severity.HIGH,
Severity.MEDIUM,
Severity.LOW,
const testTypeSeverityThreshold = [
[
TestType.APPLICATION,
[Severity.CRITICAL, Severity.HIGH, Severity.MEDIUM, Severity.LOW],
],
[TestType.CODE, [Severity.HIGH, Severity.MEDIUM, Severity.LOW]],
[
TestType.CONTAINER_IMAGE,
[Severity.CRITICAL, Severity.HIGH, Severity.MEDIUM, Severity.LOW],
],
];

it('passes validation when correct combination of severity and fail on thresholds', () => {
args.severityThreshold = Severity.LOW;
args.failOnThreshold = Severity.HIGH;
Expand All @@ -153,22 +180,51 @@ describe('TaskArgs.validate', () => {
args.validate();
});

it('throws error if invalid severity threshold', () => {
it('throws error if invalid severity threshold for blank testType defaulted to app', () => {
expect(() => {
args.severityThreshold = 'hey';
args.validate();
}).toThrow(
new Error(
"If set, severityThreshold must be 'critical' or 'high' or 'medium' or 'low' (case insensitive). If not set, the default is 'low'.",
"If set, severityThreshold must be one from [critical,high,medium,low] (case insensitive). If not set, the default is 'low'.",
),
);
});

it.each(validSeverityThresholds)(
'passes validation for ${level}',
(level) => {
args.severityThreshold = level;
it('throws error if invalid severity threshold for container testType', () => {
expect(() => {
args.severityThreshold = 'hey';
args.testType = TestType.CONTAINER_IMAGE;
args.validate();
}).toThrow(
new Error(
"If set, severityThreshold must be one from [critical,high,medium,low] (case insensitive). If not set, the default is 'low'.",
),
);
});

it('throws error if invalid severity threshold for code', () => {
expect(() => {
args.codeSeverityThreshold = 'hey';
args.testType = TestType.CODE;
args.validate();
}).toThrow(
new Error(
"If set, severityThreshold must be one from [high,medium,low] (case insensitive). If not set, the default is 'low'.",
),
);
});

it.each(testTypeSeverityThreshold)(
'passes validation for each test type severity threshold',
(a, b) => {
args.testType = a as TestType;
for (const sev of b) {
args.severityThreshold = sev as Severity;
args.codeSeverityThreshold = sev as Severity;
args.failOnThreshold = sev as Severity;
args.validate();
}
},
);
});
Expand Down
54 changes: 49 additions & 5 deletions snykTask/src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import {
HTML_ATTACHMENT_TYPE,
doVulnerabilitiesExistForFailureThreshold,
Severity,
TestType,
} from './task-lib';
import * as fs from 'fs';
import * as path from 'path';
Expand Down Expand Up @@ -74,6 +75,7 @@ function parseInputArgs(): TaskArgs {
failOnIssues: tl.getBoolInput('failOnIssues', true),
});

taskArgs.testType = tl.getInput('testType', false) || TestType.APPLICATION;
taskArgs.targetFile = tl.getInput('targetFile', false);
taskArgs.dockerImageName = tl.getInput('dockerImageName', false);
taskArgs.dockerfilePath = tl.getInput('dockerfilePath', false);
Expand All @@ -90,6 +92,7 @@ function parseInputArgs(): TaskArgs {
tl.getInput('additionalArguments', false) || '';
taskArgs.testDirectory = tl.getInput('testDirectory', false);
taskArgs.severityThreshold = tl.getInput('severityThreshold', false);
taskArgs.codeSeverityThreshold = tl.getInput('codeSeverityThreshold', false);
taskArgs.failOnThreshold =
tl.getInput('failOnThreshold', false) || Severity.LOW;
taskArgs.ignoreUnknownCA = tl.getBoolInput('ignoreUnknownCA', false);
Expand All @@ -104,10 +107,14 @@ function parseInputArgs(): TaskArgs {
}

const logAllTaskArgs = (taskArgs: TaskArgs) => {
console.log(`taskArgs.testType: ${taskArgs.testType}`);
console.log(`taskArgs.targetFile: ${taskArgs.targetFile}`);
console.log(`taskArgs.dockerImageName: ${taskArgs.dockerImageName}`);
console.log(`taskArgs.dockerfilePath: ${taskArgs.dockerfilePath}`);
console.log(`taskArgs.severityThreshold: ${taskArgs.severityThreshold}`);
console.log(
`taskArgs.codeSeverityThreshold: ${taskArgs.codeSeverityThreshold}`,
);
console.log(`taskArgs.failOnThreshold: ${taskArgs.failOnThreshold}`);
console.log(`taskArgs.projectName: ${taskArgs.projectName}`);
console.log(`taskArgs.organization: ${taskArgs.organization}`);
Expand Down Expand Up @@ -146,12 +153,20 @@ async function runSnykTest(

const snykTestToolRunner = tl
.tool(snykPath)
.argIf(taskArgs.testType == TestType.CODE, 'code')
.argIf(
taskArgs.dockerImageName || taskArgs.testType == TestType.CONTAINER_IMAGE,
'container',
)
.arg('test')
.argIf(
taskArgs.severityThreshold,
taskArgs.testType != TestType.CODE && taskArgs.severityThreshold,
`--severity-threshold=${taskArgs.severityThreshold}`,
)
.argIf(taskArgs.dockerImageName, `--docker`)
.argIf(
taskArgs.testType == TestType.CODE && taskArgs.codeSeverityThreshold,
`--severity-threshold=${taskArgs.codeSeverityThreshold}`,
)
.argIf(taskArgs.dockerImageName, `${taskArgs.dockerImageName}`)
.argIf(fileArg, `--file=${fileArg}`)
.argIf(taskArgs.ignoreUnknownCA, `--insecure`)
Expand Down Expand Up @@ -179,9 +194,34 @@ async function runSnykTest(
if (snykTestExitCode >= CLI_EXIT_CODE_INVALID_USE) {
code = snykTestExitCode;
errorMsg =
'failing task because `snyk test` was improperly used or had other errors';
'failing task because `snyk` was improperly used or had other errors';
}
const snykOutput: SnykOutput = { code: code, message: errorMsg };

// handle snyk code no-issues-found non-existent json by rerunning --json stdout to piped file
if (
taskArgs.testType == TestType.CODE &&
!fs.existsSync(jsonReportOutputPath) &&
snykTestExitCode === CLI_EXIT_CODE_SUCCESS
) {
const echoToolRunner = tl.tool('echo');
const snykCodeTestToolRunner = tl
.tool(snykPath)
.arg('code')
.arg('test')
.arg('--json')
.argIf(
taskArgs.codeSeverityThreshold,
`--severity-threshold=${taskArgs.codeSeverityThreshold}`,
)
.argIf(taskArgs.ignoreUnknownCA, `--insecure`)
.argIf(taskArgs.organization, `--org=${taskArgs.organization}`)
.argIf(taskArgs.projectName, `--project-name=${projectNameArg}`)
.line(taskArgs.additionalArguments)
.pipeExecOutputToTool(echoToolRunner, jsonReportOutputPath);
await snykCodeTestToolRunner.exec(options);
}

removeRegexFromFile(
jsonReportOutputPath,
regexForRemoveCommandLine,
Expand Down Expand Up @@ -215,7 +255,7 @@ const runSnykToHTML = async (
if (snykToHTMLExitCode >= CLI_EXIT_CODE_INVALID_USE) {
code = snykToHTMLExitCode;
errorMsg =
'failing task because `snyk test` was improperly used or had other errors';
'failing task because `snyk` was improperly used or had other errors';
}
const snykOutput: SnykOutput = { code: code, message: errorMsg };
removeRegexFromFile(
Expand All @@ -241,10 +281,14 @@ async function runSnykMonitor(
taskVersion,
snykToken,
);
// not handling snyk code cli upload which is still a closed beta
const snykMonitorToolRunner = tl
.tool(snykPath)
.argIf(
taskArgs.dockerImageName || taskArgs.testType == TestType.CONTAINER_IMAGE,
'container',
)
.arg('monitor')
.argIf(taskArgs.dockerImageName, `--docker`)
.argIf(taskArgs.dockerImageName, `${taskArgs.dockerImageName}`)
.argIf(fileArg, `--file=${fileArg}`)
.argIf(taskArgs.organization, `--org=${taskArgs.organization}`)
Expand Down
53 changes: 32 additions & 21 deletions snykTask/src/task-args.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
*/

import * as tl from 'azure-pipelines-task-lib';
import { Severity } from './task-lib';
import { Severity, TestType, testTypeSeverityThreshold } from './task-lib';
export type MonitorWhen = 'never' | 'noIssuesFound' | 'always';
class TaskArgs {
testType: string | undefined = '';
Expand All @@ -35,6 +35,8 @@ class TaskArgs {
testDirectory: string | undefined = '';
additionalArguments: string = '';
ignoreUnknownCA: boolean = false;
// Snyk Code severity with its own pickList (no critical)
codeSeverityThreshold: string | undefined = '';

delayAfterReportGenerationSeconds: number = 0;

Expand All @@ -46,7 +48,10 @@ class TaskArgs {
public setMonitorWhen(rawInput?: string) {
if (rawInput) {
const lowerCaseInput = rawInput.toLowerCase();
if (lowerCaseInput === 'never' || lowerCaseInput === 'always') {
if (this.testType == TestType.CODE) {
console.log('Snyk Code publishes results using --report workflow');
this.monitorWhen = 'never';
} else if (lowerCaseInput === 'never' || lowerCaseInput === 'always') {
this.monitorWhen = lowerCaseInput;
} else if (lowerCaseInput === 'noissuesfound') {
this.monitorWhen = 'noIssuesFound';
Expand All @@ -57,8 +62,12 @@ class TaskArgs {
}
}
}

// disallow snyk code monitor which follows --report workflow
public shouldRunMonitor(snykTestSuccess: boolean): boolean {
if (this.monitorWhen === 'always') {
if (this.testType == TestType.CODE) {
return false;
} else if (this.monitorWhen === 'always') {
return true;
} else if (this.monitorWhen === 'never') {
return false;
Expand Down Expand Up @@ -102,32 +111,34 @@ class TaskArgs {
return this.projectName;
}

// validate based on testTypeSeverityThreshold applicable thresholds
public validate() {
const taskTestType = this.testType || TestType.APPLICATION;
const taskTestTypeThreshold = testTypeSeverityThreshold.get(taskTestType);

if (this.failOnThreshold) {
if (this.isNotValidThreshold(this.failOnThreshold)) {
const errorMsg = `If set, failOnThreshold must be '${Severity.CRITICAL}' or '${Severity.HIGH}' or '${Severity.MEDIUM}' or '${Severity.LOW}' (case insensitive). If not set, the default is '${Severity.LOW}'.`;
if (
!taskTestTypeThreshold?.includes(this.failOnThreshold.toLowerCase())
) {
const errorMsg = `If set, failOnThreshold must be one from [${taskTestTypeThreshold}] (case insensitive). If not set, the default is '${Severity.LOW}'.`;
throw new Error(errorMsg);
}
}

if (this.severityThreshold) {
if (this.isNotValidThreshold(this.severityThreshold)) {
const errorMsg = `If set, severityThreshold must be '${Severity.CRITICAL}' or '${Severity.HIGH}' or '${Severity.MEDIUM}' or '${Severity.LOW}' (case insensitive). If not set, the default is '${Severity.LOW}'.`;
throw new Error(errorMsg);
}
if (
(this.severityThreshold &&
!taskTestTypeThreshold?.includes(
this.severityThreshold.toLowerCase(),
)) ||
(this.codeSeverityThreshold &&
!taskTestTypeThreshold?.includes(
this.codeSeverityThreshold.toLowerCase(),
))
) {
const errorMsg = `If set, severityThreshold must be one from [${taskTestTypeThreshold}] (case insensitive). If not set, the default is '${Severity.LOW}'.`;
throw new Error(errorMsg);
}
}

private isNotValidThreshold(threshold: string) {
const severityThresholdLowerCase = threshold.toLowerCase();

return (
severityThresholdLowerCase !== Severity.CRITICAL &&
severityThresholdLowerCase !== Severity.HIGH &&
severityThresholdLowerCase !== Severity.MEDIUM &&
severityThresholdLowerCase !== Severity.LOW
);
}
}

export function getAuthToken() {
Expand Down
18 changes: 18 additions & 0 deletions snykTask/src/task-lib.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,24 @@ export enum Severity {
LOW = 'low',
}

export enum TestType {
APPLICATION = 'app',
CODE = 'code',
CONTAINER_IMAGE = 'container',
}

export const testTypeSeverityThreshold = new Map<string, Array<string>>([
[
TestType.APPLICATION,
[Severity.CRITICAL, Severity.HIGH, Severity.MEDIUM, Severity.LOW],
],
[TestType.CODE, [Severity.HIGH, Severity.MEDIUM, Severity.LOW]],
[
TestType.CONTAINER_IMAGE,
[Severity.CRITICAL, Severity.HIGH, Severity.MEDIUM, Severity.LOW],
],
]);

export function getSeverityOrdinal(severity: string): number {
switch (severity) {
case Severity.CRITICAL:
Expand Down
Loading

0 comments on commit e8d0ad7

Please sign in to comment.