conformance wip

.github/actions/specs-report/__tests__/main.test.ts

@@ -7,11 +7,20 @@
  */
 
 import * as core from '@actions/core'
+import * as junit2json from 'junit2json'
+
 import * as main from '../src/main'
 import * as files from '../src/files'
 import * as prComment from '../src/pr-comment'
-import * as junit2json from 'junit2json'
-import { TBDEX_TEST_VECTORS_FILES, SUCCESS_MOCK, FAILURE_MOCK } from './mocks'
+import { ConformanceData } from '../src/spec-release'
+
+import {
+  TBDEX_TEST_VECTORS_FILES,
+  SUCCESS_MOCK,
+  FAILURE_MOCK,
+  WEB5_TEST_VECTORS_FILES,
+  WEB5_TEST_VECTORS_FEATURES
+} from './mocks'
 
 // Mock the action's main function
 const runMock = jest.spyOn(main, 'run')
@@ -29,19 +38,49 @@ let readJsonFileMock: jest.SpiedFunction<typeof files.readJsonFile>
 let junitParseMock: jest.SpiedFunction<typeof junit2json.parse>
 let addCommentToPrMock: jest.SpiedFunction<typeof prComment.addCommentToPr>
 
-const defaultGetInputMockImplementation = (): void => {
+// Mock Octokit
+const mockOctokit = {
+  rest: {
+    repos: {
+      createOrUpdateFileContents: jest.fn()
+    }
+  }
+}
+
+// Mock fetch
+const mockFetch = jest.fn()
+global.fetch = mockFetch
+
+jest.mock('@actions/github', () => ({
+  getOctokit: jest.fn().mockImplementation(() => mockOctokit),
+  context: {
+    eventName: 'pull_request',
+    payload: { pull_request: { number: 123 } },
+    repo: { owner: 'TBD54566975', repo: 'sdk-report-runner' }
+  }
+}))
+
+const defaultGetInputMockImplementation = (
+  releaseMode?: 'spec' | 'sdk',
+  releaseParams?: {
+    releaseRepo?: string
+    releaseTag?: string
+    specName?: string
+    specTag?: string
+  }
+): void => {
   getInputMock.mockImplementation(name => {
     switch (name) {
       case 'junit-report-paths':
-        return '500'
+        return './whatever'
       case 'suite-name-regex':
         return 'TbdexTestVector' // or 'Web5TestVector' for Web5
       case 'feature-regex':
         return 'TbdexTestVectors(\\w+)'
       case 'vector-regex':
         return 'TbdexTestVectors(\\w+) (\\w+)'
       case 'spec-path':
-        return '500'
+        return './whatever'
       case 'git-token':
         return 'fake-token'
       case 'comment-on-pr':
@@ -50,6 +89,16 @@ const defaultGetInputMockImplementation = (): void => {
         return 'false'
       case 'fail-on-failed-test-cases':
         return 'true'
+      case 'release-mode':
+        return releaseMode || ''
+      case 'release-repo':
+        return releaseParams?.releaseRepo || ''
+      case 'release-tag':
+        return releaseParams?.releaseTag || ''
+      case 'spec-name':
+        return releaseParams?.specName || ''
+      case 'spec-tag':
+        return releaseParams?.specTag || ''
       default:
         return ''
     }
@@ -79,6 +128,7 @@ describe('action', () => {
 
     getFilesMock = jest.spyOn(files, 'getFiles').mockImplementation()
     readJsonFileMock = jest.spyOn(files, 'readJsonFile').mockImplementation()
+    readJsonFileMock.mockReturnValue(VALID_SPEC_JSON_BODY)
 
     addCommentToPrMock = jest
       .spyOn(prComment, 'addCommentToPr')
@@ -94,7 +144,6 @@ describe('action', () => {
       .mockReturnValueOnce(Promise.resolve(SUCCESS_MOCK.junitFiles))
       // get test vector json
       .mockReturnValueOnce(Promise.resolve(TBDEX_TEST_VECTORS_FILES))
-    readJsonFileMock.mockReturnValue(VALID_SPEC_JSON_BODY)
 
     const rawJunitFile = files.readFile(SUCCESS_MOCK.junitFiles[0])
 
@@ -139,7 +188,6 @@ describe('action', () => {
       .mockReturnValueOnce(Promise.resolve(FAILURE_MOCK.junitFiles))
       // get test vector json
       .mockReturnValueOnce(Promise.resolve(TBDEX_TEST_VECTORS_FILES))
-    readJsonFileMock.mockReturnValue(VALID_SPEC_JSON_BODY)
 
     const rawJunitFile = files.readFile(FAILURE_MOCK.junitFiles[0])
 
@@ -172,4 +220,64 @@ describe('action', () => {
     const testVectorReport = JSON.parse(output['test-vector-report'])
     expect(testVectorReport).toEqual(expectedReport)
   })
+
+  it('generates a spec release json file successfully', async () => {
+    defaultGetInputMockImplementation('spec', {
+      releaseRepo: 'TBD54566975/web5-spec',
+      specName: 'web5-spec',
+      specTag: 'v2.0'
+    })
+
+    //   defaultGetInputMockImplementation('sdk', {
+    //     releaseRepo: 'TBD54566975/web5-kt',
+    //     releaseTag: 'v1.9.3',
+    //     specName: 'web5-spec',
+    //     specTag: 'v2.0'
+    //   })
+
+    getFilesMock
+      // get test vector json
+      .mockReturnValueOnce(Promise.resolve(WEB5_TEST_VECTORS_FILES))
+
+    const expectedConformanceData: ConformanceData = {
+      specReleases: [
+        {
+          version: 'v2.0',
+          releaseLink:
+            'https://github.com/TBD54566975/web5-spec/releases/tag/v2.0',
+          testVectors: {
+            srcLink:
+              'https://github.com/TBD54566975/web5-spec/tree/v2.0/test-vectors',
+            cases: WEB5_TEST_VECTORS_FEATURES
+          },
+          sdks: {}
+        }
+      ]
+    }
+
+    mockFetch.mockReturnValue(
+      Promise.resolve({
+        status: 404
+      })
+    )
+
+    await main.run()
+    expect(runMock).toHaveReturned()
+
+    expect(
+      mockOctokit.rest.repos.createOrUpdateFileContents
+    ).toHaveBeenCalledTimes(1)
+    expect(
+      mockOctokit.rest.repos.createOrUpdateFileContents
+    ).toHaveBeenCalledWith({
+      owner: 'TBD54566975',
+      repo: 'sdk-report-runner',
+      path: 'spec-conformance-web5-spec.json',
+      message: `[ci] Update spec-conformance-web5-spec.json: [email protected]`,
+      content: Buffer.from(
+        JSON.stringify(expectedConformanceData, null, 2)
+      ).toString('base64'),
+      branch: 'gh-pages'
+    })
+  })
 })

.github/actions/specs-report/__tests__/mocks.ts

@@ -294,6 +294,23 @@ export const WEB5_TEST_VECTORS_FILES = [
   '/home/runner/work/web5-kt/web5-kt/web5-spec/test-vectors/presentation_exchange/validate_definition.json',
   '/home/runner/work/web5-kt/web5-kt/web5-spec/test-vectors/presentation_exchange/validate_submission.json',
   '/home/runner/work/web5-kt/web5-kt/web5-spec/test-vectors/vc_jwt/decode.json',
-  '/home/runner/work/web5-kt/web5-kt/web5-spec/test-vectors/vc_jwt/verify.json',
-  '/home/runner/work/web5-kt/web5-kt/web5-spec/test-vectors/vectors.schema.json'
+  '/home/runner/work/web5-kt/web5-kt/web5-spec/test-vectors/vc_jwt/verify.json'
 ]
+
+export const WEB5_TEST_VECTORS_FEATURES = {
+  Credentials: ['create', 'verify'],
+  CryptoEd25519: ['sign', 'verify'],
+  CryptoEs256k: ['sign', 'verify'],
+  DidDht: ['create', 'resolve'],
+  DidJwk: ['resolve'],
+  DidWeb: ['resolve'],
+  PortableDid: ['parse'],
+  PresentationExchange: [
+    'create_presentation_from_credentials',
+    'evaluate_presentation',
+    'select_credentials',
+    'validate_definition',
+    'validate_submission'
+  ],
+  VcJwt: ['decode', 'verify']
+}

.github/actions/specs-report/__tests__/test-vectors.test.ts

@@ -39,12 +39,12 @@ describe('test-vectors', () => {
     expect(parseJunitTestCasesMock).toHaveBeenCalledWith(WEB5_KT_JUNIT_FILES)
 
     expect(report.totalJunitFiles).toBe(2)
-    expect(report.totalTestVectors).toBe(19)
+    expect(report.totalTestVectors).toBe(18)
     expect(report.totalJunitTestCases).toBe(7)
     expect(report.specTestCases).toBe(6)
     expect(report.specFailedTestCases).toBe(0)
     expect(report.specPassedTestCases).toBe(6)
-    expect(report.missingVectors).toHaveLength(13)
+    expect(report.missingVectors).toHaveLength(12)
     expect(report.failedVectors).toHaveLength(0)
     expect(report.skippedVectors).toHaveLength(0)
     expect(report.successVectors).toEqual([
@@ -144,12 +144,12 @@ describe('test-vectors', () => {
     expect(parseJunitTestCasesMock).toHaveBeenCalledWith(swiftJunitFiles)
 
     expect(report.totalJunitFiles).toBe(1)
-    expect(report.totalTestVectors).toBe(19)
+    expect(report.totalTestVectors).toBe(18)
     expect(report.totalJunitTestCases).toBe(113)
     expect(report.specTestCases).toBe(7)
     expect(report.specFailedTestCases).toBe(0)
     expect(report.specPassedTestCases).toBe(7)
-    expect(report.missingVectors).toHaveLength(12)
+    expect(report.missingVectors).toHaveLength(11)
     expect(report.failedVectors).toHaveLength(0)
     expect(report.skippedVectors).toHaveLength(0)
     expect(report.successVectors).toEqual([

.github/actions/specs-report/action.yml

@@ -7,10 +7,10 @@ inputs:
     description:
       'Glob Path with the JUnit test vectors report, can be a single path or a
       list of paths in an array'
-    required: true
+    required: true # required if release-mode is "none"
   spec-path:
     description: 'The relative path to the tbd spec submodule folder'
-    required: true
+    required: true # required if release-mode is "none"
   suite-name-regex:
     description: 'The regex to filter the suite name'
     required: false
@@ -33,6 +33,30 @@ inputs:
   fail-on-failed-test-cases:
     description: 'Whether to fail the job if failed test cases are found'
     required: false
+  release-mode:
+    description: 'Set to "spec" or "sdk" to handle release updates'
+    required: false
+    default: 'none'
+  release-repo:
+    description:
+      'The release repo, eg: TBD54566975/web5-spec for spec releases,
+      TBD54566975/web5-rs for a sdk release'
+    required: false
+    default: 'none'
+  release-package-name:
+    description:
+      'The name of the package that is being released (useful for monorepos, eg:
+      web5-rs releases web5-core-kt)'
+    required: false
+  release-tag:
+    description: 'The tag of the release'
+    required: false
+  spec-name:
+    description: 'The name of the spec, eg: web5-spec'
+    required: false
+  spec-tag:
+    description: 'The tag of the spec release'
+    required: false # required if release-mode is "spec"
 
 # Define your outputs here.
 outputs: