From 4820ca6ededc7674c946ee9ea70968e2b3ef3559 Mon Sep 17 00:00:00 2001 From: thim81 Date: Thu, 20 Jan 2022 04:45:52 +0100 Subject: [PATCH] Filter - remove "unused" component items (#4) * Filter - remove "unused" component items Co-authored-by: Tim <> --- asyncapi-format.js | 112 ++++++++- bin/cli.js | 2 +- defaultFilter.json | 3 +- defaultSortComponents.json | 1 + test/test.js | 2 +- .../customFilter.yaml | 18 ++ test/yaml-filter-unused-components/input.yaml | 224 ++++++++++++++++++ .../options.yaml | 4 + .../yaml-filter-unused-components/output.yaml | 147 ++++++++++++ 9 files changed, 503 insertions(+), 10 deletions(-) create mode 100644 defaultSortComponents.json create mode 100644 test/yaml-filter-unused-components/customFilter.yaml create mode 100644 test/yaml-filter-unused-components/input.yaml create mode 100644 test/yaml-filter-unused-components/options.yaml create mode 100644 test/yaml-filter-unused-components/output.yaml diff --git a/asyncapi-format.js b/asyncapi-format.js index 381b3f2..623cd1d 100644 --- a/asyncapi-format.js +++ b/asyncapi-format.js @@ -3,7 +3,7 @@ const fs = require('fs'); const traverse = require('traverse'); -const {isString, isObject, isArray} = require("./util-types"); +const {isString, isArray, isObject} = require("./util-types"); const { adaCase, camelCase, @@ -93,13 +93,22 @@ function asyncapiSort(oaObj, options) { let jsonObj = JSON.parse(JSON.stringify(oaObj)); // Deep copy of the schema object let sortSet = options.sortSet || JSON.parse(fs.readFileSync(__dirname + "/defaultSort.json", 'utf8')); - + let sortComponentsSet = options.sortComponentsSet || JSON.parse(fs.readFileSync(__dirname + "/defaultSortComponents.json", 'utf8')); let debugStep = '' // uncomment // debugStep below to see which sort part is triggered // Recursive traverse through AsyncAPI document traverse(jsonObj).forEach(function (node) { // if (obj.hasOwnProperty(this.key) && obj[this.key] && typeof obj[this.key] === 'object') { if (typeof node === 'object') { + + // Components sorting by alphabet + if (this.parent && this.parent.key && this.parent.key && this.parent.key === 'components' + && sortComponentsSet.length > 0 && sortComponentsSet.includes(this.key) + ) { + // debugStep = 'Component sorting by alphabet' + node = prioritySort(node, []); + } + // Generic sorting if (sortSet.hasOwnProperty(this.key) && Array.isArray(sortSet[this.key])) { @@ -153,27 +162,70 @@ function asyncapiFilter(oaObj, options) { let defaultFilter = JSON.parse(fs.readFileSync(__dirname + "/defaultFilter.json", 'utf8')) let filterSet = Object.assign({}, defaultFilter, options.filterSet); const operationVerbs = ["subscribe", "publish"]; + options.unusedDepth = options.unusedDepth || 0; // Merge object filters const filterKeys = [...filterSet.operations]; const filterArray = [...filterSet.tags]; const filterProps = [...filterSet.operationIds, ...filterSet.flags]; + const stripUnused = [...filterSet.unusedComponents]; + + // Initiate components tracking + const comps = { + schemas: {}, + messages: {}, + parameters: {}, + messageTraits: {}, + operationTraits: {}, + meta: {total: 0} + } // Prepare unused components let unusedComp = { schemas: [], - responses: [], + messages: [], parameters: [], - examples: [], - requestBodies: [], - headers: [], + messageTraits: [], + operationTraits: [], meta: {total: 0} } + // Use options.unusedComp to collect unused components during multiple recursion + if (!options.unusedComp) options.unusedComp = JSON.parse(JSON.stringify(unusedComp)); let debugFilterStep = '' // uncomment // debugFilterStep below to see which sort part is triggered traverse(jsonObj).forEach(function (node) { + // Register components presence + if (get(this, 'parent.parent.key') && this.parent.parent.key === 'components') { + if (get(this, 'parent.key') && this.parent.key && comps[this.parent.key]) { + comps[this.parent.key][this.key] = {...comps[this.parent.key][this.key], present: true}; + comps.meta.total = comps.meta.total++; + } + } + // Register components usage + if (this.key === '$ref') { + if (node.startsWith('#/components/schemas/')) { + const compSchema = node.replace('#/components/schemas/', ''); + comps.schemas[compSchema] = {...comps.schemas[compSchema], used: true}; + } + if (node.startsWith('#/components/messages/')) { + const compMess = node.replace('#/components/messages/', ''); + comps.messages[compMess] = {...comps.messages[compMess], used: true}; + } + if (node.startsWith('#/components/parameters/')) { + const compParam = node.replace('#/components/parameters/', ''); + comps.parameters[compParam] = {...comps.parameters[compParam], used: true}; + } + if (node.startsWith('#/components/messageTraits/')) { + const compMessTraits = node.replace('#/components/messageTraits/', ''); + comps.messageTraits[compMessTraits] = {...comps.messageTraits[compMessTraits], used: true}; + } + if (node.startsWith('#/components/operationTraits/')) { + const compOpTraits = node.replace('#/components/operationTraits/', ''); + comps.operationTraits[compOpTraits] = {...comps.operationTraits[compOpTraits], used: true}; + } + } // Filter out object matching the "methods" if (filterKeys.length > 0 && filterKeys.includes(this.key)) { // debugFilterStep = 'Filter - methods' @@ -216,19 +268,50 @@ function asyncapiFilter(oaObj, options) { } }); + if (stripUnused.length > 0) { + const optFs = get(options, 'filterSet.unusedComponents', []) || []; + unusedComp.schemas = Object.keys(comps.schemas || {}).filter(key => !get(comps, `schemas[${key}].used`)); //comps.schemas[key]?.used); + if(optFs.includes('schemas')) options.unusedComp.schemas = [...options.unusedComp.schemas, ...unusedComp.schemas]; + unusedComp.messages = Object.keys(comps.messages || {}).filter(key => !get(comps, `messages[${key}].used`));//!comps.messages[key]?.used); + if(optFs.includes('messages')) options.unusedComp.messages = [...options.unusedComp.messages, ...unusedComp.messages]; + unusedComp.parameters = Object.keys(comps.parameters || {}).filter(key => !get(comps, `parameters[${key}].used`));//!comps.parameters[key]?.used); + if(optFs.includes('parameters')) options.unusedComp.parameters = [...options.unusedComp.parameters, ...unusedComp.parameters]; + unusedComp.messageTraits = Object.keys(comps.messageTraits || {}).filter(key => !get(comps, `messageTraits[${key}].used`));//!comps.messageTraits[key]?.used); + if(optFs.includes('messageTraits')) options.unusedComp.messageTraits = [...options.unusedComp.messageTraits, ...unusedComp.messageTraits]; + unusedComp.operationTraits = Object.keys(comps.operationTraits || {}).filter(key => !get(comps, `operationTraits[${key}].used`));//!comps.operationTraits[key]?.used); + if(optFs.includes('operationTraits')) options.unusedComp.operationTraits = [...options.unusedComp.operationTraits, ...unusedComp.operationTraits]; + unusedComp.meta.total = unusedComp.schemas.length + unusedComp.messages.length + unusedComp.parameters.length + unusedComp.messageTraits.length + unusedComp.operationTraits.length + } + // Clean-up jsonObj traverse(jsonObj).forEach(function (node) { + // Remove unused component + if (this.path[0] === 'components' && stripUnused.length > 0) { + if (stripUnused.includes(this.path[1]) && unusedComp[this.path[1]].includes(this.key)) { + // debugFilterStep = 'Filter - Remove unused components' + this.delete(); + } + } + // Remove empty objects if (node && Object.keys(node).length === 0 && node.constructor === Object) { // debugFilterStep = 'Filter - Remove empty objects' this.delete(); } - // Remove path items without operations + // Remove message items without operations // if (this.parent && this.parent.key === 'messages' && !operationVerbs.some(i => this.keys.includes(i))) { // this.delete(); // } }); + // Recurse to strip any remaining unusedComp, to a maximum depth of 10 + if (stripUnused.length > 0 && unusedComp.meta.total > 0 && options.unusedDepth <= 10) { + options.unusedDepth++; + const resultObj = asyncapiFilter(jsonObj, options); + jsonObj = resultObj.data; + unusedComp = JSON.parse(JSON.stringify(options.unusedComp)); + } + // Return result object return {data: jsonObj, resultData: {unusedComp: unusedComp}} } @@ -497,6 +580,21 @@ function changeCase(valueAsString, caseType) { } } +/** + * Alternative optional chaining function, to provide support for NodeJS 12 + * TODO replace this with native ?. optional chaining once NodeJS12 is deprecated. + * @param obj object + * @param path path to access the properties + * @param defaultValue + * @returns {T} + */ +function get(obj, path, defaultValue = undefined) { + const travel = regexp => String.prototype.split.call(path, regexp) + .filter(Boolean).reduce((res, key) => res !== null && res !== undefined ? res[key] : res, obj); + + const result = travel(/[,[\]]+?/) || travel(/[,[\].]+?/); + return result === undefined || result === obj ? defaultValue : result; +} module.exports = { asyncapiFilter: asyncapiFilter, diff --git a/bin/cli.js b/bin/cli.js index 2bc156b..892556c 100644 --- a/bin/cli.js +++ b/bin/cli.js @@ -197,7 +197,7 @@ async function run(asFile, options) { keys.map((comp) => { if (unusedComp && unusedComp[comp] && unusedComp[comp].length > 0) { unusedComp[comp].forEach(value => { - const spacer = (comp === 'requestBodies' ? `\t` : `\t\t`); + const spacer = (comp === 'messageTraits' || comp === 'operationTraits' ? `\t` : `\t\t`); cliOut.push(`- components/${comp}${spacer} "${value}"`); count++; }); diff --git a/defaultFilter.json b/defaultFilter.json index c752437..2e21fcf 100644 --- a/defaultFilter.json +++ b/defaultFilter.json @@ -3,5 +3,6 @@ "tags": [], "operationIds": [], "flags": [], - "flagValues": [] + "flagValues": [], + "unusedComponents": [] } diff --git a/defaultSortComponents.json b/defaultSortComponents.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/defaultSortComponents.json @@ -0,0 +1 @@ +[] diff --git a/test/test.js b/test/test.js index 9754561..d13efda 100644 --- a/test/test.js +++ b/test/test.js @@ -12,7 +12,7 @@ const tests = fs.readdirSync(__dirname).filter(file => { }); // SELECTIVE TESTING DEBUG -// const tests = ['yaml-filter-custom'] +// const tests = ['yaml-filter-unused-components'] // destroyOutput = true describe('asyncapi-format tests', () => { diff --git a/test/yaml-filter-unused-components/customFilter.yaml b/test/yaml-filter-unused-components/customFilter.yaml new file mode 100644 index 0000000..845decb --- /dev/null +++ b/test/yaml-filter-unused-components/customFilter.yaml @@ -0,0 +1,18 @@ +#methods: +# - get +#flags: +# - x-visibility +#flagValues: [] +#tags: +# - store +# - user +#operationIds: +# - addPet +# - findPetsByStatus +unusedComponents: + - schemas + - messages + - parameters + - messageTraits + - operationTraits + diff --git a/test/yaml-filter-unused-components/input.yaml b/test/yaml-filter-unused-components/input.yaml new file mode 100644 index 0000000..d7115da --- /dev/null +++ b/test/yaml-filter-unused-components/input.yaml @@ -0,0 +1,224 @@ +asyncapi: '2.2.0' +info: + title: Streetlights Kafka API + version: '1.0.0' + description: | + The Smartylighting Streetlights API allows you to remotely manage the city lights. + + ### Check out its awesome features: + + * Turn a specific streetlight on/off 🌃 + * Dim a specific streetlight 😎 + * Receive real-time information about environmental lighting conditions 📈 + license: + name: Apache 2.0 + url: https://www.apache.org/licenses/LICENSE-2.0 +servers: + production: + url: test.mosquitto.org:{port} + protocol: mqtt + description: Test broker + variables: + port: + description: Secure connection (TLS) is available through port 8883. + default: '1883' + enum: + - '1883' + - '8883' + security: + - apiKey: [] + - supportedOauthFlows: + - streetlights:on + - streetlights:off + - streetlights:dim + - openIdConnectWellKnown: [] +defaultContentType: application/json +channels: + smartylighting.streetlights.1.0.event.{streetlightId}.lighting.measured: + description: The topic on which measured values may be produced and consumed. + subscribe: + operationId: measuredStreetlight +# traits: +# - $ref: '#/components/operationTraits/kafka' + message: + $ref: '#/components/messages/turnOnOff' +# parameters: +# streetlightId: +# $ref: '#/components/parameters/streetlightId' + publish: + summary: Inform about environmental lighting conditions of a particular streetlight. + operationId: receiveLightMeasurement +# traits: +# - $ref: '#/components/operationTraits/kafka' + message: + $ref: '#/components/messages/lightMeasured' + smartylighting.streetlights.1.0.action.{streetlightId}.turn.on: +# parameters: +# streetlightId: +# $ref: '#/components/parameters/streetlightId' + subscribe: + operationId: turnOn +# traits: +# - $ref: '#/components/operationTraits/kafka' + message: + $ref: '#/components/messages/turnOnOff' + smartylighting.streetlights.1.0.action.{streetlightId}.turn.off: +# parameters: +# streetlightId: +# $ref: '#/components/parameters/streetlightId' + subscribe: + operationId: turnOff +# traits: +# - $ref: '#/components/operationTraits/kafka' +# message: +# $ref: '#/components/messages/turnOnOff' + smartylighting.streetlights.1.0.action.{streetlightId}.dim: +# parameters: +# streetlightId: +# $ref: '#/components/parameters/streetlightId' + subscribe: + operationId: dimLight +# traits: +# - $ref: '#/components/operationTraits/kafka' + message: + title: Dim light + summary: Command a particular streetlight to dim the lights. +# traits: +# - $ref: '#/components/messageTraits/commonHeaders' + payload: + type: object + properties: + percentage: + type: integer + description: Percentage to which the light should be dimmed to. + minimum: 0 + maximum: 100 +# sentAt: +# $ref: '#/components/schemas/sentAt' +components: + messages: + lightMeasured: + name: lightMeasured + title: Light measured + summary: Inform about environmental lighting conditions of a particular streetlight. + contentType: application/json +# traits: +# - $ref: '#/components/messageTraits/commonHeaders' + payload: + $ref: '#/components/schemas/lightMeasuredPayload' + turnOnOff: + name: turnOnOff + title: Turn on/off + summary: Command a particular streetlight to turn the lights on or off. +# traits: +# - $ref: '#/components/messageTraits/commonHeaders' + payload: + $ref: '#/components/schemas/turnOnOffPayload' + dimLight: + name: dimLight + title: Dim light + summary: Command a particular streetlight to dim the lights. +# traits: +# - $ref: '#/components/messageTraits/commonHeaders' + payload: + type: object + properties: + percentage: + type: integer + description: Percentage to which the light should be dimmed to. + minimum: 0 + maximum: 100 + sentAt: + $ref: '#/components/schemas/sentAt' + schemas: + lightMeasuredPayload: + type: object + properties: + lumens: + type: integer + minimum: 0 + description: Light intensity measured in lumens. + sentAt: + $ref: '#/components/schemas/sentAt' + turnOnOffPayload: + type: object + properties: + command: + type: string + enum: + - 'on' + - 'off' + description: Whether to turn on or off the light. + sentAt: + $ref: '#/components/schemas/sentAt' + dimLightPayload: + type: object + properties: + percentage: + type: integer + description: Percentage to which the light should be dimmed to. + minimum: 0 + maximum: 100 + sentAt: + $ref: '#/components/schemas/sentAt' + sentAt: + type: string + format: date-time + description: Date and time when the message was sent. + securitySchemes: + apiKey: + type: apiKey + in: user + description: Provide your API key as the user and leave the password empty. + supportedOauthFlows: + type: oauth2 + description: Flows to support OAuth 2.0 + flows: + implicit: + authorizationUrl: https://authserver.example/auth + scopes: + streetlights:on: Ability to switch lights on + streetlights:off: Ability to switch lights off + streetlights:dim: Ability to dim the lights + password: + tokenUrl: https://authserver.example/token + scopes: + streetlights:on: Ability to switch lights on + streetlights:off: Ability to switch lights off + streetlights:dim: Ability to dim the lights + clientCredentials: + tokenUrl: https://authserver.example/token + scopes: + streetlights:on: Ability to switch lights on + streetlights:off: Ability to switch lights off + streetlights:dim: Ability to dim the lights + authorizationCode: + authorizationUrl: https://authserver.example/auth + tokenUrl: https://authserver.example/token + refreshUrl: https://authserver.example/refresh + scopes: + streetlights:on: Ability to switch lights on + streetlights:off: Ability to switch lights off + streetlights:dim: Ability to dim the lights + openIdConnectWellKnown: + type: openIdConnect + openIdConnectUrl: https://authserver.example/.well-known + parameters: + streetlightId: + description: The ID of the streetlight. + schema: + type: string + messageTraits: + commonHeaders: + headers: + type: object + properties: + my-app-header: + type: integer + minimum: 0 + maximum: 100 + operationTraits: + kafka: + bindings: + kafka: + clientId: my-app-id diff --git a/test/yaml-filter-unused-components/options.yaml b/test/yaml-filter-unused-components/options.yaml new file mode 100644 index 0000000..d581280 --- /dev/null +++ b/test/yaml-filter-unused-components/options.yaml @@ -0,0 +1,4 @@ +verbose: true +no-sort: true +output: output.yaml +filterFile: customFilter.yaml diff --git a/test/yaml-filter-unused-components/output.yaml b/test/yaml-filter-unused-components/output.yaml new file mode 100644 index 0000000..a5994ee --- /dev/null +++ b/test/yaml-filter-unused-components/output.yaml @@ -0,0 +1,147 @@ +asyncapi: 2.2.0 +info: + title: Streetlights Kafka API + version: 1.0.0 + description: | + The Smartylighting Streetlights API allows you to remotely manage the city lights. + + ### Check out its awesome features: + + * Turn a specific streetlight on/off 🌃 + * Dim a specific streetlight 😎 + * Receive real-time information about environmental lighting conditions 📈 + license: + name: Apache 2.0 + url: https://www.apache.org/licenses/LICENSE-2.0 +servers: + production: + url: test.mosquitto.org:{port} + protocol: mqtt + description: Test broker + variables: + port: + description: Secure connection (TLS) is available through port 8883. + default: '1883' + enum: + - '1883' + - '8883' + security: + - apiKey: [] + - supportedOauthFlows: + - streetlights:on + - streetlights:off + - streetlights:dim + - openIdConnectWellKnown: [] +defaultContentType: application/json +channels: + smartylighting.streetlights.1.0.event.{streetlightId}.lighting.measured: + description: The topic on which measured values may be produced and consumed. + subscribe: + operationId: measuredStreetlight + message: + $ref: '#/components/messages/turnOnOff' + publish: + summary: Inform about environmental lighting conditions of a particular streetlight. + operationId: receiveLightMeasurement + message: + $ref: '#/components/messages/lightMeasured' + smartylighting.streetlights.1.0.action.{streetlightId}.turn.on: + subscribe: + operationId: turnOn + message: + $ref: '#/components/messages/turnOnOff' + smartylighting.streetlights.1.0.action.{streetlightId}.turn.off: + subscribe: + operationId: turnOff + smartylighting.streetlights.1.0.action.{streetlightId}.dim: + subscribe: + operationId: dimLight + message: + title: Dim light + summary: Command a particular streetlight to dim the lights. + payload: + type: object + properties: + percentage: + type: integer + description: Percentage to which the light should be dimmed to. + minimum: 0 + maximum: 100 +components: + messages: + lightMeasured: + name: lightMeasured + title: Light measured + summary: Inform about environmental lighting conditions of a particular streetlight. + contentType: application/json + payload: + $ref: '#/components/schemas/lightMeasuredPayload' + turnOnOff: + name: turnOnOff + title: Turn on/off + summary: Command a particular streetlight to turn the lights on or off. + payload: + $ref: '#/components/schemas/turnOnOffPayload' + schemas: + lightMeasuredPayload: + type: object + properties: + lumens: + type: integer + minimum: 0 + description: Light intensity measured in lumens. + sentAt: + $ref: '#/components/schemas/sentAt' + turnOnOffPayload: + type: object + properties: + command: + type: string + enum: + - 'on' + - 'off' + description: Whether to turn on or off the light. + sentAt: + $ref: '#/components/schemas/sentAt' + sentAt: + type: string + format: date-time + description: Date and time when the message was sent. + securitySchemes: + apiKey: + type: apiKey + in: user + description: Provide your API key as the user and leave the password empty. + supportedOauthFlows: + type: oauth2 + description: Flows to support OAuth 2.0 + flows: + implicit: + authorizationUrl: https://authserver.example/auth + scopes: + streetlights:on: Ability to switch lights on + streetlights:off: Ability to switch lights off + streetlights:dim: Ability to dim the lights + password: + tokenUrl: https://authserver.example/token + scopes: + streetlights:on: Ability to switch lights on + streetlights:off: Ability to switch lights off + streetlights:dim: Ability to dim the lights + clientCredentials: + tokenUrl: https://authserver.example/token + scopes: + streetlights:on: Ability to switch lights on + streetlights:off: Ability to switch lights off + streetlights:dim: Ability to dim the lights + authorizationCode: + authorizationUrl: https://authserver.example/auth + tokenUrl: https://authserver.example/token + refreshUrl: https://authserver.example/refresh + scopes: + streetlights:on: Ability to switch lights on + streetlights:off: Ability to switch lights off + streetlights:dim: Ability to dim the lights + openIdConnectWellKnown: + type: openIdConnect + openIdConnectUrl: https://authserver.example/.well-known