v5,6: add useAdditionalVariablesOnly parameter (#63)

CHANGELOG.md

@@ -1,4 +1,11 @@
 # Changelog
+## 5.1.0
+Task 6.1.0
+- Add _useAdditionalVariablesOnly_ parameter ([#29](https://github.com/qetza/replacetokens-task/issues/60)).
+
+Task 5.4.0
+- Add _useAdditionalVariablesOnly_ parameter ([#29](https://github.com/qetza/replacetokens-task/issues/60)).
+
 ## 5.0.8
 - Add links in README to migration documentation.
 

README.md

@@ -277,6 +277,11 @@ If you are migrating from **v3 to v6** make sure to read this documentation firs
     #
     # Optional. Default: )
     transformsSuffix: ''
+
+    # Use only variables declared in 'additionalVariables'.
+    #
+    # Optional: Default: false
+    useAdditionalVariablesOnly: ''
 ```
 
 ### Output
@@ -354,6 +359,7 @@ The following **anonymous** data is send:
   - _transforms_
   - _transformsPrefix_
   - _transformsSuffix_
+  - _useAdditionalVariablesOnly_
 - the **number of** _sources_ entries
 - the **number of** _additionalVariables_ entries referencing file
 - the **number of** _additionalVariables_ entries referencing environment variables

tasks/ReplaceTokensV5/CHANGELOG.md

@@ -1,4 +1,7 @@
 # Changelog
+## 5.4.0
+- Add _useAdditionalVariablesOnly_ parameter ([#29](https://github.com/qetza/replacetokens-task/issues/60)).
+
 ## 5.3.3
 - Fix telemetry account hash.
 

tasks/ReplaceTokensV5/index.ts

@@ -42,6 +42,7 @@ interface Options {
   readonly enableRecursion: boolean;
   readonly useLegacyEmptyFeature: boolean;
   readonly useDefaultValue: boolean;
+  readonly useAdditionalVariablesOnly: boolean;
 }
 
 interface Rule {
@@ -256,8 +257,7 @@ var replaceTokensInString = function (
     if (options.enableRecursion && names.includes(name)) throw new Error("recursion cycle with token '" + name + "'.");
 
     // replace value
-    let value: string = tl.getVariable(name);
-    if (name in externalVariables) value = externalVariables[name];
+    let value: string = name in externalVariables || options.useAdditionalVariablesOnly ? externalVariables[name] : tl.getVariable(name);
 
     let usedDefaultValue: boolean = false;
     if (
@@ -549,7 +549,8 @@ async function run() {
       enableTransforms: tl.getBoolInput('enableTransforms', false),
       enableRecursion: tl.getBoolInput('enableRecursion', false),
       useLegacyEmptyFeature: tl.getBoolInput('useLegacyEmptyFeature', false),
-      useDefaultValue: tl.getBoolInput('useDefaultValue', false)
+      useDefaultValue: tl.getBoolInput('useDefaultValue', false),
+      useAdditionalVariablesOnly: tl.getBoolInput('useAdditionalVariablesOnly', false)
     };
     const transformPrefix: string = tl.getInput('transformPrefix', false) || '(';
     const transformSuffix: string = tl.getInput('transformSuffix', false) || ')';
@@ -739,6 +740,7 @@ async function run() {
     telemetryEvent.enableRecursion = options.enableRecursion;
     telemetryEvent.useLegacyEmptyFeature = options.useLegacyEmptyFeature;
     telemetryEvent.useDefaultValue = options.useDefaultValue;
+    telemetryEvent.useAdditionalVariablesOnly = options.useAdditionalVariablesOnly;
 
     // process files
     rules.forEach(rule => {

tasks/ReplaceTokensV5/task.json

@@ -12,8 +12,8 @@
   "author": "Guillaume ROUCHON",
   "version": {
     "Major": 5,
-    "Minor": 3,
-    "Patch": 3
+    "Minor": 4,
+    "Patch": 0
   },
   "releaseNotes": "Migrate to Node10 handler (breaking change).<br/>Add Node16 handler.",
   "instanceNameFormat": "Replace tokens in $(targetFiles)",
@@ -251,6 +251,15 @@
       "required": false,
       "helpMarkDown": "The separtor to use in variable names for nested objects in inline variables or variable files.<br/>Example: {'My':{'Value':'Hello World!'}} will create a variable 'My.Value' with the value 'Hello World!'"
     },
+    {
+      "name": "useAdditionalVariablesOnly",
+      "type": "boolean",
+      "label": "Use only file and inline variables",
+      "defaultValue": "false",
+      "groupName": "variables",
+      "required": false,
+      "helpMarkDown": "Use only variables declared in variable files and inline variables."
+    },
     {
       "name": "enableRecursion",
       "type": "boolean",

tasks/ReplaceTokensV5/telemetry.ts

@@ -125,4 +125,5 @@ export class TelemetryEvent {
   enableRecursion: boolean;
   useLegacyEmptyFeature: boolean;
   useDefaultValue: boolean;
+  useAdditionalVariablesOnly: boolean;
 }

tasks/ReplaceTokensV5/tests/L0.ts

@@ -191,7 +191,7 @@ describe('ReplaceTokens v5 L0 suite', function () {
 
           tr.stdout.should.include('telemetry sent');
           tr.stdout.should.match(
-            /\[\{"account":"494d0aad9d06c4ddb51d5300620122ce55366a9382b3cc2835ed5f0e2e67b4d0","pipeline":"b98ed03d3eec376dcc015365c1a944e3ebbcc33d30e3261af3f4e4abb107aa82","host":"server","os":"Windows","actionOnMissing":"warn","encoding":"auto","keepToken":false,"pattern":"#\\\\{\\\\s\*\(\(\?:\(\?!#\\\\{\)\(\?!\\\\s\*\\\\}#\)\.\)\*\)\\\\s\*\\\\}#","result":"success","rules":1,"rulesWithInputWildcard":0,"rulesWithNegativePattern":0,"rulesWithOutputPattern":0,"tokenPrefix":"#{","tokenSuffix":"}#","variableFiles":0,"verbosity":"normal","writeBOM":true,"useLegacyPattern":false,"enableTransforms":false,"transformPrefix":"\(","transformSuffix":"\)","transformPattern":"\\\\s\*\(\.\*\)\\\\\(\\\\s\*\(\(\?:\(\?!\\\\\(\)\(\?!\\\\s\*\\\\\)\)\.\)\*\)\\\\s\*\\\\\)\\\\s\*","defaultValue":"","tokenPattern":"default","actionOnNoFiles":"continue","inlineVariables":0,"enableRecursion":false,"useLegacyEmptyFeature":false,"useDefaultValue":false,"duration":\d+(?:\.\d+)?,"tokenReplaced":1,"tokenFound":1,"defaultValueReplaced":0,"fileProcessed":1,"transformExecuted":0,"eventType":"TokensReplaced","application":"replacetokens-task","version":"5.0.0"}]/
+            /\[\{"account":"494d0aad9d06c4ddb51d5300620122ce55366a9382b3cc2835ed5f0e2e67b4d0","pipeline":"b98ed03d3eec376dcc015365c1a944e3ebbcc33d30e3261af3f4e4abb107aa82","host":"server","os":"Windows","actionOnMissing":"warn","encoding":"auto","keepToken":false,"pattern":"#\\\\{\\\\s\*\(\(\?:\(\?!#\\\\{\)\(\?!\\\\s\*\\\\}#\)\.\)\*\)\\\\s\*\\\\}#","result":"success","rules":1,"rulesWithInputWildcard":0,"rulesWithNegativePattern":0,"rulesWithOutputPattern":0,"tokenPrefix":"#{","tokenSuffix":"}#","variableFiles":0,"verbosity":"normal","writeBOM":true,"useLegacyPattern":false,"enableTransforms":false,"transformPrefix":"\(","transformSuffix":"\)","transformPattern":"\\\\s\*\(\.\*\)\\\\\(\\\\s\*\(\(\?:\(\?!\\\\\(\)\(\?!\\\\s\*\\\\\)\)\.\)\*\)\\\\s\*\\\\\)\\\\s\*","defaultValue":"","tokenPattern":"default","actionOnNoFiles":"continue","inlineVariables":0,"enableRecursion":false,"useLegacyEmptyFeature":false,"useDefaultValue":false,"useAdditionalVariablesOnly":false,"duration":\d+(?:\.\d+)?,"tokenReplaced":1,"tokenFound":1,"defaultValueReplaced":0,"fileProcessed":1,"transformExecuted":0,"eventType":"TokensReplaced","application":"replacetokens-task","version":"5.0.0"}]/
           );
         },
         tr,
@@ -1583,5 +1583,28 @@ describe('ReplaceTokens v5 L0 suite', function () {
         done
       );
     });
+
+    it('should replace only with file or inline variables when specified', function (done: Mocha.Done) {
+      // arrange
+      let tp = path.join(__dirname, 'externalVariables', 'L0_UseAdditionalVariablesOnly.js');
+      let tr: ttm.MockTestRunner = new ttm.MockTestRunner(tp);
+
+      process.env['__inputpath__'] = copyData('variables.json', 'useonlyexternalvariables.json');
+      process.env['__variablespath__'] = path.join(data, 'externalvariables1.json');
+
+      // act
+      tr.run();
+
+      // assert
+      runValidation(
+        () => {
+          tr.succeeded.should.equal(true, 'task succeeded');
+
+          assertFilesEqual(process.env['__inputpath__'], path.join(data, 'variables.useadditionalvariablesonly.expected.json'), 'replaced output');
+        },
+        tr,
+        done
+      );
+    });
   });
 });

tasks/ReplaceTokensV5/tests/_data/variables.useadditionalvariablesonly.expected.json

@@ -0,0 +1,4 @@
+{
+  "var1": "var1_value",
+  "var2": ""
+}

tasks/ReplaceTokensV5/tests/externalVariables/L0_UseAdditionalVariablesOnly.ts

@@ -0,0 +1,41 @@
+import ma = require('azure-pipelines-task-lib/mock-answer');
+import tmrm = require('azure-pipelines-task-lib/mock-run');
+import path = require('path');
+
+const taskPath = path.join(__dirname, '..', '..', 'index.js');
+const tmr: tmrm.TaskMockRunner = new tmrm.TaskMockRunner(taskPath);
+
+// variables
+process.env['VAR1'] = 'variables';
+process.env['VAR2'] = 'variables';
+
+// inputs
+tmr.setInput('enableTelemetry', 'false');
+tmr.setInput('targetFiles', 'input.json');
+tmr.setInput('writeBOM', 'true');
+tmr.setInput('variableFiles', 'variables.ext');
+tmr.setInput('useAdditionalVariablesOnly', 'true');
+
+// sdk answers
+let answers = {
+  checkPath: {},
+  findMatch: {
+    'input.json': [process.env['__inputpath__']],
+    'variables.ext': [process.env['__variablespath__']]
+  },
+  stats: {},
+  exist: {}
+};
+answers['stats'][process.env['__inputpath__']] = {
+  isDirectory: false
+};
+answers['stats'][process.env['__variablespath__']] = {
+  isDirectory: false
+};
+answers['exist'][process.env['__inputpath__']] = true;
+answers['exist'][process.env['__variablespath__']] = true;
+
+tmr.setAnswers(answers);
+
+// act
+tmr.run();

tasks/ReplaceTokensV6/CHANGELOG.md

@@ -1,4 +1,7 @@
 # Changelog
+## 6.1.0
+- Add _useAdditionalVariablesOnly_ parameter ([#29](https://github.com/qetza/replacetokens-task/issues/60)).
+
 ## 6.0.6
 - Fix default case sensitivity in sources and additional variables matching ([#29](https://github.com/qetza/replacetokens-task/issues/29)).
 - Fix default directories and files starting with a dot in sources and additional variables matching ([#29](https://github.com/qetza/replacetokens-task/issues/29)).

tasks/ReplaceTokensV6/README.md

@@ -277,6 +277,11 @@ If you are migrating from **v3 to v6** make sure to read this documentation firs
     #
     # Optional. Default: )
     transformsSuffix: ''
+
+    # Use only variables declared in 'additionalVariables'.
+    #
+    # Optional: Default: false
+    useAdditionalVariablesOnly: ''
 ```
 
 ### Output
@@ -354,6 +359,7 @@ The following **anonymous** data is send:
   - _transforms_
   - _transformsPrefix_
   - _transformsSuffix_
+  - _useAdditionalVariablesOnly_
 - the **number of** _sources_ entries
 - the **number of** _additionalVariables_ entries referencing file
 - the **number of** _additionalVariables_ entries referencing environment variables

tasks/ReplaceTokensV6/index.ts

@@ -117,6 +117,7 @@ async function run() {
     // load additional variables
     const separator = tl.getInput('variableSeparator') || rt.Defaults.Separator;
     const additionalVariables = await getAdditionalVariables(options.root, separator, options.sources.caseInsensitive, options.sources.dot);
+    const useAdditionalVariablesOnly = tl.getBoolInput('useAdditionalVariablesOnly');
 
     // set telemetry attributes
     telemetryEvent.setAttributes({
@@ -141,13 +142,18 @@ async function run() {
       transforms: options.transforms.enabled,
       'transforms-prefix': options.transforms.prefix,
       'transforms-suffix': options.transforms.suffix,
+      'use-additional-variables-only': useAdditionalVariablesOnly,
       'variable-files': variableFilesCount,
       'variable-envs': variablesEnvCount,
       'inline-variables': inlineVariablesCount
     });
 
     // replace tokens
-    const result = await rt.replaceTokens(sources, (name: string) => (name in additionalVariables ? additionalVariables[name] : tl.getVariable(name)), options);
+    const result = await rt.replaceTokens(
+      sources,
+      (name: string) => (name in additionalVariables || useAdditionalVariablesOnly ? additionalVariables[name] : tl.getVariable(name)),
+      options
+    );
 
     if (result.files === 0) {
       (msg => {

tasks/ReplaceTokensV6/task.json

@@ -12,8 +12,8 @@
   "author": "Guillaume ROUCHON",
   "version": {
     "Major": 6,
-    "Minor": 0,
-    "Patch": 6
+    "Minor": 1,
+    "Patch": 0
   },
   "releaseNotes": "breaking changes, see [changelog](https://github.com/qetza/replacetokens-task/blob/master/tasks/ReplaceTokensV6/CHANGELOG.md)",
   "instanceNameFormat": "Replace tokens",
@@ -133,6 +133,15 @@
       "label": "Separator",
       "helpMarkDown": "The separtor to use when flattening keys in variables."
     },
+    {
+      "name": "useAdditionalVariablesOnly",
+      "type": "boolean",
+      "required": false,
+      "defaultValue": false,
+      "visibleRule": "additionalVariables != \"\"",
+      "label": "Use only additional variables",
+      "helpMarkDown": "Use only variables declared in additional variables. Default: false"
+    },
     {
       "name": "escapeType",
       "aliases": ["escape"],

tasks/ReplaceTokensV6/tests/L0.ts

@@ -233,7 +233,7 @@ describe('ReplaceTokens v6 L0 suite', function () {
 
         tr.stdout.should.include('telemetry sent');
         tr.stdout.should.match(
-          /\[\{"eventType":"TokensReplaced","application":"replacetokens-task","version":"6.0.0","account":"494d0aad9d06c4ddb51d5300620122ce55366a9382b3cc2835ed5f0e2e67b4d0","pipeline":"b98ed03d3eec376dcc015365c1a944e3ebbcc33d30e3261af3f4e4abb107aa82","host":"cloud","os":"Windows","sources":3,"add-bom":false,"case-insenstive-paths":true,"encoding":"auto","escape":"auto","if-no-files-found":"ignore","include-dot-paths":true,"log-level":"info","missing-var-action":"none","missing-var-default":"","missing-var-log":"warn","recusrive":false,"separator":".","token-pattern":"default","transforms":false,"transforms-prefix":"\(","transforms-suffix":"\)","variable-files":0,"variable-envs":0,"inline-variables":0,"output-defaults":1,"output-files":2,"output-replaced":3,"output-tokens":4,"output-transforms":5,"result":"success","duration":\d+(?:\.\d+)?}]/
+          /\[\{"eventType":"TokensReplaced","application":"replacetokens-task","version":"6.0.0","account":"494d0aad9d06c4ddb51d5300620122ce55366a9382b3cc2835ed5f0e2e67b4d0","pipeline":"b98ed03d3eec376dcc015365c1a944e3ebbcc33d30e3261af3f4e4abb107aa82","host":"cloud","os":"Windows","sources":3,"add-bom":false,"case-insenstive-paths":true,"encoding":"auto","escape":"auto","if-no-files-found":"ignore","include-dot-paths":true,"log-level":"info","missing-var-action":"none","missing-var-default":"","missing-var-log":"warn","recusrive":false,"separator":".","token-pattern":"default","transforms":false,"transforms-prefix":"\(","transforms-suffix":"\)","use-additional-variables-only":false,"variable-files":0,"variable-envs":0,"inline-variables":0,"output-defaults":1,"output-files":2,"output-replaced":3,"output-tokens":4,"output-transforms":5,"result":"success","duration":\d+(?:\.\d+)?}]/
         );
       }, tr);
     } finally {
@@ -1077,4 +1077,44 @@ describe('ReplaceTokens v6 L0 suite', function () {
       actual.should.equal(expected);
     }, tr);
   });
+
+  it('useAdditionalVariablesOnly', async () => {
+    // arrange
+    const tp = path.join(__dirname, 'L0_NoMock.js');
+    const tr: ttm.MockTestRunner = new ttm.MockTestRunner(tp);
+
+    addVariables({ 'var.var': 'var', var_yaml2: 'var' });
+    addSecrets({ 'var.secret': 'secret', 'var.yml2': 'secret' });
+
+    let input = path.join(tmp, 'file.txt');
+    await fs.promises.copyFile(path.join(data, 'file.txt'), input);
+    input = path.resolve(input);
+
+    process.env['__telemetryOptout__'] = 'true';
+    process.env['__sources__'] = input;
+    process.env['__missingVarAction__'] = 'replace';
+    process.env['__missingVarDefault__'] = 'DEFAULT';
+    process.env['__root__'] = path.join(data, '..');
+    process.env['__VARS__'] = '{ "var1": "env", "var2": "env" }';
+    process.env['__useAdditionalVariablesOnly__'] = 'true';
+    process.env['__additionalVariables__'] = `
+- '@**/_data/vars.*;!**/*.xml'
+- '$__VARS__'
+- var2: inline
+  var.yml2: inline
+`;
+
+    // act
+    await tr.runAsync();
+
+    // assert
+    runValidations(() => {
+      tr.succeeded.should.be.true;
+
+      const actual = fs.readFileSync(input, 'utf8');
+      const expected = fs.readFileSync(path.join(data, 'file.only_additional_variables.expected.txt'), 'utf8');
+
+      actual.should.equal(expected);
+    }, tr);
+  });
 });

tasks/ReplaceTokensV6/tests/L0_NoMock.ts

@@ -27,6 +27,7 @@ if (process.env['__tokenSuffix__']) tmr.setInput('tokenSuffix', process.env['__t
 if (process.env['__transforms__']) tmr.setInput('enableTransforms', process.env['__transforms__']);
 if (process.env['__transformsPrefix__']) tmr.setInput('transformPrefix', process.env['__transformsPrefix__']);
 if (process.env['__transformsSuffix__']) tmr.setInput('transformSuffix', process.env['__transformsSuffix__']);
+if (process.env['__useAdditionalVariablesOnly__']) tmr.setInput('useAdditionalVariablesOnly', process.env['__useAdditionalVariablesOnly__']);
 
 // mocks
 const axiosClone = Object.assign({}, require('axios'));

tasks/ReplaceTokensV6/tests/L0_Run.ts

@@ -29,6 +29,7 @@ if (process.env['__tokenSuffix__']) tmr.setInput('tokenSuffix', process.env['__t
 if (process.env['__transforms__']) tmr.setInput('enableTransforms', process.env['__transforms__']);
 if (process.env['__transformsPrefix__']) tmr.setInput('transformPrefix', process.env['__transformsPrefix__']);
 if (process.env['__transformsSuffix__']) tmr.setInput('transformSuffix', process.env['__transformsSuffix__']);
+if (process.env['__useAdditionalVariablesOnly__']) tmr.setInput('useAdditionalVariablesOnly', process.env['__useAdditionalVariablesOnly__']);
 
 // mocks
 const rtClone = Object.assign({}, require('@qetza/replacetokens'));

tasks/ReplaceTokensV6/tests/_data/file.only_additional_variables.expected.txt

@@ -0,0 +1,10 @@
+var.var: DEFAULT
+var.secret: DEFAULT
+var1: env
+var2: inline
+var_json: file
+var_yaml1: file
+var_yaml2: file
+var.yml1: file
+var.yml2: inline
+unknown: DEFAULT

vss-extension.json

@@ -2,7 +2,7 @@
   "manifestVersion": 1,
   "id": "replacetokens",
   "name": "Replace Tokens",
-  "version": "5.0.8",
+  "version": "5.1.0",
   "public": true,
   "publisher": "qetza",
   "targets": [