feat: proposal pre/post test steps (#210)

README.md

@@ -70,6 +70,15 @@ If the proposal is _pending_ and does not yet have a number, use a letter. The p
 - `package.json` specifies what kind of proposal it is in a `agoricProposal` field. If it's a "Software Upgrade Proposal" it also includes additional parameters.
 - `use.sh` is the script that will be run in the USE stage of the build
 - `test.sh` is the script that will be _included_ in the TEST stage of the build, and run in CI
+- `test` folder contents will be copied only to the `test` image
+- `setup-test.sh` is an optional script which can be used to run setup steps _inside_ the container. It runs _before_ the chain starts.
+- `teardown-test.sh` is an optional script which can be used to run teardown steps _inside_ the container. It runs _after_ the chain stops.
+- `host` folder is the home of these scripts:
+
+  - `before-test-run.sh` is an optional script which can be used to run setup steps on the _host_ (like starting a follower). It runs _before_ the container launches.
+  - `after-test-run.sh` is an optional script which can be used to run teardown steps on the _host_ (like stopping a follower). It runs _after_ the container exits.
+
+  _`host` folder contents will never be copied to a Docker image_
 
 ## Development
 

packages/synthetic-chain/package.json

@@ -25,13 +25,15 @@
     "chalk": "^5.4.1",
     "cosmjs-types": "^0.9.0",
     "execa": "^9.5.2",
-    "glob": "^11.0.1"
+    "glob": "^11.0.1",
+    "tmp": "0.2.3"
   },
   "devDependencies": {
     "@agoric/cosmic-proto": "0.5.0-u18.5",
     "@types/better-sqlite3": "^7.6.12",
     "@types/glob": "^8.1.0",
     "@types/node": "^18.19.50",
+    "@types/tmp": "0.2.6",
     "ava": "^6.2.0",
     "ts-blank-space": "^0.5.1",
     "tsup": "^8.3.6",

packages/synthetic-chain/public/upgrade-test-scripts/run_test.sh

@@ -28,10 +28,10 @@ source ./env_setup.sh
 
 cd /usr/src/proposals/"$PROPOSAL/" || fail "Proposal $PROPOSAL does not exist"
 
-if test -f prepare-test.sh
+if test -f setup-test.sh
 then
-  echo "[$PROPOSAL] Running prepare-test.sh"
-  ./prepare-test.sh
+  echo "[$PROPOSAL] Running setup-test.sh"
+  ./setup-test.sh
 fi
 
 echo "[$PROPOSAL] Starting agd"
@@ -44,3 +44,9 @@ echo "[$PROPOSAL] Running test.sh."
 echo "[$PROPOSAL] Testing completed."
 
 killAgd
+
+if test -f teardown-test.sh
+then
+  echo "[$PROPOSAL] Running teardown-test.sh"
+  ./teardown-test.sh
+fi

packages/synthetic-chain/src/cli/cli.ts

@@ -117,7 +117,7 @@ switch (cmd) {
         console.log(chalk.cyan.bold(`Testing ${proposal.proposalName}`));
         const image = imageNameForProposal(proposal, 'test');
         bakeTarget(image.target, values.dry);
-        runTestImage(proposal);
+        runTestImage({ proposal });
         // delete the image to reclaim disk space. The next build
         // will use the build cache.
         execSync('docker system df', { stdio: 'inherit' });

packages/synthetic-chain/src/cli/dockerfileGen.ts

@@ -81,7 +81,7 @@ ENV \
     SKIP_PROPOSAL_VALIDATION=${skipProposalValidation}
 
 ${createCopyCommand(
-  ['node_modules', 'test', 'test.sh'],
+  ['host', 'node_modules', 'test', 'test.sh'],
   `./proposals/${path}`,
   `/usr/src/proposals/${path}`,
 )}
@@ -116,7 +116,7 @@ WORKDIR /usr/src/upgrade-test-scripts
 
 # base is a fresh sdk image so set up the proposal and its dependencies
 ${createCopyCommand(
-  ['node_modules', 'test', 'test.sh'],
+  ['host', 'node_modules', 'test', 'test.sh'],
   `./proposals/${path}`,
   `/usr/src/proposals/${path}`,
 )}
@@ -149,7 +149,7 @@ RUN ./run_execute.sh ${planName}
 FROM use-${lastProposal.proposalName} as eval-${proposalName}
 
 ${createCopyCommand(
-  ['node_modules', 'test', 'test.sh'],
+  ['host', 'node_modules', 'test', 'test.sh'],
   `./proposals/${path}`,
   `/usr/src/proposals/${path}`,
 )}
@@ -215,7 +215,7 @@ FROM use-${proposalName} as test-${proposalName}
 # to copy only missing files, but there may be none. Fortunately, copying extra
 # does not invalidate other images because nothing depends on this layer.
 ${createCopyCommand(
-  ['node_modules'],
+  ['host', 'node_modules'],
   `./proposals/${path}`,
   `/usr/src/proposals/${path}`,
 )}

packages/synthetic-chain/src/cli/run.ts

@@ -1,7 +1,29 @@
-import { execSync } from 'node:child_process';
-import { realpathSync } from 'node:fs';
+import { spawnSync } from 'node:child_process';
+import { existsSync, realpathSync } from 'node:fs';
+import { resolve as resolvePath } from 'node:path';
+import { fileSync as createTempFile } from 'tmp';
 import { ProposalInfo, imageNameForProposal } from './proposals.js';
 
+const createMessageFile = (proposal: ProposalInfo) =>
+  createTempFile({ prefix: proposal.proposalName });
+
+const executeHostScriptIfPresent = (
+  extraEnv: typeof process.env,
+  proposal: ProposalInfo,
+  scriptName: string,
+) => {
+  const scriptPath = `${resolvePath('.')}/proposals/${proposal.path}/host/${scriptName}`;
+  if (existsSync(scriptPath)) {
+    console.log(
+      `Running script ${scriptName} for proposal ${proposal.proposalName}`,
+    );
+    spawnSync(scriptPath, {
+      env: { ...process.env, ...extraEnv },
+      stdio: 'inherit',
+    });
+  }
+};
+
 /**
  * Used to propagate a SLOGFILE environment variable into Docker containers.
  * Any file identified by such a variable will be created if it does not already
@@ -14,40 +36,101 @@ const propagateSlogfile = env => {
   const { SLOGFILE } = env;
   if (!SLOGFILE) return [];
 
-  execSync('touch "$SLOGFILE"');
-  return ['-e', 'SLOGFILE', '-v', `"$SLOGFILE:${realpathSync(SLOGFILE)}"`];
+  spawnSync('touch', [SLOGFILE]);
+
+  return [
+    '--env',
+    `SLOGFILE=${SLOGFILE}`,
+    '--volume',
+    `${SLOGFILE}:${realpathSync(SLOGFILE)}`,
+  ];
 };
 
-export const runTestImage = (proposal: ProposalInfo) => {
-  console.log(`Running test image for proposal ${proposal.proposalName}`);
-  const { name } = imageNameForProposal(proposal, 'test');
-  const slogOpts = propagateSlogfile(process.env);
-  // 'rm' to remove the container when it exits
-  const cmd = `docker run ${slogOpts.join(' ')} --rm ${name}`;
-  execSync(cmd, { stdio: 'inherit' });
+export const runTestImage = ({
+  extraDockerArgs = [],
+  proposal,
+  removeContainerOnExit = true,
+}: {
+  extraDockerArgs?: Array<string>;
+  proposal: ProposalInfo;
+  removeContainerOnExit?: boolean;
+}) => {
+  const { name: messageFilePath, removeCallback: removeTempFileCallback } =
+    createMessageFile(proposal);
+
+  const containerFilePath = '/root/message-file-path';
+
+  try {
+    executeHostScriptIfPresent(
+      {
+        MESSAGE_FILE_PATH: messageFilePath,
+      },
+      proposal,
+      'before-test-run.sh',
+    );
+
+    console.log(`Running test image for proposal ${proposal.proposalName}`);
+    const { name } = imageNameForProposal(proposal, 'test');
+    spawnSync(
+      'docker',
+      [
+        'run',
+        '--env',
+        `MESSAGE_FILE_PATH=${containerFilePath}`,
+        '--mount',
+        `source=${messageFilePath},target=${containerFilePath},type=bind`,
+        ...(removeContainerOnExit ? ['--rm'] : []),
+        ...propagateSlogfile(process.env),
+        ...extraDockerArgs,
+        name,
+      ],
+      { stdio: 'inherit' },
+    );
+
+    executeHostScriptIfPresent(
+      {
+        MESSAGE_FILE_PATH: messageFilePath,
+      },
+      proposal,
+      'after-test-run.sh',
+    );
+  } finally {
+    removeTempFileCallback();
+  }
 };
 
 export const debugTestImage = (proposal: ProposalInfo) => {
   const { name } = imageNameForProposal(proposal, 'test');
   console.log(
     `
   Starting chain of test image for proposal ${proposal.proposalName}
-  
+
   To get an interactive shell in the container, use an IDE feature like "Attach Shell" or this command:'
-  
+
     docker exec -ti $(docker ps -q -f ancestor=${name}) bash
-  
+
   And within that shell:
     cd /usr/src/proposals/${proposal.path} && ./test.sh
-  
+
   To edit files you can use terminal tools like vim, or mount the container in your IDE.
   In VS Code the command is:
     Dev Containers: Attach to Running Container...
   `,
   );
-
-  const slogOpts = propagateSlogfile(process.env);
-  // start the chain with ports mapped
-  const cmd = `docker run ${slogOpts.join(' ')} --publish 26657:26657 --publish 1317:1317 --publish 9090:9090 --interactive --tty --entrypoint /usr/src/upgrade-test-scripts/start_agd.sh ${name}`;
-  execSync(cmd, { stdio: 'inherit' });
+  return runTestImage({
+    extraDockerArgs: [
+      '--entrypoint',
+      '/usr/src/upgrade-test-scripts/start_agd.sh',
+      '--interactive',
+      '--publish',
+      '1317:1317',
+      '--publish',
+      '9090:9090',
+      '--publish',
+      '26657:26657',
+      '--tty',
+    ],
+    proposal,
+    removeContainerOnExit: false,
+  });
 };

yarn.lock

@@ -26,12 +26,14 @@ __metadata:
     "@types/better-sqlite3": "npm:^7.6.12"
     "@types/glob": "npm:^8.1.0"
     "@types/node": "npm:^18.19.50"
+    "@types/tmp": "npm:0.2.6"
     ava: "npm:^6.2.0"
     better-sqlite3: "npm:^11.8.1"
     chalk: "npm:^5.4.1"
     cosmjs-types: "npm:^0.9.0"
     execa: "npm:^9.5.2"
     glob: "npm:^11.0.1"
+    tmp: "npm:0.2.3"
     ts-blank-space: "npm:^0.5.1"
     tsup: "npm:^8.3.6"
     typescript: "npm:^5.7.3"
@@ -626,11 +628,18 @@ __metadata:
   linkType: hard
 
 "@types/node@npm:^18.19.50":
-  version: 18.19.74
-  resolution: "@types/node@npm:18.19.74"
+  version: 18.19.50
+  resolution: "@types/node@npm:18.19.50"
   dependencies:
     undici-types: "npm:~5.26.4"
-  checksum: 10c0/365d9cc2af934965aa6a8471e24ae80add815c15dc094e42a320c57c1ea5416032f0b7ef6f23e32174c34811fbb8d89ea8eaa1396548610fbb8ba317b6e93fbf
+  checksum: 10c0/36e6bc9eb47213ce94a868dad9504465ad89fba6af9f7954e22bb27fb17a32ac495f263d0cf4fdaee74becd7b2629609a446ec8c2b59b7a07bd587567c8a4782
+  languageName: node
+  linkType: hard
+
+"@types/tmp@npm:0.2.6":
+  version: 0.2.6
+  resolution: "@types/tmp@npm:0.2.6"
+  checksum: 10c0/a11bfa2cd8eaa6c5d62f62a3569192d7a2c28efdc5c17af0b0551db85816b2afc8156f3ca15ac76f0b142ae1403f04f44279871424233a1f3390b2e5fc828cd0
   languageName: node
   linkType: hard
 
@@ -3326,6 +3335,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"tmp@npm:0.2.3":
+  version: 0.2.3
+  resolution: "tmp@npm:0.2.3"
+  checksum: 10c0/3e809d9c2f46817475b452725c2aaa5d11985cf18d32a7a970ff25b568438e2c076c2e8609224feef3b7923fa9749b74428e3e634f6b8e520c534eef2fd24125
+  languageName: node
+  linkType: hard
+
 "to-regex-range@npm:^5.0.1":
   version: 5.0.1
   resolution: "to-regex-range@npm:5.0.1"