chore: Try to improve smoke test tracing to figure failure/flaky case…

… causes [INS-5056] (#8431) * improve the tracing
Kong · Mar 7, 2025 · bbf2934 · bbf2934
1 parent 6a6483b
commit bbf2934
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 10 deletions.
diff --git a/packages/insomnia-smoke-test/playwright/test.ts b/packages/insomnia-smoke-test/playwright/test.ts
@@ -123,12 +123,25 @@ export const test = baseTest.extend<{
       await appContext.tracing.start(traceOptions);
     }
 
-    await use(electronApp);
-
-    if (captureTrace) {
-      await appContext.tracing.stop({
-        path: path.join(testInfo.outputDir, 'trace.zip'),
-      });
+    let testFailed = false;
+    try {
+      await use(electronApp);
+    } catch (error) {
+      testFailed = true;
+      throw error;
+    } finally {
+      // set testFailed to true if the test timed out or failed
+      testFailed = testFailed || testInfo.status === 'timedOut' || testInfo.status === 'failed';
+      if (traceMode === 'on' || (traceMode === 'retain-on-failure' && testFailed) || (traceMode === 'on-first-retry' && testInfo.retry === 1)) {
+        // Use a different name rather than the default trace.zip to avoid overwriting the trace.
+        // Refer: https://github.com/microsoft/playwright/issues/35005
+        await appContext.tracing.stop({
+          path: path.join(testInfo.outputDir, `trace-${testInfo.title}-${testInfo.status}.zip`),
+        });
+      } else {
+        // Discard the trace if not needed
+        await appContext.tracing.stop();
+      }
     }
 
     await electronApp.close();

diff --git a/packages/insomnia-smoke-test/tests/smoke/pre-request-script-features.test.ts b/packages/insomnia-smoke-test/tests/smoke/pre-request-script-features.test.ts
@@ -445,8 +445,8 @@ test.describe('pre-request features tests', async () => {
         await page.getByRole('tab', { name: 'Tests' }).click();
 
         const responsePane = page.getByTestId('response-pane');
-        expect(responsePane).toContainText('FAILunhappy tests | error: AssertionError: expected 199 to deeply equal 200 | ACTUAL: 199 | EXPECTED: 200Pre-request Test');
-        expect(responsePane).toContainText('PASShappy tests');
+        await expect(responsePane).toContainText('FAILunhappy tests | error: AssertionError: expected 199 to deeply equal 200 | ACTUAL: 199 | EXPECTED: 200Pre-request Test');
+        await expect(responsePane).toContainText('PASShappy tests');
     });
 
     test('environment and baseEnvironment can be persisted', async ({ page }) => {

diff --git a/packages/insomnia-smoke-test/tests/smoke/runner.test.ts b/packages/insomnia-smoke-test/tests/smoke/runner.test.ts
@@ -291,9 +291,9 @@ test.describe('runner features tests', async () => {
 
             const consoleTabContent = page.locator('.pane-two');
             if (expectToHaveLog) {
-                expect(consoleTabContent).toContainText("it won't print");
+                await expect(consoleTabContent).toContainText("it won't print");
             } else {
-                expect(consoleTabContent).not.toContainText("it won't print");
+                await expect(consoleTabContent).not.toContainText("it won't print");
             }
         }
     });