Add retry e2e tests

jestjs · SimenB · Jan 15, 2025 · Oct 25, 2023 · Oct 25, 2023 · Oct 26, 2023
commit 5f27eb732d2f812b5dae0804357c45eaf8440888
diff --git a/e2e/__tests__/__snapshots__/testRetries.test.ts.snap b/e2e/__tests__/__snapshots__/testRetries.test.ts.snap
@@ -1,5 +1,80 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`Concurrent Test Retries with flag retryImmediately retry immediately after failed test 1`] = `
+"LOGGING RETRY ERRORS  retryable test 1
+ RETRY 1 
+
+    expect(received).toBeFalsy()
+
+    Received: true
+
+      15 |     expect(true).toBeTruthy();
+      16 |   } else {
+    > 17 |     expect(true).toBeFalsy();
+         |                  ^
+      18 |   }
+      19 | });
+      20 |
+
+      at Object.toBeFalsy (__tests__/retryImmediatelyConcurrent.test.js:17:18)
+
+ RETRY 2 
+
+    expect(received).toBeFalsy()
+
+    Received: true
+
+      15 |     expect(true).toBeTruthy();
+      16 |   } else {
+    > 17 |     expect(true).toBeFalsy();
+         |                  ^
+      18 |   }
+      19 | });
+      20 |
+
+      at Object.toBeFalsy (__tests__/retryImmediatelyConcurrent.test.js:17:18)
+          at async Promise.all (index 0)
+
+ LOGGING RETRY ERRORS  retryable test 2
+ RETRY 1 
+
+    expect(received).toBeFalsy()
+
+    Received: true
+
+      26 |     expect(true).toBeTruthy();
+      27 |   } else {
+    > 28 |     expect(true).toBeFalsy();
+         |                  ^
+      29 |   }
+      30 | });
+      31 | it.concurrent('truthy test', () => {
+
+      at Object.toBeFalsy (__tests__/retryImmediatelyConcurrent.test.js:28:18)
+
+ RETRY 2 
+
+    expect(received).toBeFalsy()
+
+    Received: true
+
+      26 |     expect(true).toBeTruthy();
+      27 |   } else {
+    > 28 |     expect(true).toBeFalsy();
+         |                  ^
+      29 |   }
+      30 | });
+      31 | it.concurrent('truthy test', () => {
+
+      at Object.toBeFalsy (__tests__/retryImmediatelyConcurrent.test.js:28:18)
+          at async Promise.all (index 1)
+
+PASS __tests__/retryImmediatelyConcurrent.test.js
+  ✓ retryable test 1
+  ✓ retryable test 2
+  ✓ truthy test"
+`;
+
 exports[`Test Retries logs error(s) before retry 1`] = `
 "LOGGING RETRY ERRORS  retryTimes set
  RETRY 1 

diff --git a/e2e/__tests__/testRetries.test.ts b/e2e/__tests__/testRetries.test.ts
@@ -173,3 +173,149 @@ describe('Test Retries', () => {
     expect(jsonResult.testResults[0].testResults[0].invocations).toBe(1);
   });
 });
+
+describe('Concurrent Test Retries', () => {
+  const outputFileName = 'retries.result.json';
+  const outputFilePath = path.join(
+    process.cwd(),
+    'e2e/test-retries/',
+    outputFileName,
+  );
+  const logErrorsBeforeRetryErrorMessage = 'LOGGING RETRY ERRORS';
+
+  afterAll(() => {
+    fs.unlinkSync(outputFilePath);
+  });
+
+  it('retries failed tests', () => {
+    const result = runJest('test-retries', ['e2eConcurrent.test.js']);
+
+    expect(result.exitCode).toBe(0);
+    expect(result.failed).toBe(false);
+    expect(result.stderr).not.toContain(logErrorsBeforeRetryErrorMessage);
+  });
+
+  it('with flag retryImmediately retry immediately after failed test', () => {
+    const logMessage = `console.log
+    FIRST TRUTHY TEST
+
+      at Object.log (__tests__/retryImmediatelyConcurrent.test.js:32:11)
+
+  console.log
+    SECOND TRUTHY TEST
+
+      at Object.log (__tests__/retryImmediatelyConcurrent.test.js:14:13)
+          at async Promise.all (index 0)
+
+  console.log
+    THIRD TRUTHY TEST
+
+      at Object.log (__tests__/retryImmediatelyConcurrent.test.js:25:13)
+          at async Promise.all (index 1)`;
+
+    const result = runJest('test-retries', [
+      'retryImmediatelyConcurrent.test.js',
+    ]);
+    const stdout = result.stdout.trim();
+    expect(result.exitCode).toBe(0);
+    expect(result.failed).toBe(false);
+    expect(result.stderr).toContain(logErrorsBeforeRetryErrorMessage);
+    expect(stdout).toBe(logMessage);
+    expect(extractSummary(result.stderr).rest).toMatchSnapshot();
+  });
+
+  it('reporter shows more than 1 invocation if test is retried', () => {
+    let jsonResult;
+
+    const reporterConfig = {
+      reporters: [
+        ['<rootDir>/reporters/RetryReporter.js', {output: outputFilePath}],
+      ],
+    };
+
+    runJest('test-retries', [
+      '--config',
+      JSON.stringify(reporterConfig),
+      '__tests__/retryConcurrent.test.js',
+    ]);
+
+    const testOutput = fs.readFileSync(outputFilePath, 'utf8');
+
+    try {
+      jsonResult = JSON.parse(testOutput);
+    } catch (error: any) {
+      throw new Error(
+        `Can't parse the JSON result from ${outputFileName}, ${error.toString()}`,
+      );
+    }
+
+    expect(jsonResult.numPassedTests).toBe(1);
+    expect(jsonResult.numFailedTests).toBe(1);
+    expect(jsonResult.numPendingTests).toBe(0);
+    expect(jsonResult.testResults[0].testResults[0].invocations).toBe(4);
+    expect(jsonResult.testResults[0].testResults[1].invocations).toBe(1);
+  });
+
+  it('reporter shows 1 invocation if tests are not retried', () => {
+    let jsonResult;
+
+    const reporterConfig = {
+      reporters: [
+        ['<rootDir>/reporters/RetryReporter.js', {output: outputFilePath}],
+      ],
+    };
+
+    runJest('test-retries', [
+      '--config',
+      JSON.stringify(reporterConfig),
+      'controlConcurrent.test.js',
+    ]);
+
+    const testOutput = fs.readFileSync(outputFilePath, 'utf8');
+
+    try {
+      jsonResult = JSON.parse(testOutput);
+    } catch (error: any) {
+      throw new Error(
+        `Can't parse the JSON result from ${outputFileName}, ${error.toString()}`,
+      );
+    }
+
+    expect(jsonResult.numPassedTests).toBe(0);
+    expect(jsonResult.numFailedTests).toBe(1);
+    expect(jsonResult.numPendingTests).toBe(0);
+    expect(jsonResult.testResults[0].testResults[0].invocations).toBe(1);
+  });
+
+  it('tests are not retried if beforeAll hook failure occurs', () => {
+    let jsonResult;
+
+    const reporterConfig = {
+      reporters: [
+        ['<rootDir>/reporters/RetryReporter.js', {output: outputFilePath}],
+      ],
+    };
+
+    runJest('test-retries', [
+      '--config',
+      JSON.stringify(reporterConfig),
+      'beforeAllFailureConcurrent.test.js',
+    ]);
+
+    const testOutput = fs.readFileSync(outputFilePath, 'utf8');
+
+    try {
+      jsonResult = JSON.parse(testOutput);
+    } catch (error: any) {
+      throw new Error(
+        `Can't parse the JSON result from ${outputFileName}, ${error.toString()}`,
+      );
+    }
+
+    expect(jsonResult.numPassedTests).toBe(0);
+    expect(jsonResult.numFailedTests).toBe(2);
+    expect(jsonResult.numPendingTests).toBe(0);
+    expect(jsonResult.testResults[0].testResults[0].invocations).toBe(1);
+    expect(jsonResult.testResults[0].testResults[1].invocations).toBe(1);
+  });
+});
diff --git a/e2e/test-retries/__tests__/beforeAllFailureConcurrent.test.js b/e2e/test-retries/__tests__/beforeAllFailureConcurrent.test.js
@@ -0,0 +1,21 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+'use strict';
+
+jest.retryTimes(3);
+
+beforeAll(() => {
+  throw new Error('Failure in beforeAll');
+});
+
+it.concurrent('should not be retried because hook failure occurred', () => {
+  throw new Error('should not be invoked');
+});
+
+it.concurrent('should fail due to the beforeAll', () => {
+  expect(10).toBe(10);
+});
diff --git a/e2e/test-retries/__tests__/controlConcurrent.test.js b/e2e/test-retries/__tests__/controlConcurrent.test.js
@@ -0,0 +1,11 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+'use strict';
+
+it('retryTimes not set', () => {
+  expect(true).toBeFalsy();
+});
diff --git a/e2e/test-retries/__tests__/e2eConcurrent.test.js b/e2e/test-retries/__tests__/e2eConcurrent.test.js
@@ -0,0 +1,29 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const countPath = path.join(__dirname, '.tries');
+
+beforeAll(() => {
+  fs.writeFileSync(countPath, '0', 'utf8');
+});
+
+jest.retryTimes(3);
+
+it.concurrent('retries', () => {
+  const tries = Number.parseInt(fs.readFileSync(countPath, 'utf8'), 10);
+  fs.writeFileSync(countPath, `${tries + 1}`, 'utf8');
+  expect(tries).toBe(3);
+});
+
+afterAll(() => {
+  // cleanup
+  fs.unlinkSync(countPath);
+});
diff --git a/e2e/test-retries/__tests__/retryConcurrent.test.js b/e2e/test-retries/__tests__/retryConcurrent.test.js
@@ -0,0 +1,17 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+'use strict';
+
+jest.retryTimes(3);
+
+it.concurrent('retryTimes set', () => {
+  expect(true).toBeFalsy();
+});
+
+it.concurrent('truthy test', () => {
+  expect(true).toBeTruthy();
+});
diff --git a/e2e/test-retries/__tests__/retryImmediatelyConcurrent.test.js b/e2e/test-retries/__tests__/retryImmediatelyConcurrent.test.js
@@ -0,0 +1,34 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+'use strict';
+
+jest.retryTimes(3, {logErrorsBeforeRetry: true, retryImmediately: true});
+let i1 = 0;
+it.concurrent('retryable test 1', () => {
+  i1++;
+  if (i1 === 3) {
+    console.log('SECOND TRUTHY TEST');
+    expect(true).toBeTruthy();
+  } else {
+    expect(true).toBeFalsy();
+  }
+});
+
+let i2 = 0;
+it.concurrent('retryable test 2', () => {
+  i2++;
+  if (i2 === 3) {
+    console.log('THIRD TRUTHY TEST');
+    expect(true).toBeTruthy();
+  } else {
+    expect(true).toBeFalsy();
+  }
+});
+it.concurrent('truthy test', () => {
+  console.log('FIRST TRUTHY TEST');
+  expect(true).toBeTruthy();
+});