feat: implement flag to fail flaky tests

docs/src/test-api/class-fullproject.md

@@ -58,6 +58,12 @@ See [`property: TestProject.repeatEach`].
 
 See [`property: TestProject.retries`].
 
+## property: FullProject.failOnFlakyTests
+* since: v1.44
+- type: <[boolean]>
+
+See [`property: TestProject.failOnFlakyTests`].
+
 ## property: FullProject.teardown
 * since: v1.34
 - type: ?<[string]>

docs/src/test-api/class-test.md

@@ -521,6 +521,12 @@ Execution mode. Learn more about the execution modes [here](../test-parallel.md)
 
 The number of retries for each test.
 
+### option: Test.describe.configure.failOnFlakyTests
+* since: v1.44
+- `failOnFlakyTests` <[boolean]>
+
+Fails flaky tests.
+
 ### option: Test.describe.configure.timeout
 * since: v1.28
 - `timeout` <[int]>

docs/src/test-api/class-testconfig.md

@@ -455,6 +455,22 @@ export default defineConfig({
 });
 ```
 
+## property: TestConfig.failOnFlakyTests
+* since: v1.44
+- type: ?<[boolean]>
+
+Counts tests that initially fail but subsequently pass as failed. By default 'flaky' tests are considered as passing.
+
+**Usage**
+
+```js title="playwright.config.ts"
+import { defineConfig } from '@playwright/test';
+
+export default failOnFlakyTests({
+ failOnFlakyTests: true,
+});
+```
+
 ## property: TestConfig.shard
 * since: v1.10
 - type: ?<[null]|[Object]>

docs/src/test-api/class-testproject.md

@@ -233,6 +233,12 @@ Use [`method: Test.describe.configure`] to change the number of retries for a sp
 
 Use [`property: TestConfig.retries`] to change this option for all projects.
 
+## property: TestProject.failOnFlakyTests
+* since: v1.44
+- type: ?<[boolean]>
+
+
+Counts tests that initially fail but subsequently pass as failed. By default 'flaky' tests are considered as passing.
 
 ## property: TestProject.teardown
 * since: v1.34

packages/playwright/src/common/config.ts

@@ -176,6 +176,7 @@ export class FullProjectInternal {
  // project is top-level vs dependency. See collectProjectsAndTestFiles in loadUtils.
  repeatEach: takeFirst(projectConfig.repeatEach, config.repeatEach, 1),
  retries: takeFirst(configCLIOverrides.retries, projectConfig.retries, config.retries, 0),
+ failOnFlakyTests: takeFirst(configCLIOverrides.failOnFlakyTests, projectConfig.failOnFlakyTests, config.failOnFlakyTests, false),
  metadata: takeFirst(projectConfig.metadata, config.metadata, {}),
  name: takeFirst(projectConfig.name, config.name, ''),
  testDir,
@@ -187,6 +188,7 @@ export class FullProjectInternal {
  dependencies: projectConfig.dependencies || [],
  teardown: projectConfig.teardown,
  };
+
  this.fullyParallel = takeFirst(configCLIOverrides.fullyParallel, projectConfig.fullyParallel, config.fullyParallel, undefined);
  this.expect = takeFirst(projectConfig.expect, config.expect, {});
  if (this.expect.toHaveScreenshot?.stylePath) {

packages/playwright/src/common/configLoader.ts

@@ -251,6 +251,11 @@ function validateProject(file: string, project: Project, title: string) {
  throw errorWithFile(file, `${title}.retries must be a non-negative number`);
  }
 
+ if ('failOnFlakyTests' in project && project.failOnFlakyTests !== undefined) {
+ if (typeof project.failOnFlakyTests !== 'boolean')
+ throw errorWithFile(file, `${title}.failOnFlakyTests must be a boolean`);
+ }
+
  if ('testDir' in project && project.testDir !== undefined) {
  if (typeof project.testDir !== 'string')
  throw errorWithFile(file, `${title}.testDir must be a string`);

packages/playwright/src/common/ipc.ts

@@ -29,6 +29,7 @@ export type ConfigCLIOverrides = {
  quiet?: boolean;
  repeatEach?: number;
  retries?: number;
+ failOnFlakyTests?: boolean;
  reporter?: ReporterDescription[];
  additionalReporters?: ReporterDescription[];
  shard?: { current: number, total: number };

packages/playwright/src/common/suiteUtils.ts

@@ -61,6 +61,7 @@ export function bindFileSuiteToProject(project: FullProjectInternal, suite: Suit
  // Inherit properties from parent suites.
  let inheritedRetries: number | undefined;
  let inheritedTimeout: number | undefined;
+ let inheritedfailOnFlakyTests: boolean | undefined;
  test.annotations = [];
  for (let parentSuite: Suite | undefined = suite; parentSuite; parentSuite = parentSuite.parent) {
  if (parentSuite._staticAnnotations.length)
@@ -69,9 +70,12 @@ export function bindFileSuiteToProject(project: FullProjectInternal, suite: Suit
  inheritedRetries = parentSuite._retries;
  if (inheritedTimeout === undefined && parentSuite._timeout !== undefined)
  inheritedTimeout = parentSuite._timeout;
+ if (inheritedfailOnFlakyTests === undefined && parentSuite._failOnFlakyTests !== undefined)
+ inheritedfailOnFlakyTests = parentSuite._failOnFlakyTests;
  }
  test.retries = inheritedRetries ?? project.project.retries;
  test.timeout = inheritedTimeout ?? project.project.timeout;
+ test.failOnFlakyTests = inheritedfailOnFlakyTests ?? project.project.failOnFlakyTests;
  test.annotations.push(...test._staticAnnotations);
 
  // Skip annotations imply skipped expectedStatus.

packages/playwright/src/common/test.ts

@@ -47,6 +47,7 @@ export class Suite extends Base {
  _hooks: { type: 'beforeEach' | 'afterEach' | 'beforeAll' | 'afterAll', fn: Function, title: string, location: Location }[] = [];
  _timeout: number | undefined;
  _retries: number | undefined;
+ _failOnFlakyTests: boolean | undefined;
  // Annotations known statically before running the test, e.g. `test.describe.skip()` or `test.describe({ annotation }, body)`.
  _staticAnnotations: Annotation[] = [];
  // Explicitly declared tags that are not a part of the title.
@@ -220,6 +221,7 @@ export class Suite extends Base {
  suite._requireFile = data.requireFile;
  suite._timeout = data.timeout;
  suite._retries = data.retries;
+ suite._failOnFlakyTests = data.failOnFlakyTests;
  suite._staticAnnotations = data.staticAnnotations;
  suite._tags = data.tags;
  suite._modifiers = data.modifiers;
@@ -254,6 +256,7 @@ export class TestCase extends Base implements reporterTypes.TestCase {
  timeout = 0;
  annotations: Annotation[] = [];
  retries = 0;
+ failOnFlakyTests = false;
  repeatEachIndex = 0;
 
  _testType: TestTypeImpl;
@@ -286,7 +289,7 @@ export class TestCase extends Base implements reporterTypes.TestCase {
 
  ok(): boolean {
  const status = this.outcome();
- return status === 'expected' || status === 'flaky' || status === 'skipped';
+ return status === 'expected' || (status === 'flaky' && !this.failOnFlakyTests) || status === 'skipped';
  }
 
  get tags(): string[] {
@@ -299,6 +302,7 @@ export class TestCase extends Base implements reporterTypes.TestCase {
  id: this.id,
  title: this.title,
  retries: this.retries,
+ failOnFlakyTests: this.failOnFlakyTests,
  timeout: this.timeout,
  expectedStatus: this.expectedStatus,
  location: this.location,
@@ -317,6 +321,7 @@ export class TestCase extends Base implements reporterTypes.TestCase {
  const test = new TestCase(data.title, () => {}, rootTestType, data.location);
  test.id = data.id;
  test.retries = data.retries;
+ test.failOnFlakyTests = data.failOnFlakyTests;
  test.timeout = data.timeout;
  test.expectedStatus = data.expectedStatus;
  test._only = data.only;

packages/playwright/src/isomorphic/teleReceiver.ts

@@ -47,6 +47,7 @@ export type JsonProject = {
  outputDir: string;
  repeatEach: number;
  retries: number;
+ failOnFlakyTests: boolean;
  suites: JsonSuite[];
  teardown?: string;
  // This is relative to root dir.
@@ -316,6 +317,7 @@ export class TeleReporterReceiver {
  outputDir: this._absolutePath(project.outputDir),
  repeatEach: project.repeatEach,
  retries: project.retries,
+ failOnFlakyTests: project.failOnFlakyTests,
  testDir: this._absolutePath(project.testDir),
  testIgnore: parseRegexPatterns(project.testIgnore),
  testMatch: parseRegexPatterns(project.testMatch),

packages/playwright/src/program.ts

@@ -277,6 +277,7 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid
  quiet: options.quiet ? options.quiet : undefined,
  repeatEach: options.repeatEach ? parseInt(options.repeatEach, 10) : undefined,
  retries: options.retries ? parseInt(options.retries, 10) : undefined,
+ failOnFlakyTests: options.failOnFlakyTests ? true : undefined,
  reporter: resolveReporterOption(options.reporter),
  shard: shardPair ? { current: shardPair[0], total: shardPair[1] } : undefined,
  timeout: options.timeout ? parseInt(options.timeout, 10) : undefined,
@@ -336,6 +337,7 @@ const testOptions: [string, string][] = [
  ['--browser <browser>', `Browser to use for tests, one of "all", "chromium", "firefox" or "webkit" (default: "chromium")`],
  ['-c, --config <file>', `Configuration file, or a test directory with optional "playwright.config.{m,c}?{js,ts}"`],
  ['--debug', `Run tests with Playwright Inspector. Shortcut for "PWDEBUG=1" environment variable and "--timeout=0 --max-failures=1 --headed --workers=1" options`],
+ ['--fail-on-flaky-tests', `Fail if any test is flagged as flaky (default: false)`],
  ['--forbid-only', `Fail if test.only is called (default: false)`],
  ['--fully-parallel', `Run all tests in parallel (default: false)`],
  ['--global-timeout <timeout>', `Maximum time this test suite can run in milliseconds (default: unlimited)`],

packages/playwright/src/reporters/teleEmitter.ts

@@ -171,6 +171,7 @@ export class TeleReporterEmitter implements ReporterV2 {
  outputDir: this._relativePath(project.outputDir),
  repeatEach: project.repeatEach,
  retries: project.retries,
+ failOnFlakyTests: project.failOnFlakyTests,
  testDir: this._relativePath(project.testDir),
  testIgnore: serializeRegexPatterns(project.testIgnore),
  testMatch: serializeRegexPatterns(project.testMatch),

packages/playwright/types/test.d.ts

@@ -254,6 +254,12 @@ interface TestProject<TestArgs = {}, WorkerArgs = {}> {
  };
  };
 
+ /**
+ * Counts tests that initially fail but subsequently pass as failed. By default 'flaky' tests are considered as
+ * passing.
+ */
+ failOnFlakyTests?: boolean;
+
  /**
  * Playwright Test runs tests in parallel. In order to achieve that, it runs several worker processes that run at the
  * same time. By default, **test files** are run in parallel. Tests in a single file are run in order, in the same
@@ -644,6 +650,12 @@ export interface FullProject<TestArgs = {}, WorkerArgs = {}> {
  */
  dependencies: Array<string>;
 
+ /**
+ * See
+ * [testProject.failOnFlakyTests](https://playwright.dev/docs/api/class-testproject#test-project-fail-on-flaky-tests).
+ */
+ failOnFlakyTests: boolean;
+
  /**
  * See [testProject.grep](https://playwright.dev/docs/api/class-testproject#test-project-grep).
  */
@@ -1017,6 +1029,24 @@ interface TestConfig<TestArgs = {}, WorkerArgs = {}> {
  };
  };
 
+ /**
+ * Counts tests that initially fail but subsequently pass as failed. By default 'flaky' tests are considered as
+ * passing.
+ *
+ * **Usage**
+ *
+ * ```js
+ * // playwright.config.ts
+ * import { defineConfig } from '@playwright/test';
+ *
+ * export default failOnFlakyTests({
+ * failOnFlakyTests: true,
+ * });
+ * ```
+ *
+ */
+ failOnFlakyTests?: boolean;
+
  /**
  * Whether to exit with an error if any tests or groups are marked as
  * [test.only(title[, details, body])](https://playwright.dev/docs/api/class-test#test-only) or

tests/playwright-test/exit-code.spec.ts

@@ -128,6 +128,19 @@ test('should allow flaky', async ({ runInlineTest }) => {
  expect(result.flaky).toBe(1);
 });
 
+test('failOnFlakyTests flag disallows flaky', async ({ runInlineTest }) => {
+ const result = await runInlineTest({
+ 'a.test.js': `
+ import { test, expect } from '@playwright/test';
+ test('flake', async ({}, testInfo) => {
+ expect(testInfo.retry).toBe(1);
+ });
+ `,
+ }, { 'retries': 1, 'fail-on-flaky-tests': true });
+ expect(result.exitCode).not.toBe(0);
+ expect(result.flaky).toBe(1);
+});
+
 test('should fail on unexpected pass', async ({ runInlineTest }) => {
  const { exitCode, failed, output } = await runInlineTest({
  'unexpected-pass.spec.js': `