diff --git a/cli/CHANGELOG.md b/cli/CHANGELOG.md index e1c2cac344e..aa8a93b7c6a 100644 --- a/cli/CHANGELOG.md +++ b/cli/CHANGELOG.md @@ -11,6 +11,7 @@ - The default `keystrokeDelay` for `cy.type()` has been changed from `10` to `0` to improve performance of typing-heavy test runs. Users who relied on the implicit 10ms delay can restore it by setting `keystrokeDelay: 10` in their Cypress config or on `Cypress.Keyboard`. Addresses [#33523](https://github.com/cypress-io/cypress/issues/33523). - Rewrote the experimental `experimentalFastVisibility` algorithm to delegate to the browser's native `Element.checkVisibility()` with a zero-dimension guard. Addressed in [#33781](https://github.com/cypress-io/cypress/pull/33781). - The `experimentalFastVisibility` boolean flag has been replaced with a new `visibilityStrategy` config option that accepts `'legacy'` or `'modern'`, defaulting to `'modern'`. The modern visibility algorithm (based on `Element.checkVisibility()`) is now the default for all users. The option is deprecated — set `visibilityStrategy: 'legacy'` only if you need the old algorithm when migrating to the modern visibility algorithm. Addressed in [#33794](https://github.com/cypress-io/cypress/pull/33794). +- `viewportWidth` and `viewportHeight` can no longer be set via [`Cypress.config()`](https://on.cypress.io/config) during test execution. Mutating them this way affected the *next* test rather than the current one and was not reflected in Test Replay. Use [`cy.viewport()`](https://on.cypress.io/viewport) or set them in the test configuration of a `describe`/`context` or `it` block instead. Addresses [#31592](https://github.com/cypress-io/cypress/issues/31592). - [`cy.getCookie()`](https://on.cypress.io/getcookie) and [`cy.getCookies()`](https://on.cypress.io/getcookies) are now query commands. They re-read the cookie(s) from the browser and retry any attached assertions until they pass or the command times out, so `cy.getCookie('token').should('exist')` now waits for a cookie that is set asynchronously (for example, after a login request resolves) instead of failing on the first read. As a result, their timeout now follows `defaultCommandTimeout` (default `4000`) rather than `responseTimeout` (default `30000`), and to overwrite these commands you must now use [`Cypress.Commands.overwriteQuery()`](https://on.cypress.io/api/cypress-api/custom-queries) instead of `Cypress.Commands.overwrite()`. Addresses [#4802](https://github.com/cypress-io/cypress/issues/4802). - Removed the `experimentalSourceRewriting` configuration option. The experimental AST-based source rewriting was removed in favor of the default regex-based source rewriting, so default behavior is unchanged. You can safely remove this option from your config. Addresses [#34213](https://github.com/cypress-io/cypress/issues/34213). diff --git a/packages/config/src/browser.ts b/packages/config/src/browser.ts index c1dfe3d95ab..f58b1d781b7 100644 --- a/packages/config/src/browser.ts +++ b/packages/config/src/browser.ts @@ -10,7 +10,7 @@ import { } from './options' import type { BreakingErrResult, TestingType } from '@packages/types' -import type { BreakingOption, BreakingOptionErrorKey, OverrideLevel } from './options' +import type { BreakingOption, BreakingOptionErrorKey, CurrentOverrideLevel, OverrideLevel } from './options' import type { ErrResult } from './validation' // this export has to be done in 2 lines because of a bug in babel typescript @@ -205,7 +205,7 @@ export const validateNoBreakingTestingTypeConfig = (cfg: any, testingType: keyof return validateNoBreakingOptions(options, cfg, onWarning, onErr, testingType) } -export const validateOverridableAtRunTime = (config: any, isSuiteLevelOverride: boolean, onErr: (result: InvalidTestOverrideResult) => void) => { +export const validateOverridableAtRunTime = (config: any, currentOverrideLevel: CurrentOverrideLevel, onErr: (result: InvalidTestOverrideResult) => void) => { Object.keys(config).some((configKey) => { const overrideLevel: OverrideLevel = testOverrideLevels[configKey] @@ -237,7 +237,11 @@ export const validateOverridableAtRunTime = (config: any, isSuiteLevelOverride: // TODO: add a hook to ensure valid testing-type configuration is being set at runtime for all configuration values. // https://github.com/cypress-io/cypress/issues/24365 - if (overrideLevel === 'never' || (overrideLevel === 'suite' && !isSuiteLevelOverride)) { + if ( + overrideLevel === 'never' || + (overrideLevel === 'suite' && currentOverrideLevel !== 'suite') || + (overrideLevel === 'suiteOrTest' && currentOverrideLevel === 'runtime') + ) { onErr({ invalidConfigKey: configKey, supportedOverrideLevel: overrideLevel, diff --git a/packages/config/src/options.ts b/packages/config/src/options.ts index 5c5793469da..1a83965de79 100644 --- a/packages/config/src/options.ts +++ b/packages/config/src/options.ts @@ -41,7 +41,22 @@ type ValidationOptions = { export type BreakingOptionErrorKey = typeof BREAKING_OPTION_ERROR_KEY[number] -export type OverrideLevel = 'any' | 'suite' | 'never' +/** + * Where a configuration option is allowed to be overridden at test time: + * - `any`: suite-level overrides, test-level overrides, and at run-time via `Cypress.config()`. + * - `suiteOrTest`: suite-level and test-level overrides only. Cannot be set via `Cypress.config()` + * while a test is executing. + * - `suite`: suite-level overrides only. + * - `never`: cannot be overridden at test time. + */ +export type OverrideLevel = 'any' | 'suiteOrTest' | 'suite' | 'never' + +/** + * The context in which a test-time config change is being applied: `suite`/`test` for + * `describe`/`it` config overrides, `runtime` for a `Cypress.config()` call while a test is + * executing, and `undefined` for anything else (support/spec file load or a `test:before:run` event). + */ +export type CurrentOverrideLevel = 'suite' | 'test' | 'runtime' | undefined interface ConfigOption { name: string @@ -477,12 +492,12 @@ const driverConfigOptions: Array = [ name: 'viewportHeight', defaultValue: (options: Record = {}) => options.testingType === 'component' ? 500 : 660, validation: validate.isNumber, - overrideLevel: 'any', + overrideLevel: 'suiteOrTest', }, { name: 'viewportWidth', defaultValue: (options: Record = {}) => options.testingType === 'component' ? 500 : 1000, validation: validate.isNumber, - overrideLevel: 'any', + overrideLevel: 'suiteOrTest', }, { name: 'waitForAnimations', defaultValue: true, diff --git a/packages/config/test/index.spec.ts b/packages/config/test/index.spec.ts index ef3eed76276..637bf1d706c 100644 --- a/packages/config/test/index.spec.ts +++ b/packages/config/test/index.spec.ts @@ -245,7 +245,7 @@ describe('config/src/index', () => { it('calls onError handler if configuration override level=never', () => { const errorFn = vi.fn() - configUtil.validateOverridableAtRunTime({ chromeWebSecurity: false }, false, errorFn) + configUtil.validateOverridableAtRunTime({ chromeWebSecurity: false }, undefined, errorFn) expect(errorFn).toHaveBeenCalledTimes(1) expect(errorFn).toHaveBeenCalledWith(expect.objectContaining({ @@ -258,19 +258,27 @@ describe('config/src/index', () => { it('does not calls onError handler if validating level is suite', () => { const errorFn = vi.fn() - const isSuiteOverride = true - - configUtil.validateOverridableAtRunTime({ testIsolation: true }, isSuiteOverride, errorFn) + configUtil.validateOverridableAtRunTime({ testIsolation: true }, 'suite', errorFn) expect(errorFn).toHaveBeenCalledTimes(0) }) - it('calls onError handler if validating level is not suite', () => { + it('calls onError handler if validating level is test', () => { const errorFn = vi.fn() - const isSuiteOverride = false + configUtil.validateOverridableAtRunTime({ testIsolation: 'off' }, 'test', errorFn) + + expect(errorFn).toHaveBeenCalledTimes(1) + expect(errorFn).toHaveBeenCalledWith(expect.objectContaining({ + invalidConfigKey: 'testIsolation', + supportedOverrideLevel: 'suite', + })) + }) + + it('calls onError handler if validating level is run-time', () => { + const errorFn = vi.fn() - configUtil.validateOverridableAtRunTime({ testIsolation: 'off' }, isSuiteOverride, errorFn) + configUtil.validateOverridableAtRunTime({ testIsolation: 'off' }, 'runtime', errorFn) expect(errorFn).toHaveBeenCalledTimes(1) expect(errorFn).toHaveBeenCalledWith(expect.objectContaining({ @@ -280,10 +288,53 @@ describe('config/src/index', () => { }) }) + describe('configuration override level=suiteOrTest', () => { + it('does not call onError handler if validating level is suite', () => { + const errorFn = vi.fn() + + configUtil.validateOverridableAtRunTime({ viewportWidth: 200 }, 'suite', errorFn) + + expect(errorFn).toHaveBeenCalledTimes(0) + }) + + it('does not call onError handler if validating level is test', () => { + const errorFn = vi.fn() + + configUtil.validateOverridableAtRunTime({ viewportHeight: 100 }, 'test', errorFn) + + expect(errorFn).toHaveBeenCalledTimes(0) + }) + + it('does not call onError handler outside test execution (e.g. file load)', () => { + const errorFn = vi.fn() + + configUtil.validateOverridableAtRunTime({ viewportWidth: 200, viewportHeight: 100 }, undefined, errorFn) + + expect(errorFn).toHaveBeenCalledTimes(0) + }) + + it('calls onError handler if validating level is run-time', () => { + const errorFn = vi.fn() + + configUtil.validateOverridableAtRunTime({ viewportWidth: 200, viewportHeight: 100 }, 'runtime', errorFn) + + expect(errorFn).toHaveBeenCalledTimes(2) + expect(errorFn).toHaveBeenCalledWith(expect.objectContaining({ + invalidConfigKey: 'viewportWidth', + supportedOverrideLevel: 'suiteOrTest', + })) + + expect(errorFn).toHaveBeenCalledWith(expect.objectContaining({ + invalidConfigKey: 'viewportHeight', + supportedOverrideLevel: 'suiteOrTest', + })) + }) + }) + it(`does not call onErr if config override level=any`, () => { const errorFn = vi.fn() - configUtil.validateOverridableAtRunTime({ requestTimeout: 1000 }, false, errorFn) + configUtil.validateOverridableAtRunTime({ requestTimeout: 1000 }, undefined, errorFn) expect(errorFn).toHaveBeenCalledTimes(0) }) @@ -291,7 +342,7 @@ describe('config/src/index', () => { it('does not call onErr if configuration is a non-Cypress config option', () => { const errorFn = vi.fn() - configUtil.validateOverridableAtRunTime({ foo: 'bar' }, true, errorFn) + configUtil.validateOverridableAtRunTime({ foo: 'bar' }, 'suite', errorFn) expect(errorFn).toHaveBeenCalledTimes(0) }) diff --git a/packages/driver/cypress/e2e/e2e/testConfigOverrides.cy.js b/packages/driver/cypress/e2e/e2e/testConfigOverrides.cy.js index 89673aa41d9..b1711caa236 100644 --- a/packages/driver/cypress/e2e/e2e/testConfigOverrides.cy.js +++ b/packages/driver/cypress/e2e/e2e/testConfigOverrides.cy.js @@ -372,6 +372,28 @@ describe('cannot set override configuration options that', () => { Cypress.config('chromeWebSecurity', false) }) + // https://github.com/cypress-io/cypress/issues/31592 + it('throws if mutating viewportWidth with Cypress.config() during test execution', (done) => { + window.top.__cySkipValidateConfig = false + cy.once('fail', (err) => { + expect(err.message).to.include('`Cypress.config()` cannot override `viewportWidth` during test execution') + done() + }) + + Cypress.config('viewportWidth', 200) + }) + + // https://github.com/cypress-io/cypress/issues/31592 + it('throws if mutating viewportHeight with Cypress.config() during test execution', (done) => { + window.top.__cySkipValidateConfig = false + cy.once('fail', (err) => { + expect(err.message).to.include('`Cypress.config()` cannot override `viewportHeight` during test execution') + done() + }) + + Cypress.config('viewportHeight', 100) + }) + it('does not throw for non-Cypress config values', () => { expect(() => { Cypress.config('foo', 'bar') diff --git a/packages/driver/cypress/e2e/util/config.cy.js b/packages/driver/cypress/e2e/util/config.cy.js index 0f76a2eb13a..d0a8ad2bc24 100644 --- a/packages/driver/cypress/e2e/util/config.cy.js +++ b/packages/driver/cypress/e2e/util/config.cy.js @@ -132,6 +132,57 @@ describe('driver/src/cypress/validate_config', () => { }) }) + describe('when config override level is suiteOrTest', () => { + ['test', 'suite'].forEach((mocha_runnable) => { + it(`does not throw when runtime level is ${mocha_runnable}`, () => { + const state = $SetterGetter.create({ + duringUserTestExecution: false, + test: { + _testConfig: { applied: mocha_runnable }, + }, + specWindow: { Error }, + }) + const overrideLevel = getMochaOverrideLevel(state) + + expect(overrideLevel).to.eq(mocha_runnable) + + expect(() => { + validateConfig(state, { viewportWidth: 200, viewportHeight: 100 }) + }).not.to.throw() + }) + }) + + it('throws when mutated at run-time with Cypress.config()', () => { + const state = $SetterGetter.create({ + duringUserTestExecution: true, + specWindow: { Error }, + runnable: { type: 'test' }, + }) + const overrideLevel = getMochaOverrideLevel(state) + + expect(overrideLevel).to.be.undefined + + expect(() => { + validateConfig(state, { viewportWidth: 200 }) + }).to.throw(`\`Cypress.config()\` cannot override \`viewportWidth\` during test execution`) + }) + + it('does not throw when set outside test execution (e.g. support/spec file load)', () => { + const state = $SetterGetter.create({ + duringUserTestExecution: false, + test: undefined, + specWindow: { Error }, + }) + const overrideLevel = getMochaOverrideLevel(state) + + expect(overrideLevel).to.be.undefined + + expect(() => { + validateConfig(state, { viewportWidth: 200, viewportHeight: 100 }) + }).not.to.throw() + }) + }) + describe('when config override level is suite', () => { it('and config override is read-only', () => { const state = $SetterGetter.create({ @@ -204,8 +255,8 @@ describe('driver/src/cypress/validate_config', () => { }) expect(() => { - validateConfig(state, { viewportHeight: '300' }) - }).to.throw(`Expected \`viewportHeight\` to be a number.\n\nInstead the value was: \`"300"\``) + validateConfig(state, { defaultCommandTimeout: '300' }) + }).to.throw(`Expected \`defaultCommandTimeout\` to be a number.\n\nInstead the value was: \`"300"\``) }) }) }) diff --git a/packages/driver/src/cypress/error_messages.ts b/packages/driver/src/cypress/error_messages.ts index bbbfc01dc9c..751d1e8d3a4 100644 --- a/packages/driver/src/cypress/error_messages.ts +++ b/packages/driver/src/cypress/error_messages.ts @@ -262,6 +262,10 @@ export default { message: `\`Cypress.config()\` can never override \`{{invalidConfigKey}}\` because it is a read-only configuration option.`, docsUrl: 'https://on.cypress.io/config', }, + suite_or_test_only: { + message: `\`Cypress.config()\` cannot override \`{{invalidConfigKey}}\` during test execution because it would affect the next test rather than the current one and is not reflected in Test Replay. Set \`{{invalidConfigKey}}\` in the test configuration of a \`describe\`/\`context\` or \`it\` block instead.{{additionalInfo}}`, + docsUrl: 'https://on.cypress.io/config', + }, }, invalid_mocha_config_override: { read_only: { diff --git a/packages/driver/src/util/config.ts b/packages/driver/src/util/config.ts index 4b5b3917970..822693e1472 100644 --- a/packages/driver/src/util/config.ts +++ b/packages/driver/src/util/config.ts @@ -25,7 +25,12 @@ const omitConfigReadOnlyDifferences = (objectLikeConfig: Cypress.ObjectLike) => return } - if ((overrideLevels === 'never' && configKey !== 'isDefaultProtocolEnabled')) { + // Cross-origin syncing applies the diff via Cypress.config() at run-time, so omit any + // value that cannot be set that way. + if ( + (overrideLevels === 'never' && configKey !== 'isDefaultProtocolEnabled') || + overrideLevels === 'suiteOrTest' + ) { delete objectLikeConfig[configKey] } }) @@ -100,6 +105,13 @@ export const getMochaOverrideLevel = (state): MochaOverrideLevel | undefined => return undefined } +// Extra guidance appended to the generic `suite_or_test_only` error, keyed by config option. +// Add an entry here when a new `suiteOrTest` option needs option-specific advice. +const suiteOrTestOnlyGuidance: Record = { + viewportWidth: ' To change the viewport during a test, use `cy.viewport()`.', + viewportHeight: ' To change the viewport during a test, use `cy.viewport()`.', +} + // Configuration can be override at multiple run-time levels. Ensure the configuration keys can // be override and that the provided override values are the correct type. // @@ -115,20 +127,27 @@ export const validateConfig = (state: State, config: Record, skipCo const mochaOverrideLevel = getMochaOverrideLevel(state) if (!skipConfigOverrideValidation && mochaOverrideLevel !== 'restoring') { - const isSuiteOverride = mochaOverrideLevel === 'suite' + const currentOverrideLevel = mochaOverrideLevel === 'suite' || mochaOverrideLevel === 'test' + ? mochaOverrideLevel + : state('duringUserTestExecution') ? 'runtime' : undefined - validateOverridableAtRunTime(config, isSuiteOverride, (validationResult) => { + validateOverridableAtRunTime(config, currentOverrideLevel, (validationResult) => { let errKey = 'config.cypress_config_api.read_only' if (validationResult.supportedOverrideLevel === 'global_only') { errKey = 'config.invalid_mocha_config_override.global_only' } else if (validationResult.supportedOverrideLevel === 'suite') { errKey = 'config.invalid_mocha_config_override.suite_only' + } else if (validationResult.supportedOverrideLevel === 'suiteOrTest') { + errKey = 'config.cypress_config_api.suite_or_test_only' } else if (mochaOverrideLevel) { errKey = 'config.invalid_mocha_config_override.read_only' } - throw new (state('specWindow').Error)($errUtils.errByPath(errKey, validationResult)) + throw new (state('specWindow').Error)($errUtils.errByPath(errKey, { + ...validationResult, + additionalInfo: suiteOrTestOnlyGuidance[validationResult.invalidConfigKey] ?? '', + })) }) }