From 7c08fb589992672bb11061d04b9842f3a060eb5b Mon Sep 17 00:00:00 2001 From: bailey Date: Tue, 10 Feb 2026 15:44:12 -0700 Subject: [PATCH 01/11] POC --- src/cmap/connect.ts | 2 + src/cmap/connection.ts | 9 + src/index.ts | 1 + src/operations/execute_operation.ts | 203 +- src/operations/operation.ts | 8 + src/sdam/topology.ts | 9 +- src/sessions.ts | 39 +- src/token_bucket.ts | 39 + src/utils.ts | 20 + sync.sh | 4 + .../client-backpressure.prose.test.ts | 60 + .../client-backpressure.spec.test.ts | 16 + .../mongodb-handshake.prose.test.ts | 35 + .../retryable_writes.spec.prose.test.ts | 103 + .../transactions/transactions.spec.test.ts | 2 +- test/spec/client-backpressure/README.md | 57 + .../backpressure-retry-loop.json | 2759 +++++++++++++ .../backpressure-retry-loop.yml | 1307 +++++++ .../backpressure-retry-loop.yml.template | 101 + .../backpressure-retry-max-attempts.json | 3448 +++++++++++++++++ .../backpressure-retry-max-attempts.yml | 1711 ++++++++ ...ckpressure-retry-max-attempts.yml.template | 101 + .../client-backpressure/getMore-retried.json | 291 ++ .../client-backpressure/getMore-retried.yml | 149 + .../unified/backpressure-retryable-abort.json | 357 ++ .../unified/backpressure-retryable-abort.yml | 213 + .../backpressure-retryable-commit.json | 374 ++ .../unified/backpressure-retryable-commit.yml | 220 ++ .../unified/backpressure-retryable-reads.json | 328 ++ .../unified/backpressure-retryable-reads.yml | 192 + .../backpressure-retryable-writes.json | 440 +++ .../unified/backpressure-retryable-writes.yml | 250 ++ test/unit/index.test.ts | 1 + 33 files changed, 12785 insertions(+), 64 deletions(-) create mode 100644 src/token_bucket.ts create mode 100644 sync.sh create mode 100644 test/integration/client-backpressure/client-backpressure.prose.test.ts create mode 100644 test/integration/client-backpressure/client-backpressure.spec.test.ts create mode 100644 test/spec/client-backpressure/README.md create mode 100644 test/spec/client-backpressure/backpressure-retry-loop.json create mode 100644 test/spec/client-backpressure/backpressure-retry-loop.yml create mode 100644 test/spec/client-backpressure/backpressure-retry-loop.yml.template create mode 100644 test/spec/client-backpressure/backpressure-retry-max-attempts.json create mode 100644 test/spec/client-backpressure/backpressure-retry-max-attempts.yml create mode 100644 test/spec/client-backpressure/backpressure-retry-max-attempts.yml.template create mode 100644 test/spec/client-backpressure/getMore-retried.json create mode 100644 test/spec/client-backpressure/getMore-retried.yml create mode 100644 test/spec/transactions/unified/backpressure-retryable-abort.json create mode 100644 test/spec/transactions/unified/backpressure-retryable-abort.yml create mode 100644 test/spec/transactions/unified/backpressure-retryable-commit.json create mode 100644 test/spec/transactions/unified/backpressure-retryable-commit.yml create mode 100644 test/spec/transactions/unified/backpressure-retryable-reads.json create mode 100644 test/spec/transactions/unified/backpressure-retryable-reads.yml create mode 100644 test/spec/transactions/unified/backpressure-retryable-writes.json create mode 100644 test/spec/transactions/unified/backpressure-retryable-writes.yml diff --git a/src/cmap/connect.ts b/src/cmap/connect.ts index 544ec471a99..711335f3d1d 100644 --- a/src/cmap/connect.ts +++ b/src/cmap/connect.ts @@ -224,6 +224,7 @@ export interface HandshakeDocument extends Document { compression: string[]; saslSupportedMechs?: string; loadBalanced?: boolean; + backpressure: true; } /** @@ -241,6 +242,7 @@ export async function prepareHandshakeDocument( const handshakeDoc: HandshakeDocument = { [serverApi?.version || options.loadBalanced === true ? 'hello' : LEGACY_HELLO_COMMAND]: 1, + backpressure: true, helloOk: true, client: clientMetadata, compression: compressors diff --git a/src/cmap/connection.ts b/src/cmap/connection.ts index 71afd72c2aa..065e0dc42c4 100644 --- a/src/cmap/connection.ts +++ b/src/cmap/connection.ts @@ -586,6 +586,15 @@ export class Connection extends TypedEventEmitter { this.throwIfAborted(); } } catch (error) { + // Note for Sergey: when retrying a command with `startTransaction`, the spec says drivers must only mark the transaction + // as in progress _after_ the server responds, regardless of the command's outcome. + // Server errors are thrown from the try-block above _after_ updateSessionFromResponse is called. So, server errors already correctly update the + // session's state. + // Network errors, however, are thrown from the try-block from `sendWire()`, and as such bypass the call to `updateSessionFromResponse` above. So, we have to + // handle updating sessions for non-server errors here. + if (options.session != null && !(error instanceof MongoServerError)) { + updateSessionFromResponse(options.session, MongoDBResponse.empty); + } if (this.shouldEmitAndLogCommand) { this.emitAndLogCommand( this.monitorCommands, diff --git a/src/index.ts b/src/index.ts index 74803dfa2a0..b5fc6ecd4aa 100644 --- a/src/index.ts +++ b/src/index.ts @@ -87,6 +87,7 @@ export { MongoWriteConcernError, WriteConcernErrorResult } from './error'; +export { TokenBucket } from './token_bucket'; export { AbstractCursor, // Actual driver classes exported diff --git a/src/operations/execute_operation.ts b/src/operations/execute_operation.ts index b6ddbba2b66..27470b13025 100644 --- a/src/operations/execute_operation.ts +++ b/src/operations/execute_operation.ts @@ -1,3 +1,5 @@ +import { setTimeout } from 'timers/promises'; + import { MIN_SUPPORTED_SNAPSHOT_READS_WIRE_VERSION } from '../cmap/wire_protocol/constants'; import { isRetryableReadError, @@ -10,6 +12,7 @@ import { MongoInvalidArgumentError, MongoNetworkError, MongoNotConnectedError, + MongoOperationTimeoutError, MongoRuntimeError, MongoServerError, MongoTransactionError, @@ -26,9 +29,16 @@ import { import type { Topology } from '../sdam/topology'; import type { ClientSession } from '../sessions'; import { TimeoutContext } from '../timeout'; -import { abortable, maxWireVersion, supportsRetryableWrites } from '../utils'; +import { RETRY_COST, TOKEN_REFRESH_RATE } from '../token_bucket'; +import { + abortable, + ExponentialBackoffProvider, + maxWireVersion, + supportsRetryableWrites +} from '../utils'; import { AggregateOperation } from './aggregate'; import { AbstractOperation, Aspect } from './operation'; +import { RunCommandOperation } from './run_command'; const MMAPv1_RETRY_WRITES_ERROR_CODE = MONGODB_ERROR_CODES.IllegalOperation; const MMAPv1_RETRY_WRITES_ERROR_MESSAGE = @@ -50,7 +60,7 @@ type ResultTypeFromOperation = ReturnType< * The expectation is that this function: * - Connects the MongoClient if it has not already been connected, see {@link autoConnect} * - Creates a session if none is provided and cleans up the session it creates - * - Tries an operation and retries under certain conditions, see {@link tryOperation} + * - Tries an operation and retries under certain conditions, see {@link executeOperationWithRetries} * * @typeParam T - The operation's type * @typeParam TResult - The type of the operation's result, calculated from T @@ -120,7 +130,7 @@ export async function executeOperation< }); try { - return await tryOperation(operation, { + return await executeOperationWithRetries(operation, { topology, timeoutContext, session, @@ -183,8 +193,12 @@ type RetryOptions = { * @typeParam TResult - The type of the operation's result, calculated from T * * @param operation - The operation to execute - * */ -async function tryOperation>( + */ +// Note for Sergey: The rename here could be reverted - I just thought this was more descriptive. +async function executeOperationWithRetries< + T extends AbstractOperation, + TResult = ResultTypeFromOperation +>( operation: T, { topology, timeoutContext, session, readPreference }: RetryOptions ): Promise { @@ -233,33 +247,118 @@ async function tryOperation 0 + ? // on successful retry, deposit the retry cost + the refresh rate. + TOKEN_REFRESH_RATE + RETRY_COST + : // otherwise, just deposit the refresh rate. + TOKEN_REFRESH_RATE + ); + return operation.handleOk(result); + } catch (error) { + return operation.handleError(error); + } + + // Note for Sergey: This ended up being a larger refactor than I anticipated. But it made more sense to me to put the error handling + // that previously lived in the try-block into the catch-block. + // The primary motivator for this change was to make error tracking simpler. The post failure but pre-retry logic needs access to both the + // most recently encountered operation error _and_ the error we're storing to potentially throw, if we need to (because in some circumstances + // the error raised from here is _not_ the most recently encountered operation error). If we chose to keep the previously existing structure, + // this would force us to keep track of two errors outside of the for loop. By moving the pre-retry logic into the catch block, it gains + // access to `operationError`, and still only requires keeping track of one error outside the loop. + } catch (operationError) { + // Should never happen but if it does - propagate the error. + if (!(operationError instanceof MongoError)) throw operationError; + + if (!operationError.hasErrorLabel(MongoErrorLabel.SystemOverloadedError)) { + // if an operation fails with an error that does not contain the SystemOverloadError, deposit 1 token. + topology.tokenBucket.deposit(RETRY_COST); + } + + if (error == null) { + error = operationError; + } else { + if (!operationError.hasErrorLabel(MongoErrorLabel.NoWritesPerformed)) { + error = operationError; + } + } + + // Reset timeouts + timeoutContext.clear(); + + if (hasWriteAspect && operationError.code === MMAPv1_RETRY_WRITES_ERROR_CODE) { throw new MongoServerError({ message: MMAPv1_RETRY_WRITES_ERROR_MESSAGE, errmsg: MMAPv1_RETRY_WRITES_ERROR_MESSAGE, - originalError: previousOperationError + originalError: operationError }); } - if (operation.hasAspect(Aspect.COMMAND_BATCHING) && !operation.canRetryWrite) { - throw previousOperationError; + if (!canRetry(operation, operationError)) { + throw error; } - if (hasWriteAspect && !isRetryableWriteError(previousOperationError)) - throw previousOperationError; + maxAttempts = + shouldRetry && operationError.hasErrorLabel(MongoErrorLabel.SystemOverloadedError) + ? 6 + : maxAttempts; + + if (attempt + 1 >= maxAttempts) { + throw error; + } - if (hasReadAspect && !isRetryableReadError(previousOperationError)) { - throw previousOperationError; + if (operationError.hasErrorLabel(MongoErrorLabel.SystemOverloadedError)) { + if (!topology.tokenBucket.consume(RETRY_COST)) { + throw error; + } + + const delayMS = backoffDelayProvider.getNextBackoffDuration(); + + // if the delay would exhaust the CSOT timeout, short-circuit. + if (timeoutContext.csotEnabled() && delayMS > timeoutContext.remainingTimeMS) { + // TODO: is this the right error to throw? + throw new MongoOperationTimeoutError( + `MongoDB SystemOverload exponential backoff would exceed timeoutMS deadline: remaining CSOT deadline=${timeoutContext.remainingTimeMS}, backoff delayMS=${delayMS}`, + { + cause: error + } + ); + } + + await setTimeout(delayMS); } if ( - previousOperationError instanceof MongoNetworkError && + operationError instanceof MongoNetworkError && operation.hasAspect(Aspect.CURSOR_CREATING) && session != null && session.isPinned && @@ -268,6 +367,8 @@ async function tryOperation 0 and we are command batching we need to reset the batch. - if (tries > 0 && operation.hasAspect(Aspect.COMMAND_BATCHING)) { + // Batched operations must reset the batch before retry, + // otherwise building a command will build the _next_ batch, not the current batch. + if (operation.hasAspect(Aspect.COMMAND_BATCHING)) { operation.resetBatch(); } - - try { - const result = await server.command(operation, timeoutContext); - return operation.handleOk(result); - } catch (error) { - return operation.handleError(error); - } - } catch (operationError) { - if (!(operationError instanceof MongoError)) throw operationError; - if ( - previousOperationError != null && - operationError.hasErrorLabel(MongoErrorLabel.NoWritesPerformed) - ) { - throw previousOperationError; - } - deprioritizedServers.add(server.description); - previousOperationError = operationError; - - // Reset timeouts - timeoutContext.clear(); } } throw ( - previousOperationError ?? - new MongoRuntimeError('Tried to propagate retryability error, but no error was found.') + error ?? + new MongoRuntimeError( + 'Should never happen: operation execution loop terminated but no error was recorded.' + ) ); + + function canRetry(operation: AbstractOperation, error: MongoError) { + // always retryable + if ( + error.hasErrorLabel(MongoErrorLabel.SystemOverloadedError) && + error.hasErrorLabel(MongoErrorLabel.RetryableError) + ) { + return true; + } + + // run command is only retryable if we get retryable overload errors + if (operation instanceof RunCommandOperation) { + return false; + } + + // batch operations are only retryable if the batch is retryable + if (operation.hasAspect(Aspect.COMMAND_BATCHING)) { + return operation.canRetryWrite; + } + + return ( + (hasWriteAspect && willRetryWrite && isRetryableWriteError(error)) || + (hasReadAspect && willRetryRead && isRetryableReadError(error)) + ); + } } diff --git a/src/operations/operation.ts b/src/operations/operation.ts index cb0181ee555..573bb13039f 100644 --- a/src/operations/operation.ts +++ b/src/operations/operation.ts @@ -66,6 +66,14 @@ export abstract class AbstractOperation { /** Specifies the time an operation will run until it throws a timeout error. */ timeoutMS?: number; + // Note for Sergey: sort of a hack, one that I planned to revisit before officially opening the PR for review. + // For every operation _except_ transactions, all backpressure retries occur inside one `executeOperation` call. `commitTransaction`, however, + // currently hard-codes two retry attempts, each with a `RunCommandOperation`. Neither retry attempt is retryable by itself. + // Because we use two separate RunCommand operations, it's possible to retry up to 2 * MAX_BACKPRESSURE_RETRIES times, if the server is overloaded. + // I hacked around it here by storing the max attempts for an operation on the operation, and sharing the value between both operations. + // I don't have a solid idea of a better way to do this right now though. + maxAttempts?: number; + private _session: ClientSession | undefined; static aspects?: Set; diff --git a/src/sdam/topology.ts b/src/sdam/topology.ts index 2a158525fa0..327d4d9a14c 100644 --- a/src/sdam/topology.ts +++ b/src/sdam/topology.ts @@ -35,6 +35,7 @@ import { type Abortable, TypedEventEmitter } from '../mongo_types'; import { ReadPreference, type ReadPreferenceLike } from '../read_preference'; import type { ClientSession } from '../sessions'; import { Timeout, TimeoutContext, TimeoutError } from '../timeout'; +import { TokenBucket } from '../token_bucket'; import type { Transaction } from '../transactions'; import { addAbortListener, @@ -207,18 +208,16 @@ export type TopologyEvents = { * @internal */ export class Topology extends TypedEventEmitter { - /** @internal */ s: TopologyPrivate; - /** @internal */ waitQueue: List; - /** @internal */ hello?: Document; - /** @internal */ _type?: string; + // Note for Sergey: expect this to change, and instead be stored on the server class. + tokenBucket = new TokenBucket(1000); + client!: MongoClient; - /** @internal */ private connectionLock?: Promise; /** @event */ diff --git a/src/sessions.ts b/src/sessions.ts index 875ed93afa6..744fc8aafd0 100644 --- a/src/sessions.ts +++ b/src/sessions.ts @@ -467,7 +467,11 @@ export class ClientSession } else { const wcKeys = Object.keys(wc); if (wcKeys.length > 2 || (!wcKeys.includes('wtimeoutMS') && !wcKeys.includes('wTimeoutMS'))) - // if the write concern was specified with wTimeoutMS, then we set both wtimeoutMS and wTimeoutMS, guaranteeing at least two keys, so if we have more than two keys, then we can automatically assume that we should add the write concern to the command. If it has 2 or fewer keys, we need to check that those keys aren't the wtimeoutMS or wTimeoutMS options before we add the write concern to the command + // if the write concern was specified with wTimeoutMS, then we set both wtimeoutMS + // and wTimeoutMS, guaranteeing at least two keys, so if we have more than two keys, + // then we can automatically assume that we should add the write concern to the command. + // If it has 2 or fewer keys, we need to check that those keys aren't the wtimeoutMS + // or wTimeoutMS options before we add the write concern to the command WriteConcern.apply(command, { ...wc, wtimeoutMS: undefined }); } } @@ -493,6 +497,7 @@ export class ClientSession readPreference: ReadPreference.primary, bypassPinningCheck: true }); + operation.maxAttempts = 5; const timeoutContext = this.timeoutContext ?? @@ -510,6 +515,8 @@ export class ClientSession return; } catch (firstCommitError) { this.commitAttempted = true; + // Backpressure exhausted all retries on the initial retry loop. + if (operation.maxAttempts === 5) throw firstCommitError; if (firstCommitError instanceof MongoError && isRetryableWriteError(firstCommitError)) { // SPEC-1185: apply majority write concern when retrying commitTransaction WriteConcern.apply(command, { wtimeoutMS: 10000, ...wc, w: 'majority' }); @@ -517,15 +524,13 @@ export class ClientSession this.unpin({ force: true }); try { - await executeOperation( - this.client, - new RunCommandOperation(new MongoDBNamespace('admin'), command, { - session: this, - readPreference: ReadPreference.primary, - bypassPinningCheck: true - }), - timeoutContext - ); + const op = new RunCommandOperation(new MongoDBNamespace('admin'), command, { + session: this, + readPreference: ReadPreference.primary, + bypassPinningCheck: true + }); + op.maxAttempts = operation.maxAttempts; + await executeOperation(this.client, op, timeoutContext); return; } catch (retryCommitError) { // If the retry failed, we process that error instead of the original @@ -1202,7 +1207,6 @@ export function applySession( command.autocommit = false; if (session.transaction.state === TxnState.STARTING_TRANSACTION) { - session.transaction.transition(TxnState.TRANSACTION_IN_PROGRESS); command.startTransaction = true; const readConcern = @@ -1240,4 +1244,17 @@ export function updateSessionFromResponse(session: ClientSession, document: Mong session.snapshotTime = atClusterTime; } } + + if (session.transaction.state === TxnState.STARTING_TRANSACTION) { + if (document.ok === 1) { + session.transaction.transition(TxnState.TRANSACTION_IN_PROGRESS); + } else { + const error = new MongoServerError(document.toObject()); + const isBackpressureError = error.hasErrorLabel(MongoErrorLabel.RetryableError); + + if (!isBackpressureError) { + session.transaction.transition(TxnState.TRANSACTION_IN_PROGRESS); + } + } + } } diff --git a/src/token_bucket.ts b/src/token_bucket.ts new file mode 100644 index 00000000000..76e88b660f5 --- /dev/null +++ b/src/token_bucket.ts @@ -0,0 +1,39 @@ +/** + * @internal + */ +export class TokenBucket { + private budget: number; + private capacity: number; + + constructor(allowance: number) { + this.budget = allowance; + this.capacity = allowance; + } + + deposit(tokens: number) { + this.budget = Math.min(this.budget + tokens, this.capacity); + } + + consume(tokens: number): boolean { + if (tokens > this.budget) return false; + + this.budget -= tokens; + return true; + } +} + +/** + * @internal + * The amount to deposit on successful operations, as defined in the backpressure specification. + */ +export const TOKEN_REFRESH_RATE = 0.1; +/** + * @internal + * The initial size of the token bucket, as defined in the backpressure specification. + */ +export const INITIAL_TOKEN_BUCKET_SIZE = 1000; +/** + * @internal + * The cost of a retry, as defined in the backpressure specification. + */ +export const RETRY_COST = 1; diff --git a/src/utils.ts b/src/utils.ts index 5aadbf4e624..05e93702dcf 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1419,3 +1419,23 @@ export async function abortable( abortListener?.[kDispose](); } } + +// Note for Sergey: overkill, I know - but sometimes you gotta over-engineer a bit for fun and revert before opening for review ¯\_(ツ)_/¯ +// feel free to remove this class. Or leave it. If you keep it, we should use this for the convenient transactions' API as well. +export class ExponentialBackoffProvider { + constructor( + public readonly maxBackoff: number, + public readonly baseBackoff: number, + public readonly backoffIncreaseRate: number, + public iteration = 0 + ) {} + + getNextBackoffDuration(): number { + const iteration = this.iteration; + this.iteration += 1; + return ( + Math.random() * + Math.min(this.maxBackoff, this.baseBackoff * this.backoffIncreaseRate ** iteration) + ); + } +} diff --git a/sync.sh b/sync.sh new file mode 100644 index 00000000000..b99d9ed1086 --- /dev/null +++ b/sync.sh @@ -0,0 +1,4 @@ + + +cp ~/dev/specifications/source/client-backpressure/tests/* ~/dev/node-mongodb-native/test/spec/client-backpressure +cp ~/dev/specifications/source/transactions/tests/unified/backpressure* ~/dev/node-mongodb-native/test/spec/transactions/unified/ \ No newline at end of file diff --git a/test/integration/client-backpressure/client-backpressure.prose.test.ts b/test/integration/client-backpressure/client-backpressure.prose.test.ts new file mode 100644 index 00000000000..7c437c28ce1 --- /dev/null +++ b/test/integration/client-backpressure/client-backpressure.prose.test.ts @@ -0,0 +1,60 @@ +import { expect } from 'chai'; +import * as sinon from 'sinon'; + +import { type Collection, type MongoClient, MongoServerError } from '../../../src'; +import { clearFailPoint, configureFailPoint, measureDuration } from '../../tools/utils'; + +describe('Client Backpressure (Prose)', function () { + let client: MongoClient; + let collection: Collection; + + beforeEach(async function () { + client = this.configuration.newClient(); + await client.connect(); + + collection = client.db('foo').collection('bar'); + }); + + afterEach(async function () { + await client.close(); + await clearFailPoint(this.configuration); + }); + + it( + 'Test 1: Operation Retry Uses Exponential Backoff', + { + requires: { + mongodb: '>=4.4' + } + }, + async function () { + await configureFailPoint(this.configuration, { + configureFailPoint: 'failCommand', + mode: 'alwaysOn', + data: { + failCommands: ['insert'], + errorCode: 2, + errorLabels: ['SystemOverloadedError', 'RetryableError'] + } + }); + + const stub = sinon.stub(Math, 'random'); + + stub.returns(0); + + const { duration: durationNoBackoff } = await measureDuration(async () => { + const error = await collection.insertOne({ a: 1 }).catch(e => e); + expect(error).to.be.instanceof(MongoServerError); + }); + + stub.returns(1); + + const { duration: durationBackoff } = await measureDuration(async () => { + const error = await collection.insertOne({ a: 1 }).catch(e => e); + expect(error).to.be.instanceof(MongoServerError); + }); + + expect(durationBackoff - durationNoBackoff).to.be.within(3100 - 1000, 3100 + 1000); + } + ); +}); diff --git a/test/integration/client-backpressure/client-backpressure.spec.test.ts b/test/integration/client-backpressure/client-backpressure.spec.test.ts new file mode 100644 index 00000000000..9485c46a96c --- /dev/null +++ b/test/integration/client-backpressure/client-backpressure.spec.test.ts @@ -0,0 +1,16 @@ +import { loadSpecTests } from '../../spec'; +import { runUnifiedSuite } from '../../tools/unified-spec-runner/runner'; +import { type Test } from '../../tools/unified-spec-runner/schema'; + +const skippedTests = { + 'collection.dropIndexes retries at most maxAttempts=5 times': + 'TODO(NODE-6517): dropIndexes squashes all errors other than ns not found' +}; + +function shouldSkip({ description }: Test) { + return skippedTests[description] ?? false; +} + +describe('Client Backpressure (spec)', function () { + runUnifiedSuite(loadSpecTests('client-backpressure'), shouldSkip); +}); diff --git a/test/integration/mongodb-handshake/mongodb-handshake.prose.test.ts b/test/integration/mongodb-handshake/mongodb-handshake.prose.test.ts index 6b9dad60995..6656e39f029 100644 --- a/test/integration/mongodb-handshake/mongodb-handshake.prose.test.ts +++ b/test/integration/mongodb-handshake/mongodb-handshake.prose.test.ts @@ -949,3 +949,38 @@ describe('Client Metadata Update Prose Tests', function () { } }); }); + +// TODO: add prose test descriptions here to align the test with the spec. +describe('Backpressure Metadata', function () { + let client: MongoClient; + let spy: sinon.SinonSpy>; + + beforeEach(async function () { + client = this.configuration.newClient(); + spy = sinon.spy(Connection.prototype, 'command'); + await client.connect(); + + // run an operation to force a connection establishment, + // if we're testing noauth load balanced mode. + await client.db('foo').collection('bar').insertOne({ name: 'bumpy' }); + }); + + afterEach(async function () { + sinon.restore(); + await client?.close(); + }); + + it('includes backpressure in the handshake document', function () { + const isHello = (cmd: Document): cmd is HandshakeDocument => + `hello` in cmd || LEGACY_HELLO_COMMAND in cmd; + + const hellos = spy.args.map(([_ns, command, _options]) => command).filter(isHello); + + expect(hellos.length).to.be.greaterThan(0); + + expect( + hellos.every(hello => hello.backpressure === true), + `some handshake documents did not specify backpressure: true` + ); + }); +}); diff --git a/test/integration/retryable-writes/retryable_writes.spec.prose.test.ts b/test/integration/retryable-writes/retryable_writes.spec.prose.test.ts index 22f475978d0..a494a3695a8 100644 --- a/test/integration/retryable-writes/retryable_writes.spec.prose.test.ts +++ b/test/integration/retryable-writes/retryable_writes.spec.prose.test.ts @@ -8,6 +8,7 @@ import { type Collection, type MongoClient, MongoError, + MongoErrorLabel, MongoServerError, MongoWriteConcernError, Server @@ -344,4 +345,106 @@ describe('Retryable Writes Spec Prose', () => { } ); }); + + describe('6. Test that drivers return the original error after encountering multiple WriteConcernErrors with a RetryableWriteError label', () => { + let client: MongoClient; + let collection: Collection<{ _id: 1 }>; + + beforeEach(async function () { + // This test MUST: + // - be implemented by any driver that implements the Command Monitoring specification, + // - only run against replica sets as mongos does not propagate the NoWritesPerformed label to the drivers. + // - be run against server versions 6.0 and above. + // - be implemented by any driver that has implemented the Client Backpressure specification. + + // Additionally, this test requires drivers to set a fail point after an `insertOne` operation but before the subsequent + // retry. Drivers that are unable to set a failCommand after the CommandFailedEvent SHOULD use mocking or write a unit test + // to cover the same sequence of events. + + // 1. Create a client with `retryWrites=true`. + client = this.configuration.newClient({ monitorCommands: true, retryWrites: true }); + await client + .db() + .collection('retryReturnsOriginal') + .drop() + .catch(() => null); + collection = client.db().collection('retryReturnsOriginal'); + }); + + afterEach(async function () { + // 5. Disable the fail point: + // ```javascript + // { + // configureFailPoint: "failCommand", + // mode: "off" + // } + // ``` + + // (we don't use a failPoint, so we use sinon.restore instead) + sinon.restore(); + await client.close(); + }); + + it( + 'when a retry attempt fails with an error labeled NoWritesPerformed, drivers MUST return the original error', + { requires: { topology: 'replicaset', mongodb: '>=4.2.9' } }, + async () => { + // 2. Configure a fail point with error code `91` (ShutdownInProgress) with the `RetryableError` and + // `SystemOverloadedError` error labels: + + // ```javascript + // { + // configureFailPoint: "failCommand", + // mode: {times: 1}, + // data: { + // failCommands: ["insert"], + // errorLabels: ["RetryableError", "SystemOverloadedError", "NoWritesPerformed"], + // errorCode: 91 + // } + // } + // ``` + + // 3. Via the command monitoring CommandFailedEvent, configure a fail point with error code `10107` (NotWritablePrimary) + // and a NoWritesPerformed label: + + // ```javascript + // { + // configureFailPoint: "failCommand", + // mode: "alwaysOn", + // data: { + // failCommands: ["insert"], + // errorCode: 10107, + // errorLabels: ["RetryableError", "SystemOverloadedError", , "NoWritesPerformed"] + // } + // } + // ``` + + // Drivers SHOULD only configure the `10107` fail point command if the the failed event is for the `91` error configured + // in step 2. + const serverCommandStub = sinon + .stub(Server.prototype, 'command') + .callsFake(async function () { + throw new MongoServerError({ + message: 'Server Error', + errorLabels: [ + MongoErrorLabel.RetryableError, + MongoErrorLabel.SystemOverloadedError, + MongoErrorLabel.NoWritesPerformed + ], + code: serverCommandStub.callCount === 1 ? 91 : 10107, + ok: 0 + }); + }); + + const insertResult = await collection.insertOne({ _id: 1 }).catch(error => error); + + expect(serverCommandStub.callCount).to.equal(6); + + // 4. Attempt an `insertOne` operation on any record for any database and collection. Expect the `insertOne` to fail with a + // server error. Assert that the error code of the server error is 91. + expect(insertResult).to.be.instanceOf(MongoServerError); + expect(insertResult).to.have.property('code', 91); + } + ); + }); }); diff --git a/test/integration/transactions/transactions.spec.test.ts b/test/integration/transactions/transactions.spec.test.ts index ab6f84b1e94..b24f1a667cc 100644 --- a/test/integration/transactions/transactions.spec.test.ts +++ b/test/integration/transactions/transactions.spec.test.ts @@ -13,7 +13,7 @@ const SKIPPED_TESTS = [ 'client bulkWrite with writeConcern in a transaction causes a transaction error' ]; -describe('Transactions Spec Unified Tests', function () { +describe.only('Transactions Spec Unified Tests', function () { runUnifiedSuite(loadSpecTests(path.join('transactions', 'unified')), test => { return SKIPPED_TESTS.includes(test.description) ? 'TODO(NODE-5924): Skipping failing transaction tests' diff --git a/test/spec/client-backpressure/README.md b/test/spec/client-backpressure/README.md new file mode 100644 index 00000000000..e9048e1b4a3 --- /dev/null +++ b/test/spec/client-backpressure/README.md @@ -0,0 +1,57 @@ +# Client Backpressure Tests + +______________________________________________________________________ + +## Introduction + +The YAML and JSON files in this directory are platform-independent tests meant to exercise a driver's implementation of +retryable reads. These tests utilize the [Unified Test Format](../../unified-test-format/unified-test-format.md). + +Several prose tests, which are not easily expressed in YAML, are also presented in this file. Those tests will need to +be manually implemented by each driver. + +### Prose Tests + +#### Test 1: Operation Retry Uses Exponential Backoff + +Drivers should test that retries do not occur immediately when a SystemOverloadedError is encountered. + +1. Let `client` be a `MongoClient` +2. Let `collection` be a collection +3. Now, run transactions without backoff: + 1. Configure the random number generator used for jitter to always return `0` -- this effectively disables backoff. + + 2. Configure the following failPoint: + + ```javascript + { + configureFailPoint: 'failCommand', + mode: 'alwaysOn', + data: { + failCommands: ['insert'], + errorCode: 2, + errorLabels: ['SystemOverloadedError', 'RetryableError'] + } + } + ``` + + 3. Insert the document `{ a: 1 }`. Expect that the command errors. Measure the duration of the command execution. + + ```javascript + const start = performance.now(); + expect( + await coll.insertOne({ a: 1 }).catch(e => e) + ).to.be.an.instanceof(MongoServerError); + const end = performance.now(); + ``` + + 4. Configure the random number generator used for jitter to always return `1`. + + 5. Execute step 3 again. + + 6. Compare the two time between the two runs. + ```python + assertTrue(with_backoff_time - no_backoff_time >= 2.1) + ``` + The sum of 5 backoffs is 3.1 seconds. There is a 1-second window to account for potential variance between the two + runs. diff --git a/test/spec/client-backpressure/backpressure-retry-loop.json b/test/spec/client-backpressure/backpressure-retry-loop.json new file mode 100644 index 00000000000..8c01020ca52 --- /dev/null +++ b/test/spec/client-backpressure/backpressure-retry-loop.json @@ -0,0 +1,2759 @@ +{ + "description": "tests that operations respect overload backoff retry loop", + "schemaVersion": "1.3", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "client": { + "id": "internal_client", + "useMultipleMongoses": false + } + }, + { + "database": { + "id": "internal_db", + "client": "internal_client", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "retryable-writes-tests", + "database": "internal_db", + "collectionName": "coll" + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "coll" + } + } + ], + "initialData": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [] + } + ], + "_yamlAnchors": { + "bulWriteInsertNamespace": "retryable-writes-tests.coll" + }, + "tests": [ + { + "description": "client.listDatabases retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "listDatabases", + "object": "client", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandSucceededEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, + { + "description": "client.listDatabaseNames retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "listDatabaseNames", + "object": "client" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandSucceededEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, + { + "description": "client.createChangeStream retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "createChangeStream", + "object": "client", + "arguments": { + "pipeline": [] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "client.clientBulkWrite retries using operation loop", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "clientBulkWrite", + "object": "client", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-tests.coll", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandSucceededEvent": { + "commandName": "bulkWrite" + } + } + ] + } + ] + }, + { + "description": "database.aggregate retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "aggregate", + "object": "database", + "arguments": { + "pipeline": [ + { + "$listLocalSessions": {} + }, + { + "$limit": 1 + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "database.listCollections retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "listCollections", + "object": "database", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandSucceededEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.listCollectionNames retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "listCollectionNames", + "object": "database", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandSucceededEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.runCommand retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "ping" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "runCommand", + "object": "database", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandSucceededEvent": { + "commandName": "ping" + } + } + ] + } + ] + }, + { + "description": "database.createChangeStream retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "createChangeStream", + "object": "database", + "arguments": { + "pipeline": [] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.aggregate retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "aggregate", + "object": "collection", + "arguments": { + "pipeline": [] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.countDocuments retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "countDocuments", + "object": "collection", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.estimatedDocumentCount retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "count" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "estimatedDocumentCount", + "object": "collection" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandSucceededEvent": { + "commandName": "count" + } + } + ] + } + ] + }, + { + "description": "collection.distinct retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "distinct" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "distinct", + "object": "collection", + "arguments": { + "fieldName": "x", + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandSucceededEvent": { + "commandName": "distinct" + } + } + ] + } + ] + }, + { + "description": "collection.find retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "find", + "object": "collection", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandSucceededEvent": { + "commandName": "find" + } + } + ] + } + ] + }, + { + "description": "collection.findOne retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "findOne", + "object": "collection", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandSucceededEvent": { + "commandName": "find" + } + } + ] + } + ] + }, + { + "description": "collection.listIndexes retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "listIndexes", + "object": "collection" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.listIndexNames retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "listIndexNames", + "object": "collection" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.createChangeStream retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "createChangeStream", + "object": "collection", + "arguments": { + "pipeline": [] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.insertOne retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "insertOne", + "object": "collection", + "arguments": { + "document": { + "_id": 2, + "x": 22 + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandSucceededEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.insertMany retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "insertMany", + "object": "collection", + "arguments": { + "documents": [ + { + "_id": 2, + "x": 22 + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandSucceededEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.deleteOne retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "deleteOne", + "object": "collection", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandSucceededEvent": { + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "collection.deleteMany retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "deleteMany", + "object": "collection", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandSucceededEvent": { + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "collection.replaceOne retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "replaceOne", + "object": "collection", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandSucceededEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateOne retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "updateOne", + "object": "collection", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandSucceededEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateMany retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "updateMany", + "object": "collection", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandSucceededEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndDelete retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "findOneAndDelete", + "object": "collection", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandSucceededEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndReplace retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "findOneAndReplace", + "object": "collection", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandSucceededEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndUpdate retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "findOneAndUpdate", + "object": "collection", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandSucceededEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.bulkWrite retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "bulkWrite", + "object": "collection", + "arguments": { + "requests": [ + { + "insertOne": { + "document": { + "_id": 2, + "x": 22 + } + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandSucceededEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.createIndex retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "createIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "createIndex", + "object": "collection", + "arguments": { + "keys": { + "x": 11 + }, + "name": "x_11" + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "createIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.dropIndex retries using operation loop", + "operations": [ + { + "name": "createIndex", + "object": "retryable-writes-tests", + "arguments": { + "keys": { + "x": 11 + }, + "name": "x_11" + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "dropIndex", + "object": "collection", + "arguments": { + "name": "x_11" + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.dropIndexes retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "dropIndexes", + "object": "collection" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] + } + ] +} diff --git a/test/spec/client-backpressure/backpressure-retry-loop.yml b/test/spec/client-backpressure/backpressure-retry-loop.yml new file mode 100644 index 00000000000..eda57d4979d --- /dev/null +++ b/test/spec/client-backpressure/backpressure-retry-loop.yml @@ -0,0 +1,1307 @@ +# Tests in this file are generated from backpressure-retry-loop.yml.template. + +description: tests that operations respect overload backoff retry loop + +schemaVersion: '1.3' + +runOnRequirements: + - minServerVersion: '4.4' # failCommand + topologies: [replicaset, sharded, load-balanced] + +createEntities: + - client: + id: &client client + useMultipleMongoses: false + observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] + + - client: + id: &internal_client internal_client + useMultipleMongoses: false + + - database: + id: &internal_db internal_db + client: *internal_client + databaseName: &database_name retryable-writes-tests + + - collection: + id: &internal_collection retryable-writes-tests + database: *internal_db + collectionName: &collection_name coll + + - database: + id: &database database + client: *client + databaseName: *database_name + + - collection: + id: &collection collection + database: *database + collectionName: *collection_name + +initialData: +- collectionName: *collection_name + databaseName: *database_name + documents: [] + +_yamlAnchors: + bulWriteInsertNamespace: &client_bulk_write_ns retryable-writes-tests.coll + +tests: + - description: 'client.listDatabases retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listDatabases] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: listDatabases + object: *client + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandSucceededEvent: + commandName: listDatabases + + - description: 'client.listDatabaseNames retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listDatabases] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: listDatabaseNames + object: *client + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandSucceededEvent: + commandName: listDatabases + + - description: 'client.createChangeStream retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: createChangeStream + object: *client + arguments: + pipeline: [] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - description: 'client.clientBulkWrite retries using operation loop' + runOnRequirements: + - minServerVersion: '8.0' # client bulk write added to server in 8.0 + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [bulkWrite] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: clientBulkWrite + object: *client + arguments: + models: + - insertOne: + namespace: *client_bulk_write_ns + document: { _id: 8, x: 88 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandSucceededEvent: + commandName: bulkWrite + + - description: 'database.aggregate retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: aggregate + object: *database + arguments: + pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - description: 'database.listCollections retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listCollections] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: listCollections + object: *database + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandSucceededEvent: + commandName: listCollections + + - description: 'database.listCollectionNames retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listCollections] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: listCollectionNames + object: *database + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandSucceededEvent: + commandName: listCollections + + - description: 'database.runCommand retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [ping] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: runCommand + object: *database + arguments: + command: { ping: 1 } + commandName: ping + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandSucceededEvent: + commandName: ping + + - description: 'database.createChangeStream retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: createChangeStream + object: *database + arguments: + pipeline: [] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - description: 'collection.aggregate retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: aggregate + object: *collection + arguments: + pipeline: [] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - description: 'collection.countDocuments retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: countDocuments + object: *collection + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - description: 'collection.estimatedDocumentCount retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [count] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: estimatedDocumentCount + object: *collection + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandSucceededEvent: + commandName: count + + - description: 'collection.distinct retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [distinct] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: distinct + object: *collection + arguments: + fieldName: x + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandSucceededEvent: + commandName: distinct + + - description: 'collection.find retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [find] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: find + object: *collection + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandSucceededEvent: + commandName: find + + - description: 'collection.findOne retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [find] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: findOne + object: *collection + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandSucceededEvent: + commandName: find + + - description: 'collection.listIndexes retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: listIndexes + object: *collection + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandSucceededEvent: + commandName: listIndexes + + - description: 'collection.listIndexNames retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: listIndexNames + object: *collection + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandSucceededEvent: + commandName: listIndexes + + - description: 'collection.createChangeStream retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: createChangeStream + object: *collection + arguments: + pipeline: [] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - description: 'collection.insertOne retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [insert] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: insertOne + object: *collection + arguments: + document: { _id: 2, x: 22 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandSucceededEvent: + commandName: insert + + - description: 'collection.insertMany retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [insert] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: insertMany + object: *collection + arguments: + documents: + - { _id: 2, x: 22 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandSucceededEvent: + commandName: insert + + - description: 'collection.deleteOne retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [delete] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: deleteOne + object: *collection + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandSucceededEvent: + commandName: delete + + - description: 'collection.deleteMany retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [delete] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: deleteMany + object: *collection + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandSucceededEvent: + commandName: delete + + - description: 'collection.replaceOne retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [update] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: replaceOne + object: *collection + arguments: + filter: {} + replacement: { x: 22 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandSucceededEvent: + commandName: update + + - description: 'collection.updateOne retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [update] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: updateOne + object: *collection + arguments: + filter: {} + update: { $set: { x: 22 } } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandSucceededEvent: + commandName: update + + - description: 'collection.updateMany retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [update] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: updateMany + object: *collection + arguments: + filter: {} + update: { $set: { x: 22 } } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandSucceededEvent: + commandName: update + + - description: 'collection.findOneAndDelete retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [findAndModify] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: findOneAndDelete + object: *collection + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandSucceededEvent: + commandName: findAndModify + + - description: 'collection.findOneAndReplace retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [findAndModify] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: findOneAndReplace + object: *collection + arguments: + filter: {} + replacement: { x: 22 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandSucceededEvent: + commandName: findAndModify + + - description: 'collection.findOneAndUpdate retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [findAndModify] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: findOneAndUpdate + object: *collection + arguments: + filter: {} + update: { $set: { x: 22 } } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandSucceededEvent: + commandName: findAndModify + + - description: 'collection.bulkWrite retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [insert] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: bulkWrite + object: *collection + arguments: + requests: + - insertOne: + document: { _id: 2, x: 22 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandSucceededEvent: + commandName: insert + + - description: 'collection.createIndex retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [createIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: createIndex + object: *collection + arguments: + keys: { x: 11 } + name: "x_11" + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandSucceededEvent: + commandName: createIndexes + + - description: 'collection.dropIndex retries using operation loop' + operations: + - name: createIndex + object: *internal_collection + arguments: + keys: { x: 11 } + name: "x_11" + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [dropIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: dropIndex + object: *collection + arguments: + name: "x_11" + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandSucceededEvent: + commandName: dropIndexes + + - description: 'collection.dropIndexes retries using operation loop' + operations: + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [dropIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: dropIndexes + object: *collection + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandSucceededEvent: + commandName: dropIndexes diff --git a/test/spec/client-backpressure/backpressure-retry-loop.yml.template b/test/spec/client-backpressure/backpressure-retry-loop.yml.template new file mode 100644 index 00000000000..bd64827d4e6 --- /dev/null +++ b/test/spec/client-backpressure/backpressure-retry-loop.yml.template @@ -0,0 +1,101 @@ +# Tests in this file are generated from backpressure-retry-loop.yml.template. + +description: tests that operations respect overload backoff retry loop + +schemaVersion: '1.3' + +runOnRequirements: + - minServerVersion: '4.4' # failCommand + topologies: [replicaset, sharded, load-balanced] + +createEntities: + - client: + id: &client client + useMultipleMongoses: false + observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] + + - client: + id: &internal_client internal_client + useMultipleMongoses: false + + - database: + id: &internal_db internal_db + client: *internal_client + databaseName: &database_name retryable-writes-tests + + - collection: + id: &internal_collection retryable-writes-tests + database: *internal_db + collectionName: &collection_name coll + + - database: + id: &database database + client: *client + databaseName: *database_name + + - collection: + id: &collection collection + database: *database + collectionName: *collection_name + +initialData: +- collectionName: *collection_name + databaseName: *database_name + documents: [] + +_yamlAnchors: + bulWriteInsertNamespace: &client_bulk_write_ns retryable-writes-tests.coll + +tests: {% for operation in operations %} + - description: '{{operation.object}}.{{operation.operation_name}} retries using operation loop' {%- if ((operation.operation_name == 'clientBulkWrite')) %} + runOnRequirements: + - minServerVersion: '8.0' # client bulk write added to server in 8.0 + {%- endif %} + operations: {%- if operation.operation_name == "dropIndex" %} + - name: createIndex + object: *internal_collection + arguments: + keys: { x: 11 } + name: "x_11" + {%- endif %} + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [{{operation.command_name}}] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: {{operation.operation_name}} + object: *{{operation.object}} + {%- if operation.arguments|length > 0 %} + arguments: + {%- for arg in operation.arguments %} + {{arg}} + {%- endfor -%} + {%- endif %} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandSucceededEvent: + commandName: {{operation.command_name}} +{% endfor -%} diff --git a/test/spec/client-backpressure/backpressure-retry-max-attempts.json b/test/spec/client-backpressure/backpressure-retry-max-attempts.json new file mode 100644 index 00000000000..efde5426212 --- /dev/null +++ b/test/spec/client-backpressure/backpressure-retry-max-attempts.json @@ -0,0 +1,3448 @@ +{ + "description": "tests that operations retry at most maxAttempts=5 times", + "schemaVersion": "1.3", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "client": { + "id": "fail_point_client", + "useMultipleMongoses": false + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "coll" + } + } + ], + "_yamlAnchors": { + "bulkWriteInsertNamespace": "retryable-writes-tests.coll" + }, + "initialData": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ], + "tests": [ + { + "description": "client.listDatabases retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "listDatabases", + "object": "client", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, + { + "description": "client.listDatabaseNames retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "listDatabaseNames", + "object": "client", + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, + { + "description": "client.createChangeStream retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "createChangeStream", + "object": "client", + "arguments": { + "pipeline": [] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "client.clientBulkWrite retries at most maxAttempts=5 times", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "clientBulkWrite", + "object": "client", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-tests.coll", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + } + ] + } + ] + }, + { + "description": "database.aggregate retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "aggregate", + "object": "database", + "arguments": { + "pipeline": [ + { + "$listLocalSessions": {} + }, + { + "$limit": 1 + } + ] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "database.listCollections retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "listCollections", + "object": "database", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.listCollectionNames retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "listCollectionNames", + "object": "database", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.runCommand retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "ping" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "runCommand", + "object": "database", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + } + ] + } + ] + }, + { + "description": "database.createChangeStream retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "createChangeStream", + "object": "database", + "arguments": { + "pipeline": [] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.aggregate retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "aggregate", + "object": "collection", + "arguments": { + "pipeline": [] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.countDocuments retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "countDocuments", + "object": "collection", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.estimatedDocumentCount retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "count" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "estimatedDocumentCount", + "object": "collection", + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + } + ] + } + ] + }, + { + "description": "collection.distinct retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "distinct" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "distinct", + "object": "collection", + "arguments": { + "fieldName": "x", + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + } + ] + } + ] + }, + { + "description": "collection.find retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "find", + "object": "collection", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + } + ] + } + ] + }, + { + "description": "collection.findOne retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "findOne", + "object": "collection", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + } + ] + } + ] + }, + { + "description": "collection.listIndexes retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "listIndexes", + "object": "collection", + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.listIndexNames retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "listIndexNames", + "object": "collection", + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.createChangeStream retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "createChangeStream", + "object": "collection", + "arguments": { + "pipeline": [] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.insertOne retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "insertOne", + "object": "collection", + "arguments": { + "document": { + "_id": 2, + "x": 22 + } + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.insertMany retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "insertMany", + "object": "collection", + "arguments": { + "documents": [ + { + "_id": 2, + "x": 22 + } + ] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.deleteOne retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "deleteOne", + "object": "collection", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "collection.deleteMany retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "deleteMany", + "object": "collection", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "collection.replaceOne retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "replaceOne", + "object": "collection", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateOne retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "updateOne", + "object": "collection", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateMany retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "updateMany", + "object": "collection", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndDelete retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "findOneAndDelete", + "object": "collection", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndReplace retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "findOneAndReplace", + "object": "collection", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndUpdate retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "findOneAndUpdate", + "object": "collection", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.bulkWrite retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "bulkWrite", + "object": "collection", + "arguments": { + "requests": [ + { + "insertOne": { + "document": { + "_id": 2, + "x": 22 + } + } + } + ] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.createIndex retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "createIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "createIndex", + "object": "collection", + "arguments": { + "keys": { + "x": 11 + }, + "name": "x_11" + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.dropIndex retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "dropIndex", + "object": "collection", + "arguments": { + "name": "x_11" + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.dropIndexes retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "dropIndexes", + "object": "collection", + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] + } + ] +} diff --git a/test/spec/client-backpressure/backpressure-retry-max-attempts.yml b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml new file mode 100644 index 00000000000..39e4859368a --- /dev/null +++ b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml @@ -0,0 +1,1711 @@ +# Tests in this file are generated from backpressure-retry-max-attempts.yml.template. + +description: tests that operations retry at most maxAttempts=5 times + +schemaVersion: '1.3' + +runOnRequirements: + - + minServerVersion: '4.4' # failCommand + topologies: [replicaset, sharded, load-balanced] + +createEntities: + - client: + id: &client client + useMultipleMongoses: false + observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] + + - client: + id: &fail_point_client fail_point_client + useMultipleMongoses: false + + - database: + id: &database database + client: *client + databaseName: &database_name retryable-writes-tests + + - collection: + id: &collection collection + database: *database + collectionName: &collection_name coll + +_yamlAnchors: + bulkWriteInsertNamespace: &client_bulk_write_ns retryable-writes-tests.coll + +initialData: + - collectionName: *collection_name + databaseName: *database_name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +tests: + - description: 'client.listDatabases retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [listDatabases] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: listDatabases + object: *client + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + + - description: 'client.listDatabaseNames retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [listDatabases] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: listDatabaseNames + object: *client + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + + - description: 'client.createChangeStream retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: createChangeStream + object: *client + arguments: + pipeline: [] + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + - description: 'client.clientBulkWrite retries at most maxAttempts=5 times' + runOnRequirements: + - minServerVersion: '8.0' # client bulk write added to server in 8.0 + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [bulkWrite] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: clientBulkWrite + object: *client + arguments: + models: + - insertOne: + namespace: *client_bulk_write_ns + document: { _id: 8, x: 88 } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + + - description: 'database.aggregate retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: aggregate + object: *database + arguments: + pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + - description: 'database.listCollections retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [listCollections] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: listCollections + object: *database + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + + - description: 'database.listCollectionNames retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [listCollections] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: listCollectionNames + object: *database + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + + - description: 'database.runCommand retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [ping] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: runCommand + object: *database + arguments: + command: { ping: 1 } + commandName: ping + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + + - description: 'database.createChangeStream retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: createChangeStream + object: *database + arguments: + pipeline: [] + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + - description: 'collection.aggregate retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: aggregate + object: *collection + arguments: + pipeline: [] + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + - description: 'collection.countDocuments retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: countDocuments + object: *collection + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + - description: 'collection.estimatedDocumentCount retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [count] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: estimatedDocumentCount + object: *collection + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + + - description: 'collection.distinct retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [distinct] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: distinct + object: *collection + arguments: + fieldName: x + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + + - description: 'collection.find retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [find] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: find + object: *collection + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + + - description: 'collection.findOne retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [find] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: findOne + object: *collection + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + + - description: 'collection.listIndexes retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [listIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: listIndexes + object: *collection + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + + - description: 'collection.listIndexNames retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [listIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: listIndexNames + object: *collection + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + + - description: 'collection.createChangeStream retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: createChangeStream + object: *collection + arguments: + pipeline: [] + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + - description: 'collection.insertOne retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [insert] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: insertOne + object: *collection + arguments: + document: { _id: 2, x: 22 } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + + - description: 'collection.insertMany retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [insert] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: insertMany + object: *collection + arguments: + documents: + - { _id: 2, x: 22 } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + + - description: 'collection.deleteOne retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [delete] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: deleteOne + object: *collection + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + + - description: 'collection.deleteMany retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [delete] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: deleteMany + object: *collection + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + + - description: 'collection.replaceOne retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [update] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: replaceOne + object: *collection + arguments: + filter: {} + replacement: { x: 22 } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + + - description: 'collection.updateOne retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [update] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: updateOne + object: *collection + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + + - description: 'collection.updateMany retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [update] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: updateMany + object: *collection + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + + - description: 'collection.findOneAndDelete retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [findAndModify] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: findOneAndDelete + object: *collection + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + + - description: 'collection.findOneAndReplace retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [findAndModify] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: findOneAndReplace + object: *collection + arguments: + filter: {} + replacement: { x: 22 } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + + - description: 'collection.findOneAndUpdate retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [findAndModify] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: findOneAndUpdate + object: *collection + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + + - description: 'collection.bulkWrite retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [insert] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: bulkWrite + object: *collection + arguments: + requests: + - insertOne: + document: { _id: 2, x: 22 } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + + - description: 'collection.createIndex retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [createIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: createIndex + object: *collection + arguments: + keys: { x: 11 } + name: "x_11" + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + + - description: 'collection.dropIndex retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [dropIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: dropIndex + object: *collection + arguments: + name: "x_11" + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + + - description: 'collection.dropIndexes retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [dropIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: dropIndexes + object: *collection + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes diff --git a/test/spec/client-backpressure/backpressure-retry-max-attempts.yml.template b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml.template new file mode 100644 index 00000000000..3a9cb27ab53 --- /dev/null +++ b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml.template @@ -0,0 +1,101 @@ +# Tests in this file are generated from backpressure-retry-max-attempts.yml.template. + +description: tests that operations retry at most maxAttempts=5 times + +schemaVersion: '1.3' + +runOnRequirements: + - + minServerVersion: '4.4' # failCommand + topologies: [replicaset, sharded, load-balanced] + +createEntities: + - client: + id: &client client + useMultipleMongoses: false + observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] + + - client: + id: &fail_point_client fail_point_client + useMultipleMongoses: false + + - database: + id: &database database + client: *client + databaseName: &database_name retryable-writes-tests + + - collection: + id: &collection collection + database: *database + collectionName: &collection_name coll + +_yamlAnchors: + bulkWriteInsertNamespace: &client_bulk_write_ns retryable-writes-tests.coll + +initialData: + - collectionName: *collection_name + databaseName: *database_name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +tests: {% for operation in operations %} + - description: '{{operation.object}}.{{operation.operation_name}} retries at most maxAttempts=5 times' + {%- if ((operation.operation_name == 'clientBulkWrite')) %} + runOnRequirements: + - minServerVersion: '8.0' # client bulk write added to server in 8.0 + {%- endif %} + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [{{operation.command_name}}] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: {{operation.operation_name}} + object: *{{operation.object}} {%- if operation.arguments|length > 0 %} + arguments: + {%- for arg in operation.arguments %} + {{arg}} + {%- endfor -%} + {%- endif %} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} +{% endfor -%} diff --git a/test/spec/client-backpressure/getMore-retried.json b/test/spec/client-backpressure/getMore-retried.json new file mode 100644 index 00000000000..fed6ab8cb95 --- /dev/null +++ b/test/spec/client-backpressure/getMore-retried.json @@ -0,0 +1,291 @@ +{ + "description": "getMore-retried-backpressure", + "schemaVersion": "1.3", + "runOnRequirements": [ + { + "minServerVersion": "4.4" + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent", + "commandFailedEvent", + "commandSucceededEvent" + ] + } + }, + { + "client": { + "id": "failPointClient" + } + }, + { + "database": { + "id": "db", + "client": "client0", + "databaseName": "default" + } + }, + { + "collection": { + "id": "coll", + "database": "db", + "collectionName": "default" + } + } + ], + "initialData": [ + { + "databaseName": "default", + "collectionName": "default", + "documents": [ + { + "a": 1 + }, + { + "a": 2 + }, + { + "a": 3 + } + ] + } + ], + "tests": [ + { + "description": "getMores are retried", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "getMore" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "find", + "object": "coll", + "arguments": { + "batchSize": 2, + "filter": {}, + "sort": { + "a": 1 + } + }, + "expectResult": [ + { + "a": 1 + }, + { + "a": 2 + }, + { + "a": 3 + } + ] + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandSucceededEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandFailedEvent": { + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandFailedEvent": { + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandFailedEvent": { + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandSucceededEvent": { + "commandName": "getMore" + } + } + ] + } + ] + }, + { + "description": "getMores are retried maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "getMore" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "find", + "arguments": { + "batchSize": 2, + "filter": {} + }, + "object": "coll", + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandSucceededEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandFailedEvent": { + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandFailedEvent": { + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandFailedEvent": { + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandFailedEvent": { + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandFailedEvent": { + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandFailedEvent": { + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "commandName": "killCursors" + } + }, + { + "commandSucceededEvent": { + "commandName": "killCursors" + } + } + ] + } + ] + } + ] +} diff --git a/test/spec/client-backpressure/getMore-retried.yml b/test/spec/client-backpressure/getMore-retried.yml new file mode 100644 index 00000000000..45f5ad9deb6 --- /dev/null +++ b/test/spec/client-backpressure/getMore-retried.yml @@ -0,0 +1,149 @@ +description: getMore-retried-backpressure +schemaVersion: "1.3" +runOnRequirements: + - minServerVersion: '4.4' # failCommand +createEntities: + - client: + id: &client client0 + observeEvents: + - commandStartedEvent + - commandFailedEvent + - commandSucceededEvent + - client: + id: &failPointClient failPointClient + - database: + id: db + client: *client + databaseName: &dbName default + - collection: + id: &collection coll + database: db + collectionName: &collectionName default +initialData: + - databaseName: *dbName + collectionName: *collectionName + documents: + - { a: 1 } + - { a: 2 } + - { a: 3 } + +tests: + - description: "getMores are retried" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [getMore] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: find + object: *collection + arguments: + # batch size of 2 with 3 docs in the collection ensures exactly one find + one getMore exhaust the cursor + batchSize: 2 + filter: {} + # ensure stable ordering of result documents + sort: { a: 1 } + expectResult: + - { a: 1 } + - { a: 2 } + - { a: 3 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: find + - commandSucceededEvent: + commandName: find + # first attempt + - commandStartedEvent: + commandName: getMore + - commandFailedEvent: + commandName: getMore + # second attempt + - commandStartedEvent: + commandName: getMore + - commandFailedEvent: + commandName: getMore + # third attempt + - commandStartedEvent: + commandName: getMore + - commandFailedEvent: + commandName: getMore + # success + - commandStartedEvent: + commandName: getMore + - commandSucceededEvent: + commandName: getMore + + - description: "getMores are retried maxAttempts=5 times" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [getMore] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: find + arguments: + batchSize: 2 + filter: {} + object: *collection + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: find + - commandSucceededEvent: + commandName: find + # first attempt + - commandStartedEvent: + commandName: getMore + - commandFailedEvent: + commandName: getMore + # second attempt + - commandStartedEvent: + commandName: getMore + - commandFailedEvent: + commandName: getMore + # third attempt + - commandStartedEvent: + commandName: getMore + - commandFailedEvent: + commandName: getMore + # fourth attempt + - commandStartedEvent: + commandName: getMore + - commandFailedEvent: + commandName: getMore + # fifth attempt + - commandStartedEvent: + commandName: getMore + - commandFailedEvent: + commandName: getMore + # final attempt + - commandStartedEvent: + commandName: getMore + - commandFailedEvent: + commandName: getMore + - commandStartedEvent: + commandName: killCursors + - commandSucceededEvent: + commandName: killCursors \ No newline at end of file diff --git a/test/spec/transactions/unified/backpressure-retryable-abort.json b/test/spec/transactions/unified/backpressure-retryable-abort.json new file mode 100644 index 00000000000..53fc9c6f090 --- /dev/null +++ b/test/spec/transactions/unified/backpressure-retryable-abort.json @@ -0,0 +1,357 @@ +{ + "description": "backpressure-retryable-abort", + "schemaVersion": "1.3", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "transaction-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + }, + { + "session": { + "id": "session0", + "client": "client0" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ], + "tests": [ + { + "description": "abortTransaction retries if backpressure labels are added", + "operations": [ + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 2 + }, + "data": { + "failCommands": [ + "abortTransaction" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 112 + } + } + } + }, + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + }, + { + "object": "session0", + "name": "abortTransaction" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "abortTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "abortTransaction", + "databaseName": "admin" + } + }, + { + "commandStartedEvent": { + "command": { + "abortTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "abortTransaction", + "databaseName": "admin" + } + }, + { + "commandStartedEvent": { + "command": { + "abortTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "abortTransaction", + "databaseName": "admin" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ] + }, + { + "description": "abortTransaction is retried maxAttempts=5 times if backpressure labels are added", + "operations": [ + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "abortTransaction" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 112 + } + } + } + }, + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + }, + { + "object": "session0", + "name": "abortTransaction" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "abortTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "abortTransaction", + "databaseName": "admin" + } + }, + { + "commandStartedEvent": { + "commandName": "abortTransaction" + } + }, + { + "commandStartedEvent": { + "commandName": "abortTransaction" + } + }, + { + "commandStartedEvent": { + "commandName": "abortTransaction" + } + }, + { + "commandStartedEvent": { + "commandName": "abortTransaction" + } + }, + { + "commandStartedEvent": { + "commandName": "abortTransaction" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ] + } + ] +} diff --git a/test/spec/transactions/unified/backpressure-retryable-abort.yml b/test/spec/transactions/unified/backpressure-retryable-abort.yml new file mode 100644 index 00000000000..85532e1f605 --- /dev/null +++ b/test/spec/transactions/unified/backpressure-retryable-abort.yml @@ -0,0 +1,213 @@ +description: backpressure-retryable-abort +schemaVersion: "1.3" +runOnRequirements: + - minServerVersion: "4.4" + topologies: + - replicaset + - sharded + - load-balanced +createEntities: + - + client: + id: &client0 client0 + useMultipleMongoses: false + observeEvents: + - commandStartedEvent + - + database: + id: &database0 database0 + client: *client0 + databaseName: &database_name transaction-tests + - + collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection_name test + - + session: + id: &session0 session0 + client: *client0 + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: [] +tests: + - description: abortTransaction retries if backpressure labels are added + operations: + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: + times: 2 + data: + failCommands: + - abortTransaction + errorLabels: + - RetryableError + - SystemOverloadedError + errorCode: 112 + - object: *session0 + name: startTransaction + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 1 + expectResult: + $$unsetOrMatches: + insertedId: + $$unsetOrMatches: 1 + - object: *session0 + name: abortTransaction + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + insert: test + documents: + - _id: 1 + ordered: true + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: true + autocommit: false + writeConcern: + $$exists: false + commandName: insert + databaseName: *database_name + - commandStartedEvent: + command: + abortTransaction: 1 + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: abortTransaction + databaseName: admin + - commandStartedEvent: + command: + abortTransaction: 1 + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: abortTransaction + databaseName: admin + - commandStartedEvent: + command: + abortTransaction: 1 + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: abortTransaction + databaseName: admin + outcome: + - collectionName: *collection_name + databaseName: *database_name + documents: [] + - description: abortTransaction is retried maxAttempts=5 times if backpressure labels are added + operations: + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: + - abortTransaction + errorLabels: + - RetryableError + - SystemOverloadedError + errorCode: 112 + - object: *session0 + name: startTransaction + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 1 + expectResult: + $$unsetOrMatches: + insertedId: + $$unsetOrMatches: 1 + - object: *session0 + name: abortTransaction + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + insert: test + documents: + - _id: 1 + ordered: true + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: true + autocommit: false + writeConcern: + $$exists: false + commandName: insert + databaseName: *database_name + - commandStartedEvent: + command: + abortTransaction: 1 + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: abortTransaction + databaseName: admin + - commandStartedEvent: + commandName: abortTransaction + - commandStartedEvent: + commandName: abortTransaction + - commandStartedEvent: + commandName: abortTransaction + - commandStartedEvent: + commandName: abortTransaction + - commandStartedEvent: + commandName: abortTransaction + outcome: + - collectionName: *collection_name + databaseName: *database_name + documents: [] diff --git a/test/spec/transactions/unified/backpressure-retryable-commit.json b/test/spec/transactions/unified/backpressure-retryable-commit.json new file mode 100644 index 00000000000..ae873561a99 --- /dev/null +++ b/test/spec/transactions/unified/backpressure-retryable-commit.json @@ -0,0 +1,374 @@ +{ + "description": "backpressure-retryable-commit", + "schemaVersion": "1.4", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "sharded", + "replicaset", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "transaction-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + }, + { + "session": { + "id": "session0", + "client": "client0" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ], + "tests": [ + { + "description": "commitTransaction retries if backpressure labels are added", + "runOnRequirements": [ + { + "serverless": "forbid" + } + ], + "operations": [ + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 2 + }, + "data": { + "failCommands": [ + "commitTransaction" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 112 + } + } + } + }, + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + }, + { + "object": "session0", + "name": "commitTransaction" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1 + } + ] + } + ] + }, + { + "description": "commitTransaction is retried maxAttempts=5 times if backpressure labels are added", + "runOnRequirements": [ + { + "serverless": "forbid" + } + ], + "operations": [ + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "commitTransaction" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 112 + } + } + } + }, + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + }, + { + "object": "session0", + "name": "commitTransaction", + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + }, + { + "commandStartedEvent": { + "commandName": "commitTransaction" + } + }, + { + "commandStartedEvent": { + "commandName": "commitTransaction" + } + }, + { + "commandStartedEvent": { + "commandName": "commitTransaction" + } + }, + { + "commandStartedEvent": { + "commandName": "commitTransaction" + } + }, + { + "commandStartedEvent": { + "commandName": "commitTransaction" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ] + } + ] +} diff --git a/test/spec/transactions/unified/backpressure-retryable-commit.yml b/test/spec/transactions/unified/backpressure-retryable-commit.yml new file mode 100644 index 00000000000..8099e1c1eba --- /dev/null +++ b/test/spec/transactions/unified/backpressure-retryable-commit.yml @@ -0,0 +1,220 @@ +description: backpressure-retryable-commit +schemaVersion: "1.4" +runOnRequirements: + - minServerVersion: "4.4" + topologies: + - sharded + - replicaset + - load-balanced +createEntities: + - + client: + id: &client0 client0 + useMultipleMongoses: false + observeEvents: + - commandStartedEvent + - + database: + id: &database0 database0 + client: *client0 + databaseName: &database_name transaction-tests + - + collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection_name test + - + session: + id: &session0 session0 + client: *client0 + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: [] +tests: + - description: commitTransaction retries if backpressure labels are added + runOnRequirements: + - serverless: forbid + operations: + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: + times: 2 + data: + failCommands: + - commitTransaction + errorLabels: + - RetryableError + - SystemOverloadedError + errorCode: 112 + - object: *session0 + name: startTransaction + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 1 + expectResult: + $$unsetOrMatches: + insertedId: + $$unsetOrMatches: 1 + - object: *session0 + name: commitTransaction + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + insert: test + documents: + - _id: 1 + ordered: true + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: true + autocommit: false + writeConcern: + $$exists: false + commandName: insert + databaseName: *database_name + - commandStartedEvent: + command: + commitTransaction: 1 + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: commitTransaction + databaseName: admin + - commandStartedEvent: + command: + commitTransaction: 1 + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: commitTransaction + databaseName: admin + - commandStartedEvent: + command: + commitTransaction: 1 + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: commitTransaction + databaseName: admin + outcome: + - collectionName: *collection_name + databaseName: *database_name + documents: + - _id: 1 + - description: commitTransaction is retried maxAttempts=5 times if backpressure labels are added + runOnRequirements: + - serverless: forbid + operations: + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: + - commitTransaction + errorLabels: + - RetryableError + - SystemOverloadedError + errorCode: 112 + - object: *session0 + name: startTransaction + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 1 + expectResult: + $$unsetOrMatches: + insertedId: + $$unsetOrMatches: 1 + - object: *session0 + name: commitTransaction + expectError: + isError: true + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + insert: test + documents: + - _id: 1 + ordered: true + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: true + autocommit: false + writeConcern: + $$exists: false + commandName: insert + databaseName: *database_name + - commandStartedEvent: + command: + commitTransaction: 1 + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: commitTransaction + databaseName: admin + - commandStartedEvent: + commandName: commitTransaction + - commandStartedEvent: + commandName: commitTransaction + - commandStartedEvent: + commandName: commitTransaction + - commandStartedEvent: + commandName: commitTransaction + - commandStartedEvent: + commandName: commitTransaction + outcome: + - collectionName: *collection_name + databaseName: *database_name + documents: [] diff --git a/test/spec/transactions/unified/backpressure-retryable-reads.json b/test/spec/transactions/unified/backpressure-retryable-reads.json new file mode 100644 index 00000000000..731762830e3 --- /dev/null +++ b/test/spec/transactions/unified/backpressure-retryable-reads.json @@ -0,0 +1,328 @@ +{ + "description": "backpressure-retryable-reads", + "schemaVersion": "1.3", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "transaction-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + }, + { + "session": { + "id": "session0", + "client": "client0" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ], + "tests": [ + { + "description": "reads are retried if backpressure labels are added", + "operations": [ + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + }, + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 112 + } + } + } + }, + { + "object": "collection0", + "name": "find", + "arguments": { + "filter": {}, + "session": "session0" + } + }, + { + "object": "session0", + "name": "commitTransaction" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "test", + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "find", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "test", + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "find", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "abortTransaction": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + } + ] + } + ] + }, + { + "description": "reads are retried maxAttempts=5 times if backpressure labels are added", + "operations": [ + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + }, + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 112 + } + } + } + }, + { + "object": "collection0", + "name": "find", + "arguments": { + "filter": {}, + "session": "session0" + }, + "expectError": { + "isError": true + } + }, + { + "object": "session0", + "name": "abortTransaction" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "abortTransaction" + } + } + ] + } + ] + } + ] +} diff --git a/test/spec/transactions/unified/backpressure-retryable-reads.yml b/test/spec/transactions/unified/backpressure-retryable-reads.yml new file mode 100644 index 00000000000..18bbdaadbfa --- /dev/null +++ b/test/spec/transactions/unified/backpressure-retryable-reads.yml @@ -0,0 +1,192 @@ +description: backpressure-retryable-reads +schemaVersion: "1.3" +runOnRequirements: + - minServerVersion: "4.4" + topologies: + - replicaset + - sharded + - load-balanced +createEntities: + - + client: + id: &client0 client0 + useMultipleMongoses: false + observeEvents: + - commandStartedEvent + - + database: + id: &database0 database0 + client: *client0 + databaseName: &database_name transaction-tests + - + collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection_name test + - + session: + id: &session0 session0 + client: *client0 + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: [] +tests: + - description: reads are retried if backpressure labels are added + operations: + - object: *session0 + name: startTransaction + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 1 + expectResult: + $$unsetOrMatches: + insertedId: + $$unsetOrMatches: 1 + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: + times: 1 + data: + failCommands: + - find + errorLabels: + - RetryableError + - SystemOverloadedError + errorCode: 112 + - object: *collection0 + name: find + arguments: + filter: {} + session: *session0 + - object: *session0 + name: commitTransaction + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + insert: test + documents: + - _id: 1 + ordered: true + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: true + autocommit: false + writeConcern: + $$exists: false + commandName: insert + databaseName: *database_name + - commandStartedEvent: + command: + find: test + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + autocommit: false + writeConcern: + $$exists: false + commandName: find + databaseName: *database_name + - commandStartedEvent: + command: + find: test + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + autocommit: false + writeConcern: + $$exists: false + commandName: find + databaseName: *database_name + - commandStartedEvent: + command: + abortTransaction: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: commitTransaction + databaseName: admin + - description: reads are retried maxAttempts=5 times if backpressure labels are added + operations: + - object: *session0 + name: startTransaction + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 1 + expectResult: + $$unsetOrMatches: + insertedId: + $$unsetOrMatches: 1 + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: + - find + errorLabels: + - RetryableError + - SystemOverloadedError + errorCode: 112 + - object: *collection0 + name: find + arguments: + filter: {} + session: *session0 + expectError: + isError: true + - object: *session0 + name: abortTransaction + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandStartedEvent: + commandName: abortTransaction diff --git a/test/spec/transactions/unified/backpressure-retryable-writes.json b/test/spec/transactions/unified/backpressure-retryable-writes.json new file mode 100644 index 00000000000..0817e03f2f7 --- /dev/null +++ b/test/spec/transactions/unified/backpressure-retryable-writes.json @@ -0,0 +1,440 @@ +{ + "description": "backpressure-retryable-writes", + "schemaVersion": "1.3", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "transaction-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + }, + { + "session": { + "id": "session0", + "client": "client0" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ], + "tests": [ + { + "description": "writes are retried if backpressure labels are added", + "operations": [ + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + }, + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 112 + } + } + } + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 2 + } + } + }, + { + "object": "session0", + "name": "commitTransaction" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 2 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 2 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "abortTransaction": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ] + }, + { + "description": "writes are retried maxAttempts=5 times if backpressure labels are added", + "operations": [ + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + }, + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 112 + } + } + } + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 2 + } + }, + "expectError": { + "isError": true + } + }, + { + "object": "session0", + "name": "abortTransaction" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "abortTransaction" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ] + }, + { + "description": "retry succeeds if backpressure labels are added to the first operation in a transaction", + "operations": [ + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 112 + } + } + } + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 2 + } + } + }, + { + "object": "session0", + "name": "abortTransaction" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "abortTransaction" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ] + } + ] +} diff --git a/test/spec/transactions/unified/backpressure-retryable-writes.yml b/test/spec/transactions/unified/backpressure-retryable-writes.yml new file mode 100644 index 00000000000..630c9d9694b --- /dev/null +++ b/test/spec/transactions/unified/backpressure-retryable-writes.yml @@ -0,0 +1,250 @@ +description: backpressure-retryable-writes +schemaVersion: "1.3" +runOnRequirements: + - minServerVersion: "4.4" + topologies: + - replicaset + - sharded + - load-balanced +createEntities: + - + client: + id: &client0 client0 + useMultipleMongoses: false + observeEvents: + - commandStartedEvent + - + database: + id: &database0 database0 + client: *client0 + databaseName: &database_name transaction-tests + - + collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection_name test + - + session: + id: &session0 session0 + client: *client0 + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: [] +tests: + - description: writes are retried if backpressure labels are added + operations: + - object: *session0 + name: startTransaction + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 1 + expectResult: + $$unsetOrMatches: + insertedId: + $$unsetOrMatches: 1 + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: + times: 1 + data: + failCommands: + - insert + errorLabels: + - RetryableError + - SystemOverloadedError + errorCode: 112 + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 2 + - object: *session0 + name: commitTransaction + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + insert: test + documents: + - _id: 1 + ordered: true + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: true + autocommit: false + writeConcern: + $$exists: false + commandName: insert + databaseName: *database_name + - commandStartedEvent: + command: + insert: test + documents: + - _id: 2 + ordered: true + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + autocommit: false + writeConcern: + $$exists: false + commandName: insert + databaseName: *database_name + - commandStartedEvent: + command: + insert: test + documents: + - _id: 2 + ordered: true + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + autocommit: false + writeConcern: + $$exists: false + commandName: insert + databaseName: *database_name + - commandStartedEvent: + command: + abortTransaction: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: commitTransaction + databaseName: admin + outcome: + - collectionName: *collection_name + databaseName: *database_name + documents: + - { _id: 1 } + - { _id: 2 } + - description: writes are retried maxAttempts=5 times if backpressure labels are added + operations: + - object: *session0 + name: startTransaction + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 1 + expectResult: + $$unsetOrMatches: + insertedId: + $$unsetOrMatches: 1 + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: + - insert + errorLabels: + - RetryableError + - SystemOverloadedError + errorCode: 112 + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 2 + expectError: + isError: true + - object: *session0 + name: abortTransaction + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: abortTransaction + outcome: + - collectionName: *collection_name + databaseName: *database_name + documents: [] + - description: retry succeeds if backpressure labels are added to the first operation in a transaction + operations: + - object: *session0 + name: startTransaction + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: + times: 1 + data: + failCommands: + - insert + errorLabels: + - RetryableError + - SystemOverloadedError + errorCode: 112 + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 2 + - object: *session0 + name: abortTransaction + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: abortTransaction + outcome: + - collectionName: *collection_name + databaseName: *database_name + documents: [] diff --git a/test/unit/index.test.ts b/test/unit/index.test.ts index 8f32cc3698c..723198ca63d 100644 --- a/test/unit/index.test.ts +++ b/test/unit/index.test.ts @@ -132,6 +132,7 @@ const EXPECTED_EXPORTS = [ 'SeverityLevel', 'SrvPollingEvent', 'Timestamp', + 'TokenBucket', 'TopologyClosedEvent', 'TopologyDescriptionChangedEvent', 'TopologyOpeningEvent', From 4100da71575bc2c5c60538b0d7632260625af376 Mon Sep 17 00:00:00 2001 From: Sergey Zelenov Date: Thu, 19 Feb 2026 10:08:04 +0100 Subject: [PATCH 02/11] deposit retry tokens only on retries, update maxAttempts property conditionally --- package-lock.json | 17 +--------- src/cmap/connection.ts | 6 ---- src/operations/execute_operation.ts | 50 +++++++---------------------- src/operations/operation.ts | 10 +++--- src/sdam/topology.ts | 1 - src/sessions.ts | 12 ++++--- src/token_bucket.ts | 2 +- src/utils.ts | 20 ------------ 8 files changed, 26 insertions(+), 92 deletions(-) diff --git a/package-lock.json b/package-lock.json index b1def2b859d..286cc04d576 100644 --- a/package-lock.json +++ b/package-lock.json @@ -847,7 +847,6 @@ "integrity": "sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/generator": "^7.28.5", @@ -3026,8 +3025,7 @@ "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.3.20.tgz", "integrity": "sha512-/pC9HAB5I/xMlc5FP77qjCnI16ChlJfW0tGa0IUcFn38VJrTV6DeZ60NU5KZBtaOZqjdpwTWohz5HU1RrhiYxQ==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@types/chai-subset": { "version": "1.3.6", @@ -3129,7 +3127,6 @@ "integrity": "sha512-BICHQ67iqxQGFSzfCFTT7MRQ5XcBjG5aeKh5Ok38UBbPe5fxTyE+aHFxwVrGyr8GNlqFMLKD1D3P2K/1ks8tog==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~6.21.0" } @@ -3261,7 +3258,6 @@ "integrity": "sha512-sbaQ27XBUopBkRiuY/P9sWGOWUW4rl8fDoHIUmLpZd8uldsTyB4/Zg6bWTegPoTLnKj9Hqgn3QD6cjPNB32Odw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/regexpp": "^4.10.0", "@typescript-eslint/scope-manager": "8.46.3", @@ -3292,7 +3288,6 @@ "integrity": "sha512-6m1I5RmHBGTnUGS113G04DMu3CpSdxCAU/UvtjNWL4Nuf3MW9tQhiJqRlHzChIkhy6kZSAQmc+I1bcGjE3yNKg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.46.3", "@typescript-eslint/types": "8.46.3", @@ -3538,7 +3533,6 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -3599,7 +3593,6 @@ "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "json-schema-traverse": "^1.0.0", @@ -3915,7 +3908,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.8.19", "caniuse-lite": "^1.0.30001751", @@ -4093,7 +4085,6 @@ "integrity": "sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "assertion-error": "^1.1.0", "check-error": "^1.0.3", @@ -4700,7 +4691,6 @@ "integrity": "sha512-BhHmn2yNOFA9H9JmmIVKJmd288g9hrVRDkdoIgRCRuSySRUHH7r/DI6aAXW9T1WwUuY3DFgrcaqB+deURBLR5g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -4761,7 +4751,6 @@ "integrity": "sha512-82GZUjRS0p/jganf6q1rEO25VSoHH0hKPCTrgillPjdI/3bgBhAE1QzHrHTizjpRvy6pGAvKjDJtk2pF9NDq8w==", "dev": true, "license": "MIT", - "peer": true, "bin": { "eslint-config-prettier": "bin/cli.js" }, @@ -7336,7 +7325,6 @@ "integrity": "sha512-U42vQ4czpKa0QdI1hu950XuNhYqgoM+ZF1HT+VuUHL9hPfDPVvNQyltmMqdE9bUHMVa+8yNbc3QKTj8zQhlVxQ==", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "@istanbuljs/load-nyc-config": "^1.0.0", "@istanbuljs/schema": "^0.1.2", @@ -8021,7 +8009,6 @@ "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", "dev": true, "license": "MIT", - "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -8877,7 +8864,6 @@ "integrity": "sha512-a2N2TDY1uGviajJ6r4D1CyRAkzE9NNVlYOV1wX5xQDuAk0ONgzgRl0EjCQuRCPxOwp13ghsMwt9Gdldujs39qw==", "dev": true, "license": "BSD-3-Clause", - "peer": true, "dependencies": { "@sinonjs/commons": "^3.0.1", "@sinonjs/fake-timers": "11.2.2", @@ -9660,7 +9646,6 @@ "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" diff --git a/src/cmap/connection.ts b/src/cmap/connection.ts index 065e0dc42c4..0c099b8bd8c 100644 --- a/src/cmap/connection.ts +++ b/src/cmap/connection.ts @@ -586,12 +586,6 @@ export class Connection extends TypedEventEmitter { this.throwIfAborted(); } } catch (error) { - // Note for Sergey: when retrying a command with `startTransaction`, the spec says drivers must only mark the transaction - // as in progress _after_ the server responds, regardless of the command's outcome. - // Server errors are thrown from the try-block above _after_ updateSessionFromResponse is called. So, server errors already correctly update the - // session's state. - // Network errors, however, are thrown from the try-block from `sendWire()`, and as such bypass the call to `updateSessionFromResponse` above. So, we have to - // handle updating sessions for non-server errors here. if (options.session != null && !(error instanceof MongoServerError)) { updateSessionFromResponse(options.session, MongoDBResponse.empty); } diff --git a/src/operations/execute_operation.ts b/src/operations/execute_operation.ts index 27470b13025..1486cb2c1c0 100644 --- a/src/operations/execute_operation.ts +++ b/src/operations/execute_operation.ts @@ -12,7 +12,6 @@ import { MongoInvalidArgumentError, MongoNetworkError, MongoNotConnectedError, - MongoOperationTimeoutError, MongoRuntimeError, MongoServerError, MongoTransactionError, @@ -29,13 +28,8 @@ import { import type { Topology } from '../sdam/topology'; import type { ClientSession } from '../sessions'; import { TimeoutContext } from '../timeout'; -import { RETRY_COST, TOKEN_REFRESH_RATE } from '../token_bucket'; -import { - abortable, - ExponentialBackoffProvider, - maxWireVersion, - supportsRetryableWrites -} from '../utils'; +import { RETRY_COST, RETRY_TOKEN_RETURN_RATE } from '../token_bucket'; +import { abortable, maxWireVersion, supportsRetryableWrites } from '../utils'; import { AggregateOperation } from './aggregate'; import { AbstractOperation, Aspect } from './operation'; import { RunCommandOperation } from './run_command'; @@ -194,7 +188,6 @@ type RetryOptions = { * * @param operation - The operation to execute */ -// Note for Sergey: The rename here could be reverted - I just thought this was more descriptive. async function executeOperationWithRetries< T extends AbstractOperation, TResult = ResultTypeFromOperation @@ -248,11 +241,6 @@ async function executeOperationWithRetries< } const deprioritizedServers = new DeprioritizedServers(); - const backoffDelayProvider = new ExponentialBackoffProvider( - 10_000, // MAX_BACKOFF - 100, // base backoff - 2 // backoff rate - ); let maxAttempts = typeof operation.maxAttempts === 'number' @@ -271,6 +259,7 @@ async function executeOperationWithRetries< let error: MongoError | null = null; for (let attempt = 0; attempt < maxAttempts; attempt++) { + operation.attemptsMade = attempt + 1; operation.server = server; try { @@ -279,28 +268,20 @@ async function executeOperationWithRetries< topology.tokenBucket.deposit( attempt > 0 ? // on successful retry, deposit the retry cost + the refresh rate. - TOKEN_REFRESH_RATE + RETRY_COST + RETRY_TOKEN_RETURN_RATE + RETRY_COST : // otherwise, just deposit the refresh rate. - TOKEN_REFRESH_RATE + RETRY_TOKEN_RETURN_RATE ); return operation.handleOk(result); } catch (error) { return operation.handleError(error); } - - // Note for Sergey: This ended up being a larger refactor than I anticipated. But it made more sense to me to put the error handling - // that previously lived in the try-block into the catch-block. - // The primary motivator for this change was to make error tracking simpler. The post failure but pre-retry logic needs access to both the - // most recently encountered operation error _and_ the error we're storing to potentially throw, if we need to (because in some circumstances - // the error raised from here is _not_ the most recently encountered operation error). If we chose to keep the previously existing structure, - // this would force us to keep track of two errors outside of the for loop. By moving the pre-retry logic into the catch block, it gains - // access to `operationError`, and still only requires keeping track of one error outside the loop. } catch (operationError) { // Should never happen but if it does - propagate the error. if (!(operationError instanceof MongoError)) throw operationError; - if (!operationError.hasErrorLabel(MongoErrorLabel.SystemOverloadedError)) { - // if an operation fails with an error that does not contain the SystemOverloadError, deposit 1 token. + if (attempt > 0 && !operationError.hasErrorLabel(MongoErrorLabel.SystemOverloadedError)) { + // if a retry attempt fails with a non-overload error, deposit 1 token. topology.tokenBucket.deposit(RETRY_COST); } @@ -327,10 +308,9 @@ async function executeOperationWithRetries< throw error; } - maxAttempts = - shouldRetry && operationError.hasErrorLabel(MongoErrorLabel.SystemOverloadedError) - ? 6 - : maxAttempts; + if (operationError.hasErrorLabel(MongoErrorLabel.SystemOverloadedError)) { + maxAttempts = Math.min(6, operation.maxAttempts ?? 6); + } if (attempt + 1 >= maxAttempts) { throw error; @@ -341,17 +321,11 @@ async function executeOperationWithRetries< throw error; } - const delayMS = backoffDelayProvider.getNextBackoffDuration(); + const delayMS = Math.random() * Math.min(10_000, 100 * 2 ** attempt); // if the delay would exhaust the CSOT timeout, short-circuit. if (timeoutContext.csotEnabled() && delayMS > timeoutContext.remainingTimeMS) { - // TODO: is this the right error to throw? - throw new MongoOperationTimeoutError( - `MongoDB SystemOverload exponential backoff would exceed timeoutMS deadline: remaining CSOT deadline=${timeoutContext.remainingTimeMS}, backoff delayMS=${delayMS}`, - { - cause: error - } - ); + throw error; } await setTimeout(delayMS); diff --git a/src/operations/operation.ts b/src/operations/operation.ts index 573bb13039f..cb7da4f8001 100644 --- a/src/operations/operation.ts +++ b/src/operations/operation.ts @@ -66,14 +66,12 @@ export abstract class AbstractOperation { /** Specifies the time an operation will run until it throws a timeout error. */ timeoutMS?: number; - // Note for Sergey: sort of a hack, one that I planned to revisit before officially opening the PR for review. - // For every operation _except_ transactions, all backpressure retries occur inside one `executeOperation` call. `commitTransaction`, however, - // currently hard-codes two retry attempts, each with a `RunCommandOperation`. Neither retry attempt is retryable by itself. - // Because we use two separate RunCommand operations, it's possible to retry up to 2 * MAX_BACKPRESSURE_RETRIES times, if the server is overloaded. - // I hacked around it here by storing the max attempts for an operation on the operation, and sharing the value between both operations. - // I don't have a solid idea of a better way to do this right now though. + /** @internal Used by commitTransaction to share the retry budget across two executeOperatin calls. */ maxAttempts?: number; + /** @internal Tracks how many attempts were made in the last executeOperation call. */ + attemptsMade?: number; + private _session: ClientSession | undefined; static aspects?: Set; diff --git a/src/sdam/topology.ts b/src/sdam/topology.ts index 327d4d9a14c..16288261ebc 100644 --- a/src/sdam/topology.ts +++ b/src/sdam/topology.ts @@ -213,7 +213,6 @@ export class Topology extends TypedEventEmitter { hello?: Document; _type?: string; - // Note for Sergey: expect this to change, and instead be stored on the server class. tokenBucket = new TokenBucket(1000); client!: MongoClient; diff --git a/src/sessions.ts b/src/sessions.ts index 744fc8aafd0..41c1c7641ef 100644 --- a/src/sessions.ts +++ b/src/sessions.ts @@ -497,7 +497,7 @@ export class ClientSession readPreference: ReadPreference.primary, bypassPinningCheck: true }); - operation.maxAttempts = 5; + operation.maxAttempts = 6; const timeoutContext = this.timeoutContext ?? @@ -515,8 +515,12 @@ export class ClientSession return; } catch (firstCommitError) { this.commitAttempted = true; - // Backpressure exhausted all retries on the initial retry loop. - if (operation.maxAttempts === 5) throw firstCommitError; + + const remainingAttempts = 6 - (operation.attemptsMade ?? 1); + if (remainingAttempts <= 0) { + throw firstCommitError; + } + if (firstCommitError instanceof MongoError && isRetryableWriteError(firstCommitError)) { // SPEC-1185: apply majority write concern when retrying commitTransaction WriteConcern.apply(command, { wtimeoutMS: 10000, ...wc, w: 'majority' }); @@ -529,7 +533,7 @@ export class ClientSession readPreference: ReadPreference.primary, bypassPinningCheck: true }); - op.maxAttempts = operation.maxAttempts; + op.maxAttempts = remainingAttempts; await executeOperation(this.client, op, timeoutContext); return; } catch (retryCommitError) { diff --git a/src/token_bucket.ts b/src/token_bucket.ts index 76e88b660f5..6a6013ccbd4 100644 --- a/src/token_bucket.ts +++ b/src/token_bucket.ts @@ -26,7 +26,7 @@ export class TokenBucket { * @internal * The amount to deposit on successful operations, as defined in the backpressure specification. */ -export const TOKEN_REFRESH_RATE = 0.1; +export const RETRY_TOKEN_RETURN_RATE = 0.1; /** * @internal * The initial size of the token bucket, as defined in the backpressure specification. diff --git a/src/utils.ts b/src/utils.ts index 05e93702dcf..5aadbf4e624 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1419,23 +1419,3 @@ export async function abortable( abortListener?.[kDispose](); } } - -// Note for Sergey: overkill, I know - but sometimes you gotta over-engineer a bit for fun and revert before opening for review ¯\_(ツ)_/¯ -// feel free to remove this class. Or leave it. If you keep it, we should use this for the convenient transactions' API as well. -export class ExponentialBackoffProvider { - constructor( - public readonly maxBackoff: number, - public readonly baseBackoff: number, - public readonly backoffIncreaseRate: number, - public iteration = 0 - ) {} - - getNextBackoffDuration(): number { - const iteration = this.iteration; - this.iteration += 1; - return ( - Math.random() * - Math.min(this.maxBackoff, this.baseBackoff * this.backoffIncreaseRate ** iteration) - ); - } -} From 091e98977acf9d8179295e7fb3d94d1ec00d1f77 Mon Sep 17 00:00:00 2001 From: Sergey Zelenov Date: Thu, 19 Feb 2026 10:59:47 +0100 Subject: [PATCH 03/11] update to latest, small clean-ups --- src/index.ts | 1 - src/operations/execute_operation.ts | 5 ----- src/operations/operation.ts | 2 +- src/sdam/topology.ts | 4 ++-- src/sessions.ts | 4 ++-- sync.sh | 4 ++-- .../client-backpressure.prose.test.ts | 1 + .../mongodb-handshake.prose.test.ts | 2 ++ test/spec/client-backpressure/README.md | 13 +++++++++++++ 9 files changed, 23 insertions(+), 13 deletions(-) mode change 100644 => 100755 sync.sh diff --git a/src/index.ts b/src/index.ts index b5fc6ecd4aa..74803dfa2a0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -87,7 +87,6 @@ export { MongoWriteConcernError, WriteConcernErrorResult } from './error'; -export { TokenBucket } from './token_bucket'; export { AbstractCursor, // Actual driver classes exported diff --git a/src/operations/execute_operation.ts b/src/operations/execute_operation.ts index 1486cb2c1c0..865bfe58630 100644 --- a/src/operations/execute_operation.ts +++ b/src/operations/execute_operation.ts @@ -251,11 +251,6 @@ async function executeOperationWithRetries< : 2 : 1; - const shouldRetry = - (operation.hasAspect(Aspect.READ_OPERATION) && topology.s.options.retryReads) || - ((operation.hasAspect(Aspect.WRITE_OPERATION) || operation instanceof RunCommandOperation) && - topology.s.options.retryWrites); - let error: MongoError | null = null; for (let attempt = 0; attempt < maxAttempts; attempt++) { diff --git a/src/operations/operation.ts b/src/operations/operation.ts index cb7da4f8001..9bee2e480cc 100644 --- a/src/operations/operation.ts +++ b/src/operations/operation.ts @@ -66,7 +66,7 @@ export abstract class AbstractOperation { /** Specifies the time an operation will run until it throws a timeout error. */ timeoutMS?: number; - /** @internal Used by commitTransaction to share the retry budget across two executeOperatin calls. */ + /** @internal Used by commitTransaction to share the retry budget across two executeOperation calls. */ maxAttempts?: number; /** @internal Tracks how many attempts were made in the last executeOperation call. */ diff --git a/src/sdam/topology.ts b/src/sdam/topology.ts index 16288261ebc..2de0db757d4 100644 --- a/src/sdam/topology.ts +++ b/src/sdam/topology.ts @@ -35,7 +35,7 @@ import { type Abortable, TypedEventEmitter } from '../mongo_types'; import { ReadPreference, type ReadPreferenceLike } from '../read_preference'; import type { ClientSession } from '../sessions'; import { Timeout, TimeoutContext, TimeoutError } from '../timeout'; -import { TokenBucket } from '../token_bucket'; +import { INITIAL_TOKEN_BUCKET_SIZE, TokenBucket } from '../token_bucket'; import type { Transaction } from '../transactions'; import { addAbortListener, @@ -213,7 +213,7 @@ export class Topology extends TypedEventEmitter { hello?: Document; _type?: string; - tokenBucket = new TokenBucket(1000); + tokenBucket = new TokenBucket(INITIAL_TOKEN_BUCKET_SIZE); client!: MongoClient; diff --git a/src/sessions.ts b/src/sessions.ts index 41c1c7641ef..47b9ea28ffa 100644 --- a/src/sessions.ts +++ b/src/sessions.ts @@ -1254,9 +1254,9 @@ export function updateSessionFromResponse(session: ClientSession, document: Mong session.transaction.transition(TxnState.TRANSACTION_IN_PROGRESS); } else { const error = new MongoServerError(document.toObject()); - const isBackpressureError = error.hasErrorLabel(MongoErrorLabel.RetryableError); + const isRetryableError = error.hasErrorLabel(MongoErrorLabel.RetryableError); - if (!isBackpressureError) { + if (!isRetryableError) { session.transaction.transition(TxnState.TRANSACTION_IN_PROGRESS); } } diff --git a/sync.sh b/sync.sh old mode 100644 new mode 100755 index b99d9ed1086..b9b457b4830 --- a/sync.sh +++ b/sync.sh @@ -1,4 +1,4 @@ -cp ~/dev/specifications/source/client-backpressure/tests/* ~/dev/node-mongodb-native/test/spec/client-backpressure -cp ~/dev/specifications/source/transactions/tests/unified/backpressure* ~/dev/node-mongodb-native/test/spec/transactions/unified/ \ No newline at end of file +cp ~/code/specifications/source/client-backpressure/tests/* ~/code/node-mongodb-native/test/spec/client-backpressure +cp ~/code/specifications/source/transactions/tests/unified/backpressure* ~/code/node-mongodb-native/test/spec/transactions/unified/ \ No newline at end of file diff --git a/test/integration/client-backpressure/client-backpressure.prose.test.ts b/test/integration/client-backpressure/client-backpressure.prose.test.ts index 7c437c28ce1..fcd855fb34d 100644 --- a/test/integration/client-backpressure/client-backpressure.prose.test.ts +++ b/test/integration/client-backpressure/client-backpressure.prose.test.ts @@ -16,6 +16,7 @@ describe('Client Backpressure (Prose)', function () { }); afterEach(async function () { + sinon.restore(); await client.close(); await clearFailPoint(this.configuration); }); diff --git a/test/integration/mongodb-handshake/mongodb-handshake.prose.test.ts b/test/integration/mongodb-handshake/mongodb-handshake.prose.test.ts index 6656e39f029..c73ef9b73a4 100644 --- a/test/integration/mongodb-handshake/mongodb-handshake.prose.test.ts +++ b/test/integration/mongodb-handshake/mongodb-handshake.prose.test.ts @@ -5,8 +5,10 @@ import * as sinon from 'sinon'; import { type ClientMetadata, Connection, + type Document, type DriverInfo, getFAASEnv, + type HandshakeDocument, Int32, isDriverInfoEqual, LEGACY_HELLO_COMMAND, diff --git a/test/spec/client-backpressure/README.md b/test/spec/client-backpressure/README.md index e9048e1b4a3..22efe29b207 100644 --- a/test/spec/client-backpressure/README.md +++ b/test/spec/client-backpressure/README.md @@ -50,8 +50,21 @@ Drivers should test that retries do not occur immediately when a SystemOverloade 5. Execute step 3 again. 6. Compare the two time between the two runs. + ```python assertTrue(with_backoff_time - no_backoff_time >= 2.1) ``` + The sum of 5 backoffs is 3.1 seconds. There is a 1-second window to account for potential variance between the two runs. + +#### Test 2: Token Bucket Capacity is Enforced + +Drivers should test that retry token buckets are created at their maximum capacity and that that capacity is enforced. + +1. Let `client` be a `MongoClient`. +2. Assert that the client's retry token bucket is at full capacity and that the capacity is + `DEFAULT_RETRY_TOKEN_CAPACITY`. +3. Using `client`, execute a successful `ping` command. +4. Assert that the successful command did not increase the number of tokens in the bucket above + `DEFAULT_RETRY_TOKEN_CAPACITY`. From 680ee083b0c73c4c488f08e70abddfef8c3f7748 Mon Sep 17 00:00:00 2001 From: Sergey Zelenov Date: Thu, 19 Feb 2026 11:47:15 +0100 Subject: [PATCH 04/11] add timeoutContext to retry selectServer --- src/operations/execute_operation.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/operations/execute_operation.ts b/src/operations/execute_operation.ts index 865bfe58630..0529963eb84 100644 --- a/src/operations/execute_operation.ts +++ b/src/operations/execute_operation.ts @@ -342,7 +342,8 @@ async function executeOperationWithRetries< session, operationName: operation.commandName, deprioritizedServers, - signal: operation.options.signal + signal: operation.options.signal, + timeoutContext }); if ( From 0c36556f2da7607e29a6d68421fe3171d9afd97e Mon Sep 17 00:00:00 2001 From: Sergey Zelenov Date: Thu, 19 Feb 2026 11:50:48 +0100 Subject: [PATCH 05/11] revert non-compliant spec change --- src/operations/execute_operation.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/operations/execute_operation.ts b/src/operations/execute_operation.ts index 0529963eb84..865bfe58630 100644 --- a/src/operations/execute_operation.ts +++ b/src/operations/execute_operation.ts @@ -342,8 +342,7 @@ async function executeOperationWithRetries< session, operationName: operation.commandName, deprioritizedServers, - signal: operation.options.signal, - timeoutContext + signal: operation.options.signal }); if ( From a4caf387b8e6d4df68309e16d65597686fabd28f Mon Sep 17 00:00:00 2001 From: Sergey Zelenov Date: Thu, 19 Feb 2026 12:59:17 +0100 Subject: [PATCH 06/11] always cleanup local timeout (also for sharded clusters) --- src/sdam/topology.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/sdam/topology.ts b/src/sdam/topology.ts index 2de0db757d4..e0804eac4d8 100644 --- a/src/sdam/topology.ts +++ b/src/sdam/topology.ts @@ -593,7 +593,11 @@ export class Topology extends TypedEventEmitter { ) ); } - if (options.timeoutContext?.clearServerSelectionTimeout) timeout?.clear(); + + if (!options.timeoutContext || options.timeoutContext?.clearServerSelectionTimeout) { + timeout?.clear(); + } + return transaction.server; } From 89eafde282699db4e68c363cacb120022079d235 Mon Sep 17 00:00:00 2001 From: Sergey Zelenov Date: Thu, 19 Feb 2026 13:03:27 +0100 Subject: [PATCH 07/11] do not leak the timeout also in normal path finally block --- src/sdam/topology.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/sdam/topology.ts b/src/sdam/topology.ts index e0804eac4d8..fb86423929d 100644 --- a/src/sdam/topology.ts +++ b/src/sdam/topology.ts @@ -594,7 +594,7 @@ export class Topology extends TypedEventEmitter { ); } - if (!options.timeoutContext || options.timeoutContext?.clearServerSelectionTimeout) { + if (!options.timeoutContext || options.timeoutContext.clearServerSelectionTimeout) { timeout?.clear(); } @@ -668,7 +668,9 @@ export class Topology extends TypedEventEmitter { throw error; } finally { abortListener?.[kDispose](); - if (options.timeoutContext?.clearServerSelectionTimeout) timeout?.clear(); + if (!options.timeoutContext || options.timeoutContext.clearServerSelectionTimeout) { + timeout?.clear(); + } } } /** From 50d5c2cf13ec88812ff0ac4efb058e382385ad92 Mon Sep 17 00:00:00 2001 From: Sergey Zelenov Date: Thu, 19 Feb 2026 13:25:27 +0100 Subject: [PATCH 08/11] run all tests --- src/operations/execute_operation.ts | 6 ++---- sync.sh | 4 ---- test/integration/transactions/transactions.spec.test.ts | 2 +- test/unit/index.test.ts | 1 - 4 files changed, 3 insertions(+), 10 deletions(-) delete mode 100755 sync.sh diff --git a/src/operations/execute_operation.ts b/src/operations/execute_operation.ts index 865bfe58630..83aaf35cd40 100644 --- a/src/operations/execute_operation.ts +++ b/src/operations/execute_operation.ts @@ -262,10 +262,8 @@ async function executeOperationWithRetries< const result = await server.command(operation, timeoutContext); topology.tokenBucket.deposit( attempt > 0 - ? // on successful retry, deposit the retry cost + the refresh rate. - RETRY_TOKEN_RETURN_RATE + RETRY_COST - : // otherwise, just deposit the refresh rate. - RETRY_TOKEN_RETURN_RATE + ? RETRY_TOKEN_RETURN_RATE + RETRY_COST // on successful retry + : RETRY_TOKEN_RETURN_RATE // otherwise ); return operation.handleOk(result); } catch (error) { diff --git a/sync.sh b/sync.sh deleted file mode 100755 index b9b457b4830..00000000000 --- a/sync.sh +++ /dev/null @@ -1,4 +0,0 @@ - - -cp ~/code/specifications/source/client-backpressure/tests/* ~/code/node-mongodb-native/test/spec/client-backpressure -cp ~/code/specifications/source/transactions/tests/unified/backpressure* ~/code/node-mongodb-native/test/spec/transactions/unified/ \ No newline at end of file diff --git a/test/integration/transactions/transactions.spec.test.ts b/test/integration/transactions/transactions.spec.test.ts index b24f1a667cc..ab6f84b1e94 100644 --- a/test/integration/transactions/transactions.spec.test.ts +++ b/test/integration/transactions/transactions.spec.test.ts @@ -13,7 +13,7 @@ const SKIPPED_TESTS = [ 'client bulkWrite with writeConcern in a transaction causes a transaction error' ]; -describe.only('Transactions Spec Unified Tests', function () { +describe('Transactions Spec Unified Tests', function () { runUnifiedSuite(loadSpecTests(path.join('transactions', 'unified')), test => { return SKIPPED_TESTS.includes(test.description) ? 'TODO(NODE-5924): Skipping failing transaction tests' diff --git a/test/unit/index.test.ts b/test/unit/index.test.ts index 723198ca63d..8f32cc3698c 100644 --- a/test/unit/index.test.ts +++ b/test/unit/index.test.ts @@ -132,7 +132,6 @@ const EXPECTED_EXPORTS = [ 'SeverityLevel', 'SrvPollingEvent', 'Timestamp', - 'TokenBucket', 'TopologyClosedEvent', 'TopologyDescriptionChangedEvent', 'TopologyOpeningEvent', From 85f10c304c4d29bb2d81c6c14b55f010a29a0272 Mon Sep 17 00:00:00 2001 From: Sergey Zelenov Date: Thu, 19 Feb 2026 14:52:58 +0100 Subject: [PATCH 09/11] define constants --- src/operations/execute_operation.ts | 18 ++++++++++++------ src/sdam/topology.ts | 1 + src/sessions.ts | 5 +++-- src/token_bucket.ts | 17 ++++++++++++++++- 4 files changed, 32 insertions(+), 9 deletions(-) diff --git a/src/operations/execute_operation.ts b/src/operations/execute_operation.ts index 83aaf35cd40..26debb92728 100644 --- a/src/operations/execute_operation.ts +++ b/src/operations/execute_operation.ts @@ -28,7 +28,13 @@ import { import type { Topology } from '../sdam/topology'; import type { ClientSession } from '../sessions'; import { TimeoutContext } from '../timeout'; -import { RETRY_COST, RETRY_TOKEN_RETURN_RATE } from '../token_bucket'; +import { + BASE_BACKOFF_MS, + MAX_BACKOFF_MS, + MAX_RETRIES, + RETRY_COST, + RETRY_TOKEN_RETURN_RATE +} from '../token_bucket'; import { abortable, maxWireVersion, supportsRetryableWrites } from '../utils'; import { AggregateOperation } from './aggregate'; import { AbstractOperation, Aspect } from './operation'; @@ -302,7 +308,7 @@ async function executeOperationWithRetries< } if (operationError.hasErrorLabel(MongoErrorLabel.SystemOverloadedError)) { - maxAttempts = Math.min(6, operation.maxAttempts ?? 6); + maxAttempts = Math.min(MAX_RETRIES + 1, operation.maxAttempts ?? MAX_RETRIES + 1); } if (attempt + 1 >= maxAttempts) { @@ -314,14 +320,14 @@ async function executeOperationWithRetries< throw error; } - const delayMS = Math.random() * Math.min(10_000, 100 * 2 ** attempt); + const backoffMS = Math.random() * Math.min(MAX_BACKOFF_MS, BASE_BACKOFF_MS * 2 ** attempt); - // if the delay would exhaust the CSOT timeout, short-circuit. - if (timeoutContext.csotEnabled() && delayMS > timeoutContext.remainingTimeMS) { + // if the backoff would exhaust the CSOT timeout, short-circuit. + if (timeoutContext.csotEnabled() && backoffMS > timeoutContext.remainingTimeMS) { throw error; } - await setTimeout(delayMS); + await setTimeout(backoffMS); } if ( diff --git a/src/sdam/topology.ts b/src/sdam/topology.ts index fb86423929d..7e2bfc3f1f6 100644 --- a/src/sdam/topology.ts +++ b/src/sdam/topology.ts @@ -213,6 +213,7 @@ export class Topology extends TypedEventEmitter { hello?: Document; _type?: string; + /** @internal */ tokenBucket = new TokenBucket(INITIAL_TOKEN_BUCKET_SIZE); client!: MongoClient; diff --git a/src/sessions.ts b/src/sessions.ts index 47b9ea28ffa..5238ceebe9a 100644 --- a/src/sessions.ts +++ b/src/sessions.ts @@ -30,6 +30,7 @@ import { ReadConcernLevel } from './read_concern'; import { ReadPreference } from './read_preference'; import { _advanceClusterTime, type ClusterTime, TopologyType } from './sdam/common'; import { TimeoutContext } from './timeout'; +import { MAX_RETRIES } from './token_bucket'; import { isTransactionCommand, Transaction, @@ -497,7 +498,7 @@ export class ClientSession readPreference: ReadPreference.primary, bypassPinningCheck: true }); - operation.maxAttempts = 6; + operation.maxAttempts = MAX_RETRIES + 1; const timeoutContext = this.timeoutContext ?? @@ -516,7 +517,7 @@ export class ClientSession } catch (firstCommitError) { this.commitAttempted = true; - const remainingAttempts = 6 - (operation.attemptsMade ?? 1); + const remainingAttempts = MAX_RETRIES + 1 - (operation.attemptsMade ?? 1); if (remainingAttempts <= 0) { throw firstCommitError; } diff --git a/src/token_bucket.ts b/src/token_bucket.ts index 6a6013ccbd4..434b847cc6f 100644 --- a/src/token_bucket.ts +++ b/src/token_bucket.ts @@ -31,9 +31,24 @@ export const RETRY_TOKEN_RETURN_RATE = 0.1; * @internal * The initial size of the token bucket, as defined in the backpressure specification. */ -export const INITIAL_TOKEN_BUCKET_SIZE = 1000; +export const INITIAL_TOKEN_BUCKET_SIZE = 1_000; /** * @internal * The cost of a retry, as defined in the backpressure specification. */ export const RETRY_COST = 1; +/** + * @internal + * The maximum number of retries for overload errors + * */ +export const MAX_RETRIES = 5; +/** + * @internal + * The base backoff duration in milliseconds + * */ +export const BASE_BACKOFF_MS = 100; +/** + * @internal + * The maximum backoff duration in milliseconds + * */ +export const MAX_BACKOFF_MS = 10_000; From b048983b2ae3b02bed17153cd3a0abfb5644a6ed Mon Sep 17 00:00:00 2001 From: Sergey Zelenov Date: Thu, 19 Feb 2026 15:05:50 +0100 Subject: [PATCH 10/11] export TokenBucket --- src/index.ts | 1 + test/unit/index.test.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/src/index.ts b/src/index.ts index 74803dfa2a0..b5fc6ecd4aa 100644 --- a/src/index.ts +++ b/src/index.ts @@ -87,6 +87,7 @@ export { MongoWriteConcernError, WriteConcernErrorResult } from './error'; +export { TokenBucket } from './token_bucket'; export { AbstractCursor, // Actual driver classes exported diff --git a/test/unit/index.test.ts b/test/unit/index.test.ts index 8f32cc3698c..723198ca63d 100644 --- a/test/unit/index.test.ts +++ b/test/unit/index.test.ts @@ -132,6 +132,7 @@ const EXPECTED_EXPORTS = [ 'SeverityLevel', 'SrvPollingEvent', 'Timestamp', + 'TokenBucket', 'TopologyClosedEvent', 'TopologyDescriptionChangedEvent', 'TopologyOpeningEvent', From adcdc4fcffd526bb33e0d814859a06afe7d49c0e Mon Sep 17 00:00:00 2001 From: Sergey Zelenov Date: Thu, 19 Feb 2026 20:20:11 +0100 Subject: [PATCH 11/11] retry batching commands only on retryable errors --- src/operations/execute_operation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/operations/execute_operation.ts b/src/operations/execute_operation.ts index 26debb92728..120960a38f8 100644 --- a/src/operations/execute_operation.ts +++ b/src/operations/execute_operation.ts @@ -390,7 +390,7 @@ async function executeOperationWithRetries< // batch operations are only retryable if the batch is retryable if (operation.hasAspect(Aspect.COMMAND_BATCHING)) { - return operation.canRetryWrite; + return operation.canRetryWrite && isRetryableWriteError(error); } return (