diff --git a/src/ServiceControl.AcceptanceTests.RavenDB/AcceptanceTestStorageConfiguration.cs b/src/ServiceControl.AcceptanceTests.RavenDB/AcceptanceTestStorageConfiguration.cs index d0de36ab23..29ef393e10 100644 --- a/src/ServiceControl.AcceptanceTests.RavenDB/AcceptanceTestStorageConfiguration.cs +++ b/src/ServiceControl.AcceptanceTests.RavenDB/AcceptanceTestStorageConfiguration.cs @@ -8,6 +8,7 @@ using ServiceControl.AcceptanceTests.TestSupport; using ServiceControl.Persistence.Tests; using ServiceControl.RavenDB; +using TestHelper; public class AcceptanceTestStorageConfiguration : IAcceptanceTestStorageConfiguration { diff --git a/src/ServiceControl.Audit.AcceptanceTests.RavenDB/AcceptanceTestStorageConfiguration.cs b/src/ServiceControl.Audit.AcceptanceTests.RavenDB/AcceptanceTestStorageConfiguration.cs index f1aee9f300..734ec5ba45 100644 --- a/src/ServiceControl.Audit.AcceptanceTests.RavenDB/AcceptanceTestStorageConfiguration.cs +++ b/src/ServiceControl.Audit.AcceptanceTests.RavenDB/AcceptanceTestStorageConfiguration.cs @@ -8,6 +8,7 @@ using ServiceControl.Audit.Persistence.RavenDB; using ServiceControl.Audit.Persistence.Tests; using ServiceControl.RavenDB; + using TestHelper; public class AcceptanceTestStorageConfiguration { diff --git a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs index 9ec30788d6..ee655f7fc2 100644 --- a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs +++ b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs @@ -7,6 +7,7 @@ namespace ServiceControl.Audit.Persistence.Tests; using Persistence.RavenDB.Indexes; using Raven.Client.Documents.Indexes; using Raven.Client.Documents.Operations.Indexes; +using Raven.Client.Exceptions; using Raven.Client.Exceptions.Documents.Indexes; [TestFixture] @@ -109,8 +110,16 @@ async Task UpdateIndex(IAbstractIndexCreationTask index) return await WaitForIndexDefinitionUpdate(statsBefore); } + // How many consecutive RavenExceptions from the stats query below get tolerated before letting one propagate for real. + // RavenDB can throw a variety of transient errors for that race (seen so far: OperationCanceledException + // from the read transaction being cancelled, and ObjectDisposedException from the old engine's index persistence being torn down). + // A genuinely broken index should still fail the test. + const int MaxTransientRavenExceptionRetries = 2; + async Task WaitForIndexDefinitionUpdate(IndexStats oldStats) { + var transientFailures = 0; + while (true) { try @@ -127,6 +136,10 @@ async Task WaitForIndexDefinitionUpdate(IndexStats oldStats) { // keep going since we can get this if we query right when the update happens } + catch (RavenException) when (transientFailures < MaxTransientRavenExceptionRetries) + { + transientFailures++; + } #pragma warning restore PS0020 await Task.Delay(TimeSpan.FromMilliseconds(100), TestTimeoutCancellationToken); diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/PersistenceTestsContext.cs b/src/ServiceControl.Persistence.Tests.RavenDB/PersistenceTestsContext.cs index 342dbc8bce..13ae79e717 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/PersistenceTestsContext.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/PersistenceTestsContext.cs @@ -15,6 +15,7 @@ namespace ServiceControl.Persistence.Tests; using ServiceControl.Persistence; using ServiceControl.Persistence.RavenDB; using ServiceControl.RavenDB; +using TestHelper; public class PersistenceTestsContext : IPersistenceTestsContext { diff --git a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs index a2079816eb..8bdc0245ab 100644 --- a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs +++ b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs @@ -13,7 +13,6 @@ namespace ServiceControl.RavenDB using Microsoft.Extensions.Logging; using Raven.Client.Documents; using Raven.Client.Documents.Conventions; - using Raven.Client.ServerWide.Operations; using Raven.Embedded; using ServiceControl.Infrastructure; using Sparrow.Logging; @@ -180,15 +179,6 @@ public async Task Connect(CancellationToken cancellationToken = return store; } - public async Task DeleteDatabase(string dbName, CancellationToken cancellationToken = default) - { - using var store = await EmbeddedServer.Instance.GetDocumentStoreAsync(new DatabaseOptions(dbName) - { - SkipCreatingDatabase = true - }, cancellationToken); - await store.Maintenance.Server.SendAsync(new DeleteDatabasesOperation(dbName, true), cancellationToken); - } - public async Task Stop(CancellationToken cancellationToken = default) { logger.LogDebug("Stopping RavenDB server"); diff --git a/src/TestHelper/EmbeddedDatabaseExtensions.cs b/src/TestHelper/EmbeddedDatabaseExtensions.cs new file mode 100644 index 0000000000..92cf85fb0f --- /dev/null +++ b/src/TestHelper/EmbeddedDatabaseExtensions.cs @@ -0,0 +1,28 @@ +namespace TestHelper; + +using System; +using System.Threading; +using System.Threading.Tasks; +using Raven.Client.ServerWide.Operations; +using Raven.Embedded; +using ServiceControl.RavenDB; + +public static class EmbeddedDatabaseExtensions +{ + public static async Task DeleteDatabase(this EmbeddedDatabase database, string dbName, CancellationToken cancellationToken = default) + { + using var store = await EmbeddedServer.Instance.GetDocumentStoreAsync( + new DatabaseOptions(dbName) { SkipCreatingDatabase = true }, + cancellationToken); + + // RavenDB's default confirmation wait (15s) is for its Raft commit log to catch up + // with this delete command's index. Under the volume this shared embedded server sees + // in a full test run (every test creates and deletes its own database against the same + // instance), that commit can occasionally land just past the default window even though + // the delete itself isn't failing - so give it more headroom explicitly rather than + // relying on RavenDB's default. + await store.Maintenance.Server.SendAsync( + new DeleteDatabasesOperation(dbName, hardDelete: true, timeToWaitForConfirmation: TimeSpan.FromSeconds(30)), + cancellationToken); + } +} diff --git a/src/TestHelper/TestHelper.csproj b/src/TestHelper/TestHelper.csproj index ed363a805f..b6077f151d 100644 --- a/src/TestHelper/TestHelper.csproj +++ b/src/TestHelper/TestHelper.csproj @@ -7,6 +7,11 @@ + + + + +