Fix flaky IndexSetupTests on RavenDB

src/ServiceControl.AcceptanceTests.RavenDB/AcceptanceTestStorageConfiguration.cs

@@ -8,6 +8,7 @@
 using ServiceControl.AcceptanceTests.TestSupport;
 using ServiceControl.Persistence.Tests;
 using ServiceControl.RavenDB;
+using TestHelper;
 
 public class AcceptanceTestStorageConfiguration : IAcceptanceTestStorageConfiguration
 {

src/ServiceControl.Audit.AcceptanceTests.RavenDB/AcceptanceTestStorageConfiguration.cs

@@ -8,6 +8,7 @@
     using ServiceControl.Audit.Persistence.RavenDB;
     using ServiceControl.Audit.Persistence.Tests;
     using ServiceControl.RavenDB;
+    using TestHelper;
 
     public class AcceptanceTestStorageConfiguration
     {

src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs

@@ -7,6 +7,7 @@ namespace ServiceControl.Audit.Persistence.Tests;
 using Persistence.RavenDB.Indexes;
 using Raven.Client.Documents.Indexes;
 using Raven.Client.Documents.Operations.Indexes;
+using Raven.Client.Exceptions;
 using Raven.Client.Exceptions.Documents.Indexes;
 
 [TestFixture]
@@ -109,8 +110,16 @@ async Task<IndexStats> UpdateIndex(IAbstractIndexCreationTask index)
         return await WaitForIndexDefinitionUpdate(statsBefore);
     }
 
+    // How many consecutive RavenExceptions from the stats query below get tolerated before letting one propagate for real.
+    // RavenDB can throw a variety of transient errors for that race (seen so far: OperationCanceledException
+    // from the read transaction being cancelled, and ObjectDisposedException from the old engine's index persistence being torn down).
+    // A genuinely broken index should still fail the test.
+    const int MaxTransientRavenExceptionRetries = 2;
+
     async Task<IndexStats> WaitForIndexDefinitionUpdate(IndexStats oldStats)
     {
+        var transientFailures = 0;
+
         while (true)
         {
             try
@@ -127,6 +136,10 @@ async Task<IndexStats> WaitForIndexDefinitionUpdate(IndexStats oldStats)
             {
                 // keep going since we can get this if we query right when the update happens
             }
+            catch (RavenException) when (transientFailures < MaxTransientRavenExceptionRetries)
+            {
+                transientFailures++;
+            }
 #pragma warning restore PS0020
 
             await Task.Delay(TimeSpan.FromMilliseconds(100), TestTimeoutCancellationToken);

src/ServiceControl.Persistence.Tests.RavenDB/PersistenceTestsContext.cs

@@ -15,6 +15,7 @@ namespace ServiceControl.Persistence.Tests;
 using ServiceControl.Persistence;
 using ServiceControl.Persistence.RavenDB;
 using ServiceControl.RavenDB;
+using TestHelper;
 
 public class PersistenceTestsContext : IPersistenceTestsContext
 {

src/ServiceControl.RavenDB/EmbeddedDatabase.cs

@@ -13,7 +13,6 @@ namespace ServiceControl.RavenDB
     using Microsoft.Extensions.Logging;
     using Raven.Client.Documents;
     using Raven.Client.Documents.Conventions;
-    using Raven.Client.ServerWide.Operations;
     using Raven.Embedded;
     using ServiceControl.Infrastructure;
     using Sparrow.Logging;
@@ -180,15 +179,6 @@ public async Task<IDocumentStore> Connect(CancellationToken cancellationToken =
             return store;
         }
 
-        public async Task DeleteDatabase(string dbName, CancellationToken cancellationToken = default)
-        {
-            using var store = await EmbeddedServer.Instance.GetDocumentStoreAsync(new DatabaseOptions(dbName)
-            {
-                SkipCreatingDatabase = true
-            }, cancellationToken);
-            await store.Maintenance.Server.SendAsync(new DeleteDatabasesOperation(dbName, true), cancellationToken);
-        }
-
         public async Task Stop(CancellationToken cancellationToken = default)
         {
             logger.LogDebug("Stopping RavenDB server");

src/TestHelper/EmbeddedDatabaseExtensions.cs

@@ -0,0 +1,28 @@
+namespace TestHelper;
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Raven.Client.ServerWide.Operations;
+using Raven.Embedded;
+using ServiceControl.RavenDB;
+
+public static class EmbeddedDatabaseExtensions
+{
+    public static async Task DeleteDatabase(this EmbeddedDatabase database, string dbName, CancellationToken cancellationToken = default)
+    {
+        using var store = await EmbeddedServer.Instance.GetDocumentStoreAsync(
+            new DatabaseOptions(dbName) { SkipCreatingDatabase = true },
+            cancellationToken);
+
+        // RavenDB's default confirmation wait (15s) is for its Raft commit log to catch up
+        // with this delete command's index. Under the volume this shared embedded server sees
+        // in a full test run (every test creates and deletes its own database against the same
+        // instance), that commit can occasionally land just past the default window even though
+        // the delete itself isn't failing - so give it more headroom explicitly rather than
+        // relying on RavenDB's default.
+        await store.Maintenance.Server.SendAsync(
+            new DeleteDatabasesOperation(dbName, hardDelete: true, timeToWaitForConfirmation: TimeSpan.FromSeconds(30)),
+            cancellationToken);
+    }
+}

src/TestHelper/TestHelper.csproj

@@ -7,6 +7,11 @@
   <ItemGroup>
     <PackageReference Include="NUnit" />
     <PackageReference Include="NUnit.Analyzers" />
+    <PackageReference Include="RavenDB.Embedded" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\ServiceControl.RavenDB\ServiceControl.RavenDB.csproj" />
   </ItemGroup>
 
   <!-- Workaround to prevent VS test discovery error -->