diff --git a/sei-db/ledger_db/block/block_db_test.go b/sei-db/ledger_db/block/block_db_test.go index 1c07f8e703..21b50d20a5 100644 --- a/sei-db/ledger_db/block/block_db_test.go +++ b/sei-db/ledger_db/block/block_db_test.go @@ -53,8 +53,9 @@ func TestBlockDB(t *testing.T) { t.Run("EmptyDB", func(t *testing.T) { testEmptyDB(t, impl.build) }) t.Run("ReadRoundTrip", func(t *testing.T) { testReadRoundTrip(t, impl.build) }) t.Run("QCByBlockNumber", func(t *testing.T) { testQCByBlockNumber(t, impl.build) }) - t.Run("Iterators", func(t *testing.T) { testIterators(t, impl.build) }) - t.Run("IteratorSnapshot", func(t *testing.T) { testIteratorSnapshot(t, impl.build) }) + t.Run("AppProposalByBlockNumber", func(t *testing.T) { testAppProposalByBlockNumber(t, impl.build) }) + t.Run("AppQCByBlockNumber", func(t *testing.T) { testAppQCByBlockNumber(t, impl.build) }) + t.Run("ReadSuffix", func(t *testing.T) { testReadSuffix(t, impl.build) }) t.Run("RestartPersistsData", func(t *testing.T) { testRestartPersistsData(t, impl.build) }) t.Run("PruneRetainsAtOrAbove", func(t *testing.T) { testPruneRetainsAtOrAbove(t, impl.build) }) t.Run("PruneStraddleRetainsQC", func(t *testing.T) { testPruneStraddleRetainsQC(t, impl.build) }) @@ -78,18 +79,14 @@ func TestBlockDB(t *testing.T) { t.Run("WriteQCCoversNoBlocksRejected", func(t *testing.T) { testWriteQCCoversNoBlocksRejected(t, impl.build) }) - t.Run("IteratorBlockRequiresPosition", func(t *testing.T) { - testIteratorBlockRequiresPosition(t, impl.build) + t.Run("WriteAppDataOrderRejected", func(t *testing.T) { + testWriteAppDataOrderRejected(t, impl.build) + }) + t.Run("PruneWithAppQCNeverEmpties", func(t *testing.T) { + testPruneWithAppQCNeverEmpties(t, impl.build) }) t.Run("WriteBlockRequiresQC", func(t *testing.T) { testWriteBlockRequiresQC(t, impl.build) }) t.Run("ResumeAfterRestart", func(t *testing.T) { testResumeAfterRestart(t, impl.build) }) - t.Run("IteratorPositioning", func(t *testing.T) { testIteratorPositioning(t, impl.build) }) - t.Run("IteratorTail", func(t *testing.T) { testIteratorTail(t, impl.build) }) - t.Run("IteratorClampsUpToCoverage", func(t *testing.T) { - testIteratorClampsUpToCoverage(t, impl.build) - }) - t.Run("FirstBlockMidQC", func(t *testing.T) { testFirstBlockMidQC(t, impl.build) }) - t.Run("QCOnlyStoreIterates", func(t *testing.T) { testQCOnlyStoreIterates(t, impl.build) }) }) } } @@ -113,6 +110,11 @@ func restart(t *testing.T, o open, db types.BlockDB) types.BlockDB { return reopened } +func status(t *testing.T, db types.BlockDB) types.SuffixRange { + t.Helper() + return db.Status().OrPanic("non-empty BlockDB status") +} + func testEmptyDB(t *testing.T, build builder) { db, _ := openFresh(t, build) defer func() { _ = db.Close() }() @@ -129,15 +131,17 @@ func testEmptyDB(t *testing.T, build builder) { require.NoError(t, err) require.False(t, qc.IsPresent()) - require.Empty(t, drainIterator(t, openIterator(t, db)), "empty db should yield no positions") + appQC, err := db.ReadAppQCByBlockNumber(0) + require.NoError(t, err) + require.False(t, appQC.IsPresent()) - itAt, err := db.Iterator(0) + appProposal, err := db.ReadAppProposalByBlockNumber(0) require.NoError(t, err) - require.Empty(t, drainIterator(t, itAt), "empty db should yield no positions from any start") + require.False(t, appProposal.IsPresent()) + + require.Empty(t, drainSuffix(t, db), "empty db should yield no suffix records") - tips := db.Status() - require.Zero(t, tips.NextBlock, "empty db has no block write tip") - require.Zero(t, tips.NextQC, "empty db has no QC write tip") + require.False(t, db.Status().IsPresent(), "empty db has no write tips") } // iterEntry is one position observed while draining an iterator. @@ -150,38 +154,60 @@ type iterEntry struct { // blk is the block at the position; nil when no block is persisted there. blk *types.Block -} -// openIterator opens an iterator over everything retained in db. -func openIterator(t *testing.T, db types.BlockDB) types.BlockDBIterator { - t.Helper() - it, err := db.Iterator(0) - require.NoError(t, err) - return it + // appQC is the AppQC at the position; nil when no AppQC is persisted there. + appQC *types.AppQC + + // appProposal is the AppProposal at the position; nil when no AppProposal is persisted there. + appProposal *types.AppProposal } -// drainIterator walks an iterator to completion (closing it), collecting every position and -// asserting the per-position contract: QC is always present and its covered range contains the number. -func drainIterator(t *testing.T, it types.BlockDBIterator) []iterEntry { +// drainSuffix reads the recovery-visible suffix batch. +func drainSuffix(t *testing.T, db types.BlockDB) []iterEntry { t.Helper() - defer func() { require.NoError(t, it.Close()) }() + suffix, err := db.ReadSuffix() + require.NoError(t, err) var entries []iterEntry - for { - pos, ok, err := it.Next() - require.NoError(t, err) - if !ok { - break - } - n, qc := pos.Number, pos.QC - require.NotNil(t, qc, "QC must be present at every position") + floor := suffix.Status.Or(types.SuffixRange{ + First: 0, + NextBlock: 0, + NextQC: 0, + NextAppQC: 0, + NextAppProposal: 0, + }).First + for _, qc := range suffix.CommitQCs { first := qc.QC().GlobalRange().First next := first + gbn(len(qc.Headers())) - require.True(t, first <= n && n < next, "QC [%d,%d) must cover position %d", first, next, n) - blkOpt, err := it.Block() - require.NoError(t, err) - blk, present := blkOpt.Get() - require.Equal(t, pos.HasBlock, present, "HasBlock must agree with Block at position %d", n) - entries = append(entries, iterEntry{n: n, qc: qc, blk: blk}) + for n := max(first, floor); n < next; n++ { + entries = append(entries, iterEntry{n: n, qc: qc}) + } + } + for _, b := range suffix.Blocks { + found := false + for i := range entries { + if entries[i].n == b.Number { + entries[i].blk = b.Block + found = true + break + } + } + require.True(t, found, "block %d must be covered by a suffix QC", b.Number) + } + for _, appQC := range suffix.AppQCs { + gr := appQC.Proposal().GlobalRange() + for i := range entries { + if gr.Has(entries[i].n) { + entries[i].appQC = appQC + } + } + } + for _, appProposal := range suffix.AppProposals { + gr := appProposal.GlobalRange() + for i := range entries { + if gr.Has(entries[i].n) { + entries[i].appProposal = appProposal + } + } } return entries } @@ -219,9 +245,12 @@ func testStatus(t *testing.T, build builder) { defer func() { _ = db.Close() }() require.NoError(t, db.WriteQC(batches[0].qc)) - tips := db.Status() + tips := status(t, db) + require.Equal(t, batches[0].first, tips.First) require.Equal(t, batches[0].next, tips.NextQC) - require.Zero(t, tips.NextBlock, "QC-only store has no block tip") + require.Equal(t, tips.First, tips.NextBlock, "QC-only store has no block tip") + require.Equal(t, tips.First, tips.NextAppQC, "QC-only store has no AppQC tip") + require.Equal(t, tips.First, tips.NextAppProposal, "QC-only store has no AppProposal tip") assertTipsMatchPresent(t, db) for i, blk := range batches[0].blocks { @@ -229,7 +258,7 @@ func testStatus(t *testing.T, build builder) { } writeAll(t, db, batches[1:]) last := batches[len(batches)-1] - tips = db.Status() + tips = status(t, db) require.Equal(t, last.next, tips.NextBlock) require.Equal(t, last.next, tips.NextQC) assertTipsMatchPresent(t, db) @@ -239,37 +268,50 @@ func testStatus(t *testing.T, build builder) { require.Greater(t, len(batches), 1) require.NoError(t, db.PruneBefore(batches[1].first)) assertTipsMatchPresent(t, db) - tips = db.Status() + tips = status(t, db) require.Equal(t, last.next, tips.NextBlock, "prune must not move the block write tip") require.Equal(t, last.next, tips.NextQC, "prune must not move the QC write tip") db = restart(t, o, db) assertTipsMatchPresent(t, db) - tips = db.Status() + tips = status(t, db) require.Equal(t, last.next, tips.NextBlock, "block tip must survive restart") require.Equal(t, last.next, tips.NextQC, "QC tip must survive restart") } -// assertTipsMatchPresent checks Status against a full iterator scan (the records -// the public read API still serves). +// assertTipsMatchPresent checks Status against point reads for the records the +// public read API still serves. func assertTipsMatchPresent(t *testing.T, db types.BlockDB) { t.Helper() - tips := db.Status() - - highest, hasBlock := recoverHighestBlock(t, db) - if tips.NextBlock != 0 { - require.True(t, hasBlock, "Status has a block tip but the iterator yields no blocks") - require.Equal(t, highest+1, tips.NextBlock, "NextBlock must be one past the highest present block") - } else { - require.False(t, hasBlock, "the iterator yields blocks but Status has no block tip") + tips := status(t, db) + + if tips.NextBlock > tips.First { + blk, err := db.ReadBlockByNumber(tips.NextBlock - 1) + require.NoError(t, err) + require.True(t, blk.IsPresent(), "NextBlock must point past a readable block") } - lastQC, hasQC := recoverLastQC(t, db) - if tips.NextQC != 0 { - require.True(t, hasQC, "Status has a QC tip but the iterator yields no QCs") - require.Equal(t, lastQC.GlobalRange().Next, tips.NextQC, "NextQC must be Next of the highest present QC") - } else { - require.False(t, hasQC, "the iterator yields QCs but Status has no QC tip") + if tips.NextQC > tips.First { + qc, err := db.ReadQCByBlockNumber(tips.NextQC - 1) + require.NoError(t, err) + got, ok := qc.Get() + require.True(t, ok, "NextQC must point past a readable QC") + require.Equal(t, tips.NextQC, got.QC().GlobalRange().Next) + } + + if tips.NextAppQC > tips.First { + appQC, err := db.ReadAppQCByBlockNumber(tips.NextAppQC - 1) + require.NoError(t, err) + got, ok := appQC.Get() + require.True(t, ok, "NextAppQC must point past a readable AppQC") + require.Equal(t, tips.NextAppQC, got.Proposal().GlobalRange().Next) + } + if tips.NextAppProposal > tips.First { + appProposal, err := db.ReadAppProposalByBlockNumber(tips.NextAppProposal - 1) + require.NoError(t, err) + got, ok := appProposal.Get() + require.True(t, ok, "NextAppProposal must point past a readable AppProposal") + require.Equal(t, tips.NextAppProposal, got.GlobalRange().Next) } } @@ -307,6 +349,123 @@ func testQCByBlockNumber(t *testing.T, build builder) { require.False(t, miss.IsPresent()) } +func testAppProposalByBlockNumber(t *testing.T, build builder) { + committee, keys := buildCommittee() + batches := generateBatches(committee, keys) + db, o := openFresh(t, build) + defer func() { _ = db.Close() }() + writeAll(t, db, batches) + + rng := utils.TestRngFromSeed(testSeed + 90) + appProposals := []*types.AppProposal{ + appProposalForBatch(rng, batches[0]), + appProposalForBatch(rng, batches[1]), + } + for _, appProposal := range appProposals { + require.NoError(t, db.WriteAppProposal(appProposal)) + } + + for _, appProposal := range appProposals { + gr := appProposal.GlobalRange() + for n := gr.First; n < gr.Next; n++ { + opt, err := db.ReadAppProposalByBlockNumber(n) + require.NoError(t, err) + got, ok := opt.Get() + require.True(t, ok, "AppProposal covering %d should exist", n) + require.Equal(t, gr, got.GlobalRange()) + require.Equal(t, appProposal.AppHash(), got.AppHash()) + } + } + miss, err := db.ReadAppProposalByBlockNumber(batches[2].first) + require.NoError(t, err) + require.False(t, miss.IsPresent(), "CommitQCs/blocks past the AppProposal prefix should not imply AppProposal presence") + + entries := drainSuffix(t, db) + for _, e := range entries { + switch { + case e.n < batches[2].first: + require.NotNil(t, e.appProposal, "iterator should expose AppProposal at %d", e.n) + default: + require.Nil(t, e.appProposal, "iterator should not expose AppProposal past the persisted AppProposal prefix at %d", e.n) + } + } + + tips := status(t, db) + require.Equal(t, batches[1].next, tips.NextAppProposal) + db = restart(t, o, db) + tips = status(t, db) + require.Equal(t, batches[1].next, tips.NextAppProposal, "AppProposal tip must survive restart") + assertTipsMatchPresent(t, db) + + for _, appProposal := range appProposals { + gr := appProposal.GlobalRange() + opt, err := db.ReadAppProposalByBlockNumber(gr.First) + require.NoError(t, err) + got, ok := opt.Get() + require.True(t, ok, "AppProposal [%d,%d) must survive restart", gr.First, gr.Next) + require.Equal(t, gr, got.GlobalRange()) + } +} + +func testAppQCByBlockNumber(t *testing.T, build builder) { + committee, keys := buildCommittee() + batches := generateBatches(committee, keys) + db, o := openFresh(t, build) + defer func() { _ = db.Close() }() + writeAll(t, db, batches) + + rng := utils.TestRngFromSeed(testSeed + 100) + appQCs := []*types.AppQC{ + appQCForBatch(rng, keys, batches[0]), + appQCForBatch(rng, keys, batches[1]), + } + for _, appQC := range appQCs { + require.NoError(t, db.WriteAppProposal(appQC.Proposal())) + require.NoError(t, db.WriteAppQC(appQC)) + } + + for _, appQC := range appQCs { + gr := appQC.Proposal().GlobalRange() + for n := gr.First; n < gr.Next; n++ { + opt, err := db.ReadAppQCByBlockNumber(n) + require.NoError(t, err) + got, ok := opt.Get() + require.True(t, ok, "AppQC covering %d should exist", n) + require.Equal(t, gr, got.Proposal().GlobalRange()) + require.Equal(t, appQC.Proposal().AppHash(), got.Proposal().AppHash()) + } + } + miss, err := db.ReadAppQCByBlockNumber(batches[2].first) + require.NoError(t, err) + require.False(t, miss.IsPresent(), "CommitQCs/blocks past the AppQC prefix should not imply AppQC presence") + + entries := drainSuffix(t, db) + for _, e := range entries { + switch { + case e.n < batches[2].first: + require.NotNil(t, e.appQC, "iterator should expose AppQC at %d", e.n) + default: + require.Nil(t, e.appQC, "iterator should not expose AppQC past the persisted AppQC prefix at %d", e.n) + } + } + + tips := status(t, db) + require.Equal(t, batches[1].next, tips.NextAppQC) + db = restart(t, o, db) + tips = status(t, db) + require.Equal(t, batches[1].next, tips.NextAppQC, "AppQC tip must survive restart") + assertTipsMatchPresent(t, db) + + for _, appQC := range appQCs { + gr := appQC.Proposal().GlobalRange() + opt, err := db.ReadAppQCByBlockNumber(gr.First) + require.NoError(t, err) + got, ok := opt.Get() + require.True(t, ok, "AppQC [%d,%d) must survive restart", gr.First, gr.Next) + require.Equal(t, gr, got.Proposal().GlobalRange()) + } +} + func testIterators(t *testing.T, build builder) { committee, keys := buildCommittee() batches := generateBatches(committee, keys) @@ -418,6 +577,7 @@ func testPruneRefusesBelowWatermark(t *testing.T, build builder) { db, _ := openFresh(t, build) defer func() { _ = db.Close() }() writeAll(t, db, batches) + writeAppData(t, db, utils.TestRngFromSeed(testSeed+500), keys, batches) // Prune at the start of the second batch: all of the first batch is below it. watermark := batches[1].first @@ -435,9 +595,17 @@ func testPruneRefusesBelowWatermark(t *testing.T, build builder) { byHash, err := db.ReadBlockByHash(blk.Header().Hash()) require.NoError(t, err) require.False(t, byHash.IsPresent(), "block %d below watermark %d must not be served by hash", n, watermark) + + appProposal, err := db.ReadAppProposalByBlockNumber(n) + require.ErrorIs(t, err, types.ErrPruned, "AppProposal at block %d below watermark %d must be reported pruned", n, watermark) + require.False(t, appProposal.IsPresent(), "AppProposal at block %d below watermark %d must not be served", n, watermark) + + appQC, err := db.ReadAppQCByBlockNumber(n) + require.ErrorIs(t, err, types.ErrPruned, "AppQC at block %d below watermark %d must be reported pruned", n, watermark) + require.False(t, appQC.IsPresent(), "AppQC at block %d below watermark %d must not be served", n, watermark) } - for _, e := range drainIterator(t, openIterator(t, db)) { + for _, e := range drainSuffix(t, db) { require.GreaterOrEqual(t, e.n, watermark, "iterator must not yield position %d below watermark %d", e.n, watermark) } @@ -460,6 +628,7 @@ func testPrunedDistinctFromNotFound(t *testing.T, build builder) { db, _ := openFresh(t, build) defer func() { _ = db.Close() }() writeAll(t, db, batches) + writeAppData(t, db, utils.TestRngFromSeed(testSeed+501), keys, batches) straddled := batches[1] pruneAt := straddled.first + 2 @@ -475,6 +644,12 @@ func testPrunedDistinctFromNotFound(t *testing.T, build builder) { opt, err := db.ReadBlockByNumber(n) require.NoError(t, err, "block %d in the straddled cohort must not report ErrPruned", n) require.True(t, opt.IsPresent(), "block %d in the straddled cohort must remain served", n) + appProposal, err := db.ReadAppProposalByBlockNumber(n) + require.NoError(t, err, "AppProposal at block %d in the straddled cohort must not report ErrPruned", n) + require.True(t, appProposal.IsPresent(), "AppProposal at block %d in the straddled cohort must remain served", n) + appQC, err := db.ReadAppQCByBlockNumber(n) + require.NoError(t, err, "AppQC at block %d in the straddled cohort must not report ErrPruned", n) + require.True(t, appQC.IsPresent(), "AppQC at block %d in the straddled cohort must remain served", n) } qcOpt, err := db.ReadQCByBlockNumber(straddled.first) require.NoError(t, err, "straddled cohort's QC must not report ErrPruned") @@ -491,6 +666,12 @@ func testPrunedDistinctFromNotFound(t *testing.T, build builder) { qc, err := db.ReadQCByBlockNumber(belowNum) require.ErrorIs(t, err, types.ErrPruned, "below-watermark QC must report ErrPruned") require.False(t, qc.IsPresent()) + appProposal, err := db.ReadAppProposalByBlockNumber(belowNum) + require.ErrorIs(t, err, types.ErrPruned, "below-watermark AppProposal must report ErrPruned") + require.False(t, appProposal.IsPresent()) + appQC, err := db.ReadAppQCByBlockNumber(belowNum) + require.ErrorIs(t, err, types.ErrPruned, "below-watermark AppQC must report ErrPruned") + require.False(t, appQC.IsPresent()) // Above the watermark but never written: not pruned, just absent. unwritten := batches[len(batches)-1].next + 1000 @@ -500,6 +681,12 @@ func testPrunedDistinctFromNotFound(t *testing.T, build builder) { missQC, err := db.ReadQCByBlockNumber(unwritten) require.NoError(t, err, "never-written height must not report ErrPruned") require.False(t, missQC.IsPresent()) + missAppProposal, err := db.ReadAppProposalByBlockNumber(unwritten) + require.NoError(t, err, "never-written AppProposal height must not report ErrPruned") + require.False(t, missAppProposal.IsPresent()) + missAppQC, err := db.ReadAppQCByBlockNumber(unwritten) + require.NoError(t, err, "never-written AppQC height must not report ErrPruned") + require.False(t, missAppQC.IsPresent()) } // testPruneIdempotentMonotonic asserts PruneBefore is idempotent and the @@ -579,6 +766,7 @@ func testPruneNeverEmpties(t *testing.T, build builder) { db, _ := openFresh(t, build) defer func() { _ = db.Close() }() writeAll(t, db, batches) + writeAppData(t, db, utils.TestRngFromSeed(testSeed+502), keys, batches) require.NoError(t, db.PruneBefore(prune)) @@ -601,6 +789,14 @@ func testPruneNeverEmpties(t *testing.T, build builder) { qc, err := db.ReadQCByBlockNumber(n) require.NoError(t, err) require.True(t, qc.IsPresent(), "the QC covering the newest cohort must survive") + + appProposal, err := db.ReadAppProposalByBlockNumber(n) + require.NoError(t, err) + require.True(t, appProposal.IsPresent(), "the AppProposal covering the newest cohort must survive") + + appQC, err := db.ReadAppQCByBlockNumber(n) + require.NoError(t, err) + require.True(t, appQC.IsPresent(), "the AppQC covering the newest cohort must survive") } // A block below the newest cohort is gone (clamped watermark refuses/removes it). @@ -609,22 +805,71 @@ func testPruneNeverEmpties(t *testing.T, build builder) { below, err := db.ReadBlockByNumber(belowBatch.first) require.ErrorIs(t, err, types.ErrPruned, "blocks below the newest cohort must be reported pruned") require.False(t, below.IsPresent(), "blocks below the newest cohort must not be served") - - // The iterator yields exactly the newest cohort's numbers, every one with a - // block, all covered by the single remaining QC. - var expected []types.GlobalBlockNumber - for i := range last.blocks { - expected = append(expected, last.first+gbn(i)) - } - entries := drainIterator(t, openIterator(t, db)) - require.Equal(t, expected, presentBlockNumbers(entries), - "exactly the newest cohort must remain after PruneBefore(%d)", prune) + belowAppProposal, err := db.ReadAppProposalByBlockNumber(belowBatch.first) + require.ErrorIs(t, err, types.ErrPruned, "AppProposals below the newest cohort must be reported pruned") + require.False(t, belowAppProposal.IsPresent(), "AppProposals below the newest cohort must not be served") + belowAppQC, err := db.ReadAppQCByBlockNumber(belowBatch.first) + require.ErrorIs(t, err, types.ErrPruned, "AppQCs below the newest cohort must be reported pruned") + require.False(t, belowAppQC.IsPresent(), "AppQCs below the newest cohort must not be served") + + // ReadSuffix starts at recentFloor(), while the prune watermark still + // rounds down to keep the whole newest QC cohort readable. + entries := drainSuffix(t, db) + require.Equal(t, []types.GlobalBlockNumber{newest}, presentBlockNumbers(entries), + "ReadSuffix must start at recentFloor() after PruneBefore(%d)", prune) require.Equal(t, []types.GlobalBlockNumber{last.first}, qcFirsts(entries), - "exactly one QC (covering the newest cohort) must remain") + "ReadSuffix must include the QC covering the suffix floor") }) } } +func testPruneWithAppQCNeverEmpties(t *testing.T, build builder) { + committee, keys := buildCommittee() + batches := generateBatches(committee, keys) + require.GreaterOrEqual(t, len(batches), 4, "need AppQC prefix to lag CommitQC tip") + db, _ := openFresh(t, build) + defer func() { _ = db.Close() }() + writeAll(t, db, batches) + + rng := utils.TestRngFromSeed(testSeed + 300) + for _, b := range batches[:3] { + appQC := appQCForBatch(rng, keys, b) + require.NoError(t, db.WriteAppProposal(appQC.Proposal())) + require.NoError(t, db.WriteAppQC(appQC)) + } + + latestApp := batches[2] + require.NoError(t, db.PruneBefore(batches[len(batches)-1].next+1000)) + + below := batches[1] + blk, err := db.ReadBlockByNumber(below.first) + require.ErrorIs(t, err, types.ErrPruned) + require.False(t, blk.IsPresent()) + appQC, err := db.ReadAppQCByBlockNumber(below.first) + require.ErrorIs(t, err, types.ErrPruned) + require.False(t, appQC.IsPresent()) + appProposal, err := db.ReadAppProposalByBlockNumber(below.first) + require.ErrorIs(t, err, types.ErrPruned) + require.False(t, appProposal.IsPresent()) + + blk, err = db.ReadBlockByNumber(latestApp.first) + require.NoError(t, err) + require.True(t, blk.IsPresent(), "newest AppQC cohort must retain a block") + qc, err := db.ReadQCByBlockNumber(latestApp.first) + require.NoError(t, err) + require.True(t, qc.IsPresent(), "newest AppQC cohort must retain its CommitQC") + appQC, err = db.ReadAppQCByBlockNumber(latestApp.first) + require.NoError(t, err) + got, ok := appQC.Get() + require.True(t, ok, "newest AppQC cohort must retain its AppQC") + require.Equal(t, latestApp.qc.QC().GlobalRange(), got.Proposal().GlobalRange()) + appProposal, err = db.ReadAppProposalByBlockNumber(latestApp.first) + require.NoError(t, err) + gotProposal, ok := appProposal.Get() + require.True(t, ok, "newest AppQC cohort must retain its AppProposal") + require.Equal(t, latestApp.qc.QC().GlobalRange(), gotProposal.GlobalRange()) +} + // testPruneQCAheadOfBlocks pins the min() guard in the prune clamp. QCs are // written before the blocks they cover, so between writing a QC and its first // block — and after a crash that persisted a QC but not its blocks — the newest @@ -697,31 +942,33 @@ func testPruneQCOnlyThenWriteBlock(t *testing.T, build builder) { require.True(t, qc.IsPresent(), "covering QC of block %d must survive the earlier prune", b0.first) } -// testIteratorSnapshot asserts that an iterator observes only the records present -// when it was created — writes made afterward are invisible to it. -func testIteratorSnapshot(t *testing.T, build builder) { +func testReadSuffix(t *testing.T, build builder) { committee, keys := buildCommittee() batches := generateBatches(committee, keys) db, _ := openFresh(t, build) defer func() { _ = db.Close() }() - // Write only the first batch, then snapshot an iterator over it. - first := batches[0] - require.NoError(t, db.WriteQC(first.qc)) - for i, blk := range first.blocks { - require.NoError(t, db.WriteBlock(first.first+gbn(i), blk)) + writeAll(t, db, batches[:2]) + appQC := appQCForBatch(utils.TestRngFromSeed(testSeed+400), keys, batches[0]) + require.NoError(t, db.WriteAppProposal(appQC.Proposal())) + require.NoError(t, db.WriteAppQC(appQC)) + require.NoError(t, db.WriteQC(batches[2].qc)) + for i, blk := range batches[2].blocks { + require.NoError(t, db.WriteBlock(batches[2].first+gbn(i), blk)) } - it := openIterator(t, db) - - // Write the remaining batches AFTER the iterator was created. - writeAll(t, db, batches[1:]) - - entries := drainIterator(t, it) - require.Len(t, presentBlockNumbers(entries), len(first.blocks), - "iterator must not observe blocks written after creation") - require.Equal(t, []types.GlobalBlockNumber{first.first}, qcFirsts(entries), - "iterator must not observe QCs written after creation") + suffix, err := db.ReadSuffix() + require.NoError(t, err) + require.Equal(t, status(t, db), suffix.Status.OrPanic("suffix status")) + require.NotEmpty(t, suffix.AppQCs) + gotAppQC := suffix.AppQCs[0] + require.Equal(t, appQC.Proposal().RoadIndex(), gotAppQC.Proposal().RoadIndex()) + entries := drainSuffix(t, db) + recoveryFloor := appQC.Proposal().GlobalRange().Next - 1 + require.Equal(t, []types.GlobalBlockNumber{batches[0].first, batches[1].first, batches[2].first}, qcFirsts(entries)) + require.Equal(t, batches[2].next-recoveryFloor, types.GlobalBlockNumber(len(entries))) + require.Equal(t, recoveryFloor, entries[0].n) + require.Len(t, presentBlockNumbers(entries), int(batches[2].next-recoveryFloor)) } func testWriteOrderRejected(t *testing.T, build builder) { @@ -751,6 +998,97 @@ func testWriteOrderRejected(t *testing.T, build builder) { require.True(t, opt.IsPresent()) } +func testWriteAppDataOrderRejected(t *testing.T, build builder) { + t.Run("AppProposal", func(t *testing.T) { + testWriteAppProposalOrderRejected(t, build) + }) + t.Run("AppQC", func(t *testing.T) { + testWriteAppQCOrderRejected(t, build) + }) +} + +func testWriteAppProposalOrderRejected(t *testing.T, build builder) { + committee, keys := buildCommittee() + batches := generateBatches(committee, keys) + db, _ := openFresh(t, build) + defer func() { _ = db.Close() }() + rng := utils.TestRngFromSeed(testSeed + 190) + + b0 := batches[0] + b1 := batches[1] + b2 := batches[2] + + err := db.WriteAppProposal(appProposalForBatch(rng, b0)) + require.ErrorIs(t, err, types.ErrAppProposalMissingQC, "AppProposal before CommitQC must fail") + + require.NoError(t, db.WriteQC(b0.qc)) + require.NoError(t, db.WriteQC(b1.qc)) + err = db.WriteAppProposal(appProposalForBatch(rng, b1)) + require.ErrorIs(t, err, types.ErrAppProposalNonContiguous, "first AppProposal must start at retained QC floor") + + for i, blk := range b0.blocks { + require.NoError(t, db.WriteBlock(b0.first+gbn(i), blk)) + } + appProposal0 := appProposalForBatch(rng, b0) + require.NoError(t, db.WriteAppProposal(appProposal0)) + + err = db.WriteAppProposal(appProposal0) + require.ErrorIs(t, err, types.ErrAppProposalNonContiguous, "duplicate AppProposal write must fail") + + require.NoError(t, db.WriteQC(b2.qc)) + err = db.WriteAppProposal(appProposalForBatch(rng, b2)) + require.ErrorIs(t, err, types.ErrAppProposalNonContiguous, "AppProposal gap must fail") + + for i, blk := range b1.blocks { + require.NoError(t, db.WriteBlock(b1.first+gbn(i), blk)) + } + require.NoError(t, db.WriteAppProposal(appProposalForBatch(rng, b1))) + tips := status(t, db) + require.Equal(t, b1.next, tips.NextAppProposal) +} + +func testWriteAppQCOrderRejected(t *testing.T, build builder) { + committee, keys := buildCommittee() + batches := generateBatches(committee, keys) + db, _ := openFresh(t, build) + defer func() { _ = db.Close() }() + rng := utils.TestRngFromSeed(testSeed + 200) + + b0 := batches[0] + b1 := batches[1] + b2 := batches[2] + + err := db.WriteAppQC(appQCForBatch(rng, keys, b0)) + require.ErrorIs(t, err, types.ErrAppQCMissingQC, "AppQC before CommitQC must fail") + + require.NoError(t, db.WriteQC(b0.qc)) + require.NoError(t, db.WriteQC(b1.qc)) + err = db.WriteAppQC(appQCForBatch(rng, keys, b1)) + require.ErrorIs(t, err, types.ErrAppQCNonContiguous, "first AppQC must start at retained QC floor") + + for i, blk := range b0.blocks { + require.NoError(t, db.WriteBlock(b0.first+gbn(i), blk)) + } + require.NoError(t, db.WriteAppProposal(appProposalForBatch(rng, b0))) + appQC0 := appQCForBatch(rng, keys, b0) + require.NoError(t, db.WriteAppQC(appQC0)) + + err = db.WriteAppQC(appQC0) + require.ErrorIs(t, err, types.ErrAppQCNonContiguous, "duplicate AppQC write must fail") + + require.NoError(t, db.WriteQC(b2.qc)) + err = db.WriteAppQC(appQCForBatch(rng, keys, b2)) + require.ErrorIs(t, err, types.ErrAppQCNonContiguous, "AppQC gap must fail") + + for i, blk := range b1.blocks { + require.NoError(t, db.WriteBlock(b1.first+gbn(i), blk)) + } + require.NoError(t, db.WriteAppProposal(appProposalForBatch(rng, b1))) + require.NoError(t, db.WriteAppQC(appQCForBatch(rng, keys, b1))) + tips := status(t, db) + require.Equal(t, b1.next, tips.NextAppQC) +} + // testWriteOrderRejectedAfterRestart asserts the write-order cursors are // reloaded from persisted state on reopen. After a restart a freshly opened DB // must still reject an out-of-order block and a non-contiguous QC, and must @@ -829,7 +1167,7 @@ func testResumeAfterRestart(t *testing.T, build builder) { require.Equal(t, last.first, prevQC.GlobalRange().First, "recovered QC must be the last persisted QC") require.Equal(t, last.next, prevQC.GlobalRange().Next) - tips := db.Status() + tips := status(t, db) require.Equal(t, highest+1, tips.NextBlock, "Status block tip must match the iterator scan") require.Equal(t, prevQC.GlobalRange().Next, tips.NextQC, "Status QC tip must match the iterator scan") covering, err := db.ReadQCByBlockNumber(tips.NextQC - 1) @@ -854,7 +1192,7 @@ func testResumeAfterRestart(t *testing.T, build builder) { // verification; production resume uses Status (see blocksim.recoverResumeState). func recoverHighestBlock(t *testing.T, db types.BlockDB) (types.GlobalBlockNumber, bool) { t.Helper() - present := presentBlockNumbers(drainIterator(t, openIterator(t, db))) + present := presentBlockNumbers(drainSuffix(t, db)) if len(present) == 0 { return 0, false } @@ -866,284 +1204,24 @@ func recoverHighestBlock(t *testing.T, db types.BlockDB) (types.GlobalBlockNumbe // verification; production resume uses Status (see blocksim.recoverResumeState). func recoverLastQC(t *testing.T, db types.BlockDB) (*types.CommitQC, bool) { t.Helper() - entries := drainIterator(t, openIterator(t, db)) + entries := drainSuffix(t, db) if len(entries) == 0 { return nil, false } return entries[len(entries)-1].qc.QC(), true } -// testIteratorPositioning asserts that Iterator positions at a given height: it yields the -// (clamped) start and every higher covered number, densely ascending, with the whole covering QC -// available even when the start falls mid-range. A start past the last covered number yields -// nothing; a start below the watermark clamps up to the watermark. The positioning assertions run -// twice: on the live store right after the writes (consensus reads the tip while writing) and -// again after a restart (the resume use case, where the backing index is rebuilt). -func testIteratorPositioning(t *testing.T, build builder) { - committee, keys := buildCommittee() - batches := generateBatches(committee, keys) - db, o := openFresh(t, build) - defer func() { _ = db.Close() }() - writeAll(t, db, batches) - - // Pick a start strictly inside a middle QC's range to exercise covering-QC positioning. - mid := batches[len(batches)/2] - require.Greater(t, mid.next, mid.first+1, "need a multi-block QC range") - start := mid.first + 1 - last := batches[len(batches)-1] - - assertPositions := func() { - it, err := db.Iterator(start) - require.NoError(t, err) - entries := drainIterator(t, it) - require.NotEmpty(t, entries) - require.Equal(t, start, entries[0].n, "Iterator must begin at the requested height") - require.Equal(t, mid.first, entries[0].qc.QC().GlobalRange().First, - "a mid-range start must expose the whole covering QC") - require.Equal(t, last.next-1, entries[len(entries)-1].n, "iteration must reach the last covered number") - for i := 1; i < len(entries); i++ { - require.Equal(t, entries[i-1].n+1, entries[i].n, "positions must be densely ascending") - } - for _, e := range entries { - require.NotNil(t, e.blk, "every covered number has a block in a fully-written store") - } - - // The covering QC and every later QC, ascending by First. - var wantFirsts []types.GlobalBlockNumber - for _, b := range batches { - if b.next > start { - wantFirsts = append(wantFirsts, b.first) - } - } - require.Equal(t, wantFirsts, qcFirsts(entries)) - - // A start past the last covered number yields nothing. - itPast, err := db.Iterator(last.next + 100) - require.NoError(t, err) - require.Empty(t, drainIterator(t, itPast)) - } - - assertPositions() - db = restart(t, o, db) - assertPositions() - - // A start below the watermark clamps up to the watermark. - watermark := batches[1].first - require.NoError(t, db.PruneBefore(watermark)) - it, err := db.Iterator(0) - require.NoError(t, err) - clamped := drainIterator(t, it) - require.NotEmpty(t, clamped) - require.Equal(t, watermark, clamped[0].n, "start below the watermark must clamp to the watermark") - for _, e := range clamped { - require.GreaterOrEqual(t, e.n, watermark, "Iterator must never yield a position below the watermark") - } -} - -// testFirstBlockMidQC asserts that iteration opens on a block that exists. WriteBlock lets the very -// first block start anywhere inside its covering QC, so a store can hold a QC whose lower numbers -// carry no block. Iteration must begin at that first block rather than at the QC's start — every -// yielded position then carries a block, so the leading numbers are simply not part of the scan. -// The mirror of testIteratorTail, which covers the blockless run at the other end. -func testFirstBlockMidQC(t *testing.T, build builder) { - committee, keys := buildCommittee() - batches := generateBatches(committee, keys) - db, o := openFresh(t, build) - closed := false - defer func() { - if !closed { - require.NoError(t, db.Close()) - } - }() - - // One QC, but blocks only from the middle of its range onward. - b0 := batches[0] - mid := b0.first + (b0.next-b0.first)/2 - require.Greater(t, mid, b0.first, "need a blockless prefix inside the cohort") - require.NoError(t, db.WriteQC(b0.qc)) - for n := mid; n < b0.next; n++ { - require.NoError(t, db.WriteBlock(n, b0.blocks[n-b0.first])) - } - - assertOpensOnFirstBlock := func(t *testing.T, db types.BlockDB) { - t.Helper() - entries := drainIterator(t, openIterator(t, db)) - require.Equal(t, b0.next-mid, types.GlobalBlockNumber(len(entries)), - "the scan must cover exactly [firstBlock, QC end)") - require.Equal(t, mid, entries[0].n, "the scan must open on the first block that exists") - for i, e := range entries { - require.Equal(t, mid+gbn(i), e.n, "positions must be densely ascending") - require.NotNil(t, e.blk, "every yielded position must carry a block") - } - - // A start below the first block clamps up to it; a start above it is honoured. - below := drainIterator(t, mustIteratorAt(t, db, b0.first)) - require.Equal(t, mid, below[0].n, "a start below the first block clamps up to it") - above := drainIterator(t, mustIteratorAt(t, db, mid+1)) - require.Equal(t, mid+1, above[0].n, "a start above the first block is honoured") - } - - assertOpensOnFirstBlock(t, db) - - // Restart: a durable backend must re-derive the same floor on open. - require.NoError(t, db.Flush()) - require.NoError(t, db.Close()) - closed = true - reopened, err := o() - require.NoError(t, err) - defer func() { require.NoError(t, reopened.Close()) }() - assertOpensOnFirstBlock(t, reopened) -} - -// mustIteratorAt opens an iterator at n, failing the test on error. -func mustIteratorAt(t *testing.T, db types.BlockDB, n types.GlobalBlockNumber) types.BlockDBIterator { +// recoverLastAppQC returns the most recently persisted AppQC via a full +// iterator scan (false if the store has no AppQCs). +func recoverLastAppQC(t *testing.T, db types.BlockDB) (*types.AppQC, bool) { t.Helper() - it, err := db.Iterator(n) - require.NoError(t, err) - return it -} - -// testIteratorTail asserts the QC-ahead-of-blocks shape: when a QC is persisted but (some of) its -// blocks are not — a crash between the QC write and the block writes leaves exactly this — the -// iterator still yields every covered number, with the covering QC present and Block None on the -// trailing positions. This is what lets replay restore trailing QCs from the same single scan. -func testIteratorTail(t *testing.T, build builder) { - committee, keys := buildCommittee() - batches := generateBatches(committee, keys) - require.GreaterOrEqual(t, len(batches), 3, "need a filled prefix plus unfilled tail cohorts") - db, _ := openFresh(t, build) - defer func() { _ = db.Close() }() - - // Fill the first cohort completely and the second only partially; write the - // third cohort's QC with no blocks at all. - b0 := batches[0] - b1 := batches[1] - b2 := batches[2] - require.NoError(t, db.WriteQC(b0.qc)) - for i, blk := range b0.blocks { - require.NoError(t, db.WriteBlock(b0.first+gbn(i), blk)) - } - require.NoError(t, db.WriteQC(b1.qc)) - partial := len(b1.blocks) / 2 - require.Greater(t, partial, 0, "need at least one block in the partially-filled cohort") - for i := 0; i < partial; i++ { - require.NoError(t, db.WriteBlock(b1.first+gbn(i), b1.blocks[i])) - } - require.NoError(t, db.WriteQC(b2.qc)) - - lastBlock := b1.first + gbn(partial-1) - - entries := drainIterator(t, openIterator(t, db)) - require.Equal(t, b2.next-b0.first, types.GlobalBlockNumber(len(entries)), - "the iterator must yield every QC-covered number") - for i, e := range entries { - require.Equal(t, b0.first+gbn(i), e.n, "positions must be densely ascending") - if e.n <= lastBlock { - require.NotNil(t, e.blk, "position %d is below the block tip and must have a block", e.n) - } else { - require.Nil(t, e.blk, "position %d is past the block tip and must be block-less", e.n) - } - } - require.Equal(t, []types.GlobalBlockNumber{b0.first, b1.first, b2.first}, qcFirsts(entries), - "trailing QCs must be observed even where no block survives") - - // An iterator positioned inside the block-less tail still serves the covering QC. - it, err := db.Iterator(b2.first + 1) - require.NoError(t, err) - tail := drainIterator(t, it) - require.NotEmpty(t, tail) - require.Equal(t, b2.first+1, tail[0].n) - require.Equal(t, b2.first, tail[0].qc.QC().GlobalRange().First) - for _, e := range tail { - require.Nil(t, e.blk, "tail positions have no blocks") - } -} - -// testIteratorClampsUpToCoverage asserts the below-coverage clamp: on a store whose first QC -// begins above zero (an unpruned store with a genesis offset), Iterator(0) begins at the first -// covered number rather than yielding nothing. Only a start past the coverage is empty. -func testIteratorClampsUpToCoverage(t *testing.T, build builder) { - db, _ := openFresh(t, build) - defer func() { _ = db.Close() }() - - rng := utils.TestRngFromSeed(testSeed + 99) - first := types.GlobalBlockNumber(100) - next := types.GlobalBlockNumber(105) - require.NoError(t, db.WriteQC(types.GenFullCommitQCRange(rng, first, next))) - for n := first; n < next; n++ { - require.NoError(t, db.WriteBlock(n, types.GenBlock(rng))) - } - - // A start below all coverage clamps up to the first covered number. - it, err := db.Iterator(0) - require.NoError(t, err) - entries := drainIterator(t, it) - require.Len(t, entries, int(next-first)) - require.Equal(t, first, entries[0].n, "a start below coverage must clamp up to the first covered number") - - // A mid-range start begins exactly there. - it, err = db.Iterator(first + 2) - require.NoError(t, err) - entries = drainIterator(t, it) - require.NotEmpty(t, entries) - require.Equal(t, first+2, entries[0].n) - - // A start past the coverage yields nothing. - it, err = db.Iterator(next) - require.NoError(t, err) - require.Empty(t, drainIterator(t, it)) -} - -// testQCOnlyStoreIterates asserts the shape of a store that holds QCs and no blocks at all — what a -// crash between a QC flush and the first block write leaves behind. Every covered number must still be -// yielded with its covering QC and no block, so replay can restore those QCs from the same single scan. -// Distinct from testIteratorTail, where block-less QCs trail a store that does have blocks. -func testQCOnlyStoreIterates(t *testing.T, build builder) { - committee, keys := buildCommittee() - batches := generateBatches(committee, keys) - require.GreaterOrEqual(t, len(batches), 2, "need two cohorts to cover a multi-QC walk") - db, o := openFresh(t, build) - closed := false - defer func() { - if !closed { - require.NoError(t, db.Close()) + entries := drainSuffix(t, db) + for i := len(entries) - 1; i >= 0; i-- { + if entries[i].appQC != nil { + return entries[i].appQC, true } - }() - - // Two QCs, no blocks whatsoever. - b0, b1 := batches[0], batches[1] - require.NoError(t, db.WriteQC(b0.qc)) - require.NoError(t, db.WriteQC(b1.qc)) - - assertQCOnlyShape := func(t *testing.T, db types.BlockDB) { - t.Helper() - // drainIterator cross-checks HasBlock against Block() at every position. - entries := drainIterator(t, openIterator(t, db)) - require.Equal(t, b1.next-b0.first, types.GlobalBlockNumber(len(entries)), - "every number both QCs cover must be yielded") - for i, e := range entries { - require.Equal(t, b0.first+gbn(i), e.n, "positions must be densely ascending") - require.Nil(t, e.blk, "no block exists anywhere in this store") - } - require.Equal(t, []types.GlobalBlockNumber{b0.first, b1.first}, qcFirsts(entries), - "both QCs must be observed in one pass") - - // A mid-range start is honoured, and a start past coverage yields nothing. - mid := drainIterator(t, mustIteratorAt(t, db, b0.first+1)) - require.Equal(t, b0.first+1, mid[0].n) - require.Empty(t, drainIterator(t, mustIteratorAt(t, db, b1.next))) } - - assertQCOnlyShape(t, db) - - // Restart: the shape must survive a reopen, where a durable backend re-derives its cursors. - require.NoError(t, db.Flush()) - require.NoError(t, db.Close()) - closed = true - reopened, err := o() - require.NoError(t, err) - defer func() { require.NoError(t, reopened.Close()) }() - assertQCOnlyShape(t, reopened) + return nil, false } // testWriteBlockRequiresQC asserts the QC-before-block contract: a block may @@ -1188,108 +1266,14 @@ func testWriteQCCoversNoBlocksRejected(t *testing.T, build builder) { // The rejection persisted nothing: the store is still empty, so a QC that // does cover blocks is still accepted at 0. - require.Zero(t, db.Status().NextQC) + require.False(t, db.Status().IsPresent()) require.NoError(t, db.WriteQC(types.GenFullCommitQCRange(rng, 0, 3))) - require.Equal(t, gbn(3), db.Status().NextQC) -} - -// testIteratorBlockRequiresPosition asserts the one precondition the iterator API -// still carries, identically on every backend. Number, QC and presence come out of -// Next by value, so they cannot be read out of window at all; Block is the only -// accessor left with a positioned precondition (it is the only one that performs -// IO, which is why it is not a Position field). Every window in which it can be -// called without a position must report misuse rather than answer for a stale one. -func testIteratorBlockRequiresPosition(t *testing.T, build builder) { - committee, keys := buildCommittee() - batches := generateBatches(committee, keys) - db, _ := openFresh(t, build) - defer func() { _ = db.Close() }() - writeAll(t, db, batches[:1]) - - t.Run("BeforeFirstNext", func(t *testing.T) { - it := openIterator(t, db) - defer func() { _ = it.Close() }() - _, err := it.Block() - require.Error(t, err, "Block before the first Next must report misuse") - }) - - t.Run("AfterExhaustion", func(t *testing.T) { - it := openIterator(t, db) - defer func() { _ = it.Close() }() - for { - _, ok, err := it.Next() - require.NoError(t, err) - if !ok { - break - } - } - _, err := it.Block() - require.Error(t, err, "Block after exhaustion must report misuse, not repeat the last position") - }) - - t.Run("AfterCloseOnBlocklessPosition", func(t *testing.T) { - // Closing on a block-less position is the shape that slips past AfterClose below, which - // deliberately closes on a held block. With no block held there is no record to read - // through, so an implementation relying on the read failing has nothing to fail on: Next - // must reject the call on its own, and Block must not answer for the position it hands back. - blockless, _ := openFresh(t, build) - defer func() { _ = blockless.Close() }() - b := batches[0] - require.NoError(t, blockless.WriteQC(b.qc)) - require.NoError(t, blockless.WriteBlock(b.first, b.blocks[0])) - - it := openIterator(t, blockless) - var pos types.Position - for { - p, ok, err := it.Next() - require.NoError(t, err) - require.True(t, ok, "expected to reach a block-less position before exhaustion") - if !p.HasBlock { - pos = p - break - } - } - require.False(t, pos.HasBlock, "must be parked on a block-less position for this case to bite") - - require.NoError(t, it.Close()) - - _, ok, err := it.Next() - require.NoError(t, err, "Next after Close must not error") - require.False(t, ok, "Next after Close must report exhaustion, not yield a fresh position") - _, err = it.Block() - require.Error(t, err, "Block after Close must report misuse") - }) - - t.Run("AfterClose", func(t *testing.T) { - it := openIterator(t, db) - pos, ok, err := it.Next() - require.NoError(t, err) - require.True(t, ok) - require.True(t, pos.HasBlock, "the first position must hold a block for this case to bite") - require.NoError(t, it.Close()) - _, err = it.Block() - require.Error(t, err, "Block after Close must report misuse, not read a released snapshot") - }) - - t.Run("EmptyIterator", func(t *testing.T) { - empty, _ := openFresh(t, build) - defer func() { _ = empty.Close() }() - it := openIterator(t, empty) - defer func() { _ = it.Close() }() - pos, ok, err := it.Next() - require.NoError(t, err, "an empty store exhausts cleanly") - require.False(t, ok) - require.Equal(t, types.Position{}, pos, "an unyielded position must be the zero value") - _, err = it.Block() - require.Error(t, err, "Block on an empty iterator must report misuse") - }) + require.Equal(t, gbn(3), status(t, db).NextQC) } // testWriteBlockGapRejected asserts that blocks must be written densely: a // number that skips past lastBlockNumber+1 is rejected with ErrBlockOutOfOrder -// and persists nothing, even when the covering QC allows it. Density is what -// makes BlockDBIterator's tail-only-None contract exact (an absent block below -// the highest persisted one can only be corruption). +// and persists nothing, even when the covering QC allows it. func testWriteBlockGapRejected(t *testing.T, build builder) { committee, keys := buildCommittee() batches := generateBatches(committee, keys) @@ -1324,6 +1308,7 @@ func TestMemblockPruneRemovesBelowWatermark(t *testing.T) { batches := generateBatches(committee, keys) db := memblock.NewBlockDB() writeAll(t, db, batches) + writeAppData(t, db, utils.TestRngFromSeed(testSeed+503), keys, batches) watermark := batches[1].first require.NoError(t, db.PruneBefore(watermark)) @@ -1345,7 +1330,7 @@ func TestMemblockPruneRemovesBelowWatermark(t *testing.T) { require.True(t, opt.IsPresent()) // The iterator must skip the pruned records entirely. - for _, e := range drainIterator(t, openIterator(t, db)) { + for _, e := range drainSuffix(t, db) { require.GreaterOrEqual(t, e.n, watermark, "iterator must not surface pruned positions") require.GreaterOrEqual(t, e.qc.QC().GlobalRange().First, watermark, "iterator must not surface pruned QCs") @@ -1363,6 +1348,7 @@ func TestMemblockPruneIntoCohortRoundsDown(t *testing.T) { batches := generateBatches(committee, keys) db := memblock.NewBlockDB() writeAll(t, db, batches) + writeAppData(t, db, utils.TestRngFromSeed(testSeed+504), keys, batches) straddled := batches[1] pruneAt := straddled.first + 2 @@ -1457,7 +1443,7 @@ func assertIterators(t *testing.T, db types.BlockDB, committee *types.Committee, totalBlocks += len(b.blocks) } - entries := drainIterator(t, openIterator(t, db)) + entries := drainSuffix(t, db) require.Len(t, entries, totalBlocks, "one position per covered number in a fully-written store") require.Equal(t, batches[0].first, entries[0].n, "the scan must begin at the first covered number") for i, e := range entries { @@ -1511,6 +1497,15 @@ func writeAll(t *testing.T, db types.BlockDB, batches []batch) { } } +func writeAppData(t *testing.T, db types.BlockDB, rng utils.Rng, keys []types.SecretKey, batches []batch) { + t.Helper() + for _, b := range batches { + appQC := appQCForBatch(rng, keys, b) + require.NoError(t, db.WriteAppProposal(appQC.Proposal())) + require.NoError(t, db.WriteAppQC(appQC)) + } +} + // buildCommittee returns a deterministic round-robin committee (global numbering // from 0) and the secret keys that sign its QCs. func buildCommittee() (*types.Committee, []types.SecretKey) { @@ -1570,15 +1565,8 @@ func buildFullCommitQC( } } } - var appQC utils.Option[*types.AppQC] - if cqc, ok := prev.Get(); ok { - p := types.NewAppProposal(cqc.GlobalRange().Next-1, types.NextIndexOpt(prev), types.GenAppHash(rng), cqc.Proposal().EpochIndex()) - appQC = utils.Some(testAppQC(keys, p)) - } else { - appQC = utils.None[*types.AppQC]() - } ep := types.NewEpoch(0, types.OpenRoadRange(), genesisTime, committee, 0) - cqc := types.BuildCommitQC(ep, keys, prev, laneQCs, appQC) + cqc := types.BuildCommitQC(ep, keys, prev, laneQCs) return types.NewFullCommitQC(cqc, headers), blockList } @@ -1591,6 +1579,14 @@ func testLaneQC(keys []types.SecretKey, header *types.BlockHeader) *types.LaneQC return types.NewLaneQC(votes) } +func appQCForBatch(rng utils.Rng, keys []types.SecretKey, b batch) *types.AppQC { + return testAppQC(keys, appProposalForBatch(rng, b)) +} + +func appProposalForBatch(rng utils.Rng, b batch) *types.AppProposal { + return types.NewAppProposal(b.qc.QC().Proposal(), types.GenAppHash(rng)) +} + func testAppQC(keys []types.SecretKey, proposal *types.AppProposal) *types.AppQC { vote := types.NewAppVote(proposal) votes := make([]*types.Signed[*types.AppVote], 0, len(keys)) diff --git a/sei-db/ledger_db/block/blocksim/block_generator.go b/sei-db/ledger_db/block/blocksim/block_generator.go index 0b13f9ad0e..596e9b78f7 100644 --- a/sei-db/ledger_db/block/blocksim/block_generator.go +++ b/sei-db/ledger_db/block/blocksim/block_generator.go @@ -170,19 +170,11 @@ func (g *BlockGenerator) buildFullCommitQC() (*types.FullCommitQC, []*types.Bloc viewSpec := types.ViewSpec{CommitQC: prev, Epoch: types.NewEpoch(0, types.OpenRoadRange(), genesisTime, committee, 0)} leader := committee.Leader(viewSpec.View()) - appQC := func() utils.Option[*types.AppQC] { - if n := viewSpec.NextGlobalBlock(); n > 0 { - p := types.NewAppProposal(n-1, viewSpec.View().Index, types.AppHash(g.rand.Bytes(hashSizeBytes)), viewSpec.Epoch.EpochIndex()) - return utils.Some(g.fakeAppQC(p)) - } - return utils.None[*types.AppQC]() - }() proposal := utils.OrPanic1(types.NewProposalForTesting( committee, viewSpec, time.Now(), laneQCs, - appQC, g.fakeSig(leader), )) commitVote := types.NewCommitVote(proposal.Proposal().Msg()) diff --git a/sei-db/ledger_db/block/blocksim/blocksim.go b/sei-db/ledger_db/block/blocksim/blocksim.go index 2e74e261bf..513b48c755 100644 --- a/sei-db/ledger_db/block/blocksim/blocksim.go +++ b/sei-db/ledger_db/block/blocksim/blocksim.go @@ -178,29 +178,11 @@ func NewBlockSim( // countExistingState scans the ledger to count the persisted blocks and QCs, // exercising the replay path at startup. func countExistingState(db types.BlockDB) (blocks int, qcs int, err error) { - it, err := db.Iterator(0) + suffix, err := db.ReadSuffix() if err != nil { - return 0, 0, fmt.Errorf("failed to open ledger iterator: %w", err) + return 0, 0, fmt.Errorf("failed to read suffix ledger data: %w", err) } - defer func() { _ = it.Close() }() - for { - pos, ok, err := it.Next() - if err != nil { - return 0, 0, fmt.Errorf("failed to advance ledger iterator: %w", err) - } - if !ok { - break - } - if pos.QC.QC().GlobalRange().First == pos.Number { - // The scan entered a new QC's range. - qcs++ - } - // Presence comes off the position, so counting never reads a block value. - if pos.HasBlock { - blocks++ - } - } - return blocks, qcs, nil + return len(suffix.Blocks), len(suffix.CommitQCs), nil } // recoverResumeState reads the persisted tail so the benchmark resumes appending @@ -218,8 +200,11 @@ func recoverResumeState( prev := tmutils.None[*types.CommitQC]() highest := tmutils.None[uint64]() - status := db.Status() - if status.NextBlock > 0 { + status, ok := db.Status().Get() + if !ok { + return prev, highest, nil + } + if status.NextBlock > status.First { highest = tmutils.Some(uint64(status.NextBlock - 1)) } if status.NextQC > 0 { diff --git a/sei-db/ledger_db/block/littblock/codec.go b/sei-db/ledger_db/block/littblock/codec.go index a536e64061..d106f33cd0 100644 --- a/sei-db/ledger_db/block/littblock/codec.go +++ b/sei-db/ledger_db/block/littblock/codec.go @@ -16,9 +16,13 @@ import ( // - kindBlock 'b' + 8-byte big-endian GlobalBlockNumber (block primary key) // - kindBlockHash 'h' + 32-byte header hash (block hash alias) // - kindQC 'q' + 8-byte big-endian GlobalBlockNumber (QC primary + covered aliases) +// - kindAppQC 'a' + 8-byte big-endian GlobalBlockNumber (AppQC primary + covered aliases) +// - kindAppProp 'p' + 8-byte big-endian GlobalBlockNumber (AppProposal primary + covered aliases) const ( + kindAppQC byte = 'a' kindBlock byte = 'b' kindBlockHash byte = 'h' + kindAppProp byte = 'p' kindQC byte = 'q' ) @@ -32,8 +36,8 @@ func encodeKey(n types.GlobalBlockNumber) []byte { } // decodeKey decodes an 8-byte value produced by encodeKey. -func decodeKey(b []byte) types.GlobalBlockNumber { - return types.GlobalBlockNumber(binary.BigEndian.Uint64(b)) +func decodeKey(b [8]byte) types.GlobalBlockNumber { + return types.GlobalBlockNumber(binary.BigEndian.Uint64(b[:])) } // blockKey returns the primary key under which a block at number n is stored. @@ -53,15 +57,28 @@ func qcKey(n types.GlobalBlockNumber) []byte { return append([]byte{kindQC}, encodeKey(n)...) } +// appQCKey returns the key for AppQC number n — used both for an AppQC's +// primary key and for each covered-number alias. +func appQCKey(n types.GlobalBlockNumber) []byte { + return append([]byte{kindAppQC}, encodeKey(n)...) +} + +// appProposalKey returns the key for AppProposal number n — used both for an +// AppProposal's primary key and for each covered-number alias. +func appProposalKey(n types.GlobalBlockNumber) []byte { + return append([]byte{kindAppProp}, encodeKey(n)...) +} + // keyKind returns the kind prefix byte of a stored key. func keyKind(key []byte) byte { return key[0] } -// decodeNumberKey decodes the GlobalBlockNumber from a kindBlock or kindQC key -// (i.e. a key whose prefix is followed by an 8-byte big-endian number). +// decodeNumberKey decodes the GlobalBlockNumber from a kindBlock, kindQC, or +// kindAppQC key (i.e. a key whose prefix is followed by an 8-byte big-endian +// number). Panics if key is not 9 bytes (1B kind + 8B GlobalBlockNumber). func decodeNumberKey(key []byte) types.GlobalBlockNumber { - return decodeKey(key[1:]) + return decodeKey([8]byte(key[1:])) } // Serialization version for blocks. @@ -70,6 +87,12 @@ const blockSerializationVersion byte = 1 // Serialization version for QCs. const qcSerializationVersion byte = 1 +// Serialization version for AppQCs. +const appQCSerializationVersion byte = 1 + +// Serialization version for AppProposals. +const appProposalSerializationVersion byte = 1 + // blockValuePrefixLen is the fixed header preceding a block's proto bytes: one // version byte followed by the 8-byte big-endian GlobalBlockNumber. const blockValuePrefixLen = 1 + 8 @@ -124,8 +147,8 @@ func encodeQC(qc *types.FullCommitQC) []byte { // so deriving from the encoded fields is what makes a QC's range identical // before and after a round trip through the table. func coveredRange(qc *types.FullCommitQC) (types.GlobalBlockNumber, types.GlobalBlockNumber) { - first := qc.QC().GlobalRange().First - return first, first + types.GlobalBlockNumber(len(qc.Headers())) + gr := qc.QC().GlobalRange() + return gr.First, gr.Next } // decodeQC unmarshals a FullCommitQC from the value produced by encodeQC. @@ -142,3 +165,54 @@ func decodeQC(value []byte) (*types.FullCommitQC, error) { } return qc, nil } + +// encodeAppProposal marshals an AppProposal to the bytes stored as its table +// value, framed as [version:1][proto(AppProposal)]. +func encodeAppProposal(appProposal *types.AppProposal) []byte { + proto := types.AppProposalConv.Marshal(appProposal) + value := make([]byte, 0, 1+len(proto)) + value = append(value, appProposalSerializationVersion) + value = append(value, proto...) + return value +} + +// decodeAppProposal unmarshals an AppProposal from the value produced by +// encodeAppProposal. +func decodeAppProposal(value []byte) (*types.AppProposal, error) { + if len(value) < 1 { + return nil, fmt.Errorf("appProposal value too short: %d bytes", len(value)) + } + if value[0] != appProposalSerializationVersion { + return nil, fmt.Errorf("unsupported appProposal serialization version %d", value[0]) + } + appProposal, err := types.AppProposalConv.Unmarshal(value[1:]) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal appProposal: %w", err) + } + return appProposal, nil +} + +// encodeAppQC marshals an AppQC to the bytes stored as its table value, +// framed as [version:1][proto(AppQC)]. +func encodeAppQC(appQC *types.AppQC) []byte { + proto := types.AppQCConv.Marshal(appQC) + value := make([]byte, 0, 1+len(proto)) + value = append(value, appQCSerializationVersion) + value = append(value, proto...) + return value +} + +// decodeAppQC unmarshals an AppQC from the value produced by encodeAppQC. +func decodeAppQC(value []byte) (*types.AppQC, error) { + if len(value) < 1 { + return nil, fmt.Errorf("appQC value too short: %d bytes", len(value)) + } + if value[0] != appQCSerializationVersion { + return nil, fmt.Errorf("unsupported appQC serialization version %d", value[0]) + } + appQC, err := types.AppQCConv.Unmarshal(value[1:]) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal appQC: %w", err) + } + return appQC, nil +} diff --git a/sei-db/ledger_db/block/littblock/codec_test.go b/sei-db/ledger_db/block/littblock/codec_test.go index 3c5aff9eca..5eee568cfe 100644 --- a/sei-db/ledger_db/block/littblock/codec_test.go +++ b/sei-db/ledger_db/block/littblock/codec_test.go @@ -23,7 +23,7 @@ func TestKeyRoundTrip(t *testing.T) { for _, n := range cases { key := encodeKey(n) require.Len(t, key, 8, "key must be 8 bytes") - require.Equal(t, n, decodeKey(key)) + require.Equal(t, n, decodeKey([8]byte(key))) } } diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index 68cf7af46a..d19c3a0dc5 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "slices" "sync" "sync/atomic" @@ -34,8 +35,8 @@ type blockDB struct { // watermark is a retention floor, always a QC boundary (a GlobalRange().First): // PruneBefore rounds a requested prune point down to the start of the cohort - // containing it, and startup re-derives it as the lowest surviving QC's First - // (see cohortStart and recoverReadFloors). Keeping it on a cohort boundary + // containing it, and startup re-derives it as the lowest surviving FullCommitQC's First + // (see clampPruneBoundary and recoverWatermark). Keeping it on a cohort boundary // is what makes a QC's blocks change readability atomically — the gate never // splits a cohort. // @@ -48,36 +49,10 @@ type blockDB struct { // goroutine, so accessed atomically. watermark atomic.Uint64 - // Write-order cursors (see types.BlockDB contract). Guarded by mu. - mu sync.Mutex - hasBlocks bool - lastBlockNumber types.GlobalBlockNumber - hasQC bool - lastQCNext types.GlobalBlockNumber - - // latestQCStartBlock is the most recently written QC's starting block number. - latestQCStartBlock types.GlobalBlockNumber - - // firstBlockNumber is the lowest block number this handle has seen. Iterator clamps its - // start up to it so a scan always opens on a block that exists: the first block may be - // written anywhere inside its covering QC, so this can sit above oldestQCStart with no - // block in between. Set when the first block is written and re-derived on open (see - // recoverReadFloors). - // - // Like oldestQCStart it is a floor, not an exact value — pruning may reclaim the block it - // names — but by then watermark has advanced past it and Iterator clamps to both. - // Meaningful only while hasBlocks. - firstBlockNumber types.GlobalBlockNumber - - // oldestQCStart is where the oldest QC this handle has seen begins. Iterator clamps its - // start up to it, which is what lets the positioned lookup always land on a retained QC - // record: a start below every QC's range has no key to position at. Set when the first QC - // is written and re-derived on open (see recoverReadFloors). - // - // It is a floor, not an exact value — GC may later reclaim that QC — but by then - // PruneBefore has advanced watermark past it, and Iterator clamps to both. Meaningful - // only while hasQC. - oldestQCStart types.GlobalBlockNumber + // status is the explicit write-order/recovery suffix cursor (see + // types.BlockDB contract). None means the DB is empty. Guarded by mu. + mu sync.Mutex + status utils.Option[types.SuffixRange] } // NewBlockDB opens (or creates) a LittDB-backed types.BlockDB from config. The @@ -117,17 +92,17 @@ func NewBlockDB(config *BlockDBConfig) (types.BlockDB, error) { _ = db.Close() return nil, fmt.Errorf("failed to build ledger table: %w", err) } - s.table = table - - if err := s.recoverCursors(); err != nil { + if err := s.recoverWatermark(); err != nil { _ = db.Close() - return nil, fmt.Errorf("failed to recover write cursors: %w", err) + return nil, fmt.Errorf("failed to recover watermark: %w", err) } - if err := s.recoverReadFloors(); err != nil { + suffix, err := s.ReadSuffix() + if err != nil { _ = db.Close() - return nil, fmt.Errorf("failed to recover read floors: %w", err) + return nil, fmt.Errorf("ReadSuffix(): %w", err) } + s.status = suffix.Status return s, nil } @@ -154,74 +129,21 @@ func refuseLegacyTable(paths []string) error { return nil } -// recoverCursors reloads the write-order cursors (lastBlockNumber, lastQCNext, -// and their presence flags) from on-disk state. Without this, a reopened DB -// would treat itself as empty and let WriteBlock/WriteQC silently accept -// out-of-order or non-contiguous writes that overwrite or gap persisted data. -func (s *blockDB) recoverCursors() error { - it, err := s.table.Iterator(true) - if err != nil { - return fmt.Errorf("failed to open recovery iterator: %w", err) - } - defer func() { _ = it.Close() }() - - for !s.hasBlocks || !s.hasQC { - ok, err := it.Next() - if err != nil { - return fmt.Errorf("failed to advance recovery iterator: %w", err) - } - if !ok { - break - } - key, isPrimary, err := it.GetKey() - if err != nil { - return fmt.Errorf("failed to read recovery key: %w", err) - } - if !isPrimary { - continue - } - switch keyKind(key) { - case kindBlock: - if !s.hasBlocks { - s.lastBlockNumber = decodeNumberKey(key) - s.hasBlocks = true - } - case kindQC: - if !s.hasQC { - value, err := it.GetValue() - if err != nil { - return fmt.Errorf("failed to read newest qc value: %w", err) - } - qc, err := decodeQC(value) - if err != nil { - return fmt.Errorf("failed to unmarshal newest qc: %w", err) - } - s.latestQCStartBlock, s.lastQCNext = coveredRange(qc) - s.hasQC = true - } - } - } - return nil -} - -// recoverReadFloors re-derives the read floors on open: the watermark (with oldestQCStart) from -// the oldest surviving QC, and firstBlockNumber from the oldest surviving block. Both are -// in-memory only, so a restart forgets every PruneBefore. That is fine for reclamation (nothing -// new is deleted), but we must protect against showing un-pruned blocks with pruned QCs. +// recoverWatermark re-derives the read watermark on open from the oldest +// surviving FullCommitQC. It is in-memory only, so a restart forgets every PruneBefore. +// That is fine for reclamation (nothing new is deleted), but we must protect +// against showing un-pruned blocks with pruned QCs. // -// One forward pass serves both. QCs are written before the blocks they cover, so the oldest -// surviving record is normally a QC and the first block follows shortly after. The block search -// is skipped when the store holds no blocks — hasBlocks comes from recoverCursors, which runs -// first — so a QC-only store does not walk the whole table looking for a block that is not there. -func (s *blockDB) recoverReadFloors() error { +// FullCommitQCs are written before any other data for the same index. +// If no FullCommitQC survives but the table is non-empty, the store is corrupt and must not reopen. +func (s *blockDB) recoverWatermark() error { it, err := s.table.Iterator(false) if err != nil { return fmt.Errorf("failed to open read floor recovery iterator: %w", err) } defer func() { _ = it.Close() }() - - needQC, needBlock := true, s.hasBlocks - for needQC || needBlock { + empty := true + for { ok, err := it.Next() if err != nil { return fmt.Errorf("failed to advance read floor recovery iterator: %w", err) @@ -229,6 +151,7 @@ func (s *blockDB) recoverReadFloors() error { if !ok { break } + empty = false key, isPrimary, err := it.GetKey() if err != nil { return fmt.Errorf("failed to read read floor recovery key: %w", err) @@ -236,29 +159,15 @@ func (s *blockDB) recoverReadFloors() error { if !isPrimary { continue } - switch keyKind(key) { - case kindQC: - if needQC { - oldest := decodeNumberKey(key) - s.watermark.Store(uint64(oldest)) - s.oldestQCStart = oldest - needQC = false - } - case kindBlock: - if needBlock { - s.firstBlockNumber = decodeNumberKey(key) - needBlock = false - } + if keyKind(key) != kindQC { + continue } + s.watermark.Store(uint64(decodeNumberKey(key))) + return nil } - - if needQC && s.hasBlocks { - // No QC survives. The never-empty prune invariant guarantees at least one - // (block, QC) pair is always retained, so blocks-without-QC is unreachable - // through normal operation — it means the store is corrupt (e.g. a QC WAL - // file was removed out of band). Refuse to open rather than serve blocks we - // can no longer trust. - return fmt.Errorf("corrupt store: newest block %d has no surviving QC covering it", s.lastBlockNumber) + // No FullCommitQC survives. Refuse to open rather than serve blocks we can no longer trust. + if !empty { + return fmt.Errorf("corrupt store: no QC in non-empty store") } return nil } @@ -266,17 +175,17 @@ func (s *blockDB) recoverReadFloors() error { func (s *blockDB) WriteBlock(n types.GlobalBlockNumber, blk *types.Block) error { s.mu.Lock() defer s.mu.Unlock() - if s.hasBlocks && n != s.lastBlockNumber+1 { - return fmt.Errorf("block number %d not contiguous with last written %d: %w", - n, s.lastBlockNumber, types.ErrBlockOutOfOrder) - } + status, ok := s.status.Get() // A covering QC must already be written. Since QCs are contiguous and blocks - // strictly ascending, n is covered iff n < lastQCNext. This guard also fixes + // strictly ascending, n is covered iff n < status.NextQC. This guard also fixes // the QC-before-block write order: the covering QC's Put has already issued - // under this mutex, so on a crash a surviving block implies a surviving QC. - if !s.hasQC || n >= s.lastQCNext { - return fmt.Errorf("block number %d not covered by any written QC (next QC bound %d): %w", - n, s.lastQCNext, types.ErrBlockMissingQC) + // under this mutex, so on a crash a surviving block implies a surviving FullCommitQC. + if !ok || n >= status.NextQC { + return fmt.Errorf("block number %d not covered by any written QC: %w", n, types.ErrBlockMissingQC) + } + if n != status.NextBlock { + return fmt.Errorf("block number %d not contiguous with last written %d: %w", + n, status.NextBlock-1, types.ErrBlockOutOfOrder) } value := encodeBlock(n, blk) @@ -289,49 +198,125 @@ func (s *blockDB) WriteBlock(n types.GlobalBlockNumber, blk *types.Block) error if err := s.table.Put(blockKey(n), value, hashAlias); err != nil { return fmt.Errorf("failed to put block %d: %w", n, err) } - - if !s.hasBlocks { - s.firstBlockNumber = n - } - s.lastBlockNumber = n - s.hasBlocks = true + status.NextBlock = n + 1 + s.status = utils.Some(status) return nil } func (s *blockDB) WriteQC(qc *types.FullCommitQC) error { - first, next := coveredRange(qc) - if first >= next { - return fmt.Errorf("QC at %d covers no blocks: %w", first, types.ErrQCNonContiguous) + gr := qc.QC().GlobalRange() + if gr.Len() == 0 { + return fmt.Errorf("QC at %d covers no blocks: %w", gr.First, types.ErrQCNonContiguous) } s.mu.Lock() defer s.mu.Unlock() - if s.hasQC && first != s.lastQCNext { + status, ok := s.status.Get() + if ok && status.NextQC != gr.First { return fmt.Errorf("QC starts at %d, expected %d: %w", - first, s.lastQCNext, types.ErrQCNonContiguous) + gr.First, status.NextQC, types.ErrQCNonContiguous) } value := encodeQC(qc) var aliases []*litttypes.SecondaryKey - for m := first + 1; m < next; m++ { + for m := gr.First + 1; m < gr.Next; m++ { aliases = append(aliases, &litttypes.SecondaryKey{ Key: qcKey(m), Offset: 0, Length: uint32(len(value)), //nolint:gosec // value length fits u32 (litt value cap is 2^32) }) } - if err := s.table.Put(qcKey(first), value, aliases...); err != nil { - return fmt.Errorf("failed to put QC [%d,%d): %w", first, next, err) + if err := s.table.Put(qcKey(gr.First), value, aliases...); err != nil { + return fmt.Errorf("failed to put QC [%d,%d): %w", gr.First, gr.Next, err) } - if !s.hasQC { + if !ok { // The first QC may start anywhere its caller allows, and nothing below it will ever // be written. Record where coverage begins so Iterator can clamp to it without // discovering it by scanning; a reopen re-derives the same value. - s.oldestQCStart = first + status = types.SuffixRange{ + First: gr.First, + NextAppQC: gr.First, + NextAppProposal: gr.First, + NextBlock: gr.First, + NextQC: gr.Next, + } + } else { + status.NextQC = gr.Next } - s.latestQCStartBlock = first - s.lastQCNext = next - s.hasQC = true + s.status = utils.Some(status) + return nil +} + +func (s *blockDB) WriteAppQC(appQC *types.AppQC) error { + gr := appQC.Proposal().GlobalRange() + if gr.Len() == 0 { + return fmt.Errorf("AppQC at %d covers no blocks: %w", gr.First, types.ErrAppQCNonContiguous) + } + s.mu.Lock() + defer s.mu.Unlock() + status, ok := s.status.Get() + if !ok { + return fmt.Errorf("AppQC [%d,%d) has no matching QC: %w", gr.First, gr.Next, types.ErrAppQCMissingQC) + } + if status.NextAppQC != gr.First { + return fmt.Errorf("AppQC starts at %d, expected %d: %w", + gr.First, status.NextAppQC, types.ErrAppQCNonContiguous) + } + if gr.Next > status.NextAppProposal { + return fmt.Errorf("AppQC [%d,%d) is not covered by written AppProposals: %w", gr.First, gr.Next, types.ErrAppQCMissingQC) + } + + value := encodeAppQC(appQC) + var aliases []*litttypes.SecondaryKey + for m := gr.First + 1; m < gr.Next; m++ { + aliases = append(aliases, &litttypes.SecondaryKey{ + Key: appQCKey(m), + Offset: 0, + Length: uint32(len(value)), //nolint:gosec // value length fits u32 (litt value cap is 2^32) + }) + } + if err := s.table.Put(appQCKey(gr.First), value, aliases...); err != nil { + return fmt.Errorf("failed to put AppQC [%d,%d): %w", gr.First, gr.Next, err) + } + status.NextAppQC = gr.Next + status.First = status.NextAppQC - 1 + s.status = utils.Some(status) + return nil +} + +func (s *blockDB) WriteAppProposal(appProposal *types.AppProposal) error { + gr := appProposal.GlobalRange() + if gr.Len() == 0 { + return fmt.Errorf("AppProposal at %d covers no blocks: %w", gr.First, types.ErrAppProposalNonContiguous) + } + s.mu.Lock() + defer s.mu.Unlock() + status, ok := s.status.Get() + if !ok { + return fmt.Errorf("AppProposal [%d,%d) has no matching QC: %w", gr.First, gr.Next, types.ErrAppProposalMissingQC) + } + if status.NextAppProposal != gr.First { + return fmt.Errorf("AppProposal starts at %d, expected %d: %w", + gr.First, status.NextAppProposal, types.ErrAppProposalNonContiguous) + } + if gr.Next > status.NextBlock { + return fmt.Errorf("AppProposal [%d,%d) is not covered by written blocks: %w", gr.First, gr.Next, types.ErrAppProposalMissingQC) + } + value := encodeAppProposal(appProposal) + var aliases []*litttypes.SecondaryKey + for m := gr.First + 1; m < gr.Next; m++ { + aliases = append(aliases, &litttypes.SecondaryKey{ + Key: appProposalKey(m), + Offset: 0, + Length: uint32(len(value)), //nolint:gosec // value length fits u32 (litt value cap is 2^32) + }) + } + if err := s.table.Put(appProposalKey(gr.First), value, aliases...); err != nil { + return fmt.Errorf("failed to put AppProposal [%d,%d): %w", gr.First, gr.Next, err) + } + + status.NextAppProposal = gr.Next + s.status = utils.Some(status) return nil } @@ -339,18 +324,15 @@ func (s *blockDB) PruneBefore(blockHeight types.GlobalBlockNumber) error { s.mu.Lock() defer s.mu.Unlock() - if !s.hasBlocks { + status, ok := s.status.Get() + if !ok { // Ignore prune requests if we've not got any data yet. Simplifies several edge cases // and is technically a legal implementation of the contract in the godocs. return nil } - if ceiling := min(s.latestQCStartBlock, s.lastBlockNumber); blockHeight > ceiling { - blockHeight = ceiling - } - // Round the watermark down to the start of a QC's range, to avoid pruning a QC before its blocks. - blockHeight, err := s.clampPruneBoundary(blockHeight) + blockHeight, err := s.clampPruneBoundary(min(blockHeight, status.First)) if err != nil { return err } @@ -364,6 +346,10 @@ func (s *blockDB) PruneBefore(blockHeight types.GlobalBlockNumber) error { return nil } +func (s *blockDB) First() types.GlobalBlockNumber { + return types.GlobalBlockNumber(s.watermark.Load()) +} + // clampPruneBoundary returns the start of the QC that covers n, or n if there is no QC covering N // (which can happen if you prune the same n twice). func (s *blockDB) clampPruneBoundary(blockHeight types.GlobalBlockNumber) (types.GlobalBlockNumber, error) { @@ -386,16 +372,16 @@ func (s *blockDB) clampPruneBoundary(blockHeight types.GlobalBlockNumber) (types // // - block-number keys are reclaimable once the block number is strictly below // the prune watermark; -// - QC keys (the primary First and every per-covered-number secondary) are +// - QC, AppProposal, and AppQC keys (the primary First and every per-covered-number secondary) are // reclaimable once their number is below the watermark, so a QC's segment is // reclaimable only once its highest covered number (Next-1) is below the -// watermark — i.e. once Next <= watermark; a QC straddling the watermark is -// retained; +// watermark — i.e. once Next <= watermark; a QC/AppProposal/AppQC straddling the +// watermark is retained; // - header-hash aliases share their block's segment, so they always pass — the // block's primary number key is what actually gates segment reclamation. func (s *blockDB) gcFilter(key []byte, _ bool) (bool, error) { switch keyKind(key) { - case kindBlock, kindQC: + case kindBlock, kindQC, kindAppProp, kindAppQC: return uint64(decodeNumberKey(key)) < s.watermark.Load(), nil case kindBlockHash: return true, nil @@ -411,94 +397,130 @@ func (s *blockDB) Flush() error { return nil } -func (s *blockDB) Status() types.DBStatus { +func (s *blockDB) Status() utils.Option[types.SuffixRange] { s.mu.Lock() defer s.mu.Unlock() - var tips types.DBStatus - if s.hasBlocks { - tips.NextBlock = s.lastBlockNumber + 1 - } - if s.hasQC { - tips.NextQC = s.lastQCNext - } - return tips + return s.status } -func (s *blockDB) Iterator(n types.GlobalBlockNumber) (types.BlockDBIterator, error) { - // One consistent read of the cursors. The watermark stays on its atomic because the GC goroutine - // writes it, but everything else is taken together so the floors below cannot disagree about - // which instant they describe. +// ReadSuffix reads the materialized startup-recovery suffix. +func (s *blockDB) ReadSuffix() (types.Suffix, error) { + // Suffix is computed under lock, so that GC cannot malform it: + // locked => no new data can be appended => watermark doesn't move => GC doesn't consume data of the suffix. s.mu.Lock() - hasQC, nextQC, oldestQCStart := s.hasQC, s.lastQCNext, s.oldestQCStart - hasBlocks, firstBlock := s.hasBlocks, s.firstBlockNumber - s.mu.Unlock() - watermark := types.GlobalBlockNumber(s.watermark.Load()) - - if !hasQC { - // An empty store covers nothing. - return &blockDBIterator{}, nil - } - - // Clamp up to the lowest number this store can serve. The watermark is the retention gate and - // oldestQCStart is where coverage begins on a store that never had data below it (bootstrapped - // mid-chain); either may be the higher. - start := max(n, watermark, oldestQCStart) - - if !hasBlocks { - // No block has ever been written, so there is no block to open on and the QC floor governs. - // This is the one case blockDBIterator cannot serve safely — see simpleIterator. - if start >= nextQC { - return &blockDBIterator{}, nil - } - return newSimpleIterator(s.table, start, nextQC) - } - - // firstBlock is where the block history begins, which can sit inside its covering QC's range - // because the first block is free to start there. Clamping to it is what makes the scan open on a - // block that exists rather than on blockless numbers below it. - start = max(start, firstBlock) - - if start >= nextQC { - // Nothing is covered at or above start. - return &blockDBIterator{}, nil - } - - // A QC is stored under its First as the primary key with a covered-number alias for every - // other number in its range, and an alias carries the full QC value. Positioning the scan at - // qcKey(start) therefore lands on the covering QC no matter where start falls in its range. - it, found, err := s.table.IteratorAt(qcKey(start), false) + defer s.mu.Unlock() + it, err := s.table.Iterator(true) if err != nil { - return nil, fmt.Errorf("failed to open iterator at %d: %w", start, err) - } - if !found { - // start was clamped into [oldestQCStart, NextQC) and retained QCs cover that interval - // contiguously, so some QC's primary or covered-number alias is stored under qcKey(start) - // — unless the retention floor moved past start while we were positioning. Either way we - // refuse rather than scanning from the beginning of the table, which would read exactly - // the history the caller passed n to skip. Which of the two it was decides the diagnosis, - // and getting that wrong is expensive: reporting corruption on a healthy store sends an - // operator hunting for damage that isn't there. - if s.watermark.Load() > uint64(start) { - // A concurrent PruneBefore advanced the floor past start, and GC reclaimed the record - // (litt surfaces a prune/GC boundary as not-found — see DiskTable.IteratorAt). GC's - // filter only clears a segment once the watermark exceeds every number in it, so a - // watermark above start is exactly the condition that makes this benign rather than - // corrupt. Racing a pruner has no deterministic answer, so report the floor honestly - // and let the caller decide whether to retry. - return nil, fmt.Errorf("%w: start %d fell below the retention floor while positioning", - types.ErrPruned, start) + return types.Suffix{}, fmt.Errorf("failed to open suffix iterator: %w", err) + } + defer func() { _ = it.Close() }() + var suffix types.Suffix + var status types.SuffixRange + var oldestQC *types.FullCommitQC + var gotBlock, gotQC, gotAppProposal, gotAppQC bool + for !gotAppQC || !gotQC || status.NextAppQC <= oldestQC.QC().GlobalRange().Next { + ok, err := it.Next() + if err != nil { + return types.Suffix{}, fmt.Errorf("failed to advance suffix iterator: %w", err) + } + if !ok { + break + } + key, isPrimary, err := it.GetKey() + if err != nil { + return types.Suffix{}, fmt.Errorf("failed to read suffix key: %w", err) + } + if !isPrimary { + continue + } + value, err := it.GetValue() + if err != nil { + return types.Suffix{}, fmt.Errorf("failed to read suffix value: %w", err) + } + switch keyKind(key) { + case kindBlock: + n, block, err := decodeBlock(value) + if err != nil { + return types.Suffix{}, fmt.Errorf("failed to decode suffix block: %w", err) + } + if !gotBlock { + status.NextBlock = n + 1 + gotBlock = true + } + suffix.Blocks = append(suffix.Blocks, types.SuffixBlock{Number: n, Block: block}) + case kindAppQC: + appQC, err := decodeAppQC(value) + if err != nil { + return types.Suffix{}, fmt.Errorf("failed to decode suffix AppQC: %w", err) + } + gr := appQC.Proposal().GlobalRange() + if !gotAppQC { + status.NextAppQC = gr.Next + status.First = gr.Next - 1 + gotAppQC = true + } + suffix.AppQCs = append(suffix.AppQCs, appQC) + case kindAppProp: + appProposal, err := decodeAppProposal(value) + if err != nil { + return types.Suffix{}, fmt.Errorf("failed to decode suffix AppProposal: %w", err) + } + gr := appProposal.GlobalRange() + if !gotAppProposal { + status.NextAppProposal = gr.Next + gotAppProposal = true + } + suffix.AppProposals = append(suffix.AppProposals, appProposal) + case kindQC: + qc, err := decodeQC(value) + if err != nil { + return types.Suffix{}, fmt.Errorf("failed to decode suffix CommitQC: %w", err) + } + gr := qc.QC().GlobalRange() + if !gotQC { + status.NextQC = gr.Next + gotQC = true + } + oldestQC = qc + suffix.CommitQCs = append(suffix.CommitQCs, qc) } - return nil, fmt.Errorf("corrupt store: no QC record at %d despite coverage to %d", start, nextQC) } - return &blockDBIterator{ - it: it, - startN: start, - expectStartQC: true, - }, nil + if !gotQC { + // Empty db. + return types.Suffix{}, nil + } + // Set fields for missing resources. + first := oldestQC.QC().GlobalRange().First + status.NextQC = max(status.NextQC, first) + status.NextBlock = max(status.NextBlock, first) + status.NextAppProposal = max(status.NextAppProposal, first) + status.NextAppQC = max(status.NextAppQC, first) + status.First = max(status.First, first) + suffix.Status = utils.Some(status) + + // Prune resources fully below status.First. + suffix.CommitQCs = slices.DeleteFunc(suffix.CommitQCs, func(qc *types.FullCommitQC) bool { + return qc.QC().GlobalRange().Next <= status.First + }) + suffix.Blocks = slices.DeleteFunc(suffix.Blocks, func(block types.SuffixBlock) bool { + return block.Number < status.First + }) + suffix.AppProposals = slices.DeleteFunc(suffix.AppProposals, func(appProposal *types.AppProposal) bool { + return appProposal.GlobalRange().Next <= status.First + }) + suffix.AppQCs = slices.DeleteFunc(suffix.AppQCs, func(appQC *types.AppQC) bool { + return appQC.Proposal().GlobalRange().Next <= status.First + }) + // Put resources in increasing order. + slices.Reverse(suffix.CommitQCs) + slices.Reverse(suffix.Blocks) + slices.Reverse(suffix.AppProposals) + slices.Reverse(suffix.AppQCs) + return suffix, nil } func (s *blockDB) ReadBlockByNumber(n types.GlobalBlockNumber) (utils.Option[*types.Block], error) { - // Refuse below-watermark blocks: they may be stranded (covering QC reclaimed). + // Data below watermark should not be visible to the caller, even though it is pruned asynchronously. if uint64(n) < s.watermark.Load() { return utils.None[*types.Block](), types.ErrPruned } @@ -517,8 +539,8 @@ func (s *blockDB) ReadBlockByHash(hash types.BlockHeaderHash) (utils.Option[type if err != nil { return utils.None[types.BlockWithNumber](), err } - // The number is not known until the block is resolved; refuse it if it turns - // out to be below the watermark (potentially stranded from its covering QC). + // The number is not known until the block is resolved; + // Data below watermark should not be visible to the caller, even though it is pruned asynchronously. if bwn, ok := result.Get(); ok && uint64(bwn.Number) < s.watermark.Load() { return utils.None[types.BlockWithNumber](), nil } @@ -561,6 +583,46 @@ func (s *blockDB) ReadQCByBlockNumber( return utils.Some(qc), nil } +func (s *blockDB) ReadAppProposalByBlockNumber( + n types.GlobalBlockNumber, +) (utils.Option[*types.AppProposal], error) { + if uint64(n) < s.watermark.Load() { + return utils.None[*types.AppProposal](), types.ErrPruned + } + value, exists, err := s.table.Get(appProposalKey(n)) + if err != nil { + return utils.None[*types.AppProposal](), fmt.Errorf("failed to read AppProposal: %w", err) + } + if !exists { + return utils.None[*types.AppProposal](), nil + } + appProposal, err := decodeAppProposal(value) + if err != nil { + return utils.None[*types.AppProposal](), fmt.Errorf("failed to unmarshal AppProposal: %w", err) + } + return utils.Some(appProposal), nil +} + +func (s *blockDB) ReadAppQCByBlockNumber( + n types.GlobalBlockNumber, +) (utils.Option[*types.AppQC], error) { + if uint64(n) < s.watermark.Load() { + return utils.None[*types.AppQC](), types.ErrPruned + } + value, exists, err := s.table.Get(appQCKey(n)) + if err != nil { + return utils.None[*types.AppQC](), fmt.Errorf("failed to read AppQC: %w", err) + } + if !exists { + return utils.None[*types.AppQC](), nil + } + appQC, err := decodeAppQC(value) + if err != nil { + return utils.None[*types.AppQC](), fmt.Errorf("failed to unmarshal AppQC: %w", err) + } + return utils.Some(appQC), nil +} + func (s *blockDB) Close() error { if err := s.db.Close(); err != nil { return fmt.Errorf("failed to close litt db: %w", err) diff --git a/sei-db/ledger_db/block/littblock/litt_block_gc.go b/sei-db/ledger_db/block/littblock/litt_block_gc.go index ff48ba9c32..a7f5769466 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_gc.go +++ b/sei-db/ledger_db/block/littblock/litt_block_gc.go @@ -49,8 +49,8 @@ func (s *blockDB) GetRollbackFloor(rollbackWindow uint64) uint64 { func (s *blockDB) GetLatestBlock() (uint64, error) { s.mu.Lock() defer s.mu.Unlock() - if !s.hasBlocks { - return 0, nil + if status, ok := s.status.Get(); ok && types.GlobalBlockNumber(s.watermark.Load()) < status.NextBlock { + return uint64(status.NextBlock - 1), nil } - return uint64(s.lastBlockNumber), nil + return 0, nil } diff --git a/sei-db/ledger_db/block/littblock/litt_block_iterator.go b/sei-db/ledger_db/block/littblock/litt_block_iterator.go deleted file mode 100644 index bc7629e84c..0000000000 --- a/sei-db/ledger_db/block/littblock/litt_block_iterator.go +++ /dev/null @@ -1,281 +0,0 @@ -package littblock - -import ( - "fmt" - - littdb "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt" - "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" - "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" -) - -var _ types.BlockDBIterator = (*blockDBIterator)(nil) - -// coveredQC is a decoded QC together with the half-open number range it covers. -type coveredQC struct { - // qc is the decoded QC. - qc *types.FullCommitQC - - // first is the lowest number the QC covers (its GlobalRange().First). - first types.GlobalBlockNumber - - // next is one past the highest number the QC covers (its GlobalRange().Next). - next types.GlobalBlockNumber -} - -// blockDBIterator implements types.BlockDBIterator over the shared ledger table. -// -// It steps through consecutive block numbers, driven by a single forward litt scan. The scan -// visits records in insertion order, and the write path inserts every QC before the blocks it -// covers, so by the time the number cursor reaches n the covering QC has always been decoded -// already: the underlying cursor is advanced lazily, collecting QC records into a small pending -// queue and holding position on each block record until the number cursor consumes it. -// -// A nil it represents an empty iterator (produced by Iterator when the requested start is past -// the persisted coverage): Next reports exhaustion immediately and Close is a no-op. -type blockDBIterator struct { - // it is the underlying litt scan; nil for an empty iterator. - it littdb.Iterator - - // startN is the first number the iterator may yield, and doubles as the retention floor: - // blockDB.Iterator clamps it up to the prune watermark and to the start of the block history, - // so a block below startN is either below the start or stranded from a reclaimed QC, and is - // skipped either way. Because of the block-history floor, startN names a block that exists - // whenever the store holds one — it may sit inside its covering QC's range rather than on that - // range's first number. - startN types.GlobalBlockNumber - - // expectStartQC is one-shot state, not a mode: every non-empty iterator is positioned at - // qcKey(startN), so the scan's very first record is the covering QC's primary or - // covered-number alias rather than a record the normal dispatch handles. Cleared once that - // record is consumed. - expectStartQC bool - - // started is false until the first Next call establishes the start position. - started bool - - // positioned is true only while the iterator sits on a number Next yielded. Block - // rejects calls made when it is false. - positioned bool - - // n is the current number; valid while positioned (after Next has returned true). - n types.GlobalBlockNumber - - // current is the QC covering n; nil only before the first Next call. - current *coveredQC - - // pending holds QCs the scan has passed whose ranges begin at or above current.next. QCs are - // written ahead of the blocks they cover, so several can precede the block records that - // consume them. Ranges are ascending and contiguous with current. - pending []*coveredQC - - // heldBlock is true when the underlying cursor is positioned on a block record the number - // cursor has not consumed yet; heldNumber is that record's number. - heldBlock bool - - // heldNumber is the number of the held block record; meaningful only while heldBlock is true. - heldNumber types.GlobalBlockNumber - - // exhausted is true once the underlying scan has no more records. - exhausted bool - - // closed is true once Close has been called. Next and Block reject calls made afterward. - // Checking it explicitly is what makes that rejection uniform: on a trailing block-less position - // the scan holds no record, so Next would otherwise skip fill() entirely and hand back a fresh - // position without ever touching the closed cursor. - closed bool -} - -func (l *blockDBIterator) Next() (types.Position, bool, error) { - // Any exit other than a yielded position leaves the iterator unpositioned, so a - // subsequent Block() reports misuse rather than answering for a stale position. - l.positioned = false - - if l.closed || l.it == nil { - return types.Position{}, false, nil - } - - var next types.GlobalBlockNumber - if l.started { - next = l.n + 1 - if l.heldBlock && l.heldNumber == l.n { - // Leaving a position whose block record the scan still holds; release it so the - // scan can advance past it. - l.heldBlock = false - } - } else { - // The start position needs the first covering QC in hand before a number can be yielded. - if err := l.fill(); err != nil { - return types.Position{}, false, err - } - if l.current == nil { - // Unreachable: the scan is positioned at qcKey(startN), so its first record is the - // covering QC — fill dispatches it through expectStartQC, which errors if it is not - // a QC record, and collectQC always adopts it as current because its range contains - // startN. Asserted rather than dereferenced blindly below. - return types.Position{}, false, - fmt.Errorf("ledger scan at %d established no covering QC", l.startN) - } - // current's range contains startN (the scan was positioned inside it), so no clamp - // up to current.first is needed. - next = l.startN - } - - // Establish coverage for next, promoting across QC range boundaries. - for next >= l.current.next { - if len(l.pending) > 0 { - l.current = l.pending[0] - l.pending = l.pending[1:] - continue - } - if l.heldBlock { - // A block record at heldNumber >= next is waiting, but no QC covers next. The write - // path guarantees a covering QC precedes every block, so this is corruption. - return types.Position{}, false, - fmt.Errorf("corrupt store: block %d has no QC coverage", l.heldNumber) - } - if l.exhausted { - return types.Position{}, false, nil - } - if err := l.fill(); err != nil { - return types.Position{}, false, err - } - } - - // Position the scan to answer Block() at next: it must hold the next block record (if any - // remains) so presence is decidable. - if !l.heldBlock && !l.exhausted { - if err := l.fill(); err != nil { - return types.Position{}, false, err - } - } - if l.heldBlock && l.heldNumber != next { - // Blocks are written densely, so a missing number below the highest persisted block can - // only be corruption. - return types.Position{}, false, - fmt.Errorf("%w: corrupt store: block gap at %d (next persisted block is %d)", - types.ErrBlockGap, next, l.heldNumber) - } - - l.n = next - l.started = true - l.positioned = true - return types.Position{Number: next, QC: l.current.qc, HasBlock: l.heldBlock}, true, nil -} - -// fill advances the underlying scan until it holds an unconsumed block record or exhausts, -// decoding every QC record it passes into the covering-QC state. -func (l *blockDBIterator) fill() error { - for !l.heldBlock && !l.exhausted { - ok, err := l.it.Next() - if err != nil { - return fmt.Errorf("failed to advance ledger scan: %w", err) - } - if !ok { - l.exhausted = true - return nil - } - key, isPrimary, err := l.it.GetKey() - if err != nil { - return fmt.Errorf("failed to read ledger key: %w", err) - } - switch { - case l.expectStartQC: - // The scan was positioned at qcKey(start): the first record is the covering - // QC — its primary when start is a range's First, otherwise a covered-number - // alias, whose value is the same full QC either way. - l.expectStartQC = false - if keyKind(key) != kindQC { - return fmt.Errorf("ledger scan positioned at %d is not on a QC record", l.startN) - } - if err := l.collectQC(); err != nil { - return err - } - case !isPrimary: - // Secondary records (block hash aliases, QC covered-number aliases) duplicate a - // primary the scan handles elsewhere. - case keyKind(key) == kindBlock: - number := decodeNumberKey(key) - if number < l.startN { - // Below the start: either a start that lands mid-cohort, whose cohort's earlier - // blocks still follow the covering QC in the scan, or a block stranded below the - // retention floor. startN is never below that floor, so one test covers both. - continue - } - l.heldBlock = true - l.heldNumber = number - case keyKind(key) == kindQC: - if err := l.collectQC(); err != nil { - return err - } - default: - return fmt.Errorf("unknown ledger key kind %q", keyKind(key)) - } - } - return nil -} - -// collectQC decodes the QC record at the scan's current position into the covering-QC state: it -// becomes current when no current QC is set, and otherwise joins the pending queue. -func (l *blockDBIterator) collectQC() error { - value, err := l.it.GetValue() - if err != nil { - return fmt.Errorf("failed to read QC value: %w", err) - } - qc, err := decodeQC(value) - if err != nil { - return fmt.Errorf("failed to unmarshal QC: %w", err) - } - first, next := coveredRange(qc) - entry := &coveredQC{qc: qc, first: first, next: next} - if l.current == nil { - l.current = entry - return nil - } - // QCs are written contiguously, so each surviving range must extend the previous bound. - tailNext := l.current.next - if len(l.pending) > 0 { - tailNext = l.pending[len(l.pending)-1].next - } - if entry.first != tailNext { - return fmt.Errorf("corrupt store: QC range [%d,%d) does not extend previous bound %d", - entry.first, entry.next, tailNext) - } - l.pending = append(l.pending, entry) - return nil -} - -func (l *blockDBIterator) Block() (utils.Option[*types.Block], error) { - if l.closed { - return utils.None[*types.Block](), fmt.Errorf("iterator is closed") - } - if !l.positioned { - return utils.None[*types.Block](), fmt.Errorf("iterator is not positioned on a block number") - } - if !l.heldBlock { - // The tail of the ledger: the covering QC is persisted but this block is not. - return utils.None[*types.Block](), nil - } - value, err := l.it.GetValue() - if err != nil { - return utils.None[*types.Block](), fmt.Errorf("failed to read block value: %w", err) - } - _, blk, err := decodeBlock(value) - if err != nil { - return utils.None[*types.Block](), fmt.Errorf("failed to unmarshal block: %w", err) - } - return utils.Some(blk), nil -} - -func (l *blockDBIterator) Close() error { - // A closed iterator holds no position, so Next() reports exhaustion and Block() reports misuse - // rather than reading through a released snapshot. - l.closed = true - l.positioned = false - if l.it == nil { - return nil - } - if err := l.it.Close(); err != nil { - return fmt.Errorf("failed to close ledger iterator: %w", err) - } - return nil -} diff --git a/sei-db/ledger_db/block/littblock/litt_block_iterator_test.go b/sei-db/ledger_db/block/littblock/litt_block_iterator_test.go deleted file mode 100644 index 324f45fe1c..0000000000 --- a/sei-db/ledger_db/block/littblock/litt_block_iterator_test.go +++ /dev/null @@ -1,345 +0,0 @@ -package littblock - -import ( - "sync" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" - "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" -) - -// TestLittblockIteratorConcurrentFirstBlockIsNotAGap pins the reason simpleIterator exists. A store -// holding a QC and no blocks has no block for a scan to open on, and the first block may legally land -// anywhere inside its covering QC. If Iterator opened a table snapshot in that state, a WriteBlock -// racing it could put a block above the cursor into the snapshot and the scan would report it as a gap -// — a corruption error on a healthy store. Iterator must therefore take no snapshot until a block -// exists. -// -// The interleaving is not directly forceable, so this hammers the window instead: many rounds of a -// fresh QC-only store with Iterator and WriteBlock started together. Any ErrBlockGap is a regression. -func TestLittblockIteratorConcurrentFirstBlockIsNotAGap(t *testing.T) { - rng := utils.TestRngFromSeed(31) - - for round := 0; round < 64; round++ { - db, err := NewBlockDB(strandingConfig(t, t.TempDir(), 1024)) - require.NoError(t, err) - - // One QC covering [0,6) and no blocks: the state where the scan has nothing to anchor on. - require.NoError(t, db.WriteQC(types.GenFullCommitQCRange(rng, 0, 6))) - blk := types.GenBlock(rng) - - var wg sync.WaitGroup - wg.Add(2) - var writeErr, iterErr error - go func() { - defer wg.Done() - // The first block, landing mid-QC — permitted by WriteBlock. - writeErr = db.WriteBlock(2, blk) - }() - go func() { - defer wg.Done() - it, err := db.Iterator(0) - if err != nil { - iterErr = err - return - } - defer func() { _ = it.Close() }() - for { - _, ok, err := it.Next() - if err != nil { - iterErr = err - return - } - if !ok { - return - } - } - }() - wg.Wait() - - require.NoError(t, writeErr, "round %d: the first block may start anywhere inside its QC", round) - require.NotErrorIs(t, iterErr, types.ErrBlockGap, - "round %d: a concurrent first block must never read as a gap", round) - require.NoError(t, iterErr, "round %d", round) - require.NoError(t, db.Close()) - } -} - -// TestLittblockIteratorGapAboveMidQCStartIsCorruption pins that clamping the scan's start up to the -// first existing block does not weaken gap detection above it. A store may legitimately begin partway -// into its covering QC, but blocks are dense from that point up, so an interior hole above it is still -// corruption. A clamp that overshot the first block would silently skip past such a hole. This cannot -// be written against the shared BlockDB contract: producing the hole needs a raw table write that -// bypasses WriteBlock's contiguity cursor. -func TestLittblockIteratorGapAboveMidQCStartIsCorruption(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRngFromSeed(21) - - db, err := NewBlockDB(strandingConfig(t, dir, 1024)) - require.NoError(t, err) - defer func() { _ = db.Close() }() - impl := db.(*blockDB) - - // One QC covering [0,6) whose block history begins at 2, so the scan opens at 2. - require.NoError(t, db.WriteQC(types.GenFullCommitQCRange(rng, 0, 6))) - require.NoError(t, db.WriteBlock(2, types.GenBlock(rng))) - require.NoError(t, db.WriteBlock(3, types.GenBlock(rng))) - - // Corrupt the store above the start: a block at 5 with none at 4. - require.NoError(t, impl.table.Put(blockKey(5), encodeBlock(5, types.GenBlock(rng)))) - - it, err := db.Iterator(0) - require.NoError(t, err) - defer func() { _ = it.Close() }() - - for _, want := range []types.GlobalBlockNumber{2, 3} { - pos, ok, err := it.Next() - require.NoError(t, err) - require.True(t, ok) - require.Equal(t, want, pos.Number, "the scan must open on the first existing block") - require.True(t, pos.HasBlock) - } - - // Position 4 has no block while a later one exists: corruption, not a legitimate hole. - _, _, err = it.Next() - require.ErrorIs(t, err, types.ErrBlockGap) -} - -// TestLittblockIteratorGapIsCorruption pins the iterator's gap detection: blocks are written -// densely (WriteBlock enforces contiguity), so an interior missing block on disk can only be -// corruption and must surface as an error rather than a silent None. The gap is injected by -// writing a block record directly to the raw table, bypassing WriteBlock's contiguity cursor. -func TestLittblockIteratorGapIsCorruption(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRngFromSeed(7) - - db, err := NewBlockDB(strandingConfig(t, dir, 1024)) - require.NoError(t, err) - defer func() { _ = db.Close() }() - impl := db.(*blockDB) - - // One QC covering [0,5) with blocks 0 and 1 written normally. - require.NoError(t, db.WriteQC(types.GenFullCommitQCRange(rng, 0, 5))) - require.NoError(t, db.WriteBlock(0, types.GenBlock(rng))) - require.NoError(t, db.WriteBlock(1, types.GenBlock(rng))) - - // Corrupt the store: a block at 3 with no block at 2, injected past WriteBlock. - blk := types.GenBlock(rng) - require.NoError(t, impl.table.Put(blockKey(3), encodeBlock(3, blk))) - - it, err := db.Iterator(0) - require.NoError(t, err) - defer func() { _ = it.Close() }() - - // Positions 0 and 1 are intact; advancing to 2 must surface the gap as corruption. - for _, want := range []types.GlobalBlockNumber{0, 1} { - pos, ok, err := it.Next() - require.NoError(t, err) - require.True(t, ok) - require.Equal(t, want, pos.Number) - } - _, _, err = it.Next() - require.ErrorIs(t, err, types.ErrBlockGap) -} - -// TestLittblockIteratorUncoveredBlockIsCorruption pins the iterator's coverage check: a QC -// covering every block is always written first, so a block record beyond every QC's range can -// only be corruption and must surface as an error. The uncovered block is injected by writing a -// block record directly to the raw table, bypassing WriteBlock's coverage check. -func TestLittblockIteratorUncoveredBlockIsCorruption(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRngFromSeed(8) - - db, err := NewBlockDB(strandingConfig(t, dir, 1024)) - require.NoError(t, err) - defer func() { _ = db.Close() }() - impl := db.(*blockDB) - - // One QC covering [0,2), fully blocked. - require.NoError(t, db.WriteQC(types.GenFullCommitQCRange(rng, 0, 2))) - require.NoError(t, db.WriteBlock(0, types.GenBlock(rng))) - require.NoError(t, db.WriteBlock(1, types.GenBlock(rng))) - - // Corrupt the store: a block at 2, past every QC's range, injected past WriteBlock. - blk := types.GenBlock(rng) - require.NoError(t, impl.table.Put(blockKey(2), encodeBlock(2, blk))) - - it, err := db.Iterator(0) - require.NoError(t, err) - defer func() { _ = it.Close() }() - - for _, want := range []types.GlobalBlockNumber{0, 1} { - pos, ok, err := it.Next() - require.NoError(t, err) - require.True(t, ok) - require.Equal(t, want, pos.Number) - } - _, _, err = it.Next() - require.Error(t, err, "a block with no QC coverage must surface as corruption") - require.Contains(t, err.Error(), "no QC coverage") -} - -// TestLittblockIteratorMidChainStartNeedsNoScan pins that a store whose coverage begins above 0 is -// positioned directly, without the full-scan fallback that used to serve this case. It must behave -// identically before and after a restart: in the writing session the clamp comes from -// oldestQCStart, after a reopen recoverReadFloors derives the same floor into the watermark. -func TestLittblockIteratorMidChainStartNeedsNoScan(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRngFromSeed(23) - cfg := strandingConfig(t, dir, 1<<20) - - db, err := NewBlockDB(cfg) - require.NoError(t, err) - - // Coverage begins at 100 and nothing below it was ever written. - require.NoError(t, db.WriteQC(types.GenFullCommitQCRange(rng, 100, 103))) - for n := types.GlobalBlockNumber(100); n < 103; n++ { - require.NoError(t, db.WriteBlock(n, types.GenBlock(rng))) - } - require.NoError(t, db.Flush()) - - drain := func(t *testing.T, d types.BlockDB, n types.GlobalBlockNumber) []types.GlobalBlockNumber { - t.Helper() - it, err := d.Iterator(n) - require.NoError(t, err) - defer func() { _ = it.Close() }() - var got []types.GlobalBlockNumber - for { - pos, ok, err := it.Next() - require.NoError(t, err) - if !ok { - return got - } - got = append(got, pos.Number) - } - } - - want := []types.GlobalBlockNumber{100, 101, 102} - for _, start := range []types.GlobalBlockNumber{0, 50, 100} { - require.Equal(t, want, drain(t, db, start), "same session, Iterator(%d)", start) - } - require.Equal(t, []types.GlobalBlockNumber{101, 102}, drain(t, db, 101), "same session, mid-cohort start") - - require.NoError(t, db.Close()) - db2, err := NewBlockDB(cfg) - require.NoError(t, err) - defer func() { _ = db2.Close() }() - - for _, start := range []types.GlobalBlockNumber{0, 50, 100} { - require.Equal(t, want, drain(t, db2, start), "after restart, Iterator(%d)", start) - } -} - -// TestLittblockIteratorMissingStartQCIsCorruption pins that a positioned lookup which misses inside -// known coverage is an error, not a silent full-scan fallback. The clamp guarantees some QC record -// is stored under qcKey(start), so a miss means a record that must exist does not. -// -// It also pins the negative side of the prune/corruption discriminator: with the retention floor at -// or below start, the miss must be reported as corruption and NOT relabelled ErrPruned. The positive -// side (floor above start ⇒ ErrPruned) needs the watermark to advance between Iterator's own -// watermark load and its positioning call, which no deterministic test can arrange without a -// production seam — so it is verified by inspection against gcFilter's reclamation condition -// (littblock's gcFilter clears a key only once the watermark exceeds its number). -func TestLittblockIteratorMissingStartQCIsCorruption(t *testing.T) { - rng := utils.TestRngFromSeed(24) - db, err := NewBlockDB(strandingConfig(t, t.TempDir(), 1<<20)) - require.NoError(t, err) - defer func() { _ = db.Close() }() - impl := db.(*blockDB) - - require.NoError(t, db.WriteQC(types.GenFullCommitQCRange(rng, 0, 3))) - for n := types.GlobalBlockNumber(0); n < 3; n++ { - require.NoError(t, db.WriteBlock(n, types.GenBlock(rng))) - } - require.NoError(t, db.Flush()) - - // Claim coverage runs to 10 while only [0,3) is stored, so qcKey(5) has no record. This is the - // shape a truncated key file or a segment missing from the snapshot presents as. - impl.mu.Lock() - impl.lastQCNext = 10 - impl.mu.Unlock() - - _, err = db.Iterator(5) - require.Error(t, err, "a missing start QC inside claimed coverage must not fall back to a full scan") - require.Contains(t, err.Error(), "corrupt store") - require.NotErrorIs(t, err, types.ErrPruned, - "the floor is at or below start, so this is corruption and must not be excused as pruning") -} - -// TestLittblockIteratorDoesNotServeBlockBelowCoverage pins the start-clamp boundary for a block -// that no retained QC covers: it is not served, and not reported as corruption either. -// -// This test previously asserted the opposite. It was written when Iterator fell back to a plain -// full scan whenever the positioned lookup missed, which meant Iterator(0) walked over the -// uncovered block and flagged it. Iterator now clamps its start up to oldestQCStart and refuses to -// scan below it, so the block is never visited. That is the intended semantics on both counts: -// -// - Iterator(n) is contractually clamped up to the lowest number a retained QC covers, and this -// block is below every retained QC's range, so it is not a position the iterator may yield. -// - Detecting corruption strictly below the requested start is exactly what a positioned -// iterator exists to avoid (see the audit's finding 6), and the detection here was incidental -// to the fallback rather than designed. -// -// The detection is also not recoverable in principle: this same on-disk state — a block below the -// oldest retained QC — is the *legitimate* stranded state after a prune whose GC pass reclaimed -// the covering QC, and littblock does not persist the watermark, so after a restart the two are -// indistinguishable. The check only ever fired in the window before the watermark advanced. -// -// What still holds: an uncovered block at or above the start is corruption and errors — see -// TestLittblockIteratorUncoveredBlockIsCorruption. And finding 2's actual hazard does not return, -// because this iterator is not empty: loadFromBlockDB replays 5..7 rather than seeing an empty -// scan and silently restarting from committee genesis. -func TestLittblockIteratorDoesNotServeBlockBelowCoverage(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRngFromSeed(9) - - db, err := NewBlockDB(strandingConfig(t, dir, 1024)) - require.NoError(t, err) - defer func() { _ = db.Close() }() - impl := db.(*blockDB) - - // A block at 0 injected past WriteBlock's coverage check while no QC exists at all, so the - // record precedes every QC in insertion order. - require.NoError(t, impl.table.Put(blockKey(0), encodeBlock(0, types.GenBlock(rng)))) - - // A well-formed cohort above it, written normally. This sets oldestQCStart to 5. - require.NoError(t, db.WriteQC(types.GenFullCommitQCRange(rng, 5, 8))) - for n := types.GlobalBlockNumber(5); n < 8; n++ { - require.NoError(t, db.WriteBlock(n, types.GenBlock(rng))) - } - - it, err := db.Iterator(0) - require.NoError(t, err) - defer func() { _ = it.Close() }() - - var got []types.GlobalBlockNumber - for { - pos, ok, err := it.Next() - require.NoError(t, err, "the uncovered block below the clamped start must not be visited") - if !ok { - break - } - got = append(got, pos.Number) - } - require.Equal(t, []types.GlobalBlockNumber{5, 6, 7}, got, - "Iterator(0) must clamp up to the first retained QC and yield only covered numbers") -} - -// TestLittblockIteratorEmptyStoreIsCleanExhaustion pins the negative of the coverage check: with no -// block record held, no retained QC means there is genuinely nothing to serve, which must stay a -// clean (false, nil) rather than becoming a corruption error. -func TestLittblockIteratorEmptyStoreIsCleanExhaustion(t *testing.T) { - db, err := NewBlockDB(strandingConfig(t, t.TempDir(), 1024)) - require.NoError(t, err) - defer func() { _ = db.Close() }() - - it, err := db.Iterator(0) - require.NoError(t, err) - defer func() { _ = it.Close() }() - - _, ok, err := it.Next() - require.NoError(t, err) - require.False(t, ok) -} diff --git a/sei-db/ledger_db/block/littblock/litt_block_simple_iterator.go b/sei-db/ledger_db/block/littblock/litt_block_simple_iterator.go deleted file mode 100644 index 956ff783d1..0000000000 --- a/sei-db/ledger_db/block/littblock/litt_block_simple_iterator.go +++ /dev/null @@ -1,121 +0,0 @@ -package littblock - -import ( - "fmt" - - "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" - "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" -) - -var _ types.BlockDBIterator = (*simpleIterator)(nil) - -// simpleIterator serves one narrow case: a store holding QCs but no blocks at all. That happens between -// the first WriteQC and the first WriteBlock, and after a crash that made a QC durable before any block. -// It yields every number those QCs cover, each with no block. -// -// It exists so blockDBIterator does not have to serve that case. That iterator is driven by a single -// forward scan of the shared ledger table, and this is the one case where the scan has no block to anchor -// its start on: its snapshot can pick up a first block written concurrently and report it as a gap below -// the cursor, since the first block may legally land anywhere inside its covering QC. Handling the case -// here is simpler than building a contract around that behaviour — this iterator takes no table snapshot, -// so there is nothing to race, and the scanning path keeps its plain invariant that iteration starts on a -// block that exists. -// -// Every position is read and decoded up front. Nothing is pinned: no table snapshot, no segment -// reservation, no open handle. Close exists only to satisfy the interface and leaking one costs nothing. -// The case is tiny by construction — no block has been written yet, so coverage spans at most the few -// cohorts a single persistence batch queued. -type simpleIterator struct { - // positions is every position this iterator will yield, ascending, built at construction. - positions []types.Position - - // idx is the current position; -1 before the first Next and len(positions) once exhausted. - idx int - - // closed is true once Close has been called. Block rejects calls made afterward. - closed bool -} - -// newSimpleIterator reads the QCs covering [start, nextQC) and materializes a position for every number -// they cover at or above start, each with no block. -// -// QCs are contiguous, so this walks cohort by cohort: one point read per QC, whose decoded value is -// shared by every position in its range. Sharing matters — types.Position documents QC as the same -// pointer across a range, and callers key "the scan entered a new QC" on that identity. -func newSimpleIterator( - table qcReader, - start types.GlobalBlockNumber, - nextQC types.GlobalBlockNumber, -) (*simpleIterator, error) { - var positions []types.Position - for n := start; n < nextQC; { - qc, err := readQCCovering(table, n) - if err != nil { - return nil, err - } - first, next := coveredRange(qc) - if next <= n { - // Would not advance; a QC whose range does not contain n means coverage is not the - // contiguous span the write path guarantees. - return nil, fmt.Errorf("corrupt store: QC at %d covers [%d,%d), which does not reach it", - n, first, next) - } - for m := n; m < next && m < nextQC; m++ { - positions = append(positions, types.Position{Number: m, QC: qc, HasBlock: false}) - } - n = next - } - return &simpleIterator{positions: positions, idx: -1}, nil -} - -func (s *simpleIterator) Next() (types.Position, bool, error) { - if s.idx < len(s.positions) { - s.idx++ - } - if s.closed || s.idx < 0 || s.idx >= len(s.positions) { - return types.Position{}, false, nil - } - return s.positions[s.idx], true, nil -} - -// Block always reports absence: this iterator only ever covers numbers whose blocks have not been -// written, so every position it yields has HasBlock false. -func (s *simpleIterator) Block() (utils.Option[*types.Block], error) { - if s.closed { - return utils.None[*types.Block](), fmt.Errorf("iterator is closed") - } - if s.idx < 0 || s.idx >= len(s.positions) { - return utils.None[*types.Block](), fmt.Errorf("iterator is not positioned on a block number") - } - return utils.None[*types.Block](), nil -} - -// Close releases nothing — see the type doc. It only marks the iterator unusable so misuse after close -// is reported rather than silently answered. -func (s *simpleIterator) Close() error { - s.closed = true - return nil -} - -// qcReader is the slice of littdb.Table that newSimpleIterator needs, so tests can supply QCs without -// building a table. -type qcReader interface { - Get(key []byte) ([]byte, bool, error) -} - -// readQCCovering point-reads and decodes the QC covering n. Every covered number carries a QC alias key -// holding the full QC value, so any number inside a retained range resolves. -func readQCCovering(table qcReader, n types.GlobalBlockNumber) (*types.FullCommitQC, error) { - value, exists, err := table.Get(qcKey(n)) - if err != nil { - return nil, fmt.Errorf("failed to read covering QC for %d: %w", n, err) - } - if !exists { - return nil, fmt.Errorf("corrupt store: no QC record at %d despite coverage past it", n) - } - qc, err := decodeQC(value) - if err != nil { - return nil, fmt.Errorf("failed to decode covering QC for %d: %w", n, err) - } - return qc, nil -} diff --git a/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go b/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go index 002433a083..ecf0bb0907 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go +++ b/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go @@ -26,20 +26,77 @@ func strandingConfig(t *testing.T, dir string, maxSegmentKeyCount uint32) *Block return cfg } -// writeSyntheticBatches writes numBatches contiguous batches of perQC blocks each -// (global numbers 0.., QC ranges [0,perQC), [perQC,2*perQC), ...). Each QC's own -// GlobalRange matches the range it is written at, so littblock's clamp can trust -// it; littblock does not verify signatures, so no committee is needed. func writeSyntheticBatches(t *testing.T, db types.BlockDB, rng utils.Rng, numBatches int, perQC int) { - for i := 0; i < numBatches; i++ { - first := types.GlobalBlockNumber(i * perQC) //nolint:gosec // small test indices - next := first + types.GlobalBlockNumber(perQC) - qc := types.GenFullCommitQCRange(rng, first, next) + t.Helper() + committee, keys := types.GenCommittee(rng, 4) + prev := utils.None[*types.CommitQC]() + qcs := make([]*types.FullCommitQC, 0, numBatches) + for range numBatches { + qc, blocks := syntheticFullCommitQC(rng, committee, keys, prev, perQC) + first := qc.QC().GlobalRange().First require.NoError(t, db.WriteQC(qc)) - for j := 0; j < perQC; j++ { - require.NoError(t, db.WriteBlock(first+types.GlobalBlockNumber(j), types.GenBlock(rng))) //nolint:gosec + for j, block := range blocks { + require.NoError(t, db.WriteBlock(first+types.GlobalBlockNumber(j), block)) //nolint:gosec } + qcs = append(qcs, qc) + prev = utils.Some(qc.QC()) } + + for _, qc := range qcs { + proposal := types.NewAppProposal(qc.QC().Proposal(), types.GenAppHash(rng)) + require.NoError(t, db.WriteAppProposal(proposal)) + vote := types.NewAppVote(proposal) + appQC := types.NewAppQC([]*types.Signed[*types.AppVote]{ + types.Sign(types.GenSecretKey(rng), vote), + }) + require.NoError(t, db.WriteAppQC(appQC)) + } +} + +func syntheticFullCommitQC( + rng utils.Rng, + committee *types.Committee, + keys []types.SecretKey, + prev utils.Option[*types.CommitQC], + perQC int, +) (*types.FullCommitQC, []*types.Block) { + blocksByLane := map[types.LaneID][]*types.Block{} + makeBlock := func(producer types.LaneID) *types.Block { + if blocks := blocksByLane[producer]; len(blocks) > 0 { + parent := blocks[len(blocks)-1] + return types.NewBlock(producer, parent.Header().Next(), parent.Header().Hash(), types.GenPayload(rng)) + } + return types.NewBlock(producer, types.LaneRangeOpt(prev, producer).Next(), types.GenBlockHeaderHash(rng), types.GenPayload(rng)) + } + for range perQC { + producer := committee.Lanes().At(rng.Intn(committee.Lanes().Len())) + blocksByLane[producer] = append(blocksByLane[producer], makeBlock(producer)) + } + + laneQCs := map[types.LaneID]*types.LaneQC{} + headers := make([]*types.BlockHeader, 0, perQC) + blocks := make([]*types.Block, 0, perQC) + for lane := range committee.Lanes().All() { + if bs := blocksByLane[lane]; len(bs) > 0 { + laneQCs[lane] = syntheticLaneQC(keys, bs[len(bs)-1].Header()) + for _, block := range bs { + headers = append(headers, block.Header()) + blocks = append(blocks, block) + } + } + } + epoch := types.NewEpoch(0, types.OpenRoadRange(), time.Unix(1_700_000_000, 0), committee, 0) + qc := types.BuildCommitQC(epoch, keys, prev, laneQCs) + return types.NewFullCommitQC(qc, headers), blocks +} + +func syntheticLaneQC(keys []types.SecretKey, header *types.BlockHeader) *types.LaneQC { + vote := types.NewLaneVote(header) + votes := make([]*types.Signed[*types.LaneVote], 0, len(keys)) + for _, key := range keys { + votes = append(votes, types.Sign(key, vote)) + } + return types.NewLaneQC(votes) } // physicallyPresent reports whether a key exists in the raw table, bypassing the @@ -101,7 +158,7 @@ func TestLittblockStrandedBlockNotServedAfterRestart(t *testing.T) { require.Equal(t, uint64(5), impl.watermark.Load(), "recovered watermark must be the lowest surviving QC's First") // The gate refuses the stranded blocks and their (absent) QCs. - for n := types.GlobalBlockNumber(0); n < 5; n++ { + for n := range types.GlobalBlockNumber(5) { blk, err := db3.ReadBlockByNumber(n) require.ErrorIs(t, err, types.ErrPruned, "stranded/pruned block %d must be reported pruned", n) require.False(t, blk.IsPresent(), "stranded/pruned block %d must not be served", n) @@ -120,23 +177,16 @@ func TestLittblockStrandedBlockNotServedAfterRestart(t *testing.T) { require.True(t, qc.IsPresent(), "covering QC for served block %d must be readable", n) } - // The ledger never yields a stranded position, and every yielded position - // has a covering QC. - it, err := db3.Iterator(0) + // ReadSuffix never includes stranded blocks, and every returned + // block has a covering QC. + suffix, err := db3.ReadSuffix() require.NoError(t, err) - defer func() { _ = it.Close() }() - for { - pos, ok, err := it.Next() - require.NoError(t, err) - if !ok { - break - } - n := pos.Number - require.GreaterOrEqual(t, uint64(n), uint64(5), "ledger must not yield stranded position %d", n) - require.NotNil(t, pos.QC, "position %d must have a covering QC", n) - blkOpt, err := it.Block() + for _, block := range suffix.Blocks { + n := block.Number + require.GreaterOrEqual(t, uint64(n), uint64(5), "suffix data must not include stranded block %d", n) + qc, err := db3.ReadQCByBlockNumber(n) require.NoError(t, err) - require.True(t, blkOpt.IsPresent(), "position %d must have a block", n) + require.True(t, qc.IsPresent(), "block %d must have a covering QC", n) } } @@ -172,7 +222,7 @@ func TestLittblockReclaimsAcrossRestart(t *testing.T) { require.NoError(t, ForceGC(db2)) // Blocks 0..14 and their QC keys are physically reclaimed. - for n := types.GlobalBlockNumber(0); n < 15; n++ { + for n := range types.GlobalBlockNumber(15) { require.False(t, physicallyPresent(t, impl, blockKey(n)), "block %d must be reclaimed", n) require.False(t, physicallyPresent(t, impl, qcKey(n)), "QC key %d must be reclaimed", n) } @@ -278,7 +328,7 @@ func TestLittblockPruneIntoCohortRoundsDown(t *testing.T) { require.True(t, qc.IsPresent(), "covering QC for served block %d must be readable", n) } // The fully-below cohort [0,5) is pruned. - for n := types.GlobalBlockNumber(0); n < 5; n++ { + for n := range types.GlobalBlockNumber(5) { blk, err := db2.ReadBlockByNumber(n) require.ErrorIs(t, err, types.ErrPruned, "block %d below the cohort must be reported pruned", n) require.False(t, blk.IsPresent(), "block %d below the cohort must not be served", n) @@ -286,7 +336,7 @@ func TestLittblockPruneIntoCohortRoundsDown(t *testing.T) { } // TestLittblockRefusesToOpenWithStrandedBlocks verifies the corruption guard in -// recoverReadFloors. The never-empty prune invariant guarantees at least one +// recoverWatermark. The never-empty prune invariant guarantees at least one // (block, QC) pair is always retained, so a store holding a block with no // surviving QC is corrupt (e.g. a QC WAL file removed out of band). Rather than // serve blocks it can no longer trust, the store refuses to open. @@ -311,7 +361,6 @@ func TestLittblockRefusesToOpenWithStrandedBlocks(t *testing.T) { // Reopen: recovery finds the block but no QC, so it refuses to open. _, err = NewBlockDB(strandingConfig(t, dir, 8)) require.Error(t, err) - require.ErrorContains(t, err, "no surviving QC") } // TestLittblockEmptyStorePruneDoesNotReclaimLaterWrites is the regression for the @@ -338,7 +387,7 @@ func TestLittblockEmptyStorePruneDoesNotReclaimLaterWrites(t *testing.T) { require.NoError(t, db.Flush()) // Every written block survives GC on disk and is served by the read gate. - for n := types.GlobalBlockNumber(0); n < 10; n++ { + for n := range types.GlobalBlockNumber(10) { require.True(t, physicallyPresent(t, impl, blockKey(n)), "block %d must survive GC", n) blk, err := db.ReadBlockByNumber(n) require.NoError(t, err) diff --git a/sei-db/ledger_db/block/littblock_crash_test.go b/sei-db/ledger_db/block/littblock_crash_test.go index 4bdba321f9..857740b615 100644 --- a/sei-db/ledger_db/block/littblock_crash_test.go +++ b/sei-db/ledger_db/block/littblock_crash_test.go @@ -71,27 +71,15 @@ func TestLittblockNoBlockWithoutQCAfterTornTail(t *testing.T) { totalBlocks += len(b.blocks) } - it, err := db2.Iterator(0) + suffix, err := db2.ReadSuffix() require.NoError(t, err) - defer func() { _ = it.Close() }() present := 0 - for { - pos, ok, err := it.Next() - require.NoError(t, err) - if !ok { - break - } - // pos.QC being non-nil at every position (guaranteed by the iterator - // contract) is the covering-QC invariant; cross-check via the point-read - // path too. - n := pos.Number - require.NotNil(t, pos.QC, "position %d has no covering QC", n) + for _, block := range suffix.Blocks { + n := block.Number qc, err := db2.ReadQCByBlockNumber(n) require.NoError(t, err) - require.True(t, qc.IsPresent(), "position %d survived but its covering QC was lost", n) - if pos.HasBlock { - present++ - } + require.True(t, qc.IsPresent(), "block %d survived but its covering QC was lost", n) + present++ } // The truncation must have actually dropped at least one block, otherwise the @@ -225,24 +213,15 @@ func TestLittblockFlushSurvivesHardKill(t *testing.T) { totalBlocks += len(b.blocks) } - it, err := db.Iterator(0) + suffix, err := db.ReadSuffix() require.NoError(t, err) - defer func() { _ = it.Close() }() present := 0 - for { - pos, ok, err := it.Next() - require.NoError(t, err) - if !ok { - break - } - n := pos.Number - require.NotNil(t, pos.QC, "position %d has no covering QC", n) + for _, block := range suffix.Blocks { + n := block.Number qc, err := db.ReadQCByBlockNumber(n) require.NoError(t, err) - require.True(t, qc.IsPresent(), "position %d lost its covering QC after hard kill", n) - if pos.HasBlock { - present++ - } + require.True(t, qc.IsPresent(), "block %d lost its covering QC after hard kill", n) + present++ } // Unlike the torn-tail test (which expects loss), a clean Flush before the kill diff --git a/sei-db/ledger_db/block/memblock/mem_block_db.go b/sei-db/ledger_db/block/memblock/mem_block_db.go index ece2e6ed7c..f682fc7361 100644 --- a/sei-db/ledger_db/block/memblock/mem_block_db.go +++ b/sei-db/ledger_db/block/memblock/mem_block_db.go @@ -2,7 +2,6 @@ package memblock import ( "fmt" - "sort" "sync" "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" @@ -11,274 +10,277 @@ import ( var _ types.BlockDB = (*blockDB)(nil) -// qcEntry pairs a QC with the half-open range [lower, upper) it covers, as -// derived from the QC itself by coveredRange. -type qcEntry struct { - qc *types.FullCommitQC - lower types.GlobalBlockNumber - upper types.GlobalBlockNumber -} - -// hashEntry pairs a block with its GlobalBlockNumber so ReadBlockByHash can -// return the number, mirroring the littblock implementation which embeds it in -// the stored value. type hashEntry struct { blk *types.Block n types.GlobalBlockNumber } -// blockDB is an in-memory types.BlockDB. It holds blocks and QCs by pointer (no -// marshaling) and is intended as a test/benchmark fixture, not a durable -// implementation. +// blockDB is an in-memory types.BlockDB. It mirrors littblock's write/status +// rules, but stores already-decoded values in maps instead of LittDB records. type blockDB struct { - mu sync.RWMutex - byNumber map[types.GlobalBlockNumber]*types.Block - byHash map[types.BlockHeaderHash]hashEntry - qcsByLower map[types.GlobalBlockNumber]qcEntry - - // Write-order cursors (see types.BlockDB contract). - hasBlocks bool - lastBlockNumber types.GlobalBlockNumber - hasQC bool - lastQCNext types.GlobalBlockNumber - - // latestQCStartBlock is the most recently written QC's starting block number — - // the lowest block number in the newest cohort. PruneBefore clamps to it (see - // littblock). - latestQCStartBlock types.GlobalBlockNumber - - // firstBlockNumber is the lowest block number written. Iterator clamps its start up to - // it so a scan always opens on a block that exists; the first block may land anywhere - // inside its covering QC, so this can sit above that QC's start with no block in - // between. Meaningful only while hasBlocks. - firstBlockNumber types.GlobalBlockNumber - - // watermark is the (clamped) retention floor set by PruneBefore. Reads - // strictly below it are refused with types.ErrPruned; because pruned entries - // are deleted eagerly, this is the only record of where the floor sits and - // so the only way to tell a pruned block from one never written. + mu sync.RWMutex + + blocksByNumber map[types.GlobalBlockNumber]*types.Block + blocksByHash map[types.BlockHeaderHash]hashEntry + + qcsByBlock map[types.GlobalBlockNumber]*types.FullCommitQC + + appProposalsByBlock map[types.GlobalBlockNumber]*types.AppProposal + + appQCsByBlock map[types.GlobalBlockNumber]*types.AppQC + watermark types.GlobalBlockNumber + status utils.Option[types.SuffixRange] } // NewBlockDB returns an in-memory types.BlockDB. func NewBlockDB() types.BlockDB { return &blockDB{ - byNumber: make(map[types.GlobalBlockNumber]*types.Block), - byHash: make(map[types.BlockHeaderHash]hashEntry), - qcsByLower: make(map[types.GlobalBlockNumber]qcEntry), + blocksByNumber: make(map[types.GlobalBlockNumber]*types.Block), + blocksByHash: make(map[types.BlockHeaderHash]hashEntry), + qcsByBlock: make(map[types.GlobalBlockNumber]*types.FullCommitQC), + appProposalsByBlock: make(map[types.GlobalBlockNumber]*types.AppProposal), + appQCsByBlock: make(map[types.GlobalBlockNumber]*types.AppQC), } } func (s *blockDB) WriteBlock(n types.GlobalBlockNumber, blk *types.Block) error { s.mu.Lock() defer s.mu.Unlock() - if s.hasBlocks && n != s.lastBlockNumber+1 { - return fmt.Errorf("block number %d not contiguous with last written %d: %w", - n, s.lastBlockNumber, types.ErrBlockOutOfOrder) - } - // A covering QC must already be written. QCs are contiguous and blocks - // strictly ascending, so n is covered iff n < lastQCNext. - if !s.hasQC || n >= s.lastQCNext { - return fmt.Errorf("block number %d not covered by any written QC (next QC bound %d): %w", - n, s.lastQCNext, types.ErrBlockMissingQC) + + status, ok := s.status.Get() + if !ok || n >= status.NextQC { + return fmt.Errorf("block number %d not covered by any written QC: %w", n, types.ErrBlockMissingQC) } - s.byNumber[n] = blk - s.byHash[blk.Header().Hash()] = hashEntry{blk: blk, n: n} - if !s.hasBlocks { - s.firstBlockNumber = n + if n != status.NextBlock { + return fmt.Errorf("block number %d not contiguous with last written %d: %w", + n, status.NextBlock-1, types.ErrBlockOutOfOrder) } - s.lastBlockNumber = n - s.hasBlocks = true - return nil -} -// coveredRange returns the half-open global block number range the QC covers, -// as specified by types.BlockDB.WriteQC: [First, First+len(Headers())). Derived -// identically in littblock — see the comment there for why the bound comes from -// the header count rather than from GlobalRange().Next. -func coveredRange(qc *types.FullCommitQC) (types.GlobalBlockNumber, types.GlobalBlockNumber) { - first := qc.QC().GlobalRange().First - return first, first + types.GlobalBlockNumber(len(qc.Headers())) + s.blocksByNumber[n] = blk + s.blocksByHash[blk.Header().Hash()] = hashEntry{blk: blk, n: n} + status.NextBlock = n + 1 + s.status = utils.Some(status) + return nil } func (s *blockDB) WriteQC(qc *types.FullCommitQC) error { - first, next := coveredRange(qc) - if first >= next { - return fmt.Errorf("QC at %d covers no blocks: %w", first, types.ErrQCNonContiguous) + gr := qc.QC().GlobalRange() + if gr.Len() == 0 { + return fmt.Errorf("QC at %d covers no blocks: %w", gr.First, types.ErrQCNonContiguous) } + s.mu.Lock() defer s.mu.Unlock() - if s.hasQC && first != s.lastQCNext { + + status, ok := s.status.Get() + if ok && status.NextQC != gr.First { return fmt.Errorf("QC starts at %d, expected %d: %w", - first, s.lastQCNext, types.ErrQCNonContiguous) + gr.First, status.NextQC, types.ErrQCNonContiguous) + } + + for n := gr.First; n < gr.Next; n++ { + s.qcsByBlock[n] = qc } - s.qcsByLower[first] = qcEntry{qc: qc, lower: first, upper: next} - s.latestQCStartBlock = first - s.lastQCNext = next - s.hasQC = true + + if !ok { + status = types.SuffixRange{ + First: gr.First, + NextAppQC: gr.First, + NextAppProposal: gr.First, + NextBlock: gr.First, + NextQC: gr.Next, + } + } else { + status.NextQC = gr.Next + } + s.status = utils.Some(status) return nil } -func (s *blockDB) PruneBefore(n types.GlobalBlockNumber) error { +func (s *blockDB) WriteAppProposal(appProposal *types.AppProposal) error { + gr := appProposal.GlobalRange() + if gr.Len() == 0 { + return fmt.Errorf("AppProposal at %d covers no blocks: %w", gr.First, types.ErrAppProposalNonContiguous) + } + s.mu.Lock() defer s.mu.Unlock() - if !s.hasBlocks { - // No blocks yet: nothing to prune, and deleting QCs here would strand a - // future block whose coverage check still passes. Mirrors littblock. - return nil + + status, ok := s.status.Get() + if !ok { + return fmt.Errorf("AppProposal [%d,%d) has no matching QC: %w", gr.First, gr.Next, types.ErrAppProposalMissingQC) } - // Never let the watermark enter the newest block's cohort: clamp its ceiling - // at the cohort's first block (latestQCStartBlock), guarded by lastBlockNumber - // for a QC written ahead of its blocks. Keeps the newest cohort whole and - // pruning monotonic. See littblock and the BlockDB PruneBefore contract. - if ceiling := min(s.latestQCStartBlock, s.lastBlockNumber); n > ceiling { - n = ceiling - } - // Round the watermark down to the covering QC's First. A QC's cohort of - // blocks changes readability atomically, so the watermark must never fall - // strictly inside a QC's range (see littblock): otherwise a read would - // refuse the cohort's low blocks while still serving its high blocks (which - // pruning must retain). - for _, e := range s.qcsByLower { - if e.lower <= n && n < e.upper { - n = e.lower - break - } + if status.NextAppProposal != gr.First { + return fmt.Errorf("AppProposal starts at %d, expected %d: %w", + gr.First, status.NextAppProposal, types.ErrAppProposalNonContiguous) } - s.watermark = max(s.watermark, n) - for num, blk := range s.byNumber { - if num < s.watermark { - delete(s.byNumber, num) - delete(s.byHash, blk.Header().Hash()) - } + if gr.Next > status.NextBlock { + return fmt.Errorf("AppProposal [%d,%d) is not covered by written blocks: %w", gr.First, gr.Next, types.ErrAppProposalMissingQC) } - for lower, e := range s.qcsByLower { - if e.upper <= s.watermark { - delete(s.qcsByLower, lower) - } + + for n := gr.First; n < gr.Next; n++ { + s.appProposalsByBlock[n] = appProposal } + status.NextAppProposal = gr.Next + s.status = utils.Some(status) return nil } -func (s *blockDB) Flush() error { return nil } +func (s *blockDB) WriteAppQC(appQC *types.AppQC) error { + gr := appQC.Proposal().GlobalRange() + if gr.Len() == 0 { + return fmt.Errorf("AppQC at %d covers no blocks: %w", gr.First, types.ErrAppQCNonContiguous) + } -func (s *blockDB) Status() types.DBStatus { - s.mu.RLock() - defer s.mu.RUnlock() - var tips types.DBStatus - if s.hasBlocks { - tips.NextBlock = s.lastBlockNumber + 1 + s.mu.Lock() + defer s.mu.Unlock() + + status, ok := s.status.Get() + if !ok { + return fmt.Errorf("AppQC [%d,%d) has no matching QC: %w", gr.First, gr.Next, types.ErrAppQCMissingQC) + } + if status.NextAppQC != gr.First { + return fmt.Errorf("AppQC starts at %d, expected %d: %w", + gr.First, status.NextAppQC, types.ErrAppQCNonContiguous) } - if s.hasQC { - tips.NextQC = s.lastQCNext + if gr.Next > status.NextAppProposal { + return fmt.Errorf("AppQC [%d,%d) is not covered by written AppProposals: %w", gr.First, gr.Next, types.ErrAppQCMissingQC) } - return tips + + for n := gr.First; n < gr.Next; n++ { + s.appQCsByBlock[n] = appQC + } + status.NextAppQC = gr.Next + status.First = status.NextAppQC - 1 + s.status = utils.Some(status) + return nil } -func (s *blockDB) Iterator(n types.GlobalBlockNumber) (types.BlockDBIterator, error) { - s.mu.RLock() - defer s.mu.RUnlock() +func (s *blockDB) PruneBefore(n types.GlobalBlockNumber) error { + s.mu.Lock() + defer s.mu.Unlock() - // Clamp up to the lowest number this store can serve: the retention gate, and the start of - // the block history so a scan opens on a block that exists rather than on blockless numbers - // below it (the first block may land inside its covering QC's range). With no block at all - // the per-entry clamp in iteratorLocked governs, so a QC written ahead of its blocks is - // still iterable. - start := max(n, s.watermark) - if s.hasBlocks { - start = max(start, s.firstBlockNumber) - } - entries := s.sortedQCsLocked() - if len(entries) == 0 || start >= entries[len(entries)-1].upper { - // Nothing is covered at or above the (clamped) start. - return &memBlockDBIterator{idx: -1}, nil - } - return s.iteratorLocked(entries, start), nil -} + status, ok := s.status.Get() + if !ok { + return nil + } -// sortedQCsLocked returns the retained QC entries ascending by lower bound. Caller holds mu. -func (s *blockDB) sortedQCsLocked() []qcEntry { - entries := make([]qcEntry, 0, len(s.qcsByLower)) - for _, e := range s.qcsByLower { - entries = append(entries, e) + n = min(n, status.First) + if qc, ok := s.qcsByBlock[n]; ok { + n = qc.QC().GlobalRange().First + } + if n <= s.watermark { + return nil } - sort.Slice(entries, func(i, j int) bool { return entries[i].lower < entries[j].lower }) - return entries -} -// iteratorLocked snapshots every covered number from start upward (clamping up to the first -// covered number when start falls below all coverage), pairing each with its covering QC and -// (possibly absent) block. Caller holds mu and guarantees some entry's range ends above start. -// -// Copying the whole range up front is how this store satisfies the iterator's snapshot -// guarantee: the records live in maps that later writes and prunes mutate in place, so a -// lazy walk would observe them. Residency is not something BlockDB.Iterator promises, and a -// store that already holds every record in memory has nothing to stream anyway. -func (s *blockDB) iteratorLocked(entries []qcEntry, start types.GlobalBlockNumber) *memBlockDBIterator { - it := &memBlockDBIterator{idx: -1} - for _, e := range entries { - for num := max(e.lower, start); num < e.upper; num++ { - it.nums = append(it.nums, num) - it.qcs = append(it.qcs, e.qc) - it.blocks = append(it.blocks, s.byNumber[num]) + s.watermark = n + for num, blk := range s.blocksByNumber { + if num < s.watermark { + delete(s.blocksByNumber, num) + delete(s.blocksByHash, blk.Header().Hash()) } } - return it + pruneRanges(s.watermark, s.qcsByBlock, func(qc *types.FullCommitQC) types.GlobalRange { + return qc.QC().GlobalRange() + }) + pruneRanges(s.watermark, s.appProposalsByBlock, func(appProposal *types.AppProposal) types.GlobalRange { + return appProposal.GlobalRange() + }) + pruneRanges(s.watermark, s.appQCsByBlock, func(appQC *types.AppQC) types.GlobalRange { + return appQC.Proposal().GlobalRange() + }) + return nil } -var _ types.BlockDBIterator = (*memBlockDBIterator)(nil) - -// memBlockDBIterator steps through a snapshot of covered numbers captured at creation. -type memBlockDBIterator struct { - // nums holds every covered number, ascending. - nums []types.GlobalBlockNumber - - // qcs holds the covering QC per position. - qcs []*types.FullCommitQC +func (s *blockDB) First() types.GlobalBlockNumber { + s.mu.RLock() + defer s.mu.RUnlock() + return s.watermark +} - // blocks holds the block per position; nil where no block is persisted. - blocks []*types.Block +func pruneRanges[T any]( + watermark types.GlobalBlockNumber, + byBlock map[types.GlobalBlockNumber]T, + globalRange func(T) types.GlobalRange, +) { + for n, value := range byBlock { + if globalRange(value).Next <= watermark { + delete(byBlock, n) + } + } +} - // idx is the current position; -1 before the first Next and len(nums) once exhausted. - idx int +func (s *blockDB) Flush() error { return nil } - // closed is true once Close has been called. Block rejects calls made afterward. - closed bool +func (s *blockDB) Status() utils.Option[types.SuffixRange] { + s.mu.RLock() + defer s.mu.RUnlock() + return s.status } -func (it *memBlockDBIterator) Next() (types.Position, bool, error) { - if it.idx < len(it.nums) { - it.idx++ +func (s *blockDB) ReadSuffix() (types.Suffix, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + status, ok := s.status.Get() + if !ok { + return types.Suffix{}, nil } - if !it.positioned() { - return types.Position{}, false, nil + + suffix := types.Suffix{Status: utils.Some(status)} + suffix.CommitQCs = appendSuffixRanges( + s.qcsByBlock, + status.First, + status.NextQC, + func(qc *types.FullCommitQC) types.GlobalRange { return qc.QC().GlobalRange() }, + ) + suffix.AppProposals = appendSuffixRanges( + s.appProposalsByBlock, + status.First, + status.NextAppProposal, + func(appProposal *types.AppProposal) types.GlobalRange { return appProposal.GlobalRange() }, + ) + suffix.AppQCs = appendSuffixRanges( + s.appQCsByBlock, + status.First, + status.NextAppQC, + func(appQC *types.AppQC) types.GlobalRange { return appQC.Proposal().GlobalRange() }, + ) + for n := status.First; n < status.NextBlock; n++ { + if block, ok := s.blocksByNumber[n]; ok { + suffix.Blocks = append(suffix.Blocks, types.SuffixBlock{Number: n, Block: block}) + } } - return types.Position{ - Number: it.nums[it.idx], - QC: it.qcs[it.idx], - HasBlock: it.blocks[it.idx] != nil, - }, true, nil + return suffix, nil } -func (it *memBlockDBIterator) Block() (utils.Option[*types.Block], error) { - if !it.positioned() { - return utils.None[*types.Block](), fmt.Errorf("iterator is not positioned on a block number") +func appendSuffixRanges[T any]( + byBlock map[types.GlobalBlockNumber]T, + floor types.GlobalBlockNumber, + next types.GlobalBlockNumber, + globalRange func(T) types.GlobalRange, +) []T { + if next <= floor { + return nil } - if it.blocks[it.idx] == nil { - return utils.None[*types.Block](), nil + values := make([]T, 0) + var lastFirst types.GlobalBlockNumber + haveLast := false + for n := floor; n < next; n++ { + value, ok := byBlock[n] + if !ok { + continue + } + gr := globalRange(value) + if gr.Next <= floor || haveLast && gr.First == lastFirst { + continue + } + values = append(values, value) + lastFirst = gr.First + haveLast = true } - return utils.Some(it.blocks[it.idx]), nil -} - -func (it *memBlockDBIterator) Close() error { - // Mirrors littblock: a closed iterator holds no position, so Block reports misuse. - it.closed = true - return nil -} - -// positioned reports whether the iterator sits on a number Next yielded. -func (it *memBlockDBIterator) positioned() bool { - return !it.closed && it.idx >= 0 && it.idx < len(it.nums) + return values } func (s *blockDB) ReadBlockByNumber( @@ -289,7 +291,7 @@ func (s *blockDB) ReadBlockByNumber( if n < s.watermark { return utils.None[*types.Block](), types.ErrPruned } - if blk, ok := s.byNumber[n]; ok { + if blk, ok := s.blocksByNumber[n]; ok { return utils.Some(blk), nil } return utils.None[*types.Block](), nil @@ -300,7 +302,7 @@ func (s *blockDB) ReadBlockByHash( ) (utils.Option[types.BlockWithNumber], error) { s.mu.RLock() defer s.mu.RUnlock() - if e, ok := s.byHash[hash]; ok { + if e, ok := s.blocksByHash[hash]; ok && e.n >= s.watermark { return utils.Some(types.BlockWithNumber{Block: e.blk, Number: e.n}), nil } return utils.None[types.BlockWithNumber](), nil @@ -314,12 +316,38 @@ func (s *blockDB) ReadQCByBlockNumber( if n < s.watermark { return utils.None[*types.FullCommitQC](), types.ErrPruned } - for _, e := range s.qcsByLower { - if e.lower <= n && n < e.upper { - return utils.Some(e.qc), nil - } + if qc, ok := s.qcsByBlock[n]; ok { + return utils.Some(qc), nil } return utils.None[*types.FullCommitQC](), nil } +func (s *blockDB) ReadAppProposalByBlockNumber( + n types.GlobalBlockNumber, +) (utils.Option[*types.AppProposal], error) { + s.mu.RLock() + defer s.mu.RUnlock() + if n < s.watermark { + return utils.None[*types.AppProposal](), types.ErrPruned + } + if appProposal, ok := s.appProposalsByBlock[n]; ok { + return utils.Some(appProposal), nil + } + return utils.None[*types.AppProposal](), nil +} + +func (s *blockDB) ReadAppQCByBlockNumber( + n types.GlobalBlockNumber, +) (utils.Option[*types.AppQC], error) { + s.mu.RLock() + defer s.mu.RUnlock() + if n < s.watermark { + return utils.None[*types.AppQC](), types.ErrPruned + } + if appQC, ok := s.appQCsByBlock[n]; ok { + return utils.Some(appQC), nil + } + return utils.None[*types.AppQC](), nil +} + func (s *blockDB) Close() error { return nil } diff --git a/sei-tendermint/autobahn/types/app_proposal.go b/sei-tendermint/autobahn/types/app_proposal.go index e383cc3591..dcbe936ec3 100644 --- a/sei-tendermint/autobahn/types/app_proposal.go +++ b/sei-tendermint/autobahn/types/app_proposal.go @@ -15,20 +15,22 @@ type AppHash []byte // AppProposal . type AppProposal struct { utils.ReadOnly - globalNumber GlobalBlockNumber - roadIndex RoadIndex - appHash AppHash - epochIndex EpochIndex + epochIndex EpochIndex + roadIndex RoadIndex + globalRange GlobalRange + appHash AppHash } // NewAppProposal creates a new AppProposal. -func NewAppProposal(globalNumber GlobalBlockNumber, roadIndex RoadIndex, appHash AppHash, epochIndex EpochIndex) *AppProposal { - return &AppProposal{globalNumber: globalNumber, roadIndex: roadIndex, appHash: appHash, epochIndex: epochIndex} +func NewAppProposal(proposal *Proposal, appHash AppHash) *AppProposal { + return &AppProposal{ + globalRange: proposal.GlobalRange(), + roadIndex: proposal.Index(), + appHash: appHash, + epochIndex: proposal.EpochIndex(), + } } -// GlobalNumber . -func (m *AppProposal) GlobalNumber() GlobalBlockNumber { return m.globalNumber } - // RoadIndex returns the road index of the proposal. func (m *AppProposal) RoadIndex() RoadIndex { return m.roadIndex } @@ -38,7 +40,10 @@ func (m *AppProposal) AppHash() AppHash { return m.appHash } // EpochIndex returns the epoch this proposal belongs to. func (m *AppProposal) EpochIndex() EpochIndex { return m.epochIndex } -// Next is the next global block number to compute AppHash for. +// GlobalRange returns the global block range covered by the proposal. +func (m *AppProposal) GlobalRange() GlobalRange { return m.globalRange } + +// Next is the next road index after this proposal. func (m *AppProposal) Next() RoadIndex { return m.RoadIndex() + 1 } @@ -48,12 +53,12 @@ func (m *AppProposal) Verify(qc *CommitQC) error { if got, want := m.RoadIndex(), qc.Proposal().Index(); got != want { return fmt.Errorf("roadIndex() = %v, want %v", got, want) } - if got, want := m.GlobalNumber(), qc.GlobalRange(); got < want.First || got >= want.Next { - return fmt.Errorf("globalNumber() = %v, want in range [%v,%v)", got, want.First, want.Next) - } if got, want := m.EpochIndex(), qc.Proposal().EpochIndex(); got != want { return fmt.Errorf("epoch_index = %d, want %d", got, want) } + if got, want := m.GlobalRange(), qc.GlobalRange(); got != want { + return fmt.Errorf("global_range = %v, want %v", got, want) + } return nil } @@ -61,27 +66,34 @@ func (m *AppProposal) Verify(qc *CommitQC) error { var AppProposalConv = protoutils.Conv[*AppProposal, *pb.AppProposal]{ Encode: func(m *AppProposal) *pb.AppProposal { return &pb.AppProposal{ - GlobalNumber: utils.Alloc(uint64(m.globalNumber)), - RoadIndex: utils.Alloc(uint64(m.roadIndex)), - AppHash: m.appHash, - EpochIndex: utils.Alloc(uint64(m.epochIndex)), + RoadIndex: utils.Alloc(uint64(m.roadIndex)), + AppHash: m.appHash, + EpochIndex: utils.Alloc(uint64(m.epochIndex)), + GlobalFirst: utils.Alloc(uint64(m.globalRange.First)), + GlobalNext: utils.Alloc(uint64(m.globalRange.Next)), } }, Decode: func(m *pb.AppProposal) (*AppProposal, error) { - if m.GlobalNumber == nil { - return nil, fmt.Errorf("global_number: missing") - } if m.RoadIndex == nil { return nil, fmt.Errorf("road_index: missing") } if m.EpochIndex == nil { return nil, fmt.Errorf("epoch_index: missing") } + if m.GlobalFirst == nil { + return nil, fmt.Errorf("global_first: missing") + } + if m.GlobalNext == nil { + return nil, fmt.Errorf("global_next: missing") + } return &AppProposal{ - globalNumber: GlobalBlockNumber(*m.GlobalNumber), - roadIndex: RoadIndex(*m.RoadIndex), - appHash: AppHash(m.AppHash), - epochIndex: EpochIndex(*m.EpochIndex), + epochIndex: EpochIndex(*m.EpochIndex), + roadIndex: RoadIndex(*m.RoadIndex), + globalRange: GlobalRange{ + First: GlobalBlockNumber(*m.GlobalFirst), + Next: GlobalBlockNumber(*m.GlobalNext), + }, + appHash: AppHash(m.AppHash), }, nil }, } diff --git a/sei-tendermint/autobahn/types/block.go b/sei-tendermint/autobahn/types/block.go index d3e283bd23..951a37f049 100644 --- a/sei-tendermint/autobahn/types/block.go +++ b/sei-tendermint/autobahn/types/block.go @@ -117,8 +117,6 @@ type GlobalBlock struct { Timestamp time.Time GlobalNumber GlobalBlockNumber Payload *Payload - // Highest known finalized state. - FinalAppState utils.Option[*AppProposal] } // NewBlock creates a new Block. diff --git a/sei-tendermint/autobahn/types/block_db.go b/sei-tendermint/autobahn/types/block_db.go index b2f62b0936..138414d435 100644 --- a/sei-tendermint/autobahn/types/block_db.go +++ b/sei-tendermint/autobahn/types/block_db.go @@ -4,22 +4,22 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" ) -// BlockDB is the durable backing store for data.State. It persists the two -// kinds of finalized records the consensus state machine produces — -// finalized blocks (indexed by GlobalBlockNumber and by header hash) and -// FullCommitQCs (each covering a contiguous range of GlobalBlockNumbers) — +// BlockDB is the durable backing store for data.State. It persists the +// finalized records the consensus state machine produces — finalized blocks +// (indexed by GlobalBlockNumber and by header hash), FullCommitQCs (each +// covering a contiguous range of GlobalBlockNumbers), AppProposals and AppQCs // and provides the read API needed for crash recovery and runtime lookups. // // # Concurrency // // All methods are safe for concurrent use. Implementations should expect -// concurrent writes (WriteBlock + WriteQC interleaved from a single +// concurrent writes (WriteBlock + WriteQC + WriteAppQC interleaved from a single // background persistence loop) and concurrent reads from RPC handlers // and peer-sync streams. // // # Durability and crash safety // -// Writes are two-phase: WriteBlock and WriteQC return without +// Writes are two-phase: WriteBlock, WriteQC, and WriteAppQC return without // guaranteeing the record is on disk. Flush blocks until all // previously-returned Writes are durable. // @@ -38,27 +38,33 @@ import ( // Writes must be ordered, and the contract is enforced (not merely // expected): // -// - Blocks must be written densely: each block's number must be exactly -// one greater than the previously written block's (the first block may -// start anywhere its covering QC allows). WriteBlock returns -// ErrBlockOutOfOrder otherwise. -// - QCs must be written contiguously — each QC's GlobalRange().First -// must equal the previous QC's GlobalRange().Next. WriteQC returns -// ErrQCNonContiguous otherwise. -// - QCs must be written before blocks. A QC covering a block must -// be written before that block is written. +// - All data must be written contiguously: +// - FullCommitQC.GlobalRange().First must equal the previous FullCommitQC.GlobalRange().Next +// - Block.Number must equal the previous Block.Number + 1 +// - AppProposal.GlobalRange().First must equal the previous AppProposal.GlobalRange().Next +// - AppQC.Proposal().GlobalRange().First must equal the previous AppQC.Proposal().GlobalRange().Next +// - All data must be written in order: for every GlobalBlockNumber X, writes need to happen in order: +// - FullCommitQC covering X +// - Block X +// - AppProposal covering X +// - AppQC covering X +// - Call to PruneBefore(X) makes data before X inaccessible, however it still might be accessible after BlockDB is +// reopened, as the pruning may happen asynchronously. +// Only full rows (X such that FullCommitQC, Block, AppProposal and AppQC are in BlockDB) are eligible for pruning, +// and at least 1 full row (once written) needs to stay in BlockDB at all times. +// In particular: +// - PruneBefore is a noop until the first AppQC is written +// - If PruneBefore(X) called and the highest full row is Y, then only data in rows = r.First && idx <= r.Last } +// Has reports whether idx falls within this range. +func (r RoadRange) Has(idx RoadIndex) bool { return r.First <= idx && idx < r.Next } // Epoch holds the complete context for a single epoch. // Retrieved from the local Registry; never transmitted on the wire. diff --git a/sei-tendermint/autobahn/types/errors.go b/sei-tendermint/autobahn/types/errors.go index 64376fd724..e06a64f9a2 100644 --- a/sei-tendermint/autobahn/types/errors.go +++ b/sei-tendermint/autobahn/types/errors.go @@ -8,9 +8,8 @@ import "errors" // means the height is below the retention / eviction floor. var ErrNotFound = errors.New("not found") -// ErrBlockGap is returned when the persisted blocks are not contiguous, -// surfaced by BlockDBIterator.Next during a scan. WriteBlock rejects gapped -// writes, so a gap on disk indicates store corruption. +// ErrBlockGap is returned when persisted blocks are not contiguous. WriteBlock +// rejects gapped writes, so a gap on disk indicates store corruption. var ErrBlockGap = errors.New("block gap in BlockDB") // ErrBlockOutOfOrder is returned by WriteBlock when the supplied @@ -28,6 +27,27 @@ var ErrQCNonContiguous = errors.New("block: WriteQC non-contiguous") // before that block (see the BlockDB ordering contract). var ErrBlockMissingQC = errors.New("block: WriteBlock without covering QC") +// ErrAppProposalNonContiguous is returned by WriteAppProposal when the supplied +// AppProposal does not extend the existing AppProposal prefix. AppProposals must +// be written as a contiguous, ascending sequence aligned with the retained +// CommitQC prefix. +var ErrAppProposalNonContiguous = errors.New("block: WriteAppProposal non-contiguous") + +// ErrAppProposalMissingQC is returned by WriteAppProposal when no previously +// written CommitQC exactly matches the AppProposal's GlobalRange. The matching +// CommitQC must be written before the AppProposal. +var ErrAppProposalMissingQC = errors.New("block: WriteAppProposal without matching CommitQC") + +// ErrAppQCNonContiguous is returned by WriteAppQC when the supplied AppQC does +// not extend the existing AppQC prefix. AppQCs must be written as a contiguous, +// ascending sequence aligned with the retained CommitQC prefix. +var ErrAppQCNonContiguous = errors.New("block: WriteAppQC non-contiguous") + +// ErrAppQCMissingQC is returned by WriteAppQC when no previously written +// CommitQC exactly matches the AppQC's GlobalRange. The matching CommitQC must +// be written before the AppQC. +var ErrAppQCMissingQC = errors.New("block: WriteAppQC without matching CommitQC") + // ErrPruned is returned when a requested record is below the current retention // / eviction floor and is not served. Used for BlockDB by-number reads below // the store watermark, for BlockDB.Iterator when a concurrent PruneBefore moves diff --git a/sei-tendermint/autobahn/types/proposal.go b/sei-tendermint/autobahn/types/proposal.go index bb78f668c1..fadb9b0a49 100644 --- a/sei-tendermint/autobahn/types/proposal.go +++ b/sei-tendermint/autobahn/types/proposal.go @@ -68,6 +68,10 @@ type GlobalRange struct { Next GlobalBlockNumber } +func (g GlobalRange) String() string { + return fmt.Sprintf("[%v,%v)", g.First, g.Next) +} + // Len returns the number of global blocks in the range. func (g GlobalRange) Len() uint64 { return uint64(g.Next - g.First) @@ -110,7 +114,7 @@ func (v View) Verify(ep *Epoch) error { return fmt.Errorf("epoch_index = %d, want %d", got, want) } if rr := ep.RoadRange(); !rr.Has(v.Index) { - return fmt.Errorf("road_index %v not in epoch roads [%v, %v]", v.Index, rr.First, rr.Last) + return fmt.Errorf("road_index %v not in epoch roads [%v, %v)", v.Index, rr.First, rr.Next) } return nil } @@ -168,11 +172,10 @@ type Proposal struct { view View timestamp time.Time laneRanges map[LaneID]*LaneRange - app utils.Option[*AppProposal] globalRange GlobalRange } -func newProposal(view View, timestamp time.Time, laneRanges []*LaneRange, app utils.Option[*AppProposal], globalFirst GlobalBlockNumber) *Proposal { +func newProposal(view View, timestamp time.Time, laneRanges []*LaneRange, globalFirst GlobalBlockNumber) *Proposal { laneRangesM := map[LaneID]*LaneRange{} gr := GlobalRange{First: globalFirst, Next: globalFirst} for _, r := range laneRanges { @@ -186,7 +189,6 @@ func newProposal(view View, timestamp time.Time, laneRanges []*LaneRange, app ut timestamp: timestamp, laneRanges: laneRangesM, globalRange: gr, - app: app, } } @@ -199,9 +201,6 @@ func (m *Proposal) View() View { return m.view } // Timestamp of the proposal. func (m *Proposal) Timestamp() time.Time { return m.timestamp } -// App . -func (m *Proposal) App() utils.Option[*AppProposal] { return m.app } - // EpochIndex returns the epoch index encoded in the proposal. func (m *Proposal) EpochIndex() EpochIndex { return m.view.EpochIndex } @@ -266,7 +265,6 @@ type FullProposal struct { utils.ReadOnly proposal *Signed[*Proposal] laneQCs map[LaneID]*LaneQC - appQC utils.Option[*AppQC] timeoutQC utils.Option[*TimeoutQC] } @@ -297,7 +295,6 @@ func NewProposal( viewSpec ViewSpec, timestamp time.Time, laneQCs map[LaneID]*LaneQC, - appQC utils.Option[*AppQC], ) (*FullProposal, error) { committee := viewSpec.Epoch.Committee() if got, want := key.Public(), committee.Leader(viewSpec.View()); got != want { @@ -306,14 +303,13 @@ func NewProposal( if p, ok := NewReproposal(key, viewSpec); ok { return p, nil } - proposal, appQC, err := buildProposal(committee, viewSpec, timestamp, laneQCs, appQC) + proposal, err := buildProposal(committee, viewSpec, timestamp, laneQCs) if err != nil { return nil, err } return &FullProposal{ proposal: Sign(key, proposal), laneQCs: laneQCs, - appQC: appQC, timeoutQC: viewSpec.TimeoutQC, }, nil } @@ -326,45 +322,32 @@ func buildProposal( viewSpec ViewSpec, timestamp time.Time, laneQCs map[LaneID]*LaneQC, - appQC utils.Option[*AppQC], -) (*Proposal, utils.Option[*AppQC], error) { +) (*Proposal, error) { var laneRanges []*LaneRange for lane := range committee.Lanes().All() { first := LaneRangeOpt(viewSpec.CommitQC, lane).Next() if lQC, ok := laneQCs[lane]; ok { if lQC.Header().Lane() != lane { - return nil, appQC, fmt.Errorf("laneQC %v for lane %v", lQC.Header().Lane(), lane) + return nil, fmt.Errorf("laneQC %v for lane %v", lQC.Header().Lane(), lane) } laneRange := NewLaneRange(lane, first, utils.Some(lQC.Header())) if got := laneRange.Len(); got > MaxLaneRangeInProposal { - return nil, appQC, fmt.Errorf("laneRange[%v].Len() = %d, want <= %d", lane, got, MaxLaneRangeInProposal) + return nil, fmt.Errorf("laneRange[%v].Len() = %d, want <= %d", lane, got, MaxLaneRangeInProposal) } laneRanges = append(laneRanges, laneRange) } else { laneRanges = append(laneRanges, NewLaneRange(lane, first, utils.None[*BlockHeader]())) } } - app := ProposalOpt(appQC) - // If the new appProposal is not later than the previous one, then clear appQC. - if old := AppOpt(ProposalOpt(viewSpec.CommitQC)); NextOpt(app) <= NextOpt(old) { - app = old - appQC = utils.None[*AppQC]() - } - // If the new appProposal is from the future (which may happen if this node is behind), then clear appQC. - // The proposal will be useless in this case, but at least it will be valid. - if a, ok := app.Get(); ok && a.GlobalNumber() >= viewSpec.NextGlobalBlock() { - app = utils.None[*AppProposal]() - appQC = utils.None[*AppQC]() - } // Normalize the creation timestamp. if wantMin := viewSpec.NextTimestamp(); timestamp.Before(wantMin) { timestamp = wantMin } - proposal := newProposal(viewSpec.View(), timestamp, laneRanges, app, viewSpec.NextGlobalBlock()) + proposal := newProposal(viewSpec.View(), timestamp, laneRanges, viewSpec.NextGlobalBlock()) if proposal.GlobalRange().Len() == 0 { - return nil, appQC, errors.New("empty tipcut: need at least one LaneQC") + return nil, errors.New("empty tipcut: need at least one LaneQC") } - return proposal, appQC, nil + return proposal, nil } // NewProposalForTesting builds a FullProposal exactly like NewProposal but attaches the @@ -376,17 +359,15 @@ func NewProposalForTesting( viewSpec ViewSpec, timestamp time.Time, laneQCs map[LaneID]*LaneQC, - appQC utils.Option[*AppQC], sig *Signature, ) (*FullProposal, error) { - proposal, appQC, err := buildProposal(committee, viewSpec, timestamp, laneQCs, appQC) + proposal, err := buildProposal(committee, viewSpec, timestamp, laneQCs) if err != nil { return nil, err } return &FullProposal{ proposal: newSigned(proposal, sig), laneQCs: laneQCs, - appQC: appQC, timeoutQC: viewSpec.TimeoutQC, }, nil } @@ -447,7 +428,7 @@ func (m *FullProposal) Verify(vs ViewSpec) error { }) // Is this a reproposal? if want, ok := tQC.reproposal(); ok { - if len(m.laneQCs) > 0 || m.appQC.IsPresent() { + if len(m.laneQCs) > 0 { return errors.New("unnecessary data when reproposing") } if NewHashed(want).Hash() != m.proposal.hashed.hash { @@ -489,36 +470,6 @@ func (m *FullProposal) Verify(vs ViewSpec) error { }) } } - // Verify the appQC. - if got, wantMin := NextOpt(m.proposal.Msg().App()), NextOpt(AppOpt(ProposalOpt(vs.CommitQC))); got < wantMin { - return errors.New("AppProposal lower than in previous CommitQC") - } else if got == wantMin { - if m.appQC.IsPresent() { - return errors.New("unnecessary appQC") - } - } else { - app, _ := m.proposal.Msg().App().Get() - // TODO: relax to allow current_epoch-1 once epoch transitions are wired up. - if got, want := app.EpochIndex(), m.proposal.Msg().EpochIndex(); got != want { - return fmt.Errorf("app epoch_index %d != proposal epoch_index %d", got, want) - } - appQC, ok := m.appQC.Get() - if !ok { - return errors.New("appQC missing") - } - if appQC.vote.hash != NewHashed(NewAppVote(app)).hash { - return errors.New("appQC doesn't match the proposal") - } - s.Spawn(func() error { - if err := appQC.Verify(c); err != nil { - return fmt.Errorf("appQC: %w", err) - } - return nil - }) - if got, want := appQC.Proposal().GlobalNumber(), vs.NextGlobalBlock(); got >= want { - return fmt.Errorf("appQC for block %v, while only %v blocks were finalized", got, want) - } - } return nil }) } @@ -596,7 +547,6 @@ var ProposalConv = protoutils.Conv[*Proposal, *pb.Proposal]{ View: ViewConv.Encode(m.view), Timestamp: TimeConv.Encode(m.timestamp), LaneRanges: LaneRangeConv.EncodeSlice(laneRanges), - App: AppProposalConv.EncodeOpt(m.app), GlobalFirst: utils.Alloc(uint64(m.globalRange.First)), } }, @@ -613,17 +563,13 @@ var ProposalConv = protoutils.Conv[*Proposal, *pb.Proposal]{ if err != nil { return nil, fmt.Errorf("timestamp: %w", err) } - app, err := AppProposalConv.DecodeOpt(m.App) - if err != nil { - return nil, fmt.Errorf("appQC: %w", err) - } // Hard-reject messages with absent global_first/epoch_index. // Autobahn is pre-production; there is no rolling-upgrade path from // messages encoded before these fields were added. if m.GlobalFirst == nil { return nil, fmt.Errorf("global_first: missing") } - proposal := newProposal(view, timestamp, laneRanges, app, GlobalBlockNumber(*m.GlobalFirst)) + proposal := newProposal(view, timestamp, laneRanges, GlobalBlockNumber(*m.GlobalFirst)) if len(proposal.laneRanges) != len(laneRanges) { return nil, fmt.Errorf("laneRanges: duplicate ranges") } @@ -641,7 +587,6 @@ var FullProposalConv = protoutils.Conv[*FullProposal, *pb.FullProposal]{ return &pb.FullProposal{ ProposalV2: SignedProposalConv.Encode(m.proposal), LaneQcs: LaneQCConv.EncodeSlice(laneQCs), - AppQc: AppQCConv.EncodeOpt(m.appQC), TimeoutQc: TimeoutQCConv.EncodeOpt(m.timeoutQC), } }, @@ -658,14 +603,10 @@ var FullProposalConv = protoutils.Conv[*FullProposal, *pb.FullProposal]{ for _, qc := range laneQCs { laneQCsMap[qc.Header().Lane()] = qc } - appQC, err := AppQCConv.DecodeOpt(m.AppQc) - if err != nil { - return nil, fmt.Errorf("appQC: %w", err) - } timeoutQC, err := TimeoutQCConv.DecodeOpt(m.TimeoutQc) if err != nil { return nil, fmt.Errorf("timeoutQC: %w", err) } - return &FullProposal{proposal: proposal, laneQCs: laneQCsMap, appQC: appQC, timeoutQC: timeoutQC}, nil + return &FullProposal{proposal: proposal, laneQCs: laneQCsMap, timeoutQC: timeoutQC}, nil }, } diff --git a/sei-tendermint/autobahn/types/proposal_test.go b/sei-tendermint/autobahn/types/proposal_test.go index 0228ccc1c3..58a2b15154 100644 --- a/sei-tendermint/autobahn/types/proposal_test.go +++ b/sei-tendermint/autobahn/types/proposal_test.go @@ -60,17 +60,6 @@ func makeCommitQCFromProposal(keys []SecretKey, fp *FullProposal) *CommitQC { return NewCommitQC(votes) } -// makeAppQCFor creates an AppQC for the given parameters, signed by all keys. -func makeAppQCFor(keys []SecretKey, globalNum GlobalBlockNumber, roadIdx RoadIndex, appHash AppHash, epochIdx EpochIndex) *AppQC { - appProposal := NewAppProposal(globalNum, roadIdx, appHash, epochIdx) - vote := NewAppVote(appProposal) - var votes []*Signed[*AppVote] - for _, k := range keys { - votes = append(votes, Sign(k, vote)) - } - return NewAppQC(votes) -} - func TestProposalVerifyRejectsEmptyTipcut(t *testing.T) { rng := utils.TestRng() committee, _ := GenCommittee(rng, 4) @@ -80,7 +69,7 @@ func TestProposalVerifyRejectsEmptyTipcut(t *testing.T) { // Direct Proposal.Verify rejects empty tipcuts (no LaneQCs / zero GlobalRange). // Local propose waits via WaitForLaneQCs; verification must refuse empty tipcuts // from peers as well. - empty := newProposal(vs.View(), time.Now(), nil, utils.None[*AppProposal](), vs.NextGlobalBlock()) + empty := newProposal(vs.View(), time.Now(), nil, vs.NextGlobalBlock()) require.Equal(t, uint64(0), empty.GlobalRange().Len()) require.Error(t, empty.Verify(ep)) } @@ -105,7 +94,7 @@ func TestProposalVerifyFreshWithBlocks(t *testing.T) { laneQC := makeLaneQC(rng, committee, keys, lane, 0, GenBlockHeaderHash(rng)) fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), - map[LaneID]*LaneQC{lane: laneQC}, utils.None[*AppQC]())) + map[LaneID]*LaneQC{lane: laneQC})) require.NoError(t, fp.Verify(vs)) } @@ -123,7 +112,6 @@ func TestNewProposalRejectsLaneRangeLongerThanMaxLaneRangeInProposal(t *testing. vs, time.Now(), map[LaneID]*LaneQC{lane: laneQC}, - utils.None[*AppQC](), ) require.Error(t, err) } @@ -143,7 +131,6 @@ func TestProposalBlockTimestampStrictlyMonotone(t *testing.T) { map[LaneID]*LaneQC{ lane: makeLaneQC(rng, committee, keys, lane, 2, GenBlockHeaderHash(rng)), }, - utils.None[*AppQC](), )) p0 := firstProposal.Proposal().Msg() gr0 := p0.GlobalRange() @@ -165,7 +152,6 @@ func TestProposalBlockTimestampStrictlyMonotone(t *testing.T) { map[LaneID]*LaneQC{ lane: makeLaneQC(rng, committee, keys, lane, 3, GenBlockHeaderHash(rng)), }, - utils.None[*AppQC](), )) p1 := secondProposal.Proposal().Msg() gr1 := p1.GlobalRange() @@ -183,7 +169,7 @@ func TestProposalVerifyRejectsNonMonotoneTimestamp(t *testing.T) { ep := NewEpoch(GenEpochIndex(rng), OpenRoadRange(), genesisTimestamp, committee, GlobalBlockNumber(rng.Uint64()%1000000)+1) vs := ViewSpec{Epoch: ep} k := leaderKey(committee, keys, vs.View()) - fp := utils.OrPanic1(NewProposal(k, vs, genesisTimestamp, oneLaneQCMap(rng, committee, keys, vs), utils.None[*AppQC]())) + fp := utils.OrPanic1(NewProposal(k, vs, genesisTimestamp, oneLaneQCMap(rng, committee, keys, vs))) require.NoError(t, fp.Verify(vs)) vsLater := vs @@ -204,13 +190,11 @@ func TestProposalVerifyRejectsNonMonotoneTimestamp(t *testing.T) { proposer0, vs0, time.Now(), map[LaneID]*LaneQC{lane: lQC}, - utils.None[*AppQC](), )) fp0b := utils.OrPanic1(NewProposal( proposer0, vs0, fp0a.Proposal().Msg().NextTimestamp().Add(time.Hour), map[LaneID]*LaneQC{lane: lQC}, - utils.None[*AppQC](), )) vs1a := ViewSpec{CommitQC: utils.Some(makeCommitQCFromProposal(keys, fp0a)), Epoch: ep} @@ -221,7 +205,6 @@ func TestProposalVerifyRejectsNonMonotoneTimestamp(t *testing.T) { proposer1, vs1a, fp0a.Proposal().Msg().NextTimestamp(), oneLaneQCMap(rng, committee, keys, vs1a), - utils.None[*AppQC](), )) require.NoError(t, fp1a.Verify(vs1a)) @@ -237,7 +220,7 @@ func TestProposalVerifyRejectsViewMismatch(t *testing.T) { // Build a valid proposal at genesis view (0, 0). vs0 := ViewSpec{Epoch: ep} leader0 := leaderKey(committee, keys, vs0.View()) - fp := utils.OrPanic1(NewProposal(leader0, vs0, time.Now(), oneLaneQCMap(rng, committee, keys, vs0), utils.None[*AppQC]())) + fp := utils.OrPanic1(NewProposal(leader0, vs0, time.Now(), oneLaneQCMap(rng, committee, keys, vs0))) // Verify it against a different ViewSpec (view 1, 0). commitQC := makeCommitQCFromProposal(keys, fp) @@ -254,8 +237,8 @@ func TestProposalVerifyRejectsForgedSignature(t *testing.T) { proposerKey := leaderKey(committee, keys, vs.View()) // Build two valid proposals with different timestamps. - fp1 := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.None[*AppQC]())) - fp2 := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now().Add(time.Hour), oneLaneQCMap(rng, committee, keys, vs), utils.None[*AppQC]())) + fp1 := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs))) + fp2 := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now().Add(time.Hour), oneLaneQCMap(rng, committee, keys, vs))) // Graft fp1's signature onto fp2 (different content). fp2.proposal.sig = fp1.proposal.sig @@ -270,7 +253,7 @@ func TestProposalVerifyRejectsWrongProposer(t *testing.T) { vs := ViewSpec{Epoch: ep} correctLeader := leaderKey(committee, keys, vs.View()) - fp := utils.OrPanic1(NewProposal(correctLeader, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.None[*AppQC]())) + fp := utils.OrPanic1(NewProposal(correctLeader, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs))) // Re-sign the same proposal with a different (non-leader) key. var wrongKey SecretKey @@ -283,7 +266,6 @@ func TestProposalVerifyRejectsWrongProposer(t *testing.T) { tamperedFP := &FullProposal{ proposal: Sign(wrongKey, fp.Proposal().Msg()), laneQCs: fp.laneQCs, - appQC: fp.appQC, timeoutQC: fp.timeoutQC, } err := tamperedFP.Verify(vs) @@ -297,7 +279,7 @@ func TestProposalVerifyRejectsInconsistentTimeoutQC(t *testing.T) { vs := ViewSpec{Epoch: ep} // no timeoutQC proposerKey := leaderKey(committee, keys, vs.View()) - fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.None[*AppQC]())) + fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs))) // Attach a timeoutQC that the ViewSpec doesn't expect. var timeoutVotes []*FullTimeoutVote @@ -309,7 +291,6 @@ func TestProposalVerifyRejectsInconsistentTimeoutQC(t *testing.T) { tamperedFP := &FullProposal{ proposal: fp.proposal, laneQCs: fp.laneQCs, - appQC: fp.appQC, timeoutQC: utils.Some(tQC), } err := tamperedFP.Verify(vs) @@ -323,7 +304,7 @@ func TestProposalVerifyRejectsNonCommitteeLane(t *testing.T) { vs := ViewSpec{Epoch: ep} proposerKey := leaderKey(committee, keys, vs.View()) - fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.None[*AppQC]())) + fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs))) // Keep the non-empty committee tipcut and add a non-committee lane. // LaneRange.Verify rejects X because it's not a committee lane. @@ -337,11 +318,10 @@ func TestProposalVerifyRejectsNonCommitteeLane(t *testing.T) { } tamperedRanges = append(tamperedRanges, NewLaneRange(extraLane, 0, utils.None[*BlockHeader]())) - tamperedProposal := newProposal(origProposal.view, origProposal.timestamp, tamperedRanges, origProposal.app, origProposal.GlobalRange().First) + tamperedProposal := newProposal(origProposal.view, origProposal.timestamp, tamperedRanges, origProposal.GlobalRange().First) maliciousFP := &FullProposal{ proposal: Sign(proposerKey, tamperedProposal), laneQCs: fp.laneQCs, - appQC: fp.appQC, timeoutQC: fp.timeoutQC, } err := maliciousFP.Verify(vs) @@ -355,7 +335,7 @@ func TestProposalVerifyAcceptsImplicitLaneRange(t *testing.T) { vs := ViewSpec{Epoch: ep} proposerKey := leaderKey(committee, keys, vs.View()) - fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.None[*AppQC]())) + fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs))) // Drop one empty lane — the omitted lane gets an implicit [0, 0) range, // which matches the expected first=0 at genesis. Keep the non-empty range @@ -372,7 +352,7 @@ func TestProposalVerifyAcceptsImplicitLaneRange(t *testing.T) { } require.True(t, droppedEmpty) - shortProposal := newProposal(origP.view, origP.timestamp, keptRanges, origP.app, origP.GlobalRange().First) + shortProposal := newProposal(origP.view, origP.timestamp, keptRanges, origP.GlobalRange().First) shortFP := &FullProposal{ proposal: Sign(proposerKey, shortProposal), laneQCs: fp.laneQCs, @@ -387,7 +367,7 @@ func TestProposalVerifyAcceptsNonContiguousImplicitRanges(t *testing.T) { vs := ViewSpec{Epoch: ep} proposerKey := leaderKey(committee, keys, vs.View()) - fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.None[*AppQC]())) + fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs))) // Drop every other empty lane (keep the non-empty range and its LaneQC). origP := fp.Proposal().Msg() @@ -404,7 +384,7 @@ func TestProposalVerifyAcceptsNonContiguousImplicitRanges(t *testing.T) { keptRanges = append(keptRanges, r) } - shortProposal := newProposal(origP.view, origP.timestamp, keptRanges, origP.app, origP.GlobalRange().First) + shortProposal := newProposal(origP.view, origP.timestamp, keptRanges, origP.GlobalRange().First) shortFP := &FullProposal{ proposal: Sign(proposerKey, shortProposal), laneQCs: fp.laneQCs, @@ -419,7 +399,7 @@ func TestProposalVerifyRejectsLaneRangeFirstMismatch(t *testing.T) { vs := ViewSpec{Epoch: ep} proposerKey := leaderKey(committee, keys, vs.View()) - fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.None[*AppQC]())) + fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs))) // Tamper the non-empty lane's First (genesis expects 0) while keeping a // non-empty range and a matching LaneQC so Verify reaches the first-mismatch check. @@ -441,7 +421,7 @@ func TestProposalVerifyRejectsLaneRangeFirstMismatch(t *testing.T) { tamperedRanges = append(tamperedRanges, r) } } - tamperedProposal := newProposal(origP.view, origP.timestamp, tamperedRanges, origP.app, origP.GlobalRange().First) + tamperedProposal := newProposal(origP.view, origP.timestamp, tamperedRanges, origP.GlobalRange().First) tamperedFP := &FullProposal{ proposal: Sign(proposerKey, tamperedProposal), laneQCs: map[LaneID]*LaneQC{target: badQC}, @@ -462,7 +442,7 @@ func TestProposalVerifyRejectsMissingLaneQC(t *testing.T) { // Build a valid proposal with a block, then strip the laneQC. fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), - map[LaneID]*LaneQC{lane: laneQC}, utils.None[*AppQC]())) + map[LaneID]*LaneQC{lane: laneQC})) tamperedFP := &FullProposal{ proposal: fp.proposal, @@ -484,7 +464,7 @@ func TestProposalVerifyRejectsLaneQCBlockNumberMismatch(t *testing.T) { // Build a valid proposal with a QC certifying block 1 (range [0, 2)). goodQC := makeLaneQC(rng, committee, keys, lane, 1, GenBlockHeaderHash(rng)) fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), - map[LaneID]*LaneQC{lane: goodQC}, utils.None[*AppQC]())) + map[LaneID]*LaneQC{lane: goodQC})) // Swap in a QC certifying block 0 — range expects block 1. wrongQC := makeLaneQC(rng, committee, keys, lane, 0, GenBlockHeaderHash(rng)) @@ -519,7 +499,7 @@ func TestProposalVerifyRejectsInvalidLaneQCSignature(t *testing.T) { badLaneQC := NewLaneQC(badVotes) fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), - map[LaneID]*LaneQC{lane: badLaneQC}, utils.None[*AppQC]())) + map[LaneID]*LaneQC{lane: badLaneQC})) err := fp.Verify(vs) require.Error(t, err) @@ -547,7 +527,6 @@ func TestProposalVerifyRejectsLaneRangeLongerThanMaxLaneRangeInProposal(t *testi View{}, time.Now(), []*LaneRange{NewLaneRange(lane, 0, utils.Some(NewBlock(lane, MaxLaneRangeInProposal, GenBlockHeaderHash(rng), GenPayload(rng)).Header()))}, - utils.None[*AppProposal](), ep.FirstBlock(), ) require.Error(t, oversized.Verify(ep)) @@ -566,7 +545,6 @@ func makeFullProposal( leaderKey(committee, keys, vs.View()), vs, time.Now(), laneQCs, - appQC, )) } @@ -579,144 +557,6 @@ func makeCommitQC(keys []SecretKey, fullProposal *FullProposal) *CommitQC { return NewCommitQC(votes) } -func TestProposalVerifyRejectsAppProposalLowerThanPrevious(t *testing.T) { - rng := utils.TestRng() - committee, keys := GenCommittee(rng, 4) - ep := genFreshEpoch(rng, committee) - - // Construct commitQC for index 1 with AppProposal - // and Proposal for index 2 without any app proposal. - // Such a proposal should fail validation, because app proposals need to be monotone. - l := keys[0].Public() - lQCs := map[LaneID]*LaneQC{l: makeLaneQC(rng, committee, keys, l, 0, GenBlockHeaderHash(rng))} - commitQC0 := makeCommitQC(keys, makeFullProposal(ep, keys, utils.None[*CommitQC](), lQCs, utils.None[*AppQC]())) - appQC0 := makeAppQCFor(keys, commitQC0.GlobalRange().First, 0, GenAppHash(rng), ep.EpochIndex()) - vs1 := ViewSpec{CommitQC: utils.Some(commitQC0), Epoch: ep} - commitQC1a := makeCommitQC(keys, makeFullProposal(ep, keys, utils.Some(commitQC0), oneLaneQCMap(rng, committee, keys, vs1), utils.Some(appQC0))) - commitQC1b := makeCommitQC(keys, makeFullProposal(ep, keys, utils.Some(commitQC0), oneLaneQCMap(rng, committee, keys, vs1), utils.None[*AppQC]())) - vs := ViewSpec{CommitQC: utils.Some(commitQC1a), Epoch: ep} - fp2a := makeFullProposal(ep, keys, utils.Some(commitQC1a), oneLaneQCMap(rng, committee, keys, vs), utils.None[*AppQC]()) - fp2b := makeFullProposal(ep, keys, utils.Some(commitQC1b), oneLaneQCMap(rng, committee, keys, vs), utils.None[*AppQC]()) - - // We construct the invalid proposal by constructing 2 alternative futures: one with appQC, one without. - require.NoError(t, fp2a.Verify(vs)) - require.Error(t, fp2b.Verify(vs)) -} - -func TestProposalVerifyRejectsUnnecessaryAppQC(t *testing.T) { - rng := utils.TestRng() - committee, keys := GenCommittee(rng, 4) - ep := genFreshEpoch(rng, committee) - vs := ViewSpec{Epoch: ep} // no previous commitQC, so app starts at None - - leader := leaderKey(committee, keys, vs.View()) - fp := utils.OrPanic1(NewProposal(leader, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.None[*AppQC]())) - - // Attach an unrequested AppQC. - appQC := makeAppQCFor(keys, ep.FirstBlock(), 0, GenAppHash(rng), ep.EpochIndex()) - tamperedFP := &FullProposal{ - proposal: fp.proposal, - laneQCs: fp.laneQCs, - appQC: utils.Some(appQC), - timeoutQC: fp.timeoutQC, - } - err := tamperedFP.Verify(vs) - require.Error(t, err) -} - -func TestProposalVerifyRejectsMissingAppQC(t *testing.T) { - rng := utils.TestRng() - committee, keys := GenCommittee(rng, 4) - ep := genFreshEpoch(rng, committee) // firstBlock >= 1, so firstBlock-1 is valid - vs := ViewSpec{Epoch: ep} // no previous commitQC - leader := leaderKey(committee, keys, vs.View()) - - // Build a valid proposal with an AppQC, then strip it. - goodAppQC := makeAppQCFor(keys, ep.FirstBlock()-1, 0, GenAppHash(rng), ep.EpochIndex()) - fp := utils.OrPanic1(NewProposal(leader, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.Some(goodAppQC))) - - tamperedFP := &FullProposal{ - proposal: fp.proposal, - laneQCs: fp.laneQCs, - } - err := tamperedFP.Verify(vs) - require.Error(t, err) -} - -func TestProposalVerifyRejectsAppQCMismatch(t *testing.T) { - rng := utils.TestRng() - committee, keys := GenCommittee(rng, 4) - ep := genFreshEpoch(rng, committee) - vs := ViewSpec{Epoch: ep} - leader := leaderKey(committee, keys, vs.View()) - - // Build a valid proposal with an AppQC, then swap in a different one. - goodAppQC := makeAppQCFor(keys, ep.FirstBlock(), 0, GenAppHash(rng), ep.EpochIndex()) - fp := utils.OrPanic1(NewProposal(leader, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.Some(goodAppQC))) - - differentAppQC := makeAppQCFor(keys, ep.FirstBlock(), 0, GenAppHash(rng), ep.EpochIndex()) - tamperedFP := &FullProposal{ - proposal: fp.proposal, - laneQCs: fp.laneQCs, - appQC: utils.Some(differentAppQC), - } - err := tamperedFP.Verify(vs) - require.Error(t, err) -} - -func TestProposalVerifyRejectsAppProposalWrongEpoch(t *testing.T) { - rng := utils.TestRng() - committee, keys := GenCommittee(rng, 4) - - // firstBlock=1 so NextGlobalBlock()=1 and globalNumber=0 is a valid app target. - vs := ViewSpec{Epoch: NewEpoch(0, OpenRoadRange(), time.Time{}, committee, 1)} - leader := leaderKey(committee, keys, vs.View()) - - makeAppQCWithEpoch := func(epochIdx EpochIndex) *AppQC { - p := NewAppProposal(0, 0, GenAppHash(rng), epochIdx) - v := NewAppVote(p) - var votes []*Signed[*AppVote] - for _, k := range keys { - votes = append(votes, Sign(k, v)) - } - return NewAppQC(votes) - } - - // app epoch matches proposal epoch — accepted. - fp := utils.OrPanic1(NewProposal(leader, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.Some(makeAppQCWithEpoch(0)))) - require.NoError(t, fp.Verify(vs)) - - // app epoch differs from proposal epoch — rejected. - fpWrong := utils.OrPanic1(NewProposal(leader, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.Some(makeAppQCWithEpoch(1)))) - require.Error(t, fpWrong.Verify(vs)) -} - -func TestProposalVerifyRejectsInvalidAppQCSignature(t *testing.T) { - rng := utils.TestRng() - committee, keys := GenCommittee(rng, 4) - ep := genFreshEpoch(rng, committee) - vs := ViewSpec{Epoch: ep} - leader := leaderKey(committee, keys, vs.View()) - - appHash := GenAppHash(rng) - goodAppQC := makeAppQCFor(keys, ep.FirstBlock(), 0, appHash, ep.EpochIndex()) - fp := utils.OrPanic1(NewProposal(leader, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.Some(goodAppQC))) - - // Swap in an AppQC signed by NON-committee keys (same hash). - otherKeys := make([]SecretKey, len(keys)) - for i := range otherKeys { - otherKeys[i] = GenSecretKey(rng) - } - badAppQC := makeAppQCFor(otherKeys, ep.FirstBlock(), 0, appHash, ep.EpochIndex()) - tamperedFP := &FullProposal{ - proposal: fp.proposal, - laneQCs: fp.laneQCs, - appQC: utils.Some(badAppQC), - } - err := tamperedFP.Verify(vs) - require.Error(t, err) -} - func TestProposalVerifyRejectsLaneQCHeaderHashMismatch(t *testing.T) { rng := utils.TestRng() committee, keys := GenCommittee(rng, 4) @@ -729,7 +569,7 @@ func TestProposalVerifyRejectsLaneQCHeaderHashMismatch(t *testing.T) { // Build a valid proposal with a QC for block 0. realQC := makeLaneQC(rng, committee, keys, lane, 0, GenBlockHeaderHash(rng)) fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), - map[LaneID]*LaneQC{lane: realQC}, utils.None[*AppQC]())) + map[LaneID]*LaneQC{lane: realQC})) // Swap in a different QC for block 0 (different payload → different hash). differentQC := makeLaneQC(rng, committee, keys, lane, 0, GenBlockHeaderHash(rng)) @@ -755,7 +595,7 @@ func TestProposalVerifyValidReproposal(t *testing.T) { lane := committee.Leader(vs0.View()) laneQC0 := makeLaneQC(rng, committee, keys, lane, 0, GenBlockHeaderHash(rng)) fp0 := utils.OrPanic1(NewProposal(leader0, vs0, time.Now(), - map[LaneID]*LaneQC{lane: laneQC0}, utils.None[*AppQC]())) + map[LaneID]*LaneQC{lane: laneQC0})) // Build a PrepareQC for the proposal at (0, 0). var prepareVotes []*Signed[*PrepareVote] @@ -775,7 +615,7 @@ func TestProposalVerifyValidReproposal(t *testing.T) { require.Equal(t, View{Index: 0, Number: 1, EpochIndex: ep.EpochIndex()}, vs1.View()) leader1 := leaderKey(committee, keys, vs1.View()) - reproposal := utils.OrPanic1(NewProposal(leader1, vs1, time.Now(), oneLaneQCMap(rng, committee, keys, vs1), utils.None[*AppQC]())) + reproposal := utils.OrPanic1(NewProposal(leader1, vs1, time.Now(), oneLaneQCMap(rng, committee, keys, vs1))) // Reproposal must carry the same GlobalRange as the original. require.Equal(t, fp0.Proposal().Msg().GlobalRange(), reproposal.Proposal().Msg().GlobalRange()) @@ -790,7 +630,7 @@ func TestProposalVerifyRejectsReproposalWithUnnecessaryData(t *testing.T) { // Build a PrepareQC at (0, 0). vs0 := ViewSpec{Epoch: ep} leader0 := leaderKey(committee, keys, vs0.View()) - fp0 := utils.OrPanic1(NewProposal(leader0, vs0, time.Now(), oneLaneQCMap(rng, committee, keys, vs0), utils.None[*AppQC]())) + fp0 := utils.OrPanic1(NewProposal(leader0, vs0, time.Now(), oneLaneQCMap(rng, committee, keys, vs0))) var prepareVotes []*Signed[*PrepareVote] for _, k := range keys { @@ -808,7 +648,7 @@ func TestProposalVerifyRejectsReproposalWithUnnecessaryData(t *testing.T) { leader1 := leaderKey(committee, keys, vs1.View()) // Create a valid reproposal, then tamper it with unnecessary laneQCs. - reproposal := utils.OrPanic1(NewProposal(leader1, vs1, time.Now(), oneLaneQCMap(rng, committee, keys, vs1), utils.None[*AppQC]())) + reproposal := utils.OrPanic1(NewProposal(leader1, vs1, time.Now(), oneLaneQCMap(rng, committee, keys, vs1))) lane := keys[0].Public() laneQC := makeLaneQC(rng, committee, keys, lane, 0, GenBlockHeaderHash(rng)) @@ -829,7 +669,7 @@ func TestProposalVerifyRejectsReproposalHashMismatch(t *testing.T) { // Build a PrepareQC at (0, 0). vs0 := ViewSpec{Epoch: ep} leader0 := leaderKey(committee, keys, vs0.View()) - fp0 := utils.OrPanic1(NewProposal(leader0, vs0, time.Now(), oneLaneQCMap(rng, committee, keys, vs0), utils.None[*AppQC]())) + fp0 := utils.OrPanic1(NewProposal(leader0, vs0, time.Now(), oneLaneQCMap(rng, committee, keys, vs0))) var prepareVotes []*Signed[*PrepareVote] for _, k := range keys { @@ -847,14 +687,14 @@ func TestProposalVerifyRejectsReproposalHashMismatch(t *testing.T) { leader1 := leaderKey(committee, keys, vs1.View()) // Build the valid reproposal, then tamper its timestamp to get a different hash. - reproposal := utils.OrPanic1(NewProposal(leader1, vs1, time.Now(), oneLaneQCMap(rng, committee, keys, vs1), utils.None[*AppQC]())) + reproposal := utils.OrPanic1(NewProposal(leader1, vs1, time.Now(), oneLaneQCMap(rng, committee, keys, vs1))) origP := reproposal.Proposal().Msg() var ranges []*LaneRange for _, r := range origP.laneRanges { ranges = append(ranges, r) } - wrongP := newProposal(origP.view, time.Now().Add(time.Hour), ranges, origP.app, origP.GlobalRange().First) + wrongP := newProposal(origP.view, time.Now().Add(time.Hour), ranges, origP.GlobalRange().First) wrongFP := &FullProposal{ proposal: Sign(leader1, wrongP), timeoutQC: reproposal.timeoutQC, @@ -882,7 +722,7 @@ func TestProposalVerifyRejectsInvalidTimeoutQCSignature(t *testing.T) { vs := ViewSpec{TimeoutQC: utils.Some(badTimeoutQC), Epoch: ep} leader := leaderKey(committee, keys, vs.View()) - fp := utils.OrPanic1(NewProposal(leader, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.None[*AppQC]())) + fp := utils.OrPanic1(NewProposal(leader, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs))) err := fp.Verify(vs) require.Error(t, err) diff --git a/sei-tendermint/autobahn/types/testonly.go b/sei-tendermint/autobahn/types/testonly.go index ffc3d5a73d..e33fef6a46 100644 --- a/sei-tendermint/autobahn/types/testonly.go +++ b/sei-tendermint/autobahn/types/testonly.go @@ -21,7 +21,6 @@ func BuildCommitQC( keys []SecretKey, prev utils.Option[*CommitQC], laneQCs map[LaneID]*LaneQC, - appQC utils.Option[*AppQC], ) *CommitQC { vs := ViewSpec{CommitQC: prev, Epoch: epoch} if len(laneQCs) == 0 { @@ -35,7 +34,7 @@ func BuildCommitQC( break } } - proposal := utils.OrPanic1(NewProposal(leaderKey, vs, time.Now(), laneQCs, appQC)) + proposal := utils.OrPanic1(NewProposal(leaderKey, vs, time.Now(), laneQCs)) votes := make([]*Signed[*CommitVote], 0, len(keys)) for _, k := range keys { votes = append(votes, Sign(k, NewCommitVote(proposal.Proposal().Msg()))) @@ -295,7 +294,7 @@ func GenEpochWithCommittee(rng utils.Rng, committee *Committee) *Epoch { first := RoadIndex(rng.Uint64() % 1000) return NewEpoch( GenEpochIndex(rng), - RoadRange{First: first, Last: first + RoadIndex(rng.Uint64()%10000) + 10}, + RoadRange{First: first, Next: first + RoadIndex(rng.Uint64()%10000) + 10}, utils.GenTimestamp(rng), committee, GlobalBlockNumber(rng.Uint64()%1000000)+1, @@ -314,12 +313,12 @@ func CommitQCAt(ep *Epoch, keys []SecretKey) *CommitQC { // GenProposal generates a random Proposal. func GenProposal(rng utils.Rng) *Proposal { - return newProposal(GenView(rng), utils.GenTimestamp(rng), utils.GenSlice(rng, GenLaneRange), utils.Some(GenAppProposal(rng)), GlobalBlockNumber(rng.Uint64())) + return newProposal(GenView(rng), utils.GenTimestamp(rng), utils.GenSlice(rng, GenLaneRange), GlobalBlockNumber(rng.Uint64())) } // GenProposalAt generates a Proposal at a specific view. func GenProposalAt(rng utils.Rng, view View) *Proposal { - return newProposal(view, utils.GenTimestamp(rng), utils.GenSlice(rng, GenLaneRange), utils.Some(GenAppProposal(rng)), GlobalBlockNumber(rng.Uint64())) + return newProposal(view, utils.GenTimestamp(rng), utils.GenSlice(rng, GenLaneRange), GlobalBlockNumber(rng.Uint64())) } // ProposalAt returns a minimal non-empty Proposal at view, consistent with ep. @@ -330,7 +329,7 @@ func ProposalAt(ep *Epoch, view View) *Proposal { view.EpochIndex = ep.EpochIndex() lane := ep.Committee().Lanes().At(0) header := NewBlock(lane, 0, BlockHeaderHash{}, &Payload{}).Header() - return newProposal(view, time.Time{}, []*LaneRange{NewLaneRange(lane, 0, utils.Some(header))}, utils.None[*AppProposal](), ep.FirstBlock()) + return newProposal(view, time.Time{}, []*LaneRange{NewLaneRange(lane, 0, utils.Some(header))}, ep.FirstBlock()) } // GenProposalForEpoch generates a Proposal at a specific view whose epochIndex, @@ -340,7 +339,7 @@ func GenProposalForEpoch(rng utils.Rng, ep *Epoch, view View) *Proposal { view.EpochIndex = ep.EpochIndex() c := ep.Committee() laneRanges := utils.GenSlice(rng, func(rng utils.Rng) *LaneRange { return GenLaneRangeFor(rng, c) }) - return newProposal(view, utils.GenTimestamp(rng), laneRanges, utils.Some(GenAppProposal(rng)), ep.FirstBlock()) + return newProposal(view, utils.GenTimestamp(rng), laneRanges, ep.FirstBlock()) } // GenAppHash generates a random AppHash. @@ -350,7 +349,7 @@ func GenAppHash(rng utils.Rng) AppHash { // GenAppProposal generates a random AppProposal. func GenAppProposal(rng utils.Rng) *AppProposal { - return NewAppProposal(GenGlobalBlockNumber(rng), GenRoadIndex(rng), GenAppHash(rng), GenEpochIndex(rng)) + return NewAppProposal(GenProposal(rng), GenAppHash(rng)) } // GenAppVote generates a random AppVote. @@ -376,7 +375,6 @@ func GenFullProposal(rng utils.Rng) *FullProposal { return &FullProposal{ proposal: GenSigned(rng, GenProposal(rng)), laneQCs: laneQCs, - appQC: utils.Some(GenAppQC(rng)), timeoutQC: utils.Some(GenTimeoutQC(rng)), } } @@ -389,9 +387,8 @@ func GenGlobalBlockNumber(rng utils.Rng) GlobalBlockNumber { // GenGlobalBlock generates a random GlobalBlock. func GenGlobalBlock(rng utils.Rng) *GlobalBlock { return &GlobalBlock{ - GlobalNumber: GenGlobalBlockNumber(rng), - Payload: GenPayload(rng), - FinalAppState: utils.Some(GenAppProposal(rng)), + GlobalNumber: GenGlobalBlockNumber(rng), + Payload: GenPayload(rng), } } diff --git a/sei-tendermint/autobahn/types/timeout.go b/sei-tendermint/autobahn/types/timeout.go index d870891a13..f043796ca4 100644 --- a/sei-tendermint/autobahn/types/timeout.go +++ b/sei-tendermint/autobahn/types/timeout.go @@ -215,7 +215,7 @@ func (m *TimeoutQC) reproposal() (*Proposal, bool) { for _, l := range p.laneRanges { laneRanges = append(laneRanges, l) } - return newProposal(m.View().Next(), p.Timestamp(), laneRanges, p.App(), p.GlobalRange().First), true + return newProposal(m.View().Next(), p.Timestamp(), laneRanges, p.GlobalRange().First), true } // TimeoutVoteConv is the protobuf converter for TimeoutVote. diff --git a/sei-tendermint/autobahn/types/types_test.go b/sei-tendermint/autobahn/types/types_test.go index b82de8462c..7f57a70c99 100644 --- a/sei-tendermint/autobahn/types/types_test.go +++ b/sei-tendermint/autobahn/types/types_test.go @@ -116,7 +116,7 @@ func TestNewTimeoutQC(t *testing.T) { Number: GenViewNumber(rng) % view.Number, EpochIndex: view.EpochIndex, } - p := newProposal(pView, utils.GenTimestamp(rng), utils.GenSlice(rng, GenLaneRange), utils.Some(GenAppProposal(rng)), GlobalBlockNumber(rng.Uint64())) + p := newProposal(pView, utils.GenTimestamp(rng), utils.GenSlice(rng, GenLaneRange), GlobalBlockNumber(rng.Uint64())) if wantView.Less(pView) { wantView = pView } diff --git a/sei-tendermint/autobahn/types/wireguard_test.go b/sei-tendermint/autobahn/types/wireguard_test.go index cbebd32535..cd6745e232 100644 --- a/sei-tendermint/autobahn/types/wireguard_test.go +++ b/sei-tendermint/autobahn/types/wireguard_test.go @@ -143,7 +143,7 @@ func TestFullCommitQCWireguardAcceptsMaxValidatorsAndHeaders(t *testing.T) { } laneRanges = append(laneRanges, NewLaneRange(lane, 0, utils.Some(lastHeader))) } - proposal := newProposal(View{}, time.Unix(1, 2), laneRanges, utils.None[*AppProposal](), 0) + proposal := newProposal(View{}, time.Unix(1, 2), laneRanges, 0) vote := NewCommitVote(proposal) votes := make([]*Signed[*CommitVote], len(keys)) for i, key := range keys { @@ -200,7 +200,6 @@ func TestFullProposalWireguardAcceptsMaxValidators(t *testing.T) { ViewSpec{Epoch: NewEpoch(0, OpenRoadRange(), time.Time{}, committee, 0)}, time.Unix(1, 2), laneQCs, - utils.None[*AppQC](), ) require.NoError(t, err) diff --git a/sei-tendermint/internal/autobahn/autobahn.proto b/sei-tendermint/internal/autobahn/autobahn.proto index 3b04dd242f..c08b2bc15e 100644 --- a/sei-tendermint/internal/autobahn/autobahn.proto +++ b/sei-tendermint/internal/autobahn/autobahn.proto @@ -215,16 +215,21 @@ message AppQC { } message AppProposal { + reserved 1; + reserved "global_number"; option (hashable.hashable) = true; option (wireguard.sized) = true; - // Global block number. - optional uint64 global_number = 1; // required + + // Epoch this proposal belongs to. + optional uint64 epoch_index = 4; // required // Index of the commit qc finalizing the block. optional uint64 road_index = 2; // required - // App hash at that block. + // Range of global blocks convered by this proposal. + // Has to match the corresponding CommitQC. + optional uint64 global_first = 5; // required + optional uint64 global_next = 6; // required + // App hash of the state at the end of the range above. optional bytes app_hash = 3 [(wireguard.max_size) = 32]; // required - // Epoch this proposal belongs to. - optional uint64 epoch_index = 4; // required } // This is the signable message. diff --git a/sei-tendermint/internal/autobahn/avail/app_votes.go b/sei-tendermint/internal/autobahn/avail/app_votes.go index 7dce877719..b80ae9107e 100644 --- a/sei-tendermint/internal/autobahn/avail/app_votes.go +++ b/sei-tendermint/internal/autobahn/avail/app_votes.go @@ -4,6 +4,7 @@ import ( "log/slog" "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/sei-protocol/seilog" ) @@ -14,40 +15,48 @@ type voteSet[V any] struct { votes []V } -type appVotes struct { - byKey map[types.PublicKey]*types.Signed[*types.AppVote] - byHash map[types.Hash[*types.AppVote]]*voteSet[*types.Signed[*types.AppVote]] +type road struct { + epoch *types.Epoch + commitQC *types.CommitQC + appByKey map[types.PublicKey]struct{} + appByHash map[types.Hash[*types.AppVote]]*voteSet[*types.Signed[*types.AppVote]] + appQC utils.Option[*types.AppQC] } -func newAppVotes() appVotes { - return appVotes{ - byKey: map[types.PublicKey]*types.Signed[*types.AppVote]{}, - byHash: map[types.Hash[*types.AppVote]]*voteSet[*types.Signed[*types.AppVote]]{}, +func newRoad(commitQC *types.CommitQC, epoch *types.Epoch) *road { + return &road{ + epoch: epoch, + commitQC: commitQC, + appByKey: map[types.PublicKey]struct{}{}, + appByHash: map[types.Hash[*types.AppVote]]*voteSet[*types.Signed[*types.AppVote]]{}, } } // Returns qc if a new qc has been reached. -func (av appVotes) pushVote(c *types.Committee, vote *types.Signed[*types.AppVote]) (*types.AppQC, bool) { +func (r *road) pushAppVote(vote *types.Signed[*types.AppVote]) bool { + if r.appQC.IsPresent() { + return false + } k := vote.Key() - if _, ok := av.byKey[k]; ok { - return nil, false + if _, ok := r.appByKey[k]; ok { + return false } - av.byKey[k] = vote - byHash, ok := av.byHash[vote.Hash()] + r.appByKey[k] = struct{}{} + byHash, ok := r.appByHash[vote.Hash()] if !ok { - if len(av.byHash) == 1 { - logger.Error("appHash mismatch", slog.Uint64("n", uint64(vote.Msg().Proposal().GlobalNumber()))) + if len(r.appByHash) == 1 { + // Log an error just at the first conflicting hash. + logger.Error("appHash mismatch", slog.Uint64("n", uint64(vote.Msg().Proposal().RoadIndex()))) } byHash = &voteSet[*types.Signed[*types.AppVote]]{} - av.byHash[vote.Hash()] = byHash - } - if byHash.weight >= c.AppQuorum() { - return nil, false + r.appByHash[vote.Hash()] = byHash } + c := r.epoch.Committee() byHash.weight += c.Weight(k) byHash.votes = append(byHash.votes, vote) if byHash.weight >= c.AppQuorum() { - return types.NewAppQC(byHash.votes), true + r.appQC = utils.Some(types.NewAppQC(byHash.votes)) + return true } - return nil, false + return false } diff --git a/sei-tendermint/internal/autobahn/avail/conv_test.go b/sei-tendermint/internal/autobahn/avail/conv_test.go deleted file mode 100644 index ca753e5ee5..0000000000 --- a/sei-tendermint/internal/autobahn/avail/conv_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package avail - -import ( - "testing" - - "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/epoch" - "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" - "github.com/stretchr/testify/require" - "google.golang.org/protobuf/proto" -) - -func TestPruneAnchorConv(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - - lane := keys[0].Public() - block := types.NewBlock(lane, 0, types.BlockHeaderHash{}, types.GenPayload(rng)) - laneQCs := map[types.LaneID]*types.LaneQC{ - lane: types.NewLaneQC(makeLaneVotes(keys, block.Header())), - } - commitQC := makeCommitQC(registry.LatestEpoch(), keys, utils.None[*types.CommitQC](), laneQCs, utils.None[*types.AppQC]()) - appProposal := types.NewAppProposal(commitQC.GlobalRange().First, commitQC.Proposal().Index(), types.GenAppHash(rng), commitQC.Proposal().EpochIndex()) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - anchor := &PruneAnchor{AppQC: appQC, CommitQC: commitQC} - pb1 := PruneAnchorConv.Encode(anchor) - decoded, err := PruneAnchorConv.Decode(pb1) - require.NoError(t, err) - require.True(t, proto.Equal(pb1, PruneAnchorConv.Encode(decoded))) -} diff --git a/sei-tendermint/internal/autobahn/avail/inner.go b/sei-tendermint/internal/autobahn/avail/inner.go index d24eec2a2a..5f126df1f8 100644 --- a/sei-tendermint/internal/autobahn/avail/inner.go +++ b/sei-tendermint/internal/autobahn/avail/inner.go @@ -2,27 +2,26 @@ package avail import ( "fmt" - "log/slog" "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/avail/metrics" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/consensus/persist" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/data" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" ) // TODO: when dynamic committee changes are supported, newly joined members -// must be added to blocks, votes, nextBlockToPersist, and persistedBlockStart. +// must be added to blocks, votes, and nextBlockToPersist. // Currently all four are initialized once in newInner from c.Lanes().All(). -// BlockPersister creates lane WALs lazily inside MaybePruneAndPersistLane, but the new +// BlockPersister creates lane WALs lazily inside PruneAndPersist, but the new // member must also appear in inner.blocks before the next persist cycle. type inner struct { - epoch *types.Epoch - latestAppQC utils.Option[*types.AppQC] - latestCommitQC utils.AtomicSend[utils.Option[*types.CommitQC]] - appVotes *queue[types.GlobalBlockNumber, appVotes] - commitQCs *queue[types.RoadIndex, *types.CommitQC] - blocks map[types.LaneID]*queue[types.BlockNumber, *types.Signed[*types.LaneProposal]] - votes map[types.LaneID]*queue[types.BlockNumber, blockVotes] + persistedCommitQC utils.AtomicSend[utils.Option[*types.CommitQC]] // latest persisted CommitQC + roads *queue[types.RoadIndex, *road] + + // Epoch is the current epoch for blocks votes collection. + epoch *types.Epoch + blocks map[types.LaneID]*queue[types.BlockNumber, *types.Signed[*types.LaneProposal]] + votes map[types.LaneID]*queue[types.BlockNumber, blockVotes] // nextBlockToPersist tracks per-lane how far block persistence has progressed. // RecvBatch only yields blocks below this cursor for voting. // Always initialized (even when persistence is disabled — the no-op persist @@ -36,117 +35,95 @@ type inner struct { // ideal. Only RecvBatch needs to be notified of cursor changes; // collectPersistBatch is in the same goroutine and reads it directly. nextBlockToPersist map[types.LaneID]types.BlockNumber - - // persistedBlockStart is the per-lane block number derived from the last - // durably persisted prune anchor. Block admission (PushBlock, ProduceBlock, - // WaitForCapacity, PushVote) uses persistedBlockStart + BlocksPerLane as - // the capacity limit, ensuring we never admit more blocks than can be - // recovered after a crash. - persistedBlockStart map[types.LaneID]types.BlockNumber } -// loadedAvailState holds data loaded from disk on restart. -// pruneAnchor is the decoded prune anchor (if any). -// commitQCs and blocks are pre-filtered: stale entries below the -// anchor have already been removed by loadPersistedState. +// loadedState holds data loaded from disk on restart. // commitQCs are sorted by road index; blocks are sorted by number per lane. // newInner requires both to be contiguous and returns an error on gaps. That // requirement is what makes persist.contiguousSuffix safe: it silently drops // everything before the last hole it finds, so this is the only thing that // distinguishes a lazily pruned record from genuinely lost data. -type loadedAvailState struct { - pruneAnchor utils.Option[*PruneAnchor] - commitQCs []persist.LoadedCommitQC - blocks map[types.LaneID][]persist.LoadedBlock +// newInner requires both to be contiguous and returns an error on gaps. +type loadedState struct { + commitQCs []*types.CommitQC + blocks map[types.LaneID][]persist.LoadedBlock } -func newInner(epoch *types.Epoch, loaded utils.Option[*loadedAvailState]) (*inner, error) { - votes := map[types.LaneID]*queue[types.BlockNumber, blockVotes]{} - blocks := map[types.LaneID]*queue[types.BlockNumber, *types.Signed[*types.LaneProposal]]{} - for lane := range epoch.Committee().Lanes().All() { - votes[lane] = newQueue[types.BlockNumber, blockVotes]() - blocks[lane] = newQueue[types.BlockNumber, *types.Signed[*types.LaneProposal]]() - } - +func newInner(ds *data.State, loaded *loadedState) (*inner, error) { + epoch := ds.Registry().LatestEpoch() i := &inner{ - epoch: epoch, - latestAppQC: utils.None[*types.AppQC](), - latestCommitQC: utils.NewAtomicSend(utils.None[*types.CommitQC]()), - appVotes: newQueue[types.GlobalBlockNumber, appVotes](), - commitQCs: newQueue[types.RoadIndex, *types.CommitQC](), - blocks: blocks, - votes: votes, - nextBlockToPersist: make(map[types.LaneID]types.BlockNumber, len(votes)), - persistedBlockStart: make(map[types.LaneID]types.BlockNumber, len(votes)), + persistedCommitQC: utils.NewAtomicSend(utils.None[*types.CommitQC]()), + roads: newQueue[types.RoadIndex, *road](), + epoch: epoch, + blocks: map[types.LaneID]*queue[types.BlockNumber, *types.Signed[*types.LaneProposal]]{}, + votes: map[types.LaneID]*queue[types.BlockNumber, blockVotes]{}, + nextBlockToPersist: map[types.LaneID]types.BlockNumber{}, } - i.appVotes.prune(epoch.FirstBlock()) - - l, ok := loaded.Get() - if !ok { - return i, nil + for lane := range epoch.Committee().Lanes().All() { + i.blocks[lane] = newQueue[types.BlockNumber, *types.Signed[*types.LaneProposal]]() + i.votes[lane] = newQueue[types.BlockNumber, blockVotes]() } - // Apply the persisted prune anchor first: prune() positions all queues - // (commitQCs, blocks, votes) so that subsequent pushBack calls insert - // at the correct indices without needing reset(). - if anchor, ok := l.pruneAnchor.Get(); ok { - logger.Info("loaded persisted prune anchor", - slog.Uint64("roadIndex", uint64(anchor.AppQC.Proposal().RoadIndex())), - slog.Uint64("globalNumber", uint64(anchor.AppQC.Proposal().GlobalNumber())), - ) - // TODO: use the committee of the anchor's epoch once epoch transitions are wired up. - if _, err := i.prune(epoch.Committee(), anchor.AppQC, anchor.CommitQC); err != nil { - return nil, fmt.Errorf("prune: %w", err) - } - for lane := range i.blocks { - i.persistedBlockStart[lane] = anchor.CommitQC.LaneRange(lane).First() - } + // Apply the persisted prune anchor from the data.State: + // avail.State can drop everything below AppQC persisted in data.State. + if anchor, ok := ds.Anchor().Load().Get(); ok { + i.prune(anchor) } // Restore persisted CommitQCs. prune() may have already pushed the // anchor's CommitQC, so skip entries below commitQCs.next. - for _, lqc := range l.commitQCs { - if lqc.Index < i.commitQCs.next { + for _, qc := range loaded.commitQCs { + if qc.Index() < i.roads.next { continue } - if lqc.Index != i.commitQCs.next { - return nil, fmt.Errorf("non-contiguous persisted commitQCs: expected %d, got %d", i.commitQCs.next, lqc.Index) + if qc.Index() != i.roads.next { + return nil, fmt.Errorf("non-contiguous persisted commitQCs: expected %d, got %d", i.roads.next, qc.Index()) + } + epoch, ok := ds.Registry().EpochByIndex(qc.Proposal().EpochIndex()) + if !ok { + return nil, fmt.Errorf("epoch not found") } - i.commitQCs.pushBack(lqc.QC) + i.roads.pushBack(newRoad(qc, epoch)) } - if i.commitQCs.next > i.commitQCs.first { - i.latestCommitQC.Store(utils.Some(i.commitQCs.q[i.commitQCs.next-1])) + // It may happen that data.State has progressed beyond avail state. + // In this case the whole persisted avail.State is invalidated and anchor.CommitQC + // is NOT stored in avail.State. We need it to get persisted before we update persistedCommitQC. + if i.roads.Len() > 0 { + i.persistedCommitQC.Store(utils.Some(i.roads.q[i.roads.next-1].commitQC)) } // Restore persisted blocks. Since the anchor is persisted first and // blocks are written sequentially per lane, gaps, parent-hash // mismatches, and over-capacity indicate corruption or a bug. - for lane, bs := range l.blocks { + for lane, bs := range loaded.blocks { q, ok := i.blocks[lane] if !ok || len(bs) == 0 { continue } - var lastHash types.BlockHeaderHash - for j, b := range bs { + for _, b := range bs { if q.Len() >= BlocksPerLane { return nil, fmt.Errorf("lane %s: loaded %d blocks exceeds capacity %d", lane, len(bs), BlocksPerLane) } + if b.Number < q.next { + continue + } if b.Number != q.next { return nil, fmt.Errorf("lane %s: non-contiguous persisted blocks: expected %d, got %d", lane, q.next, b.Number) } - if j > 0 { - if got := b.Proposal.Msg().Block().Header().ParentHash(); got != lastHash { + // We check the parent hash only for the blocks above the anchor, because: + // * node can cast LaneVote for the block of the lane without checking the parent hash, + // in case the previous block was already (executed and) pruned from memory. + // * current WAL implementation is lazily pruning on disk, so old executed blocks might be loaded on startup. + if q.Len() > 0 { + ph := b.Proposal.Msg().Block().Header().ParentHash() + if q.q[q.next-1].Msg().Block().Header().Hash() != ph { return nil, fmt.Errorf("lane %s: parent hash mismatch at block %d", lane, b.Number) } } - lastHash = b.Proposal.Msg().Block().Header().Hash() q.pushBack(b.Proposal) } - if q.next > q.first { - i.nextBlockToPersist[lane] = q.next - } + i.nextBlockToPersist[lane] = q.next } - return i, nil } @@ -161,31 +138,23 @@ func (i *inner) laneQC(lane types.LaneID, n types.BlockNumber) (*types.LaneQC, b return nil, false } -// prune advances the state to account for a new AppQC/CommitQC pair. -// Returns true if pruning occurred, false if the QC was stale. -func (i *inner) prune(c *types.Committee, appQC *types.AppQC, commitQC *types.CommitQC) (bool, error) { - idx := appQC.Proposal().RoadIndex() - if idx != commitQC.Proposal().Index() { - return false, fmt.Errorf("mismatched QCs: appQC index %v, commitQC index %v", idx, commitQC.Proposal().Index()) - } - if idx < types.NextOpt(i.latestAppQC) { - return false, nil +// prune advances the state up to Anchor of the data state. +// Returns true iff pruning occurred. +func (i *inner) prune(anchor data.Anchor) { + idx := anchor.CommitQC.Index() + if idx < i.roads.first { + return } - i.latestAppQC = utils.Some(appQC) - metrics.ObserveAppQC(appQC) - i.commitQCs.prune(idx) - if i.commitQCs.next == idx { - i.commitQCs.pushBack(commitQC) - metrics.ObserveCommitQC(commitQC) - } - i.appVotes.prune(commitQC.GlobalRange().First) + i.roads.prune(idx + 1) for lane := range i.votes { - lr := commitQC.LaneRange(lane) - i.votes[lr.Lane()].prune(lr.First()) - i.blocks[lr.Lane()].prune(lr.First()) - if i.nextBlockToPersist[lr.Lane()] < lr.First() { - i.nextBlockToPersist[lr.Lane()] = lr.First() + lr := anchor.CommitQC.LaneRange(lane) + i.votes[lr.Lane()].prune(lr.Next()) + i.blocks[lr.Lane()].prune(lr.Next()) + if i.nextBlockToPersist[lr.Lane()] < lr.Next() { + i.nextBlockToPersist[lr.Lane()] = lr.Next() } } - return true, nil + if i.roads.Len() == 0 { + i.persistedCommitQC.Store(utils.Some(anchor.CommitQC)) + } } diff --git a/sei-tendermint/internal/autobahn/avail/inner_test.go b/sei-tendermint/internal/autobahn/avail/inner_test.go index a6deb0a076..d040076d1f 100644 --- a/sei-tendermint/internal/autobahn/avail/inner_test.go +++ b/sei-tendermint/internal/autobahn/avail/inner_test.go @@ -8,7 +8,6 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/consensus/persist" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/data" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/epoch" - pb "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/pb" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/stretchr/testify/require" ) @@ -17,52 +16,6 @@ func newTestDataState(cfg *data.Config) *data.State { return utils.OrPanic1(data.NewState(cfg, memblock.NewBlockDB())) } -func TestPruneMismatchedIndices(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - - makeCommitQC := func(prev utils.Option[*types.CommitQC]) *types.CommitQC { - l := keys[0].Public() - lr := types.LaneRangeOpt(prev, l) - b := types.NewBlock(l, lr.Next(), lr.LastHash(), types.GenPayload(rng)) - lqcs := map[types.LaneID]*types.LaneQC{ - l: types.NewLaneQC(makeLaneVotes(keys, b.Header())), - } - return makeCommitQC(registry.LatestEpoch(), keys, prev, lqcs, utils.None[*types.AppQC]()) - } - makeAppQC := func(qcForRange *types.CommitQC, qcForIndex *types.CommitQC) *types.AppQC { - gr := qcForRange.GlobalRange() - require.True(t, gr.Len() > 0) - ap := types.NewAppProposal(gr.First, qcForIndex.Index(), types.GenAppHash(rng), 0) - return types.NewAppQC(makeAppVotes(keys, ap)) - } - - qc0 := makeCommitQC(utils.None[*types.CommitQC]()) - qc1 := makeCommitQC(utils.Some(qc0)) - - t.Logf("test State.PushAppQC") - ds := newTestDataState(&data.Config{Registry: registry}) - state, err := NewState(keys[0], ds, utils.Some(t.TempDir())) - require.NoError(t, err) - require.Error(t, state.PushAppQC(makeAppQC(qc0, qc0), qc1), "bad range, bad index should fail") - require.Error(t, state.PushAppQC(makeAppQC(qc1, qc0), qc1), "good range, bad index should fail") - require.Error(t, state.PushAppQC(makeAppQC(qc0, qc1), qc1), "bad range, good index should fail") - require.NoError(t, state.PushAppQC(makeAppQC(qc1, qc1), qc1), "good range, good index should succeed") - - t.Logf("test inner.prune") - ds = newTestDataState(&data.Config{Registry: registry}) - state, err = NewState(keys[0], ds, utils.Some(t.TempDir())) - require.NoError(t, err) - for inner := range state.inner.Lock() { - _, err := inner.prune(registry.LatestEpoch().Committee(), makeAppQC(qc1, qc0), qc1) - require.Error(t, err, "good range, bad index should fail") - require.False(t, inner.latestAppQC.IsPresent(), "latestAppQC should not have been updated") - _, err = inner.prune(registry.LatestEpoch().Committee(), makeAppQC(qc1, qc1), qc1) - require.NoError(t, err, "good range, good index should succeed") - } -} - -// testSignedBlock creates a signed lane proposal for a given lane, block number, and parent hash. func testSignedBlock(key types.SecretKey, lane types.LaneID, n types.BlockNumber, parent types.BlockHeaderHash, rng utils.Rng) *types.Signed[*types.LaneProposal] { block := types.NewBlock(lane, n, parent, types.GenPayload(rng)) return types.Sign(key, types.NewLaneProposal(block)) @@ -72,15 +25,12 @@ func TestNewInnerFreshStart(t *testing.T) { rng := utils.TestRng() registry, _ := epoch.GenRegistry(rng, 4) - i, err := newInner(registry.LatestEpoch(), utils.None[*loadedAvailState]()) + i, err := newInner(newTestDataState(&data.Config{Registry: registry}), &loadedState{}) require.NoError(t, err) - require.False(t, i.latestAppQC.IsPresent()) + require.Equal(t, types.RoadIndex(0), i.roads.first) + require.Equal(t, types.RoadIndex(0), i.roads.next) require.NotNil(t, i.nextBlockToPersist) - require.Equal(t, types.RoadIndex(0), i.commitQCs.first) - require.Equal(t, types.RoadIndex(0), i.commitQCs.next) - require.Equal(t, registry.FirstBlock(), i.appVotes.first) - require.Equal(t, registry.FirstBlock(), i.appVotes.next) for lane := range registry.LatestEpoch().Committee().Lanes().All() { require.Equal(t, types.BlockNumber(0), i.blocks[lane].first) require.Equal(t, types.BlockNumber(0), i.blocks[lane].next) @@ -89,41 +39,11 @@ func TestNewInnerFreshStart(t *testing.T) { } } -func TestDecodePruneAnchorIncomplete(t *testing.T) { - rng := utils.TestRng() - _, keys := epoch.GenRegistry(rng, 4) - - appProposal := types.NewAppProposal(42, 5, types.GenAppHash(rng), 0) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - _, err := PruneAnchorConv.Decode(&pb.PersistedAvailPruneAnchor{ - AppQc: types.AppQCConv.Encode(appQC), - }) - require.Error(t, err) - require.Contains(t, err.Error(), "incomplete prune anchor") -} - -func TestNewInnerLoadedNoAnchor(t *testing.T) { - rng := utils.TestRng() - registry, _ := epoch.GenRegistry(rng, 4) - - loaded := &loadedAvailState{} - - i, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) - require.NoError(t, err) - - // No anchor loaded, app votes should start at the registry's first block. - require.False(t, i.latestAppQC.IsPresent()) - require.Equal(t, types.RoadIndex(0), i.commitQCs.first) - require.Equal(t, registry.FirstBlock(), i.appVotes.first) -} - func TestNewInnerLoadedBlocksContiguous(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 4) lane := keys[0].Public() - // Build 3 contiguous blocks: 0, 1, 2. var parent types.BlockHeaderHash var bs []persist.LoadedBlock for n := range types.BlockNumber(3) { @@ -132,11 +52,9 @@ func TestNewInnerLoadedBlocksContiguous(t *testing.T) { bs = append(bs, persist.LoadedBlock{Number: n, Proposal: b}) } - loaded := &loadedAvailState{ + i, err := newInner(newTestDataState(&data.Config{Registry: registry}), &loadedState{ blocks: map[types.LaneID][]persist.LoadedBlock{lane: bs}, - } - - i, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) + }) require.NoError(t, err) q := i.blocks[lane] @@ -145,9 +63,6 @@ func TestNewInnerLoadedBlocksContiguous(t *testing.T) { for j, b := range bs { require.Equal(t, b.Proposal, q.q[types.BlockNumber(j)]) } - - // nextBlockToPersist: loaded lane at q.next, other lanes at 0 (map zero-value). - require.NotNil(t, i.nextBlockToPersist) require.Equal(t, types.BlockNumber(3), i.nextBlockToPersist[lane]) for other := range registry.LatestEpoch().Committee().Lanes().All() { if other != lane { @@ -161,11 +76,9 @@ func TestNewInnerLoadedBlocksEmptySlice(t *testing.T) { registry, keys := epoch.GenRegistry(rng, 4) lane := keys[0].Public() - loaded := &loadedAvailState{ + i, err := newInner(newTestDataState(&data.Config{Registry: registry}), &loadedState{ blocks: map[types.LaneID][]persist.LoadedBlock{lane: {}}, - } - - i, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) + }) require.NoError(t, err) q := i.blocks[lane] @@ -179,13 +92,11 @@ func TestNewInnerLoadedBlocksUnknownLane(t *testing.T) { unknownKey := types.GenSecretKey(rng) unknownLane := unknownKey.Public() - b := testSignedBlock(unknownKey, unknownLane, 0, types.BlockHeaderHash{}, rng) - loaded := &loadedAvailState{ - blocks: map[types.LaneID][]persist.LoadedBlock{unknownLane: {{Number: 0, Proposal: b}}}, - } - i, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) + i, err := newInner(newTestDataState(&data.Config{Registry: registry}), &loadedState{ + blocks: map[types.LaneID][]persist.LoadedBlock{unknownLane: {{Number: 0, Proposal: b}}}, + }) require.NoError(t, err) for lane := range registry.LatestEpoch().Committee().Lanes().All() { @@ -218,22 +129,13 @@ func TestNewInnerLoadedBlocksMultipleLanes(t *testing.T) { bs1 = append(bs1, persist.LoadedBlock{Number: n, Proposal: b}) } - loaded := &loadedAvailState{ + i, err := newInner(newTestDataState(&data.Config{Registry: registry}), &loadedState{ blocks: map[types.LaneID][]persist.LoadedBlock{lane0: bs0, lane1: bs1}, - } - - i, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) + }) require.NoError(t, err) - q0 := i.blocks[lane0] - require.Equal(t, types.BlockNumber(0), q0.first) - require.Equal(t, types.BlockNumber(2), q0.next) - - q1 := i.blocks[lane1] - require.Equal(t, types.BlockNumber(0), q1.first) - require.Equal(t, types.BlockNumber(3), q1.next) - - // nextBlockToPersist reflects q.next per loaded lane. + require.Equal(t, types.BlockNumber(2), i.blocks[lane0].next) + require.Equal(t, types.BlockNumber(3), i.blocks[lane1].next) require.Equal(t, types.BlockNumber(2), i.nextBlockToPersist[lane0]) require.Equal(t, types.BlockNumber(3), i.nextBlockToPersist[lane1]) } @@ -242,307 +144,33 @@ func TestNewInnerLoadedCommitQCsNoAppQC(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 4) - // Create 3 sequential CommitQCs. qcs := make([]*types.CommitQC, 3) prev := utils.None[*types.CommitQC]() for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) + qcs[i] = types.BuildCommitQC(registry.LatestEpoch(), keys, prev, nil) prev = utils.Some(qcs[i]) } - var loadedQCs []persist.LoadedCommitQC - for i, qc := range qcs { - loadedQCs = append(loadedQCs, persist.LoadedCommitQC{Index: types.RoadIndex(i), QC: qc}) - } - - loaded := &loadedAvailState{ - commitQCs: loadedQCs, - } - - inner, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) + inner, err := newInner(newTestDataState(&data.Config{Registry: registry}), &loadedState{commitQCs: qcs}) require.NoError(t, err) - // Without anchor, commitQCs.first = 0. All 3 should be restored. - require.Equal(t, types.RoadIndex(0), inner.commitQCs.first) - require.Equal(t, types.RoadIndex(3), inner.commitQCs.next) + require.Equal(t, types.RoadIndex(0), inner.roads.first) + require.Equal(t, types.RoadIndex(3), inner.roads.next) for i, qc := range qcs { - require.NoError(t, utils.TestDiff(qc, inner.commitQCs.q[types.RoadIndex(i)])) - } - - // latestCommitQC should be set to the last loaded one. - latest, ok := inner.latestCommitQC.Load().Get() - require.True(t, ok) - require.NoError(t, utils.TestDiff(qcs[2], latest)) -} - -func TestNewInnerLoadedCommitQCsWithAppQC(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - - // AppQC at road index 2. - roadIdx := types.RoadIndex(2) - globalNum := types.GlobalBlockNumber(10) - appProposal := types.NewAppProposal(globalNum, roadIdx, types.GenAppHash(rng), 0) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - // Create 5 sequential CommitQCs (indices 0-4). - qcs := make([]*types.CommitQC, 5) - prev := utils.None[*types.CommitQC]() - for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[i]) - } - - // Pre-filtered: only commitQCs >= anchor road index (2). - loadedQCs := []persist.LoadedCommitQC{ - {Index: 2, QC: qcs[2]}, - {Index: 3, QC: qcs[3]}, - {Index: 4, QC: qcs[4]}, - } - - loaded := &loadedAvailState{ - pruneAnchor: utils.Some(&PruneAnchor{AppQC: appQC, CommitQC: qcs[2]}), - commitQCs: loadedQCs, - } - - inner, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) - require.NoError(t, err) - - // latestAppQC should be set by prune. - aq, ok := inner.latestAppQC.Get() - require.True(t, ok) - require.Equal(t, roadIdx, aq.Proposal().RoadIndex()) - - // inner.prune(appQC@2, commitQC@2) sets commitQCs.first = 2. - // Indices 2, 3 and 4 remain; earlier ones are pruned. - require.Equal(t, types.RoadIndex(2), inner.commitQCs.first) - require.Equal(t, types.RoadIndex(5), inner.commitQCs.next) - require.NoError(t, utils.TestDiff(qcs[2], inner.commitQCs.q[2])) - require.NoError(t, utils.TestDiff(qcs[3], inner.commitQCs.q[3])) - require.NoError(t, utils.TestDiff(qcs[4], inner.commitQCs.q[4])) - - // latestCommitQC should be the last restored one (index 4). - latest, ok := inner.latestCommitQC.Load().Get() - require.True(t, ok) - require.NoError(t, utils.TestDiff(qcs[4], latest)) -} - -func TestNewInnerLoadedAllThree(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - lane := keys[0].Public() - - // AppQC at road index 2. - roadIdx := types.RoadIndex(2) - appProposal := types.NewAppProposal(10, roadIdx, types.GenAppHash(rng), 0) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - // CommitQCs 0-4. - qcs := make([]*types.CommitQC, 5) - prev := utils.None[*types.CommitQC]() - for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[i]) - } - // Pre-filtered: only commitQCs >= anchor road index (2). - loadedQCs := []persist.LoadedCommitQC{ - {Index: 2, QC: qcs[2]}, - {Index: 3, QC: qcs[3]}, - {Index: 4, QC: qcs[4]}, - } - - // Blocks 0-2 on one lane (nil laneQCs → lr.First()=0 after prune). - var parent types.BlockHeaderHash - var bs []persist.LoadedBlock - for n := range types.BlockNumber(3) { - b := testSignedBlock(keys[0], lane, n, parent, rng) - parent = b.Msg().Block().Header().Hash() - bs = append(bs, persist.LoadedBlock{Number: n, Proposal: b}) - } - - loaded := &loadedAvailState{ - pruneAnchor: utils.Some(&PruneAnchor{AppQC: appQC, CommitQC: qcs[2]}), - commitQCs: loadedQCs, - blocks: map[types.LaneID][]persist.LoadedBlock{lane: bs}, - } - - inner, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) - require.NoError(t, err) - - // AppQC restored. - aq, ok := inner.latestAppQC.Get() - require.True(t, ok) - require.Equal(t, roadIdx, aq.Proposal().RoadIndex()) - - // CommitQCs: prune pushed qcs[2], loading skipped it, added 3 and 4. - require.Equal(t, types.RoadIndex(2), inner.commitQCs.first) - require.Equal(t, types.RoadIndex(5), inner.commitQCs.next) - - // Blocks loaded. - q := inner.blocks[lane] - require.Equal(t, types.BlockNumber(0), q.first) - require.Equal(t, types.BlockNumber(3), q.next) - require.Equal(t, types.BlockNumber(3), inner.nextBlockToPersist[lane]) - - // latestCommitQC is the last loaded one. - latest, ok := inner.latestCommitQC.Load().Get() - require.True(t, ok) - require.NoError(t, utils.TestDiff(qcs[4], latest)) -} - -func TestPruneAdvancesNextBlockToPersist(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - lane := keys[0].Public() - - i, err := newInner(registry.LatestEpoch(), utils.None[*loadedAvailState]()) - require.NoError(t, err) - - // Push blocks 0-4 on one lane. - var parent types.BlockHeaderHash - for n := range types.BlockNumber(5) { - b := testSignedBlock(keys[0], lane, n, parent, rng) - parent = b.Msg().Block().Header().Hash() - i.blocks[lane].pushBack(b) - } - // Simulate partial persistence: only block 0 persisted. - i.nextBlockToPersist[lane] = 1 - - // Build CommitQCs with lane ranges that reference actual blocks. - // Each CommitQC covers one block on the lane via a LaneQC. - qcs := make([]*types.CommitQC, 3) - prev := utils.None[*types.CommitQC]() - for j := range qcs { - bn := types.BlockNumber(j) - h := i.blocks[lane].q[bn].Msg().Block().Header() - laneQCs := map[types.LaneID]*types.LaneQC{ - lane: types.NewLaneQC(makeLaneVotes( - types.TestKeysWithWeight(registry.LatestEpoch().Committee(), keys, registry.LatestEpoch().Committee().LaneQuorum()), - h, - )), - } - qcs[j] = makeCommitQC(registry.LatestEpoch(), keys, prev, laneQCs, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[j]) - i.commitQCs.pushBack(qcs[j]) - } - - // Verify QC@2's lane range actually covers blocks (First > 0). - lr := qcs[2].LaneRange(lane) - require.Greater(t, lr.First(), types.BlockNumber(0), - "CommitQC lane range should reference blocks for this test to be meaningful") - - // AppQC at index 2 → prune will fast-forward blocks past the cursor. - appProposal := types.NewAppProposal(10, 2, types.GenAppHash(rng), 0) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - updated, err := i.prune(registry.LatestEpoch().Committee(), appQC, qcs[2]) - require.NoError(t, err) - require.True(t, updated) - - // nextBlockToPersist must have advanced to at least the lane's new first - // (determined by CommitQC@2's lane range). Without this fix, it would - // stay at 1, causing the persist goroutine to busy-loop. - laneFirst := i.blocks[lane].first - require.Greater(t, laneFirst, types.BlockNumber(1), - "prune should have advanced blocks.first past the old cursor") - require.GreaterOrEqual(t, i.nextBlockToPersist[lane], laneFirst, - "nextBlockToPersist should advance when prune moves blocks.first past it") -} - -func TestNewInnerLoadedCommitQCsAllBeforeAppQCArePruned(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - - // Build 6 CommitQCs (indices 0-5). Anchor at index 5. - // All stale commitQCs (0-4) were already filtered by loadPersistedState, - // so newInner receives an empty commitQC slice. - qcs := make([]*types.CommitQC, 6) - prev := utils.None[*types.CommitQC]() - for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[i]) - } - - appProposal := types.NewAppProposal(20, 5, types.GenAppHash(rng), 0) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - loaded := &loadedAvailState{ - pruneAnchor: utils.Some(&PruneAnchor{AppQC: appQC, CommitQC: qcs[5]}), - } - - inner, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) - require.NoError(t, err) - - // prune() pushes the anchor's CommitQC into the queue. - require.Equal(t, types.RoadIndex(5), inner.commitQCs.first) - require.Equal(t, types.RoadIndex(6), inner.commitQCs.next) - require.NoError(t, utils.TestDiff(qcs[5], inner.commitQCs.q[5])) -} - -func TestNewInnerAnchorWithNoCommitQCFiles(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - - // Simulate crash between anchor write and CommitQC file write: - // anchor has AppQC@3 + CommitQC@3, but no CommitQC files on disk. - qcs := make([]*types.CommitQC, 4) - prev := utils.None[*types.CommitQC]() - for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[i]) - } - - appProposal := types.NewAppProposal(20, 3, types.GenAppHash(rng), 0) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - loaded := &loadedAvailState{ - pruneAnchor: utils.Some(&PruneAnchor{AppQC: appQC, CommitQC: qcs[3]}), - } - - inner, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) - require.NoError(t, err) - - // prune() should push the anchor's CommitQC into the queue. - require.Equal(t, types.RoadIndex(3), inner.commitQCs.first) - require.Equal(t, types.RoadIndex(4), inner.commitQCs.next) - require.NoError(t, utils.TestDiff(qcs[3], inner.commitQCs.q[3])) - - // latestAppQC should be set. - aq, ok := inner.latestAppQC.Get() - require.True(t, ok) - require.Equal(t, types.RoadIndex(3), aq.Proposal().RoadIndex()) - - // persistedBlockStart should be initialized from the anchor's CommitQC. - for lane := range registry.LatestEpoch().Committee().Lanes().All() { - expected := qcs[3].LaneRange(lane).First() - require.Equal(t, expected, inner.persistedBlockStart[lane]) + require.NoError(t, utils.TestDiff(qc, inner.roads.q[types.RoadIndex(i)].commitQC)) } + require.NoError(t, utils.TestDiff(utils.Some(qcs[2]), inner.persistedCommitQC.Load())) } func TestNewInnerLoadedCommitQCsGapReturnsError(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 4) - qcs := make([]*types.CommitQC, 3) - prev := utils.None[*types.CommitQC]() - for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[i]) - } + qc0 := types.BuildCommitQC(registry.LatestEpoch(), keys, utils.None[*types.CommitQC](), nil) + qc1 := types.BuildCommitQC(registry.LatestEpoch(), keys, utils.Some(qc0), nil) + qc2 := types.BuildCommitQC(registry.LatestEpoch(), keys, utils.Some(qc1), nil) - // Gap: indices 0, 1, 3 (missing 2). Since the anchor is persisted first, - // a gap in committed QCs is a bug — newInner should return an error. - loadedQCs := []persist.LoadedCommitQC{ - {Index: 0, QC: qcs[0]}, - {Index: 1, QC: qcs[1]}, - {Index: 3, QC: qcs[2]}, - } - - loaded := &loadedAvailState{ - commitQCs: loadedQCs, - } - - _, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) + _, err := newInner(newTestDataState(&data.Config{Registry: registry}), &loadedState{commitQCs: []*types.CommitQC{qc0, qc2}}) require.Error(t, err) require.Contains(t, err.Error(), "non-contiguous") } @@ -551,158 +179,28 @@ func TestNewInnerLoadedCommitQCsEmpty(t *testing.T) { rng := utils.TestRng() registry, _ := epoch.GenRegistry(rng, 4) - loaded := &loadedAvailState{ - commitQCs: nil, - } - - inner, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) + inner, err := newInner(newTestDataState(&data.Config{Registry: registry}), &loadedState{}) require.NoError(t, err) - require.Equal(t, types.RoadIndex(0), inner.commitQCs.first) - require.Equal(t, types.RoadIndex(0), inner.commitQCs.next) - _, ok := inner.latestCommitQC.Load().Get() + require.Equal(t, types.RoadIndex(0), inner.roads.first) + require.Equal(t, types.RoadIndex(0), inner.roads.next) + _, ok := inner.persistedCommitQC.Load().Get() require.False(t, ok) } -func TestNewInnerLoadedCommitQCsGapWithAppQCAnchor(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - - // Simulate crash scenario: disk had stale QCs [0,1,2] and a new QC at - // index 10. loadPersistedState pre-filters stale entries, so newInner - // only receives [10]. - qcs := make([]*types.CommitQC, 11) - prev := utils.None[*types.CommitQC]() - for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[i]) - } - - appProposal := types.NewAppProposal(50, 10, types.GenAppHash(rng), 0) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - loadedQCs := []persist.LoadedCommitQC{ - {Index: 10, QC: qcs[10]}, - } - - loaded := &loadedAvailState{ - pruneAnchor: utils.Some(&PruneAnchor{AppQC: appQC, CommitQC: qcs[10]}), - commitQCs: loadedQCs, - } - - inner, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) - require.NoError(t, err) - - // Only QC@10 loaded. - require.Equal(t, types.RoadIndex(10), inner.commitQCs.first) - require.Equal(t, types.RoadIndex(11), inner.commitQCs.next) - require.NoError(t, utils.TestDiff(qcs[10], inner.commitQCs.q[10])) - - latest, ok := inner.latestCommitQC.Load().Get() - require.True(t, ok) - require.NoError(t, utils.TestDiff(qcs[10], latest)) - - // AppQC should be applied via prune. - aq, ok := inner.latestAppQC.Get() - require.True(t, ok) - require.Equal(t, types.RoadIndex(10), aq.Proposal().RoadIndex()) -} - -func TestNewInnerLoadedCommitQCsBelowAnchorSkipped(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - - // Build 6 CommitQCs (0-5). Anchor at index 3. - // Loaded list includes stale entries [1, 2] below the anchor plus [3, 4, 5]. - // In production loadPersistedState filters these, but newInner should - // handle them gracefully via the lqc.Index < commitQCs.next skip. - qcs := make([]*types.CommitQC, 6) - prev := utils.None[*types.CommitQC]() - for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[i]) - } - - appProposal := types.NewAppProposal(20, 3, types.GenAppHash(rng), 0) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - loadedQCs := []persist.LoadedCommitQC{ - {Index: 1, QC: qcs[1]}, - {Index: 2, QC: qcs[2]}, - {Index: 3, QC: qcs[3]}, - {Index: 4, QC: qcs[4]}, - {Index: 5, QC: qcs[5]}, - } - - loaded := &loadedAvailState{ - pruneAnchor: utils.Some(&PruneAnchor{AppQC: appQC, CommitQC: qcs[3]}), - commitQCs: loadedQCs, - } - - inner, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) - require.NoError(t, err) - - // prune(3) pushes QC@3 (next=4). Indices 1,2,3 are skipped. 4,5 pushed. - require.Equal(t, types.RoadIndex(3), inner.commitQCs.first) - require.Equal(t, types.RoadIndex(6), inner.commitQCs.next) - latest, ok := inner.latestCommitQC.Load().Get() - require.True(t, ok) - require.NoError(t, utils.TestDiff(qcs[5], latest)) -} - -func TestNewInnerLoadedCommitQCsGapAfterAnchorReturnsError(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - - // Anchor at index 2. Loaded commitQCs are [2, 3, 5] — gap at 4. - // After prune(2), next=3. Index 2 is skipped, 3 pushed (next=4), - // then 5 != 4 → error. - qcs := make([]*types.CommitQC, 6) - prev := utils.None[*types.CommitQC]() - for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[i]) - } - - appProposal := types.NewAppProposal(10, 2, types.GenAppHash(rng), 0) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - loadedQCs := []persist.LoadedCommitQC{ - {Index: 2, QC: qcs[2]}, - {Index: 3, QC: qcs[3]}, - {Index: 5, QC: qcs[5]}, - } - - loaded := &loadedAvailState{ - pruneAnchor: utils.Some(&PruneAnchor{AppQC: appQC, CommitQC: qcs[2]}), - commitQCs: loadedQCs, - } - - _, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) - require.Error(t, err) - require.Contains(t, err.Error(), "non-contiguous") -} - func TestNewInnerLoadedBlocksGapReturnsError(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 4) lane := keys[0].Public() - // Blocks 3, 4, 6, 7 with no anchor — queue starts at 0, so block 3 - // fails the contiguity check immediately (expected 0, got 3). - var parent types.BlockHeaderHash var bs []persist.LoadedBlock for _, n := range []types.BlockNumber{3, 4, 6, 7} { - b := testSignedBlock(keys[0], lane, n, parent, rng) - parent = b.Msg().Block().Header().Hash() - bs = append(bs, persist.LoadedBlock{Number: n, Proposal: b}) + bs = append(bs, persist.LoadedBlock{Number: n, Proposal: testSignedBlock(keys[0], lane, n, types.BlockHeaderHash{}, rng)}) } - loaded := &loadedAvailState{ + _, err := newInner(newTestDataState(&data.Config{Registry: registry}), &loadedState{ blocks: map[types.LaneID][]persist.LoadedBlock{lane: bs}, - } - - _, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) + }) require.Error(t, err) require.Contains(t, err.Error(), "non-contiguous") } @@ -712,25 +210,19 @@ func TestNewInnerLoadedBlocksParentHashMismatchReturnsError(t *testing.T) { registry, keys := epoch.GenRegistry(rng, 4) lane := keys[0].Public() - // Build blocks 0, 1 with correct chaining, then block 2 with wrong parent. var parent types.BlockHeaderHash b0 := testSignedBlock(keys[0], lane, 0, parent, rng) parent = b0.Msg().Block().Header().Hash() b1 := testSignedBlock(keys[0], lane, 1, parent, rng) - wrongParent := types.GenBlockHeaderHash(rng) - b2 := testSignedBlock(keys[0], lane, 2, wrongParent, rng) - - bs := []persist.LoadedBlock{ - {Number: 0, Proposal: b0}, - {Number: 1, Proposal: b1}, - {Number: 2, Proposal: b2}, - } - - loaded := &loadedAvailState{ - blocks: map[types.LaneID][]persist.LoadedBlock{lane: bs}, - } - - _, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) + b2 := testSignedBlock(keys[0], lane, 2, types.GenBlockHeaderHash(rng), rng) + + _, err := newInner(newTestDataState(&data.Config{Registry: registry}), &loadedState{ + blocks: map[types.LaneID][]persist.LoadedBlock{lane: { + {Number: 0, Proposal: b0}, + {Number: 1, Proposal: b1}, + {Number: 2, Proposal: b2}, + }}, + }) require.Error(t, err) require.Contains(t, err.Error(), "parent hash mismatch") } @@ -740,9 +232,6 @@ func TestNewInnerLoadedBlocksOverCapacityReturnsError(t *testing.T) { registry, keys := epoch.GenRegistry(rng, 4) lane := keys[0].Public() - // Build BlocksPerLane + 5 contiguous blocks — more than the lane capacity. - // Since runtime enforces the capacity limit, exceeding it on disk indicates - // corruption or a bug. count := BlocksPerLane + 5 var parent types.BlockHeaderHash var bs []persist.LoadedBlock @@ -752,94 +241,9 @@ func TestNewInnerLoadedBlocksOverCapacityReturnsError(t *testing.T) { bs = append(bs, persist.LoadedBlock{Number: n, Proposal: b}) } - loaded := &loadedAvailState{ + _, err := newInner(newTestDataState(&data.Config{Registry: registry}), &loadedState{ blocks: map[types.LaneID][]persist.LoadedBlock{lane: bs}, - } - - _, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) + }) require.Error(t, err) require.Contains(t, err.Error(), "exceeds capacity") } - -func TestNewInnerPruneAnchorPrunesBlockQueues(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - initialBlock := types.GlobalBlockNumber(0) - - // Build CommitQCs 0-2. - qcs := make([]*types.CommitQC, 3) - prev := utils.None[*types.CommitQC]() - for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[i]) - } - - // AppQC at road index 2, prune anchor is CommitQC[2]. - appProposal := types.NewAppProposal(initialBlock, 2, types.GenAppHash(rng), 0) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - pruneQC := qcs[2] - - lane := keys[0].Public() - - // Persist some blocks starting at the lane range for the prune CommitQC. - lrFirst := pruneQC.LaneRange(lane).First() - var parent types.BlockHeaderHash - var bs []persist.LoadedBlock - for n := lrFirst; n < lrFirst+3; n++ { - b := testSignedBlock(keys[0], lane, n, parent, rng) - parent = b.Msg().Block().Header().Hash() - bs = append(bs, persist.LoadedBlock{Number: n, Proposal: b}) - } - - loaded := &loadedAvailState{ - pruneAnchor: utils.Some(&PruneAnchor{AppQC: appQC, CommitQC: pruneQC}), - commitQCs: []persist.LoadedCommitQC{ - {Index: 2, QC: qcs[2]}, - }, - blocks: map[types.LaneID][]persist.LoadedBlock{lane: bs}, - } - - i, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) - require.NoError(t, err) - - // prune() should advance block queue first to the prune anchor's lane range. - for l := range registry.LatestEpoch().Committee().Lanes().All() { - expected := pruneQC.LaneRange(l).First() - require.Equal(t, expected, i.blocks[l].first, - "blocks[%v].first should be advanced by prune to prune anchor lane range", l) - } -} - -func TestNewInnerPruneAnchorCommitQCUsedForPrune(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - initialBlock := types.GlobalBlockNumber(0) - - // Build CommitQCs 0-2. - qcs := make([]*types.CommitQC, 3) - prev := utils.None[*types.CommitQC]() - for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[i]) - } - - // AppQC at road index 1, prune anchor is CommitQC[1]. - appProposal := types.NewAppProposal(initialBlock, 1, types.GenAppHash(rng), 0) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - loaded := &loadedAvailState{ - pruneAnchor: utils.Some(&PruneAnchor{AppQC: appQC, CommitQC: qcs[1]}), - commitQCs: []persist.LoadedCommitQC{ - {Index: 1, QC: qcs[1]}, - {Index: 2, QC: qcs[2]}, - }, - } - - i, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) - require.NoError(t, err) - - // prune(appQC@1, pruneQC@1) should advance commitQCs.first to 1. - require.Equal(t, types.RoadIndex(1), i.commitQCs.first) - // CommitQCs 1 and 2 should still be loaded. - require.Equal(t, types.RoadIndex(3), i.commitQCs.next) -} diff --git a/sei-tendermint/internal/autobahn/avail/metrics/metrics.gen.go b/sei-tendermint/internal/autobahn/avail/metrics/metrics.gen.go index 4aa8151859..1341ba9240 100644 --- a/sei-tendermint/internal/autobahn/avail/metrics/metrics.gen.go +++ b/sei-tendermint/internal/autobahn/avail/metrics/metrics.gen.go @@ -12,9 +12,7 @@ var Global = newMetrics() func init() { prometheus.MustRegister( Global.commitRoadIndex, - Global.appRoadIndex, Global.commitGlobalBlockNumber, - Global.appGlobalBlockNumber, Global.proposalToCommitLatency, Global.commitToCommitLatency, ) @@ -28,24 +26,12 @@ func newMetrics() *metrics { Name: "commit_road_index", Help: "Road index of the highest observed commitQC.", }, nil), - appRoadIndex: tmprometheus.NewGaugeIntVec(prometheus.GaugeOpts{ - Namespace: MetricsNamespace, - Subsystem: MetricsSubsystem, - Name: "app_road_index", - Help: "Road index of the highest observed appQC.", - }, nil), commitGlobalBlockNumber: tmprometheus.NewGaugeIntVec(prometheus.GaugeOpts{ Namespace: MetricsNamespace, Subsystem: MetricsSubsystem, Name: "commit_global_block_number", Help: "Global block number of the highest observed commitQC.", }, nil), - appGlobalBlockNumber: tmprometheus.NewGaugeIntVec(prometheus.GaugeOpts{ - Namespace: MetricsNamespace, - Subsystem: MetricsSubsystem, - Name: "app_global_block_number", - Help: "Global block number of the highest observed appQC.", - }, nil), proposalToCommitLatency: tmprometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: MetricsNamespace, Subsystem: MetricsSubsystem, @@ -66,18 +52,10 @@ func (m *metrics) commitRoadIndexAt() *tmprometheus.GaugeInt { return m.commitRoadIndex.WithLabelValues() } -func (m *metrics) appRoadIndexAt() *tmprometheus.GaugeInt { - return m.appRoadIndex.WithLabelValues() -} - func (m *metrics) commitGlobalBlockNumberAt() *tmprometheus.GaugeInt { return m.commitGlobalBlockNumber.WithLabelValues() } -func (m *metrics) appGlobalBlockNumberAt() *tmprometheus.GaugeInt { - return m.appGlobalBlockNumber.WithLabelValues() -} - func (m *metrics) proposalToCommitLatencyAt() *tmprometheus.Histogram { return m.proposalToCommitLatency.WithLabelValues() } diff --git a/sei-tendermint/internal/autobahn/avail/metrics/metrics.go b/sei-tendermint/internal/autobahn/avail/metrics/metrics.go index acfb7a6c6f..7492041eb8 100644 --- a/sei-tendermint/internal/autobahn/avail/metrics/metrics.go +++ b/sei-tendermint/internal/autobahn/avail/metrics/metrics.go @@ -16,13 +16,9 @@ const MetricsSubsystem = "internal_autobahn_avail" type metrics struct { // Road index of the highest observed commitQC. commitRoadIndex prometheus.GaugeIntVec - // Road index of the highest observed appQC. - appRoadIndex prometheus.GaugeIntVec // Global block number of the highest observed commitQC. commitGlobalBlockNumber prometheus.GaugeIntVec - // Global block number of the highest observed appQC. - appGlobalBlockNumber prometheus.GaugeIntVec // Latency from proposal being constructed to commit being observed. proposalToCommitLatency prometheus.HistogramVec `metrics_buckets:"exp(0.01, 1.2, 35)"` @@ -40,7 +36,6 @@ func newObserved[T any]() utils.Mutex[*utils.Option[observed[T]]] { } var observedCommitQC = newObserved[*types.CommitQC]() -var observedAppQC = newObserved[*types.AppQC]() // ObserveCommitQC observes the CommitQC latency. func ObserveCommitQC(qc *types.CommitQC) { @@ -63,16 +58,3 @@ func ObserveCommitQC(qc *types.CommitQC) { *mLast = utils.Some(observed[*types.CommitQC]{now, qc}) } } - -func ObserveAppQC(qc *types.AppQC) { - now := time.Now() - for mLast := range observedAppQC.Lock() { - if last, ok := mLast.Get(); ok && last.val.Proposal().GlobalNumber() >= qc.Proposal().GlobalNumber() { - return - } - Global.appRoadIndexAt().Set(int64(qc.Proposal().RoadIndex())) // nolint: gosec - // +1 is for consistency with commitGlobalBlockNumber - Global.appGlobalBlockNumberAt().Set(int64(qc.Proposal().GlobalNumber() + 1)) // nolint: gosec - *mLast = utils.Some(observed[*types.AppQC]{now, qc}) - } -} diff --git a/sei-tendermint/internal/autobahn/avail/state.go b/sei-tendermint/internal/autobahn/avail/state.go index 8a2e172c44..fb4ab2800d 100644 --- a/sei-tendermint/internal/autobahn/avail/state.go +++ b/sei-tendermint/internal/autobahn/avail/state.go @@ -10,8 +10,6 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/avail/metrics" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/consensus/persist" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/data" - pb "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/pb" - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/protoutils" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/scope" ) @@ -37,7 +35,7 @@ type State struct { // persisters groups all disk persistence components. // Always initialized: real when stateDir is set, no-op otherwise. - persisters persisters + persisters *persisters } func (s *State) PublicKey() types.PublicKey { @@ -48,9 +46,8 @@ func (s *State) PublicKey() types.PublicKey { // (real I/O) or all are no-op (testing). It is a pure I/O struct — all inner // state access goes through State methods. type persisters struct { - pruneAnchor persist.Persister[*pb.PersistedAvailPruneAnchor] - blocks *persist.BlockPersister - commitQCs *persist.CommitQCPersister + blocks *persist.BlockPersister + commitQCs *persist.CommitQCPersister } // close releases the WALs these persisters own, and with them the exclusive lock each holds on its @@ -59,121 +56,26 @@ type persisters struct { // The fields are nil-checked because loadPersistedState fills them one at a time and closes what it // has when a later open fails. func (p persisters) close() error { - var errs []error - if p.blocks != nil { - errs = append(errs, p.blocks.Close()) - } - if p.commitQCs != nil { - errs = append(errs, p.commitQCs.Close()) - } - return errors.Join(errs...) -} - -// innerFile is the A/B file prefix for avail inner state persistence. -const innerFile = "avail_inner" - -// PruneAnchor is the decoded form of the persisted prune anchor -// (AppQC + matching CommitQC pair). It serves as the crash-recovery -// pruning watermark. -type PruneAnchor struct { - AppQC *types.AppQC - CommitQC *types.CommitQC -} - -// PruneAnchorConv converts between PruneAnchor and its protobuf representation. -var PruneAnchorConv = protoutils.Conv[*PruneAnchor, *pb.PersistedAvailPruneAnchor]{ - Encode: func(a *PruneAnchor) *pb.PersistedAvailPruneAnchor { - return &pb.PersistedAvailPruneAnchor{ - AppQc: types.AppQCConv.Encode(a.AppQC), - CommitQc: types.CommitQCConv.Encode(a.CommitQC), - } - }, - Decode: func(p *pb.PersistedAvailPruneAnchor) (*PruneAnchor, error) { - if p.AppQc == nil || p.CommitQc == nil { - return nil, fmt.Errorf("incomplete prune anchor: AppQC=%v CommitQC=%v", p.AppQc != nil, p.CommitQc != nil) - } - appQC, err := types.AppQCConv.Decode(p.AppQc) - if err != nil { - return nil, fmt.Errorf("decode AppQC: %w", err) - } - commitQC, err := types.CommitQCConv.Decode(p.CommitQc) - if err != nil { - return nil, fmt.Errorf("decode CommitQC: %w", err) - } - return &PruneAnchor{AppQC: appQC, CommitQC: commitQC}, nil - }, + return errors.Join(p.blocks.Close(), p.commitQCs.Close()) } // loadPersistedState creates persisters for the given directory option and loads // any existing state from disk. When dir is None, all persisters are no-op // and no state is loaded. When a prune anchor is present, stale commitQCs and // blocks below the anchor are filtered out before returning. -// -// On failure the returned persisters are empty: whatever was already open is released first, because -// each WAL holds an exclusive lock on its directory that a later open of the same directory in this -// process would block on. -func loadPersistedState(dir utils.Option[string]) (_ utils.Option[*loadedAvailState], pers persisters, err error) { - // Every error return below hands back the persisters opened so far so this can release them. - defer func() { - if err != nil { - err = errors.Join(err, pers.close()) - pers = persisters{} - } - }() - - prunePersister, persistedPruneAnchor, err := persist.NewPersister[*pb.PersistedAvailPruneAnchor](dir, innerFile) - if err != nil { - return utils.None[*loadedAvailState](), pers, fmt.Errorf("NewPersister %s: %w", innerFile, err) - } - pers.pruneAnchor = prunePersister - +func loadPersistedState(dir utils.Option[string]) (*loadedState, *persisters, error) { bp, blocks, err := persist.NewBlockPersister(dir) if err != nil { - return utils.None[*loadedAvailState](), pers, fmt.Errorf("NewBlockPersister: %w", err) + return nil, nil, fmt.Errorf("NewBlockPersister: %w", err) } - pers.blocks = bp - cp, commitQCs, err := persist.NewCommitQCPersister(dir) if err != nil { - return utils.None[*loadedAvailState](), pers, fmt.Errorf("NewCommitQCPersister: %w", err) + _ = bp.Close() + return nil, nil, fmt.Errorf("NewCommitQCPersister: %w", err) } - pers.commitQCs = cp - - if _, ok := dir.Get(); !ok { - return utils.None[*loadedAvailState](), pers, nil - } - - loaded := &loadedAvailState{commitQCs: commitQCs, blocks: blocks} - - if raw, ok := persistedPruneAnchor.Get(); ok { - anchor, decodeErr := PruneAnchorConv.Decode(raw) - if decodeErr != nil { - return utils.None[*loadedAvailState](), pers, fmt.Errorf("decode prune anchor: %w", decodeErr) - } - loaded.pruneAnchor = utils.Some(anchor) - - anchorIdx := anchor.AppQC.Proposal().RoadIndex() - filtered := commitQCs[:0] - for _, lqc := range commitQCs { - if lqc.Index >= anchorIdx { - filtered = append(filtered, lqc) - } - } - loaded.commitQCs = filtered - - for lane, bs := range blocks { - first := anchor.CommitQC.LaneRange(lane).First() - j := 0 - for j < len(bs) && bs[j].Number < first { - j++ - } - if j > 0 { - loaded.blocks[lane] = bs[j:] - } - } - } - - return utils.Some(loaded), pers, nil + pers := &persisters{blocks: bp, commitQCs: cp} + loaded := &loadedState{commitQCs: commitQCs, blocks: blocks} + return loaded, pers, nil } // NewState constructs a new availability state. @@ -184,36 +86,11 @@ func NewState(key types.SecretKey, data *data.State, stateDir utils.Option[strin if err != nil { return nil, err } - - // pers owns the WALs from here on, so every failure below has to release them: each WAL holds an - // exclusive lock on its directory that a later open of the same state directory would block on. - defer func() { - if err != nil { - err = errors.Join(err, pers.close()) - } - }() - - ep := data.Registry().LatestEpoch() - inner, err := newInner(ep, loaded) + inner, err := newInner(data, loaded) if err != nil { + _ = pers.close() return nil, err } - - // Truncate WAL entries below the prune anchor that were filtered out by - // loadPersistedState. - if ls, ok := loaded.Get(); ok { - if anchor, ok := ls.pruneAnchor.Get(); ok { - for lane := range ep.Committee().Lanes().All() { - if err := pers.blocks.MaybePruneAndPersistLane(lane, utils.Some(anchor.CommitQC), nil, utils.None[func(*types.Signed[*types.LaneProposal])]()); err != nil { - return nil, fmt.Errorf("prune stale block WAL entries: %w", err) - } - } - if err := pers.commitQCs.MaybePruneAndPersist(utils.Some(anchor.CommitQC), nil, utils.None[func(*types.CommitQC)]()); err != nil { - return nil, fmt.Errorf("prune stale commitQC WAL entries: %w", err) - } - } - } - return &State{ key: key, data: data, @@ -236,9 +113,9 @@ func (s *State) Close() error { return s.persisters.close() } -func (s *State) FirstCommitQC() types.RoadIndex { +func (s *State) First() types.RoadIndex { for inner := range s.inner.Lock() { - return inner.commitQCs.first + return inner.roads.first } panic("unreachable") } @@ -251,85 +128,75 @@ func (s *State) Data() *data.State { // LastCommitQC returns receiver of the LastCommitQC. func (s *State) LastCommitQC() utils.AtomicRecv[utils.Option[*types.CommitQC]] { for inner := range s.inner.Lock() { - return inner.latestCommitQC.Subscribe() + return inner.persistedCommitQC.Subscribe() } panic("unreachable") } -func (s *State) waitForCommitQC(ctx context.Context, idx types.RoadIndex) error { - _, err := s.LastCommitQC().Wait(ctx, func(qc utils.Option[*types.CommitQC]) bool { - return types.NextIndexOpt(qc) > idx - }) - return err -} - -// LastAppQC returns the latest observed AppQC. -func (s *State) LastAppQC() utils.Option[*types.AppQC] { - for inner := range s.inner.Lock() { - return inner.latestAppQC - } - panic("unreachable") -} - -// WaitForAppQC waits until there is an AppQC for the given index or higher. -// Returns this AppQC and the corresponding CommitQC. -// Together they provide enough information to prune the availability state. -func (s *State) WaitForAppQC(ctx context.Context, idx types.RoadIndex) (*types.AppQC, *types.CommitQC, error) { +func (s *State) appQC(ctx context.Context, idx types.RoadIndex) (*types.AppQC, error) { for inner, ctrl := range s.inner.Lock() { for { - if appQC, ok := inner.latestAppQC.Get(); ok { - if x := appQC.Proposal().RoadIndex(); x >= idx && inner.commitQCs.next > x { - return appQC, inner.commitQCs.q[x], nil + if idx < inner.roads.first { + return nil, types.ErrPruned + } + if idx < inner.roads.next { + if appQC, ok := inner.roads.q[idx].appQC.Get(); ok { + return appQC, nil } } if err := ctrl.Wait(ctx); err != nil { - return nil, nil, err + return nil, err } } } panic("unreachable") } -// CommitQC returns the CommitQC for the given index. -func (s *State) CommitQC(ctx context.Context, idx types.RoadIndex) (*types.CommitQC, error) { - if err := s.waitForCommitQC(ctx, idx); err != nil { - return nil, err - } - for inner := range s.inner.Lock() { - if idx < inner.commitQCs.first { - return nil, types.ErrPruned +func (s *State) commitQC(ctx context.Context, idx types.RoadIndex) (*types.Epoch, *types.CommitQC, error) { + for inner, ctrl := range s.inner.Lock() { + if err := ctrl.WaitUntil(ctx, func() bool { return idx < inner.roads.next }); err != nil { + return nil, nil, err + } + if idx < inner.roads.first { + return nil, nil, types.ErrPruned } - return inner.commitQCs.q[idx], nil + r := inner.roads.q[idx] + return r.epoch, r.commitQC, nil } panic("unreachable") } +func (s *State) CommitQC(ctx context.Context, idx types.RoadIndex) (*types.CommitQC, error) { + _, qc, err := s.commitQC(ctx, idx) + return qc, err +} + // PushCommitQC pushes a CommitQC to the state. // Waits until all previous CommitQCs are pushed. func (s *State) PushCommitQC(ctx context.Context, qc *types.CommitQC) error { idx := qc.Proposal().Index() - if idx > 0 { - if err := s.waitForCommitQC(ctx, idx-1); err != nil { + for inner, ctrl := range s.inner.Lock() { + if err := ctrl.WaitUntil(ctx, func() bool { return idx <= inner.roads.next }); err != nil { return err } + if inner.roads.next > idx { + return nil + } } - ep, ok := s.data.Registry().EpochByIndex(qc.Proposal().EpochIndex()) + epoch, ok := s.data.Registry().EpochByIndex(qc.Proposal().EpochIndex()) if !ok { return fmt.Errorf("unknown epoch_index %d", qc.Proposal().EpochIndex()) } - if err := qc.Verify(ep); err != nil { + if err := qc.Verify(epoch); err != nil { return fmt.Errorf("qc.Verify(): %w", err) } for inner, ctrl := range s.inner.Lock() { - if idx != inner.commitQCs.next { + if idx != inner.roads.next { return nil } - if qc.Proposal().EpochIndex() != inner.epoch.EpochIndex() { - return fmt.Errorf("commitQC epoch_index %d != current epoch %d", qc.Proposal().EpochIndex(), inner.epoch.EpochIndex()) - } - inner.commitQCs.pushBack(qc) + inner.roads.pushBack(newRoad(qc, epoch)) metrics.ObserveCommitQC(qc) - // The persist goroutine publishes latestCommitQC after writing to disk + // The persist goroutine publishes persistedCommitQC after writing to disk // (or immediately for no-op persisters), so consensus won't advance // until the CommitQC is durable. ctrl.Updated() @@ -340,86 +207,26 @@ func (s *State) PushCommitQC(ctx context.Context, qc *types.CommitQC) error { // PushAppVote pushes an AppVote to the state. func (s *State) PushAppVote(ctx context.Context, v *types.Signed[*types.AppVote]) error { - ep, ok := s.data.Registry().EpochByIndex(v.Msg().Proposal().EpochIndex()) - if !ok { - return fmt.Errorf("unknown epoch_index %d", v.Msg().Proposal().EpochIndex()) - } - committee := ep.Committee() - idx := v.Msg().Proposal().RoadIndex() - if err := v.VerifySig(committee); err != nil { - return fmt.Errorf("v.VerifySig(): %w", err) - } // Wait for the corresponding commitQC. - if err := s.waitForCommitQC(ctx, idx); err != nil { - return err - } - for inner, ctrl := range s.inner.Lock() { - // Early exit if not useful (we collect <=1 AppQC per road index). - if idx < types.NextOpt(inner.latestAppQC) { - return nil - } - // Verify the vote against the CommitQC. - qc := inner.commitQCs.q[idx] - if err := v.Msg().Proposal().Verify(qc); err != nil { - return fmt.Errorf("invalid vote: %w", err) - } - // Push the vote. - n := v.Msg().Proposal().GlobalNumber() - q := inner.appVotes - for q.next <= n { - q.pushBack(newAppVotes()) - } - appQC, ok := q.q[n].pushVote(committee, v) - if !ok { - return nil - } - updated, err := inner.prune(committee, appQC, qc) - if err != nil { - return err - } - if updated { - ctrl.Updated() - } - } - return nil -} - -// PushAppQC pushes an AppQC to the state. It requires a corresponding CommitQC -// as a justification. -func (s *State) PushAppQC(appQC *types.AppQC, commitQC *types.CommitQC) error { - // Check whether it is needed before verifying. - for inner := range s.inner.Lock() { - if types.NextOpt(inner.latestAppQC) > appQC.Proposal().RoadIndex() { + idx := v.Msg().Proposal().RoadIndex() + epoch, commitQC, err := s.commitQC(ctx, idx) + if err != nil { + if errors.Is(err, types.ErrPruned) { return nil } + return err } - ep, ok := s.data.Registry().EpochByIndex(commitQC.Proposal().EpochIndex()) - if !ok { - return fmt.Errorf("unknown epoch_index %d", commitQC.Proposal().EpochIndex()) - } - if err := appQC.Verify(ep.Committee()); err != nil { - return fmt.Errorf("appQC.Verify(): %w", err) - } - if err := commitQC.Verify(ep); err != nil { - return fmt.Errorf("commitQC.Verify(): %w", err) - } - if appQC.Proposal().RoadIndex() != commitQC.Proposal().Index() { - return fmt.Errorf("mismatched QCs: appQC index %v, commitQC index %v", appQC.Proposal().RoadIndex(), commitQC.Proposal().Index()) - } - if got, want := appQC.Proposal().EpochIndex(), commitQC.Proposal().EpochIndex(); got != want { - return fmt.Errorf("appQC epoch_index %d != commitQC epoch_index %d", got, want) + if err := v.Msg().Proposal().Verify(commitQC); err != nil { + return fmt.Errorf("invalid vote: %w", err) } - // Defense-in-depth check, it should never happen that >f validators sign - // a proposal which does not match the commitQC's global range. - if !commitQC.GlobalRange().Has(appQC.Proposal().GlobalNumber()) { - return fmt.Errorf("appQC GlobalNumber not in commitQC range") + if err := v.VerifySig(epoch.Committee()); err != nil { + return fmt.Errorf("v.VerifySig(): %w", err) } for inner, ctrl := range s.inner.Lock() { - updated, err := inner.prune(ep.Committee(), appQC, commitQC) - if err != nil { - return err + if idx < inner.roads.first || inner.roads.next <= idx { + return nil } - if updated { + if inner.roads.q[idx].pushAppVote(v) { ctrl.Updated() } } @@ -477,7 +284,7 @@ func (s *State) PushBlock(ctx context.Context, p *types.Signed[*types.LanePropos return ErrBadLane } if err := ctrl.WaitUntil(ctx, func() bool { - return h.BlockNumber() <= min(q.next, inner.persistedBlockStart[h.Lane()]+BlocksPerLane-1) + return h.BlockNumber() <= min(q.next, q.first+BlocksPerLane-1) }); err != nil { return err } @@ -515,14 +322,20 @@ func (s *State) PushBlock(ctx context.Context, p *types.Signed[*types.LanePropos // Waits until the lane has enough capacity for the new vote. // It does NOT wait for the previous votes. func (s *State) PushVote(ctx context.Context, vote *types.Signed[*types.LaneVote]) error { - if _, err := s.data.Registry().VerifyInWindow(func(c *types.Committee) error { - if err := vote.Msg().Verify(c); err != nil { + var epoch *types.Epoch + for inner, ctrl := range s.inner.Lock() { + // TODO(gprusak): we should wait only if LaneID is from the future. + if err := ctrl.WaitUntil(ctx, func() bool { return inner.epoch.Committee().HasLane(vote.Key()) }); err != nil { return err } - return vote.VerifySig(c) - }); err != nil { + epoch = inner.epoch + } + if err := vote.Msg().Verify(epoch.Committee()); err != nil { return fmt.Errorf("vote.Verify(): %w", err) } + if err := vote.VerifySig(epoch.Committee()); err != nil { + return fmt.Errorf("vote.VerifySig(): %w", err) + } h := vote.Msg().Header() for inner, ctrl := range s.inner.Lock() { q, ok := inner.votes[h.Lane()] @@ -530,7 +343,7 @@ func (s *State) PushVote(ctx context.Context, vote *types.Signed[*types.LaneVote return ErrBadLane } if err := ctrl.WaitUntil(ctx, func() bool { - return h.BlockNumber() < inner.persistedBlockStart[h.Lane()]+BlocksPerLane + return h.BlockNumber() < q.first+BlocksPerLane }); err != nil { return err } @@ -581,34 +394,31 @@ func (s *State) headers(ctx context.Context, lr *types.LaneRange) ([]*types.Bloc return headers, nil } -func (s *State) fullCommitQC(ctx context.Context, n types.RoadIndex) (*types.FullCommitQC, error) { +func (s *State) fullCommitQC(ctx context.Context, n types.RoadIndex) (*types.Epoch, *types.FullCommitQC, error) { // Collect the CommitQC. - qc, err := s.CommitQC(ctx, n) + epoch, qc, err := s.commitQC(ctx, n) if err != nil { - return nil, err + return nil, nil, err } // Collect the headers from the votes. var commitHeaders []*types.BlockHeader - ep, ok := s.data.Registry().EpochByIndex(qc.Proposal().EpochIndex()) - if !ok { - return nil, fmt.Errorf("unknown epoch_index %d", qc.Proposal().EpochIndex()) - } - for lane := range ep.Committee().Lanes().All() { + for lane := range epoch.Committee().Lanes().All() { headers, err := s.headers(ctx, qc.LaneRange(lane)) if err != nil { - return nil, err + return nil, nil, err } commitHeaders = append(commitHeaders, headers...) } - return types.NewFullCommitQC(qc, commitHeaders), nil + return epoch, types.NewFullCommitQC(qc, commitHeaders), nil } // WaitForLocalCapacity waits until the lane owned by this node has capacity for toProduce block. func (s *State) WaitForLocalCapacity(ctx context.Context, toProduce types.BlockNumber) error { lane := s.key.Public() for inner, ctrl := range s.inner.Lock() { + q := inner.blocks[lane] if err := ctrl.WaitUntil(ctx, func() bool { - return toProduce < inner.persistedBlockStart[lane]+BlocksPerLane + return toProduce < q.first+BlocksPerLane }); err != nil { return err } @@ -660,7 +470,7 @@ func (s *State) produceLocalBlock(n types.BlockNumber, key types.SecretKey, payl if !ok { return nil, ErrBadLane } - if n >= inner.persistedBlockStart[lane]+BlocksPerLane { + if n >= q.first+BlocksPerLane { return nil, fmt.Errorf("lane full") } if q.next != n { @@ -677,135 +487,113 @@ func (s *State) produceLocalBlock(n types.BlockNumber, key types.SecretKey, payl return result, nil } -// Run runs the background tasks of the state. -// -// Goroutines: this method spawns long-lived goroutines via scope.SpawnNamed -// (the persist loop and the FullCommitQC→data-state pusher). Inside -// runPersist, scope.Parallel spawns short-lived goroutines for concurrent -// per-lane block and commit-QC persistence. The persist package itself does -// not spawn goroutines. -func (s *State) Run(ctx context.Context) error { - return scope.Run(ctx, func(ctx context.Context, scope scope.Scope) error { - scope.SpawnNamed("persist", func() error { - return s.runPersist(ctx, s.persisters) - }) - // Task inserting FullCommitQCs and local blocks to data state. - scope.SpawnNamed("s.data.PushQC", func() error { - for n := types.RoadIndex(0); ; n = max(n+1, s.FirstCommitQC()) { - qc, err := s.fullCommitQC(ctx, n) - if err != nil { - if errors.Is(err, types.ErrPruned) { - continue - } - return err - } +// Task inserting CommitQCs and local blocks to data state. +func (s *State) runPushQC(ctx context.Context) error { + for n := types.RoadIndex(0); ; n = max(n+1, s.First()) { + epoch, qc, err := s.fullCommitQC(ctx, n) + if err != nil { + if errors.Is(err, types.ErrPruned) { + continue + } + return err + } - // Collect the blocks we have locally. - ep, ok := s.data.Registry().EpochByIndex(qc.QC().Proposal().EpochIndex()) - if !ok { - return fmt.Errorf("unknown epoch_index %d", qc.QC().Proposal().EpochIndex()) - } - c := ep.Committee() - var blocks []*types.Block - for inner := range s.inner.Lock() { - for lane := range c.Lanes().All() { - lr := qc.QC().LaneRange(lane) - for n := lr.First(); n < lr.Next(); n++ { - // We are not expected to have all the blocks locally - only the available ones. - if b, ok := inner.blocks[lr.Lane()].q[n]; ok { - // We don't need to check the blocks against the headers, - // as bad blocks will be filtered out by PushQC anyway. - blocks = append(blocks, b.Msg().Block()) - } - } + // Collect the blocks we have locally. + c := epoch.Committee() + var blocks []*types.Block + for inner := range s.inner.Lock() { + for lane := range c.Lanes().All() { + lr := qc.QC().LaneRange(lane) + for n := lr.First(); n < lr.Next(); n++ { + // We are not expected to have all the blocks locally - only the available ones. + if b, ok := inner.blocks[lr.Lane()].q[n]; ok { + // We don't need to check the blocks against the headers, + // as bad blocks will be filtered out by PushQC anyway. + blocks = append(blocks, b.Msg().Block()) } } - if err := s.data.PushQC(ctx, qc, blocks); err != nil { - return fmt.Errorf("s.data.PushQC(): %w", err) - } } - }) - return nil - }) + } + if err := s.data.PushQC(ctx, qc, blocks); err != nil { + return fmt.Errorf("s.data.PushQC(): %w", err) + } + } } -// runPersist is the main loop for the persist goroutine. -// Write order: -// 1. Prune anchor (AppQC + CommitQC pair) — the crash-recovery watermark (sequential). -// 2. commitQCs.MaybePruneAndPersist and each lane's blocks.MaybePruneAndPersistLane run -// concurrently via scope.Parallel (separate WALs, no early cancellation; first error -// is returned after all tasks finish). -// Each path appends its whole batch, flushes once, then publishes -// (markCommitQCsPersisted / markBlockPersisted) for every entry in the batch. Publishing -// trails the flush because an append is not durable before it, and voting must never be -// unblocked by an entry a crash could still lose. -// -// The prune anchor is a pruning watermark: on restart we resume from it. -// -// TODO: use a single WAL for anchor and CommitQCs to make -// this atomic rather than relying on write order. -func (s *State) runPersist(ctx context.Context, pers persisters) error { - var lastPersistedAppQCNext types.RoadIndex - for { - batch, err := s.collectPersistBatch(ctx, lastPersistedAppQCNext) +// Task inserting AppQCs to data state. +func (s *State) runPushAppQC(ctx context.Context) error { + for n := types.RoadIndex(0); ; n = max(n+1, s.First()) { + appQC, err := s.appQC(ctx, n) if err != nil { + if errors.Is(err, types.ErrPruned) { + continue + } return err } + if err := s.data.PushAppQC(ctx, appQC); err != nil { + return fmt.Errorf("s.data.PushAppQC(): %w", err) + } + } +} - // Prune CommitQC anchor: same Option drives commit-QC WAL and per-lane block WAL - // (truncate-then-append below this QC). - var anchorQC utils.Option[*types.CommitQC] - // 1. Persist prune anchor first — establishes the crash-recovery watermark. - if anchor, ok := batch.pruneAnchor.Get(); ok { - if err := pers.pruneAnchor.Persist(PruneAnchorConv.Encode(anchor)); err != nil { - return fmt.Errorf("persist prune anchor: %w", err) +func (s *State) runEvict(ctx context.Context) error { + return s.data.Anchor().Iter(ctx, func(ctx context.Context, anchor utils.Option[data.Anchor]) error { + if anchor, ok := anchor.Get(); ok { + for inner, ctrl := range s.inner.Lock() { + inner.prune(anchor) + ctrl.Updated() } - s.advancePersistedBlockStart(anchor.CommitQC) - lastPersistedAppQCNext = anchor.CommitQC.Proposal().Index() + 1 - anchorQC = utils.Some(anchor.CommitQC) } + return nil + }) +} - markBlock := func(p *types.Signed[*types.LaneProposal]) { - header := p.Msg().Block().Header() - s.markBlockPersisted(header.Lane(), header.BlockNumber()+1) - } +// Run runs the background tasks of the state. +func (s *State) Run(ctx context.Context) error { + return scope.Run(ctx, func(ctx context.Context, scope scope.Scope) error { + scope.SpawnNamed("runEvict", func() error { return s.runEvict(ctx) }) + scope.SpawnNamed("runPersist", func() error { return s.runPersist(ctx) }) + scope.SpawnNamed("runPushQC", func() error { return s.runPushQC(ctx) }) + scope.SpawnNamed("runPushAppQC", func() error { return s.runPushAppQC(ctx) }) + return nil + }) +} - blocksByLane := make(map[types.LaneID][]*types.Signed[*types.LaneProposal]) - for _, proposal := range batch.blocks { - lane := proposal.Msg().Block().Header().Lane() - blocksByLane[lane] = append(blocksByLane[lane], proposal) +// runPersist is the main loop for the persist goroutine. +// 2. commitQCs.PruneAndPersist and each lane's blocks.PruneAndPersist run +// concurrently via scope.Parallel (separate WALs, no early cancellation; first error +// is returned after all tasks finish). +// Each path publishes (markCommitQCsPersisted / markBlockPersisted) per entry so voting +// unblocks ASAP. +func (s *State) runPersist(ctx context.Context) error { + for ctx.Err() == nil { + batch, err := s.collectPersistBatch(ctx) + if err != nil { + return err } // 2. Persist commit-QCs and per-lane blocks in parallel. // Callees handle empty inputs gracefully (no-op when nothing to write/truncate). if err := scope.Parallel(func(ps scope.ParallelScope) error { ps.Spawn(func() error { - return pers.commitQCs.MaybePruneAndPersist(anchorQC, batch.commitQCs, utils.Some(func(qc *types.CommitQC) { - s.markCommitQCsPersisted(qc) - })) - }) - // Collect lanes: any lane with blocks in this batch, plus all lanes - // in the anchor epoch (for WAL pruning). - // TODO: when epoch transitions land, also union in lanes from all - // epochs that appear in batch.commitQCs so new-epoch lanes are - // never skipped in a cross-epoch batch. - batchLanes := map[types.LaneID]struct{}{} - for lane := range blocksByLane { - batchLanes[lane] = struct{}{} - } - if anchor, ok := anchorQC.Get(); ok { - ep, epOK := s.data.Registry().EpochByIndex(anchor.Proposal().EpochIndex()) - if !epOK { - return fmt.Errorf("unknown epoch_index %d", anchor.Proposal().EpochIndex()) + if err := s.persisters.commitQCs.PruneAndPersist(batch.commitQCs.first, batch.commitQCs.tail); err != nil { + return err } - for lane := range ep.Committee().Lanes().All() { - batchLanes[lane] = struct{}{} + if t := batch.commitQCs.tail; len(t) > 0 { + s.markCommitQCsPersisted(t[len(t)-1]) } - } - for lane := range batchLanes { - proposals := blocksByLane[lane] + return nil + }) + for lane, batch := range batch.blocks { ps.Spawn(func() error { - return pers.blocks.MaybePruneAndPersistLane(lane, anchorQC, proposals, utils.Some(markBlock)) + if err := s.persisters.blocks.PruneAndPersist(lane, batch.first, batch.tail); err != nil { + return err + } + if n := len(batch.tail); n > 0 { + header := batch.tail[n-1].Msg().Block().Header() + s.markBlockPersisted(lane, header.BlockNumber()+1) + } + return nil }) } return nil @@ -813,28 +601,22 @@ func (s *State) runPersist(ctx context.Context, pers persisters) error { return err } } + return ctx.Err() } -// persistBatch holds the data collected under lock for one persist iteration. -type persistBatch struct { - blocks []*types.Signed[*types.LaneProposal] - commitQCs []*types.CommitQC - pruneAnchor utils.Option[*PruneAnchor] +type batch[I any, T any] struct { + first I + tail []T } -// advancePersistedBlockStart updates the per-lane block admission watermark -// after durably writing the prune anchor. This unblocks PushBlock/ProduceBlock -// waiters that are gated on persistedBlockStart + BlocksPerLane. -func (s *State) advancePersistedBlockStart(commitQC *types.CommitQC) { - for inner, ctrl := range s.inner.Lock() { - for lane := range inner.blocks { - start := commitQC.LaneRange(lane).First() - if start > inner.persistedBlockStart[lane] { - inner.persistedBlockStart[lane] = start - } - } - ctrl.Updated() - } +type blocksBatch = batch[types.BlockNumber, *types.Signed[*types.LaneProposal]] +type commitQCsBatch = batch[types.RoadIndex, *types.CommitQC] + +// persistBatch holds the data collected under lock for one persist iteration. +type persistBatch struct { + epoch *types.Epoch + blocks map[types.LaneID]blocksBatch + commitQCs commitQCsBatch } // markBlockPersisted advances the per-lane block persistence cursor. @@ -851,57 +633,46 @@ func (s *State) markBlockPersisted(lane types.LaneID, next types.BlockNumber) { // markCommitQCsPersisted publishes the latest persisted CommitQC, // gating consensus from advancing until the QC is durable. func (s *State) markCommitQCsPersisted(qc *types.CommitQC) { - for inner, ctrl := range s.inner.Lock() { - inner.latestCommitQC.Store(utils.Some(qc)) - ctrl.Updated() + for inner := range s.inner.Lock() { + inner.persistedCommitQC.Store(utils.Some(qc)) } } // collectPersistBatch waits for new blocks or commitQCs and collects them under lock. -func (s *State) collectPersistBatch(ctx context.Context, lastPersistedAppQCNext types.RoadIndex) (persistBatch, error) { - var b persistBatch +func (s *State) collectPersistBatch(ctx context.Context) (*persistBatch, error) { for inner, ctrl := range s.inner.Lock() { - // Derive the CommitQC persist cursor from latestCommitQC. This is - // safe because latestCommitQC is only advanced by markCommitQCsPersisted + // Derive the CommitQC persist cursor from persistedCommitQC. This is + // safe because persistedCommitQC is only advanced by markCommitQCsPersisted // (after disk write) and on startup (from disk). prune() does NOT - // update latestCommitQC, so this always reflects persistence state. + // update persistedCommitQC, so this always reflects persistence state. // The max clamp with commitQCs.first handles the case where prune() // fast-forwarded the queue past the cursor. - commitQCNext := types.NextIndexOpt(inner.latestCommitQC.Load()) + next := types.NextIndexOpt(inner.persistedCommitQC.Load()) if err := ctrl.WaitUntil(ctx, func() bool { - if types.NextOpt(inner.latestAppQC) != lastPersistedAppQCNext { - return true - } for lane, q := range inner.blocks { if inner.nextBlockToPersist[lane] < q.next { return true } } - return commitQCNext < inner.commitQCs.next + return next < inner.roads.next }); err != nil { - return b, err + return nil, err } - for lane, q := range inner.blocks { - start := max(inner.nextBlockToPersist[lane], q.first) - for n := start; n < q.next; n++ { - b.blocks = append(b.blocks, q.q[n]) - } + b := &persistBatch{ + blocks: map[types.LaneID]blocksBatch{}, + commitQCs: commitQCsBatch{first: inner.roads.first}, } - commitQCNext = max(commitQCNext, inner.commitQCs.first) - for n := commitQCNext; n < inner.commitQCs.next; n++ { - b.commitQCs = append(b.commitQCs, inner.commitQCs.q[n]) - } - if types.NextOpt(inner.latestAppQC) != lastPersistedAppQCNext { - if appQC, ok := inner.latestAppQC.Get(); ok { - idx := appQC.Proposal().RoadIndex() - if qc, ok := inner.commitQCs.q[idx]; ok { - b.pruneAnchor = utils.Some(&PruneAnchor{ - AppQC: appQC, - CommitQC: qc, - }) - } + for n := max(next, inner.roads.first); n < inner.roads.next; n++ { + b.commitQCs.tail = append(b.commitQCs.tail, inner.roads.q[n].commitQC) + } + for lane, q := range inner.blocks { + bb := blocksBatch{first: q.first} + for n := max(inner.nextBlockToPersist[lane], q.first); n < q.next; n++ { + bb.tail = append(bb.tail, q.q[n]) } + b.blocks[lane] = bb } + return b, nil } - return b, nil + panic("unreachable") } diff --git a/sei-tendermint/internal/autobahn/avail/state_test.go b/sei-tendermint/internal/autobahn/avail/state_test.go index 7411cd0c45..76267051b7 100644 --- a/sei-tendermint/internal/autobahn/avail/state_test.go +++ b/sei-tendermint/internal/autobahn/avail/state_test.go @@ -4,27 +4,17 @@ import ( "context" "errors" "fmt" - "os" - "path/filepath" "testing" - "time" - "github.com/sei-protocol/sei-chain/sei-db/ledger_db/block/memblock" "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/consensus/persist" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/data" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/epoch" - pb "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/pb" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/scope" "github.com/stretchr/testify/require" ) -var ( - noBlockCB = utils.None[func(*types.Signed[*types.LaneProposal])]() - noCommitQCCB = utils.None[func(*types.CommitQC)]() -) - type byLane[T any] map[types.LaneID][]T func makeAppVotes(keys []types.SecretKey, proposal *types.AppProposal) []*types.Signed[*types.AppVote] { @@ -36,40 +26,6 @@ func makeAppVotes(keys []types.SecretKey, proposal *types.AppProposal) []*types. return votes } -func TestSubscribeAppVotesJumpsToDataFloor(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 3) - qc, blocks := data.TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) - gr := qc.QC().GlobalRange() - require.Greater(t, gr.Len(), uint64(2)) - - db := memblock.NewBlockDB() - t.Cleanup(func() { require.NoError(t, db.Close()) }) - require.NoError(t, db.WriteQC(qc)) - for i, n := 0, gr.First; n < gr.Next; i, n = i+1, n+1 { - require.NoError(t, db.WriteBlock(n, blocks[i])) - } - require.NoError(t, db.Flush()) - - first := gr.First + types.GlobalBlockNumber(gr.Len()/2) - ds, err := data.NewState(&data.Config{ - Registry: registry, - LastExecutedBlock: utils.Some(first), - }, db) - require.NoError(t, err) - appHash := types.GenAppHash(rng) - require.NoError(t, ds.PushAppHash(t.Context(), first, appHash)) - - state, err := NewState(keys[0], ds, utils.None[string]()) - require.NoError(t, err) - recv := state.SubscribeAppVotes() - require.Equal(t, types.GlobalBlockNumber(0), recv.next) - - vote, err := recv.Recv(t.Context()) - require.NoError(t, err) - require.Equal(t, first, vote.Msg().Proposal().GlobalNumber()) -} - func makeLaneVotes(keys []types.SecretKey, h *types.BlockHeader) []*types.Signed[*types.LaneVote] { var votes []*types.Signed[*types.LaneVote] for _, k := range keys { @@ -78,26 +34,6 @@ func makeLaneVotes(keys []types.SecretKey, h *types.BlockHeader) []*types.Signed return votes } -func leaderKey(committee *types.Committee, keys []types.SecretKey, view types.View) types.SecretKey { - leader := committee.Leader(view) - for _, k := range keys { - if k.Public() == leader { - return k - } - } - panic("leader not in keys") -} - -func makeCommitQC( - ep *types.Epoch, - keys []types.SecretKey, - prev utils.Option[*types.CommitQC], - laneQCs map[types.LaneID]*types.LaneQC, - appQC utils.Option[*types.AppQC], -) *types.CommitQC { - return types.BuildCommitQC(ep, keys, prev, laneQCs, appQC) -} - func qcPayloadHashes(qc *types.FullCommitQC) byLane[types.PayloadHash] { x := byLane[types.PayloadHash]{} for _, h := range qc.Headers() { @@ -107,7 +43,10 @@ func qcPayloadHashes(qc *types.FullCommitQC) byLane[types.PayloadHash] { } func TestState(t *testing.T) { - testState(t, utils.None[string]()) + rng := utils.TestRng() + for range 5 { + testState(t, rng, utils.None[string]()) + } } // TestStateWithPersistence runs the same flow as TestState but with disk @@ -115,28 +54,84 @@ func TestState(t *testing.T) { // run concurrently, exercising the cursor-clamp logic that prevents reading // pruned map entries. func TestStateWithPersistence(t *testing.T) { + rng := utils.TestRng() for range 5 { - testState(t, utils.Some(t.TempDir())) + testState(t, rng, utils.Some(t.TempDir())) } } -func testState(t *testing.T, stateDir utils.Option[string]) { - t.Helper() +// Test checking that State can correctly start collecting CommitQCs starting from arbitrary anchor. +func TestAnchorResetsState(t *testing.T) { ctx := t.Context() rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 3) + epoch := registry.LatestEpoch() + require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { + t.Logf("data.Run()") + ds := newTestDataState(&data.Config{Registry: registry}) + s.SpawnBgNamed("data.Run", func() error { return utils.IgnoreCancel(ds.Run(ctx)) }) + + t.Logf("Push FullCommitQC, blocks, AppHash, AppQC to data") + qc, blocks := data.TestCommitQC(rng, epoch, keys, utils.None[*types.CommitQC]()) + if err := ds.PushQC(ctx, qc, blocks); err != nil { + return err + } + appHash := types.GenAppHash(rng) + if err := ds.PushAppHash(ctx, qc.QC().GlobalRange().Next-1, appHash); err != nil { + return err + } + appQC := data.TestAppQC(keys, types.NewAppProposal(qc.QC().Proposal(), appHash)) + if err := ds.PushAppQC(ctx, appQC); err != nil { + return err + } + + t.Logf("wait for anchor to be updated") + if _, err := ds.Anchor().Wait(ctx, func(anchor utils.Option[data.Anchor]) bool { + a, ok := anchor.Get() + return ok && a.CommitQC.Index() == qc.QC().Index() + }); err != nil { + return err + } + + t.Logf("NewState() should load the anchor") + state, err := NewState(keys[0], ds, utils.None[string]()) + if err != nil { + return fmt.Errorf("NewState(): %w", err) + } + if err := utils.TestDiff(utils.Some(qc.QC()), state.LastCommitQC().Load()); err != nil { + return err + } + t.Logf("avail.Run()") + s.SpawnBgNamed("avail.Run", func() error { return utils.IgnoreCancel(state.Run(ctx)) }) + + t.Logf("push next CommitQC to avail") + qc, _ = data.TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc.QC())) + if err := state.PushCommitQC(ctx, qc.QC()); err != nil { + return err + } + t.Logf("wait for this CommitQC to be persisted in avail") + _, err = state.LastCommitQC().Wait(ctx, func(got utils.Option[*types.CommitQC]) bool { + gotQC, ok := got.Get() + return ok && gotQC.Index() == qc.QC().Index() + }) + return err + })) +} + +func testState(t *testing.T, rng utils.Rng, stateDir utils.Option[string]) { + t.Helper() + ctx := t.Context() + registry, keys := epoch.GenRegistry(rng, 3) committee := registry.LatestEpoch().Committee() if err := scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { ds := newTestDataState(&data.Config{Registry: registry}) - s.SpawnBgNamed("data.State.Run()", func() error { - return utils.IgnoreCancel(ds.Run(ctx)) - }) + s.SpawnBgNamed("ds.Run()", func() error { return utils.IgnoreCancel(ds.Run(ctx)) }) state, err := NewState(keys[0], ds, stateDir) - require.NoError(t, err) - s.SpawnBgNamed("da.State.Run()", func() error { - return utils.IgnoreCancel(state.Run(ctx)) - }) + if err != nil { + return fmt.Errorf("NewState(): %w", err) + } + s.SpawnBgNamed("state.Run()", func() error { return utils.IgnoreCancel(state.Run(ctx)) }) for i := range 3 { t.Logf("iteration %v", i) @@ -179,47 +174,49 @@ func testState(t *testing.T, stateDir utils.Option[string]) { if err != nil { return fmt.Errorf("state.WaitForNewLaneQCs(): %w", err) } - qc := makeCommitQC(registry.LatestEpoch(), keys, prev, laneQCs, state.LastAppQC()) + qc := types.BuildCommitQC(registry.LatestEpoch(), keys, prev, laneQCs) if err := state.PushCommitQC(ctx, qc); err != nil { return fmt.Errorf("state.PushCommitQC(): %w", err) } + t.Logf("Check that a CommitQC was successfully reconstructed.") + _, got, err := state.fullCommitQC(ctx, qc.Proposal().Index()) + if err != nil { + return fmt.Errorf("state.fullCommitQC(): %w", err) + } + if err := utils.TestDiff(want, qcPayloadHashes(got)); err != nil { + return fmt.Errorf("snapshot: %w", err) + } + t.Logf("Push app votes.") - appProposal := types.NewAppProposal(qc.GlobalRange().Next-1, qc.Proposal().Index(), types.GenAppHash(rng), 0) + appHash := types.GenAppHash(rng) + appProposal := types.NewAppProposal(qc.Proposal(), appHash) + appGR := appProposal.GlobalRange() + if err := ds.PushAppHash(ctx, appGR.Next-1, appHash); err != nil { + return fmt.Errorf("ds.PushAppHash(): %w", err) + } for _, vote := range makeAppVotes(keys, appProposal) { if err := state.PushAppVote(ctx, vote); err != nil { return fmt.Errorf("state.PushAppVote(): %w", err) } } - t.Logf("Previous one should be pruned because of appQC.") - if _, _, err := state.WaitForAppQC(ctx, appProposal.RoadIndex()); err != nil { - return fmt.Errorf("state.WaitForAppQC(): %w", err) - } - if prev, ok := prev.Get(); ok { - if _, err := state.CommitQC(ctx, prev.Proposal().Index()); !errors.Is(err, types.ErrPruned) { - return fmt.Errorf("state.CommitQC(): %w, want %v", err, types.ErrPruned) + t.Logf("Executed CommitQC should be eventually evicted") + for inner, ctrl := range state.inner.Lock() { + if err := ctrl.WaitUntil(ctx, func() bool { return inner.roads.first == appProposal.RoadIndex()+1 }); err != nil { + return err } } t.Logf("Check that the executed local blocks have been pruned") for lane := range committee.Lanes().All() { - if lr := types.LaneRangeOpt(prev, lane); lr.Next() > 0 { + if lr := qc.LaneRange(lane); lr.Next() > 0 { if _, err := state.Block(ctx, lane, lr.Next()-1); !errors.Is(err, types.ErrPruned) { return fmt.Errorf("state.Block(): %w, want %v", err, types.ErrPruned) } } } - t.Logf("Check that a CommitQC was successfully reconstructed.") - got, err := state.fullCommitQC(ctx, qc.Proposal().Index()) - if err != nil { - return fmt.Errorf("state.fullCommitQC(): %w", err) - } - if err := utils.TestDiff(want, qcPayloadHashes(got)); err != nil { - return fmt.Errorf("snapshot: %w", err) - } - t.Logf("Check that the blocks were successfully pushed to data state.") gr := got.QC().GlobalRange() for i := gr.First; i < gr.Next; i++ { @@ -243,10 +240,9 @@ func testState(t *testing.T, stateDir utils.Option[string]) { // from the same directory. This verifies that what the runtime persist // goroutine writes can be correctly loaded back by loadPersistedState/newInner. // -// After iteration 0's AppQC prunes old data, iteration 1 writes new blocks -// and commitQCs at higher indices. If WAL truncation hasn't cleaned up the -// stale entries by shutdown, restart exercises the gap-filtering path in -// loadPersistedState (stale entries below the prune anchor are discarded). +// The restarted state uses a fresh in-memory data.State, so this covers the +// availability persisters themselves: CommitQCs and local blocks are loaded back +// and lane next-block cursors resume where they left off. func TestStateRestartFromPersisted(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 3) @@ -274,9 +270,9 @@ func TestStateRestartFromPersisted(t *testing.T) { return utils.IgnoreCancel(state.Run(ctx)) }) + var prev utils.Option[*types.CommitQC] for i := range 2 { t.Logf("iteration %d", i) - prev := state.LastCommitQC().Load() for range 5 { key := keys[rng.Intn(len(keys))] @@ -304,20 +300,21 @@ func TestStateRestartFromPersisted(t *testing.T) { if err != nil { return fmt.Errorf("WaitForLaneQCs: %w", err) } - qc := makeCommitQC(registry.LatestEpoch(), keys, prev, laneQCs, state.LastAppQC()) + qc := types.BuildCommitQC(registry.LatestEpoch(), keys, prev, laneQCs) if err := state.PushCommitQC(ctx, qc); err != nil { return fmt.Errorf("PushCommitQC: %w", err) } - appProposal := types.NewAppProposal(qc.GlobalRange().Next-1, qc.Proposal().Index(), types.GenAppHash(rng), 0) + appProposal := types.NewAppProposal(qc.Proposal(), types.GenAppHash(rng)) for _, vote := range makeAppVotes(keys, appProposal) { if err := state.PushAppVote(ctx, vote); err != nil { return fmt.Errorf("PushAppVote: %w", err) } } - if _, _, err := state.WaitForAppQC(ctx, appProposal.RoadIndex()); err != nil { + if _, err := state.appQC(ctx, appProposal.RoadIndex()); err != nil { return fmt.Errorf("WaitForAppQC: %w", err) } + prev = utils.Some(qc) wantAppQCIdx = appProposal.RoadIndex() } @@ -325,7 +322,9 @@ func TestStateRestartFromPersisted(t *testing.T) { // all commitQCs in the batch are on disk. Block goroutines may still // be in flight, but scope.Parallel in runPersist ensures they complete // before the next batch, so the data is durable by scope exit. - if err := state.waitForCommitQC(ctx, wantAppQCIdx); err != nil { + if _, err := state.LastCommitQC().Wait(ctx, func(qc utils.Option[*types.CommitQC]) bool { + return types.NextIndexOpt(qc) > wantAppQCIdx + }); err != nil { return fmt.Errorf("waitForCommitQC: %w", err) } @@ -344,13 +343,7 @@ func TestStateRestartFromPersisted(t *testing.T) { state2, err := NewState(keys[0], ds2, utils.Some(dir)) require.NoError(t, err) - got, ok := state2.LastAppQC().Get() - require.True(t, ok, "AppQC should be restored after restart") - require.Equal(t, wantAppQCIdx, got.Proposal().RoadIndex()) - - require.GreaterOrEqual(t, state2.FirstCommitQC(), wantAppQCIdx) - - _, ok = state2.LastCommitQC().Load().Get() + _, ok := state2.LastCommitQC().Load().Get() require.True(t, ok, "LastCommitQC should be set after restart") for lane := range committee.Lanes().All() { @@ -359,62 +352,6 @@ func TestStateRestartFromPersisted(t *testing.T) { } } -func TestStateMismatchedQCs(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - committee := registry.LatestEpoch().Committee() - initialBlock := registry.FirstBlock() - - ds := newTestDataState(&data.Config{Registry: registry}) - state, err := NewState(keys[0], ds, utils.Some(t.TempDir())) - require.NoError(t, err) - - // Helper to create a CommitQC for a specific index - makeQC := func(prev utils.Option[*types.CommitQC], laneQCs map[types.LaneID]*types.LaneQC) *types.CommitQC { - vs := types.ViewSpec{CommitQC: prev, Epoch: types.NewEpoch(0, types.OpenRoadRange(), time.Time{}, committee, initialBlock)} - fullProposal := utils.OrPanic1(types.NewProposal( - leaderKey(committee, keys, vs.View()), - vs, - time.Now(), - laneQCs, - utils.None[*types.AppQC](), - )) - vote := types.NewCommitVote(fullProposal.Proposal().Msg()) - var votes []*types.Signed[*types.CommitVote] - for _, k := range keys { - votes = append(votes, types.Sign(k, vote)) - } - return types.NewCommitQC(votes) - } - - // 1. Produce a block so we have a non-empty range - lane := keys[0].Public() - p := types.GenPayload(rng) - b, err := state.ProduceLocalBlock(state.NextBlock(lane), p) - require.NoError(t, err) - - // 2. Form a LaneQC for it - laneQC := types.NewLaneQC(makeLaneVotes( - types.TestKeysWithWeight(committee, keys, committee.LaneQuorum()), - b.Msg().Block().Header(), - )) - - // 3. Create CommitQC for index 0 (finalizes block 0) - qc0 := makeQC(utils.None[*types.CommitQC](), map[types.LaneID]*types.LaneQC{lane: laneQC}) - require.Equal(t, initialBlock, qc0.GlobalRange().First) - require.Equal(t, initialBlock+1, qc0.GlobalRange().Next) - - t.Run("PushAppQC mismatch", func(t *testing.T) { - require := require.New(t) - // AppQC for index 1, but paired with CommitQC for index 0 - appProposal1 := types.NewAppProposal(initialBlock, 1, types.GenAppHash(rng), 0) - appQC1 := types.NewAppQC(makeAppVotes(keys, appProposal1)) - - err := state.PushAppQC(appQC1, qc0) - require.Error(err) - }) -} - func TestPushBlockRejectsBadParentHash(t *testing.T) { ctx := t.Context() rng := utils.TestRng() @@ -458,8 +395,6 @@ func TestPushBlockRejectsWrongSigner(t *testing.T) { func TestNewStateWithPersistence(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 4) - initialBlock := registry.FirstBlock() - t.Run("empty dir loads fresh state", func(t *testing.T) { dir := t.TempDir() ds := newTestDataState(&data.Config{Registry: registry}) @@ -467,54 +402,8 @@ func TestNewStateWithPersistence(t *testing.T) { state, err := NewState(keys[0], ds, utils.Some(dir)) require.NoError(t, err) - // No persisted AppQC → None. - require.False(t, state.LastAppQC().IsPresent()) // Queues start at 0. - require.Equal(t, types.RoadIndex(0), state.FirstCommitQC()) - }) - - t.Run("loads persisted AppQC", func(t *testing.T) { - dir := t.TempDir() - ds := newTestDataState(&data.Config{Registry: registry}) - - roadIdx := types.RoadIndex(7) - globalNum := types.GlobalBlockNumber(50) - appProposal := types.NewAppProposal(globalNum, roadIdx, types.GenAppHash(rng), 0) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - // Persist commitQCs 0-7 so the matching one at roadIdx exists. - cp, _, err := persist.NewCommitQCPersister(utils.Some(dir)) - require.NoError(t, err) - prev := utils.None[*types.CommitQC]() - var pruneQC *types.CommitQC - for i := types.RoadIndex(0); i <= roadIdx; i++ { - qc := makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qc) - require.NoError(t, cp.MaybePruneAndPersist(utils.None[*types.CommitQC](), []*types.CommitQC{qc}, noCommitQCCB)) - pruneQC = qc - } - - // Persist prune anchor (AppQC + CommitQC pair). - prunePers, _, err := persist.NewPersister[*pb.PersistedAvailPruneAnchor](utils.Some(dir), innerFile) - require.NoError(t, err) - require.NoError(t, prunePers.Persist(&pb.PersistedAvailPruneAnchor{ - AppQc: types.AppQCConv.Encode(appQC), - CommitQc: types.CommitQCConv.Encode(pruneQC), - })) - - // Release the seeding persister's WAL locks before NewState opens the same directory. - require.NoError(t, cp.Close()) - - state, err := NewState(keys[0], ds, utils.Some(dir)) - require.NoError(t, err) - - aq := state.LastAppQC() - got, ok := aq.Get() - require.True(t, ok) - require.Equal(t, roadIdx, got.Proposal().RoadIndex()) - require.Equal(t, globalNum, got.Proposal().GlobalNumber()) - - require.Equal(t, roadIdx, state.FirstCommitQC()) + require.Equal(t, types.RoadIndex(0), state.First()) }) t.Run("loads persisted blocks", func(t *testing.T) { @@ -531,7 +420,7 @@ func TestNewStateWithPersistence(t *testing.T) { block := types.NewBlock(lane, n, parent, types.GenPayload(rng)) signed := types.Sign(keys[0], types.NewLaneProposal(block)) parent = block.Header().Hash() - require.NoError(t, bp.MaybePruneAndPersistLane(lane, utils.None[*types.CommitQC](), []*types.Signed[*types.LaneProposal]{signed}, noBlockCB)) + require.NoError(t, bp.PruneAndPersist(lane, 0, []*types.Signed[*types.LaneProposal]{signed})) } // Release the seeding persister's WAL locks before NewState opens the same directory. @@ -544,63 +433,6 @@ func TestNewStateWithPersistence(t *testing.T) { require.Equal(t, types.BlockNumber(3), state.NextBlock(lane)) }) - t.Run("loads persisted AppQC and blocks together", func(t *testing.T) { - dir := t.TempDir() - ds := newTestDataState(&data.Config{Registry: registry}) - lane := keys[0].Public() - - roadIdx := types.RoadIndex(2) - globalNum := types.GlobalBlockNumber(5) - appProposal := types.NewAppProposal(globalNum, roadIdx, types.GenAppHash(rng), 0) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - // Persist commitQCs 0-2 so the matching one at roadIdx exists. - cp, _, err := persist.NewCommitQCPersister(utils.Some(dir)) - require.NoError(t, err) - prev := utils.None[*types.CommitQC]() - var pruneQC *types.CommitQC - for range roadIdx + 1 { - qc := makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qc) - require.NoError(t, cp.MaybePruneAndPersist(utils.None[*types.CommitQC](), []*types.CommitQC{qc}, noCommitQCCB)) - pruneQC = qc - } - - // Persist prune anchor (AppQC + CommitQC pair). - prunePers, _, err := persist.NewPersister[*pb.PersistedAvailPruneAnchor](utils.Some(dir), innerFile) - require.NoError(t, err) - require.NoError(t, prunePers.Persist(&pb.PersistedAvailPruneAnchor{ - AppQc: types.AppQCConv.Encode(appQC), - CommitQc: types.CommitQCConv.Encode(pruneQC), - })) - - // Persist blocks starting at 0 (nil laneQCs → lr.First()=0 after prune). - bp, _, err := persist.NewBlockPersister(utils.Some(dir)) - require.NoError(t, err) - - var parent types.BlockHeaderHash - for n := range types.BlockNumber(3) { - block := types.NewBlock(lane, n, parent, types.GenPayload(rng)) - signed := types.Sign(keys[0], types.NewLaneProposal(block)) - parent = block.Header().Hash() - require.NoError(t, bp.MaybePruneAndPersistLane(lane, utils.None[*types.CommitQC](), []*types.Signed[*types.LaneProposal]{signed}, noBlockCB)) - } - - // Release the seeding persisters' WAL locks before NewState opens the same directory. - require.NoError(t, cp.Close()) - require.NoError(t, bp.Close()) - - state, err := NewState(keys[0], ds, utils.Some(dir)) - require.NoError(t, err) - - got, ok := state.LastAppQC().Get() - require.True(t, ok) - require.Equal(t, roadIdx, got.Proposal().RoadIndex()) - - require.Equal(t, types.BlockNumber(3), state.NextBlock(lane)) - require.Equal(t, roadIdx, state.FirstCommitQC()) - }) - t.Run("loads persisted commitQCs", func(t *testing.T) { dir := t.TempDir() ds := newTestDataState(&data.Config{Registry: registry}) @@ -612,9 +444,9 @@ func TestNewStateWithPersistence(t *testing.T) { qcs := make([]*types.CommitQC, 3) prev := utils.None[*types.CommitQC]() for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) + qcs[i] = types.BuildCommitQC(registry.LatestEpoch(), keys, prev, nil) prev = utils.Some(qcs[i]) - require.NoError(t, cp.MaybePruneAndPersist(utils.None[*types.CommitQC](), []*types.CommitQC{qcs[i]}, noCommitQCCB)) + require.NoError(t, cp.PruneAndPersist(0, []*types.CommitQC{qcs[i]})) } // Release the seeding persister's WAL locks before NewState opens the same directory. @@ -624,52 +456,11 @@ func TestNewStateWithPersistence(t *testing.T) { require.NoError(t, err) // All 3 commitQCs should be loaded (no AppQC to skip past). - require.Equal(t, types.RoadIndex(0), state.FirstCommitQC()) + require.Equal(t, types.RoadIndex(0), state.First()) // LastCommitQC should be set to the last loaded one. require.NoError(t, utils.TestDiff(utils.Some(qcs[2]), state.LastCommitQC().Load())) }) - t.Run("loads persisted commitQCs with AppQC", func(t *testing.T) { - dir := t.TempDir() - ds := newTestDataState(&data.Config{Registry: registry}) - - // Persist AppQC at road index 1. - roadIdx := types.RoadIndex(1) - globalNum := types.GlobalBlockNumber(5) - appProposal := types.NewAppProposal(globalNum, roadIdx, types.GenAppHash(rng), 0) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - // Persist CommitQCs 0-4. - cp, _, err := persist.NewCommitQCPersister(utils.Some(dir)) - require.NoError(t, err) - - qcs := make([]*types.CommitQC, 5) - prev := utils.None[*types.CommitQC]() - for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[i]) - require.NoError(t, cp.MaybePruneAndPersist(utils.None[*types.CommitQC](), []*types.CommitQC{qcs[i]}, noCommitQCCB)) - } - - // Persist prune anchor (AppQC + CommitQC pair at roadIdx). - prunePers, _, err := persist.NewPersister[*pb.PersistedAvailPruneAnchor](utils.Some(dir), innerFile) - require.NoError(t, err) - require.NoError(t, prunePers.Persist(&pb.PersistedAvailPruneAnchor{ - AppQc: types.AppQCConv.Encode(appQC), - CommitQc: types.CommitQCConv.Encode(qcs[roadIdx]), - })) - - // Release the seeding persister's WAL locks before NewState opens the same directory. - require.NoError(t, cp.Close()) - - state, err := NewState(keys[0], ds, utils.Some(dir)) - require.NoError(t, err) - - // inner.prune(appQC@1, commitQC@1) sets commitQCs.first = 1. - require.Equal(t, types.RoadIndex(1), state.FirstCommitQC()) - require.NoError(t, utils.TestDiff(utils.Some(qcs[4]), state.LastCommitQC().Load())) - }) - t.Run("non-contiguous commitQC files return error", func(t *testing.T) { dir := t.TempDir() @@ -677,184 +468,21 @@ func TestNewStateWithPersistence(t *testing.T) { allQCs := make([]*types.CommitQC, 6) prev := utils.None[*types.CommitQC]() for i := range allQCs { - allQCs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) + allQCs[i] = types.BuildCommitQC(registry.LatestEpoch(), keys, prev, nil) prev = utils.Some(allQCs[i]) } - // Persist prune anchor (AppQC + CommitQC pair at road index 0). - appProposal := types.NewAppProposal(initialBlock, 0, types.GenAppHash(rng), 0) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - prunePers, _, err := persist.NewPersister[*pb.PersistedAvailPruneAnchor](utils.Some(dir), innerFile) - require.NoError(t, err) - require.NoError(t, prunePers.Persist(&pb.PersistedAvailPruneAnchor{ - AppQc: types.AppQCConv.Encode(appQC), - CommitQc: types.CommitQCConv.Encode(allQCs[0]), - })) - // Persist QCs 0, 1, 2 contiguously, then try to skip to 5. - // MaybePruneAndPersist enforces strict sequential order, so the gap + // Persist enforces strict sequential order, so the gap // is caught at write time rather than at load time. cp, _, err := persist.NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) for i := range 3 { - require.NoError(t, cp.MaybePruneAndPersist(utils.None[*types.CommitQC](), []*types.CommitQC{allQCs[i]}, noCommitQCCB)) + require.NoError(t, cp.PruneAndPersist(0, []*types.CommitQC{allQCs[i]})) } - err = cp.MaybePruneAndPersist(utils.None[*types.CommitQC](), []*types.CommitQC{allQCs[5]}, noCommitQCCB) + err = cp.PruneAndPersist(0, []*types.CommitQC{allQCs[5]}) require.Error(t, err) require.Contains(t, err.Error(), "out of sequence") require.NoError(t, cp.Close()) }) - - t.Run("anchor past all persisted commitQCs truncates WAL", func(t *testing.T) { - dir := t.TempDir() - ds := newTestDataState(&data.Config{Registry: registry}) - - // Build a chain of 10 CommitQCs (indices 0-9). - qcs := make([]*types.CommitQC, 10) - prev := utils.None[*types.CommitQC]() - for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[i]) - } - - // Persist only indices 0-4 to the CommitQC WAL. - cp, _, err := persist.NewCommitQCPersister(utils.Some(dir)) - require.NoError(t, err) - for i := range 5 { - require.NoError(t, cp.MaybePruneAndPersist(utils.None[*types.CommitQC](), []*types.CommitQC{qcs[i]}, noCommitQCCB)) - } - require.NoError(t, cp.Close()) - - // Persist a prune anchor at index 9 — well past the persisted range. - appProposal := types.NewAppProposal(50, 9, types.GenAppHash(rng), 0) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - prunePers, _, err := persist.NewPersister[*pb.PersistedAvailPruneAnchor](utils.Some(dir), innerFile) - require.NoError(t, err) - require.NoError(t, prunePers.Persist(&pb.PersistedAvailPruneAnchor{ - AppQc: types.AppQCConv.Encode(appQC), - CommitQc: types.CommitQCConv.Encode(qcs[9]), - })) - - // NewState should succeed: MaybePruneAndPersist truncates the stale WAL - // and internally re-persists the anchor's CommitQC for crash recovery. - state, err := NewState(keys[0], ds, utils.Some(dir)) - require.NoError(t, err) - - require.Equal(t, types.RoadIndex(9), state.FirstCommitQC()) - require.NoError(t, utils.TestDiff(utils.Some(qcs[9]), state.LastCommitQC().Load())) - - got, ok := state.LastAppQC().Get() - require.True(t, ok) - require.Equal(t, types.RoadIndex(9), got.Proposal().RoadIndex()) - }) - - t.Run("anchor past all persisted blocks truncates lane WAL", func(t *testing.T) { - dir := t.TempDir() - ds := newTestDataState(&data.Config{Registry: registry}) - lane := keys[0].Public() - - // Persist commitQCs 0-9 and blocks 0-2 for one lane. - qcs := make([]*types.CommitQC, 10) - prev := utils.None[*types.CommitQC]() - cp, _, err := persist.NewCommitQCPersister(utils.Some(dir)) - require.NoError(t, err) - for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[i]) - require.NoError(t, cp.MaybePruneAndPersist(utils.None[*types.CommitQC](), []*types.CommitQC{qcs[i]}, noCommitQCCB)) - } - require.NoError(t, cp.Close()) - - bp, _, err := persist.NewBlockPersister(utils.Some(dir)) - require.NoError(t, err) - var parent types.BlockHeaderHash - for n := range types.BlockNumber(3) { - block := types.NewBlock(lane, n, parent, types.GenPayload(rng)) - signed := types.Sign(keys[0], types.NewLaneProposal(block)) - parent = block.Header().Hash() - require.NoError(t, bp.MaybePruneAndPersistLane(lane, utils.None[*types.CommitQC](), []*types.Signed[*types.LaneProposal]{signed}, noBlockCB)) - } - - // Persist a prune anchor at index 9 with a laneRange that starts past - // all persisted blocks — MaybePruneAndPersistLane prunes the whole block WAL. - appProposal := types.NewAppProposal(50, 9, types.GenAppHash(rng), 0) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - prunePers, _, err := persist.NewPersister[*pb.PersistedAvailPruneAnchor](utils.Some(dir), innerFile) - require.NoError(t, err) - require.NoError(t, prunePers.Persist(&pb.PersistedAvailPruneAnchor{ - AppQc: types.AppQCConv.Encode(appQC), - CommitQc: types.CommitQCConv.Encode(qcs[9]), - })) - - // Release the seeding persister's WAL locks before NewState opens the same directory. - require.NoError(t, bp.Close()) - - // NewState should succeed: block WAL gets truncated, lane starts clean. - state, err := NewState(keys[0], ds, utils.Some(dir)) - require.NoError(t, err) - - require.Equal(t, types.RoadIndex(9), state.FirstCommitQC()) - got, ok := state.LastAppQC().Get() - require.True(t, ok) - require.Equal(t, types.RoadIndex(9), got.Proposal().RoadIndex()) - }) - - t.Run("corrupt AppQC data returns error", func(t *testing.T) { - dir := t.TempDir() - ds := newTestDataState(&data.Config{Registry: registry}) - - // Create a throwaway persister to discover the A/B filenames, - // then corrupt them so NewState fails on load. - _, _, err := persist.NewPersister[*pb.PersistedAvailPruneAnchor](utils.Some(dir), innerFile) - require.NoError(t, err) - entries, err := os.ReadDir(dir) - require.NoError(t, err) - for _, e := range entries { - require.NoError(t, os.WriteFile(filepath.Join(dir, e.Name()), []byte("corrupt"), 0600)) - } - - _, err = NewState(keys[0], ds, utils.Some(dir)) - require.Error(t, err) - }) - - // A NewState that fails after opening the WALs must release them. Each WAL holds an exclusive lock on - // its directory for its lifetime, so an abandoned one makes every later open of the same state - // directory in this process fail — which is exactly the restart that Close exists to support. - t.Run("failed NewState releases WAL locks", func(t *testing.T) { - dir := t.TempDir() - ds := newTestDataState(&data.Config{Registry: registry}) - lane := keys[0].Public() - - // Seed one lane so the failing NewState below has a lane WAL to leak, then release the seeder. - bp, _, err := persist.NewBlockPersister(utils.Some(dir)) - require.NoError(t, err) - var parent types.BlockHeaderHash - block := types.NewBlock(lane, 0, parent, types.GenPayload(rng)) - proposals := []*types.Signed[*types.LaneProposal]{types.Sign(keys[0], types.NewLaneProposal(block))} - require.NoError(t, bp.MaybePruneAndPersistLane( - lane, utils.None[*types.CommitQC](), proposals, noBlockCB)) - require.NoError(t, bp.Close()) - - // A prune anchor missing its CommitQC unmarshals as proto but fails PruneAnchorConv.Decode, so - // NewState fails only after both the lane WAL and the commitQC WAL are open. - appProposal := types.NewAppProposal(50, 0, types.GenAppHash(rng), 0) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - prunePers, _, err := persist.NewPersister[*pb.PersistedAvailPruneAnchor](utils.Some(dir), innerFile) - require.NoError(t, err) - require.NoError(t, prunePers.Persist(&pb.PersistedAvailPruneAnchor{ - AppQc: types.AppQCConv.Encode(appQC), - })) - - _, err = NewState(keys[0], ds, utils.Some(dir)) - require.Error(t, err) - - // Reopening either WAL proves its lock was released; otherwise this fails as lock-unavailable. - reopenedBlocks, _, err := persist.NewBlockPersister(utils.Some(dir)) - require.NoError(t, err, "failed NewState leaked the lane block WAL lock") - require.NoError(t, reopenedBlocks.Close()) - - reopenedQCs, _, err := persist.NewCommitQCPersister(utils.Some(dir)) - require.NoError(t, err, "failed NewState leaked the commitQC WAL lock") - require.NoError(t, reopenedQCs.Close()) - }) } diff --git a/sei-tendermint/internal/autobahn/avail/subscriptions.go b/sei-tendermint/internal/autobahn/avail/subscriptions.go index 76c6634409..8883ff6010 100644 --- a/sei-tendermint/internal/autobahn/avail/subscriptions.go +++ b/sei-tendermint/internal/autobahn/avail/subscriptions.go @@ -76,29 +76,20 @@ type AppVotesRecv struct { } func (s *State) SubscribeAppVotes() *AppVotesRecv { - return &AppVotesRecv{s, 0} + return &AppVotesRecv{s, s.data.NextAppQC()} } func (r *AppVotesRecv) Recv(ctx context.Context) (*types.Signed[*types.AppVote], error) { for { - // If needed, fast forward to the first global number without known AppQC. - if qc, ok := r.state.LastAppQC().Get(); ok { - r.next = max(r.next, qc.Proposal().GlobalNumber()+1) - } - // Fetch the proposal. - p, err := r.state.data.AppProposal(ctx, r.next) + vote, err := r.state.data.AppVote(ctx, r.next) if err != nil { if errors.Is(err, types.ErrPruned) { - r.next = max(r.next+1, r.state.data.FirstAppProposal()) + r.next = max(r.next, r.state.data.NextAppQC()) continue } return nil, err } - // AppProposal currently might return a proposal with a higher global number than the one we requested. - // Correct the n in such a case. - // TODO(gprusak): perhaps it would be possible to require AppHash at every block from the execution engine. - // This would simplify the data state. - r.next = p.GlobalNumber() + 1 - return types.Sign(r.state.key, types.NewAppVote(p)), nil + r.next = vote.Proposal().GlobalRange().Next + return types.Sign(r.state.key, vote), nil } } diff --git a/sei-tendermint/internal/autobahn/avail/testonly.go b/sei-tendermint/internal/autobahn/avail/testonly.go index aee8eead1d..27690e4813 100644 --- a/sei-tendermint/internal/autobahn/avail/testonly.go +++ b/sei-tendermint/internal/autobahn/avail/testonly.go @@ -56,7 +56,7 @@ func RunTestNetwork(ctx context.Context, states []*State) error { qc, err := from.CommitQC(ctx, next) if err != nil { if errors.Is(err, types.ErrPruned) { - next = from.FirstCommitQC() + next = from.First() continue } return err @@ -68,21 +68,6 @@ func RunTestNetwork(ctx context.Context, states []*State) error { } }) } - s.Spawn(func() error { - next := types.RoadIndex(0) - for { - appQC, commitQC, err := from.WaitForAppQC(ctx, next) - if err != nil { - return err - } - next = appQC.Next() - for _, to := range states { - if err := to.PushAppQC(appQC, commitQC); err != nil { - return err - } - } - } - }) } return nil }) diff --git a/sei-tendermint/internal/autobahn/consensus/persist/blocks.go b/sei-tendermint/internal/autobahn/consensus/persist/blocks.go index 5500e2b410..a25e3a44ba 100644 --- a/sei-tendermint/internal/autobahn/consensus/persist/blocks.go +++ b/sei-tendermint/internal/autobahn/consensus/persist/blocks.go @@ -116,17 +116,14 @@ type laneWAL struct { state utils.Mutex[*laneWALState] } -func (lw *laneWAL) maybePruneAndPersist( +func (lw *laneWAL) persist( lane types.LaneID, - anchor utils.Option[*types.CommitQC], + first types.BlockNumber, proposals []*types.Signed[*types.LaneProposal], - afterEach utils.Option[func(*types.Signed[*types.LaneProposal])], ) error { for s := range lw.state.Lock() { - if qc, ok := anchor.Get(); ok { - if err := s.truncateForAnchor(lane, qc.LaneRange(lane).First()); err != nil { - return err - } + if err := s.truncateForAnchor(lane, first); err != nil { + return err } for _, p := range proposals { if p.Msg().Block().Header().Lane() != lane { @@ -143,11 +140,6 @@ func (lw *laneWAL) maybePruneAndPersist( return err } } - if fn, ok := afterEach.Get(); ok { - for _, p := range proposals { - fn(p) - } - } return nil } panic("unreachable") @@ -170,11 +162,11 @@ func (lw *laneWAL) close() error { // // All public methods are safe for concurrent use. The lanes map is protected // by an RWMutex; each laneWAL has its own Mutex for write serialization. -// MaybePruneAndPersistLane holds the per-lane lock for the entire +// PruneAndPersist holds the per-lane lock for the entire // truncate-then-append sequence, so concurrent calls on the same lane // serialize correctly. Different lanes are fully parallel. // -// NOTE: MaybePruneAndPersistLane releases the map RLock before acquiring +// NOTE: PruneAndPersist releases the map RLock before acquiring // the per-lane lock. This is safe because lanes are only added, never // removed. If lane deletion is added in the future, the map RLock must be // held through the WAL write. @@ -294,7 +286,7 @@ func (bp *BlockPersister) getOrCreateLane(lane types.LaneID) (*laneWAL, error) { panic("unreachable") } -// MaybePruneAndPersistLane optionally truncates the lane's WAL and/or appends +// PruneAndPersist optionally truncates the lane's WAL and/or appends // new proposals, depending on which arguments are present: // // - anchor set, proposals non-empty: truncate WAL below anchor, then append (runtime path). @@ -302,27 +294,17 @@ func (bp *BlockPersister) getOrCreateLane(lane types.LaneID) (*laneWAL, error) { // - anchor empty, proposals non-empty: append only, no truncation. // - anchor empty, proposals empty: no-op. // -// afterEach, when present, is called once per appended proposal in order, after the whole batch has -// been flushed — never before, because an append is not durable until then and afterEach is what -// releases a block to the rest of consensus. It is invoked while the per-lane lock is held, so it must -// not re-enter the persister. If any append fails, afterEach is not called for the batch at all. -// No-op persister (dir=None): skips disk I/O but still invokes afterEach. +// No-op persister (dir=None): skips disk I/O. // Does not spawn goroutines — the caller schedules parallelism per lane. // // The per-lane lock is held for the entire truncate-then-append sequence, // so concurrent calls on the same lane serialize correctly. -func (bp *BlockPersister) MaybePruneAndPersistLane( +func (bp *BlockPersister) PruneAndPersist( lane types.LaneID, - anchor utils.Option[*types.CommitQC], + first types.BlockNumber, proposals []*types.Signed[*types.LaneProposal], - afterEach utils.Option[func(*types.Signed[*types.LaneProposal])], ) error { if _, ok := bp.dir.Get(); !ok { - if fn, ok := afterEach.Get(); ok { - for _, p := range proposals { - fn(p) - } - } return nil } @@ -330,7 +312,7 @@ func (bp *BlockPersister) MaybePruneAndPersistLane( if err != nil { return err } - return lw.maybePruneAndPersist(lane, anchor, proposals, afterEach) + return lw.persist(lane, first, proposals) } // Close shuts down all per-lane WALs, releasing the exclusive lock each one holds on its directory. diff --git a/sei-tendermint/internal/autobahn/consensus/persist/blocks_test.go b/sei-tendermint/internal/autobahn/consensus/persist/blocks_test.go index 8c52225f6e..a1fb15450c 100644 --- a/sei-tendermint/internal/autobahn/consensus/persist/blocks_test.go +++ b/sei-tendermint/internal/autobahn/consensus/persist/blocks_test.go @@ -17,8 +17,6 @@ func testSignedProposal(rng utils.Rng, key types.SecretKey, n types.BlockNumber) return types.Sign(key, types.NewLaneProposal(block)) } -var noBlockCB = utils.None[func(*types.Signed[*types.LaneProposal])]() - // liveBlocks drops blocks the prune anchor has moved past, mirroring the filter loadPersistedState // applies in the avail package. Pruning reclaims whole WAL files, so a pruned block can still be on // disk when the persister reloads; only what the anchor considers live is asserted on here. @@ -33,37 +31,13 @@ func liveBlocks(loaded []LoadedBlock, first types.BlockNumber) []LoadedBlock { func testPersistBlock(t *testing.T, bp *BlockPersister, p *types.Signed[*types.LaneProposal]) { t.Helper() - require.NoError(t, bp.MaybePruneAndPersistLane( + require.NoError(t, bp.PruneAndPersist( p.Msg().Block().Header().Lane(), - utils.None[*types.CommitQC](), + 0, []*types.Signed[*types.LaneProposal]{p}, - noBlockCB, )) } -// testDeleteBefore is a test helper that truncates lane WALs using a plain -// map, avoiding the need to construct a full CommitQC. -func testDeleteBefore(bp *BlockPersister, laneFirsts map[types.LaneID]types.BlockNumber) error { - for lanes := range bp.lanes.RLock() { - return scope.Parallel(func(ps scope.ParallelScope) error { - for lane, first := range laneFirsts { - lw, ok := lanes[lane] - if !ok { - continue - } - ps.Spawn(func() error { - for s := range lw.state.Lock() { - return s.truncateForAnchor(lane, first) - } - panic("unreachable") - }) - } - return nil - }) - } - panic("unreachable") -} - func TestNewBlockPersisterEmptyDir(t *testing.T) { dir := t.TempDir() bp, blocks, err := NewBlockPersister(utils.Some(dir)) @@ -142,7 +116,7 @@ func TestDeleteBeforeRemovesOldKeepsNew(t *testing.T) { testPersistBlock(t, bp, testSignedProposal(rng, key, i)) } - require.NoError(t, testDeleteBefore(bp, map[types.LaneID]types.BlockNumber{lane: 3})) + require.NoError(t, bp.PruneAndPersist(lane, 3, nil)) require.NoError(t, bp.Close()) _, blocks, err := NewBlockPersister(utils.Some(dir)) @@ -172,7 +146,9 @@ func TestDeleteBeforeAndRestart(t *testing.T) { } // lane1: truncate old blocks, lane2: delete nothing (first=0), lane3: empty (no WAL). - require.NoError(t, testDeleteBefore(bp, map[types.LaneID]types.BlockNumber{lane1: 2, lane2: 0, lane3: 0})) + require.NoError(t, bp.PruneAndPersist(lane1, 2, nil)) + require.NoError(t, bp.PruneAndPersist(lane2, 0, nil)) + require.NoError(t, bp.PruneAndPersist(lane3, 0, nil)) require.NoError(t, bp.Close()) // Restart — verify varied lane states load correctly. @@ -213,16 +189,8 @@ func TestNoOpBlockPersister(t *testing.T) { proposals[i] = testSignedProposal(rng, key, types.BlockNumber(i)) } - // Persist and prune with anchor + new proposals in no-op mode. - // Verify afterEach is still invoked for every proposal. - var called int - cb := utils.Some(func(_ *types.Signed[*types.LaneProposal]) { called++ }) - require.NoError(t, bp.MaybePruneAndPersistLane(lane, utils.None[*types.CommitQC](), proposals[:3], cb)) - require.Equal(t, 3, called) - - called = 0 - require.NoError(t, bp.MaybePruneAndPersistLane(lane, utils.None[*types.CommitQC](), proposals[3:], cb)) - require.Equal(t, 2, called) + require.NoError(t, bp.PruneAndPersist(lane, 0, proposals[:3])) + require.NoError(t, bp.PruneAndPersist(lane, 0, proposals[3:])) require.NoError(t, bp.Close()) } @@ -240,7 +208,7 @@ func TestDeleteBeforeThenPersistMore(t *testing.T) { for i := range types.BlockNumber(5) { testPersistBlock(t, bp, testSignedProposal(rng, key, i)) } - require.NoError(t, testDeleteBefore(bp, map[types.LaneID]types.BlockNumber{lane: 3})) + require.NoError(t, bp.PruneAndPersist(lane, 3, nil)) testPersistBlock(t, bp, testSignedProposal(rng, key, 5)) require.NoError(t, bp.Close()) @@ -267,7 +235,7 @@ func TestDeleteBeforePastAllBlocks(t *testing.T) { } // Anchor advanced past everything (nextBlockNum is 3, first=10). - require.NoError(t, testDeleteBefore(bp, map[types.LaneID]types.BlockNumber{lane: 10})) + require.NoError(t, bp.PruneAndPersist(lane, 10, nil)) // Lane WAL is now empty; new writes starting from 10 should work. testPersistBlock(t, bp, testSignedProposal(rng, key, 10)) @@ -296,11 +264,11 @@ func TestDeleteBeforePastAllRejectsStaleBlock(t *testing.T) { } // Anchor advanced past everything; nextBlockNum re-anchored to 10. - require.NoError(t, testDeleteBefore(bp, map[types.LaneID]types.BlockNumber{lane: 10})) + require.NoError(t, bp.PruneAndPersist(lane, 10, nil)) // Writing a stale block number (0) should be rejected. stale := testSignedProposal(rng, key, 0) - err = bp.MaybePruneAndPersistLane(lane, utils.None[*types.CommitQC](), []*types.Signed[*types.LaneProposal]{stale}, noBlockCB) + err = bp.PruneAndPersist(lane, 10, []*types.Signed[*types.LaneProposal]{stale}) require.Error(t, err) require.Contains(t, err.Error(), "out of sequence") @@ -323,12 +291,12 @@ func TestTruncateOnEmptyWALAdvancesCursor(t *testing.T) { } // First truncation empties the WAL (first=10 > nextBlockNum=3). - require.NoError(t, testDeleteBefore(bp, map[types.LaneID]types.BlockNumber{lane: 10})) + require.NoError(t, bp.PruneAndPersist(lane, 10, nil)) // Second truncation on the already-empty WAL (first=15). // Before the fix, nextBlockNum would stay at 10 and block 15 would // be rejected as out of sequence. - require.NoError(t, testDeleteBefore(bp, map[types.LaneID]types.BlockNumber{lane: 15})) + require.NoError(t, bp.PruneAndPersist(lane, 15, nil)) testPersistBlock(t, bp, testSignedProposal(rng, key, 15)) require.NoError(t, bp.Close()) @@ -404,13 +372,13 @@ func TestPersistBlockOutOfSequence(t *testing.T) { // Gap: skip block 1, try block 2. gap := testSignedProposal(rng, key, 2) - err = bp.MaybePruneAndPersistLane(lane, utils.None[*types.CommitQC](), []*types.Signed[*types.LaneProposal]{gap}, noBlockCB) + err = bp.PruneAndPersist(lane, 0, []*types.Signed[*types.LaneProposal]{gap}) require.Error(t, err) require.Contains(t, err.Error(), "out of sequence") // Duplicate: try block 0 again. dup := testSignedProposal(rng, key, 0) - err = bp.MaybePruneAndPersistLane(lane, utils.None[*types.CommitQC](), []*types.Signed[*types.LaneProposal]{dup}, noBlockCB) + err = bp.PruneAndPersist(lane, 0, []*types.Signed[*types.LaneProposal]{dup}) require.Error(t, err) require.Contains(t, err.Error(), "out of sequence") @@ -506,35 +474,6 @@ func TestPruneReclaimsSealedFiles(t *testing.T) { require.Equal(t, types.BlockNumber(total), s2.nextBlockNum) } -// TestPersistBlockInvokesAfterEachOncePerBlock covers the on-disk path: appends are flushed as a batch -// and afterEach then reports every block in it, exactly once and in order. -func TestPersistBlockInvokesAfterEachOncePerBlock(t *testing.T) { - rng := utils.TestRng() - dir := t.TempDir() - - key := types.GenSecretKey(rng) - lane := key.Public() - bp, _, err := NewBlockPersister(utils.Some(dir)) - require.NoError(t, err) - - proposals := make([]*types.Signed[*types.LaneProposal], 5) - for i := range proposals { - proposals[i] = testSignedProposal(rng, key, types.BlockNumber(i)) - } - - var seen []types.BlockNumber - cb := utils.Some(func(p *types.Signed[*types.LaneProposal]) { - seen = append(seen, p.Msg().Block().Header().BlockNumber()) - }) - require.NoError(t, bp.MaybePruneAndPersistLane(lane, utils.None[*types.CommitQC](), proposals, cb)) - require.NoError(t, bp.Close()) - - require.Equal(t, len(proposals), len(seen)) - for i := range seen { - require.Equal(t, types.BlockNumber(i), seen[i]) - } -} - func TestPersistBlockConcurrentDistinctLanes(t *testing.T) { rng := utils.TestRng() dir := t.TempDir() @@ -562,7 +501,7 @@ func TestPersistBlockConcurrentDistinctLanes(t *testing.T) { for i := range numLanes { lane := keys[i].Public() ps.Spawn(func() error { - return bp.MaybePruneAndPersistLane(lane, utils.None[*types.CommitQC](), proposals[i], noBlockCB) + return bp.PruneAndPersist(lane, 0, proposals[i]) }) } return nil diff --git a/sei-tendermint/internal/autobahn/consensus/persist/commitqcs.go b/sei-tendermint/internal/autobahn/consensus/persist/commitqcs.go index 0eb249413e..6d27e73391 100644 --- a/sei-tendermint/internal/autobahn/consensus/persist/commitqcs.go +++ b/sei-tendermint/internal/autobahn/consensus/persist/commitqcs.go @@ -19,16 +19,10 @@ const commitqcsWALName = "autobahn_commitqcs" // unlike the lane WALs (see blocksWALName) it has nothing to collide with. const commitqcsWALMetrics = true -// LoadedCommitQC is a CommitQC loaded from disk during state restoration. -type LoadedCommitQC struct { - Index types.RoadIndex - QC *types.CommitQC -} - // commitQCState is the mutable state protected by CommitQCPersister's mutex. type commitQCState struct { - wal utils.Option[seiwal.WAL[*types.CommitQC]] - next types.RoadIndex + wal utils.Option[seiwal.WAL[*types.CommitQC]] + persisted types.RoadRange // Whether a QC has been appended since the last flush, so a prune that re-persists its anchor is // still made durable while a run of duplicates costs no fsync. unflushed bool @@ -38,13 +32,13 @@ type commitQCState struct { // until flush returns. Caller must hold the lock. // Duplicates (idx < next) are silently ignored for idempotent startup. // Gaps (idx > next) return an error. -func (s *commitQCState) persistCommitQC(qc *types.CommitQC) error { +func (s *commitQCState) persist(qc *types.CommitQC) error { idx := qc.Index() - if idx < s.next { + if idx < s.persisted.Next { return nil } - if idx > s.next { - return fmt.Errorf("commitqc %d out of sequence (next=%d)", idx, s.next) + if idx > s.persisted.Next { + return fmt.Errorf("commitqc %d out of sequence (next=%d)", idx, s.persisted.Next) } if w, ok := s.wal.Get(); ok { if err := w.Append(uint64(idx), qc); err != nil { @@ -52,7 +46,7 @@ func (s *commitQCState) persistCommitQC(qc *types.CommitQC) error { } s.unflushed = true } - s.next = idx + 1 + s.persisted.Next += 1 return nil } @@ -72,19 +66,18 @@ func (s *commitQCState) flush() error { // deleteBefore prunes WAL entries below the anchor's index, then re-persists the anchor for crash // recovery. Pruning is lazy, so entries below the anchor may remain on disk until the file holding // them falls entirely below the threshold. Caller must hold the lock. -func (s *commitQCState) deleteBefore(anchor *types.CommitQC) error { - idx := anchor.Index() +func (s *commitQCState) deleteBefore(idx types.RoadIndex) error { + if idx <= s.persisted.First { + return nil + } if w, ok := s.wal.Get(); ok { if err := w.PruneBefore(uint64(idx)); err != nil { return fmt.Errorf("prune commitqc WAL before %d: %w", idx, err) } } - if idx > s.next { - // The anchor moved past every QC persisted so far, so the next QC to persist is the anchor - // itself. That leaves a gap, which the WAL is configured to permit. - s.next = idx - } - return s.persistCommitQC(anchor) + s.persisted.First = idx + s.persisted.Next = max(s.persisted.First, s.persisted.Next) + return nil } // CommitQCPersister manages CommitQC persistence using a WAL. @@ -109,9 +102,9 @@ type CommitQCPersister struct { // When stateDir is None, returns a no-op persister. // // After crash recovery with an empty WAL, LoadNext() returns 0. The caller MUST -// use MaybePruneAndPersist with the prune CommitQC in Anchor to re-establish the +// use PruneAndPersist with the prune CommitQC in Anchor to re-establish the // cursor and re-persist the anchor's CommitQC before appending more QCs. -func NewCommitQCPersister(stateDir utils.Option[string]) (*CommitQCPersister, []LoadedCommitQC, error) { +func NewCommitQCPersister(stateDir utils.Option[string]) (*CommitQCPersister, []*types.CommitQC, error) { sd, ok := stateDir.Get() if !ok { return &CommitQCPersister{state: utils.NewMutex(&commitQCState{})}, nil, nil @@ -129,21 +122,24 @@ func NewCommitQCPersister(stateDir utils.Option[string]) (*CommitQCPersister, [] return nil, nil, err } if len(loaded) > 0 { - s.next = loaded[len(loaded)-1].Index + 1 + s.persisted = types.RoadRange{ + First: loaded[0].Index(), + Next: loaded[len(loaded)-1].Index() + 1, + } } return &CommitQCPersister{state: utils.NewMutex(s)}, loaded, nil } // LoadNext returns the road index of the first CommitQC that has not been // persisted (exclusive upper bound of what's on disk). -func (cp *CommitQCPersister) LoadNext() types.RoadIndex { +func (cp *CommitQCPersister) Next() types.RoadIndex { for s := range cp.state.Lock() { - return s.next + return s.persisted.Next } panic("unreachable") } -// MaybePruneAndPersist optionally truncates the WAL and/or appends new +// PruneAndPersist optionally truncates the WAL and/or appends new // CommitQCs, depending on which arguments are present: // // - anchor set, commitQCs non-empty: truncate WAL below anchor, re-persist @@ -158,23 +154,15 @@ func (cp *CommitQCPersister) LoadNext() types.RoadIndex { // // The lock is held for the entire prune-then-append sequence, so callers // need not coordinate ordering. -// afterEach, when present, is called once per QC in commitQCs in order, after the batch has been -// flushed — never before, because an append is not durable until then and afterEach is what releases a -// QC to the rest of consensus. It is invoked while the lock is held, so it must not re-enter the -// persister. If any append fails, afterEach is not called for the batch at all. -func (cp *CommitQCPersister) MaybePruneAndPersist( - anchor utils.Option[*types.CommitQC], - commitQCs []*types.CommitQC, - afterEach utils.Option[func(*types.CommitQC)], -) error { +// afterEach, when present, is called after each successful append. It is +// invoked while the lock is held, so it must not re-enter the persister. +func (cp *CommitQCPersister) PruneAndPersist(deleteBefore types.RoadIndex, commitQCs []*types.CommitQC) error { for s := range cp.state.Lock() { - if qc, ok := anchor.Get(); ok { - if err := s.deleteBefore(qc); err != nil { - return err - } + if err := s.deleteBefore(deleteBefore); err != nil { + return err } for _, c := range commitQCs { - if err := s.persistCommitQC(c); err != nil { + if err := s.persist(c); err != nil { return err } } @@ -183,11 +171,6 @@ func (cp *CommitQCPersister) MaybePruneAndPersist( if err := s.flush(); err != nil { return err } - if fn, ok := afterEach.Get(); ok { - for _, c := range commitQCs { - fn(c) - } - } return nil } panic("unreachable") @@ -210,7 +193,7 @@ func (cp *CommitQCPersister) Close() error { // // QCs stranded below a lazy prune are discarded: a gap in the stored road indices marks where pruning // has already logically removed everything before it. -func loadAllCommitQCs(s *commitQCState) ([]LoadedCommitQC, error) { +func loadAllCommitQCs(s *commitQCState) ([]*types.CommitQC, error) { w, ok := s.wal.Get() if !ok { return nil, nil // no-op persister (persistence disabled) @@ -219,11 +202,9 @@ func loadAllCommitQCs(s *commitQCState) ([]LoadedCommitQC, error) { if err != nil { return nil, fmt.Errorf("read commitqc WAL: %w", err) } - live := contiguousSuffix(entries) - - loaded := make([]LoadedCommitQC, 0, len(live)) - for _, entry := range live { - loaded = append(loaded, LoadedCommitQC{Index: entry.value.Index(), QC: entry.value}) + var loaded []*types.CommitQC + for _, entry := range contiguousSuffix(entries) { + loaded = append(loaded, entry.value) } return loaded, nil } diff --git a/sei-tendermint/internal/autobahn/consensus/persist/commitqcs_test.go b/sei-tendermint/internal/autobahn/consensus/persist/commitqcs_test.go index bc3c5245bb..f9e867ae0c 100644 --- a/sei-tendermint/internal/autobahn/consensus/persist/commitqcs_test.go +++ b/sei-tendermint/internal/autobahn/consensus/persist/commitqcs_test.go @@ -12,67 +12,30 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/require" ) -var noQC = utils.None[*types.CommitQC]() -var noCommitQCCB = utils.None[func(*types.CommitQC)]() - // liveCommitQCs drops QCs the prune anchor has moved past, mirroring the filter loadPersistedState // applies in the avail package. Pruning reclaims whole WAL files, so a pruned QC can still be on disk // when the persister reloads; only what the anchor considers live is asserted on here. -func liveCommitQCs(loaded []LoadedCommitQC, first types.RoadIndex) []LoadedCommitQC { +func liveCommitQCs(loaded []*types.CommitQC, first types.RoadIndex) []*types.CommitQC { for i, lqc := range loaded { - if lqc.Index >= first { + if lqc.Index() >= first { return loaded[i:] } } return nil } -func testCommitQC( - committee *types.Committee, - keys []types.SecretKey, - prev utils.Option[*types.CommitQC], - laneQCs map[types.LaneID]*types.LaneQC, - appQC utils.Option[*types.AppQC], -) *types.CommitQC { - ep := types.NewEpoch(0, types.OpenRoadRange(), time.Time{}, committee, 0) - return types.BuildCommitQC(ep, keys, prev, laneQCs, appQC) -} - -func makeSequentialCommitQCs( - committee *types.Committee, - keys []types.SecretKey, - count int, -) []*types.CommitQC { +func makeSequentialCommitQCs(committee *types.Committee, keys []types.SecretKey, count int) []*types.CommitQC { var qcs []*types.CommitQC prev := utils.None[*types.CommitQC]() + ep := types.NewEpoch(0, types.OpenRoadRange(), time.Time{}, committee, 0) for range count { - qc := testCommitQC(committee, keys, prev, nil, utils.None[*types.AppQC]()) + qc := types.BuildCommitQC(ep, keys, prev, nil) qcs = append(qcs, qc) prev = utils.Some(qc) } return qcs } -// testPersistCommitQC persists a single CommitQC via the public API. -func testPersistCommitQC(t *testing.T, cp *CommitQCPersister, qc *types.CommitQC) { - t.Helper() - require.NoError(t, cp.MaybePruneAndPersist( - utils.None[*types.CommitQC](), - []*types.CommitQC{qc}, - noCommitQCCB, - )) -} - -// testDeleteCommitQCsBefore truncates the WAL below the anchor's index and -// re-persists the anchor for crash recovery. -func testDeleteCommitQCsBefore(t *testing.T, cp *CommitQCPersister, anchor *types.CommitQC) { - t.Helper() - for s := range cp.state.Lock() { - require.NoError(t, s.deleteBefore(anchor)) - return - } -} - // clearCommitQCWAL removes all WAL files to simulate a crash between // WAL truncation and the subsequent anchor write. func clearCommitQCWAL(t *testing.T, dir string) { @@ -88,7 +51,7 @@ func TestNewCommitQCPersisterEmptyDir(t *testing.T) { require.NoError(t, err) require.NotNil(t, cp) require.Equal(t, 0, len(loaded)) - require.Equal(t, types.RoadIndex(0), cp.LoadNext()) + require.Equal(t, types.RoadIndex(0), cp.Next()) fi, err := os.Stat(filepath.Join(dir, commitqcsDir)) require.NoError(t, err) @@ -107,10 +70,8 @@ func TestPersistCommitQCAndLoad(t *testing.T) { cp, _, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) - for _, qc := range qcs { - testPersistCommitQC(t, cp, qc) - } - require.Equal(t, types.RoadIndex(3), cp.LoadNext()) + require.NoError(t, cp.PruneAndPersist(0, qcs)) + require.Equal(t, types.RoadIndex(3), cp.Next()) require.NoError(t, cp.Close()) cp2, loaded, err := NewCommitQCPersister(utils.Some(dir)) @@ -118,10 +79,10 @@ func TestPersistCommitQCAndLoad(t *testing.T) { require.NotNil(t, cp2) require.Equal(t, 3, len(loaded)) for i, lqc := range loaded { - require.Equal(t, types.RoadIndex(i), lqc.Index) - require.NoError(t, utils.TestDiff(qcs[i], lqc.QC)) + require.Equal(t, types.RoadIndex(i), lqc.Index()) + require.NoError(t, utils.TestDiff(qcs[i], lqc)) } - require.Equal(t, types.RoadIndex(3), cp2.LoadNext()) + require.Equal(t, types.RoadIndex(3), cp2.Next()) require.NoError(t, cp2.Close()) } @@ -134,19 +95,17 @@ func TestCommitQCDeleteBeforeRemovesOldKeepsNew(t *testing.T) { qcs := makeSequentialCommitQCs(committee, keys, 5) cp, _, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) - for _, qc := range qcs { - testPersistCommitQC(t, cp, qc) - } + require.NoError(t, cp.PruneAndPersist(0, qcs)) - testDeleteCommitQCsBefore(t, cp, qcs[3]) + require.NoError(t, cp.PruneAndPersist(qcs[3].Index(), nil)) require.NoError(t, cp.Close()) _, loaded, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) live := liveCommitQCs(loaded, 3) require.Equal(t, 2, len(live), "should have indices 3 and 4") - require.Equal(t, types.RoadIndex(3), live[0].Index) - require.Equal(t, types.RoadIndex(4), live[1].Index) + require.Equal(t, types.RoadIndex(3), live[0].Index()) + require.Equal(t, types.RoadIndex(4), live[1].Index()) } func TestCommitQCDeleteBeforeZero(t *testing.T) { @@ -159,21 +118,18 @@ func TestCommitQCDeleteBeforeZero(t *testing.T) { cp, _, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) - for _, qc := range qcs[:2] { - testPersistCommitQC(t, cp, qc) - } + require.NoError(t, cp.PruneAndPersist(0, qcs[:2])) - // deleteBefore with anchor at index 0 should persist the anchor - // (which is a duplicate here) and leave everything intact. - testDeleteCommitQCsBefore(t, cp, qcs[0]) + // deleteBefore with index 0 should leave everything intact. + require.NoError(t, cp.PruneAndPersist(qcs[0].Index(), nil)) require.NoError(t, cp.Close()) cp2, loaded, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) require.Equal(t, 2, len(loaded)) - testPersistCommitQC(t, cp2, qcs[2]) - require.Equal(t, types.RoadIndex(3), cp2.LoadNext()) + require.NoError(t, cp2.PruneAndPersist(0, []*types.CommitQC{qcs[2]})) + require.Equal(t, types.RoadIndex(3), cp2.Next()) require.NoError(t, cp2.Close()) } @@ -187,11 +143,11 @@ func TestCommitQCPersistDuplicateIsNoOp(t *testing.T) { cp, _, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) - testPersistCommitQC(t, cp, qcs[0]) - testPersistCommitQC(t, cp, qcs[1]) + require.NoError(t, cp.PruneAndPersist(0, []*types.CommitQC{qcs[0]})) + require.NoError(t, cp.PruneAndPersist(0, []*types.CommitQC{qcs[1]})) // Persisting qcs[0] again is a no-op (idx < next). - testPersistCommitQC(t, cp, qcs[0]) - require.Equal(t, types.RoadIndex(2), cp.LoadNext()) + require.NoError(t, cp.PruneAndPersist(0, []*types.CommitQC{qcs[0]})) + require.Equal(t, types.RoadIndex(2), cp.Next()) require.NoError(t, cp.Close()) } @@ -205,10 +161,10 @@ func TestCommitQCPersistGapRejected(t *testing.T) { cp, _, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) - testPersistCommitQC(t, cp, qcs[0]) - testPersistCommitQC(t, cp, qcs[1]) + require.NoError(t, cp.PruneAndPersist(0, []*types.CommitQC{qcs[0]})) + require.NoError(t, cp.PruneAndPersist(0, []*types.CommitQC{qcs[1]})) // Skip qcs[2], try to persist qcs[3] — should fail because idx(3) != next(2). - err = cp.MaybePruneAndPersist(noQC, []*types.CommitQC{qcs[3]}, noCommitQCCB) + err = cp.PruneAndPersist(0, []*types.CommitQC{qcs[3]}) require.Error(t, err) require.Contains(t, err.Error(), "out of sequence") require.NoError(t, cp.Close()) @@ -239,7 +195,7 @@ func TestLoadAllDropsCommitQCsBehindGap(t *testing.T) { _, loaded, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) require.Equal(t, 1, len(loaded), "index 0 sits behind the gap and is dropped") - require.Equal(t, types.RoadIndex(2), loaded[0].Index) + require.Equal(t, types.RoadIndex(2), loaded[0].Index()) } func TestNoOpCommitQCPersister(t *testing.T) { @@ -248,31 +204,18 @@ func TestNoOpCommitQCPersister(t *testing.T) { committee := registry.LatestEpoch().Committee() qcs := makeSequentialCommitQCs(committee, keys, 11) - // Fresh no-op persister: prune with anchor at index 0 (idx==0, - // s.next==0). Should persist the anchor and advance to 1. + // Fresh no-op persister: persist sequential QCs and track Next. cp, loaded, err := NewCommitQCPersister(utils.None[string]()) require.NoError(t, err) require.NotNil(t, cp) require.Equal(t, 0, len(loaded)) - require.NoError(t, cp.MaybePruneAndPersist( - utils.Some(qcs[0]), - qcs[1:5], - noCommitQCCB, - )) - require.Equal(t, types.RoadIndex(5), cp.LoadNext()) - - // Prune with a future anchor (index 8 > s.next=5). deleteBefore - // advances s.next to 8, then persistCommitQC persists the anchor - // and advances s.next to 9. The remaining QCs (9,10) follow. - // Before the fix, deleteBefore in no-op mode wouldn't advance - // s.next, so re-persisting the anchor QC would fail with "out of - // sequence". - require.NoError(t, cp.MaybePruneAndPersist( - utils.Some(qcs[8]), - qcs[9:], - noCommitQCCB, - )) - require.Equal(t, types.RoadIndex(11), cp.LoadNext()) + require.NoError(t, cp.PruneAndPersist(0, qcs[:5])) + require.Equal(t, types.RoadIndex(5), cp.Next()) + + // Prune with a future index. deleteBefore advances persisted.Next, + // so the remaining QCs follow the new bound. + require.NoError(t, cp.PruneAndPersist(8, qcs[8:])) + require.Equal(t, types.RoadIndex(11), cp.Next()) require.NoError(t, cp.Close()) } @@ -285,24 +228,23 @@ func TestCommitQCDeleteBeforePastAll(t *testing.T) { qcs := makeSequentialCommitQCs(committee, keys, 12) cp, _, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) - for i := range 3 { - testPersistCommitQC(t, cp, qcs[i]) - } - // next is 3; deleteBefore with anchor at 10 truncates the WAL, - // advances the cursor to 10, and re-persists the anchor (next → 11). - testDeleteCommitQCsBefore(t, cp, qcs[10]) - require.Equal(t, types.RoadIndex(11), cp.LoadNext()) - - // New write starting from 11 should work. - testPersistCommitQC(t, cp, qcs[11]) + require.NoError(t, cp.PruneAndPersist(0, qcs[:3])) + // next is 3; deleteBefore at 10 truncates the WAL and advances the + // cursor to 10. + require.NoError(t, cp.PruneAndPersist(qcs[10].Index(), nil)) + require.Equal(t, types.RoadIndex(10), cp.Next()) + + // New writes starting from 10 should work. + require.NoError(t, cp.PruneAndPersist(0, []*types.CommitQC{qcs[10]})) + require.NoError(t, cp.PruneAndPersist(0, []*types.CommitQC{qcs[11]})) require.NoError(t, cp.Close()) // Reopen — the fast-forward left a gap, so only the entries after it are live. _, loaded, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) require.Equal(t, 2, len(loaded)) - require.Equal(t, types.RoadIndex(10), loaded[0].Index) - require.Equal(t, types.RoadIndex(11), loaded[1].Index) + require.Equal(t, types.RoadIndex(10), loaded[0].Index()) + require.Equal(t, types.RoadIndex(11), loaded[1].Index()) } // TestCommitQCDeleteBeforePastAllCrashRecovery simulates a crash between WAL @@ -317,9 +259,7 @@ func TestCommitQCDeleteBeforePastAllCrashRecovery(t *testing.T) { qcs := makeSequentialCommitQCs(committee, keys, 12) cp, _, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) - for i := range 3 { - testPersistCommitQC(t, cp, qcs[i]) - } + require.NoError(t, cp.PruneAndPersist(0, qcs[:3])) require.NoError(t, cp.Close()) // Simulate crash: clear the WAL as if the prune reclaimed everything but the @@ -330,23 +270,19 @@ func TestCommitQCDeleteBeforePastAllCrashRecovery(t *testing.T) { cp2, loaded, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) require.Empty(t, loaded) - require.Equal(t, types.RoadIndex(0), cp2.LoadNext()) + require.Equal(t, types.RoadIndex(0), cp2.Next()) - // MaybePruneAndPersist with anchor at 10 re-establishes the cursor + // Persist with anchor at 10 re-establishes the cursor // and appends new QCs. - require.NoError(t, cp2.MaybePruneAndPersist( - utils.Some(qcs[10]), - []*types.CommitQC{qcs[11]}, - noCommitQCCB, - )) - require.Equal(t, types.RoadIndex(12), cp2.LoadNext()) + require.NoError(t, cp2.PruneAndPersist(qcs[10].Index(), []*types.CommitQC{qcs[10], qcs[11]})) + require.Equal(t, types.RoadIndex(12), cp2.Next()) require.NoError(t, cp2.Close()) _, loaded, err = NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) require.Equal(t, 2, len(loaded)) - require.Equal(t, types.RoadIndex(10), loaded[0].Index) - require.Equal(t, types.RoadIndex(11), loaded[1].Index) + require.Equal(t, types.RoadIndex(10), loaded[0].Index()) + require.Equal(t, types.RoadIndex(11), loaded[1].Index()) } // TestCommitQCDeleteBeforeWithAnchorRecovers verifies that after a crash @@ -359,36 +295,26 @@ func TestCommitQCDeleteBeforeWithAnchorRecovers(t *testing.T) { dir := t.TempDir() qcs := makeSequentialCommitQCs(committee, keys, 5) - cp, _, err := NewCommitQCPersister(utils.Some(dir)) - require.NoError(t, err) - for _, qc := range qcs { - testPersistCommitQC(t, cp, qc) - } - require.NoError(t, cp.Close()) - - // Simulate crash: clear WAL. - clearCommitQCWAL(t, dir) - // Restart: WAL is empty. Pass the anchor QC (index 4) through deleteBefore. - cp2, loaded, err := NewCommitQCPersister(utils.Some(dir)) + // WAL is empty. Pass the anchor QC (index 4) through deleteBefore. + cp, loaded, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) require.Empty(t, loaded) - // deleteBefore advances cursor to 4, then re-persists qcs[4] via anchor. - testDeleteCommitQCsBefore(t, cp2, qcs[4]) - require.Equal(t, types.RoadIndex(5), cp2.LoadNext()) + // deleteBefore advances cursor to 4. + require.NoError(t, cp.PruneAndPersist(qcs[4].Index(), utils.Slice(qcs[4]))) + require.Equal(t, types.RoadIndex(5), cp.Next()) // Continue writing from 5. - testPersistCommitQC(t, cp2, qcs[4]) // duplicate — no-op - require.Equal(t, types.RoadIndex(5), cp2.LoadNext()) - require.NoError(t, cp2.Close()) + require.NoError(t, cp.PruneAndPersist(0, utils.Slice(qcs[4]))) // duplicate — no-op + require.Equal(t, types.RoadIndex(5), cp.Next()) + require.NoError(t, cp.Close()) // Reopen — anchor QC should be on disk. _, loaded, err = NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) require.Equal(t, 1, len(loaded)) - require.Equal(t, types.RoadIndex(4), loaded[0].Index) - require.NoError(t, utils.TestDiff(qcs[4], loaded[0].QC)) + require.NoError(t, utils.TestDiff(qcs[4], loaded[0])) } func TestCommitQCDeleteBeforeThenPersistMore(t *testing.T) { @@ -402,20 +328,18 @@ func TestCommitQCDeleteBeforeThenPersistMore(t *testing.T) { require.NoError(t, err) // Persist 0..4, delete before 3, then persist 5. - for i := range 5 { - testPersistCommitQC(t, cp, qcs[i]) - } - testDeleteCommitQCsBefore(t, cp, qcs[3]) - testPersistCommitQC(t, cp, qcs[5]) + require.NoError(t, cp.PruneAndPersist(0, qcs[:5])) + require.NoError(t, cp.PruneAndPersist(qcs[3].Index(), nil)) + require.NoError(t, cp.PruneAndPersist(0, []*types.CommitQC{qcs[5]})) require.NoError(t, cp.Close()) _, loaded, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) live := liveCommitQCs(loaded, 3) require.Equal(t, 3, len(live), "should have indices 3, 4, 5") - require.Equal(t, types.RoadIndex(3), live[0].Index) - require.Equal(t, types.RoadIndex(4), live[1].Index) - require.Equal(t, types.RoadIndex(5), live[2].Index) + require.Equal(t, types.RoadIndex(3), live[0].Index()) + require.Equal(t, types.RoadIndex(4), live[1].Index()) + require.Equal(t, types.RoadIndex(5), live[2].Index()) } func TestCommitQCDeleteBeforeAlreadyPruned(t *testing.T) { @@ -427,16 +351,14 @@ func TestCommitQCDeleteBeforeAlreadyPruned(t *testing.T) { qcs := makeSequentialCommitQCs(committee, keys, 5) cp, _, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) - for _, qc := range qcs { - testPersistCommitQC(t, cp, qc) - } + require.NoError(t, cp.PruneAndPersist(0, qcs)) // Prune up to index 3. - testDeleteCommitQCsBefore(t, cp, qcs[3]) + require.NoError(t, cp.PruneAndPersist(qcs[3].Index(), nil)) // Pruning at or below the current first should be a no-op. - testDeleteCommitQCsBefore(t, cp, qcs[2]) - testDeleteCommitQCsBefore(t, cp, qcs[3]) + require.NoError(t, cp.PruneAndPersist(qcs[2].Index(), nil)) + require.NoError(t, cp.PruneAndPersist(qcs[3].Index(), nil)) require.NoError(t, cp.Close()) // Verify nothing extra was pruned. @@ -444,8 +366,8 @@ func TestCommitQCDeleteBeforeAlreadyPruned(t *testing.T) { require.NoError(t, err) live := liveCommitQCs(loaded, 3) require.Equal(t, 2, len(live), "should still have indices 3 and 4") - require.Equal(t, types.RoadIndex(3), live[0].Index) - require.Equal(t, types.RoadIndex(4), live[1].Index) + require.Equal(t, types.RoadIndex(3), live[0].Index()) + require.Equal(t, types.RoadIndex(4), live[1].Index()) } func TestCommitQCProgressiveDeleteBefore(t *testing.T) { @@ -457,16 +379,14 @@ func TestCommitQCProgressiveDeleteBefore(t *testing.T) { qcs := makeSequentialCommitQCs(committee, keys, 8) cp, _, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) - for _, qc := range qcs { - testPersistCommitQC(t, cp, qc) - } + require.NoError(t, cp.PruneAndPersist(0, qcs)) // First prune: remove 0, 1. - testDeleteCommitQCsBefore(t, cp, qcs[2]) - require.Equal(t, types.RoadIndex(8), cp.LoadNext()) + require.NoError(t, cp.PruneAndPersist(qcs[2].Index(), nil)) + require.Equal(t, types.RoadIndex(8), cp.Next()) // Second prune: remove 2, 3, 4. - testDeleteCommitQCsBefore(t, cp, qcs[5]) + require.NoError(t, cp.PruneAndPersist(qcs[5].Index(), nil)) require.NoError(t, cp.Close()) // Verify indices 5, 6, 7 survive. @@ -474,7 +394,7 @@ func TestCommitQCProgressiveDeleteBefore(t *testing.T) { require.NoError(t, err) live := liveCommitQCs(loaded, 5) require.Equal(t, 3, len(live)) - require.Equal(t, types.RoadIndex(5), live[0].Index) - require.Equal(t, types.RoadIndex(6), live[1].Index) - require.Equal(t, types.RoadIndex(7), live[2].Index) + require.Equal(t, types.RoadIndex(5), live[0].Index()) + require.Equal(t, types.RoadIndex(6), live[1].Index()) + require.Equal(t, types.RoadIndex(7), live[2].Index()) } diff --git a/sei-tendermint/internal/autobahn/consensus/state.go b/sei-tendermint/internal/autobahn/consensus/state.go index 4323eb4292..42b2ab0d7b 100644 --- a/sei-tendermint/internal/autobahn/consensus/state.go +++ b/sei-tendermint/internal/autobahn/consensus/state.go @@ -239,7 +239,6 @@ func (s *State) runPropose(ctx context.Context) error { vs, time.Now(), laneQCsMap, - s.avail.LastAppQC(), ) if err != nil { return fmt.Errorf("s.avail.WaitForProposal(): %w", err) diff --git a/sei-tendermint/internal/autobahn/data/metrics/metrics.go b/sei-tendermint/internal/autobahn/data/metrics/metrics.go index bf230ad8fd..338d61130b 100644 --- a/sei-tendermint/internal/autobahn/data/metrics/metrics.go +++ b/sei-tendermint/internal/autobahn/data/metrics/metrics.go @@ -16,6 +16,7 @@ type metrics struct { type resourceMetrics struct { Receive *prometheus.Histogram Execute *prometheus.Histogram + Certify *prometheus.Histogram } type Metrics struct { @@ -28,6 +29,7 @@ func Get() *Metrics { return resourceMetrics{ Receive: Global.latencyAt(resource, "receive"), Execute: Global.latencyAt(resource, "execute"), + Certify: Global.latencyAt(resource, "certify"), } } return &Metrics{ diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index f5bbd6ca32..46c58c1659 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -1,6 +1,7 @@ package data import ( + "bytes" "context" "errors" "fmt" @@ -14,25 +15,16 @@ import ( const blocksCacheSize = 4000 +// ErrOutOfOrder is returned when PushAppHash receives an app hash past the +// next CommitQC range waiting for execution. +var ErrOutOfOrder = errors.New("out of order") + // Config is the config for the data State. type Config struct { // Registry is the authoritative source of committee and stake information. Registry *epoch.Registry - // LastExecutedBlock is the app's committed global height if any. - LastExecutedBlock utils.Option[types.GlobalBlockNumber] -} - -// StateAPI is the interface of the State for consuming global blocks -// and reporting AppHashes. -type StateAPI interface { - GlobalBlock(ctx context.Context, n types.GlobalBlockNumber) (*types.GlobalBlock, error) - // PushAppHash blocks until block n and its QC are durably persisted, - // ensuring AppVotes are only issued for data that survives a crash. - PushAppHash(ctx context.Context, n types.GlobalBlockNumber, hash types.AppHash) error } -var _ StateAPI = (*State)(nil) - // blockEntry is a (number, block) pair collected in runPersist batches. type blockEntry struct { n types.GlobalBlockNumber @@ -40,72 +32,32 @@ type blockEntry struct { } type inner struct { - // Map key ranges (low end = first): - // qcs: [first, nextQC) - // blocks: [first, nextBlock) + gap-fills in [nextBlock, nextQC) - // appProposals: [first, nextAppProposal) - // blockHashes: mirrors blocks (insertBlock / evictBelowBound) - // - // Durable copies below first live in BlockDB. AppProposals are not - // persisted; they are rebuilt via PushAppHash / re-execution after restart. - qcs map[types.GlobalBlockNumber]*types.FullCommitQC - blocks map[types.GlobalBlockNumber]*types.Block - appProposals map[types.GlobalBlockNumber]*types.AppProposal - blockHashes map[types.BlockHeaderHash]types.GlobalBlockNumber - - // first is the exclusive low end of retained in-memory state: maps keep - // [first, next*). Set by newInner / skipTo; advanced by evictBelowBound to - // min(nextAppProposal, App.GlobalNumber()+1) when a CommitQC.App exists. - // nextToExecute reads the next (or tip) QC from maps — it does not need - // nextAppProposal-1 retained after eviction. - // - // first <= nextAppProposal <= nextBlockToPersist <= nextBlock <= nextQC + qcs map[types.GlobalBlockNumber]*types.FullCommitQC // [first, nextQC) + blocks map[types.GlobalBlockNumber]*types.Block // [first, nextBlock) + gap-fills in [nextBlock, nextQC) + appProposals map[types.GlobalBlockNumber]*types.AppProposal // [first, nextAppProposal) + appQCs map[types.GlobalBlockNumber]*types.AppQC // [first, nextAppQC) + blockHashes map[types.BlockHeaderHash]types.GlobalBlockNumber // blockHashes mirrors blocks (insertBlock / setPersisted) + + // first is the exclusive low end of retained in-memory state: maps keep [first, next*). + // Advanced by runPersist(). Durable copies below first live in BlockDB. // - // AppProposals require persistence (nextAppProposal <= nextBlockToPersist). - // BlockDB prune status lives only in the store watermark (see PruneBefore). - first types.GlobalBlockNumber - nextAppProposal types.GlobalBlockNumber - nextBlockToPersist types.GlobalBlockNumber - nextBlock types.GlobalBlockNumber - nextQC types.GlobalBlockNumber -} - -func newInner(firstBlock types.GlobalBlockNumber) *inner { - return &inner{ - qcs: map[types.GlobalBlockNumber]*types.FullCommitQC{}, - blocks: map[types.GlobalBlockNumber]*types.Block{}, - appProposals: map[types.GlobalBlockNumber]*types.AppProposal{}, - blockHashes: map[types.BlockHeaderHash]types.GlobalBlockNumber{}, - first: firstBlock, - nextAppProposal: firstBlock, - nextBlockToPersist: firstBlock, - nextBlock: firstBlock, - nextQC: firstBlock, - } -} - -// skipTo advances all cursors to n, discarding everything before it. -// Used on recovery when the first loaded QC starts past committee.FirstBlock() -// (i.e. data before n was pruned in a previous run). -func (i *inner) skipTo(n types.GlobalBlockNumber) { - i.first = n - i.nextAppProposal = n - i.nextBlockToPersist = n - i.nextBlock = n - i.nextQC = n + // first <= nextAppQC <= nextAppProposal <= nextBlock <= nextQC + first types.GlobalBlockNumber + nextAppQC types.GlobalBlockNumber + nextAppProposal types.GlobalBlockNumber + nextBlock types.GlobalBlockNumber + nextQC types.GlobalBlockNumber + persisted types.SuffixRange + + // Anchor represents the highest fully processed row: + // CommitQC, Blocks, AppProposal, AppQC present and persisted. + anchor utils.AtomicSend[utils.Option[Anchor]] } // insertQC verifies and inserts a FullCommitQC into the inner state. // Accepts QCs whose range starts at or before nextQC (partially pruned // prefix is silently skipped). Rejects gaps where gr.First > nextQC. func (i *inner) insertQC(registry *epoch.Registry, qc *types.FullCommitQC) error { - e, ok := registry.EpochByIndex(qc.QC().Proposal().EpochIndex()) - if !ok { - return fmt.Errorf("unknown epoch_index %d", qc.QC().Proposal().EpochIndex()) - } - if err := qc.Verify(e); err != nil { - return fmt.Errorf("qc.Verify(): %w", err) - } gr := qc.QC().GlobalRange() if gr.Next <= i.nextQC { return nil // fully behind, skip @@ -113,6 +65,13 @@ func (i *inner) insertQC(registry *epoch.Registry, qc *types.FullCommitQC) error if gr.First > i.nextQC { return fmt.Errorf("QC gap: expected first<=%d, got %d", i.nextQC, gr.First) } + e, ok := registry.EpochByIndex(qc.QC().Proposal().EpochIndex()) + if !ok { + return fmt.Errorf("unknown epoch_index %d", qc.QC().Proposal().EpochIndex()) + } + if err := qc.Verify(e); err != nil { + return fmt.Errorf("qc.Verify(): %w", err) + } for i.nextQC < gr.Next { i.qcs[i.nextQC] = qc i.nextQC++ @@ -120,6 +79,53 @@ func (i *inner) insertQC(registry *epoch.Registry, qc *types.FullCommitQC) error return nil } +func (i *inner) insertAppQC(registry *epoch.Registry, appQC *types.AppQC) error { + gr := appQC.Proposal().GlobalRange() + if gr.Next <= i.nextAppQC { + return nil + } + if gr.Next > i.nextAppProposal { + return fmt.Errorf("missing AppProposal for this AppQC") + } + if gr.First > i.nextAppQC { + return fmt.Errorf("AppQC gap: expected first<=%d, got %d", i.nextAppQC, gr.First) + } + ei := appQC.Proposal().EpochIndex() + epoch, ok := registry.EpochByIndex(ei) + if !ok { + return fmt.Errorf("unknown epoch_index %d", ei) + } + if err := appQC.Verify(epoch.Committee()); err != nil { + return fmt.Errorf("appQC.Verify(): %w", err) + } + for i.nextAppQC < gr.Next { + i.appQCs[i.nextAppQC] = appQC + i.nextAppQC++ + } + return nil +} + +func (i *inner) insertAppProposal(appProposal *types.AppProposal) error { + gr := appProposal.GlobalRange() + if gr.Next <= i.nextAppProposal { + return nil + } + if gr.Next > i.nextQC { + return fmt.Errorf("missing CommitQC for this AppProposal") + } + if gr.First > i.nextAppProposal { + return fmt.Errorf("AppProposal gap: expected first<=%d, got %d", i.nextAppProposal, gr.First) + } + if err := appProposal.Verify(i.qcs[i.nextAppProposal].QC()); err != nil { + return fmt.Errorf("appProposal.Verify(): %w", err) + } + for i.nextAppProposal < gr.Next { + i.appProposals[i.nextAppProposal] = appProposal + i.nextAppProposal++ + } + return nil +} + // insertBlock inserts a pre-verified block into the inner state. // Requires a QC to already be present for block n. Callers must verify // the block signature before calling (unlike insertQC, which verifies). @@ -166,15 +172,7 @@ func (i *inner) updateNextBlock(m *metrics.Metrics) { } // State of the chain. -// Contains blocks in global order and proofs of their finality. -// -// Invariant: a CommitQC's embedded AppProposal (when present) always refers to -// a global number from a *past* CommitQC — strictly below that tip QC's -// GlobalRange.First (enforced in Proposal.Verify). Together with BlockDB's -// never-empty retention and eviction at min(nextAppProposal, App+1), in-memory -// maps therefore always retain at least the certified tip QC after a -// CommitQC.App appears. nextToExecute uses qc[nextAppProposal] (or the tip QC -// when fully caught up), so it does not require retaining nextAppProposal-1. +// Contains blocks in global order and proofs of sequencing: (CommitQC) and execution result (AppQC). type State struct { cfg *Config metrics *metrics.Metrics @@ -186,146 +184,91 @@ type State struct { // Use memblock.NewBlockDB() for an in-memory store (testing / no persistent dir). // The caller owns blockDB and must close it after State.Run returns (nodeImpl // owns this in production); State never closes it. -// Recovery starts at cfg.LastExecutedBlock and handles a non-zero CommitQC tip -// via loadFromBlockDB (skipTo). func NewState(cfg *Config, blockDB types.BlockDB) (*State, error) { - s := &State{ + inner, err := loadFromBlockDB(cfg, blockDB) + if err != nil { + return nil, fmt.Errorf("loadFromBlockDB: %w", err) + } + return &State{ cfg: cfg, metrics: metrics.Get(), - inner: utils.NewWatch(newInner(cfg.Registry.FirstBlock())), + inner: utils.NewWatch(inner), blockDB: blockDB, - } - if err := s.loadFromBlockDB(blockDB); err != nil { - return nil, fmt.Errorf("loadFromBlockDB: %w", err) - } - return s, nil + }, nil } -// loadFromBlockDB replays QCs and blocks from blockDB into s.inner. -// Called from NewState before any goroutines are spawned; the lock is acquired -// only to satisfy the Watch API. -// -// Recovery starts at the app tip so runExecute can replay its AppHash. If the -// app tip equals BlockDB's next block, recovery starts at the last stored block: -// app.Commit may finish before BlockDB is durable. A larger gap violates the -// PushAppHash durability invariant. An empty BlockDB allows only no app tip or -// the first committed block. -// -// Without an app tip, recovery starts at the registry's first block. -// BlockDB.Iterator clamps the start to its retained floor. skipTo uses the first -// returned position, even inside a QC, to keep blocks dense over -// [first, nextBlock). -// -// Each iterator position has its covering QC and an optional block. Missing -// blocks are allowed only at the tail. BlockDB enforces other consistency; this -// method only rejects a first QC before committee genesis. -func (s *State) loadFromBlockDB(blockDB types.BlockDB) error { - for in := range s.inner.Lock() { - err := func() error { - firstBlock := s.cfg.Registry.FirstBlock() - dbNextBlock := blockDB.Status().NextBlock - recoveryStart := firstBlock - if lastExecutedBlock, executed := s.cfg.LastExecutedBlock.Get(); executed { - if lastExecutedBlock > max(dbNextBlock, firstBlock) { - return fmt.Errorf( - "BlockDB next block %d is behind app tip %d by more than the recoverable crash window; restore matching BlockDB data or state-sync the node: %w", - dbNextBlock, lastExecutedBlock, types.ErrNotFound, - ) - } - recoveryStart = lastExecutedBlock - if dbNextBlock != 0 && lastExecutedBlock == dbNextBlock { - // app.Commit completed before the app tip became durable. - recoveryStart = dbNextBlock - 1 - } - } - it, err := blockDB.Iterator(recoveryStart) - if err != nil { - return fmt.Errorf("open block db iterator: %w", err) - } - defer func() { _ = it.Close() }() - var lastQC *types.FullCommitQC - for { - pos, ok, err := it.Next() - if err != nil { - return fmt.Errorf("advance block db iterator: %w", err) - } - if !ok { - break - } - n, qc := pos.Number, pos.QC - gr := qc.QC().GlobalRange() - if lastQC == nil { - // First position: skipTo it to advance past any pruned prefix. Nothing has - // been inserted yet, which is what makes it legal for skipTo to move first - // without deleting. Subsequent QCs must be consecutive — insertQC errors on - // any gap. - if gr.First < in.nextQC { - return fmt.Errorf("QC in BlockDB predates committee genesis %d: got %d", - in.nextQC, gr.First) - } - if n > in.nextQC { - in.skipTo(n) - } - } - if qc != lastQC { - // The scan entered a new QC's range. Position.QC hands back the same pointer - // for every number in a range, so identity is the exact test — and unlike - // gr.First == n it still fires for the covering QC when the scan opened - // inside that QC's range. insertQC clips it to [nextQC, gr.Next). - lastQC = qc - if err := in.insertQC(s.cfg.Registry, qc); err != nil { - return fmt.Errorf("load QC from BlockDB: %w", err) - } - } - if !pos.HasBlock { - // The iteration tail: the covering QC is persisted but this block is - // not (lost in a crash, or written ahead of its blocks). - continue - } - blkOpt, err := it.Block() - if err != nil { - return fmt.Errorf("read block %d from BlockDB: %w", n, err) - } - blk := blkOpt.OrPanic(fmt.Sprintf("block %d absent at a HasBlock position", n)) - e, ok := s.cfg.Registry.EpochByIndex(qc.QC().Proposal().EpochIndex()) - if !ok { - return fmt.Errorf("unknown epoch_index %d", qc.QC().Proposal().EpochIndex()) - } - if err := blk.Verify(e.Committee()); err != nil { - return fmt.Errorf("verify block %d from BlockDB: %w", n, err) - } - if err := in.insertBlock(n, blk); err != nil { - return fmt.Errorf("insert block %d from BlockDB: %w", n, err) - } - } - return nil - }() - if err != nil { - return err +// loadFromBlockDB replays the persisted suffix from blockDB into s.inner. +// Called from NewState before any goroutines are spawned. +func loadFromBlockDB(cfg *Config, blockDB types.BlockDB) (*inner, error) { + suffix, err := blockDB.ReadSuffix() + if err != nil { + return nil, fmt.Errorf("blockDB.ReadSuffix(): %w", err) + } + firstBlock := cfg.Registry.FirstBlock() + status := suffix.Status.Or(types.SuffixRange{ + First: firstBlock, + NextQC: firstBlock, + NextAppProposal: firstBlock, + NextAppQC: firstBlock, + NextBlock: firstBlock, + }) + inner := &inner{ + qcs: map[types.GlobalBlockNumber]*types.FullCommitQC{}, + blocks: map[types.GlobalBlockNumber]*types.Block{}, + appQCs: map[types.GlobalBlockNumber]*types.AppQC{}, + appProposals: map[types.GlobalBlockNumber]*types.AppProposal{}, + blockHashes: map[types.BlockHeaderHash]types.GlobalBlockNumber{}, + first: status.First, + nextAppProposal: status.First, + nextAppQC: status.First, + nextBlock: status.First, + nextQC: status.First, + persisted: status, + anchor: utils.NewAtomicSend(utils.None[Anchor]()), + } + for _, qc := range suffix.CommitQCs { + if err := inner.insertQC(cfg.Registry, qc); err != nil { + return nil, fmt.Errorf("load QC from BlockDB: %w", err) + } + } + for _, b := range suffix.Blocks { + qc := inner.qcs[b.Number] + ei := qc.QC().Proposal().EpochIndex() + e, ok := cfg.Registry.EpochByIndex(ei) + if !ok { + return nil, fmt.Errorf("unknown epoch_index %d", ei) } - - // Advance nextBlock through contiguous loaded blocks. Don't use - // updateNextBlock: stale timestamps would skew metrics. - for ; in.blocks[in.nextBlock] != nil; in.nextBlock++ { + if err := b.Block.Verify(e.Committee()); err != nil { + return nil, fmt.Errorf("verify block %d from BlockDB: %w", b.Number, err) + } + if err := inner.insertBlock(b.Number, b.Block); err != nil { + return nil, fmt.Errorf("insert block %d from BlockDB: %w", b.Number, err) } - // Data loaded from BlockDB was already durably persisted. - in.nextBlockToPersist = in.nextBlock } - return nil + // Advance nextBlock through contiguous loaded blocks. Don't use + // updateNextBlock: stale timestamps would skew metrics. + inner.nextBlock = max(inner.first, status.NextBlock) + for _, appProposal := range suffix.AppProposals { + if err := inner.insertAppProposal(appProposal); err != nil { + return nil, fmt.Errorf("load AppProposal from BlockDB: %w", err) + } + } + for _, appQC := range suffix.AppQCs { + if err := inner.insertAppQC(cfg.Registry, appQC); err != nil { + return nil, fmt.Errorf("load AppQC from BlockDB: %w", err) + } + } + inner.setAnchor() + return inner, nil +} + +func (s *State) First() types.GlobalBlockNumber { + return max(s.blockDB.First(), s.cfg.Registry.FirstBlock()) } // Registry returns the epoch registry. func (s *State) Registry() *epoch.Registry { return s.cfg.Registry } -// FirstAppProposal is the first global number for which AppProposal may become -// available. Requests below it return ErrPruned. -func (s *State) FirstAppProposal() types.GlobalBlockNumber { - for inner := range s.inner.Lock() { - return inner.first - } - panic("unreachable") -} - // insertBlocksByHash matches byHash against stored (already verified) QC // headers over gr ∩ [nextBlock, nextQC) and inserts hits. Advances nextBlock // when the contiguous prefix grows. Caller must hold inner's lock. @@ -356,7 +299,7 @@ func (s *State) PushQC(ctx context.Context, qc *types.FullCommitQC, blocks []*ty needQC, err := func() (bool, error) { for inner, ctrl := range s.inner.Lock() { if err := ctrl.WaitUntil(ctx, func() bool { - return gr.First <= inner.nextQC && gr.First < inner.nextAppProposal+blocksCacheSize + return gr.First <= inner.nextQC && gr.First < inner.first+blocksCacheSize }); err != nil { return false, err } @@ -396,10 +339,6 @@ func (s *State) PushQC(ctx context.Context, qc *types.FullCommitQC, blocks []*ty } ctrl.Updated() } - // Only a newly accepted QC can advance CommitQC.App / the eviction floor. - if needQC { - evictBelowBound(inner) - } } return nil } @@ -468,10 +407,18 @@ func (s *State) NextBlock() types.GlobalBlockNumber { panic("unreachable") } +// NextAppQC returns the index of the next AppQC to be pushed. +func (s *State) NextAppQC() types.GlobalBlockNumber { + for inner := range s.inner.Lock() { + return inner.nextAppQC + } + panic("unreachable") +} + // GlobalBlockByHash returns the finalized GlobalBlock whose stored header // hashes to the given value, or None if no such block is currently retained. // Non-blocking. Serves from RAM whenever the hash is still indexed (contiguous -// prefix, gap-fills, and executed heights not yet dropped by evictBelowBound). +// prefix, gap-fills, and executed heights not yet dropped by setPersisted). // Falls back to BlockDB only after eviction removes the hash — matching // Block/TryBlock/QC, which also prefer maps before the store. Gap-fills are // not written to BlockDB until nextBlock catches up, so they must be served @@ -552,11 +499,10 @@ func (s *State) NeedBlock(n types.GlobalBlockNumber) bool { func assembleGlobalBlock(n types.GlobalBlockNumber, b *types.Block, fqc *types.FullCommitQC) *types.GlobalBlock { qc := fqc.QC() return &types.GlobalBlock{ - GlobalNumber: n, - Timestamp: qc.Proposal().BlockTimestamp(n).OrPanic("global block not in QC"), - Header: b.Header(), - Payload: b.Payload(), - FinalAppState: qc.Proposal().App(), + GlobalNumber: n, + Timestamp: qc.Proposal().BlockTimestamp(n).OrPanic("global block not in QC"), + Header: b.Header(), + Payload: b.Payload(), } } @@ -605,6 +551,17 @@ func (s *State) qcFromDB(n types.GlobalBlockNumber) (*types.FullCommitQC, error) return qc, nil } +func (s *State) appQCFromDB(n types.GlobalBlockNumber) (*types.AppQC, error) { + opt, err := s.blockDB.ReadAppQCByBlockNumber(n) + if err != nil { + return nil, fmt.Errorf("blockDB.ReadAppQCByBlockNumber(%d): %w", n, err) + } + if appQC, ok := opt.Get(); ok { + return appQC, nil + } + return nil, types.ErrPruned +} + func (s *State) globalBlockFromDB(n types.GlobalBlockNumber) (*types.GlobalBlock, error) { b, err := s.blockFromDB(n) if err != nil { @@ -636,24 +593,24 @@ func (s *State) globalBlockByHashFromDB(hash types.BlockHeaderHash) (utils.Optio return utils.Some(assembleGlobalBlock(bn.Number, bn.Block, qc)), nil } -// PushAppHash marks blocks up to n as executed. Hash is the execution result. -// Waits for the block to be durably persisted before proceeding. +// PushAppHash marks blocks up to n as executed. func (s *State) PushAppHash(ctx context.Context, n types.GlobalBlockNumber, hash types.AppHash) error { for inner, ctrl := range s.inner.Lock() { + if err := ctrl.WaitUntil(ctx, func() bool { return n < inner.nextBlock }); err != nil { + return err + } if n < inner.nextAppProposal { - return fmt.Errorf("received app proposal out of order: got %v, want >= %v", n, inner.nextAppProposal) + return nil } - if err := ctrl.WaitUntil(ctx, func() bool { - return n < inner.nextBlockToPersist - }); err != nil { - return err + p := inner.qcs[n].QC().Proposal() + if next, first := inner.nextAppProposal, p.GlobalRange().First; next < first { + // We expect the AppHashes to be pushed in order. + return fmt.Errorf("received appHash for %v: %w", n, ErrOutOfOrder) + } else if n != p.GlobalRange().Next-1 { + // We only care about the AppHash of the last block of the range. + return nil } - proposal := types.NewAppProposal( - n, - inner.qcs[n].QC().Proposal().Index(), - hash, - inner.qcs[n].QC().Proposal().EpochIndex(), - ) + proposal := types.NewAppProposal(p, hash) t := time.Now() for inner.nextAppProposal <= n { b := inner.blocks[inner.nextAppProposal] @@ -663,62 +620,96 @@ func (s *State) PushAppHash(ctx context.Context, n types.GlobalBlockNumber, hash inner.appProposals[inner.nextAppProposal] = proposal inner.nextAppProposal += 1 } - evictBelowBound(inner) ctrl.Updated() + // CRITICAL: We need to persist AppHash before we return and start executing the next block, + // otherwise we lose the apphash on restart. + // TODO(gprusak): this is a temporary measure, until AppHashes are persisted outside of BlockDB. + if err := ctrl.WaitUntil(ctx, func() bool { return n < inner.persisted.NextAppProposal }); err != nil { + return err + } } return nil } -// AppProposal returns the lowest AppProposal containing the block n. -// WARNING: currently we do not enforce all blocks to have AppProposal, therefore -// an AppProposal for a later block might be returned instead. -func (s *State) AppProposal(ctx context.Context, n types.GlobalBlockNumber) (*types.AppProposal, error) { +// AppVote returns an appVote for a block >= n. +// Vote is available ONLY once AppHash has been pushed AND persisted. +// This prevents any possible equivocation and ensures that local node has the executed blocks persisted (since nextAppProposa <= nextBlock). +func (s *State) AppVote(ctx context.Context, n types.GlobalBlockNumber) (*types.AppVote, error) { for inner, ctrl := range s.inner.Lock() { - if err := ctrl.WaitUntil(ctx, func() bool { return n < inner.nextAppProposal }); err != nil { + if err := ctrl.WaitUntil(ctx, func() bool { return n < inner.persisted.NextAppProposal }); err != nil { return nil, err } if n < inner.first { return nil, types.ErrPruned } - ap, ok := inner.appProposals[n] - if !ok { - return nil, types.ErrPruned + return types.NewAppVote(inner.appProposals[n]), nil + } + panic("unreachable") +} + +// PushAppQC pushes an AppQC to the state and advances the AppQC cursor. +func (s *State) PushAppQC(ctx context.Context, appQC *types.AppQC) error { + gr := appQC.Proposal().GlobalRange() + for inner, ctrl := range s.inner.Lock() { + if err := ctrl.WaitUntil(ctx, func() bool { + return gr.Next <= inner.nextAppProposal + }); err != nil { + return err + } + if gr.First < inner.nextAppQC { + return nil + } + if err := inner.insertAppQC(s.cfg.Registry, appQC); err != nil { + return err + } + t := time.Now() + for n := gr.First; n < gr.Next; n++ { + b := inner.blocks[n] + latency := t.Sub(b.Payload().CreatedAt()).Seconds() + s.metrics.Blocks.Certify.Observe(latency) + s.metrics.Txs.Certify.ObserveWithWeight(latency, uint64(len(b.Payload().Txs()))) + } + ctrl.Updated() + return nil + } + panic("unreachable") +} + +func (s *State) AppQC(ctx context.Context, n types.GlobalBlockNumber) (*types.AppQC, error) { + for inner, ctrl := range s.inner.Lock() { + if err := ctrl.WaitUntil(ctx, func() bool { return n < inner.nextAppQC }); err != nil { + return nil, err + } + if inner.first <= n { + return inner.appQCs[n], nil } - return ap, nil + } + return s.appQCFromDB(n) +} + +type Anchor struct { + CommitQC *types.CommitQC + AppQC *types.AppQC +} + +// Anchor represents the AppQC/CommitQC covering inner.first. +// It is used by avail.State. +func (s *State) Anchor() utils.AtomicRecv[utils.Option[Anchor]] { + for inner := range s.inner.Lock() { + return inner.anchor.Subscribe() } panic("unreachable") } func (i *inner) nextToExecute(lane types.LaneID) types.BlockNumber { - // TODO(gprusak): decide whether 0 is a good result in this case in general. - // Empty maps (first == nextQC) only on fresh start / after skipTo with no QC. - if i.first == i.nextQC { - return 0 + if i.nextAppProposal < i.nextQC { + return i.qcs[i.nextAppProposal].QC().LaneRange(lane).First() } - // Fully executed through the certified tip: next lane block is past the tip QC. - if i.nextAppProposal == i.nextQC { + if i.first < i.nextAppProposal { return i.qcs[i.nextAppProposal-1].QC().LaneRange(lane).Next() } - // nextAppProposal < nextQC: derive from the next global block to execute - // (header from FullCommitQC — works even if blocks[n] was never gap-filled). - n := i.nextAppProposal - fqc := i.qcs[n] - qc := fqc.QC() - gr := qc.GlobalRange() - h := fqc.Headers()[n-gr.First] - r := qc.LaneRange(lane) - x := lane.Compare(h.Lane()) - // NOTE: here we assume the specific ordering of lane blocks in the CommitQC: - // TODO(gprusak): move this logic closer to CommitQC - switch { - case x < 0: - return r.Next() - case x > 0: - return r.First() - default: - // This block is not executed yet. - return h.BlockNumber() - } + // Genesis state: i.first == i.nextQC + return 0 } // Waits until lane block n is executed, returns the next block of this lane to be executed (>n) @@ -737,24 +728,24 @@ func (s *State) WaitUntilExecuted(ctx context.Context, lane types.LaneID, n type } // PruneBefore asks BlockDB to drop data before retainFrom. This is independent -// of in-memory retention: RAM is cleared only by evictBelowBound (AppQC floor), -// and AppProposals are not persisted. BlockDB enforces its own never-empty -// retention and refuses reads below its watermark. +// of in-memory retention: RAM is cleared only by setPersisted. +// BlockDB enforces its own never-empty retention and refuses reads below its +// watermark. func (s *State) PruneBefore(retainFrom types.GlobalBlockNumber) error { return s.blockDB.PruneBefore(retainFrom) } -// runPersist is a background goroutine that persists blocks and QCs to BlockDB. -// It waits for in-memory blocks to advance past the block persistence cursor, -// then writes covering QCs (first, per the BlockDB contract) and blocks, then -// flushes once per batch. nextBlockToPersist advances with the block tip to -// unblock PushAppHash only when data is durable. +// runPersist is a background goroutine that persists QCs, blocks, +// AppProposals, and AppQCs to BlockDB. It waits for in-memory data to advance +// past the SuffixRange persistence cursor, then writes each stream in cursor +// order and flushes once per batch. persisted.NextBlock advances with the +// block tip to unblock PushAppHash only when data is durable. // Errors propagate vertically (kill the component). // // Cursors seed from BlockDB.Status() when non-zero so PushQC-before-Run heights -// are not skipped. When a tip is zero, seed from post-load nextBlockToPersist -// (recovery floor), never bare registry.FirstBlock() — a QC-only store can -// skipTo past genesis while NextBlock is still zero. +// are not skipped. When a tip is zero, seed from the recovery floor, never bare +// registry.FirstBlock() — a QC-only store can skipTo past genesis while +// NextBlock is still zero. // // Under the BlockDB write contract (QC before covered blocks), NextQC is never // behind NextBlock after a successful write. Persistence is driven by the block @@ -762,123 +753,101 @@ func (s *State) PruneBefore(retainFrom types.GlobalBlockNumber) error { // is still at or past NextQC (enough coverage for each new block, not every // in-memory QC, and no rewrite of QCs already on disk). // -// In-memory eviction is driven by PushQC / PushAppHash (evictBelowBound), not here. +// In-memory block/QC/AppProposal/AppQC eviction is driven by persisted SuffixRange +// changes. func (s *State) runPersist(ctx context.Context) error { - tips := s.blockDB.Status() - var nextToPersistQC, nextToPersistBlock types.GlobalBlockNumber - for inner := range s.inner.Lock() { - // After loadFromBlockDB, nextBlockToPersist is the durable recovery tip. - nextToPersistQC = tips.NextQC - if nextToPersistQC == 0 { - nextToPersistQC = inner.nextBlockToPersist - } - nextToPersistBlock = tips.NextBlock - if nextToPersistBlock == 0 { - nextToPersistBlock = inner.nextBlockToPersist - } - } for { - type batch struct { - qcs []*types.FullCommitQC - blocks []blockEntry - nextBlock types.GlobalBlockNumber - } - var b batch + var qcs []*types.FullCommitQC + var blocks []blockEntry + var appProposals []*types.AppProposal + var appQCs []*types.AppQC + var status types.SuffixRange for inner, ctrl := range s.inner.Lock() { + status = inner.persisted + // Wait until there is anything to persist. if err := ctrl.WaitUntil(ctx, func() bool { - return nextToPersistBlock < inner.nextBlock + return status.NextQC < inner.nextQC || status.NextBlock < inner.nextBlock || + status.NextAppProposal < inner.nextAppProposal || status.NextAppQC < inner.nextAppQC }); err != nil { return err } - b.nextBlock = inner.nextBlock - // Persist blocks in [nextToPersistBlock, nextBlock). Emit each covering - // QC once at GlobalRange.First when it has not already been written - // (First >= nextToPersistQC). - for n := nextToPersistBlock; n < inner.nextBlock; n++ { - qc := inner.qcs[n] - gr := qc.QC().GlobalRange() - if n == gr.First && gr.First >= nextToPersistQC { - b.qcs = append(b.qcs, qc) - } - b.blocks = append(b.blocks, blockEntry{n: n, block: inner.blocks[n]}) + // Collect data to persist. + for status.NextQC < inner.nextQC { + qc := inner.qcs[status.NextQC] + qcs = append(qcs, qc) + status.NextQC = qc.QC().GlobalRange().Next + } + for status.NextBlock < inner.nextBlock { + blocks = append(blocks, blockEntry{n: status.NextBlock, block: inner.blocks[status.NextBlock]}) + status.NextBlock += 1 + } + for status.NextAppProposal < inner.nextAppProposal { + appProposal := inner.appProposals[status.NextAppProposal] + appProposals = append(appProposals, appProposal) + status.NextAppProposal = appProposal.GlobalRange().Next + } + for status.NextAppQC < inner.nextAppQC { + appQC := inner.appQCs[status.NextAppQC] + appQCs = append(appQCs, appQC) + status.NextAppQC = appQC.Proposal().GlobalRange().Next + status.First = status.NextAppQC - 1 } } - // Write QCs first (BlockDB contract: QC must precede covered blocks). - for _, qc := range b.qcs { - gr := qc.QC().GlobalRange() + // Write data in order: QCs,blocks,appProposals,appQCs + // to maintain the invariants. + for _, qc := range qcs { if err := s.blockDB.WriteQC(qc); err != nil { - return fmt.Errorf("write QC [%d,%d): %w", gr.First, gr.Next, err) - } - if gr.Next > nextToPersistQC { - nextToPersistQC = gr.Next + return fmt.Errorf("write QC %d: %w", qc.QC().Index(), err) } } - for _, lb := range b.blocks { + for _, lb := range blocks { if err := s.blockDB.WriteBlock(lb.n, lb.block); err != nil { return fmt.Errorf("write block %d: %w", lb.n, err) } } - // Flush once per batch before advancing nextBlockToPersist, so that - // PushAppHash only unblocks after data is crash-durable. + for _, appProposal := range appProposals { + if err := s.blockDB.WriteAppProposal(appProposal); err != nil { + return fmt.Errorf("write AppProposal %d: %w", appProposal.RoadIndex(), err) + } + } + for _, appQC := range appQCs { + if err := s.blockDB.WriteAppQC(appQC); err != nil { + return fmt.Errorf("write AppQC %d: %w", appQC.Proposal().RoadIndex(), err) + } + } + // Flush the new data. if err := s.blockDB.Flush(); err != nil { return fmt.Errorf("flush BlockDB: %w", err) } - nextToPersistBlock = b.nextBlock + // Prune the inner state. for inner, ctrl := range s.inner.Lock() { - if nextToPersistBlock > inner.nextBlockToPersist { - inner.nextBlockToPersist = nextToPersistBlock - ctrl.Updated() + inner.persisted = status + for inner.first < inner.persisted.First { + // Divergence detection + n := inner.first + if got, want := inner.appProposals[n].AppHash(), inner.appQCs[n].Proposal().AppHash(); !bytes.Equal(got, want) { + return fmt.Errorf("AppHash divergence detected at block %v: local AppHash = %v, quorum Apphash = %v", n, got, want) + } + delete(inner.blockHashes, inner.blocks[n].Header().Hash()) + delete(inner.blocks, n) + delete(inner.qcs, n) + delete(inner.appQCs, n) + delete(inner.appProposals, n) + inner.first += 1 } + inner.setAnchor() + ctrl.Updated() } } } -// certifiedAppFloor is the exclusive eviction floor from the tip CommitQC's -// embedded App, if any: App.GlobalNumber()+1. Only the tip QC is consulted -// (no backward walk). Returns 0 when maps are empty or the tip has no App. -func (i *inner) certifiedAppFloor() types.GlobalBlockNumber { - if i.first == i.nextQC { - return 0 - } - // [first, nextQC) is dense in qcs, so nextQC-1 is present. - app, ok := i.qcs[i.nextQC-1].QC().Proposal().App().Get() - if !ok { - return 0 +func (i *inner) setAnchor() { + if i.first < i.persisted.NextAppQC { + i.anchor.Store(utils.Some(Anchor{ + CommitQC: i.qcs[i.first].QC(), + AppQC: i.appQCs[i.first], + })) } - return app.GlobalNumber() + 1 -} - -// evictBelowBound advances first toward the certified App floor and drops cached -// blocks/QCs/AppProposals with n < first. No-op when there is no certified App -// or the bound would not advance first. Caller must hold inner's lock. Invoked -// from PushQC / PushAppHash. -// -// Bound is min(nextAppProposal, App.GlobalNumber()+1). A zero floor (no App / -// empty maps) yields bound 0 and is a no-op via bound <= first. With the -// past-CommitQC App invariant (see State), App+1 never exceeds the tip QC -// start, so at least one CommitQC remains. nextToExecute uses qc[nextAppProposal] -// (or the tip when caught up), so nextAppProposal-1 need not be retained. -// -// TODO: At eviction we have both a local AppProposal and a CommitQC.App, so this -// is the right place to detect local-vs-quorum AppHash inconsistency. Surface -// any mismatch from data.State.Run() (node-fatal), not from PushQC/PushAppHash — -// e.g. stash an error on State for a Run monitor, or run eviction as its own -// Run subtask. -func evictBelowBound(inner *inner) { - floor := inner.certifiedAppFloor() - bound := min(inner.nextAppProposal, floor) - if bound <= inner.first { - return - } - for n := inner.first; n < bound; n++ { - if b, ok := inner.blocks[n]; ok { - delete(inner.blockHashes, b.Header().Hash()) - delete(inner.blocks, n) - } - delete(inner.qcs, n) - delete(inner.appProposals, n) - } - inner.first = bound } // Run starts the background persistence loop. diff --git a/sei-tendermint/internal/autobahn/data/state_recovery_test.go b/sei-tendermint/internal/autobahn/data/state_recovery_test.go index 251c1b51c0..ee80268b65 100644 --- a/sei-tendermint/internal/autobahn/data/state_recovery_test.go +++ b/sei-tendermint/internal/autobahn/data/state_recovery_test.go @@ -14,16 +14,6 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/scope" ) -type recoveryStartBlockDB struct { - types.BlockDB - start types.GlobalBlockNumber -} - -func (db *recoveryStartBlockDB) Iterator(n types.GlobalBlockNumber) (types.BlockDBIterator, error) { - db.start = n - return db.BlockDB.Iterator(n) -} - // TestRecoveryEmpty verifies that NewState is a no-op on a fresh BlockDB. func TestRecoveryEmpty(t *testing.T) { rng := utils.TestRng() @@ -108,119 +98,21 @@ func TestRecoveryNormal(t *testing.T) { require.Equal(t, gr2.Next, state3.NextBlock()) } -func TestRecoveryStartsAtLastExecutedBlock(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 3) - qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) - qc2, blocks2 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc1.QC())) - gr2 := qc2.QC().GlobalRange() - require.Greater(t, gr2.Len(), 2) - - db := &recoveryStartBlockDB{BlockDB: newTestBlockDB(t, t.TempDir())} - writeToBlockDB(t, db, - []*types.FullCommitQC{qc1, qc2}, - [][]*types.Block{blocks1, blocks2}) - - offset := gr2.Len() / 2 - lastExecuted := gr2.First + types.GlobalBlockNumber(offset) - state := newTestState(t, &Config{ - Registry: registry, - LastExecutedBlock: utils.Some(lastExecuted), - }, db) - - require.Equal(t, lastExecuted, db.start) - require.Equal(t, lastExecuted, state.FirstAppProposal()) - require.Equal(t, gr2.Next, state.NextBlock()) - got, err := state.TryBlock(lastExecuted) - require.NoError(t, err) - require.Equal(t, blocks2[offset].Header().Hash(), got.Header().Hash()) - - appHash := types.GenAppHash(rng) - require.NoError(t, state.PushAppHash(t.Context(), lastExecuted, appHash)) - proposal, err := state.AppProposal(t.Context(), lastExecuted) - require.NoError(t, err) - require.Equal(t, appHash, proposal.AppHash()) -} - -func TestRecoveryCapsAppTipAtLastBlockInBlockDB(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 3) - qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) - qc2, blocks2 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc1.QC())) - gr1 := qc1.QC().GlobalRange() - gr2 := qc2.QC().GlobalRange() - - dir := t.TempDir() - db1 := newTestBlockDB(t, dir) - writeToBlockDB(t, db1, []*types.FullCommitQC{qc1}, [][]*types.Block{blocks1}) - require.NoError(t, db1.Close()) - - db := &recoveryStartBlockDB{BlockDB: newTestBlockDB(t, dir)} - - state, err := NewState(&Config{ - Registry: registry, - LastExecutedBlock: utils.Some(gr2.First), - }, db) - require.NoError(t, err) - require.Equal(t, gr1.Next-1, db.start) - require.Equal(t, gr2.First, state.NextBlock()) - - require.NoError(t, state.PushQC(t.Context(), qc2, blocks2)) - got, err := state.GlobalBlock(t.Context(), gr2.First) - require.NoError(t, err) - require.Equal(t, blocks2[0].Header().Hash(), got.Header.Hash()) - - require.NoError(t, pushAppHashesRunning(t.Context(), state, rng, gr2.First, gr2.First+1)) -} - -func TestRecoveryRejectsAppTipBeyondCrashWindow(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 3) - qc, blocks := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) - - db := newTestBlockDB(t, t.TempDir()) - writeToBlockDB(t, db, []*types.FullCommitQC{qc}, [][]*types.Block{blocks}) - dbNextBlock := db.Status().NextBlock - lastExecuted := dbNextBlock + 1 - - _, err := NewState(&Config{ - Registry: registry, - LastExecutedBlock: utils.Some(lastExecuted), - }, db) - require.ErrorIs(t, err, types.ErrNotFound) -} - func TestRecoveryStartsAtRegistryFloorWhenBlockDBMissingFirstCommittedBlock(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 3) qc, _ := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) gr := qc.QC().GlobalRange() - db := &recoveryStartBlockDB{BlockDB: newTestBlockDB(t, t.TempDir())} + db := newTestBlockDB(t, t.TempDir()) require.NoError(t, db.WriteQC(qc)) require.NoError(t, db.Flush()) - state, err := NewState(&Config{ - Registry: registry, - LastExecutedBlock: utils.Some(gr.First), - }, db) + state, err := NewState(&Config{Registry: registry}, db) require.NoError(t, err) - require.Equal(t, registry.FirstBlock(), db.start) require.Equal(t, gr.First, state.NextBlock()) } -func TestRecoveryRejectsEmptyBlockDBAfterFirstCommittedBlock(t *testing.T) { - rng := utils.TestRng() - registry, _ := epoch.GenRegistry(rng, 3) - lastExecuted := registry.FirstBlock() + 1 - - _, err := NewState(&Config{ - Registry: registry, - LastExecutedBlock: utils.Some(lastExecuted), - }, newTestBlockDB(t, t.TempDir())) - require.ErrorIs(t, err, types.ErrNotFound) -} - func TestRecoveryLeavesAppTipBelowPruneFloorUnreadable(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 3) @@ -231,20 +123,18 @@ func TestRecoveryLeavesAppTipBelowPruneFloorUnreadable(t *testing.T) { writeToBlockDB(t, db, []*types.FullCommitQC{qc1, qc2}, [][]*types.Block{blocks1, blocks2}) + writeAppDataToBlockDB(t, rng, db, keys, qc1, qc2) require.NoError(t, db.PruneBefore(qc2.QC().GlobalRange().First)) - state, err := NewState(&Config{ - Registry: registry, - LastExecutedBlock: utils.Some(qc1.QC().GlobalRange().First), - }, db) + state, err := NewState(&Config{Registry: registry}, db) require.NoError(t, err) _, err = state.GlobalBlock(t.Context(), qc1.QC().GlobalRange().First) require.ErrorIs(t, err, types.ErrPruned) } -// TestPruningDiscards verifies that PruneBefore advances BlockDB's watermark so -// TryBlock returns ErrPruned for the discarded range, while later blocks stay -// accessible. Memory is cleared by evictBelowBound (from PushQC/PushAppHash). +// TestPruningDiscards verifies that PruneBefore advances BlockDB's watermark but +// does not discard RAM-retained blocks. Memory is cleared only by the AppQC +// eviction floor. func TestPruningDiscards(t *testing.T) { ctx := t.Context() rng := utils.TestRng() @@ -269,8 +159,9 @@ func TestPruningDiscards(t *testing.T) { require.NoError(t, state.PruneBefore(gr2.First)) for n := gr1.First; n < gr2.First; n++ { - _, err := state.TryBlock(n) - require.ErrorIs(t, err, types.ErrPruned) + got, err := state.TryBlock(n) + require.NoError(t, err) + require.NotNil(t, got) } for n := gr2.First; n < gr3.Next; n++ { got, err := state.TryBlock(n) @@ -300,8 +191,8 @@ func TestRecoveryAfterPruning(t *testing.T) { require.NoError(t, db1.Close()) // Recovery skipTo(gr2.First); qc1 heights are absent from BlockDB → ErrPruned. - // With no CommitQC.App yet, first stays at the recovery floor (not advanced - // to 0), so below-floor reads fall through to BlockDB instead of nil maps. + // With no AppQC yet, first stays at the recovery floor (not advanced to 0), + // so below-floor reads fall through to BlockDB instead of nil maps. db2 := newTestBlockDB(t, dir) state2 := newTestState(t, &Config{Registry: registry}, db2) @@ -371,58 +262,6 @@ func TestRecoveryBlocksBehind(t *testing.T) { require.Equal(t, gr2.Next, state2.NextBlock()) } -// TestRecoveryPartialQCPrefix verifies recovery from a store whose blocks begin partway into -// their covering QC. The first block may be written anywhere inside that QC (see -// types.BlockDB.WriteBlock), so BlockDB.Iterator opens on it and the recovery floor follows — -// landing on the first present block, not on the QC's start. Flooring at the QC start would -// leave the blockless prefix inside [first, nextBlock), which inner's density invariant forbids. -func TestRecoveryPartialQCPrefix(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 3) - dir := t.TempDir() - - qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) - gr1 := qc1.QC().GlobalRange() - if gr1.Next-gr1.First < 3 { - t.Skip("need at least 3 blocks in QC range to test split") - } - - // Write the QC for the full range, but write blocks only from mid onwards. - mid := gr1.First + (gr1.Next-gr1.First)/2 - db1 := newTestBlockDB(t, dir) - require.NoError(t, db1.WriteQC(qc1)) - for i, n := 0, gr1.First; n < gr1.Next; n++ { - if n >= mid { - require.NoError(t, db1.WriteBlock(n, blocks1[i])) - } - i++ - } - require.NoError(t, db1.Flush()) - require.NoError(t, db1.Close()) - - state2 := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, dir)) - - // The floor is the first present block, and the contiguous prefix runs from there to the - // end of the QC's coverage. - require.Equal(t, gr1.Next, state2.NextBlock()) - for inner := range state2.inner.Lock() { - require.Equal(t, mid, inner.first, "floor must be the first present block") - require.Equal(t, mid, inner.nextAppProposal) - require.Equal(t, gr1.Next, inner.nextQC) - } - - // Nothing was ever written below the floor, so those heights are not served. - for n := gr1.First; n < mid; n++ { - _, err := state2.TryBlock(n) - require.ErrorIs(t, err, types.ErrPruned) - } - for n := mid; n < gr1.Next; n++ { - got, err := state2.TryBlock(n) - require.NoError(t, err) - require.NotNil(t, got) - } -} - // TestRecoveryAfterPruneNoGC verifies that restarting before async GC reclaims // pruned entries does not cause NewState to fail. Blocks and QCs share the same // GC filter in littblock, so below-watermark blocks never survive past their @@ -500,9 +339,9 @@ func TestRecoveryQCsNoBlocks(t *testing.T) { } // TestRunPersistSeedsFromRecoveryFloor verifies that runPersist does not walk -// [genesis, recoveryFloor) when Status lacks NextBlock (QC-only store -// whose first QC starts past FirstBlock). Seeding from nextBlockToPersist -// avoids collecting nil block pointers. +// [genesis, recoveryFloor) for a QC-only store whose first QC starts past +// FirstBlock. Seeding persisted SuffixRange from the recovery floor avoids +// collecting nil block pointers. func TestRunPersistSeedsFromRecoveryFloor(t *testing.T) { ctx := t.Context() rng := utils.TestRng() @@ -521,8 +360,9 @@ func TestRunPersistSeedsFromRecoveryFloor(t *testing.T) { require.NoError(t, db1.Close()) db2 := newTestBlockDB(t, dir) - tips := db2.Status() - require.Zero(t, tips.NextBlock) + tips := db2.Status().OrPanic("non-empty BlockDB status") + require.Equal(t, gr2.First, tips.First) + require.Equal(t, tips.First, tips.NextBlock) require.NotZero(t, tips.NextQC) state := newTestState(t, &Config{Registry: registry}, db2) @@ -552,10 +392,7 @@ func TestRunPersistSeedsFromRecoveryFloor(t *testing.T) { // TestRecoveryBlockGap verifies that a block gap can never enter BlockDB in the // first place: WriteBlock enforces contiguity, so skipping a covered number is -// rejected at write time. (A gap on disk can therefore only be corruption, which -// the ledger iterator reports as ErrBlockGap during replay — pinned by -// littblock's TestLittblockIteratorGapIsCorruption — and loadFromBlockDB -// propagates.) +// rejected at write time. func TestRecoveryBlockGap(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 3) diff --git a/sei-tendermint/internal/autobahn/data/state_test.go b/sei-tendermint/internal/autobahn/data/state_test.go index d1893ef689..30554a7abc 100644 --- a/sei-tendermint/internal/autobahn/data/state_test.go +++ b/sei-tendermint/internal/autobahn/data/state_test.go @@ -33,10 +33,7 @@ func newSnapshot() Snapshot { func snapshot(s *State) Snapshot { for inner := range s.inner.Lock() { - aps := map[types.GlobalBlockNumber]*types.AppProposal{} - for n, ap := range inner.appProposals { - aps[n] = ap - } + aps := maps.Clone(inner.appProposals) return Snapshot{ QCs: maps.Clone(inner.qcs), Blocks: maps.Clone(inner.blocks), @@ -81,26 +78,39 @@ func writeToBlockDB(t *testing.T, db types.BlockDB, qcs []*types.FullCommitQC, b utils.OrPanic(db.Flush()) } +func writeAppDataToBlockDB(t testing.TB, rng utils.Rng, db types.BlockDB, keys []types.SecretKey, qcs ...*types.FullCommitQC) { + t.Helper() + for _, qc := range qcs { + appProposal := types.NewAppProposal(qc.QC().Proposal(), types.GenAppHash(rng)) + utils.OrPanic(db.WriteAppProposal(appProposal)) + utils.OrPanic(db.WriteAppQC(TestAppQC(keys, appProposal))) + } + utils.OrPanic(db.Flush()) +} + // pushAppHashesRunning runs state.Run under scope.Run long enough to accept // PushAppHash for [first, next), then cancels Run. Prefers scope.Run over a // raw goroutine so cleanup is structured. func pushAppHashesRunning(ctx context.Context, state *State, rng utils.Rng, first, next types.GlobalBlockNumber) error { return scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { - runCtx, cancel := context.WithCancel(ctx) - s.SpawnBgNamed("state.Run", func() error { - return utils.IgnoreCancel(state.Run(runCtx)) - }) + s.SpawnBgNamed("state.Run", func() error { return utils.IgnoreCancel(state.Run(ctx)) }) for n := first; n < next; n++ { if err := state.PushAppHash(ctx, n, types.GenAppHash(rng)); err != nil { - cancel() return err } } - cancel() return nil }) } +func pushAppQCForBlock(ctx context.Context, state *State, keys []types.SecretKey, n types.GlobalBlockNumber) error { + vote, err := state.AppVote(ctx, n) + if err != nil { + return err + } + return state.PushAppQC(ctx, TestAppQC(keys, vote.Proposal())) +} + func TestState(t *testing.T) { ctx := t.Context() rng := utils.TestRng() @@ -147,11 +157,10 @@ func TestState(t *testing.T) { } wantG := &types.GlobalBlock{ - GlobalNumber: n, - Timestamp: want.QCs[n].QC().Proposal().BlockTimestamp(n).OrPanic("global block not in QC"), - Header: wantB.Header(), - Payload: wantB.Payload(), - FinalAppState: want.QCs[n].QC().Proposal().App(), + GlobalNumber: n, + Timestamp: want.QCs[n].QC().Proposal().BlockTimestamp(n).OrPanic("global block not in QC"), + Header: wantB.Header(), + Payload: wantB.Payload(), } gotG, err := state.GlobalBlock(ctx, n) if err != nil { @@ -235,7 +244,6 @@ func TestPushConflictingBadCommitQC(t *testing.T) { viewSpec, time.Now(), laneQCs, - utils.None[*types.AppQC](), )) malGR := proposal.Proposal().Msg().GlobalRange() require.Less(t, malGR.First, gr1.Next, "test setup: malicious gr.First must be < nextQC") @@ -348,15 +356,12 @@ func TestExecution(t *testing.T) { shortCtx, cancel := context.WithTimeout(ctx, 10*time.Millisecond) if err := state.PushAppHash(shortCtx, gr.Next, types.GenAppHash(rng)); err == nil { cancel() - return errors.New("PushAppProposal expected to fail on non-finalized blocks") + return errors.New("PushAppHash expected to fail on non-finalized blocks") } cancel() for n := gr.First; n < gr.Next; n += 1 { if err := state.PushAppHash(ctx, n, types.GenAppHash(rng)); err != nil { - return fmt.Errorf("state.PushAppProposal(): %w", err) - } - if err := state.PushAppHash(ctx, n, types.GenAppHash(rng)); err == nil { - return errors.New("PushAppProposal expected to fail on duplicate proposal") + return fmt.Errorf("state.PushAppHash(): %w", err) } } } @@ -366,6 +371,57 @@ func TestExecution(t *testing.T) { } } +func TestPushAppHashRejectsJumpOverCommitQCRange(t *testing.T) { + ctx := t.Context() + rng := utils.TestRng() + registry, keys := epoch.GenRegistry(rng, 3) + + state := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) + require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { + s.SpawnBgNamed("state.Run()", func() error { return utils.IgnoreCancel(state.Run(ctx)) }) + epoch := registry.LatestEpoch() + var qcs []*types.CommitQC + for range 3 { + var prev utils.Option[*types.CommitQC] + if len(qcs) > 0 { + prev = utils.Some(qcs[len(qcs)-1]) + } + qc, blocks := TestCommitQC(rng, epoch, keys, prev) + if err := state.PushQC(ctx, qc, blocks); err != nil { + return fmt.Errorf("PushQC(): %w", err) + } + qcs = append(qcs, qc.QC()) + } + if err := state.PushAppHash(ctx, qcs[0].GlobalRange().Next-1, types.GenAppHash(rng)); err != nil { + return fmt.Errorf("PushAppHash(qc1): %w", err) + } + if qcs[2].GlobalRange().Len() < 2 { + panic("qcs[2].Len() is too small for this test") + } + if err := state.PushAppHash(ctx, qcs[2].GlobalRange().Next-2, types.GenAppHash(rng)); !errors.Is(err, ErrOutOfOrder) { + return fmt.Errorf("PushAppHash(qc3 before qc2) error = %w, want %w", err, ErrOutOfOrder) + } + if err := state.PushAppHash(ctx, qcs[2].GlobalRange().Next-1, types.GenAppHash(rng)); !errors.Is(err, ErrOutOfOrder) { + return fmt.Errorf("PushAppHash(qc3 before qc2) error = %w, want %w", err, ErrOutOfOrder) + } + + if err := state.PushAppHash(ctx, qcs[1].GlobalRange().Next-1, types.GenAppHash(rng)); err != nil { + return fmt.Errorf("PushAppHash(qc2): %w", err) + } + if err := state.PushAppHash(ctx, qcs[2].GlobalRange().Next-1, types.GenAppHash(rng)); err != nil { + return fmt.Errorf("PushAppHash(qc3): %w", err) + } + // Inserting old stuff should be a noop. + if err := state.PushAppHash(ctx, qcs[1].GlobalRange().Next-1, types.GenAppHash(rng)); err != nil { + return fmt.Errorf("PushAppHash(qc2): %w", err) + } + if err := state.PushAppHash(ctx, qcs[2].GlobalRange().Next-2, types.GenAppHash(rng)); err != nil { + return fmt.Errorf("PushAppHash(qc2): %w", err) + } + return nil + })) +} + func TestPushBlockAcceptsBlockWithQC(t *testing.T) { ctx := t.Context() rng := utils.TestRng() @@ -440,27 +496,20 @@ func TestPushQCBeforeRunPersistsToBlockDB(t *testing.T) { // Transport-race window: PushQC before data.Run / runPersist starts. require.NoError(t, state.PushQC(ctx, qc1, blocks1)) - tips := db.Status() - require.Zero(t, tips.NextBlock, "PushQC must not write BlockDB before Run") - require.Zero(t, tips.NextQC) + require.False(t, db.Status().IsPresent(), "PushQC must not write BlockDB before Run") require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { - runCtx, cancel := context.WithCancel(ctx) - s.SpawnBgNamed("state.Run", func() error { - return utils.IgnoreCancel(state.Run(runCtx)) - }) - // PushAppHash waits on nextBlockToPersist, so success implies Flush. + s.SpawnBgNamed("state.Run", func() error { return utils.IgnoreCancel(state.Run(ctx)) }) + // PushAppHash waits on persisted.NextBlock, so success implies Flush. for n := gr1.First; n < gr1.Next; n++ { if err := state.PushAppHash(ctx, n, types.GenAppHash(rng)); err != nil { - cancel() return fmt.Errorf("PushAppHash(%d): %w", n, err) } } - cancel() return nil })) - tips = db.Status() + tips := db.Status().OrPanic("non-empty BlockDB status") require.Equal(t, gr1.Next, tips.NextBlock) require.Equal(t, gr1.Next, tips.NextQC) @@ -475,79 +524,170 @@ func TestPushQCBeforeRunPersistsToBlockDB(t *testing.T) { } } -// TestEvictionWaitsForCommitQCApp checks that evictBelowBound does not drop -// AppProposals until a later CommitQC embeds an App (certifying AppQC), and -// that once that App exists, heights below min(NAP, App+1) are evicted. -func TestEvictionWaitsForCommitQCApp(t *testing.T) { +// TestEvictionWaitsForAppQC checks that setPersisted does not drop +// AppProposals until AppQC is persisted, and that once it is, heights below +// persisted.First are evicted. +func TestEvictionWaitsForAppQC(t *testing.T) { ctx := t.Context() rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 3) qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) gr1 := qc1.QC().GlobalRange() - require.False(t, qc1.QC().Proposal().App().IsPresent(), "genesis CommitQC has no App") - qc2, blocks2 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc1.QC())) - app, ok := qc2.QC().Proposal().App().Get() - require.True(t, ok, "second CommitQC embeds App for qc1 tip") - appFloor := app.GlobalNumber() - require.Equal(t, gr1.Next-1, appFloor) gr2 := qc2.QC().GlobalRange() state := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { - runCtx, cancel := context.WithCancel(ctx) - defer cancel() - s.SpawnBgNamed("state.Run", func() error { - return utils.IgnoreCancel(state.Run(runCtx)) - }) + s.SpawnBgNamed("state.Run", func() error { return utils.IgnoreCancel(state.Run(ctx)) }) - require.NoError(t, state.PushQC(ctx, qc1, blocks1)) + if err := state.PushQC(ctx, qc1, blocks1); err != nil { + return fmt.Errorf("PushQC(qc1): %w", err) + } for n := gr1.First; n < gr1.Next; n++ { if err := state.PushAppHash(ctx, n, types.GenAppHash(rng)); err != nil { - return err + return fmt.Errorf("PushAppHash(%d): %w", n, err) } } - // No CommitQC.App yet → eviction must not strip AppProposals; first stays put. + // No AppQC yet -> eviction must not strip AppProposals; first stays put. for inner := range state.inner.Lock() { - require.Equal(t, gr1.First, inner.first, "no certified App → first unchanged") + if inner.first != gr1.First { + return fmt.Errorf("no certified App: first = %d, want %d", inner.first, gr1.First) + } for n := gr1.First; n < gr1.Next; n++ { _, ok := inner.appProposals[n] - require.True(t, ok, "AppProposal %d must survive without CommitQC.App", n) + if !ok { + return fmt.Errorf("AppProposal %d missing before AppQC", n) + } } } - require.NoError(t, state.PushQC(ctx, qc2, blocks2)) + if err := pushAppQCForBlock(ctx, state, keys, gr1.First); err != nil { + return fmt.Errorf("pushAppQCForBlock(%d): %w", gr1.First, err) + } + if _, err := state.Anchor().Wait(ctx, func(anchor utils.Option[Anchor]) bool { + if anchor, ok := anchor.Get(); ok { + return anchor.AppQC.Proposal().RoadIndex() >= qc1.Index() + } + return false + }); err != nil { + return fmt.Errorf("state.Anchor.Wait(): %w", err) + } + + if err := state.PushQC(ctx, qc2, blocks2); err != nil { + return fmt.Errorf("PushQC(qc2): %w", err) + } for n := gr2.First; n < gr2.Next; n++ { if err := state.PushAppHash(ctx, n, types.GenAppHash(rng)); err != nil { - return err + return fmt.Errorf("PushAppHash(%d): %w", n, err) } } for inner := range state.inner.Lock() { - require.Equal(t, appFloor+1, inner.first, "after catching up, first reaches App+1") + evictionBound := inner.persisted.First + if inner.first != evictionBound { + return fmt.Errorf("after catching up, first = %d, want eviction bound %d", inner.first, evictionBound) + } + if anchor, ok := inner.anchor.Load().Get(); !ok || anchor.AppQC != inner.appQCs[inner.first] || anchor.CommitQC != inner.qcs[inner.first].QC() { + return fmt.Errorf("anchor must cover inner.first %d", inner.first) + } for n := gr1.First; n < inner.first; n++ { _, ok := inner.appProposals[n] - require.False(t, ok, "AppProposal %d should be evicted (< first)", n) + if ok { + return fmt.Errorf("AppProposal %d present below first %d", n, inner.first) + } } // Heights at/above exclusive floor stay until executed further. for n := inner.first; n < inner.nextAppProposal; n++ { _, ok := inner.appProposals[n] - require.True(t, ok, "AppProposal %d must remain (>= first)", n) + if !ok { + return fmt.Errorf("AppProposal %d missing at/above first %d", n, inner.first) + } } // Tip QC (nextQC-1) stays; nextToExecute uses maps at/above first. - require.GreaterOrEqual(t, inner.nextQC-1, inner.first) - _, ok = inner.qcs[inner.nextQC-1] - require.True(t, ok, "tip QC must stay in maps") + if inner.nextQC-1 < inner.first { + return fmt.Errorf("tip QC height %d below first %d", inner.nextQC-1, inner.first) + } + _, ok := inner.qcs[inner.nextQC-1] + if !ok { + return fmt.Errorf("tip QC %d missing from maps", inner.nextQC-1) + } + } + return nil + })) +} + +func TestEvictionWaitsForPersistedAppQC(t *testing.T) { + ctx := t.Context() + rng := utils.TestRng() + registry, keys := epoch.GenRegistry(rng, 3) + + qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) + gr1 := qc1.QC().GlobalRange() + + state := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) + require.NoError(t, state.PushQC(ctx, qc1, blocks1)) + require.NoError(t, pushAppHashesRunning(ctx, state, rng, gr1.First, gr1.Next)) + require.NoError(t, pushAppQCForBlock(ctx, state, keys, gr1.First)) + + for inner := range state.inner.Lock() { + require.Equal(t, gr1.Next, inner.nextAppQC) + require.Equal(t, gr1.First, inner.persisted.NextAppQC) + require.Equal(t, gr1.First, inner.first, "accepted but unpersisted AppQC must not advance eviction") + for n := gr1.First; n < gr1.Next; n++ { + _, ok := inner.appProposals[n] + require.True(t, ok, "AppProposal %d must survive until AppQC is persisted", n) + } + } +} + +func TestPushAppHashBelowAnchorSucceeds(t *testing.T) { + ctx := t.Context() + rng := utils.TestRng() + registry, keys := epoch.GenRegistry(rng, 3) + epoch := registry.LatestEpoch() + + require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { + state := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) + s.SpawnBgNamed("state.Run", func() error { return utils.IgnoreCancel(state.Run(ctx)) }) + + prev := utils.None[*types.CommitQC]() + for range 2 { + qc, blocks := TestCommitQC(rng, epoch, keys, prev) + prev = utils.Some(qc.QC()) + gr := qc.QC().GlobalRange() + if err := state.PushQC(ctx, qc, blocks); err != nil { + return fmt.Errorf("PushQC: %w", err) + } + if err := state.PushAppHash(ctx, gr.Next-1, types.GenAppHash(rng)); err != nil { + return fmt.Errorf("PushAppHash(tip): %w", err) + } + if err := pushAppQCForBlock(ctx, state, keys, gr.First); err != nil { + return fmt.Errorf("pushAppQCForBlock(%d): %w", gr.First, err) + } + } + // Wait for anchor to progress past first block. + if _, err := state.Anchor().Wait(ctx, func(anchor utils.Option[Anchor]) bool { + if anchor, ok := anchor.Get(); ok { + return registry.FirstBlock() < anchor.AppQC.Proposal().GlobalRange().First + } + return false + }); err != nil { + return fmt.Errorf("state.Anchor.Wait(): %w", err) + } + // Pushing apphash for height below the anchor should NOT expolode. + if err := state.PushAppHash(ctx, registry.FirstBlock(), types.GenAppHash(rng)); err != nil { + return fmt.Errorf("PushAppHash below anchor: %w", err) } return nil })) } // TestNextToExecuteAfterAppEviction checks WaitUntilExecuted / nextToExecute -// still work when PushQC embeds an App that aggressively evicts through -// nextAppProposal (first = App+1 = NAP). nextToExecute uses qc[NAP], not NAP-1. +// still work when persisted AppQC aggressively evicts through nextAppProposal +// (first = persisted.First). +// nextToExecute uses the retained boundary QC. func TestNextToExecuteAfterAppEviction(t *testing.T) { ctx := t.Context() rng := utils.TestRng() @@ -556,45 +696,64 @@ func TestNextToExecuteAfterAppEviction(t *testing.T) { qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) gr1 := qc1.QC().GlobalRange() qc2, blocks2 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc1.QC())) - app, ok := qc2.QC().Proposal().App().Get() - require.True(t, ok) - require.Equal(t, gr1.Next-1, app.GlobalNumber()) state := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { - runCtx, cancel := context.WithCancel(ctx) - defer cancel() - s.SpawnBgNamed("state.Run", func() error { - return utils.IgnoreCancel(state.Run(runCtx)) - }) + s.SpawnBgNamed("state.Run", func() error { return utils.IgnoreCancel(state.Run(ctx)) }) - require.NoError(t, state.PushQC(ctx, qc1, blocks1)) + if err := state.PushQC(ctx, qc1, blocks1); err != nil { + return fmt.Errorf("PushQC(qc1): %w", err) + } for n := gr1.First; n < gr1.Next; n++ { if err := state.PushAppHash(ctx, n, types.GenAppHash(rng)); err != nil { - return err + return fmt.Errorf("PushAppHash(%d): %w", n, err) } } - // Sticky case: App floor == nextAppProposal. first advances to NAP; - // NAP-1 is gone; nextToExecute reads qc[NAP]. - require.NoError(t, state.PushQC(ctx, qc2, blocks2)) + // Sticky case: persisted.NextAppQC == nextAppProposal. first advances to + // NAP-1; NAP-2 is gone; nextToExecute reads qc[NAP-1] until the next QC executes. + if err := pushAppQCForBlock(ctx, state, keys, gr1.First); err != nil { + return fmt.Errorf("pushAppQCForBlock(%d): %w", gr1.First, err) + } + if _, err := state.Anchor().Wait(ctx, func(anchor utils.Option[Anchor]) bool { + if anchor, ok := anchor.Get(); ok { + return anchor.AppQC.Proposal().RoadIndex() >= qc1.Index() + } + return false + }); err != nil { + return fmt.Errorf("state.Anchor.Wait(): %w", err) + } + if err := state.PushQC(ctx, qc2, blocks2); err != nil { + return fmt.Errorf("PushQC(qc2): %w", err) + } var tipLane types.LaneID var tipBlockNum types.BlockNumber for inner := range state.inner.Lock() { - require.Equal(t, gr1.Next, inner.nextAppProposal) - require.Equal(t, app.GlobalNumber()+1, inner.first, - "eviction advances to App+1 == NAP") - _, ok := inner.blocks[inner.nextAppProposal-1] - require.False(t, ok, "NAP-1 must be evicted") - require.Less(t, inner.nextAppProposal, inner.nextQC) + if inner.nextAppProposal != gr1.Next { + return fmt.Errorf("nextAppProposal = %d, want %d", inner.nextAppProposal, gr1.Next) + } + evictionBound := inner.persisted.First + if inner.first != evictionBound { + return fmt.Errorf("first = %d, want eviction bound %d", inner.first, evictionBound) + } + _, ok := inner.blocks[evictionBound-1] + if ok { + return fmt.Errorf("block %d present, want evicted", inner.nextAppProposal-1) + } + if inner.nextAppProposal >= inner.nextQC { + return fmt.Errorf("nextAppProposal = %d, want < nextQC %d", inner.nextAppProposal, inner.nextQC) + } fqc := inner.qcs[inner.nextAppProposal] - require.NotNil(t, fqc) + if fqc == nil { + return fmt.Errorf("QC %d missing", inner.nextAppProposal) + } gr := fqc.QC().GlobalRange() h := fqc.Headers()[inner.nextAppProposal-gr.First] tipLane = h.Lane() tipBlockNum = h.BlockNumber() - require.Equal(t, tipBlockNum, inner.nextToExecute(tipLane), - "nextToExecute should be the next block's lane number") + if got := inner.nextToExecute(tipLane); got != tipBlockNum { + return fmt.Errorf("nextToExecute(%d) = %d, want %d", tipLane, got, tipBlockNum) + } } // WaitUntilExecuted(n) returns when nextToExecute > n. waitFrom := tipBlockNum @@ -602,16 +761,78 @@ func TestNextToExecuteAfterAppEviction(t *testing.T) { waitFrom-- } next, err := state.WaitUntilExecuted(ctx, tipLane, waitFrom) - require.NoError(t, err) - require.Equal(t, tipBlockNum, next) + if err != nil { + return fmt.Errorf("WaitUntilExecuted(%d, %d): %w", tipLane, waitFrom, err) + } + if next != tipBlockNum { + return fmt.Errorf("WaitUntilExecuted(%d, %d) = %d, want %d", tipLane, waitFrom, next, tipBlockNum) + } return nil })) } +func TestPushAppQCPersistsAndRecovers(t *testing.T) { + ctx := t.Context() + rng := utils.TestRng() + registry, keys := epoch.GenRegistry(rng, 3) + dir := t.TempDir() + + qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) + gr1 := qc1.QC().GlobalRange() + + db1 := newTestBlockDB(t, dir) + state1 := newTestState(t, &Config{Registry: registry}, db1) + require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { + s.SpawnBgNamed("state.Run", func() error { return utils.IgnoreCancel(state1.Run(ctx)) }) + + if err := state1.PushQC(ctx, qc1, blocks1); err != nil { + return fmt.Errorf("PushQC(qc1): %w", err) + } + for n := gr1.First; n < gr1.Next; n++ { + if err := state1.PushAppHash(ctx, n, types.GenAppHash(rng)); err != nil { + return fmt.Errorf("PushAppHash(%d): %w", n, err) + } + } + if err := pushAppQCForBlock(ctx, state1, keys, gr1.First); err != nil { + return fmt.Errorf("pushAppQCForBlock(%d): %w", gr1.First, err) + } + if _, err := state1.Anchor().Wait(ctx, func(anchor utils.Option[Anchor]) bool { + if anchor, ok := anchor.Get(); ok { + return anchor.AppQC.Proposal().RoadIndex() >= qc1.Index() + } + return false + }); err != nil { + return fmt.Errorf("state.Anchor.Wait(): %w", err) + } + return nil + })) + + storedProposal, err := db1.ReadAppProposalByBlockNumber(gr1.First) + require.NoError(t, err) + require.True(t, storedProposal.IsPresent(), "PushAppHash must persist the AppProposal") + stored, err := db1.ReadAppQCByBlockNumber(gr1.First) + require.NoError(t, err) + require.True(t, stored.IsPresent(), "PushAppQC must persist the AppQC") + require.NoError(t, db1.Close()) + + db2 := newTestBlockDB(t, dir) + state2 := newTestState(t, &Config{Registry: registry}, db2) + for inner := range state2.inner.Lock() { + require.Equal(t, gr1.Next, inner.nextAppProposal) + require.Equal(t, gr1.Next, inner.nextAppQC) + } + appQC, err := state2.AppQC(ctx, gr1.First) + require.NoError(t, err) + fQC, err := state2.QC(ctx, gr1.First) + require.NoError(t, err) + require.Equal(t, gr1, appQC.Proposal().GlobalRange()) + require.Equal(t, gr1, fQC.QC().GlobalRange()) + require.NoError(t, db2.Close()) +} + // TestPruningKeepsLastQCRange verifies BlockDB's never-empty prune: asking to -// prune past the tip still leaves the newest cohort readable. A QC retaining -// only a suffix of its blocks recovers with the floor on that suffix; a -// consistent range recovers from the QC start. +// prune past the tip still leaves the newest cohort readable, and a consistent +// range recovers from the QC start. func TestPruningKeepsLastQCRange(t *testing.T) { ctx := t.Context() rng := utils.TestRng() @@ -632,21 +853,6 @@ func TestPruningKeepsLastQCRange(t *testing.T) { require.NotNil(t, got) } - // A QC covering a range with only its last block present: the first block is free to - // start inside its covering QC, so iteration opens there and the recovery floor follows. - survivor := gr1.Next - 1 - dirSuffix := t.TempDir() - dbSuffix := newTestBlockDB(t, dirSuffix) - require.NoError(t, dbSuffix.WriteQC(qc1)) - require.NoError(t, dbSuffix.WriteBlock(survivor, blocks1[survivor-gr1.First])) - require.NoError(t, dbSuffix.Flush()) - require.NoError(t, dbSuffix.Close()) - suffixState := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, dirSuffix)) - require.Equal(t, gr1.Next, suffixState.NextBlock()) - for inner := range suffixState.inner.Lock() { - require.Equal(t, survivor, inner.first, "floor must be the surviving block") - } - // Consistent post-GC shape: full QC range of blocks. Restart recovers at QC start. dir := t.TempDir() db := newTestBlockDB(t, dir) @@ -669,8 +875,8 @@ func TestPruningKeepsLastQCRange(t *testing.T) { // readability), so a mid-range prune does not refuse heights inside that QC. // // PruneBefore is BlockDB-only: heights still retained in RAM for AppVotes -// (at/above CommitQC.App+1 exclusive floor) remain readable via TryBlock even -// after the store watermark advances past them. +// (at/above persisted.First) remain +// readable via TryBlock even after the store watermark advances past them. func TestPruningWithPartialQCRange(t *testing.T) { ctx := t.Context() rng := utils.TestRng() @@ -680,16 +886,32 @@ func TestPruningWithPartialQCRange(t *testing.T) { qc2, blocks2 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc1.QC())) gr1 := qc1.QC().GlobalRange() gr2 := qc2.QC().GlobalRange() - app, ok := qc2.QC().Proposal().App().Get() - require.True(t, ok) - appFloor := app.GlobalNumber() - exclusiveFloor := appFloor + 1 state1 := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) require.NoError(t, state1.PushQC(ctx, qc1, blocks1)) require.NoError(t, state1.PushQC(ctx, qc2, blocks2)) require.NoError(t, pushAppHashesRunning(ctx, state1, rng, gr1.First, gr2.Next)) + var exclusiveFloor types.GlobalBlockNumber + require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { + s.SpawnBgNamed("state.Run", func() error { return utils.IgnoreCancel(state1.Run(ctx)) }) + if err := pushAppQCForBlock(ctx, state1, keys, gr1.First); err != nil { + return err + } + if _, err := state1.Anchor().Wait(ctx, func(anchor utils.Option[Anchor]) bool { + if anchor, ok := anchor.Get(); ok { + return anchor.AppQC.Proposal().RoadIndex() >= qc1.Index() + } + return false + }); err != nil { + return fmt.Errorf("state.Anchor.Wait(): %w", err) + } + return nil + })) + for inner := range state1.inner.Lock() { + exclusiveFloor = inner.persisted.First + require.Equal(t, exclusiveFloor, inner.first) + } // Mid-QC prune clamps to gr1.First, so the whole qc1 cohort stays readable. midQC1 := gr1.First + (gr1.Next-gr1.First)/2 @@ -702,12 +924,15 @@ func TestPruningWithPartialQCRange(t *testing.T) { } } - // Prune past qc1 entirely; BlockDB never-empty keeps the newest cohort (qc2). + // Prune past qc1 entirely. Because qc1 now has a persisted AppQC, BlockDB's + // never-empty rule keeps that newest AppQC+CommitQC+Block cohort readable. require.NoError(t, state1.PruneBefore(gr2.Next)) - // Evicted heights (< exclusive App floor) fall through to BlockDB → ErrPruned. + // Evicted heights (< exclusive App floor) fall through to BlockDB, but the + // persisted AppQC cohort is retained by the prune cap. for n := gr1.First; n < exclusiveFloor; n++ { - _, err := state1.TryBlock(n) - require.ErrorIs(t, err, types.ErrPruned) + got, err := state1.TryBlock(n) + require.NoError(t, err) + require.NotNil(t, got) } // Exclusive floor and above stay cached for AppVotes despite BlockDB prune. // ByHash must match TryBlock here — not fall through to a pruned BlockDB. @@ -722,20 +947,6 @@ func TestPruningWithPartialQCRange(t *testing.T) { require.Equal(t, n, gb.GlobalNumber) } - // A lone qc2 suffix: iteration opens on the surviving block and the floor follows. - survivor := gr2.Next - 1 - dirSuffix := t.TempDir() - dbSuffix := newTestBlockDB(t, dirSuffix) - require.NoError(t, dbSuffix.WriteQC(qc2)) - require.NoError(t, dbSuffix.WriteBlock(survivor, blocks2[survivor-gr2.First])) - require.NoError(t, dbSuffix.Flush()) - require.NoError(t, dbSuffix.Close()) - suffixState := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, dirSuffix)) - require.Equal(t, gr2.Next, suffixState.NextBlock()) - for inner := range suffixState.inner.Lock() { - require.Equal(t, survivor, inner.first, "floor must be the surviving block") - } - // Consistent retained range: full qc2. dir := t.TempDir() db := newTestBlockDB(t, dir) diff --git a/sei-tendermint/internal/autobahn/data/testonly.go b/sei-tendermint/internal/autobahn/data/testonly.go index 2595b3a25a..29f2fff5b9 100644 --- a/sei-tendermint/internal/autobahn/data/testonly.go +++ b/sei-tendermint/internal/autobahn/data/testonly.go @@ -1,10 +1,6 @@ package data import ( - "context" - "errors" - "fmt" - "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" ) @@ -58,88 +54,6 @@ func TestCommitQC( } } } - var appQC utils.Option[*types.AppQC] - if cqc, ok := prev.Get(); ok { - vs := types.ViewSpec{CommitQC: prev, Epoch: ep} - p := types.NewAppProposal(cqc.GlobalRange().Next-1, vs.View().Index, types.GenAppHash(rng), ep.EpochIndex()) - appQC = utils.Some(TestAppQC(keys, p)) - } - cqc := types.BuildCommitQC(ep, keys, prev, laneQCs, appQC) + cqc := types.BuildCommitQC(ep, keys, prev, laneQCs) return types.NewFullCommitQC(cqc, headers), blockList } - -var _ StateAPI = (*MockState)(nil) - -type innerMockState struct { - blocks map[types.GlobalBlockNumber]*types.GlobalBlock // [first,next) - first types.GlobalBlockNumber - next types.GlobalBlockNumber -} - -// MockState is a mock implementation of the StateAPI interface. -// Allows for pushing global blocks directly (without going through consensus). -type MockState struct { - capacity uint64 - inner utils.Watch[*innerMockState] -} - -// NewMockState creates a new MockState with the given block capacity. -func NewMockState(capacity uint64) *MockState { - return &MockState{ - capacity: capacity, - inner: utils.NewWatch(&innerMockState{ - blocks: make(map[types.GlobalBlockNumber]*types.GlobalBlock), - next: 0, - }), - } -} - -// GlobalBlock returns the global block with the given number. -func (s *MockState) GlobalBlock(ctx context.Context, n types.GlobalBlockNumber) (*types.GlobalBlock, error) { - for inner, ctrl := range s.inner.Lock() { - if err := ctrl.WaitUntil(ctx, func() bool { return inner.next > n }); err != nil { - return nil, err - } - if inner.first > n { - return nil, types.ErrPruned - } - return inner.blocks[n], nil - } - panic("unreachable") -} - -// ProduceBlock appends a new global block with the given payload. -func (s *MockState) ProduceBlock(ctx context.Context, payload *types.Payload) error { - for inner, ctrl := range s.inner.Lock() { - if err := ctrl.WaitUntil(ctx, func() bool { - return uint64(inner.next-inner.first) < s.capacity - }); err != nil { - return err - } - inner.blocks[inner.next] = &types.GlobalBlock{ - GlobalNumber: inner.next, - Payload: payload, - } - inner.next += 1 - ctrl.Updated() - } - return nil -} - -// PushAppHash marks all blocks up to n as executed. -func (s *MockState) PushAppHash(_ context.Context, n types.GlobalBlockNumber, appHash types.AppHash) error { - for inner, ctrl := range s.inner.Lock() { - if got, wantMin := n, inner.first; got < wantMin { - return fmt.Errorf("received app proposal out of order: got %v, want >= %v", got, wantMin) - } - if n >= inner.next { - return errors.New("proposal for block which hasn't been received yet") - } - for inner.first <= n { - delete(inner.blocks, inner.first) - inner.first += 1 - } - ctrl.Updated() - } - return nil -} diff --git a/sei-tendermint/internal/autobahn/pb/autobahn.pb.go b/sei-tendermint/internal/autobahn/pb/autobahn.pb.go index 1348b18d6b..490c46f968 100644 --- a/sei-tendermint/internal/autobahn/pb/autobahn.pb.go +++ b/sei-tendermint/internal/autobahn/pb/autobahn.pb.go @@ -1493,14 +1493,16 @@ func (x *AppQC) GetSigs() []*Signature { type AppProposal struct { state protoimpl.MessageState `protogen:"open.v1"` - // Global block number. - GlobalNumber *uint64 `protobuf:"varint,1,opt,name=global_number,json=globalNumber,proto3,oneof" json:"global_number,omitempty"` // required + // Epoch this proposal belongs to. + EpochIndex *uint64 `protobuf:"varint,4,opt,name=epoch_index,json=epochIndex,proto3,oneof" json:"epoch_index,omitempty"` // required // Index of the commit qc finalizing the block. RoadIndex *uint64 `protobuf:"varint,2,opt,name=road_index,json=roadIndex,proto3,oneof" json:"road_index,omitempty"` // required - // App hash at that block. - AppHash []byte `protobuf:"bytes,3,opt,name=app_hash,json=appHash,proto3,oneof" json:"app_hash,omitempty"` // required - // Epoch this proposal belongs to. - EpochIndex *uint64 `protobuf:"varint,4,opt,name=epoch_index,json=epochIndex,proto3,oneof" json:"epoch_index,omitempty"` // required + // Range of global blocks convered by this proposal. + // Has to match the corresponding CommitQC. + GlobalFirst *uint64 `protobuf:"varint,5,opt,name=global_first,json=globalFirst,proto3,oneof" json:"global_first,omitempty"` // required + GlobalNext *uint64 `protobuf:"varint,6,opt,name=global_next,json=globalNext,proto3,oneof" json:"global_next,omitempty"` // required + // App hash of the state at the end of the range above. + AppHash []byte `protobuf:"bytes,3,opt,name=app_hash,json=appHash,proto3,oneof" json:"app_hash,omitempty"` // required unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1535,9 +1537,9 @@ func (*AppProposal) Descriptor() ([]byte, []int) { return file_autobahn_autobahn_proto_rawDescGZIP(), []int{25} } -func (x *AppProposal) GetGlobalNumber() uint64 { - if x != nil && x.GlobalNumber != nil { - return *x.GlobalNumber +func (x *AppProposal) GetEpochIndex() uint64 { + if x != nil && x.EpochIndex != nil { + return *x.EpochIndex } return 0 } @@ -1549,20 +1551,27 @@ func (x *AppProposal) GetRoadIndex() uint64 { return 0 } -func (x *AppProposal) GetAppHash() []byte { - if x != nil { - return x.AppHash +func (x *AppProposal) GetGlobalFirst() uint64 { + if x != nil && x.GlobalFirst != nil { + return *x.GlobalFirst } - return nil + return 0 } -func (x *AppProposal) GetEpochIndex() uint64 { - if x != nil && x.EpochIndex != nil { - return *x.EpochIndex +func (x *AppProposal) GetGlobalNext() uint64 { + if x != nil && x.GlobalNext != nil { + return *x.GlobalNext } return 0 } +func (x *AppProposal) GetAppHash() []byte { + if x != nil { + return x.AppHash + } + return nil +} + // This is the signable message. // To sign ConsensusMsg/BlockMsg, you need to embed it in Msg first. type Msg struct { @@ -2339,18 +2348,21 @@ const file_autobahn_autobahn_proto_rawDesc = "" + "_commit_qc\"k\n" + "\x05AppQC\x12)\n" + "\x04vote\x18\x01 \x01(\v2\x15.autobahn.AppProposalR\x04vote\x12/\n" + - "\x04sigs\x18\x02 \x03(\v2\x13.autobahn.SignatureB\x06Ј\xe2\xab\fdR\x04sigs:\x06\xe8\x88\xe2\xab\f\x01\"\xf5\x01\n" + - "\vAppProposal\x12(\n" + - "\rglobal_number\x18\x01 \x01(\x04H\x00R\fglobalNumber\x88\x01\x01\x12\"\n" + + "\x04sigs\x18\x02 \x03(\v2\x13.autobahn.SignatureB\x06Ј\xe2\xab\fdR\x04sigs:\x06\xe8\x88\xe2\xab\f\x01\"\xbd\x02\n" + + "\vAppProposal\x12$\n" + + "\vepoch_index\x18\x04 \x01(\x04H\x00R\n" + + "epochIndex\x88\x01\x01\x12\"\n" + "\n" + "road_index\x18\x02 \x01(\x04H\x01R\troadIndex\x88\x01\x01\x12&\n" + - "\bapp_hash\x18\x03 \x01(\fB\x06؈\xe2\xab\f H\x02R\aappHash\x88\x01\x01\x12$\n" + - "\vepoch_index\x18\x04 \x01(\x04H\x03R\n" + - "epochIndex\x88\x01\x01:\fȈ\xe2\xab\f\x01\xe8\x88\xe2\xab\f\x01B\x10\n" + - "\x0e_global_numberB\r\n" + - "\v_road_indexB\v\n" + - "\t_app_hashB\x0e\n" + - "\f_epoch_index\"\x98\x03\n" + + "\fglobal_first\x18\x05 \x01(\x04H\x02R\vglobalFirst\x88\x01\x01\x12$\n" + + "\vglobal_next\x18\x06 \x01(\x04H\x03R\n" + + "globalNext\x88\x01\x01\x12&\n" + + "\bapp_hash\x18\x03 \x01(\fB\x06؈\xe2\xab\f H\x04R\aappHash\x88\x01\x01:\fȈ\xe2\xab\f\x01\xe8\x88\xe2\xab\f\x01B\x0e\n" + + "\f_epoch_indexB\r\n" + + "\v_road_indexB\x0f\n" + + "\r_global_firstB\x0e\n" + + "\f_global_nextB\v\n" + + "\t_app_hashJ\x04\b\x01\x10\x02R\rglobal_number\"\x98\x03\n" + "\x03Msg\x126\n" + "\rlane_proposal\x18\x01 \x01(\v2\x0f.autobahn.BlockH\x00R\flaneProposal\x124\n" + "\tlane_vote\x18\x02 \x01(\v2\x15.autobahn.BlockHeaderH\x00R\blaneVote\x120\n" + diff --git a/sei-tendermint/internal/autobahn/pb/autobahn.wireguard.go b/sei-tendermint/internal/autobahn/pb/autobahn.wireguard.go index 99b476ba5d..55d8c736b0 100644 --- a/sei-tendermint/internal/autobahn/pb/autobahn.wireguard.go +++ b/sei-tendermint/internal/autobahn/pb/autobahn.wireguard.go @@ -44,23 +44,23 @@ func (*View) MaxSize() int { } func (*Proposal) MaxSize() int { - return 9539 + return 9550 } func (*FullProposal) MaxSize() int { - return 1107571 + return 1107604 } func (*PrepareQC) MaxSize() int { - return 19942 + return 19953 } func (*CommitQC) MaxSize() int { - return 19942 + return 19953 } func (*FullCommitQC) MaxSize() int { - return 136946 + return 136957 } func (*TimeoutVote) MaxSize() int { @@ -68,19 +68,19 @@ func (*TimeoutVote) MaxSize() int { } func (*TimeoutQC) MaxSize() int { - return 35446 + return 35457 } func (*FullTimeoutVote) MaxSize() int { - return 20101 + return 20112 } func (*AppQC) MaxSize() int { - return 10469 + return 10480 } func (*AppProposal) MaxSize() int { - return 67 + return 78 } func (*Msg) MaxSize() int { @@ -88,7 +88,7 @@ func (*Msg) MaxSize() int { } func (*SignedProposal) MaxSize() int { - return 9646 + return 9657 } func (*SignedTimeoutVote) MaxSize() int { @@ -96,7 +96,7 @@ func (*SignedTimeoutVote) MaxSize() int { } func (*SignedAppVote) MaxSize() int { - return 173 + return 184 } func (*SignedBlock) MaxSize() int { @@ -108,11 +108,11 @@ func (*SignedBlockHeader) MaxSize() int { } func (*SignedAppProposal) MaxSize() int { - return 173 + return 184 } func (*ConsensusReq) MaxSize() int { - return 1107575 + return 1107608 } func init() { @@ -287,10 +287,11 @@ func init() { // Register the wireguard.Schema generated for autobahn.AppProposal. runtime.MustRegister[*AppProposal](runtime.Schema{ - 1: {MaxCount: 1}, + 4: {MaxCount: 1}, 2: {MaxCount: 1}, + 5: {MaxCount: 1}, + 6: {MaxCount: 1}, 3: {MaxCount: 1, MaxSize: 32}, - 4: {MaxCount: 1}, }) // Register the wireguard.Schema generated for autobahn.Msg. diff --git a/sei-tendermint/internal/p2p/giga/api.go b/sei-tendermint/internal/p2p/giga/api.go index cc7637d71f..e8bfef757d 100644 --- a/sei-tendermint/internal/p2p/giga/api.go +++ b/sei-tendermint/internal/p2p/giga/api.go @@ -36,11 +36,6 @@ var StreamAppVotes = rpc.Register[API](4, "stream_app_votes", rpc.Msg[*pb.StreamAppVotesReq]{MsgSize: kB, Window: 1}, rpc.Msg[*pb.AppVote]{MsgSize: 10 * kB, Window: 100}, ) -var StreamAppQCs = rpc.Register[API](5, "stream_app_qcs", - rpc.Limit{Rate: 1, Concurrent: 1}, - rpc.Msg[*pb.StreamAppQCsReq]{MsgSize: kB, Window: 1}, - rpc.Msg[*pb.StreamAppQCsResp]{MsgSize: 30 * kB, Window: 20}, -) var Consensus = rpc.Register[API](6, "consensus", // Consensus streams are special in a sense that // * each stream sends just 1 message per view @@ -57,6 +52,11 @@ var StreamFullCommitQCs = rpc.Register[API](7, "stream_full_commit_qcs", rpc.Msg[*pb.StreamFullCommitQCsReq]{MsgSize: kB, Window: 1}, rpc.Msg[*apb.FullCommitQC]{MsgSize: 300 * kB, Window: 20}, ) +var StreamAppQCs = rpc.Register[API](5, "stream_app_qcs", + rpc.Limit{Rate: 1, Concurrent: 1}, + rpc.Msg[*pb.StreamAppQCsReq]{MsgSize: kB, Window: 1}, + rpc.Msg[*apb.AppQC]{MsgSize: 30 * kB, Window: 20}, +) var GetBlock = rpc.Register[API](8, "get_block", rpc.Limit{Rate: 10, Concurrent: 10}, rpc.Msg[*pb.GetBlockReq]{MsgSize: 10 * kB, Window: 1}, diff --git a/sei-tendermint/internal/p2p/giga/api.proto b/sei-tendermint/internal/p2p/giga/api.proto index 10ac870470..63ab4f1813 100644 --- a/sei-tendermint/internal/p2p/giga/api.proto +++ b/sei-tendermint/internal/p2p/giga/api.proto @@ -45,15 +45,6 @@ message StreamLaneProposalsReq { uint64 first_block_number = 1; } -message StreamAppQCsReq { - option (wireguard.sized) = true; -} -message StreamAppQCsResp { - option (wireguard.sized) = true; - autobahn.AppQC app_qc = 1; // required - autobahn.CommitQC commit_qc = 2; // required -} - message StreamCommitQCsReq { option (wireguard.sized) = true; } @@ -80,3 +71,8 @@ message StreamFullCommitQCsReq { option (wireguard.sized) = true; uint64 next_block = 1; } + +message StreamAppQCsReq { + option (wireguard.sized) = true; + uint64 next_block = 1; +} diff --git a/sei-tendermint/internal/p2p/giga/avail.go b/sei-tendermint/internal/p2p/giga/avail.go index b6a09ccdb3..8d4e12048f 100644 --- a/sei-tendermint/internal/p2p/giga/avail.go +++ b/sei-tendermint/internal/p2p/giga/avail.go @@ -11,7 +11,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p/rpc" ) -func (x *Service) serverStreamLaneProposals(ctx context.Context, server rpc.Server[API]) error { +func (x *validatorService) serverStreamLaneProposals(ctx context.Context, server rpc.Server[API]) error { return StreamLaneProposals.Serve(ctx, server, func(ctx context.Context, stream rpc.Stream[*pb.LaneProposal, *pb.StreamLaneProposalsReq]) error { reqRaw, err := stream.Recv(ctx) if err != nil { @@ -21,7 +21,7 @@ func (x *Service) serverStreamLaneProposals(ctx context.Context, server rpc.Serv if err != nil { return fmt.Errorf("StreamLaneProposalsReqConv.Decode(): %w", err) } - sub := x.validatorState().Avail().SubscribeLaneProposals(req.FirstBlockNumber) + sub := x.state.Avail().SubscribeLaneProposals(req.FirstBlockNumber) for { p, err := sub.Recv(ctx) if err != nil { @@ -34,14 +34,14 @@ func (x *Service) serverStreamLaneProposals(ctx context.Context, server rpc.Serv }) } -func (x *Service) serverStreamLaneVotes(ctx context.Context, server rpc.Server[API]) error { +func (x *validatorService) serverStreamLaneVotes(ctx context.Context, server rpc.Server[API]) error { return StreamLaneVotes.Serve(ctx, server, func(ctx context.Context, stream rpc.Stream[*pb.LaneVote, *pb.StreamLaneVotesReq]) error { reqRaw, err := stream.Recv(ctx) if err != nil { return err } _ = reqRaw - sub := x.validatorState().Avail().SubscribeLaneVotes() + sub := x.state.Avail().SubscribeLaneVotes() for { batch, err := sub.RecvBatch(ctx) if err != nil { @@ -56,14 +56,14 @@ func (x *Service) serverStreamLaneVotes(ctx context.Context, server rpc.Server[A }) } -func (x *Service) serverStreamAppVotes(ctx context.Context, server rpc.Server[API]) error { +func (x *validatorService) serverStreamAppVotes(ctx context.Context, server rpc.Server[API]) error { return StreamAppVotes.Serve(ctx, server, func(ctx context.Context, stream rpc.Stream[*pb.AppVote, *pb.StreamAppVotesReq]) error { reqRaw, err := stream.Recv(ctx) if err != nil { return err } _ = reqRaw - sub := x.validatorState().Avail().SubscribeAppVotes() + sub := x.state.Avail().SubscribeAppVotes() for { vote, err := sub.Recv(ctx) if err != nil { @@ -76,41 +76,17 @@ func (x *Service) serverStreamAppVotes(ctx context.Context, server rpc.Server[AP }) } -func (x *Service) serverStreamAppQCs(ctx context.Context, server rpc.Server[API]) error { - return StreamAppQCs.Serve(ctx, server, func(ctx context.Context, stream rpc.Stream[*pb.StreamAppQCsResp, *pb.StreamAppQCsReq]) error { - reqRaw, err := stream.Recv(ctx) - if err != nil { - return err - } - _ = reqRaw - next := types.RoadIndex(0) - for { - appQC, commitQC, err := x.validatorState().Avail().WaitForAppQC(ctx, next) - if err != nil { - return fmt.Errorf("x.validatorState().Avail().WaitForAppQC(): %w", err) - } - next = appQC.Next() - if err := stream.Send(ctx, StreamAppQCsRespConv.Encode(&StreamAppQCsResp{ - AppQC: appQC, - CommitQC: commitQC, - })); err != nil { - return fmt.Errorf("stream.Send(): %w", err) - } - } - }) -} - -func (x *Service) serverStreamCommitQCs(ctx context.Context, server rpc.Server[API]) error { +func (x *validatorService) serverStreamCommitQCs(ctx context.Context, server rpc.Server[API]) error { return StreamCommitQCs.Serve(ctx, server, func(ctx context.Context, stream rpc.Stream[*apb.CommitQC, *pb.StreamCommitQCsReq]) error { next := types.RoadIndex(0) for { - qc, err := x.validatorState().Avail().CommitQC(ctx, next) + qc, err := x.state.Avail().CommitQC(ctx, next) if err != nil { if errors.Is(err, types.ErrPruned) { - next = x.validatorState().Avail().FirstCommitQC() + next = x.state.Avail().First() continue } - return fmt.Errorf("x.validatorState().Avail().FirstCommitQC(): %w", err) + return fmt.Errorf("x.state.Avail().CommitQC(): %w", err) } next = qc.Index() + 1 if err := stream.Send(ctx, types.CommitQCConv.Encode(qc)); err != nil { @@ -120,7 +96,7 @@ func (x *Service) serverStreamCommitQCs(ctx context.Context, server rpc.Server[A }) } -func (x *Service) clientStreamLaneProposals(ctx context.Context, c rpc.Client[API]) error { +func (x *validatorService) clientStreamLaneProposals(ctx context.Context, c rpc.Client[API]) error { stream, err := StreamLaneProposals.Call(ctx, c) if err != nil { return err @@ -148,13 +124,13 @@ func (x *Service) clientStreamLaneProposals(ctx context.Context, c rpc.Client[AP /*if got, want := proposal.Msg().Block().Header().Lane(), c.cfg.GetKey(); got != want { return fmt.Errorf("producer = %q, want %q", got, want) }*/ - if err := x.validatorState().Avail().PushBlock(ctx, proposal); err != nil { + if err := x.state.Avail().PushBlock(ctx, proposal); err != nil { return fmt.Errorf("s.PushLaneProposal(): %w", err) } } } -func (x *Service) clientStreamLaneVotes(ctx context.Context, c rpc.Client[API]) error { +func (x *validatorService) clientStreamLaneVotes(ctx context.Context, c rpc.Client[API]) error { stream, err := StreamLaneVotes.Call(ctx, c) if err != nil { return fmt.Errorf("client.StreamLaneVotes(): %w", err) @@ -172,13 +148,13 @@ func (x *Service) clientStreamLaneVotes(ctx context.Context, c rpc.Client[API]) if err != nil { return fmt.Errorf("LaneVoteConv.Decode(): %w", err) } - if err := x.validatorState().Avail().PushVote(ctx, vote); err != nil { + if err := x.state.Avail().PushVote(ctx, vote); err != nil { return fmt.Errorf("s.PushLaneVote(): %w", err) } } } -func (x *Service) clientStreamCommitQCs(ctx context.Context, c rpc.Client[API]) error { +func (x *validatorService) clientStreamCommitQCs(ctx context.Context, c rpc.Client[API]) error { stream, err := StreamCommitQCs.Call(ctx, c) if err != nil { return fmt.Errorf("client.StreamCommitQCs(): %w", err) @@ -196,13 +172,13 @@ func (x *Service) clientStreamCommitQCs(ctx context.Context, c rpc.Client[API]) if err != nil { return fmt.Errorf("types.CommitQCConv.Decode(): %w", err) } - if err := x.validatorState().Avail().PushCommitQC(ctx, qc); err != nil { - return fmt.Errorf("s.PushFirstCommitQC(): %w", err) + if err := x.state.Avail().PushCommitQC(ctx, qc); err != nil { + return fmt.Errorf("s.PushCommitQC(): %w", err) } } } -func (x *Service) clientStreamAppVotes(ctx context.Context, c rpc.Client[API]) error { +func (x *validatorService) clientStreamAppVotes(ctx context.Context, c rpc.Client[API]) error { stream, err := StreamAppVotes.Call(ctx, c) if err != nil { return fmt.Errorf("client.StreamAppVotes(): %w", err) @@ -220,32 +196,8 @@ func (x *Service) clientStreamAppVotes(ctx context.Context, c rpc.Client[API]) e if err != nil { return fmt.Errorf("AppVoteConv.Decode(): %w", err) } - if err := x.validatorState().Avail().PushAppVote(ctx, vote); err != nil { + if err := x.state.Avail().PushAppVote(ctx, vote); err != nil { return fmt.Errorf("s.PushLaneVote(): %w", err) } } } - -func (x *Service) clientStreamAppQCs(ctx context.Context, c rpc.Client[API]) error { - stream, err := StreamAppQCs.Call(ctx, c) - if err != nil { - return fmt.Errorf("client.StreamAppQCs(): %w", err) - } - defer stream.Close() - if err := stream.Send(ctx, &pb.StreamAppQCsReq{}); err != nil { - return err - } - for { - resp, err := stream.Recv(ctx) - if err != nil { - return fmt.Errorf("stream.Recv(): %w", err) - } - msg, err := StreamAppQCsRespConv.Decode(resp) - if err != nil { - return fmt.Errorf("StreamAppQCsRespConv.Decode(): %w", err) - } - if err := x.validatorState().Avail().PushAppQC(msg.AppQC, msg.CommitQC); err != nil { - return fmt.Errorf("s.PushFirstCommitQC(): %w", err) - } - } -} diff --git a/sei-tendermint/internal/p2p/giga/consensus.go b/sei-tendermint/internal/p2p/giga/consensus.go index 6d5d2bace7..8d31dbb9c3 100644 --- a/sei-tendermint/internal/p2p/giga/consensus.go +++ b/sei-tendermint/internal/p2p/giga/consensus.go @@ -3,7 +3,6 @@ package giga import ( "context" "fmt" - "time" "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" apb "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/pb" @@ -42,66 +41,21 @@ func sendUpdates[T interface { } } -const pingInterval = 10 * time.Second -const pingTimeout = 5 * time.Second - -// sendPings periodically sends Ping messages. -func (x *Service) clientPing(ctx context.Context, client rpc.Client[API]) error { - for { - if err := utils.Sleep(ctx, pingInterval); err != nil { - return err - } - if err := utils.WithTimeout(ctx, pingTimeout, func(ctx context.Context) error { - stream, err := Ping.Call(ctx, client) - if err != nil { - return fmt.Errorf("p.client.Ping(): %w", err) - } - defer stream.Close() - // TODO(gprusak): add random payload to actually verify roundtrip latency. - if err := stream.Send(ctx, &pb.PingReq{}); err != nil { - return fmt.Errorf("stream.Send(): %w", err) - } - _, err = stream.Recv(ctx) - if err != nil { - return fmt.Errorf("stream.Recv(): %w", err) - } - // - return nil - }); err != nil { - return err - } - } -} - // Run sends newest consensus messages to the peer. -func (x *Service) clientConsensus(ctx context.Context, c rpc.Client[API]) error { +func (x *validatorService) clientConsensus(ctx context.Context, c rpc.Client[API]) error { return scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { // Send updates about new consensus messages. - s.Spawn(func() error { return sendUpdates(ctx, c, x.validatorState().SubscribeProposal()) }) - s.Spawn(func() error { return sendUpdates(ctx, c, x.validatorState().SubscribePrepareVote()) }) - s.Spawn(func() error { return sendUpdates(ctx, c, x.validatorState().SubscribeCommitVote()) }) - s.Spawn(func() error { return sendUpdates(ctx, c, x.validatorState().SubscribeTimeoutVote()) }) - s.Spawn(func() error { return sendUpdates(ctx, c, x.validatorState().SubscribeTimeoutQC()) }) - return nil - }) -} - -// Ping implements pb.StreamAPIServer. -// Note that we use streaming RPC, because unary RPC apparently causes 10ms extra delay on avg (empirically tested). -func (x *Service) serverPing(ctx context.Context, server rpc.Server[API]) error { - return Ping.Serve(ctx, server, func(ctx context.Context, stream rpc.Stream[*pb.PingResp, *pb.PingReq]) error { - if _, err := stream.Recv(ctx); err != nil { - return fmt.Errorf("stream.Recv(): %w", err) - } - if err := stream.Send(ctx, &pb.PingResp{}); err != nil { - return fmt.Errorf("stream.Send(): %w", err) - } + s.Spawn(func() error { return sendUpdates(ctx, c, x.state.SubscribeProposal()) }) + s.Spawn(func() error { return sendUpdates(ctx, c, x.state.SubscribePrepareVote()) }) + s.Spawn(func() error { return sendUpdates(ctx, c, x.state.SubscribeCommitVote()) }) + s.Spawn(func() error { return sendUpdates(ctx, c, x.state.SubscribeTimeoutVote()) }) + s.Spawn(func() error { return sendUpdates(ctx, c, x.state.SubscribeTimeoutQC()) }) return nil }) } // Consensus implements pb.StreaAPIServer. -func (x *Service) serverConsensus(ctx context.Context, server rpc.Server[API]) error { +func (x *validatorService) serverConsensus(ctx context.Context, server rpc.Server[API]) error { return Consensus.Serve(ctx, server, func(ctx context.Context, stream rpc.Stream[*pb.ConsensusResp, *apb.ConsensusReq]) error { for { reqRaw, err := stream.Recv(ctx) @@ -114,24 +68,24 @@ func (x *Service) serverConsensus(ctx context.Context, server rpc.Server[API]) e } switch req := req.(type) { case *types.ConsensusReqPrepareVote: - if err := x.validatorState().PushPrepareVote(req.Signed); err != nil { - return fmt.Errorf("x.validatorState().PushPrepareVote(): %w", err) + if err := x.state.PushPrepareVote(req.Signed); err != nil { + return fmt.Errorf("x.state.PushPrepareVote(): %w", err) } case *types.ConsensusReqCommitVote: - if err := x.validatorState().PushCommitVote(req.Signed); err != nil { - return fmt.Errorf("x.validatorState().PushCommitVote(): %w", err) + if err := x.state.PushCommitVote(req.Signed); err != nil { + return fmt.Errorf("x.state.PushCommitVote(): %w", err) } case *types.FullTimeoutVote: - if err := x.validatorState().PushTimeoutVote(req); err != nil { - return fmt.Errorf("x.validatorState().PushTimeoutVote(): %w", err) + if err := x.state.PushTimeoutVote(req); err != nil { + return fmt.Errorf("x.state.PushTimeoutVote(): %w", err) } case *types.FullProposal: - if err := x.validatorState().PushProposal(ctx, req); err != nil { - return fmt.Errorf("x.validatorState().PushProposal(): %w", err) + if err := x.state.PushProposal(ctx, req); err != nil { + return fmt.Errorf("x.state.PushProposal(): %w", err) } case *types.TimeoutQC: - if err := x.validatorState().PushTimeoutQC(ctx, req); err != nil { - return fmt.Errorf("x.validatorState().PushTimeoutQC(): %w", err) + if err := x.state.PushTimeoutQC(ctx, req); err != nil { + return fmt.Errorf("x.state.PushTimeoutQC(): %w", err) } default: return fmt.Errorf("unknown consensus request type: %T", req) diff --git a/sei-tendermint/internal/p2p/giga/consensus_test.go b/sei-tendermint/internal/p2p/giga/consensus_test.go index a80fa5aa00..8d5d4169ca 100644 --- a/sei-tendermint/internal/p2p/giga/consensus_test.go +++ b/sei-tendermint/internal/p2p/giga/consensus_test.go @@ -25,7 +25,6 @@ func TestConsensusClientServer(t *testing.T) { firstBlock := nodes[0].data.Registry().FirstBlock() if err := scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { s.SpawnBg(func() error { return env.Run(ctx) }) - var wantAppProposal utils.Option[*types.AppProposal] for offset := range types.GlobalBlockNumber(20) { idx := firstBlock + offset t.Logf("[%v] Push a block.", idx) @@ -36,18 +35,11 @@ func TestConsensusClientServer(t *testing.T) { return fmt.Errorf("ds.ProduceLocalBlock(): %w", err) } want := &types.GlobalBlock{ - Header: b.Msg().Block().Header(), - Payload: b.Msg().Block().Payload(), - GlobalNumber: idx, - FinalAppState: wantAppProposal, + Header: b.Msg().Block().Header(), + Payload: b.Msg().Block().Payload(), + GlobalNumber: idx, } - p := types.NewAppProposal( - idx, - types.RoadIndex(offset), - types.GenAppHash(rng), - registry.LatestEpoch().EpochIndex(), - ) - wantAppProposal = utils.Some(p) + wantAppProposal := utils.None[*types.AppProposal]() for _, n := range nodes { t.Logf("[%v] Wait for it to be final.", idx) got, err := n.data.GlobalBlock(ctx, idx) @@ -62,13 +54,18 @@ func TestConsensusClientServer(t *testing.T) { if err := utils.TestDiff(want, got); err != nil { return err } + if !wantAppProposal.IsPresent() { + wantAppProposal = utils.Some(types.NewAppProposal(qc.QC().Proposal(), types.GenAppHash(rng))) + } + p := wantAppProposal.OrPanic("missing app proposal") if err := n.data.PushAppHash(ctx, idx, p.AppHash()); err != nil { return fmt.Errorf("ds.PushAppProposal(): %w", err) } } for _, n := range nodes { t.Logf("[%v] Wait for AppHash consensus.", idx) - got, _, err := n.consensus.Avail().WaitForAppQC(ctx, p.RoadIndex()) + p := wantAppProposal.OrPanic("missing app proposal") + got, err := n.data.AppQC(ctx, idx) if err != nil { return fmt.Errorf("cs.avail.WaitForAppQC(): %w", err) } diff --git a/sei-tendermint/internal/p2p/giga/data.go b/sei-tendermint/internal/p2p/giga/data.go index c79134c5af..e347f208da 100644 --- a/sei-tendermint/internal/p2p/giga/data.go +++ b/sei-tendermint/internal/p2p/giga/data.go @@ -42,6 +42,32 @@ func (s *Service) clientStreamFullCommitQCs(ctx context.Context, client rpc.Clie return ctx.Err() } +func (x *Service) clientStreamAppQCs(ctx context.Context, c rpc.Client[API]) error { + stream, err := StreamAppQCs.Call(ctx, c) + if err != nil { + return fmt.Errorf("client.StreamAppQCs(): %w", err) + } + defer stream.Close() + if err := stream.Send(ctx, StreamAppQCsReqConv.Encode(&StreamAppQCsReq{ + NextBlock: x.data.NextAppQC(), + })); err != nil { + return err + } + for { + resp, err := stream.Recv(ctx) + if err != nil { + return fmt.Errorf("stream.Recv(): %w", err) + } + appQC, err := types.AppQCConv.Decode(resp) + if err != nil { + return fmt.Errorf("StreamAppQCsRespConv.Decode(): %w", err) + } + if err := x.data.PushAppQC(ctx, appQC); err != nil { + return fmt.Errorf("s.PushFirstCommitQC(): %w", err) + } + } +} + // MaxConcurrentBlockFetches is the maximum number of blocks that client fetches concurrently. const MaxConcurrentBlockFetches = 100 @@ -157,17 +183,17 @@ func (s *Service) serverStreamFullCommitQCs(ctx context.Context, server rpc.Serv if err != nil { return fmt.Errorf("StreamFullCommitQCsReqConv.Decode(): %w", err) } - prev := utils.None[*types.FullCommitQC]() - for i := req.NextBlock; ; i++ { - qc, err := s.data.QC(ctx, i) + for next := req.NextBlock; ; { + qc, err := s.data.QC(ctx, next) if err != nil { + if errors.Is(err, types.ErrPruned) { + next = s.data.First() + continue + } return fmt.Errorf("s.data.QC(): %w", err) } // Don't send the same QC twice. - if types.NextIndexOpt(prev) > qc.Index() { - continue - } - prev = utils.Some(qc) + next = qc.QC().GlobalRange().Next if err := stream.Send(ctx, types.FullCommitQCConv.Encode(qc)); err != nil { return fmt.Errorf("stream.Send(): %w", err) } @@ -175,6 +201,33 @@ func (s *Service) serverStreamFullCommitQCs(ctx context.Context, server rpc.Serv }) } +func (s *Service) serverStreamAppQCs(ctx context.Context, server rpc.Server[API]) error { + return StreamAppQCs.Serve(ctx, server, func(ctx context.Context, stream rpc.Stream[*apb.AppQC, *pb.StreamAppQCsReq]) error { + reqRaw, err := stream.Recv(ctx) + if err != nil { + return err + } + req, err := StreamAppQCsReqConv.Decode(reqRaw) + if err != nil { + return fmt.Errorf("StreamFullCommitQCsReqConv.Decode(): %w", err) + } + for next := req.NextBlock; ; { + appQC, err := s.data.AppQC(ctx, next) + if err != nil { + if errors.Is(err, types.ErrPruned) { + next = s.data.First() + continue + } + return fmt.Errorf("x.validatorState().Data().AppQC(): %w", err) + } + next = appQC.Proposal().GlobalRange().Next + if err := stream.Send(ctx, types.AppQCConv.Encode(appQC)); err != nil { + return fmt.Errorf("stream.Send(): %w", err) + } + } + }) +} + func (x *Service) serverGetBlock(ctx context.Context, server rpc.Server[API]) error { return GetBlock.Serve(ctx, server, func(ctx context.Context, stream rpc.Stream[*pb.GetBlockResp, *pb.GetBlockReq]) error { reqRaw, err := stream.Recv(ctx) diff --git a/sei-tendermint/internal/p2p/giga/data_test.go b/sei-tendermint/internal/p2p/giga/data_test.go index 1e90e455ac..c4ad706578 100644 --- a/sei-tendermint/internal/p2p/giga/data_test.go +++ b/sei-tendermint/internal/p2p/giga/data_test.go @@ -86,7 +86,7 @@ func (e *testEnv) Run(ctx context.Context) error { client := rpc.NewClient[API]() s.SpawnNamed("mux server", func() error { return server.Run(ctx, xConn) }) s.SpawnNamed("mux client", func() error { return client.Run(ctx, yConn) }) - s.SpawnNamed("RunServer", func() error { return x.service.RunServer(ctx, server) }) + s.SpawnNamed("RunServer", func() error { return x.service.RunServer(ctx, server, true) }) s.SpawnNamed("RunClient", func() error { return y.service.RunClient(ctx, client, true) }) } } diff --git a/sei-tendermint/internal/p2p/giga/pb/api.pb.go b/sei-tendermint/internal/p2p/giga/pb/api.pb.go index 21987c8ed9..d3c14daeca 100644 --- a/sei-tendermint/internal/p2p/giga/pb/api.pb.go +++ b/sei-tendermint/internal/p2p/giga/pb/api.pb.go @@ -307,94 +307,6 @@ func (x *StreamLaneProposalsReq) GetFirstBlockNumber() uint64 { return 0 } -type StreamAppQCsReq struct { - state protoimpl.MessageState `protogen:"open.v1"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *StreamAppQCsReq) Reset() { - *x = StreamAppQCsReq{} - mi := &file_p2p_giga_api_proto_msgTypes[7] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) -} - -func (x *StreamAppQCsReq) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*StreamAppQCsReq) ProtoMessage() {} - -func (x *StreamAppQCsReq) ProtoReflect() protoreflect.Message { - mi := &file_p2p_giga_api_proto_msgTypes[7] - if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use StreamAppQCsReq.ProtoReflect.Descriptor instead. -func (*StreamAppQCsReq) Descriptor() ([]byte, []int) { - return file_p2p_giga_api_proto_rawDescGZIP(), []int{7} -} - -type StreamAppQCsResp struct { - state protoimpl.MessageState `protogen:"open.v1"` - AppQc *pb.AppQC `protobuf:"bytes,1,opt,name=app_qc,json=appQc,proto3" json:"app_qc,omitempty"` // required - CommitQc *pb.CommitQC `protobuf:"bytes,2,opt,name=commit_qc,json=commitQc,proto3" json:"commit_qc,omitempty"` // required - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *StreamAppQCsResp) Reset() { - *x = StreamAppQCsResp{} - mi := &file_p2p_giga_api_proto_msgTypes[8] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) -} - -func (x *StreamAppQCsResp) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*StreamAppQCsResp) ProtoMessage() {} - -func (x *StreamAppQCsResp) ProtoReflect() protoreflect.Message { - mi := &file_p2p_giga_api_proto_msgTypes[8] - if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use StreamAppQCsResp.ProtoReflect.Descriptor instead. -func (*StreamAppQCsResp) Descriptor() ([]byte, []int) { - return file_p2p_giga_api_proto_rawDescGZIP(), []int{8} -} - -func (x *StreamAppQCsResp) GetAppQc() *pb.AppQC { - if x != nil { - return x.AppQc - } - return nil -} - -func (x *StreamAppQCsResp) GetCommitQc() *pb.CommitQC { - if x != nil { - return x.CommitQc - } - return nil -} - type StreamCommitQCsReq struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -403,7 +315,7 @@ type StreamCommitQCsReq struct { func (x *StreamCommitQCsReq) Reset() { *x = StreamCommitQCsReq{} - mi := &file_p2p_giga_api_proto_msgTypes[9] + mi := &file_p2p_giga_api_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -415,7 +327,7 @@ func (x *StreamCommitQCsReq) String() string { func (*StreamCommitQCsReq) ProtoMessage() {} func (x *StreamCommitQCsReq) ProtoReflect() protoreflect.Message { - mi := &file_p2p_giga_api_proto_msgTypes[9] + mi := &file_p2p_giga_api_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -428,7 +340,7 @@ func (x *StreamCommitQCsReq) ProtoReflect() protoreflect.Message { // Deprecated: Use StreamCommitQCsReq.ProtoReflect.Descriptor instead. func (*StreamCommitQCsReq) Descriptor() ([]byte, []int) { - return file_p2p_giga_api_proto_rawDescGZIP(), []int{9} + return file_p2p_giga_api_proto_rawDescGZIP(), []int{7} } type StreamLaneVotesReq struct { @@ -439,7 +351,7 @@ type StreamLaneVotesReq struct { func (x *StreamLaneVotesReq) Reset() { *x = StreamLaneVotesReq{} - mi := &file_p2p_giga_api_proto_msgTypes[10] + mi := &file_p2p_giga_api_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -451,7 +363,7 @@ func (x *StreamLaneVotesReq) String() string { func (*StreamLaneVotesReq) ProtoMessage() {} func (x *StreamLaneVotesReq) ProtoReflect() protoreflect.Message { - mi := &file_p2p_giga_api_proto_msgTypes[10] + mi := &file_p2p_giga_api_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -464,7 +376,7 @@ func (x *StreamLaneVotesReq) ProtoReflect() protoreflect.Message { // Deprecated: Use StreamLaneVotesReq.ProtoReflect.Descriptor instead. func (*StreamLaneVotesReq) Descriptor() ([]byte, []int) { - return file_p2p_giga_api_proto_rawDescGZIP(), []int{10} + return file_p2p_giga_api_proto_rawDescGZIP(), []int{8} } type StreamAppVotesReq struct { @@ -475,7 +387,7 @@ type StreamAppVotesReq struct { func (x *StreamAppVotesReq) Reset() { *x = StreamAppVotesReq{} - mi := &file_p2p_giga_api_proto_msgTypes[11] + mi := &file_p2p_giga_api_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -487,7 +399,7 @@ func (x *StreamAppVotesReq) String() string { func (*StreamAppVotesReq) ProtoMessage() {} func (x *StreamAppVotesReq) ProtoReflect() protoreflect.Message { - mi := &file_p2p_giga_api_proto_msgTypes[11] + mi := &file_p2p_giga_api_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -500,7 +412,7 @@ func (x *StreamAppVotesReq) ProtoReflect() protoreflect.Message { // Deprecated: Use StreamAppVotesReq.ProtoReflect.Descriptor instead. func (*StreamAppVotesReq) Descriptor() ([]byte, []int) { - return file_p2p_giga_api_proto_rawDescGZIP(), []int{11} + return file_p2p_giga_api_proto_rawDescGZIP(), []int{9} } type GetBlockReq struct { @@ -512,7 +424,7 @@ type GetBlockReq struct { func (x *GetBlockReq) Reset() { *x = GetBlockReq{} - mi := &file_p2p_giga_api_proto_msgTypes[12] + mi := &file_p2p_giga_api_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -524,7 +436,7 @@ func (x *GetBlockReq) String() string { func (*GetBlockReq) ProtoMessage() {} func (x *GetBlockReq) ProtoReflect() protoreflect.Message { - mi := &file_p2p_giga_api_proto_msgTypes[12] + mi := &file_p2p_giga_api_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -537,7 +449,7 @@ func (x *GetBlockReq) ProtoReflect() protoreflect.Message { // Deprecated: Use GetBlockReq.ProtoReflect.Descriptor instead. func (*GetBlockReq) Descriptor() ([]byte, []int) { - return file_p2p_giga_api_proto_rawDescGZIP(), []int{12} + return file_p2p_giga_api_proto_rawDescGZIP(), []int{10} } func (x *GetBlockReq) GetGlobalNumber() uint64 { @@ -556,7 +468,7 @@ type GetBlockResp struct { func (x *GetBlockResp) Reset() { *x = GetBlockResp{} - mi := &file_p2p_giga_api_proto_msgTypes[13] + mi := &file_p2p_giga_api_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -568,7 +480,7 @@ func (x *GetBlockResp) String() string { func (*GetBlockResp) ProtoMessage() {} func (x *GetBlockResp) ProtoReflect() protoreflect.Message { - mi := &file_p2p_giga_api_proto_msgTypes[13] + mi := &file_p2p_giga_api_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -581,7 +493,7 @@ func (x *GetBlockResp) ProtoReflect() protoreflect.Message { // Deprecated: Use GetBlockResp.ProtoReflect.Descriptor instead. func (*GetBlockResp) Descriptor() ([]byte, []int) { - return file_p2p_giga_api_proto_rawDescGZIP(), []int{13} + return file_p2p_giga_api_proto_rawDescGZIP(), []int{11} } func (x *GetBlockResp) GetBlock() *pb.Block { @@ -600,7 +512,7 @@ type StreamFullCommitQCsReq struct { func (x *StreamFullCommitQCsReq) Reset() { *x = StreamFullCommitQCsReq{} - mi := &file_p2p_giga_api_proto_msgTypes[14] + mi := &file_p2p_giga_api_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -612,7 +524,7 @@ func (x *StreamFullCommitQCsReq) String() string { func (*StreamFullCommitQCsReq) ProtoMessage() {} func (x *StreamFullCommitQCsReq) ProtoReflect() protoreflect.Message { - mi := &file_p2p_giga_api_proto_msgTypes[14] + mi := &file_p2p_giga_api_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -625,7 +537,7 @@ func (x *StreamFullCommitQCsReq) ProtoReflect() protoreflect.Message { // Deprecated: Use StreamFullCommitQCsReq.ProtoReflect.Descriptor instead. func (*StreamFullCommitQCsReq) Descriptor() ([]byte, []int) { - return file_p2p_giga_api_proto_rawDescGZIP(), []int{14} + return file_p2p_giga_api_proto_rawDescGZIP(), []int{12} } func (x *StreamFullCommitQCsReq) GetNextBlock() uint64 { @@ -635,6 +547,50 @@ func (x *StreamFullCommitQCsReq) GetNextBlock() uint64 { return 0 } +type StreamAppQCsReq struct { + state protoimpl.MessageState `protogen:"open.v1"` + NextBlock uint64 `protobuf:"varint,1,opt,name=next_block,json=nextBlock,proto3" json:"next_block,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StreamAppQCsReq) Reset() { + *x = StreamAppQCsReq{} + mi := &file_p2p_giga_api_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StreamAppQCsReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamAppQCsReq) ProtoMessage() {} + +func (x *StreamAppQCsReq) ProtoReflect() protoreflect.Message { + mi := &file_p2p_giga_api_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamAppQCsReq.ProtoReflect.Descriptor instead. +func (*StreamAppQCsReq) Descriptor() ([]byte, []int) { + return file_p2p_giga_api_proto_rawDescGZIP(), []int{13} +} + +func (x *StreamAppQCsReq) GetNextBlock() uint64 { + if x != nil { + return x.NextBlock + } + return 0 +} + var File_p2p_giga_api_proto protoreflect.FileDescriptor const file_p2p_giga_api_proto_rawDesc = "" + @@ -651,11 +607,7 @@ const file_p2p_giga_api_proto_rawDesc = "" + "\aAppVote\x127\n" + "\vapp_vote_v2\x18\x02 \x01(\v2\x17.autobahn.SignedAppVoteR\tappVoteV2:\x06\xe8\x88\xe2\xab\f\x01J\x04\b\x01\x10\x02R\bapp_vote\"N\n" + "\x16StreamLaneProposalsReq\x12,\n" + - "\x12first_block_number\x18\x01 \x01(\x04R\x10firstBlockNumber:\x06\xe8\x88\xe2\xab\f\x01\"\x19\n" + - "\x0fStreamAppQCsReq:\x06\xe8\x88\xe2\xab\f\x01\"s\n" + - "\x10StreamAppQCsResp\x12&\n" + - "\x06app_qc\x18\x01 \x01(\v2\x0f.autobahn.AppQCR\x05appQc\x12/\n" + - "\tcommit_qc\x18\x02 \x01(\v2\x12.autobahn.CommitQCR\bcommitQc:\x06\xe8\x88\xe2\xab\f\x01\"\x1c\n" + + "\x12first_block_number\x18\x01 \x01(\x04R\x10firstBlockNumber:\x06\xe8\x88\xe2\xab\f\x01\"\x1c\n" + "\x12StreamCommitQCsReq:\x06\xe8\x88\xe2\xab\f\x01\"\x1c\n" + "\x12StreamLaneVotesReq:\x06\xe8\x88\xe2\xab\f\x01\"\x1b\n" + "\x11StreamAppVotesReq:\x06\xe8\x88\xe2\xab\f\x01\":\n" + @@ -666,6 +618,9 @@ const file_p2p_giga_api_proto_rawDesc = "" + "\x06_block\"?\n" + "\x16StreamFullCommitQCsReq\x12\x1d\n" + "\n" + + "next_block\x18\x01 \x01(\x04R\tnextBlock:\x06\xe8\x88\xe2\xab\f\x01\"8\n" + + "\x0fStreamAppQCsReq\x12\x1d\n" + + "\n" + "next_block\x18\x01 \x01(\x04R\tnextBlock:\x06\xe8\x88\xe2\xab\f\x01BGZEgithub.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p/giga/pbb\x06proto3" var ( @@ -680,7 +635,7 @@ func file_p2p_giga_api_proto_rawDescGZIP() []byte { return file_p2p_giga_api_proto_rawDescData } -var file_p2p_giga_api_proto_msgTypes = make([]protoimpl.MessageInfo, 15) +var file_p2p_giga_api_proto_msgTypes = make([]protoimpl.MessageInfo, 14) var file_p2p_giga_api_proto_goTypes = []any{ (*ConsensusResp)(nil), // 0: p2p.giga.ConsensusResp (*PingReq)(nil), // 1: p2p.giga.PingReq @@ -689,33 +644,28 @@ var file_p2p_giga_api_proto_goTypes = []any{ (*LaneProposal)(nil), // 4: p2p.giga.LaneProposal (*AppVote)(nil), // 5: p2p.giga.AppVote (*StreamLaneProposalsReq)(nil), // 6: p2p.giga.StreamLaneProposalsReq - (*StreamAppQCsReq)(nil), // 7: p2p.giga.StreamAppQCsReq - (*StreamAppQCsResp)(nil), // 8: p2p.giga.StreamAppQCsResp - (*StreamCommitQCsReq)(nil), // 9: p2p.giga.StreamCommitQCsReq - (*StreamLaneVotesReq)(nil), // 10: p2p.giga.StreamLaneVotesReq - (*StreamAppVotesReq)(nil), // 11: p2p.giga.StreamAppVotesReq - (*GetBlockReq)(nil), // 12: p2p.giga.GetBlockReq - (*GetBlockResp)(nil), // 13: p2p.giga.GetBlockResp - (*StreamFullCommitQCsReq)(nil), // 14: p2p.giga.StreamFullCommitQCsReq - (*pb.SignedBlockHeader)(nil), // 15: autobahn.SignedBlockHeader - (*pb.SignedBlock)(nil), // 16: autobahn.SignedBlock - (*pb.SignedAppVote)(nil), // 17: autobahn.SignedAppVote - (*pb.AppQC)(nil), // 18: autobahn.AppQC - (*pb.CommitQC)(nil), // 19: autobahn.CommitQC - (*pb.Block)(nil), // 20: autobahn.Block + (*StreamCommitQCsReq)(nil), // 7: p2p.giga.StreamCommitQCsReq + (*StreamLaneVotesReq)(nil), // 8: p2p.giga.StreamLaneVotesReq + (*StreamAppVotesReq)(nil), // 9: p2p.giga.StreamAppVotesReq + (*GetBlockReq)(nil), // 10: p2p.giga.GetBlockReq + (*GetBlockResp)(nil), // 11: p2p.giga.GetBlockResp + (*StreamFullCommitQCsReq)(nil), // 12: p2p.giga.StreamFullCommitQCsReq + (*StreamAppQCsReq)(nil), // 13: p2p.giga.StreamAppQCsReq + (*pb.SignedBlockHeader)(nil), // 14: autobahn.SignedBlockHeader + (*pb.SignedBlock)(nil), // 15: autobahn.SignedBlock + (*pb.SignedAppVote)(nil), // 16: autobahn.SignedAppVote + (*pb.Block)(nil), // 17: autobahn.Block } var file_p2p_giga_api_proto_depIdxs = []int32{ - 15, // 0: p2p.giga.LaneVote.lane_vote_v2:type_name -> autobahn.SignedBlockHeader - 16, // 1: p2p.giga.LaneProposal.lane_proposal_v2:type_name -> autobahn.SignedBlock - 17, // 2: p2p.giga.AppVote.app_vote_v2:type_name -> autobahn.SignedAppVote - 18, // 3: p2p.giga.StreamAppQCsResp.app_qc:type_name -> autobahn.AppQC - 19, // 4: p2p.giga.StreamAppQCsResp.commit_qc:type_name -> autobahn.CommitQC - 20, // 5: p2p.giga.GetBlockResp.block:type_name -> autobahn.Block - 6, // [6:6] is the sub-list for method output_type - 6, // [6:6] is the sub-list for method input_type - 6, // [6:6] is the sub-list for extension type_name - 6, // [6:6] is the sub-list for extension extendee - 0, // [0:6] is the sub-list for field type_name + 14, // 0: p2p.giga.LaneVote.lane_vote_v2:type_name -> autobahn.SignedBlockHeader + 15, // 1: p2p.giga.LaneProposal.lane_proposal_v2:type_name -> autobahn.SignedBlock + 16, // 2: p2p.giga.AppVote.app_vote_v2:type_name -> autobahn.SignedAppVote + 17, // 3: p2p.giga.GetBlockResp.block:type_name -> autobahn.Block + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name } func init() { file_p2p_giga_api_proto_init() } @@ -723,14 +673,14 @@ func file_p2p_giga_api_proto_init() { if File_p2p_giga_api_proto != nil { return } - file_p2p_giga_api_proto_msgTypes[13].OneofWrappers = []any{} + file_p2p_giga_api_proto_msgTypes[11].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_p2p_giga_api_proto_rawDesc), len(file_p2p_giga_api_proto_rawDesc)), NumEnums: 0, - NumMessages: 15, + NumMessages: 14, NumExtensions: 0, NumServices: 0, }, diff --git a/sei-tendermint/internal/p2p/giga/pb/api.wireguard.go b/sei-tendermint/internal/p2p/giga/pb/api.wireguard.go index 4e40412ad9..4882b5c1a6 100644 --- a/sei-tendermint/internal/p2p/giga/pb/api.wireguard.go +++ b/sei-tendermint/internal/p2p/giga/pb/api.wireguard.go @@ -29,21 +29,13 @@ func (*LaneProposal) MaxSize() int { } func (*AppVote) MaxSize() int { - return 176 + return 187 } func (*StreamLaneProposalsReq) MaxSize() int { return 11 } -func (*StreamAppQCsReq) MaxSize() int { - return 0 -} - -func (*StreamAppQCsResp) MaxSize() int { - return 30418 -} - func (*StreamCommitQCsReq) MaxSize() int { return 0 } @@ -68,6 +60,10 @@ func (*StreamFullCommitQCsReq) MaxSize() int { return 11 } +func (*StreamAppQCsReq) MaxSize() int { + return 11 +} + func init() { // Register the wireguard.Schema generated for p2p.giga.ConsensusResp. runtime.MustRegister[*ConsensusResp](runtime.Schema{}) @@ -98,15 +94,6 @@ func init() { 1: {MaxCount: 1}, }) - // Register the wireguard.Schema generated for p2p.giga.StreamAppQCsReq. - runtime.MustRegister[*StreamAppQCsReq](runtime.Schema{}) - - // Register the wireguard.Schema generated for p2p.giga.StreamAppQCsResp. - runtime.MustRegister[*StreamAppQCsResp](runtime.Schema{ - 1: {MaxCount: 1, Nested: utils.Some(reflect.TypeFor[*pb.AppQC]())}, - 2: {MaxCount: 1, Nested: utils.Some(reflect.TypeFor[*pb.CommitQC]())}, - }) - // Register the wireguard.Schema generated for p2p.giga.StreamCommitQCsReq. runtime.MustRegister[*StreamCommitQCsReq](runtime.Schema{}) @@ -131,4 +118,9 @@ func init() { 1: {MaxCount: 1}, }) + // Register the wireguard.Schema generated for p2p.giga.StreamAppQCsReq. + runtime.MustRegister[*StreamAppQCsReq](runtime.Schema{ + 1: {MaxCount: 1}, + }) + } diff --git a/sei-tendermint/internal/p2p/giga/service.go b/sei-tendermint/internal/p2p/giga/service.go index 72cff7cc95..d62210247c 100644 --- a/sei-tendermint/internal/p2p/giga/service.go +++ b/sei-tendermint/internal/p2p/giga/service.go @@ -3,25 +3,28 @@ package giga import ( "context" "fmt" + "time" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/consensus" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/data" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p/giga/pb" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p/rpc" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/scope" ) -// Service serves the giga RPC API. NewService builds a full validator -// service (all streams); NewBlockSyncService builds a block-sync-only -// service (StreamFullCommitQCs + GetBlock) for fullnodes. state is None -// on block-sync-only services; the consensus / avail handlers reach it -// via validatorState() and panic if invoked outside RunServer / RunClient. +// Service serves the giga RPC API. type Service struct { getBlockReqs chan req data *data.State state utils.Option[*consensus.State] } +type validatorService struct { + state *consensus.State +} + +// NewService constructs service with all streams. func NewService(state *consensus.State) *Service { return &Service{ getBlockReqs: make(chan req), @@ -30,95 +33,113 @@ func NewService(state *consensus.State) *Service { } } -// NewBlockSyncService constructs a Service that only serves and consumes -// block-sync streams (no consensus / avail). -func NewBlockSyncService(d *data.State) *Service { +// NewFullNodeService constructs a Service that only serves and consumes +// fullnode streams (no consensus / avail). +func NewFullNodeService(d *data.State) *Service { return &Service{ getBlockReqs: make(chan req), data: d, } } -// RunInbound dispatches an inbound peer to the right handler set. -// Non-committee peers get the block-sync subset. Committee peers get the -// full RunServer on a validator (state present); on a non-validator the -// connection is refused — committee members shouldn't be dialing -// fullnodes in any healthy configuration, and we don't want a stale -// autobahn.json entry to take down RPC nodes via a reachable panic. -func (x *Service) RunInbound(ctx context.Context, server rpc.Server[API], isCommittee bool) error { - if !isCommittee { - return x.RunBlockSyncServer(ctx, server) - } - if !x.state.IsPresent() { - return fmt.Errorf("committee peer dialed a non-validator service") - } - return x.RunServer(ctx, server) -} - -// validatorState unwraps state for the validator-only handlers. Panics if -// called from a block-sync-only Service — which is structurally impossible -// because those handlers are only spawned by RunServer / RunClient. -func (x *Service) validatorState() *consensus.State { - return x.state.OrPanic("Service.state called from block-sync-only mode") -} - func (x *Service) Run(ctx context.Context) error { return x.runBlockFetcher(ctx) } -func (x *Service) RunServer(ctx context.Context, server rpc.Server[API]) error { +func (x *validatorService) RunServer(ctx context.Context, server rpc.Server[API]) error { return scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { - s.Spawn(func() error { return x.serverPing(ctx, server) }) s.Spawn(func() error { return x.serverConsensus(ctx, server) }) - s.Spawn(func() error { return x.serverStreamFullCommitQCs(ctx, server) }) - s.Spawn(func() error { return x.serverGetBlock(ctx, server) }) s.Spawn(func() error { return x.serverStreamLaneProposals(ctx, server) }) s.Spawn(func() error { return x.serverStreamLaneVotes(ctx, server) }) s.Spawn(func() error { return x.serverStreamCommitQCs(ctx, server) }) s.Spawn(func() error { return x.serverStreamAppVotes(ctx, server) }) - s.Spawn(func() error { return x.serverStreamAppQCs(ctx, server) }) return nil }) } -func (x *Service) RunClient(ctx context.Context, client rpc.Client[API], getBlock bool) error { - // TODO: implement a uniform robust GetBlock peer-selection / retry strategy - // so connections that lack a height (including self) do not need a separate - // getBlock=false path to avoid starving the shared fetch queue. +func (x *validatorService) RunClient(ctx context.Context, client rpc.Client[API]) error { return scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { - s.Spawn(func() error { return x.clientPing(ctx, client) }) s.Spawn(func() error { return x.clientConsensus(ctx, client) }) - s.Spawn(func() error { return x.clientStreamFullCommitQCs(ctx, client) }) s.Spawn(func() error { return x.clientStreamLaneProposals(ctx, client) }) s.Spawn(func() error { return x.clientStreamLaneVotes(ctx, client) }) s.Spawn(func() error { return x.clientStreamCommitQCs(ctx, client) }) s.Spawn(func() error { return x.clientStreamAppVotes(ctx, client) }) - s.Spawn(func() error { return x.clientStreamAppQCs(ctx, client) }) - if getBlock { - s.Spawn(func() error { return x.clientGetBlock(ctx, client) }) - } return nil }) } -// RunBlockSyncServer spawns only the block-sync server handlers. Used on -// both validator and fullnode inbound connections from non-committee peers. -func (x *Service) RunBlockSyncServer(ctx context.Context, server rpc.Server[API]) error { +func (x *Service) RunServer(ctx context.Context, server rpc.Server[API], isCommittee bool) error { return scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { s.Spawn(func() error { return x.serverPing(ctx, server) }) s.Spawn(func() error { return x.serverStreamFullCommitQCs(ctx, server) }) s.Spawn(func() error { return x.serverGetBlock(ctx, server) }) + s.Spawn(func() error { return x.serverStreamAppQCs(ctx, server) }) + if c, ok := x.state.Get(); ok && isCommittee { + return (&validatorService{c}).RunServer(ctx, server) + } return nil }) } -// RunBlockSyncClient spawns only the block-sync client handlers. Used by -// fullnodes dialing committee members. -func (x *Service) RunBlockSyncClient(ctx context.Context, client rpc.Client[API]) error { +func (x *Service) RunClient(ctx context.Context, client rpc.Client[API], getBlock bool) error { return scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { s.Spawn(func() error { return x.clientPing(ctx, client) }) s.Spawn(func() error { return x.clientStreamFullCommitQCs(ctx, client) }) - s.Spawn(func() error { return x.clientGetBlock(ctx, client) }) + s.Spawn(func() error { return x.clientStreamAppQCs(ctx, client) }) + // TODO: implement a uniform robust GetBlock peer-selection / retry strategy + // so connections that lack a height (including self) do not need a separate + // getBlock=false path to avoid starving the shared fetch queue. + if getBlock { + s.Spawn(func() error { return x.clientGetBlock(ctx, client) }) + } + if c, ok := x.state.Get(); ok { + return (&validatorService{c}).RunClient(ctx, client) + } return nil }) } + +// Ping implements pb.StreamAPIServer. +// Note that we use streaming RPC, because unary RPC apparently causes 10ms extra delay on avg (empirically tested). +func (x *Service) serverPing(ctx context.Context, server rpc.Server[API]) error { + return Ping.Serve(ctx, server, func(ctx context.Context, stream rpc.Stream[*pb.PingResp, *pb.PingReq]) error { + if _, err := stream.Recv(ctx); err != nil { + return fmt.Errorf("stream.Recv(): %w", err) + } + if err := stream.Send(ctx, &pb.PingResp{}); err != nil { + return fmt.Errorf("stream.Send(): %w", err) + } + return nil + }) +} + +const pingInterval = 10 * time.Second +const pingTimeout = 5 * time.Second + +// sendPings periodically sends Ping messages. +func (x *Service) clientPing(ctx context.Context, client rpc.Client[API]) error { + for { + if err := utils.Sleep(ctx, pingInterval); err != nil { + return err + } + if err := utils.WithTimeout(ctx, pingTimeout, func(ctx context.Context) error { + stream, err := Ping.Call(ctx, client) + if err != nil { + return fmt.Errorf("p.client.Ping(): %w", err) + } + defer stream.Close() + // TODO(gprusak): add random payload to actually verify roundtrip latency. + if err := stream.Send(ctx, &pb.PingReq{}); err != nil { + return fmt.Errorf("stream.Send(): %w", err) + } + _, err = stream.Recv(ctx) + if err != nil { + return fmt.Errorf("stream.Recv(): %w", err) + } + // + return nil + }); err != nil { + return err + } + } +} diff --git a/sei-tendermint/internal/p2p/giga/types.go b/sei-tendermint/internal/p2p/giga/types.go index d742224f45..172b05ba9f 100644 --- a/sei-tendermint/internal/p2p/giga/types.go +++ b/sei-tendermint/internal/p2p/giga/types.go @@ -26,6 +26,10 @@ type StreamFullCommitQCsReq struct { NextBlock types.GlobalBlockNumber } +type StreamAppQCsReq struct { + NextBlock types.GlobalBlockNumber +} + var LaneVoteConv = protoutils.Conv[*types.Signed[*types.LaneVote], *pb.LaneVote]{ Encode: func(m *types.Signed[*types.LaneVote]) *pb.LaneVote { return &pb.LaneVote{ @@ -80,26 +84,6 @@ var StreamLaneProposalsReqConv = protoutils.Conv[*StreamLaneProposalsReq, *pb.St }, } -var StreamAppQCsRespConv = protoutils.Conv[*StreamAppQCsResp, *pb.StreamAppQCsResp]{ - Encode: func(m *StreamAppQCsResp) *pb.StreamAppQCsResp { - return &pb.StreamAppQCsResp{ - AppQc: types.AppQCConv.Encode(m.AppQC), - CommitQc: types.CommitQCConv.Encode(m.CommitQC), - } - }, - Decode: func(m *pb.StreamAppQCsResp) (*StreamAppQCsResp, error) { - appQC, err := types.AppQCConv.DecodeReq(m.AppQc) - if err != nil { - return nil, fmt.Errorf("appQC: %w", err) - } - commitQC, err := types.CommitQCConv.DecodeReq(m.CommitQc) - if err != nil { - return nil, fmt.Errorf("commitQC: %w", err) - } - return &StreamAppQCsResp{AppQC: appQC, CommitQC: commitQC}, nil - }, -} - var GetBlockReqConv = protoutils.Conv[*GetBlockReq, *pb.GetBlockReq]{ Encode: func(m *GetBlockReq) *pb.GetBlockReq { return &pb.GetBlockReq{GlobalNumber: uint64(m.GlobalNumber)} @@ -130,3 +114,12 @@ var StreamFullCommitQCsReqConv = protoutils.Conv[*StreamFullCommitQCsReq, *pb.St return &StreamFullCommitQCsReq{NextBlock: types.GlobalBlockNumber(m.NextBlock)}, nil }, } + +var StreamAppQCsReqConv = protoutils.Conv[*StreamAppQCsReq, *pb.StreamAppQCsReq]{ + Encode: func(m *StreamAppQCsReq) *pb.StreamAppQCsReq { + return &pb.StreamAppQCsReq{NextBlock: uint64(m.NextBlock)} + }, + Decode: func(m *pb.StreamAppQCsReq) (*StreamAppQCsReq, error) { + return &StreamAppQCsReq{NextBlock: types.GlobalBlockNumber(m.NextBlock)}, nil + }, +} diff --git a/sei-tendermint/internal/p2p/giga/types_test.go b/sei-tendermint/internal/p2p/giga/types_test.go index 4b5bfb19e3..8b8db6dc92 100644 --- a/sei-tendermint/internal/p2p/giga/types_test.go +++ b/sei-tendermint/internal/p2p/giga/types_test.go @@ -16,14 +16,11 @@ func TestConv(t *testing.T) { LaneProposalConv.Test(types.GenSigned(rng, types.GenLaneProposal(rng))), AppVoteConv.Test(types.GenSigned(rng, types.GenAppVote(rng))), StreamLaneProposalsReqConv.Test(&StreamLaneProposalsReq{FirstBlockNumber: types.GenBlockNumber(rng)}), - StreamAppQCsRespConv.Test(&StreamAppQCsResp{ - AppQC: types.GenAppQC(rng), - CommitQC: types.GenCommitQC(rng), - }), GetBlockReqConv.Test(&GetBlockReq{GlobalNumber: types.GenGlobalBlockNumber(rng)}), GetBlockRespConv.Test(utils.None[*types.Block]()), GetBlockRespConv.Test(utils.Some(types.GenBlock(rng))), StreamFullCommitQCsReqConv.Test(&StreamFullCommitQCsReq{NextBlock: types.GenGlobalBlockNumber(rng)}), + StreamAppQCsReqConv.Test(&StreamAppQCsReq{NextBlock: types.GenGlobalBlockNumber(rng)}), )) } } diff --git a/sei-tendermint/internal/p2p/giga_router_common.go b/sei-tendermint/internal/p2p/giga_router_common.go index ff8996c683..04a2f14d86 100644 --- a/sei-tendermint/internal/p2p/giga_router_common.go +++ b/sei-tendermint/internal/p2p/giga_router_common.go @@ -66,15 +66,6 @@ func BuildDataState(cfg *GigaRouterCommonConfig, blockDB atypes.BlockDB) (*data. if cfg.MaxInboundFullnodePeers < 0 || cfg.MaxInboundFullnodePeers > maxInboundFullnodePeers { return nil, fmt.Errorf("GigaRouterCommonConfig.MaxInboundFullnodePeers = %v, want 0..%v", cfg.MaxInboundFullnodePeers, maxInboundFullnodePeers) } - lastExecutedHeight := cfg.App.Info().LastBlockHeight - lastExecutedBlock := utils.None[atypes.GlobalBlockNumber]() - if lastExecutedHeight != 0 { - n, ok := utils.SafeCast[atypes.GlobalBlockNumber](lastExecutedHeight) - if !ok { - return nil, fmt.Errorf("invalid App.Info().LastBlockHeight = %v", lastExecutedHeight) - } - lastExecutedBlock = utils.Some(n) - } firstBlock := atypes.GlobalBlockNumber(cfg.GenDoc.InitialHeight) // nolint:gosec // verified to be positive. genesisWeights := map[atypes.PublicKey]uint64{} for k := range cfg.ValidatorAddrs { @@ -88,10 +79,7 @@ func BuildDataState(cfg *GigaRouterCommonConfig, blockDB atypes.BlockDB) (*data. if err != nil { return nil, fmt.Errorf("epoch.NewRegistry(): %w", err) } - ds, err := data.NewState(&data.Config{ - Registry: registry, - LastExecutedBlock: lastExecutedBlock, - }, blockDB) + ds, err := data.NewState(&data.Config{Registry: registry}, blockDB) if err != nil { return nil, fmt.Errorf("data.NewState: %w", err) } @@ -515,7 +503,7 @@ func (r *gigaRouterCommon) RunInboundConn(ctx context.Context, hConn *handshaked Global.gigaNewConnsAt("in").Add(1) Global.gigaConnsAt("in").Add(1) defer Global.gigaConnsAt("in").Add(-1) - if err := r.service.RunInbound(ctx, server, isCommittee); err != nil { + if err := r.service.RunServer(ctx, server, isCommittee); err != nil { return fmt.Errorf("inbound from %v: %w", key, err) } return nil diff --git a/sei-tendermint/internal/p2p/giga_router_common_test.go b/sei-tendermint/internal/p2p/giga_router_common_test.go index 4e340a157c..4cc991438c 100644 --- a/sei-tendermint/internal/p2p/giga_router_common_test.go +++ b/sei-tendermint/internal/p2p/giga_router_common_test.go @@ -114,7 +114,6 @@ func TestBuildDataStateStartsRecoveryAtAppTip(t *testing.T) { App: proxy.New(&fixedHeightApp{height: int64(last)}), }, db) require.NoError(t, err) - require.Equal(t, last, state.FirstAppProposal()) got, err := state.TryBlock(last) require.NoError(t, err) require.Equal(t, blocks[gr.Len()/2].Header().Hash(), got.Header().Hash()) diff --git a/sei-tendermint/internal/p2p/giga_router_fullnode.go b/sei-tendermint/internal/p2p/giga_router_fullnode.go index 890b899845..7a6b49608b 100644 --- a/sei-tendermint/internal/p2p/giga_router_fullnode.go +++ b/sei-tendermint/internal/p2p/giga_router_fullnode.go @@ -27,7 +27,7 @@ func NewGigaFullnodeRouter(cfg *GigaRouterCommonConfig, key NodeSecretKey, dataS cfg: cfg, key: key, data: dataState, - service: giga.NewBlockSyncService(dataState), + service: giga.NewFullNodeService(dataState), poolIn: giga.NewPool[NodePublicKey, rpc.Server[giga.API]](), poolOut: giga.NewPool[NodePublicKey, rpc.Client[giga.API]](), app: cfg.App, @@ -72,7 +72,9 @@ func (r *gigaFullnodeRouter) runFullnodeSubscriber(ctx context.Context) error { rand.Shuffle(len(addrs), func(i, j int) { addrs[i], addrs[j] = addrs[j], addrs[i] }) for i := 0; ; i = (i + 1) % len(addrs) { addr := addrs[i] - err := r.dialAndRunConn(ctx, utils.None[NodePublicKey](), addr.HostPort, r.service.RunBlockSyncClient) + err := r.dialAndRunConn(ctx, utils.Some(addr.Key), addr.HostPort, func(ctx context.Context, client rpc.Client[giga.API]) error { + return r.service.RunClient(ctx, client, true) + }) logger.Info("fullnode giga connection ended; failing over", "addr", addr, "err", err) if err := utils.Sleep(ctx, r.cfg.DialInterval); err != nil { return err