From 164832cc285ac23cd4902bed911cc5b401171109 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 4 Aug 2026 13:40:33 +0200 Subject: [PATCH 01/61] WIP --- sei-tendermint/autobahn/types/app_proposal.go | 20 +- sei-tendermint/autobahn/types/block.go | 2 - .../autobahn/types/committee_test.go | 10 +- sei-tendermint/autobahn/types/proposal.go | 87 +------ .../autobahn/types/proposal_test.go | 165 +------------ sei-tendermint/autobahn/types/testonly.go | 15 +- sei-tendermint/autobahn/types/timeout.go | 2 +- sei-tendermint/autobahn/types/types_test.go | 2 +- .../autobahn/types/wireguard_test.go | 2 +- .../internal/autobahn/avail/app_votes.go | 45 ++-- .../internal/autobahn/avail/inner.go | 91 ++++--- .../autobahn/avail/metrics/metrics.gen.go | 11 - .../autobahn/avail/metrics/metrics.go | 6 +- .../internal/autobahn/avail/state.go | 222 ++++-------------- .../internal/autobahn/avail/subscriptions.go | 4 +- .../internal/autobahn/data/state.go | 101 +++----- .../internal/autobahn/data/testonly.go | 8 +- 17 files changed, 194 insertions(+), 599 deletions(-) diff --git a/sei-tendermint/autobahn/types/app_proposal.go b/sei-tendermint/autobahn/types/app_proposal.go index e383cc3591..2b45630434 100644 --- a/sei-tendermint/autobahn/types/app_proposal.go +++ b/sei-tendermint/autobahn/types/app_proposal.go @@ -15,20 +15,16 @@ type AppHash []byte // AppProposal . type AppProposal struct { utils.ReadOnly - globalNumber GlobalBlockNumber + epochIndex EpochIndex roadIndex RoadIndex appHash AppHash - epochIndex EpochIndex } // NewAppProposal creates a new AppProposal. -func NewAppProposal(globalNumber GlobalBlockNumber, roadIndex RoadIndex, appHash AppHash, epochIndex EpochIndex) *AppProposal { - return &AppProposal{globalNumber: globalNumber, roadIndex: roadIndex, appHash: appHash, epochIndex: epochIndex} +func NewAppProposal(roadIndex RoadIndex, appHash AppHash, epochIndex EpochIndex) *AppProposal { + return &AppProposal{roadIndex: roadIndex, appHash: appHash, epochIndex: epochIndex} } -// GlobalNumber . -func (m *AppProposal) GlobalNumber() GlobalBlockNumber { return m.globalNumber } - // RoadIndex returns the road index of the proposal. func (m *AppProposal) RoadIndex() RoadIndex { return m.roadIndex } @@ -48,9 +44,6 @@ func (m *AppProposal) Verify(qc *CommitQC) error { if got, want := m.RoadIndex(), qc.Proposal().Index(); got != want { return fmt.Errorf("roadIndex() = %v, want %v", got, want) } - if got, want := m.GlobalNumber(), qc.GlobalRange(); got < want.First || got >= want.Next { - return fmt.Errorf("globalNumber() = %v, want in range [%v,%v)", got, want.First, want.Next) - } if got, want := m.EpochIndex(), qc.Proposal().EpochIndex(); got != want { return fmt.Errorf("epoch_index = %d, want %d", got, want) } @@ -61,16 +54,12 @@ func (m *AppProposal) Verify(qc *CommitQC) error { var AppProposalConv = protoutils.Conv[*AppProposal, *pb.AppProposal]{ Encode: func(m *AppProposal) *pb.AppProposal { return &pb.AppProposal{ - GlobalNumber: utils.Alloc(uint64(m.globalNumber)), RoadIndex: utils.Alloc(uint64(m.roadIndex)), AppHash: m.appHash, EpochIndex: utils.Alloc(uint64(m.epochIndex)), } }, Decode: func(m *pb.AppProposal) (*AppProposal, error) { - if m.GlobalNumber == nil { - return nil, fmt.Errorf("global_number: missing") - } if m.RoadIndex == nil { return nil, fmt.Errorf("road_index: missing") } @@ -78,10 +67,9 @@ var AppProposalConv = protoutils.Conv[*AppProposal, *pb.AppProposal]{ return nil, fmt.Errorf("epoch_index: missing") } return &AppProposal{ - globalNumber: GlobalBlockNumber(*m.GlobalNumber), + epochIndex: EpochIndex(*m.EpochIndex), roadIndex: RoadIndex(*m.RoadIndex), appHash: AppHash(m.AppHash), - epochIndex: EpochIndex(*m.EpochIndex), }, nil }, } diff --git a/sei-tendermint/autobahn/types/block.go b/sei-tendermint/autobahn/types/block.go index d3e283bd23..951a37f049 100644 --- a/sei-tendermint/autobahn/types/block.go +++ b/sei-tendermint/autobahn/types/block.go @@ -117,8 +117,6 @@ type GlobalBlock struct { Timestamp time.Time GlobalNumber GlobalBlockNumber Payload *Payload - // Highest known finalized state. - FinalAppState utils.Option[*AppProposal] } // NewBlock creates a new Block. diff --git a/sei-tendermint/autobahn/types/committee_test.go b/sei-tendermint/autobahn/types/committee_test.go index 996e2a812a..db9e3a3587 100644 --- a/sei-tendermint/autobahn/types/committee_test.go +++ b/sei-tendermint/autobahn/types/committee_test.go @@ -129,10 +129,10 @@ func TestPrepareQCVerifyChecksEpochBinding(t *testing.T) { require.NoError(t, sign(ProposalAt(ep, View{Index: ep.RoadRange().First})).Verify(ep)) - wrongEpoch := newProposal(View{Index: ep.RoadRange().First, EpochIndex: ep.EpochIndex() + 1}, time.Time{}, nil, utils.None[*AppProposal](), ep.FirstBlock()) + wrongEpoch := newProposal(View{Index: ep.RoadRange().First, EpochIndex: ep.EpochIndex() + 1}, time.Time{}, nil, ep.FirstBlock()) require.Error(t, sign(wrongEpoch).Verify(ep)) - outOfRoads := newProposal(View{Index: ep.RoadRange().Last + 1, EpochIndex: ep.EpochIndex()}, time.Time{}, nil, utils.None[*AppProposal](), ep.FirstBlock()) + outOfRoads := newProposal(View{Index: ep.RoadRange().Last + 1, EpochIndex: ep.EpochIndex()}, time.Time{}, nil, ep.FirstBlock()) require.Error(t, sign(outOfRoads).Verify(ep)) } @@ -145,10 +145,10 @@ func TestCommitQCVerifyChecksEpochBinding(t *testing.T) { require.NoError(t, sign(ProposalAt(ep, View{Index: ep.RoadRange().First})).Verify(ep)) - wrongEpoch := newProposal(View{Index: ep.RoadRange().First, EpochIndex: ep.EpochIndex() + 1}, time.Time{}, nil, utils.None[*AppProposal](), ep.FirstBlock()) + wrongEpoch := newProposal(View{Index: ep.RoadRange().First, EpochIndex: ep.EpochIndex() + 1}, time.Time{}, nil, ep.FirstBlock()) require.Error(t, sign(wrongEpoch).Verify(ep)) - outOfRoads := newProposal(View{Index: ep.RoadRange().Last + 1, EpochIndex: ep.EpochIndex()}, time.Time{}, nil, utils.None[*AppProposal](), ep.FirstBlock()) + outOfRoads := newProposal(View{Index: ep.RoadRange().Last + 1, EpochIndex: ep.EpochIndex()}, time.Time{}, nil, ep.FirstBlock()) require.Error(t, sign(outOfRoads).Verify(ep)) } @@ -171,7 +171,7 @@ func TestCommitQCVerifyChecksWeight(t *testing.T) { func TestAppQCVerifyChecksWeight(t *testing.T) { rng := utils.TestRng() ep, keys := makeEpoch(rng) - vote := NewAppVote(NewAppProposal(0, 0, GenAppHash(rng), ep.EpochIndex())) + vote := NewAppVote(NewAppProposal(0, GenAppHash(rng), ep.EpochIndex())) heavyOnly := NewAppQC([]*Signed[*AppVote]{ Sign(keys[0], vote), diff --git a/sei-tendermint/autobahn/types/proposal.go b/sei-tendermint/autobahn/types/proposal.go index bb78f668c1..70079b8117 100644 --- a/sei-tendermint/autobahn/types/proposal.go +++ b/sei-tendermint/autobahn/types/proposal.go @@ -168,11 +168,10 @@ type Proposal struct { view View timestamp time.Time laneRanges map[LaneID]*LaneRange - app utils.Option[*AppProposal] globalRange GlobalRange } -func newProposal(view View, timestamp time.Time, laneRanges []*LaneRange, app utils.Option[*AppProposal], globalFirst GlobalBlockNumber) *Proposal { +func newProposal(view View, timestamp time.Time, laneRanges []*LaneRange, globalFirst GlobalBlockNumber) *Proposal { laneRangesM := map[LaneID]*LaneRange{} gr := GlobalRange{First: globalFirst, Next: globalFirst} for _, r := range laneRanges { @@ -186,7 +185,6 @@ func newProposal(view View, timestamp time.Time, laneRanges []*LaneRange, app ut timestamp: timestamp, laneRanges: laneRangesM, globalRange: gr, - app: app, } } @@ -199,9 +197,6 @@ func (m *Proposal) View() View { return m.view } // Timestamp of the proposal. func (m *Proposal) Timestamp() time.Time { return m.timestamp } -// App . -func (m *Proposal) App() utils.Option[*AppProposal] { return m.app } - // EpochIndex returns the epoch index encoded in the proposal. func (m *Proposal) EpochIndex() EpochIndex { return m.view.EpochIndex } @@ -266,7 +261,6 @@ type FullProposal struct { utils.ReadOnly proposal *Signed[*Proposal] laneQCs map[LaneID]*LaneQC - appQC utils.Option[*AppQC] timeoutQC utils.Option[*TimeoutQC] } @@ -297,7 +291,6 @@ func NewProposal( viewSpec ViewSpec, timestamp time.Time, laneQCs map[LaneID]*LaneQC, - appQC utils.Option[*AppQC], ) (*FullProposal, error) { committee := viewSpec.Epoch.Committee() if got, want := key.Public(), committee.Leader(viewSpec.View()); got != want { @@ -306,14 +299,13 @@ func NewProposal( if p, ok := NewReproposal(key, viewSpec); ok { return p, nil } - proposal, appQC, err := buildProposal(committee, viewSpec, timestamp, laneQCs, appQC) + proposal, err := buildProposal(committee, viewSpec, timestamp, laneQCs) if err != nil { return nil, err } return &FullProposal{ proposal: Sign(key, proposal), laneQCs: laneQCs, - appQC: appQC, timeoutQC: viewSpec.TimeoutQC, }, nil } @@ -326,45 +318,32 @@ func buildProposal( viewSpec ViewSpec, timestamp time.Time, laneQCs map[LaneID]*LaneQC, - appQC utils.Option[*AppQC], -) (*Proposal, utils.Option[*AppQC], error) { +) (*Proposal, error) { var laneRanges []*LaneRange for lane := range committee.Lanes().All() { first := LaneRangeOpt(viewSpec.CommitQC, lane).Next() if lQC, ok := laneQCs[lane]; ok { if lQC.Header().Lane() != lane { - return nil, appQC, fmt.Errorf("laneQC %v for lane %v", lQC.Header().Lane(), lane) + return nil, fmt.Errorf("laneQC %v for lane %v", lQC.Header().Lane(), lane) } laneRange := NewLaneRange(lane, first, utils.Some(lQC.Header())) if got := laneRange.Len(); got > MaxLaneRangeInProposal { - return nil, appQC, fmt.Errorf("laneRange[%v].Len() = %d, want <= %d", lane, got, MaxLaneRangeInProposal) + return nil, fmt.Errorf("laneRange[%v].Len() = %d, want <= %d", lane, got, MaxLaneRangeInProposal) } laneRanges = append(laneRanges, laneRange) } else { laneRanges = append(laneRanges, NewLaneRange(lane, first, utils.None[*BlockHeader]())) } } - app := ProposalOpt(appQC) - // If the new appProposal is not later than the previous one, then clear appQC. - if old := AppOpt(ProposalOpt(viewSpec.CommitQC)); NextOpt(app) <= NextOpt(old) { - app = old - appQC = utils.None[*AppQC]() - } - // If the new appProposal is from the future (which may happen if this node is behind), then clear appQC. - // The proposal will be useless in this case, but at least it will be valid. - if a, ok := app.Get(); ok && a.GlobalNumber() >= viewSpec.NextGlobalBlock() { - app = utils.None[*AppProposal]() - appQC = utils.None[*AppQC]() - } // Normalize the creation timestamp. if wantMin := viewSpec.NextTimestamp(); timestamp.Before(wantMin) { timestamp = wantMin } - proposal := newProposal(viewSpec.View(), timestamp, laneRanges, app, viewSpec.NextGlobalBlock()) + proposal := newProposal(viewSpec.View(), timestamp, laneRanges, viewSpec.NextGlobalBlock()) if proposal.GlobalRange().Len() == 0 { - return nil, appQC, errors.New("empty tipcut: need at least one LaneQC") + return nil, errors.New("empty tipcut: need at least one LaneQC") } - return proposal, appQC, nil + return proposal, nil } // NewProposalForTesting builds a FullProposal exactly like NewProposal but attaches the @@ -376,17 +355,15 @@ func NewProposalForTesting( viewSpec ViewSpec, timestamp time.Time, laneQCs map[LaneID]*LaneQC, - appQC utils.Option[*AppQC], sig *Signature, ) (*FullProposal, error) { - proposal, appQC, err := buildProposal(committee, viewSpec, timestamp, laneQCs, appQC) + proposal, err := buildProposal(committee, viewSpec, timestamp, laneQCs) if err != nil { return nil, err } return &FullProposal{ proposal: newSigned(proposal, sig), laneQCs: laneQCs, - appQC: appQC, timeoutQC: viewSpec.TimeoutQC, }, nil } @@ -447,7 +424,7 @@ func (m *FullProposal) Verify(vs ViewSpec) error { }) // Is this a reproposal? if want, ok := tQC.reproposal(); ok { - if len(m.laneQCs) > 0 || m.appQC.IsPresent() { + if len(m.laneQCs) > 0 { return errors.New("unnecessary data when reproposing") } if NewHashed(want).Hash() != m.proposal.hashed.hash { @@ -489,36 +466,6 @@ func (m *FullProposal) Verify(vs ViewSpec) error { }) } } - // Verify the appQC. - if got, wantMin := NextOpt(m.proposal.Msg().App()), NextOpt(AppOpt(ProposalOpt(vs.CommitQC))); got < wantMin { - return errors.New("AppProposal lower than in previous CommitQC") - } else if got == wantMin { - if m.appQC.IsPresent() { - return errors.New("unnecessary appQC") - } - } else { - app, _ := m.proposal.Msg().App().Get() - // TODO: relax to allow current_epoch-1 once epoch transitions are wired up. - if got, want := app.EpochIndex(), m.proposal.Msg().EpochIndex(); got != want { - return fmt.Errorf("app epoch_index %d != proposal epoch_index %d", got, want) - } - appQC, ok := m.appQC.Get() - if !ok { - return errors.New("appQC missing") - } - if appQC.vote.hash != NewHashed(NewAppVote(app)).hash { - return errors.New("appQC doesn't match the proposal") - } - s.Spawn(func() error { - if err := appQC.Verify(c); err != nil { - return fmt.Errorf("appQC: %w", err) - } - return nil - }) - if got, want := appQC.Proposal().GlobalNumber(), vs.NextGlobalBlock(); got >= want { - return fmt.Errorf("appQC for block %v, while only %v blocks were finalized", got, want) - } - } return nil }) } @@ -596,7 +543,6 @@ var ProposalConv = protoutils.Conv[*Proposal, *pb.Proposal]{ View: ViewConv.Encode(m.view), Timestamp: TimeConv.Encode(m.timestamp), LaneRanges: LaneRangeConv.EncodeSlice(laneRanges), - App: AppProposalConv.EncodeOpt(m.app), GlobalFirst: utils.Alloc(uint64(m.globalRange.First)), } }, @@ -613,17 +559,13 @@ var ProposalConv = protoutils.Conv[*Proposal, *pb.Proposal]{ if err != nil { return nil, fmt.Errorf("timestamp: %w", err) } - app, err := AppProposalConv.DecodeOpt(m.App) - if err != nil { - return nil, fmt.Errorf("appQC: %w", err) - } // Hard-reject messages with absent global_first/epoch_index. // Autobahn is pre-production; there is no rolling-upgrade path from // messages encoded before these fields were added. if m.GlobalFirst == nil { return nil, fmt.Errorf("global_first: missing") } - proposal := newProposal(view, timestamp, laneRanges, app, GlobalBlockNumber(*m.GlobalFirst)) + proposal := newProposal(view, timestamp, laneRanges, GlobalBlockNumber(*m.GlobalFirst)) if len(proposal.laneRanges) != len(laneRanges) { return nil, fmt.Errorf("laneRanges: duplicate ranges") } @@ -641,7 +583,6 @@ var FullProposalConv = protoutils.Conv[*FullProposal, *pb.FullProposal]{ return &pb.FullProposal{ ProposalV2: SignedProposalConv.Encode(m.proposal), LaneQcs: LaneQCConv.EncodeSlice(laneQCs), - AppQc: AppQCConv.EncodeOpt(m.appQC), TimeoutQc: TimeoutQCConv.EncodeOpt(m.timeoutQC), } }, @@ -658,14 +599,10 @@ var FullProposalConv = protoutils.Conv[*FullProposal, *pb.FullProposal]{ for _, qc := range laneQCs { laneQCsMap[qc.Header().Lane()] = qc } - appQC, err := AppQCConv.DecodeOpt(m.AppQc) - if err != nil { - return nil, fmt.Errorf("appQC: %w", err) - } timeoutQC, err := TimeoutQCConv.DecodeOpt(m.TimeoutQc) if err != nil { return nil, fmt.Errorf("timeoutQC: %w", err) } - return &FullProposal{proposal: proposal, laneQCs: laneQCsMap, appQC: appQC, timeoutQC: timeoutQC}, nil + return &FullProposal{proposal: proposal, laneQCs: laneQCsMap, timeoutQC: timeoutQC}, nil }, } diff --git a/sei-tendermint/autobahn/types/proposal_test.go b/sei-tendermint/autobahn/types/proposal_test.go index 0228ccc1c3..d4c6538de1 100644 --- a/sei-tendermint/autobahn/types/proposal_test.go +++ b/sei-tendermint/autobahn/types/proposal_test.go @@ -60,17 +60,6 @@ func makeCommitQCFromProposal(keys []SecretKey, fp *FullProposal) *CommitQC { return NewCommitQC(votes) } -// makeAppQCFor creates an AppQC for the given parameters, signed by all keys. -func makeAppQCFor(keys []SecretKey, globalNum GlobalBlockNumber, roadIdx RoadIndex, appHash AppHash, epochIdx EpochIndex) *AppQC { - appProposal := NewAppProposal(globalNum, roadIdx, appHash, epochIdx) - vote := NewAppVote(appProposal) - var votes []*Signed[*AppVote] - for _, k := range keys { - votes = append(votes, Sign(k, vote)) - } - return NewAppQC(votes) -} - func TestProposalVerifyRejectsEmptyTipcut(t *testing.T) { rng := utils.TestRng() committee, _ := GenCommittee(rng, 4) @@ -80,7 +69,7 @@ func TestProposalVerifyRejectsEmptyTipcut(t *testing.T) { // Direct Proposal.Verify rejects empty tipcuts (no LaneQCs / zero GlobalRange). // Local propose waits via WaitForLaneQCs; verification must refuse empty tipcuts // from peers as well. - empty := newProposal(vs.View(), time.Now(), nil, utils.None[*AppProposal](), vs.NextGlobalBlock()) + empty := newProposal(vs.View(), time.Now(), nil, vs.NextGlobalBlock()) require.Equal(t, uint64(0), empty.GlobalRange().Len()) require.Error(t, empty.Verify(ep)) } @@ -283,7 +272,6 @@ func TestProposalVerifyRejectsWrongProposer(t *testing.T) { tamperedFP := &FullProposal{ proposal: Sign(wrongKey, fp.Proposal().Msg()), laneQCs: fp.laneQCs, - appQC: fp.appQC, timeoutQC: fp.timeoutQC, } err := tamperedFP.Verify(vs) @@ -309,7 +297,6 @@ func TestProposalVerifyRejectsInconsistentTimeoutQC(t *testing.T) { tamperedFP := &FullProposal{ proposal: fp.proposal, laneQCs: fp.laneQCs, - appQC: fp.appQC, timeoutQC: utils.Some(tQC), } err := tamperedFP.Verify(vs) @@ -337,11 +324,10 @@ func TestProposalVerifyRejectsNonCommitteeLane(t *testing.T) { } tamperedRanges = append(tamperedRanges, NewLaneRange(extraLane, 0, utils.None[*BlockHeader]())) - tamperedProposal := newProposal(origProposal.view, origProposal.timestamp, tamperedRanges, origProposal.app, origProposal.GlobalRange().First) + tamperedProposal := newProposal(origProposal.view, origProposal.timestamp, tamperedRanges, origProposal.GlobalRange().First) maliciousFP := &FullProposal{ proposal: Sign(proposerKey, tamperedProposal), laneQCs: fp.laneQCs, - appQC: fp.appQC, timeoutQC: fp.timeoutQC, } err := maliciousFP.Verify(vs) @@ -372,7 +358,7 @@ func TestProposalVerifyAcceptsImplicitLaneRange(t *testing.T) { } require.True(t, droppedEmpty) - shortProposal := newProposal(origP.view, origP.timestamp, keptRanges, origP.app, origP.GlobalRange().First) + shortProposal := newProposal(origP.view, origP.timestamp, keptRanges, origP.GlobalRange().First) shortFP := &FullProposal{ proposal: Sign(proposerKey, shortProposal), laneQCs: fp.laneQCs, @@ -404,7 +390,7 @@ func TestProposalVerifyAcceptsNonContiguousImplicitRanges(t *testing.T) { keptRanges = append(keptRanges, r) } - shortProposal := newProposal(origP.view, origP.timestamp, keptRanges, origP.app, origP.GlobalRange().First) + shortProposal := newProposal(origP.view, origP.timestamp, keptRanges, origP.GlobalRange().First) shortFP := &FullProposal{ proposal: Sign(proposerKey, shortProposal), laneQCs: fp.laneQCs, @@ -441,7 +427,7 @@ func TestProposalVerifyRejectsLaneRangeFirstMismatch(t *testing.T) { tamperedRanges = append(tamperedRanges, r) } } - tamperedProposal := newProposal(origP.view, origP.timestamp, tamperedRanges, origP.app, origP.GlobalRange().First) + tamperedProposal := newProposal(origP.view, origP.timestamp, tamperedRanges, origP.GlobalRange().First) tamperedFP := &FullProposal{ proposal: Sign(proposerKey, tamperedProposal), laneQCs: map[LaneID]*LaneQC{target: badQC}, @@ -547,7 +533,6 @@ func TestProposalVerifyRejectsLaneRangeLongerThanMaxLaneRangeInProposal(t *testi View{}, time.Now(), []*LaneRange{NewLaneRange(lane, 0, utils.Some(NewBlock(lane, MaxLaneRangeInProposal, GenBlockHeaderHash(rng), GenPayload(rng)).Header()))}, - utils.None[*AppProposal](), ep.FirstBlock(), ) require.Error(t, oversized.Verify(ep)) @@ -579,144 +564,6 @@ func makeCommitQC(keys []SecretKey, fullProposal *FullProposal) *CommitQC { return NewCommitQC(votes) } -func TestProposalVerifyRejectsAppProposalLowerThanPrevious(t *testing.T) { - rng := utils.TestRng() - committee, keys := GenCommittee(rng, 4) - ep := genFreshEpoch(rng, committee) - - // Construct commitQC for index 1 with AppProposal - // and Proposal for index 2 without any app proposal. - // Such a proposal should fail validation, because app proposals need to be monotone. - l := keys[0].Public() - lQCs := map[LaneID]*LaneQC{l: makeLaneQC(rng, committee, keys, l, 0, GenBlockHeaderHash(rng))} - commitQC0 := makeCommitQC(keys, makeFullProposal(ep, keys, utils.None[*CommitQC](), lQCs, utils.None[*AppQC]())) - appQC0 := makeAppQCFor(keys, commitQC0.GlobalRange().First, 0, GenAppHash(rng), ep.EpochIndex()) - vs1 := ViewSpec{CommitQC: utils.Some(commitQC0), Epoch: ep} - commitQC1a := makeCommitQC(keys, makeFullProposal(ep, keys, utils.Some(commitQC0), oneLaneQCMap(rng, committee, keys, vs1), utils.Some(appQC0))) - commitQC1b := makeCommitQC(keys, makeFullProposal(ep, keys, utils.Some(commitQC0), oneLaneQCMap(rng, committee, keys, vs1), utils.None[*AppQC]())) - vs := ViewSpec{CommitQC: utils.Some(commitQC1a), Epoch: ep} - fp2a := makeFullProposal(ep, keys, utils.Some(commitQC1a), oneLaneQCMap(rng, committee, keys, vs), utils.None[*AppQC]()) - fp2b := makeFullProposal(ep, keys, utils.Some(commitQC1b), oneLaneQCMap(rng, committee, keys, vs), utils.None[*AppQC]()) - - // We construct the invalid proposal by constructing 2 alternative futures: one with appQC, one without. - require.NoError(t, fp2a.Verify(vs)) - require.Error(t, fp2b.Verify(vs)) -} - -func TestProposalVerifyRejectsUnnecessaryAppQC(t *testing.T) { - rng := utils.TestRng() - committee, keys := GenCommittee(rng, 4) - ep := genFreshEpoch(rng, committee) - vs := ViewSpec{Epoch: ep} // no previous commitQC, so app starts at None - - leader := leaderKey(committee, keys, vs.View()) - fp := utils.OrPanic1(NewProposal(leader, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.None[*AppQC]())) - - // Attach an unrequested AppQC. - appQC := makeAppQCFor(keys, ep.FirstBlock(), 0, GenAppHash(rng), ep.EpochIndex()) - tamperedFP := &FullProposal{ - proposal: fp.proposal, - laneQCs: fp.laneQCs, - appQC: utils.Some(appQC), - timeoutQC: fp.timeoutQC, - } - err := tamperedFP.Verify(vs) - require.Error(t, err) -} - -func TestProposalVerifyRejectsMissingAppQC(t *testing.T) { - rng := utils.TestRng() - committee, keys := GenCommittee(rng, 4) - ep := genFreshEpoch(rng, committee) // firstBlock >= 1, so firstBlock-1 is valid - vs := ViewSpec{Epoch: ep} // no previous commitQC - leader := leaderKey(committee, keys, vs.View()) - - // Build a valid proposal with an AppQC, then strip it. - goodAppQC := makeAppQCFor(keys, ep.FirstBlock()-1, 0, GenAppHash(rng), ep.EpochIndex()) - fp := utils.OrPanic1(NewProposal(leader, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.Some(goodAppQC))) - - tamperedFP := &FullProposal{ - proposal: fp.proposal, - laneQCs: fp.laneQCs, - } - err := tamperedFP.Verify(vs) - require.Error(t, err) -} - -func TestProposalVerifyRejectsAppQCMismatch(t *testing.T) { - rng := utils.TestRng() - committee, keys := GenCommittee(rng, 4) - ep := genFreshEpoch(rng, committee) - vs := ViewSpec{Epoch: ep} - leader := leaderKey(committee, keys, vs.View()) - - // Build a valid proposal with an AppQC, then swap in a different one. - goodAppQC := makeAppQCFor(keys, ep.FirstBlock(), 0, GenAppHash(rng), ep.EpochIndex()) - fp := utils.OrPanic1(NewProposal(leader, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.Some(goodAppQC))) - - differentAppQC := makeAppQCFor(keys, ep.FirstBlock(), 0, GenAppHash(rng), ep.EpochIndex()) - tamperedFP := &FullProposal{ - proposal: fp.proposal, - laneQCs: fp.laneQCs, - appQC: utils.Some(differentAppQC), - } - err := tamperedFP.Verify(vs) - require.Error(t, err) -} - -func TestProposalVerifyRejectsAppProposalWrongEpoch(t *testing.T) { - rng := utils.TestRng() - committee, keys := GenCommittee(rng, 4) - - // firstBlock=1 so NextGlobalBlock()=1 and globalNumber=0 is a valid app target. - vs := ViewSpec{Epoch: NewEpoch(0, OpenRoadRange(), time.Time{}, committee, 1)} - leader := leaderKey(committee, keys, vs.View()) - - makeAppQCWithEpoch := func(epochIdx EpochIndex) *AppQC { - p := NewAppProposal(0, 0, GenAppHash(rng), epochIdx) - v := NewAppVote(p) - var votes []*Signed[*AppVote] - for _, k := range keys { - votes = append(votes, Sign(k, v)) - } - return NewAppQC(votes) - } - - // app epoch matches proposal epoch — accepted. - fp := utils.OrPanic1(NewProposal(leader, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.Some(makeAppQCWithEpoch(0)))) - require.NoError(t, fp.Verify(vs)) - - // app epoch differs from proposal epoch — rejected. - fpWrong := utils.OrPanic1(NewProposal(leader, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.Some(makeAppQCWithEpoch(1)))) - require.Error(t, fpWrong.Verify(vs)) -} - -func TestProposalVerifyRejectsInvalidAppQCSignature(t *testing.T) { - rng := utils.TestRng() - committee, keys := GenCommittee(rng, 4) - ep := genFreshEpoch(rng, committee) - vs := ViewSpec{Epoch: ep} - leader := leaderKey(committee, keys, vs.View()) - - appHash := GenAppHash(rng) - goodAppQC := makeAppQCFor(keys, ep.FirstBlock(), 0, appHash, ep.EpochIndex()) - fp := utils.OrPanic1(NewProposal(leader, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.Some(goodAppQC))) - - // Swap in an AppQC signed by NON-committee keys (same hash). - otherKeys := make([]SecretKey, len(keys)) - for i := range otherKeys { - otherKeys[i] = GenSecretKey(rng) - } - badAppQC := makeAppQCFor(otherKeys, ep.FirstBlock(), 0, appHash, ep.EpochIndex()) - tamperedFP := &FullProposal{ - proposal: fp.proposal, - laneQCs: fp.laneQCs, - appQC: utils.Some(badAppQC), - } - err := tamperedFP.Verify(vs) - require.Error(t, err) -} - func TestProposalVerifyRejectsLaneQCHeaderHashMismatch(t *testing.T) { rng := utils.TestRng() committee, keys := GenCommittee(rng, 4) @@ -854,7 +701,7 @@ func TestProposalVerifyRejectsReproposalHashMismatch(t *testing.T) { for _, r := range origP.laneRanges { ranges = append(ranges, r) } - wrongP := newProposal(origP.view, time.Now().Add(time.Hour), ranges, origP.app, origP.GlobalRange().First) + wrongP := newProposal(origP.view, time.Now().Add(time.Hour), ranges, origP.GlobalRange().First) wrongFP := &FullProposal{ proposal: Sign(leader1, wrongP), timeoutQC: reproposal.timeoutQC, diff --git a/sei-tendermint/autobahn/types/testonly.go b/sei-tendermint/autobahn/types/testonly.go index ffc3d5a73d..635d779d6d 100644 --- a/sei-tendermint/autobahn/types/testonly.go +++ b/sei-tendermint/autobahn/types/testonly.go @@ -21,7 +21,6 @@ func BuildCommitQC( keys []SecretKey, prev utils.Option[*CommitQC], laneQCs map[LaneID]*LaneQC, - appQC utils.Option[*AppQC], ) *CommitQC { vs := ViewSpec{CommitQC: prev, Epoch: epoch} if len(laneQCs) == 0 { @@ -35,7 +34,7 @@ func BuildCommitQC( break } } - proposal := utils.OrPanic1(NewProposal(leaderKey, vs, time.Now(), laneQCs, appQC)) + proposal := utils.OrPanic1(NewProposal(leaderKey, vs, time.Now(), laneQCs)) votes := make([]*Signed[*CommitVote], 0, len(keys)) for _, k := range keys { votes = append(votes, Sign(k, NewCommitVote(proposal.Proposal().Msg()))) @@ -314,12 +313,12 @@ func CommitQCAt(ep *Epoch, keys []SecretKey) *CommitQC { // GenProposal generates a random Proposal. func GenProposal(rng utils.Rng) *Proposal { - return newProposal(GenView(rng), utils.GenTimestamp(rng), utils.GenSlice(rng, GenLaneRange), utils.Some(GenAppProposal(rng)), GlobalBlockNumber(rng.Uint64())) + return newProposal(GenView(rng), utils.GenTimestamp(rng), utils.GenSlice(rng, GenLaneRange), GlobalBlockNumber(rng.Uint64())) } // GenProposalAt generates a Proposal at a specific view. func GenProposalAt(rng utils.Rng, view View) *Proposal { - return newProposal(view, utils.GenTimestamp(rng), utils.GenSlice(rng, GenLaneRange), utils.Some(GenAppProposal(rng)), GlobalBlockNumber(rng.Uint64())) + return newProposal(view, utils.GenTimestamp(rng), utils.GenSlice(rng, GenLaneRange), GlobalBlockNumber(rng.Uint64())) } // ProposalAt returns a minimal non-empty Proposal at view, consistent with ep. @@ -330,7 +329,7 @@ func ProposalAt(ep *Epoch, view View) *Proposal { view.EpochIndex = ep.EpochIndex() lane := ep.Committee().Lanes().At(0) header := NewBlock(lane, 0, BlockHeaderHash{}, &Payload{}).Header() - return newProposal(view, time.Time{}, []*LaneRange{NewLaneRange(lane, 0, utils.Some(header))}, utils.None[*AppProposal](), ep.FirstBlock()) + return newProposal(view, time.Time{}, []*LaneRange{NewLaneRange(lane, 0, utils.Some(header))}, ep.FirstBlock()) } // GenProposalForEpoch generates a Proposal at a specific view whose epochIndex, @@ -340,7 +339,7 @@ func GenProposalForEpoch(rng utils.Rng, ep *Epoch, view View) *Proposal { view.EpochIndex = ep.EpochIndex() c := ep.Committee() laneRanges := utils.GenSlice(rng, func(rng utils.Rng) *LaneRange { return GenLaneRangeFor(rng, c) }) - return newProposal(view, utils.GenTimestamp(rng), laneRanges, utils.Some(GenAppProposal(rng)), ep.FirstBlock()) + return newProposal(view, utils.GenTimestamp(rng), laneRanges, ep.FirstBlock()) } // GenAppHash generates a random AppHash. @@ -350,7 +349,7 @@ func GenAppHash(rng utils.Rng) AppHash { // GenAppProposal generates a random AppProposal. func GenAppProposal(rng utils.Rng) *AppProposal { - return NewAppProposal(GenGlobalBlockNumber(rng), GenRoadIndex(rng), GenAppHash(rng), GenEpochIndex(rng)) + return NewAppProposal(GenRoadIndex(rng), GenAppHash(rng), GenEpochIndex(rng)) } // GenAppVote generates a random AppVote. @@ -376,7 +375,6 @@ func GenFullProposal(rng utils.Rng) *FullProposal { return &FullProposal{ proposal: GenSigned(rng, GenProposal(rng)), laneQCs: laneQCs, - appQC: utils.Some(GenAppQC(rng)), timeoutQC: utils.Some(GenTimeoutQC(rng)), } } @@ -391,7 +389,6 @@ func GenGlobalBlock(rng utils.Rng) *GlobalBlock { return &GlobalBlock{ GlobalNumber: GenGlobalBlockNumber(rng), Payload: GenPayload(rng), - FinalAppState: utils.Some(GenAppProposal(rng)), } } diff --git a/sei-tendermint/autobahn/types/timeout.go b/sei-tendermint/autobahn/types/timeout.go index d870891a13..f043796ca4 100644 --- a/sei-tendermint/autobahn/types/timeout.go +++ b/sei-tendermint/autobahn/types/timeout.go @@ -215,7 +215,7 @@ func (m *TimeoutQC) reproposal() (*Proposal, bool) { for _, l := range p.laneRanges { laneRanges = append(laneRanges, l) } - return newProposal(m.View().Next(), p.Timestamp(), laneRanges, p.App(), p.GlobalRange().First), true + return newProposal(m.View().Next(), p.Timestamp(), laneRanges, p.GlobalRange().First), true } // TimeoutVoteConv is the protobuf converter for TimeoutVote. diff --git a/sei-tendermint/autobahn/types/types_test.go b/sei-tendermint/autobahn/types/types_test.go index b82de8462c..7f57a70c99 100644 --- a/sei-tendermint/autobahn/types/types_test.go +++ b/sei-tendermint/autobahn/types/types_test.go @@ -116,7 +116,7 @@ func TestNewTimeoutQC(t *testing.T) { Number: GenViewNumber(rng) % view.Number, EpochIndex: view.EpochIndex, } - p := newProposal(pView, utils.GenTimestamp(rng), utils.GenSlice(rng, GenLaneRange), utils.Some(GenAppProposal(rng)), GlobalBlockNumber(rng.Uint64())) + p := newProposal(pView, utils.GenTimestamp(rng), utils.GenSlice(rng, GenLaneRange), GlobalBlockNumber(rng.Uint64())) if wantView.Less(pView) { wantView = pView } diff --git a/sei-tendermint/autobahn/types/wireguard_test.go b/sei-tendermint/autobahn/types/wireguard_test.go index cbebd32535..86383ab405 100644 --- a/sei-tendermint/autobahn/types/wireguard_test.go +++ b/sei-tendermint/autobahn/types/wireguard_test.go @@ -143,7 +143,7 @@ func TestFullCommitQCWireguardAcceptsMaxValidatorsAndHeaders(t *testing.T) { } laneRanges = append(laneRanges, NewLaneRange(lane, 0, utils.Some(lastHeader))) } - proposal := newProposal(View{}, time.Unix(1, 2), laneRanges, utils.None[*AppProposal](), 0) + proposal := newProposal(View{}, time.Unix(1, 2), laneRanges, 0) vote := NewCommitVote(proposal) votes := make([]*Signed[*CommitVote], len(keys)) for i, key := range keys { diff --git a/sei-tendermint/internal/autobahn/avail/app_votes.go b/sei-tendermint/internal/autobahn/avail/app_votes.go index 7dce877719..f60727b07c 100644 --- a/sei-tendermint/internal/autobahn/avail/app_votes.go +++ b/sei-tendermint/internal/autobahn/avail/app_votes.go @@ -4,6 +4,7 @@ import ( "log/slog" "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/sei-protocol/seilog" ) @@ -14,40 +15,42 @@ type voteSet[V any] struct { votes []V } -type appVotes struct { - byKey map[types.PublicKey]*types.Signed[*types.AppVote] - byHash map[types.Hash[*types.AppVote]]*voteSet[*types.Signed[*types.AppVote]] +type road struct { + epoch *types.Epoch + commitQC *types.CommitQC + appByKey map[types.PublicKey]struct{} + appByHash map[types.Hash[*types.AppVote]]*voteSet[*types.Signed[*types.AppVote]] + appQC utils.Option[*types.AppQC] } -func newAppVotes() appVotes { - return appVotes{ - byKey: map[types.PublicKey]*types.Signed[*types.AppVote]{}, - byHash: map[types.Hash[*types.AppVote]]*voteSet[*types.Signed[*types.AppVote]]{}, +func newRoad(commitQC *types.CommitQC, epoch *types.Epoch) *road { + return &road{ + epoch: epoch, + commitQC: commitQC, + appByKey: map[types.PublicKey]struct{}{}, + appByHash: map[types.Hash[*types.AppVote]]*voteSet[*types.Signed[*types.AppVote]]{}, } } // Returns qc if a new qc has been reached. -func (av appVotes) pushVote(c *types.Committee, vote *types.Signed[*types.AppVote]) (*types.AppQC, bool) { +func (r *road) pushAppVote(vote *types.Signed[*types.AppVote]) { + if r.appQC.IsPresent() { return } k := vote.Key() - if _, ok := av.byKey[k]; ok { - return nil, false - } - av.byKey[k] = vote - byHash, ok := av.byHash[vote.Hash()] + if _, ok := r.appByKey[k]; ok { return } + r.appByKey[k] = struct{}{} + byHash, ok := r.appByHash[vote.Hash()] if !ok { - if len(av.byHash) == 1 { - logger.Error("appHash mismatch", slog.Uint64("n", uint64(vote.Msg().Proposal().GlobalNumber()))) + if len(r.appByHash) == 1 { + // Log an error just at the first conflicting hash. + logger.Error("appHash mismatch", slog.Uint64("n", uint64(vote.Msg().Proposal().RoadIndex()))) } byHash = &voteSet[*types.Signed[*types.AppVote]]{} - av.byHash[vote.Hash()] = byHash - } - if byHash.weight >= c.AppQuorum() { - return nil, false + r.appByHash[vote.Hash()] = byHash } + c := r.epoch.Committee() byHash.weight += c.Weight(k) byHash.votes = append(byHash.votes, vote) if byHash.weight >= c.AppQuorum() { - return types.NewAppQC(byHash.votes), true + r.appQC = utils.Some(types.NewAppQC(byHash.votes)) } - return nil, false } diff --git a/sei-tendermint/internal/autobahn/avail/inner.go b/sei-tendermint/internal/autobahn/avail/inner.go index f25d8bf461..b288564318 100644 --- a/sei-tendermint/internal/autobahn/avail/inner.go +++ b/sei-tendermint/internal/autobahn/avail/inner.go @@ -2,10 +2,9 @@ package avail import ( "fmt" - "log/slog" "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/avail/metrics" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/data" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/consensus/persist" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" ) @@ -16,11 +15,12 @@ import ( // BlockPersister creates lane WALs lazily inside MaybePruneAndPersistLane, but the new // member must also appear in inner.blocks before the next persist cycle. type inner struct { + latestCommitQC utils.AtomicSend[utils.Option[*types.CommitQC]] // latest persisted CommitQC + roads *queue[types.RoadIndex, *road] + nextAppQC types.RoadIndex + + // Epoch is the current epoch for blocks votes collection. epoch *types.Epoch - latestAppQC utils.Option[*types.AppQC] - latestCommitQC utils.AtomicSend[utils.Option[*types.CommitQC]] - appVotes *queue[types.GlobalBlockNumber, appVotes] - commitQCs *queue[types.RoadIndex, *types.CommitQC] blocks map[types.LaneID]*queue[types.BlockNumber, *types.Signed[*types.LaneProposal]] votes map[types.LaneID]*queue[types.BlockNumber, blockVotes] // nextBlockToPersist tracks per-lane how far block persistence has progressed. @@ -52,12 +52,12 @@ type inner struct { // commitQCs are sorted by road index; blocks are sorted by number per lane. // newInner requires both to be contiguous and returns an error on gaps. type loadedAvailState struct { - pruneAnchor utils.Option[*PruneAnchor] commitQCs []persist.LoadedCommitQC blocks map[types.LaneID][]persist.LoadedBlock } -func newInner(epoch *types.Epoch, loaded utils.Option[*loadedAvailState]) (*inner, error) { +func newInner(ds *data.State, loaded utils.Option[*loadedAvailState]) (*inner, error) { + epoch := ds.Registry().LatestEpoch() votes := map[types.LaneID]*queue[types.BlockNumber, blockVotes]{} blocks := map[types.LaneID]*queue[types.BlockNumber, *types.Signed[*types.LaneProposal]]{} for lane := range epoch.Committee().Lanes().All() { @@ -66,18 +66,14 @@ func newInner(epoch *types.Epoch, loaded utils.Option[*loadedAvailState]) (*inne } i := &inner{ - epoch: epoch, - latestAppQC: utils.None[*types.AppQC](), latestCommitQC: utils.NewAtomicSend(utils.None[*types.CommitQC]()), - appVotes: newQueue[types.GlobalBlockNumber, appVotes](), - commitQCs: newQueue[types.RoadIndex, *types.CommitQC](), + roads: newQueue[types.RoadIndex, *road](), + epoch: epoch, blocks: blocks, votes: votes, nextBlockToPersist: make(map[types.LaneID]types.BlockNumber, len(votes)), persistedBlockStart: make(map[types.LaneID]types.BlockNumber, len(votes)), } - i.appVotes.prune(epoch.FirstBlock()) - l, ok := loaded.Get() if !ok { return i, nil @@ -86,33 +82,30 @@ func newInner(epoch *types.Epoch, loaded utils.Option[*loadedAvailState]) (*inne // Apply the persisted prune anchor first: prune() positions all queues // (commitQCs, blocks, votes) so that subsequent pushBack calls insert // at the correct indices without needing reset(). - if anchor, ok := l.pruneAnchor.Get(); ok { - logger.Info("loaded persisted prune anchor", - slog.Uint64("roadIndex", uint64(anchor.AppQC.Proposal().RoadIndex())), - slog.Uint64("globalNumber", uint64(anchor.AppQC.Proposal().GlobalNumber())), - ) - // TODO: use the committee of the anchor's epoch once epoch transitions are wired up. - if _, err := i.prune(epoch.Committee(), anchor.AppQC, anchor.CommitQC); err != nil { - return nil, fmt.Errorf("prune: %w", err) - } - for lane := range i.blocks { - i.persistedBlockStart[lane] = anchor.CommitQC.LaneRange(lane).First() - } + appQC, fQC := ds.LastAppQC() + r := newRoad(fQC.QC(),epoch) + r.appQC = utils.Some(appQC) + i.roads.prune(fQC.Index()) + i.roads.pushBack(r) + i.nextAppQC = fQC.Index()+1 + + for lane := range i.blocks { + i.persistedBlockStart[lane] = fQC.QC().LaneRange(lane).Next() } // Restore persisted CommitQCs. prune() may have already pushed the // anchor's CommitQC, so skip entries below commitQCs.next. for _, lqc := range l.commitQCs { - if lqc.Index < i.commitQCs.next { + if lqc.Index < i.roads.next { continue } - if lqc.Index != i.commitQCs.next { - return nil, fmt.Errorf("non-contiguous persisted commitQCs: expected %d, got %d", i.commitQCs.next, lqc.Index) + if lqc.Index != i.roads.next { + return nil, fmt.Errorf("non-contiguous persisted commitQCs: expected %d, got %d", i.roads.next, lqc.Index) } - i.commitQCs.pushBack(lqc.QC) + i.roads.pushBack(newRoad(lqc.QC, epoch)) } - if i.commitQCs.next > i.commitQCs.first { - i.latestCommitQC.Store(utils.Some(i.commitQCs.q[i.commitQCs.next-1])) + if i.roads.Len()>0 { + i.latestCommitQC.Store(utils.Some(i.roads.q[i.roads.next-1].commitQC)) } // Restore persisted blocks. Since the anchor is persisted first and @@ -158,24 +151,27 @@ func (i *inner) laneQC(lane types.LaneID, n types.BlockNumber) (*types.LaneQC, b return nil, false } +func (i *inner) updateNextAppQC() bool { + updated := false + for i.nextAppQC < i.roads.next && i.roads.q[i.nextAppQC].appQC.IsPresent() { + i.nextAppQC += 1 + updated = true + } + return updated +} + // prune advances the state to account for a new AppQC/CommitQC pair. // Returns true if pruning occurred, false if the QC was stale. -func (i *inner) prune(c *types.Committee, appQC *types.AppQC, commitQC *types.CommitQC) (bool, error) { - idx := appQC.Proposal().RoadIndex() - if idx != commitQC.Proposal().Index() { - return false, fmt.Errorf("mismatched QCs: appQC index %v, commitQC index %v", idx, commitQC.Proposal().Index()) - } - if idx < types.NextOpt(i.latestAppQC) { - return false, nil - } - i.latestAppQC = utils.Some(appQC) - metrics.ObserveAppQC(appQC) - i.commitQCs.prune(idx) - if i.commitQCs.next == idx { - i.commitQCs.pushBack(commitQC) - metrics.ObserveCommitQC(commitQC) +func (i *inner) prune(epoch *types.Epoch, commitQC *types.CommitQC, appQC *types.AppQC) { + idx := commitQC.Index() + if idx < i.roads.first { return } + i.roads.prune(idx) + i.nextAppQC = max(idx,i.nextAppQC) + if idx == i.roads.next { + i.roads.pushBack(newRoad(commitQC,epoch)) } - i.appVotes.prune(commitQC.GlobalRange().First) + i.roads.q[idx].appQC = utils.Some(appQC) + i.updateNextAppQC() for lane := range i.votes { lr := commitQC.LaneRange(lane) i.votes[lr.Lane()].prune(lr.First()) @@ -184,5 +180,4 @@ func (i *inner) prune(c *types.Committee, appQC *types.AppQC, commitQC *types.Co i.nextBlockToPersist[lr.Lane()] = lr.First() } } - return true, nil } diff --git a/sei-tendermint/internal/autobahn/avail/metrics/metrics.gen.go b/sei-tendermint/internal/autobahn/avail/metrics/metrics.gen.go index 4aa8151859..46d58380c9 100644 --- a/sei-tendermint/internal/autobahn/avail/metrics/metrics.gen.go +++ b/sei-tendermint/internal/autobahn/avail/metrics/metrics.gen.go @@ -14,7 +14,6 @@ func init() { Global.commitRoadIndex, Global.appRoadIndex, Global.commitGlobalBlockNumber, - Global.appGlobalBlockNumber, Global.proposalToCommitLatency, Global.commitToCommitLatency, ) @@ -40,12 +39,6 @@ func newMetrics() *metrics { Name: "commit_global_block_number", Help: "Global block number of the highest observed commitQC.", }, nil), - appGlobalBlockNumber: tmprometheus.NewGaugeIntVec(prometheus.GaugeOpts{ - Namespace: MetricsNamespace, - Subsystem: MetricsSubsystem, - Name: "app_global_block_number", - Help: "Global block number of the highest observed appQC.", - }, nil), proposalToCommitLatency: tmprometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: MetricsNamespace, Subsystem: MetricsSubsystem, @@ -74,10 +67,6 @@ func (m *metrics) commitGlobalBlockNumberAt() *tmprometheus.GaugeInt { return m.commitGlobalBlockNumber.WithLabelValues() } -func (m *metrics) appGlobalBlockNumberAt() *tmprometheus.GaugeInt { - return m.appGlobalBlockNumber.WithLabelValues() -} - func (m *metrics) proposalToCommitLatencyAt() *tmprometheus.Histogram { return m.proposalToCommitLatency.WithLabelValues() } diff --git a/sei-tendermint/internal/autobahn/avail/metrics/metrics.go b/sei-tendermint/internal/autobahn/avail/metrics/metrics.go index acfb7a6c6f..61c32f4142 100644 --- a/sei-tendermint/internal/autobahn/avail/metrics/metrics.go +++ b/sei-tendermint/internal/autobahn/avail/metrics/metrics.go @@ -21,8 +21,6 @@ type metrics struct { // Global block number of the highest observed commitQC. commitGlobalBlockNumber prometheus.GaugeIntVec - // Global block number of the highest observed appQC. - appGlobalBlockNumber prometheus.GaugeIntVec // Latency from proposal being constructed to commit being observed. proposalToCommitLatency prometheus.HistogramVec `metrics_buckets:"exp(0.01, 1.2, 35)"` @@ -67,12 +65,10 @@ func ObserveCommitQC(qc *types.CommitQC) { func ObserveAppQC(qc *types.AppQC) { now := time.Now() for mLast := range observedAppQC.Lock() { - if last, ok := mLast.Get(); ok && last.val.Proposal().GlobalNumber() >= qc.Proposal().GlobalNumber() { + if last, ok := mLast.Get(); ok && last.val.Proposal().RoadIndex() >= qc.Proposal().RoadIndex() { return } Global.appRoadIndexAt().Set(int64(qc.Proposal().RoadIndex())) // nolint: gosec - // +1 is for consistency with commitGlobalBlockNumber - Global.appGlobalBlockNumberAt().Set(int64(qc.Proposal().GlobalNumber() + 1)) // nolint: gosec *mLast = utils.Some(observed[*types.AppQC]{now, qc}) } } diff --git a/sei-tendermint/internal/autobahn/avail/state.go b/sei-tendermint/internal/autobahn/avail/state.go index 40adf5834a..10f72c8149 100644 --- a/sei-tendermint/internal/autobahn/avail/state.go +++ b/sei-tendermint/internal/autobahn/avail/state.go @@ -48,7 +48,6 @@ func (s *State) PublicKey() types.PublicKey { // (real I/O) or all are no-op (testing). It is a pure I/O struct — all inner // state access goes through State methods. type persisters struct { - pruneAnchor persist.Persister[*pb.PersistedAvailPruneAnchor] blocks *persist.BlockPersister commitQCs *persist.CommitQCPersister } @@ -93,57 +92,19 @@ var PruneAnchorConv = protoutils.Conv[*PruneAnchor, *pb.PersistedAvailPruneAncho // and no state is loaded. When a prune anchor is present, stale commitQCs and // blocks below the anchor are filtered out before returning. func loadPersistedState(dir utils.Option[string]) (utils.Option[*loadedAvailState], persisters, error) { - prunePersister, persistedPruneAnchor, err := persist.NewPersister[*pb.PersistedAvailPruneAnchor](dir, innerFile) - if err != nil { - return utils.None[*loadedAvailState](), persisters{}, fmt.Errorf("NewPersister %s: %w", innerFile, err) - } - bp, blocks, err := persist.NewBlockPersister(dir) if err != nil { return utils.None[*loadedAvailState](), persisters{}, fmt.Errorf("NewBlockPersister: %w", err) } - cp, commitQCs, err := persist.NewCommitQCPersister(dir) if err != nil { return utils.None[*loadedAvailState](), persisters{}, fmt.Errorf("NewCommitQCPersister: %w", err) } - - pers := persisters{pruneAnchor: prunePersister, blocks: bp, commitQCs: cp} - + pers := persisters{blocks: bp, commitQCs: cp} if _, ok := dir.Get(); !ok { return utils.None[*loadedAvailState](), pers, nil } - loaded := &loadedAvailState{commitQCs: commitQCs, blocks: blocks} - - if raw, ok := persistedPruneAnchor.Get(); ok { - anchor, err := PruneAnchorConv.Decode(raw) - if err != nil { - return utils.None[*loadedAvailState](), persisters{}, fmt.Errorf("decode prune anchor: %w", err) - } - loaded.pruneAnchor = utils.Some(anchor) - - anchorIdx := anchor.AppQC.Proposal().RoadIndex() - filtered := commitQCs[:0] - for _, lqc := range commitQCs { - if lqc.Index >= anchorIdx { - filtered = append(filtered, lqc) - } - } - loaded.commitQCs = filtered - - for lane, bs := range blocks { - first := anchor.CommitQC.LaneRange(lane).First() - j := 0 - for j < len(bs) && bs[j].Number < first { - j++ - } - if j > 0 { - loaded.blocks[lane] = bs[j:] - } - } - } - return utils.Some(loaded), pers, nil } @@ -155,28 +116,10 @@ func NewState(key types.SecretKey, data *data.State, stateDir utils.Option[strin if err != nil { return nil, err } - - ep := data.Registry().LatestEpoch() - inner, err := newInner(ep, loaded) + inner, err := newInner(data, loaded) if err != nil { return nil, err } - - // Truncate WAL entries below the prune anchor that were filtered out by - // loadPersistedState. - if ls, ok := loaded.Get(); ok { - if anchor, ok := ls.pruneAnchor.Get(); ok { - for lane := range ep.Committee().Lanes().All() { - if err := pers.blocks.MaybePruneAndPersistLane(lane, utils.Some(anchor.CommitQC), nil, utils.None[func(*types.Signed[*types.LaneProposal])]()); err != nil { - return nil, fmt.Errorf("prune stale block WAL entries: %w", err) - } - } - if err := pers.commitQCs.MaybePruneAndPersist(utils.Some(anchor.CommitQC), nil, utils.None[func(*types.CommitQC)]()); err != nil { - return nil, fmt.Errorf("prune stale commitQC WAL entries: %w", err) - } - } - } - return &State{ key: key, data: data, @@ -187,7 +130,7 @@ func NewState(key types.SecretKey, data *data.State, stateDir utils.Option[strin func (s *State) FirstCommitQC() types.RoadIndex { for inner := range s.inner.Lock() { - return inner.commitQCs.first + return inner.roads.first } panic("unreachable") } @@ -205,78 +148,56 @@ func (s *State) LastCommitQC() utils.AtomicRecv[utils.Option[*types.CommitQC]] { panic("unreachable") } -func (s *State) waitForCommitQC(ctx context.Context, idx types.RoadIndex) error { - _, err := s.LastCommitQC().Wait(ctx, func(qc utils.Option[*types.CommitQC]) bool { - return types.NextIndexOpt(qc) > idx - }) - return err -} - -// LastAppQC returns the latest observed AppQC. -func (s *State) LastAppQC() utils.Option[*types.AppQC] { - for inner := range s.inner.Lock() { - return inner.latestAppQC +func (s *State) commitQC(ctx context.Context, idx types.RoadIndex) (*types.Epoch, *types.CommitQC, error) { + for inner,ctrl := range s.inner.Lock() { + if err:=ctrl.WaitUntil(ctx, func() bool{ return idx < inner.roads.next }); err!=nil { return nil,nil,err } + if idx < inner.roads.first { return nil,nil,types.ErrPruned } + r := inner.roads.q[idx] + return r.epoch,r.commitQC,nil } panic("unreachable") } +func (s *State) CommitQC(ctx context.Context, idx types.RoadIndex) (*types.CommitQC,error) { + _,qc,err := s.commitQC(ctx,idx) + return qc,err +} + // WaitForAppQC waits until there is an AppQC for the given index or higher. // Returns this AppQC and the corresponding CommitQC. // Together they provide enough information to prune the availability state. func (s *State) WaitForAppQC(ctx context.Context, idx types.RoadIndex) (*types.AppQC, *types.CommitQC, error) { for inner, ctrl := range s.inner.Lock() { - for { - if appQC, ok := inner.latestAppQC.Get(); ok { - if x := appQC.Proposal().RoadIndex(); x >= idx && inner.commitQCs.next > x { - return appQC, inner.commitQCs.q[x], nil - } - } - if err := ctrl.Wait(ctx); err != nil { - return nil, nil, err - } - } + if err:=ctrl.WaitUntil(ctx, func() bool { return idx 0 { - if err := s.waitForCommitQC(ctx, idx-1); err != nil { - return err - } + for inner,ctrl := range s.inner.Lock() { + if err:=ctrl.WaitUntil(ctx,func() bool { return idx <= inner.roads.next }); err!=nil { return err } + if inner.roads.next > idx { return nil } } - ep, ok := s.data.Registry().EpochByIndex(qc.Proposal().EpochIndex()) + epoch, ok := s.data.Registry().EpochByIndex(qc.Proposal().EpochIndex()) if !ok { return fmt.Errorf("unknown epoch_index %d", qc.Proposal().EpochIndex()) } - if err := qc.Verify(ep); err != nil { + if err := qc.Verify(epoch); err != nil { return fmt.Errorf("qc.Verify(): %w", err) } for inner, ctrl := range s.inner.Lock() { - if idx != inner.commitQCs.next { - return nil - } - if qc.Proposal().EpochIndex() != inner.epoch.EpochIndex() { - return fmt.Errorf("commitQC epoch_index %d != current epoch %d", qc.Proposal().EpochIndex(), inner.epoch.EpochIndex()) - } - inner.commitQCs.pushBack(qc) + if idx != inner.roads.next { return nil } + inner.roads.pushBack(newRoad(qc,epoch)) metrics.ObserveCommitQC(qc) // The persist goroutine publishes latestCommitQC after writing to disk // (or immediately for no-op persisters), so consensus won't advance @@ -289,44 +210,22 @@ func (s *State) PushCommitQC(ctx context.Context, qc *types.CommitQC) error { // PushAppVote pushes an AppVote to the state. func (s *State) PushAppVote(ctx context.Context, v *types.Signed[*types.AppVote]) error { - ep, ok := s.data.Registry().EpochByIndex(v.Msg().Proposal().EpochIndex()) - if !ok { - return fmt.Errorf("unknown epoch_index %d", v.Msg().Proposal().EpochIndex()) - } - committee := ep.Committee() + // Wait for the corresponding commitQC. idx := v.Msg().Proposal().RoadIndex() - if err := v.VerifySig(committee); err != nil { - return fmt.Errorf("v.VerifySig(): %w", err) + epoch, commitQC,err := s.commitQC(ctx, idx) + if err != nil { return ignorePruned(err) } + if err := v.Msg().Proposal().Verify(commitQC); err != nil { + return fmt.Errorf("invalid vote: %w", err) } - // Wait for the corresponding commitQC. - if err := s.waitForCommitQC(ctx, idx); err != nil { - return err + if err := v.VerifySig(epoch.Committee()); err != nil { + return fmt.Errorf("v.VerifySig(): %w", err) } for inner, ctrl := range s.inner.Lock() { - // Early exit if not useful (we collect <=1 AppQC per road index). - if idx < types.NextOpt(inner.latestAppQC) { + if idx < inner.roads.first || inner.roads.next >= idx { return nil } - // Verify the vote against the CommitQC. - qc := inner.commitQCs.q[idx] - if err := v.Msg().Proposal().Verify(qc); err != nil { - return fmt.Errorf("invalid vote: %w", err) - } - // Push the vote. - n := v.Msg().Proposal().GlobalNumber() - q := inner.appVotes - for q.next <= n { - q.pushBack(newAppVotes()) - } - appQC, ok := q.q[n].pushVote(committee, v) - if !ok { - return nil - } - updated, err := inner.prune(committee, appQC, qc) - if err != nil { - return err - } - if updated { + inner.roads.q[idx].pushAppVote(v) + if inner.updateNextAppQC() { ctrl.Updated() } } @@ -335,42 +234,32 @@ func (s *State) PushAppVote(ctx context.Context, v *types.Signed[*types.AppVote] // PushAppQC pushes an AppQC to the state. It requires a corresponding CommitQC // as a justification. -func (s *State) PushAppQC(appQC *types.AppQC, commitQC *types.CommitQC) error { +func (s *State) prune(appQC *types.AppQC, commitQC *types.CommitQC) error { // Check whether it is needed before verifying. for inner := range s.inner.Lock() { - if types.NextOpt(inner.latestAppQC) > appQC.Proposal().RoadIndex() { + if commitQC.Index() <= inner.roads.first { return nil } } - ep, ok := s.data.Registry().EpochByIndex(commitQC.Proposal().EpochIndex()) + if got, want := appQC.Proposal().EpochIndex(), commitQC.Proposal().EpochIndex(); got != want { + return fmt.Errorf("appQC epoch_index %d != commitQC epoch_index %d", got, want) + } + if appQC.Proposal().RoadIndex() != commitQC.Proposal().Index() { + return fmt.Errorf("mismatched QCs: appQC index %v, commitQC index %v", appQC.Proposal().RoadIndex(), commitQC.Proposal().Index()) + } + epoch, ok := s.data.Registry().EpochByIndex(commitQC.Proposal().EpochIndex()) if !ok { return fmt.Errorf("unknown epoch_index %d", commitQC.Proposal().EpochIndex()) } - if err := appQC.Verify(ep.Committee()); err != nil { + if err := appQC.Verify(epoch.Committee()); err != nil { return fmt.Errorf("appQC.Verify(): %w", err) } - if err := commitQC.Verify(ep); err != nil { + if err := commitQC.Verify(epoch); err != nil { return fmt.Errorf("commitQC.Verify(): %w", err) } - if appQC.Proposal().RoadIndex() != commitQC.Proposal().Index() { - return fmt.Errorf("mismatched QCs: appQC index %v, commitQC index %v", appQC.Proposal().RoadIndex(), commitQC.Proposal().Index()) - } - if got, want := appQC.Proposal().EpochIndex(), commitQC.Proposal().EpochIndex(); got != want { - return fmt.Errorf("appQC epoch_index %d != commitQC epoch_index %d", got, want) - } - // Defense-in-depth check, it should never happen that >f validators sign - // a proposal which does not match the commitQC's global range. - if !commitQC.GlobalRange().Has(appQC.Proposal().GlobalNumber()) { - return fmt.Errorf("appQC GlobalNumber not in commitQC range") - } for inner, ctrl := range s.inner.Lock() { - updated, err := inner.prune(ep.Committee(), appQC, commitQC) - if err != nil { - return err - } - if updated { - ctrl.Updated() - } + inner.prune(epoch, commitQC, appQC) + ctrl.Updated() } return nil } @@ -702,15 +591,6 @@ func (s *State) runPersist(ctx context.Context, pers persisters) error { // Prune CommitQC anchor: same Option drives commit-QC WAL and per-lane block WAL // (truncate-then-append below this QC). var anchorQC utils.Option[*types.CommitQC] - // 1. Persist prune anchor first — establishes the crash-recovery watermark. - if anchor, ok := batch.pruneAnchor.Get(); ok { - if err := pers.pruneAnchor.Persist(PruneAnchorConv.Encode(anchor)); err != nil { - return fmt.Errorf("persist prune anchor: %w", err) - } - s.advancePersistedBlockStart(anchor.CommitQC) - lastPersistedAppQCNext = anchor.CommitQC.Proposal().Index() + 1 - anchorQC = utils.Some(anchor.CommitQC) - } markBlock := func(p *types.Signed[*types.LaneProposal]) { header := p.Msg().Block().Header() @@ -766,7 +646,7 @@ func (s *State) runPersist(ctx context.Context, pers persisters) error { type persistBatch struct { blocks []*types.Signed[*types.LaneProposal] commitQCs []*types.CommitQC - pruneAnchor utils.Option[*PruneAnchor] + appQCs []*types.AppQC } // advancePersistedBlockStart updates the per-lane block admission watermark diff --git a/sei-tendermint/internal/autobahn/avail/subscriptions.go b/sei-tendermint/internal/autobahn/avail/subscriptions.go index 76c6634409..6c4468ab21 100644 --- a/sei-tendermint/internal/autobahn/avail/subscriptions.go +++ b/sei-tendermint/internal/autobahn/avail/subscriptions.go @@ -72,7 +72,7 @@ func (r *LaneVotesRecv) RecvBatch(ctx context.Context) ([]*types.Signed[*types.L type AppVotesRecv struct { state *State - next types.GlobalBlockNumber + next types.RoadIndex } func (s *State) SubscribeAppVotes() *AppVotesRecv { @@ -83,7 +83,7 @@ func (r *AppVotesRecv) Recv(ctx context.Context) (*types.Signed[*types.AppVote], for { // If needed, fast forward to the first global number without known AppQC. if qc, ok := r.state.LastAppQC().Get(); ok { - r.next = max(r.next, qc.Proposal().GlobalNumber()+1) + r.next = max(r.next, qc.Proposal().RoadIndex()+1) } // Fetch the proposal. p, err := r.state.data.AppProposal(ctx, r.next) diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index f5bbd6ca32..c9c2ba0cc4 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -43,6 +43,7 @@ type inner struct { // Map key ranges (low end = first): // qcs: [first, nextQC) // blocks: [first, nextBlock) + gap-fills in [nextBlock, nextQC) + // appQCS: [first, nextAppQC) // appProposals: [first, nextAppProposal) // blockHashes: mirrors blocks (insertBlock / evictBelowBound) // @@ -51,20 +52,22 @@ type inner struct { qcs map[types.GlobalBlockNumber]*types.FullCommitQC blocks map[types.GlobalBlockNumber]*types.Block appProposals map[types.GlobalBlockNumber]*types.AppProposal + appQCs map[types.GlobalBlockNumber]*types.AppQC blockHashes map[types.BlockHeaderHash]types.GlobalBlockNumber // first is the exclusive low end of retained in-memory state: maps keep // [first, next*). Set by newInner / skipTo; advanced by evictBelowBound to - // min(nextAppProposal, App.GlobalNumber()+1) when a CommitQC.App exists. + // min(nextAppProposal, nextAppQC). // nextToExecute reads the next (or tip) QC from maps — it does not need // nextAppProposal-1 retained after eviction. // - // first <= nextAppProposal <= nextBlockToPersist <= nextBlock <= nextQC + // first <= nextAppProposal,nextAppQC <= nextBlockToPersist <= nextBlock <= nextQC // // AppProposals require persistence (nextAppProposal <= nextBlockToPersist). // BlockDB prune status lives only in the store watermark (see PruneBefore). first types.GlobalBlockNumber nextAppProposal types.GlobalBlockNumber + nextAppQC types.GlobalBlockNumber nextBlockToPersist types.GlobalBlockNumber nextBlock types.GlobalBlockNumber nextQC types.GlobalBlockNumber @@ -74,6 +77,7 @@ func newInner(firstBlock types.GlobalBlockNumber) *inner { return &inner{ qcs: map[types.GlobalBlockNumber]*types.FullCommitQC{}, blocks: map[types.GlobalBlockNumber]*types.Block{}, + appQCs: map[types.GlobalBlockNumber]*types.AppQC{}, appProposals: map[types.GlobalBlockNumber]*types.AppProposal{}, blockHashes: map[types.BlockHeaderHash]types.GlobalBlockNumber{}, first: firstBlock, @@ -556,7 +560,6 @@ func assembleGlobalBlock(n types.GlobalBlockNumber, b *types.Block, fqc *types.F Timestamp: qc.Proposal().BlockTimestamp(n).OrPanic("global block not in QC"), Header: b.Header(), Payload: b.Payload(), - FinalAppState: qc.Proposal().App(), } } @@ -639,23 +642,24 @@ func (s *State) globalBlockByHashFromDB(hash types.BlockHeaderHash) (utils.Optio // PushAppHash marks blocks up to n as executed. Hash is the execution result. // Waits for the block to be durably persisted before proceeding. func (s *State) PushAppHash(ctx context.Context, n types.GlobalBlockNumber, hash types.AppHash) error { - for inner, ctrl := range s.inner.Lock() { - if n < inner.nextAppProposal { - return fmt.Errorf("received app proposal out of order: got %v, want >= %v", n, inner.nextAppProposal) - } + for inner, ctrl := range s.inner.Lock() { if err := ctrl.WaitUntil(ctx, func() bool { return n < inner.nextBlockToPersist }); err != nil { return err } - proposal := types.NewAppProposal( - n, - inner.qcs[n].QC().Proposal().Index(), - hash, - inner.qcs[n].QC().Proposal().EpochIndex(), - ) + p := inner.qcs[n].QC().Proposal() + gr := p.GlobalRange() + if gr.First!=inner.nextAppProposal { + return fmt.Errorf("unexpected app proposal : got %v, want in [%v;%v)", n, gr.First,gr.Next) + } + // We only care about the AppHash of the last block of the CommitQC. + if gr.Next!=n+1 { + return nil + } + proposal := types.NewAppProposal(p.Index(),hash,p.EpochIndex()) t := time.Now() - for inner.nextAppProposal <= n { + for inner.nextAppProposal < gr.Next { b := inner.blocks[inner.nextAppProposal] latency := t.Sub(b.Payload().CreatedAt()).Seconds() s.metrics.Blocks.Execute.Observe(latency) @@ -669,22 +673,27 @@ func (s *State) PushAppHash(ctx context.Context, n types.GlobalBlockNumber, hash return nil } -// AppProposal returns the lowest AppProposal containing the block n. +// AppProposal returns the AppProposal containing the block n. // WARNING: currently we do not enforce all blocks to have AppProposal, therefore // an AppProposal for a later block might be returned instead. -func (s *State) AppProposal(ctx context.Context, n types.GlobalBlockNumber) (*types.AppProposal, error) { +func (s *State) AppProposal(ctx context.Context, n types.GlobalBlockNumber) (*types.AppProposal, *types.FullCommitQC, error) { for inner, ctrl := range s.inner.Lock() { if err := ctrl.WaitUntil(ctx, func() bool { return n < inner.nextAppProposal }); err != nil { - return nil, err + return nil, nil, err } if n < inner.first { - return nil, types.ErrPruned + return nil, nil, types.ErrPruned } - ap, ok := inner.appProposals[n] - if !ok { - return nil, types.ErrPruned - } - return ap, nil + return inner.appProposals[n], inner.qcs[n], nil + } + panic("unreachable") +} + +func (s *State) LastAppQC() (*types.AppQC,*types.FullCommitQC) { + for i := range s.inner.Lock() { + // TODO: currently no guarantee that there is >=1 element. + n := i.nextAppQC-1 + return i.appQCs[n],i.qcs[n] } panic("unreachable") } @@ -692,33 +701,10 @@ func (s *State) AppProposal(ctx context.Context, n types.GlobalBlockNumber) (*ty func (i *inner) nextToExecute(lane types.LaneID) types.BlockNumber { // TODO(gprusak): decide whether 0 is a good result in this case in general. // Empty maps (first == nextQC) only on fresh start / after skipTo with no QC. - if i.first == i.nextQC { + if i.first == i.nextAppProposal { return 0 } - // Fully executed through the certified tip: next lane block is past the tip QC. - if i.nextAppProposal == i.nextQC { - return i.qcs[i.nextAppProposal-1].QC().LaneRange(lane).Next() - } - // nextAppProposal < nextQC: derive from the next global block to execute - // (header from FullCommitQC — works even if blocks[n] was never gap-filled). - n := i.nextAppProposal - fqc := i.qcs[n] - qc := fqc.QC() - gr := qc.GlobalRange() - h := fqc.Headers()[n-gr.First] - r := qc.LaneRange(lane) - x := lane.Compare(h.Lane()) - // NOTE: here we assume the specific ordering of lane blocks in the CommitQC: - // TODO(gprusak): move this logic closer to CommitQC - switch { - case x < 0: - return r.Next() - case x > 0: - return r.First() - default: - // This block is not executed yet. - return h.BlockNumber() - } + return i.qcs[i.nextAppProposal-1].QC().LaneRange(lane).Next() } // Waits until lane block n is executed, returns the next block of this lane to be executed (>n) @@ -833,21 +819,6 @@ func (s *State) runPersist(ctx context.Context) error { } } -// certifiedAppFloor is the exclusive eviction floor from the tip CommitQC's -// embedded App, if any: App.GlobalNumber()+1. Only the tip QC is consulted -// (no backward walk). Returns 0 when maps are empty or the tip has no App. -func (i *inner) certifiedAppFloor() types.GlobalBlockNumber { - if i.first == i.nextQC { - return 0 - } - // [first, nextQC) is dense in qcs, so nextQC-1 is present. - app, ok := i.qcs[i.nextQC-1].QC().Proposal().App().Get() - if !ok { - return 0 - } - return app.GlobalNumber() + 1 -} - // evictBelowBound advances first toward the certified App floor and drops cached // blocks/QCs/AppProposals with n < first. No-op when there is no certified App // or the bound would not advance first. Caller must hold inner's lock. Invoked @@ -865,8 +836,7 @@ func (i *inner) certifiedAppFloor() types.GlobalBlockNumber { // e.g. stash an error on State for a Run monitor, or run eviction as its own // Run subtask. func evictBelowBound(inner *inner) { - floor := inner.certifiedAppFloor() - bound := min(inner.nextAppProposal, floor) + bound := min(inner.nextAppProposal, inner.nextAppQC) if bound <= inner.first { return } @@ -876,6 +846,7 @@ func evictBelowBound(inner *inner) { delete(inner.blocks, n) } delete(inner.qcs, n) + delete(inner.appQCs, n) delete(inner.appProposals, n) } inner.first = bound diff --git a/sei-tendermint/internal/autobahn/data/testonly.go b/sei-tendermint/internal/autobahn/data/testonly.go index 2595b3a25a..d0aafffd1c 100644 --- a/sei-tendermint/internal/autobahn/data/testonly.go +++ b/sei-tendermint/internal/autobahn/data/testonly.go @@ -58,13 +58,7 @@ func TestCommitQC( } } } - var appQC utils.Option[*types.AppQC] - if cqc, ok := prev.Get(); ok { - vs := types.ViewSpec{CommitQC: prev, Epoch: ep} - p := types.NewAppProposal(cqc.GlobalRange().Next-1, vs.View().Index, types.GenAppHash(rng), ep.EpochIndex()) - appQC = utils.Some(TestAppQC(keys, p)) - } - cqc := types.BuildCommitQC(ep, keys, prev, laneQCs, appQC) + cqc := types.BuildCommitQC(ep, keys, prev, laneQCs) return types.NewFullCommitQC(cqc, headers), blockList } From 7a8d996ea1af4b0f3ed089e59a58d47f1ba762e3 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 4 Aug 2026 14:43:25 +0200 Subject: [PATCH 02/61] roughly compiles --- sei-tendermint/autobahn/types/epoch.go | 6 +- sei-tendermint/autobahn/types/proposal.go | 2 +- sei-tendermint/autobahn/types/testonly.go | 2 +- .../internal/autobahn/avail/inner.go | 14 ++- .../internal/autobahn/avail/state.go | 112 +++++++----------- .../internal/autobahn/avail/subscriptions.go | 21 +--- .../autobahn/consensus/persist/blocks.go | 16 ++- .../autobahn/consensus/persist/commitqcs.go | 69 +++++------ .../internal/autobahn/data/state.go | 23 +--- 9 files changed, 100 insertions(+), 165 deletions(-) diff --git a/sei-tendermint/autobahn/types/epoch.go b/sei-tendermint/autobahn/types/epoch.go index 3ab13df54a..f5d92283b0 100644 --- a/sei-tendermint/autobahn/types/epoch.go +++ b/sei-tendermint/autobahn/types/epoch.go @@ -12,15 +12,15 @@ type EpochIndex uint64 // RoadRange is an inclusive range of RoadIndex values [First, Last]. type RoadRange struct { First RoadIndex - Last RoadIndex + Next RoadIndex } // OpenRoadRange returns a RoadRange covering all road indices from 0. // Use in tests and genesis epochs where no upper bound is known yet. -func OpenRoadRange() RoadRange { return RoadRange{First: 0, Last: utils.Max[RoadIndex]()} } +func OpenRoadRange() RoadRange { return RoadRange{First: 0, Next: utils.Max[RoadIndex]()} } // Has reports whether idx falls within this range (inclusive on both ends). -func (r RoadRange) Has(idx RoadIndex) bool { return idx >= r.First && idx <= r.Last } +func (r RoadRange) Has(idx RoadIndex) bool { return r.First <= idx && idx < r.Next } // Epoch holds the complete context for a single epoch. // Retrieved from the local Registry; never transmitted on the wire. diff --git a/sei-tendermint/autobahn/types/proposal.go b/sei-tendermint/autobahn/types/proposal.go index 70079b8117..e44825d7bc 100644 --- a/sei-tendermint/autobahn/types/proposal.go +++ b/sei-tendermint/autobahn/types/proposal.go @@ -110,7 +110,7 @@ func (v View) Verify(ep *Epoch) error { return fmt.Errorf("epoch_index = %d, want %d", got, want) } if rr := ep.RoadRange(); !rr.Has(v.Index) { - return fmt.Errorf("road_index %v not in epoch roads [%v, %v]", v.Index, rr.First, rr.Last) + return fmt.Errorf("road_index %v not in epoch roads [%v, %v)", v.Index, rr.First, rr.Next) } return nil } diff --git a/sei-tendermint/autobahn/types/testonly.go b/sei-tendermint/autobahn/types/testonly.go index 635d779d6d..97f03c1603 100644 --- a/sei-tendermint/autobahn/types/testonly.go +++ b/sei-tendermint/autobahn/types/testonly.go @@ -294,7 +294,7 @@ func GenEpochWithCommittee(rng utils.Rng, committee *Committee) *Epoch { first := RoadIndex(rng.Uint64() % 1000) return NewEpoch( GenEpochIndex(rng), - RoadRange{First: first, Last: first + RoadIndex(rng.Uint64()%10000) + 10}, + RoadRange{First: first, Next: first + RoadIndex(rng.Uint64()%10000) + 10}, utils.GenTimestamp(rng), committee, GlobalBlockNumber(rng.Uint64()%1000000)+1, diff --git a/sei-tendermint/internal/autobahn/avail/inner.go b/sei-tendermint/internal/autobahn/avail/inner.go index b288564318..7bd958cb91 100644 --- a/sei-tendermint/internal/autobahn/avail/inner.go +++ b/sei-tendermint/internal/autobahn/avail/inner.go @@ -52,7 +52,7 @@ type inner struct { // commitQCs are sorted by road index; blocks are sorted by number per lane. // newInner requires both to be contiguous and returns an error on gaps. type loadedAvailState struct { - commitQCs []persist.LoadedCommitQC + commitQCs []*types.CommitQC blocks map[types.LaneID][]persist.LoadedBlock } @@ -95,14 +95,16 @@ func newInner(ds *data.State, loaded utils.Option[*loadedAvailState]) (*inner, e // Restore persisted CommitQCs. prune() may have already pushed the // anchor's CommitQC, so skip entries below commitQCs.next. - for _, lqc := range l.commitQCs { - if lqc.Index < i.roads.next { + for _, qc := range l.commitQCs { + if qc.Index() < i.roads.next { continue } - if lqc.Index != i.roads.next { - return nil, fmt.Errorf("non-contiguous persisted commitQCs: expected %d, got %d", i.roads.next, lqc.Index) + if qc.Index() != i.roads.next { + return nil, fmt.Errorf("non-contiguous persisted commitQCs: expected %d, got %d", i.roads.next, qc.Index()) } - i.roads.pushBack(newRoad(lqc.QC, epoch)) + epoch,ok := ds.Registry().EpochByIndex(qc.Proposal().EpochIndex()) + if !ok { return nil, fmt.Errorf("epoch not found") } + i.roads.pushBack(newRoad(qc, epoch)) } if i.roads.Len()>0 { i.latestCommitQC.Store(utils.Some(i.roads.q[i.roads.next-1].commitQC)) diff --git a/sei-tendermint/internal/autobahn/avail/state.go b/sei-tendermint/internal/autobahn/avail/state.go index 10f72c8149..c9edc8441c 100644 --- a/sei-tendermint/internal/autobahn/avail/state.go +++ b/sei-tendermint/internal/autobahn/avail/state.go @@ -264,6 +264,13 @@ func (s *State) prune(appQC *types.AppQC, commitQC *types.CommitQC) error { return nil } +func (s *State) nextAppQC() types.RoadIndex { + for inner := range s.inner.Lock() { + return inner.nextAppQC + } + panic("unreachable") +} + // NextBlock returns the index of the next missing block in local storage for the given lane. func (s *State) NextBlock(lane types.LaneID) types.BlockNumber { for inner := range s.inner.Lock() { @@ -581,58 +588,30 @@ func (s *State) Run(ctx context.Context) error { // TODO: use a single WAL for anchor and CommitQCs to make // this atomic rather than relying on write order. func (s *State) runPersist(ctx context.Context, pers persisters) error { - var lastPersistedAppQCNext types.RoadIndex for { - batch, err := s.collectPersistBatch(ctx, lastPersistedAppQCNext) + batch, err := s.collectPersistBatch(ctx) if err != nil { return err } - // Prune CommitQC anchor: same Option drives commit-QC WAL and per-lane block WAL - // (truncate-then-append below this QC). - var anchorQC utils.Option[*types.CommitQC] - markBlock := func(p *types.Signed[*types.LaneProposal]) { header := p.Msg().Block().Header() s.markBlockPersisted(header.Lane(), header.BlockNumber()+1) } - blocksByLane := make(map[types.LaneID][]*types.Signed[*types.LaneProposal]) - for _, proposal := range batch.blocks { - lane := proposal.Msg().Block().Header().Lane() - blocksByLane[lane] = append(blocksByLane[lane], proposal) - } - // 2. Persist commit-QCs and per-lane blocks in parallel. // Callees handle empty inputs gracefully (no-op when nothing to write/truncate). if err := scope.Parallel(func(ps scope.ParallelScope) error { ps.Spawn(func() error { - return pers.commitQCs.MaybePruneAndPersist(anchorQC, batch.commitQCs, utils.Some(func(qc *types.CommitQC) { - s.markCommitQCsPersisted(qc) - })) - }) - // Collect lanes: any lane with blocks in this batch, plus all lanes - // in the anchor epoch (for WAL pruning). - // TODO: when epoch transitions land, also union in lanes from all - // epochs that appear in batch.commitQCs so new-epoch lanes are - // never skipped in a cross-epoch batch. - batchLanes := map[types.LaneID]struct{}{} - for lane := range blocksByLane { - batchLanes[lane] = struct{}{} - } - if anchor, ok := anchorQC.Get(); ok { - ep, epOK := s.data.Registry().EpochByIndex(anchor.Proposal().EpochIndex()) - if !epOK { - return fmt.Errorf("unknown epoch_index %d", anchor.Proposal().EpochIndex()) - } - for lane := range ep.Committee().Lanes().All() { - batchLanes[lane] = struct{}{} + if err:=pers.commitQCs.PruneAndPersist(batch.commitQCs.first, batch.commitQCs.tail); err!=nil { return err } + if t:= batch.commitQCs.tail; len(t)>0 { + s.markCommitQCsPersisted(t[len(t)-1]) } - } - for lane := range batchLanes { - proposals := blocksByLane[lane] + return nil + }) + for lane,batch := range batch.blocks { ps.Spawn(func() error { - return pers.blocks.MaybePruneAndPersistLane(lane, anchorQC, proposals, utils.Some(markBlock)) + return pers.blocks.Persist(lane, batch.first, batch.tail, utils.Some(markBlock)) }) } return nil @@ -642,11 +621,19 @@ func (s *State) runPersist(ctx context.Context, pers persisters) error { } } +type batch[I any, T any] struct { + first I + tail []T +} + +type blocksBatch = batch[types.BlockNumber,*types.Signed[*types.LaneProposal]] +type commitQCsBatch = batch[types.RoadIndex,*types.CommitQC] + // persistBatch holds the data collected under lock for one persist iteration. type persistBatch struct { - blocks []*types.Signed[*types.LaneProposal] - commitQCs []*types.CommitQC - appQCs []*types.AppQC + epoch *types.Epoch + blocks map[types.LaneID]blocksBatch + commitQCs commitQCsBatch } // advancePersistedBlockStart updates the per-lane block admission watermark @@ -678,15 +665,13 @@ func (s *State) markBlockPersisted(lane types.LaneID, next types.BlockNumber) { // markCommitQCsPersisted publishes the latest persisted CommitQC, // gating consensus from advancing until the QC is durable. func (s *State) markCommitQCsPersisted(qc *types.CommitQC) { - for inner, ctrl := range s.inner.Lock() { + for inner := range s.inner.Lock() { inner.latestCommitQC.Store(utils.Some(qc)) - ctrl.Updated() } } // collectPersistBatch waits for new blocks or commitQCs and collects them under lock. -func (s *State) collectPersistBatch(ctx context.Context, lastPersistedAppQCNext types.RoadIndex) (persistBatch, error) { - var b persistBatch +func (s *State) collectPersistBatch(ctx context.Context) (*persistBatch, error) { for inner, ctrl := range s.inner.Lock() { // Derive the CommitQC persist cursor from latestCommitQC. This is // safe because latestCommitQC is only advanced by markCommitQCsPersisted @@ -694,41 +679,32 @@ func (s *State) collectPersistBatch(ctx context.Context, lastPersistedAppQCNext // update latestCommitQC, so this always reflects persistence state. // The max clamp with commitQCs.first handles the case where prune() // fast-forwarded the queue past the cursor. - commitQCNext := types.NextIndexOpt(inner.latestCommitQC.Load()) + next := types.NextIndexOpt(inner.latestCommitQC.Load()) if err := ctrl.WaitUntil(ctx, func() bool { - if types.NextOpt(inner.latestAppQC) != lastPersistedAppQCNext { - return true - } for lane, q := range inner.blocks { if inner.nextBlockToPersist[lane] < q.next { return true } } - return commitQCNext < inner.commitQCs.next + return next < inner.roads.next }); err != nil { - return b, err + return nil, err } - for lane, q := range inner.blocks { - start := max(inner.nextBlockToPersist[lane], q.first) - for n := start; n < q.next; n++ { - b.blocks = append(b.blocks, q.q[n]) - } + b := &persistBatch { + blocks: map[types.LaneID]blocksBatch{}, + commitQCs: commitQCsBatch{first: inner.roads.first}, } - commitQCNext = max(commitQCNext, inner.commitQCs.first) - for n := commitQCNext; n < inner.commitQCs.next; n++ { - b.commitQCs = append(b.commitQCs, inner.commitQCs.q[n]) - } - if types.NextOpt(inner.latestAppQC) != lastPersistedAppQCNext { - if appQC, ok := inner.latestAppQC.Get(); ok { - idx := appQC.Proposal().RoadIndex() - if qc, ok := inner.commitQCs.q[idx]; ok { - b.pruneAnchor = utils.Some(&PruneAnchor{ - AppQC: appQC, - CommitQC: qc, - }) - } + for n := max(next, inner.roads.first); n < inner.roads.next; n++ { + b.commitQCs.tail = append(b.commitQCs.tail, inner.roads.q[n].commitQC) + } + for lane, q := range inner.blocks { + bb := blocksBatch{first: q.first} + for n := max(inner.nextBlockToPersist[lane], q.first); n < q.next; n++ { + bb.tail = append(bb.tail, q.q[n]) } + b.blocks[lane] = bb } + return b, nil } - return b, nil + panic("unreachable") } diff --git a/sei-tendermint/internal/autobahn/avail/subscriptions.go b/sei-tendermint/internal/autobahn/avail/subscriptions.go index 6c4468ab21..be405b3252 100644 --- a/sei-tendermint/internal/autobahn/avail/subscriptions.go +++ b/sei-tendermint/internal/autobahn/avail/subscriptions.go @@ -72,7 +72,7 @@ func (r *LaneVotesRecv) RecvBatch(ctx context.Context) ([]*types.Signed[*types.L type AppVotesRecv struct { state *State - next types.RoadIndex + next types.GlobalBlockNumber } func (s *State) SubscribeAppVotes() *AppVotesRecv { @@ -81,24 +81,11 @@ func (s *State) SubscribeAppVotes() *AppVotesRecv { func (r *AppVotesRecv) Recv(ctx context.Context) (*types.Signed[*types.AppVote], error) { for { - // If needed, fast forward to the first global number without known AppQC. - if qc, ok := r.state.LastAppQC().Get(); ok { - r.next = max(r.next, qc.Proposal().RoadIndex()+1) - } - // Fetch the proposal. - p, err := r.state.data.AppProposal(ctx, r.next) + vote, qc, err := r.state.data.AppVote(ctx, r.next) if err != nil { - if errors.Is(err, types.ErrPruned) { - r.next = max(r.next+1, r.state.data.FirstAppProposal()) - continue - } return nil, err } - // AppProposal currently might return a proposal with a higher global number than the one we requested. - // Correct the n in such a case. - // TODO(gprusak): perhaps it would be possible to require AppHash at every block from the execution engine. - // This would simplify the data state. - r.next = p.GlobalNumber() + 1 - return types.Sign(r.state.key, types.NewAppVote(p)), nil + r.next = qc.QC().GlobalRange().Next + return types.Sign(r.state.key, vote), nil } } diff --git a/sei-tendermint/internal/autobahn/consensus/persist/blocks.go b/sei-tendermint/internal/autobahn/consensus/persist/blocks.go index 585eb1d74b..b5cd6ba58a 100644 --- a/sei-tendermint/internal/autobahn/consensus/persist/blocks.go +++ b/sei-tendermint/internal/autobahn/consensus/persist/blocks.go @@ -116,17 +116,15 @@ type laneWAL struct { state utils.Mutex[*laneWALState] } -func (lw *laneWAL) maybePruneAndPersist( +func (lw *laneWAL) persist( lane types.LaneID, - anchor utils.Option[*types.CommitQC], + first types.BlockNumber, proposals []*types.Signed[*types.LaneProposal], afterEach utils.Option[func(*types.Signed[*types.LaneProposal])], ) error { for s := range lw.state.Lock() { - if qc, ok := anchor.Get(); ok { - if err := s.truncateForAnchor(lane, qc.LaneRange(lane).First()); err != nil { - return err - } + if err := s.truncateForAnchor(lane, first); err != nil { + return err } for _, p := range proposals { if p.Msg().Block().Header().Lane() != lane { @@ -294,9 +292,9 @@ func (bp *BlockPersister) getOrCreateLane(lane types.LaneID) (*laneWAL, error) { // // The per-lane lock is held for the entire truncate-then-append sequence, // so concurrent calls on the same lane serialize correctly. -func (bp *BlockPersister) MaybePruneAndPersistLane( +func (bp *BlockPersister) Persist( lane types.LaneID, - anchor utils.Option[*types.CommitQC], + first types.BlockNumber, proposals []*types.Signed[*types.LaneProposal], afterEach utils.Option[func(*types.Signed[*types.LaneProposal])], ) error { @@ -313,7 +311,7 @@ func (bp *BlockPersister) MaybePruneAndPersistLane( if err != nil { return err } - return lw.maybePruneAndPersist(lane, anchor, proposals, afterEach) + return lw.persist(lane, first, proposals, afterEach) } // close shuts down all per-lane WALs. Internal: only used by tests and diff --git a/sei-tendermint/internal/autobahn/consensus/persist/commitqcs.go b/sei-tendermint/internal/autobahn/consensus/persist/commitqcs.go index d0291410b7..5c8d6e082d 100644 --- a/sei-tendermint/internal/autobahn/consensus/persist/commitqcs.go +++ b/sei-tendermint/internal/autobahn/consensus/persist/commitqcs.go @@ -11,54 +11,46 @@ import ( const commitqcsDir = "commitqcs" -// LoadedCommitQC is a CommitQC loaded from disk during state restoration. -type LoadedCommitQC struct { - Index types.RoadIndex - QC *types.CommitQC -} - // commitQCState is the mutable state protected by CommitQCPersister's mutex. type commitQCState struct { iw utils.Option[*indexedWAL[*types.CommitQC]] - next types.RoadIndex + persisted types.RoadRange } // persistCommitQC writes a CommitQC to the WAL. Caller must hold the lock. // Duplicates (idx < next) are silently ignored for idempotent startup. // Gaps (idx > next) return an error (breaks linear index mapping). -func (s *commitQCState) persistCommitQC(qc *types.CommitQC) error { +func (s *commitQCState) persist(qc *types.CommitQC) error { idx := qc.Index() - if idx < s.next { + if idx < s.persisted.Next { return nil } - if idx > s.next { - return fmt.Errorf("commitqc %d out of sequence (next=%d)", idx, s.next) + if idx > s.persisted.Next { + return fmt.Errorf("commitqc %d out of sequence (next=%d)", idx, s.persisted.Next) } if iw, ok := s.iw.Get(); ok { if err := iw.Write(qc); err != nil { return fmt.Errorf("persist commitqc %d: %w", idx, err) } } - s.next = idx + 1 + s.persisted.Next += 1 return nil } // deleteBefore truncates WAL entries below the anchor's index, then // re-persists the anchor for crash recovery. Caller must hold the lock. -func (s *commitQCState) deleteBefore(anchor *types.CommitQC) error { - idx := anchor.Index() +func (s *commitQCState) deleteBefore(idx types.RoadIndex) error { iw, ok := s.iw.Get() - if idx >= s.next { - s.next = idx + if idx >= s.persisted.Next { + s.persisted = types.RoadRange{First:idx,Next:idx} if ok && iw.Count() > 0 { if err := iw.TruncateAll(); err != nil { return err } } } else if ok && iw.Count() > 0 { - firstRoadIndex := s.next - types.RoadIndex(iw.Count()) - if idx > firstRoadIndex { - walIdx := iw.FirstIdx() + uint64(idx-firstRoadIndex) + if s.persisted.First < idx { + walIdx := iw.FirstIdx() + uint64(idx-s.persisted.First) if err := iw.TruncateBefore(walIdx, func(entry *types.CommitQC) error { if entry.Index() != idx { return fmt.Errorf("commitqc at WAL index %d has road index %d, expected %d (index mapping broken)", walIdx, entry.Index(), idx) @@ -69,7 +61,7 @@ func (s *commitQCState) deleteBefore(anchor *types.CommitQC) error { } } } - return s.persistCommitQC(anchor) + return nil } // CommitQCPersister manages CommitQC persistence using a WAL. @@ -90,7 +82,7 @@ type CommitQCPersister struct { // new write followed), LoadNext() returns 0. The caller MUST use // MaybePruneAndPersist with the prune CommitQC in Anchor to re-establish the // cursor and re-persist the anchor's CommitQC before appending more QCs. -func NewCommitQCPersister(stateDir utils.Option[string]) (*CommitQCPersister, []LoadedCommitQC, error) { +func NewCommitQCPersister(stateDir utils.Option[string]) (*CommitQCPersister, []*types.CommitQC, error) { sd, ok := stateDir.Get() if !ok { return &CommitQCPersister{state: utils.NewMutex(&commitQCState{})}, nil, nil @@ -108,16 +100,19 @@ func NewCommitQCPersister(stateDir utils.Option[string]) (*CommitQCPersister, [] return nil, nil, err } if len(loaded) > 0 { - s.next = loaded[len(loaded)-1].Index + 1 + s.persisted = types.RoadRange { + First: loaded[0].Index(), + Next: loaded[len(loaded)-1].Index() + 1, + } } return &CommitQCPersister{state: utils.NewMutex(s)}, loaded, nil } // LoadNext returns the road index of the first CommitQC that has not been // persisted (exclusive upper bound of what's on disk). -func (cp *CommitQCPersister) LoadNext() types.RoadIndex { +func (cp *CommitQCPersister) Next() types.RoadIndex { for s := range cp.state.Lock() { - return s.next + return s.persisted.Next } panic("unreachable") } @@ -136,25 +131,15 @@ func (cp *CommitQCPersister) LoadNext() types.RoadIndex { // need not coordinate ordering. // afterEach, when present, is called after each successful append. It is // invoked while the lock is held, so it must not re-enter the persister. -func (cp *CommitQCPersister) MaybePruneAndPersist( - anchor utils.Option[*types.CommitQC], - commitQCs []*types.CommitQC, - afterEach utils.Option[func(*types.CommitQC)], -) error { +func (cp *CommitQCPersister) PruneAndPersist(deleteBefore types.RoadIndex, commitQCs []*types.CommitQC) error { for s := range cp.state.Lock() { - if qc, ok := anchor.Get(); ok { - if err := s.deleteBefore(qc); err != nil { - return err - } + if err := s.deleteBefore(deleteBefore); err != nil { + return err } - fn, hasFn := afterEach.Get() for _, c := range commitQCs { - if err := s.persistCommitQC(c); err != nil { + if err := s.persist(c); err != nil { return err } - if hasFn { - fn(c) - } } return nil } @@ -174,7 +159,7 @@ func (cp *CommitQCPersister) Close() error { panic("unreachable") } -func loadAllCommitQCs(s *commitQCState) ([]LoadedCommitQC, error) { +func loadAllCommitQCs(s *commitQCState) ([]*types.CommitQC, error) { iw, ok := s.iw.Get() if !ok { return nil, nil // no-op persister (persistence disabled) @@ -183,12 +168,12 @@ func loadAllCommitQCs(s *commitQCState) ([]LoadedCommitQC, error) { if err != nil { return nil, err } - loaded := make([]LoadedCommitQC, 0, len(entries)) + loaded := make([]*types.CommitQC, 0, len(entries)) for i, qc := range entries { - if i > 0 && qc.Index() != loaded[i-1].Index+1 { + if i > 0 && qc.Index() != loaded[i-1].Index()+1 { return nil, fmt.Errorf("gap in commitqcs: index %d follows %d", qc.Index(), loaded[i-1].Index) } - loaded = append(loaded, LoadedCommitQC{Index: qc.Index(), QC: qc}) + loaded = append(loaded, qc) } return loaded, nil } diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index c9c2ba0cc4..dfc61f262d 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -321,15 +321,6 @@ func (s *State) loadFromBlockDB(blockDB types.BlockDB) error { // Registry returns the epoch registry. func (s *State) Registry() *epoch.Registry { return s.cfg.Registry } -// FirstAppProposal is the first global number for which AppProposal may become -// available. Requests below it return ErrPruned. -func (s *State) FirstAppProposal() types.GlobalBlockNumber { - for inner := range s.inner.Lock() { - return inner.first - } - panic("unreachable") -} - // insertBlocksByHash matches byHash against stored (already verified) QC // headers over gr ∩ [nextBlock, nextQC) and inserts hits. Advances nextBlock // when the contiguous prefix grows. Caller must hold inner's lock. @@ -673,18 +664,14 @@ func (s *State) PushAppHash(ctx context.Context, n types.GlobalBlockNumber, hash return nil } -// AppProposal returns the AppProposal containing the block n. -// WARNING: currently we do not enforce all blocks to have AppProposal, therefore -// an AppProposal for a later block might be returned instead. -func (s *State) AppProposal(ctx context.Context, n types.GlobalBlockNumber) (*types.AppProposal, *types.FullCommitQC, error) { +// AppVote returns an appVote for a block >= n. +func (s *State) AppVote(ctx context.Context, n types.GlobalBlockNumber) (*types.AppVote, *types.FullCommitQC, error) { for inner, ctrl := range s.inner.Lock() { - if err := ctrl.WaitUntil(ctx, func() bool { return n < inner.nextAppProposal }); err != nil { + if err := ctrl.WaitUntil(ctx, func() bool { return max(inner.nextAppQC,n) < inner.nextAppProposal }); err != nil { return nil, nil, err } - if n < inner.first { - return nil, nil, types.ErrPruned - } - return inner.appProposals[n], inner.qcs[n], nil + n := max(inner.nextAppQC,n) + return types.NewAppVote(inner.appProposals[n]), inner.qcs[n], nil } panic("unreachable") } From 74243c484676949927ce0011e1eb0fc931f25d35 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 4 Aug 2026 15:25:31 +0200 Subject: [PATCH 03/61] snapshot --- .../internal/autobahn/data/state.go | 12 + sei-tendermint/internal/p2p/giga/api.go | 10 +- sei-tendermint/internal/p2p/giga/api.proto | 14 +- sei-tendermint/internal/p2p/giga/avail.go | 24 -- sei-tendermint/internal/p2p/giga/data.go | 31 ++- sei-tendermint/internal/p2p/giga/pb/api.pb.go | 228 +++++++----------- .../internal/p2p/giga/pb/api.wireguard.go | 26 +- sei-tendermint/internal/p2p/giga/types.go | 20 -- 8 files changed, 145 insertions(+), 220 deletions(-) diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index dfc61f262d..1c2c850cc3 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -676,9 +676,21 @@ func (s *State) AppVote(ctx context.Context, n types.GlobalBlockNumber) (*types. panic("unreachable") } +func (s *State) AppQC(ctx context.Context, n types.GlobalBlockNumber) (*types.AppQC, *types.FullCommitQC, error) { + for inner, ctrl := range s.inner.Lock() { + if err := ctrl.WaitUntil(ctx, func() bool { return n < inner.nextAppQC }); err!=nil { return nil,nil,err } + if inner.first <= n { + return inner.appQCs[n],inner.qcs[n],nil + } + } + // TODO: we should fallback to blocksDB + panic("unreachable") +} + func (s *State) LastAppQC() (*types.AppQC,*types.FullCommitQC) { for i := range s.inner.Lock() { // TODO: currently no guarantee that there is >=1 element. + // TODO: nextAppQC is NOT good enough, we need it to be persisted. n := i.nextAppQC-1 return i.appQCs[n],i.qcs[n] } diff --git a/sei-tendermint/internal/p2p/giga/api.go b/sei-tendermint/internal/p2p/giga/api.go index cc7637d71f..e8bfef757d 100644 --- a/sei-tendermint/internal/p2p/giga/api.go +++ b/sei-tendermint/internal/p2p/giga/api.go @@ -36,11 +36,6 @@ var StreamAppVotes = rpc.Register[API](4, "stream_app_votes", rpc.Msg[*pb.StreamAppVotesReq]{MsgSize: kB, Window: 1}, rpc.Msg[*pb.AppVote]{MsgSize: 10 * kB, Window: 100}, ) -var StreamAppQCs = rpc.Register[API](5, "stream_app_qcs", - rpc.Limit{Rate: 1, Concurrent: 1}, - rpc.Msg[*pb.StreamAppQCsReq]{MsgSize: kB, Window: 1}, - rpc.Msg[*pb.StreamAppQCsResp]{MsgSize: 30 * kB, Window: 20}, -) var Consensus = rpc.Register[API](6, "consensus", // Consensus streams are special in a sense that // * each stream sends just 1 message per view @@ -57,6 +52,11 @@ var StreamFullCommitQCs = rpc.Register[API](7, "stream_full_commit_qcs", rpc.Msg[*pb.StreamFullCommitQCsReq]{MsgSize: kB, Window: 1}, rpc.Msg[*apb.FullCommitQC]{MsgSize: 300 * kB, Window: 20}, ) +var StreamAppQCs = rpc.Register[API](5, "stream_app_qcs", + rpc.Limit{Rate: 1, Concurrent: 1}, + rpc.Msg[*pb.StreamAppQCsReq]{MsgSize: kB, Window: 1}, + rpc.Msg[*apb.AppQC]{MsgSize: 30 * kB, Window: 20}, +) var GetBlock = rpc.Register[API](8, "get_block", rpc.Limit{Rate: 10, Concurrent: 10}, rpc.Msg[*pb.GetBlockReq]{MsgSize: 10 * kB, Window: 1}, diff --git a/sei-tendermint/internal/p2p/giga/api.proto b/sei-tendermint/internal/p2p/giga/api.proto index 10ac870470..63ab4f1813 100644 --- a/sei-tendermint/internal/p2p/giga/api.proto +++ b/sei-tendermint/internal/p2p/giga/api.proto @@ -45,15 +45,6 @@ message StreamLaneProposalsReq { uint64 first_block_number = 1; } -message StreamAppQCsReq { - option (wireguard.sized) = true; -} -message StreamAppQCsResp { - option (wireguard.sized) = true; - autobahn.AppQC app_qc = 1; // required - autobahn.CommitQC commit_qc = 2; // required -} - message StreamCommitQCsReq { option (wireguard.sized) = true; } @@ -80,3 +71,8 @@ message StreamFullCommitQCsReq { option (wireguard.sized) = true; uint64 next_block = 1; } + +message StreamAppQCsReq { + option (wireguard.sized) = true; + uint64 next_block = 1; +} diff --git a/sei-tendermint/internal/p2p/giga/avail.go b/sei-tendermint/internal/p2p/giga/avail.go index b6a09ccdb3..c231b87da3 100644 --- a/sei-tendermint/internal/p2p/giga/avail.go +++ b/sei-tendermint/internal/p2p/giga/avail.go @@ -76,30 +76,6 @@ func (x *Service) serverStreamAppVotes(ctx context.Context, server rpc.Server[AP }) } -func (x *Service) serverStreamAppQCs(ctx context.Context, server rpc.Server[API]) error { - return StreamAppQCs.Serve(ctx, server, func(ctx context.Context, stream rpc.Stream[*pb.StreamAppQCsResp, *pb.StreamAppQCsReq]) error { - reqRaw, err := stream.Recv(ctx) - if err != nil { - return err - } - _ = reqRaw - next := types.RoadIndex(0) - for { - appQC, commitQC, err := x.validatorState().Avail().WaitForAppQC(ctx, next) - if err != nil { - return fmt.Errorf("x.validatorState().Avail().WaitForAppQC(): %w", err) - } - next = appQC.Next() - if err := stream.Send(ctx, StreamAppQCsRespConv.Encode(&StreamAppQCsResp{ - AppQC: appQC, - CommitQC: commitQC, - })); err != nil { - return fmt.Errorf("stream.Send(): %w", err) - } - } - }) -} - func (x *Service) serverStreamCommitQCs(ctx context.Context, server rpc.Server[API]) error { return StreamCommitQCs.Serve(ctx, server, func(ctx context.Context, stream rpc.Stream[*apb.CommitQC, *pb.StreamCommitQCsReq]) error { next := types.RoadIndex(0) diff --git a/sei-tendermint/internal/p2p/giga/data.go b/sei-tendermint/internal/p2p/giga/data.go index c79134c5af..318c677837 100644 --- a/sei-tendermint/internal/p2p/giga/data.go +++ b/sei-tendermint/internal/p2p/giga/data.go @@ -157,17 +157,13 @@ func (s *Service) serverStreamFullCommitQCs(ctx context.Context, server rpc.Serv if err != nil { return fmt.Errorf("StreamFullCommitQCsReqConv.Decode(): %w", err) } - prev := utils.None[*types.FullCommitQC]() - for i := req.NextBlock; ; i++ { + for next := req.NextBlock;; { qc, err := s.data.QC(ctx, i) if err != nil { return fmt.Errorf("s.data.QC(): %w", err) } // Don't send the same QC twice. - if types.NextIndexOpt(prev) > qc.Index() { - continue - } - prev = utils.Some(qc) + next = qc.GlobalRange().Next() if err := stream.Send(ctx, types.FullCommitQCConv.Encode(qc)); err != nil { return fmt.Errorf("stream.Send(): %w", err) } @@ -175,6 +171,29 @@ func (s *Service) serverStreamFullCommitQCs(ctx context.Context, server rpc.Serv }) } +func (x *Service) serverStreamAppQCs(ctx context.Context, server rpc.Server[API]) error { + return StreamAppQCs.Serve(ctx, server, func(ctx context.Context, stream rpc.Stream[*apb.AppQC, *pb.StreamAppQCsReq]) error { + reqRaw, err := stream.Recv(ctx) + if err != nil { + return err + } + req, err := StreamAppQCsReqConv.Decode(reqRaw) + if err != nil { + return fmt.Errorf("StreamFullCommitQCsReqConv.Decode(): %w", err) + } + for next:=req.NextBlock;; { + appQC, commitQC, err := x.validatorState().Data().AppQC(ctx, next) + if err != nil { + return fmt.Errorf("x.validatorState().Avail().WaitForAppQC(): %w", err) + } + next = commitQC.QC().GlobalRange().Next + if err := stream.Send(ctx, types.AppQCConv.Encode(appQC)); err != nil { + return fmt.Errorf("stream.Send(): %w", err) + } + } + }) +} + func (x *Service) serverGetBlock(ctx context.Context, server rpc.Server[API]) error { return GetBlock.Serve(ctx, server, func(ctx context.Context, stream rpc.Stream[*pb.GetBlockResp, *pb.GetBlockReq]) error { reqRaw, err := stream.Recv(ctx) diff --git a/sei-tendermint/internal/p2p/giga/pb/api.pb.go b/sei-tendermint/internal/p2p/giga/pb/api.pb.go index 21987c8ed9..d3c14daeca 100644 --- a/sei-tendermint/internal/p2p/giga/pb/api.pb.go +++ b/sei-tendermint/internal/p2p/giga/pb/api.pb.go @@ -307,94 +307,6 @@ func (x *StreamLaneProposalsReq) GetFirstBlockNumber() uint64 { return 0 } -type StreamAppQCsReq struct { - state protoimpl.MessageState `protogen:"open.v1"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *StreamAppQCsReq) Reset() { - *x = StreamAppQCsReq{} - mi := &file_p2p_giga_api_proto_msgTypes[7] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) -} - -func (x *StreamAppQCsReq) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*StreamAppQCsReq) ProtoMessage() {} - -func (x *StreamAppQCsReq) ProtoReflect() protoreflect.Message { - mi := &file_p2p_giga_api_proto_msgTypes[7] - if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use StreamAppQCsReq.ProtoReflect.Descriptor instead. -func (*StreamAppQCsReq) Descriptor() ([]byte, []int) { - return file_p2p_giga_api_proto_rawDescGZIP(), []int{7} -} - -type StreamAppQCsResp struct { - state protoimpl.MessageState `protogen:"open.v1"` - AppQc *pb.AppQC `protobuf:"bytes,1,opt,name=app_qc,json=appQc,proto3" json:"app_qc,omitempty"` // required - CommitQc *pb.CommitQC `protobuf:"bytes,2,opt,name=commit_qc,json=commitQc,proto3" json:"commit_qc,omitempty"` // required - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *StreamAppQCsResp) Reset() { - *x = StreamAppQCsResp{} - mi := &file_p2p_giga_api_proto_msgTypes[8] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) -} - -func (x *StreamAppQCsResp) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*StreamAppQCsResp) ProtoMessage() {} - -func (x *StreamAppQCsResp) ProtoReflect() protoreflect.Message { - mi := &file_p2p_giga_api_proto_msgTypes[8] - if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use StreamAppQCsResp.ProtoReflect.Descriptor instead. -func (*StreamAppQCsResp) Descriptor() ([]byte, []int) { - return file_p2p_giga_api_proto_rawDescGZIP(), []int{8} -} - -func (x *StreamAppQCsResp) GetAppQc() *pb.AppQC { - if x != nil { - return x.AppQc - } - return nil -} - -func (x *StreamAppQCsResp) GetCommitQc() *pb.CommitQC { - if x != nil { - return x.CommitQc - } - return nil -} - type StreamCommitQCsReq struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -403,7 +315,7 @@ type StreamCommitQCsReq struct { func (x *StreamCommitQCsReq) Reset() { *x = StreamCommitQCsReq{} - mi := &file_p2p_giga_api_proto_msgTypes[9] + mi := &file_p2p_giga_api_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -415,7 +327,7 @@ func (x *StreamCommitQCsReq) String() string { func (*StreamCommitQCsReq) ProtoMessage() {} func (x *StreamCommitQCsReq) ProtoReflect() protoreflect.Message { - mi := &file_p2p_giga_api_proto_msgTypes[9] + mi := &file_p2p_giga_api_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -428,7 +340,7 @@ func (x *StreamCommitQCsReq) ProtoReflect() protoreflect.Message { // Deprecated: Use StreamCommitQCsReq.ProtoReflect.Descriptor instead. func (*StreamCommitQCsReq) Descriptor() ([]byte, []int) { - return file_p2p_giga_api_proto_rawDescGZIP(), []int{9} + return file_p2p_giga_api_proto_rawDescGZIP(), []int{7} } type StreamLaneVotesReq struct { @@ -439,7 +351,7 @@ type StreamLaneVotesReq struct { func (x *StreamLaneVotesReq) Reset() { *x = StreamLaneVotesReq{} - mi := &file_p2p_giga_api_proto_msgTypes[10] + mi := &file_p2p_giga_api_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -451,7 +363,7 @@ func (x *StreamLaneVotesReq) String() string { func (*StreamLaneVotesReq) ProtoMessage() {} func (x *StreamLaneVotesReq) ProtoReflect() protoreflect.Message { - mi := &file_p2p_giga_api_proto_msgTypes[10] + mi := &file_p2p_giga_api_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -464,7 +376,7 @@ func (x *StreamLaneVotesReq) ProtoReflect() protoreflect.Message { // Deprecated: Use StreamLaneVotesReq.ProtoReflect.Descriptor instead. func (*StreamLaneVotesReq) Descriptor() ([]byte, []int) { - return file_p2p_giga_api_proto_rawDescGZIP(), []int{10} + return file_p2p_giga_api_proto_rawDescGZIP(), []int{8} } type StreamAppVotesReq struct { @@ -475,7 +387,7 @@ type StreamAppVotesReq struct { func (x *StreamAppVotesReq) Reset() { *x = StreamAppVotesReq{} - mi := &file_p2p_giga_api_proto_msgTypes[11] + mi := &file_p2p_giga_api_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -487,7 +399,7 @@ func (x *StreamAppVotesReq) String() string { func (*StreamAppVotesReq) ProtoMessage() {} func (x *StreamAppVotesReq) ProtoReflect() protoreflect.Message { - mi := &file_p2p_giga_api_proto_msgTypes[11] + mi := &file_p2p_giga_api_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -500,7 +412,7 @@ func (x *StreamAppVotesReq) ProtoReflect() protoreflect.Message { // Deprecated: Use StreamAppVotesReq.ProtoReflect.Descriptor instead. func (*StreamAppVotesReq) Descriptor() ([]byte, []int) { - return file_p2p_giga_api_proto_rawDescGZIP(), []int{11} + return file_p2p_giga_api_proto_rawDescGZIP(), []int{9} } type GetBlockReq struct { @@ -512,7 +424,7 @@ type GetBlockReq struct { func (x *GetBlockReq) Reset() { *x = GetBlockReq{} - mi := &file_p2p_giga_api_proto_msgTypes[12] + mi := &file_p2p_giga_api_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -524,7 +436,7 @@ func (x *GetBlockReq) String() string { func (*GetBlockReq) ProtoMessage() {} func (x *GetBlockReq) ProtoReflect() protoreflect.Message { - mi := &file_p2p_giga_api_proto_msgTypes[12] + mi := &file_p2p_giga_api_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -537,7 +449,7 @@ func (x *GetBlockReq) ProtoReflect() protoreflect.Message { // Deprecated: Use GetBlockReq.ProtoReflect.Descriptor instead. func (*GetBlockReq) Descriptor() ([]byte, []int) { - return file_p2p_giga_api_proto_rawDescGZIP(), []int{12} + return file_p2p_giga_api_proto_rawDescGZIP(), []int{10} } func (x *GetBlockReq) GetGlobalNumber() uint64 { @@ -556,7 +468,7 @@ type GetBlockResp struct { func (x *GetBlockResp) Reset() { *x = GetBlockResp{} - mi := &file_p2p_giga_api_proto_msgTypes[13] + mi := &file_p2p_giga_api_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -568,7 +480,7 @@ func (x *GetBlockResp) String() string { func (*GetBlockResp) ProtoMessage() {} func (x *GetBlockResp) ProtoReflect() protoreflect.Message { - mi := &file_p2p_giga_api_proto_msgTypes[13] + mi := &file_p2p_giga_api_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -581,7 +493,7 @@ func (x *GetBlockResp) ProtoReflect() protoreflect.Message { // Deprecated: Use GetBlockResp.ProtoReflect.Descriptor instead. func (*GetBlockResp) Descriptor() ([]byte, []int) { - return file_p2p_giga_api_proto_rawDescGZIP(), []int{13} + return file_p2p_giga_api_proto_rawDescGZIP(), []int{11} } func (x *GetBlockResp) GetBlock() *pb.Block { @@ -600,7 +512,7 @@ type StreamFullCommitQCsReq struct { func (x *StreamFullCommitQCsReq) Reset() { *x = StreamFullCommitQCsReq{} - mi := &file_p2p_giga_api_proto_msgTypes[14] + mi := &file_p2p_giga_api_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -612,7 +524,7 @@ func (x *StreamFullCommitQCsReq) String() string { func (*StreamFullCommitQCsReq) ProtoMessage() {} func (x *StreamFullCommitQCsReq) ProtoReflect() protoreflect.Message { - mi := &file_p2p_giga_api_proto_msgTypes[14] + mi := &file_p2p_giga_api_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -625,7 +537,7 @@ func (x *StreamFullCommitQCsReq) ProtoReflect() protoreflect.Message { // Deprecated: Use StreamFullCommitQCsReq.ProtoReflect.Descriptor instead. func (*StreamFullCommitQCsReq) Descriptor() ([]byte, []int) { - return file_p2p_giga_api_proto_rawDescGZIP(), []int{14} + return file_p2p_giga_api_proto_rawDescGZIP(), []int{12} } func (x *StreamFullCommitQCsReq) GetNextBlock() uint64 { @@ -635,6 +547,50 @@ func (x *StreamFullCommitQCsReq) GetNextBlock() uint64 { return 0 } +type StreamAppQCsReq struct { + state protoimpl.MessageState `protogen:"open.v1"` + NextBlock uint64 `protobuf:"varint,1,opt,name=next_block,json=nextBlock,proto3" json:"next_block,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StreamAppQCsReq) Reset() { + *x = StreamAppQCsReq{} + mi := &file_p2p_giga_api_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StreamAppQCsReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamAppQCsReq) ProtoMessage() {} + +func (x *StreamAppQCsReq) ProtoReflect() protoreflect.Message { + mi := &file_p2p_giga_api_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamAppQCsReq.ProtoReflect.Descriptor instead. +func (*StreamAppQCsReq) Descriptor() ([]byte, []int) { + return file_p2p_giga_api_proto_rawDescGZIP(), []int{13} +} + +func (x *StreamAppQCsReq) GetNextBlock() uint64 { + if x != nil { + return x.NextBlock + } + return 0 +} + var File_p2p_giga_api_proto protoreflect.FileDescriptor const file_p2p_giga_api_proto_rawDesc = "" + @@ -651,11 +607,7 @@ const file_p2p_giga_api_proto_rawDesc = "" + "\aAppVote\x127\n" + "\vapp_vote_v2\x18\x02 \x01(\v2\x17.autobahn.SignedAppVoteR\tappVoteV2:\x06\xe8\x88\xe2\xab\f\x01J\x04\b\x01\x10\x02R\bapp_vote\"N\n" + "\x16StreamLaneProposalsReq\x12,\n" + - "\x12first_block_number\x18\x01 \x01(\x04R\x10firstBlockNumber:\x06\xe8\x88\xe2\xab\f\x01\"\x19\n" + - "\x0fStreamAppQCsReq:\x06\xe8\x88\xe2\xab\f\x01\"s\n" + - "\x10StreamAppQCsResp\x12&\n" + - "\x06app_qc\x18\x01 \x01(\v2\x0f.autobahn.AppQCR\x05appQc\x12/\n" + - "\tcommit_qc\x18\x02 \x01(\v2\x12.autobahn.CommitQCR\bcommitQc:\x06\xe8\x88\xe2\xab\f\x01\"\x1c\n" + + "\x12first_block_number\x18\x01 \x01(\x04R\x10firstBlockNumber:\x06\xe8\x88\xe2\xab\f\x01\"\x1c\n" + "\x12StreamCommitQCsReq:\x06\xe8\x88\xe2\xab\f\x01\"\x1c\n" + "\x12StreamLaneVotesReq:\x06\xe8\x88\xe2\xab\f\x01\"\x1b\n" + "\x11StreamAppVotesReq:\x06\xe8\x88\xe2\xab\f\x01\":\n" + @@ -666,6 +618,9 @@ const file_p2p_giga_api_proto_rawDesc = "" + "\x06_block\"?\n" + "\x16StreamFullCommitQCsReq\x12\x1d\n" + "\n" + + "next_block\x18\x01 \x01(\x04R\tnextBlock:\x06\xe8\x88\xe2\xab\f\x01\"8\n" + + "\x0fStreamAppQCsReq\x12\x1d\n" + + "\n" + "next_block\x18\x01 \x01(\x04R\tnextBlock:\x06\xe8\x88\xe2\xab\f\x01BGZEgithub.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p/giga/pbb\x06proto3" var ( @@ -680,7 +635,7 @@ func file_p2p_giga_api_proto_rawDescGZIP() []byte { return file_p2p_giga_api_proto_rawDescData } -var file_p2p_giga_api_proto_msgTypes = make([]protoimpl.MessageInfo, 15) +var file_p2p_giga_api_proto_msgTypes = make([]protoimpl.MessageInfo, 14) var file_p2p_giga_api_proto_goTypes = []any{ (*ConsensusResp)(nil), // 0: p2p.giga.ConsensusResp (*PingReq)(nil), // 1: p2p.giga.PingReq @@ -689,33 +644,28 @@ var file_p2p_giga_api_proto_goTypes = []any{ (*LaneProposal)(nil), // 4: p2p.giga.LaneProposal (*AppVote)(nil), // 5: p2p.giga.AppVote (*StreamLaneProposalsReq)(nil), // 6: p2p.giga.StreamLaneProposalsReq - (*StreamAppQCsReq)(nil), // 7: p2p.giga.StreamAppQCsReq - (*StreamAppQCsResp)(nil), // 8: p2p.giga.StreamAppQCsResp - (*StreamCommitQCsReq)(nil), // 9: p2p.giga.StreamCommitQCsReq - (*StreamLaneVotesReq)(nil), // 10: p2p.giga.StreamLaneVotesReq - (*StreamAppVotesReq)(nil), // 11: p2p.giga.StreamAppVotesReq - (*GetBlockReq)(nil), // 12: p2p.giga.GetBlockReq - (*GetBlockResp)(nil), // 13: p2p.giga.GetBlockResp - (*StreamFullCommitQCsReq)(nil), // 14: p2p.giga.StreamFullCommitQCsReq - (*pb.SignedBlockHeader)(nil), // 15: autobahn.SignedBlockHeader - (*pb.SignedBlock)(nil), // 16: autobahn.SignedBlock - (*pb.SignedAppVote)(nil), // 17: autobahn.SignedAppVote - (*pb.AppQC)(nil), // 18: autobahn.AppQC - (*pb.CommitQC)(nil), // 19: autobahn.CommitQC - (*pb.Block)(nil), // 20: autobahn.Block + (*StreamCommitQCsReq)(nil), // 7: p2p.giga.StreamCommitQCsReq + (*StreamLaneVotesReq)(nil), // 8: p2p.giga.StreamLaneVotesReq + (*StreamAppVotesReq)(nil), // 9: p2p.giga.StreamAppVotesReq + (*GetBlockReq)(nil), // 10: p2p.giga.GetBlockReq + (*GetBlockResp)(nil), // 11: p2p.giga.GetBlockResp + (*StreamFullCommitQCsReq)(nil), // 12: p2p.giga.StreamFullCommitQCsReq + (*StreamAppQCsReq)(nil), // 13: p2p.giga.StreamAppQCsReq + (*pb.SignedBlockHeader)(nil), // 14: autobahn.SignedBlockHeader + (*pb.SignedBlock)(nil), // 15: autobahn.SignedBlock + (*pb.SignedAppVote)(nil), // 16: autobahn.SignedAppVote + (*pb.Block)(nil), // 17: autobahn.Block } var file_p2p_giga_api_proto_depIdxs = []int32{ - 15, // 0: p2p.giga.LaneVote.lane_vote_v2:type_name -> autobahn.SignedBlockHeader - 16, // 1: p2p.giga.LaneProposal.lane_proposal_v2:type_name -> autobahn.SignedBlock - 17, // 2: p2p.giga.AppVote.app_vote_v2:type_name -> autobahn.SignedAppVote - 18, // 3: p2p.giga.StreamAppQCsResp.app_qc:type_name -> autobahn.AppQC - 19, // 4: p2p.giga.StreamAppQCsResp.commit_qc:type_name -> autobahn.CommitQC - 20, // 5: p2p.giga.GetBlockResp.block:type_name -> autobahn.Block - 6, // [6:6] is the sub-list for method output_type - 6, // [6:6] is the sub-list for method input_type - 6, // [6:6] is the sub-list for extension type_name - 6, // [6:6] is the sub-list for extension extendee - 0, // [0:6] is the sub-list for field type_name + 14, // 0: p2p.giga.LaneVote.lane_vote_v2:type_name -> autobahn.SignedBlockHeader + 15, // 1: p2p.giga.LaneProposal.lane_proposal_v2:type_name -> autobahn.SignedBlock + 16, // 2: p2p.giga.AppVote.app_vote_v2:type_name -> autobahn.SignedAppVote + 17, // 3: p2p.giga.GetBlockResp.block:type_name -> autobahn.Block + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name } func init() { file_p2p_giga_api_proto_init() } @@ -723,14 +673,14 @@ func file_p2p_giga_api_proto_init() { if File_p2p_giga_api_proto != nil { return } - file_p2p_giga_api_proto_msgTypes[13].OneofWrappers = []any{} + file_p2p_giga_api_proto_msgTypes[11].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_p2p_giga_api_proto_rawDesc), len(file_p2p_giga_api_proto_rawDesc)), NumEnums: 0, - NumMessages: 15, + NumMessages: 14, NumExtensions: 0, NumServices: 0, }, diff --git a/sei-tendermint/internal/p2p/giga/pb/api.wireguard.go b/sei-tendermint/internal/p2p/giga/pb/api.wireguard.go index 4e40412ad9..e56f868527 100644 --- a/sei-tendermint/internal/p2p/giga/pb/api.wireguard.go +++ b/sei-tendermint/internal/p2p/giga/pb/api.wireguard.go @@ -36,14 +36,6 @@ func (*StreamLaneProposalsReq) MaxSize() int { return 11 } -func (*StreamAppQCsReq) MaxSize() int { - return 0 -} - -func (*StreamAppQCsResp) MaxSize() int { - return 30418 -} - func (*StreamCommitQCsReq) MaxSize() int { return 0 } @@ -68,6 +60,10 @@ func (*StreamFullCommitQCsReq) MaxSize() int { return 11 } +func (*StreamAppQCsReq) MaxSize() int { + return 11 +} + func init() { // Register the wireguard.Schema generated for p2p.giga.ConsensusResp. runtime.MustRegister[*ConsensusResp](runtime.Schema{}) @@ -98,15 +94,6 @@ func init() { 1: {MaxCount: 1}, }) - // Register the wireguard.Schema generated for p2p.giga.StreamAppQCsReq. - runtime.MustRegister[*StreamAppQCsReq](runtime.Schema{}) - - // Register the wireguard.Schema generated for p2p.giga.StreamAppQCsResp. - runtime.MustRegister[*StreamAppQCsResp](runtime.Schema{ - 1: {MaxCount: 1, Nested: utils.Some(reflect.TypeFor[*pb.AppQC]())}, - 2: {MaxCount: 1, Nested: utils.Some(reflect.TypeFor[*pb.CommitQC]())}, - }) - // Register the wireguard.Schema generated for p2p.giga.StreamCommitQCsReq. runtime.MustRegister[*StreamCommitQCsReq](runtime.Schema{}) @@ -131,4 +118,9 @@ func init() { 1: {MaxCount: 1}, }) + // Register the wireguard.Schema generated for p2p.giga.StreamAppQCsReq. + runtime.MustRegister[*StreamAppQCsReq](runtime.Schema{ + 1: {MaxCount: 1}, + }) + } diff --git a/sei-tendermint/internal/p2p/giga/types.go b/sei-tendermint/internal/p2p/giga/types.go index d742224f45..00d420e754 100644 --- a/sei-tendermint/internal/p2p/giga/types.go +++ b/sei-tendermint/internal/p2p/giga/types.go @@ -80,26 +80,6 @@ var StreamLaneProposalsReqConv = protoutils.Conv[*StreamLaneProposalsReq, *pb.St }, } -var StreamAppQCsRespConv = protoutils.Conv[*StreamAppQCsResp, *pb.StreamAppQCsResp]{ - Encode: func(m *StreamAppQCsResp) *pb.StreamAppQCsResp { - return &pb.StreamAppQCsResp{ - AppQc: types.AppQCConv.Encode(m.AppQC), - CommitQc: types.CommitQCConv.Encode(m.CommitQC), - } - }, - Decode: func(m *pb.StreamAppQCsResp) (*StreamAppQCsResp, error) { - appQC, err := types.AppQCConv.DecodeReq(m.AppQc) - if err != nil { - return nil, fmt.Errorf("appQC: %w", err) - } - commitQC, err := types.CommitQCConv.DecodeReq(m.CommitQc) - if err != nil { - return nil, fmt.Errorf("commitQC: %w", err) - } - return &StreamAppQCsResp{AppQC: appQC, CommitQC: commitQC}, nil - }, -} - var GetBlockReqConv = protoutils.Conv[*GetBlockReq, *pb.GetBlockReq]{ Encode: func(m *GetBlockReq) *pb.GetBlockReq { return &pb.GetBlockReq{GlobalNumber: uint64(m.GlobalNumber)} From bec2480dbbb73a1c32a384ac6a7b5e3d50435ee0 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 4 Aug 2026 15:36:09 +0200 Subject: [PATCH 04/61] streaming --- sei-tendermint/internal/p2p/giga/avail.go | 24 ---------------- sei-tendermint/internal/p2p/giga/data.go | 28 +++++++++++++++++-- sei-tendermint/internal/p2p/giga/types.go | 13 +++++++++ .../internal/p2p/giga/types_test.go | 5 +--- 4 files changed, 40 insertions(+), 30 deletions(-) diff --git a/sei-tendermint/internal/p2p/giga/avail.go b/sei-tendermint/internal/p2p/giga/avail.go index c231b87da3..3f64bdd38c 100644 --- a/sei-tendermint/internal/p2p/giga/avail.go +++ b/sei-tendermint/internal/p2p/giga/avail.go @@ -201,27 +201,3 @@ func (x *Service) clientStreamAppVotes(ctx context.Context, c rpc.Client[API]) e } } } - -func (x *Service) clientStreamAppQCs(ctx context.Context, c rpc.Client[API]) error { - stream, err := StreamAppQCs.Call(ctx, c) - if err != nil { - return fmt.Errorf("client.StreamAppQCs(): %w", err) - } - defer stream.Close() - if err := stream.Send(ctx, &pb.StreamAppQCsReq{}); err != nil { - return err - } - for { - resp, err := stream.Recv(ctx) - if err != nil { - return fmt.Errorf("stream.Recv(): %w", err) - } - msg, err := StreamAppQCsRespConv.Decode(resp) - if err != nil { - return fmt.Errorf("StreamAppQCsRespConv.Decode(): %w", err) - } - if err := x.validatorState().Avail().PushAppQC(msg.AppQC, msg.CommitQC); err != nil { - return fmt.Errorf("s.PushFirstCommitQC(): %w", err) - } - } -} diff --git a/sei-tendermint/internal/p2p/giga/data.go b/sei-tendermint/internal/p2p/giga/data.go index 318c677837..f8e2ae61ec 100644 --- a/sei-tendermint/internal/p2p/giga/data.go +++ b/sei-tendermint/internal/p2p/giga/data.go @@ -42,6 +42,30 @@ func (s *Service) clientStreamFullCommitQCs(ctx context.Context, client rpc.Clie return ctx.Err() } +func (x *Service) clientStreamAppQCs(ctx context.Context, c rpc.Client[API]) error { + stream, err := StreamAppQCs.Call(ctx, c) + if err != nil { + return fmt.Errorf("client.StreamAppQCs(): %w", err) + } + defer stream.Close() + if err := stream.Send(ctx, &pb.StreamAppQCsReq{}); err != nil { + return err + } + for { + resp, err := stream.Recv(ctx) + if err != nil { + return fmt.Errorf("stream.Recv(): %w", err) + } + appQC, err := types.AppQCConv.Decode(resp) + if err != nil { + return fmt.Errorf("StreamAppQCsRespConv.Decode(): %w", err) + } + if err := x.data.PushAppQC(ctx,appQC); err != nil { + return fmt.Errorf("s.PushFirstCommitQC(): %w", err) + } + } +} + // MaxConcurrentBlockFetches is the maximum number of blocks that client fetches concurrently. const MaxConcurrentBlockFetches = 100 @@ -158,12 +182,12 @@ func (s *Service) serverStreamFullCommitQCs(ctx context.Context, server rpc.Serv return fmt.Errorf("StreamFullCommitQCsReqConv.Decode(): %w", err) } for next := req.NextBlock;; { - qc, err := s.data.QC(ctx, i) + qc, err := s.data.QC(ctx, next) if err != nil { return fmt.Errorf("s.data.QC(): %w", err) } // Don't send the same QC twice. - next = qc.GlobalRange().Next() + next = qc.QC().GlobalRange().Next if err := stream.Send(ctx, types.FullCommitQCConv.Encode(qc)); err != nil { return fmt.Errorf("stream.Send(): %w", err) } diff --git a/sei-tendermint/internal/p2p/giga/types.go b/sei-tendermint/internal/p2p/giga/types.go index 00d420e754..172b05ba9f 100644 --- a/sei-tendermint/internal/p2p/giga/types.go +++ b/sei-tendermint/internal/p2p/giga/types.go @@ -26,6 +26,10 @@ type StreamFullCommitQCsReq struct { NextBlock types.GlobalBlockNumber } +type StreamAppQCsReq struct { + NextBlock types.GlobalBlockNumber +} + var LaneVoteConv = protoutils.Conv[*types.Signed[*types.LaneVote], *pb.LaneVote]{ Encode: func(m *types.Signed[*types.LaneVote]) *pb.LaneVote { return &pb.LaneVote{ @@ -110,3 +114,12 @@ var StreamFullCommitQCsReqConv = protoutils.Conv[*StreamFullCommitQCsReq, *pb.St return &StreamFullCommitQCsReq{NextBlock: types.GlobalBlockNumber(m.NextBlock)}, nil }, } + +var StreamAppQCsReqConv = protoutils.Conv[*StreamAppQCsReq, *pb.StreamAppQCsReq]{ + Encode: func(m *StreamAppQCsReq) *pb.StreamAppQCsReq { + return &pb.StreamAppQCsReq{NextBlock: uint64(m.NextBlock)} + }, + Decode: func(m *pb.StreamAppQCsReq) (*StreamAppQCsReq, error) { + return &StreamAppQCsReq{NextBlock: types.GlobalBlockNumber(m.NextBlock)}, nil + }, +} diff --git a/sei-tendermint/internal/p2p/giga/types_test.go b/sei-tendermint/internal/p2p/giga/types_test.go index 4b5bfb19e3..8b8db6dc92 100644 --- a/sei-tendermint/internal/p2p/giga/types_test.go +++ b/sei-tendermint/internal/p2p/giga/types_test.go @@ -16,14 +16,11 @@ func TestConv(t *testing.T) { LaneProposalConv.Test(types.GenSigned(rng, types.GenLaneProposal(rng))), AppVoteConv.Test(types.GenSigned(rng, types.GenAppVote(rng))), StreamLaneProposalsReqConv.Test(&StreamLaneProposalsReq{FirstBlockNumber: types.GenBlockNumber(rng)}), - StreamAppQCsRespConv.Test(&StreamAppQCsResp{ - AppQC: types.GenAppQC(rng), - CommitQC: types.GenCommitQC(rng), - }), GetBlockReqConv.Test(&GetBlockReq{GlobalNumber: types.GenGlobalBlockNumber(rng)}), GetBlockRespConv.Test(utils.None[*types.Block]()), GetBlockRespConv.Test(utils.Some(types.GenBlock(rng))), StreamFullCommitQCsReqConv.Test(&StreamFullCommitQCsReq{NextBlock: types.GenGlobalBlockNumber(rng)}), + StreamAppQCsReqConv.Test(&StreamAppQCsReq{NextBlock: types.GenGlobalBlockNumber(rng)}), )) } } From 1b5a061a9f4f9f01748a114d290ddf8b78782d98 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 4 Aug 2026 16:02:58 +0200 Subject: [PATCH 05/61] GlobalRange in AppProposal --- sei-tendermint/autobahn/types/app_proposal.go | 48 +++++++--- .../autobahn/types/committee_test.go | 2 +- sei-tendermint/autobahn/types/testonly.go | 6 +- .../internal/autobahn/autobahn.proto | 15 ++-- .../internal/autobahn/avail/conv_test.go | 2 +- .../internal/autobahn/avail/inner_test.go | 51 +++++------ .../internal/autobahn/avail/state.go | 87 +++++++++++-------- .../internal/autobahn/avail/state_test.go | 31 +++---- .../internal/autobahn/data/state.go | 45 +++++----- .../internal/autobahn/data/state_test.go | 24 ++--- .../internal/autobahn/pb/autobahn.pb.go | 64 ++++++++------ .../autobahn/pb/autobahn.wireguard.go | 31 +++---- .../internal/p2p/giga/pb/api.wireguard.go | 2 +- 13 files changed, 229 insertions(+), 179 deletions(-) diff --git a/sei-tendermint/autobahn/types/app_proposal.go b/sei-tendermint/autobahn/types/app_proposal.go index 2b45630434..dcbe936ec3 100644 --- a/sei-tendermint/autobahn/types/app_proposal.go +++ b/sei-tendermint/autobahn/types/app_proposal.go @@ -15,14 +15,20 @@ type AppHash []byte // AppProposal . type AppProposal struct { utils.ReadOnly - epochIndex EpochIndex - roadIndex RoadIndex - appHash AppHash + epochIndex EpochIndex + roadIndex RoadIndex + globalRange GlobalRange + appHash AppHash } // NewAppProposal creates a new AppProposal. -func NewAppProposal(roadIndex RoadIndex, appHash AppHash, epochIndex EpochIndex) *AppProposal { - return &AppProposal{roadIndex: roadIndex, appHash: appHash, epochIndex: epochIndex} +func NewAppProposal(proposal *Proposal, appHash AppHash) *AppProposal { + return &AppProposal{ + globalRange: proposal.GlobalRange(), + roadIndex: proposal.Index(), + appHash: appHash, + epochIndex: proposal.EpochIndex(), + } } // RoadIndex returns the road index of the proposal. @@ -34,7 +40,10 @@ func (m *AppProposal) AppHash() AppHash { return m.appHash } // EpochIndex returns the epoch this proposal belongs to. func (m *AppProposal) EpochIndex() EpochIndex { return m.epochIndex } -// Next is the next global block number to compute AppHash for. +// GlobalRange returns the global block range covered by the proposal. +func (m *AppProposal) GlobalRange() GlobalRange { return m.globalRange } + +// Next is the next road index after this proposal. func (m *AppProposal) Next() RoadIndex { return m.RoadIndex() + 1 } @@ -47,6 +56,9 @@ func (m *AppProposal) Verify(qc *CommitQC) error { if got, want := m.EpochIndex(), qc.Proposal().EpochIndex(); got != want { return fmt.Errorf("epoch_index = %d, want %d", got, want) } + if got, want := m.GlobalRange(), qc.GlobalRange(); got != want { + return fmt.Errorf("global_range = %v, want %v", got, want) + } return nil } @@ -54,9 +66,11 @@ func (m *AppProposal) Verify(qc *CommitQC) error { var AppProposalConv = protoutils.Conv[*AppProposal, *pb.AppProposal]{ Encode: func(m *AppProposal) *pb.AppProposal { return &pb.AppProposal{ - RoadIndex: utils.Alloc(uint64(m.roadIndex)), - AppHash: m.appHash, - EpochIndex: utils.Alloc(uint64(m.epochIndex)), + RoadIndex: utils.Alloc(uint64(m.roadIndex)), + AppHash: m.appHash, + EpochIndex: utils.Alloc(uint64(m.epochIndex)), + GlobalFirst: utils.Alloc(uint64(m.globalRange.First)), + GlobalNext: utils.Alloc(uint64(m.globalRange.Next)), } }, Decode: func(m *pb.AppProposal) (*AppProposal, error) { @@ -66,10 +80,20 @@ var AppProposalConv = protoutils.Conv[*AppProposal, *pb.AppProposal]{ if m.EpochIndex == nil { return nil, fmt.Errorf("epoch_index: missing") } + if m.GlobalFirst == nil { + return nil, fmt.Errorf("global_first: missing") + } + if m.GlobalNext == nil { + return nil, fmt.Errorf("global_next: missing") + } return &AppProposal{ - epochIndex: EpochIndex(*m.EpochIndex), - roadIndex: RoadIndex(*m.RoadIndex), - appHash: AppHash(m.AppHash), + epochIndex: EpochIndex(*m.EpochIndex), + roadIndex: RoadIndex(*m.RoadIndex), + globalRange: GlobalRange{ + First: GlobalBlockNumber(*m.GlobalFirst), + Next: GlobalBlockNumber(*m.GlobalNext), + }, + appHash: AppHash(m.AppHash), }, nil }, } diff --git a/sei-tendermint/autobahn/types/committee_test.go b/sei-tendermint/autobahn/types/committee_test.go index db9e3a3587..4627da4aeb 100644 --- a/sei-tendermint/autobahn/types/committee_test.go +++ b/sei-tendermint/autobahn/types/committee_test.go @@ -171,7 +171,7 @@ func TestCommitQCVerifyChecksWeight(t *testing.T) { func TestAppQCVerifyChecksWeight(t *testing.T) { rng := utils.TestRng() ep, keys := makeEpoch(rng) - vote := NewAppVote(NewAppProposal(0, GenAppHash(rng), ep.EpochIndex())) + vote := NewAppVote(NewAppProposal(ProposalAt(ep, View{EpochIndex: ep.EpochIndex(), Index: ep.RoadRange().First}), GenAppHash(rng))) heavyOnly := NewAppQC([]*Signed[*AppVote]{ Sign(keys[0], vote), diff --git a/sei-tendermint/autobahn/types/testonly.go b/sei-tendermint/autobahn/types/testonly.go index 97f03c1603..e33fef6a46 100644 --- a/sei-tendermint/autobahn/types/testonly.go +++ b/sei-tendermint/autobahn/types/testonly.go @@ -349,7 +349,7 @@ func GenAppHash(rng utils.Rng) AppHash { // GenAppProposal generates a random AppProposal. func GenAppProposal(rng utils.Rng) *AppProposal { - return NewAppProposal(GenRoadIndex(rng), GenAppHash(rng), GenEpochIndex(rng)) + return NewAppProposal(GenProposal(rng), GenAppHash(rng)) } // GenAppVote generates a random AppVote. @@ -387,8 +387,8 @@ func GenGlobalBlockNumber(rng utils.Rng) GlobalBlockNumber { // GenGlobalBlock generates a random GlobalBlock. func GenGlobalBlock(rng utils.Rng) *GlobalBlock { return &GlobalBlock{ - GlobalNumber: GenGlobalBlockNumber(rng), - Payload: GenPayload(rng), + GlobalNumber: GenGlobalBlockNumber(rng), + Payload: GenPayload(rng), } } diff --git a/sei-tendermint/internal/autobahn/autobahn.proto b/sei-tendermint/internal/autobahn/autobahn.proto index 3b04dd242f..e4b3cfc7e8 100644 --- a/sei-tendermint/internal/autobahn/autobahn.proto +++ b/sei-tendermint/internal/autobahn/autobahn.proto @@ -215,16 +215,21 @@ message AppQC { } message AppProposal { + reserved 1; + reserved "global_number"; option (hashable.hashable) = true; option (wireguard.sized) = true; - // Global block number. - optional uint64 global_number = 1; // required + + // Epoch this proposal belongs to. + optional uint64 epoch_index = 4; // required // Index of the commit qc finalizing the block. optional uint64 road_index = 2; // required - // App hash at that block. + // Range of global blocks convered by this proposal. + // Has to match the corresponding CommitQC. + optional uint64 global_first = 5; // required + optional uint64 global_next = 6; // required + // App hash of the state at the end of the range above. optional bytes app_hash = 3 [(wireguard.max_size) = 32]; // required - // Epoch this proposal belongs to. - optional uint64 epoch_index = 4; // required } // This is the signable message. diff --git a/sei-tendermint/internal/autobahn/avail/conv_test.go b/sei-tendermint/internal/autobahn/avail/conv_test.go index ca753e5ee5..210608664c 100644 --- a/sei-tendermint/internal/autobahn/avail/conv_test.go +++ b/sei-tendermint/internal/autobahn/avail/conv_test.go @@ -20,7 +20,7 @@ func TestPruneAnchorConv(t *testing.T) { lane: types.NewLaneQC(makeLaneVotes(keys, block.Header())), } commitQC := makeCommitQC(registry.LatestEpoch(), keys, utils.None[*types.CommitQC](), laneQCs, utils.None[*types.AppQC]()) - appProposal := types.NewAppProposal(commitQC.GlobalRange().First, commitQC.Proposal().Index(), types.GenAppHash(rng), commitQC.Proposal().EpochIndex()) + appProposal := types.NewAppProposal(commitQC.Proposal(), types.GenAppHash(rng)) appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) anchor := &PruneAnchor{AppQC: appQC, CommitQC: commitQC} diff --git a/sei-tendermint/internal/autobahn/avail/inner_test.go b/sei-tendermint/internal/autobahn/avail/inner_test.go index a6deb0a076..86ce237296 100644 --- a/sei-tendermint/internal/autobahn/avail/inner_test.go +++ b/sei-tendermint/internal/autobahn/avail/inner_test.go @@ -30,10 +30,8 @@ func TestPruneMismatchedIndices(t *testing.T) { } return makeCommitQC(registry.LatestEpoch(), keys, prev, lqcs, utils.None[*types.AppQC]()) } - makeAppQC := func(qcForRange *types.CommitQC, qcForIndex *types.CommitQC) *types.AppQC { - gr := qcForRange.GlobalRange() - require.True(t, gr.Len() > 0) - ap := types.NewAppProposal(gr.First, qcForIndex.Index(), types.GenAppHash(rng), 0) + makeAppQC := func(qc *types.CommitQC) *types.AppQC { + ap := types.NewAppProposal(qc.Proposal(), types.GenAppHash(rng)) return types.NewAppQC(makeAppVotes(keys, ap)) } @@ -44,21 +42,19 @@ func TestPruneMismatchedIndices(t *testing.T) { ds := newTestDataState(&data.Config{Registry: registry}) state, err := NewState(keys[0], ds, utils.Some(t.TempDir())) require.NoError(t, err) - require.Error(t, state.PushAppQC(makeAppQC(qc0, qc0), qc1), "bad range, bad index should fail") - require.Error(t, state.PushAppQC(makeAppQC(qc1, qc0), qc1), "good range, bad index should fail") - require.Error(t, state.PushAppQC(makeAppQC(qc0, qc1), qc1), "bad range, good index should fail") - require.NoError(t, state.PushAppQC(makeAppQC(qc1, qc1), qc1), "good range, good index should succeed") + require.Error(t, state.PushAppQC(makeAppQC(qc0), qc1), "mismatched proposal should fail") + require.NoError(t, state.PushAppQC(makeAppQC(qc1), qc1), "matching proposal should succeed") t.Logf("test inner.prune") ds = newTestDataState(&data.Config{Registry: registry}) state, err = NewState(keys[0], ds, utils.Some(t.TempDir())) require.NoError(t, err) for inner := range state.inner.Lock() { - _, err := inner.prune(registry.LatestEpoch().Committee(), makeAppQC(qc1, qc0), qc1) - require.Error(t, err, "good range, bad index should fail") + _, err := inner.prune(registry.LatestEpoch().Committee(), makeAppQC(qc0), qc1) + require.Error(t, err, "mismatched proposal should fail") require.False(t, inner.latestAppQC.IsPresent(), "latestAppQC should not have been updated") - _, err = inner.prune(registry.LatestEpoch().Committee(), makeAppQC(qc1, qc1), qc1) - require.NoError(t, err, "good range, good index should succeed") + _, err = inner.prune(registry.LatestEpoch().Committee(), makeAppQC(qc1), qc1) + require.NoError(t, err, "matching proposal should succeed") } } @@ -93,7 +89,7 @@ func TestDecodePruneAnchorIncomplete(t *testing.T) { rng := utils.TestRng() _, keys := epoch.GenRegistry(rng, 4) - appProposal := types.NewAppProposal(42, 5, types.GenAppHash(rng), 0) + appProposal := types.GenAppProposal(rng) appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) _, err := PruneAnchorConv.Decode(&pb.PersistedAvailPruneAnchor{ @@ -281,9 +277,6 @@ func TestNewInnerLoadedCommitQCsWithAppQC(t *testing.T) { // AppQC at road index 2. roadIdx := types.RoadIndex(2) - globalNum := types.GlobalBlockNumber(10) - appProposal := types.NewAppProposal(globalNum, roadIdx, types.GenAppHash(rng), 0) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) // Create 5 sequential CommitQCs (indices 0-4). qcs := make([]*types.CommitQC, 5) @@ -292,6 +285,8 @@ func TestNewInnerLoadedCommitQCsWithAppQC(t *testing.T) { qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) prev = utils.Some(qcs[i]) } + appProposal := types.NewAppProposal(qcs[roadIdx].Proposal(), types.GenAppHash(rng)) + appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) // Pre-filtered: only commitQCs >= anchor road index (2). loadedQCs := []persist.LoadedCommitQC{ @@ -334,8 +329,6 @@ func TestNewInnerLoadedAllThree(t *testing.T) { // AppQC at road index 2. roadIdx := types.RoadIndex(2) - appProposal := types.NewAppProposal(10, roadIdx, types.GenAppHash(rng), 0) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) // CommitQCs 0-4. qcs := make([]*types.CommitQC, 5) @@ -344,6 +337,8 @@ func TestNewInnerLoadedAllThree(t *testing.T) { qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) prev = utils.Some(qcs[i]) } + appProposal := types.NewAppProposal(qcs[roadIdx].Proposal(), types.GenAppHash(rng)) + appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) // Pre-filtered: only commitQCs >= anchor road index (2). loadedQCs := []persist.LoadedCommitQC{ {Index: 2, QC: qcs[2]}, @@ -432,7 +427,7 @@ func TestPruneAdvancesNextBlockToPersist(t *testing.T) { "CommitQC lane range should reference blocks for this test to be meaningful") // AppQC at index 2 → prune will fast-forward blocks past the cursor. - appProposal := types.NewAppProposal(10, 2, types.GenAppHash(rng), 0) + appProposal := types.NewAppProposal(qcs[2].Proposal(), types.GenAppHash(rng)) appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) updated, err := i.prune(registry.LatestEpoch().Committee(), appQC, qcs[2]) @@ -463,7 +458,7 @@ func TestNewInnerLoadedCommitQCsAllBeforeAppQCArePruned(t *testing.T) { prev = utils.Some(qcs[i]) } - appProposal := types.NewAppProposal(20, 5, types.GenAppHash(rng), 0) + appProposal := types.NewAppProposal(qcs[5].Proposal(), types.GenAppHash(rng)) appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) loaded := &loadedAvailState{ @@ -492,7 +487,7 @@ func TestNewInnerAnchorWithNoCommitQCFiles(t *testing.T) { prev = utils.Some(qcs[i]) } - appProposal := types.NewAppProposal(20, 3, types.GenAppHash(rng), 0) + appProposal := types.NewAppProposal(qcs[3].Proposal(), types.GenAppHash(rng)) appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) loaded := &loadedAvailState{ @@ -578,7 +573,7 @@ func TestNewInnerLoadedCommitQCsGapWithAppQCAnchor(t *testing.T) { prev = utils.Some(qcs[i]) } - appProposal := types.NewAppProposal(50, 10, types.GenAppHash(rng), 0) + appProposal := types.NewAppProposal(qcs[10].Proposal(), types.GenAppHash(rng)) appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) loadedQCs := []persist.LoadedCommitQC{ @@ -623,7 +618,7 @@ func TestNewInnerLoadedCommitQCsBelowAnchorSkipped(t *testing.T) { prev = utils.Some(qcs[i]) } - appProposal := types.NewAppProposal(20, 3, types.GenAppHash(rng), 0) + appProposal := types.NewAppProposal(qcs[3].Proposal(), types.GenAppHash(rng)) appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) loadedQCs := []persist.LoadedCommitQC{ @@ -664,7 +659,7 @@ func TestNewInnerLoadedCommitQCsGapAfterAnchorReturnsError(t *testing.T) { prev = utils.Some(qcs[i]) } - appProposal := types.NewAppProposal(10, 2, types.GenAppHash(rng), 0) + appProposal := types.NewAppProposal(qcs[2].Proposal(), types.GenAppHash(rng)) appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) loadedQCs := []persist.LoadedCommitQC{ @@ -764,7 +759,6 @@ func TestNewInnerLoadedBlocksOverCapacityReturnsError(t *testing.T) { func TestNewInnerPruneAnchorPrunesBlockQueues(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 4) - initialBlock := types.GlobalBlockNumber(0) // Build CommitQCs 0-2. qcs := make([]*types.CommitQC, 3) @@ -775,9 +769,9 @@ func TestNewInnerPruneAnchorPrunesBlockQueues(t *testing.T) { } // AppQC at road index 2, prune anchor is CommitQC[2]. - appProposal := types.NewAppProposal(initialBlock, 2, types.GenAppHash(rng), 0) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) pruneQC := qcs[2] + appProposal := types.NewAppProposal(pruneQC.Proposal(), types.GenAppHash(rng)) + appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) lane := keys[0].Public() @@ -813,7 +807,6 @@ func TestNewInnerPruneAnchorPrunesBlockQueues(t *testing.T) { func TestNewInnerPruneAnchorCommitQCUsedForPrune(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 4) - initialBlock := types.GlobalBlockNumber(0) // Build CommitQCs 0-2. qcs := make([]*types.CommitQC, 3) @@ -824,7 +817,7 @@ func TestNewInnerPruneAnchorCommitQCUsedForPrune(t *testing.T) { } // AppQC at road index 1, prune anchor is CommitQC[1]. - appProposal := types.NewAppProposal(initialBlock, 1, types.GenAppHash(rng), 0) + appProposal := types.NewAppProposal(qcs[1].Proposal(), types.GenAppHash(rng)) appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) loaded := &loadedAvailState{ diff --git a/sei-tendermint/internal/autobahn/avail/state.go b/sei-tendermint/internal/autobahn/avail/state.go index c9edc8441c..a9362936ea 100644 --- a/sei-tendermint/internal/autobahn/avail/state.go +++ b/sei-tendermint/internal/autobahn/avail/state.go @@ -48,8 +48,8 @@ func (s *State) PublicKey() types.PublicKey { // (real I/O) or all are no-op (testing). It is a pure I/O struct — all inner // state access goes through State methods. type persisters struct { - blocks *persist.BlockPersister - commitQCs *persist.CommitQCPersister + blocks *persist.BlockPersister + commitQCs *persist.CommitQCPersister } // innerFile is the A/B file prefix for avail inner state persistence. @@ -149,18 +149,22 @@ func (s *State) LastCommitQC() utils.AtomicRecv[utils.Option[*types.CommitQC]] { } func (s *State) commitQC(ctx context.Context, idx types.RoadIndex) (*types.Epoch, *types.CommitQC, error) { - for inner,ctrl := range s.inner.Lock() { - if err:=ctrl.WaitUntil(ctx, func() bool{ return idx < inner.roads.next }); err!=nil { return nil,nil,err } - if idx < inner.roads.first { return nil,nil,types.ErrPruned } + for inner, ctrl := range s.inner.Lock() { + if err := ctrl.WaitUntil(ctx, func() bool { return idx < inner.roads.next }); err != nil { + return nil, nil, err + } + if idx < inner.roads.first { + return nil, nil, types.ErrPruned + } r := inner.roads.q[idx] - return r.epoch,r.commitQC,nil + return r.epoch, r.commitQC, nil } panic("unreachable") } -func (s *State) CommitQC(ctx context.Context, idx types.RoadIndex) (*types.CommitQC,error) { - _,qc,err := s.commitQC(ctx,idx) - return qc,err +func (s *State) CommitQC(ctx context.Context, idx types.RoadIndex) (*types.CommitQC, error) { + _, qc, err := s.commitQC(ctx, idx) + return qc, err } // WaitForAppQC waits until there is an AppQC for the given index or higher. @@ -168,15 +172,19 @@ func (s *State) CommitQC(ctx context.Context, idx types.RoadIndex) (*types.Commi // Together they provide enough information to prune the availability state. func (s *State) WaitForAppQC(ctx context.Context, idx types.RoadIndex) (*types.AppQC, *types.CommitQC, error) { for inner, ctrl := range s.inner.Lock() { - if err:=ctrl.WaitUntil(ctx, func() bool { return idx idx { return nil } + for inner, ctrl := range s.inner.Lock() { + if err := ctrl.WaitUntil(ctx, func() bool { return idx <= inner.roads.next }); err != nil { + return err + } + if inner.roads.next > idx { + return nil + } } epoch, ok := s.data.Registry().EpochByIndex(qc.Proposal().EpochIndex()) if !ok { @@ -196,8 +208,10 @@ func (s *State) PushCommitQC(ctx context.Context, qc *types.CommitQC) error { return fmt.Errorf("qc.Verify(): %w", err) } for inner, ctrl := range s.inner.Lock() { - if idx != inner.roads.next { return nil } - inner.roads.pushBack(newRoad(qc,epoch)) + if idx != inner.roads.next { + return nil + } + inner.roads.pushBack(newRoad(qc, epoch)) metrics.ObserveCommitQC(qc) // The persist goroutine publishes latestCommitQC after writing to disk // (or immediately for no-op persisters), so consensus won't advance @@ -212,8 +226,10 @@ func (s *State) PushCommitQC(ctx context.Context, qc *types.CommitQC) error { func (s *State) PushAppVote(ctx context.Context, v *types.Signed[*types.AppVote]) error { // Wait for the corresponding commitQC. idx := v.Msg().Proposal().RoadIndex() - epoch, commitQC,err := s.commitQC(ctx, idx) - if err != nil { return ignorePruned(err) } + epoch, commitQC, err := s.commitQC(ctx, idx) + if err != nil { + return ignorePruned(err) + } if err := v.Msg().Proposal().Verify(commitQC); err != nil { return fmt.Errorf("invalid vote: %w", err) } @@ -241,11 +257,8 @@ func (s *State) prune(appQC *types.AppQC, commitQC *types.CommitQC) error { return nil } } - if got, want := appQC.Proposal().EpochIndex(), commitQC.Proposal().EpochIndex(); got != want { - return fmt.Errorf("appQC epoch_index %d != commitQC epoch_index %d", got, want) - } - if appQC.Proposal().RoadIndex() != commitQC.Proposal().Index() { - return fmt.Errorf("mismatched QCs: appQC index %v, commitQC index %v", appQC.Proposal().RoadIndex(), commitQC.Proposal().Index()) + if err := appQC.Proposal().Verify(commitQC); err != nil { + return fmt.Errorf("appQC proposal: %w", err) } epoch, ok := s.data.Registry().EpochByIndex(commitQC.Proposal().EpochIndex()) if !ok { @@ -603,13 +616,15 @@ func (s *State) runPersist(ctx context.Context, pers persisters) error { // Callees handle empty inputs gracefully (no-op when nothing to write/truncate). if err := scope.Parallel(func(ps scope.ParallelScope) error { ps.Spawn(func() error { - if err:=pers.commitQCs.PruneAndPersist(batch.commitQCs.first, batch.commitQCs.tail); err!=nil { return err } - if t:= batch.commitQCs.tail; len(t)>0 { + if err := pers.commitQCs.PruneAndPersist(batch.commitQCs.first, batch.commitQCs.tail); err != nil { + return err + } + if t := batch.commitQCs.tail; len(t) > 0 { s.markCommitQCsPersisted(t[len(t)-1]) } return nil }) - for lane,batch := range batch.blocks { + for lane, batch := range batch.blocks { ps.Spawn(func() error { return pers.blocks.Persist(lane, batch.first, batch.tail, utils.Some(markBlock)) }) @@ -623,17 +638,17 @@ func (s *State) runPersist(ctx context.Context, pers persisters) error { type batch[I any, T any] struct { first I - tail []T + tail []T } -type blocksBatch = batch[types.BlockNumber,*types.Signed[*types.LaneProposal]] -type commitQCsBatch = batch[types.RoadIndex,*types.CommitQC] +type blocksBatch = batch[types.BlockNumber, *types.Signed[*types.LaneProposal]] +type commitQCsBatch = batch[types.RoadIndex, *types.CommitQC] // persistBatch holds the data collected under lock for one persist iteration. type persistBatch struct { - epoch *types.Epoch - blocks map[types.LaneID]blocksBatch - commitQCs commitQCsBatch + epoch *types.Epoch + blocks map[types.LaneID]blocksBatch + commitQCs commitQCsBatch } // advancePersistedBlockStart updates the per-lane block admission watermark @@ -690,8 +705,8 @@ func (s *State) collectPersistBatch(ctx context.Context) (*persistBatch, error) }); err != nil { return nil, err } - b := &persistBatch { - blocks: map[types.LaneID]blocksBatch{}, + b := &persistBatch{ + blocks: map[types.LaneID]blocksBatch{}, commitQCs: commitQCsBatch{first: inner.roads.first}, } for n := max(next, inner.roads.first); n < inner.roads.next; n++ { diff --git a/sei-tendermint/internal/autobahn/avail/state_test.go b/sei-tendermint/internal/autobahn/avail/state_test.go index 90d0506df5..dfe29d496b 100644 --- a/sei-tendermint/internal/autobahn/avail/state_test.go +++ b/sei-tendermint/internal/autobahn/avail/state_test.go @@ -67,7 +67,7 @@ func TestSubscribeAppVotesJumpsToDataFloor(t *testing.T) { vote, err := recv.Recv(t.Context()) require.NoError(t, err) - require.Equal(t, first, vote.Msg().Proposal().GlobalNumber()) + require.Equal(t, first, vote.Msg().Proposal().GlobalFirst()) } func makeLaneVotes(keys []types.SecretKey, h *types.BlockHeader) []*types.Signed[*types.LaneVote] { @@ -185,7 +185,7 @@ func testState(t *testing.T, stateDir utils.Option[string]) { } t.Logf("Push app votes.") - appProposal := types.NewAppProposal(qc.GlobalRange().Next-1, qc.Proposal().Index(), types.GenAppHash(rng), 0) + appProposal := types.NewAppProposal(qc.Proposal(), types.GenAppHash(rng)) for _, vote := range makeAppVotes(keys, appProposal) { if err := state.PushAppVote(ctx, vote); err != nil { return fmt.Errorf("state.PushAppVote(): %w", err) @@ -305,7 +305,7 @@ func TestStateRestartFromPersisted(t *testing.T) { return fmt.Errorf("PushCommitQC: %w", err) } - appProposal := types.NewAppProposal(qc.GlobalRange().Next-1, qc.Proposal().Index(), types.GenAppHash(rng), 0) + appProposal := types.NewAppProposal(qc.Proposal(), types.GenAppHash(rng)) for _, vote := range makeAppVotes(keys, appProposal) { if err := state.PushAppVote(ctx, vote); err != nil { return fmt.Errorf("PushAppVote: %w", err) @@ -400,7 +400,7 @@ func TestStateMismatchedQCs(t *testing.T) { t.Run("PushAppQC mismatch", func(t *testing.T) { require := require.New(t) // AppQC for index 1, but paired with CommitQC for index 0 - appProposal1 := types.NewAppProposal(initialBlock, 1, types.GenAppHash(rng), 0) + appProposal1 := types.GenAppProposal(rng) appQC1 := types.NewAppQC(makeAppVotes(keys, appProposal1)) err := state.PushAppQC(appQC1, qc0) @@ -471,9 +471,6 @@ func TestNewStateWithPersistence(t *testing.T) { ds := newTestDataState(&data.Config{Registry: registry}) roadIdx := types.RoadIndex(7) - globalNum := types.GlobalBlockNumber(50) - appProposal := types.NewAppProposal(globalNum, roadIdx, types.GenAppHash(rng), 0) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) // Persist commitQCs 0-7 so the matching one at roadIdx exists. cp, _, err := persist.NewCommitQCPersister(utils.Some(dir)) @@ -486,6 +483,8 @@ func TestNewStateWithPersistence(t *testing.T) { require.NoError(t, cp.MaybePruneAndPersist(utils.None[*types.CommitQC](), []*types.CommitQC{qc}, noCommitQCCB)) pruneQC = qc } + appProposal := types.NewAppProposal(pruneQC.Proposal(), types.GenAppHash(rng)) + appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) // Persist prune anchor (AppQC + CommitQC pair). prunePers, _, err := persist.NewPersister[*pb.PersistedAvailPruneAnchor](utils.Some(dir), innerFile) @@ -502,7 +501,7 @@ func TestNewStateWithPersistence(t *testing.T) { got, ok := aq.Get() require.True(t, ok) require.Equal(t, roadIdx, got.Proposal().RoadIndex()) - require.Equal(t, globalNum, got.Proposal().GlobalNumber()) + require.Equal(t, pruneQC.GlobalRange().First, got.Proposal().GlobalFirst()) require.Equal(t, roadIdx, state.FirstCommitQC()) }) @@ -537,9 +536,6 @@ func TestNewStateWithPersistence(t *testing.T) { lane := keys[0].Public() roadIdx := types.RoadIndex(2) - globalNum := types.GlobalBlockNumber(5) - appProposal := types.NewAppProposal(globalNum, roadIdx, types.GenAppHash(rng), 0) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) // Persist commitQCs 0-2 so the matching one at roadIdx exists. cp, _, err := persist.NewCommitQCPersister(utils.Some(dir)) @@ -552,6 +548,8 @@ func TestNewStateWithPersistence(t *testing.T) { require.NoError(t, cp.MaybePruneAndPersist(utils.None[*types.CommitQC](), []*types.CommitQC{qc}, noCommitQCCB)) pruneQC = qc } + appProposal := types.NewAppProposal(pruneQC.Proposal(), types.GenAppHash(rng)) + appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) // Persist prune anchor (AppQC + CommitQC pair). prunePers, _, err := persist.NewPersister[*pb.PersistedAvailPruneAnchor](utils.Some(dir), innerFile) @@ -615,9 +613,6 @@ func TestNewStateWithPersistence(t *testing.T) { // Persist AppQC at road index 1. roadIdx := types.RoadIndex(1) - globalNum := types.GlobalBlockNumber(5) - appProposal := types.NewAppProposal(globalNum, roadIdx, types.GenAppHash(rng), 0) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) // Persist CommitQCs 0-4. cp, _, err := persist.NewCommitQCPersister(utils.Some(dir)) @@ -630,6 +625,8 @@ func TestNewStateWithPersistence(t *testing.T) { prev = utils.Some(qcs[i]) require.NoError(t, cp.MaybePruneAndPersist(utils.None[*types.CommitQC](), []*types.CommitQC{qcs[i]}, noCommitQCCB)) } + appProposal := types.NewAppProposal(qcs[roadIdx].Proposal(), types.GenAppHash(rng)) + appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) // Persist prune anchor (AppQC + CommitQC pair at roadIdx). prunePers, _, err := persist.NewPersister[*pb.PersistedAvailPruneAnchor](utils.Some(dir), innerFile) @@ -659,7 +656,7 @@ func TestNewStateWithPersistence(t *testing.T) { } // Persist prune anchor (AppQC + CommitQC pair at road index 0). - appProposal := types.NewAppProposal(initialBlock, 0, types.GenAppHash(rng), 0) + appProposal := types.NewAppProposal(allQCs[0].Proposal(), types.GenAppHash(rng)) appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) prunePers, _, err := persist.NewPersister[*pb.PersistedAvailPruneAnchor](utils.Some(dir), innerFile) require.NoError(t, err) @@ -703,7 +700,7 @@ func TestNewStateWithPersistence(t *testing.T) { require.NoError(t, cp.Close()) // Persist a prune anchor at index 9 — well past the persisted range. - appProposal := types.NewAppProposal(50, 9, types.GenAppHash(rng), 0) + appProposal := types.NewAppProposal(qcs[9].Proposal(), types.GenAppHash(rng)) appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) prunePers, _, err := persist.NewPersister[*pb.PersistedAvailPruneAnchor](utils.Some(dir), innerFile) require.NoError(t, err) @@ -754,7 +751,7 @@ func TestNewStateWithPersistence(t *testing.T) { // Persist a prune anchor at index 9 with a laneRange that starts past // all persisted blocks — MaybePruneAndPersistLane will TruncateAll the block WAL. - appProposal := types.NewAppProposal(50, 9, types.GenAppHash(rng), 0) + appProposal := types.NewAppProposal(qcs[9].Proposal(), types.GenAppHash(rng)) appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) prunePers, _, err := persist.NewPersister[*pb.PersistedAvailPruneAnchor](utils.Some(dir), innerFile) require.NoError(t, err) diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index 1c2c850cc3..006c4516ee 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -175,7 +175,8 @@ func (i *inner) updateNextBlock(m *metrics.Metrics) { // Invariant: a CommitQC's embedded AppProposal (when present) always refers to // a global number from a *past* CommitQC — strictly below that tip QC's // GlobalRange.First (enforced in Proposal.Verify). Together with BlockDB's -// never-empty retention and eviction at min(nextAppProposal, App+1), in-memory +// never-empty retention and eviction at min(nextAppProposal, App.GlobalNext), +// in-memory // maps therefore always retain at least the certified tip QC after a // CommitQC.App appears. nextToExecute uses qc[nextAppProposal] (or the tip QC // when fully caught up), so it does not require retaining nextAppProposal-1. @@ -547,10 +548,10 @@ func (s *State) NeedBlock(n types.GlobalBlockNumber) bool { func assembleGlobalBlock(n types.GlobalBlockNumber, b *types.Block, fqc *types.FullCommitQC) *types.GlobalBlock { qc := fqc.QC() return &types.GlobalBlock{ - GlobalNumber: n, - Timestamp: qc.Proposal().BlockTimestamp(n).OrPanic("global block not in QC"), - Header: b.Header(), - Payload: b.Payload(), + GlobalNumber: n, + Timestamp: qc.Proposal().BlockTimestamp(n).OrPanic("global block not in QC"), + Header: b.Header(), + Payload: b.Payload(), } } @@ -633,7 +634,7 @@ func (s *State) globalBlockByHashFromDB(hash types.BlockHeaderHash) (utils.Optio // PushAppHash marks blocks up to n as executed. Hash is the execution result. // Waits for the block to be durably persisted before proceeding. func (s *State) PushAppHash(ctx context.Context, n types.GlobalBlockNumber, hash types.AppHash) error { - for inner, ctrl := range s.inner.Lock() { + for inner, ctrl := range s.inner.Lock() { if err := ctrl.WaitUntil(ctx, func() bool { return n < inner.nextBlockToPersist }); err != nil { @@ -641,14 +642,14 @@ func (s *State) PushAppHash(ctx context.Context, n types.GlobalBlockNumber, hash } p := inner.qcs[n].QC().Proposal() gr := p.GlobalRange() - if gr.First!=inner.nextAppProposal { - return fmt.Errorf("unexpected app proposal : got %v, want in [%v;%v)", n, gr.First,gr.Next) + if gr.First != inner.nextAppProposal { + return fmt.Errorf("unexpected app proposal : got %v, want in [%v;%v)", n, gr.First, gr.Next) } // We only care about the AppHash of the last block of the CommitQC. - if gr.Next!=n+1 { + if gr.Next != n+1 { return nil } - proposal := types.NewAppProposal(p.Index(),hash,p.EpochIndex()) + proposal := types.NewAppProposal(p, hash) t := time.Now() for inner.nextAppProposal < gr.Next { b := inner.blocks[inner.nextAppProposal] @@ -664,35 +665,37 @@ func (s *State) PushAppHash(ctx context.Context, n types.GlobalBlockNumber, hash return nil } -// AppVote returns an appVote for a block >= n. +// AppVote returns an appVote for a block >= n. func (s *State) AppVote(ctx context.Context, n types.GlobalBlockNumber) (*types.AppVote, *types.FullCommitQC, error) { for inner, ctrl := range s.inner.Lock() { - if err := ctrl.WaitUntil(ctx, func() bool { return max(inner.nextAppQC,n) < inner.nextAppProposal }); err != nil { + if err := ctrl.WaitUntil(ctx, func() bool { return max(inner.nextAppQC, n) < inner.nextAppProposal }); err != nil { return nil, nil, err } - n := max(inner.nextAppQC,n) - return types.NewAppVote(inner.appProposals[n]), inner.qcs[n], nil + n := max(inner.nextAppQC, n) + return types.NewAppVote(inner.appProposals[n]), inner.qcs[n], nil } panic("unreachable") } func (s *State) AppQC(ctx context.Context, n types.GlobalBlockNumber) (*types.AppQC, *types.FullCommitQC, error) { for inner, ctrl := range s.inner.Lock() { - if err := ctrl.WaitUntil(ctx, func() bool { return n < inner.nextAppQC }); err!=nil { return nil,nil,err } + if err := ctrl.WaitUntil(ctx, func() bool { return n < inner.nextAppQC }); err != nil { + return nil, nil, err + } if inner.first <= n { - return inner.appQCs[n],inner.qcs[n],nil + return inner.appQCs[n], inner.qcs[n], nil } } // TODO: we should fallback to blocksDB panic("unreachable") } -func (s *State) LastAppQC() (*types.AppQC,*types.FullCommitQC) { +func (s *State) LastAppQC() (*types.AppQC, *types.FullCommitQC) { for i := range s.inner.Lock() { // TODO: currently no guarantee that there is >=1 element. // TODO: nextAppQC is NOT good enough, we need it to be persisted. - n := i.nextAppQC-1 - return i.appQCs[n],i.qcs[n] + n := i.nextAppQC - 1 + return i.appQCs[n], i.qcs[n] } panic("unreachable") } @@ -823,9 +826,9 @@ func (s *State) runPersist(ctx context.Context) error { // or the bound would not advance first. Caller must hold inner's lock. Invoked // from PushQC / PushAppHash. // -// Bound is min(nextAppProposal, App.GlobalNumber()+1). A zero floor (no App / +// Bound is min(nextAppProposal, App.GlobalNext()). A zero floor (no App / // empty maps) yields bound 0 and is a no-op via bound <= first. With the -// past-CommitQC App invariant (see State), App+1 never exceeds the tip QC +// past-CommitQC App invariant (see State), App.GlobalNext never exceeds the tip QC // start, so at least one CommitQC remains. nextToExecute uses qc[nextAppProposal] // (or the tip when caught up), so nextAppProposal-1 need not be retained. // diff --git a/sei-tendermint/internal/autobahn/data/state_test.go b/sei-tendermint/internal/autobahn/data/state_test.go index 23ae335fb7..374eb2f5df 100644 --- a/sei-tendermint/internal/autobahn/data/state_test.go +++ b/sei-tendermint/internal/autobahn/data/state_test.go @@ -477,7 +477,7 @@ func TestPushQCBeforeRunPersistsToBlockDB(t *testing.T) { // TestEvictionWaitsForCommitQCApp checks that evictBelowBound does not drop // AppProposals until a later CommitQC embeds an App (certifying AppQC), and -// that once that App exists, heights below min(NAP, App+1) are evicted. +// that once that App exists, heights below min(NAP, App.GlobalNext) are evicted. func TestEvictionWaitsForCommitQCApp(t *testing.T) { ctx := t.Context() rng := utils.TestRng() @@ -490,8 +490,8 @@ func TestEvictionWaitsForCommitQCApp(t *testing.T) { qc2, blocks2 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc1.QC())) app, ok := qc2.QC().Proposal().App().Get() require.True(t, ok, "second CommitQC embeds App for qc1 tip") - appFloor := app.GlobalNumber() - require.Equal(t, gr1.Next-1, appFloor) + appFloor := app.GlobalNext() + require.Equal(t, gr1.Next, appFloor) gr2 := qc2.QC().GlobalRange() state := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) @@ -526,7 +526,7 @@ func TestEvictionWaitsForCommitQCApp(t *testing.T) { } for inner := range state.inner.Lock() { - require.Equal(t, appFloor+1, inner.first, "after catching up, first reaches App+1") + require.Equal(t, appFloor, inner.first, "after catching up, first reaches App.GlobalNext") for n := gr1.First; n < inner.first; n++ { _, ok := inner.appProposals[n] require.False(t, ok, "AppProposal %d should be evicted (< first)", n) @@ -547,7 +547,8 @@ func TestEvictionWaitsForCommitQCApp(t *testing.T) { // TestNextToExecuteAfterAppEviction checks WaitUntilExecuted / nextToExecute // still work when PushQC embeds an App that aggressively evicts through -// nextAppProposal (first = App+1 = NAP). nextToExecute uses qc[NAP], not NAP-1. +// nextAppProposal (first = App.GlobalNext = NAP). nextToExecute uses qc[NAP], +// not NAP-1. func TestNextToExecuteAfterAppEviction(t *testing.T) { ctx := t.Context() rng := utils.TestRng() @@ -558,7 +559,7 @@ func TestNextToExecuteAfterAppEviction(t *testing.T) { qc2, blocks2 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc1.QC())) app, ok := qc2.QC().Proposal().App().Get() require.True(t, ok) - require.Equal(t, gr1.Next-1, app.GlobalNumber()) + require.Equal(t, gr1.Next, app.GlobalNext()) state := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { @@ -574,7 +575,7 @@ func TestNextToExecuteAfterAppEviction(t *testing.T) { return err } } - // Sticky case: App floor == nextAppProposal. first advances to NAP; + // Sticky case: App.GlobalNext == nextAppProposal. first advances to NAP; // NAP-1 is gone; nextToExecute reads qc[NAP]. require.NoError(t, state.PushQC(ctx, qc2, blocks2)) @@ -582,8 +583,8 @@ func TestNextToExecuteAfterAppEviction(t *testing.T) { var tipBlockNum types.BlockNumber for inner := range state.inner.Lock() { require.Equal(t, gr1.Next, inner.nextAppProposal) - require.Equal(t, app.GlobalNumber()+1, inner.first, - "eviction advances to App+1 == NAP") + require.Equal(t, app.GlobalNext(), inner.first, + "eviction advances to App.GlobalNext == NAP") _, ok := inner.blocks[inner.nextAppProposal-1] require.False(t, ok, "NAP-1 must be evicted") require.Less(t, inner.nextAppProposal, inner.nextQC) @@ -669,7 +670,7 @@ func TestPruningKeepsLastQCRange(t *testing.T) { // readability), so a mid-range prune does not refuse heights inside that QC. // // PruneBefore is BlockDB-only: heights still retained in RAM for AppVotes -// (at/above CommitQC.App+1 exclusive floor) remain readable via TryBlock even +// (at/above CommitQC.App.GlobalNext exclusive floor) remain readable via TryBlock even // after the store watermark advances past them. func TestPruningWithPartialQCRange(t *testing.T) { ctx := t.Context() @@ -682,8 +683,7 @@ func TestPruningWithPartialQCRange(t *testing.T) { gr2 := qc2.QC().GlobalRange() app, ok := qc2.QC().Proposal().App().Get() require.True(t, ok) - appFloor := app.GlobalNumber() - exclusiveFloor := appFloor + 1 + exclusiveFloor := app.GlobalNext() state1 := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) require.NoError(t, state1.PushQC(ctx, qc1, blocks1)) diff --git a/sei-tendermint/internal/autobahn/pb/autobahn.pb.go b/sei-tendermint/internal/autobahn/pb/autobahn.pb.go index 1348b18d6b..490c46f968 100644 --- a/sei-tendermint/internal/autobahn/pb/autobahn.pb.go +++ b/sei-tendermint/internal/autobahn/pb/autobahn.pb.go @@ -1493,14 +1493,16 @@ func (x *AppQC) GetSigs() []*Signature { type AppProposal struct { state protoimpl.MessageState `protogen:"open.v1"` - // Global block number. - GlobalNumber *uint64 `protobuf:"varint,1,opt,name=global_number,json=globalNumber,proto3,oneof" json:"global_number,omitempty"` // required + // Epoch this proposal belongs to. + EpochIndex *uint64 `protobuf:"varint,4,opt,name=epoch_index,json=epochIndex,proto3,oneof" json:"epoch_index,omitempty"` // required // Index of the commit qc finalizing the block. RoadIndex *uint64 `protobuf:"varint,2,opt,name=road_index,json=roadIndex,proto3,oneof" json:"road_index,omitempty"` // required - // App hash at that block. - AppHash []byte `protobuf:"bytes,3,opt,name=app_hash,json=appHash,proto3,oneof" json:"app_hash,omitempty"` // required - // Epoch this proposal belongs to. - EpochIndex *uint64 `protobuf:"varint,4,opt,name=epoch_index,json=epochIndex,proto3,oneof" json:"epoch_index,omitempty"` // required + // Range of global blocks convered by this proposal. + // Has to match the corresponding CommitQC. + GlobalFirst *uint64 `protobuf:"varint,5,opt,name=global_first,json=globalFirst,proto3,oneof" json:"global_first,omitempty"` // required + GlobalNext *uint64 `protobuf:"varint,6,opt,name=global_next,json=globalNext,proto3,oneof" json:"global_next,omitempty"` // required + // App hash of the state at the end of the range above. + AppHash []byte `protobuf:"bytes,3,opt,name=app_hash,json=appHash,proto3,oneof" json:"app_hash,omitempty"` // required unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1535,9 +1537,9 @@ func (*AppProposal) Descriptor() ([]byte, []int) { return file_autobahn_autobahn_proto_rawDescGZIP(), []int{25} } -func (x *AppProposal) GetGlobalNumber() uint64 { - if x != nil && x.GlobalNumber != nil { - return *x.GlobalNumber +func (x *AppProposal) GetEpochIndex() uint64 { + if x != nil && x.EpochIndex != nil { + return *x.EpochIndex } return 0 } @@ -1549,20 +1551,27 @@ func (x *AppProposal) GetRoadIndex() uint64 { return 0 } -func (x *AppProposal) GetAppHash() []byte { - if x != nil { - return x.AppHash +func (x *AppProposal) GetGlobalFirst() uint64 { + if x != nil && x.GlobalFirst != nil { + return *x.GlobalFirst } - return nil + return 0 } -func (x *AppProposal) GetEpochIndex() uint64 { - if x != nil && x.EpochIndex != nil { - return *x.EpochIndex +func (x *AppProposal) GetGlobalNext() uint64 { + if x != nil && x.GlobalNext != nil { + return *x.GlobalNext } return 0 } +func (x *AppProposal) GetAppHash() []byte { + if x != nil { + return x.AppHash + } + return nil +} + // This is the signable message. // To sign ConsensusMsg/BlockMsg, you need to embed it in Msg first. type Msg struct { @@ -2339,18 +2348,21 @@ const file_autobahn_autobahn_proto_rawDesc = "" + "_commit_qc\"k\n" + "\x05AppQC\x12)\n" + "\x04vote\x18\x01 \x01(\v2\x15.autobahn.AppProposalR\x04vote\x12/\n" + - "\x04sigs\x18\x02 \x03(\v2\x13.autobahn.SignatureB\x06Ј\xe2\xab\fdR\x04sigs:\x06\xe8\x88\xe2\xab\f\x01\"\xf5\x01\n" + - "\vAppProposal\x12(\n" + - "\rglobal_number\x18\x01 \x01(\x04H\x00R\fglobalNumber\x88\x01\x01\x12\"\n" + + "\x04sigs\x18\x02 \x03(\v2\x13.autobahn.SignatureB\x06Ј\xe2\xab\fdR\x04sigs:\x06\xe8\x88\xe2\xab\f\x01\"\xbd\x02\n" + + "\vAppProposal\x12$\n" + + "\vepoch_index\x18\x04 \x01(\x04H\x00R\n" + + "epochIndex\x88\x01\x01\x12\"\n" + "\n" + "road_index\x18\x02 \x01(\x04H\x01R\troadIndex\x88\x01\x01\x12&\n" + - "\bapp_hash\x18\x03 \x01(\fB\x06؈\xe2\xab\f H\x02R\aappHash\x88\x01\x01\x12$\n" + - "\vepoch_index\x18\x04 \x01(\x04H\x03R\n" + - "epochIndex\x88\x01\x01:\fȈ\xe2\xab\f\x01\xe8\x88\xe2\xab\f\x01B\x10\n" + - "\x0e_global_numberB\r\n" + - "\v_road_indexB\v\n" + - "\t_app_hashB\x0e\n" + - "\f_epoch_index\"\x98\x03\n" + + "\fglobal_first\x18\x05 \x01(\x04H\x02R\vglobalFirst\x88\x01\x01\x12$\n" + + "\vglobal_next\x18\x06 \x01(\x04H\x03R\n" + + "globalNext\x88\x01\x01\x12&\n" + + "\bapp_hash\x18\x03 \x01(\fB\x06؈\xe2\xab\f H\x04R\aappHash\x88\x01\x01:\fȈ\xe2\xab\f\x01\xe8\x88\xe2\xab\f\x01B\x0e\n" + + "\f_epoch_indexB\r\n" + + "\v_road_indexB\x0f\n" + + "\r_global_firstB\x0e\n" + + "\f_global_nextB\v\n" + + "\t_app_hashJ\x04\b\x01\x10\x02R\rglobal_number\"\x98\x03\n" + "\x03Msg\x126\n" + "\rlane_proposal\x18\x01 \x01(\v2\x0f.autobahn.BlockH\x00R\flaneProposal\x124\n" + "\tlane_vote\x18\x02 \x01(\v2\x15.autobahn.BlockHeaderH\x00R\blaneVote\x120\n" + diff --git a/sei-tendermint/internal/autobahn/pb/autobahn.wireguard.go b/sei-tendermint/internal/autobahn/pb/autobahn.wireguard.go index 99b476ba5d..55d8c736b0 100644 --- a/sei-tendermint/internal/autobahn/pb/autobahn.wireguard.go +++ b/sei-tendermint/internal/autobahn/pb/autobahn.wireguard.go @@ -44,23 +44,23 @@ func (*View) MaxSize() int { } func (*Proposal) MaxSize() int { - return 9539 + return 9550 } func (*FullProposal) MaxSize() int { - return 1107571 + return 1107604 } func (*PrepareQC) MaxSize() int { - return 19942 + return 19953 } func (*CommitQC) MaxSize() int { - return 19942 + return 19953 } func (*FullCommitQC) MaxSize() int { - return 136946 + return 136957 } func (*TimeoutVote) MaxSize() int { @@ -68,19 +68,19 @@ func (*TimeoutVote) MaxSize() int { } func (*TimeoutQC) MaxSize() int { - return 35446 + return 35457 } func (*FullTimeoutVote) MaxSize() int { - return 20101 + return 20112 } func (*AppQC) MaxSize() int { - return 10469 + return 10480 } func (*AppProposal) MaxSize() int { - return 67 + return 78 } func (*Msg) MaxSize() int { @@ -88,7 +88,7 @@ func (*Msg) MaxSize() int { } func (*SignedProposal) MaxSize() int { - return 9646 + return 9657 } func (*SignedTimeoutVote) MaxSize() int { @@ -96,7 +96,7 @@ func (*SignedTimeoutVote) MaxSize() int { } func (*SignedAppVote) MaxSize() int { - return 173 + return 184 } func (*SignedBlock) MaxSize() int { @@ -108,11 +108,11 @@ func (*SignedBlockHeader) MaxSize() int { } func (*SignedAppProposal) MaxSize() int { - return 173 + return 184 } func (*ConsensusReq) MaxSize() int { - return 1107575 + return 1107608 } func init() { @@ -287,10 +287,11 @@ func init() { // Register the wireguard.Schema generated for autobahn.AppProposal. runtime.MustRegister[*AppProposal](runtime.Schema{ - 1: {MaxCount: 1}, + 4: {MaxCount: 1}, 2: {MaxCount: 1}, + 5: {MaxCount: 1}, + 6: {MaxCount: 1}, 3: {MaxCount: 1, MaxSize: 32}, - 4: {MaxCount: 1}, }) // Register the wireguard.Schema generated for autobahn.Msg. diff --git a/sei-tendermint/internal/p2p/giga/pb/api.wireguard.go b/sei-tendermint/internal/p2p/giga/pb/api.wireguard.go index e56f868527..4882b5c1a6 100644 --- a/sei-tendermint/internal/p2p/giga/pb/api.wireguard.go +++ b/sei-tendermint/internal/p2p/giga/pb/api.wireguard.go @@ -29,7 +29,7 @@ func (*LaneProposal) MaxSize() int { } func (*AppVote) MaxSize() int { - return 176 + return 187 } func (*StreamLaneProposalsReq) MaxSize() int { From 027ed62dbc76c41f1e782aa6b8cc00250fb906b9 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 4 Aug 2026 16:07:30 +0200 Subject: [PATCH 06/61] fixes types tests --- .../autobahn/types/committee_test.go | 4 +- .../autobahn/types/proposal_test.go | 51 ++++++++----------- .../autobahn/types/wireguard_test.go | 1 - 3 files changed, 24 insertions(+), 32 deletions(-) diff --git a/sei-tendermint/autobahn/types/committee_test.go b/sei-tendermint/autobahn/types/committee_test.go index 4627da4aeb..005865a436 100644 --- a/sei-tendermint/autobahn/types/committee_test.go +++ b/sei-tendermint/autobahn/types/committee_test.go @@ -132,7 +132,7 @@ func TestPrepareQCVerifyChecksEpochBinding(t *testing.T) { wrongEpoch := newProposal(View{Index: ep.RoadRange().First, EpochIndex: ep.EpochIndex() + 1}, time.Time{}, nil, ep.FirstBlock()) require.Error(t, sign(wrongEpoch).Verify(ep)) - outOfRoads := newProposal(View{Index: ep.RoadRange().Last + 1, EpochIndex: ep.EpochIndex()}, time.Time{}, nil, ep.FirstBlock()) + outOfRoads := newProposal(View{Index: ep.RoadRange().Next, EpochIndex: ep.EpochIndex()}, time.Time{}, nil, ep.FirstBlock()) require.Error(t, sign(outOfRoads).Verify(ep)) } @@ -148,7 +148,7 @@ func TestCommitQCVerifyChecksEpochBinding(t *testing.T) { wrongEpoch := newProposal(View{Index: ep.RoadRange().First, EpochIndex: ep.EpochIndex() + 1}, time.Time{}, nil, ep.FirstBlock()) require.Error(t, sign(wrongEpoch).Verify(ep)) - outOfRoads := newProposal(View{Index: ep.RoadRange().Last + 1, EpochIndex: ep.EpochIndex()}, time.Time{}, nil, ep.FirstBlock()) + outOfRoads := newProposal(View{Index: ep.RoadRange().Next, EpochIndex: ep.EpochIndex()}, time.Time{}, nil, ep.FirstBlock()) require.Error(t, sign(outOfRoads).Verify(ep)) } diff --git a/sei-tendermint/autobahn/types/proposal_test.go b/sei-tendermint/autobahn/types/proposal_test.go index d4c6538de1..58a2b15154 100644 --- a/sei-tendermint/autobahn/types/proposal_test.go +++ b/sei-tendermint/autobahn/types/proposal_test.go @@ -94,7 +94,7 @@ func TestProposalVerifyFreshWithBlocks(t *testing.T) { laneQC := makeLaneQC(rng, committee, keys, lane, 0, GenBlockHeaderHash(rng)) fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), - map[LaneID]*LaneQC{lane: laneQC}, utils.None[*AppQC]())) + map[LaneID]*LaneQC{lane: laneQC})) require.NoError(t, fp.Verify(vs)) } @@ -112,7 +112,6 @@ func TestNewProposalRejectsLaneRangeLongerThanMaxLaneRangeInProposal(t *testing. vs, time.Now(), map[LaneID]*LaneQC{lane: laneQC}, - utils.None[*AppQC](), ) require.Error(t, err) } @@ -132,7 +131,6 @@ func TestProposalBlockTimestampStrictlyMonotone(t *testing.T) { map[LaneID]*LaneQC{ lane: makeLaneQC(rng, committee, keys, lane, 2, GenBlockHeaderHash(rng)), }, - utils.None[*AppQC](), )) p0 := firstProposal.Proposal().Msg() gr0 := p0.GlobalRange() @@ -154,7 +152,6 @@ func TestProposalBlockTimestampStrictlyMonotone(t *testing.T) { map[LaneID]*LaneQC{ lane: makeLaneQC(rng, committee, keys, lane, 3, GenBlockHeaderHash(rng)), }, - utils.None[*AppQC](), )) p1 := secondProposal.Proposal().Msg() gr1 := p1.GlobalRange() @@ -172,7 +169,7 @@ func TestProposalVerifyRejectsNonMonotoneTimestamp(t *testing.T) { ep := NewEpoch(GenEpochIndex(rng), OpenRoadRange(), genesisTimestamp, committee, GlobalBlockNumber(rng.Uint64()%1000000)+1) vs := ViewSpec{Epoch: ep} k := leaderKey(committee, keys, vs.View()) - fp := utils.OrPanic1(NewProposal(k, vs, genesisTimestamp, oneLaneQCMap(rng, committee, keys, vs), utils.None[*AppQC]())) + fp := utils.OrPanic1(NewProposal(k, vs, genesisTimestamp, oneLaneQCMap(rng, committee, keys, vs))) require.NoError(t, fp.Verify(vs)) vsLater := vs @@ -193,13 +190,11 @@ func TestProposalVerifyRejectsNonMonotoneTimestamp(t *testing.T) { proposer0, vs0, time.Now(), map[LaneID]*LaneQC{lane: lQC}, - utils.None[*AppQC](), )) fp0b := utils.OrPanic1(NewProposal( proposer0, vs0, fp0a.Proposal().Msg().NextTimestamp().Add(time.Hour), map[LaneID]*LaneQC{lane: lQC}, - utils.None[*AppQC](), )) vs1a := ViewSpec{CommitQC: utils.Some(makeCommitQCFromProposal(keys, fp0a)), Epoch: ep} @@ -210,7 +205,6 @@ func TestProposalVerifyRejectsNonMonotoneTimestamp(t *testing.T) { proposer1, vs1a, fp0a.Proposal().Msg().NextTimestamp(), oneLaneQCMap(rng, committee, keys, vs1a), - utils.None[*AppQC](), )) require.NoError(t, fp1a.Verify(vs1a)) @@ -226,7 +220,7 @@ func TestProposalVerifyRejectsViewMismatch(t *testing.T) { // Build a valid proposal at genesis view (0, 0). vs0 := ViewSpec{Epoch: ep} leader0 := leaderKey(committee, keys, vs0.View()) - fp := utils.OrPanic1(NewProposal(leader0, vs0, time.Now(), oneLaneQCMap(rng, committee, keys, vs0), utils.None[*AppQC]())) + fp := utils.OrPanic1(NewProposal(leader0, vs0, time.Now(), oneLaneQCMap(rng, committee, keys, vs0))) // Verify it against a different ViewSpec (view 1, 0). commitQC := makeCommitQCFromProposal(keys, fp) @@ -243,8 +237,8 @@ func TestProposalVerifyRejectsForgedSignature(t *testing.T) { proposerKey := leaderKey(committee, keys, vs.View()) // Build two valid proposals with different timestamps. - fp1 := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.None[*AppQC]())) - fp2 := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now().Add(time.Hour), oneLaneQCMap(rng, committee, keys, vs), utils.None[*AppQC]())) + fp1 := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs))) + fp2 := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now().Add(time.Hour), oneLaneQCMap(rng, committee, keys, vs))) // Graft fp1's signature onto fp2 (different content). fp2.proposal.sig = fp1.proposal.sig @@ -259,7 +253,7 @@ func TestProposalVerifyRejectsWrongProposer(t *testing.T) { vs := ViewSpec{Epoch: ep} correctLeader := leaderKey(committee, keys, vs.View()) - fp := utils.OrPanic1(NewProposal(correctLeader, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.None[*AppQC]())) + fp := utils.OrPanic1(NewProposal(correctLeader, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs))) // Re-sign the same proposal with a different (non-leader) key. var wrongKey SecretKey @@ -285,7 +279,7 @@ func TestProposalVerifyRejectsInconsistentTimeoutQC(t *testing.T) { vs := ViewSpec{Epoch: ep} // no timeoutQC proposerKey := leaderKey(committee, keys, vs.View()) - fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.None[*AppQC]())) + fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs))) // Attach a timeoutQC that the ViewSpec doesn't expect. var timeoutVotes []*FullTimeoutVote @@ -310,7 +304,7 @@ func TestProposalVerifyRejectsNonCommitteeLane(t *testing.T) { vs := ViewSpec{Epoch: ep} proposerKey := leaderKey(committee, keys, vs.View()) - fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.None[*AppQC]())) + fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs))) // Keep the non-empty committee tipcut and add a non-committee lane. // LaneRange.Verify rejects X because it's not a committee lane. @@ -341,7 +335,7 @@ func TestProposalVerifyAcceptsImplicitLaneRange(t *testing.T) { vs := ViewSpec{Epoch: ep} proposerKey := leaderKey(committee, keys, vs.View()) - fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.None[*AppQC]())) + fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs))) // Drop one empty lane — the omitted lane gets an implicit [0, 0) range, // which matches the expected first=0 at genesis. Keep the non-empty range @@ -373,7 +367,7 @@ func TestProposalVerifyAcceptsNonContiguousImplicitRanges(t *testing.T) { vs := ViewSpec{Epoch: ep} proposerKey := leaderKey(committee, keys, vs.View()) - fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.None[*AppQC]())) + fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs))) // Drop every other empty lane (keep the non-empty range and its LaneQC). origP := fp.Proposal().Msg() @@ -405,7 +399,7 @@ func TestProposalVerifyRejectsLaneRangeFirstMismatch(t *testing.T) { vs := ViewSpec{Epoch: ep} proposerKey := leaderKey(committee, keys, vs.View()) - fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.None[*AppQC]())) + fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs))) // Tamper the non-empty lane's First (genesis expects 0) while keeping a // non-empty range and a matching LaneQC so Verify reaches the first-mismatch check. @@ -448,7 +442,7 @@ func TestProposalVerifyRejectsMissingLaneQC(t *testing.T) { // Build a valid proposal with a block, then strip the laneQC. fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), - map[LaneID]*LaneQC{lane: laneQC}, utils.None[*AppQC]())) + map[LaneID]*LaneQC{lane: laneQC})) tamperedFP := &FullProposal{ proposal: fp.proposal, @@ -470,7 +464,7 @@ func TestProposalVerifyRejectsLaneQCBlockNumberMismatch(t *testing.T) { // Build a valid proposal with a QC certifying block 1 (range [0, 2)). goodQC := makeLaneQC(rng, committee, keys, lane, 1, GenBlockHeaderHash(rng)) fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), - map[LaneID]*LaneQC{lane: goodQC}, utils.None[*AppQC]())) + map[LaneID]*LaneQC{lane: goodQC})) // Swap in a QC certifying block 0 — range expects block 1. wrongQC := makeLaneQC(rng, committee, keys, lane, 0, GenBlockHeaderHash(rng)) @@ -505,7 +499,7 @@ func TestProposalVerifyRejectsInvalidLaneQCSignature(t *testing.T) { badLaneQC := NewLaneQC(badVotes) fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), - map[LaneID]*LaneQC{lane: badLaneQC}, utils.None[*AppQC]())) + map[LaneID]*LaneQC{lane: badLaneQC})) err := fp.Verify(vs) require.Error(t, err) @@ -551,7 +545,6 @@ func makeFullProposal( leaderKey(committee, keys, vs.View()), vs, time.Now(), laneQCs, - appQC, )) } @@ -576,7 +569,7 @@ func TestProposalVerifyRejectsLaneQCHeaderHashMismatch(t *testing.T) { // Build a valid proposal with a QC for block 0. realQC := makeLaneQC(rng, committee, keys, lane, 0, GenBlockHeaderHash(rng)) fp := utils.OrPanic1(NewProposal(proposerKey, vs, time.Now(), - map[LaneID]*LaneQC{lane: realQC}, utils.None[*AppQC]())) + map[LaneID]*LaneQC{lane: realQC})) // Swap in a different QC for block 0 (different payload → different hash). differentQC := makeLaneQC(rng, committee, keys, lane, 0, GenBlockHeaderHash(rng)) @@ -602,7 +595,7 @@ func TestProposalVerifyValidReproposal(t *testing.T) { lane := committee.Leader(vs0.View()) laneQC0 := makeLaneQC(rng, committee, keys, lane, 0, GenBlockHeaderHash(rng)) fp0 := utils.OrPanic1(NewProposal(leader0, vs0, time.Now(), - map[LaneID]*LaneQC{lane: laneQC0}, utils.None[*AppQC]())) + map[LaneID]*LaneQC{lane: laneQC0})) // Build a PrepareQC for the proposal at (0, 0). var prepareVotes []*Signed[*PrepareVote] @@ -622,7 +615,7 @@ func TestProposalVerifyValidReproposal(t *testing.T) { require.Equal(t, View{Index: 0, Number: 1, EpochIndex: ep.EpochIndex()}, vs1.View()) leader1 := leaderKey(committee, keys, vs1.View()) - reproposal := utils.OrPanic1(NewProposal(leader1, vs1, time.Now(), oneLaneQCMap(rng, committee, keys, vs1), utils.None[*AppQC]())) + reproposal := utils.OrPanic1(NewProposal(leader1, vs1, time.Now(), oneLaneQCMap(rng, committee, keys, vs1))) // Reproposal must carry the same GlobalRange as the original. require.Equal(t, fp0.Proposal().Msg().GlobalRange(), reproposal.Proposal().Msg().GlobalRange()) @@ -637,7 +630,7 @@ func TestProposalVerifyRejectsReproposalWithUnnecessaryData(t *testing.T) { // Build a PrepareQC at (0, 0). vs0 := ViewSpec{Epoch: ep} leader0 := leaderKey(committee, keys, vs0.View()) - fp0 := utils.OrPanic1(NewProposal(leader0, vs0, time.Now(), oneLaneQCMap(rng, committee, keys, vs0), utils.None[*AppQC]())) + fp0 := utils.OrPanic1(NewProposal(leader0, vs0, time.Now(), oneLaneQCMap(rng, committee, keys, vs0))) var prepareVotes []*Signed[*PrepareVote] for _, k := range keys { @@ -655,7 +648,7 @@ func TestProposalVerifyRejectsReproposalWithUnnecessaryData(t *testing.T) { leader1 := leaderKey(committee, keys, vs1.View()) // Create a valid reproposal, then tamper it with unnecessary laneQCs. - reproposal := utils.OrPanic1(NewProposal(leader1, vs1, time.Now(), oneLaneQCMap(rng, committee, keys, vs1), utils.None[*AppQC]())) + reproposal := utils.OrPanic1(NewProposal(leader1, vs1, time.Now(), oneLaneQCMap(rng, committee, keys, vs1))) lane := keys[0].Public() laneQC := makeLaneQC(rng, committee, keys, lane, 0, GenBlockHeaderHash(rng)) @@ -676,7 +669,7 @@ func TestProposalVerifyRejectsReproposalHashMismatch(t *testing.T) { // Build a PrepareQC at (0, 0). vs0 := ViewSpec{Epoch: ep} leader0 := leaderKey(committee, keys, vs0.View()) - fp0 := utils.OrPanic1(NewProposal(leader0, vs0, time.Now(), oneLaneQCMap(rng, committee, keys, vs0), utils.None[*AppQC]())) + fp0 := utils.OrPanic1(NewProposal(leader0, vs0, time.Now(), oneLaneQCMap(rng, committee, keys, vs0))) var prepareVotes []*Signed[*PrepareVote] for _, k := range keys { @@ -694,7 +687,7 @@ func TestProposalVerifyRejectsReproposalHashMismatch(t *testing.T) { leader1 := leaderKey(committee, keys, vs1.View()) // Build the valid reproposal, then tamper its timestamp to get a different hash. - reproposal := utils.OrPanic1(NewProposal(leader1, vs1, time.Now(), oneLaneQCMap(rng, committee, keys, vs1), utils.None[*AppQC]())) + reproposal := utils.OrPanic1(NewProposal(leader1, vs1, time.Now(), oneLaneQCMap(rng, committee, keys, vs1))) origP := reproposal.Proposal().Msg() var ranges []*LaneRange @@ -729,7 +722,7 @@ func TestProposalVerifyRejectsInvalidTimeoutQCSignature(t *testing.T) { vs := ViewSpec{TimeoutQC: utils.Some(badTimeoutQC), Epoch: ep} leader := leaderKey(committee, keys, vs.View()) - fp := utils.OrPanic1(NewProposal(leader, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs), utils.None[*AppQC]())) + fp := utils.OrPanic1(NewProposal(leader, vs, time.Now(), oneLaneQCMap(rng, committee, keys, vs))) err := fp.Verify(vs) require.Error(t, err) diff --git a/sei-tendermint/autobahn/types/wireguard_test.go b/sei-tendermint/autobahn/types/wireguard_test.go index 86383ab405..cd6745e232 100644 --- a/sei-tendermint/autobahn/types/wireguard_test.go +++ b/sei-tendermint/autobahn/types/wireguard_test.go @@ -200,7 +200,6 @@ func TestFullProposalWireguardAcceptsMaxValidators(t *testing.T) { ViewSpec{Epoch: NewEpoch(0, OpenRoadRange(), time.Time{}, committee, 0)}, time.Unix(1, 2), laneQCs, - utils.None[*AppQC](), ) require.NoError(t, err) From 0892f8d8cb8ccbf2cece2d60221f2badd294531d Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 4 Aug 2026 16:27:38 +0200 Subject: [PATCH 07/61] data fixes --- .../internal/autobahn/avail/testonly.go | 15 ---- .../internal/autobahn/data/state.go | 58 +++++++++++++++- .../autobahn/data/state_recovery_test.go | 8 ++- .../internal/autobahn/data/state_test.go | 68 ++++++++++--------- 4 files changed, 98 insertions(+), 51 deletions(-) diff --git a/sei-tendermint/internal/autobahn/avail/testonly.go b/sei-tendermint/internal/autobahn/avail/testonly.go index aee8eead1d..eb70708076 100644 --- a/sei-tendermint/internal/autobahn/avail/testonly.go +++ b/sei-tendermint/internal/autobahn/avail/testonly.go @@ -68,21 +68,6 @@ func RunTestNetwork(ctx context.Context, states []*State) error { } }) } - s.Spawn(func() error { - next := types.RoadIndex(0) - for { - appQC, commitQC, err := from.WaitForAppQC(ctx, next) - if err != nil { - return err - } - next = appQC.Next() - for _, to := range states { - if err := to.PushAppQC(appQC, commitQC); err != nil { - return err - } - } - } - }) } return nil }) diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index 006c4516ee..251031acbb 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -82,6 +82,7 @@ func newInner(firstBlock types.GlobalBlockNumber) *inner { blockHashes: map[types.BlockHeaderHash]types.GlobalBlockNumber{}, first: firstBlock, nextAppProposal: firstBlock, + nextAppQC: firstBlock, nextBlockToPersist: firstBlock, nextBlock: firstBlock, nextQC: firstBlock, @@ -94,6 +95,7 @@ func newInner(firstBlock types.GlobalBlockNumber) *inner { func (i *inner) skipTo(n types.GlobalBlockNumber) { i.first = n i.nextAppProposal = n + i.nextAppQC = n i.nextBlockToPersist = n i.nextBlock = n i.nextQC = n @@ -677,6 +679,57 @@ func (s *State) AppVote(ctx context.Context, n types.GlobalBlockNumber) (*types. panic("unreachable") } +// PushAppQC pushes an AppQC to the state and advances the AppQC cursor. +func (s *State) PushAppQC(ctx context.Context, appQC *types.AppQC) error { + proposal := appQC.Proposal() + ep, ok := s.cfg.Registry.EpochByIndex(proposal.EpochIndex()) + if !ok { + return fmt.Errorf("unknown epoch_index %d", proposal.EpochIndex()) + } + if err := appQC.Verify(ep.Committee()); err != nil { + return fmt.Errorf("appQC.Verify(): %w", err) + } + gr := proposal.GlobalRange() + for inner, ctrl := range s.inner.Lock() { + if err := ctrl.WaitUntil(ctx, func() bool { + return gr.First <= inner.nextQC + }); err != nil { + return err + } + if gr.Next <= inner.nextAppQC { + return nil + } + if gr.First > inner.nextQC { + return fmt.Errorf("AppQC gap: expected first<=%d, got %d", inner.nextQC, gr.First) + } + if gr.Next > inner.nextQC { + return fmt.Errorf("AppQC range [%d,%d) exceeds nextQC %d", gr.First, gr.Next, inner.nextQC) + } + for n := gr.First; n < gr.Next; n++ { + qc := inner.qcs[n] + if qc == nil { + return fmt.Errorf("missing QC for AppQC block %d", n) + } + if err := proposal.Verify(qc.QC()); err != nil { + return fmt.Errorf("appQC proposal for block %d: %w", n, err) + } + } + for n := max(gr.First, inner.nextAppQC); n < gr.Next; n++ { + inner.appQCs[n] = appQC + } + for inner.nextAppQC < inner.nextQC { + if _, ok := inner.appQCs[inner.nextAppQC]; !ok { + break + } + inner.nextAppQC++ + } + evictBelowBound(inner) + ctrl.Updated() + return nil + } + panic("unreachable") +} + func (s *State) AppQC(ctx context.Context, n types.GlobalBlockNumber) (*types.AppQC, *types.FullCommitQC, error) { for inner, ctrl := range s.inner.Lock() { if err := ctrl.WaitUntil(ctx, func() bool { return n < inner.nextAppQC }); err != nil { @@ -704,7 +757,10 @@ func (i *inner) nextToExecute(lane types.LaneID) types.BlockNumber { // TODO(gprusak): decide whether 0 is a good result in this case in general. // Empty maps (first == nextQC) only on fresh start / after skipTo with no QC. if i.first == i.nextAppProposal { - return 0 + if i.first == i.nextQC { + return 0 + } + return i.qcs[i.nextAppProposal].QC().LaneRange(lane).First() } return i.qcs[i.nextAppProposal-1].QC().LaneRange(lane).Next() } diff --git a/sei-tendermint/internal/autobahn/data/state_recovery_test.go b/sei-tendermint/internal/autobahn/data/state_recovery_test.go index 6e7ebe9e31..efa83c9318 100644 --- a/sei-tendermint/internal/autobahn/data/state_recovery_test.go +++ b/sei-tendermint/internal/autobahn/data/state_recovery_test.go @@ -129,7 +129,9 @@ func TestRecoveryStartsAtLastExecutedBlock(t *testing.T) { }, db) require.Equal(t, lastExecuted, db.start) - require.Equal(t, lastExecuted, state.FirstAppProposal()) + for inner := range state.inner.Lock() { + require.Equal(t, lastExecuted, inner.nextAppProposal) + } require.Equal(t, gr2.Next, state.NextBlock()) got, err := state.TryBlock(lastExecuted) require.NoError(t, err) @@ -137,9 +139,9 @@ func TestRecoveryStartsAtLastExecutedBlock(t *testing.T) { appHash := types.GenAppHash(rng) require.NoError(t, state.PushAppHash(t.Context(), lastExecuted, appHash)) - proposal, err := state.AppProposal(t.Context(), lastExecuted) + appVote, _, err := state.AppVote(t.Context(), lastExecuted) require.NoError(t, err) - require.Equal(t, appHash, proposal.AppHash()) + require.Equal(t, appHash, appVote.Proposal().AppHash()) } func TestRecoveryCapsAppTipAtLastBlockInBlockDB(t *testing.T) { diff --git a/sei-tendermint/internal/autobahn/data/state_test.go b/sei-tendermint/internal/autobahn/data/state_test.go index 374eb2f5df..09a7a0fe6e 100644 --- a/sei-tendermint/internal/autobahn/data/state_test.go +++ b/sei-tendermint/internal/autobahn/data/state_test.go @@ -101,6 +101,14 @@ func pushAppHashesRunning(ctx context.Context, state *State, rng utils.Rng, firs }) } +func pushAppQCForBlock(ctx context.Context, state *State, keys []types.SecretKey, n types.GlobalBlockNumber) error { + vote, _, err := state.AppVote(ctx, n) + if err != nil { + return err + } + return state.PushAppQC(ctx, TestAppQC(keys, vote.Proposal())) +} + func TestState(t *testing.T) { ctx := t.Context() rng := utils.TestRng() @@ -147,11 +155,10 @@ func TestState(t *testing.T) { } wantG := &types.GlobalBlock{ - GlobalNumber: n, - Timestamp: want.QCs[n].QC().Proposal().BlockTimestamp(n).OrPanic("global block not in QC"), - Header: wantB.Header(), - Payload: wantB.Payload(), - FinalAppState: want.QCs[n].QC().Proposal().App(), + GlobalNumber: n, + Timestamp: want.QCs[n].QC().Proposal().BlockTimestamp(n).OrPanic("global block not in QC"), + Header: wantB.Header(), + Payload: wantB.Payload(), } gotG, err := state.GlobalBlock(ctx, n) if err != nil { @@ -235,7 +242,6 @@ func TestPushConflictingBadCommitQC(t *testing.T) { viewSpec, time.Now(), laneQCs, - utils.None[*types.AppQC](), )) malGR := proposal.Proposal().Msg().GlobalRange() require.Less(t, malGR.First, gr1.Next, "test setup: malicious gr.First must be < nextQC") @@ -475,23 +481,17 @@ func TestPushQCBeforeRunPersistsToBlockDB(t *testing.T) { } } -// TestEvictionWaitsForCommitQCApp checks that evictBelowBound does not drop -// AppProposals until a later CommitQC embeds an App (certifying AppQC), and -// that once that App exists, heights below min(NAP, App.GlobalNext) are evicted. -func TestEvictionWaitsForCommitQCApp(t *testing.T) { +// TestEvictionWaitsForAppQC checks that evictBelowBound does not drop +// AppProposals until AppQC advances, and that once it does, heights below +// min(nextAppProposal, nextAppQC) are evicted. +func TestEvictionWaitsForAppQC(t *testing.T) { ctx := t.Context() rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 3) qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) gr1 := qc1.QC().GlobalRange() - require.False(t, qc1.QC().Proposal().App().IsPresent(), "genesis CommitQC has no App") - qc2, blocks2 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc1.QC())) - app, ok := qc2.QC().Proposal().App().Get() - require.True(t, ok, "second CommitQC embeds App for qc1 tip") - appFloor := app.GlobalNext() - require.Equal(t, gr1.Next, appFloor) gr2 := qc2.QC().GlobalRange() state := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) @@ -509,7 +509,7 @@ func TestEvictionWaitsForCommitQCApp(t *testing.T) { } } - // No CommitQC.App yet → eviction must not strip AppProposals; first stays put. + // No AppQC yet -> eviction must not strip AppProposals; first stays put. for inner := range state.inner.Lock() { require.Equal(t, gr1.First, inner.first, "no certified App → first unchanged") for n := gr1.First; n < gr1.Next; n++ { @@ -518,6 +518,8 @@ func TestEvictionWaitsForCommitQCApp(t *testing.T) { } } + require.NoError(t, pushAppQCForBlock(ctx, state, keys, gr1.First)) + require.NoError(t, state.PushQC(ctx, qc2, blocks2)) for n := gr2.First; n < gr2.Next; n++ { if err := state.PushAppHash(ctx, n, types.GenAppHash(rng)); err != nil { @@ -526,7 +528,8 @@ func TestEvictionWaitsForCommitQCApp(t *testing.T) { } for inner := range state.inner.Lock() { - require.Equal(t, appFloor, inner.first, "after catching up, first reaches App.GlobalNext") + evictionBound := min(inner.nextAppProposal, inner.nextAppQC) + require.Equal(t, evictionBound, inner.first, "after catching up, first reaches min(nextAppProposal, nextAppQC)") for n := gr1.First; n < inner.first; n++ { _, ok := inner.appProposals[n] require.False(t, ok, "AppProposal %d should be evicted (< first)", n) @@ -538,7 +541,7 @@ func TestEvictionWaitsForCommitQCApp(t *testing.T) { } // Tip QC (nextQC-1) stays; nextToExecute uses maps at/above first. require.GreaterOrEqual(t, inner.nextQC-1, inner.first) - _, ok = inner.qcs[inner.nextQC-1] + _, ok := inner.qcs[inner.nextQC-1] require.True(t, ok, "tip QC must stay in maps") } return nil @@ -546,8 +549,8 @@ func TestEvictionWaitsForCommitQCApp(t *testing.T) { } // TestNextToExecuteAfterAppEviction checks WaitUntilExecuted / nextToExecute -// still work when PushQC embeds an App that aggressively evicts through -// nextAppProposal (first = App.GlobalNext = NAP). nextToExecute uses qc[NAP], +// still work when AppQC aggressively evicts through nextAppProposal +// (first = min(nextAppProposal, nextAppQC) = NAP). nextToExecute uses qc[NAP], // not NAP-1. func TestNextToExecuteAfterAppEviction(t *testing.T) { ctx := t.Context() @@ -557,9 +560,6 @@ func TestNextToExecuteAfterAppEviction(t *testing.T) { qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) gr1 := qc1.QC().GlobalRange() qc2, blocks2 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc1.QC())) - app, ok := qc2.QC().Proposal().App().Get() - require.True(t, ok) - require.Equal(t, gr1.Next, app.GlobalNext()) state := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { @@ -575,16 +575,17 @@ func TestNextToExecuteAfterAppEviction(t *testing.T) { return err } } - // Sticky case: App.GlobalNext == nextAppProposal. first advances to NAP; - // NAP-1 is gone; nextToExecute reads qc[NAP]. + // Sticky case: nextAppQC == nextAppProposal. first advances to NAP; + // NAP-1 is gone; nextToExecute reads qc[NAP] after the next QC arrives. + require.NoError(t, pushAppQCForBlock(ctx, state, keys, gr1.First)) require.NoError(t, state.PushQC(ctx, qc2, blocks2)) var tipLane types.LaneID var tipBlockNum types.BlockNumber for inner := range state.inner.Lock() { require.Equal(t, gr1.Next, inner.nextAppProposal) - require.Equal(t, app.GlobalNext(), inner.first, - "eviction advances to App.GlobalNext == NAP") + require.Equal(t, min(inner.nextAppProposal, inner.nextAppQC), inner.first, + "eviction advances to min(nextAppProposal, nextAppQC) == NAP") _, ok := inner.blocks[inner.nextAppProposal-1] require.False(t, ok, "NAP-1 must be evicted") require.Less(t, inner.nextAppProposal, inner.nextQC) @@ -670,7 +671,7 @@ func TestPruningKeepsLastQCRange(t *testing.T) { // readability), so a mid-range prune does not refuse heights inside that QC. // // PruneBefore is BlockDB-only: heights still retained in RAM for AppVotes -// (at/above CommitQC.App.GlobalNext exclusive floor) remain readable via TryBlock even +// (at/above min(nextAppProposal, nextAppQC) exclusive floor) remain readable via TryBlock even // after the store watermark advances past them. func TestPruningWithPartialQCRange(t *testing.T) { ctx := t.Context() @@ -681,15 +682,18 @@ func TestPruningWithPartialQCRange(t *testing.T) { qc2, blocks2 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc1.QC())) gr1 := qc1.QC().GlobalRange() gr2 := qc2.QC().GlobalRange() - app, ok := qc2.QC().Proposal().App().Get() - require.True(t, ok) - exclusiveFloor := app.GlobalNext() state1 := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) require.NoError(t, state1.PushQC(ctx, qc1, blocks1)) require.NoError(t, state1.PushQC(ctx, qc2, blocks2)) require.NoError(t, pushAppHashesRunning(ctx, state1, rng, gr1.First, gr2.Next)) + var exclusiveFloor types.GlobalBlockNumber + require.NoError(t, pushAppQCForBlock(ctx, state1, keys, gr1.First)) + for inner := range state1.inner.Lock() { + exclusiveFloor = min(inner.nextAppProposal, inner.nextAppQC) + require.Equal(t, exclusiveFloor, inner.first) + } // Mid-QC prune clamps to gr1.First, so the whole qc1 cohort stays readable. midQC1 := gr1.First + (gr1.Next-gr1.First)/2 From 0f43cb13610d521c0e6bc3046fba5d118fc324aa Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 4 Aug 2026 17:33:19 +0200 Subject: [PATCH 08/61] codex AppQC in BlockDB --- sei-db/ledger_db/block/block_db_test.go | 183 +++++++++++- .../block/blocksim/block_generator.go | 8 - sei-db/ledger_db/block/littblock/codec.go | 46 ++- .../block/littblock/litt_block_db.go | 122 +++++++- .../block/littblock/litt_block_iterator.go | 77 ++++- .../littblock/litt_block_simple_iterator.go | 10 +- .../ledger_db/block/memblock/mem_block_db.go | 101 ++++++- sei-tendermint/autobahn/types/block_db.go | 63 ++++- sei-tendermint/autobahn/types/errors.go | 10 + .../internal/autobahn/consensus/state.go | 1 - .../internal/autobahn/data/state.go | 264 +++++++++++++----- .../autobahn/data/state_recovery_test.go | 31 +- .../internal/autobahn/data/state_test.go | 184 ++++++++++-- 13 files changed, 943 insertions(+), 157 deletions(-) diff --git a/sei-db/ledger_db/block/block_db_test.go b/sei-db/ledger_db/block/block_db_test.go index 7b8bde798b..e0ce91004b 100644 --- a/sei-db/ledger_db/block/block_db_test.go +++ b/sei-db/ledger_db/block/block_db_test.go @@ -53,6 +53,7 @@ func TestBlockDB(t *testing.T) { t.Run("EmptyDB", func(t *testing.T) { testEmptyDB(t, impl.build) }) t.Run("ReadRoundTrip", func(t *testing.T) { testReadRoundTrip(t, impl.build) }) t.Run("QCByBlockNumber", func(t *testing.T) { testQCByBlockNumber(t, impl.build) }) + t.Run("AppQCByBlockNumber", func(t *testing.T) { testAppQCByBlockNumber(t, impl.build) }) t.Run("Iterators", func(t *testing.T) { testIterators(t, impl.build) }) t.Run("IteratorSnapshot", func(t *testing.T) { testIteratorSnapshot(t, impl.build) }) t.Run("RestartPersistsData", func(t *testing.T) { testRestartPersistsData(t, impl.build) }) @@ -78,6 +79,12 @@ func TestBlockDB(t *testing.T) { t.Run("WriteQCCoversNoBlocksRejected", func(t *testing.T) { testWriteQCCoversNoBlocksRejected(t, impl.build) }) + t.Run("WriteAppQCOrderRejected", func(t *testing.T) { + testWriteAppQCOrderRejected(t, impl.build) + }) + t.Run("PruneWithAppQCNeverEmpties", func(t *testing.T) { + testPruneWithAppQCNeverEmpties(t, impl.build) + }) t.Run("IteratorBlockRequiresPosition", func(t *testing.T) { testIteratorBlockRequiresPosition(t, impl.build) }) @@ -129,6 +136,10 @@ func testEmptyDB(t *testing.T, build builder) { require.NoError(t, err) require.False(t, qc.IsPresent()) + appQC, err := db.ReadAppQCByBlockNumber(0) + require.NoError(t, err) + require.False(t, appQC.IsPresent()) + require.Empty(t, drainIterator(t, openIterator(t, db)), "empty db should yield no positions") itAt, err := db.Iterator(0) @@ -138,6 +149,7 @@ func testEmptyDB(t *testing.T, build builder) { tips := db.Status() require.Zero(t, tips.NextBlock, "empty db has no block write tip") require.Zero(t, tips.NextQC, "empty db has no QC write tip") + require.Zero(t, tips.NextAppQC, "empty db has no AppQC write tip") } // iterEntry is one position observed while draining an iterator. @@ -150,6 +162,9 @@ type iterEntry struct { // blk is the block at the position; nil when no block is persisted there. blk *types.Block + + // appQC is the AppQC at the position; nil when no AppQC is persisted there. + appQC *types.AppQC } // openIterator opens an iterator over everything retained in db. @@ -181,7 +196,12 @@ func drainIterator(t *testing.T, it types.BlockDBIterator) []iterEntry { require.NoError(t, err) blk, present := blkOpt.Get() require.Equal(t, pos.HasBlock, present, "HasBlock must agree with Block at position %d", n) - entries = append(entries, iterEntry{n: n, qc: qc, blk: blk}) + require.Equal(t, pos.HasAppQC, pos.AppQC != nil, "HasAppQC must agree with AppQC at position %d", n) + if pos.AppQC != nil { + appGR := pos.AppQC.Proposal().GlobalRange() + require.True(t, appGR.Has(n), "AppQC [%d,%d) must cover position %d", appGR.First, appGR.Next, n) + } + entries = append(entries, iterEntry{n: n, qc: qc, blk: blk, appQC: pos.AppQC}) } return entries } @@ -271,6 +291,15 @@ func assertTipsMatchPresent(t *testing.T, db types.BlockDB) { } else { require.False(t, hasQC, "the iterator yields QCs but Status has no QC tip") } + + lastAppQC, hasAppQC := recoverLastAppQC(t, db) + if tips.NextAppQC != 0 { + require.True(t, hasAppQC, "Status has an AppQC tip but the iterator yields no AppQCs") + require.Equal(t, lastAppQC.Proposal().GlobalRange().Next, tips.NextAppQC, + "NextAppQC must be Next of the highest present AppQC") + } else { + require.False(t, hasAppQC, "the iterator yields AppQCs but Status has no AppQC tip") + } } func testReadRoundTrip(t *testing.T, build builder) { @@ -307,6 +336,55 @@ func testQCByBlockNumber(t *testing.T, build builder) { require.False(t, miss.IsPresent()) } +func testAppQCByBlockNumber(t *testing.T, build builder) { + committee, keys := buildCommittee() + batches := generateBatches(committee, keys) + db, o := openFresh(t, build) + defer func() { _ = db.Close() }() + writeAll(t, db, batches) + + rng := utils.TestRngFromSeed(testSeed + 100) + appQCs := []*types.AppQC{ + appQCForBatch(rng, keys, batches[0]), + appQCForBatch(rng, keys, batches[1]), + } + for _, appQC := range appQCs { + require.NoError(t, db.WriteAppQC(appQC)) + } + + for _, appQC := range appQCs { + gr := appQC.Proposal().GlobalRange() + for n := gr.First; n < gr.Next; n++ { + opt, err := db.ReadAppQCByBlockNumber(n) + require.NoError(t, err) + got, ok := opt.Get() + require.True(t, ok, "AppQC covering %d should exist", n) + require.Equal(t, gr, got.Proposal().GlobalRange()) + require.Equal(t, appQC.Proposal().AppHash(), got.Proposal().AppHash()) + } + } + miss, err := db.ReadAppQCByBlockNumber(batches[2].first) + require.NoError(t, err) + require.False(t, miss.IsPresent(), "CommitQCs/blocks past the AppQC prefix should not imply AppQC presence") + + entries := drainIterator(t, openIterator(t, db)) + for _, e := range entries { + switch { + case e.n < batches[2].first: + require.NotNil(t, e.appQC, "iterator should expose AppQC at %d", e.n) + default: + require.Nil(t, e.appQC, "iterator should not expose AppQC past the persisted AppQC prefix at %d", e.n) + } + } + + tips := db.Status() + require.Equal(t, batches[1].next, tips.NextAppQC) + db = restart(t, o, db) + tips = db.Status() + require.Equal(t, batches[1].next, tips.NextAppQC, "AppQC tip must survive restart") + assertTipsMatchPresent(t, db) +} + func testIterators(t *testing.T, build builder) { committee, keys := buildCommittee() batches := generateBatches(committee, keys) @@ -625,6 +703,43 @@ func testPruneNeverEmpties(t *testing.T, build builder) { } } +func testPruneWithAppQCNeverEmpties(t *testing.T, build builder) { + committee, keys := buildCommittee() + batches := generateBatches(committee, keys) + require.GreaterOrEqual(t, len(batches), 4, "need AppQC prefix to lag CommitQC tip") + db, _ := openFresh(t, build) + defer func() { _ = db.Close() }() + writeAll(t, db, batches) + + rng := utils.TestRngFromSeed(testSeed + 300) + for _, b := range batches[:3] { + require.NoError(t, db.WriteAppQC(appQCForBatch(rng, keys, b))) + } + + latestApp := batches[2] + require.NoError(t, db.PruneBefore(batches[len(batches)-1].next+1000)) + + below := batches[1] + blk, err := db.ReadBlockByNumber(below.first) + require.ErrorIs(t, err, types.ErrPruned) + require.False(t, blk.IsPresent()) + appQC, err := db.ReadAppQCByBlockNumber(below.first) + require.ErrorIs(t, err, types.ErrPruned) + require.False(t, appQC.IsPresent()) + + blk, err = db.ReadBlockByNumber(latestApp.first) + require.NoError(t, err) + require.True(t, blk.IsPresent(), "newest AppQC cohort must retain a block") + qc, err := db.ReadQCByBlockNumber(latestApp.first) + require.NoError(t, err) + require.True(t, qc.IsPresent(), "newest AppQC cohort must retain its CommitQC") + appQC, err = db.ReadAppQCByBlockNumber(latestApp.first) + require.NoError(t, err) + got, ok := appQC.Get() + require.True(t, ok, "newest AppQC cohort must retain its AppQC") + require.Equal(t, latestApp.qc.QC().GlobalRange(), got.Proposal().GlobalRange()) +} + // testPruneQCAheadOfBlocks pins the min() guard in the prune clamp. QCs are // written before the blocks they cover, so between writing a QC and its first // block — and after a crash that persisted a QC but not its blocks — the newest @@ -714,7 +829,8 @@ func testIteratorSnapshot(t *testing.T, build builder) { it := openIterator(t, db) - // Write the remaining batches AFTER the iterator was created. + // Write AppQC and the remaining batches AFTER the iterator was created. + require.NoError(t, db.WriteAppQC(appQCForBatch(utils.TestRngFromSeed(testSeed+400), keys, first))) writeAll(t, db, batches[1:]) entries := drainIterator(t, it) @@ -722,6 +838,9 @@ func testIteratorSnapshot(t *testing.T, build builder) { "iterator must not observe blocks written after creation") require.Equal(t, []types.GlobalBlockNumber{first.first}, qcFirsts(entries), "iterator must not observe QCs written after creation") + for _, e := range entries { + require.Nil(t, e.appQC, "iterator must not observe AppQCs written after creation") + } } func testWriteOrderRejected(t *testing.T, build builder) { @@ -751,6 +870,40 @@ func testWriteOrderRejected(t *testing.T, build builder) { require.True(t, opt.IsPresent()) } +func testWriteAppQCOrderRejected(t *testing.T, build builder) { + committee, keys := buildCommittee() + batches := generateBatches(committee, keys) + db, _ := openFresh(t, build) + defer func() { _ = db.Close() }() + rng := utils.TestRngFromSeed(testSeed + 200) + + b0 := batches[0] + b1 := batches[1] + b2 := batches[2] + + err := db.WriteAppQC(appQCForBatch(rng, keys, b0)) + require.ErrorIs(t, err, types.ErrAppQCMissingQC, "AppQC before CommitQC must fail") + + require.NoError(t, db.WriteQC(b0.qc)) + require.NoError(t, db.WriteQC(b1.qc)) + err = db.WriteAppQC(appQCForBatch(rng, keys, b1)) + require.ErrorIs(t, err, types.ErrAppQCNonContiguous, "first AppQC must start at retained QC floor") + + appQC0 := appQCForBatch(rng, keys, b0) + require.NoError(t, db.WriteAppQC(appQC0)) + + err = db.WriteAppQC(appQC0) + require.ErrorIs(t, err, types.ErrAppQCNonContiguous, "duplicate AppQC write must fail") + + require.NoError(t, db.WriteQC(b2.qc)) + err = db.WriteAppQC(appQCForBatch(rng, keys, b2)) + require.ErrorIs(t, err, types.ErrAppQCNonContiguous, "AppQC gap must fail") + + require.NoError(t, db.WriteAppQC(appQCForBatch(rng, keys, b1))) + tips := db.Status() + require.Equal(t, b1.next, tips.NextAppQC) +} + // testWriteOrderRejectedAfterRestart asserts the write-order cursors are // reloaded from persisted state on reopen. After a restart a freshly opened DB // must still reject an out-of-order block and a non-contiguous QC, and must @@ -873,6 +1026,19 @@ func recoverLastQC(t *testing.T, db types.BlockDB) (*types.CommitQC, bool) { return entries[len(entries)-1].qc.QC(), true } +// recoverLastAppQC returns the most recently persisted AppQC via a full +// iterator scan (false if the store has no AppQCs). +func recoverLastAppQC(t *testing.T, db types.BlockDB) (*types.AppQC, bool) { + t.Helper() + entries := drainIterator(t, openIterator(t, db)) + for i := len(entries) - 1; i >= 0; i-- { + if entries[i].appQC != nil { + return entries[i].appQC, true + } + } + return nil, false +} + // testIteratorPositioning asserts that Iterator positions at a given height: it yields the // (clamped) start and every higher covered number, densely ascending, with the whole covering QC // available even when the start falls mid-range. A start past the last covered number yields @@ -1570,15 +1736,8 @@ func buildFullCommitQC( } } } - var appQC utils.Option[*types.AppQC] - if cqc, ok := prev.Get(); ok { - p := types.NewAppProposal(cqc.GlobalRange().Next-1, types.NextIndexOpt(prev), types.GenAppHash(rng), cqc.Proposal().EpochIndex()) - appQC = utils.Some(testAppQC(keys, p)) - } else { - appQC = utils.None[*types.AppQC]() - } ep := types.NewEpoch(0, types.OpenRoadRange(), genesisTime, committee, 0) - cqc := types.BuildCommitQC(ep, keys, prev, laneQCs, appQC) + cqc := types.BuildCommitQC(ep, keys, prev, laneQCs) return types.NewFullCommitQC(cqc, headers), blockList } @@ -1591,6 +1750,10 @@ func testLaneQC(keys []types.SecretKey, header *types.BlockHeader) *types.LaneQC return types.NewLaneQC(votes) } +func appQCForBatch(rng utils.Rng, keys []types.SecretKey, b batch) *types.AppQC { + return testAppQC(keys, types.NewAppProposal(b.qc.QC().Proposal(), types.GenAppHash(rng))) +} + func testAppQC(keys []types.SecretKey, proposal *types.AppProposal) *types.AppQC { vote := types.NewAppVote(proposal) votes := make([]*types.Signed[*types.AppVote], 0, len(keys)) diff --git a/sei-db/ledger_db/block/blocksim/block_generator.go b/sei-db/ledger_db/block/blocksim/block_generator.go index 0b13f9ad0e..596e9b78f7 100644 --- a/sei-db/ledger_db/block/blocksim/block_generator.go +++ b/sei-db/ledger_db/block/blocksim/block_generator.go @@ -170,19 +170,11 @@ func (g *BlockGenerator) buildFullCommitQC() (*types.FullCommitQC, []*types.Bloc viewSpec := types.ViewSpec{CommitQC: prev, Epoch: types.NewEpoch(0, types.OpenRoadRange(), genesisTime, committee, 0)} leader := committee.Leader(viewSpec.View()) - appQC := func() utils.Option[*types.AppQC] { - if n := viewSpec.NextGlobalBlock(); n > 0 { - p := types.NewAppProposal(n-1, viewSpec.View().Index, types.AppHash(g.rand.Bytes(hashSizeBytes)), viewSpec.Epoch.EpochIndex()) - return utils.Some(g.fakeAppQC(p)) - } - return utils.None[*types.AppQC]() - }() proposal := utils.OrPanic1(types.NewProposalForTesting( committee, viewSpec, time.Now(), laneQCs, - appQC, g.fakeSig(leader), )) commitVote := types.NewCommitVote(proposal.Proposal().Msg()) diff --git a/sei-db/ledger_db/block/littblock/codec.go b/sei-db/ledger_db/block/littblock/codec.go index a536e64061..4abb446c3c 100644 --- a/sei-db/ledger_db/block/littblock/codec.go +++ b/sei-db/ledger_db/block/littblock/codec.go @@ -16,10 +16,12 @@ import ( // - kindBlock 'b' + 8-byte big-endian GlobalBlockNumber (block primary key) // - kindBlockHash 'h' + 32-byte header hash (block hash alias) // - kindQC 'q' + 8-byte big-endian GlobalBlockNumber (QC primary + covered aliases) +// - kindAppQC 'a' + 8-byte big-endian GlobalBlockNumber (AppQC primary + covered aliases) const ( kindBlock byte = 'b' kindBlockHash byte = 'h' kindQC byte = 'q' + kindAppQC byte = 'a' ) // encodeKey encodes a GlobalBlockNumber as an 8-byte big-endian value. Big-endian @@ -53,13 +55,20 @@ func qcKey(n types.GlobalBlockNumber) []byte { return append([]byte{kindQC}, encodeKey(n)...) } +// appQCKey returns the key for AppQC number n — used both for an AppQC's +// primary key and for each covered-number alias. +func appQCKey(n types.GlobalBlockNumber) []byte { + return append([]byte{kindAppQC}, encodeKey(n)...) +} + // keyKind returns the kind prefix byte of a stored key. func keyKind(key []byte) byte { return key[0] } -// decodeNumberKey decodes the GlobalBlockNumber from a kindBlock or kindQC key -// (i.e. a key whose prefix is followed by an 8-byte big-endian number). +// decodeNumberKey decodes the GlobalBlockNumber from a kindBlock, kindQC, or +// kindAppQC key (i.e. a key whose prefix is followed by an 8-byte big-endian +// number). func decodeNumberKey(key []byte) types.GlobalBlockNumber { return decodeKey(key[1:]) } @@ -70,6 +79,9 @@ const blockSerializationVersion byte = 1 // Serialization version for QCs. const qcSerializationVersion byte = 1 +// Serialization version for AppQCs. +const appQCSerializationVersion byte = 1 + // blockValuePrefixLen is the fixed header preceding a block's proto bytes: one // version byte followed by the 8-byte big-endian GlobalBlockNumber. const blockValuePrefixLen = 1 + 8 @@ -142,3 +154,33 @@ func decodeQC(value []byte) (*types.FullCommitQC, error) { } return qc, nil } + +func appQCRange(appQC *types.AppQC) (types.GlobalBlockNumber, types.GlobalBlockNumber) { + gr := appQC.Proposal().GlobalRange() + return gr.First, gr.Next +} + +// encodeAppQC marshals an AppQC to the bytes stored as its table value, +// framed as [version:1][proto(AppQC)]. +func encodeAppQC(appQC *types.AppQC) []byte { + proto := types.AppQCConv.Marshal(appQC) + value := make([]byte, 0, 1+len(proto)) + value = append(value, appQCSerializationVersion) + value = append(value, proto...) + return value +} + +// decodeAppQC unmarshals an AppQC from the value produced by encodeAppQC. +func decodeAppQC(value []byte) (*types.AppQC, error) { + if len(value) < 1 { + return nil, fmt.Errorf("appQC value too short: %d bytes", len(value)) + } + if value[0] != appQCSerializationVersion { + return nil, fmt.Errorf("unsupported appQC serialization version %d", value[0]) + } + appQC, err := types.AppQCConv.Unmarshal(value[1:]) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal appQC: %w", err) + } + return appQC, nil +} diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index f29576b62d..490b66caa6 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -47,10 +47,17 @@ type blockDB struct { lastBlockNumber types.GlobalBlockNumber hasQC bool lastQCNext types.GlobalBlockNumber + hasAppQC bool + lastAppQCNext types.GlobalBlockNumber // latestQCStartBlock is the most recently written QC's starting block number. latestQCStartBlock types.GlobalBlockNumber + // latestAppQCStartBlock is the most recently written AppQC's starting block + // number. When AppQCs exist, PruneBefore clamps to it so the newest AppQC + // cohort remains readable together with its CommitQC and blocks. + latestAppQCStartBlock types.GlobalBlockNumber + // firstBlockNumber is the lowest block number this handle has seen. Iterator clamps its // start up to it so a scan always opens on a block that exists: the first block may be // written anywhere inside its covering QC, so this can sit above oldestQCStart with no @@ -120,7 +127,7 @@ func NewBlockDB(config *LittBlockConfig) (types.BlockDB, error) { } // recoverCursors reloads the write-order cursors (lastBlockNumber, lastQCNext, -// and their presence flags) from on-disk state. Without this, a reopened DB +// lastAppQCNext, and their presence flags) from on-disk state. Without this, a reopened DB // would treat itself as empty and let WriteBlock/WriteQC silently accept // out-of-order or non-contiguous writes that overwrite or gap persisted data. func (s *blockDB) recoverCursors() error { @@ -130,7 +137,7 @@ func (s *blockDB) recoverCursors() error { } defer func() { _ = it.Close() }() - for !s.hasBlocks || !s.hasQC { + for !s.hasBlocks || !s.hasQC || !s.hasAppQC { ok, err := it.Next() if err != nil { return fmt.Errorf("failed to advance recovery iterator: %w", err) @@ -164,6 +171,19 @@ func (s *blockDB) recoverCursors() error { s.latestQCStartBlock, s.lastQCNext = coveredRange(qc) s.hasQC = true } + case kindAppQC: + if !s.hasAppQC { + value, err := it.GetValue() + if err != nil { + return fmt.Errorf("failed to read newest appQC value: %w", err) + } + appQC, err := decodeAppQC(value) + if err != nil { + return fmt.Errorf("failed to unmarshal newest appQC: %w", err) + } + s.latestAppQCStartBlock, s.lastAppQCNext = appQCRange(appQC) + s.hasAppQC = true + } } } return nil @@ -300,6 +320,55 @@ func (s *blockDB) WriteQC(qc *types.FullCommitQC) error { return nil } +func (s *blockDB) WriteAppQC(appQC *types.AppQC) error { + first, next := appQCRange(appQC) + if first >= next { + return fmt.Errorf("AppQC at %d covers no blocks: %w", first, types.ErrAppQCNonContiguous) + } + s.mu.Lock() + defer s.mu.Unlock() + if !s.hasQC { + return fmt.Errorf("AppQC [%d,%d) has no matching QC: %w", first, next, types.ErrAppQCMissingQC) + } + if s.hasAppQC { + if first != s.lastAppQCNext { + return fmt.Errorf("AppQC starts at %d, expected %d: %w", + first, s.lastAppQCNext, types.ErrAppQCNonContiguous) + } + } else if first != s.oldestQCStart { + return fmt.Errorf("first AppQC starts at %d, expected retained QC floor %d: %w", + first, s.oldestQCStart, types.ErrAppQCNonContiguous) + } + + qc, err := readQCCovering(s.table, first) + if err != nil { + return fmt.Errorf("read matching QC for AppQC [%d,%d): %w", first, next, err) + } + qcFirst, qcNext := coveredRange(qc) + if qcFirst != first || qcNext != next { + return fmt.Errorf("AppQC [%d,%d) does not exactly match QC [%d,%d): %w", + first, next, qcFirst, qcNext, types.ErrAppQCMissingQC) + } + + value := encodeAppQC(appQC) + var aliases []*litttypes.SecondaryKey + for m := first + 1; m < next; m++ { + aliases = append(aliases, &litttypes.SecondaryKey{ + Key: appQCKey(m), + Offset: 0, + Length: uint32(len(value)), //nolint:gosec // value length fits u32 (litt value cap is 2^32) + }) + } + if err := s.table.Put(appQCKey(first), value, aliases...); err != nil { + return fmt.Errorf("failed to put AppQC [%d,%d): %w", first, next, err) + } + + s.latestAppQCStartBlock = first + s.lastAppQCNext = next + s.hasAppQC = true + return nil +} + func (s *blockDB) PruneBefore(blockHeight types.GlobalBlockNumber) error { s.mu.Lock() defer s.mu.Unlock() @@ -310,7 +379,11 @@ func (s *blockDB) PruneBefore(blockHeight types.GlobalBlockNumber) error { return nil } - if ceiling := min(s.latestQCStartBlock, s.lastBlockNumber); blockHeight > ceiling { + ceiling := min(s.latestQCStartBlock, s.lastBlockNumber) + if s.hasAppQC { + ceiling = min(s.latestAppQCStartBlock, s.lastBlockNumber) + } + if blockHeight > ceiling { blockHeight = ceiling } @@ -351,16 +424,16 @@ func (s *blockDB) clampPruneBoundary(blockHeight types.GlobalBlockNumber) (types // // - block-number keys are reclaimable once the block number is strictly below // the prune watermark; -// - QC keys (the primary First and every per-covered-number secondary) are +// - QC and AppQC keys (the primary First and every per-covered-number secondary) are // reclaimable once their number is below the watermark, so a QC's segment is // reclaimable only once its highest covered number (Next-1) is below the -// watermark — i.e. once Next <= watermark; a QC straddling the watermark is -// retained; +// watermark — i.e. once Next <= watermark; a QC/AppQC straddling the +// watermark is retained; // - header-hash aliases share their block's segment, so they always pass — the // block's primary number key is what actually gates segment reclamation. func (s *blockDB) gcFilter(key []byte, _ bool) (bool, error) { switch keyKind(key) { - case kindBlock, kindQC: + case kindBlock, kindQC, kindAppQC: return uint64(decodeNumberKey(key)) < s.watermark.Load(), nil case kindBlockHash: return true, nil @@ -386,6 +459,9 @@ func (s *blockDB) Status() types.DBStatus { if s.hasQC { tips.NextQC = s.lastQCNext } + if s.hasAppQC { + tips.NextAppQC = s.lastAppQCNext + } return tips } @@ -415,7 +491,11 @@ func (s *blockDB) Iterator(n types.GlobalBlockNumber) (types.BlockDBIterator, er if start >= nextQC { return &blockDBIterator{}, nil } - return newSimpleIterator(s.table, start, nextQC) + appQCs, err := snapshotAppQCs(s.table, start, nextQC) + if err != nil { + return nil, err + } + return newSimpleIterator(s.table, start, nextQC, appQCs) } // firstBlock is where the block history begins, which can sit inside its covering QC's range @@ -428,6 +508,11 @@ func (s *blockDB) Iterator(n types.GlobalBlockNumber) (types.BlockDBIterator, er return &blockDBIterator{}, nil } + appQCs, err := snapshotAppQCs(s.table, start, nextQC) + if err != nil { + return nil, err + } + // A QC is stored under its First as the primary key with a covered-number alias for every // other number in its range, and an alias carries the full QC value. Positioning the scan at // qcKey(start) therefore lands on the covering QC no matter where start falls in its range. @@ -457,6 +542,7 @@ func (s *blockDB) Iterator(n types.GlobalBlockNumber) (types.BlockDBIterator, er } return &blockDBIterator{ it: it, + appQCs: appQCs, startN: start, expectStartQC: true, }, nil @@ -526,6 +612,26 @@ func (s *blockDB) ReadQCByBlockNumber( return utils.Some(qc), nil } +func (s *blockDB) ReadAppQCByBlockNumber( + n types.GlobalBlockNumber, +) (utils.Option[*types.AppQC], error) { + if uint64(n) < s.watermark.Load() { + return utils.None[*types.AppQC](), types.ErrPruned + } + value, exists, err := s.table.Get(appQCKey(n)) + if err != nil { + return utils.None[*types.AppQC](), fmt.Errorf("failed to read AppQC: %w", err) + } + if !exists { + return utils.None[*types.AppQC](), nil + } + appQC, err := decodeAppQC(value) + if err != nil { + return utils.None[*types.AppQC](), fmt.Errorf("failed to unmarshal AppQC: %w", err) + } + return utils.Some(appQC), nil +} + func (s *blockDB) Close() error { if err := s.db.Close(); err != nil { return fmt.Errorf("failed to close litt db: %w", err) diff --git a/sei-db/ledger_db/block/littblock/litt_block_iterator.go b/sei-db/ledger_db/block/littblock/litt_block_iterator.go index bc7629e84c..ea0745f190 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_iterator.go +++ b/sei-db/ledger_db/block/littblock/litt_block_iterator.go @@ -22,6 +22,12 @@ type coveredQC struct { next types.GlobalBlockNumber } +type coveredAppQC struct { + appQC *types.AppQC + first types.GlobalBlockNumber + next types.GlobalBlockNumber +} + // blockDBIterator implements types.BlockDBIterator over the shared ledger table. // // It steps through consecutive block numbers, driven by a single forward litt scan. The scan @@ -36,6 +42,12 @@ type blockDBIterator struct { // it is the underlying litt scan; nil for an empty iterator. it littdb.Iterator + // appQCs is a snapshot of AppQC coverage captured when the iterator was + // opened. AppQCs may be written after the blocks they certify, so the + // insertion-order block/QC scan cannot discover them before yielding those + // blocks without a separate snapshot. + appQCs []*coveredAppQC + // startN is the first number the iterator may yield, and doubles as the retention floor: // blockDB.Iterator clamps it up to the prune watermark and to the start of the block history, // so a block below startN is either below the start or stranded from a reclaimed QC, and is @@ -156,10 +168,18 @@ func (l *blockDBIterator) Next() (types.Position, bool, error) { types.ErrBlockGap, next, l.heldNumber) } + appQC := appQCCovering(l.appQCs, next) + l.n = next l.started = true l.positioned = true - return types.Position{Number: next, QC: l.current.qc, HasBlock: l.heldBlock}, true, nil + return types.Position{ + Number: next, + QC: l.current.qc, + HasBlock: l.heldBlock, + AppQC: appQC, + HasAppQC: appQC != nil, + }, true, nil } // fill advances the underlying scan until it holds an unconsumed block record or exhausts, @@ -207,6 +227,8 @@ func (l *blockDBIterator) fill() error { if err := l.collectQC(); err != nil { return err } + case keyKind(key) == kindAppQC: + // AppQC coverage was snapshotted when this iterator opened. default: return fmt.Errorf("unknown ledger key kind %q", keyKind(key)) } @@ -214,6 +236,59 @@ func (l *blockDBIterator) fill() error { return nil } +func appQCCovering(appQCs []*coveredAppQC, n types.GlobalBlockNumber) *types.AppQC { + for _, a := range appQCs { + if a.first <= n && n < a.next { + return a.appQC + } + } + return nil +} + +func snapshotAppQCs( + table littdb.Table, + start types.GlobalBlockNumber, + nextQC types.GlobalBlockNumber, +) ([]*coveredAppQC, error) { + it, err := table.Iterator(false) + if err != nil { + return nil, fmt.Errorf("failed to open AppQC snapshot iterator: %w", err) + } + defer func() { _ = it.Close() }() + + var appQCs []*coveredAppQC + for { + ok, err := it.Next() + if err != nil { + return nil, fmt.Errorf("failed to advance AppQC snapshot iterator: %w", err) + } + if !ok { + break + } + key, isPrimary, err := it.GetKey() + if err != nil { + return nil, fmt.Errorf("failed to read AppQC snapshot key: %w", err) + } + if !isPrimary || keyKind(key) != kindAppQC { + continue + } + value, err := it.GetValue() + if err != nil { + return nil, fmt.Errorf("failed to read AppQC snapshot value: %w", err) + } + appQC, err := decodeAppQC(value) + if err != nil { + return nil, fmt.Errorf("failed to decode AppQC snapshot value: %w", err) + } + first, next := appQCRange(appQC) + if next <= start || first >= nextQC { + continue + } + appQCs = append(appQCs, &coveredAppQC{appQC: appQC, first: first, next: next}) + } + return appQCs, nil +} + // collectQC decodes the QC record at the scan's current position into the covering-QC state: it // becomes current when no current QC is set, and otherwise joins the pending queue. func (l *blockDBIterator) collectQC() error { diff --git a/sei-db/ledger_db/block/littblock/litt_block_simple_iterator.go b/sei-db/ledger_db/block/littblock/litt_block_simple_iterator.go index 956ff783d1..04183c5a52 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_simple_iterator.go +++ b/sei-db/ledger_db/block/littblock/litt_block_simple_iterator.go @@ -46,6 +46,7 @@ func newSimpleIterator( table qcReader, start types.GlobalBlockNumber, nextQC types.GlobalBlockNumber, + appQCs []*coveredAppQC, ) (*simpleIterator, error) { var positions []types.Position for n := start; n < nextQC; { @@ -61,7 +62,14 @@ func newSimpleIterator( n, first, next) } for m := n; m < next && m < nextQC; m++ { - positions = append(positions, types.Position{Number: m, QC: qc, HasBlock: false}) + appQC := appQCCovering(appQCs, m) + positions = append(positions, types.Position{ + Number: m, + QC: qc, + HasBlock: false, + AppQC: appQC, + HasAppQC: appQC != nil, + }) } n = next } diff --git a/sei-db/ledger_db/block/memblock/mem_block_db.go b/sei-db/ledger_db/block/memblock/mem_block_db.go index ece2e6ed7c..1ae8188679 100644 --- a/sei-db/ledger_db/block/memblock/mem_block_db.go +++ b/sei-db/ledger_db/block/memblock/mem_block_db.go @@ -19,6 +19,13 @@ type qcEntry struct { upper types.GlobalBlockNumber } +// appQCEntry pairs an AppQC with the half-open range [lower, upper) it covers. +type appQCEntry struct { + appQC *types.AppQC + lower types.GlobalBlockNumber + upper types.GlobalBlockNumber +} + // hashEntry pairs a block with its GlobalBlockNumber so ReadBlockByHash can // return the number, mirroring the littblock implementation which embeds it in // the stored value. @@ -35,18 +42,26 @@ type blockDB struct { byNumber map[types.GlobalBlockNumber]*types.Block byHash map[types.BlockHeaderHash]hashEntry qcsByLower map[types.GlobalBlockNumber]qcEntry + appQCs map[types.GlobalBlockNumber]appQCEntry // Write-order cursors (see types.BlockDB contract). hasBlocks bool lastBlockNumber types.GlobalBlockNumber hasQC bool lastQCNext types.GlobalBlockNumber + hasAppQC bool + lastAppQCNext types.GlobalBlockNumber // latestQCStartBlock is the most recently written QC's starting block number — // the lowest block number in the newest cohort. PruneBefore clamps to it (see // littblock). latestQCStartBlock types.GlobalBlockNumber + // latestAppQCStartBlock is the most recently written AppQC's starting + // block number. When AppQCs exist, PruneBefore clamps to it so the newest + // AppQC cohort remains readable together with its CommitQC and blocks. + latestAppQCStartBlock types.GlobalBlockNumber + // firstBlockNumber is the lowest block number written. Iterator clamps its start up to // it so a scan always opens on a block that exists; the first block may land anywhere // inside its covering QC, so this can sit above that QC's start with no block in @@ -66,6 +81,7 @@ func NewBlockDB() types.BlockDB { byNumber: make(map[types.GlobalBlockNumber]*types.Block), byHash: make(map[types.BlockHeaderHash]hashEntry), qcsByLower: make(map[types.GlobalBlockNumber]qcEntry), + appQCs: make(map[types.GlobalBlockNumber]appQCEntry), } } @@ -119,6 +135,48 @@ func (s *blockDB) WriteQC(qc *types.FullCommitQC) error { return nil } +func appQCRange(appQC *types.AppQC) (types.GlobalBlockNumber, types.GlobalBlockNumber) { + gr := appQC.Proposal().GlobalRange() + return gr.First, gr.Next +} + +func (s *blockDB) WriteAppQC(appQC *types.AppQC) error { + first, next := appQCRange(appQC) + if first >= next { + return fmt.Errorf("AppQC at %d covers no blocks: %w", first, types.ErrAppQCNonContiguous) + } + s.mu.Lock() + defer s.mu.Unlock() + if !s.hasQC { + return fmt.Errorf("AppQC [%d,%d) has no matching QC: %w", first, next, types.ErrAppQCMissingQC) + } + if s.hasAppQC { + if first != s.lastAppQCNext { + return fmt.Errorf("AppQC starts at %d, expected %d: %w", + first, s.lastAppQCNext, types.ErrAppQCNonContiguous) + } + } else { + entries := s.sortedQCsLocked() + if len(entries) == 0 { + return fmt.Errorf("AppQC [%d,%d) has no retained QC floor: %w", first, next, types.ErrAppQCMissingQC) + } + if first != entries[0].lower { + return fmt.Errorf("first AppQC starts at %d, expected retained QC floor %d: %w", + first, entries[0].lower, types.ErrAppQCNonContiguous) + } + } + qc, ok := s.qcsByLower[first] + if !ok || qc.upper != next { + return fmt.Errorf("AppQC [%d,%d) has no exact matching QC: %w", + first, next, types.ErrAppQCMissingQC) + } + s.appQCs[first] = appQCEntry{appQC: appQC, lower: first, upper: next} + s.latestAppQCStartBlock = first + s.lastAppQCNext = next + s.hasAppQC = true + return nil +} + func (s *blockDB) PruneBefore(n types.GlobalBlockNumber) error { s.mu.Lock() defer s.mu.Unlock() @@ -131,7 +189,11 @@ func (s *blockDB) PruneBefore(n types.GlobalBlockNumber) error { // at the cohort's first block (latestQCStartBlock), guarded by lastBlockNumber // for a QC written ahead of its blocks. Keeps the newest cohort whole and // pruning monotonic. See littblock and the BlockDB PruneBefore contract. - if ceiling := min(s.latestQCStartBlock, s.lastBlockNumber); n > ceiling { + ceiling := min(s.latestQCStartBlock, s.lastBlockNumber) + if s.hasAppQC { + ceiling = min(s.latestAppQCStartBlock, s.lastBlockNumber) + } + if n > ceiling { n = ceiling } // Round the watermark down to the covering QC's First. A QC's cohort of @@ -157,6 +219,11 @@ func (s *blockDB) PruneBefore(n types.GlobalBlockNumber) error { delete(s.qcsByLower, lower) } } + for lower, e := range s.appQCs { + if e.upper <= s.watermark { + delete(s.appQCs, lower) + } + } return nil } @@ -172,6 +239,9 @@ func (s *blockDB) Status() types.DBStatus { if s.hasQC { tips.NextQC = s.lastQCNext } + if s.hasAppQC { + tips.NextAppQC = s.lastAppQCNext + } return tips } @@ -221,11 +291,21 @@ func (s *blockDB) iteratorLocked(entries []qcEntry, start types.GlobalBlockNumbe it.nums = append(it.nums, num) it.qcs = append(it.qcs, e.qc) it.blocks = append(it.blocks, s.byNumber[num]) + it.appQCs = append(it.appQCs, s.appQCCoveringLocked(num)) } } return it } +func (s *blockDB) appQCCoveringLocked(n types.GlobalBlockNumber) *types.AppQC { + for _, e := range s.appQCs { + if e.lower <= n && n < e.upper { + return e.appQC + } + } + return nil +} + var _ types.BlockDBIterator = (*memBlockDBIterator)(nil) // memBlockDBIterator steps through a snapshot of covered numbers captured at creation. @@ -239,6 +319,9 @@ type memBlockDBIterator struct { // blocks holds the block per position; nil where no block is persisted. blocks []*types.Block + // appQCs holds the AppQC per position; nil where no AppQC is persisted. + appQCs []*types.AppQC + // idx is the current position; -1 before the first Next and len(nums) once exhausted. idx int @@ -257,6 +340,8 @@ func (it *memBlockDBIterator) Next() (types.Position, bool, error) { Number: it.nums[it.idx], QC: it.qcs[it.idx], HasBlock: it.blocks[it.idx] != nil, + AppQC: it.appQCs[it.idx], + HasAppQC: it.appQCs[it.idx] != nil, }, true, nil } @@ -322,4 +407,18 @@ func (s *blockDB) ReadQCByBlockNumber( return utils.None[*types.FullCommitQC](), nil } +func (s *blockDB) ReadAppQCByBlockNumber( + n types.GlobalBlockNumber, +) (utils.Option[*types.AppQC], error) { + s.mu.RLock() + defer s.mu.RUnlock() + if n < s.watermark { + return utils.None[*types.AppQC](), types.ErrPruned + } + if appQC := s.appQCCoveringLocked(n); appQC != nil { + return utils.Some(appQC), nil + } + return utils.None[*types.AppQC](), nil +} + func (s *blockDB) Close() error { return nil } diff --git a/sei-tendermint/autobahn/types/block_db.go b/sei-tendermint/autobahn/types/block_db.go index b2f62b0936..d3ec594126 100644 --- a/sei-tendermint/autobahn/types/block_db.go +++ b/sei-tendermint/autobahn/types/block_db.go @@ -4,22 +4,23 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" ) -// BlockDB is the durable backing store for data.State. It persists the two -// kinds of finalized records the consensus state machine produces — -// finalized blocks (indexed by GlobalBlockNumber and by header hash) and -// FullCommitQCs (each covering a contiguous range of GlobalBlockNumbers) — -// and provides the read API needed for crash recovery and runtime lookups. +// BlockDB is the durable backing store for data.State. It persists the +// finalized records the consensus state machine produces — finalized blocks +// (indexed by GlobalBlockNumber and by header hash), FullCommitQCs (each +// covering a contiguous range of GlobalBlockNumbers), and AppQCs (each matching +// a persisted CommitQC range) — and provides the read API needed for crash +// recovery and runtime lookups. // // # Concurrency // // All methods are safe for concurrent use. Implementations should expect -// concurrent writes (WriteBlock + WriteQC interleaved from a single +// concurrent writes (WriteBlock + WriteQC + WriteAppQC interleaved from a single // background persistence loop) and concurrent reads from RPC handlers // and peer-sync streams. // // # Durability and crash safety // -// Writes are two-phase: WriteBlock and WriteQC return without +// Writes are two-phase: WriteBlock, WriteQC, and WriteAppQC return without // guaranteeing the record is on disk. Flush blocks until all // previously-returned Writes are durable. // @@ -47,14 +48,18 @@ import ( // ErrQCNonContiguous otherwise. // - QCs must be written before blocks. A QC covering a block must // be written before that block is written. +// - AppQCs must be written contiguously as an exact prefix of retained QCs. +// The first AppQC starts at the retained QC floor; every AppQC's range must +// exactly match the next persisted QC range. // // After a crash, data not flushed may be lost, but the following invariants hold: // -// - Individual blocks and QCs are either fully persisted or not at all; there are no partial writes. +// - Individual blocks, QCs, and AppQCs are either fully persisted or not at all; there are no partial writes. // - Data is persisted in order, meaning that data loss never leaves gaps. If A is written and then B // is written, then after a crash if B is persisted then A is also persisted. -// - Since QCs must always be written before the blocks they cover, a persisted block is always covered -// by a persisted QC, but a persisted QC may or may not have its covered blocks persisted. +// - Since QCs must always be written before the blocks or AppQCs they cover, a persisted block or +// AppQC is always covered by a persisted QC, but a persisted QC may or may not have its covered +// blocks or AppQC persisted. // // # A readable block always has a readable covering QC // @@ -107,6 +112,20 @@ type BlockDB interface { // so loss of non-durable data after a crash never leaves gaps. WriteQC(qc *FullCommitQC) error + // WriteAppQC persists an AppQC. The AppQC's proposal carries the exact + // CommitQC range it certifies. A matching CommitQC must already be written: + // the CommitQC covering GlobalRange.First must have the same GlobalRange. + // + // AppQCs form a contiguous prefix aligned with retained CommitQCs. The first + // AppQC must start at the retained CommitQC floor; each subsequent AppQC's + // First must equal the previous AppQC's Next. Re-writing, gaps, overlaps, + // mid-QC starts, and ranges that do not exactly match the next persisted + // CommitQC range are rejected. + // + // May return before the AppQC is on disk. See the BlockDB type doc for the + // two-phase write/flush contract. + WriteAppQC(appQC *AppQC) error + // PruneBefore advances the retention watermark toward n and removes // everything below it: // - every block with GlobalBlockNumber < watermark @@ -241,6 +260,20 @@ type BlockDB interface { // Non-blocking. ReadQCByBlockNumber(n GlobalBlockNumber) (utils.Option[*FullCommitQC], error) + // ReadAppQCByBlockNumber returns the AppQC whose + // AppProposal.GlobalRange().First ≤ n < AppProposal.GlobalRange().Next. + // Because a single AppQC covers a CommitQC range, the same *AppQC is + // returned for every n in its range. + // + // The result is one of: + // - utils.Some with a nil error: an AppQC covering n is present. + // - ErrPruned: n is strictly below the current retention watermark. + // - utils.None with a nil error: n is at or above the watermark but no + // AppQC covers it. + // + // Non-blocking. + ReadAppQCByBlockNumber(n GlobalBlockNumber) (utils.Option[*AppQC], error) + // Close releases resources held by the store. After Close returns, // no other method may be called on the BlockDB; doing so is // undefined. @@ -260,6 +293,9 @@ type DBStatus struct { // accepted by WriteQC (the next QC's range must start here). Zero if no QC // has been written. NextQC GlobalBlockNumber + // NextAppQC is one past the highest GlobalBlockNumber covered by the last + // AppQC accepted by WriteAppQC. Zero if no AppQC has been written. + NextAppQC GlobalBlockNumber } // BlockDBIterator steps through consecutive GlobalBlockNumbers in ascending @@ -328,4 +364,11 @@ type Position struct { // whose covering QC was persisted but whose block was not (e.g. lost in // a crash, or not yet written). HasBlock bool + + // AppQC is the AppQC covering Number, if one has been persisted. It is nil + // when no AppQC covers Number. + AppQC *AppQC + + // HasAppQC reports whether AppQC is present at Number. + HasAppQC bool } diff --git a/sei-tendermint/autobahn/types/errors.go b/sei-tendermint/autobahn/types/errors.go index 64376fd724..fdade4a772 100644 --- a/sei-tendermint/autobahn/types/errors.go +++ b/sei-tendermint/autobahn/types/errors.go @@ -28,6 +28,16 @@ var ErrQCNonContiguous = errors.New("block: WriteQC non-contiguous") // before that block (see the BlockDB ordering contract). var ErrBlockMissingQC = errors.New("block: WriteBlock without covering QC") +// ErrAppQCNonContiguous is returned by WriteAppQC when the supplied AppQC does +// not extend the existing AppQC prefix. AppQCs must be written as a contiguous, +// ascending sequence aligned with the retained CommitQC prefix. +var ErrAppQCNonContiguous = errors.New("block: WriteAppQC non-contiguous") + +// ErrAppQCMissingQC is returned by WriteAppQC when no previously written +// CommitQC exactly matches the AppQC's GlobalRange. The matching CommitQC must +// be written before the AppQC. +var ErrAppQCMissingQC = errors.New("block: WriteAppQC without matching CommitQC") + // ErrPruned is returned when a requested record is below the current retention // / eviction floor and is not served. Used for BlockDB by-number reads below // the store watermark, for BlockDB.Iterator when a concurrent PruneBefore moves diff --git a/sei-tendermint/internal/autobahn/consensus/state.go b/sei-tendermint/internal/autobahn/consensus/state.go index 583ec1f270..80a7375258 100644 --- a/sei-tendermint/internal/autobahn/consensus/state.go +++ b/sei-tendermint/internal/autobahn/consensus/state.go @@ -229,7 +229,6 @@ func (s *State) runPropose(ctx context.Context) error { vs, time.Now(), laneQCsMap, - s.avail.LastAppQC(), ) if err != nil { return fmt.Errorf("s.avail.WaitForProposal(): %w", err) diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index 251031acbb..dc94433375 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -41,33 +41,31 @@ type blockEntry struct { type inner struct { // Map key ranges (low end = first): - // qcs: [first, nextQC) - // blocks: [first, nextBlock) + gap-fills in [nextBlock, nextQC) - // appQCS: [first, nextAppQC) - // appProposals: [first, nextAppProposal) - // blockHashes: mirrors blocks (insertBlock / evictBelowBound) // // Durable copies below first live in BlockDB. AppProposals are not // persisted; they are rebuilt via PushAppHash / re-execution after restart. - qcs map[types.GlobalBlockNumber]*types.FullCommitQC - blocks map[types.GlobalBlockNumber]*types.Block - appProposals map[types.GlobalBlockNumber]*types.AppProposal - appQCs map[types.GlobalBlockNumber]*types.AppQC - blockHashes map[types.BlockHeaderHash]types.GlobalBlockNumber + qcs map[types.GlobalBlockNumber]*types.FullCommitQC // [first, nextQC) + blocks map[types.GlobalBlockNumber]*types.Block // [first, nextBlock) + gap-fills in [nextBlock, nextQC) + appProposals map[types.GlobalBlockNumber]*types.AppProposal // [first, nextAppProposal) + appQCs map[types.GlobalBlockNumber]*types.AppQC // [first, nextAppQC) + blockHashes map[types.BlockHeaderHash]types.GlobalBlockNumber // blockHashes: mirrors blocks (insertBlock / evictBelowBound) // first is the exclusive low end of retained in-memory state: maps keep // [first, next*). Set by newInner / skipTo; advanced by evictBelowBound to - // min(nextAppProposal, nextAppQC). + // min(nextAppProposal, nextAppQCToPersist). // nextToExecute reads the next (or tip) QC from maps — it does not need // nextAppProposal-1 retained after eviction. // - // first <= nextAppProposal,nextAppQC <= nextBlockToPersist <= nextBlock <= nextQC + // first <= nextAppProposal <= nextBlockToPersist <= nextBlock <= nextQC + // first <= nextAppQCToPersist <= nextBlockToPersist + // first <= nextAppQC <= nextBlock // // AppProposals require persistence (nextAppProposal <= nextBlockToPersist). // BlockDB prune status lives only in the store watermark (see PruneBefore). first types.GlobalBlockNumber nextAppProposal types.GlobalBlockNumber nextAppQC types.GlobalBlockNumber + nextAppQCToPersist types.GlobalBlockNumber nextBlockToPersist types.GlobalBlockNumber nextBlock types.GlobalBlockNumber nextQC types.GlobalBlockNumber @@ -83,6 +81,7 @@ func newInner(firstBlock types.GlobalBlockNumber) *inner { first: firstBlock, nextAppProposal: firstBlock, nextAppQC: firstBlock, + nextAppQCToPersist: firstBlock, nextBlockToPersist: firstBlock, nextBlock: firstBlock, nextQC: firstBlock, @@ -96,6 +95,7 @@ func (i *inner) skipTo(n types.GlobalBlockNumber) { i.first = n i.nextAppProposal = n i.nextAppQC = n + i.nextAppQCToPersist = n i.nextBlockToPersist = n i.nextBlock = n i.nextQC = n @@ -126,6 +126,54 @@ func (i *inner) insertQC(registry *epoch.Registry, qc *types.FullCommitQC) error return nil } +func (i *inner) verifyAppQC(registry *epoch.Registry, appQC *types.AppQC) error { + proposal := appQC.Proposal() + ep, ok := registry.EpochByIndex(proposal.EpochIndex()) + if !ok { + return fmt.Errorf("unknown epoch_index %d", proposal.EpochIndex()) + } + if err := appQC.Verify(ep.Committee()); err != nil { + return fmt.Errorf("appQC.Verify(): %w", err) + } + gr := proposal.GlobalRange() + if gr.Next <= i.nextAppQC { + return nil + } + if gr.First != i.nextAppQC { + return fmt.Errorf("AppQC gap: expected first=%d, got %d", i.nextAppQC, gr.First) + } + for n := gr.First; n < gr.Next; n++ { + qc := i.qcs[n] + if qc == nil { + return fmt.Errorf("missing QC for AppQC block %d", n) + } + if err := proposal.Verify(qc.QC()); err != nil { + return fmt.Errorf("appQC proposal for block %d: %w", n, err) + } + } + return nil +} + +func (i *inner) applyAppQC(appQC *types.AppQC) { + gr := appQC.Proposal().GlobalRange() + if gr.Next <= i.nextAppQC { + return + } + for n := gr.First; n < gr.Next; n++ { + i.appQCs[n] = appQC + } + i.nextAppQC = gr.Next +} + +func (i *inner) insertAppQC(registry *epoch.Registry, appQC *types.AppQC) error { + if err := i.verifyAppQC(registry, appQC); err != nil { + return err + } + i.applyAppQC(appQC) + i.nextAppQCToPersist = i.nextAppQC + return nil +} + // insertBlock inserts a pre-verified block into the inner state. // Requires a QC to already be present for block n. Callers must verify // the block signature before calling (unlike insertQC, which verifies). @@ -177,10 +225,10 @@ func (i *inner) updateNextBlock(m *metrics.Metrics) { // Invariant: a CommitQC's embedded AppProposal (when present) always refers to // a global number from a *past* CommitQC — strictly below that tip QC's // GlobalRange.First (enforced in Proposal.Verify). Together with BlockDB's -// never-empty retention and eviction at min(nextAppProposal, App.GlobalNext), +// never-empty retention and eviction at min(nextAppProposal, nextAppQCToPersist), // in-memory -// maps therefore always retain at least the certified tip QC after a -// CommitQC.App appears. nextToExecute uses qc[nextAppProposal] (or the tip QC +// maps therefore always retain at least the certified tip QC after an AppQC is +// persisted. nextToExecute uses qc[nextAppProposal] (or the tip QC // when fully caught up), so it does not require retaining nextAppProposal-1. type State struct { cfg *Config @@ -284,6 +332,11 @@ func (s *State) loadFromBlockDB(blockDB types.BlockDB) error { return fmt.Errorf("load QC from BlockDB: %w", err) } } + if pos.AppQC != nil { + if err := in.insertAppQC(s.cfg.Registry, pos.AppQC); err != nil { + return fmt.Errorf("load AppQC from BlockDB: %w", err) + } + } if !pos.HasBlock { // The iteration tail: the covering QC is persisted but this block is // not (lost in a crash, or written ahead of its blocks). @@ -317,6 +370,9 @@ func (s *State) loadFromBlockDB(blockDB types.BlockDB) error { } // Data loaded from BlockDB was already durably persisted. in.nextBlockToPersist = in.nextBlock + if in.nextAppQCToPersist > in.nextBlockToPersist { + return fmt.Errorf("BlockDB AppQC tip %d exceeds durable block tip %d", in.nextAppQCToPersist, in.nextBlockToPersist) + } } return nil } @@ -394,7 +450,7 @@ func (s *State) PushQC(ctx context.Context, qc *types.FullCommitQC, blocks []*ty } ctrl.Updated() } - // Only a newly accepted QC can advance CommitQC.App / the eviction floor. + // Only newly accepted App data can advance the eviction floor. if needQC { evictBelowBound(inner) } @@ -602,6 +658,18 @@ func (s *State) qcFromDB(n types.GlobalBlockNumber) (*types.FullCommitQC, error) return qc, nil } +func (s *State) appQCFromDB(n types.GlobalBlockNumber) (*types.AppQC, error) { + opt, err := s.blockDB.ReadAppQCByBlockNumber(n) + if err != nil { + return nil, fmt.Errorf("blockDB.ReadAppQCByBlockNumber(%d): %w", n, err) + } + appQC, ok := opt.Get() + if !ok { + return nil, types.ErrPruned + } + return appQC, nil +} + func (s *State) globalBlockFromDB(n types.GlobalBlockNumber) (*types.GlobalBlock, error) { b, err := s.blockFromDB(n) if err != nil { @@ -681,48 +749,20 @@ func (s *State) AppVote(ctx context.Context, n types.GlobalBlockNumber) (*types. // PushAppQC pushes an AppQC to the state and advances the AppQC cursor. func (s *State) PushAppQC(ctx context.Context, appQC *types.AppQC) error { - proposal := appQC.Proposal() - ep, ok := s.cfg.Registry.EpochByIndex(proposal.EpochIndex()) - if !ok { - return fmt.Errorf("unknown epoch_index %d", proposal.EpochIndex()) - } - if err := appQC.Verify(ep.Committee()); err != nil { - return fmt.Errorf("appQC.Verify(): %w", err) - } - gr := proposal.GlobalRange() + gr := appQC.Proposal().GlobalRange() for inner, ctrl := range s.inner.Lock() { if err := ctrl.WaitUntil(ctx, func() bool { - return gr.First <= inner.nextQC + return gr.Next <= inner.nextBlock }); err != nil { return err } if gr.Next <= inner.nextAppQC { return nil } - if gr.First > inner.nextQC { - return fmt.Errorf("AppQC gap: expected first<=%d, got %d", inner.nextQC, gr.First) - } - if gr.Next > inner.nextQC { - return fmt.Errorf("AppQC range [%d,%d) exceeds nextQC %d", gr.First, gr.Next, inner.nextQC) - } - for n := gr.First; n < gr.Next; n++ { - qc := inner.qcs[n] - if qc == nil { - return fmt.Errorf("missing QC for AppQC block %d", n) - } - if err := proposal.Verify(qc.QC()); err != nil { - return fmt.Errorf("appQC proposal for block %d: %w", n, err) - } - } - for n := max(gr.First, inner.nextAppQC); n < gr.Next; n++ { - inner.appQCs[n] = appQC - } - for inner.nextAppQC < inner.nextQC { - if _, ok := inner.appQCs[inner.nextAppQC]; !ok { - break - } - inner.nextAppQC++ + if err := inner.verifyAppQC(s.cfg.Registry, appQC); err != nil { + return err } + inner.applyAppQC(appQC) evictBelowBound(inner) ctrl.Updated() return nil @@ -739,16 +779,35 @@ func (s *State) AppQC(ctx context.Context, n types.GlobalBlockNumber) (*types.Ap return inner.appQCs[n], inner.qcs[n], nil } } - // TODO: we should fallback to blocksDB - panic("unreachable") + qc, err := s.qcFromDB(n) + if err != nil { + return nil, nil, err + } + appQC, err := s.appQCFromDB(n) + if err != nil { + return nil, nil, err + } + return appQC, qc, nil } func (s *State) LastAppQC() (*types.AppQC, *types.FullCommitQC) { for i := range s.inner.Lock() { - // TODO: currently no guarantee that there is >=1 element. - // TODO: nextAppQC is NOT good enough, we need it to be persisted. + if i.nextAppQC == i.first { + return nil, nil + } n := i.nextAppQC - 1 - return i.appQCs[n], i.qcs[n] + if n >= i.first { + return i.appQCs[n], i.qcs[n] + } + appQC, err := s.appQCFromDB(n) + if err != nil { + return nil, nil + } + qc, err := s.qcFromDB(n) + if err != nil { + return nil, nil + } + return appQC, qc } panic("unreachable") } @@ -788,11 +847,12 @@ func (s *State) PruneBefore(retainFrom types.GlobalBlockNumber) error { return s.blockDB.PruneBefore(retainFrom) } -// runPersist is a background goroutine that persists blocks and QCs to BlockDB. -// It waits for in-memory blocks to advance past the block persistence cursor, -// then writes covering QCs (first, per the BlockDB contract) and blocks, then -// flushes once per batch. nextBlockToPersist advances with the block tip to -// unblock PushAppHash only when data is durable. +// runPersist is a background goroutine that persists blocks, QCs, and AppQCs to +// BlockDB. It waits for in-memory blocks to advance past the block persistence +// cursor, then writes covering QCs (first, per the BlockDB contract) and blocks, +// then flushes once per batch. nextBlockToPersist advances with the block tip +// to unblock PushAppHash only when data is durable. AppQCs are persisted later, +// once their matching CommitQC range is already durable. // Errors propagate vertically (kill the component). // // Cursors seed from BlockDB.Status() when non-zero so PushQC-before-Run heights @@ -806,10 +866,12 @@ func (s *State) PruneBefore(retainFrom types.GlobalBlockNumber) error { // is still at or past NextQC (enough coverage for each new block, not every // in-memory QC, and no rewrite of QCs already on disk). // -// In-memory eviction is driven by PushQC / PushAppHash (evictBelowBound), not here. +// In-memory block/QC eviction is driven by PushQC / PushAppHash / +// PushAppQC (evictBelowBound); AppQC entries are retained until their own +// persistence cursor catches up. func (s *State) runPersist(ctx context.Context) error { tips := s.blockDB.Status() - var nextToPersistQC, nextToPersistBlock types.GlobalBlockNumber + var nextToPersistQC, nextToPersistBlock, nextToPersistAppQC types.GlobalBlockNumber for inner := range s.inner.Lock() { // After loadFromBlockDB, nextBlockToPersist is the durable recovery tip. nextToPersistQC = tips.NextQC @@ -820,25 +882,34 @@ func (s *State) runPersist(ctx context.Context) error { if nextToPersistBlock == 0 { nextToPersistBlock = inner.nextBlockToPersist } + nextToPersistAppQC = tips.NextAppQC + if nextToPersistAppQC == 0 { + nextToPersistAppQC = inner.nextAppQCToPersist + } } for { type batch struct { qcs []*types.FullCommitQC blocks []blockEntry + appQCs []*types.AppQC nextBlock types.GlobalBlockNumber + nextAppQC types.GlobalBlockNumber } var b batch for inner, ctrl := range s.inner.Lock() { if err := ctrl.WaitUntil(ctx, func() bool { - return nextToPersistBlock < inner.nextBlock + return nextToPersistBlock < inner.nextBlock || + inner.appQCReadyToPersist(nextToPersistAppQC, nextToPersistBlock) }); err != nil { return err } - b.nextBlock = inner.nextBlock + if nextToPersistBlock < inner.nextBlock { + b.nextBlock = inner.nextBlock + } // Persist blocks in [nextToPersistBlock, nextBlock). Emit each covering // QC once at GlobalRange.First when it has not already been written // (First >= nextToPersistQC). - for n := nextToPersistBlock; n < inner.nextBlock; n++ { + for n := nextToPersistBlock; n < b.nextBlock; n++ { qc := inner.qcs[n] gr := qc.QC().GlobalRange() if n == gr.First && gr.First >= nextToPersistQC { @@ -846,6 +917,18 @@ func (s *State) runPersist(ctx context.Context) error { } b.blocks = append(b.blocks, blockEntry{n: n, block: inner.blocks[n]}) } + if b.nextBlock == 0 { + b.nextBlock = nextToPersistBlock + } + // AppQCs are written only through the durable block cursor from the + // previous flush. This preserves nextAppQCToPersist <= nextBlockToPersist + // at the write side instead of repairing it during recovery. + for inner.appQCReadyToPersist(nextToPersistAppQC, nextToPersistBlock) { + appQC := inner.appQCs[nextToPersistAppQC] + b.appQCs = append(b.appQCs, appQC) + nextToPersistAppQC = appQC.Proposal().GlobalRange().Next + } + b.nextAppQC = nextToPersistAppQC } // Write QCs first (BlockDB contract: QC must precede covered blocks). for _, qc := range b.qcs { @@ -862,8 +945,15 @@ func (s *State) runPersist(ctx context.Context) error { return fmt.Errorf("write block %d: %w", lb.n, err) } } + for _, appQC := range b.appQCs { + gr := appQC.Proposal().GlobalRange() + if err := s.blockDB.WriteAppQC(appQC); err != nil { + return fmt.Errorf("write AppQC [%d,%d): %w", gr.First, gr.Next, err) + } + } // Flush once per batch before advancing nextBlockToPersist, so that - // PushAppHash only unblocks after data is crash-durable. + // PushAppHash only unblocks after data is crash-durable. AppQCs share + // this async durability boundary. if err := s.blockDB.Flush(); err != nil { return fmt.Errorf("flush BlockDB: %w", err) } @@ -871,30 +961,54 @@ func (s *State) runPersist(ctx context.Context) error { for inner, ctrl := range s.inner.Lock() { if nextToPersistBlock > inner.nextBlockToPersist { inner.nextBlockToPersist = nextToPersistBlock - ctrl.Updated() } + if b.nextAppQC > inner.nextAppQCToPersist { + old := inner.nextAppQCToPersist + inner.nextAppQCToPersist = b.nextAppQC + evictBelowBound(inner) + inner.dropPersistedAppQCsBelowFirst(old, b.nextAppQC) + } + ctrl.Updated() } } } +func (i *inner) appQCReadyToPersist(n, nextPersistedBlock types.GlobalBlockNumber) bool { + if n >= i.nextAppQC { + return false + } + appQC := i.appQCs[n] + if appQC == nil { + return false + } + return appQC.Proposal().GlobalRange().Next <= nextPersistedBlock +} + +func (i *inner) dropPersistedAppQCsBelowFirst(first, next types.GlobalBlockNumber) { + for n := first; n < next && n < i.first; n++ { + delete(i.appQCs, n) + } +} + // evictBelowBound advances first toward the certified App floor and drops cached // blocks/QCs/AppProposals with n < first. No-op when there is no certified App // or the bound would not advance first. Caller must hold inner's lock. Invoked // from PushQC / PushAppHash. // -// Bound is min(nextAppProposal, App.GlobalNext()). A zero floor (no App / -// empty maps) yields bound 0 and is a no-op via bound <= first. With the -// past-CommitQC App invariant (see State), App.GlobalNext never exceeds the tip QC -// start, so at least one CommitQC remains. nextToExecute uses qc[nextAppProposal] -// (or the tip when caught up), so nextAppProposal-1 need not be retained. +// Bound is min(nextAppProposal, nextAppQCToPersist). A zero floor (no persisted +// AppQC / empty maps) yields bound 0 and is a no-op via bound <= first. AppQCs +// are verified against retained CommitQCs before advancing nextAppQC, and only +// persisted AppQCs advance the eviction floor, so at least one CommitQC remains. +// nextToExecute uses qc[nextAppProposal] (or the tip when caught up), so +// nextAppProposal-1 need not be retained. // -// TODO: At eviction we have both a local AppProposal and a CommitQC.App, so this +// TODO: At eviction we have both a local AppProposal and an AppQC, so this // is the right place to detect local-vs-quorum AppHash inconsistency. Surface // any mismatch from data.State.Run() (node-fatal), not from PushQC/PushAppHash — // e.g. stash an error on State for a Run monitor, or run eviction as its own // Run subtask. func evictBelowBound(inner *inner) { - bound := min(inner.nextAppProposal, inner.nextAppQC) + bound := min(inner.nextAppProposal, inner.nextAppQCToPersist) if bound <= inner.first { return } @@ -904,7 +1018,9 @@ func evictBelowBound(inner *inner) { delete(inner.blocks, n) } delete(inner.qcs, n) - delete(inner.appQCs, n) + if n < inner.nextAppQCToPersist { + delete(inner.appQCs, n) + } delete(inner.appProposals, n) } inner.first = bound diff --git a/sei-tendermint/internal/autobahn/data/state_recovery_test.go b/sei-tendermint/internal/autobahn/data/state_recovery_test.go index efa83c9318..df210639c6 100644 --- a/sei-tendermint/internal/autobahn/data/state_recovery_test.go +++ b/sei-tendermint/internal/autobahn/data/state_recovery_test.go @@ -121,8 +121,7 @@ func TestRecoveryStartsAtLastExecutedBlock(t *testing.T) { []*types.FullCommitQC{qc1, qc2}, [][]*types.Block{blocks1, blocks2}) - offset := gr2.Len() / 2 - lastExecuted := gr2.First + types.GlobalBlockNumber(offset) + lastExecuted := gr2.First state := newTestState(t, &Config{ Registry: registry, LastExecutedBlock: utils.Some(lastExecuted), @@ -135,10 +134,16 @@ func TestRecoveryStartsAtLastExecutedBlock(t *testing.T) { require.Equal(t, gr2.Next, state.NextBlock()) got, err := state.TryBlock(lastExecuted) require.NoError(t, err) - require.Equal(t, blocks2[offset].Header().Hash(), got.Header().Hash()) + require.Equal(t, blocks2[0].Header().Hash(), got.Header().Hash()) appHash := types.GenAppHash(rng) - require.NoError(t, state.PushAppHash(t.Context(), lastExecuted, appHash)) + for n := gr2.First; n < gr2.Next; n++ { + hash := types.GenAppHash(rng) + if n == gr2.Next-1 { + hash = appHash + } + require.NoError(t, state.PushAppHash(t.Context(), n, hash)) + } appVote, _, err := state.AppVote(t.Context(), lastExecuted) require.NoError(t, err) require.Equal(t, appHash, appVote.Proposal().AppHash()) @@ -172,7 +177,8 @@ func TestRecoveryCapsAppTipAtLastBlockInBlockDB(t *testing.T) { require.NoError(t, err) require.Equal(t, blocks2[0].Header().Hash(), got.Header.Hash()) - require.NoError(t, pushAppHashesRunning(t.Context(), state, rng, gr2.First, gr2.First+1)) + // A per-CommitQC AppProposal cannot be rebuilt from the capped mid-QC + // cursor; this test only pins the block/QC recovery cap. } func TestRecoveryRejectsAppTipBeyondCrashWindow(t *testing.T) { @@ -244,9 +250,9 @@ func TestRecoveryLeavesAppTipBelowPruneFloorUnreadable(t *testing.T) { require.ErrorIs(t, err, types.ErrPruned) } -// TestPruningDiscards verifies that PruneBefore advances BlockDB's watermark so -// TryBlock returns ErrPruned for the discarded range, while later blocks stay -// accessible. Memory is cleared by evictBelowBound (from PushQC/PushAppHash). +// TestPruningDiscards verifies that PruneBefore advances BlockDB's watermark but +// does not discard RAM-retained blocks. Memory is cleared only by the AppQC +// eviction floor. func TestPruningDiscards(t *testing.T) { ctx := t.Context() rng := utils.TestRng() @@ -271,8 +277,9 @@ func TestPruningDiscards(t *testing.T) { require.NoError(t, state.PruneBefore(gr2.First)) for n := gr1.First; n < gr2.First; n++ { - _, err := state.TryBlock(n) - require.ErrorIs(t, err, types.ErrPruned) + got, err := state.TryBlock(n) + require.NoError(t, err) + require.NotNil(t, got) } for n := gr2.First; n < gr3.Next; n++ { got, err := state.TryBlock(n) @@ -302,8 +309,8 @@ func TestRecoveryAfterPruning(t *testing.T) { require.NoError(t, db1.Close()) // Recovery skipTo(gr2.First); qc1 heights are absent from BlockDB → ErrPruned. - // With no CommitQC.App yet, first stays at the recovery floor (not advanced - // to 0), so below-floor reads fall through to BlockDB instead of nil maps. + // With no AppQC yet, first stays at the recovery floor (not advanced to 0), + // so below-floor reads fall through to BlockDB instead of nil maps. db2 := newTestBlockDB(t, dir) state2 := newTestState(t, &Config{Registry: registry}, db2) diff --git a/sei-tendermint/internal/autobahn/data/state_test.go b/sei-tendermint/internal/autobahn/data/state_test.go index 09a7a0fe6e..7b3ae26142 100644 --- a/sei-tendermint/internal/autobahn/data/state_test.go +++ b/sei-tendermint/internal/autobahn/data/state_test.go @@ -33,10 +33,7 @@ func newSnapshot() Snapshot { func snapshot(s *State) Snapshot { for inner := range s.inner.Lock() { - aps := map[types.GlobalBlockNumber]*types.AppProposal{} - for n, ap := range inner.appProposals { - aps[n] = ap - } + aps := maps.Clone(inner.appProposals) return Snapshot{ QCs: maps.Clone(inner.qcs), Blocks: maps.Clone(inner.blocks), @@ -354,17 +351,17 @@ func TestExecution(t *testing.T) { shortCtx, cancel := context.WithTimeout(ctx, 10*time.Millisecond) if err := state.PushAppHash(shortCtx, gr.Next, types.GenAppHash(rng)); err == nil { cancel() - return errors.New("PushAppProposal expected to fail on non-finalized blocks") + return errors.New("PushAppHash expected to fail on non-finalized blocks") } cancel() for n := gr.First; n < gr.Next; n += 1 { if err := state.PushAppHash(ctx, n, types.GenAppHash(rng)); err != nil { - return fmt.Errorf("state.PushAppProposal(): %w", err) - } - if err := state.PushAppHash(ctx, n, types.GenAppHash(rng)); err == nil { - return errors.New("PushAppProposal expected to fail on duplicate proposal") + return fmt.Errorf("state.PushAppHash(): %w", err) } } + if err := state.PushAppHash(ctx, gr.Next-1, types.GenAppHash(rng)); err == nil { + return errors.New("PushAppHash expected to fail on duplicate proposal") + } } return nil }); err != nil { @@ -482,8 +479,8 @@ func TestPushQCBeforeRunPersistsToBlockDB(t *testing.T) { } // TestEvictionWaitsForAppQC checks that evictBelowBound does not drop -// AppProposals until AppQC advances, and that once it does, heights below -// min(nextAppProposal, nextAppQC) are evicted. +// AppProposals until AppQC is persisted, and that once it is, heights below +// min(nextAppProposal, nextAppQCToPersist) are evicted. func TestEvictionWaitsForAppQC(t *testing.T) { ctx := t.Context() rng := utils.TestRng() @@ -514,11 +511,17 @@ func TestEvictionWaitsForAppQC(t *testing.T) { require.Equal(t, gr1.First, inner.first, "no certified App → first unchanged") for n := gr1.First; n < gr1.Next; n++ { _, ok := inner.appProposals[n] - require.True(t, ok, "AppProposal %d must survive without CommitQC.App", n) + require.True(t, ok, "AppProposal %d must survive without AppQC", n) } } require.NoError(t, pushAppQCForBlock(ctx, state, keys, gr1.First)) + require.Eventually(t, func() bool { + for inner := range state.inner.Lock() { + return inner.nextAppQCToPersist >= gr1.Next + } + panic("unreachable") + }, time.Second, time.Millisecond) require.NoError(t, state.PushQC(ctx, qc2, blocks2)) for n := gr2.First; n < gr2.Next; n++ { @@ -528,8 +531,8 @@ func TestEvictionWaitsForAppQC(t *testing.T) { } for inner := range state.inner.Lock() { - evictionBound := min(inner.nextAppProposal, inner.nextAppQC) - require.Equal(t, evictionBound, inner.first, "after catching up, first reaches min(nextAppProposal, nextAppQC)") + evictionBound := min(inner.nextAppProposal, inner.nextAppQCToPersist) + require.Equal(t, evictionBound, inner.first, "after catching up, first reaches min(nextAppProposal, nextAppQCToPersist)") for n := gr1.First; n < inner.first; n++ { _, ok := inner.appProposals[n] require.False(t, ok, "AppProposal %d should be evicted (< first)", n) @@ -548,10 +551,62 @@ func TestEvictionWaitsForAppQC(t *testing.T) { })) } +func TestEvictionWaitsForPersistedAppQC(t *testing.T) { + ctx := t.Context() + rng := utils.TestRng() + registry, keys := epoch.GenRegistry(rng, 3) + + qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) + gr1 := qc1.QC().GlobalRange() + + state := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) + require.NoError(t, state.PushQC(ctx, qc1, blocks1)) + require.NoError(t, pushAppHashesRunning(ctx, state, rng, gr1.First, gr1.Next)) + require.NoError(t, pushAppQCForBlock(ctx, state, keys, gr1.First)) + + for inner := range state.inner.Lock() { + require.Equal(t, gr1.Next, inner.nextAppQC) + require.Equal(t, gr1.First, inner.nextAppQCToPersist) + require.Equal(t, gr1.First, inner.first, "accepted but unpersisted AppQC must not advance eviction") + for n := gr1.First; n < gr1.Next; n++ { + _, ok := inner.appProposals[n] + require.True(t, ok, "AppProposal %d must survive until AppQC is persisted", n) + } + } +} + +func TestPushAppQCWaitsForBlocks(t *testing.T) { + ctx := t.Context() + rng := utils.TestRng() + registry, keys := epoch.GenRegistry(rng, 3) + + qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) + gr1 := qc1.QC().GlobalRange() + appProposal := types.NewAppProposal(qc1.QC().Proposal(), types.GenAppHash(rng)) + appQC := TestAppQC(keys, appProposal) + + state := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) + require.NoError(t, state.PushQC(ctx, qc1, nil)) + + shortCtx, cancel := context.WithTimeout(ctx, 10*time.Millisecond) + err := state.PushAppQC(shortCtx, appQC) + cancel() + require.ErrorIs(t, err, context.DeadlineExceeded) + + for n := gr1.First; n < gr1.Next; n++ { + require.NoError(t, state.PushBlock(ctx, n, blocks1[n-gr1.First])) + } + require.NoError(t, state.PushAppQC(ctx, appQC)) + for inner := range state.inner.Lock() { + require.Equal(t, gr1.Next, inner.nextAppQC) + require.LessOrEqual(t, inner.nextAppQC, inner.nextBlock) + } +} + // TestNextToExecuteAfterAppEviction checks WaitUntilExecuted / nextToExecute -// still work when AppQC aggressively evicts through nextAppProposal -// (first = min(nextAppProposal, nextAppQC) = NAP). nextToExecute uses qc[NAP], -// not NAP-1. +// still work when persisted AppQC aggressively evicts through nextAppProposal +// (first = min(nextAppProposal, nextAppQCToPersist) = NAP). nextToExecute uses +// qc[NAP], not NAP-1. func TestNextToExecuteAfterAppEviction(t *testing.T) { ctx := t.Context() rng := utils.TestRng() @@ -575,17 +630,23 @@ func TestNextToExecuteAfterAppEviction(t *testing.T) { return err } } - // Sticky case: nextAppQC == nextAppProposal. first advances to NAP; - // NAP-1 is gone; nextToExecute reads qc[NAP] after the next QC arrives. + // Sticky case: nextAppQCToPersist == nextAppProposal. first advances to + // NAP; NAP-1 is gone; nextToExecute reads qc[NAP] after the next QC arrives. require.NoError(t, pushAppQCForBlock(ctx, state, keys, gr1.First)) + require.Eventually(t, func() bool { + for inner := range state.inner.Lock() { + return inner.nextAppQCToPersist >= gr1.Next + } + panic("unreachable") + }, time.Second, time.Millisecond) require.NoError(t, state.PushQC(ctx, qc2, blocks2)) var tipLane types.LaneID var tipBlockNum types.BlockNumber for inner := range state.inner.Lock() { require.Equal(t, gr1.Next, inner.nextAppProposal) - require.Equal(t, min(inner.nextAppProposal, inner.nextAppQC), inner.first, - "eviction advances to min(nextAppProposal, nextAppQC) == NAP") + require.Equal(t, min(inner.nextAppProposal, inner.nextAppQCToPersist), inner.first, + "eviction advances to min(nextAppProposal, nextAppQCToPersist) == NAP") _, ok := inner.blocks[inner.nextAppProposal-1] require.False(t, ok, "NAP-1 must be evicted") require.Less(t, inner.nextAppProposal, inner.nextQC) @@ -610,6 +671,55 @@ func TestNextToExecuteAfterAppEviction(t *testing.T) { })) } +func TestPushAppQCPersistsAndRecovers(t *testing.T) { + ctx := t.Context() + rng := utils.TestRng() + registry, keys := epoch.GenRegistry(rng, 3) + dir := t.TempDir() + + qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) + gr1 := qc1.QC().GlobalRange() + + db1 := newTestBlockDB(t, dir) + state1 := newTestState(t, &Config{Registry: registry}, db1) + require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { + runCtx, cancel := context.WithCancel(ctx) + defer cancel() + s.SpawnBgNamed("state.Run", func() error { + return utils.IgnoreCancel(state1.Run(runCtx)) + }) + + require.NoError(t, state1.PushQC(ctx, qc1, blocks1)) + for n := gr1.First; n < gr1.Next; n++ { + if err := state1.PushAppHash(ctx, n, types.GenAppHash(rng)); err != nil { + return err + } + } + require.NoError(t, pushAppQCForBlock(ctx, state1, keys, gr1.First)) + require.Eventually(t, func() bool { + return db1.Status().NextAppQC >= gr1.Next + }, time.Second, time.Millisecond) + return nil + })) + + stored, err := db1.ReadAppQCByBlockNumber(gr1.First) + require.NoError(t, err) + require.True(t, stored.IsPresent(), "PushAppQC must persist the AppQC") + require.NoError(t, db1.Close()) + + db2 := newTestBlockDB(t, dir) + state2 := newTestState(t, &Config{Registry: registry}, db2) + for inner := range state2.inner.Lock() { + require.Equal(t, gr1.Next, inner.nextAppQC) + } + appQC, fQC := state2.LastAppQC() + require.NotNil(t, appQC) + require.NotNil(t, fQC) + require.Equal(t, gr1, appQC.Proposal().GlobalRange()) + require.Equal(t, gr1, fQC.QC().GlobalRange()) + require.NoError(t, db2.Close()) +} + // TestPruningKeepsLastQCRange verifies BlockDB's never-empty prune: asking to // prune past the tip still leaves the newest cohort readable. A QC retaining // only a suffix of its blocks recovers with the floor on that suffix; a @@ -671,8 +781,8 @@ func TestPruningKeepsLastQCRange(t *testing.T) { // readability), so a mid-range prune does not refuse heights inside that QC. // // PruneBefore is BlockDB-only: heights still retained in RAM for AppVotes -// (at/above min(nextAppProposal, nextAppQC) exclusive floor) remain readable via TryBlock even -// after the store watermark advances past them. +// (at/above min(nextAppProposal, nextAppQCToPersist) exclusive floor) remain +// readable via TryBlock even after the store watermark advances past them. func TestPruningWithPartialQCRange(t *testing.T) { ctx := t.Context() rng := utils.TestRng() @@ -689,9 +799,22 @@ func TestPruningWithPartialQCRange(t *testing.T) { require.NoError(t, pushAppHashesRunning(ctx, state1, rng, gr1.First, gr2.Next)) var exclusiveFloor types.GlobalBlockNumber - require.NoError(t, pushAppQCForBlock(ctx, state1, keys, gr1.First)) + require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { + runCtx, cancel := context.WithCancel(ctx) + defer cancel() + s.SpawnBgNamed("state.Run", func() error { + return utils.IgnoreCancel(state1.Run(runCtx)) + }) + if err := pushAppQCForBlock(ctx, state1, keys, gr1.First); err != nil { + return err + } + require.Eventually(t, func() bool { + return state1.blockDB.Status().NextAppQC >= gr1.Next + }, time.Second, time.Millisecond) + return nil + })) for inner := range state1.inner.Lock() { - exclusiveFloor = min(inner.nextAppProposal, inner.nextAppQC) + exclusiveFloor = min(inner.nextAppProposal, inner.nextAppQCToPersist) require.Equal(t, exclusiveFloor, inner.first) } @@ -706,12 +829,15 @@ func TestPruningWithPartialQCRange(t *testing.T) { } } - // Prune past qc1 entirely; BlockDB never-empty keeps the newest cohort (qc2). + // Prune past qc1 entirely. Because qc1 now has a persisted AppQC, BlockDB's + // never-empty rule keeps that newest AppQC+CommitQC+Block cohort readable. require.NoError(t, state1.PruneBefore(gr2.Next)) - // Evicted heights (< exclusive App floor) fall through to BlockDB → ErrPruned. + // Evicted heights (< exclusive App floor) fall through to BlockDB, but the + // persisted AppQC cohort is retained by the prune cap. for n := gr1.First; n < exclusiveFloor; n++ { - _, err := state1.TryBlock(n) - require.ErrorIs(t, err, types.ErrPruned) + got, err := state1.TryBlock(n) + require.NoError(t, err) + require.NotNil(t, got) } // Exclusive floor and above stay cached for AppVotes despite BlockDB prune. // ByHash must match TryBlock here — not fall through to a pruned BlockDB. From f5f12b775fc3b1d91bd9bae109494ec9e925008f Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 5 Aug 2026 18:45:26 +0200 Subject: [PATCH 09/61] robust tracking --- .../internal/autobahn/avail/app_votes.go | 18 +- .../internal/autobahn/avail/inner.go | 113 ++-- .../internal/autobahn/avail/state.go | 236 +++------ .../internal/autobahn/avail/subscriptions.go | 2 +- .../autobahn/consensus/persist/commitqcs.go | 10 +- .../internal/autobahn/data/state.go | 494 ++++++------------ 6 files changed, 297 insertions(+), 576 deletions(-) diff --git a/sei-tendermint/internal/autobahn/avail/app_votes.go b/sei-tendermint/internal/autobahn/avail/app_votes.go index f60727b07c..a3860af7af 100644 --- a/sei-tendermint/internal/autobahn/avail/app_votes.go +++ b/sei-tendermint/internal/autobahn/avail/app_votes.go @@ -16,17 +16,17 @@ type voteSet[V any] struct { } type road struct { - epoch *types.Epoch - commitQC *types.CommitQC + epoch *types.Epoch + commitQC *types.CommitQC appByKey map[types.PublicKey]struct{} appByHash map[types.Hash[*types.AppVote]]*voteSet[*types.Signed[*types.AppVote]] - appQC utils.Option[*types.AppQC] + appQC utils.Option[*types.AppQC] } func newRoad(commitQC *types.CommitQC, epoch *types.Epoch) *road { return &road{ - epoch: epoch, - commitQC: commitQC, + epoch: epoch, + commitQC: commitQC, appByKey: map[types.PublicKey]struct{}{}, appByHash: map[types.Hash[*types.AppVote]]*voteSet[*types.Signed[*types.AppVote]]{}, } @@ -34,9 +34,13 @@ func newRoad(commitQC *types.CommitQC, epoch *types.Epoch) *road { // Returns qc if a new qc has been reached. func (r *road) pushAppVote(vote *types.Signed[*types.AppVote]) { - if r.appQC.IsPresent() { return } + if r.appQC.IsPresent() { + return + } k := vote.Key() - if _, ok := r.appByKey[k]; ok { return } + if _, ok := r.appByKey[k]; ok { + return + } r.appByKey[k] = struct{}{} byHash, ok := r.appByHash[vote.Hash()] if !ok { diff --git a/sei-tendermint/internal/autobahn/avail/inner.go b/sei-tendermint/internal/autobahn/avail/inner.go index 7bd958cb91..f28b702729 100644 --- a/sei-tendermint/internal/autobahn/avail/inner.go +++ b/sei-tendermint/internal/autobahn/avail/inner.go @@ -4,8 +4,8 @@ import ( "fmt" "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/data" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/consensus/persist" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/data" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" ) @@ -15,14 +15,14 @@ import ( // BlockPersister creates lane WALs lazily inside MaybePruneAndPersistLane, but the new // member must also appear in inner.blocks before the next persist cycle. type inner struct { - latestCommitQC utils.AtomicSend[utils.Option[*types.CommitQC]] // latest persisted CommitQC - roads *queue[types.RoadIndex, *road] - nextAppQC types.RoadIndex + persistedCommitQC utils.AtomicSend[utils.Option[*types.CommitQC]] // latest persisted CommitQC + roads *queue[types.RoadIndex, *road] + nextAppQC types.RoadIndex // Epoch is the current epoch for blocks votes collection. - epoch *types.Epoch - blocks map[types.LaneID]*queue[types.BlockNumber, *types.Signed[*types.LaneProposal]] - votes map[types.LaneID]*queue[types.BlockNumber, blockVotes] + epoch *types.Epoch + blocks map[types.LaneID]*queue[types.BlockNumber, *types.Signed[*types.LaneProposal]] + votes map[types.LaneID]*queue[types.BlockNumber, blockVotes] // nextBlockToPersist tracks per-lane how far block persistence has progressed. // RecvBatch only yields blocks below this cursor for voting. // Always initialized (even when persistence is disabled — the no-op persist @@ -45,75 +45,66 @@ type inner struct { persistedBlockStart map[types.LaneID]types.BlockNumber } -// loadedAvailState holds data loaded from disk on restart. +// loadedState holds data loaded from disk on restart. // pruneAnchor is the decoded prune anchor (if any). // commitQCs and blocks are pre-filtered: stale entries below the // anchor have already been removed by loadPersistedState. // commitQCs are sorted by road index; blocks are sorted by number per lane. // newInner requires both to be contiguous and returns an error on gaps. -type loadedAvailState struct { - commitQCs []*types.CommitQC - blocks map[types.LaneID][]persist.LoadedBlock +type loadedState struct { + commitQCs []*types.CommitQC + blocks map[types.LaneID][]persist.LoadedBlock } -func newInner(ds *data.State, loaded utils.Option[*loadedAvailState]) (*inner, error) { +func newInner(ds *data.State, loaded *loadedState) (*inner, error) { epoch := ds.Registry().LatestEpoch() - votes := map[types.LaneID]*queue[types.BlockNumber, blockVotes]{} - blocks := map[types.LaneID]*queue[types.BlockNumber, *types.Signed[*types.LaneProposal]]{} - for lane := range epoch.Committee().Lanes().All() { - votes[lane] = newQueue[types.BlockNumber, blockVotes]() - blocks[lane] = newQueue[types.BlockNumber, *types.Signed[*types.LaneProposal]]() - } - i := &inner{ - latestCommitQC: utils.NewAtomicSend(utils.None[*types.CommitQC]()), + persistedCommitQC: utils.NewAtomicSend(utils.None[*types.CommitQC]()), roads: newQueue[types.RoadIndex, *road](), epoch: epoch, - blocks: blocks, - votes: votes, - nextBlockToPersist: make(map[types.LaneID]types.BlockNumber, len(votes)), - persistedBlockStart: make(map[types.LaneID]types.BlockNumber, len(votes)), + blocks: map[types.LaneID]*queue[types.BlockNumber, *types.Signed[*types.LaneProposal]]{}, + votes: map[types.LaneID]*queue[types.BlockNumber, blockVotes]{}, + nextBlockToPersist: map[types.LaneID]types.BlockNumber{}, + persistedBlockStart: map[types.LaneID]types.BlockNumber{}, } - l, ok := loaded.Get() - if !ok { - return i, nil + for lane := range epoch.Committee().Lanes().All() { + i.blocks[lane] = newQueue[types.BlockNumber, *types.Signed[*types.LaneProposal]]() + i.votes[lane] = newQueue[types.BlockNumber, blockVotes]() } - // Apply the persisted prune anchor first: prune() positions all queues - // (commitQCs, blocks, votes) so that subsequent pushBack calls insert - // at the correct indices without needing reset(). - appQC, fQC := ds.LastAppQC() - r := newRoad(fQC.QC(),epoch) - r.appQC = utils.Some(appQC) - i.roads.prune(fQC.Index()) - i.roads.pushBack(r) - i.nextAppQC = fQC.Index()+1 - - for lane := range i.blocks { - i.persistedBlockStart[lane] = fQC.QC().LaneRange(lane).Next() + // Apply the persisted prune anchor from the data.State: + // avail.State can drop everything below AppQC persisted in data.State. + if anchor, ok := ds.Anchor().Load().Get(); ok { + epoch, ok := ds.Registry().EpochByIndex(anchor.CommitQC.Proposal().EpochIndex()) + if !ok { + return nil, fmt.Errorf("epoch not found") + } + i.prune(epoch, anchor) } // Restore persisted CommitQCs. prune() may have already pushed the // anchor's CommitQC, so skip entries below commitQCs.next. - for _, qc := range l.commitQCs { + for _, qc := range loaded.commitQCs { if qc.Index() < i.roads.next { continue } if qc.Index() != i.roads.next { return nil, fmt.Errorf("non-contiguous persisted commitQCs: expected %d, got %d", i.roads.next, qc.Index()) } - epoch,ok := ds.Registry().EpochByIndex(qc.Proposal().EpochIndex()) - if !ok { return nil, fmt.Errorf("epoch not found") } + epoch, ok := ds.Registry().EpochByIndex(qc.Proposal().EpochIndex()) + if !ok { + return nil, fmt.Errorf("epoch not found") + } i.roads.pushBack(newRoad(qc, epoch)) } - if i.roads.Len()>0 { - i.latestCommitQC.Store(utils.Some(i.roads.q[i.roads.next-1].commitQC)) + if i.roads.Len() > 0 { + i.persistedCommitQC.Store(utils.Some(i.roads.q[i.roads.next-1].commitQC)) } // Restore persisted blocks. Since the anchor is persisted first and // blocks are written sequentially per lane, gaps, parent-hash // mismatches, and over-capacity indicate corruption or a bug. - for lane, bs := range l.blocks { + for lane, bs := range loaded.blocks { q, ok := i.blocks[lane] if !ok || len(bs) == 0 { continue @@ -123,22 +114,22 @@ func newInner(ds *data.State, loaded utils.Option[*loadedAvailState]) (*inner, e if q.Len() >= BlocksPerLane { return nil, fmt.Errorf("lane %s: loaded %d blocks exceeds capacity %d", lane, len(bs), BlocksPerLane) } - if b.Number != q.next { - return nil, fmt.Errorf("lane %s: non-contiguous persisted blocks: expected %d, got %d", lane, q.next, b.Number) - } if j > 0 { if got := b.Proposal.Msg().Block().Header().ParentHash(); got != lastHash { return nil, fmt.Errorf("lane %s: parent hash mismatch at block %d", lane, b.Number) } } lastHash = b.Proposal.Msg().Block().Header().Hash() + if b.Number < q.next { + continue + } + if b.Number != q.next { + return nil, fmt.Errorf("lane %s: non-contiguous persisted blocks: expected %d, got %d", lane, q.next, b.Number) + } q.pushBack(b.Proposal) } - if q.next > q.first { - i.nextBlockToPersist[lane] = q.next - } + i.nextBlockToPersist[lane] = q.next } - return i, nil } @@ -159,23 +150,25 @@ func (i *inner) updateNextAppQC() bool { i.nextAppQC += 1 updated = true } - return updated + return updated } // prune advances the state to account for a new AppQC/CommitQC pair. // Returns true if pruning occurred, false if the QC was stale. -func (i *inner) prune(epoch *types.Epoch, commitQC *types.CommitQC, appQC *types.AppQC) { - idx := commitQC.Index() - if idx < i.roads.first { return } +func (i *inner) prune(epoch *types.Epoch, anchor data.Anchor) { + idx := anchor.CommitQC.Index() + if idx < i.roads.first { + return + } i.roads.prune(idx) - i.nextAppQC = max(idx,i.nextAppQC) + i.nextAppQC = max(idx, i.nextAppQC) if idx == i.roads.next { - i.roads.pushBack(newRoad(commitQC,epoch)) + i.roads.pushBack(newRoad(anchor.CommitQC, epoch)) } - i.roads.q[idx].appQC = utils.Some(appQC) + i.roads.q[idx].appQC = utils.Some(anchor.AppQC) i.updateNextAppQC() for lane := range i.votes { - lr := commitQC.LaneRange(lane) + lr := anchor.CommitQC.LaneRange(lane) i.votes[lr.Lane()].prune(lr.First()) i.blocks[lr.Lane()].prune(lr.First()) if i.nextBlockToPersist[lr.Lane()] < lr.First() { diff --git a/sei-tendermint/internal/autobahn/avail/state.go b/sei-tendermint/internal/autobahn/avail/state.go index a9362936ea..204a94d61e 100644 --- a/sei-tendermint/internal/autobahn/avail/state.go +++ b/sei-tendermint/internal/autobahn/avail/state.go @@ -10,8 +10,6 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/avail/metrics" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/consensus/persist" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/data" - pb "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/pb" - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/protoutils" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/scope" ) @@ -37,7 +35,7 @@ type State struct { // persisters groups all disk persistence components. // Always initialized: real when stateDir is set, no-op otherwise. - persisters persisters + persisters *persisters } func (s *State) PublicKey() types.PublicKey { @@ -55,57 +53,22 @@ type persisters struct { // innerFile is the A/B file prefix for avail inner state persistence. const innerFile = "avail_inner" -// PruneAnchor is the decoded form of the persisted prune anchor -// (AppQC + matching CommitQC pair). It serves as the crash-recovery -// pruning watermark. -type PruneAnchor struct { - AppQC *types.AppQC - CommitQC *types.CommitQC -} - -// PruneAnchorConv converts between PruneAnchor and its protobuf representation. -var PruneAnchorConv = protoutils.Conv[*PruneAnchor, *pb.PersistedAvailPruneAnchor]{ - Encode: func(a *PruneAnchor) *pb.PersistedAvailPruneAnchor { - return &pb.PersistedAvailPruneAnchor{ - AppQc: types.AppQCConv.Encode(a.AppQC), - CommitQc: types.CommitQCConv.Encode(a.CommitQC), - } - }, - Decode: func(p *pb.PersistedAvailPruneAnchor) (*PruneAnchor, error) { - if p.AppQc == nil || p.CommitQc == nil { - return nil, fmt.Errorf("incomplete prune anchor: AppQC=%v CommitQC=%v", p.AppQc != nil, p.CommitQc != nil) - } - appQC, err := types.AppQCConv.Decode(p.AppQc) - if err != nil { - return nil, fmt.Errorf("decode AppQC: %w", err) - } - commitQC, err := types.CommitQCConv.Decode(p.CommitQc) - if err != nil { - return nil, fmt.Errorf("decode CommitQC: %w", err) - } - return &PruneAnchor{AppQC: appQC, CommitQC: commitQC}, nil - }, -} - // loadPersistedState creates persisters for the given directory option and loads // any existing state from disk. When dir is None, all persisters are no-op // and no state is loaded. When a prune anchor is present, stale commitQCs and // blocks below the anchor are filtered out before returning. -func loadPersistedState(dir utils.Option[string]) (utils.Option[*loadedAvailState], persisters, error) { +func loadPersistedState(dir utils.Option[string]) (*loadedState, *persisters, error) { bp, blocks, err := persist.NewBlockPersister(dir) if err != nil { - return utils.None[*loadedAvailState](), persisters{}, fmt.Errorf("NewBlockPersister: %w", err) + return nil, nil, fmt.Errorf("NewBlockPersister: %w", err) } cp, commitQCs, err := persist.NewCommitQCPersister(dir) if err != nil { - return utils.None[*loadedAvailState](), persisters{}, fmt.Errorf("NewCommitQCPersister: %w", err) - } - pers := persisters{blocks: bp, commitQCs: cp} - if _, ok := dir.Get(); !ok { - return utils.None[*loadedAvailState](), pers, nil + return nil, nil, fmt.Errorf("NewCommitQCPersister: %w", err) } - loaded := &loadedAvailState{commitQCs: commitQCs, blocks: blocks} - return utils.Some(loaded), pers, nil + pers := &persisters{blocks: bp, commitQCs: cp} + loaded := &loadedState{commitQCs: commitQCs, blocks: blocks} + return loaded, pers, nil } // NewState constructs a new availability state. @@ -143,7 +106,7 @@ func (s *State) Data() *data.State { // LastCommitQC returns receiver of the LastCommitQC. func (s *State) LastCommitQC() utils.AtomicRecv[utils.Option[*types.CommitQC]] { for inner := range s.inner.Lock() { - return inner.latestCommitQC.Subscribe() + return inner.persistedCommitQC.Subscribe() } panic("unreachable") } @@ -213,7 +176,7 @@ func (s *State) PushCommitQC(ctx context.Context, qc *types.CommitQC) error { } inner.roads.pushBack(newRoad(qc, epoch)) metrics.ObserveCommitQC(qc) - // The persist goroutine publishes latestCommitQC after writing to disk + // The persist goroutine publishes persistedCommitQC after writing to disk // (or immediately for no-op persisters), so consensus won't advance // until the CommitQC is durable. ctrl.Updated() @@ -248,42 +211,6 @@ func (s *State) PushAppVote(ctx context.Context, v *types.Signed[*types.AppVote] return nil } -// PushAppQC pushes an AppQC to the state. It requires a corresponding CommitQC -// as a justification. -func (s *State) prune(appQC *types.AppQC, commitQC *types.CommitQC) error { - // Check whether it is needed before verifying. - for inner := range s.inner.Lock() { - if commitQC.Index() <= inner.roads.first { - return nil - } - } - if err := appQC.Proposal().Verify(commitQC); err != nil { - return fmt.Errorf("appQC proposal: %w", err) - } - epoch, ok := s.data.Registry().EpochByIndex(commitQC.Proposal().EpochIndex()) - if !ok { - return fmt.Errorf("unknown epoch_index %d", commitQC.Proposal().EpochIndex()) - } - if err := appQC.Verify(epoch.Committee()); err != nil { - return fmt.Errorf("appQC.Verify(): %w", err) - } - if err := commitQC.Verify(epoch); err != nil { - return fmt.Errorf("commitQC.Verify(): %w", err) - } - for inner, ctrl := range s.inner.Lock() { - inner.prune(epoch, commitQC, appQC) - ctrl.Updated() - } - return nil -} - -func (s *State) nextAppQC() types.RoadIndex { - for inner := range s.inner.Lock() { - return inner.nextAppQC - } - panic("unreachable") -} - // NextBlock returns the index of the next missing block in local storage for the given lane. func (s *State) NextBlock(lane types.LaneID) types.BlockNumber { for inner := range s.inner.Lock() { @@ -439,26 +366,22 @@ func (s *State) headers(ctx context.Context, lr *types.LaneRange) ([]*types.Bloc return headers, nil } -func (s *State) fullCommitQC(ctx context.Context, n types.RoadIndex) (*types.FullCommitQC, error) { +func (s *State) fullCommitQC(ctx context.Context, n types.RoadIndex) (*types.Epoch, *types.FullCommitQC, error) { // Collect the CommitQC. - qc, err := s.CommitQC(ctx, n) + epoch, qc, err := s.commitQC(ctx, n) if err != nil { - return nil, err + return nil, nil, err } // Collect the headers from the votes. var commitHeaders []*types.BlockHeader - ep, ok := s.data.Registry().EpochByIndex(qc.Proposal().EpochIndex()) - if !ok { - return nil, fmt.Errorf("unknown epoch_index %d", qc.Proposal().EpochIndex()) - } - for lane := range ep.Committee().Lanes().All() { + for lane := range epoch.Committee().Lanes().All() { headers, err := s.headers(ctx, qc.LaneRange(lane)) if err != nil { - return nil, err + return nil, nil, err } commitHeaders = append(commitHeaders, headers...) } - return types.NewFullCommitQC(qc, commitHeaders), nil + return epoch, types.NewFullCommitQC(qc, commitHeaders), nil } // WaitForLocalCapacity waits until the lane owned by this node has capacity for toProduce block. @@ -535,72 +458,72 @@ func (s *State) produceLocalBlock(n types.BlockNumber, key types.SecretKey, payl return result, nil } -// Run runs the background tasks of the state. -// -// Goroutines: this method spawns long-lived goroutines via scope.SpawnNamed -// (the persist loop and the FullCommitQC→data-state pusher). Inside -// runPersist, scope.Parallel spawns short-lived goroutines for concurrent -// per-lane block and commit-QC persistence. The persist package itself does -// not spawn goroutines. -func (s *State) Run(ctx context.Context) error { - return scope.Run(ctx, func(ctx context.Context, scope scope.Scope) error { - scope.SpawnNamed("persist", func() error { - return s.runPersist(ctx, s.persisters) - }) - // Task inserting FullCommitQCs and local blocks to data state. - scope.SpawnNamed("s.data.PushQC", func() error { - for n := types.RoadIndex(0); ; n = max(n+1, s.FirstCommitQC()) { - qc, err := s.fullCommitQC(ctx, n) - if err != nil { - if errors.Is(err, types.ErrPruned) { - continue +// Task inserting CommitQCs and local blocks to data state. +func (s *State) runPushQC(ctx context.Context) error { + for n := types.RoadIndex(0); ; n = max(n+1, s.FirstCommitQC()) { + epoch, qc, err := s.fullCommitQC(ctx, n) + if err != nil { + if errors.Is(err, types.ErrPruned) { + continue + } + return err + } + + // Collect the blocks we have locally. + c := epoch.Committee() + var blocks []*types.Block + for inner := range s.inner.Lock() { + for lane := range c.Lanes().All() { + lr := qc.QC().LaneRange(lane) + for n := lr.First(); n < lr.Next(); n++ { + // We are not expected to have all the blocks locally - only the available ones. + if b, ok := inner.blocks[lr.Lane()].q[n]; ok { + // We don't need to check the blocks against the headers, + // as bad blocks will be filtered out by PushQC anyway. + blocks = append(blocks, b.Msg().Block()) } - return err } + } + } + if err := s.data.PushQC(ctx, qc, blocks); err != nil { + return fmt.Errorf("s.data.PushQC(): %w", err) + } + } +} - // Collect the blocks we have locally. - ep, ok := s.data.Registry().EpochByIndex(qc.QC().Proposal().EpochIndex()) +func (s *State) runEvict(ctx context.Context) error { + return s.data.Anchor().Iter(ctx, func(ctx context.Context, anchor utils.Option[data.Anchor]) error { + if anchor, ok := anchor.Get(); ok { + for inner, ctrl := range s.inner.Lock() { + epoch, ok := s.data.Registry().EpochByIndex(anchor.CommitQC.Proposal().EpochIndex()) if !ok { - return fmt.Errorf("unknown epoch_index %d", qc.QC().Proposal().EpochIndex()) - } - c := ep.Committee() - var blocks []*types.Block - for inner := range s.inner.Lock() { - for lane := range c.Lanes().All() { - lr := qc.QC().LaneRange(lane) - for n := lr.First(); n < lr.Next(); n++ { - // We are not expected to have all the blocks locally - only the available ones. - if b, ok := inner.blocks[lr.Lane()].q[n]; ok { - // We don't need to check the blocks against the headers, - // as bad blocks will be filtered out by PushQC anyway. - blocks = append(blocks, b.Msg().Block()) - } - } - } - } - if err := s.data.PushQC(ctx, qc, blocks); err != nil { - return fmt.Errorf("s.data.PushQC(): %w", err) + return fmt.Errorf("epoch not found") } + inner.prune(epoch, anchor) + ctrl.Updated() } - }) + } + return nil + }) +} + +// Run runs the background tasks of the state. +func (s *State) Run(ctx context.Context) error { + return scope.Run(ctx, func(ctx context.Context, scope scope.Scope) error { + scope.SpawnNamed("runEvict", func() error { return s.runEvict(ctx) }) + scope.SpawnNamed("runPersist", func() error { return s.runPersist(ctx) }) + scope.SpawnNamed("runPushQC", func() error { return s.runPushQC(ctx) }) return nil }) } // runPersist is the main loop for the persist goroutine. -// Write order: -// 1. Prune anchor (AppQC + CommitQC pair) — the crash-recovery watermark (sequential). // 2. commitQCs.MaybePruneAndPersist and each lane's blocks.MaybePruneAndPersistLane run // concurrently via scope.Parallel (separate WALs, no early cancellation; first error // is returned after all tasks finish). // Each path publishes (markCommitQCsPersisted / markBlockPersisted) per entry so voting // unblocks ASAP. -// -// The prune anchor is a pruning watermark: on restart we resume from it. -// -// TODO: use a single WAL for anchor and CommitQCs to make -// this atomic rather than relying on write order. -func (s *State) runPersist(ctx context.Context, pers persisters) error { +func (s *State) runPersist(ctx context.Context) error { for { batch, err := s.collectPersistBatch(ctx) if err != nil { @@ -616,7 +539,7 @@ func (s *State) runPersist(ctx context.Context, pers persisters) error { // Callees handle empty inputs gracefully (no-op when nothing to write/truncate). if err := scope.Parallel(func(ps scope.ParallelScope) error { ps.Spawn(func() error { - if err := pers.commitQCs.PruneAndPersist(batch.commitQCs.first, batch.commitQCs.tail); err != nil { + if err := s.persisters.commitQCs.Persist(batch.commitQCs.first, batch.commitQCs.tail); err != nil { return err } if t := batch.commitQCs.tail; len(t) > 0 { @@ -626,7 +549,7 @@ func (s *State) runPersist(ctx context.Context, pers persisters) error { }) for lane, batch := range batch.blocks { ps.Spawn(func() error { - return pers.blocks.Persist(lane, batch.first, batch.tail, utils.Some(markBlock)) + return s.persisters.blocks.Persist(lane, batch.first, batch.tail, utils.Some(markBlock)) }) } return nil @@ -651,21 +574,6 @@ type persistBatch struct { commitQCs commitQCsBatch } -// advancePersistedBlockStart updates the per-lane block admission watermark -// after durably writing the prune anchor. This unblocks PushBlock/ProduceBlock -// waiters that are gated on persistedBlockStart + BlocksPerLane. -func (s *State) advancePersistedBlockStart(commitQC *types.CommitQC) { - for inner, ctrl := range s.inner.Lock() { - for lane := range inner.blocks { - start := commitQC.LaneRange(lane).First() - if start > inner.persistedBlockStart[lane] { - inner.persistedBlockStart[lane] = start - } - } - ctrl.Updated() - } -} - // markBlockPersisted advances the per-lane block persistence cursor. // Called after each block is persisted so that RecvBatch (and therefore // voting) can unblock as soon as the block is durable. Safe for concurrent @@ -681,20 +589,20 @@ func (s *State) markBlockPersisted(lane types.LaneID, next types.BlockNumber) { // gating consensus from advancing until the QC is durable. func (s *State) markCommitQCsPersisted(qc *types.CommitQC) { for inner := range s.inner.Lock() { - inner.latestCommitQC.Store(utils.Some(qc)) + inner.persistedCommitQC.Store(utils.Some(qc)) } } // collectPersistBatch waits for new blocks or commitQCs and collects them under lock. func (s *State) collectPersistBatch(ctx context.Context) (*persistBatch, error) { for inner, ctrl := range s.inner.Lock() { - // Derive the CommitQC persist cursor from latestCommitQC. This is - // safe because latestCommitQC is only advanced by markCommitQCsPersisted + // Derive the CommitQC persist cursor from persistedCommitQC. This is + // safe because persistedCommitQC is only advanced by markCommitQCsPersisted // (after disk write) and on startup (from disk). prune() does NOT - // update latestCommitQC, so this always reflects persistence state. + // update persistedCommitQC, so this always reflects persistence state. // The max clamp with commitQCs.first handles the case where prune() // fast-forwarded the queue past the cursor. - next := types.NextIndexOpt(inner.latestCommitQC.Load()) + next := types.NextIndexOpt(inner.persistedCommitQC.Load()) if err := ctrl.WaitUntil(ctx, func() bool { for lane, q := range inner.blocks { if inner.nextBlockToPersist[lane] < q.next { diff --git a/sei-tendermint/internal/autobahn/avail/subscriptions.go b/sei-tendermint/internal/autobahn/avail/subscriptions.go index be405b3252..a517e5d282 100644 --- a/sei-tendermint/internal/autobahn/avail/subscriptions.go +++ b/sei-tendermint/internal/autobahn/avail/subscriptions.go @@ -85,7 +85,7 @@ func (r *AppVotesRecv) Recv(ctx context.Context) (*types.Signed[*types.AppVote], if err != nil { return nil, err } - r.next = qc.QC().GlobalRange().Next + r.next = qc.QC().GlobalRange().Next return types.Sign(r.state.key, vote), nil } } diff --git a/sei-tendermint/internal/autobahn/consensus/persist/commitqcs.go b/sei-tendermint/internal/autobahn/consensus/persist/commitqcs.go index 5c8d6e082d..75423931fa 100644 --- a/sei-tendermint/internal/autobahn/consensus/persist/commitqcs.go +++ b/sei-tendermint/internal/autobahn/consensus/persist/commitqcs.go @@ -13,7 +13,7 @@ const commitqcsDir = "commitqcs" // commitQCState is the mutable state protected by CommitQCPersister's mutex. type commitQCState struct { - iw utils.Option[*indexedWAL[*types.CommitQC]] + iw utils.Option[*indexedWAL[*types.CommitQC]] persisted types.RoadRange } @@ -42,7 +42,7 @@ func (s *commitQCState) persist(qc *types.CommitQC) error { func (s *commitQCState) deleteBefore(idx types.RoadIndex) error { iw, ok := s.iw.Get() if idx >= s.persisted.Next { - s.persisted = types.RoadRange{First:idx,Next:idx} + s.persisted = types.RoadRange{First: idx, Next: idx} if ok && iw.Count() > 0 { if err := iw.TruncateAll(); err != nil { return err @@ -100,9 +100,9 @@ func NewCommitQCPersister(stateDir utils.Option[string]) (*CommitQCPersister, [] return nil, nil, err } if len(loaded) > 0 { - s.persisted = types.RoadRange { + s.persisted = types.RoadRange{ First: loaded[0].Index(), - Next: loaded[len(loaded)-1].Index() + 1, + Next: loaded[len(loaded)-1].Index() + 1, } } return &CommitQCPersister{state: utils.NewMutex(s)}, loaded, nil @@ -131,7 +131,7 @@ func (cp *CommitQCPersister) Next() types.RoadIndex { // need not coordinate ordering. // afterEach, when present, is called after each successful append. It is // invoked while the lock is held, so it must not re-enter the persister. -func (cp *CommitQCPersister) PruneAndPersist(deleteBefore types.RoadIndex, commitQCs []*types.CommitQC) error { +func (cp *CommitQCPersister) Persist(deleteBefore types.RoadIndex, commitQCs []*types.CommitQC) error { for s := range cp.state.Lock() { if err := s.deleteBefore(deleteBefore); err != nil { return err diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index dc94433375..1ebeb51f6b 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -50,18 +50,13 @@ type inner struct { appQCs map[types.GlobalBlockNumber]*types.AppQC // [first, nextAppQC) blockHashes map[types.BlockHeaderHash]types.GlobalBlockNumber // blockHashes: mirrors blocks (insertBlock / evictBelowBound) - // first is the exclusive low end of retained in-memory state: maps keep - // [first, next*). Set by newInner / skipTo; advanced by evictBelowBound to - // min(nextAppProposal, nextAppQCToPersist). - // nextToExecute reads the next (or tip) QC from maps — it does not need - // nextAppProposal-1 retained after eviction. + // first is the exclusive low end of retained in-memory state: maps keep [first, next*). + // Advanced by evictBelowBound t (nextAppProposal, nextAppQCToPersist)-1. // // first <= nextAppProposal <= nextBlockToPersist <= nextBlock <= nextQC - // first <= nextAppQCToPersist <= nextBlockToPersist - // first <= nextAppQC <= nextBlock + // first <= nextAppQCToPersist <= nextAppQC <= nextQC // // AppProposals require persistence (nextAppProposal <= nextBlockToPersist). - // BlockDB prune status lives only in the store watermark (see PruneBefore). first types.GlobalBlockNumber nextAppProposal types.GlobalBlockNumber nextAppQC types.GlobalBlockNumber @@ -69,49 +64,31 @@ type inner struct { nextBlockToPersist types.GlobalBlockNumber nextBlock types.GlobalBlockNumber nextQC types.GlobalBlockNumber + + anchor utils.AtomicSend[utils.Option[Anchor]] } -func newInner(firstBlock types.GlobalBlockNumber) *inner { +func newInner(first types.GlobalBlockNumber) *inner { return &inner{ qcs: map[types.GlobalBlockNumber]*types.FullCommitQC{}, blocks: map[types.GlobalBlockNumber]*types.Block{}, appQCs: map[types.GlobalBlockNumber]*types.AppQC{}, appProposals: map[types.GlobalBlockNumber]*types.AppProposal{}, blockHashes: map[types.BlockHeaderHash]types.GlobalBlockNumber{}, - first: firstBlock, - nextAppProposal: firstBlock, - nextAppQC: firstBlock, - nextAppQCToPersist: firstBlock, - nextBlockToPersist: firstBlock, - nextBlock: firstBlock, - nextQC: firstBlock, + first: first, + nextAppProposal: first, + nextAppQC: first, + nextAppQCToPersist: first, + nextBlockToPersist: first, + nextBlock: first, + nextQC: first, } } -// skipTo advances all cursors to n, discarding everything before it. -// Used on recovery when the first loaded QC starts past committee.FirstBlock() -// (i.e. data before n was pruned in a previous run). -func (i *inner) skipTo(n types.GlobalBlockNumber) { - i.first = n - i.nextAppProposal = n - i.nextAppQC = n - i.nextAppQCToPersist = n - i.nextBlockToPersist = n - i.nextBlock = n - i.nextQC = n -} - // insertQC verifies and inserts a FullCommitQC into the inner state. // Accepts QCs whose range starts at or before nextQC (partially pruned // prefix is silently skipped). Rejects gaps where gr.First > nextQC. func (i *inner) insertQC(registry *epoch.Registry, qc *types.FullCommitQC) error { - e, ok := registry.EpochByIndex(qc.QC().Proposal().EpochIndex()) - if !ok { - return fmt.Errorf("unknown epoch_index %d", qc.QC().Proposal().EpochIndex()) - } - if err := qc.Verify(e); err != nil { - return fmt.Errorf("qc.Verify(): %w", err) - } gr := qc.QC().GlobalRange() if gr.Next <= i.nextQC { return nil // fully behind, skip @@ -119,6 +96,13 @@ func (i *inner) insertQC(registry *epoch.Registry, qc *types.FullCommitQC) error if gr.First > i.nextQC { return fmt.Errorf("QC gap: expected first<=%d, got %d", i.nextQC, gr.First) } + e, ok := registry.EpochByIndex(qc.QC().Proposal().EpochIndex()) + if !ok { + return fmt.Errorf("unknown epoch_index %d", qc.QC().Proposal().EpochIndex()) + } + if err := qc.Verify(e); err != nil { + return fmt.Errorf("qc.Verify(): %w", err) + } for i.nextQC < gr.Next { i.qcs[i.nextQC] = qc i.nextQC++ @@ -126,51 +110,27 @@ func (i *inner) insertQC(registry *epoch.Registry, qc *types.FullCommitQC) error return nil } -func (i *inner) verifyAppQC(registry *epoch.Registry, appQC *types.AppQC) error { - proposal := appQC.Proposal() - ep, ok := registry.EpochByIndex(proposal.EpochIndex()) - if !ok { - return fmt.Errorf("unknown epoch_index %d", proposal.EpochIndex()) - } - if err := appQC.Verify(ep.Committee()); err != nil { - return fmt.Errorf("appQC.Verify(): %w", err) - } - gr := proposal.GlobalRange() - if gr.Next <= i.nextAppQC { - return nil +func (i *inner) insertAppQC(registry *epoch.Registry, appQC *types.AppQC) error { + gr := appQC.Proposal().GlobalRange() + if gr.Next > i.nextQC { + return fmt.Errorf("Missing CommitQC for this AppQC") } if gr.First != i.nextAppQC { return fmt.Errorf("AppQC gap: expected first=%d, got %d", i.nextAppQC, gr.First) } - for n := gr.First; n < gr.Next; n++ { - qc := i.qcs[n] - if qc == nil { - return fmt.Errorf("missing QC for AppQC block %d", n) - } - if err := proposal.Verify(qc.QC()); err != nil { - return fmt.Errorf("appQC proposal for block %d: %w", n, err) - } + ei := appQC.Proposal().EpochIndex() + epoch, ok := registry.EpochByIndex(ei) + if !ok { + return fmt.Errorf("unknown epoch_index %d", ei) } - return nil -} - -func (i *inner) applyAppQC(appQC *types.AppQC) { - gr := appQC.Proposal().GlobalRange() - if gr.Next <= i.nextAppQC { - return + if err := appQC.Verify(epoch.Committee()); err != nil { + return fmt.Errorf("appQC.Verify(): %w", err) } - for n := gr.First; n < gr.Next; n++ { - i.appQCs[n] = appQC + for i.nextAppQC < gr.Next { + i.appQCs[i.nextAppQC] = appQC + i.nextAppQC++ } i.nextAppQC = gr.Next -} - -func (i *inner) insertAppQC(registry *epoch.Registry, appQC *types.AppQC) error { - if err := i.verifyAppQC(registry, appQC); err != nil { - return err - } - i.applyAppQC(appQC) - i.nextAppQCToPersist = i.nextAppQC return nil } @@ -220,16 +180,7 @@ func (i *inner) updateNextBlock(m *metrics.Metrics) { } // State of the chain. -// Contains blocks in global order and proofs of their finality. -// -// Invariant: a CommitQC's embedded AppProposal (when present) always refers to -// a global number from a *past* CommitQC — strictly below that tip QC's -// GlobalRange.First (enforced in Proposal.Verify). Together with BlockDB's -// never-empty retention and eviction at min(nextAppProposal, nextAppQCToPersist), -// in-memory -// maps therefore always retain at least the certified tip QC after an AppQC is -// persisted. nextToExecute uses qc[nextAppProposal] (or the tip QC -// when fully caught up), so it does not require retaining nextAppProposal-1. +// Contains blocks in global order and proofs of sequencing: (CommitQC) and execution result (AppQC). type State struct { cfg *Config metrics *metrics.Metrics @@ -244,21 +195,20 @@ type State struct { // Recovery starts at cfg.LastExecutedBlock and handles a non-zero CommitQC tip // via loadFromBlockDB (skipTo). func NewState(cfg *Config, blockDB types.BlockDB) (*State, error) { - s := &State{ + inner, err := loadFromBlockDB(cfg, blockDB) + if err != nil { + return nil, fmt.Errorf("loadFromBlockDB: %w", err) + } + return &State{ cfg: cfg, metrics: metrics.Get(), - inner: utils.NewWatch(newInner(cfg.Registry.FirstBlock())), + inner: utils.NewWatch(inner), blockDB: blockDB, - } - if err := s.loadFromBlockDB(blockDB); err != nil { - return nil, fmt.Errorf("loadFromBlockDB: %w", err) - } - return s, nil + }, nil } // loadFromBlockDB replays QCs and blocks from blockDB into s.inner. -// Called from NewState before any goroutines are spawned; the lock is acquired -// only to satisfy the Watch API. +// Called from NewState before any goroutines are spawned. // // Recovery starts at the app tip so runExecute can replay its AppHash. If the // app tip equals BlockDB's next block, recovery starts at the last stored block: @@ -274,107 +224,66 @@ func NewState(cfg *Config, blockDB types.BlockDB) (*State, error) { // Each iterator position has its covering QC and an optional block. Missing // blocks are allowed only at the tail. BlockDB enforces other consistency; this // method only rejects a first QC before committee genesis. -func (s *State) loadFromBlockDB(blockDB types.BlockDB) error { - for in := range s.inner.Lock() { - err := func() error { - firstBlock := s.cfg.Registry.FirstBlock() - dbNextBlock := blockDB.Status().NextBlock - recoveryStart := firstBlock - if lastExecutedBlock, executed := s.cfg.LastExecutedBlock.Get(); executed { - if lastExecutedBlock > max(dbNextBlock, firstBlock) { - return fmt.Errorf( - "BlockDB next block %d is behind app tip %d by more than the recoverable crash window; restore matching BlockDB data or state-sync the node: %w", - dbNextBlock, lastExecutedBlock, types.ErrNotFound, - ) - } - recoveryStart = lastExecutedBlock - if dbNextBlock != 0 && lastExecutedBlock == dbNextBlock { - // app.Commit completed before the app tip became durable. - recoveryStart = dbNextBlock - 1 - } +func loadFromBlockDB(cfg *Config, blockDB types.BlockDB) (*inner, error) { + firstBlock := cfg.Registry.FirstBlock() + status := blockDB.Status() + status.NextQC = max(status.NextQC, firstBlock) + status.NextBlock = max(status.NextBlock, firstBlock) + status.NextAppQC = max(status.NextAppQC, firstBlock) + first := status.NextAppQC + inner := newInner(first) + it, err := blockDB.Iterator(first) + if err != nil { + return nil, fmt.Errorf("open block db iterator: %w", err) + } + defer func() { _ = it.Close() }() + for { + pos, ok, err := it.Next() + if err != nil { + return nil, fmt.Errorf("advance block db iterator: %w", err) + } + if !ok { + break + } + if err := inner.insertQC(cfg.Registry, pos.QC); err != nil { + return nil, fmt.Errorf("load QC from BlockDB: %w", err) + } + b, err := it.Block() + if err != nil { + return nil, fmt.Errorf("read block %d from BlockDB: %w", pos.Number, err) + } + if b, ok := b.Get(); ok { + ei := pos.QC.QC().Proposal().EpochIndex() + e, ok := cfg.Registry.EpochByIndex(ei) + if !ok { + return nil, fmt.Errorf("unknown epoch_index %d", ei) } - it, err := blockDB.Iterator(recoveryStart) - if err != nil { - return fmt.Errorf("open block db iterator: %w", err) + if err := b.Verify(e.Committee()); err != nil { + return nil, fmt.Errorf("verify block %d from BlockDB: %w", pos.Number, err) } - defer func() { _ = it.Close() }() - var lastQC *types.FullCommitQC - for { - pos, ok, err := it.Next() - if err != nil { - return fmt.Errorf("advance block db iterator: %w", err) - } - if !ok { - break - } - n, qc := pos.Number, pos.QC - gr := qc.QC().GlobalRange() - if lastQC == nil { - // First position: skipTo it to advance past any pruned prefix. Nothing has - // been inserted yet, which is what makes it legal for skipTo to move first - // without deleting. Subsequent QCs must be consecutive — insertQC errors on - // any gap. - if gr.First < in.nextQC { - return fmt.Errorf("QC in BlockDB predates committee genesis %d: got %d", - in.nextQC, gr.First) - } - if n > in.nextQC { - in.skipTo(n) - } - } - if qc != lastQC { - // The scan entered a new QC's range. Position.QC hands back the same pointer - // for every number in a range, so identity is the exact test — and unlike - // gr.First == n it still fires for the covering QC when the scan opened - // inside that QC's range. insertQC clips it to [nextQC, gr.Next). - lastQC = qc - if err := in.insertQC(s.cfg.Registry, qc); err != nil { - return fmt.Errorf("load QC from BlockDB: %w", err) - } - } - if pos.AppQC != nil { - if err := in.insertAppQC(s.cfg.Registry, pos.AppQC); err != nil { - return fmt.Errorf("load AppQC from BlockDB: %w", err) - } - } - if !pos.HasBlock { - // The iteration tail: the covering QC is persisted but this block is - // not (lost in a crash, or written ahead of its blocks). - continue - } - blkOpt, err := it.Block() - if err != nil { - return fmt.Errorf("read block %d from BlockDB: %w", n, err) - } - blk := blkOpt.OrPanic(fmt.Sprintf("block %d absent at a HasBlock position", n)) - e, ok := s.cfg.Registry.EpochByIndex(qc.QC().Proposal().EpochIndex()) - if !ok { - return fmt.Errorf("unknown epoch_index %d", qc.QC().Proposal().EpochIndex()) - } - if err := blk.Verify(e.Committee()); err != nil { - return fmt.Errorf("verify block %d from BlockDB: %w", n, err) - } - if err := in.insertBlock(n, blk); err != nil { - return fmt.Errorf("insert block %d from BlockDB: %w", n, err) - } + if err := inner.insertBlock(pos.Number, b); err != nil { + return nil, fmt.Errorf("insert block %d from BlockDB: %w", pos.Number, err) } - return nil - }() - if err != nil { - return err - } - - // Advance nextBlock through contiguous loaded blocks. Don't use - // updateNextBlock: stale timestamps would skew metrics. - for ; in.blocks[in.nextBlock] != nil; in.nextBlock++ { } - // Data loaded from BlockDB was already durably persisted. - in.nextBlockToPersist = in.nextBlock - if in.nextAppQCToPersist > in.nextBlockToPersist { - return fmt.Errorf("BlockDB AppQC tip %d exceeds durable block tip %d", in.nextAppQCToPersist, in.nextBlockToPersist) + if pos.HasAppQC { + if err := inner.insertAppQC(cfg.Registry, pos.AppQC); err != nil { + return nil, fmt.Errorf("load AppQC from BlockDB: %w", err) + } } } - return nil + // Advance nextBlock through contiguous loaded blocks. Don't use + // updateNextBlock: stale timestamps would skew metrics. + inner.nextBlock = status.NextBlock + inner.nextBlockToPersist = status.NextBlock + inner.nextAppQCToPersist = status.NextAppQC + if inner.first < inner.nextAppQCToPersist { + n := inner.nextAppQCToPersist - 1 + inner.anchor.Store(utils.Some(Anchor{ + CommitQC: inner.qcs[n].QC(), + AppQC: inner.appQCs[n], + })) + } + return inner, nil } // Registry returns the epoch registry. @@ -450,10 +359,6 @@ func (s *State) PushQC(ctx context.Context, qc *types.FullCommitQC, blocks []*ty } ctrl.Updated() } - // Only newly accepted App data can advance the eviction floor. - if needQC { - evictBelowBound(inner) - } } return nil } @@ -663,11 +568,10 @@ func (s *State) appQCFromDB(n types.GlobalBlockNumber) (*types.AppQC, error) { if err != nil { return nil, fmt.Errorf("blockDB.ReadAppQCByBlockNumber(%d): %w", n, err) } - appQC, ok := opt.Get() - if !ok { - return nil, types.ErrPruned + if appQC, ok := opt.Get(); ok { + return appQC, nil } - return appQC, nil + return nil, types.ErrPruned } func (s *State) globalBlockFromDB(n types.GlobalBlockNumber) (*types.GlobalBlock, error) { @@ -729,7 +633,7 @@ func (s *State) PushAppHash(ctx context.Context, n types.GlobalBlockNumber, hash inner.appProposals[inner.nextAppProposal] = proposal inner.nextAppProposal += 1 } - evictBelowBound(inner) + inner.evict() ctrl.Updated() } return nil @@ -752,18 +656,16 @@ func (s *State) PushAppQC(ctx context.Context, appQC *types.AppQC) error { gr := appQC.Proposal().GlobalRange() for inner, ctrl := range s.inner.Lock() { if err := ctrl.WaitUntil(ctx, func() bool { - return gr.Next <= inner.nextBlock + return gr.Next <= inner.nextQC }); err != nil { return err } - if gr.Next <= inner.nextAppQC { + if gr.First < inner.nextAppQC { return nil } - if err := inner.verifyAppQC(s.cfg.Registry, appQC); err != nil { + if err := inner.insertAppQC(s.cfg.Registry, appQC); err != nil { return err } - inner.applyAppQC(appQC) - evictBelowBound(inner) ctrl.Updated() return nil } @@ -790,24 +692,14 @@ func (s *State) AppQC(ctx context.Context, n types.GlobalBlockNumber) (*types.Ap return appQC, qc, nil } -func (s *State) LastAppQC() (*types.AppQC, *types.FullCommitQC) { - for i := range s.inner.Lock() { - if i.nextAppQC == i.first { - return nil, nil - } - n := i.nextAppQC - 1 - if n >= i.first { - return i.appQCs[n], i.qcs[n] - } - appQC, err := s.appQCFromDB(n) - if err != nil { - return nil, nil - } - qc, err := s.qcFromDB(n) - if err != nil { - return nil, nil - } - return appQC, qc +type Anchor struct { + CommitQC *types.CommitQC + AppQC *types.AppQC +} + +func (s *State) Anchor() utils.AtomicRecv[utils.Option[Anchor]] { + for inner := range s.inner.Lock() { + return inner.anchor.Subscribe() } panic("unreachable") } @@ -870,162 +762,86 @@ func (s *State) PruneBefore(retainFrom types.GlobalBlockNumber) error { // PushAppQC (evictBelowBound); AppQC entries are retained until their own // persistence cursor catches up. func (s *State) runPersist(ctx context.Context) error { - tips := s.blockDB.Status() - var nextToPersistQC, nextToPersistBlock, nextToPersistAppQC types.GlobalBlockNumber - for inner := range s.inner.Lock() { - // After loadFromBlockDB, nextBlockToPersist is the durable recovery tip. - nextToPersistQC = tips.NextQC - if nextToPersistQC == 0 { - nextToPersistQC = inner.nextBlockToPersist - } - nextToPersistBlock = tips.NextBlock - if nextToPersistBlock == 0 { - nextToPersistBlock = inner.nextBlockToPersist - } - nextToPersistAppQC = tips.NextAppQC - if nextToPersistAppQC == 0 { - nextToPersistAppQC = inner.nextAppQCToPersist - } - } for { - type batch struct { - qcs []*types.FullCommitQC - blocks []blockEntry - appQCs []*types.AppQC - nextBlock types.GlobalBlockNumber - nextAppQC types.GlobalBlockNumber - } - var b batch + var qcs []*types.FullCommitQC + var blocks []blockEntry + var appQCs []*types.AppQC + var nextBlock types.GlobalBlockNumber + var nextAppQC types.GlobalBlockNumber for inner, ctrl := range s.inner.Lock() { if err := ctrl.WaitUntil(ctx, func() bool { - return nextToPersistBlock < inner.nextBlock || - inner.appQCReadyToPersist(nextToPersistAppQC, nextToPersistBlock) + return inner.nextBlockToPersist < inner.nextBlock || inner.nextAppQCToPersist < inner.nextAppQC }); err != nil { return err } - if nextToPersistBlock < inner.nextBlock { - b.nextBlock = inner.nextBlock - } - // Persist blocks in [nextToPersistBlock, nextBlock). Emit each covering - // QC once at GlobalRange.First when it has not already been written - // (First >= nextToPersistQC). - for n := nextToPersistBlock; n < b.nextBlock; n++ { - qc := inner.qcs[n] - gr := qc.QC().GlobalRange() - if n == gr.First && gr.First >= nextToPersistQC { - b.qcs = append(b.qcs, qc) + for nextBlock < inner.nextBlock { + qc := inner.qcs[nextBlock] + if nextBlock == qc.QC().GlobalRange().First { + qcs = append(qcs, qc) } - b.blocks = append(b.blocks, blockEntry{n: n, block: inner.blocks[n]}) - } - if b.nextBlock == 0 { - b.nextBlock = nextToPersistBlock + blocks = append(blocks, blockEntry{n: nextBlock, block: inner.blocks[nextBlock]}) + nextBlock++ } - // AppQCs are written only through the durable block cursor from the - // previous flush. This preserves nextAppQCToPersist <= nextBlockToPersist - // at the write side instead of repairing it during recovery. - for inner.appQCReadyToPersist(nextToPersistAppQC, nextToPersistBlock) { - appQC := inner.appQCs[nextToPersistAppQC] - b.appQCs = append(b.appQCs, appQC) - nextToPersistAppQC = appQC.Proposal().GlobalRange().Next + for nextAppQC < inner.nextAppQC { + appQC := inner.appQCs[nextAppQC] + appQCs = append(appQCs, appQC) + nextAppQC = appQC.Proposal().GlobalRange().Next } - b.nextAppQC = nextToPersistAppQC } // Write QCs first (BlockDB contract: QC must precede covered blocks). - for _, qc := range b.qcs { - gr := qc.QC().GlobalRange() + for _, qc := range qcs { if err := s.blockDB.WriteQC(qc); err != nil { - return fmt.Errorf("write QC [%d,%d): %w", gr.First, gr.Next, err) - } - if gr.Next > nextToPersistQC { - nextToPersistQC = gr.Next + return fmt.Errorf("write QC %d: %w", qc.QC().Index(), err) } } - for _, lb := range b.blocks { + for _, lb := range blocks { if err := s.blockDB.WriteBlock(lb.n, lb.block); err != nil { return fmt.Errorf("write block %d: %w", lb.n, err) } } - for _, appQC := range b.appQCs { - gr := appQC.Proposal().GlobalRange() + for _, appQC := range appQCs { if err := s.blockDB.WriteAppQC(appQC); err != nil { - return fmt.Errorf("write AppQC [%d,%d): %w", gr.First, gr.Next, err) + return fmt.Errorf("write AppQC %d: %w", appQC.Proposal().RoadIndex(), err) } } - // Flush once per batch before advancing nextBlockToPersist, so that - // PushAppHash only unblocks after data is crash-durable. AppQCs share - // this async durability boundary. if err := s.blockDB.Flush(); err != nil { return fmt.Errorf("flush BlockDB: %w", err) } - nextToPersistBlock = b.nextBlock for inner, ctrl := range s.inner.Lock() { - if nextToPersistBlock > inner.nextBlockToPersist { - inner.nextBlockToPersist = nextToPersistBlock - } - if b.nextAppQC > inner.nextAppQCToPersist { - old := inner.nextAppQCToPersist - inner.nextAppQCToPersist = b.nextAppQC - evictBelowBound(inner) - inner.dropPersistedAppQCsBelowFirst(old, b.nextAppQC) + inner.nextBlockToPersist = nextBlock + if inner.nextAppQCToPersist < nextAppQC { + inner.nextAppQCToPersist = nextAppQC + inner.anchor.Store(utils.Some(Anchor{ + CommitQC: inner.qcs[nextAppQC-1].QC(), + AppQC: inner.appQCs[nextAppQC-1], + })) } + inner.evict() ctrl.Updated() } } } -func (i *inner) appQCReadyToPersist(n, nextPersistedBlock types.GlobalBlockNumber) bool { - if n >= i.nextAppQC { - return false +// evict pushes first to min(i.nextAppProposal, i.nextAppQCToPersist-1) +// I.e. it makes sure that at least 1 persisted appQC is still in memory: +// it is passed to avail.State. +func (i *inner) evict() { + bound := i.nextAppQCToPersist + if bound > i.first { + bound -= 1 } - appQC := i.appQCs[n] - if appQC == nil { - return false - } - return appQC.Proposal().GlobalRange().Next <= nextPersistedBlock -} - -func (i *inner) dropPersistedAppQCsBelowFirst(first, next types.GlobalBlockNumber) { - for n := first; n < next && n < i.first; n++ { + bound = min(bound, i.nextAppProposal) + for i.first < bound { + n := i.first + delete(i.blockHashes, i.blocks[n].Header().Hash()) + delete(i.blocks, n) + delete(i.qcs, n) delete(i.appQCs, n) + delete(i.appProposals, n) + i.first += 1 } } -// evictBelowBound advances first toward the certified App floor and drops cached -// blocks/QCs/AppProposals with n < first. No-op when there is no certified App -// or the bound would not advance first. Caller must hold inner's lock. Invoked -// from PushQC / PushAppHash. -// -// Bound is min(nextAppProposal, nextAppQCToPersist). A zero floor (no persisted -// AppQC / empty maps) yields bound 0 and is a no-op via bound <= first. AppQCs -// are verified against retained CommitQCs before advancing nextAppQC, and only -// persisted AppQCs advance the eviction floor, so at least one CommitQC remains. -// nextToExecute uses qc[nextAppProposal] (or the tip when caught up), so -// nextAppProposal-1 need not be retained. -// -// TODO: At eviction we have both a local AppProposal and an AppQC, so this -// is the right place to detect local-vs-quorum AppHash inconsistency. Surface -// any mismatch from data.State.Run() (node-fatal), not from PushQC/PushAppHash — -// e.g. stash an error on State for a Run monitor, or run eviction as its own -// Run subtask. -func evictBelowBound(inner *inner) { - bound := min(inner.nextAppProposal, inner.nextAppQCToPersist) - if bound <= inner.first { - return - } - for n := inner.first; n < bound; n++ { - if b, ok := inner.blocks[n]; ok { - delete(inner.blockHashes, b.Header().Hash()) - delete(inner.blocks, n) - } - delete(inner.qcs, n) - if n < inner.nextAppQCToPersist { - delete(inner.appQCs, n) - } - delete(inner.appProposals, n) - } - inner.first = bound -} - // Run starts the background persistence loop. func (s *State) Run(ctx context.Context) error { return s.runPersist(ctx) From 5e11a5d4ceb66b9752507621a75ec6512319cc01 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 5 Aug 2026 18:59:35 +0200 Subject: [PATCH 10/61] pushing AppQCs --- .../internal/autobahn/avail/state.go | 56 +++++++++++-------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/sei-tendermint/internal/autobahn/avail/state.go b/sei-tendermint/internal/autobahn/avail/state.go index 204a94d61e..b897a183dc 100644 --- a/sei-tendermint/internal/autobahn/avail/state.go +++ b/sei-tendermint/internal/autobahn/avail/state.go @@ -111,6 +111,19 @@ func (s *State) LastCommitQC() utils.AtomicRecv[utils.Option[*types.CommitQC]] { panic("unreachable") } +func (s *State) appQC(ctx context.Context, idx types.RoadIndex) (*types.AppQC, error) { + for inner, ctrl := range s.inner.Lock() { + if err := ctrl.WaitUntil(ctx, func() bool { return idx < inner.nextAppQC }); err != nil { + return nil, err + } + if idx < inner.roads.first { + return nil, types.ErrPruned + } + return inner.roads.q[idx].appQC.OrPanic("missing appQC"), nil + } + panic("unreachable") +} + func (s *State) commitQC(ctx context.Context, idx types.RoadIndex) (*types.Epoch, *types.CommitQC, error) { for inner, ctrl := range s.inner.Lock() { if err := ctrl.WaitUntil(ctx, func() bool { return idx < inner.roads.next }); err != nil { @@ -130,27 +143,6 @@ func (s *State) CommitQC(ctx context.Context, idx types.RoadIndex) (*types.Commi return qc, err } -// WaitForAppQC waits until there is an AppQC for the given index or higher. -// Returns this AppQC and the corresponding CommitQC. -// Together they provide enough information to prune the availability state. -func (s *State) WaitForAppQC(ctx context.Context, idx types.RoadIndex) (*types.AppQC, *types.CommitQC, error) { - for inner, ctrl := range s.inner.Lock() { - if err := ctrl.WaitUntil(ctx, func() bool { return idx < inner.nextAppQC }); err != nil { - return nil, nil, err - } - r := inner.roads.q[max(inner.roads.first, idx)] - return r.appQC.OrPanic("missing appQC"), r.commitQC, nil - } - panic("unreachable") -} - -func ignorePruned(err error) error { - if errors.Is(err, types.ErrPruned) { - return nil - } - return err -} - // PushCommitQC pushes a CommitQC to the state. // Waits until all previous CommitQCs are pushed. func (s *State) PushCommitQC(ctx context.Context, qc *types.CommitQC) error { @@ -191,7 +183,10 @@ func (s *State) PushAppVote(ctx context.Context, v *types.Signed[*types.AppVote] idx := v.Msg().Proposal().RoadIndex() epoch, commitQC, err := s.commitQC(ctx, idx) if err != nil { - return ignorePruned(err) + if errors.Is(err, types.ErrPruned) { + return nil + } + return err } if err := v.Msg().Proposal().Verify(commitQC); err != nil { return fmt.Errorf("invalid vote: %w", err) @@ -491,6 +486,22 @@ func (s *State) runPushQC(ctx context.Context) error { } } +// Task inserting AppQCs to data state. +func (s *State) runPushAppQC(ctx context.Context) error { + for n := types.RoadIndex(0); ; n = max(n+1, s.FirstCommitQC()) { + appQC, err := s.appQC(ctx, n) + if err != nil { + if errors.Is(err, types.ErrPruned) { + continue + } + return err + } + if err := s.data.PushAppQC(ctx, appQC); err != nil { + return fmt.Errorf("s.data.PushAppQC(): %w", err) + } + } +} + func (s *State) runEvict(ctx context.Context) error { return s.data.Anchor().Iter(ctx, func(ctx context.Context, anchor utils.Option[data.Anchor]) error { if anchor, ok := anchor.Get(); ok { @@ -513,6 +524,7 @@ func (s *State) Run(ctx context.Context) error { scope.SpawnNamed("runEvict", func() error { return s.runEvict(ctx) }) scope.SpawnNamed("runPersist", func() error { return s.runPersist(ctx) }) scope.SpawnNamed("runPushQC", func() error { return s.runPushQC(ctx) }) + scope.SpawnNamed("runPushAppQC", func() error { return s.runPushAppQC(ctx) }) return nil }) } From 921892cd384b25a72c588abc232b99448da88dfc Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 5 Aug 2026 19:10:44 +0200 Subject: [PATCH 11/61] reverted metrics --- .../internal/autobahn/avail/metrics/metrics.gen.go | 11 +++++++++++ .../internal/autobahn/avail/metrics/metrics.go | 5 ++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/sei-tendermint/internal/autobahn/avail/metrics/metrics.gen.go b/sei-tendermint/internal/autobahn/avail/metrics/metrics.gen.go index 46d58380c9..4aa8151859 100644 --- a/sei-tendermint/internal/autobahn/avail/metrics/metrics.gen.go +++ b/sei-tendermint/internal/autobahn/avail/metrics/metrics.gen.go @@ -14,6 +14,7 @@ func init() { Global.commitRoadIndex, Global.appRoadIndex, Global.commitGlobalBlockNumber, + Global.appGlobalBlockNumber, Global.proposalToCommitLatency, Global.commitToCommitLatency, ) @@ -39,6 +40,12 @@ func newMetrics() *metrics { Name: "commit_global_block_number", Help: "Global block number of the highest observed commitQC.", }, nil), + appGlobalBlockNumber: tmprometheus.NewGaugeIntVec(prometheus.GaugeOpts{ + Namespace: MetricsNamespace, + Subsystem: MetricsSubsystem, + Name: "app_global_block_number", + Help: "Global block number of the highest observed appQC.", + }, nil), proposalToCommitLatency: tmprometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: MetricsNamespace, Subsystem: MetricsSubsystem, @@ -67,6 +74,10 @@ func (m *metrics) commitGlobalBlockNumberAt() *tmprometheus.GaugeInt { return m.commitGlobalBlockNumber.WithLabelValues() } +func (m *metrics) appGlobalBlockNumberAt() *tmprometheus.GaugeInt { + return m.appGlobalBlockNumber.WithLabelValues() +} + func (m *metrics) proposalToCommitLatencyAt() *tmprometheus.Histogram { return m.proposalToCommitLatency.WithLabelValues() } diff --git a/sei-tendermint/internal/autobahn/avail/metrics/metrics.go b/sei-tendermint/internal/autobahn/avail/metrics/metrics.go index 61c32f4142..760ab822ac 100644 --- a/sei-tendermint/internal/autobahn/avail/metrics/metrics.go +++ b/sei-tendermint/internal/autobahn/avail/metrics/metrics.go @@ -21,6 +21,8 @@ type metrics struct { // Global block number of the highest observed commitQC. commitGlobalBlockNumber prometheus.GaugeIntVec + // Global block number of the highest observed appQC. + appGlobalBlockNumber prometheus.GaugeIntVec // Latency from proposal being constructed to commit being observed. proposalToCommitLatency prometheus.HistogramVec `metrics_buckets:"exp(0.01, 1.2, 35)"` @@ -68,7 +70,8 @@ func ObserveAppQC(qc *types.AppQC) { if last, ok := mLast.Get(); ok && last.val.Proposal().RoadIndex() >= qc.Proposal().RoadIndex() { return } - Global.appRoadIndexAt().Set(int64(qc.Proposal().RoadIndex())) // nolint: gosec + Global.appRoadIndexAt().Set(int64(qc.Proposal().RoadIndex())) // nolint: gosec + Global.appGlobalBlockNumberAt().Set(int64(qc.Proposal().GlobalRange().Next)) // nolint: gosec *mLast = utils.Some(observed[*types.AppQC]{now, qc}) } } From 5434911a0b6c5df99a1fb0562a28d2894b9d4697 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 6 Aug 2026 16:34:23 +0200 Subject: [PATCH 12/61] WIP --- sei-db/ledger_db/block/block_db_test.go | 529 +++--------------- sei-db/ledger_db/block/blocksim/blocksim.go | 24 +- .../block/littblock/litt_block_db.go | 145 +++-- .../block/littblock/litt_block_iterator.go | 356 ------------ .../littblock/litt_block_iterator_test.go | 345 ------------ .../littblock/litt_block_simple_iterator.go | 129 ----- .../littblock/litt_block_stranding_test.go | 23 +- .../ledger_db/block/littblock_crash_test.go | 41 +- .../ledger_db/block/memblock/mem_block_db.go | 128 ++--- sei-tendermint/autobahn/types/block_db.go | 129 +---- sei-tendermint/autobahn/types/errors.go | 5 +- .../internal/autobahn/data/state.go | 105 ++-- .../autobahn/data/state_recovery_test.go | 50 +- 13 files changed, 279 insertions(+), 1730 deletions(-) delete mode 100644 sei-db/ledger_db/block/littblock/litt_block_iterator.go delete mode 100644 sei-db/ledger_db/block/littblock/litt_block_iterator_test.go delete mode 100644 sei-db/ledger_db/block/littblock/litt_block_simple_iterator.go diff --git a/sei-db/ledger_db/block/block_db_test.go b/sei-db/ledger_db/block/block_db_test.go index e0ce91004b..7909811c26 100644 --- a/sei-db/ledger_db/block/block_db_test.go +++ b/sei-db/ledger_db/block/block_db_test.go @@ -54,8 +54,7 @@ func TestBlockDB(t *testing.T) { t.Run("ReadRoundTrip", func(t *testing.T) { testReadRoundTrip(t, impl.build) }) t.Run("QCByBlockNumber", func(t *testing.T) { testQCByBlockNumber(t, impl.build) }) t.Run("AppQCByBlockNumber", func(t *testing.T) { testAppQCByBlockNumber(t, impl.build) }) - t.Run("Iterators", func(t *testing.T) { testIterators(t, impl.build) }) - t.Run("IteratorSnapshot", func(t *testing.T) { testIteratorSnapshot(t, impl.build) }) + t.Run("ReadRecent", func(t *testing.T) { testReadRecent(t, impl.build) }) t.Run("RestartPersistsData", func(t *testing.T) { testRestartPersistsData(t, impl.build) }) t.Run("PruneRetainsAtOrAbove", func(t *testing.T) { testPruneRetainsAtOrAbove(t, impl.build) }) t.Run("PruneStraddleRetainsQC", func(t *testing.T) { testPruneStraddleRetainsQC(t, impl.build) }) @@ -85,18 +84,8 @@ func TestBlockDB(t *testing.T) { t.Run("PruneWithAppQCNeverEmpties", func(t *testing.T) { testPruneWithAppQCNeverEmpties(t, impl.build) }) - t.Run("IteratorBlockRequiresPosition", func(t *testing.T) { - testIteratorBlockRequiresPosition(t, impl.build) - }) t.Run("WriteBlockRequiresQC", func(t *testing.T) { testWriteBlockRequiresQC(t, impl.build) }) t.Run("ResumeAfterRestart", func(t *testing.T) { testResumeAfterRestart(t, impl.build) }) - t.Run("IteratorPositioning", func(t *testing.T) { testIteratorPositioning(t, impl.build) }) - t.Run("IteratorTail", func(t *testing.T) { testIteratorTail(t, impl.build) }) - t.Run("IteratorClampsUpToCoverage", func(t *testing.T) { - testIteratorClampsUpToCoverage(t, impl.build) - }) - t.Run("FirstBlockMidQC", func(t *testing.T) { testFirstBlockMidQC(t, impl.build) }) - t.Run("QCOnlyStoreIterates", func(t *testing.T) { testQCOnlyStoreIterates(t, impl.build) }) }) } } @@ -140,11 +129,7 @@ func testEmptyDB(t *testing.T, build builder) { require.NoError(t, err) require.False(t, appQC.IsPresent()) - require.Empty(t, drainIterator(t, openIterator(t, db)), "empty db should yield no positions") - - itAt, err := db.Iterator(0) - require.NoError(t, err) - require.Empty(t, drainIterator(t, itAt), "empty db should yield no positions from any start") + require.Empty(t, drainRecent(t, db), "empty db should yield no recent records") tips := db.Status() require.Zero(t, tips.NextBlock, "empty db has no block write tip") @@ -167,41 +152,37 @@ type iterEntry struct { appQC *types.AppQC } -// openIterator opens an iterator over everything retained in db. -func openIterator(t *testing.T, db types.BlockDB) types.BlockDBIterator { +// drainRecent reads the recovery-visible recent batch. +func drainRecent(t *testing.T, db types.BlockDB) []iterEntry { t.Helper() - it, err := db.Iterator(0) + recent, err := db.ReadRecent() require.NoError(t, err) - return it -} - -// drainIterator walks an iterator to completion (closing it), collecting every position and -// asserting the per-position contract: QC is always present and its covered range contains the number. -func drainIterator(t *testing.T, it types.BlockDBIterator) []iterEntry { - t.Helper() - defer func() { require.NoError(t, it.Close()) }() var entries []iterEntry - for { - pos, ok, err := it.Next() - require.NoError(t, err) - if !ok { - break - } - n, qc := pos.Number, pos.QC - require.NotNil(t, qc, "QC must be present at every position") + for _, qc := range recent.CommitQCs { first := qc.QC().GlobalRange().First next := first + gbn(len(qc.Headers())) - require.True(t, first <= n && n < next, "QC [%d,%d) must cover position %d", first, next, n) - blkOpt, err := it.Block() - require.NoError(t, err) - blk, present := blkOpt.Get() - require.Equal(t, pos.HasBlock, present, "HasBlock must agree with Block at position %d", n) - require.Equal(t, pos.HasAppQC, pos.AppQC != nil, "HasAppQC must agree with AppQC at position %d", n) - if pos.AppQC != nil { - appGR := pos.AppQC.Proposal().GlobalRange() - require.True(t, appGR.Has(n), "AppQC [%d,%d) must cover position %d", appGR.First, appGR.Next, n) + for n := first; n < next; n++ { + entries = append(entries, iterEntry{n: n, qc: qc}) + } + } + for _, b := range recent.Blocks { + found := false + for i := range entries { + if entries[i].n == b.Number { + entries[i].blk = b.Block + found = true + break + } + } + require.True(t, found, "block %d must be covered by a recent QC", b.Number) + } + if appQC, ok := recent.AppQC.Get(); ok { + gr := appQC.Proposal().GlobalRange() + for i := range entries { + if gr.Has(entries[i].n) { + entries[i].appQC = appQC + } } - entries = append(entries, iterEntry{n: n, qc: qc, blk: blk, appQC: pos.AppQC}) } return entries } @@ -270,35 +251,32 @@ func testStatus(t *testing.T, build builder) { require.Equal(t, last.next, tips.NextQC, "QC tip must survive restart") } -// assertTipsMatchPresent checks Status against a full iterator scan (the records -// the public read API still serves). +// assertTipsMatchPresent checks Status against point reads for the records the +// public read API still serves. func assertTipsMatchPresent(t *testing.T, db types.BlockDB) { t.Helper() tips := db.Status() - highest, hasBlock := recoverHighestBlock(t, db) if tips.NextBlock != 0 { - require.True(t, hasBlock, "Status has a block tip but the iterator yields no blocks") - require.Equal(t, highest+1, tips.NextBlock, "NextBlock must be one past the highest present block") - } else { - require.False(t, hasBlock, "the iterator yields blocks but Status has no block tip") + blk, err := db.ReadBlockByNumber(tips.NextBlock - 1) + require.NoError(t, err) + require.True(t, blk.IsPresent(), "NextBlock must point past a readable block") } - lastQC, hasQC := recoverLastQC(t, db) if tips.NextQC != 0 { - require.True(t, hasQC, "Status has a QC tip but the iterator yields no QCs") - require.Equal(t, lastQC.GlobalRange().Next, tips.NextQC, "NextQC must be Next of the highest present QC") - } else { - require.False(t, hasQC, "the iterator yields QCs but Status has no QC tip") + qc, err := db.ReadQCByBlockNumber(tips.NextQC - 1) + require.NoError(t, err) + got, ok := qc.Get() + require.True(t, ok, "NextQC must point past a readable QC") + require.Equal(t, tips.NextQC, got.QC().GlobalRange().Next) } - lastAppQC, hasAppQC := recoverLastAppQC(t, db) if tips.NextAppQC != 0 { - require.True(t, hasAppQC, "Status has an AppQC tip but the iterator yields no AppQCs") - require.Equal(t, lastAppQC.Proposal().GlobalRange().Next, tips.NextAppQC, - "NextAppQC must be Next of the highest present AppQC") - } else { - require.False(t, hasAppQC, "the iterator yields AppQCs but Status has no AppQC tip") + appQC, err := db.ReadAppQCByBlockNumber(tips.NextAppQC - 1) + require.NoError(t, err) + got, ok := appQC.Get() + require.True(t, ok, "NextAppQC must point past a readable AppQC") + require.Equal(t, tips.NextAppQC, got.Proposal().GlobalRange().Next) } } @@ -367,7 +345,7 @@ func testAppQCByBlockNumber(t *testing.T, build builder) { require.NoError(t, err) require.False(t, miss.IsPresent(), "CommitQCs/blocks past the AppQC prefix should not imply AppQC presence") - entries := drainIterator(t, openIterator(t, db)) + entries := drainRecent(t, db) for _, e := range entries { switch { case e.n < batches[2].first: @@ -515,7 +493,7 @@ func testPruneRefusesBelowWatermark(t *testing.T, build builder) { require.False(t, byHash.IsPresent(), "block %d below watermark %d must not be served by hash", n, watermark) } - for _, e := range drainIterator(t, openIterator(t, db)) { + for _, e := range drainRecent(t, db) { require.GreaterOrEqual(t, e.n, watermark, "iterator must not yield position %d below watermark %d", e.n, watermark) } @@ -694,7 +672,7 @@ func testPruneNeverEmpties(t *testing.T, build builder) { for i := range last.blocks { expected = append(expected, last.first+gbn(i)) } - entries := drainIterator(t, openIterator(t, db)) + entries := drainRecent(t, db) require.Equal(t, expected, presentBlockNumbers(entries), "exactly the newest cohort must remain after PruneBefore(%d)", prune) require.Equal(t, []types.GlobalBlockNumber{last.first}, qcFirsts(entries), @@ -812,35 +790,29 @@ func testPruneQCOnlyThenWriteBlock(t *testing.T, build builder) { require.True(t, qc.IsPresent(), "covering QC of block %d must survive the earlier prune", b0.first) } -// testIteratorSnapshot asserts that an iterator observes only the records present -// when it was created — writes made afterward are invisible to it. -func testIteratorSnapshot(t *testing.T, build builder) { +func testReadRecent(t *testing.T, build builder) { committee, keys := buildCommittee() batches := generateBatches(committee, keys) db, _ := openFresh(t, build) defer func() { _ = db.Close() }() - // Write only the first batch, then snapshot an iterator over it. - first := batches[0] - require.NoError(t, db.WriteQC(first.qc)) - for i, blk := range first.blocks { - require.NoError(t, db.WriteBlock(first.first+gbn(i), blk)) + writeAll(t, db, batches[:2]) + appQC := appQCForBatch(utils.TestRngFromSeed(testSeed+400), keys, batches[0]) + require.NoError(t, db.WriteAppQC(appQC)) + require.NoError(t, db.WriteQC(batches[2].qc)) + for i, blk := range batches[2].blocks { + require.NoError(t, db.WriteBlock(batches[2].first+gbn(i), blk)) } - it := openIterator(t, db) - - // Write AppQC and the remaining batches AFTER the iterator was created. - require.NoError(t, db.WriteAppQC(appQCForBatch(utils.TestRngFromSeed(testSeed+400), keys, first))) - writeAll(t, db, batches[1:]) - - entries := drainIterator(t, it) - require.Len(t, presentBlockNumbers(entries), len(first.blocks), - "iterator must not observe blocks written after creation") - require.Equal(t, []types.GlobalBlockNumber{first.first}, qcFirsts(entries), - "iterator must not observe QCs written after creation") - for _, e := range entries { - require.Nil(t, e.appQC, "iterator must not observe AppQCs written after creation") - } + recent, err := db.ReadRecent() + require.NoError(t, err) + gotAppQC, ok := recent.AppQC.Get() + require.True(t, ok) + require.Equal(t, appQC.Proposal().RoadIndex(), gotAppQC.Proposal().RoadIndex()) + entries := drainRecent(t, db) + require.Equal(t, []types.GlobalBlockNumber{batches[0].first, batches[1].first, batches[2].first}, qcFirsts(entries)) + require.Equal(t, batches[2].next-batches[0].first, types.GlobalBlockNumber(len(entries))) + require.Len(t, presentBlockNumbers(entries), int(batches[2].next-batches[0].first)) } func testWriteOrderRejected(t *testing.T, build builder) { @@ -1007,7 +979,7 @@ func testResumeAfterRestart(t *testing.T, build builder) { // verification; production resume uses Status (see blocksim.recoverResumeState). func recoverHighestBlock(t *testing.T, db types.BlockDB) (types.GlobalBlockNumber, bool) { t.Helper() - present := presentBlockNumbers(drainIterator(t, openIterator(t, db))) + present := presentBlockNumbers(drainRecent(t, db)) if len(present) == 0 { return 0, false } @@ -1019,7 +991,7 @@ func recoverHighestBlock(t *testing.T, db types.BlockDB) (types.GlobalBlockNumbe // verification; production resume uses Status (see blocksim.recoverResumeState). func recoverLastQC(t *testing.T, db types.BlockDB) (*types.CommitQC, bool) { t.Helper() - entries := drainIterator(t, openIterator(t, db)) + entries := drainRecent(t, db) if len(entries) == 0 { return nil, false } @@ -1030,7 +1002,7 @@ func recoverLastQC(t *testing.T, db types.BlockDB) (*types.CommitQC, bool) { // iterator scan (false if the store has no AppQCs). func recoverLastAppQC(t *testing.T, db types.BlockDB) (*types.AppQC, bool) { t.Helper() - entries := drainIterator(t, openIterator(t, db)) + entries := drainRecent(t, db) for i := len(entries) - 1; i >= 0; i-- { if entries[i].appQC != nil { return entries[i].appQC, true @@ -1039,279 +1011,6 @@ func recoverLastAppQC(t *testing.T, db types.BlockDB) (*types.AppQC, bool) { return nil, false } -// testIteratorPositioning asserts that Iterator positions at a given height: it yields the -// (clamped) start and every higher covered number, densely ascending, with the whole covering QC -// available even when the start falls mid-range. A start past the last covered number yields -// nothing; a start below the watermark clamps up to the watermark. The positioning assertions run -// twice: on the live store right after the writes (consensus reads the tip while writing) and -// again after a restart (the resume use case, where the backing index is rebuilt). -func testIteratorPositioning(t *testing.T, build builder) { - committee, keys := buildCommittee() - batches := generateBatches(committee, keys) - db, o := openFresh(t, build) - defer func() { _ = db.Close() }() - writeAll(t, db, batches) - - // Pick a start strictly inside a middle QC's range to exercise covering-QC positioning. - mid := batches[len(batches)/2] - require.Greater(t, mid.next, mid.first+1, "need a multi-block QC range") - start := mid.first + 1 - last := batches[len(batches)-1] - - assertPositions := func() { - it, err := db.Iterator(start) - require.NoError(t, err) - entries := drainIterator(t, it) - require.NotEmpty(t, entries) - require.Equal(t, start, entries[0].n, "Iterator must begin at the requested height") - require.Equal(t, mid.first, entries[0].qc.QC().GlobalRange().First, - "a mid-range start must expose the whole covering QC") - require.Equal(t, last.next-1, entries[len(entries)-1].n, "iteration must reach the last covered number") - for i := 1; i < len(entries); i++ { - require.Equal(t, entries[i-1].n+1, entries[i].n, "positions must be densely ascending") - } - for _, e := range entries { - require.NotNil(t, e.blk, "every covered number has a block in a fully-written store") - } - - // The covering QC and every later QC, ascending by First. - var wantFirsts []types.GlobalBlockNumber - for _, b := range batches { - if b.next > start { - wantFirsts = append(wantFirsts, b.first) - } - } - require.Equal(t, wantFirsts, qcFirsts(entries)) - - // A start past the last covered number yields nothing. - itPast, err := db.Iterator(last.next + 100) - require.NoError(t, err) - require.Empty(t, drainIterator(t, itPast)) - } - - assertPositions() - db = restart(t, o, db) - assertPositions() - - // A start below the watermark clamps up to the watermark. - watermark := batches[1].first - require.NoError(t, db.PruneBefore(watermark)) - it, err := db.Iterator(0) - require.NoError(t, err) - clamped := drainIterator(t, it) - require.NotEmpty(t, clamped) - require.Equal(t, watermark, clamped[0].n, "start below the watermark must clamp to the watermark") - for _, e := range clamped { - require.GreaterOrEqual(t, e.n, watermark, "Iterator must never yield a position below the watermark") - } -} - -// testFirstBlockMidQC asserts that iteration opens on a block that exists. WriteBlock lets the very -// first block start anywhere inside its covering QC, so a store can hold a QC whose lower numbers -// carry no block. Iteration must begin at that first block rather than at the QC's start — every -// yielded position then carries a block, so the leading numbers are simply not part of the scan. -// The mirror of testIteratorTail, which covers the blockless run at the other end. -func testFirstBlockMidQC(t *testing.T, build builder) { - committee, keys := buildCommittee() - batches := generateBatches(committee, keys) - db, o := openFresh(t, build) - closed := false - defer func() { - if !closed { - require.NoError(t, db.Close()) - } - }() - - // One QC, but blocks only from the middle of its range onward. - b0 := batches[0] - mid := b0.first + (b0.next-b0.first)/2 - require.Greater(t, mid, b0.first, "need a blockless prefix inside the cohort") - require.NoError(t, db.WriteQC(b0.qc)) - for n := mid; n < b0.next; n++ { - require.NoError(t, db.WriteBlock(n, b0.blocks[n-b0.first])) - } - - assertOpensOnFirstBlock := func(t *testing.T, db types.BlockDB) { - t.Helper() - entries := drainIterator(t, openIterator(t, db)) - require.Equal(t, b0.next-mid, types.GlobalBlockNumber(len(entries)), - "the scan must cover exactly [firstBlock, QC end)") - require.Equal(t, mid, entries[0].n, "the scan must open on the first block that exists") - for i, e := range entries { - require.Equal(t, mid+gbn(i), e.n, "positions must be densely ascending") - require.NotNil(t, e.blk, "every yielded position must carry a block") - } - - // A start below the first block clamps up to it; a start above it is honoured. - below := drainIterator(t, mustIteratorAt(t, db, b0.first)) - require.Equal(t, mid, below[0].n, "a start below the first block clamps up to it") - above := drainIterator(t, mustIteratorAt(t, db, mid+1)) - require.Equal(t, mid+1, above[0].n, "a start above the first block is honoured") - } - - assertOpensOnFirstBlock(t, db) - - // Restart: a durable backend must re-derive the same floor on open. - require.NoError(t, db.Flush()) - require.NoError(t, db.Close()) - closed = true - reopened, err := o() - require.NoError(t, err) - defer func() { require.NoError(t, reopened.Close()) }() - assertOpensOnFirstBlock(t, reopened) -} - -// mustIteratorAt opens an iterator at n, failing the test on error. -func mustIteratorAt(t *testing.T, db types.BlockDB, n types.GlobalBlockNumber) types.BlockDBIterator { - t.Helper() - it, err := db.Iterator(n) - require.NoError(t, err) - return it -} - -// testIteratorTail asserts the QC-ahead-of-blocks shape: when a QC is persisted but (some of) its -// blocks are not — a crash between the QC write and the block writes leaves exactly this — the -// iterator still yields every covered number, with the covering QC present and Block None on the -// trailing positions. This is what lets replay restore trailing QCs from the same single scan. -func testIteratorTail(t *testing.T, build builder) { - committee, keys := buildCommittee() - batches := generateBatches(committee, keys) - require.GreaterOrEqual(t, len(batches), 3, "need a filled prefix plus unfilled tail cohorts") - db, _ := openFresh(t, build) - defer func() { _ = db.Close() }() - - // Fill the first cohort completely and the second only partially; write the - // third cohort's QC with no blocks at all. - b0 := batches[0] - b1 := batches[1] - b2 := batches[2] - require.NoError(t, db.WriteQC(b0.qc)) - for i, blk := range b0.blocks { - require.NoError(t, db.WriteBlock(b0.first+gbn(i), blk)) - } - require.NoError(t, db.WriteQC(b1.qc)) - partial := len(b1.blocks) / 2 - require.Greater(t, partial, 0, "need at least one block in the partially-filled cohort") - for i := 0; i < partial; i++ { - require.NoError(t, db.WriteBlock(b1.first+gbn(i), b1.blocks[i])) - } - require.NoError(t, db.WriteQC(b2.qc)) - - lastBlock := b1.first + gbn(partial-1) - - entries := drainIterator(t, openIterator(t, db)) - require.Equal(t, b2.next-b0.first, types.GlobalBlockNumber(len(entries)), - "the iterator must yield every QC-covered number") - for i, e := range entries { - require.Equal(t, b0.first+gbn(i), e.n, "positions must be densely ascending") - if e.n <= lastBlock { - require.NotNil(t, e.blk, "position %d is below the block tip and must have a block", e.n) - } else { - require.Nil(t, e.blk, "position %d is past the block tip and must be block-less", e.n) - } - } - require.Equal(t, []types.GlobalBlockNumber{b0.first, b1.first, b2.first}, qcFirsts(entries), - "trailing QCs must be observed even where no block survives") - - // An iterator positioned inside the block-less tail still serves the covering QC. - it, err := db.Iterator(b2.first + 1) - require.NoError(t, err) - tail := drainIterator(t, it) - require.NotEmpty(t, tail) - require.Equal(t, b2.first+1, tail[0].n) - require.Equal(t, b2.first, tail[0].qc.QC().GlobalRange().First) - for _, e := range tail { - require.Nil(t, e.blk, "tail positions have no blocks") - } -} - -// testIteratorClampsUpToCoverage asserts the below-coverage clamp: on a store whose first QC -// begins above zero (an unpruned store with a genesis offset), Iterator(0) begins at the first -// covered number rather than yielding nothing. Only a start past the coverage is empty. -func testIteratorClampsUpToCoverage(t *testing.T, build builder) { - db, _ := openFresh(t, build) - defer func() { _ = db.Close() }() - - rng := utils.TestRngFromSeed(testSeed + 99) - first := types.GlobalBlockNumber(100) - next := types.GlobalBlockNumber(105) - require.NoError(t, db.WriteQC(types.GenFullCommitQCRange(rng, first, next))) - for n := first; n < next; n++ { - require.NoError(t, db.WriteBlock(n, types.GenBlock(rng))) - } - - // A start below all coverage clamps up to the first covered number. - it, err := db.Iterator(0) - require.NoError(t, err) - entries := drainIterator(t, it) - require.Len(t, entries, int(next-first)) - require.Equal(t, first, entries[0].n, "a start below coverage must clamp up to the first covered number") - - // A mid-range start begins exactly there. - it, err = db.Iterator(first + 2) - require.NoError(t, err) - entries = drainIterator(t, it) - require.NotEmpty(t, entries) - require.Equal(t, first+2, entries[0].n) - - // A start past the coverage yields nothing. - it, err = db.Iterator(next) - require.NoError(t, err) - require.Empty(t, drainIterator(t, it)) -} - -// testQCOnlyStoreIterates asserts the shape of a store that holds QCs and no blocks at all — what a -// crash between a QC flush and the first block write leaves behind. Every covered number must still be -// yielded with its covering QC and no block, so replay can restore those QCs from the same single scan. -// Distinct from testIteratorTail, where block-less QCs trail a store that does have blocks. -func testQCOnlyStoreIterates(t *testing.T, build builder) { - committee, keys := buildCommittee() - batches := generateBatches(committee, keys) - require.GreaterOrEqual(t, len(batches), 2, "need two cohorts to cover a multi-QC walk") - db, o := openFresh(t, build) - closed := false - defer func() { - if !closed { - require.NoError(t, db.Close()) - } - }() - - // Two QCs, no blocks whatsoever. - b0, b1 := batches[0], batches[1] - require.NoError(t, db.WriteQC(b0.qc)) - require.NoError(t, db.WriteQC(b1.qc)) - - assertQCOnlyShape := func(t *testing.T, db types.BlockDB) { - t.Helper() - // drainIterator cross-checks HasBlock against Block() at every position. - entries := drainIterator(t, openIterator(t, db)) - require.Equal(t, b1.next-b0.first, types.GlobalBlockNumber(len(entries)), - "every number both QCs cover must be yielded") - for i, e := range entries { - require.Equal(t, b0.first+gbn(i), e.n, "positions must be densely ascending") - require.Nil(t, e.blk, "no block exists anywhere in this store") - } - require.Equal(t, []types.GlobalBlockNumber{b0.first, b1.first}, qcFirsts(entries), - "both QCs must be observed in one pass") - - // A mid-range start is honoured, and a start past coverage yields nothing. - mid := drainIterator(t, mustIteratorAt(t, db, b0.first+1)) - require.Equal(t, b0.first+1, mid[0].n) - require.Empty(t, drainIterator(t, mustIteratorAt(t, db, b1.next))) - } - - assertQCOnlyShape(t, db) - - // Restart: the shape must survive a reopen, where a durable backend re-derives its cursors. - require.NoError(t, db.Flush()) - require.NoError(t, db.Close()) - closed = true - reopened, err := o() - require.NoError(t, err) - defer func() { require.NoError(t, reopened.Close()) }() - assertQCOnlyShape(t, reopened) -} - // testWriteBlockRequiresQC asserts the QC-before-block contract: a block may // only be written once a QC covering its number has been written, otherwise // WriteBlock returns ErrBlockMissingQC. This also pins the genesis rule — the @@ -1359,103 +1058,9 @@ func testWriteQCCoversNoBlocksRejected(t *testing.T, build builder) { require.Equal(t, gbn(3), db.Status().NextQC) } -// testIteratorBlockRequiresPosition asserts the one precondition the iterator API -// still carries, identically on every backend. Number, QC and presence come out of -// Next by value, so they cannot be read out of window at all; Block is the only -// accessor left with a positioned precondition (it is the only one that performs -// IO, which is why it is not a Position field). Every window in which it can be -// called without a position must report misuse rather than answer for a stale one. -func testIteratorBlockRequiresPosition(t *testing.T, build builder) { - committee, keys := buildCommittee() - batches := generateBatches(committee, keys) - db, _ := openFresh(t, build) - defer func() { _ = db.Close() }() - writeAll(t, db, batches[:1]) - - t.Run("BeforeFirstNext", func(t *testing.T) { - it := openIterator(t, db) - defer func() { _ = it.Close() }() - _, err := it.Block() - require.Error(t, err, "Block before the first Next must report misuse") - }) - - t.Run("AfterExhaustion", func(t *testing.T) { - it := openIterator(t, db) - defer func() { _ = it.Close() }() - for { - _, ok, err := it.Next() - require.NoError(t, err) - if !ok { - break - } - } - _, err := it.Block() - require.Error(t, err, "Block after exhaustion must report misuse, not repeat the last position") - }) - - t.Run("AfterCloseOnBlocklessPosition", func(t *testing.T) { - // Closing on a block-less position is the shape that slips past AfterClose below, which - // deliberately closes on a held block. With no block held there is no record to read - // through, so an implementation relying on the read failing has nothing to fail on: Next - // must reject the call on its own, and Block must not answer for the position it hands back. - blockless, _ := openFresh(t, build) - defer func() { _ = blockless.Close() }() - b := batches[0] - require.NoError(t, blockless.WriteQC(b.qc)) - require.NoError(t, blockless.WriteBlock(b.first, b.blocks[0])) - - it := openIterator(t, blockless) - var pos types.Position - for { - p, ok, err := it.Next() - require.NoError(t, err) - require.True(t, ok, "expected to reach a block-less position before exhaustion") - if !p.HasBlock { - pos = p - break - } - } - require.False(t, pos.HasBlock, "must be parked on a block-less position for this case to bite") - - require.NoError(t, it.Close()) - - _, ok, err := it.Next() - require.NoError(t, err, "Next after Close must not error") - require.False(t, ok, "Next after Close must report exhaustion, not yield a fresh position") - _, err = it.Block() - require.Error(t, err, "Block after Close must report misuse") - }) - - t.Run("AfterClose", func(t *testing.T) { - it := openIterator(t, db) - pos, ok, err := it.Next() - require.NoError(t, err) - require.True(t, ok) - require.True(t, pos.HasBlock, "the first position must hold a block for this case to bite") - require.NoError(t, it.Close()) - _, err = it.Block() - require.Error(t, err, "Block after Close must report misuse, not read a released snapshot") - }) - - t.Run("EmptyIterator", func(t *testing.T) { - empty, _ := openFresh(t, build) - defer func() { _ = empty.Close() }() - it := openIterator(t, empty) - defer func() { _ = it.Close() }() - pos, ok, err := it.Next() - require.NoError(t, err, "an empty store exhausts cleanly") - require.False(t, ok) - require.Equal(t, types.Position{}, pos, "an unyielded position must be the zero value") - _, err = it.Block() - require.Error(t, err, "Block on an empty iterator must report misuse") - }) -} - // testWriteBlockGapRejected asserts that blocks must be written densely: a // number that skips past lastBlockNumber+1 is rejected with ErrBlockOutOfOrder -// and persists nothing, even when the covering QC allows it. Density is what -// makes BlockDBIterator's tail-only-None contract exact (an absent block below -// the highest persisted one can only be corruption). +// and persists nothing, even when the covering QC allows it. func testWriteBlockGapRejected(t *testing.T, build builder) { committee, keys := buildCommittee() batches := generateBatches(committee, keys) @@ -1511,7 +1116,7 @@ func TestMemblockPruneRemovesBelowWatermark(t *testing.T) { require.True(t, opt.IsPresent()) // The iterator must skip the pruned records entirely. - for _, e := range drainIterator(t, openIterator(t, db)) { + for _, e := range drainRecent(t, db) { require.GreaterOrEqual(t, e.n, watermark, "iterator must not surface pruned positions") require.GreaterOrEqual(t, e.qc.QC().GlobalRange().First, watermark, "iterator must not surface pruned QCs") @@ -1623,7 +1228,7 @@ func assertIterators(t *testing.T, db types.BlockDB, committee *types.Committee, totalBlocks += len(b.blocks) } - entries := drainIterator(t, openIterator(t, db)) + entries := drainRecent(t, db) require.Len(t, entries, totalBlocks, "one position per covered number in a fully-written store") require.Equal(t, batches[0].first, entries[0].n, "the scan must begin at the first covered number") for i, e := range entries { diff --git a/sei-db/ledger_db/block/blocksim/blocksim.go b/sei-db/ledger_db/block/blocksim/blocksim.go index e70e8d4a87..8a0881e6cb 100644 --- a/sei-db/ledger_db/block/blocksim/blocksim.go +++ b/sei-db/ledger_db/block/blocksim/blocksim.go @@ -178,29 +178,11 @@ func NewBlockSim( // countExistingState scans the ledger to count the persisted blocks and QCs, // exercising the replay path at startup. func countExistingState(db types.BlockDB) (blocks int, qcs int, err error) { - it, err := db.Iterator(0) + recent, err := db.ReadRecent() if err != nil { - return 0, 0, fmt.Errorf("failed to open ledger iterator: %w", err) + return 0, 0, fmt.Errorf("failed to read recent ledger data: %w", err) } - defer func() { _ = it.Close() }() - for { - pos, ok, err := it.Next() - if err != nil { - return 0, 0, fmt.Errorf("failed to advance ledger iterator: %w", err) - } - if !ok { - break - } - if pos.QC.QC().GlobalRange().First == pos.Number { - // The scan entered a new QC's range. - qcs++ - } - // Presence comes off the position, so counting never reads a block value. - if pos.HasBlock { - blocks++ - } - } - return blocks, qcs, nil + return len(recent.Blocks), len(recent.CommitQCs), nil } // recoverResumeState reads the persisted tail so the benchmark resumes appending diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index 490b66caa6..e71285b3e7 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -2,6 +2,7 @@ package littblock import ( "fmt" + "slices" "sync" "sync/atomic" @@ -465,87 +466,85 @@ func (s *blockDB) Status() types.DBStatus { return tips } -func (s *blockDB) Iterator(n types.GlobalBlockNumber) (types.BlockDBIterator, error) { - // One consistent read of the cursors. The watermark stays on its atomic because the GC goroutine - // writes it, but everything else is taken together so the floors below cannot disagree about - // which instant they describe. +// ReadRecent() reads the latest AppQC and all Blocks and CommitQCs, for indices >= AppQC.GlobalRange().First. +// WARNING: ReadRecent() will return an error if watermark is moved during iteration. +func (s *blockDB) ReadRecent() (types.RecentData, error) { + // Determine the targetFloor: it is either all the data, or data since the lastestAppQC. s.mu.Lock() - hasQC, nextQC, oldestQCStart := s.hasQC, s.lastQCNext, s.oldestQCStart - hasBlocks, firstBlock := s.hasBlocks, s.firstBlockNumber + watermark := s.watermark.Load() + targetFloor := types.GlobalBlockNumber(watermark) + if s.hasAppQC { + targetFloor = s.latestAppQCStartBlock + } s.mu.Unlock() - watermark := types.GlobalBlockNumber(s.watermark.Load()) - - if !hasQC { - // An empty store covers nothing. - return &blockDBIterator{}, nil + // Collect data >= targetFloor. + it, err := s.table.Iterator(true) + if err != nil { + return types.RecentData{}, fmt.Errorf("failed to open recent-data iterator: %w", err) } - - // Clamp up to the lowest number this store can serve. The watermark is the retention gate and - // oldestQCStart is where coverage begins on a store that never had data below it (bootstrapped - // mid-chain); either may be the higher. - start := max(n, watermark, oldestQCStart) - - if !hasBlocks { - // No block has ever been written, so there is no block to open on and the QC floor governs. - // This is the one case blockDBIterator cannot serve safely — see simpleIterator. - if start >= nextQC { - return &blockDBIterator{}, nil + defer func() { _ = it.Close() }() + var recent types.RecentData + for done := false; !done; { + ok, err := it.Next() + if err != nil { + return types.RecentData{}, fmt.Errorf("failed to advance recent-data iterator: %w", err) } - appQCs, err := snapshotAppQCs(s.table, start, nextQC) + if !ok { + break + } + key, isPrimary, err := it.GetKey() if err != nil { - return nil, err + return types.RecentData{}, fmt.Errorf("failed to read recent-data key: %w", err) + } + if !isPrimary { + continue + } + value, err := it.GetValue() + if err != nil { + return types.RecentData{}, fmt.Errorf("failed to read recent-data value: %w", err) + } + switch keyKind(key) { + case kindBlock: + n, block, err := decodeBlock(value) + if err != nil { + return types.RecentData{}, fmt.Errorf("failed to decode recent block: %w", err) + } + if targetFloor <= n { + recent.Blocks = append(recent.Blocks, types.RecentBlock{Number: n, Block: block}) + } + case kindAppQC: + appQC, err := decodeAppQC(value) + if err != nil { + return types.RecentData{}, fmt.Errorf("failed to decode recent AppQC: %w", err) + } + if targetFloor <= appQC.Proposal().GlobalRange().First { + recent.AppQC = utils.Some(appQC) + } + case kindQC: + qc, err := decodeQC(value) + if err != nil { + return types.RecentData{}, fmt.Errorf("failed to decode recent CommitQC: %w", err) + } + first := qc.QC().GlobalRange().First + if targetFloor <= first { + recent.CommitQCs = append(recent.CommitQCs, qc) + } + if first <= targetFloor { + // targetFloor has been reached - since CommitQC is persisted before covered Blocks and AppQC, + // reaching CommitQC for targetFloor means we finished the read + done = true + } + default: } - return newSimpleIterator(s.table, start, nextQC, appQCs) - } - - // firstBlock is where the block history begins, which can sit inside its covering QC's range - // because the first block is free to start there. Clamping to it is what makes the scan open on a - // block that exists rather than on blockless numbers below it. - start = max(start, firstBlock) - - if start >= nextQC { - // Nothing is covered at or above start. - return &blockDBIterator{}, nil } - - appQCs, err := snapshotAppQCs(s.table, start, nextQC) - if err != nil { - return nil, err + // Safety check: if watermark has been moved and GC happened to get executed during iteration, + // the loaded data might be inconsistent with the targetFloor we computed. + if got := s.watermark.Load(); got != watermark { + return types.RecentData{}, fmt.Errorf("watermark has moved while iterating") } - - // A QC is stored under its First as the primary key with a covered-number alias for every - // other number in its range, and an alias carries the full QC value. Positioning the scan at - // qcKey(start) therefore lands on the covering QC no matter where start falls in its range. - it, found, err := s.table.IteratorAt(qcKey(start), false) - if err != nil { - return nil, fmt.Errorf("failed to open iterator at %d: %w", start, err) - } - if !found { - // start was clamped into [oldestQCStart, NextQC) and retained QCs cover that interval - // contiguously, so some QC's primary or covered-number alias is stored under qcKey(start) - // — unless the retention floor moved past start while we were positioning. Either way we - // refuse rather than scanning from the beginning of the table, which would read exactly - // the history the caller passed n to skip. Which of the two it was decides the diagnosis, - // and getting that wrong is expensive: reporting corruption on a healthy store sends an - // operator hunting for damage that isn't there. - if s.watermark.Load() > uint64(start) { - // A concurrent PruneBefore advanced the floor past start, and GC reclaimed the record - // (litt surfaces a prune/GC boundary as not-found — see DiskTable.IteratorAt). GC's - // filter only clears a segment once the watermark exceeds every number in it, so a - // watermark above start is exactly the condition that makes this benign rather than - // corrupt. Racing a pruner has no deterministic answer, so report the floor honestly - // and let the caller decide whether to retry. - return nil, fmt.Errorf("%w: start %d fell below the retention floor while positioning", - types.ErrPruned, start) - } - return nil, fmt.Errorf("corrupt store: no QC record at %d despite coverage to %d", start, nextQC) - } - return &blockDBIterator{ - it: it, - appQCs: appQCs, - startN: start, - expectStartQC: true, - }, nil + slices.Reverse(recent.CommitQCs) + slices.Reverse(recent.Blocks) + return recent, nil } func (s *blockDB) ReadBlockByNumber(n types.GlobalBlockNumber) (utils.Option[*types.Block], error) { diff --git a/sei-db/ledger_db/block/littblock/litt_block_iterator.go b/sei-db/ledger_db/block/littblock/litt_block_iterator.go deleted file mode 100644 index ea0745f190..0000000000 --- a/sei-db/ledger_db/block/littblock/litt_block_iterator.go +++ /dev/null @@ -1,356 +0,0 @@ -package littblock - -import ( - "fmt" - - littdb "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt" - "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" - "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" -) - -var _ types.BlockDBIterator = (*blockDBIterator)(nil) - -// coveredQC is a decoded QC together with the half-open number range it covers. -type coveredQC struct { - // qc is the decoded QC. - qc *types.FullCommitQC - - // first is the lowest number the QC covers (its GlobalRange().First). - first types.GlobalBlockNumber - - // next is one past the highest number the QC covers (its GlobalRange().Next). - next types.GlobalBlockNumber -} - -type coveredAppQC struct { - appQC *types.AppQC - first types.GlobalBlockNumber - next types.GlobalBlockNumber -} - -// blockDBIterator implements types.BlockDBIterator over the shared ledger table. -// -// It steps through consecutive block numbers, driven by a single forward litt scan. The scan -// visits records in insertion order, and the write path inserts every QC before the blocks it -// covers, so by the time the number cursor reaches n the covering QC has always been decoded -// already: the underlying cursor is advanced lazily, collecting QC records into a small pending -// queue and holding position on each block record until the number cursor consumes it. -// -// A nil it represents an empty iterator (produced by Iterator when the requested start is past -// the persisted coverage): Next reports exhaustion immediately and Close is a no-op. -type blockDBIterator struct { - // it is the underlying litt scan; nil for an empty iterator. - it littdb.Iterator - - // appQCs is a snapshot of AppQC coverage captured when the iterator was - // opened. AppQCs may be written after the blocks they certify, so the - // insertion-order block/QC scan cannot discover them before yielding those - // blocks without a separate snapshot. - appQCs []*coveredAppQC - - // startN is the first number the iterator may yield, and doubles as the retention floor: - // blockDB.Iterator clamps it up to the prune watermark and to the start of the block history, - // so a block below startN is either below the start or stranded from a reclaimed QC, and is - // skipped either way. Because of the block-history floor, startN names a block that exists - // whenever the store holds one — it may sit inside its covering QC's range rather than on that - // range's first number. - startN types.GlobalBlockNumber - - // expectStartQC is one-shot state, not a mode: every non-empty iterator is positioned at - // qcKey(startN), so the scan's very first record is the covering QC's primary or - // covered-number alias rather than a record the normal dispatch handles. Cleared once that - // record is consumed. - expectStartQC bool - - // started is false until the first Next call establishes the start position. - started bool - - // positioned is true only while the iterator sits on a number Next yielded. Block - // rejects calls made when it is false. - positioned bool - - // n is the current number; valid while positioned (after Next has returned true). - n types.GlobalBlockNumber - - // current is the QC covering n; nil only before the first Next call. - current *coveredQC - - // pending holds QCs the scan has passed whose ranges begin at or above current.next. QCs are - // written ahead of the blocks they cover, so several can precede the block records that - // consume them. Ranges are ascending and contiguous with current. - pending []*coveredQC - - // heldBlock is true when the underlying cursor is positioned on a block record the number - // cursor has not consumed yet; heldNumber is that record's number. - heldBlock bool - - // heldNumber is the number of the held block record; meaningful only while heldBlock is true. - heldNumber types.GlobalBlockNumber - - // exhausted is true once the underlying scan has no more records. - exhausted bool - - // closed is true once Close has been called. Next and Block reject calls made afterward. - // Checking it explicitly is what makes that rejection uniform: on a trailing block-less position - // the scan holds no record, so Next would otherwise skip fill() entirely and hand back a fresh - // position without ever touching the closed cursor. - closed bool -} - -func (l *blockDBIterator) Next() (types.Position, bool, error) { - // Any exit other than a yielded position leaves the iterator unpositioned, so a - // subsequent Block() reports misuse rather than answering for a stale position. - l.positioned = false - - if l.closed || l.it == nil { - return types.Position{}, false, nil - } - - var next types.GlobalBlockNumber - if l.started { - next = l.n + 1 - if l.heldBlock && l.heldNumber == l.n { - // Leaving a position whose block record the scan still holds; release it so the - // scan can advance past it. - l.heldBlock = false - } - } else { - // The start position needs the first covering QC in hand before a number can be yielded. - if err := l.fill(); err != nil { - return types.Position{}, false, err - } - if l.current == nil { - // Unreachable: the scan is positioned at qcKey(startN), so its first record is the - // covering QC — fill dispatches it through expectStartQC, which errors if it is not - // a QC record, and collectQC always adopts it as current because its range contains - // startN. Asserted rather than dereferenced blindly below. - return types.Position{}, false, - fmt.Errorf("ledger scan at %d established no covering QC", l.startN) - } - // current's range contains startN (the scan was positioned inside it), so no clamp - // up to current.first is needed. - next = l.startN - } - - // Establish coverage for next, promoting across QC range boundaries. - for next >= l.current.next { - if len(l.pending) > 0 { - l.current = l.pending[0] - l.pending = l.pending[1:] - continue - } - if l.heldBlock { - // A block record at heldNumber >= next is waiting, but no QC covers next. The write - // path guarantees a covering QC precedes every block, so this is corruption. - return types.Position{}, false, - fmt.Errorf("corrupt store: block %d has no QC coverage", l.heldNumber) - } - if l.exhausted { - return types.Position{}, false, nil - } - if err := l.fill(); err != nil { - return types.Position{}, false, err - } - } - - // Position the scan to answer Block() at next: it must hold the next block record (if any - // remains) so presence is decidable. - if !l.heldBlock && !l.exhausted { - if err := l.fill(); err != nil { - return types.Position{}, false, err - } - } - if l.heldBlock && l.heldNumber != next { - // Blocks are written densely, so a missing number below the highest persisted block can - // only be corruption. - return types.Position{}, false, - fmt.Errorf("%w: corrupt store: block gap at %d (next persisted block is %d)", - types.ErrBlockGap, next, l.heldNumber) - } - - appQC := appQCCovering(l.appQCs, next) - - l.n = next - l.started = true - l.positioned = true - return types.Position{ - Number: next, - QC: l.current.qc, - HasBlock: l.heldBlock, - AppQC: appQC, - HasAppQC: appQC != nil, - }, true, nil -} - -// fill advances the underlying scan until it holds an unconsumed block record or exhausts, -// decoding every QC record it passes into the covering-QC state. -func (l *blockDBIterator) fill() error { - for !l.heldBlock && !l.exhausted { - ok, err := l.it.Next() - if err != nil { - return fmt.Errorf("failed to advance ledger scan: %w", err) - } - if !ok { - l.exhausted = true - return nil - } - key, isPrimary, err := l.it.GetKey() - if err != nil { - return fmt.Errorf("failed to read ledger key: %w", err) - } - switch { - case l.expectStartQC: - // The scan was positioned at qcKey(start): the first record is the covering - // QC — its primary when start is a range's First, otherwise a covered-number - // alias, whose value is the same full QC either way. - l.expectStartQC = false - if keyKind(key) != kindQC { - return fmt.Errorf("ledger scan positioned at %d is not on a QC record", l.startN) - } - if err := l.collectQC(); err != nil { - return err - } - case !isPrimary: - // Secondary records (block hash aliases, QC covered-number aliases) duplicate a - // primary the scan handles elsewhere. - case keyKind(key) == kindBlock: - number := decodeNumberKey(key) - if number < l.startN { - // Below the start: either a start that lands mid-cohort, whose cohort's earlier - // blocks still follow the covering QC in the scan, or a block stranded below the - // retention floor. startN is never below that floor, so one test covers both. - continue - } - l.heldBlock = true - l.heldNumber = number - case keyKind(key) == kindQC: - if err := l.collectQC(); err != nil { - return err - } - case keyKind(key) == kindAppQC: - // AppQC coverage was snapshotted when this iterator opened. - default: - return fmt.Errorf("unknown ledger key kind %q", keyKind(key)) - } - } - return nil -} - -func appQCCovering(appQCs []*coveredAppQC, n types.GlobalBlockNumber) *types.AppQC { - for _, a := range appQCs { - if a.first <= n && n < a.next { - return a.appQC - } - } - return nil -} - -func snapshotAppQCs( - table littdb.Table, - start types.GlobalBlockNumber, - nextQC types.GlobalBlockNumber, -) ([]*coveredAppQC, error) { - it, err := table.Iterator(false) - if err != nil { - return nil, fmt.Errorf("failed to open AppQC snapshot iterator: %w", err) - } - defer func() { _ = it.Close() }() - - var appQCs []*coveredAppQC - for { - ok, err := it.Next() - if err != nil { - return nil, fmt.Errorf("failed to advance AppQC snapshot iterator: %w", err) - } - if !ok { - break - } - key, isPrimary, err := it.GetKey() - if err != nil { - return nil, fmt.Errorf("failed to read AppQC snapshot key: %w", err) - } - if !isPrimary || keyKind(key) != kindAppQC { - continue - } - value, err := it.GetValue() - if err != nil { - return nil, fmt.Errorf("failed to read AppQC snapshot value: %w", err) - } - appQC, err := decodeAppQC(value) - if err != nil { - return nil, fmt.Errorf("failed to decode AppQC snapshot value: %w", err) - } - first, next := appQCRange(appQC) - if next <= start || first >= nextQC { - continue - } - appQCs = append(appQCs, &coveredAppQC{appQC: appQC, first: first, next: next}) - } - return appQCs, nil -} - -// collectQC decodes the QC record at the scan's current position into the covering-QC state: it -// becomes current when no current QC is set, and otherwise joins the pending queue. -func (l *blockDBIterator) collectQC() error { - value, err := l.it.GetValue() - if err != nil { - return fmt.Errorf("failed to read QC value: %w", err) - } - qc, err := decodeQC(value) - if err != nil { - return fmt.Errorf("failed to unmarshal QC: %w", err) - } - first, next := coveredRange(qc) - entry := &coveredQC{qc: qc, first: first, next: next} - if l.current == nil { - l.current = entry - return nil - } - // QCs are written contiguously, so each surviving range must extend the previous bound. - tailNext := l.current.next - if len(l.pending) > 0 { - tailNext = l.pending[len(l.pending)-1].next - } - if entry.first != tailNext { - return fmt.Errorf("corrupt store: QC range [%d,%d) does not extend previous bound %d", - entry.first, entry.next, tailNext) - } - l.pending = append(l.pending, entry) - return nil -} - -func (l *blockDBIterator) Block() (utils.Option[*types.Block], error) { - if l.closed { - return utils.None[*types.Block](), fmt.Errorf("iterator is closed") - } - if !l.positioned { - return utils.None[*types.Block](), fmt.Errorf("iterator is not positioned on a block number") - } - if !l.heldBlock { - // The tail of the ledger: the covering QC is persisted but this block is not. - return utils.None[*types.Block](), nil - } - value, err := l.it.GetValue() - if err != nil { - return utils.None[*types.Block](), fmt.Errorf("failed to read block value: %w", err) - } - _, blk, err := decodeBlock(value) - if err != nil { - return utils.None[*types.Block](), fmt.Errorf("failed to unmarshal block: %w", err) - } - return utils.Some(blk), nil -} - -func (l *blockDBIterator) Close() error { - // A closed iterator holds no position, so Next() reports exhaustion and Block() reports misuse - // rather than reading through a released snapshot. - l.closed = true - l.positioned = false - if l.it == nil { - return nil - } - if err := l.it.Close(); err != nil { - return fmt.Errorf("failed to close ledger iterator: %w", err) - } - return nil -} diff --git a/sei-db/ledger_db/block/littblock/litt_block_iterator_test.go b/sei-db/ledger_db/block/littblock/litt_block_iterator_test.go deleted file mode 100644 index 324f45fe1c..0000000000 --- a/sei-db/ledger_db/block/littblock/litt_block_iterator_test.go +++ /dev/null @@ -1,345 +0,0 @@ -package littblock - -import ( - "sync" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" - "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" -) - -// TestLittblockIteratorConcurrentFirstBlockIsNotAGap pins the reason simpleIterator exists. A store -// holding a QC and no blocks has no block for a scan to open on, and the first block may legally land -// anywhere inside its covering QC. If Iterator opened a table snapshot in that state, a WriteBlock -// racing it could put a block above the cursor into the snapshot and the scan would report it as a gap -// — a corruption error on a healthy store. Iterator must therefore take no snapshot until a block -// exists. -// -// The interleaving is not directly forceable, so this hammers the window instead: many rounds of a -// fresh QC-only store with Iterator and WriteBlock started together. Any ErrBlockGap is a regression. -func TestLittblockIteratorConcurrentFirstBlockIsNotAGap(t *testing.T) { - rng := utils.TestRngFromSeed(31) - - for round := 0; round < 64; round++ { - db, err := NewBlockDB(strandingConfig(t, t.TempDir(), 1024)) - require.NoError(t, err) - - // One QC covering [0,6) and no blocks: the state where the scan has nothing to anchor on. - require.NoError(t, db.WriteQC(types.GenFullCommitQCRange(rng, 0, 6))) - blk := types.GenBlock(rng) - - var wg sync.WaitGroup - wg.Add(2) - var writeErr, iterErr error - go func() { - defer wg.Done() - // The first block, landing mid-QC — permitted by WriteBlock. - writeErr = db.WriteBlock(2, blk) - }() - go func() { - defer wg.Done() - it, err := db.Iterator(0) - if err != nil { - iterErr = err - return - } - defer func() { _ = it.Close() }() - for { - _, ok, err := it.Next() - if err != nil { - iterErr = err - return - } - if !ok { - return - } - } - }() - wg.Wait() - - require.NoError(t, writeErr, "round %d: the first block may start anywhere inside its QC", round) - require.NotErrorIs(t, iterErr, types.ErrBlockGap, - "round %d: a concurrent first block must never read as a gap", round) - require.NoError(t, iterErr, "round %d", round) - require.NoError(t, db.Close()) - } -} - -// TestLittblockIteratorGapAboveMidQCStartIsCorruption pins that clamping the scan's start up to the -// first existing block does not weaken gap detection above it. A store may legitimately begin partway -// into its covering QC, but blocks are dense from that point up, so an interior hole above it is still -// corruption. A clamp that overshot the first block would silently skip past such a hole. This cannot -// be written against the shared BlockDB contract: producing the hole needs a raw table write that -// bypasses WriteBlock's contiguity cursor. -func TestLittblockIteratorGapAboveMidQCStartIsCorruption(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRngFromSeed(21) - - db, err := NewBlockDB(strandingConfig(t, dir, 1024)) - require.NoError(t, err) - defer func() { _ = db.Close() }() - impl := db.(*blockDB) - - // One QC covering [0,6) whose block history begins at 2, so the scan opens at 2. - require.NoError(t, db.WriteQC(types.GenFullCommitQCRange(rng, 0, 6))) - require.NoError(t, db.WriteBlock(2, types.GenBlock(rng))) - require.NoError(t, db.WriteBlock(3, types.GenBlock(rng))) - - // Corrupt the store above the start: a block at 5 with none at 4. - require.NoError(t, impl.table.Put(blockKey(5), encodeBlock(5, types.GenBlock(rng)))) - - it, err := db.Iterator(0) - require.NoError(t, err) - defer func() { _ = it.Close() }() - - for _, want := range []types.GlobalBlockNumber{2, 3} { - pos, ok, err := it.Next() - require.NoError(t, err) - require.True(t, ok) - require.Equal(t, want, pos.Number, "the scan must open on the first existing block") - require.True(t, pos.HasBlock) - } - - // Position 4 has no block while a later one exists: corruption, not a legitimate hole. - _, _, err = it.Next() - require.ErrorIs(t, err, types.ErrBlockGap) -} - -// TestLittblockIteratorGapIsCorruption pins the iterator's gap detection: blocks are written -// densely (WriteBlock enforces contiguity), so an interior missing block on disk can only be -// corruption and must surface as an error rather than a silent None. The gap is injected by -// writing a block record directly to the raw table, bypassing WriteBlock's contiguity cursor. -func TestLittblockIteratorGapIsCorruption(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRngFromSeed(7) - - db, err := NewBlockDB(strandingConfig(t, dir, 1024)) - require.NoError(t, err) - defer func() { _ = db.Close() }() - impl := db.(*blockDB) - - // One QC covering [0,5) with blocks 0 and 1 written normally. - require.NoError(t, db.WriteQC(types.GenFullCommitQCRange(rng, 0, 5))) - require.NoError(t, db.WriteBlock(0, types.GenBlock(rng))) - require.NoError(t, db.WriteBlock(1, types.GenBlock(rng))) - - // Corrupt the store: a block at 3 with no block at 2, injected past WriteBlock. - blk := types.GenBlock(rng) - require.NoError(t, impl.table.Put(blockKey(3), encodeBlock(3, blk))) - - it, err := db.Iterator(0) - require.NoError(t, err) - defer func() { _ = it.Close() }() - - // Positions 0 and 1 are intact; advancing to 2 must surface the gap as corruption. - for _, want := range []types.GlobalBlockNumber{0, 1} { - pos, ok, err := it.Next() - require.NoError(t, err) - require.True(t, ok) - require.Equal(t, want, pos.Number) - } - _, _, err = it.Next() - require.ErrorIs(t, err, types.ErrBlockGap) -} - -// TestLittblockIteratorUncoveredBlockIsCorruption pins the iterator's coverage check: a QC -// covering every block is always written first, so a block record beyond every QC's range can -// only be corruption and must surface as an error. The uncovered block is injected by writing a -// block record directly to the raw table, bypassing WriteBlock's coverage check. -func TestLittblockIteratorUncoveredBlockIsCorruption(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRngFromSeed(8) - - db, err := NewBlockDB(strandingConfig(t, dir, 1024)) - require.NoError(t, err) - defer func() { _ = db.Close() }() - impl := db.(*blockDB) - - // One QC covering [0,2), fully blocked. - require.NoError(t, db.WriteQC(types.GenFullCommitQCRange(rng, 0, 2))) - require.NoError(t, db.WriteBlock(0, types.GenBlock(rng))) - require.NoError(t, db.WriteBlock(1, types.GenBlock(rng))) - - // Corrupt the store: a block at 2, past every QC's range, injected past WriteBlock. - blk := types.GenBlock(rng) - require.NoError(t, impl.table.Put(blockKey(2), encodeBlock(2, blk))) - - it, err := db.Iterator(0) - require.NoError(t, err) - defer func() { _ = it.Close() }() - - for _, want := range []types.GlobalBlockNumber{0, 1} { - pos, ok, err := it.Next() - require.NoError(t, err) - require.True(t, ok) - require.Equal(t, want, pos.Number) - } - _, _, err = it.Next() - require.Error(t, err, "a block with no QC coverage must surface as corruption") - require.Contains(t, err.Error(), "no QC coverage") -} - -// TestLittblockIteratorMidChainStartNeedsNoScan pins that a store whose coverage begins above 0 is -// positioned directly, without the full-scan fallback that used to serve this case. It must behave -// identically before and after a restart: in the writing session the clamp comes from -// oldestQCStart, after a reopen recoverReadFloors derives the same floor into the watermark. -func TestLittblockIteratorMidChainStartNeedsNoScan(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRngFromSeed(23) - cfg := strandingConfig(t, dir, 1<<20) - - db, err := NewBlockDB(cfg) - require.NoError(t, err) - - // Coverage begins at 100 and nothing below it was ever written. - require.NoError(t, db.WriteQC(types.GenFullCommitQCRange(rng, 100, 103))) - for n := types.GlobalBlockNumber(100); n < 103; n++ { - require.NoError(t, db.WriteBlock(n, types.GenBlock(rng))) - } - require.NoError(t, db.Flush()) - - drain := func(t *testing.T, d types.BlockDB, n types.GlobalBlockNumber) []types.GlobalBlockNumber { - t.Helper() - it, err := d.Iterator(n) - require.NoError(t, err) - defer func() { _ = it.Close() }() - var got []types.GlobalBlockNumber - for { - pos, ok, err := it.Next() - require.NoError(t, err) - if !ok { - return got - } - got = append(got, pos.Number) - } - } - - want := []types.GlobalBlockNumber{100, 101, 102} - for _, start := range []types.GlobalBlockNumber{0, 50, 100} { - require.Equal(t, want, drain(t, db, start), "same session, Iterator(%d)", start) - } - require.Equal(t, []types.GlobalBlockNumber{101, 102}, drain(t, db, 101), "same session, mid-cohort start") - - require.NoError(t, db.Close()) - db2, err := NewBlockDB(cfg) - require.NoError(t, err) - defer func() { _ = db2.Close() }() - - for _, start := range []types.GlobalBlockNumber{0, 50, 100} { - require.Equal(t, want, drain(t, db2, start), "after restart, Iterator(%d)", start) - } -} - -// TestLittblockIteratorMissingStartQCIsCorruption pins that a positioned lookup which misses inside -// known coverage is an error, not a silent full-scan fallback. The clamp guarantees some QC record -// is stored under qcKey(start), so a miss means a record that must exist does not. -// -// It also pins the negative side of the prune/corruption discriminator: with the retention floor at -// or below start, the miss must be reported as corruption and NOT relabelled ErrPruned. The positive -// side (floor above start ⇒ ErrPruned) needs the watermark to advance between Iterator's own -// watermark load and its positioning call, which no deterministic test can arrange without a -// production seam — so it is verified by inspection against gcFilter's reclamation condition -// (littblock's gcFilter clears a key only once the watermark exceeds its number). -func TestLittblockIteratorMissingStartQCIsCorruption(t *testing.T) { - rng := utils.TestRngFromSeed(24) - db, err := NewBlockDB(strandingConfig(t, t.TempDir(), 1<<20)) - require.NoError(t, err) - defer func() { _ = db.Close() }() - impl := db.(*blockDB) - - require.NoError(t, db.WriteQC(types.GenFullCommitQCRange(rng, 0, 3))) - for n := types.GlobalBlockNumber(0); n < 3; n++ { - require.NoError(t, db.WriteBlock(n, types.GenBlock(rng))) - } - require.NoError(t, db.Flush()) - - // Claim coverage runs to 10 while only [0,3) is stored, so qcKey(5) has no record. This is the - // shape a truncated key file or a segment missing from the snapshot presents as. - impl.mu.Lock() - impl.lastQCNext = 10 - impl.mu.Unlock() - - _, err = db.Iterator(5) - require.Error(t, err, "a missing start QC inside claimed coverage must not fall back to a full scan") - require.Contains(t, err.Error(), "corrupt store") - require.NotErrorIs(t, err, types.ErrPruned, - "the floor is at or below start, so this is corruption and must not be excused as pruning") -} - -// TestLittblockIteratorDoesNotServeBlockBelowCoverage pins the start-clamp boundary for a block -// that no retained QC covers: it is not served, and not reported as corruption either. -// -// This test previously asserted the opposite. It was written when Iterator fell back to a plain -// full scan whenever the positioned lookup missed, which meant Iterator(0) walked over the -// uncovered block and flagged it. Iterator now clamps its start up to oldestQCStart and refuses to -// scan below it, so the block is never visited. That is the intended semantics on both counts: -// -// - Iterator(n) is contractually clamped up to the lowest number a retained QC covers, and this -// block is below every retained QC's range, so it is not a position the iterator may yield. -// - Detecting corruption strictly below the requested start is exactly what a positioned -// iterator exists to avoid (see the audit's finding 6), and the detection here was incidental -// to the fallback rather than designed. -// -// The detection is also not recoverable in principle: this same on-disk state — a block below the -// oldest retained QC — is the *legitimate* stranded state after a prune whose GC pass reclaimed -// the covering QC, and littblock does not persist the watermark, so after a restart the two are -// indistinguishable. The check only ever fired in the window before the watermark advanced. -// -// What still holds: an uncovered block at or above the start is corruption and errors — see -// TestLittblockIteratorUncoveredBlockIsCorruption. And finding 2's actual hazard does not return, -// because this iterator is not empty: loadFromBlockDB replays 5..7 rather than seeing an empty -// scan and silently restarting from committee genesis. -func TestLittblockIteratorDoesNotServeBlockBelowCoverage(t *testing.T) { - dir := t.TempDir() - rng := utils.TestRngFromSeed(9) - - db, err := NewBlockDB(strandingConfig(t, dir, 1024)) - require.NoError(t, err) - defer func() { _ = db.Close() }() - impl := db.(*blockDB) - - // A block at 0 injected past WriteBlock's coverage check while no QC exists at all, so the - // record precedes every QC in insertion order. - require.NoError(t, impl.table.Put(blockKey(0), encodeBlock(0, types.GenBlock(rng)))) - - // A well-formed cohort above it, written normally. This sets oldestQCStart to 5. - require.NoError(t, db.WriteQC(types.GenFullCommitQCRange(rng, 5, 8))) - for n := types.GlobalBlockNumber(5); n < 8; n++ { - require.NoError(t, db.WriteBlock(n, types.GenBlock(rng))) - } - - it, err := db.Iterator(0) - require.NoError(t, err) - defer func() { _ = it.Close() }() - - var got []types.GlobalBlockNumber - for { - pos, ok, err := it.Next() - require.NoError(t, err, "the uncovered block below the clamped start must not be visited") - if !ok { - break - } - got = append(got, pos.Number) - } - require.Equal(t, []types.GlobalBlockNumber{5, 6, 7}, got, - "Iterator(0) must clamp up to the first retained QC and yield only covered numbers") -} - -// TestLittblockIteratorEmptyStoreIsCleanExhaustion pins the negative of the coverage check: with no -// block record held, no retained QC means there is genuinely nothing to serve, which must stay a -// clean (false, nil) rather than becoming a corruption error. -func TestLittblockIteratorEmptyStoreIsCleanExhaustion(t *testing.T) { - db, err := NewBlockDB(strandingConfig(t, t.TempDir(), 1024)) - require.NoError(t, err) - defer func() { _ = db.Close() }() - - it, err := db.Iterator(0) - require.NoError(t, err) - defer func() { _ = it.Close() }() - - _, ok, err := it.Next() - require.NoError(t, err) - require.False(t, ok) -} diff --git a/sei-db/ledger_db/block/littblock/litt_block_simple_iterator.go b/sei-db/ledger_db/block/littblock/litt_block_simple_iterator.go deleted file mode 100644 index 04183c5a52..0000000000 --- a/sei-db/ledger_db/block/littblock/litt_block_simple_iterator.go +++ /dev/null @@ -1,129 +0,0 @@ -package littblock - -import ( - "fmt" - - "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" - "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" -) - -var _ types.BlockDBIterator = (*simpleIterator)(nil) - -// simpleIterator serves one narrow case: a store holding QCs but no blocks at all. That happens between -// the first WriteQC and the first WriteBlock, and after a crash that made a QC durable before any block. -// It yields every number those QCs cover, each with no block. -// -// It exists so blockDBIterator does not have to serve that case. That iterator is driven by a single -// forward scan of the shared ledger table, and this is the one case where the scan has no block to anchor -// its start on: its snapshot can pick up a first block written concurrently and report it as a gap below -// the cursor, since the first block may legally land anywhere inside its covering QC. Handling the case -// here is simpler than building a contract around that behaviour — this iterator takes no table snapshot, -// so there is nothing to race, and the scanning path keeps its plain invariant that iteration starts on a -// block that exists. -// -// Every position is read and decoded up front. Nothing is pinned: no table snapshot, no segment -// reservation, no open handle. Close exists only to satisfy the interface and leaking one costs nothing. -// The case is tiny by construction — no block has been written yet, so coverage spans at most the few -// cohorts a single persistence batch queued. -type simpleIterator struct { - // positions is every position this iterator will yield, ascending, built at construction. - positions []types.Position - - // idx is the current position; -1 before the first Next and len(positions) once exhausted. - idx int - - // closed is true once Close has been called. Block rejects calls made afterward. - closed bool -} - -// newSimpleIterator reads the QCs covering [start, nextQC) and materializes a position for every number -// they cover at or above start, each with no block. -// -// QCs are contiguous, so this walks cohort by cohort: one point read per QC, whose decoded value is -// shared by every position in its range. Sharing matters — types.Position documents QC as the same -// pointer across a range, and callers key "the scan entered a new QC" on that identity. -func newSimpleIterator( - table qcReader, - start types.GlobalBlockNumber, - nextQC types.GlobalBlockNumber, - appQCs []*coveredAppQC, -) (*simpleIterator, error) { - var positions []types.Position - for n := start; n < nextQC; { - qc, err := readQCCovering(table, n) - if err != nil { - return nil, err - } - first, next := coveredRange(qc) - if next <= n { - // Would not advance; a QC whose range does not contain n means coverage is not the - // contiguous span the write path guarantees. - return nil, fmt.Errorf("corrupt store: QC at %d covers [%d,%d), which does not reach it", - n, first, next) - } - for m := n; m < next && m < nextQC; m++ { - appQC := appQCCovering(appQCs, m) - positions = append(positions, types.Position{ - Number: m, - QC: qc, - HasBlock: false, - AppQC: appQC, - HasAppQC: appQC != nil, - }) - } - n = next - } - return &simpleIterator{positions: positions, idx: -1}, nil -} - -func (s *simpleIterator) Next() (types.Position, bool, error) { - if s.idx < len(s.positions) { - s.idx++ - } - if s.closed || s.idx < 0 || s.idx >= len(s.positions) { - return types.Position{}, false, nil - } - return s.positions[s.idx], true, nil -} - -// Block always reports absence: this iterator only ever covers numbers whose blocks have not been -// written, so every position it yields has HasBlock false. -func (s *simpleIterator) Block() (utils.Option[*types.Block], error) { - if s.closed { - return utils.None[*types.Block](), fmt.Errorf("iterator is closed") - } - if s.idx < 0 || s.idx >= len(s.positions) { - return utils.None[*types.Block](), fmt.Errorf("iterator is not positioned on a block number") - } - return utils.None[*types.Block](), nil -} - -// Close releases nothing — see the type doc. It only marks the iterator unusable so misuse after close -// is reported rather than silently answered. -func (s *simpleIterator) Close() error { - s.closed = true - return nil -} - -// qcReader is the slice of littdb.Table that newSimpleIterator needs, so tests can supply QCs without -// building a table. -type qcReader interface { - Get(key []byte) ([]byte, bool, error) -} - -// readQCCovering point-reads and decodes the QC covering n. Every covered number carries a QC alias key -// holding the full QC value, so any number inside a retained range resolves. -func readQCCovering(table qcReader, n types.GlobalBlockNumber) (*types.FullCommitQC, error) { - value, exists, err := table.Get(qcKey(n)) - if err != nil { - return nil, fmt.Errorf("failed to read covering QC for %d: %w", n, err) - } - if !exists { - return nil, fmt.Errorf("corrupt store: no QC record at %d despite coverage past it", n) - } - qc, err := decodeQC(value) - if err != nil { - return nil, fmt.Errorf("failed to decode covering QC for %d: %w", n, err) - } - return qc, nil -} diff --git a/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go b/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go index dad0065f8f..435ef0075e 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go +++ b/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go @@ -120,23 +120,16 @@ func TestLittblockStrandedBlockNotServedAfterRestart(t *testing.T) { require.True(t, qc.IsPresent(), "covering QC for served block %d must be readable", n) } - // The ledger never yields a stranded position, and every yielded position - // has a covering QC. - it, err := db3.Iterator(0) + // Recent recovery data never includes stranded blocks, and every returned + // block has a covering QC. + recent, err := db3.ReadRecent() require.NoError(t, err) - defer func() { _ = it.Close() }() - for { - pos, ok, err := it.Next() - require.NoError(t, err) - if !ok { - break - } - n := pos.Number - require.GreaterOrEqual(t, uint64(n), uint64(5), "ledger must not yield stranded position %d", n) - require.NotNil(t, pos.QC, "position %d must have a covering QC", n) - blkOpt, err := it.Block() + for _, block := range recent.Blocks { + n := block.Number + require.GreaterOrEqual(t, uint64(n), uint64(5), "recent data must not include stranded block %d", n) + qc, err := db3.ReadQCByBlockNumber(n) require.NoError(t, err) - require.True(t, blkOpt.IsPresent(), "position %d must have a block", n) + require.True(t, qc.IsPresent(), "block %d must have a covering QC", n) } } diff --git a/sei-db/ledger_db/block/littblock_crash_test.go b/sei-db/ledger_db/block/littblock_crash_test.go index 4bdba321f9..59e088bbfe 100644 --- a/sei-db/ledger_db/block/littblock_crash_test.go +++ b/sei-db/ledger_db/block/littblock_crash_test.go @@ -71,27 +71,15 @@ func TestLittblockNoBlockWithoutQCAfterTornTail(t *testing.T) { totalBlocks += len(b.blocks) } - it, err := db2.Iterator(0) + recent, err := db2.ReadRecent() require.NoError(t, err) - defer func() { _ = it.Close() }() present := 0 - for { - pos, ok, err := it.Next() - require.NoError(t, err) - if !ok { - break - } - // pos.QC being non-nil at every position (guaranteed by the iterator - // contract) is the covering-QC invariant; cross-check via the point-read - // path too. - n := pos.Number - require.NotNil(t, pos.QC, "position %d has no covering QC", n) + for _, block := range recent.Blocks { + n := block.Number qc, err := db2.ReadQCByBlockNumber(n) require.NoError(t, err) - require.True(t, qc.IsPresent(), "position %d survived but its covering QC was lost", n) - if pos.HasBlock { - present++ - } + require.True(t, qc.IsPresent(), "block %d survived but its covering QC was lost", n) + present++ } // The truncation must have actually dropped at least one block, otherwise the @@ -225,24 +213,15 @@ func TestLittblockFlushSurvivesHardKill(t *testing.T) { totalBlocks += len(b.blocks) } - it, err := db.Iterator(0) + recent, err := db.ReadRecent() require.NoError(t, err) - defer func() { _ = it.Close() }() present := 0 - for { - pos, ok, err := it.Next() - require.NoError(t, err) - if !ok { - break - } - n := pos.Number - require.NotNil(t, pos.QC, "position %d has no covering QC", n) + for _, block := range recent.Blocks { + n := block.Number qc, err := db.ReadQCByBlockNumber(n) require.NoError(t, err) - require.True(t, qc.IsPresent(), "position %d lost its covering QC after hard kill", n) - if pos.HasBlock { - present++ - } + require.True(t, qc.IsPresent(), "block %d lost its covering QC after hard kill", n) + present++ } // Unlike the torn-tail test (which expects loss), a clean Flush before the kill diff --git a/sei-db/ledger_db/block/memblock/mem_block_db.go b/sei-db/ledger_db/block/memblock/mem_block_db.go index 1ae8188679..36dcc434cd 100644 --- a/sei-db/ledger_db/block/memblock/mem_block_db.go +++ b/sei-db/ledger_db/block/memblock/mem_block_db.go @@ -62,10 +62,7 @@ type blockDB struct { // AppQC cohort remains readable together with its CommitQC and blocks. latestAppQCStartBlock types.GlobalBlockNumber - // firstBlockNumber is the lowest block number written. Iterator clamps its start up to - // it so a scan always opens on a block that exists; the first block may land anywhere - // inside its covering QC, so this can sit above that QC's start with no block in - // between. Meaningful only while hasBlocks. + // firstBlockNumber is the lowest block number written. Meaningful only while hasBlocks. firstBlockNumber types.GlobalBlockNumber // watermark is the (clamped) retention floor set by PruneBefore. Reads @@ -245,25 +242,36 @@ func (s *blockDB) Status() types.DBStatus { return tips } -func (s *blockDB) Iterator(n types.GlobalBlockNumber) (types.BlockDBIterator, error) { +func (s *blockDB) ReadRecent() (types.RecentData, error) { s.mu.RLock() defer s.mu.RUnlock() - // Clamp up to the lowest number this store can serve: the retention gate, and the start of - // the block history so a scan opens on a block that exists rather than on blockless numbers - // below it (the first block may land inside its covering QC's range). With no block at all - // the per-entry clamp in iteratorLocked governs, so a QC written ahead of its blocks is - // still iterable. - start := max(n, s.watermark) - if s.hasBlocks { - start = max(start, s.firstBlockNumber) + var recent types.RecentData + floor := s.watermark + var targetIndex types.RoadIndex + if s.hasAppQC { + appQC := s.appQCs[s.latestAppQCStartBlock].appQC + recent.AppQC = utils.Some(appQC) + floor = max(floor, s.latestAppQCStartBlock) + targetIndex = appQC.Proposal().RoadIndex() + } + + for _, e := range s.sortedQCsLocked() { + if e.upper <= s.watermark { + continue + } + if s.hasAppQC && e.qc.Index() < targetIndex { + continue + } + recent.CommitQCs = append(recent.CommitQCs, e.qc) } - entries := s.sortedQCsLocked() - if len(entries) == 0 || start >= entries[len(entries)-1].upper { - // Nothing is covered at or above the (clamped) start. - return &memBlockDBIterator{idx: -1}, nil + for _, n := range s.sortedBlockNumbersLocked() { + if n < floor { + continue + } + recent.Blocks = append(recent.Blocks, types.RecentBlock{Number: n, Block: s.byNumber[n]}) } - return s.iteratorLocked(entries, start), nil + return recent, nil } // sortedQCsLocked returns the retained QC entries ascending by lower bound. Caller holds mu. @@ -276,27 +284,6 @@ func (s *blockDB) sortedQCsLocked() []qcEntry { return entries } -// iteratorLocked snapshots every covered number from start upward (clamping up to the first -// covered number when start falls below all coverage), pairing each with its covering QC and -// (possibly absent) block. Caller holds mu and guarantees some entry's range ends above start. -// -// Copying the whole range up front is how this store satisfies the iterator's snapshot -// guarantee: the records live in maps that later writes and prunes mutate in place, so a -// lazy walk would observe them. Residency is not something BlockDB.Iterator promises, and a -// store that already holds every record in memory has nothing to stream anyway. -func (s *blockDB) iteratorLocked(entries []qcEntry, start types.GlobalBlockNumber) *memBlockDBIterator { - it := &memBlockDBIterator{idx: -1} - for _, e := range entries { - for num := max(e.lower, start); num < e.upper; num++ { - it.nums = append(it.nums, num) - it.qcs = append(it.qcs, e.qc) - it.blocks = append(it.blocks, s.byNumber[num]) - it.appQCs = append(it.appQCs, s.appQCCoveringLocked(num)) - } - } - return it -} - func (s *blockDB) appQCCoveringLocked(n types.GlobalBlockNumber) *types.AppQC { for _, e := range s.appQCs { if e.lower <= n && n < e.upper { @@ -306,64 +293,13 @@ func (s *blockDB) appQCCoveringLocked(n types.GlobalBlockNumber) *types.AppQC { return nil } -var _ types.BlockDBIterator = (*memBlockDBIterator)(nil) - -// memBlockDBIterator steps through a snapshot of covered numbers captured at creation. -type memBlockDBIterator struct { - // nums holds every covered number, ascending. - nums []types.GlobalBlockNumber - - // qcs holds the covering QC per position. - qcs []*types.FullCommitQC - - // blocks holds the block per position; nil where no block is persisted. - blocks []*types.Block - - // appQCs holds the AppQC per position; nil where no AppQC is persisted. - appQCs []*types.AppQC - - // idx is the current position; -1 before the first Next and len(nums) once exhausted. - idx int - - // closed is true once Close has been called. Block rejects calls made afterward. - closed bool -} - -func (it *memBlockDBIterator) Next() (types.Position, bool, error) { - if it.idx < len(it.nums) { - it.idx++ - } - if !it.positioned() { - return types.Position{}, false, nil +func (s *blockDB) sortedBlockNumbersLocked() []types.GlobalBlockNumber { + nums := make([]types.GlobalBlockNumber, 0, len(s.byNumber)) + for n := range s.byNumber { + nums = append(nums, n) } - return types.Position{ - Number: it.nums[it.idx], - QC: it.qcs[it.idx], - HasBlock: it.blocks[it.idx] != nil, - AppQC: it.appQCs[it.idx], - HasAppQC: it.appQCs[it.idx] != nil, - }, true, nil -} - -func (it *memBlockDBIterator) Block() (utils.Option[*types.Block], error) { - if !it.positioned() { - return utils.None[*types.Block](), fmt.Errorf("iterator is not positioned on a block number") - } - if it.blocks[it.idx] == nil { - return utils.None[*types.Block](), nil - } - return utils.Some(it.blocks[it.idx]), nil -} - -func (it *memBlockDBIterator) Close() error { - // Mirrors littblock: a closed iterator holds no position, so Block reports misuse. - it.closed = true - return nil -} - -// positioned reports whether the iterator sits on a number Next yielded. -func (it *memBlockDBIterator) positioned() bool { - return !it.closed && it.idx >= 0 && it.idx < len(it.nums) + sort.Slice(nums, func(i, j int) bool { return nums[i] < nums[j] }) + return nums } func (s *blockDB) ReadBlockByNumber( diff --git a/sei-tendermint/autobahn/types/block_db.go b/sei-tendermint/autobahn/types/block_db.go index d3ec594126..2206355c99 100644 --- a/sei-tendermint/autobahn/types/block_db.go +++ b/sei-tendermint/autobahn/types/block_db.go @@ -74,9 +74,7 @@ type BlockDB interface { // number (the first block may start anywhere its covering QC allows); // otherwise WriteBlock returns ErrBlockOutOfOrder and persists // nothing. Writes are NOT idempotent — re-writing the same (or any - // other non-contiguous) n is rejected with an error. Density is what - // lets BlockDBIterator.Block treat an absent block below the highest - // persisted one as corruption. + // other non-contiguous) n is rejected with an error. // // May return before the block is on disk. Callers that need crash // durability before some external observable action (e.g. @@ -178,43 +176,18 @@ type BlockDB interface { // Status returns a consistent snapshot of the in-memory write tips (no I/O). Status() DBStatus - // Iterator returns an iterator positioned at block number n. Iteration - // is forward-only: it steps through consecutive numbers up to the last - // persisted QC's coverage, exclusive. The start is clamped up to the - // lowest number the store can serve — the retention watermark, the first - // retained block, or the first persisted QC's range on a store holding no - // block at all — so Iterator(0) scans everything retained (startup replay) - // while a mid-history n resumes from that height without scanning what - // lies below it. See BlockDBIterator for what each position exposes. + // ReadRecent returns the materialized startup-recovery suffix. // - // Clamping to the first retained block is what makes a scan open on a - // number that has one: WriteBlock lets the first block start anywhere - // inside its covering QC, and the numbers below it were never written, so - // they are not part of the iteration. + // Implementations find the newest persisted AppQC, then collect all + // CommitQCs whose Index is greater than or equal to that AppQC's + // RoadIndex, and all blocks whose GlobalBlockNumber is greater than or + // equal to that AppQC's GlobalRange.First. If no AppQC is present, + // ReadRecent returns all retained CommitQCs and blocks. // - // If the (clamped) start is past the last persisted QC's coverage — - // including on an empty store — the iterator is empty (Next - // immediately returns false). - // - // Returns ErrPruned if a concurrent PruneBefore advances the retention - // floor past the clamped start before the iterator can be positioned. - // Racing a pruner has no deterministic answer — the floor may move - // again before the call returns — so the failure is reported rather - // than papered over, and a caller that still wants whatever is retained - // may simply call again. Distinct from the corruption error a genuinely - // missing record produces. - // - // A caller may walk an arbitrarily large retention window, and pays to - // read a block's value only where it calls Block — Number, QC and - // HasBlock come off Position for free (see BlockDBIterator). How much - // an implementation holds resident while scanning is its own affair - // and is not promised here. - // - // The iterator captures a snapshot of the records present when it is - // created; records written afterward are not observed. It is NOT safe - // for concurrent use and MUST be closed when no longer needed (see - // BlockDBIterator.Close). - Iterator(n GlobalBlockNumber) (BlockDBIterator, error) + // Returned CommitQCs and Blocks are in ascending GlobalBlockNumber order so + // data.State can replay them directly. If AppQC is present, CommitQCs + // starts with its matching CommitQC. + ReadRecent() (RecentData, error) // ReadBlockByNumber returns the block at GlobalBlockNumber n. // @@ -298,77 +271,15 @@ type DBStatus struct { NextAppQC GlobalBlockNumber } -// BlockDBIterator steps through consecutive GlobalBlockNumbers in ascending -// order, exposing at each position the covering QC (always present) and the -// block (present unless it did not survive). It is created via BlockDB.Iterator -// and captures a snapshot of the records present at creation time. -// -// The numbers yielded are exactly those covered by a retained QC, so a single -// pass observes every retained QC (via QC, which changes when the scan crosses -// a range boundary) and every retained block — including QCs written ahead of -// their blocks, which appear as trailing positions where Block returns None. -// -// A BlockDBIterator is NOT safe for concurrent use by multiple goroutines. -type BlockDBIterator interface { - // Next advances the iterator and returns the position it advanced to. ok - // is false when the iteration is complete (no number covered by a - // retained QC remains), and Position is then the zero value. It returns - // an error if advancing failed or the store is corrupt (a block missing - // below the highest persisted block — writes are dense, so a gap can - // only be corruption). After Next returns ok == false iteration is - // complete; after it returns an error the iterator must not be used - // further (other than Close). - // - // The corruption clause binds only implementations that can reach a - // corrupt state — durable ones, where a torn write, an out-of-band file - // removal or a truncated index can produce records the write path would - // have rejected. An implementation holding its records in memory cannot - // reach those states at all: the write-order guards above are the only - // way records enter it. Such an implementation satisfies this clause - // vacuously and correctly never returns an error. - Next() (pos Position, ok bool, err error) - - // Block reads and returns the block at the position most recently - // returned by Next, or None if no block is persisted there — - // equivalently, None exactly when that Position's HasBlock is false. - // - // This is the one call that may perform IO, which is why it is not a - // Position field: a caller that only needs numbers, QCs or presence - // never pays for it. Calling it without a preceding Next that returned - // ok == true, or after Close, returns an error. - Block() (utils.Option[*Block], error) - - // Close releases the resources held by the iterator. MUST be called when - // done; failure to close may leak resources in disk-backed - // implementations. - Close() error -} - -// Position is the record at one BlockDBIterator position. Every field is cheap -// — populating a Position performs no IO — so a caller can scan positions and -// materialize only the blocks it wants via BlockDBIterator.Block. -type Position struct { - // Number is the GlobalBlockNumber this position covers. +// RecentBlock is one block returned by BlockDB.ReadRecent. +type RecentBlock struct { Number GlobalBlockNumber + Block *Block +} - // QC is the FullCommitQC covering Number: its GlobalRange contains - // Number. Never nil — every yielded number is covered by construction — - // and the same pointer is returned for every position in its range. The - // value is decoded once per QC, not once per number. - QC *FullCommitQC - - // HasBlock reports whether a block is persisted at Number, and so - // whether BlockDBIterator.Block will return Some. Because QCs are - // written before the blocks they cover and blocks are written densely, - // it is false only in the trailing positions of the iteration: numbers - // whose covering QC was persisted but whose block was not (e.g. lost in - // a crash, or not yet written). - HasBlock bool - - // AppQC is the AppQC covering Number, if one has been persisted. It is nil - // when no AppQC covers Number. - AppQC *AppQC - - // HasAppQC reports whether AppQC is present at Number. - HasAppQC bool +// RecentData is the materialized suffix used by data.State startup recovery. +type RecentData struct { + CommitQCs []*FullCommitQC + Blocks []RecentBlock + AppQC utils.Option[*AppQC] } diff --git a/sei-tendermint/autobahn/types/errors.go b/sei-tendermint/autobahn/types/errors.go index fdade4a772..4009e10080 100644 --- a/sei-tendermint/autobahn/types/errors.go +++ b/sei-tendermint/autobahn/types/errors.go @@ -8,9 +8,8 @@ import "errors" // means the height is below the retention / eviction floor. var ErrNotFound = errors.New("not found") -// ErrBlockGap is returned when the persisted blocks are not contiguous, -// surfaced by BlockDBIterator.Next during a scan. WriteBlock rejects gapped -// writes, so a gap on disk indicates store corruption. +// ErrBlockGap is returned when persisted blocks are not contiguous. WriteBlock +// rejects gapped writes, so a gap on disk indicates store corruption. var ErrBlockGap = errors.New("block gap in BlockDB") // ErrBlockOutOfOrder is returned by WriteBlock when the supplied diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index 1ebeb51f6b..b0fbe8786e 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -82,6 +82,7 @@ func newInner(first types.GlobalBlockNumber) *inner { nextBlockToPersist: first, nextBlock: first, nextQC: first, + anchor: utils.NewAtomicSend(utils.None[Anchor]()), } } @@ -207,68 +208,55 @@ func NewState(cfg *Config, blockDB types.BlockDB) (*State, error) { }, nil } -// loadFromBlockDB replays QCs and blocks from blockDB into s.inner. +// loadFromBlockDB replays the recent persisted suffix from blockDB into s.inner. // Called from NewState before any goroutines are spawned. -// -// Recovery starts at the app tip so runExecute can replay its AppHash. If the -// app tip equals BlockDB's next block, recovery starts at the last stored block: -// app.Commit may finish before BlockDB is durable. A larger gap violates the -// PushAppHash durability invariant. An empty BlockDB allows only no app tip or -// the first committed block. -// -// Without an app tip, recovery starts at the registry's first block. -// BlockDB.Iterator clamps the start to its retained floor. skipTo uses the first -// returned position, even inside a QC, to keep blocks dense over -// [first, nextBlock). -// -// Each iterator position has its covering QC and an optional block. Missing -// blocks are allowed only at the tail. BlockDB enforces other consistency; this -// method only rejects a first QC before committee genesis. func loadFromBlockDB(cfg *Config, blockDB types.BlockDB) (*inner, error) { firstBlock := cfg.Registry.FirstBlock() status := blockDB.Status() - status.NextQC = max(status.NextQC, firstBlock) - status.NextBlock = max(status.NextBlock, firstBlock) - status.NextAppQC = max(status.NextAppQC, firstBlock) - first := status.NextAppQC - inner := newInner(first) - it, err := blockDB.Iterator(first) + + recent, err := blockDB.ReadRecent() if err != nil { - return nil, fmt.Errorf("open block db iterator: %w", err) + return nil, fmt.Errorf("blockDB.ReadRecent(): %w", err) } - defer func() { _ = it.Close() }() - for { - pos, ok, err := it.Next() - if err != nil { - return nil, fmt.Errorf("advance block db iterator: %w", err) - } - if !ok { - break - } - if err := inner.insertQC(cfg.Registry, pos.QC); err != nil { + + first := firstBlock + if appQC, ok := recent.AppQC.Get(); ok { + first = appQC.Proposal().GlobalRange().First + } else if len(recent.Blocks) > 0 { + first = recent.Blocks[0].Number + } else if len(recent.CommitQCs) > 0 { + first = recent.CommitQCs[0].QC().GlobalRange().First + } + if first < firstBlock { + return nil, fmt.Errorf("db contains data before genesis") + } + status.NextQC = max(status.NextQC, first) + status.NextBlock = max(status.NextBlock, first) + status.NextAppQC = max(status.NextAppQC, first) + + inner := newInner(first) + for _, qc := range recent.CommitQCs { + if err := inner.insertQC(cfg.Registry, qc); err != nil { return nil, fmt.Errorf("load QC from BlockDB: %w", err) } - b, err := it.Block() - if err != nil { - return nil, fmt.Errorf("read block %d from BlockDB: %w", pos.Number, err) + } + if appQC, ok := recent.AppQC.Get(); ok { + if err := inner.insertAppQC(cfg.Registry, appQC); err != nil { + return nil, fmt.Errorf("load AppQC from BlockDB: %w", err) } - if b, ok := b.Get(); ok { - ei := pos.QC.QC().Proposal().EpochIndex() - e, ok := cfg.Registry.EpochByIndex(ei) - if !ok { - return nil, fmt.Errorf("unknown epoch_index %d", ei) - } - if err := b.Verify(e.Committee()); err != nil { - return nil, fmt.Errorf("verify block %d from BlockDB: %w", pos.Number, err) - } - if err := inner.insertBlock(pos.Number, b); err != nil { - return nil, fmt.Errorf("insert block %d from BlockDB: %w", pos.Number, err) - } + } + for _, b := range recent.Blocks { + qc := inner.qcs[b.Number] + ei := qc.QC().Proposal().EpochIndex() + e, ok := cfg.Registry.EpochByIndex(ei) + if !ok { + return nil, fmt.Errorf("unknown epoch_index %d", ei) } - if pos.HasAppQC { - if err := inner.insertAppQC(cfg.Registry, pos.AppQC); err != nil { - return nil, fmt.Errorf("load AppQC from BlockDB: %w", err) - } + if err := b.Block.Verify(e.Committee()); err != nil { + return nil, fmt.Errorf("verify block %d from BlockDB: %w", b.Number, err) + } + if err := inner.insertBlock(b.Number, b.Block); err != nil { + return nil, fmt.Errorf("insert block %d from BlockDB: %w", b.Number, err) } } // Advance nextBlock through contiguous loaded blocks. Don't use @@ -704,6 +692,17 @@ func (s *State) Anchor() utils.AtomicRecv[utils.Option[Anchor]] { panic("unreachable") } +func (s *State) LastAppQC() (*types.AppQC, *types.FullCommitQC) { + for inner := range s.inner.Lock() { + if inner.nextAppQC <= inner.first { + return nil, nil + } + n := inner.nextAppQC - 1 + return inner.appQCs[n], inner.qcs[n] + } + panic("unreachable") +} + func (i *inner) nextToExecute(lane types.LaneID) types.BlockNumber { // TODO(gprusak): decide whether 0 is a good result in this case in general. // Empty maps (first == nextQC) only on fresh start / after skipTo with no QC. @@ -774,6 +773,8 @@ func (s *State) runPersist(ctx context.Context) error { }); err != nil { return err } + nextBlock = inner.nextBlockToPersist + nextAppQC = inner.nextAppQCToPersist for nextBlock < inner.nextBlock { qc := inner.qcs[nextBlock] if nextBlock == qc.QC().GlobalRange().First { diff --git a/sei-tendermint/internal/autobahn/data/state_recovery_test.go b/sei-tendermint/internal/autobahn/data/state_recovery_test.go index df210639c6..3755a25199 100644 --- a/sei-tendermint/internal/autobahn/data/state_recovery_test.go +++ b/sei-tendermint/internal/autobahn/data/state_recovery_test.go @@ -14,16 +14,6 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/scope" ) -type recoveryStartBlockDB struct { - types.BlockDB - start types.GlobalBlockNumber -} - -func (db *recoveryStartBlockDB) Iterator(n types.GlobalBlockNumber) (types.BlockDBIterator, error) { - db.start = n - return db.BlockDB.Iterator(n) -} - // TestRecoveryEmpty verifies that NewState is a no-op on a fresh BlockDB. func TestRecoveryEmpty(t *testing.T) { rng := utils.TestRng() @@ -113,10 +103,11 @@ func TestRecoveryStartsAtLastExecutedBlock(t *testing.T) { registry, keys := epoch.GenRegistry(rng, 3) qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) qc2, blocks2 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc1.QC())) + gr1 := qc1.QC().GlobalRange() gr2 := qc2.QC().GlobalRange() require.Greater(t, gr2.Len(), 2) - db := &recoveryStartBlockDB{BlockDB: newTestBlockDB(t, t.TempDir())} + db := newTestBlockDB(t, t.TempDir()) writeToBlockDB(t, db, []*types.FullCommitQC{qc1, qc2}, [][]*types.Block{blocks1, blocks2}) @@ -127,26 +118,13 @@ func TestRecoveryStartsAtLastExecutedBlock(t *testing.T) { LastExecutedBlock: utils.Some(lastExecuted), }, db) - require.Equal(t, lastExecuted, db.start) for inner := range state.inner.Lock() { - require.Equal(t, lastExecuted, inner.nextAppProposal) + require.Equal(t, gr1.First, inner.nextAppProposal) } require.Equal(t, gr2.Next, state.NextBlock()) got, err := state.TryBlock(lastExecuted) require.NoError(t, err) require.Equal(t, blocks2[0].Header().Hash(), got.Header().Hash()) - - appHash := types.GenAppHash(rng) - for n := gr2.First; n < gr2.Next; n++ { - hash := types.GenAppHash(rng) - if n == gr2.Next-1 { - hash = appHash - } - require.NoError(t, state.PushAppHash(t.Context(), n, hash)) - } - appVote, _, err := state.AppVote(t.Context(), lastExecuted) - require.NoError(t, err) - require.Equal(t, appHash, appVote.Proposal().AppHash()) } func TestRecoveryCapsAppTipAtLastBlockInBlockDB(t *testing.T) { @@ -154,7 +132,6 @@ func TestRecoveryCapsAppTipAtLastBlockInBlockDB(t *testing.T) { registry, keys := epoch.GenRegistry(rng, 3) qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) qc2, blocks2 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc1.QC())) - gr1 := qc1.QC().GlobalRange() gr2 := qc2.QC().GlobalRange() dir := t.TempDir() @@ -162,14 +139,13 @@ func TestRecoveryCapsAppTipAtLastBlockInBlockDB(t *testing.T) { writeToBlockDB(t, db1, []*types.FullCommitQC{qc1}, [][]*types.Block{blocks1}) require.NoError(t, db1.Close()) - db := &recoveryStartBlockDB{BlockDB: newTestBlockDB(t, dir)} + db := newTestBlockDB(t, dir) state, err := NewState(&Config{ Registry: registry, LastExecutedBlock: utils.Some(gr2.First), }, db) require.NoError(t, err) - require.Equal(t, gr1.Next-1, db.start) require.Equal(t, gr2.First, state.NextBlock()) require.NoError(t, state.PushQC(t.Context(), qc2, blocks2)) @@ -191,11 +167,12 @@ func TestRecoveryRejectsAppTipBeyondCrashWindow(t *testing.T) { dbNextBlock := db.Status().NextBlock lastExecuted := dbNextBlock + 1 - _, err := NewState(&Config{ + state, err := NewState(&Config{ Registry: registry, LastExecutedBlock: utils.Some(lastExecuted), }, db) - require.ErrorIs(t, err, types.ErrNotFound) + require.NoError(t, err) + require.Equal(t, dbNextBlock, state.NextBlock()) } func TestRecoveryStartsAtRegistryFloorWhenBlockDBMissingFirstCommittedBlock(t *testing.T) { @@ -204,7 +181,7 @@ func TestRecoveryStartsAtRegistryFloorWhenBlockDBMissingFirstCommittedBlock(t *t qc, _ := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) gr := qc.QC().GlobalRange() - db := &recoveryStartBlockDB{BlockDB: newTestBlockDB(t, t.TempDir())} + db := newTestBlockDB(t, t.TempDir()) require.NoError(t, db.WriteQC(qc)) require.NoError(t, db.Flush()) @@ -213,7 +190,6 @@ func TestRecoveryStartsAtRegistryFloorWhenBlockDBMissingFirstCommittedBlock(t *t LastExecutedBlock: utils.Some(gr.First), }, db) require.NoError(t, err) - require.Equal(t, registry.FirstBlock(), db.start) require.Equal(t, gr.First, state.NextBlock()) } @@ -222,11 +198,12 @@ func TestRecoveryRejectsEmptyBlockDBAfterFirstCommittedBlock(t *testing.T) { registry, _ := epoch.GenRegistry(rng, 3) lastExecuted := registry.FirstBlock() + 1 - _, err := NewState(&Config{ + state, err := NewState(&Config{ Registry: registry, LastExecutedBlock: utils.Some(lastExecuted), }, newTestBlockDB(t, t.TempDir())) - require.ErrorIs(t, err, types.ErrNotFound) + require.NoError(t, err) + require.Equal(t, registry.FirstBlock(), state.NextBlock()) } func TestRecoveryLeavesAppTipBelowPruneFloorUnreadable(t *testing.T) { @@ -561,10 +538,7 @@ func TestRunPersistSeedsFromRecoveryFloor(t *testing.T) { // TestRecoveryBlockGap verifies that a block gap can never enter BlockDB in the // first place: WriteBlock enforces contiguity, so skipping a covered number is -// rejected at write time. (A gap on disk can therefore only be corruption, which -// the ledger iterator reports as ErrBlockGap during replay — pinned by -// littblock's TestLittblockIteratorGapIsCorruption — and loadFromBlockDB -// propagates.) +// rejected at write time. func TestRecoveryBlockGap(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 3) From 566d1106852e16381d366988472718c3ded09bf5 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 6 Aug 2026 16:51:39 +0200 Subject: [PATCH 13/61] addresses codex review --- sei-db/ledger_db/block/littblock/qc_reader.go | 31 ++++++++++++++ .../internal/autobahn/data/state.go | 41 +++++++++---------- 2 files changed, 51 insertions(+), 21 deletions(-) create mode 100644 sei-db/ledger_db/block/littblock/qc_reader.go diff --git a/sei-db/ledger_db/block/littblock/qc_reader.go b/sei-db/ledger_db/block/littblock/qc_reader.go new file mode 100644 index 0000000000..58e08f62a3 --- /dev/null +++ b/sei-db/ledger_db/block/littblock/qc_reader.go @@ -0,0 +1,31 @@ +package littblock + +import ( + "fmt" + + "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" +) + +// qcReader is the slice of littdb.Table that readQCCovering needs, so tests can +// supply QCs without building a table. +type qcReader interface { + Get(key []byte) ([]byte, bool, error) +} + +// readQCCovering point-reads and decodes the QC covering n. Every covered +// number carries a QC alias key holding the full QC value, so any number inside +// a retained range resolves. +func readQCCovering(table qcReader, n types.GlobalBlockNumber) (*types.FullCommitQC, error) { + value, exists, err := table.Get(qcKey(n)) + if err != nil { + return nil, fmt.Errorf("failed to read covering QC for %d: %w", n, err) + } + if !exists { + return nil, fmt.Errorf("corrupt store: no QC record at %d despite coverage past it", n) + } + qc, err := decodeQC(value) + if err != nil { + return nil, fmt.Errorf("failed to decode covering QC for %d: %w", n, err) + } + return qc, nil +} diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index b0fbe8786e..702be07153 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -761,32 +761,31 @@ func (s *State) PruneBefore(retainFrom types.GlobalBlockNumber) error { // PushAppQC (evictBelowBound); AppQC entries are retained until their own // persistence cursor catches up. func (s *State) runPersist(ctx context.Context) error { + status := s.blockDB.Status() for { var qcs []*types.FullCommitQC var blocks []blockEntry var appQCs []*types.AppQC - var nextBlock types.GlobalBlockNumber - var nextAppQC types.GlobalBlockNumber for inner, ctrl := range s.inner.Lock() { if err := ctrl.WaitUntil(ctx, func() bool { - return inner.nextBlockToPersist < inner.nextBlock || inner.nextAppQCToPersist < inner.nextAppQC + return status.NextQC < inner.nextQC || status.NextBlock < inner.nextBlock || status.NextAppQC < inner.nextAppQC }); err != nil { return err } - nextBlock = inner.nextBlockToPersist - nextAppQC = inner.nextAppQCToPersist - for nextBlock < inner.nextBlock { - qc := inner.qcs[nextBlock] - if nextBlock == qc.QC().GlobalRange().First { - qcs = append(qcs, qc) - } - blocks = append(blocks, blockEntry{n: nextBlock, block: inner.blocks[nextBlock]}) - nextBlock++ + for status.NextQC < inner.nextQC { + qc := inner.qcs[status.NextQC] + qcs = append(qcs, qc) + status.NextQC = qc.QC().GlobalRange().Next } - for nextAppQC < inner.nextAppQC { - appQC := inner.appQCs[nextAppQC] + + for status.NextAppQC < inner.nextAppQC { + appQC := inner.appQCs[status.NextAppQC] appQCs = append(appQCs, appQC) - nextAppQC = appQC.Proposal().GlobalRange().Next + status.NextAppQC = appQC.Proposal().GlobalRange().Next + } + for status.NextBlock < inner.nextBlock { + blocks = append(blocks, blockEntry{n: status.NextBlock, block: inner.blocks[status.NextBlock]}) + status.NextBlock += 1 } } // Write QCs first (BlockDB contract: QC must precede covered blocks). @@ -809,15 +808,15 @@ func (s *State) runPersist(ctx context.Context) error { return fmt.Errorf("flush BlockDB: %w", err) } for inner, ctrl := range s.inner.Lock() { - inner.nextBlockToPersist = nextBlock - if inner.nextAppQCToPersist < nextAppQC { - inner.nextAppQCToPersist = nextAppQC + inner.nextBlockToPersist = status.NextBlock + if inner.nextAppQCToPersist < status.NextAppQC { + inner.nextAppQCToPersist = status.NextAppQC inner.anchor.Store(utils.Some(Anchor{ - CommitQC: inner.qcs[nextAppQC-1].QC(), - AppQC: inner.appQCs[nextAppQC-1], + CommitQC: inner.qcs[status.NextAppQC-1].QC(), + AppQC: inner.appQCs[status.NextAppQC-1], })) + inner.evict() } - inner.evict() ctrl.Updated() } } From 9c12d282c017abbe3a4a78193611a8588f7ebee3 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 6 Aug 2026 17:33:08 +0200 Subject: [PATCH 14/61] fmt --- sei-tendermint/internal/autobahn/data/state.go | 11 +++++++++-- sei-tendermint/internal/p2p/giga/data.go | 6 +++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index 702be07153..28a9044538 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -762,22 +762,29 @@ func (s *State) PruneBefore(retainFrom types.GlobalBlockNumber) error { // persistence cursor catches up. func (s *State) runPersist(ctx context.Context) error { status := s.blockDB.Status() + // Account for empty blockDB. + for inner := range s.inner.Lock() { + status.NextQC = max(status.NextQC, inner.first) + status.NextAppQC = max(status.NextAppQC, inner.first) + status.NextBlock = max(status.NextBlock, inner.first) + } for { var qcs []*types.FullCommitQC var blocks []blockEntry var appQCs []*types.AppQC for inner, ctrl := range s.inner.Lock() { + // Wait until there is anythin to persist. if err := ctrl.WaitUntil(ctx, func() bool { return status.NextQC < inner.nextQC || status.NextBlock < inner.nextBlock || status.NextAppQC < inner.nextAppQC }); err != nil { return err } + // Collect data to persist. for status.NextQC < inner.nextQC { qc := inner.qcs[status.NextQC] qcs = append(qcs, qc) status.NextQC = qc.QC().GlobalRange().Next } - for status.NextAppQC < inner.nextAppQC { appQC := inner.appQCs[status.NextAppQC] appQCs = append(appQCs, appQC) @@ -788,7 +795,7 @@ func (s *State) runPersist(ctx context.Context) error { status.NextBlock += 1 } } - // Write QCs first (BlockDB contract: QC must precede covered blocks). + // Write QCs first (BlockDB contract: QC must precede covered blocks and AppQC). for _, qc := range qcs { if err := s.blockDB.WriteQC(qc); err != nil { return fmt.Errorf("write QC %d: %w", qc.QC().Index(), err) diff --git a/sei-tendermint/internal/p2p/giga/data.go b/sei-tendermint/internal/p2p/giga/data.go index f8e2ae61ec..7a7eecc6a6 100644 --- a/sei-tendermint/internal/p2p/giga/data.go +++ b/sei-tendermint/internal/p2p/giga/data.go @@ -60,7 +60,7 @@ func (x *Service) clientStreamAppQCs(ctx context.Context, c rpc.Client[API]) err if err != nil { return fmt.Errorf("StreamAppQCsRespConv.Decode(): %w", err) } - if err := x.data.PushAppQC(ctx,appQC); err != nil { + if err := x.data.PushAppQC(ctx, appQC); err != nil { return fmt.Errorf("s.PushFirstCommitQC(): %w", err) } } @@ -181,7 +181,7 @@ func (s *Service) serverStreamFullCommitQCs(ctx context.Context, server rpc.Serv if err != nil { return fmt.Errorf("StreamFullCommitQCsReqConv.Decode(): %w", err) } - for next := req.NextBlock;; { + for next := req.NextBlock; ; { qc, err := s.data.QC(ctx, next) if err != nil { return fmt.Errorf("s.data.QC(): %w", err) @@ -205,7 +205,7 @@ func (x *Service) serverStreamAppQCs(ctx context.Context, server rpc.Server[API] if err != nil { return fmt.Errorf("StreamFullCommitQCsReqConv.Decode(): %w", err) } - for next:=req.NextBlock;; { + for next := req.NextBlock; ; { appQC, commitQC, err := x.validatorState().Data().AppQC(ctx, next) if err != nil { return fmt.Errorf("x.validatorState().Avail().WaitForAppQC(): %w", err) From 5d195eea26c3c8607c67297518ae1fa1f8bebc42 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 6 Aug 2026 17:52:36 +0200 Subject: [PATCH 15/61] fixed data tests --- .../internal/autobahn/data/state_test.go | 187 ++++++++++-------- 1 file changed, 108 insertions(+), 79 deletions(-) diff --git a/sei-tendermint/internal/autobahn/data/state_test.go b/sei-tendermint/internal/autobahn/data/state_test.go index 7b3ae26142..64c3a022f6 100644 --- a/sei-tendermint/internal/autobahn/data/state_test.go +++ b/sei-tendermint/internal/autobahn/data/state_test.go @@ -495,57 +495,77 @@ func TestEvictionWaitsForAppQC(t *testing.T) { require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { runCtx, cancel := context.WithCancel(ctx) defer cancel() - s.SpawnBgNamed("state.Run", func() error { - return utils.IgnoreCancel(state.Run(runCtx)) - }) + s.SpawnBgNamed("state.Run", func() error { return utils.IgnoreCancel(state.Run(runCtx)) }) - require.NoError(t, state.PushQC(ctx, qc1, blocks1)) + if err := state.PushQC(ctx, qc1, blocks1); err != nil { + return fmt.Errorf("PushQC(qc1): %w", err) + } for n := gr1.First; n < gr1.Next; n++ { if err := state.PushAppHash(ctx, n, types.GenAppHash(rng)); err != nil { - return err + return fmt.Errorf("PushAppHash(%d): %w", n, err) } } // No AppQC yet -> eviction must not strip AppProposals; first stays put. for inner := range state.inner.Lock() { - require.Equal(t, gr1.First, inner.first, "no certified App → first unchanged") + if inner.first != gr1.First { + return fmt.Errorf("no certified App: first = %d, want %d", inner.first, gr1.First) + } for n := gr1.First; n < gr1.Next; n++ { _, ok := inner.appProposals[n] - require.True(t, ok, "AppProposal %d must survive without AppQC", n) + if !ok { + return fmt.Errorf("AppProposal %d missing before AppQC", n) + } } } - require.NoError(t, pushAppQCForBlock(ctx, state, keys, gr1.First)) - require.Eventually(t, func() bool { - for inner := range state.inner.Lock() { - return inner.nextAppQCToPersist >= gr1.Next + if err := pushAppQCForBlock(ctx, state, keys, gr1.First); err != nil { + return fmt.Errorf("pushAppQCForBlock(%d): %w", gr1.First, err) + } + if _, err := state.Anchor().Wait(ctx, func(anchor utils.Option[Anchor]) bool { + if anchor, ok := anchor.Get(); ok { + return anchor.AppQC.Proposal().RoadIndex() >= qc1.Index() } - panic("unreachable") - }, time.Second, time.Millisecond) + return false + }); err != nil { + return fmt.Errorf("state.Anchor.Wait(): %w", err) + } - require.NoError(t, state.PushQC(ctx, qc2, blocks2)) + if err := state.PushQC(ctx, qc2, blocks2); err != nil { + return fmt.Errorf("PushQC(qc2): %w", err) + } for n := gr2.First; n < gr2.Next; n++ { if err := state.PushAppHash(ctx, n, types.GenAppHash(rng)); err != nil { - return err + return fmt.Errorf("PushAppHash(%d): %w", n, err) } } for inner := range state.inner.Lock() { - evictionBound := min(inner.nextAppProposal, inner.nextAppQCToPersist) - require.Equal(t, evictionBound, inner.first, "after catching up, first reaches min(nextAppProposal, nextAppQCToPersist)") + evictionBound := min(inner.nextAppProposal, inner.nextAppQCToPersist) - 1 + if inner.first != evictionBound { + return fmt.Errorf("after catching up, first = %d, want eviction bound %d", inner.first, evictionBound) + } for n := gr1.First; n < inner.first; n++ { _, ok := inner.appProposals[n] - require.False(t, ok, "AppProposal %d should be evicted (< first)", n) + if ok { + return fmt.Errorf("AppProposal %d present below first %d", n, inner.first) + } } // Heights at/above exclusive floor stay until executed further. for n := inner.first; n < inner.nextAppProposal; n++ { _, ok := inner.appProposals[n] - require.True(t, ok, "AppProposal %d must remain (>= first)", n) + if !ok { + return fmt.Errorf("AppProposal %d missing at/above first %d", n, inner.first) + } } // Tip QC (nextQC-1) stays; nextToExecute uses maps at/above first. - require.GreaterOrEqual(t, inner.nextQC-1, inner.first) + if inner.nextQC-1 < inner.first { + return fmt.Errorf("tip QC height %d below first %d", inner.nextQC-1, inner.first) + } _, ok := inner.qcs[inner.nextQC-1] - require.True(t, ok, "tip QC must stay in maps") + if !ok { + return fmt.Errorf("tip QC %d missing from maps", inner.nextQC-1) + } } return nil })) @@ -575,34 +595,6 @@ func TestEvictionWaitsForPersistedAppQC(t *testing.T) { } } -func TestPushAppQCWaitsForBlocks(t *testing.T) { - ctx := t.Context() - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 3) - - qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) - gr1 := qc1.QC().GlobalRange() - appProposal := types.NewAppProposal(qc1.QC().Proposal(), types.GenAppHash(rng)) - appQC := TestAppQC(keys, appProposal) - - state := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) - require.NoError(t, state.PushQC(ctx, qc1, nil)) - - shortCtx, cancel := context.WithTimeout(ctx, 10*time.Millisecond) - err := state.PushAppQC(shortCtx, appQC) - cancel() - require.ErrorIs(t, err, context.DeadlineExceeded) - - for n := gr1.First; n < gr1.Next; n++ { - require.NoError(t, state.PushBlock(ctx, n, blocks1[n-gr1.First])) - } - require.NoError(t, state.PushAppQC(ctx, appQC)) - for inner := range state.inner.Lock() { - require.Equal(t, gr1.Next, inner.nextAppQC) - require.LessOrEqual(t, inner.nextAppQC, inner.nextBlock) - } -} - // TestNextToExecuteAfterAppEviction checks WaitUntilExecuted / nextToExecute // still work when persisted AppQC aggressively evicts through nextAppProposal // (first = min(nextAppProposal, nextAppQCToPersist) = NAP). nextToExecute uses @@ -624,40 +616,59 @@ func TestNextToExecuteAfterAppEviction(t *testing.T) { return utils.IgnoreCancel(state.Run(runCtx)) }) - require.NoError(t, state.PushQC(ctx, qc1, blocks1)) + if err := state.PushQC(ctx, qc1, blocks1); err != nil { + return fmt.Errorf("PushQC(qc1): %w", err) + } for n := gr1.First; n < gr1.Next; n++ { if err := state.PushAppHash(ctx, n, types.GenAppHash(rng)); err != nil { - return err + return fmt.Errorf("PushAppHash(%d): %w", n, err) } } // Sticky case: nextAppQCToPersist == nextAppProposal. first advances to // NAP; NAP-1 is gone; nextToExecute reads qc[NAP] after the next QC arrives. - require.NoError(t, pushAppQCForBlock(ctx, state, keys, gr1.First)) - require.Eventually(t, func() bool { - for inner := range state.inner.Lock() { - return inner.nextAppQCToPersist >= gr1.Next + if err := pushAppQCForBlock(ctx, state, keys, gr1.First); err != nil { + return fmt.Errorf("pushAppQCForBlock(%d): %w", gr1.First, err) + } + if _, err := state.Anchor().Wait(ctx, func(anchor utils.Option[Anchor]) bool { + if anchor, ok := anchor.Get(); ok { + return anchor.AppQC.Proposal().RoadIndex() >= qc1.Index() } - panic("unreachable") - }, time.Second, time.Millisecond) - require.NoError(t, state.PushQC(ctx, qc2, blocks2)) + return false + }); err != nil { + return fmt.Errorf("state.Anchor.Wait(): %w", err) + } + if err := state.PushQC(ctx, qc2, blocks2); err != nil { + return fmt.Errorf("PushQC(qc2): %w", err) + } var tipLane types.LaneID var tipBlockNum types.BlockNumber for inner := range state.inner.Lock() { - require.Equal(t, gr1.Next, inner.nextAppProposal) - require.Equal(t, min(inner.nextAppProposal, inner.nextAppQCToPersist), inner.first, - "eviction advances to min(nextAppProposal, nextAppQCToPersist) == NAP") - _, ok := inner.blocks[inner.nextAppProposal-1] - require.False(t, ok, "NAP-1 must be evicted") - require.Less(t, inner.nextAppProposal, inner.nextQC) + if inner.nextAppProposal != gr1.Next { + return fmt.Errorf("nextAppProposal = %d, want %d", inner.nextAppProposal, gr1.Next) + } + evictionBound := min(inner.nextAppProposal, inner.nextAppQCToPersist) - 1 + if inner.first != evictionBound { + return fmt.Errorf("first = %d, want eviction bound %d", inner.first, evictionBound) + } + _, ok := inner.blocks[evictionBound-1] + if ok { + return fmt.Errorf("block %d present, want evicted", inner.nextAppProposal-1) + } + if inner.nextAppProposal >= inner.nextQC { + return fmt.Errorf("nextAppProposal = %d, want < nextQC %d", inner.nextAppProposal, inner.nextQC) + } fqc := inner.qcs[inner.nextAppProposal] - require.NotNil(t, fqc) + if fqc == nil { + return fmt.Errorf("QC %d missing", inner.nextAppProposal) + } gr := fqc.QC().GlobalRange() h := fqc.Headers()[inner.nextAppProposal-gr.First] tipLane = h.Lane() tipBlockNum = h.BlockNumber() - require.Equal(t, tipBlockNum, inner.nextToExecute(tipLane), - "nextToExecute should be the next block's lane number") + if got := inner.nextToExecute(tipLane); got != tipBlockNum { + return fmt.Errorf("nextToExecute(%d) = %d, want %d", tipLane, got, tipBlockNum) + } } // WaitUntilExecuted(n) returns when nextToExecute > n. waitFrom := tipBlockNum @@ -665,8 +676,12 @@ func TestNextToExecuteAfterAppEviction(t *testing.T) { waitFrom-- } next, err := state.WaitUntilExecuted(ctx, tipLane, waitFrom) - require.NoError(t, err) - require.Equal(t, tipBlockNum, next) + if err != nil { + return fmt.Errorf("WaitUntilExecuted(%d, %d): %w", tipLane, waitFrom, err) + } + if next != tipBlockNum { + return fmt.Errorf("WaitUntilExecuted(%d, %d) = %d, want %d", tipLane, waitFrom, next, tipBlockNum) + } return nil })) } @@ -689,16 +704,25 @@ func TestPushAppQCPersistsAndRecovers(t *testing.T) { return utils.IgnoreCancel(state1.Run(runCtx)) }) - require.NoError(t, state1.PushQC(ctx, qc1, blocks1)) + if err := state1.PushQC(ctx, qc1, blocks1); err != nil { + return fmt.Errorf("PushQC(qc1): %w", err) + } for n := gr1.First; n < gr1.Next; n++ { if err := state1.PushAppHash(ctx, n, types.GenAppHash(rng)); err != nil { - return err + return fmt.Errorf("PushAppHash(%d): %w", n, err) } } - require.NoError(t, pushAppQCForBlock(ctx, state1, keys, gr1.First)) - require.Eventually(t, func() bool { - return db1.Status().NextAppQC >= gr1.Next - }, time.Second, time.Millisecond) + if err := pushAppQCForBlock(ctx, state1, keys, gr1.First); err != nil { + return fmt.Errorf("pushAppQCForBlock(%d): %w", gr1.First, err) + } + if _, err := state1.Anchor().Wait(ctx, func(anchor utils.Option[Anchor]) bool { + if anchor, ok := anchor.Get(); ok { + return anchor.AppQC.Proposal().RoadIndex() >= qc1.Index() + } + return false + }); err != nil { + return fmt.Errorf("state.Anchor.Wait(): %w", err) + } return nil })) @@ -808,13 +832,18 @@ func TestPruningWithPartialQCRange(t *testing.T) { if err := pushAppQCForBlock(ctx, state1, keys, gr1.First); err != nil { return err } - require.Eventually(t, func() bool { - return state1.blockDB.Status().NextAppQC >= gr1.Next - }, time.Second, time.Millisecond) + if _, err := state1.Anchor().Wait(ctx, func(anchor utils.Option[Anchor]) bool { + if anchor, ok := anchor.Get(); ok { + return anchor.AppQC.Proposal().RoadIndex() >= qc1.Index() + } + return false + }); err != nil { + return fmt.Errorf("state.Anchor.Wait(): %w", err) + } return nil })) for inner := range state1.inner.Lock() { - exclusiveFloor = min(inner.nextAppProposal, inner.nextAppQCToPersist) + exclusiveFloor = min(inner.nextAppProposal, inner.nextAppQCToPersist) - 1 require.Equal(t, exclusiveFloor, inner.first) } From bd863e5566aa7179733a20e58ccf960c7543190b Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 6 Aug 2026 18:19:11 +0200 Subject: [PATCH 16/61] pruned avail tests --- .../internal/autobahn/avail/conv_test.go | 31 - .../internal/autobahn/avail/inner_test.go | 674 ++---------------- .../internal/autobahn/avail/state_test.go | 399 +---------- 3 files changed, 64 insertions(+), 1040 deletions(-) delete mode 100644 sei-tendermint/internal/autobahn/avail/conv_test.go diff --git a/sei-tendermint/internal/autobahn/avail/conv_test.go b/sei-tendermint/internal/autobahn/avail/conv_test.go deleted file mode 100644 index 210608664c..0000000000 --- a/sei-tendermint/internal/autobahn/avail/conv_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package avail - -import ( - "testing" - - "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/epoch" - "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" - "github.com/stretchr/testify/require" - "google.golang.org/protobuf/proto" -) - -func TestPruneAnchorConv(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - - lane := keys[0].Public() - block := types.NewBlock(lane, 0, types.BlockHeaderHash{}, types.GenPayload(rng)) - laneQCs := map[types.LaneID]*types.LaneQC{ - lane: types.NewLaneQC(makeLaneVotes(keys, block.Header())), - } - commitQC := makeCommitQC(registry.LatestEpoch(), keys, utils.None[*types.CommitQC](), laneQCs, utils.None[*types.AppQC]()) - appProposal := types.NewAppProposal(commitQC.Proposal(), types.GenAppHash(rng)) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - anchor := &PruneAnchor{AppQC: appQC, CommitQC: commitQC} - pb1 := PruneAnchorConv.Encode(anchor) - decoded, err := PruneAnchorConv.Decode(pb1) - require.NoError(t, err) - require.True(t, proto.Equal(pb1, PruneAnchorConv.Encode(decoded))) -} diff --git a/sei-tendermint/internal/autobahn/avail/inner_test.go b/sei-tendermint/internal/autobahn/avail/inner_test.go index 86ce237296..bbea34ae7a 100644 --- a/sei-tendermint/internal/autobahn/avail/inner_test.go +++ b/sei-tendermint/internal/autobahn/avail/inner_test.go @@ -8,7 +8,6 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/consensus/persist" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/data" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/epoch" - pb "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/pb" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/stretchr/testify/require" ) @@ -17,48 +16,6 @@ func newTestDataState(cfg *data.Config) *data.State { return utils.OrPanic1(data.NewState(cfg, memblock.NewBlockDB())) } -func TestPruneMismatchedIndices(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - - makeCommitQC := func(prev utils.Option[*types.CommitQC]) *types.CommitQC { - l := keys[0].Public() - lr := types.LaneRangeOpt(prev, l) - b := types.NewBlock(l, lr.Next(), lr.LastHash(), types.GenPayload(rng)) - lqcs := map[types.LaneID]*types.LaneQC{ - l: types.NewLaneQC(makeLaneVotes(keys, b.Header())), - } - return makeCommitQC(registry.LatestEpoch(), keys, prev, lqcs, utils.None[*types.AppQC]()) - } - makeAppQC := func(qc *types.CommitQC) *types.AppQC { - ap := types.NewAppProposal(qc.Proposal(), types.GenAppHash(rng)) - return types.NewAppQC(makeAppVotes(keys, ap)) - } - - qc0 := makeCommitQC(utils.None[*types.CommitQC]()) - qc1 := makeCommitQC(utils.Some(qc0)) - - t.Logf("test State.PushAppQC") - ds := newTestDataState(&data.Config{Registry: registry}) - state, err := NewState(keys[0], ds, utils.Some(t.TempDir())) - require.NoError(t, err) - require.Error(t, state.PushAppQC(makeAppQC(qc0), qc1), "mismatched proposal should fail") - require.NoError(t, state.PushAppQC(makeAppQC(qc1), qc1), "matching proposal should succeed") - - t.Logf("test inner.prune") - ds = newTestDataState(&data.Config{Registry: registry}) - state, err = NewState(keys[0], ds, utils.Some(t.TempDir())) - require.NoError(t, err) - for inner := range state.inner.Lock() { - _, err := inner.prune(registry.LatestEpoch().Committee(), makeAppQC(qc0), qc1) - require.Error(t, err, "mismatched proposal should fail") - require.False(t, inner.latestAppQC.IsPresent(), "latestAppQC should not have been updated") - _, err = inner.prune(registry.LatestEpoch().Committee(), makeAppQC(qc1), qc1) - require.NoError(t, err, "matching proposal should succeed") - } -} - -// testSignedBlock creates a signed lane proposal for a given lane, block number, and parent hash. func testSignedBlock(key types.SecretKey, lane types.LaneID, n types.BlockNumber, parent types.BlockHeaderHash, rng utils.Rng) *types.Signed[*types.LaneProposal] { block := types.NewBlock(lane, n, parent, types.GenPayload(rng)) return types.Sign(key, types.NewLaneProposal(block)) @@ -68,15 +25,13 @@ func TestNewInnerFreshStart(t *testing.T) { rng := utils.TestRng() registry, _ := epoch.GenRegistry(rng, 4) - i, err := newInner(registry.LatestEpoch(), utils.None[*loadedAvailState]()) + i, err := newInner(newTestDataState(&data.Config{Registry: registry}), &loadedState{}) require.NoError(t, err) - require.False(t, i.latestAppQC.IsPresent()) + require.Equal(t, types.RoadIndex(0), i.nextAppQC) + require.Equal(t, types.RoadIndex(0), i.roads.first) + require.Equal(t, types.RoadIndex(0), i.roads.next) require.NotNil(t, i.nextBlockToPersist) - require.Equal(t, types.RoadIndex(0), i.commitQCs.first) - require.Equal(t, types.RoadIndex(0), i.commitQCs.next) - require.Equal(t, registry.FirstBlock(), i.appVotes.first) - require.Equal(t, registry.FirstBlock(), i.appVotes.next) for lane := range registry.LatestEpoch().Committee().Lanes().All() { require.Equal(t, types.BlockNumber(0), i.blocks[lane].first) require.Equal(t, types.BlockNumber(0), i.blocks[lane].next) @@ -85,41 +40,11 @@ func TestNewInnerFreshStart(t *testing.T) { } } -func TestDecodePruneAnchorIncomplete(t *testing.T) { - rng := utils.TestRng() - _, keys := epoch.GenRegistry(rng, 4) - - appProposal := types.GenAppProposal(rng) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - _, err := PruneAnchorConv.Decode(&pb.PersistedAvailPruneAnchor{ - AppQc: types.AppQCConv.Encode(appQC), - }) - require.Error(t, err) - require.Contains(t, err.Error(), "incomplete prune anchor") -} - -func TestNewInnerLoadedNoAnchor(t *testing.T) { - rng := utils.TestRng() - registry, _ := epoch.GenRegistry(rng, 4) - - loaded := &loadedAvailState{} - - i, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) - require.NoError(t, err) - - // No anchor loaded, app votes should start at the registry's first block. - require.False(t, i.latestAppQC.IsPresent()) - require.Equal(t, types.RoadIndex(0), i.commitQCs.first) - require.Equal(t, registry.FirstBlock(), i.appVotes.first) -} - func TestNewInnerLoadedBlocksContiguous(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 4) lane := keys[0].Public() - // Build 3 contiguous blocks: 0, 1, 2. var parent types.BlockHeaderHash var bs []persist.LoadedBlock for n := range types.BlockNumber(3) { @@ -128,11 +53,9 @@ func TestNewInnerLoadedBlocksContiguous(t *testing.T) { bs = append(bs, persist.LoadedBlock{Number: n, Proposal: b}) } - loaded := &loadedAvailState{ + i, err := newInner(newTestDataState(&data.Config{Registry: registry}), &loadedState{ blocks: map[types.LaneID][]persist.LoadedBlock{lane: bs}, - } - - i, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) + }) require.NoError(t, err) q := i.blocks[lane] @@ -141,9 +64,6 @@ func TestNewInnerLoadedBlocksContiguous(t *testing.T) { for j, b := range bs { require.Equal(t, b.Proposal, q.q[types.BlockNumber(j)]) } - - // nextBlockToPersist: loaded lane at q.next, other lanes at 0 (map zero-value). - require.NotNil(t, i.nextBlockToPersist) require.Equal(t, types.BlockNumber(3), i.nextBlockToPersist[lane]) for other := range registry.LatestEpoch().Committee().Lanes().All() { if other != lane { @@ -157,11 +77,9 @@ func TestNewInnerLoadedBlocksEmptySlice(t *testing.T) { registry, keys := epoch.GenRegistry(rng, 4) lane := keys[0].Public() - loaded := &loadedAvailState{ + i, err := newInner(newTestDataState(&data.Config{Registry: registry}), &loadedState{ blocks: map[types.LaneID][]persist.LoadedBlock{lane: {}}, - } - - i, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) + }) require.NoError(t, err) q := i.blocks[lane] @@ -175,13 +93,11 @@ func TestNewInnerLoadedBlocksUnknownLane(t *testing.T) { unknownKey := types.GenSecretKey(rng) unknownLane := unknownKey.Public() - b := testSignedBlock(unknownKey, unknownLane, 0, types.BlockHeaderHash{}, rng) - loaded := &loadedAvailState{ - blocks: map[types.LaneID][]persist.LoadedBlock{unknownLane: {{Number: 0, Proposal: b}}}, - } - i, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) + i, err := newInner(newTestDataState(&data.Config{Registry: registry}), &loadedState{ + blocks: map[types.LaneID][]persist.LoadedBlock{unknownLane: {{Number: 0, Proposal: b}}}, + }) require.NoError(t, err) for lane := range registry.LatestEpoch().Committee().Lanes().All() { @@ -214,22 +130,13 @@ func TestNewInnerLoadedBlocksMultipleLanes(t *testing.T) { bs1 = append(bs1, persist.LoadedBlock{Number: n, Proposal: b}) } - loaded := &loadedAvailState{ + i, err := newInner(newTestDataState(&data.Config{Registry: registry}), &loadedState{ blocks: map[types.LaneID][]persist.LoadedBlock{lane0: bs0, lane1: bs1}, - } - - i, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) + }) require.NoError(t, err) - q0 := i.blocks[lane0] - require.Equal(t, types.BlockNumber(0), q0.first) - require.Equal(t, types.BlockNumber(2), q0.next) - - q1 := i.blocks[lane1] - require.Equal(t, types.BlockNumber(0), q1.first) - require.Equal(t, types.BlockNumber(3), q1.next) - - // nextBlockToPersist reflects q.next per loaded lane. + require.Equal(t, types.BlockNumber(2), i.blocks[lane0].next) + require.Equal(t, types.BlockNumber(3), i.blocks[lane1].next) require.Equal(t, types.BlockNumber(2), i.nextBlockToPersist[lane0]) require.Equal(t, types.BlockNumber(3), i.nextBlockToPersist[lane1]) } @@ -238,306 +145,33 @@ func TestNewInnerLoadedCommitQCsNoAppQC(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 4) - // Create 3 sequential CommitQCs. qcs := make([]*types.CommitQC, 3) prev := utils.None[*types.CommitQC]() for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) + qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil) prev = utils.Some(qcs[i]) } - var loadedQCs []persist.LoadedCommitQC - for i, qc := range qcs { - loadedQCs = append(loadedQCs, persist.LoadedCommitQC{Index: types.RoadIndex(i), QC: qc}) - } - - loaded := &loadedAvailState{ - commitQCs: loadedQCs, - } - - inner, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) + inner, err := newInner(newTestDataState(&data.Config{Registry: registry}), &loadedState{commitQCs: qcs}) require.NoError(t, err) - // Without anchor, commitQCs.first = 0. All 3 should be restored. - require.Equal(t, types.RoadIndex(0), inner.commitQCs.first) - require.Equal(t, types.RoadIndex(3), inner.commitQCs.next) + require.Equal(t, types.RoadIndex(0), inner.roads.first) + require.Equal(t, types.RoadIndex(3), inner.roads.next) for i, qc := range qcs { - require.NoError(t, utils.TestDiff(qc, inner.commitQCs.q[types.RoadIndex(i)])) - } - - // latestCommitQC should be set to the last loaded one. - latest, ok := inner.latestCommitQC.Load().Get() - require.True(t, ok) - require.NoError(t, utils.TestDiff(qcs[2], latest)) -} - -func TestNewInnerLoadedCommitQCsWithAppQC(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - - // AppQC at road index 2. - roadIdx := types.RoadIndex(2) - - // Create 5 sequential CommitQCs (indices 0-4). - qcs := make([]*types.CommitQC, 5) - prev := utils.None[*types.CommitQC]() - for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[i]) - } - appProposal := types.NewAppProposal(qcs[roadIdx].Proposal(), types.GenAppHash(rng)) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - // Pre-filtered: only commitQCs >= anchor road index (2). - loadedQCs := []persist.LoadedCommitQC{ - {Index: 2, QC: qcs[2]}, - {Index: 3, QC: qcs[3]}, - {Index: 4, QC: qcs[4]}, - } - - loaded := &loadedAvailState{ - pruneAnchor: utils.Some(&PruneAnchor{AppQC: appQC, CommitQC: qcs[2]}), - commitQCs: loadedQCs, - } - - inner, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) - require.NoError(t, err) - - // latestAppQC should be set by prune. - aq, ok := inner.latestAppQC.Get() - require.True(t, ok) - require.Equal(t, roadIdx, aq.Proposal().RoadIndex()) - - // inner.prune(appQC@2, commitQC@2) sets commitQCs.first = 2. - // Indices 2, 3 and 4 remain; earlier ones are pruned. - require.Equal(t, types.RoadIndex(2), inner.commitQCs.first) - require.Equal(t, types.RoadIndex(5), inner.commitQCs.next) - require.NoError(t, utils.TestDiff(qcs[2], inner.commitQCs.q[2])) - require.NoError(t, utils.TestDiff(qcs[3], inner.commitQCs.q[3])) - require.NoError(t, utils.TestDiff(qcs[4], inner.commitQCs.q[4])) - - // latestCommitQC should be the last restored one (index 4). - latest, ok := inner.latestCommitQC.Load().Get() - require.True(t, ok) - require.NoError(t, utils.TestDiff(qcs[4], latest)) -} - -func TestNewInnerLoadedAllThree(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - lane := keys[0].Public() - - // AppQC at road index 2. - roadIdx := types.RoadIndex(2) - - // CommitQCs 0-4. - qcs := make([]*types.CommitQC, 5) - prev := utils.None[*types.CommitQC]() - for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[i]) - } - appProposal := types.NewAppProposal(qcs[roadIdx].Proposal(), types.GenAppHash(rng)) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - // Pre-filtered: only commitQCs >= anchor road index (2). - loadedQCs := []persist.LoadedCommitQC{ - {Index: 2, QC: qcs[2]}, - {Index: 3, QC: qcs[3]}, - {Index: 4, QC: qcs[4]}, - } - - // Blocks 0-2 on one lane (nil laneQCs → lr.First()=0 after prune). - var parent types.BlockHeaderHash - var bs []persist.LoadedBlock - for n := range types.BlockNumber(3) { - b := testSignedBlock(keys[0], lane, n, parent, rng) - parent = b.Msg().Block().Header().Hash() - bs = append(bs, persist.LoadedBlock{Number: n, Proposal: b}) - } - - loaded := &loadedAvailState{ - pruneAnchor: utils.Some(&PruneAnchor{AppQC: appQC, CommitQC: qcs[2]}), - commitQCs: loadedQCs, - blocks: map[types.LaneID][]persist.LoadedBlock{lane: bs}, - } - - inner, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) - require.NoError(t, err) - - // AppQC restored. - aq, ok := inner.latestAppQC.Get() - require.True(t, ok) - require.Equal(t, roadIdx, aq.Proposal().RoadIndex()) - - // CommitQCs: prune pushed qcs[2], loading skipped it, added 3 and 4. - require.Equal(t, types.RoadIndex(2), inner.commitQCs.first) - require.Equal(t, types.RoadIndex(5), inner.commitQCs.next) - - // Blocks loaded. - q := inner.blocks[lane] - require.Equal(t, types.BlockNumber(0), q.first) - require.Equal(t, types.BlockNumber(3), q.next) - require.Equal(t, types.BlockNumber(3), inner.nextBlockToPersist[lane]) - - // latestCommitQC is the last loaded one. - latest, ok := inner.latestCommitQC.Load().Get() - require.True(t, ok) - require.NoError(t, utils.TestDiff(qcs[4], latest)) -} - -func TestPruneAdvancesNextBlockToPersist(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - lane := keys[0].Public() - - i, err := newInner(registry.LatestEpoch(), utils.None[*loadedAvailState]()) - require.NoError(t, err) - - // Push blocks 0-4 on one lane. - var parent types.BlockHeaderHash - for n := range types.BlockNumber(5) { - b := testSignedBlock(keys[0], lane, n, parent, rng) - parent = b.Msg().Block().Header().Hash() - i.blocks[lane].pushBack(b) - } - // Simulate partial persistence: only block 0 persisted. - i.nextBlockToPersist[lane] = 1 - - // Build CommitQCs with lane ranges that reference actual blocks. - // Each CommitQC covers one block on the lane via a LaneQC. - qcs := make([]*types.CommitQC, 3) - prev := utils.None[*types.CommitQC]() - for j := range qcs { - bn := types.BlockNumber(j) - h := i.blocks[lane].q[bn].Msg().Block().Header() - laneQCs := map[types.LaneID]*types.LaneQC{ - lane: types.NewLaneQC(makeLaneVotes( - types.TestKeysWithWeight(registry.LatestEpoch().Committee(), keys, registry.LatestEpoch().Committee().LaneQuorum()), - h, - )), - } - qcs[j] = makeCommitQC(registry.LatestEpoch(), keys, prev, laneQCs, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[j]) - i.commitQCs.pushBack(qcs[j]) - } - - // Verify QC@2's lane range actually covers blocks (First > 0). - lr := qcs[2].LaneRange(lane) - require.Greater(t, lr.First(), types.BlockNumber(0), - "CommitQC lane range should reference blocks for this test to be meaningful") - - // AppQC at index 2 → prune will fast-forward blocks past the cursor. - appProposal := types.NewAppProposal(qcs[2].Proposal(), types.GenAppHash(rng)) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - updated, err := i.prune(registry.LatestEpoch().Committee(), appQC, qcs[2]) - require.NoError(t, err) - require.True(t, updated) - - // nextBlockToPersist must have advanced to at least the lane's new first - // (determined by CommitQC@2's lane range). Without this fix, it would - // stay at 1, causing the persist goroutine to busy-loop. - laneFirst := i.blocks[lane].first - require.Greater(t, laneFirst, types.BlockNumber(1), - "prune should have advanced blocks.first past the old cursor") - require.GreaterOrEqual(t, i.nextBlockToPersist[lane], laneFirst, - "nextBlockToPersist should advance when prune moves blocks.first past it") -} - -func TestNewInnerLoadedCommitQCsAllBeforeAppQCArePruned(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - - // Build 6 CommitQCs (indices 0-5). Anchor at index 5. - // All stale commitQCs (0-4) were already filtered by loadPersistedState, - // so newInner receives an empty commitQC slice. - qcs := make([]*types.CommitQC, 6) - prev := utils.None[*types.CommitQC]() - for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[i]) - } - - appProposal := types.NewAppProposal(qcs[5].Proposal(), types.GenAppHash(rng)) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - loaded := &loadedAvailState{ - pruneAnchor: utils.Some(&PruneAnchor{AppQC: appQC, CommitQC: qcs[5]}), - } - - inner, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) - require.NoError(t, err) - - // prune() pushes the anchor's CommitQC into the queue. - require.Equal(t, types.RoadIndex(5), inner.commitQCs.first) - require.Equal(t, types.RoadIndex(6), inner.commitQCs.next) - require.NoError(t, utils.TestDiff(qcs[5], inner.commitQCs.q[5])) -} - -func TestNewInnerAnchorWithNoCommitQCFiles(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - - // Simulate crash between anchor write and CommitQC file write: - // anchor has AppQC@3 + CommitQC@3, but no CommitQC files on disk. - qcs := make([]*types.CommitQC, 4) - prev := utils.None[*types.CommitQC]() - for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[i]) - } - - appProposal := types.NewAppProposal(qcs[3].Proposal(), types.GenAppHash(rng)) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - loaded := &loadedAvailState{ - pruneAnchor: utils.Some(&PruneAnchor{AppQC: appQC, CommitQC: qcs[3]}), - } - - inner, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) - require.NoError(t, err) - - // prune() should push the anchor's CommitQC into the queue. - require.Equal(t, types.RoadIndex(3), inner.commitQCs.first) - require.Equal(t, types.RoadIndex(4), inner.commitQCs.next) - require.NoError(t, utils.TestDiff(qcs[3], inner.commitQCs.q[3])) - - // latestAppQC should be set. - aq, ok := inner.latestAppQC.Get() - require.True(t, ok) - require.Equal(t, types.RoadIndex(3), aq.Proposal().RoadIndex()) - - // persistedBlockStart should be initialized from the anchor's CommitQC. - for lane := range registry.LatestEpoch().Committee().Lanes().All() { - expected := qcs[3].LaneRange(lane).First() - require.Equal(t, expected, inner.persistedBlockStart[lane]) + require.NoError(t, utils.TestDiff(qc, inner.roads.q[types.RoadIndex(i)].commitQC)) } + require.NoError(t, utils.TestDiff(utils.Some(qcs[2]), inner.persistedCommitQC.Load())) } func TestNewInnerLoadedCommitQCsGapReturnsError(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 4) - qcs := make([]*types.CommitQC, 3) - prev := utils.None[*types.CommitQC]() - for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[i]) - } + qc0 := makeCommitQC(registry.LatestEpoch(), keys, utils.None[*types.CommitQC](), nil) + qc1 := makeCommitQC(registry.LatestEpoch(), keys, utils.Some(qc0), nil) + qc2 := makeCommitQC(registry.LatestEpoch(), keys, utils.Some(qc1), nil) - // Gap: indices 0, 1, 3 (missing 2). Since the anchor is persisted first, - // a gap in committed QCs is a bug — newInner should return an error. - loadedQCs := []persist.LoadedCommitQC{ - {Index: 0, QC: qcs[0]}, - {Index: 1, QC: qcs[1]}, - {Index: 3, QC: qcs[2]}, - } - - loaded := &loadedAvailState{ - commitQCs: loadedQCs, - } - - _, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) + _, err := newInner(newTestDataState(&data.Config{Registry: registry}), &loadedState{commitQCs: []*types.CommitQC{qc0, qc2}}) require.Error(t, err) require.Contains(t, err.Error(), "non-contiguous") } @@ -546,158 +180,28 @@ func TestNewInnerLoadedCommitQCsEmpty(t *testing.T) { rng := utils.TestRng() registry, _ := epoch.GenRegistry(rng, 4) - loaded := &loadedAvailState{ - commitQCs: nil, - } - - inner, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) + inner, err := newInner(newTestDataState(&data.Config{Registry: registry}), &loadedState{}) require.NoError(t, err) - require.Equal(t, types.RoadIndex(0), inner.commitQCs.first) - require.Equal(t, types.RoadIndex(0), inner.commitQCs.next) - _, ok := inner.latestCommitQC.Load().Get() + require.Equal(t, types.RoadIndex(0), inner.roads.first) + require.Equal(t, types.RoadIndex(0), inner.roads.next) + _, ok := inner.persistedCommitQC.Load().Get() require.False(t, ok) } -func TestNewInnerLoadedCommitQCsGapWithAppQCAnchor(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - - // Simulate crash scenario: disk had stale QCs [0,1,2] and a new QC at - // index 10. loadPersistedState pre-filters stale entries, so newInner - // only receives [10]. - qcs := make([]*types.CommitQC, 11) - prev := utils.None[*types.CommitQC]() - for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[i]) - } - - appProposal := types.NewAppProposal(qcs[10].Proposal(), types.GenAppHash(rng)) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - loadedQCs := []persist.LoadedCommitQC{ - {Index: 10, QC: qcs[10]}, - } - - loaded := &loadedAvailState{ - pruneAnchor: utils.Some(&PruneAnchor{AppQC: appQC, CommitQC: qcs[10]}), - commitQCs: loadedQCs, - } - - inner, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) - require.NoError(t, err) - - // Only QC@10 loaded. - require.Equal(t, types.RoadIndex(10), inner.commitQCs.first) - require.Equal(t, types.RoadIndex(11), inner.commitQCs.next) - require.NoError(t, utils.TestDiff(qcs[10], inner.commitQCs.q[10])) - - latest, ok := inner.latestCommitQC.Load().Get() - require.True(t, ok) - require.NoError(t, utils.TestDiff(qcs[10], latest)) - - // AppQC should be applied via prune. - aq, ok := inner.latestAppQC.Get() - require.True(t, ok) - require.Equal(t, types.RoadIndex(10), aq.Proposal().RoadIndex()) -} - -func TestNewInnerLoadedCommitQCsBelowAnchorSkipped(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - - // Build 6 CommitQCs (0-5). Anchor at index 3. - // Loaded list includes stale entries [1, 2] below the anchor plus [3, 4, 5]. - // In production loadPersistedState filters these, but newInner should - // handle them gracefully via the lqc.Index < commitQCs.next skip. - qcs := make([]*types.CommitQC, 6) - prev := utils.None[*types.CommitQC]() - for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[i]) - } - - appProposal := types.NewAppProposal(qcs[3].Proposal(), types.GenAppHash(rng)) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - loadedQCs := []persist.LoadedCommitQC{ - {Index: 1, QC: qcs[1]}, - {Index: 2, QC: qcs[2]}, - {Index: 3, QC: qcs[3]}, - {Index: 4, QC: qcs[4]}, - {Index: 5, QC: qcs[5]}, - } - - loaded := &loadedAvailState{ - pruneAnchor: utils.Some(&PruneAnchor{AppQC: appQC, CommitQC: qcs[3]}), - commitQCs: loadedQCs, - } - - inner, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) - require.NoError(t, err) - - // prune(3) pushes QC@3 (next=4). Indices 1,2,3 are skipped. 4,5 pushed. - require.Equal(t, types.RoadIndex(3), inner.commitQCs.first) - require.Equal(t, types.RoadIndex(6), inner.commitQCs.next) - latest, ok := inner.latestCommitQC.Load().Get() - require.True(t, ok) - require.NoError(t, utils.TestDiff(qcs[5], latest)) -} - -func TestNewInnerLoadedCommitQCsGapAfterAnchorReturnsError(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - - // Anchor at index 2. Loaded commitQCs are [2, 3, 5] — gap at 4. - // After prune(2), next=3. Index 2 is skipped, 3 pushed (next=4), - // then 5 != 4 → error. - qcs := make([]*types.CommitQC, 6) - prev := utils.None[*types.CommitQC]() - for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[i]) - } - - appProposal := types.NewAppProposal(qcs[2].Proposal(), types.GenAppHash(rng)) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - loadedQCs := []persist.LoadedCommitQC{ - {Index: 2, QC: qcs[2]}, - {Index: 3, QC: qcs[3]}, - {Index: 5, QC: qcs[5]}, - } - - loaded := &loadedAvailState{ - pruneAnchor: utils.Some(&PruneAnchor{AppQC: appQC, CommitQC: qcs[2]}), - commitQCs: loadedQCs, - } - - _, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) - require.Error(t, err) - require.Contains(t, err.Error(), "non-contiguous") -} - func TestNewInnerLoadedBlocksGapReturnsError(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 4) lane := keys[0].Public() - // Blocks 3, 4, 6, 7 with no anchor — queue starts at 0, so block 3 - // fails the contiguity check immediately (expected 0, got 3). - var parent types.BlockHeaderHash var bs []persist.LoadedBlock for _, n := range []types.BlockNumber{3, 4, 6, 7} { - b := testSignedBlock(keys[0], lane, n, parent, rng) - parent = b.Msg().Block().Header().Hash() - bs = append(bs, persist.LoadedBlock{Number: n, Proposal: b}) + bs = append(bs, persist.LoadedBlock{Number: n, Proposal: testSignedBlock(keys[0], lane, n, types.BlockHeaderHash{}, rng)}) } - loaded := &loadedAvailState{ + _, err := newInner(newTestDataState(&data.Config{Registry: registry}), &loadedState{ blocks: map[types.LaneID][]persist.LoadedBlock{lane: bs}, - } - - _, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) + }) require.Error(t, err) require.Contains(t, err.Error(), "non-contiguous") } @@ -707,25 +211,19 @@ func TestNewInnerLoadedBlocksParentHashMismatchReturnsError(t *testing.T) { registry, keys := epoch.GenRegistry(rng, 4) lane := keys[0].Public() - // Build blocks 0, 1 with correct chaining, then block 2 with wrong parent. var parent types.BlockHeaderHash b0 := testSignedBlock(keys[0], lane, 0, parent, rng) parent = b0.Msg().Block().Header().Hash() b1 := testSignedBlock(keys[0], lane, 1, parent, rng) - wrongParent := types.GenBlockHeaderHash(rng) - b2 := testSignedBlock(keys[0], lane, 2, wrongParent, rng) - - bs := []persist.LoadedBlock{ - {Number: 0, Proposal: b0}, - {Number: 1, Proposal: b1}, - {Number: 2, Proposal: b2}, - } - - loaded := &loadedAvailState{ - blocks: map[types.LaneID][]persist.LoadedBlock{lane: bs}, - } - - _, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) + b2 := testSignedBlock(keys[0], lane, 2, types.GenBlockHeaderHash(rng), rng) + + _, err := newInner(newTestDataState(&data.Config{Registry: registry}), &loadedState{ + blocks: map[types.LaneID][]persist.LoadedBlock{lane: { + {Number: 0, Proposal: b0}, + {Number: 1, Proposal: b1}, + {Number: 2, Proposal: b2}, + }}, + }) require.Error(t, err) require.Contains(t, err.Error(), "parent hash mismatch") } @@ -735,9 +233,6 @@ func TestNewInnerLoadedBlocksOverCapacityReturnsError(t *testing.T) { registry, keys := epoch.GenRegistry(rng, 4) lane := keys[0].Public() - // Build BlocksPerLane + 5 contiguous blocks — more than the lane capacity. - // Since runtime enforces the capacity limit, exceeding it on disk indicates - // corruption or a bug. count := BlocksPerLane + 5 var parent types.BlockHeaderHash var bs []persist.LoadedBlock @@ -747,92 +242,9 @@ func TestNewInnerLoadedBlocksOverCapacityReturnsError(t *testing.T) { bs = append(bs, persist.LoadedBlock{Number: n, Proposal: b}) } - loaded := &loadedAvailState{ + _, err := newInner(newTestDataState(&data.Config{Registry: registry}), &loadedState{ blocks: map[types.LaneID][]persist.LoadedBlock{lane: bs}, - } - - _, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) + }) require.Error(t, err) require.Contains(t, err.Error(), "exceeds capacity") } - -func TestNewInnerPruneAnchorPrunesBlockQueues(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - - // Build CommitQCs 0-2. - qcs := make([]*types.CommitQC, 3) - prev := utils.None[*types.CommitQC]() - for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[i]) - } - - // AppQC at road index 2, prune anchor is CommitQC[2]. - pruneQC := qcs[2] - appProposal := types.NewAppProposal(pruneQC.Proposal(), types.GenAppHash(rng)) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - lane := keys[0].Public() - - // Persist some blocks starting at the lane range for the prune CommitQC. - lrFirst := pruneQC.LaneRange(lane).First() - var parent types.BlockHeaderHash - var bs []persist.LoadedBlock - for n := lrFirst; n < lrFirst+3; n++ { - b := testSignedBlock(keys[0], lane, n, parent, rng) - parent = b.Msg().Block().Header().Hash() - bs = append(bs, persist.LoadedBlock{Number: n, Proposal: b}) - } - - loaded := &loadedAvailState{ - pruneAnchor: utils.Some(&PruneAnchor{AppQC: appQC, CommitQC: pruneQC}), - commitQCs: []persist.LoadedCommitQC{ - {Index: 2, QC: qcs[2]}, - }, - blocks: map[types.LaneID][]persist.LoadedBlock{lane: bs}, - } - - i, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) - require.NoError(t, err) - - // prune() should advance block queue first to the prune anchor's lane range. - for l := range registry.LatestEpoch().Committee().Lanes().All() { - expected := pruneQC.LaneRange(l).First() - require.Equal(t, expected, i.blocks[l].first, - "blocks[%v].first should be advanced by prune to prune anchor lane range", l) - } -} - -func TestNewInnerPruneAnchorCommitQCUsedForPrune(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - - // Build CommitQCs 0-2. - qcs := make([]*types.CommitQC, 3) - prev := utils.None[*types.CommitQC]() - for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[i]) - } - - // AppQC at road index 1, prune anchor is CommitQC[1]. - appProposal := types.NewAppProposal(qcs[1].Proposal(), types.GenAppHash(rng)) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - loaded := &loadedAvailState{ - pruneAnchor: utils.Some(&PruneAnchor{AppQC: appQC, CommitQC: qcs[1]}), - commitQCs: []persist.LoadedCommitQC{ - {Index: 1, QC: qcs[1]}, - {Index: 2, QC: qcs[2]}, - }, - } - - i, err := newInner(registry.LatestEpoch(), utils.Some(loaded)) - require.NoError(t, err) - - // prune(appQC@1, pruneQC@1) should advance commitQCs.first to 1. - require.Equal(t, types.RoadIndex(1), i.commitQCs.first) - // CommitQCs 1 and 2 should still be loaded. - require.Equal(t, types.RoadIndex(3), i.commitQCs.next) -} diff --git a/sei-tendermint/internal/autobahn/avail/state_test.go b/sei-tendermint/internal/autobahn/avail/state_test.go index dfe29d496b..0cbfba0f44 100644 --- a/sei-tendermint/internal/autobahn/avail/state_test.go +++ b/sei-tendermint/internal/autobahn/avail/state_test.go @@ -4,25 +4,19 @@ import ( "context" "errors" "fmt" - "os" - "path/filepath" "testing" - "time" - "github.com/sei-protocol/sei-chain/sei-db/ledger_db/block/memblock" "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/consensus/persist" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/data" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/epoch" - pb "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/pb" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/scope" "github.com/stretchr/testify/require" ) var ( - noBlockCB = utils.None[func(*types.Signed[*types.LaneProposal])]() - noCommitQCCB = utils.None[func(*types.CommitQC)]() + noBlockCB = utils.None[func(*types.Signed[*types.LaneProposal])]() ) type byLane[T any] map[types.LaneID][]T @@ -36,40 +30,6 @@ func makeAppVotes(keys []types.SecretKey, proposal *types.AppProposal) []*types. return votes } -func TestSubscribeAppVotesJumpsToDataFloor(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 3) - qc, blocks := data.TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) - gr := qc.QC().GlobalRange() - require.Greater(t, gr.Len(), uint64(2)) - - db := memblock.NewBlockDB() - t.Cleanup(func() { require.NoError(t, db.Close()) }) - require.NoError(t, db.WriteQC(qc)) - for i, n := 0, gr.First; n < gr.Next; i, n = i+1, n+1 { - require.NoError(t, db.WriteBlock(n, blocks[i])) - } - require.NoError(t, db.Flush()) - - first := gr.First + types.GlobalBlockNumber(gr.Len()/2) - ds, err := data.NewState(&data.Config{ - Registry: registry, - LastExecutedBlock: utils.Some(first), - }, db) - require.NoError(t, err) - appHash := types.GenAppHash(rng) - require.NoError(t, ds.PushAppHash(t.Context(), first, appHash)) - - state, err := NewState(keys[0], ds, utils.None[string]()) - require.NoError(t, err) - recv := state.SubscribeAppVotes() - require.Equal(t, types.GlobalBlockNumber(0), recv.next) - - vote, err := recv.Recv(t.Context()) - require.NoError(t, err) - require.Equal(t, first, vote.Msg().Proposal().GlobalFirst()) -} - func makeLaneVotes(keys []types.SecretKey, h *types.BlockHeader) []*types.Signed[*types.LaneVote] { var votes []*types.Signed[*types.LaneVote] for _, k := range keys { @@ -78,24 +38,13 @@ func makeLaneVotes(keys []types.SecretKey, h *types.BlockHeader) []*types.Signed return votes } -func leaderKey(committee *types.Committee, keys []types.SecretKey, view types.View) types.SecretKey { - leader := committee.Leader(view) - for _, k := range keys { - if k.Public() == leader { - return k - } - } - panic("leader not in keys") -} - func makeCommitQC( ep *types.Epoch, keys []types.SecretKey, prev utils.Option[*types.CommitQC], laneQCs map[types.LaneID]*types.LaneQC, - appQC utils.Option[*types.AppQC], ) *types.CommitQC { - return types.BuildCommitQC(ep, keys, prev, laneQCs, appQC) + return types.BuildCommitQC(ep, keys, prev, laneQCs) } func qcPayloadHashes(qc *types.FullCommitQC) byLane[types.PayloadHash] { @@ -107,6 +56,7 @@ func qcPayloadHashes(qc *types.FullCommitQC) byLane[types.PayloadHash] { } func TestState(t *testing.T) { + t.Skip("requires current AppQC/prune progression semantics to be recharacterized") testState(t, utils.None[string]()) } @@ -115,6 +65,7 @@ func TestState(t *testing.T) { // run concurrently, exercising the cursor-clamp logic that prevents reading // pruned map entries. func TestStateWithPersistence(t *testing.T) { + t.Skip("requires current AppQC/prune progression semantics to be recharacterized") for range 5 { testState(t, utils.Some(t.TempDir())) } @@ -179,7 +130,7 @@ func testState(t *testing.T, stateDir utils.Option[string]) { if err != nil { return fmt.Errorf("state.WaitForNewLaneQCs(): %w", err) } - qc := makeCommitQC(registry.LatestEpoch(), keys, prev, laneQCs, state.LastAppQC()) + qc := makeCommitQC(registry.LatestEpoch(), keys, prev, laneQCs) if err := state.PushCommitQC(ctx, qc); err != nil { return fmt.Errorf("state.PushCommitQC(): %w", err) } @@ -193,7 +144,7 @@ func testState(t *testing.T, stateDir utils.Option[string]) { } t.Logf("Previous one should be pruned because of appQC.") - if _, _, err := state.WaitForAppQC(ctx, appProposal.RoadIndex()); err != nil { + if _, err := state.appQC(ctx, appProposal.RoadIndex()); err != nil { return fmt.Errorf("state.WaitForAppQC(): %w", err) } if prev, ok := prev.Get(); ok { @@ -212,7 +163,7 @@ func testState(t *testing.T, stateDir utils.Option[string]) { } t.Logf("Check that a CommitQC was successfully reconstructed.") - got, err := state.fullCommitQC(ctx, qc.Proposal().Index()) + _, got, err := state.fullCommitQC(ctx, qc.Proposal().Index()) if err != nil { return fmt.Errorf("state.fullCommitQC(): %w", err) } @@ -248,6 +199,7 @@ func testState(t *testing.T, stateDir utils.Option[string]) { // stale entries by shutdown, restart exercises the gap-filtering path in // loadPersistedState (stale entries below the prune anchor are discarded). func TestStateRestartFromPersisted(t *testing.T) { + t.Skip("requires current restart semantics for data.State anchor and avail persistence to be recharacterized") rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 3) committee := registry.LatestEpoch().Committee() @@ -300,7 +252,7 @@ func TestStateRestartFromPersisted(t *testing.T) { if err != nil { return fmt.Errorf("WaitForLaneQCs: %w", err) } - qc := makeCommitQC(registry.LatestEpoch(), keys, prev, laneQCs, state.LastAppQC()) + qc := makeCommitQC(registry.LatestEpoch(), keys, prev, laneQCs) if err := state.PushCommitQC(ctx, qc); err != nil { return fmt.Errorf("PushCommitQC: %w", err) } @@ -311,7 +263,7 @@ func TestStateRestartFromPersisted(t *testing.T) { return fmt.Errorf("PushAppVote: %w", err) } } - if _, _, err := state.WaitForAppQC(ctx, appProposal.RoadIndex()); err != nil { + if _, err := state.appQC(ctx, appProposal.RoadIndex()); err != nil { return fmt.Errorf("WaitForAppQC: %w", err) } wantAppQCIdx = appProposal.RoadIndex() @@ -321,7 +273,9 @@ func TestStateRestartFromPersisted(t *testing.T) { // all commitQCs in the batch are on disk. Block goroutines may still // be in flight, but scope.Parallel in runPersist ensures they complete // before the next batch, so the data is durable by scope exit. - if err := state.waitForCommitQC(ctx, wantAppQCIdx); err != nil { + if _, err := state.LastCommitQC().Wait(ctx, func(qc utils.Option[*types.CommitQC]) bool { + return types.NextIndexOpt(qc) > wantAppQCIdx + }); err != nil { return fmt.Errorf("waitForCommitQC: %w", err) } @@ -337,13 +291,9 @@ func TestStateRestartFromPersisted(t *testing.T) { state2, err := NewState(keys[0], ds2, utils.Some(dir)) require.NoError(t, err) - got, ok := state2.LastAppQC().Get() - require.True(t, ok, "AppQC should be restored after restart") - require.Equal(t, wantAppQCIdx, got.Proposal().RoadIndex()) - require.GreaterOrEqual(t, state2.FirstCommitQC(), wantAppQCIdx) - _, ok = state2.LastCommitQC().Load().Get() + _, ok := state2.LastCommitQC().Load().Get() require.True(t, ok, "LastCommitQC should be set after restart") for lane := range committee.Lanes().All() { @@ -352,62 +302,6 @@ func TestStateRestartFromPersisted(t *testing.T) { } } -func TestStateMismatchedQCs(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - committee := registry.LatestEpoch().Committee() - initialBlock := registry.FirstBlock() - - ds := newTestDataState(&data.Config{Registry: registry}) - state, err := NewState(keys[0], ds, utils.Some(t.TempDir())) - require.NoError(t, err) - - // Helper to create a CommitQC for a specific index - makeQC := func(prev utils.Option[*types.CommitQC], laneQCs map[types.LaneID]*types.LaneQC) *types.CommitQC { - vs := types.ViewSpec{CommitQC: prev, Epoch: types.NewEpoch(0, types.OpenRoadRange(), time.Time{}, committee, initialBlock)} - fullProposal := utils.OrPanic1(types.NewProposal( - leaderKey(committee, keys, vs.View()), - vs, - time.Now(), - laneQCs, - utils.None[*types.AppQC](), - )) - vote := types.NewCommitVote(fullProposal.Proposal().Msg()) - var votes []*types.Signed[*types.CommitVote] - for _, k := range keys { - votes = append(votes, types.Sign(k, vote)) - } - return types.NewCommitQC(votes) - } - - // 1. Produce a block so we have a non-empty range - lane := keys[0].Public() - p := types.GenPayload(rng) - b, err := state.ProduceLocalBlock(state.NextBlock(lane), p) - require.NoError(t, err) - - // 2. Form a LaneQC for it - laneQC := types.NewLaneQC(makeLaneVotes( - types.TestKeysWithWeight(committee, keys, committee.LaneQuorum()), - b.Msg().Block().Header(), - )) - - // 3. Create CommitQC for index 0 (finalizes block 0) - qc0 := makeQC(utils.None[*types.CommitQC](), map[types.LaneID]*types.LaneQC{lane: laneQC}) - require.Equal(t, initialBlock, qc0.GlobalRange().First) - require.Equal(t, initialBlock+1, qc0.GlobalRange().Next) - - t.Run("PushAppQC mismatch", func(t *testing.T) { - require := require.New(t) - // AppQC for index 1, but paired with CommitQC for index 0 - appProposal1 := types.GenAppProposal(rng) - appQC1 := types.NewAppQC(makeAppVotes(keys, appProposal1)) - - err := state.PushAppQC(appQC1, qc0) - require.Error(err) - }) -} - func TestPushBlockRejectsBadParentHash(t *testing.T) { ctx := t.Context() rng := utils.TestRng() @@ -451,8 +345,6 @@ func TestPushBlockRejectsWrongSigner(t *testing.T) { func TestNewStateWithPersistence(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 4) - initialBlock := registry.FirstBlock() - t.Run("empty dir loads fresh state", func(t *testing.T) { dir := t.TempDir() ds := newTestDataState(&data.Config{Registry: registry}) @@ -460,52 +352,10 @@ func TestNewStateWithPersistence(t *testing.T) { state, err := NewState(keys[0], ds, utils.Some(dir)) require.NoError(t, err) - // No persisted AppQC → None. - require.False(t, state.LastAppQC().IsPresent()) // Queues start at 0. require.Equal(t, types.RoadIndex(0), state.FirstCommitQC()) }) - t.Run("loads persisted AppQC", func(t *testing.T) { - dir := t.TempDir() - ds := newTestDataState(&data.Config{Registry: registry}) - - roadIdx := types.RoadIndex(7) - - // Persist commitQCs 0-7 so the matching one at roadIdx exists. - cp, _, err := persist.NewCommitQCPersister(utils.Some(dir)) - require.NoError(t, err) - prev := utils.None[*types.CommitQC]() - var pruneQC *types.CommitQC - for i := types.RoadIndex(0); i <= roadIdx; i++ { - qc := makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qc) - require.NoError(t, cp.MaybePruneAndPersist(utils.None[*types.CommitQC](), []*types.CommitQC{qc}, noCommitQCCB)) - pruneQC = qc - } - appProposal := types.NewAppProposal(pruneQC.Proposal(), types.GenAppHash(rng)) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - // Persist prune anchor (AppQC + CommitQC pair). - prunePers, _, err := persist.NewPersister[*pb.PersistedAvailPruneAnchor](utils.Some(dir), innerFile) - require.NoError(t, err) - require.NoError(t, prunePers.Persist(&pb.PersistedAvailPruneAnchor{ - AppQc: types.AppQCConv.Encode(appQC), - CommitQc: types.CommitQCConv.Encode(pruneQC), - })) - - state, err := NewState(keys[0], ds, utils.Some(dir)) - require.NoError(t, err) - - aq := state.LastAppQC() - got, ok := aq.Get() - require.True(t, ok) - require.Equal(t, roadIdx, got.Proposal().RoadIndex()) - require.Equal(t, pruneQC.GlobalRange().First, got.Proposal().GlobalFirst()) - - require.Equal(t, roadIdx, state.FirstCommitQC()) - }) - t.Run("loads persisted blocks", func(t *testing.T) { dir := t.TempDir() ds := newTestDataState(&data.Config{Registry: registry}) @@ -520,7 +370,7 @@ func TestNewStateWithPersistence(t *testing.T) { block := types.NewBlock(lane, n, parent, types.GenPayload(rng)) signed := types.Sign(keys[0], types.NewLaneProposal(block)) parent = block.Header().Hash() - require.NoError(t, bp.MaybePruneAndPersistLane(lane, utils.None[*types.CommitQC](), []*types.Signed[*types.LaneProposal]{signed}, noBlockCB)) + require.NoError(t, bp.Persist(lane, 0, []*types.Signed[*types.LaneProposal]{signed}, noBlockCB)) } // Now construct state — it should load the blocks. @@ -530,58 +380,6 @@ func TestNewStateWithPersistence(t *testing.T) { require.Equal(t, types.BlockNumber(3), state.NextBlock(lane)) }) - t.Run("loads persisted AppQC and blocks together", func(t *testing.T) { - dir := t.TempDir() - ds := newTestDataState(&data.Config{Registry: registry}) - lane := keys[0].Public() - - roadIdx := types.RoadIndex(2) - - // Persist commitQCs 0-2 so the matching one at roadIdx exists. - cp, _, err := persist.NewCommitQCPersister(utils.Some(dir)) - require.NoError(t, err) - prev := utils.None[*types.CommitQC]() - var pruneQC *types.CommitQC - for range roadIdx + 1 { - qc := makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qc) - require.NoError(t, cp.MaybePruneAndPersist(utils.None[*types.CommitQC](), []*types.CommitQC{qc}, noCommitQCCB)) - pruneQC = qc - } - appProposal := types.NewAppProposal(pruneQC.Proposal(), types.GenAppHash(rng)) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - // Persist prune anchor (AppQC + CommitQC pair). - prunePers, _, err := persist.NewPersister[*pb.PersistedAvailPruneAnchor](utils.Some(dir), innerFile) - require.NoError(t, err) - require.NoError(t, prunePers.Persist(&pb.PersistedAvailPruneAnchor{ - AppQc: types.AppQCConv.Encode(appQC), - CommitQc: types.CommitQCConv.Encode(pruneQC), - })) - - // Persist blocks starting at 0 (nil laneQCs → lr.First()=0 after prune). - bp, _, err := persist.NewBlockPersister(utils.Some(dir)) - require.NoError(t, err) - - var parent types.BlockHeaderHash - for n := range types.BlockNumber(3) { - block := types.NewBlock(lane, n, parent, types.GenPayload(rng)) - signed := types.Sign(keys[0], types.NewLaneProposal(block)) - parent = block.Header().Hash() - require.NoError(t, bp.MaybePruneAndPersistLane(lane, utils.None[*types.CommitQC](), []*types.Signed[*types.LaneProposal]{signed}, noBlockCB)) - } - - state, err := NewState(keys[0], ds, utils.Some(dir)) - require.NoError(t, err) - - got, ok := state.LastAppQC().Get() - require.True(t, ok) - require.Equal(t, roadIdx, got.Proposal().RoadIndex()) - - require.Equal(t, types.BlockNumber(3), state.NextBlock(lane)) - require.Equal(t, roadIdx, state.FirstCommitQC()) - }) - t.Run("loads persisted commitQCs", func(t *testing.T) { dir := t.TempDir() ds := newTestDataState(&data.Config{Registry: registry}) @@ -593,9 +391,9 @@ func TestNewStateWithPersistence(t *testing.T) { qcs := make([]*types.CommitQC, 3) prev := utils.None[*types.CommitQC]() for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) + qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil) prev = utils.Some(qcs[i]) - require.NoError(t, cp.MaybePruneAndPersist(utils.None[*types.CommitQC](), []*types.CommitQC{qcs[i]}, noCommitQCCB)) + require.NoError(t, cp.Persist(0, []*types.CommitQC{qcs[i]})) } state, err := NewState(keys[0], ds, utils.Some(dir)) @@ -607,43 +405,6 @@ func TestNewStateWithPersistence(t *testing.T) { require.NoError(t, utils.TestDiff(utils.Some(qcs[2]), state.LastCommitQC().Load())) }) - t.Run("loads persisted commitQCs with AppQC", func(t *testing.T) { - dir := t.TempDir() - ds := newTestDataState(&data.Config{Registry: registry}) - - // Persist AppQC at road index 1. - roadIdx := types.RoadIndex(1) - - // Persist CommitQCs 0-4. - cp, _, err := persist.NewCommitQCPersister(utils.Some(dir)) - require.NoError(t, err) - - qcs := make([]*types.CommitQC, 5) - prev := utils.None[*types.CommitQC]() - for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[i]) - require.NoError(t, cp.MaybePruneAndPersist(utils.None[*types.CommitQC](), []*types.CommitQC{qcs[i]}, noCommitQCCB)) - } - appProposal := types.NewAppProposal(qcs[roadIdx].Proposal(), types.GenAppHash(rng)) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - - // Persist prune anchor (AppQC + CommitQC pair at roadIdx). - prunePers, _, err := persist.NewPersister[*pb.PersistedAvailPruneAnchor](utils.Some(dir), innerFile) - require.NoError(t, err) - require.NoError(t, prunePers.Persist(&pb.PersistedAvailPruneAnchor{ - AppQc: types.AppQCConv.Encode(appQC), - CommitQc: types.CommitQCConv.Encode(qcs[roadIdx]), - })) - - state, err := NewState(keys[0], ds, utils.Some(dir)) - require.NoError(t, err) - - // inner.prune(appQC@1, commitQC@1) sets commitQCs.first = 1. - require.Equal(t, types.RoadIndex(1), state.FirstCommitQC()) - require.NoError(t, utils.TestDiff(utils.Some(qcs[4]), state.LastCommitQC().Load())) - }) - t.Run("non-contiguous commitQC files return error", func(t *testing.T) { dir := t.TempDir() @@ -651,140 +412,22 @@ func TestNewStateWithPersistence(t *testing.T) { allQCs := make([]*types.CommitQC, 6) prev := utils.None[*types.CommitQC]() for i := range allQCs { - allQCs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) + allQCs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil) prev = utils.Some(allQCs[i]) } - // Persist prune anchor (AppQC + CommitQC pair at road index 0). - appProposal := types.NewAppProposal(allQCs[0].Proposal(), types.GenAppHash(rng)) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - prunePers, _, err := persist.NewPersister[*pb.PersistedAvailPruneAnchor](utils.Some(dir), innerFile) - require.NoError(t, err) - require.NoError(t, prunePers.Persist(&pb.PersistedAvailPruneAnchor{ - AppQc: types.AppQCConv.Encode(appQC), - CommitQc: types.CommitQCConv.Encode(allQCs[0]), - })) - // Persist QCs 0, 1, 2 contiguously, then try to skip to 5. - // MaybePruneAndPersist enforces strict sequential order, so the gap + // Persist enforces strict sequential order, so the gap // is caught at write time rather than at load time. cp, _, err := persist.NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) for i := range 3 { - require.NoError(t, cp.MaybePruneAndPersist(utils.None[*types.CommitQC](), []*types.CommitQC{allQCs[i]}, noCommitQCCB)) + require.NoError(t, cp.Persist(0, []*types.CommitQC{allQCs[i]})) } - err = cp.MaybePruneAndPersist(utils.None[*types.CommitQC](), []*types.CommitQC{allQCs[5]}, noCommitQCCB) + err = cp.Persist(0, []*types.CommitQC{allQCs[5]}) require.Error(t, err) require.Contains(t, err.Error(), "out of sequence") require.NoError(t, cp.Close()) }) - t.Run("anchor past all persisted commitQCs truncates WAL", func(t *testing.T) { - dir := t.TempDir() - ds := newTestDataState(&data.Config{Registry: registry}) - - // Build a chain of 10 CommitQCs (indices 0-9). - qcs := make([]*types.CommitQC, 10) - prev := utils.None[*types.CommitQC]() - for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[i]) - } - - // Persist only indices 0-4 to the CommitQC WAL. - cp, _, err := persist.NewCommitQCPersister(utils.Some(dir)) - require.NoError(t, err) - for i := range 5 { - require.NoError(t, cp.MaybePruneAndPersist(utils.None[*types.CommitQC](), []*types.CommitQC{qcs[i]}, noCommitQCCB)) - } - require.NoError(t, cp.Close()) - - // Persist a prune anchor at index 9 — well past the persisted range. - appProposal := types.NewAppProposal(qcs[9].Proposal(), types.GenAppHash(rng)) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - prunePers, _, err := persist.NewPersister[*pb.PersistedAvailPruneAnchor](utils.Some(dir), innerFile) - require.NoError(t, err) - require.NoError(t, prunePers.Persist(&pb.PersistedAvailPruneAnchor{ - AppQc: types.AppQCConv.Encode(appQC), - CommitQc: types.CommitQCConv.Encode(qcs[9]), - })) - - // NewState should succeed: MaybePruneAndPersist truncates the stale WAL - // and internally re-persists the anchor's CommitQC for crash recovery. - state, err := NewState(keys[0], ds, utils.Some(dir)) - require.NoError(t, err) - - require.Equal(t, types.RoadIndex(9), state.FirstCommitQC()) - require.NoError(t, utils.TestDiff(utils.Some(qcs[9]), state.LastCommitQC().Load())) - - got, ok := state.LastAppQC().Get() - require.True(t, ok) - require.Equal(t, types.RoadIndex(9), got.Proposal().RoadIndex()) - }) - - t.Run("anchor past all persisted blocks truncates lane WAL", func(t *testing.T) { - dir := t.TempDir() - ds := newTestDataState(&data.Config{Registry: registry}) - lane := keys[0].Public() - - // Persist commitQCs 0-9 and blocks 0-2 for one lane. - qcs := make([]*types.CommitQC, 10) - prev := utils.None[*types.CommitQC]() - cp, _, err := persist.NewCommitQCPersister(utils.Some(dir)) - require.NoError(t, err) - for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil, utils.None[*types.AppQC]()) - prev = utils.Some(qcs[i]) - require.NoError(t, cp.MaybePruneAndPersist(utils.None[*types.CommitQC](), []*types.CommitQC{qcs[i]}, noCommitQCCB)) - } - require.NoError(t, cp.Close()) - - bp, _, err := persist.NewBlockPersister(utils.Some(dir)) - require.NoError(t, err) - var parent types.BlockHeaderHash - for n := range types.BlockNumber(3) { - block := types.NewBlock(lane, n, parent, types.GenPayload(rng)) - signed := types.Sign(keys[0], types.NewLaneProposal(block)) - parent = block.Header().Hash() - require.NoError(t, bp.MaybePruneAndPersistLane(lane, utils.None[*types.CommitQC](), []*types.Signed[*types.LaneProposal]{signed}, noBlockCB)) - } - - // Persist a prune anchor at index 9 with a laneRange that starts past - // all persisted blocks — MaybePruneAndPersistLane will TruncateAll the block WAL. - appProposal := types.NewAppProposal(qcs[9].Proposal(), types.GenAppHash(rng)) - appQC := types.NewAppQC(makeAppVotes(keys, appProposal)) - prunePers, _, err := persist.NewPersister[*pb.PersistedAvailPruneAnchor](utils.Some(dir), innerFile) - require.NoError(t, err) - require.NoError(t, prunePers.Persist(&pb.PersistedAvailPruneAnchor{ - AppQc: types.AppQCConv.Encode(appQC), - CommitQc: types.CommitQCConv.Encode(qcs[9]), - })) - - // NewState should succeed: block WAL gets truncated, lane starts clean. - state, err := NewState(keys[0], ds, utils.Some(dir)) - require.NoError(t, err) - - require.Equal(t, types.RoadIndex(9), state.FirstCommitQC()) - got, ok := state.LastAppQC().Get() - require.True(t, ok) - require.Equal(t, types.RoadIndex(9), got.Proposal().RoadIndex()) - }) - - t.Run("corrupt AppQC data returns error", func(t *testing.T) { - dir := t.TempDir() - ds := newTestDataState(&data.Config{Registry: registry}) - - // Create a throwaway persister to discover the A/B filenames, - // then corrupt them so NewState fails on load. - _, _, err := persist.NewPersister[*pb.PersistedAvailPruneAnchor](utils.Some(dir), innerFile) - require.NoError(t, err) - entries, err := os.ReadDir(dir) - require.NoError(t, err) - for _, e := range entries { - require.NoError(t, os.WriteFile(filepath.Join(dir, e.Name()), []byte("corrupt"), 0600)) - } - - _, err = NewState(keys[0], ds, utils.Some(dir)) - require.Error(t, err) - }) } From d7af2d19be250f188e65b343e71466cedc924124 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 6 Aug 2026 19:05:38 +0200 Subject: [PATCH 17/61] some fixes --- .../internal/autobahn/avail/inner_test.go | 8 ++-- .../internal/autobahn/avail/state.go | 19 +++++---- .../internal/autobahn/avail/state_test.go | 41 +++++++------------ .../autobahn/data/state_recovery_test.go | 4 +- 4 files changed, 30 insertions(+), 42 deletions(-) diff --git a/sei-tendermint/internal/autobahn/avail/inner_test.go b/sei-tendermint/internal/autobahn/avail/inner_test.go index bbea34ae7a..ea6e92895e 100644 --- a/sei-tendermint/internal/autobahn/avail/inner_test.go +++ b/sei-tendermint/internal/autobahn/avail/inner_test.go @@ -148,7 +148,7 @@ func TestNewInnerLoadedCommitQCsNoAppQC(t *testing.T) { qcs := make([]*types.CommitQC, 3) prev := utils.None[*types.CommitQC]() for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil) + qcs[i] = types.BuildCommitQC(registry.LatestEpoch(), keys, prev, nil) prev = utils.Some(qcs[i]) } @@ -167,9 +167,9 @@ func TestNewInnerLoadedCommitQCsGapReturnsError(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 4) - qc0 := makeCommitQC(registry.LatestEpoch(), keys, utils.None[*types.CommitQC](), nil) - qc1 := makeCommitQC(registry.LatestEpoch(), keys, utils.Some(qc0), nil) - qc2 := makeCommitQC(registry.LatestEpoch(), keys, utils.Some(qc1), nil) + qc0 := types.BuildCommitQC(registry.LatestEpoch(), keys, utils.None[*types.CommitQC](), nil) + qc1 := types.BuildCommitQC(registry.LatestEpoch(), keys, utils.Some(qc0), nil) + qc2 := types.BuildCommitQC(registry.LatestEpoch(), keys, utils.Some(qc1), nil) _, err := newInner(newTestDataState(&data.Config{Registry: registry}), &loadedState{commitQCs: []*types.CommitQC{qc0, qc2}}) require.Error(t, err) diff --git a/sei-tendermint/internal/autobahn/avail/state.go b/sei-tendermint/internal/autobahn/avail/state.go index b897a183dc..0c32ca62b7 100644 --- a/sei-tendermint/internal/autobahn/avail/state.go +++ b/sei-tendermint/internal/autobahn/avail/state.go @@ -50,9 +50,6 @@ type persisters struct { commitQCs *persist.CommitQCPersister } -// innerFile is the A/B file prefix for avail inner state persistence. -const innerFile = "avail_inner" - // loadPersistedState creates persisters for the given directory option and loads // any existing state from disk. When dir is None, all persisters are no-op // and no state is loaded. When a prune anchor is present, stale commitQCs and @@ -195,7 +192,7 @@ func (s *State) PushAppVote(ctx context.Context, v *types.Signed[*types.AppVote] return fmt.Errorf("v.VerifySig(): %w", err) } for inner, ctrl := range s.inner.Lock() { - if idx < inner.roads.first || inner.roads.next >= idx { + if idx < inner.roads.first || inner.roads.next <= idx { return nil } inner.roads.q[idx].pushAppVote(v) @@ -295,14 +292,20 @@ func (s *State) PushBlock(ctx context.Context, p *types.Signed[*types.LanePropos // Waits until the lane has enough capacity for the new vote. // It does NOT wait for the previous votes. func (s *State) PushVote(ctx context.Context, vote *types.Signed[*types.LaneVote]) error { - if _, err := s.data.Registry().VerifyInWindow(func(c *types.Committee) error { - if err := vote.Msg().Verify(c); err != nil { + var epoch *types.Epoch + for inner,ctrl := range s.inner.Lock() { + // TODO(gprusak): we should wait only if LaneID is from the future. + if err:=ctrl.WaitUntil(ctx, func() bool { return inner.epoch.Committee().HasLane(vote.Key()) }); err!=nil { return err } - return vote.VerifySig(c) - }); err != nil { + epoch = inner.epoch + } + if err := vote.Msg().Verify(epoch.Committee()); err != nil { return fmt.Errorf("vote.Verify(): %w", err) } + if err := vote.VerifySig(epoch.Committee()); err!=nil { + return fmt.Errorf("vote.VerifySig(): %w", err) + } h := vote.Msg().Header() for inner, ctrl := range s.inner.Lock() { q, ok := inner.votes[h.Lane()] diff --git a/sei-tendermint/internal/autobahn/avail/state_test.go b/sei-tendermint/internal/autobahn/avail/state_test.go index 0cbfba0f44..e2eab29b5f 100644 --- a/sei-tendermint/internal/autobahn/avail/state_test.go +++ b/sei-tendermint/internal/autobahn/avail/state_test.go @@ -38,15 +38,6 @@ func makeLaneVotes(keys []types.SecretKey, h *types.BlockHeader) []*types.Signed return votes } -func makeCommitQC( - ep *types.Epoch, - keys []types.SecretKey, - prev utils.Option[*types.CommitQC], - laneQCs map[types.LaneID]*types.LaneQC, -) *types.CommitQC { - return types.BuildCommitQC(ep, keys, prev, laneQCs) -} - func qcPayloadHashes(qc *types.FullCommitQC) byLane[types.PayloadHash] { x := byLane[types.PayloadHash]{} for _, h := range qc.Headers() { @@ -56,8 +47,8 @@ func qcPayloadHashes(qc *types.FullCommitQC) byLane[types.PayloadHash] { } func TestState(t *testing.T) { - t.Skip("requires current AppQC/prune progression semantics to be recharacterized") - testState(t, utils.None[string]()) + rng := utils.TestRng() + testState(t, rng, utils.None[string]()) } // TestStateWithPersistence runs the same flow as TestState but with disk @@ -65,29 +56,26 @@ func TestState(t *testing.T) { // run concurrently, exercising the cursor-clamp logic that prevents reading // pruned map entries. func TestStateWithPersistence(t *testing.T) { - t.Skip("requires current AppQC/prune progression semantics to be recharacterized") + rng := utils.TestRng() for range 5 { - testState(t, utils.Some(t.TempDir())) + testState(t, rng, utils.Some(t.TempDir())) } } -func testState(t *testing.T, stateDir utils.Option[string]) { +func testState(t *testing.T, rng utils.Rng, stateDir utils.Option[string]) { t.Helper() ctx := t.Context() - rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 3) committee := registry.LatestEpoch().Committee() if err := scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { ds := newTestDataState(&data.Config{Registry: registry}) - s.SpawnBgNamed("data.State.Run()", func() error { - return utils.IgnoreCancel(ds.Run(ctx)) - }) + s.SpawnBgNamed("ds.Run()", func() error { return utils.IgnoreCancel(ds.Run(ctx)) }) state, err := NewState(keys[0], ds, stateDir) - require.NoError(t, err) - s.SpawnBgNamed("da.State.Run()", func() error { - return utils.IgnoreCancel(state.Run(ctx)) - }) + if err != nil { + return fmt.Errorf("NewState(): %w", err) + } + s.SpawnBgNamed("state.Run()", func() error { return utils.IgnoreCancel(state.Run(ctx)) }) for i := range 3 { t.Logf("iteration %v", i) @@ -130,7 +118,7 @@ func testState(t *testing.T, stateDir utils.Option[string]) { if err != nil { return fmt.Errorf("state.WaitForNewLaneQCs(): %w", err) } - qc := makeCommitQC(registry.LatestEpoch(), keys, prev, laneQCs) + qc := types.BuildCommitQC(registry.LatestEpoch(), keys, prev, laneQCs) if err := state.PushCommitQC(ctx, qc); err != nil { return fmt.Errorf("state.PushCommitQC(): %w", err) } @@ -199,7 +187,6 @@ func testState(t *testing.T, stateDir utils.Option[string]) { // stale entries by shutdown, restart exercises the gap-filtering path in // loadPersistedState (stale entries below the prune anchor are discarded). func TestStateRestartFromPersisted(t *testing.T) { - t.Skip("requires current restart semantics for data.State anchor and avail persistence to be recharacterized") rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 3) committee := registry.LatestEpoch().Committee() @@ -252,7 +239,7 @@ func TestStateRestartFromPersisted(t *testing.T) { if err != nil { return fmt.Errorf("WaitForLaneQCs: %w", err) } - qc := makeCommitQC(registry.LatestEpoch(), keys, prev, laneQCs) + qc := types.BuildCommitQC(registry.LatestEpoch(), keys, prev, laneQCs) if err := state.PushCommitQC(ctx, qc); err != nil { return fmt.Errorf("PushCommitQC: %w", err) } @@ -391,7 +378,7 @@ func TestNewStateWithPersistence(t *testing.T) { qcs := make([]*types.CommitQC, 3) prev := utils.None[*types.CommitQC]() for i := range qcs { - qcs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil) + qcs[i] = types.BuildCommitQC(registry.LatestEpoch(), keys, prev, nil) prev = utils.Some(qcs[i]) require.NoError(t, cp.Persist(0, []*types.CommitQC{qcs[i]})) } @@ -412,7 +399,7 @@ func TestNewStateWithPersistence(t *testing.T) { allQCs := make([]*types.CommitQC, 6) prev := utils.None[*types.CommitQC]() for i := range allQCs { - allQCs[i] = makeCommitQC(registry.LatestEpoch(), keys, prev, nil) + allQCs[i] = types.BuildCommitQC(registry.LatestEpoch(), keys, prev, nil) prev = utils.Some(allQCs[i]) } diff --git a/sei-tendermint/internal/autobahn/data/state_recovery_test.go b/sei-tendermint/internal/autobahn/data/state_recovery_test.go index 3755a25199..90efcdf372 100644 --- a/sei-tendermint/internal/autobahn/data/state_recovery_test.go +++ b/sei-tendermint/internal/autobahn/data/state_recovery_test.go @@ -369,9 +369,7 @@ func TestRecoveryPartialQCPrefix(t *testing.T) { qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) gr1 := qc1.QC().GlobalRange() - if gr1.Next-gr1.First < 3 { - t.Skip("need at least 3 blocks in QC range to test split") - } + require.True(t,gr1.Next-gr1.First < 3, "need at least 3 blocks in QC range to test split") // Write the QC for the full range, but write blocks only from mid onwards. mid := gr1.First + (gr1.Next-gr1.First)/2 From 519733f82e1d047839543b98b9b9bb893d58f6e0 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 7 Aug 2026 12:09:17 +0200 Subject: [PATCH 18/61] some test fixes --- .../internal/autobahn/avail/inner.go | 9 ++++++++- .../internal/autobahn/avail/state.go | 6 +++--- .../internal/autobahn/avail/state_test.go | 18 +++++++++--------- .../internal/autobahn/avail/testonly.go | 2 +- sei-tendermint/internal/autobahn/data/state.go | 2 ++ 5 files changed, 23 insertions(+), 14 deletions(-) diff --git a/sei-tendermint/internal/autobahn/avail/inner.go b/sei-tendermint/internal/autobahn/avail/inner.go index f28b702729..1c41c7f3c1 100644 --- a/sei-tendermint/internal/autobahn/avail/inner.go +++ b/sei-tendermint/internal/autobahn/avail/inner.go @@ -154,7 +154,14 @@ func (i *inner) updateNextAppQC() bool { } // prune advances the state to account for a new AppQC/CommitQC pair. -// Returns true if pruning occurred, false if the QC was stale. +// Returns true iff pruning occurred. +// It is safe to prune on data.Anchor, because it proves that: +// * AppQC was formed for the given height (and it will be available on restart) +// * some honest nodes have voted for AppHash +// * AppHash voting is allowed only after persisting the executed blocks i blockDB. +// * blocks and FullCommitQC (sequencing proof) are available in data.State. +// TODO(gprusak): consider simplifying this invariant by making Anchor require +// locally persisted blocks as well. func (i *inner) prune(epoch *types.Epoch, anchor data.Anchor) { idx := anchor.CommitQC.Index() if idx < i.roads.first { diff --git a/sei-tendermint/internal/autobahn/avail/state.go b/sei-tendermint/internal/autobahn/avail/state.go index 0c32ca62b7..77c60688b3 100644 --- a/sei-tendermint/internal/autobahn/avail/state.go +++ b/sei-tendermint/internal/autobahn/avail/state.go @@ -88,7 +88,7 @@ func NewState(key types.SecretKey, data *data.State, stateDir utils.Option[strin }, nil } -func (s *State) FirstCommitQC() types.RoadIndex { +func (s *State) First() types.RoadIndex { for inner := range s.inner.Lock() { return inner.roads.first } @@ -458,7 +458,7 @@ func (s *State) produceLocalBlock(n types.BlockNumber, key types.SecretKey, payl // Task inserting CommitQCs and local blocks to data state. func (s *State) runPushQC(ctx context.Context) error { - for n := types.RoadIndex(0); ; n = max(n+1, s.FirstCommitQC()) { + for n := types.RoadIndex(0); ; n = max(n+1, s.First()) { epoch, qc, err := s.fullCommitQC(ctx, n) if err != nil { if errors.Is(err, types.ErrPruned) { @@ -491,7 +491,7 @@ func (s *State) runPushQC(ctx context.Context) error { // Task inserting AppQCs to data state. func (s *State) runPushAppQC(ctx context.Context) error { - for n := types.RoadIndex(0); ; n = max(n+1, s.FirstCommitQC()) { + for n := types.RoadIndex(0); ; n = max(n+1, s.First()) { appQC, err := s.appQC(ctx, n) if err != nil { if errors.Is(err, types.ErrPruned) { diff --git a/sei-tendermint/internal/autobahn/avail/state_test.go b/sei-tendermint/internal/autobahn/avail/state_test.go index e2eab29b5f..01e2ce3ab8 100644 --- a/sei-tendermint/internal/autobahn/avail/state_test.go +++ b/sei-tendermint/internal/autobahn/avail/state_test.go @@ -131,15 +131,15 @@ func testState(t *testing.T, rng utils.Rng, stateDir utils.Option[string]) { } } - t.Logf("Previous one should be pruned because of appQC.") + t.Logf("Previous one should be eventually evicted") + for inner,ctrl := range state.inner.Lock() { + if err:=ctrl.WaitUntil(ctx, func() bool { return inner.roads.first == appProposal.RoadIndex() }); err!=nil { + return err + } + } if _, err := state.appQC(ctx, appProposal.RoadIndex()); err != nil { return fmt.Errorf("state.WaitForAppQC(): %w", err) } - if prev, ok := prev.Get(); ok { - if _, err := state.CommitQC(ctx, prev.Proposal().Index()); !errors.Is(err, types.ErrPruned) { - return fmt.Errorf("state.CommitQC(): %w, want %v", err, types.ErrPruned) - } - } t.Logf("Check that the executed local blocks have been pruned") for lane := range committee.Lanes().All() { @@ -278,7 +278,7 @@ func TestStateRestartFromPersisted(t *testing.T) { state2, err := NewState(keys[0], ds2, utils.Some(dir)) require.NoError(t, err) - require.GreaterOrEqual(t, state2.FirstCommitQC(), wantAppQCIdx) + require.GreaterOrEqual(t, state2.First(), wantAppQCIdx) _, ok := state2.LastCommitQC().Load().Get() require.True(t, ok, "LastCommitQC should be set after restart") @@ -340,7 +340,7 @@ func TestNewStateWithPersistence(t *testing.T) { require.NoError(t, err) // Queues start at 0. - require.Equal(t, types.RoadIndex(0), state.FirstCommitQC()) + require.Equal(t, types.RoadIndex(0), state.First()) }) t.Run("loads persisted blocks", func(t *testing.T) { @@ -387,7 +387,7 @@ func TestNewStateWithPersistence(t *testing.T) { require.NoError(t, err) // All 3 commitQCs should be loaded (no AppQC to skip past). - require.Equal(t, types.RoadIndex(0), state.FirstCommitQC()) + require.Equal(t, types.RoadIndex(0), state.First()) // LastCommitQC should be set to the last loaded one. require.NoError(t, utils.TestDiff(utils.Some(qcs[2]), state.LastCommitQC().Load())) }) diff --git a/sei-tendermint/internal/autobahn/avail/testonly.go b/sei-tendermint/internal/autobahn/avail/testonly.go index eb70708076..27690e4813 100644 --- a/sei-tendermint/internal/autobahn/avail/testonly.go +++ b/sei-tendermint/internal/autobahn/avail/testonly.go @@ -56,7 +56,7 @@ func RunTestNetwork(ctx context.Context, states []*State) error { qc, err := from.CommitQC(ctx, next) if err != nil { if errors.Is(err, types.ErrPruned) { - next = from.FirstCommitQC() + next = from.First() continue } return err diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index 28a9044538..9d045cc41b 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -685,6 +685,8 @@ type Anchor struct { AppQC *types.AppQC } +// Anchor represents the latest persisted AppQC. +// It is used by avail.State. func (s *State) Anchor() utils.AtomicRecv[utils.Option[Anchor]] { for inner := range s.inner.Lock() { return inner.anchor.Subscribe() From 85efdec999c7342b5697c66bfdcb48c95ad1adfe Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 7 Aug 2026 12:42:43 +0200 Subject: [PATCH 19/61] appProposal persistence. --- sei-db/ledger_db/block/block_db_test.go | 90 +++++- sei-db/ledger_db/block/littblock/codec.go | 42 ++- .../block/littblock/litt_block_db.go | 274 ++++++++++++------ .../ledger_db/block/memblock/mem_block_db.go | 130 ++++++++- sei-tendermint/autobahn/types/block_db.go | 41 ++- sei-tendermint/autobahn/types/errors.go | 11 + .../internal/autobahn/data/state.go | 69 ++++- .../autobahn/data/state_recovery_test.go | 2 +- 8 files changed, 540 insertions(+), 119 deletions(-) diff --git a/sei-db/ledger_db/block/block_db_test.go b/sei-db/ledger_db/block/block_db_test.go index 7909811c26..e346bda966 100644 --- a/sei-db/ledger_db/block/block_db_test.go +++ b/sei-db/ledger_db/block/block_db_test.go @@ -53,6 +53,7 @@ func TestBlockDB(t *testing.T) { t.Run("EmptyDB", func(t *testing.T) { testEmptyDB(t, impl.build) }) t.Run("ReadRoundTrip", func(t *testing.T) { testReadRoundTrip(t, impl.build) }) t.Run("QCByBlockNumber", func(t *testing.T) { testQCByBlockNumber(t, impl.build) }) + t.Run("AppProposalByBlockNumber", func(t *testing.T) { testAppProposalByBlockNumber(t, impl.build) }) t.Run("AppQCByBlockNumber", func(t *testing.T) { testAppQCByBlockNumber(t, impl.build) }) t.Run("ReadRecent", func(t *testing.T) { testReadRecent(t, impl.build) }) t.Run("RestartPersistsData", func(t *testing.T) { testRestartPersistsData(t, impl.build) }) @@ -129,12 +130,17 @@ func testEmptyDB(t *testing.T, build builder) { require.NoError(t, err) require.False(t, appQC.IsPresent()) + appProposal, err := db.ReadAppProposalByBlockNumber(0) + require.NoError(t, err) + require.False(t, appProposal.IsPresent()) + require.Empty(t, drainRecent(t, db), "empty db should yield no recent records") tips := db.Status() require.Zero(t, tips.NextBlock, "empty db has no block write tip") require.Zero(t, tips.NextQC, "empty db has no QC write tip") require.Zero(t, tips.NextAppQC, "empty db has no AppQC write tip") + require.Zero(t, tips.NextAppProposal, "empty db has no AppProposal write tip") } // iterEntry is one position observed while draining an iterator. @@ -150,6 +156,9 @@ type iterEntry struct { // appQC is the AppQC at the position; nil when no AppQC is persisted there. appQC *types.AppQC + + // appProposal is the AppProposal at the position; nil when no AppProposal is persisted there. + appProposal *types.AppProposal } // drainRecent reads the recovery-visible recent batch. @@ -161,7 +170,7 @@ func drainRecent(t *testing.T, db types.BlockDB) []iterEntry { for _, qc := range recent.CommitQCs { first := qc.QC().GlobalRange().First next := first + gbn(len(qc.Headers())) - for n := first; n < next; n++ { + for n := max(first, recent.First); n < next; n++ { entries = append(entries, iterEntry{n: n, qc: qc}) } } @@ -184,6 +193,14 @@ func drainRecent(t *testing.T, db types.BlockDB) []iterEntry { } } } + for _, appProposal := range recent.AppProposals { + gr := appProposal.GlobalRange() + for i := range entries { + if gr.Has(entries[i].n) { + entries[i].appProposal = appProposal + } + } + } return entries } @@ -278,6 +295,13 @@ func assertTipsMatchPresent(t *testing.T, db types.BlockDB) { require.True(t, ok, "NextAppQC must point past a readable AppQC") require.Equal(t, tips.NextAppQC, got.Proposal().GlobalRange().Next) } + if tips.NextAppProposal != 0 { + appProposal, err := db.ReadAppProposalByBlockNumber(tips.NextAppProposal - 1) + require.NoError(t, err) + got, ok := appProposal.Get() + require.True(t, ok, "NextAppProposal must point past a readable AppProposal") + require.Equal(t, tips.NextAppProposal, got.GlobalRange().Next) + } } func testReadRoundTrip(t *testing.T, build builder) { @@ -314,6 +338,56 @@ func testQCByBlockNumber(t *testing.T, build builder) { require.False(t, miss.IsPresent()) } +func testAppProposalByBlockNumber(t *testing.T, build builder) { + committee, keys := buildCommittee() + batches := generateBatches(committee, keys) + db, o := openFresh(t, build) + defer func() { _ = db.Close() }() + writeAll(t, db, batches) + + rng := utils.TestRngFromSeed(testSeed + 90) + appProposals := []*types.AppProposal{ + appProposalForBatch(rng, batches[0]), + appProposalForBatch(rng, batches[1]), + } + for _, appProposal := range appProposals { + require.NoError(t, db.WriteAppProposal(appProposal)) + } + + for _, appProposal := range appProposals { + gr := appProposal.GlobalRange() + for n := gr.First; n < gr.Next; n++ { + opt, err := db.ReadAppProposalByBlockNumber(n) + require.NoError(t, err) + got, ok := opt.Get() + require.True(t, ok, "AppProposal covering %d should exist", n) + require.Equal(t, gr, got.GlobalRange()) + require.Equal(t, appProposal.AppHash(), got.AppHash()) + } + } + miss, err := db.ReadAppProposalByBlockNumber(batches[2].first) + require.NoError(t, err) + require.False(t, miss.IsPresent(), "CommitQCs/blocks past the AppProposal prefix should not imply AppProposal presence") + + entries := drainRecent(t, db) + for _, e := range entries { + switch { + case e.n < batches[2].first: + require.NotNil(t, e.appProposal, "iterator should expose AppProposal at %d", e.n) + default: + require.Nil(t, e.appProposal, "iterator should not expose AppProposal past the persisted AppProposal prefix at %d", e.n) + } + } + + tips := db.Status() + require.Equal(t, batches[1].next, tips.NextAppProposal) + db = restart(t, o, db) + tips = db.Status() + require.Equal(t, batches[1].next, tips.NextAppProposal, "AppProposal tip must survive restart") + assertTipsMatchPresent(t, db) + +} + func testAppQCByBlockNumber(t *testing.T, build builder) { committee, keys := buildCommittee() batches := generateBatches(committee, keys) @@ -327,6 +401,7 @@ func testAppQCByBlockNumber(t *testing.T, build builder) { appQCForBatch(rng, keys, batches[1]), } for _, appQC := range appQCs { + require.NoError(t, db.WriteAppProposal(appQC.Proposal())) require.NoError(t, db.WriteAppQC(appQC)) } @@ -798,6 +873,7 @@ func testReadRecent(t *testing.T, build builder) { writeAll(t, db, batches[:2]) appQC := appQCForBatch(utils.TestRngFromSeed(testSeed+400), keys, batches[0]) + require.NoError(t, db.WriteAppProposal(appQC.Proposal())) require.NoError(t, db.WriteAppQC(appQC)) require.NoError(t, db.WriteQC(batches[2].qc)) for i, blk := range batches[2].blocks { @@ -810,9 +886,11 @@ func testReadRecent(t *testing.T, build builder) { require.True(t, ok) require.Equal(t, appQC.Proposal().RoadIndex(), gotAppQC.Proposal().RoadIndex()) entries := drainRecent(t, db) + recoveryFloor := appQC.Proposal().GlobalRange().Next - 1 require.Equal(t, []types.GlobalBlockNumber{batches[0].first, batches[1].first, batches[2].first}, qcFirsts(entries)) - require.Equal(t, batches[2].next-batches[0].first, types.GlobalBlockNumber(len(entries))) - require.Len(t, presentBlockNumbers(entries), int(batches[2].next-batches[0].first)) + require.Equal(t, batches[2].next-recoveryFloor, types.GlobalBlockNumber(len(entries))) + require.Equal(t, recoveryFloor, entries[0].n) + require.Len(t, presentBlockNumbers(entries), int(batches[2].next-recoveryFloor)) } func testWriteOrderRejected(t *testing.T, build builder) { @@ -1356,7 +1434,11 @@ func testLaneQC(keys []types.SecretKey, header *types.BlockHeader) *types.LaneQC } func appQCForBatch(rng utils.Rng, keys []types.SecretKey, b batch) *types.AppQC { - return testAppQC(keys, types.NewAppProposal(b.qc.QC().Proposal(), types.GenAppHash(rng))) + return testAppQC(keys, appProposalForBatch(rng, b)) +} + +func appProposalForBatch(rng utils.Rng, b batch) *types.AppProposal { + return types.NewAppProposal(b.qc.QC().Proposal(), types.GenAppHash(rng)) } func testAppQC(keys []types.SecretKey, proposal *types.AppProposal) *types.AppQC { diff --git a/sei-db/ledger_db/block/littblock/codec.go b/sei-db/ledger_db/block/littblock/codec.go index 4abb446c3c..7ec54d2801 100644 --- a/sei-db/ledger_db/block/littblock/codec.go +++ b/sei-db/ledger_db/block/littblock/codec.go @@ -17,11 +17,13 @@ import ( // - kindBlockHash 'h' + 32-byte header hash (block hash alias) // - kindQC 'q' + 8-byte big-endian GlobalBlockNumber (QC primary + covered aliases) // - kindAppQC 'a' + 8-byte big-endian GlobalBlockNumber (AppQC primary + covered aliases) +// - kindAppProp 'p' + 8-byte big-endian GlobalBlockNumber (AppProposal primary + covered aliases) const ( kindBlock byte = 'b' kindBlockHash byte = 'h' kindQC byte = 'q' kindAppQC byte = 'a' + kindAppProp byte = 'p' ) // encodeKey encodes a GlobalBlockNumber as an 8-byte big-endian value. Big-endian @@ -61,6 +63,12 @@ func appQCKey(n types.GlobalBlockNumber) []byte { return append([]byte{kindAppQC}, encodeKey(n)...) } +// appProposalKey returns the key for AppProposal number n — used both for an +// AppProposal's primary key and for each covered-number alias. +func appProposalKey(n types.GlobalBlockNumber) []byte { + return append([]byte{kindAppProp}, encodeKey(n)...) +} + // keyKind returns the kind prefix byte of a stored key. func keyKind(key []byte) byte { return key[0] @@ -82,6 +90,9 @@ const qcSerializationVersion byte = 1 // Serialization version for AppQCs. const appQCSerializationVersion byte = 1 +// Serialization version for AppProposals. +const appProposalSerializationVersion byte = 1 + // blockValuePrefixLen is the fixed header preceding a block's proto bytes: one // version byte followed by the 8-byte big-endian GlobalBlockNumber. const blockValuePrefixLen = 1 + 8 @@ -136,8 +147,8 @@ func encodeQC(qc *types.FullCommitQC) []byte { // so deriving from the encoded fields is what makes a QC's range identical // before and after a round trip through the table. func coveredRange(qc *types.FullCommitQC) (types.GlobalBlockNumber, types.GlobalBlockNumber) { - first := qc.QC().GlobalRange().First - return first, first + types.GlobalBlockNumber(len(qc.Headers())) + gr := qc.QC().GlobalRange() + return gr.First, gr.Next } // decodeQC unmarshals a FullCommitQC from the value produced by encodeQC. @@ -155,9 +166,30 @@ func decodeQC(value []byte) (*types.FullCommitQC, error) { return qc, nil } -func appQCRange(appQC *types.AppQC) (types.GlobalBlockNumber, types.GlobalBlockNumber) { - gr := appQC.Proposal().GlobalRange() - return gr.First, gr.Next +// encodeAppProposal marshals an AppProposal to the bytes stored as its table +// value, framed as [version:1][proto(AppProposal)]. +func encodeAppProposal(appProposal *types.AppProposal) []byte { + proto := types.AppProposalConv.Marshal(appProposal) + value := make([]byte, 0, 1+len(proto)) + value = append(value, appProposalSerializationVersion) + value = append(value, proto...) + return value +} + +// decodeAppProposal unmarshals an AppProposal from the value produced by +// encodeAppProposal. +func decodeAppProposal(value []byte) (*types.AppProposal, error) { + if len(value) < 1 { + return nil, fmt.Errorf("appProposal value too short: %d bytes", len(value)) + } + if value[0] != appProposalSerializationVersion { + return nil, fmt.Errorf("unsupported appProposal serialization version %d", value[0]) + } + appProposal, err := types.AppProposalConv.Unmarshal(value[1:]) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal appProposal: %w", err) + } + return appProposal, nil } // encodeAppQC marshals an AppQC to the bytes stored as its table value, diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index e71285b3e7..7debf5b89e 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -46,18 +46,9 @@ type blockDB struct { mu sync.Mutex hasBlocks bool lastBlockNumber types.GlobalBlockNumber - hasQC bool - lastQCNext types.GlobalBlockNumber - hasAppQC bool - lastAppQCNext types.GlobalBlockNumber - - // latestQCStartBlock is the most recently written QC's starting block number. - latestQCStartBlock types.GlobalBlockNumber - - // latestAppQCStartBlock is the most recently written AppQC's starting block - // number. When AppQCs exist, PruneBefore clamps to it so the newest AppQC - // cohort remains readable together with its CommitQC and blocks. - latestAppQCStartBlock types.GlobalBlockNumber + lastQC utils.Option[*types.FullCommitQC] + lastAppProposal utils.Option[*types.AppProposal] + lastAppQC utils.Option[*types.AppQC] // firstBlockNumber is the lowest block number this handle has seen. Iterator clamps its // start up to it so a scan always opens on a block that exists: the first block may be @@ -138,7 +129,7 @@ func (s *blockDB) recoverCursors() error { } defer func() { _ = it.Close() }() - for !s.hasBlocks || !s.hasQC || !s.hasAppQC { + for !s.hasBlocks || !s.lastQC.IsPresent() || !s.lastAppProposal.IsPresent() || !s.lastAppQC.IsPresent() { ok, err := it.Next() if err != nil { return fmt.Errorf("failed to advance recovery iterator: %w", err) @@ -160,7 +151,7 @@ func (s *blockDB) recoverCursors() error { s.hasBlocks = true } case kindQC: - if !s.hasQC { + if !s.lastQC.IsPresent() { value, err := it.GetValue() if err != nil { return fmt.Errorf("failed to read newest qc value: %w", err) @@ -169,11 +160,22 @@ func (s *blockDB) recoverCursors() error { if err != nil { return fmt.Errorf("failed to unmarshal newest qc: %w", err) } - s.latestQCStartBlock, s.lastQCNext = coveredRange(qc) - s.hasQC = true + s.lastQC = utils.Some(qc) + } + case kindAppProp: + if !s.lastAppProposal.IsPresent() { + value, err := it.GetValue() + if err != nil { + return fmt.Errorf("failed to read newest appProposal value: %w", err) + } + appProposal, err := decodeAppProposal(value) + if err != nil { + return fmt.Errorf("failed to unmarshal newest appProposal: %w", err) + } + s.lastAppProposal = utils.Some(appProposal) } case kindAppQC: - if !s.hasAppQC { + if !s.lastAppQC.IsPresent() { value, err := it.GetValue() if err != nil { return fmt.Errorf("failed to read newest appQC value: %w", err) @@ -182,8 +184,7 @@ func (s *blockDB) recoverCursors() error { if err != nil { return fmt.Errorf("failed to unmarshal newest appQC: %w", err) } - s.latestAppQCStartBlock, s.lastAppQCNext = appQCRange(appQC) - s.hasAppQC = true + s.lastAppQC = utils.Some(appQC) } } } @@ -260,9 +261,8 @@ func (s *blockDB) WriteBlock(n types.GlobalBlockNumber, blk *types.Block) error // strictly ascending, n is covered iff n < lastQCNext. This guard also fixes // the QC-before-block write order: the covering QC's Put has already issued // under this mutex, so on a crash a surviving block implies a surviving QC. - if !s.hasQC || n >= s.lastQCNext { - return fmt.Errorf("block number %d not covered by any written QC (next QC bound %d): %w", - n, s.lastQCNext, types.ErrBlockMissingQC) + if qc, ok := s.lastQC.Get(); !ok || n >= qc.QC().GlobalRange().Next { + return fmt.Errorf("block number %d not covered by any written QC: %w", n, types.ErrBlockMissingQC) } value := encodeBlock(n, blk) @@ -285,88 +285,132 @@ func (s *blockDB) WriteBlock(n types.GlobalBlockNumber, blk *types.Block) error } func (s *blockDB) WriteQC(qc *types.FullCommitQC) error { - first, next := coveredRange(qc) - if first >= next { - return fmt.Errorf("QC at %d covers no blocks: %w", first, types.ErrQCNonContiguous) + gr := qc.QC().GlobalRange() + if gr.Len() == 0 { + return fmt.Errorf("QC at %d covers no blocks: %w", gr.First, types.ErrQCNonContiguous) } s.mu.Lock() defer s.mu.Unlock() - if s.hasQC && first != s.lastQCNext { + if qc, ok := s.lastQC.Get(); ok && qc.QC().GlobalRange().Next != gr.First { return fmt.Errorf("QC starts at %d, expected %d: %w", - first, s.lastQCNext, types.ErrQCNonContiguous) + gr.First, qc.QC().GlobalRange().Next, types.ErrQCNonContiguous) } value := encodeQC(qc) var aliases []*litttypes.SecondaryKey - for m := first + 1; m < next; m++ { + for m := gr.First + 1; m < gr.Next; m++ { aliases = append(aliases, &litttypes.SecondaryKey{ Key: qcKey(m), Offset: 0, Length: uint32(len(value)), //nolint:gosec // value length fits u32 (litt value cap is 2^32) }) } - if err := s.table.Put(qcKey(first), value, aliases...); err != nil { - return fmt.Errorf("failed to put QC [%d,%d): %w", first, next, err) + if err := s.table.Put(qcKey(gr.First), value, aliases...); err != nil { + return fmt.Errorf("failed to put QC [%d,%d): %w", gr.First, gr.Next, err) } - if !s.hasQC { + if !s.lastQC.IsPresent() { // The first QC may start anywhere its caller allows, and nothing below it will ever // be written. Record where coverage begins so Iterator can clamp to it without // discovering it by scanning; a reopen re-derives the same value. - s.oldestQCStart = first + s.oldestQCStart = gr.First } - s.latestQCStartBlock = first - s.lastQCNext = next - s.hasQC = true + s.lastQC = utils.Some(qc) return nil } func (s *blockDB) WriteAppQC(appQC *types.AppQC) error { - first, next := appQCRange(appQC) - if first >= next { - return fmt.Errorf("AppQC at %d covers no blocks: %w", first, types.ErrAppQCNonContiguous) + gr := appQC.Proposal().GlobalRange() + if gr.Len() == 0 { + return fmt.Errorf("AppQC at %d covers no blocks: %w", gr.First, types.ErrAppQCNonContiguous) } s.mu.Lock() defer s.mu.Unlock() - if !s.hasQC { - return fmt.Errorf("AppQC [%d,%d) has no matching QC: %w", first, next, types.ErrAppQCMissingQC) + if !s.lastQC.IsPresent() { + return fmt.Errorf("AppQC [%d,%d) has no matching QC: %w", gr.First, gr.Next, types.ErrAppQCMissingQC) } - if s.hasAppQC { - if first != s.lastAppQCNext { + if lastAppQC, ok := s.lastAppQC.Get(); ok { + if want := lastAppQC.Proposal().GlobalRange().Next; want != gr.First { return fmt.Errorf("AppQC starts at %d, expected %d: %w", - first, s.lastAppQCNext, types.ErrAppQCNonContiguous) + gr.First, want, types.ErrAppQCNonContiguous) } - } else if first != s.oldestQCStart { + } else if gr.First != s.oldestQCStart { return fmt.Errorf("first AppQC starts at %d, expected retained QC floor %d: %w", - first, s.oldestQCStart, types.ErrAppQCNonContiguous) + gr.First, s.oldestQCStart, types.ErrAppQCNonContiguous) } - qc, err := readQCCovering(s.table, first) + qc, err := readQCCovering(s.table, gr.First) if err != nil { - return fmt.Errorf("read matching QC for AppQC [%d,%d): %w", first, next, err) + return fmt.Errorf("read matching QC for AppQC [%d,%d): %w", gr.First, gr.Next, err) } - qcFirst, qcNext := coveredRange(qc) - if qcFirst != first || qcNext != next { + if want := qc.QC().GlobalRange(); gr != want { return fmt.Errorf("AppQC [%d,%d) does not exactly match QC [%d,%d): %w", - first, next, qcFirst, qcNext, types.ErrAppQCMissingQC) + gr.First, gr.Next, want.First, want.Next, types.ErrAppQCMissingQC) } value := encodeAppQC(appQC) var aliases []*litttypes.SecondaryKey - for m := first + 1; m < next; m++ { + for m := gr.First + 1; m < gr.Next; m++ { aliases = append(aliases, &litttypes.SecondaryKey{ Key: appQCKey(m), Offset: 0, Length: uint32(len(value)), //nolint:gosec // value length fits u32 (litt value cap is 2^32) }) } - if err := s.table.Put(appQCKey(first), value, aliases...); err != nil { - return fmt.Errorf("failed to put AppQC [%d,%d): %w", first, next, err) + if err := s.table.Put(appQCKey(gr.First), value, aliases...); err != nil { + return fmt.Errorf("failed to put AppQC [%d,%d): %w", gr.First, gr.Next, err) + } + + s.lastAppQC = utils.Some(appQC) + return nil +} + +func (s *blockDB) WriteAppProposal(appProposal *types.AppProposal) error { + gr := appProposal.GlobalRange() + if gr.Len() == 0 { + return fmt.Errorf("AppProposal at %d covers no blocks: %w", gr.First, types.ErrAppProposalNonContiguous) + } + s.mu.Lock() + defer s.mu.Unlock() + if !s.lastQC.IsPresent() { + return fmt.Errorf("AppProposal [%d,%d) has no matching QC: %w", gr.First, gr.Next, types.ErrAppProposalMissingQC) + } + if lastAppProposal, ok := s.lastAppProposal.Get(); ok { + if want := lastAppProposal.GlobalRange().Next; want != gr.First { + return fmt.Errorf("AppProposal starts at %d, expected %d: %w", + gr.First, want, types.ErrAppProposalNonContiguous) + } + } else if gr.First != s.oldestQCStart { + return fmt.Errorf("first AppProposal starts at %d, expected retained QC floor %d: %w", + gr.First, s.oldestQCStart, types.ErrAppProposalNonContiguous) + } + + qc, err := readQCCovering(s.table, gr.First) + if err != nil { + return fmt.Errorf("read matching QC for AppProposal [%d,%d): %w", gr.First, gr.Next, err) + } + if want := qc.QC().GlobalRange(); gr != want { + return fmt.Errorf("AppProposal [%d,%d) does not exactly match QC [%d,%d): %w", + gr.First, gr.Next, want.First, want.Next, types.ErrAppProposalMissingQC) + } + if err := appProposal.Verify(qc.QC()); err != nil { + return fmt.Errorf("AppProposal [%d,%d) does not verify against matching QC: %w", gr.First, gr.Next, err) + } + + value := encodeAppProposal(appProposal) + var aliases []*litttypes.SecondaryKey + for m := gr.First + 1; m < gr.Next; m++ { + aliases = append(aliases, &litttypes.SecondaryKey{ + Key: appProposalKey(m), + Offset: 0, + Length: uint32(len(value)), //nolint:gosec // value length fits u32 (litt value cap is 2^32) + }) + } + if err := s.table.Put(appProposalKey(gr.First), value, aliases...); err != nil { + return fmt.Errorf("failed to put AppProposal [%d,%d): %w", gr.First, gr.Next, err) } - s.latestAppQCStartBlock = first - s.lastAppQCNext = next - s.hasAppQC = true + s.lastAppProposal = utils.Some(appProposal) return nil } @@ -380,11 +424,7 @@ func (s *blockDB) PruneBefore(blockHeight types.GlobalBlockNumber) error { return nil } - ceiling := min(s.latestQCStartBlock, s.lastBlockNumber) - if s.hasAppQC { - ceiling = min(s.latestAppQCStartBlock, s.lastBlockNumber) - } - if blockHeight > ceiling { + if ceiling := s.recentFloorLocked(); blockHeight > ceiling { blockHeight = ceiling } @@ -425,16 +465,16 @@ func (s *blockDB) clampPruneBoundary(blockHeight types.GlobalBlockNumber) (types // // - block-number keys are reclaimable once the block number is strictly below // the prune watermark; -// - QC and AppQC keys (the primary First and every per-covered-number secondary) are +// - QC, AppProposal, and AppQC keys (the primary First and every per-covered-number secondary) are // reclaimable once their number is below the watermark, so a QC's segment is // reclaimable only once its highest covered number (Next-1) is below the -// watermark — i.e. once Next <= watermark; a QC/AppQC straddling the +// watermark — i.e. once Next <= watermark; a QC/AppProposal/AppQC straddling the // watermark is retained; // - header-hash aliases share their block's segment, so they always pass — the // block's primary number key is what actually gates segment reclamation. func (s *blockDB) gcFilter(key []byte, _ bool) (bool, error) { switch keyKind(key) { - case kindBlock, kindQC, kindAppQC: + case kindBlock, kindQC, kindAppProp, kindAppQC: return uint64(decodeNumberKey(key)) < s.watermark.Load(), nil case kindBlockHash: return true, nil @@ -453,38 +493,59 @@ func (s *blockDB) Flush() error { func (s *blockDB) Status() types.DBStatus { s.mu.Lock() defer s.mu.Unlock() - var tips types.DBStatus + var status types.DBStatus if s.hasBlocks { - tips.NextBlock = s.lastBlockNumber + 1 + status.NextBlock = s.lastBlockNumber + 1 + } + if qc, ok := s.lastQC.Get(); ok { + status.NextQC = qc.QC().GlobalRange().Next + } + if appQC, ok := s.lastAppQC.Get(); ok { + status.NextAppQC = appQC.Proposal().GlobalRange().Next } - if s.hasQC { - tips.NextQC = s.lastQCNext + if appProposal, ok := s.lastAppProposal.Get(); ok { + status.NextAppProposal = appProposal.GlobalRange().Next } - if s.hasAppQC { - tips.NextAppQC = s.lastAppQCNext + return status +} + +// recentFloor returns the recovery floor under s.mu. When AppProposals, +// AppQCs, and Blocks are persisted, data.State eviction keeps one height before +// the lower of their durable tips. That height can be inside a QC range. +func (s *blockDB) recentFloorLocked() types.GlobalBlockNumber { + floor := types.GlobalBlockNumber(s.watermark.Load()) + appProposal, hasAppProposal := s.lastAppProposal.Get() + appQC, hasAppQC := s.lastAppQC.Get() + if hasAppProposal && hasAppQC && s.hasBlocks { + nextAppProposal := appProposal.GlobalRange().Next + nextAppQC := appQC.Proposal().GlobalRange().Next + nextBlock := s.lastBlockNumber + 1 + evictionBound := min(nextAppProposal, nextAppQC, nextBlock) + if evictionBound > floor { + floor = evictionBound - 1 + } } - return tips + return floor } -// ReadRecent() reads the latest AppQC and all Blocks and CommitQCs, for indices >= AppQC.GlobalRange().First. +func (s *blockDB) recentFloor() types.GlobalBlockNumber { + s.mu.Lock() + defer s.mu.Unlock() + return s.recentFloorLocked() +} + +// ReadRecent() reads the latest AppQC/AppProposal recovery suffix. // WARNING: ReadRecent() will return an error if watermark is moved during iteration. func (s *blockDB) ReadRecent() (types.RecentData, error) { - // Determine the targetFloor: it is either all the data, or data since the lastestAppQC. - s.mu.Lock() - watermark := s.watermark.Load() - targetFloor := types.GlobalBlockNumber(watermark) - if s.hasAppQC { - targetFloor = s.latestAppQCStartBlock - } - s.mu.Unlock() + targetFloor := s.recentFloor() // Collect data >= targetFloor. it, err := s.table.Iterator(true) if err != nil { return types.RecentData{}, fmt.Errorf("failed to open recent-data iterator: %w", err) } defer func() { _ = it.Close() }() - var recent types.RecentData - for done := false; !done; { + recent := types.RecentData{First: targetFloor} + for { ok, err := it.Next() if err != nil { return types.RecentData{}, fmt.Errorf("failed to advance recent-data iterator: %w", err) @@ -517,33 +578,44 @@ func (s *blockDB) ReadRecent() (types.RecentData, error) { if err != nil { return types.RecentData{}, fmt.Errorf("failed to decode recent AppQC: %w", err) } - if targetFloor <= appQC.Proposal().GlobalRange().First { + gr := appQC.Proposal().GlobalRange() + if gr.First <= targetFloor && targetFloor < gr.Next { + recent.AppQC = utils.Some(appQC) + } else if targetFloor <= gr.First { recent.AppQC = utils.Some(appQC) } + case kindAppProp: + appProposal, err := decodeAppProposal(value) + if err != nil { + return types.RecentData{}, fmt.Errorf("failed to decode recent AppProposal: %w", err) + } + gr := appProposal.GlobalRange() + if targetFloor < gr.Next { + recent.AppProposals = append(recent.AppProposals, appProposal) + } case kindQC: qc, err := decodeQC(value) if err != nil { return types.RecentData{}, fmt.Errorf("failed to decode recent CommitQC: %w", err) } - first := qc.QC().GlobalRange().First - if targetFloor <= first { + _, next := coveredRange(qc) + if targetFloor < next { recent.CommitQCs = append(recent.CommitQCs, qc) } - if first <= targetFloor { - // targetFloor has been reached - since CommitQC is persisted before covered Blocks and AppQC, - // reaching CommitQC for targetFloor means we finished the read - done = true - } default: } } // Safety check: if watermark has been moved and GC happened to get executed during iteration, // the loaded data might be inconsistent with the targetFloor we computed. - if got := s.watermark.Load(); got != watermark { + if got := s.recentFloor(); got != targetFloor { return types.RecentData{}, fmt.Errorf("watermark has moved while iterating") } slices.Reverse(recent.CommitQCs) slices.Reverse(recent.Blocks) + slices.Reverse(recent.AppProposals) + if len(recent.Blocks) > 0 { + recent.First = recent.Blocks[0].Number + } return recent, nil } @@ -611,6 +683,26 @@ func (s *blockDB) ReadQCByBlockNumber( return utils.Some(qc), nil } +func (s *blockDB) ReadAppProposalByBlockNumber( + n types.GlobalBlockNumber, +) (utils.Option[*types.AppProposal], error) { + if uint64(n) < s.watermark.Load() { + return utils.None[*types.AppProposal](), types.ErrPruned + } + value, exists, err := s.table.Get(appProposalKey(n)) + if err != nil { + return utils.None[*types.AppProposal](), fmt.Errorf("failed to read AppProposal: %w", err) + } + if !exists { + return utils.None[*types.AppProposal](), nil + } + appProposal, err := decodeAppProposal(value) + if err != nil { + return utils.None[*types.AppProposal](), fmt.Errorf("failed to unmarshal AppProposal: %w", err) + } + return utils.Some(appProposal), nil +} + func (s *blockDB) ReadAppQCByBlockNumber( n types.GlobalBlockNumber, ) (utils.Option[*types.AppQC], error) { diff --git a/sei-db/ledger_db/block/memblock/mem_block_db.go b/sei-db/ledger_db/block/memblock/mem_block_db.go index 36dcc434cd..c2dafc51e0 100644 --- a/sei-db/ledger_db/block/memblock/mem_block_db.go +++ b/sei-db/ledger_db/block/memblock/mem_block_db.go @@ -26,6 +26,14 @@ type appQCEntry struct { upper types.GlobalBlockNumber } +// appProposalEntry pairs an AppProposal with the half-open range [lower, upper) +// it covers. +type appProposalEntry struct { + appProposal *types.AppProposal + lower types.GlobalBlockNumber + upper types.GlobalBlockNumber +} + // hashEntry pairs a block with its GlobalBlockNumber so ReadBlockByHash can // return the number, mirroring the littblock implementation which embeds it in // the stored value. @@ -43,12 +51,15 @@ type blockDB struct { byHash map[types.BlockHeaderHash]hashEntry qcsByLower map[types.GlobalBlockNumber]qcEntry appQCs map[types.GlobalBlockNumber]appQCEntry + appProps map[types.GlobalBlockNumber]appProposalEntry // Write-order cursors (see types.BlockDB contract). hasBlocks bool lastBlockNumber types.GlobalBlockNumber hasQC bool lastQCNext types.GlobalBlockNumber + hasAppProposal bool + lastAppPropNext types.GlobalBlockNumber hasAppQC bool lastAppQCNext types.GlobalBlockNumber @@ -62,6 +73,10 @@ type blockDB struct { // AppQC cohort remains readable together with its CommitQC and blocks. latestAppQCStartBlock types.GlobalBlockNumber + // latestAppProposalStartBlock is the most recently written AppProposal's + // starting block number. + latestAppProposalStartBlock types.GlobalBlockNumber + // firstBlockNumber is the lowest block number written. Meaningful only while hasBlocks. firstBlockNumber types.GlobalBlockNumber @@ -79,6 +94,7 @@ func NewBlockDB() types.BlockDB { byHash: make(map[types.BlockHeaderHash]hashEntry), qcsByLower: make(map[types.GlobalBlockNumber]qcEntry), appQCs: make(map[types.GlobalBlockNumber]appQCEntry), + appProps: make(map[types.GlobalBlockNumber]appProposalEntry), } } @@ -133,10 +149,54 @@ func (s *blockDB) WriteQC(qc *types.FullCommitQC) error { } func appQCRange(appQC *types.AppQC) (types.GlobalBlockNumber, types.GlobalBlockNumber) { - gr := appQC.Proposal().GlobalRange() + return appProposalRange(appQC.Proposal()) +} + +func appProposalRange(appProposal *types.AppProposal) (types.GlobalBlockNumber, types.GlobalBlockNumber) { + gr := appProposal.GlobalRange() return gr.First, gr.Next } +func (s *blockDB) WriteAppProposal(appProposal *types.AppProposal) error { + first, next := appProposalRange(appProposal) + if first >= next { + return fmt.Errorf("AppProposal at %d covers no blocks: %w", first, types.ErrAppProposalNonContiguous) + } + s.mu.Lock() + defer s.mu.Unlock() + if !s.hasQC { + return fmt.Errorf("AppProposal [%d,%d) has no matching QC: %w", first, next, types.ErrAppProposalMissingQC) + } + if s.hasAppProposal { + if first != s.lastAppPropNext { + return fmt.Errorf("AppProposal starts at %d, expected %d: %w", + first, s.lastAppPropNext, types.ErrAppProposalNonContiguous) + } + } else { + entries := s.sortedQCsLocked() + if len(entries) == 0 { + return fmt.Errorf("AppProposal [%d,%d) has no retained QC floor: %w", first, next, types.ErrAppProposalMissingQC) + } + if first != entries[0].lower { + return fmt.Errorf("first AppProposal starts at %d, expected retained QC floor %d: %w", + first, entries[0].lower, types.ErrAppProposalNonContiguous) + } + } + qc, ok := s.qcsByLower[first] + if !ok || qc.upper != next { + return fmt.Errorf("AppProposal [%d,%d) has no exact matching QC: %w", + first, next, types.ErrAppProposalMissingQC) + } + if err := appProposal.Verify(qc.qc.QC()); err != nil { + return fmt.Errorf("AppProposal [%d,%d) does not verify against matching QC: %w", first, next, err) + } + s.appProps[first] = appProposalEntry{appProposal: appProposal, lower: first, upper: next} + s.latestAppProposalStartBlock = first + s.lastAppPropNext = next + s.hasAppProposal = true + return nil +} + func (s *blockDB) WriteAppQC(appQC *types.AppQC) error { first, next := appQCRange(appQC) if first >= next { @@ -187,6 +247,9 @@ func (s *blockDB) PruneBefore(n types.GlobalBlockNumber) error { // for a QC written ahead of its blocks. Keeps the newest cohort whole and // pruning monotonic. See littblock and the BlockDB PruneBefore contract. ceiling := min(s.latestQCStartBlock, s.lastBlockNumber) + if s.hasAppProposal { + ceiling = min(s.latestAppProposalStartBlock, s.lastBlockNumber) + } if s.hasAppQC { ceiling = min(s.latestAppQCStartBlock, s.lastBlockNumber) } @@ -221,6 +284,11 @@ func (s *blockDB) PruneBefore(n types.GlobalBlockNumber) error { delete(s.appQCs, lower) } } + for lower, e := range s.appProps { + if e.upper <= s.watermark { + delete(s.appProps, lower) + } + } return nil } @@ -239,6 +307,9 @@ func (s *blockDB) Status() types.DBStatus { if s.hasAppQC { tips.NextAppQC = s.lastAppQCNext } + if s.hasAppProposal { + tips.NextAppProposal = s.lastAppPropNext + } return tips } @@ -249,12 +320,24 @@ func (s *blockDB) ReadRecent() (types.RecentData, error) { var recent types.RecentData floor := s.watermark var targetIndex types.RoadIndex + bounded := false if s.hasAppQC { - appQC := s.appQCs[s.latestAppQCStartBlock].appQC + evictionBound := min(s.lastAppPropNext, s.lastAppQCNext) + if s.hasAppProposal && evictionBound > floor { + floor = evictionBound - 1 + bounded = true + } + appQC := s.appQCCoveringLocked(floor) + if appQC == nil { + appQC = s.appQCs[s.latestAppQCStartBlock].appQC + } recent.AppQC = utils.Some(appQC) - floor = max(floor, s.latestAppQCStartBlock) targetIndex = appQC.Proposal().RoadIndex() } + if !bounded && s.hasBlocks { + floor = max(floor, s.firstBlockNumber) + } + recent.First = floor for _, e := range s.sortedQCsLocked() { if e.upper <= s.watermark { @@ -265,12 +348,21 @@ func (s *blockDB) ReadRecent() (types.RecentData, error) { } recent.CommitQCs = append(recent.CommitQCs, e.qc) } + for _, e := range s.sortedAppProposalsLocked() { + if e.upper <= floor { + continue + } + recent.AppProposals = append(recent.AppProposals, e.appProposal) + } for _, n := range s.sortedBlockNumbersLocked() { if n < floor { continue } recent.Blocks = append(recent.Blocks, types.RecentBlock{Number: n, Block: s.byNumber[n]}) } + if !bounded && len(recent.Blocks) > 0 { + recent.First = recent.Blocks[0].Number + } return recent, nil } @@ -284,6 +376,15 @@ func (s *blockDB) sortedQCsLocked() []qcEntry { return entries } +func (s *blockDB) sortedAppProposalsLocked() []appProposalEntry { + entries := make([]appProposalEntry, 0, len(s.appProps)) + for _, e := range s.appProps { + entries = append(entries, e) + } + sort.Slice(entries, func(i, j int) bool { return entries[i].lower < entries[j].lower }) + return entries +} + func (s *blockDB) appQCCoveringLocked(n types.GlobalBlockNumber) *types.AppQC { for _, e := range s.appQCs { if e.lower <= n && n < e.upper { @@ -293,6 +394,15 @@ func (s *blockDB) appQCCoveringLocked(n types.GlobalBlockNumber) *types.AppQC { return nil } +func (s *blockDB) appProposalCoveringLocked(n types.GlobalBlockNumber) *types.AppProposal { + for _, e := range s.appProps { + if e.lower <= n && n < e.upper { + return e.appProposal + } + } + return nil +} + func (s *blockDB) sortedBlockNumbersLocked() []types.GlobalBlockNumber { nums := make([]types.GlobalBlockNumber, 0, len(s.byNumber)) for n := range s.byNumber { @@ -343,6 +453,20 @@ func (s *blockDB) ReadQCByBlockNumber( return utils.None[*types.FullCommitQC](), nil } +func (s *blockDB) ReadAppProposalByBlockNumber( + n types.GlobalBlockNumber, +) (utils.Option[*types.AppProposal], error) { + s.mu.RLock() + defer s.mu.RUnlock() + if n < s.watermark { + return utils.None[*types.AppProposal](), types.ErrPruned + } + if appProposal := s.appProposalCoveringLocked(n); appProposal != nil { + return utils.Some(appProposal), nil + } + return utils.None[*types.AppProposal](), nil +} + func (s *blockDB) ReadAppQCByBlockNumber( n types.GlobalBlockNumber, ) (utils.Option[*types.AppQC], error) { diff --git a/sei-tendermint/autobahn/types/block_db.go b/sei-tendermint/autobahn/types/block_db.go index 2206355c99..69bcf90c9b 100644 --- a/sei-tendermint/autobahn/types/block_db.go +++ b/sei-tendermint/autobahn/types/block_db.go @@ -110,6 +110,21 @@ type BlockDB interface { // so loss of non-durable data after a crash never leaves gaps. WriteQC(qc *FullCommitQC) error + // WriteAppProposal persists an AppProposal. The AppProposal carries the + // exact CommitQC range it certifies by execution result. A matching CommitQC + // must already be written: the CommitQC covering GlobalRange.First must have + // the same GlobalRange. + // + // AppProposals form a contiguous prefix aligned with retained CommitQCs. The + // first AppProposal must start at the retained CommitQC floor; each + // subsequent AppProposal's First must equal the previous AppProposal's Next. + // Re-writing, gaps, overlaps, mid-QC starts, and ranges that do not exactly + // match the next persisted CommitQC range are rejected. + // + // May return before the AppProposal is on disk. See the BlockDB type doc for + // the two-phase write/flush contract. + WriteAppProposal(appProposal *AppProposal) error + // WriteAppQC persists an AppQC. The AppQC's proposal carries the exact // CommitQC range it certifies. A matching CommitQC must already be written: // the CommitQC covering GlobalRange.First must have the same GlobalRange. @@ -233,6 +248,20 @@ type BlockDB interface { // Non-blocking. ReadQCByBlockNumber(n GlobalBlockNumber) (utils.Option[*FullCommitQC], error) + // ReadAppProposalByBlockNumber returns the AppProposal whose + // GlobalRange().First ≤ n < GlobalRange().Next. Because a single AppProposal + // covers a CommitQC range, the same *AppProposal is returned for every n in + // its range. + // + // The result is one of: + // - utils.Some with a nil error: an AppProposal covering n is present. + // - ErrPruned: n is strictly below the current retention watermark. + // - utils.None with a nil error: n is at or above the watermark but no + // AppProposal covers it. + // + // Non-blocking. + ReadAppProposalByBlockNumber(n GlobalBlockNumber) (utils.Option[*AppProposal], error) + // ReadAppQCByBlockNumber returns the AppQC whose // AppProposal.GlobalRange().First ≤ n < AppProposal.GlobalRange().Next. // Because a single AppQC covers a CommitQC range, the same *AppQC is @@ -269,6 +298,10 @@ type DBStatus struct { // NextAppQC is one past the highest GlobalBlockNumber covered by the last // AppQC accepted by WriteAppQC. Zero if no AppQC has been written. NextAppQC GlobalBlockNumber + // NextAppProposal is one past the highest GlobalBlockNumber covered by the + // last AppProposal accepted by WriteAppProposal. Zero if no AppProposal has + // been written. + NextAppProposal GlobalBlockNumber } // RecentBlock is one block returned by BlockDB.ReadRecent. @@ -279,7 +312,9 @@ type RecentBlock struct { // RecentData is the materialized suffix used by data.State startup recovery. type RecentData struct { - CommitQCs []*FullCommitQC - Blocks []RecentBlock - AppQC utils.Option[*AppQC] + First GlobalBlockNumber + CommitQCs []*FullCommitQC + Blocks []RecentBlock + AppProposals []*AppProposal + AppQC utils.Option[*AppQC] } diff --git a/sei-tendermint/autobahn/types/errors.go b/sei-tendermint/autobahn/types/errors.go index 4009e10080..e06a64f9a2 100644 --- a/sei-tendermint/autobahn/types/errors.go +++ b/sei-tendermint/autobahn/types/errors.go @@ -27,6 +27,17 @@ var ErrQCNonContiguous = errors.New("block: WriteQC non-contiguous") // before that block (see the BlockDB ordering contract). var ErrBlockMissingQC = errors.New("block: WriteBlock without covering QC") +// ErrAppProposalNonContiguous is returned by WriteAppProposal when the supplied +// AppProposal does not extend the existing AppProposal prefix. AppProposals must +// be written as a contiguous, ascending sequence aligned with the retained +// CommitQC prefix. +var ErrAppProposalNonContiguous = errors.New("block: WriteAppProposal non-contiguous") + +// ErrAppProposalMissingQC is returned by WriteAppProposal when no previously +// written CommitQC exactly matches the AppProposal's GlobalRange. The matching +// CommitQC must be written before the AppProposal. +var ErrAppProposalMissingQC = errors.New("block: WriteAppProposal without matching CommitQC") + // ErrAppQCNonContiguous is returned by WriteAppQC when the supplied AppQC does // not extend the existing AppQC prefix. AppQCs must be written as a contiguous, // ascending sequence aligned with the retained CommitQC prefix. diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index 9d045cc41b..7ab6072fe2 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -42,8 +42,7 @@ type blockEntry struct { type inner struct { // Map key ranges (low end = first): // - // Durable copies below first live in BlockDB. AppProposals are not - // persisted; they are rebuilt via PushAppHash / re-execution after restart. + // Durable copies below first live in BlockDB. qcs map[types.GlobalBlockNumber]*types.FullCommitQC // [first, nextQC) blocks map[types.GlobalBlockNumber]*types.Block // [first, nextBlock) + gap-fills in [nextBlock, nextQC) appProposals map[types.GlobalBlockNumber]*types.AppProposal // [first, nextAppProposal) @@ -113,11 +112,14 @@ func (i *inner) insertQC(registry *epoch.Registry, qc *types.FullCommitQC) error func (i *inner) insertAppQC(registry *epoch.Registry, appQC *types.AppQC) error { gr := appQC.Proposal().GlobalRange() + if gr.Next <= i.nextAppQC { + return nil + } if gr.Next > i.nextQC { return fmt.Errorf("Missing CommitQC for this AppQC") } - if gr.First != i.nextAppQC { - return fmt.Errorf("AppQC gap: expected first=%d, got %d", i.nextAppQC, gr.First) + if gr.First > i.nextAppQC { + return fmt.Errorf("AppQC gap: expected first<=%d, got %d", i.nextAppQC, gr.First) } ei := appQC.Proposal().EpochIndex() epoch, ok := registry.EpochByIndex(ei) @@ -135,6 +137,27 @@ func (i *inner) insertAppQC(registry *epoch.Registry, appQC *types.AppQC) error return nil } +func (i *inner) insertAppProposal(appProposal *types.AppProposal) error { + gr := appProposal.GlobalRange() + if gr.Next <= i.nextAppProposal { + return nil + } + if gr.Next > i.nextQC { + return fmt.Errorf("Missing CommitQC for this AppProposal") + } + if gr.First > i.nextAppProposal { + return fmt.Errorf("AppProposal gap: expected first<=%d, got %d", i.nextAppProposal, gr.First) + } + if err := appProposal.Verify(i.qcs[i.nextAppProposal].QC()); err != nil { + return fmt.Errorf("appProposal.Verify(): %w", err) + } + for i.nextAppProposal < gr.Next { + i.appProposals[i.nextAppProposal] = appProposal + i.nextAppProposal++ + } + return nil +} + // insertBlock inserts a pre-verified block into the inner state. // Requires a QC to already be present for block n. Callers must verify // the block signature before calling (unlike insertQC, which verifies). @@ -220,7 +243,10 @@ func loadFromBlockDB(cfg *Config, blockDB types.BlockDB) (*inner, error) { } first := firstBlock - if appQC, ok := recent.AppQC.Get(); ok { + if (len(recent.CommitQCs) > 0 || len(recent.Blocks) > 0 || len(recent.AppProposals) > 0 || recent.AppQC.IsPresent()) && + recent.First >= firstBlock { + first = recent.First + } else if appQC, ok := recent.AppQC.Get(); ok { first = appQC.Proposal().GlobalRange().First } else if len(recent.Blocks) > 0 { first = recent.Blocks[0].Number @@ -233,6 +259,7 @@ func loadFromBlockDB(cfg *Config, blockDB types.BlockDB) (*inner, error) { status.NextQC = max(status.NextQC, first) status.NextBlock = max(status.NextBlock, first) status.NextAppQC = max(status.NextAppQC, first) + status.NextAppProposal = max(status.NextAppProposal, first) inner := newInner(first) for _, qc := range recent.CommitQCs { @@ -245,6 +272,11 @@ func loadFromBlockDB(cfg *Config, blockDB types.BlockDB) (*inner, error) { return nil, fmt.Errorf("load AppQC from BlockDB: %w", err) } } + for _, appProposal := range recent.AppProposals { + if err := inner.insertAppProposal(appProposal); err != nil { + return nil, fmt.Errorf("load AppProposal from BlockDB: %w", err) + } + } for _, b := range recent.Blocks { qc := inner.qcs[b.Number] ei := qc.QC().Proposal().EpochIndex() @@ -263,6 +295,7 @@ func loadFromBlockDB(cfg *Config, blockDB types.BlockDB) (*inner, error) { // updateNextBlock: stale timestamps would skew metrics. inner.nextBlock = status.NextBlock inner.nextBlockToPersist = status.NextBlock + inner.nextAppProposal = status.NextAppProposal inner.nextAppQCToPersist = status.NextAppQC if inner.first < inner.nextAppQCToPersist { n := inner.nextAppQCToPersist - 1 @@ -733,9 +766,9 @@ func (s *State) WaitUntilExecuted(ctx context.Context, lane types.LaneID, n type } // PruneBefore asks BlockDB to drop data before retainFrom. This is independent -// of in-memory retention: RAM is cleared only by evictBelowBound (AppQC floor), -// and AppProposals are not persisted. BlockDB enforces its own never-empty -// retention and refuses reads below its watermark. +// of in-memory retention: RAM is cleared only by evictBelowBound (AppQC floor). +// BlockDB enforces its own never-empty retention and refuses reads below its +// watermark. func (s *State) PruneBefore(retainFrom types.GlobalBlockNumber) error { return s.blockDB.PruneBefore(retainFrom) } @@ -768,16 +801,19 @@ func (s *State) runPersist(ctx context.Context) error { for inner := range s.inner.Lock() { status.NextQC = max(status.NextQC, inner.first) status.NextAppQC = max(status.NextAppQC, inner.first) + status.NextAppProposal = max(status.NextAppProposal, inner.first) status.NextBlock = max(status.NextBlock, inner.first) } for { var qcs []*types.FullCommitQC var blocks []blockEntry + var appProposals []*types.AppProposal var appQCs []*types.AppQC for inner, ctrl := range s.inner.Lock() { // Wait until there is anythin to persist. if err := ctrl.WaitUntil(ctx, func() bool { - return status.NextQC < inner.nextQC || status.NextBlock < inner.nextBlock || status.NextAppQC < inner.nextAppQC + return status.NextQC < inner.nextQC || status.NextBlock < inner.nextBlock || + status.NextAppProposal < inner.nextAppProposal || status.NextAppQC < inner.nextAppQC }); err != nil { return err } @@ -792,6 +828,11 @@ func (s *State) runPersist(ctx context.Context) error { appQCs = append(appQCs, appQC) status.NextAppQC = appQC.Proposal().GlobalRange().Next } + for status.NextAppProposal < inner.nextAppProposal { + appProposal := inner.appProposals[status.NextAppProposal] + appProposals = append(appProposals, appProposal) + status.NextAppProposal = appProposal.GlobalRange().Next + } for status.NextBlock < inner.nextBlock { blocks = append(blocks, blockEntry{n: status.NextBlock, block: inner.blocks[status.NextBlock]}) status.NextBlock += 1 @@ -808,6 +849,11 @@ func (s *State) runPersist(ctx context.Context) error { return fmt.Errorf("write block %d: %w", lb.n, err) } } + for _, appProposal := range appProposals { + if err := s.blockDB.WriteAppProposal(appProposal); err != nil { + return fmt.Errorf("write AppProposal %d: %w", appProposal.RoadIndex(), err) + } + } for _, appQC := range appQCs { if err := s.blockDB.WriteAppQC(appQC); err != nil { return fmt.Errorf("write AppQC %d: %w", appQC.Proposal().RoadIndex(), err) @@ -831,15 +877,14 @@ func (s *State) runPersist(ctx context.Context) error { } } -// evict pushes first to min(i.nextAppProposal, i.nextAppQCToPersist-1) +// evict pushes first to min(i.nextAppProposal, i.nextAppQCToPersist)-1 // I.e. it makes sure that at least 1 persisted appQC is still in memory: // it is passed to avail.State. func (i *inner) evict() { - bound := i.nextAppQCToPersist + bound := min(i.nextAppQCToPersist, i.nextAppProposal) if bound > i.first { bound -= 1 } - bound = min(bound, i.nextAppProposal) for i.first < bound { n := i.first delete(i.blockHashes, i.blocks[n].Header().Hash()) diff --git a/sei-tendermint/internal/autobahn/data/state_recovery_test.go b/sei-tendermint/internal/autobahn/data/state_recovery_test.go index 90efcdf372..9ce1c48080 100644 --- a/sei-tendermint/internal/autobahn/data/state_recovery_test.go +++ b/sei-tendermint/internal/autobahn/data/state_recovery_test.go @@ -369,7 +369,7 @@ func TestRecoveryPartialQCPrefix(t *testing.T) { qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) gr1 := qc1.QC().GlobalRange() - require.True(t,gr1.Next-gr1.First < 3, "need at least 3 blocks in QC range to test split") + require.True(t, gr1.Next-gr1.First >= 3, "need at least 3 blocks in QC range to test split") // Write the QC for the full range, but write blocks only from mid onwards. mid := gr1.First + (gr1.Next-gr1.First)/2 From 00fe049a400000bf76569a70c07222fdfe5b98e6 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 7 Aug 2026 13:15:00 +0200 Subject: [PATCH 20/61] test fixes --- sei-db/ledger_db/block/block_db_test.go | 30 ++++--- .../littblock/litt_block_stranding_test.go | 86 ++++++++++++++++--- 2 files changed, 96 insertions(+), 20 deletions(-) diff --git a/sei-db/ledger_db/block/block_db_test.go b/sei-db/ledger_db/block/block_db_test.go index e346bda966..b634d13920 100644 --- a/sei-db/ledger_db/block/block_db_test.go +++ b/sei-db/ledger_db/block/block_db_test.go @@ -549,6 +549,7 @@ func testPruneRefusesBelowWatermark(t *testing.T, build builder) { db, _ := openFresh(t, build) defer func() { _ = db.Close() }() writeAll(t, db, batches) + writeAppData(t, db, utils.TestRngFromSeed(testSeed+500), keys, batches) // Prune at the start of the second batch: all of the first batch is below it. watermark := batches[1].first @@ -591,6 +592,7 @@ func testPrunedDistinctFromNotFound(t *testing.T, build builder) { db, _ := openFresh(t, build) defer func() { _ = db.Close() }() writeAll(t, db, batches) + writeAppData(t, db, utils.TestRngFromSeed(testSeed+501), keys, batches) straddled := batches[1] pruneAt := straddled.first + 2 @@ -710,6 +712,7 @@ func testPruneNeverEmpties(t *testing.T, build builder) { db, _ := openFresh(t, build) defer func() { _ = db.Close() }() writeAll(t, db, batches) + writeAppData(t, db, utils.TestRngFromSeed(testSeed+502), keys, batches) require.NoError(t, db.PruneBefore(prune)) @@ -741,17 +744,13 @@ func testPruneNeverEmpties(t *testing.T, build builder) { require.ErrorIs(t, err, types.ErrPruned, "blocks below the newest cohort must be reported pruned") require.False(t, below.IsPresent(), "blocks below the newest cohort must not be served") - // The iterator yields exactly the newest cohort's numbers, every one with a - // block, all covered by the single remaining QC. - var expected []types.GlobalBlockNumber - for i := range last.blocks { - expected = append(expected, last.first+gbn(i)) - } + // ReadRecent starts at recentFloor(), while the prune watermark still + // rounds down to keep the whole newest QC cohort readable. entries := drainRecent(t, db) - require.Equal(t, expected, presentBlockNumbers(entries), - "exactly the newest cohort must remain after PruneBefore(%d)", prune) + require.Equal(t, []types.GlobalBlockNumber{newest}, presentBlockNumbers(entries), + "ReadRecent must start at recentFloor() after PruneBefore(%d)", prune) require.Equal(t, []types.GlobalBlockNumber{last.first}, qcFirsts(entries), - "exactly one QC (covering the newest cohort) must remain") + "ReadRecent must include the QC covering the recent floor") }) } } @@ -766,7 +765,9 @@ func testPruneWithAppQCNeverEmpties(t *testing.T, build builder) { rng := utils.TestRngFromSeed(testSeed + 300) for _, b := range batches[:3] { - require.NoError(t, db.WriteAppQC(appQCForBatch(rng, keys, b))) + appQC := appQCForBatch(rng, keys, b) + require.NoError(t, db.WriteAppProposal(appQC.Proposal())) + require.NoError(t, db.WriteAppQC(appQC)) } latestApp := batches[2] @@ -1360,6 +1361,15 @@ func writeAll(t *testing.T, db types.BlockDB, batches []batch) { } } +func writeAppData(t *testing.T, db types.BlockDB, rng utils.Rng, keys []types.SecretKey, batches []batch) { + t.Helper() + for _, b := range batches { + appQC := appQCForBatch(rng, keys, b) + require.NoError(t, db.WriteAppProposal(appQC.Proposal())) + require.NoError(t, db.WriteAppQC(appQC)) + } +} + // buildCommittee returns a deterministic round-robin committee (global numbering // from 0) and the secret keys that sign its QCs. func buildCommittee() (*types.Committee, []types.SecretKey) { diff --git a/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go b/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go index 435ef0075e..e0b6a947ed 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go +++ b/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go @@ -26,20 +26,83 @@ func strandingConfig(t *testing.T, dir string, maxSegmentKeyCount uint32) *LittB return cfg } -// writeSyntheticBatches writes numBatches contiguous batches of perQC blocks each -// (global numbers 0.., QC ranges [0,perQC), [perQC,2*perQC), ...). Each QC's own -// GlobalRange matches the range it is written at, so littblock's clamp can trust -// it; littblock does not verify signatures, so no committee is needed. func writeSyntheticBatches(t *testing.T, db types.BlockDB, rng utils.Rng, numBatches int, perQC int) { - for i := 0; i < numBatches; i++ { - first := types.GlobalBlockNumber(i * perQC) //nolint:gosec // small test indices - next := first + types.GlobalBlockNumber(perQC) - qc := types.GenFullCommitQCRange(rng, first, next) + t.Helper() + committee, keys := types.GenCommittee(rng, 4) + prev := utils.None[*types.CommitQC]() + for range numBatches { + qc, blocks := syntheticFullCommitQC(rng, committee, keys, prev, perQC) + first := qc.QC().GlobalRange().First require.NoError(t, db.WriteQC(qc)) - for j := 0; j < perQC; j++ { - require.NoError(t, db.WriteBlock(first+types.GlobalBlockNumber(j), types.GenBlock(rng))) //nolint:gosec + for j, block := range blocks { + require.NoError(t, db.WriteBlock(first+types.GlobalBlockNumber(j), block)) //nolint:gosec + } + prev = utils.Some(qc.QC()) + } +} + +func syntheticFullCommitQC( + rng utils.Rng, + committee *types.Committee, + keys []types.SecretKey, + prev utils.Option[*types.CommitQC], + perQC int, +) (*types.FullCommitQC, []*types.Block) { + blocksByLane := map[types.LaneID][]*types.Block{} + makeBlock := func(producer types.LaneID) *types.Block { + if blocks := blocksByLane[producer]; len(blocks) > 0 { + parent := blocks[len(blocks)-1] + return types.NewBlock(producer, parent.Header().Next(), parent.Header().Hash(), types.GenPayload(rng)) + } + return types.NewBlock(producer, types.LaneRangeOpt(prev, producer).Next(), types.GenBlockHeaderHash(rng), types.GenPayload(rng)) + } + for range perQC { + producer := committee.Lanes().At(rng.Intn(committee.Lanes().Len())) + blocksByLane[producer] = append(blocksByLane[producer], makeBlock(producer)) + } + + laneQCs := map[types.LaneID]*types.LaneQC{} + headers := make([]*types.BlockHeader, 0, perQC) + blocks := make([]*types.Block, 0, perQC) + for lane := range committee.Lanes().All() { + if bs := blocksByLane[lane]; len(bs) > 0 { + laneQCs[lane] = syntheticLaneQC(keys, bs[len(bs)-1].Header()) + for _, block := range bs { + headers = append(headers, block.Header()) + blocks = append(blocks, block) + } } } + epoch := types.NewEpoch(0, types.OpenRoadRange(), time.Unix(1_700_000_000, 0), committee, 0) + qc := types.BuildCommitQC(epoch, keys, prev, laneQCs) + return types.NewFullCommitQC(qc, headers), blocks +} + +func syntheticLaneQC(keys []types.SecretKey, header *types.BlockHeader) *types.LaneQC { + vote := types.NewLaneVote(header) + votes := make([]*types.Signed[*types.LaneVote], 0, len(keys)) + for _, key := range keys { + votes = append(votes, types.Sign(key, vote)) + } + return types.NewLaneQC(votes) +} + +func writeSyntheticAppData(t *testing.T, db types.BlockDB, rng utils.Rng, numBatches int, perQC int) { + t.Helper() + for i := 0; i < numBatches; i++ { + first := types.GlobalBlockNumber(i * perQC) //nolint:gosec // small test indices + qc, err := db.ReadQCByBlockNumber(first) + require.NoError(t, err) + gotQC, ok := qc.Get() + require.True(t, ok, "synthetic QC %d must exist before writing app data", first) + proposal := types.NewAppProposal(gotQC.QC().Proposal(), types.GenAppHash(rng)) + require.NoError(t, db.WriteAppProposal(proposal)) + vote := types.NewAppVote(proposal) + appQC := types.NewAppQC([]*types.Signed[*types.AppVote]{ + types.Sign(types.GenSecretKey(rng), vote), + }) + require.NoError(t, db.WriteAppQC(appQC)) + } } // physicallyPresent reports whether a key exists in the raw table, bypassing the @@ -71,6 +134,7 @@ func TestLittblockStrandedBlockNotServedAfterRestart(t *testing.T) { db, err := NewBlockDB(strandingConfig(t, dir, 8)) require.NoError(t, err) writeSyntheticBatches(t, db, rng, 4, 5) // blocks 0..19; QCs [0,5),[5,10),[10,15),[15,20) + writeSyntheticAppData(t, db, rng, 4, 5) require.NoError(t, db.Flush()) require.NoError(t, db.Close()) @@ -152,6 +216,7 @@ func TestLittblockReclaimsAcrossRestart(t *testing.T) { db, err := NewBlockDB(strandingConfig(t, dir, 8)) require.NoError(t, err) writeSyntheticBatches(t, db, rng, 4, 5) // blocks 0..19 + writeSyntheticAppData(t, db, rng, 4, 5) require.NoError(t, db.Flush()) require.NoError(t, db.Close()) @@ -226,6 +291,7 @@ func TestLittblockPruneIntoCohortRoundsDown(t *testing.T) { db, err := NewBlockDB(strandingConfig(t, dir, 8)) require.NoError(t, err) writeSyntheticBatches(t, db, rng, 4, 5) // blocks 0..19; QC[5,10) covers blocks 5..9 + writeSyntheticAppData(t, db, rng, 4, 5) require.NoError(t, db.Flush()) require.NoError(t, db.Close()) From 7fc04f0b86a3c94c62c227edbd530b35f9d0c8dd Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 7 Aug 2026 13:20:04 +0200 Subject: [PATCH 21/61] uniform coverage of blockDB data types --- sei-db/ledger_db/block/block_db_test.go | 112 +++++++++++++++++++++++- 1 file changed, 110 insertions(+), 2 deletions(-) diff --git a/sei-db/ledger_db/block/block_db_test.go b/sei-db/ledger_db/block/block_db_test.go index b634d13920..4428370483 100644 --- a/sei-db/ledger_db/block/block_db_test.go +++ b/sei-db/ledger_db/block/block_db_test.go @@ -79,8 +79,8 @@ func TestBlockDB(t *testing.T) { t.Run("WriteQCCoversNoBlocksRejected", func(t *testing.T) { testWriteQCCoversNoBlocksRejected(t, impl.build) }) - t.Run("WriteAppQCOrderRejected", func(t *testing.T) { - testWriteAppQCOrderRejected(t, impl.build) + t.Run("WriteAppDataOrderRejected", func(t *testing.T) { + testWriteAppDataOrderRejected(t, impl.build) }) t.Run("PruneWithAppQCNeverEmpties", func(t *testing.T) { testPruneWithAppQCNeverEmpties(t, impl.build) @@ -386,6 +386,14 @@ func testAppProposalByBlockNumber(t *testing.T, build builder) { require.Equal(t, batches[1].next, tips.NextAppProposal, "AppProposal tip must survive restart") assertTipsMatchPresent(t, db) + for _, appProposal := range appProposals { + gr := appProposal.GlobalRange() + opt, err := db.ReadAppProposalByBlockNumber(gr.First) + require.NoError(t, err) + got, ok := opt.Get() + require.True(t, ok, "AppProposal [%d,%d) must survive restart", gr.First, gr.Next) + require.Equal(t, gr, got.GlobalRange()) + } } func testAppQCByBlockNumber(t *testing.T, build builder) { @@ -436,6 +444,15 @@ func testAppQCByBlockNumber(t *testing.T, build builder) { tips = db.Status() require.Equal(t, batches[1].next, tips.NextAppQC, "AppQC tip must survive restart") assertTipsMatchPresent(t, db) + + for _, appQC := range appQCs { + gr := appQC.Proposal().GlobalRange() + opt, err := db.ReadAppQCByBlockNumber(gr.First) + require.NoError(t, err) + got, ok := opt.Get() + require.True(t, ok, "AppQC [%d,%d) must survive restart", gr.First, gr.Next) + require.Equal(t, gr, got.Proposal().GlobalRange()) + } } func testIterators(t *testing.T, build builder) { @@ -567,6 +584,14 @@ func testPruneRefusesBelowWatermark(t *testing.T, build builder) { byHash, err := db.ReadBlockByHash(blk.Header().Hash()) require.NoError(t, err) require.False(t, byHash.IsPresent(), "block %d below watermark %d must not be served by hash", n, watermark) + + appProposal, err := db.ReadAppProposalByBlockNumber(n) + require.ErrorIs(t, err, types.ErrPruned, "AppProposal at block %d below watermark %d must be reported pruned", n, watermark) + require.False(t, appProposal.IsPresent(), "AppProposal at block %d below watermark %d must not be served", n, watermark) + + appQC, err := db.ReadAppQCByBlockNumber(n) + require.ErrorIs(t, err, types.ErrPruned, "AppQC at block %d below watermark %d must be reported pruned", n, watermark) + require.False(t, appQC.IsPresent(), "AppQC at block %d below watermark %d must not be served", n, watermark) } for _, e := range drainRecent(t, db) { @@ -608,6 +633,12 @@ func testPrunedDistinctFromNotFound(t *testing.T, build builder) { opt, err := db.ReadBlockByNumber(n) require.NoError(t, err, "block %d in the straddled cohort must not report ErrPruned", n) require.True(t, opt.IsPresent(), "block %d in the straddled cohort must remain served", n) + appProposal, err := db.ReadAppProposalByBlockNumber(n) + require.NoError(t, err, "AppProposal at block %d in the straddled cohort must not report ErrPruned", n) + require.True(t, appProposal.IsPresent(), "AppProposal at block %d in the straddled cohort must remain served", n) + appQC, err := db.ReadAppQCByBlockNumber(n) + require.NoError(t, err, "AppQC at block %d in the straddled cohort must not report ErrPruned", n) + require.True(t, appQC.IsPresent(), "AppQC at block %d in the straddled cohort must remain served", n) } qcOpt, err := db.ReadQCByBlockNumber(straddled.first) require.NoError(t, err, "straddled cohort's QC must not report ErrPruned") @@ -624,6 +655,12 @@ func testPrunedDistinctFromNotFound(t *testing.T, build builder) { qc, err := db.ReadQCByBlockNumber(belowNum) require.ErrorIs(t, err, types.ErrPruned, "below-watermark QC must report ErrPruned") require.False(t, qc.IsPresent()) + appProposal, err := db.ReadAppProposalByBlockNumber(belowNum) + require.ErrorIs(t, err, types.ErrPruned, "below-watermark AppProposal must report ErrPruned") + require.False(t, appProposal.IsPresent()) + appQC, err := db.ReadAppQCByBlockNumber(belowNum) + require.ErrorIs(t, err, types.ErrPruned, "below-watermark AppQC must report ErrPruned") + require.False(t, appQC.IsPresent()) // Above the watermark but never written: not pruned, just absent. unwritten := batches[len(batches)-1].next + 1000 @@ -633,6 +670,12 @@ func testPrunedDistinctFromNotFound(t *testing.T, build builder) { missQC, err := db.ReadQCByBlockNumber(unwritten) require.NoError(t, err, "never-written height must not report ErrPruned") require.False(t, missQC.IsPresent()) + missAppProposal, err := db.ReadAppProposalByBlockNumber(unwritten) + require.NoError(t, err, "never-written AppProposal height must not report ErrPruned") + require.False(t, missAppProposal.IsPresent()) + missAppQC, err := db.ReadAppQCByBlockNumber(unwritten) + require.NoError(t, err, "never-written AppQC height must not report ErrPruned") + require.False(t, missAppQC.IsPresent()) } // testPruneIdempotentMonotonic asserts PruneBefore is idempotent and the @@ -735,6 +778,14 @@ func testPruneNeverEmpties(t *testing.T, build builder) { qc, err := db.ReadQCByBlockNumber(n) require.NoError(t, err) require.True(t, qc.IsPresent(), "the QC covering the newest cohort must survive") + + appProposal, err := db.ReadAppProposalByBlockNumber(n) + require.NoError(t, err) + require.True(t, appProposal.IsPresent(), "the AppProposal covering the newest cohort must survive") + + appQC, err := db.ReadAppQCByBlockNumber(n) + require.NoError(t, err) + require.True(t, appQC.IsPresent(), "the AppQC covering the newest cohort must survive") } // A block below the newest cohort is gone (clamped watermark refuses/removes it). @@ -743,6 +794,12 @@ func testPruneNeverEmpties(t *testing.T, build builder) { below, err := db.ReadBlockByNumber(belowBatch.first) require.ErrorIs(t, err, types.ErrPruned, "blocks below the newest cohort must be reported pruned") require.False(t, below.IsPresent(), "blocks below the newest cohort must not be served") + belowAppProposal, err := db.ReadAppProposalByBlockNumber(belowBatch.first) + require.ErrorIs(t, err, types.ErrPruned, "AppProposals below the newest cohort must be reported pruned") + require.False(t, belowAppProposal.IsPresent(), "AppProposals below the newest cohort must not be served") + belowAppQC, err := db.ReadAppQCByBlockNumber(belowBatch.first) + require.ErrorIs(t, err, types.ErrPruned, "AppQCs below the newest cohort must be reported pruned") + require.False(t, belowAppQC.IsPresent(), "AppQCs below the newest cohort must not be served") // ReadRecent starts at recentFloor(), while the prune watermark still // rounds down to keep the whole newest QC cohort readable. @@ -780,6 +837,9 @@ func testPruneWithAppQCNeverEmpties(t *testing.T, build builder) { appQC, err := db.ReadAppQCByBlockNumber(below.first) require.ErrorIs(t, err, types.ErrPruned) require.False(t, appQC.IsPresent()) + appProposal, err := db.ReadAppProposalByBlockNumber(below.first) + require.ErrorIs(t, err, types.ErrPruned) + require.False(t, appProposal.IsPresent()) blk, err = db.ReadBlockByNumber(latestApp.first) require.NoError(t, err) @@ -792,6 +852,11 @@ func testPruneWithAppQCNeverEmpties(t *testing.T, build builder) { got, ok := appQC.Get() require.True(t, ok, "newest AppQC cohort must retain its AppQC") require.Equal(t, latestApp.qc.QC().GlobalRange(), got.Proposal().GlobalRange()) + appProposal, err = db.ReadAppProposalByBlockNumber(latestApp.first) + require.NoError(t, err) + gotProposal, ok := appProposal.Get() + require.True(t, ok, "newest AppQC cohort must retain its AppProposal") + require.Equal(t, latestApp.qc.QC().GlobalRange(), gotProposal.GlobalRange()) } // testPruneQCAheadOfBlocks pins the min() guard in the prune clamp. QCs are @@ -921,6 +986,49 @@ func testWriteOrderRejected(t *testing.T, build builder) { require.True(t, opt.IsPresent()) } +func testWriteAppDataOrderRejected(t *testing.T, build builder) { + t.Run("AppProposal", func(t *testing.T) { + testWriteAppProposalOrderRejected(t, build) + }) + t.Run("AppQC", func(t *testing.T) { + testWriteAppQCOrderRejected(t, build) + }) +} + +func testWriteAppProposalOrderRejected(t *testing.T, build builder) { + committee, keys := buildCommittee() + batches := generateBatches(committee, keys) + db, _ := openFresh(t, build) + defer func() { _ = db.Close() }() + rng := utils.TestRngFromSeed(testSeed + 190) + + b0 := batches[0] + b1 := batches[1] + b2 := batches[2] + + err := db.WriteAppProposal(appProposalForBatch(rng, b0)) + require.ErrorIs(t, err, types.ErrAppProposalMissingQC, "AppProposal before CommitQC must fail") + + require.NoError(t, db.WriteQC(b0.qc)) + require.NoError(t, db.WriteQC(b1.qc)) + err = db.WriteAppProposal(appProposalForBatch(rng, b1)) + require.ErrorIs(t, err, types.ErrAppProposalNonContiguous, "first AppProposal must start at retained QC floor") + + appProposal0 := appProposalForBatch(rng, b0) + require.NoError(t, db.WriteAppProposal(appProposal0)) + + err = db.WriteAppProposal(appProposal0) + require.ErrorIs(t, err, types.ErrAppProposalNonContiguous, "duplicate AppProposal write must fail") + + require.NoError(t, db.WriteQC(b2.qc)) + err = db.WriteAppProposal(appProposalForBatch(rng, b2)) + require.ErrorIs(t, err, types.ErrAppProposalNonContiguous, "AppProposal gap must fail") + + require.NoError(t, db.WriteAppProposal(appProposalForBatch(rng, b1))) + tips := db.Status() + require.Equal(t, b1.next, tips.NextAppProposal) +} + func testWriteAppQCOrderRejected(t *testing.T, build builder) { committee, keys := buildCommittee() batches := generateBatches(committee, keys) From ea1df5cde41167cc5450697fc07f88a052313fe0 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 7 Aug 2026 13:23:38 +0200 Subject: [PATCH 22/61] consistent recent floor --- sei-db/ledger_db/block/block_db_test.go | 2 + .../ledger_db/block/memblock/mem_block_db.go | 39 ++++++++++--------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/sei-db/ledger_db/block/block_db_test.go b/sei-db/ledger_db/block/block_db_test.go index 4428370483..400fe26a36 100644 --- a/sei-db/ledger_db/block/block_db_test.go +++ b/sei-db/ledger_db/block/block_db_test.go @@ -1282,6 +1282,7 @@ func TestMemblockPruneRemovesBelowWatermark(t *testing.T) { batches := generateBatches(committee, keys) db := memblock.NewBlockDB() writeAll(t, db, batches) + writeAppData(t, db, utils.TestRngFromSeed(testSeed+503), keys, batches) watermark := batches[1].first require.NoError(t, db.PruneBefore(watermark)) @@ -1321,6 +1322,7 @@ func TestMemblockPruneIntoCohortRoundsDown(t *testing.T) { batches := generateBatches(committee, keys) db := memblock.NewBlockDB() writeAll(t, db, batches) + writeAppData(t, db, utils.TestRngFromSeed(testSeed+504), keys, batches) straddled := batches[1] pruneAt := straddled.first + 2 diff --git a/sei-db/ledger_db/block/memblock/mem_block_db.go b/sei-db/ledger_db/block/memblock/mem_block_db.go index c2dafc51e0..c41d6e505e 100644 --- a/sei-db/ledger_db/block/memblock/mem_block_db.go +++ b/sei-db/ledger_db/block/memblock/mem_block_db.go @@ -242,18 +242,7 @@ func (s *blockDB) PruneBefore(n types.GlobalBlockNumber) error { // future block whose coverage check still passes. Mirrors littblock. return nil } - // Never let the watermark enter the newest block's cohort: clamp its ceiling - // at the cohort's first block (latestQCStartBlock), guarded by lastBlockNumber - // for a QC written ahead of its blocks. Keeps the newest cohort whole and - // pruning monotonic. See littblock and the BlockDB PruneBefore contract. - ceiling := min(s.latestQCStartBlock, s.lastBlockNumber) - if s.hasAppProposal { - ceiling = min(s.latestAppProposalStartBlock, s.lastBlockNumber) - } - if s.hasAppQC { - ceiling = min(s.latestAppQCStartBlock, s.lastBlockNumber) - } - if n > ceiling { + if ceiling := s.recentFloorLocked(); n > ceiling { n = ceiling } // Round the watermark down to the covering QC's First. A QC's cohort of @@ -292,6 +281,23 @@ func (s *blockDB) PruneBefore(n types.GlobalBlockNumber) error { return nil } +// recentFloorLocked returns the recovery floor under s.mu. When AppProposals, +// AppQCs, and Blocks are persisted, data.State eviction keeps one height before +// the lower of their durable tips. That height can be inside a QC range. +func (s *blockDB) recentFloorLocked() types.GlobalBlockNumber { + floor := s.watermark + if s.hasAppProposal && s.hasAppQC && s.hasBlocks { + nextAppProposal := s.lastAppPropNext + nextAppQC := s.lastAppQCNext + nextBlock := s.lastBlockNumber + 1 + evictionBound := min(nextAppProposal, nextAppQC, nextBlock) + if evictionBound > floor { + floor = evictionBound - 1 + } + } + return floor +} + func (s *blockDB) Flush() error { return nil } func (s *blockDB) Status() types.DBStatus { @@ -318,15 +324,10 @@ func (s *blockDB) ReadRecent() (types.RecentData, error) { defer s.mu.RUnlock() var recent types.RecentData - floor := s.watermark + floor := s.recentFloorLocked() var targetIndex types.RoadIndex - bounded := false + bounded := s.hasAppProposal && s.hasAppQC && s.hasBlocks if s.hasAppQC { - evictionBound := min(s.lastAppPropNext, s.lastAppQCNext) - if s.hasAppProposal && evictionBound > floor { - floor = evictionBound - 1 - bounded = true - } appQC := s.appQCCoveringLocked(floor) if appQC == nil { appQC = s.appQCs[s.latestAppQCStartBlock].appQC From 06bcfd513622bf17b58368b39f566c760d40fa72 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 7 Aug 2026 14:10:29 +0200 Subject: [PATCH 23/61] simplifications --- .../block/littblock/litt_block_db.go | 31 +--- .../ledger_db/block/memblock/mem_block_db.go | 29 +--- sei-tendermint/autobahn/types/block_db.go | 11 ++ sei-tendermint/autobahn/types/types_test.go | 45 ++++++ .../internal/autobahn/data/state.go | 141 ++++++++---------- .../autobahn/data/state_recovery_test.go | 5 +- .../internal/autobahn/data/state_test.go | 41 +++-- 7 files changed, 166 insertions(+), 137 deletions(-) diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index 7debf5b89e..ba87127e05 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -424,12 +424,8 @@ func (s *blockDB) PruneBefore(blockHeight types.GlobalBlockNumber) error { return nil } - if ceiling := s.recentFloorLocked(); blockHeight > ceiling { - blockHeight = ceiling - } - // Round the watermark down to the start of a QC's range, to avoid pruning a QC before its blocks. - blockHeight, err := s.clampPruneBoundary(blockHeight) + blockHeight, err := s.clampPruneBoundary(min(blockHeight, s.statusLocked().Floor())) if err != nil { return err } @@ -493,6 +489,10 @@ func (s *blockDB) Flush() error { func (s *blockDB) Status() types.DBStatus { s.mu.Lock() defer s.mu.Unlock() + return s.statusLocked() +} + +func (s *blockDB) statusLocked() types.DBStatus { var status types.DBStatus if s.hasBlocks { status.NextBlock = s.lastBlockNumber + 1 @@ -509,29 +509,10 @@ func (s *blockDB) Status() types.DBStatus { return status } -// recentFloor returns the recovery floor under s.mu. When AppProposals, -// AppQCs, and Blocks are persisted, data.State eviction keeps one height before -// the lower of their durable tips. That height can be inside a QC range. -func (s *blockDB) recentFloorLocked() types.GlobalBlockNumber { - floor := types.GlobalBlockNumber(s.watermark.Load()) - appProposal, hasAppProposal := s.lastAppProposal.Get() - appQC, hasAppQC := s.lastAppQC.Get() - if hasAppProposal && hasAppQC && s.hasBlocks { - nextAppProposal := appProposal.GlobalRange().Next - nextAppQC := appQC.Proposal().GlobalRange().Next - nextBlock := s.lastBlockNumber + 1 - evictionBound := min(nextAppProposal, nextAppQC, nextBlock) - if evictionBound > floor { - floor = evictionBound - 1 - } - } - return floor -} - func (s *blockDB) recentFloor() types.GlobalBlockNumber { s.mu.Lock() defer s.mu.Unlock() - return s.recentFloorLocked() + return s.statusLocked().Floor() } // ReadRecent() reads the latest AppQC/AppProposal recovery suffix. diff --git a/sei-db/ledger_db/block/memblock/mem_block_db.go b/sei-db/ledger_db/block/memblock/mem_block_db.go index c41d6e505e..0350430378 100644 --- a/sei-db/ledger_db/block/memblock/mem_block_db.go +++ b/sei-db/ledger_db/block/memblock/mem_block_db.go @@ -4,6 +4,7 @@ import ( "fmt" "sort" "sync" + "slices" "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" @@ -242,9 +243,7 @@ func (s *blockDB) PruneBefore(n types.GlobalBlockNumber) error { // future block whose coverage check still passes. Mirrors littblock. return nil } - if ceiling := s.recentFloorLocked(); n > ceiling { - n = ceiling - } + n = min(n,s.statusLocked().Floor()) // Round the watermark down to the covering QC's First. A QC's cohort of // blocks changes readability atomically, so the watermark must never fall // strictly inside a QC's range (see littblock): otherwise a read would @@ -281,28 +280,16 @@ func (s *blockDB) PruneBefore(n types.GlobalBlockNumber) error { return nil } -// recentFloorLocked returns the recovery floor under s.mu. When AppProposals, -// AppQCs, and Blocks are persisted, data.State eviction keeps one height before -// the lower of their durable tips. That height can be inside a QC range. -func (s *blockDB) recentFloorLocked() types.GlobalBlockNumber { - floor := s.watermark - if s.hasAppProposal && s.hasAppQC && s.hasBlocks { - nextAppProposal := s.lastAppPropNext - nextAppQC := s.lastAppQCNext - nextBlock := s.lastBlockNumber + 1 - evictionBound := min(nextAppProposal, nextAppQC, nextBlock) - if evictionBound > floor { - floor = evictionBound - 1 - } - } - return floor -} func (s *blockDB) Flush() error { return nil } func (s *blockDB) Status() types.DBStatus { s.mu.RLock() defer s.mu.RUnlock() + return s.statusLocked() +} + +func (s *blockDB) statusLocked() types.DBStatus { var tips types.DBStatus if s.hasBlocks { tips.NextBlock = s.lastBlockNumber + 1 @@ -323,8 +310,8 @@ func (s *blockDB) ReadRecent() (types.RecentData, error) { s.mu.RLock() defer s.mu.RUnlock() + floor := s.statusLocked().Floor() var recent types.RecentData - floor := s.recentFloorLocked() var targetIndex types.RoadIndex bounded := s.hasAppProposal && s.hasAppQC && s.hasBlocks if s.hasAppQC { @@ -409,7 +396,7 @@ func (s *blockDB) sortedBlockNumbersLocked() []types.GlobalBlockNumber { for n := range s.byNumber { nums = append(nums, n) } - sort.Slice(nums, func(i, j int) bool { return nums[i] < nums[j] }) + slices.Sort(nums) return nums } diff --git a/sei-tendermint/autobahn/types/block_db.go b/sei-tendermint/autobahn/types/block_db.go index 69bcf90c9b..3ff7f54a96 100644 --- a/sei-tendermint/autobahn/types/block_db.go +++ b/sei-tendermint/autobahn/types/block_db.go @@ -304,6 +304,17 @@ type DBStatus struct { NextAppProposal GlobalBlockNumber } +// Floor returns the startup recovery floor implied by the durable data tips. +// Until blocks, AppProposals, and AppQCs are all present, there is no app +// recovery floor yet, so Floor returns zero. +func (s DBStatus) Floor() GlobalBlockNumber { + f := min(s.NextQC, s.NextBlock, s.NextAppProposal, s.NextAppQC) + if f > 0 { + f -= 1 + } + return f +} + // RecentBlock is one block returned by BlockDB.ReadRecent. type RecentBlock struct { Number GlobalBlockNumber diff --git a/sei-tendermint/autobahn/types/types_test.go b/sei-tendermint/autobahn/types/types_test.go index 7f57a70c99..faf2bdb23d 100644 --- a/sei-tendermint/autobahn/types/types_test.go +++ b/sei-tendermint/autobahn/types/types_test.go @@ -96,6 +96,51 @@ func TestMarshal(t *testing.T) { } } +func TestDBStatusFloor(t *testing.T) { + for _, tc := range []struct { + name string + in DBStatus + want GlobalBlockNumber + }{ + { + name: "empty", + in: DBStatus{ + NextBlock: 0, + NextQC: 0, + NextAppProposal: 0, + NextAppQC: 0, + }, + want: 0, + }, + { + name: "missing app proposal", + in: DBStatus{ + NextBlock: 10, + NextQC: 10, + NextAppProposal: 0, + NextAppQC: 8, + }, + want: 0, + }, + { + name: "minimum durable tip minus one", + in: DBStatus{ + NextBlock: 12, + NextQC: 12, + NextAppProposal: 10, + NextAppQC: 8, + }, + want: 7, + }, + } { + t.Run(tc.name, func(t *testing.T) { + if got := tc.in.Floor(); got != tc.want { + t.Fatalf("Floor() = %d, want %d", got, tc.want) + } + }) + } +} + func makePrepareQC(keys []SecretKey, vote *PrepareVote) *PrepareQC { var votes []*Signed[*PrepareVote] for _, k := range keys { diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index 7ab6072fe2..0a3c86778f 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -47,41 +47,45 @@ type inner struct { blocks map[types.GlobalBlockNumber]*types.Block // [first, nextBlock) + gap-fills in [nextBlock, nextQC) appProposals map[types.GlobalBlockNumber]*types.AppProposal // [first, nextAppProposal) appQCs map[types.GlobalBlockNumber]*types.AppQC // [first, nextAppQC) - blockHashes map[types.BlockHeaderHash]types.GlobalBlockNumber // blockHashes: mirrors blocks (insertBlock / evictBelowBound) + blockHashes map[types.BlockHeaderHash]types.GlobalBlockNumber // blockHashes mirrors blocks (insertBlock / setPersisted) // first is the exclusive low end of retained in-memory state: maps keep [first, next*). - // Advanced by evictBelowBound t (nextAppProposal, nextAppQCToPersist)-1. + // Advanced by setPersisted to persisted.Floor(). // - // first <= nextAppProposal <= nextBlockToPersist <= nextBlock <= nextQC - // first <= nextAppQCToPersist <= nextAppQC <= nextQC + // first <= persisted.NextBlock <= nextBlock <= nextQC + // first <= persisted.NextAppProposal <= nextAppProposal <= nextQC + // first <= persisted.NextAppQC <= nextAppQC <= nextQC // - // AppProposals require persistence (nextAppProposal <= nextBlockToPersist). - first types.GlobalBlockNumber - nextAppProposal types.GlobalBlockNumber - nextAppQC types.GlobalBlockNumber - nextAppQCToPersist types.GlobalBlockNumber - nextBlockToPersist types.GlobalBlockNumber - nextBlock types.GlobalBlockNumber - nextQC types.GlobalBlockNumber + // AppProposals require block persistence (nextAppProposal <= persisted.NextBlock). + first types.GlobalBlockNumber + nextAppProposal types.GlobalBlockNumber + nextAppQC types.GlobalBlockNumber + nextBlock types.GlobalBlockNumber + nextQC types.GlobalBlockNumber + persisted types.DBStatus anchor utils.AtomicSend[utils.Option[Anchor]] } func newInner(first types.GlobalBlockNumber) *inner { return &inner{ - qcs: map[types.GlobalBlockNumber]*types.FullCommitQC{}, - blocks: map[types.GlobalBlockNumber]*types.Block{}, - appQCs: map[types.GlobalBlockNumber]*types.AppQC{}, - appProposals: map[types.GlobalBlockNumber]*types.AppProposal{}, - blockHashes: map[types.BlockHeaderHash]types.GlobalBlockNumber{}, - first: first, - nextAppProposal: first, - nextAppQC: first, - nextAppQCToPersist: first, - nextBlockToPersist: first, - nextBlock: first, - nextQC: first, - anchor: utils.NewAtomicSend(utils.None[Anchor]()), + qcs: map[types.GlobalBlockNumber]*types.FullCommitQC{}, + blocks: map[types.GlobalBlockNumber]*types.Block{}, + appQCs: map[types.GlobalBlockNumber]*types.AppQC{}, + appProposals: map[types.GlobalBlockNumber]*types.AppProposal{}, + blockHashes: map[types.BlockHeaderHash]types.GlobalBlockNumber{}, + first: first, + nextAppProposal: first, + nextAppQC: first, + nextBlock: first, + nextQC: first, + persisted: types.DBStatus{ + NextQC: first, + NextAppProposal: first, + NextAppQC: first, + NextBlock: first, + }, + anchor: utils.NewAtomicSend(utils.None[Anchor]()), } } @@ -294,16 +298,8 @@ func loadFromBlockDB(cfg *Config, blockDB types.BlockDB) (*inner, error) { // Advance nextBlock through contiguous loaded blocks. Don't use // updateNextBlock: stale timestamps would skew metrics. inner.nextBlock = status.NextBlock - inner.nextBlockToPersist = status.NextBlock inner.nextAppProposal = status.NextAppProposal - inner.nextAppQCToPersist = status.NextAppQC - if inner.first < inner.nextAppQCToPersist { - n := inner.nextAppQCToPersist - 1 - inner.anchor.Store(utils.Some(Anchor{ - CommitQC: inner.qcs[n].QC(), - AppQC: inner.appQCs[n], - })) - } + inner.setPersisted(status) return inner, nil } @@ -451,7 +447,7 @@ func (s *State) NextBlock() types.GlobalBlockNumber { // GlobalBlockByHash returns the finalized GlobalBlock whose stored header // hashes to the given value, or None if no such block is currently retained. // Non-blocking. Serves from RAM whenever the hash is still indexed (contiguous -// prefix, gap-fills, and executed heights not yet dropped by evictBelowBound). +// prefix, gap-fills, and executed heights not yet dropped by setPersisted). // Falls back to BlockDB only after eviction removes the hash — matching // Block/TryBlock/QC, which also prefer maps before the store. Gap-fills are // not written to BlockDB until nextBlock catches up, so they must be served @@ -631,7 +627,7 @@ func (s *State) globalBlockByHashFromDB(hash types.BlockHeaderHash) (utils.Optio func (s *State) PushAppHash(ctx context.Context, n types.GlobalBlockNumber, hash types.AppHash) error { for inner, ctrl := range s.inner.Lock() { if err := ctrl.WaitUntil(ctx, func() bool { - return n < inner.nextBlockToPersist + return n < inner.persisted.NextBlock }); err != nil { return err } @@ -654,7 +650,6 @@ func (s *State) PushAppHash(ctx context.Context, n types.GlobalBlockNumber, hash inner.appProposals[inner.nextAppProposal] = proposal inner.nextAppProposal += 1 } - inner.evict() ctrl.Updated() } return nil @@ -718,7 +713,7 @@ type Anchor struct { AppQC *types.AppQC } -// Anchor represents the latest persisted AppQC. +// Anchor represents the AppQC/CommitQC covering inner.first. // It is used by avail.State. func (s *State) Anchor() utils.AtomicRecv[utils.Option[Anchor]] { for inner := range s.inner.Lock() { @@ -766,25 +761,26 @@ func (s *State) WaitUntilExecuted(ctx context.Context, lane types.LaneID, n type } // PruneBefore asks BlockDB to drop data before retainFrom. This is independent -// of in-memory retention: RAM is cleared only by evictBelowBound (AppQC floor). +// of in-memory retention: RAM is cleared only by setPersisted. // BlockDB enforces its own never-empty retention and refuses reads below its // watermark. func (s *State) PruneBefore(retainFrom types.GlobalBlockNumber) error { return s.blockDB.PruneBefore(retainFrom) } -// runPersist is a background goroutine that persists blocks, QCs, and AppQCs to -// BlockDB. It waits for in-memory blocks to advance past the block persistence -// cursor, then writes covering QCs (first, per the BlockDB contract) and blocks, -// then flushes once per batch. nextBlockToPersist advances with the block tip -// to unblock PushAppHash only when data is durable. AppQCs are persisted later, -// once their matching CommitQC range is already durable. +// runPersist is a background goroutine that persists blocks, QCs, +// AppProposals, and AppQCs to BlockDB. It waits for in-memory data to advance +// past the DBStatus persistence cursor, then writes covering QCs (first, per +// the BlockDB contract), blocks, AppProposals, and AppQCs, then flushes once per +// batch. persisted.NextBlock advances with the block tip to unblock PushAppHash +// only when data is durable. AppProposals and AppQCs are persisted once their +// matching CommitQC range is already durable. // Errors propagate vertically (kill the component). // // Cursors seed from BlockDB.Status() when non-zero so PushQC-before-Run heights -// are not skipped. When a tip is zero, seed from post-load nextBlockToPersist -// (recovery floor), never bare registry.FirstBlock() — a QC-only store can -// skipTo past genesis while NextBlock is still zero. +// are not skipped. When a tip is zero, seed from the recovery floor, never bare +// registry.FirstBlock() — a QC-only store can skipTo past genesis while +// NextBlock is still zero. // // Under the BlockDB write contract (QC before covered blocks), NextQC is never // behind NextBlock after a successful write. Persistence is driven by the block @@ -792,25 +788,19 @@ func (s *State) PruneBefore(retainFrom types.GlobalBlockNumber) error { // is still at or past NextQC (enough coverage for each new block, not every // in-memory QC, and no rewrite of QCs already on disk). // -// In-memory block/QC eviction is driven by PushQC / PushAppHash / -// PushAppQC (evictBelowBound); AppQC entries are retained until their own -// persistence cursor catches up. +// In-memory block/QC/AppProposal/AppQC eviction is driven by persisted DBStatus +// changes. Entries are retained until all three durable data streams (blocks, +// AppProposals, and AppQCs) have caught up. func (s *State) runPersist(ctx context.Context) error { - status := s.blockDB.Status() - // Account for empty blockDB. - for inner := range s.inner.Lock() { - status.NextQC = max(status.NextQC, inner.first) - status.NextAppQC = max(status.NextAppQC, inner.first) - status.NextAppProposal = max(status.NextAppProposal, inner.first) - status.NextBlock = max(status.NextBlock, inner.first) - } for { var qcs []*types.FullCommitQC var blocks []blockEntry var appProposals []*types.AppProposal var appQCs []*types.AppQC + var status types.DBStatus for inner, ctrl := range s.inner.Lock() { - // Wait until there is anythin to persist. + status = inner.persisted + // Wait until there is anything to persist. if err := ctrl.WaitUntil(ctx, func() bool { return status.NextQC < inner.nextQC || status.NextBlock < inner.nextBlock || status.NextAppProposal < inner.nextAppProposal || status.NextAppQC < inner.nextAppQC @@ -863,28 +853,19 @@ func (s *State) runPersist(ctx context.Context) error { return fmt.Errorf("flush BlockDB: %w", err) } for inner, ctrl := range s.inner.Lock() { - inner.nextBlockToPersist = status.NextBlock - if inner.nextAppQCToPersist < status.NextAppQC { - inner.nextAppQCToPersist = status.NextAppQC - inner.anchor.Store(utils.Some(Anchor{ - CommitQC: inner.qcs[status.NextAppQC-1].QC(), - AppQC: inner.appQCs[status.NextAppQC-1], - })) - inner.evict() - } + inner.setPersisted(status) ctrl.Updated() } } } -// evict pushes first to min(i.nextAppProposal, i.nextAppQCToPersist)-1 -// I.e. it makes sure that at least 1 persisted appQC is still in memory: -// it is passed to avail.State. -func (i *inner) evict() { - bound := min(i.nextAppQCToPersist, i.nextAppProposal) - if bound > i.first { - bound -= 1 - } +// setPersisted publishes a new durable cursor and pushes first to +// persisted.Floor(). +// I.e. it keeps the same recovery suffix in memory that BlockDB.ReadRecent would +// return. +func (i *inner) setPersisted(persisted types.DBStatus) { + i.persisted = persisted + bound := persisted.Floor() for i.first < bound { n := i.first delete(i.blockHashes, i.blocks[n].Header().Hash()) @@ -894,6 +875,12 @@ func (i *inner) evict() { delete(i.appProposals, n) i.first += 1 } + if i.first < persisted.NextAppQC { + i.anchor.Store(utils.Some(Anchor{ + CommitQC: i.qcs[i.first].QC(), + AppQC: i.appQCs[i.first], + })) + } } // Run starts the background persistence loop. diff --git a/sei-tendermint/internal/autobahn/data/state_recovery_test.go b/sei-tendermint/internal/autobahn/data/state_recovery_test.go index 9ce1c48080..56c135e133 100644 --- a/sei-tendermint/internal/autobahn/data/state_recovery_test.go +++ b/sei-tendermint/internal/autobahn/data/state_recovery_test.go @@ -216,6 +216,7 @@ func TestRecoveryLeavesAppTipBelowPruneFloorUnreadable(t *testing.T) { writeToBlockDB(t, db, []*types.FullCommitQC{qc1, qc2}, [][]*types.Block{blocks1, blocks2}) + writeAppDataToBlockDB(t, rng, db, keys, qc1, qc2) require.NoError(t, db.PruneBefore(qc2.QC().GlobalRange().First)) state, err := NewState(&Config{ @@ -485,8 +486,8 @@ func TestRecoveryQCsNoBlocks(t *testing.T) { // TestRunPersistSeedsFromRecoveryFloor verifies that runPersist does not walk // [genesis, recoveryFloor) when Status lacks NextBlock (QC-only store -// whose first QC starts past FirstBlock). Seeding from nextBlockToPersist -// avoids collecting nil block pointers. +// whose first QC starts past FirstBlock). Seeding persisted DBStatus from the +// recovery floor avoids collecting nil block pointers. func TestRunPersistSeedsFromRecoveryFloor(t *testing.T) { ctx := t.Context() rng := utils.TestRng() diff --git a/sei-tendermint/internal/autobahn/data/state_test.go b/sei-tendermint/internal/autobahn/data/state_test.go index 64c3a022f6..485c281f44 100644 --- a/sei-tendermint/internal/autobahn/data/state_test.go +++ b/sei-tendermint/internal/autobahn/data/state_test.go @@ -78,6 +78,16 @@ func writeToBlockDB(t *testing.T, db types.BlockDB, qcs []*types.FullCommitQC, b utils.OrPanic(db.Flush()) } +func writeAppDataToBlockDB(t testing.TB, rng utils.Rng, db types.BlockDB, keys []types.SecretKey, qcs ...*types.FullCommitQC) { + t.Helper() + for _, qc := range qcs { + appProposal := types.NewAppProposal(qc.QC().Proposal(), types.GenAppHash(rng)) + utils.OrPanic(db.WriteAppProposal(appProposal)) + utils.OrPanic(db.WriteAppQC(TestAppQC(keys, appProposal))) + } + utils.OrPanic(db.Flush()) +} + // pushAppHashesRunning runs state.Run under scope.Run long enough to accept // PushAppHash for [first, next), then cancels Run. Prefers scope.Run over a // raw goroutine so cleanup is structured. @@ -452,7 +462,7 @@ func TestPushQCBeforeRunPersistsToBlockDB(t *testing.T) { s.SpawnBgNamed("state.Run", func() error { return utils.IgnoreCancel(state.Run(runCtx)) }) - // PushAppHash waits on nextBlockToPersist, so success implies Flush. + // PushAppHash waits on persisted.NextBlock, so success implies Flush. for n := gr1.First; n < gr1.Next; n++ { if err := state.PushAppHash(ctx, n, types.GenAppHash(rng)); err != nil { cancel() @@ -478,9 +488,9 @@ func TestPushQCBeforeRunPersistsToBlockDB(t *testing.T) { } } -// TestEvictionWaitsForAppQC checks that evictBelowBound does not drop +// TestEvictionWaitsForAppQC checks that setPersisted does not drop // AppProposals until AppQC is persisted, and that once it is, heights below -// min(nextAppProposal, nextAppQCToPersist) are evicted. +// persisted.Floor() are evicted. func TestEvictionWaitsForAppQC(t *testing.T) { ctx := t.Context() rng := utils.TestRng() @@ -541,10 +551,13 @@ func TestEvictionWaitsForAppQC(t *testing.T) { } for inner := range state.inner.Lock() { - evictionBound := min(inner.nextAppProposal, inner.nextAppQCToPersist) - 1 + evictionBound := inner.persisted.Floor() if inner.first != evictionBound { return fmt.Errorf("after catching up, first = %d, want eviction bound %d", inner.first, evictionBound) } + if anchor, ok := inner.anchor.Load().Get(); !ok || anchor.AppQC != inner.appQCs[inner.first] || anchor.CommitQC != inner.qcs[inner.first].QC() { + return fmt.Errorf("anchor must cover inner.first %d", inner.first) + } for n := gr1.First; n < inner.first; n++ { _, ok := inner.appProposals[n] if ok { @@ -586,7 +599,7 @@ func TestEvictionWaitsForPersistedAppQC(t *testing.T) { for inner := range state.inner.Lock() { require.Equal(t, gr1.Next, inner.nextAppQC) - require.Equal(t, gr1.First, inner.nextAppQCToPersist) + require.Equal(t, gr1.First, inner.persisted.NextAppQC) require.Equal(t, gr1.First, inner.first, "accepted but unpersisted AppQC must not advance eviction") for n := gr1.First; n < gr1.Next; n++ { _, ok := inner.appProposals[n] @@ -597,8 +610,8 @@ func TestEvictionWaitsForPersistedAppQC(t *testing.T) { // TestNextToExecuteAfterAppEviction checks WaitUntilExecuted / nextToExecute // still work when persisted AppQC aggressively evicts through nextAppProposal -// (first = min(nextAppProposal, nextAppQCToPersist) = NAP). nextToExecute uses -// qc[NAP], not NAP-1. +// (first = persisted.Floor()). +// nextToExecute uses the retained boundary QC. func TestNextToExecuteAfterAppEviction(t *testing.T) { ctx := t.Context() rng := utils.TestRng() @@ -624,8 +637,8 @@ func TestNextToExecuteAfterAppEviction(t *testing.T) { return fmt.Errorf("PushAppHash(%d): %w", n, err) } } - // Sticky case: nextAppQCToPersist == nextAppProposal. first advances to - // NAP; NAP-1 is gone; nextToExecute reads qc[NAP] after the next QC arrives. + // Sticky case: persisted.NextAppQC == nextAppProposal. first advances to + // NAP-1; NAP-2 is gone; nextToExecute reads qc[NAP-1] until the next QC executes. if err := pushAppQCForBlock(ctx, state, keys, gr1.First); err != nil { return fmt.Errorf("pushAppQCForBlock(%d): %w", gr1.First, err) } @@ -647,7 +660,7 @@ func TestNextToExecuteAfterAppEviction(t *testing.T) { if inner.nextAppProposal != gr1.Next { return fmt.Errorf("nextAppProposal = %d, want %d", inner.nextAppProposal, gr1.Next) } - evictionBound := min(inner.nextAppProposal, inner.nextAppQCToPersist) - 1 + evictionBound := inner.persisted.Floor() if inner.first != evictionBound { return fmt.Errorf("first = %d, want eviction bound %d", inner.first, evictionBound) } @@ -726,6 +739,9 @@ func TestPushAppQCPersistsAndRecovers(t *testing.T) { return nil })) + storedProposal, err := db1.ReadAppProposalByBlockNumber(gr1.First) + require.NoError(t, err) + require.True(t, storedProposal.IsPresent(), "PushAppHash must persist the AppProposal") stored, err := db1.ReadAppQCByBlockNumber(gr1.First) require.NoError(t, err) require.True(t, stored.IsPresent(), "PushAppQC must persist the AppQC") @@ -734,6 +750,7 @@ func TestPushAppQCPersistsAndRecovers(t *testing.T) { db2 := newTestBlockDB(t, dir) state2 := newTestState(t, &Config{Registry: registry}, db2) for inner := range state2.inner.Lock() { + require.Equal(t, gr1.Next, inner.nextAppProposal) require.Equal(t, gr1.Next, inner.nextAppQC) } appQC, fQC := state2.LastAppQC() @@ -805,7 +822,7 @@ func TestPruningKeepsLastQCRange(t *testing.T) { // readability), so a mid-range prune does not refuse heights inside that QC. // // PruneBefore is BlockDB-only: heights still retained in RAM for AppVotes -// (at/above min(nextAppProposal, nextAppQCToPersist) exclusive floor) remain +// (at/above persisted.Floor()) remain // readable via TryBlock even after the store watermark advances past them. func TestPruningWithPartialQCRange(t *testing.T) { ctx := t.Context() @@ -843,7 +860,7 @@ func TestPruningWithPartialQCRange(t *testing.T) { return nil })) for inner := range state1.inner.Lock() { - exclusiveFloor = min(inner.nextAppProposal, inner.nextAppQCToPersist) - 1 + exclusiveFloor = inner.persisted.Floor() require.Equal(t, exclusiveFloor, inner.first) } From fe34f24e8746b5fc2e890cf9de618408cfa329e3 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 7 Aug 2026 14:27:12 +0200 Subject: [PATCH 24/61] WIP --- sei-db/ledger_db/block/block_db_test.go | 4 +++- .../block/littblock/litt_block_db.go | 14 ++++++++----- .../ledger_db/block/memblock/mem_block_db.go | 14 +++++-------- sei-tendermint/autobahn/types/block_db.go | 7 ++++--- .../internal/autobahn/avail/inner.go | 9 +-------- .../internal/autobahn/data/state.go | 20 +++++++++---------- 6 files changed, 31 insertions(+), 37 deletions(-) diff --git a/sei-db/ledger_db/block/block_db_test.go b/sei-db/ledger_db/block/block_db_test.go index 400fe26a36..ed1e6a9e37 100644 --- a/sei-db/ledger_db/block/block_db_test.go +++ b/sei-db/ledger_db/block/block_db_test.go @@ -167,10 +167,11 @@ func drainRecent(t *testing.T, db types.BlockDB) []iterEntry { recent, err := db.ReadRecent() require.NoError(t, err) var entries []iterEntry + floor := recent.Status.Floor() for _, qc := range recent.CommitQCs { first := qc.QC().GlobalRange().First next := first + gbn(len(qc.Headers())) - for n := max(first, recent.First); n < next; n++ { + for n := max(first, floor); n < next; n++ { entries = append(entries, iterEntry{n: n, qc: qc}) } } @@ -948,6 +949,7 @@ func testReadRecent(t *testing.T, build builder) { recent, err := db.ReadRecent() require.NoError(t, err) + require.Equal(t, db.Status(), recent.Status) gotAppQC, ok := recent.AppQC.Get() require.True(t, ok) require.Equal(t, appQC.Proposal().RoadIndex(), gotAppQC.Proposal().RoadIndex()) diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index ba87127e05..ffcaa91736 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -515,17 +515,24 @@ func (s *blockDB) recentFloor() types.GlobalBlockNumber { return s.statusLocked().Floor() } +func (s *blockDB) recentFloorAndStatus() (types.GlobalBlockNumber, types.DBStatus) { + s.mu.Lock() + defer s.mu.Unlock() + status := s.statusLocked() + return status.Floor(), status +} + // ReadRecent() reads the latest AppQC/AppProposal recovery suffix. // WARNING: ReadRecent() will return an error if watermark is moved during iteration. func (s *blockDB) ReadRecent() (types.RecentData, error) { - targetFloor := s.recentFloor() + targetFloor, status := s.recentFloorAndStatus() // Collect data >= targetFloor. it, err := s.table.Iterator(true) if err != nil { return types.RecentData{}, fmt.Errorf("failed to open recent-data iterator: %w", err) } defer func() { _ = it.Close() }() - recent := types.RecentData{First: targetFloor} + recent := types.RecentData{Status: status} for { ok, err := it.Next() if err != nil { @@ -594,9 +601,6 @@ func (s *blockDB) ReadRecent() (types.RecentData, error) { slices.Reverse(recent.CommitQCs) slices.Reverse(recent.Blocks) slices.Reverse(recent.AppProposals) - if len(recent.Blocks) > 0 { - recent.First = recent.Blocks[0].Number - } return recent, nil } diff --git a/sei-db/ledger_db/block/memblock/mem_block_db.go b/sei-db/ledger_db/block/memblock/mem_block_db.go index 0350430378..afa3f982e9 100644 --- a/sei-db/ledger_db/block/memblock/mem_block_db.go +++ b/sei-db/ledger_db/block/memblock/mem_block_db.go @@ -2,9 +2,9 @@ package memblock import ( "fmt" + "slices" "sort" "sync" - "slices" "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" @@ -243,7 +243,7 @@ func (s *blockDB) PruneBefore(n types.GlobalBlockNumber) error { // future block whose coverage check still passes. Mirrors littblock. return nil } - n = min(n,s.statusLocked().Floor()) + n = min(n, s.statusLocked().Floor()) // Round the watermark down to the covering QC's First. A QC's cohort of // blocks changes readability atomically, so the watermark must never fall // strictly inside a QC's range (see littblock): otherwise a read would @@ -280,7 +280,6 @@ func (s *blockDB) PruneBefore(n types.GlobalBlockNumber) error { return nil } - func (s *blockDB) Flush() error { return nil } func (s *blockDB) Status() types.DBStatus { @@ -310,8 +309,9 @@ func (s *blockDB) ReadRecent() (types.RecentData, error) { s.mu.RLock() defer s.mu.RUnlock() - floor := s.statusLocked().Floor() - var recent types.RecentData + status := s.statusLocked() + floor := status.Floor() + recent := types.RecentData{Status: status} var targetIndex types.RoadIndex bounded := s.hasAppProposal && s.hasAppQC && s.hasBlocks if s.hasAppQC { @@ -325,7 +325,6 @@ func (s *blockDB) ReadRecent() (types.RecentData, error) { if !bounded && s.hasBlocks { floor = max(floor, s.firstBlockNumber) } - recent.First = floor for _, e := range s.sortedQCsLocked() { if e.upper <= s.watermark { @@ -348,9 +347,6 @@ func (s *blockDB) ReadRecent() (types.RecentData, error) { } recent.Blocks = append(recent.Blocks, types.RecentBlock{Number: n, Block: s.byNumber[n]}) } - if !bounded && len(recent.Blocks) > 0 { - recent.First = recent.Blocks[0].Number - } return recent, nil } diff --git a/sei-tendermint/autobahn/types/block_db.go b/sei-tendermint/autobahn/types/block_db.go index 3ff7f54a96..caeca82fb7 100644 --- a/sei-tendermint/autobahn/types/block_db.go +++ b/sei-tendermint/autobahn/types/block_db.go @@ -283,8 +283,8 @@ type BlockDB interface { } // DBStatus is the in-memory write tips returned by BlockDB.Status. -// Both fields are exclusive "next to write" cursors (matching data.State's -// nextQC / nextBlock). Zero means no write of that kind has occurred yet +// Its fields are exclusive "next to write" cursors (matching data.State's +// DBStatus). Zero means no write of that kind has occurred yet // (NextBlock/NextQC are never zero after a successful write: the first // written block number N yields NextBlock = N+1 ≥ 1). type DBStatus struct { @@ -323,7 +323,8 @@ type RecentBlock struct { // RecentData is the materialized suffix used by data.State startup recovery. type RecentData struct { - First GlobalBlockNumber + // Status is the durable write status observed while selecting the recent suffix. + Status DBStatus CommitQCs []*FullCommitQC Blocks []RecentBlock AppProposals []*AppProposal diff --git a/sei-tendermint/internal/autobahn/avail/inner.go b/sei-tendermint/internal/autobahn/avail/inner.go index 1c41c7f3c1..baad7e7e01 100644 --- a/sei-tendermint/internal/autobahn/avail/inner.go +++ b/sei-tendermint/internal/autobahn/avail/inner.go @@ -153,15 +153,8 @@ func (i *inner) updateNextAppQC() bool { return updated } -// prune advances the state to account for a new AppQC/CommitQC pair. +// prune advances the state up to Anchor of the data state. // Returns true iff pruning occurred. -// It is safe to prune on data.Anchor, because it proves that: -// * AppQC was formed for the given height (and it will be available on restart) -// * some honest nodes have voted for AppHash -// * AppHash voting is allowed only after persisting the executed blocks i blockDB. -// * blocks and FullCommitQC (sequencing proof) are available in data.State. -// TODO(gprusak): consider simplifying this invariant by making Anchor require -// locally persisted blocks as well. func (i *inner) prune(epoch *types.Epoch, anchor data.Anchor) { idx := anchor.CommitQC.Index() if idx < i.roads.first { diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index 0a3c86778f..4dd1c2a666 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -55,8 +55,6 @@ type inner struct { // first <= persisted.NextBlock <= nextBlock <= nextQC // first <= persisted.NextAppProposal <= nextAppProposal <= nextQC // first <= persisted.NextAppQC <= nextAppQC <= nextQC - // - // AppProposals require block persistence (nextAppProposal <= persisted.NextBlock). first types.GlobalBlockNumber nextAppProposal types.GlobalBlockNumber nextAppQC types.GlobalBlockNumber @@ -238,28 +236,28 @@ func NewState(cfg *Config, blockDB types.BlockDB) (*State, error) { // loadFromBlockDB replays the recent persisted suffix from blockDB into s.inner. // Called from NewState before any goroutines are spawned. func loadFromBlockDB(cfg *Config, blockDB types.BlockDB) (*inner, error) { - firstBlock := cfg.Registry.FirstBlock() - status := blockDB.Status() - recent, err := blockDB.ReadRecent() if err != nil { return nil, fmt.Errorf("blockDB.ReadRecent(): %w", err) } - + firstBlock := cfg.Registry.FirstBlock() first := firstBlock - if (len(recent.CommitQCs) > 0 || len(recent.Blocks) > 0 || len(recent.AppProposals) > 0 || recent.AppQC.IsPresent()) && - recent.First >= firstBlock { - first = recent.First + if len(recent.Blocks) > 0 { + first = recent.Blocks[0].Number } else if appQC, ok := recent.AppQC.Get(); ok { first = appQC.Proposal().GlobalRange().First - } else if len(recent.Blocks) > 0 { - first = recent.Blocks[0].Number + } else if len(recent.AppProposals) > 0 { + first = recent.AppProposals[0].GlobalRange().First } else if len(recent.CommitQCs) > 0 { first = recent.CommitQCs[0].QC().GlobalRange().First + } else if floor := recent.Status.Floor(); floor >= firstBlock { + first = floor } if first < firstBlock { return nil, fmt.Errorf("db contains data before genesis") } + + status := recent.Status status.NextQC = max(status.NextQC, first) status.NextBlock = max(status.NextBlock, first) status.NextAppQC = max(status.NextAppQC, first) From 90d264fbcfebf64ba23bcd5f1e6ab36ed0766d38 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 7 Aug 2026 14:32:44 +0200 Subject: [PATCH 25/61] floor --- .../internal/autobahn/data/state.go | 30 ++----------------- 1 file changed, 3 insertions(+), 27 deletions(-) diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index 4dd1c2a666..c3a095d37f 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -240,30 +240,7 @@ func loadFromBlockDB(cfg *Config, blockDB types.BlockDB) (*inner, error) { if err != nil { return nil, fmt.Errorf("blockDB.ReadRecent(): %w", err) } - firstBlock := cfg.Registry.FirstBlock() - first := firstBlock - if len(recent.Blocks) > 0 { - first = recent.Blocks[0].Number - } else if appQC, ok := recent.AppQC.Get(); ok { - first = appQC.Proposal().GlobalRange().First - } else if len(recent.AppProposals) > 0 { - first = recent.AppProposals[0].GlobalRange().First - } else if len(recent.CommitQCs) > 0 { - first = recent.CommitQCs[0].QC().GlobalRange().First - } else if floor := recent.Status.Floor(); floor >= firstBlock { - first = floor - } - if first < firstBlock { - return nil, fmt.Errorf("db contains data before genesis") - } - - status := recent.Status - status.NextQC = max(status.NextQC, first) - status.NextBlock = max(status.NextBlock, first) - status.NextAppQC = max(status.NextAppQC, first) - status.NextAppProposal = max(status.NextAppProposal, first) - - inner := newInner(first) + inner := newInner(max(cfg.Registry.FirstBlock(),recent.Status.Floor())) for _, qc := range recent.CommitQCs { if err := inner.insertQC(cfg.Registry, qc); err != nil { return nil, fmt.Errorf("load QC from BlockDB: %w", err) @@ -295,9 +272,8 @@ func loadFromBlockDB(cfg *Config, blockDB types.BlockDB) (*inner, error) { } // Advance nextBlock through contiguous loaded blocks. Don't use // updateNextBlock: stale timestamps would skew metrics. - inner.nextBlock = status.NextBlock - inner.nextAppProposal = status.NextAppProposal - inner.setPersisted(status) + inner.nextBlock = max(inner.first,recent.Status.NextBlock) + inner.setPersisted(recent.Status) return inner, nil } From 8ad28820268df20e1f07ebe5b13094c672f3c5b8 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 7 Aug 2026 14:58:09 +0200 Subject: [PATCH 26/61] more precise status --- sei-db/ledger_db/block/block_db_test.go | 4 +- .../block/littblock/litt_block_db.go | 45 +++++++++-------- .../ledger_db/block/memblock/mem_block_db.go | 31 +++++++----- sei-tendermint/autobahn/types/block_db.go | 2 +- .../internal/autobahn/data/state.go | 49 +++++++++---------- 5 files changed, 68 insertions(+), 63 deletions(-) diff --git a/sei-db/ledger_db/block/block_db_test.go b/sei-db/ledger_db/block/block_db_test.go index ed1e6a9e37..5b8ab5b74c 100644 --- a/sei-db/ledger_db/block/block_db_test.go +++ b/sei-db/ledger_db/block/block_db_test.go @@ -167,7 +167,7 @@ func drainRecent(t *testing.T, db types.BlockDB) []iterEntry { recent, err := db.ReadRecent() require.NoError(t, err) var entries []iterEntry - floor := recent.Status.Floor() + floor := recent.Status.Or(types.DBStatus{}).Floor() for _, qc := range recent.CommitQCs { first := qc.QC().GlobalRange().First next := first + gbn(len(qc.Headers())) @@ -949,7 +949,7 @@ func testReadRecent(t *testing.T, build builder) { recent, err := db.ReadRecent() require.NoError(t, err) - require.Equal(t, db.Status(), recent.Status) + require.Equal(t, db.Status(), recent.Status.OrPanic("recent status")) gotAppQC, ok := recent.AppQC.Get() require.True(t, ok) require.Equal(t, appQC.Proposal().RoadIndex(), gotAppQC.Proposal().RoadIndex()) diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index ffcaa91736..b9eac4897a 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -425,7 +425,7 @@ func (s *blockDB) PruneBefore(blockHeight types.GlobalBlockNumber) error { } // Round the watermark down to the start of a QC's range, to avoid pruning a QC before its blocks. - blockHeight, err := s.clampPruneBoundary(min(blockHeight, s.statusLocked().Floor())) + blockHeight, err := s.clampPruneBoundary(min(blockHeight, s.statusLocked().Or(types.DBStatus{}).Floor())) if err != nil { return err } @@ -489,50 +489,49 @@ func (s *blockDB) Flush() error { func (s *blockDB) Status() types.DBStatus { s.mu.Lock() defer s.mu.Unlock() - return s.statusLocked() + return s.statusLocked().Or(types.DBStatus{}) } -func (s *blockDB) statusLocked() types.DBStatus { - var status types.DBStatus +func (s *blockDB) statusLocked() utils.Option[types.DBStatus] { + qc, ok := s.lastQC.Get() + if !ok { + return utils.None[types.DBStatus]() + } + status := types.DBStatus{ + NextBlock: s.oldestQCStart, + NextQC: qc.QC().GlobalRange().Next, + NextAppQC: s.oldestQCStart, + NextAppProposal: s.oldestQCStart, + } if s.hasBlocks { status.NextBlock = s.lastBlockNumber + 1 } - if qc, ok := s.lastQC.Get(); ok { - status.NextQC = qc.QC().GlobalRange().Next - } if appQC, ok := s.lastAppQC.Get(); ok { status.NextAppQC = appQC.Proposal().GlobalRange().Next } if appProposal, ok := s.lastAppProposal.Get(); ok { status.NextAppProposal = appProposal.GlobalRange().Next } - return status + return utils.Some(status) } -func (s *blockDB) recentFloor() types.GlobalBlockNumber { - s.mu.Lock() - defer s.mu.Unlock() - return s.statusLocked().Floor() -} - -func (s *blockDB) recentFloorAndStatus() (types.GlobalBlockNumber, types.DBStatus) { - s.mu.Lock() - defer s.mu.Unlock() - status := s.statusLocked() - return status.Floor(), status -} // ReadRecent() reads the latest AppQC/AppProposal recovery suffix. // WARNING: ReadRecent() will return an error if watermark is moved during iteration. func (s *blockDB) ReadRecent() (types.RecentData, error) { - targetFloor, status := s.recentFloorAndStatus() + s.mu.Lock() + status,ok := s.statusLocked().Get() + s.mu.Unlock() + if !ok { return types.RecentData{},nil } + targetFloor := status.Floor() + // Collect data >= targetFloor. it, err := s.table.Iterator(true) if err != nil { return types.RecentData{}, fmt.Errorf("failed to open recent-data iterator: %w", err) } defer func() { _ = it.Close() }() - recent := types.RecentData{Status: status} + recent := types.RecentData{Status: utils.Some(status)} for { ok, err := it.Next() if err != nil { @@ -595,7 +594,7 @@ func (s *blockDB) ReadRecent() (types.RecentData, error) { } // Safety check: if watermark has been moved and GC happened to get executed during iteration, // the loaded data might be inconsistent with the targetFloor we computed. - if got := s.recentFloor(); got != targetFloor { + if newFloor := s.Status().Floor(); newFloor != targetFloor { return types.RecentData{}, fmt.Errorf("watermark has moved while iterating") } slices.Reverse(recent.CommitQCs) diff --git a/sei-db/ledger_db/block/memblock/mem_block_db.go b/sei-db/ledger_db/block/memblock/mem_block_db.go index afa3f982e9..7fb29ff8e2 100644 --- a/sei-db/ledger_db/block/memblock/mem_block_db.go +++ b/sei-db/ledger_db/block/memblock/mem_block_db.go @@ -243,7 +243,7 @@ func (s *blockDB) PruneBefore(n types.GlobalBlockNumber) error { // future block whose coverage check still passes. Mirrors littblock. return nil } - n = min(n, s.statusLocked().Floor()) + n = min(n, s.statusLocked().Or(types.DBStatus{}).Floor()) // Round the watermark down to the covering QC's First. A QC's cohort of // blocks changes readability atomically, so the watermark must never fall // strictly inside a QC's range (see littblock): otherwise a read would @@ -285,24 +285,31 @@ func (s *blockDB) Flush() error { return nil } func (s *blockDB) Status() types.DBStatus { s.mu.RLock() defer s.mu.RUnlock() - return s.statusLocked() + return s.statusLocked().Or(types.DBStatus{}) } -func (s *blockDB) statusLocked() types.DBStatus { - var tips types.DBStatus - if s.hasBlocks { - tips.NextBlock = s.lastBlockNumber + 1 +func (s *blockDB) statusLocked() utils.Option[types.DBStatus] { + entries := s.sortedQCsLocked() + if len(entries) == 0 { + return utils.None[types.DBStatus]() + } + oldestQCStart := entries[0].lower + status := types.DBStatus{ + NextBlock: oldestQCStart, + NextQC: s.lastQCNext, + NextAppQC: oldestQCStart, + NextAppProposal: oldestQCStart, } - if s.hasQC { - tips.NextQC = s.lastQCNext + if s.hasBlocks { + status.NextBlock = s.lastBlockNumber + 1 } if s.hasAppQC { - tips.NextAppQC = s.lastAppQCNext + status.NextAppQC = s.lastAppQCNext } if s.hasAppProposal { - tips.NextAppProposal = s.lastAppPropNext + status.NextAppProposal = s.lastAppPropNext } - return tips + return utils.Some(status) } func (s *blockDB) ReadRecent() (types.RecentData, error) { @@ -310,7 +317,7 @@ func (s *blockDB) ReadRecent() (types.RecentData, error) { defer s.mu.RUnlock() status := s.statusLocked() - floor := status.Floor() + floor := status.Or(types.DBStatus{}).Floor() recent := types.RecentData{Status: status} var targetIndex types.RoadIndex bounded := s.hasAppProposal && s.hasAppQC && s.hasBlocks diff --git a/sei-tendermint/autobahn/types/block_db.go b/sei-tendermint/autobahn/types/block_db.go index caeca82fb7..167863b622 100644 --- a/sei-tendermint/autobahn/types/block_db.go +++ b/sei-tendermint/autobahn/types/block_db.go @@ -324,7 +324,7 @@ type RecentBlock struct { // RecentData is the materialized suffix used by data.State startup recovery. type RecentData struct { // Status is the durable write status observed while selecting the recent suffix. - Status DBStatus + Status utils.Option[DBStatus] CommitQCs []*FullCommitQC Blocks []RecentBlock AppProposals []*AppProposal diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index c3a095d37f..8e6c6d8864 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -65,28 +65,6 @@ type inner struct { anchor utils.AtomicSend[utils.Option[Anchor]] } -func newInner(first types.GlobalBlockNumber) *inner { - return &inner{ - qcs: map[types.GlobalBlockNumber]*types.FullCommitQC{}, - blocks: map[types.GlobalBlockNumber]*types.Block{}, - appQCs: map[types.GlobalBlockNumber]*types.AppQC{}, - appProposals: map[types.GlobalBlockNumber]*types.AppProposal{}, - blockHashes: map[types.BlockHeaderHash]types.GlobalBlockNumber{}, - first: first, - nextAppProposal: first, - nextAppQC: first, - nextBlock: first, - nextQC: first, - persisted: types.DBStatus{ - NextQC: first, - NextAppProposal: first, - NextAppQC: first, - NextBlock: first, - }, - anchor: utils.NewAtomicSend(utils.None[Anchor]()), - } -} - // insertQC verifies and inserts a FullCommitQC into the inner state. // Accepts QCs whose range starts at or before nextQC (partially pruned // prefix is silently skipped). Rejects gaps where gr.First > nextQC. @@ -240,7 +218,28 @@ func loadFromBlockDB(cfg *Config, blockDB types.BlockDB) (*inner, error) { if err != nil { return nil, fmt.Errorf("blockDB.ReadRecent(): %w", err) } - inner := newInner(max(cfg.Registry.FirstBlock(),recent.Status.Floor())) + firstBlock := cfg.Registry.FirstBlock() + status := recent.Status.Or(types.DBStatus{ + NextQC: firstBlock, + NextAppProposal: firstBlock, + NextAppQC: firstBlock, + NextBlock: firstBlock, + }) + first := status.Floor() + inner := &inner{ + qcs: map[types.GlobalBlockNumber]*types.FullCommitQC{}, + blocks: map[types.GlobalBlockNumber]*types.Block{}, + appQCs: map[types.GlobalBlockNumber]*types.AppQC{}, + appProposals: map[types.GlobalBlockNumber]*types.AppProposal{}, + blockHashes: map[types.BlockHeaderHash]types.GlobalBlockNumber{}, + first: first, + nextAppProposal: first, + nextAppQC: first, + nextBlock: first, + nextQC: first, + persisted: status, + anchor: utils.NewAtomicSend(utils.None[Anchor]()), + } for _, qc := range recent.CommitQCs { if err := inner.insertQC(cfg.Registry, qc); err != nil { return nil, fmt.Errorf("load QC from BlockDB: %w", err) @@ -272,8 +271,8 @@ func loadFromBlockDB(cfg *Config, blockDB types.BlockDB) (*inner, error) { } // Advance nextBlock through contiguous loaded blocks. Don't use // updateNextBlock: stale timestamps would skew metrics. - inner.nextBlock = max(inner.first,recent.Status.NextBlock) - inner.setPersisted(recent.Status) + inner.nextBlock = max(inner.first, status.NextBlock) + inner.setPersisted(status) return inner, nil } From cd348cd24e5e0e91aa2c13355f208582fad132b5 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 7 Aug 2026 15:12:38 +0200 Subject: [PATCH 27/61] floor fixed --- sei-db/ledger_db/block/block_db_test.go | 21 ++++++++++---- .../block/littblock/litt_block_db.go | 28 ++++++++++++++---- .../ledger_db/block/memblock/mem_block_db.go | 29 +++++++++++++++++-- sei-tendermint/autobahn/types/block_db.go | 10 +++++-- sei-tendermint/autobahn/types/types_test.go | 18 ++++++++++-- .../internal/autobahn/data/state.go | 1 + .../autobahn/data/state_recovery_test.go | 9 +++--- 7 files changed, 93 insertions(+), 23 deletions(-) diff --git a/sei-db/ledger_db/block/block_db_test.go b/sei-db/ledger_db/block/block_db_test.go index 5b8ab5b74c..e9e0c15225 100644 --- a/sei-db/ledger_db/block/block_db_test.go +++ b/sei-db/ledger_db/block/block_db_test.go @@ -167,7 +167,13 @@ func drainRecent(t *testing.T, db types.BlockDB) []iterEntry { recent, err := db.ReadRecent() require.NoError(t, err) var entries []iterEntry - floor := recent.Status.Or(types.DBStatus{}).Floor() + floor := recent.Status.Or(types.DBStatus{ + First: 0, + NextBlock: 0, + NextQC: 0, + NextAppQC: 0, + NextAppProposal: 0, + }).Floor() for _, qc := range recent.CommitQCs { first := qc.QC().GlobalRange().First next := first + gbn(len(qc.Headers())) @@ -239,8 +245,11 @@ func testStatus(t *testing.T, build builder) { require.NoError(t, db.WriteQC(batches[0].qc)) tips := db.Status() + require.Equal(t, batches[0].first, tips.First) require.Equal(t, batches[0].next, tips.NextQC) - require.Zero(t, tips.NextBlock, "QC-only store has no block tip") + require.Equal(t, tips.First, tips.NextBlock, "QC-only store has no block tip") + require.Equal(t, tips.First, tips.NextAppQC, "QC-only store has no AppQC tip") + require.Equal(t, tips.First, tips.NextAppProposal, "QC-only store has no AppProposal tip") assertTipsMatchPresent(t, db) for i, blk := range batches[0].blocks { @@ -275,13 +284,13 @@ func assertTipsMatchPresent(t *testing.T, db types.BlockDB) { t.Helper() tips := db.Status() - if tips.NextBlock != 0 { + if tips.NextBlock > tips.First { blk, err := db.ReadBlockByNumber(tips.NextBlock - 1) require.NoError(t, err) require.True(t, blk.IsPresent(), "NextBlock must point past a readable block") } - if tips.NextQC != 0 { + if tips.NextQC > tips.First { qc, err := db.ReadQCByBlockNumber(tips.NextQC - 1) require.NoError(t, err) got, ok := qc.Get() @@ -289,14 +298,14 @@ func assertTipsMatchPresent(t *testing.T, db types.BlockDB) { require.Equal(t, tips.NextQC, got.QC().GlobalRange().Next) } - if tips.NextAppQC != 0 { + if tips.NextAppQC > tips.First { appQC, err := db.ReadAppQCByBlockNumber(tips.NextAppQC - 1) require.NoError(t, err) got, ok := appQC.Get() require.True(t, ok, "NextAppQC must point past a readable AppQC") require.Equal(t, tips.NextAppQC, got.Proposal().GlobalRange().Next) } - if tips.NextAppProposal != 0 { + if tips.NextAppProposal > tips.First { appProposal, err := db.ReadAppProposalByBlockNumber(tips.NextAppProposal - 1) require.NoError(t, err) got, ok := appProposal.Get() diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index b9eac4897a..8320e413c3 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -425,7 +425,13 @@ func (s *blockDB) PruneBefore(blockHeight types.GlobalBlockNumber) error { } // Round the watermark down to the start of a QC's range, to avoid pruning a QC before its blocks. - blockHeight, err := s.clampPruneBoundary(min(blockHeight, s.statusLocked().Or(types.DBStatus{}).Floor())) + blockHeight, err := s.clampPruneBoundary(min(blockHeight, s.statusLocked().Or(types.DBStatus{ + First: 0, + NextBlock: 0, + NextQC: 0, + NextAppQC: 0, + NextAppProposal: 0, + }).Floor())) if err != nil { return err } @@ -489,7 +495,13 @@ func (s *blockDB) Flush() error { func (s *blockDB) Status() types.DBStatus { s.mu.Lock() defer s.mu.Unlock() - return s.statusLocked().Or(types.DBStatus{}) + return s.statusLocked().Or(types.DBStatus{ + First: 0, + NextBlock: 0, + NextQC: 0, + NextAppQC: 0, + NextAppProposal: 0, + }) } func (s *blockDB) statusLocked() utils.Option[types.DBStatus] { @@ -497,7 +509,12 @@ func (s *blockDB) statusLocked() utils.Option[types.DBStatus] { if !ok { return utils.None[types.DBStatus]() } + first := max(s.oldestQCStart, types.GlobalBlockNumber(s.watermark.Load())) + if s.hasBlocks { + first = max(first, s.firstBlockNumber) + } status := types.DBStatus{ + First: first, NextBlock: s.oldestQCStart, NextQC: qc.QC().GlobalRange().Next, NextAppQC: s.oldestQCStart, @@ -515,14 +532,15 @@ func (s *blockDB) statusLocked() utils.Option[types.DBStatus] { return utils.Some(status) } - // ReadRecent() reads the latest AppQC/AppProposal recovery suffix. // WARNING: ReadRecent() will return an error if watermark is moved during iteration. func (s *blockDB) ReadRecent() (types.RecentData, error) { s.mu.Lock() - status,ok := s.statusLocked().Get() + status, ok := s.statusLocked().Get() s.mu.Unlock() - if !ok { return types.RecentData{},nil } + if !ok { + return types.RecentData{}, nil + } targetFloor := status.Floor() // Collect data >= targetFloor. diff --git a/sei-db/ledger_db/block/memblock/mem_block_db.go b/sei-db/ledger_db/block/memblock/mem_block_db.go index 7fb29ff8e2..8ba7ffbd65 100644 --- a/sei-db/ledger_db/block/memblock/mem_block_db.go +++ b/sei-db/ledger_db/block/memblock/mem_block_db.go @@ -243,7 +243,13 @@ func (s *blockDB) PruneBefore(n types.GlobalBlockNumber) error { // future block whose coverage check still passes. Mirrors littblock. return nil } - n = min(n, s.statusLocked().Or(types.DBStatus{}).Floor()) + n = min(n, s.statusLocked().Or(types.DBStatus{ + First: 0, + NextBlock: 0, + NextQC: 0, + NextAppQC: 0, + NextAppProposal: 0, + }).Floor()) // Round the watermark down to the covering QC's First. A QC's cohort of // blocks changes readability atomically, so the watermark must never fall // strictly inside a QC's range (see littblock): otherwise a read would @@ -285,7 +291,13 @@ func (s *blockDB) Flush() error { return nil } func (s *blockDB) Status() types.DBStatus { s.mu.RLock() defer s.mu.RUnlock() - return s.statusLocked().Or(types.DBStatus{}) + return s.statusLocked().Or(types.DBStatus{ + First: 0, + NextBlock: 0, + NextQC: 0, + NextAppQC: 0, + NextAppProposal: 0, + }) } func (s *blockDB) statusLocked() utils.Option[types.DBStatus] { @@ -294,7 +306,12 @@ func (s *blockDB) statusLocked() utils.Option[types.DBStatus] { return utils.None[types.DBStatus]() } oldestQCStart := entries[0].lower + first := max(oldestQCStart, s.watermark) + if blockNumbers := s.sortedBlockNumbersLocked(); len(blockNumbers) > 0 { + first = max(first, blockNumbers[0]) + } status := types.DBStatus{ + First: first, NextBlock: oldestQCStart, NextQC: s.lastQCNext, NextAppQC: oldestQCStart, @@ -317,7 +334,13 @@ func (s *blockDB) ReadRecent() (types.RecentData, error) { defer s.mu.RUnlock() status := s.statusLocked() - floor := status.Or(types.DBStatus{}).Floor() + floor := status.Or(types.DBStatus{ + First: 0, + NextBlock: 0, + NextQC: 0, + NextAppQC: 0, + NextAppProposal: 0, + }).Floor() recent := types.RecentData{Status: status} var targetIndex types.RoadIndex bounded := s.hasAppProposal && s.hasAppQC && s.hasBlocks diff --git a/sei-tendermint/autobahn/types/block_db.go b/sei-tendermint/autobahn/types/block_db.go index 167863b622..62e18ff6da 100644 --- a/sei-tendermint/autobahn/types/block_db.go +++ b/sei-tendermint/autobahn/types/block_db.go @@ -288,6 +288,10 @@ type BlockDB interface { // (NextBlock/NextQC are never zero after a successful write: the first // written block number N yields NextBlock = N+1 ≥ 1). type DBStatus struct { + // First is the recovery floor used by Floor. It is the oldest readable block + // when blocks are present, otherwise the oldest retained CommitQC start. + // Zero if no QC has been written. + First GlobalBlockNumber // NextBlock is one past the highest GlobalBlockNumber accepted by WriteBlock // (the next block number that may be written). Zero if no block has been written. NextBlock GlobalBlockNumber @@ -309,10 +313,10 @@ type DBStatus struct { // recovery floor yet, so Floor returns zero. func (s DBStatus) Floor() GlobalBlockNumber { f := min(s.NextQC, s.NextBlock, s.NextAppProposal, s.NextAppQC) - if f > 0 { - f -= 1 + if f <= s.First { + return s.First } - return f + return f - 1 } // RecentBlock is one block returned by BlockDB.ReadRecent. diff --git a/sei-tendermint/autobahn/types/types_test.go b/sei-tendermint/autobahn/types/types_test.go index faf2bdb23d..1866ca1c63 100644 --- a/sei-tendermint/autobahn/types/types_test.go +++ b/sei-tendermint/autobahn/types/types_test.go @@ -105,6 +105,7 @@ func TestDBStatusFloor(t *testing.T) { { name: "empty", in: DBStatus{ + First: 0, NextBlock: 0, NextQC: 0, NextAppProposal: 0, @@ -115,16 +116,18 @@ func TestDBStatusFloor(t *testing.T) { { name: "missing app proposal", in: DBStatus{ + First: 5, NextBlock: 10, NextQC: 10, - NextAppProposal: 0, + NextAppProposal: 5, NextAppQC: 8, }, - want: 0, + want: 5, }, { name: "minimum durable tip minus one", in: DBStatus{ + First: 5, NextBlock: 12, NextQC: 12, NextAppProposal: 10, @@ -132,6 +135,17 @@ func TestDBStatusFloor(t *testing.T) { }, want: 7, }, + { + name: "first lower bound", + in: DBStatus{ + First: 10, + NextBlock: 12, + NextQC: 15, + NextAppProposal: 5, + NextAppQC: 10, + }, + want: 10, + }, } { t.Run(tc.name, func(t *testing.T) { if got := tc.in.Floor(); got != tc.want { diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index 8e6c6d8864..325ccebc88 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -220,6 +220,7 @@ func loadFromBlockDB(cfg *Config, blockDB types.BlockDB) (*inner, error) { } firstBlock := cfg.Registry.FirstBlock() status := recent.Status.Or(types.DBStatus{ + First: firstBlock, NextQC: firstBlock, NextAppProposal: firstBlock, NextAppQC: firstBlock, diff --git a/sei-tendermint/internal/autobahn/data/state_recovery_test.go b/sei-tendermint/internal/autobahn/data/state_recovery_test.go index 56c135e133..01de2c61b2 100644 --- a/sei-tendermint/internal/autobahn/data/state_recovery_test.go +++ b/sei-tendermint/internal/autobahn/data/state_recovery_test.go @@ -485,9 +485,9 @@ func TestRecoveryQCsNoBlocks(t *testing.T) { } // TestRunPersistSeedsFromRecoveryFloor verifies that runPersist does not walk -// [genesis, recoveryFloor) when Status lacks NextBlock (QC-only store -// whose first QC starts past FirstBlock). Seeding persisted DBStatus from the -// recovery floor avoids collecting nil block pointers. +// [genesis, recoveryFloor) for a QC-only store whose first QC starts past +// FirstBlock. Seeding persisted DBStatus from the recovery floor avoids +// collecting nil block pointers. func TestRunPersistSeedsFromRecoveryFloor(t *testing.T) { ctx := t.Context() rng := utils.TestRng() @@ -507,7 +507,8 @@ func TestRunPersistSeedsFromRecoveryFloor(t *testing.T) { db2 := newTestBlockDB(t, dir) tips := db2.Status() - require.Zero(t, tips.NextBlock) + require.Equal(t, gr2.First, tips.First) + require.Equal(t, tips.First, tips.NextBlock) require.NotZero(t, tips.NextQC) state := newTestState(t, &Config{Registry: registry}, db2) From 063e6067a79746b8934e23497d8148347303b30c Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 7 Aug 2026 15:21:11 +0200 Subject: [PATCH 28/61] test fixes --- .../internal/autobahn/avail/state_test.go | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/sei-tendermint/internal/autobahn/avail/state_test.go b/sei-tendermint/internal/autobahn/avail/state_test.go index 01e2ce3ab8..c5c051df66 100644 --- a/sei-tendermint/internal/autobahn/avail/state_test.go +++ b/sei-tendermint/internal/autobahn/avail/state_test.go @@ -124,7 +124,12 @@ func testState(t *testing.T, rng utils.Rng, stateDir utils.Option[string]) { } t.Logf("Push app votes.") - appProposal := types.NewAppProposal(qc.Proposal(), types.GenAppHash(rng)) + appHash := types.GenAppHash(rng) + appProposal := types.NewAppProposal(qc.Proposal(), appHash) + appGR := appProposal.GlobalRange() + if err := ds.PushAppHash(ctx, appGR.Next-1, appHash); err != nil { + return fmt.Errorf("ds.PushAppHash(): %w", err) + } for _, vote := range makeAppVotes(keys, appProposal) { if err := state.PushAppVote(ctx, vote); err != nil { return fmt.Errorf("state.PushAppVote(): %w", err) @@ -132,8 +137,8 @@ func testState(t *testing.T, rng utils.Rng, stateDir utils.Option[string]) { } t.Logf("Previous one should be eventually evicted") - for inner,ctrl := range state.inner.Lock() { - if err:=ctrl.WaitUntil(ctx, func() bool { return inner.roads.first == appProposal.RoadIndex() }); err!=nil { + for inner, ctrl := range state.inner.Lock() { + if err := ctrl.WaitUntil(ctx, func() bool { return inner.roads.first == appProposal.RoadIndex() }); err != nil { return err } } @@ -182,10 +187,9 @@ func testState(t *testing.T, rng utils.Rng, stateDir utils.Option[string]) { // from the same directory. This verifies that what the runtime persist // goroutine writes can be correctly loaded back by loadPersistedState/newInner. // -// After iteration 0's AppQC prunes old data, iteration 1 writes new blocks -// and commitQCs at higher indices. If WAL truncation hasn't cleaned up the -// stale entries by shutdown, restart exercises the gap-filtering path in -// loadPersistedState (stale entries below the prune anchor are discarded). +// The restarted state uses a fresh in-memory data.State, so this covers the +// availability persisters themselves: CommitQCs and local blocks are loaded back +// and lane next-block cursors resume where they left off. func TestStateRestartFromPersisted(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 3) @@ -278,8 +282,6 @@ func TestStateRestartFromPersisted(t *testing.T) { state2, err := NewState(keys[0], ds2, utils.Some(dir)) require.NoError(t, err) - require.GreaterOrEqual(t, state2.First(), wantAppQCIdx) - _, ok := state2.LastCommitQC().Load().Get() require.True(t, ok, "LastCommitQC should be set after restart") From c9ac367a1e695a0785961f53c32956e298603214 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 7 Aug 2026 15:26:53 +0200 Subject: [PATCH 29/61] codex test fixes --- .../autobahn/consensus/persist/blocks_test.go | 57 ++--- .../autobahn/consensus/persist/commitqcs.go | 2 +- .../consensus/persist/commitqcs_test.go | 219 ++++-------------- 3 files changed, 68 insertions(+), 210 deletions(-) diff --git a/sei-tendermint/internal/autobahn/consensus/persist/blocks_test.go b/sei-tendermint/internal/autobahn/consensus/persist/blocks_test.go index e799dcf363..0711d89c5f 100644 --- a/sei-tendermint/internal/autobahn/consensus/persist/blocks_test.go +++ b/sei-tendermint/internal/autobahn/consensus/persist/blocks_test.go @@ -21,37 +21,14 @@ var noBlockCB = utils.None[func(*types.Signed[*types.LaneProposal])]() func testPersistBlock(t *testing.T, bp *BlockPersister, p *types.Signed[*types.LaneProposal]) { t.Helper() - require.NoError(t, bp.MaybePruneAndPersistLane( + require.NoError(t, bp.Persist( p.Msg().Block().Header().Lane(), - utils.None[*types.CommitQC](), + 0, []*types.Signed[*types.LaneProposal]{p}, noBlockCB, )) } -// testDeleteBefore is a test helper that truncates lane WALs using a plain -// map, avoiding the need to construct a full CommitQC. -func testDeleteBefore(bp *BlockPersister, laneFirsts map[types.LaneID]types.BlockNumber) error { - for lanes := range bp.lanes.RLock() { - return scope.Parallel(func(ps scope.ParallelScope) error { - for lane, first := range laneFirsts { - lw, ok := lanes[lane] - if !ok { - continue - } - ps.Spawn(func() error { - for s := range lw.state.Lock() { - return s.truncateForAnchor(lane, first) - } - panic("unreachable") - }) - } - return nil - }) - } - panic("unreachable") -} - func TestNewBlockPersisterEmptyDir(t *testing.T) { dir := t.TempDir() bp, blocks, err := NewBlockPersister(utils.Some(dir)) @@ -130,7 +107,7 @@ func TestDeleteBeforeRemovesOldKeepsNew(t *testing.T) { testPersistBlock(t, bp, testSignedProposal(rng, key, i)) } - require.NoError(t, testDeleteBefore(bp, map[types.LaneID]types.BlockNumber{lane: 3})) + require.NoError(t, bp.Persist(lane, 3, nil, noBlockCB)) require.NoError(t, bp.close()) _, blocks, err := NewBlockPersister(utils.Some(dir)) @@ -159,7 +136,9 @@ func TestDeleteBeforeAndRestart(t *testing.T) { } // lane1: truncate old blocks, lane2: delete nothing (first=0), lane3: empty (no WAL). - require.NoError(t, testDeleteBefore(bp, map[types.LaneID]types.BlockNumber{lane1: 2, lane2: 0, lane3: 0})) + require.NoError(t, bp.Persist(lane1, 2, nil, noBlockCB)) + require.NoError(t, bp.Persist(lane2, 0, nil, noBlockCB)) + require.NoError(t, bp.Persist(lane3, 0, nil, noBlockCB)) require.NoError(t, bp.close()) // Restart — verify varied lane states load correctly. @@ -198,15 +177,15 @@ func TestNoOpBlockPersister(t *testing.T) { proposals[i] = testSignedProposal(rng, key, types.BlockNumber(i)) } - // Persist and prune with anchor + new proposals in no-op mode. + // Persist and prune with first + new proposals in no-op mode. // Verify afterEach is still invoked for every proposal. var called int cb := utils.Some(func(_ *types.Signed[*types.LaneProposal]) { called++ }) - require.NoError(t, bp.MaybePruneAndPersistLane(lane, utils.None[*types.CommitQC](), proposals[:3], cb)) + require.NoError(t, bp.Persist(lane, 0, proposals[:3], cb)) require.Equal(t, 3, called) called = 0 - require.NoError(t, bp.MaybePruneAndPersistLane(lane, utils.None[*types.CommitQC](), proposals[3:], cb)) + require.NoError(t, bp.Persist(lane, 0, proposals[3:], cb)) require.Equal(t, 2, called) require.NoError(t, bp.close()) @@ -225,7 +204,7 @@ func TestDeleteBeforeThenPersistMore(t *testing.T) { for i := range types.BlockNumber(5) { testPersistBlock(t, bp, testSignedProposal(rng, key, i)) } - require.NoError(t, testDeleteBefore(bp, map[types.LaneID]types.BlockNumber{lane: 3})) + require.NoError(t, bp.Persist(lane, 3, nil, noBlockCB)) testPersistBlock(t, bp, testSignedProposal(rng, key, 5)) require.NoError(t, bp.close()) @@ -251,7 +230,7 @@ func TestDeleteBeforePastAllBlocks(t *testing.T) { } // Anchor advanced past everything (nextBlockNum is 3, first=10). - require.NoError(t, testDeleteBefore(bp, map[types.LaneID]types.BlockNumber{lane: 10})) + require.NoError(t, bp.Persist(lane, 10, nil, noBlockCB)) // Lane WAL is now empty; new writes starting from 10 should work. testPersistBlock(t, bp, testSignedProposal(rng, key, 10)) @@ -280,11 +259,11 @@ func TestDeleteBeforePastAllRejectsStaleBlock(t *testing.T) { } // Anchor advanced past everything; nextBlockNum re-anchored to 10. - require.NoError(t, testDeleteBefore(bp, map[types.LaneID]types.BlockNumber{lane: 10})) + require.NoError(t, bp.Persist(lane, 10, nil, noBlockCB)) // Writing a stale block number (0) should be rejected. stale := testSignedProposal(rng, key, 0) - err = bp.MaybePruneAndPersistLane(lane, utils.None[*types.CommitQC](), []*types.Signed[*types.LaneProposal]{stale}, noBlockCB) + err = bp.Persist(lane, 10, []*types.Signed[*types.LaneProposal]{stale}, noBlockCB) require.Error(t, err) require.Contains(t, err.Error(), "out of sequence") @@ -307,12 +286,12 @@ func TestTruncateOnEmptyWALAdvancesCursor(t *testing.T) { } // First truncation empties the WAL (first=10 > nextBlockNum=3). - require.NoError(t, testDeleteBefore(bp, map[types.LaneID]types.BlockNumber{lane: 10})) + require.NoError(t, bp.Persist(lane, 10, nil, noBlockCB)) // Second truncation on the already-empty WAL (first=15). // Before the fix, nextBlockNum would stay at 10 and block 15 would // be rejected as out of sequence. - require.NoError(t, testDeleteBefore(bp, map[types.LaneID]types.BlockNumber{lane: 15})) + require.NoError(t, bp.Persist(lane, 15, nil, noBlockCB)) testPersistBlock(t, bp, testSignedProposal(rng, key, 15)) require.NoError(t, bp.close()) @@ -388,13 +367,13 @@ func TestPersistBlockOutOfSequence(t *testing.T) { // Gap: skip block 1, try block 2. gap := testSignedProposal(rng, key, 2) - err = bp.MaybePruneAndPersistLane(lane, utils.None[*types.CommitQC](), []*types.Signed[*types.LaneProposal]{gap}, noBlockCB) + err = bp.Persist(lane, 0, []*types.Signed[*types.LaneProposal]{gap}, noBlockCB) require.Error(t, err) require.Contains(t, err.Error(), "out of sequence") // Duplicate: try block 0 again. dup := testSignedProposal(rng, key, 0) - err = bp.MaybePruneAndPersistLane(lane, utils.None[*types.CommitQC](), []*types.Signed[*types.LaneProposal]{dup}, noBlockCB) + err = bp.Persist(lane, 0, []*types.Signed[*types.LaneProposal]{dup}, noBlockCB) require.Error(t, err) require.Contains(t, err.Error(), "out of sequence") @@ -474,7 +453,7 @@ func TestPersistBlockConcurrentDistinctLanes(t *testing.T) { for i := range numLanes { lane := keys[i].Public() ps.Spawn(func() error { - return bp.MaybePruneAndPersistLane(lane, utils.None[*types.CommitQC](), proposals[i], noBlockCB) + return bp.Persist(lane, 0, proposals[i], noBlockCB) }) } return nil diff --git a/sei-tendermint/internal/autobahn/consensus/persist/commitqcs.go b/sei-tendermint/internal/autobahn/consensus/persist/commitqcs.go index 75423931fa..76701dabea 100644 --- a/sei-tendermint/internal/autobahn/consensus/persist/commitqcs.go +++ b/sei-tendermint/internal/autobahn/consensus/persist/commitqcs.go @@ -171,7 +171,7 @@ func loadAllCommitQCs(s *commitQCState) ([]*types.CommitQC, error) { loaded := make([]*types.CommitQC, 0, len(entries)) for i, qc := range entries { if i > 0 && qc.Index() != loaded[i-1].Index()+1 { - return nil, fmt.Errorf("gap in commitqcs: index %d follows %d", qc.Index(), loaded[i-1].Index) + return nil, fmt.Errorf("gap in commitqcs: index %d follows %d", qc.Index(), loaded[i-1].Index()) } loaded = append(loaded, qc) } diff --git a/sei-tendermint/internal/autobahn/consensus/persist/commitqcs_test.go b/sei-tendermint/internal/autobahn/consensus/persist/commitqcs_test.go index a2231430c7..b4ace783ea 100644 --- a/sei-tendermint/internal/autobahn/consensus/persist/commitqcs_test.go +++ b/sei-tendermint/internal/autobahn/consensus/persist/commitqcs_test.go @@ -12,18 +12,14 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/require" ) -var noQC = utils.None[*types.CommitQC]() -var noCommitQCCB = utils.None[func(*types.CommitQC)]() - func testCommitQC( committee *types.Committee, keys []types.SecretKey, prev utils.Option[*types.CommitQC], laneQCs map[types.LaneID]*types.LaneQC, - appQC utils.Option[*types.AppQC], ) *types.CommitQC { ep := types.NewEpoch(0, types.OpenRoadRange(), time.Time{}, committee, 0) - return types.BuildCommitQC(ep, keys, prev, laneQCs, appQC) + return types.BuildCommitQC(ep, keys, prev, laneQCs) } func makeSequentialCommitQCs( @@ -34,7 +30,7 @@ func makeSequentialCommitQCs( var qcs []*types.CommitQC prev := utils.None[*types.CommitQC]() for range count { - qc := testCommitQC(committee, keys, prev, nil, utils.None[*types.AppQC]()) + qc := testCommitQC(committee, keys, prev, nil) qcs = append(qcs, qc) prev = utils.Some(qc) } @@ -44,30 +40,12 @@ func makeSequentialCommitQCs( // testPersistCommitQC persists a single CommitQC via the public API. func testPersistCommitQC(t *testing.T, cp *CommitQCPersister, qc *types.CommitQC) { t.Helper() - require.NoError(t, cp.MaybePruneAndPersist( - utils.None[*types.CommitQC](), - []*types.CommitQC{qc}, - noCommitQCCB, - )) + require.NoError(t, cp.Persist(0, []*types.CommitQC{qc})) } -// testDeleteCommitQCsBefore truncates the WAL below the anchor's index and -// re-persists the anchor for crash recovery. -func testDeleteCommitQCsBefore(t *testing.T, cp *CommitQCPersister, anchor *types.CommitQC) { +func testDeleteCommitQCsBefore(t *testing.T, cp *CommitQCPersister, idx types.RoadIndex) { t.Helper() - for s := range cp.state.Lock() { - require.NoError(t, s.deleteBefore(anchor)) - return - } -} - -// clearCommitQCWAL removes all WAL files to simulate a crash between -// WAL truncation and the subsequent anchor write. -func clearCommitQCWAL(t *testing.T, dir string) { - t.Helper() - walDir := filepath.Join(dir, commitqcsDir) - require.NoError(t, os.RemoveAll(walDir)) - require.NoError(t, os.MkdirAll(walDir, 0700)) + require.NoError(t, cp.Persist(idx, nil)) } func TestNewCommitQCPersisterEmptyDir(t *testing.T) { @@ -76,7 +54,7 @@ func TestNewCommitQCPersisterEmptyDir(t *testing.T) { require.NoError(t, err) require.NotNil(t, cp) require.Equal(t, 0, len(loaded)) - require.Equal(t, types.RoadIndex(0), cp.LoadNext()) + require.Equal(t, types.RoadIndex(0), cp.Next()) fi, err := os.Stat(filepath.Join(dir, commitqcsDir)) require.NoError(t, err) @@ -98,7 +76,7 @@ func TestPersistCommitQCAndLoad(t *testing.T) { for _, qc := range qcs { testPersistCommitQC(t, cp, qc) } - require.Equal(t, types.RoadIndex(3), cp.LoadNext()) + require.Equal(t, types.RoadIndex(3), cp.Next()) require.NoError(t, cp.Close()) cp2, loaded, err := NewCommitQCPersister(utils.Some(dir)) @@ -106,10 +84,10 @@ func TestPersistCommitQCAndLoad(t *testing.T) { require.NotNil(t, cp2) require.Equal(t, 3, len(loaded)) for i, lqc := range loaded { - require.Equal(t, types.RoadIndex(i), lqc.Index) - require.NoError(t, utils.TestDiff(qcs[i], lqc.QC)) + require.Equal(t, types.RoadIndex(i), lqc.Index()) + require.NoError(t, utils.TestDiff(qcs[i], lqc)) } - require.Equal(t, types.RoadIndex(3), cp2.LoadNext()) + require.Equal(t, types.RoadIndex(3), cp2.Next()) require.NoError(t, cp2.Close()) } @@ -126,14 +104,14 @@ func TestCommitQCDeleteBeforeRemovesOldKeepsNew(t *testing.T) { testPersistCommitQC(t, cp, qc) } - testDeleteCommitQCsBefore(t, cp, qcs[3]) + testDeleteCommitQCsBefore(t, cp, qcs[3].Index()) require.NoError(t, cp.Close()) _, loaded, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) require.Equal(t, 2, len(loaded), "should have indices 3 and 4") - require.Equal(t, types.RoadIndex(3), loaded[0].Index) - require.Equal(t, types.RoadIndex(4), loaded[1].Index) + require.Equal(t, types.RoadIndex(3), loaded[0].Index()) + require.Equal(t, types.RoadIndex(4), loaded[1].Index()) } func TestCommitQCDeleteBeforeZero(t *testing.T) { @@ -150,9 +128,8 @@ func TestCommitQCDeleteBeforeZero(t *testing.T) { testPersistCommitQC(t, cp, qc) } - // deleteBefore with anchor at index 0 should persist the anchor - // (which is a duplicate here) and leave everything intact. - testDeleteCommitQCsBefore(t, cp, qcs[0]) + // deleteBefore with index 0 should leave everything intact. + testDeleteCommitQCsBefore(t, cp, qcs[0].Index()) require.NoError(t, cp.Close()) cp2, loaded, err := NewCommitQCPersister(utils.Some(dir)) @@ -160,7 +137,7 @@ func TestCommitQCDeleteBeforeZero(t *testing.T) { require.Equal(t, 2, len(loaded)) testPersistCommitQC(t, cp2, qcs[2]) - require.Equal(t, types.RoadIndex(3), cp2.LoadNext()) + require.Equal(t, types.RoadIndex(3), cp2.Next()) require.NoError(t, cp2.Close()) } @@ -178,7 +155,7 @@ func TestCommitQCPersistDuplicateIsNoOp(t *testing.T) { testPersistCommitQC(t, cp, qcs[1]) // Persisting qcs[0] again is a no-op (idx < next). testPersistCommitQC(t, cp, qcs[0]) - require.Equal(t, types.RoadIndex(2), cp.LoadNext()) + require.Equal(t, types.RoadIndex(2), cp.Next()) require.NoError(t, cp.Close()) } @@ -195,7 +172,7 @@ func TestCommitQCPersistGapRejected(t *testing.T) { testPersistCommitQC(t, cp, qcs[0]) testPersistCommitQC(t, cp, qcs[1]) // Skip qcs[2], try to persist qcs[3] — should fail because idx(3) != next(2). - err = cp.MaybePruneAndPersist(noQC, []*types.CommitQC{qcs[3]}, noCommitQCCB) + err = cp.Persist(0, []*types.CommitQC{qcs[3]}) require.Error(t, err) require.Contains(t, err.Error(), "out of sequence") require.NoError(t, cp.Close()) @@ -231,31 +208,18 @@ func TestNoOpCommitQCPersister(t *testing.T) { committee := registry.LatestEpoch().Committee() qcs := makeSequentialCommitQCs(committee, keys, 11) - // Fresh no-op persister: prune with anchor at index 0 (idx==0, - // s.next==0). Should persist the anchor and advance to 1. + // Fresh no-op persister: persist sequential QCs and track Next. cp, loaded, err := NewCommitQCPersister(utils.None[string]()) require.NoError(t, err) require.NotNil(t, cp) require.Equal(t, 0, len(loaded)) - require.NoError(t, cp.MaybePruneAndPersist( - utils.Some(qcs[0]), - qcs[1:5], - noCommitQCCB, - )) - require.Equal(t, types.RoadIndex(5), cp.LoadNext()) - - // Prune with a future anchor (index 8 > s.next=5). deleteBefore - // advances s.next to 8, then persistCommitQC persists the anchor - // and advances s.next to 9. The remaining QCs (9,10) follow. - // Before the fix, deleteBefore in no-op mode wouldn't advance - // s.next, so re-persisting the anchor QC would fail with "out of - // sequence". - require.NoError(t, cp.MaybePruneAndPersist( - utils.Some(qcs[8]), - qcs[9:], - noCommitQCCB, - )) - require.Equal(t, types.RoadIndex(11), cp.LoadNext()) + require.NoError(t, cp.Persist(0, qcs[:5])) + require.Equal(t, types.RoadIndex(5), cp.Next()) + + // Prune with a future index. deleteBefore advances persisted.Next, + // so the remaining QCs follow the new bound. + require.NoError(t, cp.Persist(8, qcs[8:])) + require.Equal(t, types.RoadIndex(11), cp.Next()) require.NoError(t, cp.Close()) } @@ -271,12 +235,13 @@ func TestCommitQCDeleteBeforePastAll(t *testing.T) { for i := range 3 { testPersistCommitQC(t, cp, qcs[i]) } - // next is 3; deleteBefore with anchor at 10 truncates the WAL, - // advances the cursor to 10, and re-persists the anchor (next → 11). - testDeleteCommitQCsBefore(t, cp, qcs[10]) - require.Equal(t, types.RoadIndex(11), cp.LoadNext()) + // next is 3; deleteBefore at 10 truncates the WAL and advances the + // cursor to 10. + testDeleteCommitQCsBefore(t, cp, qcs[10].Index()) + require.Equal(t, types.RoadIndex(10), cp.Next()) - // New write starting from 11 should work. + // New writes starting from 10 should work. + testPersistCommitQC(t, cp, qcs[10]) testPersistCommitQC(t, cp, qcs[11]) require.NoError(t, cp.Close()) @@ -284,94 +249,8 @@ func TestCommitQCDeleteBeforePastAll(t *testing.T) { _, loaded, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) require.Equal(t, 2, len(loaded)) - require.Equal(t, types.RoadIndex(10), loaded[0].Index) - require.Equal(t, types.RoadIndex(11), loaded[1].Index) -} - -// TestCommitQCDeleteBeforePastAllCrashRecovery simulates a crash between WAL -// TruncateAll and the anchor write: on restart the WAL is empty and the anchor -// must re-establish the cursor so subsequent persists succeed. -func TestCommitQCDeleteBeforePastAllCrashRecovery(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - committee := registry.LatestEpoch().Committee() - dir := t.TempDir() - - qcs := makeSequentialCommitQCs(committee, keys, 12) - cp, _, err := NewCommitQCPersister(utils.Some(dir)) - require.NoError(t, err) - for i := range 3 { - testPersistCommitQC(t, cp, qcs[i]) - } - require.NoError(t, cp.Close()) - - // Simulate crash: clear the WAL as if TruncateAll completed but the - // subsequent anchor write never happened. - clearCommitQCWAL(t, dir) - - // Restart: WAL is empty, next will be 0. - cp2, loaded, err := NewCommitQCPersister(utils.Some(dir)) - require.NoError(t, err) - require.Empty(t, loaded) - require.Equal(t, types.RoadIndex(0), cp2.LoadNext()) - - // MaybePruneAndPersist with anchor at 10 re-establishes the cursor - // and appends new QCs. - require.NoError(t, cp2.MaybePruneAndPersist( - utils.Some(qcs[10]), - []*types.CommitQC{qcs[11]}, - noCommitQCCB, - )) - require.Equal(t, types.RoadIndex(12), cp2.LoadNext()) - require.NoError(t, cp2.Close()) - - _, loaded, err = NewCommitQCPersister(utils.Some(dir)) - require.NoError(t, err) - require.Equal(t, 2, len(loaded)) - require.Equal(t, types.RoadIndex(10), loaded[0].Index) - require.Equal(t, types.RoadIndex(11), loaded[1].Index) -} - -// TestCommitQCDeleteBeforeWithAnchorRecovers verifies that after a crash -// leaves the WAL empty, passing an anchor QC re-persists it and -// re-establishes the cursor for subsequent writes. -func TestCommitQCDeleteBeforeWithAnchorRecovers(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 4) - committee := registry.LatestEpoch().Committee() - dir := t.TempDir() - - qcs := makeSequentialCommitQCs(committee, keys, 5) - cp, _, err := NewCommitQCPersister(utils.Some(dir)) - require.NoError(t, err) - for _, qc := range qcs { - testPersistCommitQC(t, cp, qc) - } - require.NoError(t, cp.Close()) - - // Simulate crash: clear WAL. - clearCommitQCWAL(t, dir) - - // Restart: WAL is empty. Pass the anchor QC (index 4) through deleteBefore. - cp2, loaded, err := NewCommitQCPersister(utils.Some(dir)) - require.NoError(t, err) - require.Empty(t, loaded) - - // deleteBefore advances cursor to 4, then re-persists qcs[4] via anchor. - testDeleteCommitQCsBefore(t, cp2, qcs[4]) - require.Equal(t, types.RoadIndex(5), cp2.LoadNext()) - - // Continue writing from 5. - testPersistCommitQC(t, cp2, qcs[4]) // duplicate — no-op - require.Equal(t, types.RoadIndex(5), cp2.LoadNext()) - require.NoError(t, cp2.Close()) - - // Reopen — anchor QC should be on disk. - _, loaded, err = NewCommitQCPersister(utils.Some(dir)) - require.NoError(t, err) - require.Equal(t, 1, len(loaded)) - require.Equal(t, types.RoadIndex(4), loaded[0].Index) - require.NoError(t, utils.TestDiff(qcs[4], loaded[0].QC)) + require.Equal(t, types.RoadIndex(10), loaded[0].Index()) + require.Equal(t, types.RoadIndex(11), loaded[1].Index()) } func TestCommitQCDeleteBeforeThenPersistMore(t *testing.T) { @@ -388,16 +267,16 @@ func TestCommitQCDeleteBeforeThenPersistMore(t *testing.T) { for i := range 5 { testPersistCommitQC(t, cp, qcs[i]) } - testDeleteCommitQCsBefore(t, cp, qcs[3]) + testDeleteCommitQCsBefore(t, cp, qcs[3].Index()) testPersistCommitQC(t, cp, qcs[5]) require.NoError(t, cp.Close()) _, loaded, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) require.Equal(t, 3, len(loaded), "should have indices 3, 4, 5") - require.Equal(t, types.RoadIndex(3), loaded[0].Index) - require.Equal(t, types.RoadIndex(4), loaded[1].Index) - require.Equal(t, types.RoadIndex(5), loaded[2].Index) + require.Equal(t, types.RoadIndex(3), loaded[0].Index()) + require.Equal(t, types.RoadIndex(4), loaded[1].Index()) + require.Equal(t, types.RoadIndex(5), loaded[2].Index()) } func TestCommitQCDeleteBeforeAlreadyPruned(t *testing.T) { @@ -414,19 +293,19 @@ func TestCommitQCDeleteBeforeAlreadyPruned(t *testing.T) { } // Prune up to index 3. - testDeleteCommitQCsBefore(t, cp, qcs[3]) + testDeleteCommitQCsBefore(t, cp, qcs[3].Index()) // Pruning at or below the current first should be a no-op. - testDeleteCommitQCsBefore(t, cp, qcs[2]) - testDeleteCommitQCsBefore(t, cp, qcs[3]) + testDeleteCommitQCsBefore(t, cp, qcs[2].Index()) + testDeleteCommitQCsBefore(t, cp, qcs[3].Index()) require.NoError(t, cp.Close()) // Verify nothing extra was pruned. _, loaded, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) require.Equal(t, 2, len(loaded), "should still have indices 3 and 4") - require.Equal(t, types.RoadIndex(3), loaded[0].Index) - require.Equal(t, types.RoadIndex(4), loaded[1].Index) + require.Equal(t, types.RoadIndex(3), loaded[0].Index()) + require.Equal(t, types.RoadIndex(4), loaded[1].Index()) } func TestCommitQCProgressiveDeleteBefore(t *testing.T) { @@ -443,18 +322,18 @@ func TestCommitQCProgressiveDeleteBefore(t *testing.T) { } // First prune: remove 0, 1. - testDeleteCommitQCsBefore(t, cp, qcs[2]) - require.Equal(t, types.RoadIndex(8), cp.LoadNext()) + testDeleteCommitQCsBefore(t, cp, qcs[2].Index()) + require.Equal(t, types.RoadIndex(8), cp.Next()) // Second prune: remove 2, 3, 4. - testDeleteCommitQCsBefore(t, cp, qcs[5]) + testDeleteCommitQCsBefore(t, cp, qcs[5].Index()) require.NoError(t, cp.Close()) // Verify indices 5, 6, 7 survive. _, loaded, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) require.Equal(t, 3, len(loaded)) - require.Equal(t, types.RoadIndex(5), loaded[0].Index) - require.Equal(t, types.RoadIndex(6), loaded[1].Index) - require.Equal(t, types.RoadIndex(7), loaded[2].Index) + require.Equal(t, types.RoadIndex(5), loaded[0].Index()) + require.Equal(t, types.RoadIndex(6), loaded[1].Index()) + require.Equal(t, types.RoadIndex(7), loaded[2].Index()) } From cc64f08effc390301c0e68597d5919755b1051a5 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 7 Aug 2026 15:45:59 +0200 Subject: [PATCH 30/61] removed persistedBlockStart --- .../internal/autobahn/avail/inner.go | 22 ++++++------------ .../internal/autobahn/avail/state.go | 15 ++++++------ .../autobahn/consensus/persist/commitqcs.go | 23 ++++++++++--------- 3 files changed, 27 insertions(+), 33 deletions(-) diff --git a/sei-tendermint/internal/autobahn/avail/inner.go b/sei-tendermint/internal/autobahn/avail/inner.go index baad7e7e01..a707b343b8 100644 --- a/sei-tendermint/internal/autobahn/avail/inner.go +++ b/sei-tendermint/internal/autobahn/avail/inner.go @@ -10,7 +10,7 @@ import ( ) // TODO: when dynamic committee changes are supported, newly joined members -// must be added to blocks, votes, nextBlockToPersist, and persistedBlockStart. +// must be added to blocks, votes, and nextBlockToPersist. // Currently all four are initialized once in newInner from c.Lanes().All(). // BlockPersister creates lane WALs lazily inside MaybePruneAndPersistLane, but the new // member must also appear in inner.blocks before the next persist cycle. @@ -36,13 +36,6 @@ type inner struct { // ideal. Only RecvBatch needs to be notified of cursor changes; // collectPersistBatch is in the same goroutine and reads it directly. nextBlockToPersist map[types.LaneID]types.BlockNumber - - // persistedBlockStart is the per-lane block number derived from the last - // durably persisted prune anchor. Block admission (PushBlock, ProduceBlock, - // WaitForCapacity, PushVote) uses persistedBlockStart + BlocksPerLane as - // the capacity limit, ensuring we never admit more blocks than can be - // recovered after a crash. - persistedBlockStart map[types.LaneID]types.BlockNumber } // loadedState holds data loaded from disk on restart. @@ -59,13 +52,12 @@ type loadedState struct { func newInner(ds *data.State, loaded *loadedState) (*inner, error) { epoch := ds.Registry().LatestEpoch() i := &inner{ - persistedCommitQC: utils.NewAtomicSend(utils.None[*types.CommitQC]()), - roads: newQueue[types.RoadIndex, *road](), - epoch: epoch, - blocks: map[types.LaneID]*queue[types.BlockNumber, *types.Signed[*types.LaneProposal]]{}, - votes: map[types.LaneID]*queue[types.BlockNumber, blockVotes]{}, - nextBlockToPersist: map[types.LaneID]types.BlockNumber{}, - persistedBlockStart: map[types.LaneID]types.BlockNumber{}, + persistedCommitQC: utils.NewAtomicSend(utils.None[*types.CommitQC]()), + roads: newQueue[types.RoadIndex, *road](), + epoch: epoch, + blocks: map[types.LaneID]*queue[types.BlockNumber, *types.Signed[*types.LaneProposal]]{}, + votes: map[types.LaneID]*queue[types.BlockNumber, blockVotes]{}, + nextBlockToPersist: map[types.LaneID]types.BlockNumber{}, } for lane := range epoch.Committee().Lanes().All() { i.blocks[lane] = newQueue[types.BlockNumber, *types.Signed[*types.LaneProposal]]() diff --git a/sei-tendermint/internal/autobahn/avail/state.go b/sei-tendermint/internal/autobahn/avail/state.go index 77c60688b3..23f84c25c7 100644 --- a/sei-tendermint/internal/autobahn/avail/state.go +++ b/sei-tendermint/internal/autobahn/avail/state.go @@ -254,7 +254,7 @@ func (s *State) PushBlock(ctx context.Context, p *types.Signed[*types.LanePropos return ErrBadLane } if err := ctrl.WaitUntil(ctx, func() bool { - return h.BlockNumber() <= min(q.next, inner.persistedBlockStart[h.Lane()]+BlocksPerLane-1) + return h.BlockNumber() <= min(q.next, q.first+BlocksPerLane-1) }); err != nil { return err } @@ -293,9 +293,9 @@ func (s *State) PushBlock(ctx context.Context, p *types.Signed[*types.LanePropos // It does NOT wait for the previous votes. func (s *State) PushVote(ctx context.Context, vote *types.Signed[*types.LaneVote]) error { var epoch *types.Epoch - for inner,ctrl := range s.inner.Lock() { + for inner, ctrl := range s.inner.Lock() { // TODO(gprusak): we should wait only if LaneID is from the future. - if err:=ctrl.WaitUntil(ctx, func() bool { return inner.epoch.Committee().HasLane(vote.Key()) }); err!=nil { + if err := ctrl.WaitUntil(ctx, func() bool { return inner.epoch.Committee().HasLane(vote.Key()) }); err != nil { return err } epoch = inner.epoch @@ -303,7 +303,7 @@ func (s *State) PushVote(ctx context.Context, vote *types.Signed[*types.LaneVote if err := vote.Msg().Verify(epoch.Committee()); err != nil { return fmt.Errorf("vote.Verify(): %w", err) } - if err := vote.VerifySig(epoch.Committee()); err!=nil { + if err := vote.VerifySig(epoch.Committee()); err != nil { return fmt.Errorf("vote.VerifySig(): %w", err) } h := vote.Msg().Header() @@ -313,7 +313,7 @@ func (s *State) PushVote(ctx context.Context, vote *types.Signed[*types.LaneVote return ErrBadLane } if err := ctrl.WaitUntil(ctx, func() bool { - return h.BlockNumber() < inner.persistedBlockStart[h.Lane()]+BlocksPerLane + return h.BlockNumber() < q.first+BlocksPerLane }); err != nil { return err } @@ -386,8 +386,9 @@ func (s *State) fullCommitQC(ctx context.Context, n types.RoadIndex) (*types.Epo func (s *State) WaitForLocalCapacity(ctx context.Context, toProduce types.BlockNumber) error { lane := s.key.Public() for inner, ctrl := range s.inner.Lock() { + q := inner.blocks[lane] if err := ctrl.WaitUntil(ctx, func() bool { - return toProduce < inner.persistedBlockStart[lane]+BlocksPerLane + return toProduce < q.first+BlocksPerLane }); err != nil { return err } @@ -439,7 +440,7 @@ func (s *State) produceLocalBlock(n types.BlockNumber, key types.SecretKey, payl if !ok { return nil, ErrBadLane } - if n >= inner.persistedBlockStart[lane]+BlocksPerLane { + if n >= q.first+BlocksPerLane { return nil, fmt.Errorf("lane full") } if q.next != n { diff --git a/sei-tendermint/internal/autobahn/consensus/persist/commitqcs.go b/sei-tendermint/internal/autobahn/consensus/persist/commitqcs.go index 76701dabea..4f9c6c2fef 100644 --- a/sei-tendermint/internal/autobahn/consensus/persist/commitqcs.go +++ b/sei-tendermint/internal/autobahn/consensus/persist/commitqcs.go @@ -37,9 +37,11 @@ func (s *commitQCState) persist(qc *types.CommitQC) error { return nil } -// deleteBefore truncates WAL entries below the anchor's index, then -// re-persists the anchor for crash recovery. Caller must hold the lock. +// deleteBefore truncates WAL entries below idx. Caller must hold the lock. func (s *commitQCState) deleteBefore(idx types.RoadIndex) error { + if idx <= s.persisted.First { + return nil + } iw, ok := s.iw.Get() if idx >= s.persisted.Next { s.persisted = types.RoadRange{First: idx, Next: idx} @@ -49,17 +51,16 @@ func (s *commitQCState) deleteBefore(idx types.RoadIndex) error { } } } else if ok && iw.Count() > 0 { - if s.persisted.First < idx { - walIdx := iw.FirstIdx() + uint64(idx-s.persisted.First) - if err := iw.TruncateBefore(walIdx, func(entry *types.CommitQC) error { - if entry.Index() != idx { - return fmt.Errorf("commitqc at WAL index %d has road index %d, expected %d (index mapping broken)", walIdx, entry.Index(), idx) - } - return nil - }); err != nil { - return fmt.Errorf("truncate commitqc WAL before %d: %w", walIdx, err) + walIdx := iw.FirstIdx() + uint64(idx-s.persisted.First) + if err := iw.TruncateBefore(walIdx, func(entry *types.CommitQC) error { + if entry.Index() != idx { + return fmt.Errorf("commitqc at WAL index %d has road index %d, expected %d (index mapping broken)", walIdx, entry.Index(), idx) } + return nil + }); err != nil { + return fmt.Errorf("truncate commitqc WAL before %d: %w", walIdx, err) } + s.persisted.First = idx } return nil } From 1c2f9b1ffb2303b0c3f8b2aed1336f758a7615f7 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 7 Aug 2026 15:50:21 +0200 Subject: [PATCH 31/61] p2p tests still fail --- sei-tendermint/internal/p2p/giga/avail.go | 6 ++--- .../internal/p2p/giga/consensus_test.go | 23 ++++++++----------- sei-tendermint/internal/p2p/giga/data.go | 2 +- .../internal/p2p/giga_router_common_test.go | 1 - 4 files changed, 14 insertions(+), 18 deletions(-) diff --git a/sei-tendermint/internal/p2p/giga/avail.go b/sei-tendermint/internal/p2p/giga/avail.go index 3f64bdd38c..ca951ca81d 100644 --- a/sei-tendermint/internal/p2p/giga/avail.go +++ b/sei-tendermint/internal/p2p/giga/avail.go @@ -83,10 +83,10 @@ func (x *Service) serverStreamCommitQCs(ctx context.Context, server rpc.Server[A qc, err := x.validatorState().Avail().CommitQC(ctx, next) if err != nil { if errors.Is(err, types.ErrPruned) { - next = x.validatorState().Avail().FirstCommitQC() + next = x.validatorState().Avail().First() continue } - return fmt.Errorf("x.validatorState().Avail().FirstCommitQC(): %w", err) + return fmt.Errorf("x.validatorState().Avail().CommitQC(): %w", err) } next = qc.Index() + 1 if err := stream.Send(ctx, types.CommitQCConv.Encode(qc)); err != nil { @@ -173,7 +173,7 @@ func (x *Service) clientStreamCommitQCs(ctx context.Context, c rpc.Client[API]) return fmt.Errorf("types.CommitQCConv.Decode(): %w", err) } if err := x.validatorState().Avail().PushCommitQC(ctx, qc); err != nil { - return fmt.Errorf("s.PushFirstCommitQC(): %w", err) + return fmt.Errorf("s.PushCommitQC(): %w", err) } } } diff --git a/sei-tendermint/internal/p2p/giga/consensus_test.go b/sei-tendermint/internal/p2p/giga/consensus_test.go index a80fa5aa00..50a7b09cf7 100644 --- a/sei-tendermint/internal/p2p/giga/consensus_test.go +++ b/sei-tendermint/internal/p2p/giga/consensus_test.go @@ -25,7 +25,6 @@ func TestConsensusClientServer(t *testing.T) { firstBlock := nodes[0].data.Registry().FirstBlock() if err := scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { s.SpawnBg(func() error { return env.Run(ctx) }) - var wantAppProposal utils.Option[*types.AppProposal] for offset := range types.GlobalBlockNumber(20) { idx := firstBlock + offset t.Logf("[%v] Push a block.", idx) @@ -36,18 +35,11 @@ func TestConsensusClientServer(t *testing.T) { return fmt.Errorf("ds.ProduceLocalBlock(): %w", err) } want := &types.GlobalBlock{ - Header: b.Msg().Block().Header(), - Payload: b.Msg().Block().Payload(), - GlobalNumber: idx, - FinalAppState: wantAppProposal, + Header: b.Msg().Block().Header(), + Payload: b.Msg().Block().Payload(), + GlobalNumber: idx, } - p := types.NewAppProposal( - idx, - types.RoadIndex(offset), - types.GenAppHash(rng), - registry.LatestEpoch().EpochIndex(), - ) - wantAppProposal = utils.Some(p) + wantAppProposal := utils.None[*types.AppProposal]() for _, n := range nodes { t.Logf("[%v] Wait for it to be final.", idx) got, err := n.data.GlobalBlock(ctx, idx) @@ -62,13 +54,18 @@ func TestConsensusClientServer(t *testing.T) { if err := utils.TestDiff(want, got); err != nil { return err } + if !wantAppProposal.IsPresent() { + wantAppProposal = utils.Some(types.NewAppProposal(qc.QC().Proposal(), types.GenAppHash(rng))) + } + p := wantAppProposal.OrPanic("missing app proposal") if err := n.data.PushAppHash(ctx, idx, p.AppHash()); err != nil { return fmt.Errorf("ds.PushAppProposal(): %w", err) } } for _, n := range nodes { t.Logf("[%v] Wait for AppHash consensus.", idx) - got, _, err := n.consensus.Avail().WaitForAppQC(ctx, p.RoadIndex()) + p := wantAppProposal.OrPanic("missing app proposal") + got, _, err := n.data.AppQC(ctx, idx) if err != nil { return fmt.Errorf("cs.avail.WaitForAppQC(): %w", err) } diff --git a/sei-tendermint/internal/p2p/giga/data.go b/sei-tendermint/internal/p2p/giga/data.go index 7a7eecc6a6..9cd6de84c8 100644 --- a/sei-tendermint/internal/p2p/giga/data.go +++ b/sei-tendermint/internal/p2p/giga/data.go @@ -208,7 +208,7 @@ func (x *Service) serverStreamAppQCs(ctx context.Context, server rpc.Server[API] for next := req.NextBlock; ; { appQC, commitQC, err := x.validatorState().Data().AppQC(ctx, next) if err != nil { - return fmt.Errorf("x.validatorState().Avail().WaitForAppQC(): %w", err) + return fmt.Errorf("x.validatorState().Data().AppQC(): %w", err) } next = commitQC.QC().GlobalRange().Next if err := stream.Send(ctx, types.AppQCConv.Encode(appQC)); err != nil { diff --git a/sei-tendermint/internal/p2p/giga_router_common_test.go b/sei-tendermint/internal/p2p/giga_router_common_test.go index 4e340a157c..4cc991438c 100644 --- a/sei-tendermint/internal/p2p/giga_router_common_test.go +++ b/sei-tendermint/internal/p2p/giga_router_common_test.go @@ -114,7 +114,6 @@ func TestBuildDataStateStartsRecoveryAtAppTip(t *testing.T) { App: proxy.New(&fixedHeightApp{height: int64(last)}), }, db) require.NoError(t, err) - require.Equal(t, last, state.FirstAppProposal()) got, err := state.TryBlock(last) require.NoError(t, err) require.Equal(t, blocks[gr.Len()/2].Header().Hash(), got.Header().Hash()) From 94369ac60442c439da6c7d93611a29095674a81c Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 7 Aug 2026 20:25:26 +0200 Subject: [PATCH 32/61] should be fine --- sei-db/ledger_db/block/block_db_test.go | 22 +++- .../block/littblock/litt_block_db.go | 59 +++------ .../ledger_db/block/memblock/mem_block_db.go | 47 ++++--- sei-tendermint/autobahn/types/block_db.go | 47 +++---- sei-tendermint/autobahn/types/types_test.go | 59 --------- .../internal/autobahn/data/state.go | 122 ++++++++---------- .../internal/autobahn/data/state_test.go | 12 +- .../internal/autobahn/data/testonly.go | 80 ------------ 8 files changed, 139 insertions(+), 309 deletions(-) diff --git a/sei-db/ledger_db/block/block_db_test.go b/sei-db/ledger_db/block/block_db_test.go index e9e0c15225..e088f0a3a6 100644 --- a/sei-db/ledger_db/block/block_db_test.go +++ b/sei-db/ledger_db/block/block_db_test.go @@ -173,7 +173,7 @@ func drainRecent(t *testing.T, db types.BlockDB) []iterEntry { NextQC: 0, NextAppQC: 0, NextAppProposal: 0, - }).Floor() + }).First for _, qc := range recent.CommitQCs { first := qc.QC().GlobalRange().First next := first + gbn(len(qc.Headers())) @@ -192,7 +192,7 @@ func drainRecent(t *testing.T, db types.BlockDB) []iterEntry { } require.True(t, found, "block %d must be covered by a recent QC", b.Number) } - if appQC, ok := recent.AppQC.Get(); ok { + for _, appQC := range recent.AppQCs { gr := appQC.Proposal().GlobalRange() for i := range entries { if gr.Has(entries[i].n) { @@ -959,8 +959,8 @@ func testReadRecent(t *testing.T, build builder) { recent, err := db.ReadRecent() require.NoError(t, err) require.Equal(t, db.Status(), recent.Status.OrPanic("recent status")) - gotAppQC, ok := recent.AppQC.Get() - require.True(t, ok) + require.NotEmpty(t, recent.AppQCs) + gotAppQC := recent.AppQCs[0] require.Equal(t, appQC.Proposal().RoadIndex(), gotAppQC.Proposal().RoadIndex()) entries := drainRecent(t, db) recoveryFloor := appQC.Proposal().GlobalRange().Next - 1 @@ -1025,6 +1025,9 @@ func testWriteAppProposalOrderRejected(t *testing.T, build builder) { err = db.WriteAppProposal(appProposalForBatch(rng, b1)) require.ErrorIs(t, err, types.ErrAppProposalNonContiguous, "first AppProposal must start at retained QC floor") + for i, blk := range b0.blocks { + require.NoError(t, db.WriteBlock(b0.first+gbn(i), blk)) + } appProposal0 := appProposalForBatch(rng, b0) require.NoError(t, db.WriteAppProposal(appProposal0)) @@ -1035,6 +1038,9 @@ func testWriteAppProposalOrderRejected(t *testing.T, build builder) { err = db.WriteAppProposal(appProposalForBatch(rng, b2)) require.ErrorIs(t, err, types.ErrAppProposalNonContiguous, "AppProposal gap must fail") + for i, blk := range b1.blocks { + require.NoError(t, db.WriteBlock(b1.first+gbn(i), blk)) + } require.NoError(t, db.WriteAppProposal(appProposalForBatch(rng, b1))) tips := db.Status() require.Equal(t, b1.next, tips.NextAppProposal) @@ -1059,6 +1065,10 @@ func testWriteAppQCOrderRejected(t *testing.T, build builder) { err = db.WriteAppQC(appQCForBatch(rng, keys, b1)) require.ErrorIs(t, err, types.ErrAppQCNonContiguous, "first AppQC must start at retained QC floor") + for i, blk := range b0.blocks { + require.NoError(t, db.WriteBlock(b0.first+gbn(i), blk)) + } + require.NoError(t, db.WriteAppProposal(appProposalForBatch(rng, b0))) appQC0 := appQCForBatch(rng, keys, b0) require.NoError(t, db.WriteAppQC(appQC0)) @@ -1069,6 +1079,10 @@ func testWriteAppQCOrderRejected(t *testing.T, build builder) { err = db.WriteAppQC(appQCForBatch(rng, keys, b2)) require.ErrorIs(t, err, types.ErrAppQCNonContiguous, "AppQC gap must fail") + for i, blk := range b1.blocks { + require.NoError(t, db.WriteBlock(b1.first+gbn(i), blk)) + } + require.NoError(t, db.WriteAppProposal(appProposalForBatch(rng, b1))) require.NoError(t, db.WriteAppQC(appQCForBatch(rng, keys, b1))) tips := db.Status() require.Equal(t, b1.next, tips.NextAppQC) diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index 8320e413c3..c3182a770c 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -338,14 +338,8 @@ func (s *blockDB) WriteAppQC(appQC *types.AppQC) error { return fmt.Errorf("first AppQC starts at %d, expected retained QC floor %d: %w", gr.First, s.oldestQCStart, types.ErrAppQCNonContiguous) } - - qc, err := readQCCovering(s.table, gr.First) - if err != nil { - return fmt.Errorf("read matching QC for AppQC [%d,%d): %w", gr.First, gr.Next, err) - } - if want := qc.QC().GlobalRange(); gr != want { - return fmt.Errorf("AppQC [%d,%d) does not exactly match QC [%d,%d): %w", - gr.First, gr.Next, want.First, want.Next, types.ErrAppQCMissingQC) + if lastAppProposal, ok := s.lastAppProposal.Get(); !ok || gr.Next > lastAppProposal.GlobalRange().Next { + return fmt.Errorf("AppQC [%d,%d) is not covered by written AppProposals: %w", gr.First, gr.Next, types.ErrAppQCMissingQC) } value := encodeAppQC(appQC) @@ -360,7 +354,6 @@ func (s *blockDB) WriteAppQC(appQC *types.AppQC) error { if err := s.table.Put(appQCKey(gr.First), value, aliases...); err != nil { return fmt.Errorf("failed to put AppQC [%d,%d): %w", gr.First, gr.Next, err) } - s.lastAppQC = utils.Some(appQC) return nil } @@ -384,19 +377,9 @@ func (s *blockDB) WriteAppProposal(appProposal *types.AppProposal) error { return fmt.Errorf("first AppProposal starts at %d, expected retained QC floor %d: %w", gr.First, s.oldestQCStart, types.ErrAppProposalNonContiguous) } - - qc, err := readQCCovering(s.table, gr.First) - if err != nil { - return fmt.Errorf("read matching QC for AppProposal [%d,%d): %w", gr.First, gr.Next, err) - } - if want := qc.QC().GlobalRange(); gr != want { - return fmt.Errorf("AppProposal [%d,%d) does not exactly match QC [%d,%d): %w", - gr.First, gr.Next, want.First, want.Next, types.ErrAppProposalMissingQC) + if !s.hasBlocks || gr.Next > s.lastBlockNumber+1 { + return fmt.Errorf("AppProposal [%d,%d) is not covered by written blocks: %w", gr.First, gr.Next, types.ErrAppProposalMissingQC) } - if err := appProposal.Verify(qc.QC()); err != nil { - return fmt.Errorf("AppProposal [%d,%d) does not verify against matching QC: %w", gr.First, gr.Next, err) - } - value := encodeAppProposal(appProposal) var aliases []*litttypes.SecondaryKey for m := gr.First + 1; m < gr.Next; m++ { @@ -418,20 +401,15 @@ func (s *blockDB) PruneBefore(blockHeight types.GlobalBlockNumber) error { s.mu.Lock() defer s.mu.Unlock() - if !s.hasBlocks { + status,ok := s.statusLocked().Get() + if !ok { // Ignore prune requests if we've not got any data yet. Simplifies several edge cases // and is technically a legal implementation of the contract in the godocs. return nil } // Round the watermark down to the start of a QC's range, to avoid pruning a QC before its blocks. - blockHeight, err := s.clampPruneBoundary(min(blockHeight, s.statusLocked().Or(types.DBStatus{ - First: 0, - NextBlock: 0, - NextQC: 0, - NextAppQC: 0, - NextAppProposal: 0, - }).Floor())) + blockHeight, err := s.clampPruneBoundary(min(blockHeight, status.First)) if err != nil { return err } @@ -509,22 +487,23 @@ func (s *blockDB) statusLocked() utils.Option[types.DBStatus] { if !ok { return utils.None[types.DBStatus]() } - first := max(s.oldestQCStart, types.GlobalBlockNumber(s.watermark.Load())) + first := s.oldestQCStart if s.hasBlocks { first = max(first, s.firstBlockNumber) } status := types.DBStatus{ First: first, - NextBlock: s.oldestQCStart, + NextAppQC: first, + NextAppProposal: first, + NextBlock: first, NextQC: qc.QC().GlobalRange().Next, - NextAppQC: s.oldestQCStart, - NextAppProposal: s.oldestQCStart, } if s.hasBlocks { status.NextBlock = s.lastBlockNumber + 1 } if appQC, ok := s.lastAppQC.Get(); ok { status.NextAppQC = appQC.Proposal().GlobalRange().Next + status.First = status.NextAppQC - 1 } if appProposal, ok := s.lastAppProposal.Get(); ok { status.NextAppProposal = appProposal.GlobalRange().Next @@ -541,7 +520,7 @@ func (s *blockDB) ReadRecent() (types.RecentData, error) { if !ok { return types.RecentData{}, nil } - targetFloor := status.Floor() + targetFloor := status.First // Collect data >= targetFloor. it, err := s.table.Iterator(true) @@ -584,10 +563,8 @@ func (s *blockDB) ReadRecent() (types.RecentData, error) { return types.RecentData{}, fmt.Errorf("failed to decode recent AppQC: %w", err) } gr := appQC.Proposal().GlobalRange() - if gr.First <= targetFloor && targetFloor < gr.Next { - recent.AppQC = utils.Some(appQC) - } else if targetFloor <= gr.First { - recent.AppQC = utils.Some(appQC) + if targetFloor < gr.Next { + recent.AppQCs = append(recent.AppQCs, appQC) } case kindAppProp: appProposal, err := decodeAppProposal(value) @@ -603,8 +580,7 @@ func (s *blockDB) ReadRecent() (types.RecentData, error) { if err != nil { return types.RecentData{}, fmt.Errorf("failed to decode recent CommitQC: %w", err) } - _, next := coveredRange(qc) - if targetFloor < next { + if targetFloor < qc.QC().GlobalRange().Next { recent.CommitQCs = append(recent.CommitQCs, qc) } default: @@ -612,12 +588,13 @@ func (s *blockDB) ReadRecent() (types.RecentData, error) { } // Safety check: if watermark has been moved and GC happened to get executed during iteration, // the loaded data might be inconsistent with the targetFloor we computed. - if newFloor := s.Status().Floor(); newFloor != targetFloor { + if newFloor := s.Status().First; newFloor != targetFloor { return types.RecentData{}, fmt.Errorf("watermark has moved while iterating") } slices.Reverse(recent.CommitQCs) slices.Reverse(recent.Blocks) slices.Reverse(recent.AppProposals) + slices.Reverse(recent.AppQCs) return recent, nil } diff --git a/sei-db/ledger_db/block/memblock/mem_block_db.go b/sei-db/ledger_db/block/memblock/mem_block_db.go index 8ba7ffbd65..3dd83d0bd6 100644 --- a/sei-db/ledger_db/block/memblock/mem_block_db.go +++ b/sei-db/ledger_db/block/memblock/mem_block_db.go @@ -183,6 +183,9 @@ func (s *blockDB) WriteAppProposal(appProposal *types.AppProposal) error { first, entries[0].lower, types.ErrAppProposalNonContiguous) } } + if !s.hasBlocks || next > s.lastBlockNumber+1 { + return fmt.Errorf("AppProposal [%d,%d) is not covered by written blocks: %w", first, next, types.ErrAppProposalMissingQC) + } qc, ok := s.qcsByLower[first] if !ok || qc.upper != next { return fmt.Errorf("AppProposal [%d,%d) has no exact matching QC: %w", @@ -223,6 +226,9 @@ func (s *blockDB) WriteAppQC(appQC *types.AppQC) error { first, entries[0].lower, types.ErrAppQCNonContiguous) } } + if !s.hasAppProposal || next > s.lastAppPropNext { + return fmt.Errorf("AppQC [%d,%d) is not covered by written AppProposals: %w", first, next, types.ErrAppQCMissingQC) + } qc, ok := s.qcsByLower[first] if !ok || qc.upper != next { return fmt.Errorf("AppQC [%d,%d) has no exact matching QC: %w", @@ -249,7 +255,7 @@ func (s *blockDB) PruneBefore(n types.GlobalBlockNumber) error { NextQC: 0, NextAppQC: 0, NextAppProposal: 0, - }).Floor()) + }).First) // Round the watermark down to the covering QC's First. A QC's cohort of // blocks changes readability atomically, so the watermark must never fall // strictly inside a QC's range (see littblock): otherwise a read would @@ -312,16 +318,17 @@ func (s *blockDB) statusLocked() utils.Option[types.DBStatus] { } status := types.DBStatus{ First: first, - NextBlock: oldestQCStart, + NextBlock: first, NextQC: s.lastQCNext, - NextAppQC: oldestQCStart, - NextAppProposal: oldestQCStart, + NextAppQC: first, + NextAppProposal: first, } if s.hasBlocks { status.NextBlock = s.lastBlockNumber + 1 } if s.hasAppQC { status.NextAppQC = s.lastAppQCNext + status.First = status.NextAppQC - 1 } if s.hasAppProposal { status.NextAppProposal = s.lastAppPropNext @@ -340,31 +347,24 @@ func (s *blockDB) ReadRecent() (types.RecentData, error) { NextQC: 0, NextAppQC: 0, NextAppProposal: 0, - }).Floor() + }).First recent := types.RecentData{Status: status} - var targetIndex types.RoadIndex - bounded := s.hasAppProposal && s.hasAppQC && s.hasBlocks - if s.hasAppQC { - appQC := s.appQCCoveringLocked(floor) - if appQC == nil { - appQC = s.appQCs[s.latestAppQCStartBlock].appQC - } - recent.AppQC = utils.Some(appQC) - targetIndex = appQC.Proposal().RoadIndex() - } - if !bounded && s.hasBlocks { - floor = max(floor, s.firstBlockNumber) - } for _, e := range s.sortedQCsLocked() { if e.upper <= s.watermark { continue } - if s.hasAppQC && e.qc.Index() < targetIndex { + if e.upper <= floor { continue } recent.CommitQCs = append(recent.CommitQCs, e.qc) } + for _, e := range s.sortedAppQCsLocked() { + if e.upper <= floor { + continue + } + recent.AppQCs = append(recent.AppQCs, e.appQC) + } for _, e := range s.sortedAppProposalsLocked() { if e.upper <= floor { continue @@ -399,6 +399,15 @@ func (s *blockDB) sortedAppProposalsLocked() []appProposalEntry { return entries } +func (s *blockDB) sortedAppQCsLocked() []appQCEntry { + entries := make([]appQCEntry, 0, len(s.appQCs)) + for _, e := range s.appQCs { + entries = append(entries, e) + } + sort.Slice(entries, func(i, j int) bool { return entries[i].lower < entries[j].lower }) + return entries +} + func (s *blockDB) appQCCoveringLocked(n types.GlobalBlockNumber) *types.AppQC { for _, e := range s.appQCs { if e.lower <= n && n < e.upper { diff --git a/sei-tendermint/autobahn/types/block_db.go b/sei-tendermint/autobahn/types/block_db.go index 62e18ff6da..2a5ccc6ea3 100644 --- a/sei-tendermint/autobahn/types/block_db.go +++ b/sei-tendermint/autobahn/types/block_db.go @@ -282,23 +282,14 @@ type BlockDB interface { Close() error } -// DBStatus is the in-memory write tips returned by BlockDB.Status. -// Its fields are exclusive "next to write" cursors (matching data.State's -// DBStatus). Zero means no write of that kind has occurred yet -// (NextBlock/NextQC are never zero after a successful write: the first -// written block number N yields NextBlock = N+1 ≥ 1). +// DBStatus represents the suffix of BlockDB data that data.State can append to/would load on recovery. +// Elements since the last anchor (last full row which contains AppQC,AppProposal,Block,QC) to +// the tips persisted in the DB. These are the elements that would be loaded by data.State on restart +// via BlockDB.ReadRecent. +// First <= NextAppQC <= NextAppProposal <= NextBlock <= NextQC type DBStatus struct { - // First is the recovery floor used by Floor. It is the oldest readable block - // when blocks are present, otherwise the oldest retained CommitQC start. - // Zero if no QC has been written. + // First is either NextAppQC, or NextAppQC-1, depending on whether there is at least 1 AppQC in the BlockDB. First GlobalBlockNumber - // NextBlock is one past the highest GlobalBlockNumber accepted by WriteBlock - // (the next block number that may be written). Zero if no block has been written. - NextBlock GlobalBlockNumber - // NextQC is one past the highest GlobalBlockNumber covered by the last QC - // accepted by WriteQC (the next QC's range must start here). Zero if no QC - // has been written. - NextQC GlobalBlockNumber // NextAppQC is one past the highest GlobalBlockNumber covered by the last // AppQC accepted by WriteAppQC. Zero if no AppQC has been written. NextAppQC GlobalBlockNumber @@ -306,17 +297,13 @@ type DBStatus struct { // last AppProposal accepted by WriteAppProposal. Zero if no AppProposal has // been written. NextAppProposal GlobalBlockNumber -} - -// Floor returns the startup recovery floor implied by the durable data tips. -// Until blocks, AppProposals, and AppQCs are all present, there is no app -// recovery floor yet, so Floor returns zero. -func (s DBStatus) Floor() GlobalBlockNumber { - f := min(s.NextQC, s.NextBlock, s.NextAppProposal, s.NextAppQC) - if f <= s.First { - return s.First - } - return f - 1 + // NextBlock is one past the highest GlobalBlockNumber accepted by WriteBlock + // (the next block number that may be written). Zero if no block has been written. + NextBlock GlobalBlockNumber + // NextQC is one past the highest GlobalBlockNumber covered by the last QC + // accepted by WriteQC (the next QC's range must start here). Zero if no QC + // has been written. + NextQC GlobalBlockNumber } // RecentBlock is one block returned by BlockDB.ReadRecent. @@ -327,10 +314,12 @@ type RecentBlock struct { // RecentData is the materialized suffix used by data.State startup recovery. type RecentData struct { - // Status is the durable write status observed while selecting the recent suffix. - Status utils.Option[DBStatus] + // Ranges of elements in the suffix. + // None if the BlockDB is empty. + Status utils.Option[DBStatus] + // Elements which constitute the suffix. CommitQCs []*FullCommitQC Blocks []RecentBlock AppProposals []*AppProposal - AppQC utils.Option[*AppQC] + AppQCs []*AppQC } diff --git a/sei-tendermint/autobahn/types/types_test.go b/sei-tendermint/autobahn/types/types_test.go index 1866ca1c63..7f57a70c99 100644 --- a/sei-tendermint/autobahn/types/types_test.go +++ b/sei-tendermint/autobahn/types/types_test.go @@ -96,65 +96,6 @@ func TestMarshal(t *testing.T) { } } -func TestDBStatusFloor(t *testing.T) { - for _, tc := range []struct { - name string - in DBStatus - want GlobalBlockNumber - }{ - { - name: "empty", - in: DBStatus{ - First: 0, - NextBlock: 0, - NextQC: 0, - NextAppProposal: 0, - NextAppQC: 0, - }, - want: 0, - }, - { - name: "missing app proposal", - in: DBStatus{ - First: 5, - NextBlock: 10, - NextQC: 10, - NextAppProposal: 5, - NextAppQC: 8, - }, - want: 5, - }, - { - name: "minimum durable tip minus one", - in: DBStatus{ - First: 5, - NextBlock: 12, - NextQC: 12, - NextAppProposal: 10, - NextAppQC: 8, - }, - want: 7, - }, - { - name: "first lower bound", - in: DBStatus{ - First: 10, - NextBlock: 12, - NextQC: 15, - NextAppProposal: 5, - NextAppQC: 10, - }, - want: 10, - }, - } { - t.Run(tc.name, func(t *testing.T) { - if got := tc.in.Floor(); got != tc.want { - t.Fatalf("Floor() = %d, want %d", got, tc.want) - } - }) - } -} - func makePrepareQC(keys []SecretKey, vote *PrepareVote) *PrepareQC { var votes []*Signed[*PrepareVote] for _, k := range keys { diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index 325ccebc88..7a26512c8a 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -22,17 +22,6 @@ type Config struct { LastExecutedBlock utils.Option[types.GlobalBlockNumber] } -// StateAPI is the interface of the State for consuming global blocks -// and reporting AppHashes. -type StateAPI interface { - GlobalBlock(ctx context.Context, n types.GlobalBlockNumber) (*types.GlobalBlock, error) - // PushAppHash blocks until block n and its QC are durably persisted, - // ensuring AppVotes are only issued for data that survives a crash. - PushAppHash(ctx context.Context, n types.GlobalBlockNumber, hash types.AppHash) error -} - -var _ StateAPI = (*State)(nil) - // blockEntry is a (number, block) pair collected in runPersist batches. type blockEntry struct { n types.GlobalBlockNumber @@ -50,14 +39,12 @@ type inner struct { blockHashes map[types.BlockHeaderHash]types.GlobalBlockNumber // blockHashes mirrors blocks (insertBlock / setPersisted) // first is the exclusive low end of retained in-memory state: maps keep [first, next*). - // Advanced by setPersisted to persisted.Floor(). + // Advanced by runPersist() // - // first <= persisted.NextBlock <= nextBlock <= nextQC - // first <= persisted.NextAppProposal <= nextAppProposal <= nextQC - // first <= persisted.NextAppQC <= nextAppQC <= nextQC + // first <= nextAppQC <= nextAppProposal <= nextBlock <= nextQC first types.GlobalBlockNumber - nextAppProposal types.GlobalBlockNumber nextAppQC types.GlobalBlockNumber + nextAppProposal types.GlobalBlockNumber nextBlock types.GlobalBlockNumber nextQC types.GlobalBlockNumber persisted types.DBStatus @@ -95,8 +82,8 @@ func (i *inner) insertAppQC(registry *epoch.Registry, appQC *types.AppQC) error if gr.Next <= i.nextAppQC { return nil } - if gr.Next > i.nextQC { - return fmt.Errorf("Missing CommitQC for this AppQC") + if gr.Next > i.nextAppProposal { + return fmt.Errorf("Missing AppProposal for this AppQC") } if gr.First > i.nextAppQC { return fmt.Errorf("AppQC gap: expected first<=%d, got %d", i.nextAppQC, gr.First) @@ -226,7 +213,7 @@ func loadFromBlockDB(cfg *Config, blockDB types.BlockDB) (*inner, error) { NextAppQC: firstBlock, NextBlock: firstBlock, }) - first := status.Floor() + first := status.First inner := &inner{ qcs: map[types.GlobalBlockNumber]*types.FullCommitQC{}, blocks: map[types.GlobalBlockNumber]*types.Block{}, @@ -246,16 +233,6 @@ func loadFromBlockDB(cfg *Config, blockDB types.BlockDB) (*inner, error) { return nil, fmt.Errorf("load QC from BlockDB: %w", err) } } - if appQC, ok := recent.AppQC.Get(); ok { - if err := inner.insertAppQC(cfg.Registry, appQC); err != nil { - return nil, fmt.Errorf("load AppQC from BlockDB: %w", err) - } - } - for _, appProposal := range recent.AppProposals { - if err := inner.insertAppProposal(appProposal); err != nil { - return nil, fmt.Errorf("load AppProposal from BlockDB: %w", err) - } - } for _, b := range recent.Blocks { qc := inner.qcs[b.Number] ei := qc.QC().Proposal().EpochIndex() @@ -273,7 +250,17 @@ func loadFromBlockDB(cfg *Config, blockDB types.BlockDB) (*inner, error) { // Advance nextBlock through contiguous loaded blocks. Don't use // updateNextBlock: stale timestamps would skew metrics. inner.nextBlock = max(inner.first, status.NextBlock) - inner.setPersisted(status) + for _, appProposal := range recent.AppProposals { + if err := inner.insertAppProposal(appProposal); err != nil { + return nil, fmt.Errorf("load AppProposal from BlockDB: %w", err) + } + } + for _, appQC := range recent.AppQCs { + if err := inner.insertAppQC(cfg.Registry, appQC); err != nil { + return nil, fmt.Errorf("load AppQC from BlockDB: %w", err) + } + } + inner.setAnchor() return inner, nil } @@ -310,7 +297,7 @@ func (s *State) PushQC(ctx context.Context, qc *types.FullCommitQC, blocks []*ty needQC, err := func() (bool, error) { for inner, ctrl := range s.inner.Lock() { if err := ctrl.WaitUntil(ctx, func() bool { - return gr.First <= inner.nextQC && gr.First < inner.nextAppProposal+blocksCacheSize + return gr.First <= inner.nextQC && gr.First < inner.first+blocksCacheSize }); err != nil { return false, err } @@ -596,13 +583,10 @@ func (s *State) globalBlockByHashFromDB(hash types.BlockHeaderHash) (utils.Optio return utils.Some(assembleGlobalBlock(bn.Number, bn.Block, qc)), nil } -// PushAppHash marks blocks up to n as executed. Hash is the execution result. -// Waits for the block to be durably persisted before proceeding. +// PushAppHash marks blocks up to n as executed. func (s *State) PushAppHash(ctx context.Context, n types.GlobalBlockNumber, hash types.AppHash) error { for inner, ctrl := range s.inner.Lock() { - if err := ctrl.WaitUntil(ctx, func() bool { - return n < inner.persisted.NextBlock - }); err != nil { + if err := ctrl.WaitUntil(ctx, func() bool { return n < inner.nextBlock }); err != nil { return err } p := inner.qcs[n].QC().Proposal() @@ -646,7 +630,7 @@ func (s *State) PushAppQC(ctx context.Context, appQC *types.AppQC) error { gr := appQC.Proposal().GlobalRange() for inner, ctrl := range s.inner.Lock() { if err := ctrl.WaitUntil(ctx, func() bool { - return gr.Next <= inner.nextQC + return gr.Next <= inner.nextAppProposal }); err != nil { return err } @@ -742,13 +726,11 @@ func (s *State) PruneBefore(retainFrom types.GlobalBlockNumber) error { return s.blockDB.PruneBefore(retainFrom) } -// runPersist is a background goroutine that persists blocks, QCs, +// runPersist is a background goroutine that persists QCs, blocks, // AppProposals, and AppQCs to BlockDB. It waits for in-memory data to advance -// past the DBStatus persistence cursor, then writes covering QCs (first, per -// the BlockDB contract), blocks, AppProposals, and AppQCs, then flushes once per -// batch. persisted.NextBlock advances with the block tip to unblock PushAppHash -// only when data is durable. AppProposals and AppQCs are persisted once their -// matching CommitQC range is already durable. +// past the DBStatus persistence cursor, then writes each stream in cursor +// order and flushes once per batch. persisted.NextBlock advances with the +// block tip to unblock PushAppHash only when data is durable. // Errors propagate vertically (kill the component). // // Cursors seed from BlockDB.Status() when non-zero so PushQC-before-Run heights @@ -763,8 +745,7 @@ func (s *State) PruneBefore(retainFrom types.GlobalBlockNumber) error { // in-memory QC, and no rewrite of QCs already on disk). // // In-memory block/QC/AppProposal/AppQC eviction is driven by persisted DBStatus -// changes. Entries are retained until all three durable data streams (blocks, -// AppProposals, and AppQCs) have caught up. +// changes. func (s *State) runPersist(ctx context.Context) error { for { var qcs []*types.FullCommitQC @@ -787,22 +768,24 @@ func (s *State) runPersist(ctx context.Context) error { qcs = append(qcs, qc) status.NextQC = qc.QC().GlobalRange().Next } - for status.NextAppQC < inner.nextAppQC { - appQC := inner.appQCs[status.NextAppQC] - appQCs = append(appQCs, appQC) - status.NextAppQC = appQC.Proposal().GlobalRange().Next + for status.NextBlock < inner.nextBlock { + blocks = append(blocks, blockEntry{n: status.NextBlock, block: inner.blocks[status.NextBlock]}) + status.NextBlock += 1 } for status.NextAppProposal < inner.nextAppProposal { appProposal := inner.appProposals[status.NextAppProposal] appProposals = append(appProposals, appProposal) status.NextAppProposal = appProposal.GlobalRange().Next } - for status.NextBlock < inner.nextBlock { - blocks = append(blocks, blockEntry{n: status.NextBlock, block: inner.blocks[status.NextBlock]}) - status.NextBlock += 1 + for status.NextAppQC < inner.nextAppQC { + appQC := inner.appQCs[status.NextAppQC] + appQCs = append(appQCs, appQC) + status.NextAppQC = appQC.Proposal().GlobalRange().Next + status.First = status.NextAppQC - 1 } } - // Write QCs first (BlockDB contract: QC must precede covered blocks and AppQC). + // Write data in order: QCs,blocks,appProposals,appQCs + // to maintain the invariants. for _, qc := range qcs { if err := s.blockDB.WriteQC(qc); err != nil { return fmt.Errorf("write QC %d: %w", qc.QC().Index(), err) @@ -823,33 +806,30 @@ func (s *State) runPersist(ctx context.Context) error { return fmt.Errorf("write AppQC %d: %w", appQC.Proposal().RoadIndex(), err) } } + // Flush the new data. if err := s.blockDB.Flush(); err != nil { return fmt.Errorf("flush BlockDB: %w", err) } + // Prune the inner state. for inner, ctrl := range s.inner.Lock() { - inner.setPersisted(status) + inner.persisted = status + for inner.first < inner.persisted.First { + n := inner.first + delete(inner.blockHashes, inner.blocks[n].Header().Hash()) + delete(inner.blocks, n) + delete(inner.qcs, n) + delete(inner.appQCs, n) + delete(inner.appProposals, n) + inner.first += 1 + } + inner.setAnchor() ctrl.Updated() } } } -// setPersisted publishes a new durable cursor and pushes first to -// persisted.Floor(). -// I.e. it keeps the same recovery suffix in memory that BlockDB.ReadRecent would -// return. -func (i *inner) setPersisted(persisted types.DBStatus) { - i.persisted = persisted - bound := persisted.Floor() - for i.first < bound { - n := i.first - delete(i.blockHashes, i.blocks[n].Header().Hash()) - delete(i.blocks, n) - delete(i.qcs, n) - delete(i.appQCs, n) - delete(i.appProposals, n) - i.first += 1 - } - if i.first < persisted.NextAppQC { +func (i *inner) setAnchor() { + if i.first < i.persisted.NextAppQC { i.anchor.Store(utils.Some(Anchor{ CommitQC: i.qcs[i.first].QC(), AppQC: i.appQCs[i.first], diff --git a/sei-tendermint/internal/autobahn/data/state_test.go b/sei-tendermint/internal/autobahn/data/state_test.go index 485c281f44..06698a6a61 100644 --- a/sei-tendermint/internal/autobahn/data/state_test.go +++ b/sei-tendermint/internal/autobahn/data/state_test.go @@ -490,7 +490,7 @@ func TestPushQCBeforeRunPersistsToBlockDB(t *testing.T) { // TestEvictionWaitsForAppQC checks that setPersisted does not drop // AppProposals until AppQC is persisted, and that once it is, heights below -// persisted.Floor() are evicted. +// persisted.First are evicted. func TestEvictionWaitsForAppQC(t *testing.T) { ctx := t.Context() rng := utils.TestRng() @@ -551,7 +551,7 @@ func TestEvictionWaitsForAppQC(t *testing.T) { } for inner := range state.inner.Lock() { - evictionBound := inner.persisted.Floor() + evictionBound := inner.persisted.First if inner.first != evictionBound { return fmt.Errorf("after catching up, first = %d, want eviction bound %d", inner.first, evictionBound) } @@ -610,7 +610,7 @@ func TestEvictionWaitsForPersistedAppQC(t *testing.T) { // TestNextToExecuteAfterAppEviction checks WaitUntilExecuted / nextToExecute // still work when persisted AppQC aggressively evicts through nextAppProposal -// (first = persisted.Floor()). +// (first = persisted.First). // nextToExecute uses the retained boundary QC. func TestNextToExecuteAfterAppEviction(t *testing.T) { ctx := t.Context() @@ -660,7 +660,7 @@ func TestNextToExecuteAfterAppEviction(t *testing.T) { if inner.nextAppProposal != gr1.Next { return fmt.Errorf("nextAppProposal = %d, want %d", inner.nextAppProposal, gr1.Next) } - evictionBound := inner.persisted.Floor() + evictionBound := inner.persisted.First if inner.first != evictionBound { return fmt.Errorf("first = %d, want eviction bound %d", inner.first, evictionBound) } @@ -822,7 +822,7 @@ func TestPruningKeepsLastQCRange(t *testing.T) { // readability), so a mid-range prune does not refuse heights inside that QC. // // PruneBefore is BlockDB-only: heights still retained in RAM for AppVotes -// (at/above persisted.Floor()) remain +// (at/above persisted.First) remain // readable via TryBlock even after the store watermark advances past them. func TestPruningWithPartialQCRange(t *testing.T) { ctx := t.Context() @@ -860,7 +860,7 @@ func TestPruningWithPartialQCRange(t *testing.T) { return nil })) for inner := range state1.inner.Lock() { - exclusiveFloor = inner.persisted.Floor() + exclusiveFloor = inner.persisted.First require.Equal(t, exclusiveFloor, inner.first) } diff --git a/sei-tendermint/internal/autobahn/data/testonly.go b/sei-tendermint/internal/autobahn/data/testonly.go index d0aafffd1c..29f2fff5b9 100644 --- a/sei-tendermint/internal/autobahn/data/testonly.go +++ b/sei-tendermint/internal/autobahn/data/testonly.go @@ -1,10 +1,6 @@ package data import ( - "context" - "errors" - "fmt" - "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" ) @@ -61,79 +57,3 @@ func TestCommitQC( cqc := types.BuildCommitQC(ep, keys, prev, laneQCs) return types.NewFullCommitQC(cqc, headers), blockList } - -var _ StateAPI = (*MockState)(nil) - -type innerMockState struct { - blocks map[types.GlobalBlockNumber]*types.GlobalBlock // [first,next) - first types.GlobalBlockNumber - next types.GlobalBlockNumber -} - -// MockState is a mock implementation of the StateAPI interface. -// Allows for pushing global blocks directly (without going through consensus). -type MockState struct { - capacity uint64 - inner utils.Watch[*innerMockState] -} - -// NewMockState creates a new MockState with the given block capacity. -func NewMockState(capacity uint64) *MockState { - return &MockState{ - capacity: capacity, - inner: utils.NewWatch(&innerMockState{ - blocks: make(map[types.GlobalBlockNumber]*types.GlobalBlock), - next: 0, - }), - } -} - -// GlobalBlock returns the global block with the given number. -func (s *MockState) GlobalBlock(ctx context.Context, n types.GlobalBlockNumber) (*types.GlobalBlock, error) { - for inner, ctrl := range s.inner.Lock() { - if err := ctrl.WaitUntil(ctx, func() bool { return inner.next > n }); err != nil { - return nil, err - } - if inner.first > n { - return nil, types.ErrPruned - } - return inner.blocks[n], nil - } - panic("unreachable") -} - -// ProduceBlock appends a new global block with the given payload. -func (s *MockState) ProduceBlock(ctx context.Context, payload *types.Payload) error { - for inner, ctrl := range s.inner.Lock() { - if err := ctrl.WaitUntil(ctx, func() bool { - return uint64(inner.next-inner.first) < s.capacity - }); err != nil { - return err - } - inner.blocks[inner.next] = &types.GlobalBlock{ - GlobalNumber: inner.next, - Payload: payload, - } - inner.next += 1 - ctrl.Updated() - } - return nil -} - -// PushAppHash marks all blocks up to n as executed. -func (s *MockState) PushAppHash(_ context.Context, n types.GlobalBlockNumber, appHash types.AppHash) error { - for inner, ctrl := range s.inner.Lock() { - if got, wantMin := n, inner.first; got < wantMin { - return fmt.Errorf("received app proposal out of order: got %v, want >= %v", got, wantMin) - } - if n >= inner.next { - return errors.New("proposal for block which hasn't been received yet") - } - for inner.first <= n { - delete(inner.blocks, inner.first) - inner.first += 1 - } - ctrl.Updated() - } - return nil -} From 911462441bc37f0ba391c682e09b3fc2448a23c4 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 7 Aug 2026 20:31:21 +0200 Subject: [PATCH 33/61] voting only on persisted apphashes --- sei-db/ledger_db/block/littblock/litt_block_db.go | 2 +- sei-tendermint/internal/autobahn/data/state.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index c3182a770c..bef43112e8 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -401,7 +401,7 @@ func (s *blockDB) PruneBefore(blockHeight types.GlobalBlockNumber) error { s.mu.Lock() defer s.mu.Unlock() - status,ok := s.statusLocked().Get() + status, ok := s.statusLocked().Get() if !ok { // Ignore prune requests if we've not got any data yet. Simplifies several edge cases // and is technically a legal implementation of the contract in the godocs. diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index 7a26512c8a..d5f7ddb306 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -614,12 +614,13 @@ func (s *State) PushAppHash(ctx context.Context, n types.GlobalBlockNumber, hash } // AppVote returns an appVote for a block >= n. +// Vote is available ONLY once AppHash has been pushed AND persisted. +// This prevents any possible equivocation and ensures that local node has the executed blocks persisted (since nextAppProposa <= nextBlock). func (s *State) AppVote(ctx context.Context, n types.GlobalBlockNumber) (*types.AppVote, *types.FullCommitQC, error) { for inner, ctrl := range s.inner.Lock() { - if err := ctrl.WaitUntil(ctx, func() bool { return max(inner.nextAppQC, n) < inner.nextAppProposal }); err != nil { + if err := ctrl.WaitUntil(ctx, func() bool { return n < inner.persisted.NextAppProposal }); err != nil { return nil, nil, err } - n := max(inner.nextAppQC, n) return types.NewAppVote(inner.appProposals[n]), inner.qcs[n], nil } panic("unreachable") From 13442850039f3d98f318bd79b908584fdf0793b9 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 7 Aug 2026 20:45:20 +0200 Subject: [PATCH 34/61] simplified db --- .../block/littblock/litt_block_db.go | 368 +++++++++--------- sei-db/ledger_db/block/littblock/qc_reader.go | 31 -- 2 files changed, 178 insertions(+), 221 deletions(-) delete mode 100644 sei-db/ledger_db/block/littblock/qc_reader.go diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index bef43112e8..b91514254a 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -42,34 +42,10 @@ type blockDB struct { // goroutine, so accessed atomically. watermark atomic.Uint64 - // Write-order cursors (see types.BlockDB contract). Guarded by mu. - mu sync.Mutex - hasBlocks bool - lastBlockNumber types.GlobalBlockNumber - lastQC utils.Option[*types.FullCommitQC] - lastAppProposal utils.Option[*types.AppProposal] - lastAppQC utils.Option[*types.AppQC] - - // firstBlockNumber is the lowest block number this handle has seen. Iterator clamps its - // start up to it so a scan always opens on a block that exists: the first block may be - // written anywhere inside its covering QC, so this can sit above oldestQCStart with no - // block in between. Set when the first block is written and re-derived on open (see - // recoverReadFloors). - // - // Like oldestQCStart it is a floor, not an exact value — pruning may reclaim the block it - // names — but by then watermark has advanced past it and Iterator clamps to both. - // Meaningful only while hasBlocks. - firstBlockNumber types.GlobalBlockNumber - - // oldestQCStart is where the oldest QC this handle has seen begins. Iterator clamps its - // start up to it, which is what lets the positioned lookup always land on a retained QC - // record: a start below every QC's range has no key to position at. Set when the first QC - // is written and re-derived on open (see recoverReadFloors). - // - // It is a floor, not an exact value — GC may later reclaim that QC — but by then - // PruneBefore has advanced watermark past it, and Iterator clamps to both. Meaningful - // only while hasQC. - oldestQCStart types.GlobalBlockNumber + // status is the explicit write-order/recovery suffix cursor (see + // types.BlockDB contract). None means the DB is empty. Guarded by mu. + mu sync.Mutex + status utils.Option[types.DBStatus] } // NewBlockDB opens (or creates) a LittDB-backed types.BlockDB from config. The @@ -118,96 +94,47 @@ func NewBlockDB(config *LittBlockConfig) (types.BlockDB, error) { return s, nil } -// recoverCursors reloads the write-order cursors (lastBlockNumber, lastQCNext, -// lastAppQCNext, and their presence flags) from on-disk state. Without this, a reopened DB -// would treat itself as empty and let WriteBlock/WriteQC silently accept -// out-of-order or non-contiguous writes that overwrite or gap persisted data. +// recoverCursors reloads the write-order cursors from on-disk state. Without +// this, a reopened DB would treat itself as empty and let writes silently accept +// out-of-order or non-contiguous data that overwrite or gap persisted data. func (s *blockDB) recoverCursors() error { - it, err := s.table.Iterator(true) + recent, err := s.readRecent() if err != nil { - return fmt.Errorf("failed to open recovery iterator: %w", err) + return fmt.Errorf("read recent data: %w", err) } - defer func() { _ = it.Close() }() - - for !s.hasBlocks || !s.lastQC.IsPresent() || !s.lastAppProposal.IsPresent() || !s.lastAppQC.IsPresent() { - ok, err := it.Next() - if err != nil { - return fmt.Errorf("failed to advance recovery iterator: %w", err) - } - if !ok { - break - } - key, isPrimary, err := it.GetKey() - if err != nil { - return fmt.Errorf("failed to read recovery key: %w", err) - } - if !isPrimary { - continue - } - switch keyKind(key) { - case kindBlock: - if !s.hasBlocks { - s.lastBlockNumber = decodeNumberKey(key) - s.hasBlocks = true - } - case kindQC: - if !s.lastQC.IsPresent() { - value, err := it.GetValue() - if err != nil { - return fmt.Errorf("failed to read newest qc value: %w", err) - } - qc, err := decodeQC(value) - if err != nil { - return fmt.Errorf("failed to unmarshal newest qc: %w", err) - } - s.lastQC = utils.Some(qc) - } - case kindAppProp: - if !s.lastAppProposal.IsPresent() { - value, err := it.GetValue() - if err != nil { - return fmt.Errorf("failed to read newest appProposal value: %w", err) - } - appProposal, err := decodeAppProposal(value) - if err != nil { - return fmt.Errorf("failed to unmarshal newest appProposal: %w", err) - } - s.lastAppProposal = utils.Some(appProposal) - } - case kindAppQC: - if !s.lastAppQC.IsPresent() { - value, err := it.GetValue() - if err != nil { - return fmt.Errorf("failed to read newest appQC value: %w", err) - } - appQC, err := decodeAppQC(value) - if err != nil { - return fmt.Errorf("failed to unmarshal newest appQC: %w", err) - } - s.lastAppQC = utils.Some(appQC) - } + status, ok := recent.Status.Get() + if !ok { + if len(recent.Blocks) > 0 { + return fmt.Errorf("corrupt store: newest block %d has no surviving QC covering it", recent.Blocks[len(recent.Blocks)-1].Number) } + return nil } + s.status = utils.Some(status) return nil } -// recoverReadFloors re-derives the read floors on open: the watermark (with oldestQCStart) from -// the oldest surviving QC, and firstBlockNumber from the oldest surviving block. Both are -// in-memory only, so a restart forgets every PruneBefore. That is fine for reclamation (nothing -// new is deleted), but we must protect against showing un-pruned blocks with pruned QCs. +// recoverReadFloors re-derives the read watermark on open from the oldest +// surviving QC. It is in-memory only, so a restart forgets every PruneBefore. +// That is fine for reclamation (nothing new is deleted), but we must protect +// against showing un-pruned blocks with pruned QCs. // // One forward pass serves both. QCs are written before the blocks they cover, so the oldest -// surviving record is normally a QC and the first block follows shortly after. The block search -// is skipped when the store holds no blocks — hasBlocks comes from recoverCursors, which runs -// first — so a QC-only store does not walk the whole table looking for a block that is not there. +// surviving record is normally a QC and the first block follows shortly after. +// The block search is skipped when the store holds no blocks — status.NextBlock +// comes from recoverCursors, which runs first — so a QC-only store does not walk +// the whole table looking for a block that is not there. func (s *blockDB) recoverReadFloors() error { + status, ok := s.status.Get() + if !ok { + return nil + } it, err := s.table.Iterator(false) if err != nil { return fmt.Errorf("failed to open read floor recovery iterator: %w", err) } defer func() { _ = it.Close() }() - needQC, needBlock := true, s.hasBlocks + needQC, needBlock := true, status.NextBlock != 0 for needQC || needBlock { ok, err := it.Next() if err != nil { @@ -228,40 +155,53 @@ func (s *blockDB) recoverReadFloors() error { if needQC { oldest := decodeNumberKey(key) s.watermark.Store(uint64(oldest)) - s.oldestQCStart = oldest needQC = false } case kindBlock: if needBlock { - s.firstBlockNumber = decodeNumberKey(key) needBlock = false } } } - if needQC && s.hasBlocks { + if needQC && status.NextBlock != 0 { // No QC survives. The never-empty prune invariant guarantees at least one // (block, QC) pair is always retained, so blocks-without-QC is unreachable // through normal operation — it means the store is corrupt (e.g. a QC WAL // file was removed out of band). Refuse to open rather than serve blocks we // can no longer trust. - return fmt.Errorf("corrupt store: newest block %d has no surviving QC covering it", s.lastBlockNumber) + return fmt.Errorf("corrupt store: newest block %d has no surviving QC covering it", status.NextBlock-1) } return nil } +func setUnstartedFloor(status *types.DBStatus, first types.GlobalBlockNumber) { + oldFirst := status.First + if status.NextAppQC == oldFirst { + status.NextAppQC = first + } + if status.NextAppProposal == oldFirst { + status.NextAppProposal = first + } + if status.NextBlock == oldFirst { + status.NextBlock = first + } + status.First = first +} + func (s *blockDB) WriteBlock(n types.GlobalBlockNumber, blk *types.Block) error { s.mu.Lock() defer s.mu.Unlock() - if s.hasBlocks && n != s.lastBlockNumber+1 { + status, ok := s.status.Get() + if ok && status.NextBlock > status.First && n != status.NextBlock { return fmt.Errorf("block number %d not contiguous with last written %d: %w", - n, s.lastBlockNumber, types.ErrBlockOutOfOrder) + n, status.NextBlock-1, types.ErrBlockOutOfOrder) } // A covering QC must already be written. Since QCs are contiguous and blocks - // strictly ascending, n is covered iff n < lastQCNext. This guard also fixes + // strictly ascending, n is covered iff n < status.NextQC. This guard also fixes // the QC-before-block write order: the covering QC's Put has already issued // under this mutex, so on a crash a surviving block implies a surviving QC. - if qc, ok := s.lastQC.Get(); !ok || n >= qc.QC().GlobalRange().Next { + if !ok || n >= status.NextQC { return fmt.Errorf("block number %d not covered by any written QC: %w", n, types.ErrBlockMissingQC) } @@ -276,11 +216,11 @@ func (s *blockDB) WriteBlock(n types.GlobalBlockNumber, blk *types.Block) error return fmt.Errorf("failed to put block %d: %w", n, err) } - if !s.hasBlocks { - s.firstBlockNumber = n + if status.NextBlock == status.First { + setUnstartedFloor(&status, n) } - s.lastBlockNumber = n - s.hasBlocks = true + status.NextBlock = n + 1 + s.status = utils.Some(status) return nil } @@ -291,9 +231,10 @@ func (s *blockDB) WriteQC(qc *types.FullCommitQC) error { } s.mu.Lock() defer s.mu.Unlock() - if qc, ok := s.lastQC.Get(); ok && qc.QC().GlobalRange().Next != gr.First { + status, ok := s.status.Get() + if ok && status.NextQC != gr.First { return fmt.Errorf("QC starts at %d, expected %d: %w", - gr.First, qc.QC().GlobalRange().Next, types.ErrQCNonContiguous) + gr.First, status.NextQC, types.ErrQCNonContiguous) } value := encodeQC(qc) @@ -309,13 +250,21 @@ func (s *blockDB) WriteQC(qc *types.FullCommitQC) error { return fmt.Errorf("failed to put QC [%d,%d): %w", gr.First, gr.Next, err) } - if !s.lastQC.IsPresent() { + if !ok { // The first QC may start anywhere its caller allows, and nothing below it will ever // be written. Record where coverage begins so Iterator can clamp to it without // discovering it by scanning; a reopen re-derives the same value. - s.oldestQCStart = gr.First + status = types.DBStatus{ + First: gr.First, + NextAppQC: gr.First, + NextAppProposal: gr.First, + NextBlock: gr.First, + NextQC: gr.Next, + } + } else { + status.NextQC = gr.Next } - s.lastQC = utils.Some(qc) + s.status = utils.Some(status) return nil } @@ -326,19 +275,15 @@ func (s *blockDB) WriteAppQC(appQC *types.AppQC) error { } s.mu.Lock() defer s.mu.Unlock() - if !s.lastQC.IsPresent() { + status, ok := s.status.Get() + if !ok { return fmt.Errorf("AppQC [%d,%d) has no matching QC: %w", gr.First, gr.Next, types.ErrAppQCMissingQC) } - if lastAppQC, ok := s.lastAppQC.Get(); ok { - if want := lastAppQC.Proposal().GlobalRange().Next; want != gr.First { - return fmt.Errorf("AppQC starts at %d, expected %d: %w", - gr.First, want, types.ErrAppQCNonContiguous) - } - } else if gr.First != s.oldestQCStart { - return fmt.Errorf("first AppQC starts at %d, expected retained QC floor %d: %w", - gr.First, s.oldestQCStart, types.ErrAppQCNonContiguous) + if status.NextAppQC != gr.First { + return fmt.Errorf("AppQC starts at %d, expected %d: %w", + gr.First, status.NextAppQC, types.ErrAppQCNonContiguous) } - if lastAppProposal, ok := s.lastAppProposal.Get(); !ok || gr.Next > lastAppProposal.GlobalRange().Next { + if gr.Next > status.NextAppProposal { return fmt.Errorf("AppQC [%d,%d) is not covered by written AppProposals: %w", gr.First, gr.Next, types.ErrAppQCMissingQC) } @@ -354,7 +299,9 @@ func (s *blockDB) WriteAppQC(appQC *types.AppQC) error { if err := s.table.Put(appQCKey(gr.First), value, aliases...); err != nil { return fmt.Errorf("failed to put AppQC [%d,%d): %w", gr.First, gr.Next, err) } - s.lastAppQC = utils.Some(appQC) + status.NextAppQC = gr.Next + status.First = status.NextAppQC - 1 + s.status = utils.Some(status) return nil } @@ -365,19 +312,15 @@ func (s *blockDB) WriteAppProposal(appProposal *types.AppProposal) error { } s.mu.Lock() defer s.mu.Unlock() - if !s.lastQC.IsPresent() { + status, ok := s.status.Get() + if !ok { return fmt.Errorf("AppProposal [%d,%d) has no matching QC: %w", gr.First, gr.Next, types.ErrAppProposalMissingQC) } - if lastAppProposal, ok := s.lastAppProposal.Get(); ok { - if want := lastAppProposal.GlobalRange().Next; want != gr.First { - return fmt.Errorf("AppProposal starts at %d, expected %d: %w", - gr.First, want, types.ErrAppProposalNonContiguous) - } - } else if gr.First != s.oldestQCStart { - return fmt.Errorf("first AppProposal starts at %d, expected retained QC floor %d: %w", - gr.First, s.oldestQCStart, types.ErrAppProposalNonContiguous) + if status.NextAppProposal != gr.First { + return fmt.Errorf("AppProposal starts at %d, expected %d: %w", + gr.First, status.NextAppProposal, types.ErrAppProposalNonContiguous) } - if !s.hasBlocks || gr.Next > s.lastBlockNumber+1 { + if gr.Next > status.NextBlock { return fmt.Errorf("AppProposal [%d,%d) is not covered by written blocks: %w", gr.First, gr.Next, types.ErrAppProposalMissingQC) } value := encodeAppProposal(appProposal) @@ -393,7 +336,8 @@ func (s *blockDB) WriteAppProposal(appProposal *types.AppProposal) error { return fmt.Errorf("failed to put AppProposal [%d,%d): %w", gr.First, gr.Next, err) } - s.lastAppProposal = utils.Some(appProposal) + status.NextAppProposal = gr.Next + s.status = utils.Some(status) return nil } @@ -401,7 +345,7 @@ func (s *blockDB) PruneBefore(blockHeight types.GlobalBlockNumber) error { s.mu.Lock() defer s.mu.Unlock() - status, ok := s.statusLocked().Get() + status, ok := s.status.Get() if !ok { // Ignore prune requests if we've not got any data yet. Simplifies several edge cases // and is technically a legal implementation of the contract in the godocs. @@ -473,7 +417,7 @@ func (s *blockDB) Flush() error { func (s *blockDB) Status() types.DBStatus { s.mu.Lock() defer s.mu.Unlock() - return s.statusLocked().Or(types.DBStatus{ + return s.status.Or(types.DBStatus{ First: 0, NextBlock: 0, NextQC: 0, @@ -482,53 +426,38 @@ func (s *blockDB) Status() types.DBStatus { }) } -func (s *blockDB) statusLocked() utils.Option[types.DBStatus] { - qc, ok := s.lastQC.Get() - if !ok { - return utils.None[types.DBStatus]() - } - first := s.oldestQCStart - if s.hasBlocks { - first = max(first, s.firstBlockNumber) - } - status := types.DBStatus{ - First: first, - NextAppQC: first, - NextAppProposal: first, - NextBlock: first, - NextQC: qc.QC().GlobalRange().Next, - } - if s.hasBlocks { - status.NextBlock = s.lastBlockNumber + 1 - } - if appQC, ok := s.lastAppQC.Get(); ok { - status.NextAppQC = appQC.Proposal().GlobalRange().Next - status.First = status.NextAppQC - 1 - } - if appProposal, ok := s.lastAppProposal.Get(); ok { - status.NextAppProposal = appProposal.GlobalRange().Next - } - return utils.Some(status) -} - // ReadRecent() reads the latest AppQC/AppProposal recovery suffix. // WARNING: ReadRecent() will return an error if watermark is moved during iteration. func (s *blockDB) ReadRecent() (types.RecentData, error) { - s.mu.Lock() - status, ok := s.statusLocked().Get() - s.mu.Unlock() + recent, err := s.readRecent() + if err != nil { + return types.RecentData{}, err + } + status, ok := recent.Status.Get() if !ok { - return types.RecentData{}, nil + return recent, nil + } + // Safety check: if watermark has been moved and GC happened to get executed during iteration, + // the loaded data might be inconsistent with the targetFloor we computed. + if current := s.Status(); current.First != status.First { + return types.RecentData{}, fmt.Errorf("watermark has moved while iterating: recovered status %+v, current status %+v", status, current) } - targetFloor := status.First + return recent, nil +} - // Collect data >= targetFloor. +func (s *blockDB) readRecent() (types.RecentData, error) { it, err := s.table.Iterator(true) if err != nil { return types.RecentData{}, fmt.Errorf("failed to open recent-data iterator: %w", err) } defer func() { _ = it.Close() }() - recent := types.RecentData{Status: utils.Some(status)} + var recent types.RecentData + var recoveredStatus types.DBStatus + var oldestQCStart types.GlobalBlockNumber + var oldestBlock types.GlobalBlockNumber + var anchorFloor types.GlobalBlockNumber + var gotBlock, gotQC, gotAppProposal, gotAppQC bool + var gotAnchorQC bool for { ok, err := it.Next() if err != nil { @@ -554,42 +483,76 @@ func (s *blockDB) ReadRecent() (types.RecentData, error) { if err != nil { return types.RecentData{}, fmt.Errorf("failed to decode recent block: %w", err) } - if targetFloor <= n { - recent.Blocks = append(recent.Blocks, types.RecentBlock{Number: n, Block: block}) + if !gotBlock { + recoveredStatus.NextBlock = n + 1 + gotBlock = true } + oldestBlock = n + recent.Blocks = append(recent.Blocks, types.RecentBlock{Number: n, Block: block}) case kindAppQC: appQC, err := decodeAppQC(value) if err != nil { return types.RecentData{}, fmt.Errorf("failed to decode recent AppQC: %w", err) } gr := appQC.Proposal().GlobalRange() - if targetFloor < gr.Next { - recent.AppQCs = append(recent.AppQCs, appQC) + if !gotAppQC { + recoveredStatus.NextAppQC = gr.Next + recoveredStatus.First = gr.Next - 1 + anchorFloor = recoveredStatus.First + gotAnchorQC = commitQCCovers(recent.CommitQCs, anchorFloor) + gotAppQC = true } + recent.AppQCs = append(recent.AppQCs, appQC) case kindAppProp: appProposal, err := decodeAppProposal(value) if err != nil { return types.RecentData{}, fmt.Errorf("failed to decode recent AppProposal: %w", err) } gr := appProposal.GlobalRange() - if targetFloor < gr.Next { - recent.AppProposals = append(recent.AppProposals, appProposal) + if !gotAppProposal { + recoveredStatus.NextAppProposal = gr.Next + gotAppProposal = true } + recent.AppProposals = append(recent.AppProposals, appProposal) case kindQC: qc, err := decodeQC(value) if err != nil { return types.RecentData{}, fmt.Errorf("failed to decode recent CommitQC: %w", err) } - if targetFloor < qc.QC().GlobalRange().Next { - recent.CommitQCs = append(recent.CommitQCs, qc) + gr := qc.QC().GlobalRange() + if !gotQC { + recoveredStatus.NextQC = gr.Next + gotQC = true + } + oldestQCStart = gr.First + if gotAppQC && gr.First <= anchorFloor && anchorFloor < gr.Next { + gotAnchorQC = true } + recent.CommitQCs = append(recent.CommitQCs, qc) default: } + if gotAppQC && gotAnchorQC { + break + } } - // Safety check: if watermark has been moved and GC happened to get executed during iteration, - // the loaded data might be inconsistent with the targetFloor we computed. - if newFloor := s.Status().First; newFloor != targetFloor { - return types.RecentData{}, fmt.Errorf("watermark has moved while iterating") + if gotQC { + if !gotAppQC { + recoveredStatus.First = oldestQCStart + if gotBlock { + recoveredStatus.First = max(recoveredStatus.First, oldestBlock) + } + } + if !gotBlock { + recoveredStatus.NextBlock = recoveredStatus.First + } + if !gotAppProposal { + recoveredStatus.NextAppProposal = recoveredStatus.First + } + if !gotAppQC { + recoveredStatus.NextAppQC = recoveredStatus.First + } + recent.Status = utils.Some(recoveredStatus) + filterRecent(&recent, recoveredStatus.First) } slices.Reverse(recent.CommitQCs) slices.Reverse(recent.Blocks) @@ -598,6 +561,31 @@ func (s *blockDB) ReadRecent() (types.RecentData, error) { return recent, nil } +func commitQCCovers(qcs []*types.FullCommitQC, n types.GlobalBlockNumber) bool { + for _, qc := range qcs { + gr := qc.QC().GlobalRange() + if gr.First <= n && n < gr.Next { + return true + } + } + return false +} + +func filterRecent(recent *types.RecentData, floor types.GlobalBlockNumber) { + recent.CommitQCs = slices.DeleteFunc(recent.CommitQCs, func(qc *types.FullCommitQC) bool { + return qc.QC().GlobalRange().Next <= floor + }) + recent.Blocks = slices.DeleteFunc(recent.Blocks, func(block types.RecentBlock) bool { + return block.Number < floor + }) + recent.AppProposals = slices.DeleteFunc(recent.AppProposals, func(appProposal *types.AppProposal) bool { + return appProposal.GlobalRange().Next <= floor + }) + recent.AppQCs = slices.DeleteFunc(recent.AppQCs, func(appQC *types.AppQC) bool { + return appQC.Proposal().GlobalRange().Next <= floor + }) +} + func (s *blockDB) ReadBlockByNumber(n types.GlobalBlockNumber) (utils.Option[*types.Block], error) { // Refuse below-watermark blocks: they may be stranded (covering QC reclaimed). if uint64(n) < s.watermark.Load() { diff --git a/sei-db/ledger_db/block/littblock/qc_reader.go b/sei-db/ledger_db/block/littblock/qc_reader.go deleted file mode 100644 index 58e08f62a3..0000000000 --- a/sei-db/ledger_db/block/littblock/qc_reader.go +++ /dev/null @@ -1,31 +0,0 @@ -package littblock - -import ( - "fmt" - - "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" -) - -// qcReader is the slice of littdb.Table that readQCCovering needs, so tests can -// supply QCs without building a table. -type qcReader interface { - Get(key []byte) ([]byte, bool, error) -} - -// readQCCovering point-reads and decodes the QC covering n. Every covered -// number carries a QC alias key holding the full QC value, so any number inside -// a retained range resolves. -func readQCCovering(table qcReader, n types.GlobalBlockNumber) (*types.FullCommitQC, error) { - value, exists, err := table.Get(qcKey(n)) - if err != nil { - return nil, fmt.Errorf("failed to read covering QC for %d: %w", n, err) - } - if !exists { - return nil, fmt.Errorf("corrupt store: no QC record at %d despite coverage past it", n) - } - qc, err := decodeQC(value) - if err != nil { - return nil, fmt.Errorf("failed to decode covering QC for %d: %w", n, err) - } - return qc, nil -} From febf53e88dc6bc2d6dd6cae43bc0803b6f6b1273 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 7 Aug 2026 21:33:20 +0200 Subject: [PATCH 35/61] fix from codex --- .../internal/autobahn/avail/subscriptions.go | 10 ++++--- .../internal/autobahn/data/state.go | 27 +++++++++---------- .../internal/autobahn/data/state_test.go | 7 +++-- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/sei-tendermint/internal/autobahn/avail/subscriptions.go b/sei-tendermint/internal/autobahn/avail/subscriptions.go index a517e5d282..51b0f1e532 100644 --- a/sei-tendermint/internal/autobahn/avail/subscriptions.go +++ b/sei-tendermint/internal/autobahn/avail/subscriptions.go @@ -76,16 +76,20 @@ type AppVotesRecv struct { } func (s *State) SubscribeAppVotes() *AppVotesRecv { - return &AppVotesRecv{s, 0} + return &AppVotesRecv{s, s.data.First()} } func (r *AppVotesRecv) Recv(ctx context.Context) (*types.Signed[*types.AppVote], error) { for { - vote, qc, err := r.state.data.AppVote(ctx, r.next) + vote, err := r.state.data.AppVote(ctx, r.next) if err != nil { + if errors.Is(err, types.ErrPruned) { + r.next = max(r.next, r.state.data.First()) + continue + } return nil, err } - r.next = qc.QC().GlobalRange().Next + r.next = vote.Proposal().GlobalRange().Next return types.Sign(r.state.key, vote), nil } } diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index d5f7ddb306..107a2f3976 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -264,6 +264,13 @@ func loadFromBlockDB(cfg *Config, blockDB types.BlockDB) (*inner, error) { return inner, nil } +func (s *State) First() types.GlobalBlockNumber { + for inner := range s.inner.Lock() { + return inner.first + } + panic("unreachable") +} + // Registry returns the epoch registry. func (s *State) Registry() *epoch.Registry { return s.cfg.Registry } @@ -616,12 +623,15 @@ func (s *State) PushAppHash(ctx context.Context, n types.GlobalBlockNumber, hash // AppVote returns an appVote for a block >= n. // Vote is available ONLY once AppHash has been pushed AND persisted. // This prevents any possible equivocation and ensures that local node has the executed blocks persisted (since nextAppProposa <= nextBlock). -func (s *State) AppVote(ctx context.Context, n types.GlobalBlockNumber) (*types.AppVote, *types.FullCommitQC, error) { +func (s *State) AppVote(ctx context.Context, n types.GlobalBlockNumber) (*types.AppVote, error) { for inner, ctrl := range s.inner.Lock() { if err := ctrl.WaitUntil(ctx, func() bool { return n < inner.persisted.NextAppProposal }); err != nil { - return nil, nil, err + return nil, err } - return types.NewAppVote(inner.appProposals[n]), inner.qcs[n], nil + if n < inner.first { + return nil, types.ErrPruned + } + return types.NewAppVote(inner.appProposals[n]), nil } panic("unreachable") } @@ -681,17 +691,6 @@ func (s *State) Anchor() utils.AtomicRecv[utils.Option[Anchor]] { panic("unreachable") } -func (s *State) LastAppQC() (*types.AppQC, *types.FullCommitQC) { - for inner := range s.inner.Lock() { - if inner.nextAppQC <= inner.first { - return nil, nil - } - n := inner.nextAppQC - 1 - return inner.appQCs[n], inner.qcs[n] - } - panic("unreachable") -} - func (i *inner) nextToExecute(lane types.LaneID) types.BlockNumber { // TODO(gprusak): decide whether 0 is a good result in this case in general. // Empty maps (first == nextQC) only on fresh start / after skipTo with no QC. diff --git a/sei-tendermint/internal/autobahn/data/state_test.go b/sei-tendermint/internal/autobahn/data/state_test.go index 06698a6a61..b95c567e61 100644 --- a/sei-tendermint/internal/autobahn/data/state_test.go +++ b/sei-tendermint/internal/autobahn/data/state_test.go @@ -109,7 +109,7 @@ func pushAppHashesRunning(ctx context.Context, state *State, rng utils.Rng, firs } func pushAppQCForBlock(ctx context.Context, state *State, keys []types.SecretKey, n types.GlobalBlockNumber) error { - vote, _, err := state.AppVote(ctx, n) + vote, err := state.AppVote(ctx, n) if err != nil { return err } @@ -753,9 +753,8 @@ func TestPushAppQCPersistsAndRecovers(t *testing.T) { require.Equal(t, gr1.Next, inner.nextAppProposal) require.Equal(t, gr1.Next, inner.nextAppQC) } - appQC, fQC := state2.LastAppQC() - require.NotNil(t, appQC) - require.NotNil(t, fQC) + appQC, fQC, err := state2.AppQC(ctx, gr1.First) + require.NoError(t, err) require.Equal(t, gr1, appQC.Proposal().GlobalRange()) require.Equal(t, gr1, fQC.QC().GlobalRange()) require.NoError(t, db2.Close()) From ddddac9d359ce1b2ed07c72c2f97b55f6bebdd03 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 7 Aug 2026 22:09:43 +0200 Subject: [PATCH 36/61] simplified readRecent() --- .../block/littblock/litt_block_db.go | 174 ++++++------------ 1 file changed, 52 insertions(+), 122 deletions(-) diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index b91514254a..c1ec3cbd35 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -80,16 +80,14 @@ func NewBlockDB(config *LittBlockConfig) (types.BlockDB, error) { _ = db.Close() return nil, fmt.Errorf("failed to build ledger table: %w", err) } - s.table = table - - if err := s.recoverCursors(); err != nil { + if err := s.recoverWatermark(); err != nil { _ = db.Close() - return nil, fmt.Errorf("failed to recover write cursors: %w", err) + return nil, fmt.Errorf("failed to recover watermark: %w", err) } - if err := s.recoverReadFloors(); err != nil { + if err := s.recoverCursors(); err != nil { _ = db.Close() - return nil, fmt.Errorf("failed to recover read floors: %w", err) + return nil, fmt.Errorf("failed to recover write cursors: %w", err) } return s, nil } @@ -102,14 +100,7 @@ func (s *blockDB) recoverCursors() error { if err != nil { return fmt.Errorf("read recent data: %w", err) } - status, ok := recent.Status.Get() - if !ok { - if len(recent.Blocks) > 0 { - return fmt.Errorf("corrupt store: newest block %d has no surviving QC covering it", recent.Blocks[len(recent.Blocks)-1].Number) - } - return nil - } - s.status = utils.Some(status) + s.status = recent.Status return nil } @@ -123,7 +114,7 @@ func (s *blockDB) recoverCursors() error { // The block search is skipped when the store holds no blocks — status.NextBlock // comes from recoverCursors, which runs first — so a QC-only store does not walk // the whole table looking for a block that is not there. -func (s *blockDB) recoverReadFloors() error { +func (s *blockDB) recoverWatermark() error { status, ok := s.status.Get() if !ok { return nil @@ -133,9 +124,7 @@ func (s *blockDB) recoverReadFloors() error { return fmt.Errorf("failed to open read floor recovery iterator: %w", err) } defer func() { _ = it.Close() }() - - needQC, needBlock := true, status.NextBlock != 0 - for needQC || needBlock { + for { ok, err := it.Next() if err != nil { return fmt.Errorf("failed to advance read floor recovery iterator: %w", err) @@ -150,53 +139,24 @@ func (s *blockDB) recoverReadFloors() error { if !isPrimary { continue } - switch keyKind(key) { - case kindQC: - if needQC { - oldest := decodeNumberKey(key) - s.watermark.Store(uint64(oldest)) - needQC = false - } - case kindBlock: - if needBlock { - needBlock = false - } + if keyKind(key) != kindQC { + continue } + s.watermark.Store(uint64(decodeNumberKey(key))) + return nil } - - if needQC && status.NextBlock != 0 { - // No QC survives. The never-empty prune invariant guarantees at least one - // (block, QC) pair is always retained, so blocks-without-QC is unreachable - // through normal operation — it means the store is corrupt (e.g. a QC WAL - // file was removed out of band). Refuse to open rather than serve blocks we - // can no longer trust. - return fmt.Errorf("corrupt store: newest block %d has no surviving QC covering it", status.NextBlock-1) - } - return nil -} - -func setUnstartedFloor(status *types.DBStatus, first types.GlobalBlockNumber) { - oldFirst := status.First - if status.NextAppQC == oldFirst { - status.NextAppQC = first - } - if status.NextAppProposal == oldFirst { - status.NextAppProposal = first - } - if status.NextBlock == oldFirst { - status.NextBlock = first - } - status.First = first + // No QC survives. The never-empty prune invariant guarantees at least one + // (block, QC) pair is always retained, so blocks-without-QC is unreachable + // through normal operation — it means the store is corrupt (e.g. a QC WAL + // file was removed out of band). Refuse to open rather than serve blocks we + // can no longer trust. + return fmt.Errorf("corrupt store: newest block %d has no surviving QC covering it", status.NextBlock-1) } func (s *blockDB) WriteBlock(n types.GlobalBlockNumber, blk *types.Block) error { s.mu.Lock() defer s.mu.Unlock() status, ok := s.status.Get() - if ok && status.NextBlock > status.First && n != status.NextBlock { - return fmt.Errorf("block number %d not contiguous with last written %d: %w", - n, status.NextBlock-1, types.ErrBlockOutOfOrder) - } // A covering QC must already be written. Since QCs are contiguous and blocks // strictly ascending, n is covered iff n < status.NextQC. This guard also fixes // the QC-before-block write order: the covering QC's Put has already issued @@ -204,6 +164,10 @@ func (s *blockDB) WriteBlock(n types.GlobalBlockNumber, blk *types.Block) error if !ok || n >= status.NextQC { return fmt.Errorf("block number %d not covered by any written QC: %w", n, types.ErrBlockMissingQC) } + if n != status.NextBlock { + return fmt.Errorf("block number %d not contiguous with last written %d: %w", + n, status.NextBlock-1, types.ErrBlockOutOfOrder) + } value := encodeBlock(n, blk) hash := blk.Header().Hash() @@ -215,10 +179,6 @@ func (s *blockDB) WriteBlock(n types.GlobalBlockNumber, blk *types.Block) error if err := s.table.Put(blockKey(n), value, hashAlias); err != nil { return fmt.Errorf("failed to put block %d: %w", n, err) } - - if status.NextBlock == status.First { - setUnstartedFloor(&status, n) - } status.NextBlock = n + 1 s.status = utils.Some(status) return nil @@ -452,13 +412,10 @@ func (s *blockDB) readRecent() (types.RecentData, error) { } defer func() { _ = it.Close() }() var recent types.RecentData - var recoveredStatus types.DBStatus - var oldestQCStart types.GlobalBlockNumber - var oldestBlock types.GlobalBlockNumber - var anchorFloor types.GlobalBlockNumber + var status types.DBStatus + var oldestQC *types.FullCommitQC var gotBlock, gotQC, gotAppProposal, gotAppQC bool - var gotAnchorQC bool - for { + for !gotAppQC || !gotQC || status.NextAppQC <= oldestQC.QC().GlobalRange().Next { ok, err := it.Next() if err != nil { return types.RecentData{}, fmt.Errorf("failed to advance recent-data iterator: %w", err) @@ -484,10 +441,9 @@ func (s *blockDB) readRecent() (types.RecentData, error) { return types.RecentData{}, fmt.Errorf("failed to decode recent block: %w", err) } if !gotBlock { - recoveredStatus.NextBlock = n + 1 + status.NextBlock = n + 1 gotBlock = true } - oldestBlock = n recent.Blocks = append(recent.Blocks, types.RecentBlock{Number: n, Block: block}) case kindAppQC: appQC, err := decodeAppQC(value) @@ -496,10 +452,8 @@ func (s *blockDB) readRecent() (types.RecentData, error) { } gr := appQC.Proposal().GlobalRange() if !gotAppQC { - recoveredStatus.NextAppQC = gr.Next - recoveredStatus.First = gr.Next - 1 - anchorFloor = recoveredStatus.First - gotAnchorQC = commitQCCovers(recent.CommitQCs, anchorFloor) + status.NextAppQC = gr.Next + status.First = gr.Next - 1 gotAppQC = true } recent.AppQCs = append(recent.AppQCs, appQC) @@ -510,7 +464,7 @@ func (s *blockDB) readRecent() (types.RecentData, error) { } gr := appProposal.GlobalRange() if !gotAppProposal { - recoveredStatus.NextAppProposal = gr.Next + status.NextAppProposal = gr.Next gotAppProposal = true } recent.AppProposals = append(recent.AppProposals, appProposal) @@ -521,69 +475,45 @@ func (s *blockDB) readRecent() (types.RecentData, error) { } gr := qc.QC().GlobalRange() if !gotQC { - recoveredStatus.NextQC = gr.Next + status.NextQC = gr.Next gotQC = true } - oldestQCStart = gr.First - if gotAppQC && gr.First <= anchorFloor && anchorFloor < gr.Next { - gotAnchorQC = true - } + oldestQC = qc recent.CommitQCs = append(recent.CommitQCs, qc) - default: - } - if gotAppQC && gotAnchorQC { - break - } - } - if gotQC { - if !gotAppQC { - recoveredStatus.First = oldestQCStart - if gotBlock { - recoveredStatus.First = max(recoveredStatus.First, oldestBlock) - } - } - if !gotBlock { - recoveredStatus.NextBlock = recoveredStatus.First - } - if !gotAppProposal { - recoveredStatus.NextAppProposal = recoveredStatus.First } - if !gotAppQC { - recoveredStatus.NextAppQC = recoveredStatus.First - } - recent.Status = utils.Some(recoveredStatus) - filterRecent(&recent, recoveredStatus.First) } - slices.Reverse(recent.CommitQCs) - slices.Reverse(recent.Blocks) - slices.Reverse(recent.AppProposals) - slices.Reverse(recent.AppQCs) - return recent, nil -} - -func commitQCCovers(qcs []*types.FullCommitQC, n types.GlobalBlockNumber) bool { - for _, qc := range qcs { - gr := qc.QC().GlobalRange() - if gr.First <= n && n < gr.Next { - return true - } + if !gotQC { + // Empty db. + return types.RecentData{}, nil } - return false -} + // Set fields for missing resources. + first := oldestQC.QC().GlobalRange().First + status.NextQC = max(status.NextQC, first) + status.NextBlock = max(status.NextBlock, first) + status.NextAppProposal = max(status.NextAppProposal, first) + status.NextAppQC = max(status.NextAppQC, first) + status.First = max(status.First, first) + recent.Status = utils.Some(status) -func filterRecent(recent *types.RecentData, floor types.GlobalBlockNumber) { + // Prune resources fully below status.First. recent.CommitQCs = slices.DeleteFunc(recent.CommitQCs, func(qc *types.FullCommitQC) bool { - return qc.QC().GlobalRange().Next <= floor + return qc.QC().GlobalRange().Next <= status.First }) recent.Blocks = slices.DeleteFunc(recent.Blocks, func(block types.RecentBlock) bool { - return block.Number < floor + return block.Number < status.First }) recent.AppProposals = slices.DeleteFunc(recent.AppProposals, func(appProposal *types.AppProposal) bool { - return appProposal.GlobalRange().Next <= floor + return appProposal.GlobalRange().Next <= status.First }) recent.AppQCs = slices.DeleteFunc(recent.AppQCs, func(appQC *types.AppQC) bool { - return appQC.Proposal().GlobalRange().Next <= floor + return appQC.Proposal().GlobalRange().Next <= status.First }) + // Put resources in increasing order. + slices.Reverse(recent.CommitQCs) + slices.Reverse(recent.Blocks) + slices.Reverse(recent.AppProposals) + slices.Reverse(recent.AppQCs) + return recent, nil } func (s *blockDB) ReadBlockByNumber(n types.GlobalBlockNumber) (utils.Option[*types.Block], error) { From 86bacf63a02bea683039fb42ded6945a2addd00f Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 7 Aug 2026 22:18:22 +0200 Subject: [PATCH 37/61] simplified --- sei-db/ledger_db/block/littblock/litt_block_db.go | 11 ++++++----- .../block/littblock/litt_block_stranding_test.go | 1 - 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index c1ec3cbd35..94b3d29035 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -115,15 +115,12 @@ func (s *blockDB) recoverCursors() error { // comes from recoverCursors, which runs first — so a QC-only store does not walk // the whole table looking for a block that is not there. func (s *blockDB) recoverWatermark() error { - status, ok := s.status.Get() - if !ok { - return nil - } it, err := s.table.Iterator(false) if err != nil { return fmt.Errorf("failed to open read floor recovery iterator: %w", err) } defer func() { _ = it.Close() }() + empty := true for { ok, err := it.Next() if err != nil { @@ -132,6 +129,7 @@ func (s *blockDB) recoverWatermark() error { if !ok { break } + empty = false key, isPrimary, err := it.GetKey() if err != nil { return fmt.Errorf("failed to read read floor recovery key: %w", err) @@ -150,7 +148,10 @@ func (s *blockDB) recoverWatermark() error { // through normal operation — it means the store is corrupt (e.g. a QC WAL // file was removed out of band). Refuse to open rather than serve blocks we // can no longer trust. - return fmt.Errorf("corrupt store: newest block %d has no surviving QC covering it", status.NextBlock-1) + if !empty { + return fmt.Errorf("corrupt store: no QC in non-empty store") + } + return nil } func (s *blockDB) WriteBlock(n types.GlobalBlockNumber, blk *types.Block) error { diff --git a/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go b/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go index e0b6a947ed..028db28432 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go +++ b/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go @@ -370,7 +370,6 @@ func TestLittblockRefusesToOpenWithStrandedBlocks(t *testing.T) { // Reopen: recovery finds the block but no QC, so it refuses to open. _, err = NewBlockDB(strandingConfig(t, dir, 8)) require.Error(t, err) - require.ErrorContains(t, err, "no surviving QC") } // TestLittblockEmptyStorePruneDoesNotReclaimLaterWrites is the regression for the From b59451f41bca79fda9d96e5d088cd6830c4141f1 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 7 Aug 2026 22:27:38 +0200 Subject: [PATCH 38/61] removed useless tests --- .../autobahn/data/state_recovery_test.go | 50 ------------------- .../internal/autobahn/data/state_test.go | 34 +------------ 2 files changed, 2 insertions(+), 82 deletions(-) diff --git a/sei-tendermint/internal/autobahn/data/state_recovery_test.go b/sei-tendermint/internal/autobahn/data/state_recovery_test.go index 01de2c61b2..590852b302 100644 --- a/sei-tendermint/internal/autobahn/data/state_recovery_test.go +++ b/sei-tendermint/internal/autobahn/data/state_recovery_test.go @@ -358,56 +358,6 @@ func TestRecoveryBlocksBehind(t *testing.T) { require.Equal(t, gr2.Next, state2.NextBlock()) } -// TestRecoveryPartialQCPrefix verifies recovery from a store whose blocks begin partway into -// their covering QC. The first block may be written anywhere inside that QC (see -// types.BlockDB.WriteBlock), so BlockDB.Iterator opens on it and the recovery floor follows — -// landing on the first present block, not on the QC's start. Flooring at the QC start would -// leave the blockless prefix inside [first, nextBlock), which inner's density invariant forbids. -func TestRecoveryPartialQCPrefix(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 3) - dir := t.TempDir() - - qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) - gr1 := qc1.QC().GlobalRange() - require.True(t, gr1.Next-gr1.First >= 3, "need at least 3 blocks in QC range to test split") - - // Write the QC for the full range, but write blocks only from mid onwards. - mid := gr1.First + (gr1.Next-gr1.First)/2 - db1 := newTestBlockDB(t, dir) - require.NoError(t, db1.WriteQC(qc1)) - for i, n := 0, gr1.First; n < gr1.Next; n++ { - if n >= mid { - require.NoError(t, db1.WriteBlock(n, blocks1[i])) - } - i++ - } - require.NoError(t, db1.Flush()) - require.NoError(t, db1.Close()) - - state2 := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, dir)) - - // The floor is the first present block, and the contiguous prefix runs from there to the - // end of the QC's coverage. - require.Equal(t, gr1.Next, state2.NextBlock()) - for inner := range state2.inner.Lock() { - require.Equal(t, mid, inner.first, "floor must be the first present block") - require.Equal(t, mid, inner.nextAppProposal) - require.Equal(t, gr1.Next, inner.nextQC) - } - - // Nothing was ever written below the floor, so those heights are not served. - for n := gr1.First; n < mid; n++ { - _, err := state2.TryBlock(n) - require.ErrorIs(t, err, types.ErrPruned) - } - for n := mid; n < gr1.Next; n++ { - got, err := state2.TryBlock(n) - require.NoError(t, err) - require.NotNil(t, got) - } -} - // TestRecoveryAfterPruneNoGC verifies that restarting before async GC reclaims // pruned entries does not cause NewState to fail. Blocks and QCs share the same // GC filter in littblock, so below-watermark blocks never survive past their diff --git a/sei-tendermint/internal/autobahn/data/state_test.go b/sei-tendermint/internal/autobahn/data/state_test.go index b95c567e61..cbd5be6437 100644 --- a/sei-tendermint/internal/autobahn/data/state_test.go +++ b/sei-tendermint/internal/autobahn/data/state_test.go @@ -761,9 +761,8 @@ func TestPushAppQCPersistsAndRecovers(t *testing.T) { } // TestPruningKeepsLastQCRange verifies BlockDB's never-empty prune: asking to -// prune past the tip still leaves the newest cohort readable. A QC retaining -// only a suffix of its blocks recovers with the floor on that suffix; a -// consistent range recovers from the QC start. +// prune past the tip still leaves the newest cohort readable, and a consistent +// range recovers from the QC start. func TestPruningKeepsLastQCRange(t *testing.T) { ctx := t.Context() rng := utils.TestRng() @@ -784,21 +783,6 @@ func TestPruningKeepsLastQCRange(t *testing.T) { require.NotNil(t, got) } - // A QC covering a range with only its last block present: the first block is free to - // start inside its covering QC, so iteration opens there and the recovery floor follows. - survivor := gr1.Next - 1 - dirSuffix := t.TempDir() - dbSuffix := newTestBlockDB(t, dirSuffix) - require.NoError(t, dbSuffix.WriteQC(qc1)) - require.NoError(t, dbSuffix.WriteBlock(survivor, blocks1[survivor-gr1.First])) - require.NoError(t, dbSuffix.Flush()) - require.NoError(t, dbSuffix.Close()) - suffixState := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, dirSuffix)) - require.Equal(t, gr1.Next, suffixState.NextBlock()) - for inner := range suffixState.inner.Lock() { - require.Equal(t, survivor, inner.first, "floor must be the surviving block") - } - // Consistent post-GC shape: full QC range of blocks. Restart recovers at QC start. dir := t.TempDir() db := newTestBlockDB(t, dir) @@ -897,20 +881,6 @@ func TestPruningWithPartialQCRange(t *testing.T) { require.Equal(t, n, gb.GlobalNumber) } - // A lone qc2 suffix: iteration opens on the surviving block and the floor follows. - survivor := gr2.Next - 1 - dirSuffix := t.TempDir() - dbSuffix := newTestBlockDB(t, dirSuffix) - require.NoError(t, dbSuffix.WriteQC(qc2)) - require.NoError(t, dbSuffix.WriteBlock(survivor, blocks2[survivor-gr2.First])) - require.NoError(t, dbSuffix.Flush()) - require.NoError(t, dbSuffix.Close()) - suffixState := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, dirSuffix)) - require.Equal(t, gr2.Next, suffixState.NextBlock()) - for inner := range suffixState.inner.Lock() { - require.Equal(t, survivor, inner.first, "floor must be the surviving block") - } - // Consistent retained range: full qc2. dir := t.TempDir() db := newTestBlockDB(t, dir) From 6b678035329f72a9dd33bc31b73a67bcf90c7cdf Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 7 Aug 2026 22:36:24 +0200 Subject: [PATCH 39/61] simplified inmem blockDB --- .../ledger_db/block/memblock/mem_block_db.go | 524 +++++++----------- 1 file changed, 185 insertions(+), 339 deletions(-) diff --git a/sei-db/ledger_db/block/memblock/mem_block_db.go b/sei-db/ledger_db/block/memblock/mem_block_db.go index 3dd83d0bd6..6dad1b8325 100644 --- a/sei-db/ledger_db/block/memblock/mem_block_db.go +++ b/sei-db/ledger_db/block/memblock/mem_block_db.go @@ -2,8 +2,6 @@ package memblock import ( "fmt" - "slices" - "sort" "sync" "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" @@ -12,284 +10,199 @@ import ( var _ types.BlockDB = (*blockDB)(nil) -// qcEntry pairs a QC with the half-open range [lower, upper) it covers, as -// derived from the QC itself by coveredRange. -type qcEntry struct { - qc *types.FullCommitQC - lower types.GlobalBlockNumber - upper types.GlobalBlockNumber -} - -// appQCEntry pairs an AppQC with the half-open range [lower, upper) it covers. -type appQCEntry struct { - appQC *types.AppQC - lower types.GlobalBlockNumber - upper types.GlobalBlockNumber -} - -// appProposalEntry pairs an AppProposal with the half-open range [lower, upper) -// it covers. -type appProposalEntry struct { - appProposal *types.AppProposal - lower types.GlobalBlockNumber - upper types.GlobalBlockNumber -} - -// hashEntry pairs a block with its GlobalBlockNumber so ReadBlockByHash can -// return the number, mirroring the littblock implementation which embeds it in -// the stored value. type hashEntry struct { blk *types.Block n types.GlobalBlockNumber } -// blockDB is an in-memory types.BlockDB. It holds blocks and QCs by pointer (no -// marshaling) and is intended as a test/benchmark fixture, not a durable -// implementation. +// blockDB is an in-memory types.BlockDB. It mirrors littblock's write/status +// rules, but stores already-decoded values in maps instead of LittDB records. type blockDB struct { - mu sync.RWMutex - byNumber map[types.GlobalBlockNumber]*types.Block - byHash map[types.BlockHeaderHash]hashEntry - qcsByLower map[types.GlobalBlockNumber]qcEntry - appQCs map[types.GlobalBlockNumber]appQCEntry - appProps map[types.GlobalBlockNumber]appProposalEntry - - // Write-order cursors (see types.BlockDB contract). - hasBlocks bool - lastBlockNumber types.GlobalBlockNumber - hasQC bool - lastQCNext types.GlobalBlockNumber - hasAppProposal bool - lastAppPropNext types.GlobalBlockNumber - hasAppQC bool - lastAppQCNext types.GlobalBlockNumber - - // latestQCStartBlock is the most recently written QC's starting block number — - // the lowest block number in the newest cohort. PruneBefore clamps to it (see - // littblock). - latestQCStartBlock types.GlobalBlockNumber - - // latestAppQCStartBlock is the most recently written AppQC's starting - // block number. When AppQCs exist, PruneBefore clamps to it so the newest - // AppQC cohort remains readable together with its CommitQC and blocks. - latestAppQCStartBlock types.GlobalBlockNumber - - // latestAppProposalStartBlock is the most recently written AppProposal's - // starting block number. - latestAppProposalStartBlock types.GlobalBlockNumber - - // firstBlockNumber is the lowest block number written. Meaningful only while hasBlocks. - firstBlockNumber types.GlobalBlockNumber - - // watermark is the (clamped) retention floor set by PruneBefore. Reads - // strictly below it are refused with types.ErrPruned; because pruned entries - // are deleted eagerly, this is the only record of where the floor sits and - // so the only way to tell a pruned block from one never written. + mu sync.RWMutex + + blocksByNumber map[types.GlobalBlockNumber]*types.Block + blocksByHash map[types.BlockHeaderHash]hashEntry + + qcsByBlock map[types.GlobalBlockNumber]*types.FullCommitQC + + appProposalsByBlock map[types.GlobalBlockNumber]*types.AppProposal + + appQCsByBlock map[types.GlobalBlockNumber]*types.AppQC + watermark types.GlobalBlockNumber + status utils.Option[types.DBStatus] } // NewBlockDB returns an in-memory types.BlockDB. func NewBlockDB() types.BlockDB { return &blockDB{ - byNumber: make(map[types.GlobalBlockNumber]*types.Block), - byHash: make(map[types.BlockHeaderHash]hashEntry), - qcsByLower: make(map[types.GlobalBlockNumber]qcEntry), - appQCs: make(map[types.GlobalBlockNumber]appQCEntry), - appProps: make(map[types.GlobalBlockNumber]appProposalEntry), + blocksByNumber: make(map[types.GlobalBlockNumber]*types.Block), + blocksByHash: make(map[types.BlockHeaderHash]hashEntry), + qcsByBlock: make(map[types.GlobalBlockNumber]*types.FullCommitQC), + appProposalsByBlock: make(map[types.GlobalBlockNumber]*types.AppProposal), + appQCsByBlock: make(map[types.GlobalBlockNumber]*types.AppQC), } } func (s *blockDB) WriteBlock(n types.GlobalBlockNumber, blk *types.Block) error { s.mu.Lock() defer s.mu.Unlock() - if s.hasBlocks && n != s.lastBlockNumber+1 { - return fmt.Errorf("block number %d not contiguous with last written %d: %w", - n, s.lastBlockNumber, types.ErrBlockOutOfOrder) - } - // A covering QC must already be written. QCs are contiguous and blocks - // strictly ascending, so n is covered iff n < lastQCNext. - if !s.hasQC || n >= s.lastQCNext { - return fmt.Errorf("block number %d not covered by any written QC (next QC bound %d): %w", - n, s.lastQCNext, types.ErrBlockMissingQC) + + status, ok := s.status.Get() + if !ok || n >= status.NextQC { + return fmt.Errorf("block number %d not covered by any written QC: %w", n, types.ErrBlockMissingQC) } - s.byNumber[n] = blk - s.byHash[blk.Header().Hash()] = hashEntry{blk: blk, n: n} - if !s.hasBlocks { - s.firstBlockNumber = n + if n != status.NextBlock { + return fmt.Errorf("block number %d not contiguous with last written %d: %w", + n, status.NextBlock-1, types.ErrBlockOutOfOrder) } - s.lastBlockNumber = n - s.hasBlocks = true - return nil -} -// coveredRange returns the half-open global block number range the QC covers, -// as specified by types.BlockDB.WriteQC: [First, First+len(Headers())). Derived -// identically in littblock — see the comment there for why the bound comes from -// the header count rather than from GlobalRange().Next. -func coveredRange(qc *types.FullCommitQC) (types.GlobalBlockNumber, types.GlobalBlockNumber) { - first := qc.QC().GlobalRange().First - return first, first + types.GlobalBlockNumber(len(qc.Headers())) + s.blocksByNumber[n] = blk + s.blocksByHash[blk.Header().Hash()] = hashEntry{blk: blk, n: n} + status.NextBlock = n + 1 + s.status = utils.Some(status) + return nil } func (s *blockDB) WriteQC(qc *types.FullCommitQC) error { - first, next := coveredRange(qc) - if first >= next { - return fmt.Errorf("QC at %d covers no blocks: %w", first, types.ErrQCNonContiguous) + gr := qc.QC().GlobalRange() + if gr.Len() == 0 { + return fmt.Errorf("QC at %d covers no blocks: %w", gr.First, types.ErrQCNonContiguous) } + s.mu.Lock() defer s.mu.Unlock() - if s.hasQC && first != s.lastQCNext { + + status, ok := s.status.Get() + if ok && status.NextQC != gr.First { return fmt.Errorf("QC starts at %d, expected %d: %w", - first, s.lastQCNext, types.ErrQCNonContiguous) + gr.First, status.NextQC, types.ErrQCNonContiguous) } - s.qcsByLower[first] = qcEntry{qc: qc, lower: first, upper: next} - s.latestQCStartBlock = first - s.lastQCNext = next - s.hasQC = true - return nil -} -func appQCRange(appQC *types.AppQC) (types.GlobalBlockNumber, types.GlobalBlockNumber) { - return appProposalRange(appQC.Proposal()) -} + for n := gr.First; n < gr.Next; n++ { + s.qcsByBlock[n] = qc + } -func appProposalRange(appProposal *types.AppProposal) (types.GlobalBlockNumber, types.GlobalBlockNumber) { - gr := appProposal.GlobalRange() - return gr.First, gr.Next + if !ok { + status = types.DBStatus{ + First: gr.First, + NextAppQC: gr.First, + NextAppProposal: gr.First, + NextBlock: gr.First, + NextQC: gr.Next, + } + } else { + status.NextQC = gr.Next + } + s.status = utils.Some(status) + return nil } func (s *blockDB) WriteAppProposal(appProposal *types.AppProposal) error { - first, next := appProposalRange(appProposal) - if first >= next { - return fmt.Errorf("AppProposal at %d covers no blocks: %w", first, types.ErrAppProposalNonContiguous) + gr := appProposal.GlobalRange() + if gr.Len() == 0 { + return fmt.Errorf("AppProposal at %d covers no blocks: %w", gr.First, types.ErrAppProposalNonContiguous) } + s.mu.Lock() defer s.mu.Unlock() - if !s.hasQC { - return fmt.Errorf("AppProposal [%d,%d) has no matching QC: %w", first, next, types.ErrAppProposalMissingQC) - } - if s.hasAppProposal { - if first != s.lastAppPropNext { - return fmt.Errorf("AppProposal starts at %d, expected %d: %w", - first, s.lastAppPropNext, types.ErrAppProposalNonContiguous) - } - } else { - entries := s.sortedQCsLocked() - if len(entries) == 0 { - return fmt.Errorf("AppProposal [%d,%d) has no retained QC floor: %w", first, next, types.ErrAppProposalMissingQC) - } - if first != entries[0].lower { - return fmt.Errorf("first AppProposal starts at %d, expected retained QC floor %d: %w", - first, entries[0].lower, types.ErrAppProposalNonContiguous) - } + + status, ok := s.status.Get() + if !ok { + return fmt.Errorf("AppProposal [%d,%d) has no matching QC: %w", gr.First, gr.Next, types.ErrAppProposalMissingQC) } - if !s.hasBlocks || next > s.lastBlockNumber+1 { - return fmt.Errorf("AppProposal [%d,%d) is not covered by written blocks: %w", first, next, types.ErrAppProposalMissingQC) + if status.NextAppProposal != gr.First { + return fmt.Errorf("AppProposal starts at %d, expected %d: %w", + gr.First, status.NextAppProposal, types.ErrAppProposalNonContiguous) } - qc, ok := s.qcsByLower[first] - if !ok || qc.upper != next { - return fmt.Errorf("AppProposal [%d,%d) has no exact matching QC: %w", - first, next, types.ErrAppProposalMissingQC) + if gr.Next > status.NextBlock { + return fmt.Errorf("AppProposal [%d,%d) is not covered by written blocks: %w", gr.First, gr.Next, types.ErrAppProposalMissingQC) } - if err := appProposal.Verify(qc.qc.QC()); err != nil { - return fmt.Errorf("AppProposal [%d,%d) does not verify against matching QC: %w", first, next, err) + + for n := gr.First; n < gr.Next; n++ { + s.appProposalsByBlock[n] = appProposal } - s.appProps[first] = appProposalEntry{appProposal: appProposal, lower: first, upper: next} - s.latestAppProposalStartBlock = first - s.lastAppPropNext = next - s.hasAppProposal = true + status.NextAppProposal = gr.Next + s.status = utils.Some(status) return nil } func (s *blockDB) WriteAppQC(appQC *types.AppQC) error { - first, next := appQCRange(appQC) - if first >= next { - return fmt.Errorf("AppQC at %d covers no blocks: %w", first, types.ErrAppQCNonContiguous) + gr := appQC.Proposal().GlobalRange() + if gr.Len() == 0 { + return fmt.Errorf("AppQC at %d covers no blocks: %w", gr.First, types.ErrAppQCNonContiguous) } + s.mu.Lock() defer s.mu.Unlock() - if !s.hasQC { - return fmt.Errorf("AppQC [%d,%d) has no matching QC: %w", first, next, types.ErrAppQCMissingQC) + + status, ok := s.status.Get() + if !ok { + return fmt.Errorf("AppQC [%d,%d) has no matching QC: %w", gr.First, gr.Next, types.ErrAppQCMissingQC) } - if s.hasAppQC { - if first != s.lastAppQCNext { - return fmt.Errorf("AppQC starts at %d, expected %d: %w", - first, s.lastAppQCNext, types.ErrAppQCNonContiguous) - } - } else { - entries := s.sortedQCsLocked() - if len(entries) == 0 { - return fmt.Errorf("AppQC [%d,%d) has no retained QC floor: %w", first, next, types.ErrAppQCMissingQC) - } - if first != entries[0].lower { - return fmt.Errorf("first AppQC starts at %d, expected retained QC floor %d: %w", - first, entries[0].lower, types.ErrAppQCNonContiguous) - } + if status.NextAppQC != gr.First { + return fmt.Errorf("AppQC starts at %d, expected %d: %w", + gr.First, status.NextAppQC, types.ErrAppQCNonContiguous) } - if !s.hasAppProposal || next > s.lastAppPropNext { - return fmt.Errorf("AppQC [%d,%d) is not covered by written AppProposals: %w", first, next, types.ErrAppQCMissingQC) + if gr.Next > status.NextAppProposal { + return fmt.Errorf("AppQC [%d,%d) is not covered by written AppProposals: %w", gr.First, gr.Next, types.ErrAppQCMissingQC) } - qc, ok := s.qcsByLower[first] - if !ok || qc.upper != next { - return fmt.Errorf("AppQC [%d,%d) has no exact matching QC: %w", - first, next, types.ErrAppQCMissingQC) + + for n := gr.First; n < gr.Next; n++ { + s.appQCsByBlock[n] = appQC } - s.appQCs[first] = appQCEntry{appQC: appQC, lower: first, upper: next} - s.latestAppQCStartBlock = first - s.lastAppQCNext = next - s.hasAppQC = true + status.NextAppQC = gr.Next + status.First = status.NextAppQC - 1 + s.status = utils.Some(status) return nil } func (s *blockDB) PruneBefore(n types.GlobalBlockNumber) error { s.mu.Lock() defer s.mu.Unlock() - if !s.hasBlocks { - // No blocks yet: nothing to prune, and deleting QCs here would strand a - // future block whose coverage check still passes. Mirrors littblock. + + status, ok := s.status.Get() + if !ok { return nil } - n = min(n, s.statusLocked().Or(types.DBStatus{ - First: 0, - NextBlock: 0, - NextQC: 0, - NextAppQC: 0, - NextAppProposal: 0, - }).First) - // Round the watermark down to the covering QC's First. A QC's cohort of - // blocks changes readability atomically, so the watermark must never fall - // strictly inside a QC's range (see littblock): otherwise a read would - // refuse the cohort's low blocks while still serving its high blocks (which - // pruning must retain). - for _, e := range s.qcsByLower { - if e.lower <= n && n < e.upper { - n = e.lower - break - } - } - s.watermark = max(s.watermark, n) - for num, blk := range s.byNumber { - if num < s.watermark { - delete(s.byNumber, num) - delete(s.byHash, blk.Header().Hash()) - } + + n = min(n, status.First) + if qc, ok := s.qcsByBlock[n]; ok { + n = qc.QC().GlobalRange().First } - for lower, e := range s.qcsByLower { - if e.upper <= s.watermark { - delete(s.qcsByLower, lower) - } + if n <= s.watermark { + return nil } - for lower, e := range s.appQCs { - if e.upper <= s.watermark { - delete(s.appQCs, lower) + + s.watermark = n + for num, blk := range s.blocksByNumber { + if num < s.watermark { + delete(s.blocksByNumber, num) + delete(s.blocksByHash, blk.Header().Hash()) } } - for lower, e := range s.appProps { - if e.upper <= s.watermark { - delete(s.appProps, lower) + pruneRanges(s.watermark, s.qcsByBlock, func(qc *types.FullCommitQC) types.GlobalRange { + return qc.QC().GlobalRange() + }) + pruneRanges(s.watermark, s.appProposalsByBlock, func(appProposal *types.AppProposal) types.GlobalRange { + return appProposal.GlobalRange() + }) + pruneRanges(s.watermark, s.appQCsByBlock, func(appQC *types.AppQC) types.GlobalRange { + return appQC.Proposal().GlobalRange() + }) + return nil +} + +func pruneRanges[T any]( + watermark types.GlobalBlockNumber, + byBlock map[types.GlobalBlockNumber]T, + globalRange func(T) types.GlobalRange, +) { + for n, value := range byBlock { + if globalRange(value).Next <= watermark { + delete(byBlock, n) } } - return nil } func (s *blockDB) Flush() error { return nil } @@ -297,142 +210,77 @@ func (s *blockDB) Flush() error { return nil } func (s *blockDB) Status() types.DBStatus { s.mu.RLock() defer s.mu.RUnlock() - return s.statusLocked().Or(types.DBStatus{ + return s.status.Or(types.DBStatus{ First: 0, - NextBlock: 0, - NextQC: 0, NextAppQC: 0, NextAppProposal: 0, + NextBlock: 0, + NextQC: 0, }) } -func (s *blockDB) statusLocked() utils.Option[types.DBStatus] { - entries := s.sortedQCsLocked() - if len(entries) == 0 { - return utils.None[types.DBStatus]() - } - oldestQCStart := entries[0].lower - first := max(oldestQCStart, s.watermark) - if blockNumbers := s.sortedBlockNumbersLocked(); len(blockNumbers) > 0 { - first = max(first, blockNumbers[0]) - } - status := types.DBStatus{ - First: first, - NextBlock: first, - NextQC: s.lastQCNext, - NextAppQC: first, - NextAppProposal: first, - } - if s.hasBlocks { - status.NextBlock = s.lastBlockNumber + 1 - } - if s.hasAppQC { - status.NextAppQC = s.lastAppQCNext - status.First = status.NextAppQC - 1 - } - if s.hasAppProposal { - status.NextAppProposal = s.lastAppPropNext - } - return utils.Some(status) -} - func (s *blockDB) ReadRecent() (types.RecentData, error) { s.mu.RLock() defer s.mu.RUnlock() - status := s.statusLocked() - floor := status.Or(types.DBStatus{ - First: 0, - NextBlock: 0, - NextQC: 0, - NextAppQC: 0, - NextAppProposal: 0, - }).First - recent := types.RecentData{Status: status} - - for _, e := range s.sortedQCsLocked() { - if e.upper <= s.watermark { - continue - } - if e.upper <= floor { - continue + status, ok := s.status.Get() + if !ok { + return types.RecentData{}, nil + } + + recent := types.RecentData{Status: utils.Some(status)} + recent.CommitQCs = appendSuffixRanges( + s.qcsByBlock, + status.First, + status.NextQC, + func(qc *types.FullCommitQC) types.GlobalRange { return qc.QC().GlobalRange() }, + ) + recent.AppProposals = appendSuffixRanges( + s.appProposalsByBlock, + status.First, + status.NextAppProposal, + func(appProposal *types.AppProposal) types.GlobalRange { return appProposal.GlobalRange() }, + ) + recent.AppQCs = appendSuffixRanges( + s.appQCsByBlock, + status.First, + status.NextAppQC, + func(appQC *types.AppQC) types.GlobalRange { return appQC.Proposal().GlobalRange() }, + ) + for n := status.First; n < status.NextBlock; n++ { + if block, ok := s.blocksByNumber[n]; ok { + recent.Blocks = append(recent.Blocks, types.RecentBlock{Number: n, Block: block}) } - recent.CommitQCs = append(recent.CommitQCs, e.qc) - } - for _, e := range s.sortedAppQCsLocked() { - if e.upper <= floor { - continue - } - recent.AppQCs = append(recent.AppQCs, e.appQC) - } - for _, e := range s.sortedAppProposalsLocked() { - if e.upper <= floor { - continue - } - recent.AppProposals = append(recent.AppProposals, e.appProposal) - } - for _, n := range s.sortedBlockNumbersLocked() { - if n < floor { - continue - } - recent.Blocks = append(recent.Blocks, types.RecentBlock{Number: n, Block: s.byNumber[n]}) } return recent, nil } -// sortedQCsLocked returns the retained QC entries ascending by lower bound. Caller holds mu. -func (s *blockDB) sortedQCsLocked() []qcEntry { - entries := make([]qcEntry, 0, len(s.qcsByLower)) - for _, e := range s.qcsByLower { - entries = append(entries, e) - } - sort.Slice(entries, func(i, j int) bool { return entries[i].lower < entries[j].lower }) - return entries -} - -func (s *blockDB) sortedAppProposalsLocked() []appProposalEntry { - entries := make([]appProposalEntry, 0, len(s.appProps)) - for _, e := range s.appProps { - entries = append(entries, e) - } - sort.Slice(entries, func(i, j int) bool { return entries[i].lower < entries[j].lower }) - return entries -} - -func (s *blockDB) sortedAppQCsLocked() []appQCEntry { - entries := make([]appQCEntry, 0, len(s.appQCs)) - for _, e := range s.appQCs { - entries = append(entries, e) +func appendSuffixRanges[T any]( + byBlock map[types.GlobalBlockNumber]T, + floor types.GlobalBlockNumber, + next types.GlobalBlockNumber, + globalRange func(T) types.GlobalRange, +) []T { + if next <= floor { + return nil } - sort.Slice(entries, func(i, j int) bool { return entries[i].lower < entries[j].lower }) - return entries -} - -func (s *blockDB) appQCCoveringLocked(n types.GlobalBlockNumber) *types.AppQC { - for _, e := range s.appQCs { - if e.lower <= n && n < e.upper { - return e.appQC + values := make([]T, 0) + var lastFirst types.GlobalBlockNumber + haveLast := false + for n := floor; n < next; n++ { + value, ok := byBlock[n] + if !ok { + continue } - } - return nil -} - -func (s *blockDB) appProposalCoveringLocked(n types.GlobalBlockNumber) *types.AppProposal { - for _, e := range s.appProps { - if e.lower <= n && n < e.upper { - return e.appProposal + gr := globalRange(value) + if gr.Next <= floor || haveLast && gr.First == lastFirst { + continue } + values = append(values, value) + lastFirst = gr.First + haveLast = true } - return nil -} - -func (s *blockDB) sortedBlockNumbersLocked() []types.GlobalBlockNumber { - nums := make([]types.GlobalBlockNumber, 0, len(s.byNumber)) - for n := range s.byNumber { - nums = append(nums, n) - } - slices.Sort(nums) - return nums + return values } func (s *blockDB) ReadBlockByNumber( @@ -443,7 +291,7 @@ func (s *blockDB) ReadBlockByNumber( if n < s.watermark { return utils.None[*types.Block](), types.ErrPruned } - if blk, ok := s.byNumber[n]; ok { + if blk, ok := s.blocksByNumber[n]; ok { return utils.Some(blk), nil } return utils.None[*types.Block](), nil @@ -454,7 +302,7 @@ func (s *blockDB) ReadBlockByHash( ) (utils.Option[types.BlockWithNumber], error) { s.mu.RLock() defer s.mu.RUnlock() - if e, ok := s.byHash[hash]; ok { + if e, ok := s.blocksByHash[hash]; ok && e.n >= s.watermark { return utils.Some(types.BlockWithNumber{Block: e.blk, Number: e.n}), nil } return utils.None[types.BlockWithNumber](), nil @@ -468,10 +316,8 @@ func (s *blockDB) ReadQCByBlockNumber( if n < s.watermark { return utils.None[*types.FullCommitQC](), types.ErrPruned } - for _, e := range s.qcsByLower { - if e.lower <= n && n < e.upper { - return utils.Some(e.qc), nil - } + if qc, ok := s.qcsByBlock[n]; ok { + return utils.Some(qc), nil } return utils.None[*types.FullCommitQC](), nil } @@ -484,7 +330,7 @@ func (s *blockDB) ReadAppProposalByBlockNumber( if n < s.watermark { return utils.None[*types.AppProposal](), types.ErrPruned } - if appProposal := s.appProposalCoveringLocked(n); appProposal != nil { + if appProposal, ok := s.appProposalsByBlock[n]; ok { return utils.Some(appProposal), nil } return utils.None[*types.AppProposal](), nil @@ -498,7 +344,7 @@ func (s *blockDB) ReadAppQCByBlockNumber( if n < s.watermark { return utils.None[*types.AppQC](), types.ErrPruned } - if appQC := s.appQCCoveringLocked(n); appQC != nil { + if appQC, ok := s.appQCsByBlock[n]; ok { return utils.Some(appQC), nil } return utils.None[*types.AppQC](), nil From a7aaa6982b2cbed6618bc84ca7989b8dedf63770 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 7 Aug 2026 22:39:06 +0200 Subject: [PATCH 40/61] Option[Status] --- sei-db/ledger_db/block/block_db_test.go | 41 ++++++++++--------- sei-db/ledger_db/block/blocksim/blocksim.go | 5 ++- .../block/littblock/litt_block_db.go | 13 ++---- .../ledger_db/block/memblock/mem_block_db.go | 10 +---- sei-tendermint/autobahn/types/block_db.go | 3 +- .../autobahn/data/state_recovery_test.go | 4 +- .../internal/autobahn/data/state_test.go | 6 +-- 7 files changed, 37 insertions(+), 45 deletions(-) diff --git a/sei-db/ledger_db/block/block_db_test.go b/sei-db/ledger_db/block/block_db_test.go index e088f0a3a6..2fcc2364a9 100644 --- a/sei-db/ledger_db/block/block_db_test.go +++ b/sei-db/ledger_db/block/block_db_test.go @@ -110,6 +110,11 @@ func restart(t *testing.T, o open, db types.BlockDB) types.BlockDB { return reopened } +func status(t *testing.T, db types.BlockDB) types.DBStatus { + t.Helper() + return db.Status().OrPanic("non-empty BlockDB status") +} + func testEmptyDB(t *testing.T, build builder) { db, _ := openFresh(t, build) defer func() { _ = db.Close() }() @@ -136,11 +141,7 @@ func testEmptyDB(t *testing.T, build builder) { require.Empty(t, drainRecent(t, db), "empty db should yield no recent records") - tips := db.Status() - require.Zero(t, tips.NextBlock, "empty db has no block write tip") - require.Zero(t, tips.NextQC, "empty db has no QC write tip") - require.Zero(t, tips.NextAppQC, "empty db has no AppQC write tip") - require.Zero(t, tips.NextAppProposal, "empty db has no AppProposal write tip") + require.False(t, db.Status().IsPresent(), "empty db has no write tips") } // iterEntry is one position observed while draining an iterator. @@ -244,7 +245,7 @@ func testStatus(t *testing.T, build builder) { defer func() { _ = db.Close() }() require.NoError(t, db.WriteQC(batches[0].qc)) - tips := db.Status() + tips := status(t, db) require.Equal(t, batches[0].first, tips.First) require.Equal(t, batches[0].next, tips.NextQC) require.Equal(t, tips.First, tips.NextBlock, "QC-only store has no block tip") @@ -257,7 +258,7 @@ func testStatus(t *testing.T, build builder) { } writeAll(t, db, batches[1:]) last := batches[len(batches)-1] - tips = db.Status() + tips = status(t, db) require.Equal(t, last.next, tips.NextBlock) require.Equal(t, last.next, tips.NextQC) assertTipsMatchPresent(t, db) @@ -267,13 +268,13 @@ func testStatus(t *testing.T, build builder) { require.Greater(t, len(batches), 1) require.NoError(t, db.PruneBefore(batches[1].first)) assertTipsMatchPresent(t, db) - tips = db.Status() + tips = status(t, db) require.Equal(t, last.next, tips.NextBlock, "prune must not move the block write tip") require.Equal(t, last.next, tips.NextQC, "prune must not move the QC write tip") db = restart(t, o, db) assertTipsMatchPresent(t, db) - tips = db.Status() + tips = status(t, db) require.Equal(t, last.next, tips.NextBlock, "block tip must survive restart") require.Equal(t, last.next, tips.NextQC, "QC tip must survive restart") } @@ -282,7 +283,7 @@ func testStatus(t *testing.T, build builder) { // public read API still serves. func assertTipsMatchPresent(t *testing.T, db types.BlockDB) { t.Helper() - tips := db.Status() + tips := status(t, db) if tips.NextBlock > tips.First { blk, err := db.ReadBlockByNumber(tips.NextBlock - 1) @@ -389,10 +390,10 @@ func testAppProposalByBlockNumber(t *testing.T, build builder) { } } - tips := db.Status() + tips := status(t, db) require.Equal(t, batches[1].next, tips.NextAppProposal) db = restart(t, o, db) - tips = db.Status() + tips = status(t, db) require.Equal(t, batches[1].next, tips.NextAppProposal, "AppProposal tip must survive restart") assertTipsMatchPresent(t, db) @@ -448,10 +449,10 @@ func testAppQCByBlockNumber(t *testing.T, build builder) { } } - tips := db.Status() + tips := status(t, db) require.Equal(t, batches[1].next, tips.NextAppQC) db = restart(t, o, db) - tips = db.Status() + tips = status(t, db) require.Equal(t, batches[1].next, tips.NextAppQC, "AppQC tip must survive restart") assertTipsMatchPresent(t, db) @@ -958,7 +959,7 @@ func testReadRecent(t *testing.T, build builder) { recent, err := db.ReadRecent() require.NoError(t, err) - require.Equal(t, db.Status(), recent.Status.OrPanic("recent status")) + require.Equal(t, status(t, db), recent.Status.OrPanic("recent status")) require.NotEmpty(t, recent.AppQCs) gotAppQC := recent.AppQCs[0] require.Equal(t, appQC.Proposal().RoadIndex(), gotAppQC.Proposal().RoadIndex()) @@ -1042,7 +1043,7 @@ func testWriteAppProposalOrderRejected(t *testing.T, build builder) { require.NoError(t, db.WriteBlock(b1.first+gbn(i), blk)) } require.NoError(t, db.WriteAppProposal(appProposalForBatch(rng, b1))) - tips := db.Status() + tips := status(t, db) require.Equal(t, b1.next, tips.NextAppProposal) } @@ -1084,7 +1085,7 @@ func testWriteAppQCOrderRejected(t *testing.T, build builder) { } require.NoError(t, db.WriteAppProposal(appProposalForBatch(rng, b1))) require.NoError(t, db.WriteAppQC(appQCForBatch(rng, keys, b1))) - tips := db.Status() + tips := status(t, db) require.Equal(t, b1.next, tips.NextAppQC) } @@ -1166,7 +1167,7 @@ func testResumeAfterRestart(t *testing.T, build builder) { require.Equal(t, last.first, prevQC.GlobalRange().First, "recovered QC must be the last persisted QC") require.Equal(t, last.next, prevQC.GlobalRange().Next) - tips := db.Status() + tips := status(t, db) require.Equal(t, highest+1, tips.NextBlock, "Status block tip must match the iterator scan") require.Equal(t, prevQC.GlobalRange().Next, tips.NextQC, "Status QC tip must match the iterator scan") covering, err := db.ReadQCByBlockNumber(tips.NextQC - 1) @@ -1265,9 +1266,9 @@ func testWriteQCCoversNoBlocksRejected(t *testing.T, build builder) { // The rejection persisted nothing: the store is still empty, so a QC that // does cover blocks is still accepted at 0. - require.Zero(t, db.Status().NextQC) + require.False(t, db.Status().IsPresent()) require.NoError(t, db.WriteQC(types.GenFullCommitQCRange(rng, 0, 3))) - require.Equal(t, gbn(3), db.Status().NextQC) + require.Equal(t, gbn(3), status(t, db).NextQC) } // testWriteBlockGapRejected asserts that blocks must be written densely: a diff --git a/sei-db/ledger_db/block/blocksim/blocksim.go b/sei-db/ledger_db/block/blocksim/blocksim.go index 8a0881e6cb..e8b94540e2 100644 --- a/sei-db/ledger_db/block/blocksim/blocksim.go +++ b/sei-db/ledger_db/block/blocksim/blocksim.go @@ -200,7 +200,10 @@ func recoverResumeState( prev := tmutils.None[*types.CommitQC]() highest := tmutils.None[uint64]() - status := db.Status() + status, ok := db.Status().Get() + if !ok { + return prev, highest, nil + } if status.NextBlock > 0 { highest = tmutils.Some(uint64(status.NextBlock - 1)) } diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index 94b3d29035..be237b7d1b 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -375,16 +375,10 @@ func (s *blockDB) Flush() error { return nil } -func (s *blockDB) Status() types.DBStatus { +func (s *blockDB) Status() utils.Option[types.DBStatus] { s.mu.Lock() defer s.mu.Unlock() - return s.status.Or(types.DBStatus{ - First: 0, - NextBlock: 0, - NextQC: 0, - NextAppQC: 0, - NextAppProposal: 0, - }) + return s.status } // ReadRecent() reads the latest AppQC/AppProposal recovery suffix. @@ -400,7 +394,8 @@ func (s *blockDB) ReadRecent() (types.RecentData, error) { } // Safety check: if watermark has been moved and GC happened to get executed during iteration, // the loaded data might be inconsistent with the targetFloor we computed. - if current := s.Status(); current.First != status.First { + current, ok := s.Status().Get() + if !ok || current.First != status.First { return types.RecentData{}, fmt.Errorf("watermark has moved while iterating: recovered status %+v, current status %+v", status, current) } return recent, nil diff --git a/sei-db/ledger_db/block/memblock/mem_block_db.go b/sei-db/ledger_db/block/memblock/mem_block_db.go index 6dad1b8325..159317561a 100644 --- a/sei-db/ledger_db/block/memblock/mem_block_db.go +++ b/sei-db/ledger_db/block/memblock/mem_block_db.go @@ -207,16 +207,10 @@ func pruneRanges[T any]( func (s *blockDB) Flush() error { return nil } -func (s *blockDB) Status() types.DBStatus { +func (s *blockDB) Status() utils.Option[types.DBStatus] { s.mu.RLock() defer s.mu.RUnlock() - return s.status.Or(types.DBStatus{ - First: 0, - NextAppQC: 0, - NextAppProposal: 0, - NextBlock: 0, - NextQC: 0, - }) + return s.status } func (s *blockDB) ReadRecent() (types.RecentData, error) { diff --git a/sei-tendermint/autobahn/types/block_db.go b/sei-tendermint/autobahn/types/block_db.go index 2a5ccc6ea3..6c1afdfe8a 100644 --- a/sei-tendermint/autobahn/types/block_db.go +++ b/sei-tendermint/autobahn/types/block_db.go @@ -189,7 +189,8 @@ type BlockDB interface { Flush() error // Status returns a consistent snapshot of the in-memory write tips (no I/O). - Status() DBStatus + // None means the DB is empty. + Status() utils.Option[DBStatus] // ReadRecent returns the materialized startup-recovery suffix. // diff --git a/sei-tendermint/internal/autobahn/data/state_recovery_test.go b/sei-tendermint/internal/autobahn/data/state_recovery_test.go index 590852b302..6bfe49bc8c 100644 --- a/sei-tendermint/internal/autobahn/data/state_recovery_test.go +++ b/sei-tendermint/internal/autobahn/data/state_recovery_test.go @@ -164,7 +164,7 @@ func TestRecoveryRejectsAppTipBeyondCrashWindow(t *testing.T) { db := newTestBlockDB(t, t.TempDir()) writeToBlockDB(t, db, []*types.FullCommitQC{qc}, [][]*types.Block{blocks}) - dbNextBlock := db.Status().NextBlock + dbNextBlock := db.Status().OrPanic("non-empty BlockDB status").NextBlock lastExecuted := dbNextBlock + 1 state, err := NewState(&Config{ @@ -456,7 +456,7 @@ func TestRunPersistSeedsFromRecoveryFloor(t *testing.T) { require.NoError(t, db1.Close()) db2 := newTestBlockDB(t, dir) - tips := db2.Status() + tips := db2.Status().OrPanic("non-empty BlockDB status") require.Equal(t, gr2.First, tips.First) require.Equal(t, tips.First, tips.NextBlock) require.NotZero(t, tips.NextQC) diff --git a/sei-tendermint/internal/autobahn/data/state_test.go b/sei-tendermint/internal/autobahn/data/state_test.go index cbd5be6437..0dfb7f9e6d 100644 --- a/sei-tendermint/internal/autobahn/data/state_test.go +++ b/sei-tendermint/internal/autobahn/data/state_test.go @@ -453,9 +453,7 @@ func TestPushQCBeforeRunPersistsToBlockDB(t *testing.T) { // Transport-race window: PushQC before data.Run / runPersist starts. require.NoError(t, state.PushQC(ctx, qc1, blocks1)) - tips := db.Status() - require.Zero(t, tips.NextBlock, "PushQC must not write BlockDB before Run") - require.Zero(t, tips.NextQC) + require.False(t, db.Status().IsPresent(), "PushQC must not write BlockDB before Run") require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { runCtx, cancel := context.WithCancel(ctx) @@ -473,7 +471,7 @@ func TestPushQCBeforeRunPersistsToBlockDB(t *testing.T) { return nil })) - tips = db.Status() + tips := db.Status().OrPanic("non-empty BlockDB status") require.Equal(t, gr1.Next, tips.NextBlock) require.Equal(t, gr1.Next, tips.NextQC) From 882a8a5bfbd9fea75c03eb1dbeefca89ec9152b9 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Mon, 10 Aug 2026 10:54:50 +0200 Subject: [PATCH 41/61] rename done --- sei-db/ledger_db/block/block_db_test.go | 60 +++++----- sei-db/ledger_db/block/blocksim/blocksim.go | 6 +- sei-db/ledger_db/block/littblock/codec.go | 12 +- .../block/littblock/litt_block_db.go | 111 +++++++----------- .../littblock/litt_block_stranding_test.go | 10 +- .../ledger_db/block/littblock_crash_test.go | 8 +- .../ledger_db/block/memblock/mem_block_db.go | 22 ++-- sei-tendermint/autobahn/types/block_db.go | 26 ++-- .../internal/autobahn/data/state.go | 24 ++-- .../autobahn/data/state_recovery_test.go | 2 +- 10 files changed, 125 insertions(+), 156 deletions(-) diff --git a/sei-db/ledger_db/block/block_db_test.go b/sei-db/ledger_db/block/block_db_test.go index 2fcc2364a9..657082914c 100644 --- a/sei-db/ledger_db/block/block_db_test.go +++ b/sei-db/ledger_db/block/block_db_test.go @@ -55,7 +55,7 @@ func TestBlockDB(t *testing.T) { t.Run("QCByBlockNumber", func(t *testing.T) { testQCByBlockNumber(t, impl.build) }) t.Run("AppProposalByBlockNumber", func(t *testing.T) { testAppProposalByBlockNumber(t, impl.build) }) t.Run("AppQCByBlockNumber", func(t *testing.T) { testAppQCByBlockNumber(t, impl.build) }) - t.Run("ReadRecent", func(t *testing.T) { testReadRecent(t, impl.build) }) + t.Run("ReadSuffix", func(t *testing.T) { testReadSuffix(t, impl.build) }) t.Run("RestartPersistsData", func(t *testing.T) { testRestartPersistsData(t, impl.build) }) t.Run("PruneRetainsAtOrAbove", func(t *testing.T) { testPruneRetainsAtOrAbove(t, impl.build) }) t.Run("PruneStraddleRetainsQC", func(t *testing.T) { testPruneStraddleRetainsQC(t, impl.build) }) @@ -110,7 +110,7 @@ func restart(t *testing.T, o open, db types.BlockDB) types.BlockDB { return reopened } -func status(t *testing.T, db types.BlockDB) types.DBStatus { +func status(t *testing.T, db types.BlockDB) types.SuffixRange { t.Helper() return db.Status().OrPanic("non-empty BlockDB status") } @@ -139,7 +139,7 @@ func testEmptyDB(t *testing.T, build builder) { require.NoError(t, err) require.False(t, appProposal.IsPresent()) - require.Empty(t, drainRecent(t, db), "empty db should yield no recent records") + require.Empty(t, drainSuffix(t, db), "empty db should yield no suffix records") require.False(t, db.Status().IsPresent(), "empty db has no write tips") } @@ -162,27 +162,27 @@ type iterEntry struct { appProposal *types.AppProposal } -// drainRecent reads the recovery-visible recent batch. -func drainRecent(t *testing.T, db types.BlockDB) []iterEntry { +// drainSuffix reads the recovery-visible suffix batch. +func drainSuffix(t *testing.T, db types.BlockDB) []iterEntry { t.Helper() - recent, err := db.ReadRecent() + suffix, err := db.ReadSuffix() require.NoError(t, err) var entries []iterEntry - floor := recent.Status.Or(types.DBStatus{ + floor := suffix.Status.Or(types.SuffixRange{ First: 0, NextBlock: 0, NextQC: 0, NextAppQC: 0, NextAppProposal: 0, }).First - for _, qc := range recent.CommitQCs { + for _, qc := range suffix.CommitQCs { first := qc.QC().GlobalRange().First next := first + gbn(len(qc.Headers())) for n := max(first, floor); n < next; n++ { entries = append(entries, iterEntry{n: n, qc: qc}) } } - for _, b := range recent.Blocks { + for _, b := range suffix.Blocks { found := false for i := range entries { if entries[i].n == b.Number { @@ -191,9 +191,9 @@ func drainRecent(t *testing.T, db types.BlockDB) []iterEntry { break } } - require.True(t, found, "block %d must be covered by a recent QC", b.Number) + require.True(t, found, "block %d must be covered by a suffix QC", b.Number) } - for _, appQC := range recent.AppQCs { + for _, appQC := range suffix.AppQCs { gr := appQC.Proposal().GlobalRange() for i := range entries { if gr.Has(entries[i].n) { @@ -201,7 +201,7 @@ func drainRecent(t *testing.T, db types.BlockDB) []iterEntry { } } } - for _, appProposal := range recent.AppProposals { + for _, appProposal := range suffix.AppProposals { gr := appProposal.GlobalRange() for i := range entries { if gr.Has(entries[i].n) { @@ -380,7 +380,7 @@ func testAppProposalByBlockNumber(t *testing.T, build builder) { require.NoError(t, err) require.False(t, miss.IsPresent(), "CommitQCs/blocks past the AppProposal prefix should not imply AppProposal presence") - entries := drainRecent(t, db) + entries := drainSuffix(t, db) for _, e := range entries { switch { case e.n < batches[2].first: @@ -439,7 +439,7 @@ func testAppQCByBlockNumber(t *testing.T, build builder) { require.NoError(t, err) require.False(t, miss.IsPresent(), "CommitQCs/blocks past the AppQC prefix should not imply AppQC presence") - entries := drainRecent(t, db) + entries := drainSuffix(t, db) for _, e := range entries { switch { case e.n < batches[2].first: @@ -605,7 +605,7 @@ func testPruneRefusesBelowWatermark(t *testing.T, build builder) { require.False(t, appQC.IsPresent(), "AppQC at block %d below watermark %d must not be served", n, watermark) } - for _, e := range drainRecent(t, db) { + for _, e := range drainSuffix(t, db) { require.GreaterOrEqual(t, e.n, watermark, "iterator must not yield position %d below watermark %d", e.n, watermark) } @@ -812,13 +812,13 @@ func testPruneNeverEmpties(t *testing.T, build builder) { require.ErrorIs(t, err, types.ErrPruned, "AppQCs below the newest cohort must be reported pruned") require.False(t, belowAppQC.IsPresent(), "AppQCs below the newest cohort must not be served") - // ReadRecent starts at recentFloor(), while the prune watermark still + // ReadSuffix starts at recentFloor(), while the prune watermark still // rounds down to keep the whole newest QC cohort readable. - entries := drainRecent(t, db) + entries := drainSuffix(t, db) require.Equal(t, []types.GlobalBlockNumber{newest}, presentBlockNumbers(entries), - "ReadRecent must start at recentFloor() after PruneBefore(%d)", prune) + "ReadSuffix must start at recentFloor() after PruneBefore(%d)", prune) require.Equal(t, []types.GlobalBlockNumber{last.first}, qcFirsts(entries), - "ReadRecent must include the QC covering the recent floor") + "ReadSuffix must include the QC covering the suffix floor") }) } } @@ -942,7 +942,7 @@ func testPruneQCOnlyThenWriteBlock(t *testing.T, build builder) { require.True(t, qc.IsPresent(), "covering QC of block %d must survive the earlier prune", b0.first) } -func testReadRecent(t *testing.T, build builder) { +func testReadSuffix(t *testing.T, build builder) { committee, keys := buildCommittee() batches := generateBatches(committee, keys) db, _ := openFresh(t, build) @@ -957,13 +957,13 @@ func testReadRecent(t *testing.T, build builder) { require.NoError(t, db.WriteBlock(batches[2].first+gbn(i), blk)) } - recent, err := db.ReadRecent() + suffix, err := db.ReadSuffix() require.NoError(t, err) - require.Equal(t, status(t, db), recent.Status.OrPanic("recent status")) - require.NotEmpty(t, recent.AppQCs) - gotAppQC := recent.AppQCs[0] + require.Equal(t, status(t, db), suffix.Status.OrPanic("suffix status")) + require.NotEmpty(t, suffix.AppQCs) + gotAppQC := suffix.AppQCs[0] require.Equal(t, appQC.Proposal().RoadIndex(), gotAppQC.Proposal().RoadIndex()) - entries := drainRecent(t, db) + entries := drainSuffix(t, db) recoveryFloor := appQC.Proposal().GlobalRange().Next - 1 require.Equal(t, []types.GlobalBlockNumber{batches[0].first, batches[1].first, batches[2].first}, qcFirsts(entries)) require.Equal(t, batches[2].next-recoveryFloor, types.GlobalBlockNumber(len(entries))) @@ -1192,7 +1192,7 @@ func testResumeAfterRestart(t *testing.T, build builder) { // verification; production resume uses Status (see blocksim.recoverResumeState). func recoverHighestBlock(t *testing.T, db types.BlockDB) (types.GlobalBlockNumber, bool) { t.Helper() - present := presentBlockNumbers(drainRecent(t, db)) + present := presentBlockNumbers(drainSuffix(t, db)) if len(present) == 0 { return 0, false } @@ -1204,7 +1204,7 @@ func recoverHighestBlock(t *testing.T, db types.BlockDB) (types.GlobalBlockNumbe // verification; production resume uses Status (see blocksim.recoverResumeState). func recoverLastQC(t *testing.T, db types.BlockDB) (*types.CommitQC, bool) { t.Helper() - entries := drainRecent(t, db) + entries := drainSuffix(t, db) if len(entries) == 0 { return nil, false } @@ -1215,7 +1215,7 @@ func recoverLastQC(t *testing.T, db types.BlockDB) (*types.CommitQC, bool) { // iterator scan (false if the store has no AppQCs). func recoverLastAppQC(t *testing.T, db types.BlockDB) (*types.AppQC, bool) { t.Helper() - entries := drainRecent(t, db) + entries := drainSuffix(t, db) for i := len(entries) - 1; i >= 0; i-- { if entries[i].appQC != nil { return entries[i].appQC, true @@ -1330,7 +1330,7 @@ func TestMemblockPruneRemovesBelowWatermark(t *testing.T) { require.True(t, opt.IsPresent()) // The iterator must skip the pruned records entirely. - for _, e := range drainRecent(t, db) { + for _, e := range drainSuffix(t, db) { require.GreaterOrEqual(t, e.n, watermark, "iterator must not surface pruned positions") require.GreaterOrEqual(t, e.qc.QC().GlobalRange().First, watermark, "iterator must not surface pruned QCs") @@ -1443,7 +1443,7 @@ func assertIterators(t *testing.T, db types.BlockDB, committee *types.Committee, totalBlocks += len(b.blocks) } - entries := drainRecent(t, db) + entries := drainSuffix(t, db) require.Len(t, entries, totalBlocks, "one position per covered number in a fully-written store") require.Equal(t, batches[0].first, entries[0].n, "the scan must begin at the first covered number") for i, e := range entries { diff --git a/sei-db/ledger_db/block/blocksim/blocksim.go b/sei-db/ledger_db/block/blocksim/blocksim.go index e8b94540e2..306fa21211 100644 --- a/sei-db/ledger_db/block/blocksim/blocksim.go +++ b/sei-db/ledger_db/block/blocksim/blocksim.go @@ -178,11 +178,11 @@ func NewBlockSim( // countExistingState scans the ledger to count the persisted blocks and QCs, // exercising the replay path at startup. func countExistingState(db types.BlockDB) (blocks int, qcs int, err error) { - recent, err := db.ReadRecent() + suffix, err := db.ReadSuffix() if err != nil { - return 0, 0, fmt.Errorf("failed to read recent ledger data: %w", err) + return 0, 0, fmt.Errorf("failed to read suffix ledger data: %w", err) } - return len(recent.Blocks), len(recent.CommitQCs), nil + return len(suffix.Blocks), len(suffix.CommitQCs), nil } // recoverResumeState reads the persisted tail so the benchmark resumes appending diff --git a/sei-db/ledger_db/block/littblock/codec.go b/sei-db/ledger_db/block/littblock/codec.go index 7ec54d2801..d106f33cd0 100644 --- a/sei-db/ledger_db/block/littblock/codec.go +++ b/sei-db/ledger_db/block/littblock/codec.go @@ -19,11 +19,11 @@ import ( // - kindAppQC 'a' + 8-byte big-endian GlobalBlockNumber (AppQC primary + covered aliases) // - kindAppProp 'p' + 8-byte big-endian GlobalBlockNumber (AppProposal primary + covered aliases) const ( + kindAppQC byte = 'a' kindBlock byte = 'b' kindBlockHash byte = 'h' - kindQC byte = 'q' - kindAppQC byte = 'a' kindAppProp byte = 'p' + kindQC byte = 'q' ) // encodeKey encodes a GlobalBlockNumber as an 8-byte big-endian value. Big-endian @@ -36,8 +36,8 @@ func encodeKey(n types.GlobalBlockNumber) []byte { } // decodeKey decodes an 8-byte value produced by encodeKey. -func decodeKey(b []byte) types.GlobalBlockNumber { - return types.GlobalBlockNumber(binary.BigEndian.Uint64(b)) +func decodeKey(b [8]byte) types.GlobalBlockNumber { + return types.GlobalBlockNumber(binary.BigEndian.Uint64(b[:])) } // blockKey returns the primary key under which a block at number n is stored. @@ -76,9 +76,9 @@ func keyKind(key []byte) byte { // decodeNumberKey decodes the GlobalBlockNumber from a kindBlock, kindQC, or // kindAppQC key (i.e. a key whose prefix is followed by an 8-byte big-endian -// number). +// number). Panics if key is not 9 bytes (1B kind + 8B GlobalBlockNumber). func decodeNumberKey(key []byte) types.GlobalBlockNumber { - return decodeKey(key[1:]) + return decodeKey([8]byte(key[1:])) } // Serialization version for blocks. diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index be237b7d1b..3351abec45 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -29,7 +29,7 @@ type blockDB struct { // watermark is a retention floor, always a QC boundary (a GlobalRange().First): // PruneBefore rounds a requested prune point down to the start of the cohort // containing it, and startup re-derives it as the lowest surviving QC's First - // (see cohortStart and recoverReadFloors). Keeping it on a cohort boundary + // (see clampPruneBoundary and recoverWatermark). Keeping it on a cohort boundary // is what makes a QC's blocks change readability atomically — the gate never // splits a cohort. // @@ -45,7 +45,7 @@ type blockDB struct { // status is the explicit write-order/recovery suffix cursor (see // types.BlockDB contract). None means the DB is empty. Guarded by mu. mu sync.Mutex - status utils.Option[types.DBStatus] + status utils.Option[types.SuffixRange] } // NewBlockDB opens (or creates) a LittDB-backed types.BlockDB from config. The @@ -85,35 +85,22 @@ func NewBlockDB(config *LittBlockConfig) (types.BlockDB, error) { _ = db.Close() return nil, fmt.Errorf("failed to recover watermark: %w", err) } - if err := s.recoverCursors(); err != nil { - _ = db.Close() - return nil, fmt.Errorf("failed to recover write cursors: %w", err) - } - return s, nil -} - -// recoverCursors reloads the write-order cursors from on-disk state. Without -// this, a reopened DB would treat itself as empty and let writes silently accept -// out-of-order or non-contiguous data that overwrite or gap persisted data. -func (s *blockDB) recoverCursors() error { - recent, err := s.readRecent() + suffix, err := s.ReadSuffix() if err != nil { - return fmt.Errorf("read recent data: %w", err) + return nil, fmt.Errorf("ReadSuffix(): %w", err) } - s.status = recent.Status - return nil + s.status = suffix.Status + return s,nil } -// recoverReadFloors re-derives the read watermark on open from the oldest +// recoverWatermark re-derives the read watermark on open from the oldest // surviving QC. It is in-memory only, so a restart forgets every PruneBefore. // That is fine for reclamation (nothing new is deleted), but we must protect // against showing un-pruned blocks with pruned QCs. // -// One forward pass serves both. QCs are written before the blocks they cover, so the oldest -// surviving record is normally a QC and the first block follows shortly after. -// The block search is skipped when the store holds no blocks — status.NextBlock -// comes from recoverCursors, which runs first — so a QC-only store does not walk -// the whole table looking for a block that is not there. +// QCs are written before the blocks they cover, so the oldest surviving record +// is normally a QC and the first block follows shortly after. If no QC survives +// but the table is non-empty, the store is corrupt and must not reopen. func (s *blockDB) recoverWatermark() error { it, err := s.table.Iterator(false) if err != nil { @@ -215,7 +202,7 @@ func (s *blockDB) WriteQC(qc *types.FullCommitQC) error { // The first QC may start anywhere its caller allows, and nothing below it will ever // be written. Record where coverage begins so Iterator can clamp to it without // discovering it by scanning; a reopen re-derives the same value. - status = types.DBStatus{ + status = types.SuffixRange{ First: gr.First, NextAppQC: gr.First, NextAppProposal: gr.First, @@ -375,76 +362,58 @@ func (s *blockDB) Flush() error { return nil } -func (s *blockDB) Status() utils.Option[types.DBStatus] { +func (s *blockDB) Status() utils.Option[types.SuffixRange] { s.mu.Lock() defer s.mu.Unlock() return s.status } -// ReadRecent() reads the latest AppQC/AppProposal recovery suffix. -// WARNING: ReadRecent() will return an error if watermark is moved during iteration. -func (s *blockDB) ReadRecent() (types.RecentData, error) { - recent, err := s.readRecent() - if err != nil { - return types.RecentData{}, err - } - status, ok := recent.Status.Get() - if !ok { - return recent, nil - } - // Safety check: if watermark has been moved and GC happened to get executed during iteration, - // the loaded data might be inconsistent with the targetFloor we computed. - current, ok := s.Status().Get() - if !ok || current.First != status.First { - return types.RecentData{}, fmt.Errorf("watermark has moved while iterating: recovered status %+v, current status %+v", status, current) - } - return recent, nil -} - -func (s *blockDB) readRecent() (types.RecentData, error) { +// ReadSuffix reads the materialized startup-recovery suffix. +// WARNING: ReadSuffix() will return an error if watermark is moved during iteration. +func (s *blockDB) ReadSuffix() (types.Suffix, error) { it, err := s.table.Iterator(true) if err != nil { - return types.RecentData{}, fmt.Errorf("failed to open recent-data iterator: %w", err) + return types.Suffix{}, fmt.Errorf("failed to open suffix iterator: %w", err) } defer func() { _ = it.Close() }() - var recent types.RecentData - var status types.DBStatus + var suffix types.Suffix + var status types.SuffixRange var oldestQC *types.FullCommitQC var gotBlock, gotQC, gotAppProposal, gotAppQC bool for !gotAppQC || !gotQC || status.NextAppQC <= oldestQC.QC().GlobalRange().Next { ok, err := it.Next() if err != nil { - return types.RecentData{}, fmt.Errorf("failed to advance recent-data iterator: %w", err) + return types.Suffix{}, fmt.Errorf("failed to advance suffix iterator: %w", err) } if !ok { break } key, isPrimary, err := it.GetKey() if err != nil { - return types.RecentData{}, fmt.Errorf("failed to read recent-data key: %w", err) + return types.Suffix{}, fmt.Errorf("failed to read suffix key: %w", err) } if !isPrimary { continue } value, err := it.GetValue() if err != nil { - return types.RecentData{}, fmt.Errorf("failed to read recent-data value: %w", err) + return types.Suffix{}, fmt.Errorf("failed to read suffix value: %w", err) } switch keyKind(key) { case kindBlock: n, block, err := decodeBlock(value) if err != nil { - return types.RecentData{}, fmt.Errorf("failed to decode recent block: %w", err) + return types.Suffix{}, fmt.Errorf("failed to decode suffix block: %w", err) } if !gotBlock { status.NextBlock = n + 1 gotBlock = true } - recent.Blocks = append(recent.Blocks, types.RecentBlock{Number: n, Block: block}) + suffix.Blocks = append(suffix.Blocks, types.SuffixBlock{Number: n, Block: block}) case kindAppQC: appQC, err := decodeAppQC(value) if err != nil { - return types.RecentData{}, fmt.Errorf("failed to decode recent AppQC: %w", err) + return types.Suffix{}, fmt.Errorf("failed to decode suffix AppQC: %w", err) } gr := appQC.Proposal().GlobalRange() if !gotAppQC { @@ -452,22 +421,22 @@ func (s *blockDB) readRecent() (types.RecentData, error) { status.First = gr.Next - 1 gotAppQC = true } - recent.AppQCs = append(recent.AppQCs, appQC) + suffix.AppQCs = append(suffix.AppQCs, appQC) case kindAppProp: appProposal, err := decodeAppProposal(value) if err != nil { - return types.RecentData{}, fmt.Errorf("failed to decode recent AppProposal: %w", err) + return types.Suffix{}, fmt.Errorf("failed to decode suffix AppProposal: %w", err) } gr := appProposal.GlobalRange() if !gotAppProposal { status.NextAppProposal = gr.Next gotAppProposal = true } - recent.AppProposals = append(recent.AppProposals, appProposal) + suffix.AppProposals = append(suffix.AppProposals, appProposal) case kindQC: qc, err := decodeQC(value) if err != nil { - return types.RecentData{}, fmt.Errorf("failed to decode recent CommitQC: %w", err) + return types.Suffix{}, fmt.Errorf("failed to decode suffix CommitQC: %w", err) } gr := qc.QC().GlobalRange() if !gotQC { @@ -475,12 +444,12 @@ func (s *blockDB) readRecent() (types.RecentData, error) { gotQC = true } oldestQC = qc - recent.CommitQCs = append(recent.CommitQCs, qc) + suffix.CommitQCs = append(suffix.CommitQCs, qc) } } if !gotQC { // Empty db. - return types.RecentData{}, nil + return types.Suffix{}, nil } // Set fields for missing resources. first := oldestQC.QC().GlobalRange().First @@ -489,27 +458,27 @@ func (s *blockDB) readRecent() (types.RecentData, error) { status.NextAppProposal = max(status.NextAppProposal, first) status.NextAppQC = max(status.NextAppQC, first) status.First = max(status.First, first) - recent.Status = utils.Some(status) + suffix.Status = utils.Some(status) // Prune resources fully below status.First. - recent.CommitQCs = slices.DeleteFunc(recent.CommitQCs, func(qc *types.FullCommitQC) bool { + suffix.CommitQCs = slices.DeleteFunc(suffix.CommitQCs, func(qc *types.FullCommitQC) bool { return qc.QC().GlobalRange().Next <= status.First }) - recent.Blocks = slices.DeleteFunc(recent.Blocks, func(block types.RecentBlock) bool { + suffix.Blocks = slices.DeleteFunc(suffix.Blocks, func(block types.SuffixBlock) bool { return block.Number < status.First }) - recent.AppProposals = slices.DeleteFunc(recent.AppProposals, func(appProposal *types.AppProposal) bool { + suffix.AppProposals = slices.DeleteFunc(suffix.AppProposals, func(appProposal *types.AppProposal) bool { return appProposal.GlobalRange().Next <= status.First }) - recent.AppQCs = slices.DeleteFunc(recent.AppQCs, func(appQC *types.AppQC) bool { + suffix.AppQCs = slices.DeleteFunc(suffix.AppQCs, func(appQC *types.AppQC) bool { return appQC.Proposal().GlobalRange().Next <= status.First }) // Put resources in increasing order. - slices.Reverse(recent.CommitQCs) - slices.Reverse(recent.Blocks) - slices.Reverse(recent.AppProposals) - slices.Reverse(recent.AppQCs) - return recent, nil + slices.Reverse(suffix.CommitQCs) + slices.Reverse(suffix.Blocks) + slices.Reverse(suffix.AppProposals) + slices.Reverse(suffix.AppQCs) + return suffix, nil } func (s *blockDB) ReadBlockByNumber(n types.GlobalBlockNumber) (utils.Option[*types.Block], error) { diff --git a/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go b/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go index 028db28432..1f1871316a 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go +++ b/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go @@ -184,13 +184,13 @@ func TestLittblockStrandedBlockNotServedAfterRestart(t *testing.T) { require.True(t, qc.IsPresent(), "covering QC for served block %d must be readable", n) } - // Recent recovery data never includes stranded blocks, and every returned + // ReadSuffix never includes stranded blocks, and every returned // block has a covering QC. - recent, err := db3.ReadRecent() + suffix, err := db3.ReadSuffix() require.NoError(t, err) - for _, block := range recent.Blocks { + for _, block := range suffix.Blocks { n := block.Number - require.GreaterOrEqual(t, uint64(n), uint64(5), "recent data must not include stranded block %d", n) + require.GreaterOrEqual(t, uint64(n), uint64(5), "suffix data must not include stranded block %d", n) qc, err := db3.ReadQCByBlockNumber(n) require.NoError(t, err) require.True(t, qc.IsPresent(), "block %d must have a covering QC", n) @@ -345,7 +345,7 @@ func TestLittblockPruneIntoCohortRoundsDown(t *testing.T) { } // TestLittblockRefusesToOpenWithStrandedBlocks verifies the corruption guard in -// recoverReadFloors. The never-empty prune invariant guarantees at least one +// recoverWatermark. The never-empty prune invariant guarantees at least one // (block, QC) pair is always retained, so a store holding a block with no // surviving QC is corrupt (e.g. a QC WAL file removed out of band). Rather than // serve blocks it can no longer trust, the store refuses to open. diff --git a/sei-db/ledger_db/block/littblock_crash_test.go b/sei-db/ledger_db/block/littblock_crash_test.go index 59e088bbfe..857740b615 100644 --- a/sei-db/ledger_db/block/littblock_crash_test.go +++ b/sei-db/ledger_db/block/littblock_crash_test.go @@ -71,10 +71,10 @@ func TestLittblockNoBlockWithoutQCAfterTornTail(t *testing.T) { totalBlocks += len(b.blocks) } - recent, err := db2.ReadRecent() + suffix, err := db2.ReadSuffix() require.NoError(t, err) present := 0 - for _, block := range recent.Blocks { + for _, block := range suffix.Blocks { n := block.Number qc, err := db2.ReadQCByBlockNumber(n) require.NoError(t, err) @@ -213,10 +213,10 @@ func TestLittblockFlushSurvivesHardKill(t *testing.T) { totalBlocks += len(b.blocks) } - recent, err := db.ReadRecent() + suffix, err := db.ReadSuffix() require.NoError(t, err) present := 0 - for _, block := range recent.Blocks { + for _, block := range suffix.Blocks { n := block.Number qc, err := db.ReadQCByBlockNumber(n) require.NoError(t, err) diff --git a/sei-db/ledger_db/block/memblock/mem_block_db.go b/sei-db/ledger_db/block/memblock/mem_block_db.go index 159317561a..1466fb64a2 100644 --- a/sei-db/ledger_db/block/memblock/mem_block_db.go +++ b/sei-db/ledger_db/block/memblock/mem_block_db.go @@ -30,7 +30,7 @@ type blockDB struct { appQCsByBlock map[types.GlobalBlockNumber]*types.AppQC watermark types.GlobalBlockNumber - status utils.Option[types.DBStatus] + status utils.Option[types.SuffixRange] } // NewBlockDB returns an in-memory types.BlockDB. @@ -84,7 +84,7 @@ func (s *blockDB) WriteQC(qc *types.FullCommitQC) error { } if !ok { - status = types.DBStatus{ + status = types.SuffixRange{ First: gr.First, NextAppQC: gr.First, NextAppProposal: gr.First, @@ -207,35 +207,35 @@ func pruneRanges[T any]( func (s *blockDB) Flush() error { return nil } -func (s *blockDB) Status() utils.Option[types.DBStatus] { +func (s *blockDB) Status() utils.Option[types.SuffixRange] { s.mu.RLock() defer s.mu.RUnlock() return s.status } -func (s *blockDB) ReadRecent() (types.RecentData, error) { +func (s *blockDB) ReadSuffix() (types.Suffix, error) { s.mu.RLock() defer s.mu.RUnlock() status, ok := s.status.Get() if !ok { - return types.RecentData{}, nil + return types.Suffix{}, nil } - recent := types.RecentData{Status: utils.Some(status)} - recent.CommitQCs = appendSuffixRanges( + suffix := types.Suffix{Status: utils.Some(status)} + suffix.CommitQCs = appendSuffixRanges( s.qcsByBlock, status.First, status.NextQC, func(qc *types.FullCommitQC) types.GlobalRange { return qc.QC().GlobalRange() }, ) - recent.AppProposals = appendSuffixRanges( + suffix.AppProposals = appendSuffixRanges( s.appProposalsByBlock, status.First, status.NextAppProposal, func(appProposal *types.AppProposal) types.GlobalRange { return appProposal.GlobalRange() }, ) - recent.AppQCs = appendSuffixRanges( + suffix.AppQCs = appendSuffixRanges( s.appQCsByBlock, status.First, status.NextAppQC, @@ -243,10 +243,10 @@ func (s *blockDB) ReadRecent() (types.RecentData, error) { ) for n := status.First; n < status.NextBlock; n++ { if block, ok := s.blocksByNumber[n]; ok { - recent.Blocks = append(recent.Blocks, types.RecentBlock{Number: n, Block: block}) + suffix.Blocks = append(suffix.Blocks, types.SuffixBlock{Number: n, Block: block}) } } - return recent, nil + return suffix, nil } func appendSuffixRanges[T any]( diff --git a/sei-tendermint/autobahn/types/block_db.go b/sei-tendermint/autobahn/types/block_db.go index 6c1afdfe8a..b88c403ed9 100644 --- a/sei-tendermint/autobahn/types/block_db.go +++ b/sei-tendermint/autobahn/types/block_db.go @@ -190,20 +190,20 @@ type BlockDB interface { // Status returns a consistent snapshot of the in-memory write tips (no I/O). // None means the DB is empty. - Status() utils.Option[DBStatus] + Status() utils.Option[SuffixRange] - // ReadRecent returns the materialized startup-recovery suffix. + // ReadSuffix returns the materialized startup-recovery suffix. // // Implementations find the newest persisted AppQC, then collect all // CommitQCs whose Index is greater than or equal to that AppQC's // RoadIndex, and all blocks whose GlobalBlockNumber is greater than or // equal to that AppQC's GlobalRange.First. If no AppQC is present, - // ReadRecent returns all retained CommitQCs and blocks. + // ReadSuffix returns all retained CommitQCs and blocks. // // Returned CommitQCs and Blocks are in ascending GlobalBlockNumber order so // data.State can replay them directly. If AppQC is present, CommitQCs // starts with its matching CommitQC. - ReadRecent() (RecentData, error) + ReadSuffix() (Suffix, error) // ReadBlockByNumber returns the block at GlobalBlockNumber n. // @@ -283,12 +283,12 @@ type BlockDB interface { Close() error } -// DBStatus represents the suffix of BlockDB data that data.State can append to/would load on recovery. +// SuffixRange represents the suffix of BlockDB data that data.State can append to/would load on recovery. // Elements since the last anchor (last full row which contains AppQC,AppProposal,Block,QC) to // the tips persisted in the DB. These are the elements that would be loaded by data.State on restart -// via BlockDB.ReadRecent. +// via BlockDB.ReadSuffix. // First <= NextAppQC <= NextAppProposal <= NextBlock <= NextQC -type DBStatus struct { +type SuffixRange struct { // First is either NextAppQC, or NextAppQC-1, depending on whether there is at least 1 AppQC in the BlockDB. First GlobalBlockNumber // NextAppQC is one past the highest GlobalBlockNumber covered by the last @@ -307,20 +307,20 @@ type DBStatus struct { NextQC GlobalBlockNumber } -// RecentBlock is one block returned by BlockDB.ReadRecent. -type RecentBlock struct { +// SuffixBlock is one block returned by BlockDB.ReadSuffix. +type SuffixBlock struct { Number GlobalBlockNumber Block *Block } -// RecentData is the materialized suffix used by data.State startup recovery. -type RecentData struct { +// Suffix is the materialized suffix used by data.State startup recovery. +type Suffix struct { // Ranges of elements in the suffix. // None if the BlockDB is empty. - Status utils.Option[DBStatus] + Status utils.Option[SuffixRange] // Elements which constitute the suffix. CommitQCs []*FullCommitQC - Blocks []RecentBlock + Blocks []SuffixBlock AppProposals []*AppProposal AppQCs []*AppQC } diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index 107a2f3976..30a606a888 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -47,7 +47,7 @@ type inner struct { nextAppProposal types.GlobalBlockNumber nextBlock types.GlobalBlockNumber nextQC types.GlobalBlockNumber - persisted types.DBStatus + persisted types.SuffixRange anchor utils.AtomicSend[utils.Option[Anchor]] } @@ -198,15 +198,15 @@ func NewState(cfg *Config, blockDB types.BlockDB) (*State, error) { }, nil } -// loadFromBlockDB replays the recent persisted suffix from blockDB into s.inner. +// loadFromBlockDB replays the persisted suffix from blockDB into s.inner. // Called from NewState before any goroutines are spawned. func loadFromBlockDB(cfg *Config, blockDB types.BlockDB) (*inner, error) { - recent, err := blockDB.ReadRecent() + suffix, err := blockDB.ReadSuffix() if err != nil { - return nil, fmt.Errorf("blockDB.ReadRecent(): %w", err) + return nil, fmt.Errorf("blockDB.ReadSuffix(): %w", err) } firstBlock := cfg.Registry.FirstBlock() - status := recent.Status.Or(types.DBStatus{ + status := suffix.Status.Or(types.SuffixRange{ First: firstBlock, NextQC: firstBlock, NextAppProposal: firstBlock, @@ -228,12 +228,12 @@ func loadFromBlockDB(cfg *Config, blockDB types.BlockDB) (*inner, error) { persisted: status, anchor: utils.NewAtomicSend(utils.None[Anchor]()), } - for _, qc := range recent.CommitQCs { + for _, qc := range suffix.CommitQCs { if err := inner.insertQC(cfg.Registry, qc); err != nil { return nil, fmt.Errorf("load QC from BlockDB: %w", err) } } - for _, b := range recent.Blocks { + for _, b := range suffix.Blocks { qc := inner.qcs[b.Number] ei := qc.QC().Proposal().EpochIndex() e, ok := cfg.Registry.EpochByIndex(ei) @@ -250,12 +250,12 @@ func loadFromBlockDB(cfg *Config, blockDB types.BlockDB) (*inner, error) { // Advance nextBlock through contiguous loaded blocks. Don't use // updateNextBlock: stale timestamps would skew metrics. inner.nextBlock = max(inner.first, status.NextBlock) - for _, appProposal := range recent.AppProposals { + for _, appProposal := range suffix.AppProposals { if err := inner.insertAppProposal(appProposal); err != nil { return nil, fmt.Errorf("load AppProposal from BlockDB: %w", err) } } - for _, appQC := range recent.AppQCs { + for _, appQC := range suffix.AppQCs { if err := inner.insertAppQC(cfg.Registry, appQC); err != nil { return nil, fmt.Errorf("load AppQC from BlockDB: %w", err) } @@ -728,7 +728,7 @@ func (s *State) PruneBefore(retainFrom types.GlobalBlockNumber) error { // runPersist is a background goroutine that persists QCs, blocks, // AppProposals, and AppQCs to BlockDB. It waits for in-memory data to advance -// past the DBStatus persistence cursor, then writes each stream in cursor +// past the SuffixRange persistence cursor, then writes each stream in cursor // order and flushes once per batch. persisted.NextBlock advances with the // block tip to unblock PushAppHash only when data is durable. // Errors propagate vertically (kill the component). @@ -744,7 +744,7 @@ func (s *State) PruneBefore(retainFrom types.GlobalBlockNumber) error { // is still at or past NextQC (enough coverage for each new block, not every // in-memory QC, and no rewrite of QCs already on disk). // -// In-memory block/QC/AppProposal/AppQC eviction is driven by persisted DBStatus +// In-memory block/QC/AppProposal/AppQC eviction is driven by persisted SuffixRange // changes. func (s *State) runPersist(ctx context.Context) error { for { @@ -752,7 +752,7 @@ func (s *State) runPersist(ctx context.Context) error { var blocks []blockEntry var appProposals []*types.AppProposal var appQCs []*types.AppQC - var status types.DBStatus + var status types.SuffixRange for inner, ctrl := range s.inner.Lock() { status = inner.persisted // Wait until there is anything to persist. diff --git a/sei-tendermint/internal/autobahn/data/state_recovery_test.go b/sei-tendermint/internal/autobahn/data/state_recovery_test.go index 6bfe49bc8c..ad6c9405c7 100644 --- a/sei-tendermint/internal/autobahn/data/state_recovery_test.go +++ b/sei-tendermint/internal/autobahn/data/state_recovery_test.go @@ -436,7 +436,7 @@ func TestRecoveryQCsNoBlocks(t *testing.T) { // TestRunPersistSeedsFromRecoveryFloor verifies that runPersist does not walk // [genesis, recoveryFloor) for a QC-only store whose first QC starts past -// FirstBlock. Seeding persisted DBStatus from the recovery floor avoids +// FirstBlock. Seeding persisted SuffixRange from the recovery floor avoids // collecting nil block pointers. func TestRunPersistSeedsFromRecoveryFloor(t *testing.T) { ctx := t.Context() From 6262ca25a9072f1f24640a29d3a27639b0c56714 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Mon, 10 Aug 2026 11:30:49 +0200 Subject: [PATCH 42/61] WIP --- .../block/littblock/litt_block_db.go | 28 +++++++++--------- .../internal/autobahn/avail/subscriptions.go | 4 +-- .../internal/autobahn/data/state.go | 29 +++++++++---------- sei-tendermint/internal/p2p/giga/data.go | 16 +++++++--- 4 files changed, 40 insertions(+), 37 deletions(-) diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index 3351abec45..9087e287af 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -28,7 +28,7 @@ type blockDB struct { // watermark is a retention floor, always a QC boundary (a GlobalRange().First): // PruneBefore rounds a requested prune point down to the start of the cohort - // containing it, and startup re-derives it as the lowest surviving QC's First + // containing it, and startup re-derives it as the lowest surviving FullCommitQC's First // (see clampPruneBoundary and recoverWatermark). Keeping it on a cohort boundary // is what makes a QC's blocks change readability atomically — the gate never // splits a cohort. @@ -94,13 +94,12 @@ func NewBlockDB(config *LittBlockConfig) (types.BlockDB, error) { } // recoverWatermark re-derives the read watermark on open from the oldest -// surviving QC. It is in-memory only, so a restart forgets every PruneBefore. +// surviving FullCommitQC. It is in-memory only, so a restart forgets every PruneBefore. // That is fine for reclamation (nothing new is deleted), but we must protect // against showing un-pruned blocks with pruned QCs. // -// QCs are written before the blocks they cover, so the oldest surviving record -// is normally a QC and the first block follows shortly after. If no QC survives -// but the table is non-empty, the store is corrupt and must not reopen. +// FullCommitQCs are written before any other data for the same index. +// If no FullCommitQC survives but the table is non-empty, the store is corrupt and must not reopen. func (s *blockDB) recoverWatermark() error { it, err := s.table.Iterator(false) if err != nil { @@ -130,11 +129,7 @@ func (s *blockDB) recoverWatermark() error { s.watermark.Store(uint64(decodeNumberKey(key))) return nil } - // No QC survives. The never-empty prune invariant guarantees at least one - // (block, QC) pair is always retained, so blocks-without-QC is unreachable - // through normal operation — it means the store is corrupt (e.g. a QC WAL - // file was removed out of band). Refuse to open rather than serve blocks we - // can no longer trust. + // No FullCommitQC survives. Refuse to open rather than serve blocks we can no longer trust. if !empty { return fmt.Errorf("corrupt store: no QC in non-empty store") } @@ -148,7 +143,7 @@ func (s *blockDB) WriteBlock(n types.GlobalBlockNumber, blk *types.Block) error // A covering QC must already be written. Since QCs are contiguous and blocks // strictly ascending, n is covered iff n < status.NextQC. This guard also fixes // the QC-before-block write order: the covering QC's Put has already issued - // under this mutex, so on a crash a surviving block implies a surviving QC. + // under this mutex, so on a crash a surviving block implies a surviving FullCommitQC. if !ok || n >= status.NextQC { return fmt.Errorf("block number %d not covered by any written QC: %w", n, types.ErrBlockMissingQC) } @@ -369,8 +364,11 @@ func (s *blockDB) Status() utils.Option[types.SuffixRange] { } // ReadSuffix reads the materialized startup-recovery suffix. -// WARNING: ReadSuffix() will return an error if watermark is moved during iteration. func (s *blockDB) ReadSuffix() (types.Suffix, error) { + // Suffix is computed under lock, so that GC cannot malform it: + // locked => no new data can be appended => watermark doesn't move => GC doesn't consume data of the suffix.:w + s.mu.Lock() + defer s.mu.Unlock() it, err := s.table.Iterator(true) if err != nil { return types.Suffix{}, fmt.Errorf("failed to open suffix iterator: %w", err) @@ -482,7 +480,7 @@ func (s *blockDB) ReadSuffix() (types.Suffix, error) { } func (s *blockDB) ReadBlockByNumber(n types.GlobalBlockNumber) (utils.Option[*types.Block], error) { - // Refuse below-watermark blocks: they may be stranded (covering QC reclaimed). + // Data below watermark should not be visible to the caller, even though it is pruned asynchronously. if uint64(n) < s.watermark.Load() { return utils.None[*types.Block](), types.ErrPruned } @@ -501,8 +499,8 @@ func (s *blockDB) ReadBlockByHash(hash types.BlockHeaderHash) (utils.Option[type if err != nil { return utils.None[types.BlockWithNumber](), err } - // The number is not known until the block is resolved; refuse it if it turns - // out to be below the watermark (potentially stranded from its covering QC). + // The number is not known until the block is resolved; + // Data below watermark should not be visible to the caller, even though it is pruned asynchronously. if bwn, ok := result.Get(); ok && uint64(bwn.Number) < s.watermark.Load() { return utils.None[types.BlockWithNumber](), nil } diff --git a/sei-tendermint/internal/autobahn/avail/subscriptions.go b/sei-tendermint/internal/autobahn/avail/subscriptions.go index 51b0f1e532..8883ff6010 100644 --- a/sei-tendermint/internal/autobahn/avail/subscriptions.go +++ b/sei-tendermint/internal/autobahn/avail/subscriptions.go @@ -76,7 +76,7 @@ type AppVotesRecv struct { } func (s *State) SubscribeAppVotes() *AppVotesRecv { - return &AppVotesRecv{s, s.data.First()} + return &AppVotesRecv{s, s.data.NextAppQC()} } func (r *AppVotesRecv) Recv(ctx context.Context) (*types.Signed[*types.AppVote], error) { @@ -84,7 +84,7 @@ func (r *AppVotesRecv) Recv(ctx context.Context) (*types.Signed[*types.AppVote], vote, err := r.state.data.AppVote(ctx, r.next) if err != nil { if errors.Is(err, types.ErrPruned) { - r.next = max(r.next, r.state.data.First()) + r.next = max(r.next, r.state.data.NextAppQC()) continue } return nil, err diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index 30a606a888..a74384431d 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -265,10 +265,7 @@ func loadFromBlockDB(cfg *Config, blockDB types.BlockDB) (*inner, error) { } func (s *State) First() types.GlobalBlockNumber { - for inner := range s.inner.Lock() { - return inner.first - } - panic("unreachable") + return max(s.blockDB.First(),s.cfg.Registry.FirstBlock()) } // Registry returns the epoch registry. @@ -412,6 +409,14 @@ func (s *State) NextBlock() types.GlobalBlockNumber { panic("unreachable") } +// NextBlock returns the index of the next block to be pushed. +func (s *State) NextAppQC() types.GlobalBlockNumber { + for inner := range s.inner.Lock() { + return inner.nextAppQC + } + panic("unreachable") +} + // GlobalBlockByHash returns the finalized GlobalBlock whose stored header // hashes to the given value, or None if no such block is currently retained. // Non-blocking. Serves from RAM whenever the hash is still indexed (contiguous @@ -657,24 +662,16 @@ func (s *State) PushAppQC(ctx context.Context, appQC *types.AppQC) error { panic("unreachable") } -func (s *State) AppQC(ctx context.Context, n types.GlobalBlockNumber) (*types.AppQC, *types.FullCommitQC, error) { +func (s *State) AppQC(ctx context.Context, n types.GlobalBlockNumber) (*types.AppQC, error) { for inner, ctrl := range s.inner.Lock() { if err := ctrl.WaitUntil(ctx, func() bool { return n < inner.nextAppQC }); err != nil { - return nil, nil, err + return nil, err } if inner.first <= n { - return inner.appQCs[n], inner.qcs[n], nil + return inner.appQCs[n], nil } } - qc, err := s.qcFromDB(n) - if err != nil { - return nil, nil, err - } - appQC, err := s.appQCFromDB(n) - if err != nil { - return nil, nil, err - } - return appQC, qc, nil + return s.appQCFromDB(n) } type Anchor struct { diff --git a/sei-tendermint/internal/p2p/giga/data.go b/sei-tendermint/internal/p2p/giga/data.go index 9cd6de84c8..a7d14ff612 100644 --- a/sei-tendermint/internal/p2p/giga/data.go +++ b/sei-tendermint/internal/p2p/giga/data.go @@ -48,7 +48,9 @@ func (x *Service) clientStreamAppQCs(ctx context.Context, c rpc.Client[API]) err return fmt.Errorf("client.StreamAppQCs(): %w", err) } defer stream.Close() - if err := stream.Send(ctx, &pb.StreamAppQCsReq{}); err != nil { + if err := stream.Send(ctx, StreamAppQCsReqConv.Encode(&StreamAppQCsReq{ + NextBlock: x.data.NextAppQC(), + })); err != nil { return err } for { @@ -184,6 +186,9 @@ func (s *Service) serverStreamFullCommitQCs(ctx context.Context, server rpc.Serv for next := req.NextBlock; ; { qc, err := s.data.QC(ctx, next) if err != nil { + if errors.Is(err,types.ErrPruned) { + next = s.data.First() + } return fmt.Errorf("s.data.QC(): %w", err) } // Don't send the same QC twice. @@ -195,7 +200,7 @@ func (s *Service) serverStreamFullCommitQCs(ctx context.Context, server rpc.Serv }) } -func (x *Service) serverStreamAppQCs(ctx context.Context, server rpc.Server[API]) error { +func (s *Service) serverStreamAppQCs(ctx context.Context, server rpc.Server[API]) error { return StreamAppQCs.Serve(ctx, server, func(ctx context.Context, stream rpc.Stream[*apb.AppQC, *pb.StreamAppQCsReq]) error { reqRaw, err := stream.Recv(ctx) if err != nil { @@ -206,11 +211,14 @@ func (x *Service) serverStreamAppQCs(ctx context.Context, server rpc.Server[API] return fmt.Errorf("StreamFullCommitQCsReqConv.Decode(): %w", err) } for next := req.NextBlock; ; { - appQC, commitQC, err := x.validatorState().Data().AppQC(ctx, next) + appQC, err := s.data.AppQC(ctx, next) if err != nil { + if errors.Is(err,types.ErrPruned) { + next = s.data.First() + } return fmt.Errorf("x.validatorState().Data().AppQC(): %w", err) } - next = commitQC.QC().GlobalRange().Next + next = appQC.Proposal().GlobalRange().Next if err := stream.Send(ctx, types.AppQCConv.Encode(appQC)); err != nil { return fmt.Errorf("stream.Send(): %w", err) } From 520f74d57aa0956196d2ec48892317babbcc0e1a Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Mon, 10 Aug 2026 11:37:59 +0200 Subject: [PATCH 43/61] fixed data.State.First() --- sei-db/ledger_db/block/littblock/codec_test.go | 2 +- sei-db/ledger_db/block/littblock/litt_block_db.go | 12 ++++++++---- sei-db/ledger_db/block/memblock/mem_block_db.go | 6 ++++++ sei-tendermint/autobahn/types/block_db.go | 4 ++++ sei-tendermint/internal/autobahn/data/state.go | 2 +- sei-tendermint/internal/autobahn/data/state_test.go | 4 +++- sei-tendermint/internal/p2p/giga/consensus_test.go | 2 +- 7 files changed, 24 insertions(+), 8 deletions(-) diff --git a/sei-db/ledger_db/block/littblock/codec_test.go b/sei-db/ledger_db/block/littblock/codec_test.go index 3c5aff9eca..5eee568cfe 100644 --- a/sei-db/ledger_db/block/littblock/codec_test.go +++ b/sei-db/ledger_db/block/littblock/codec_test.go @@ -23,7 +23,7 @@ func TestKeyRoundTrip(t *testing.T) { for _, n := range cases { key := encodeKey(n) require.Len(t, key, 8, "key must be 8 bytes") - require.Equal(t, n, decodeKey(key)) + require.Equal(t, n, decodeKey([8]byte(key))) } } diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index 9087e287af..25b539e7f7 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -90,7 +90,7 @@ func NewBlockDB(config *LittBlockConfig) (types.BlockDB, error) { return nil, fmt.Errorf("ReadSuffix(): %w", err) } s.status = suffix.Status - return s,nil + return s, nil } // recoverWatermark re-derives the read watermark on open from the oldest @@ -310,6 +310,10 @@ func (s *blockDB) PruneBefore(blockHeight types.GlobalBlockNumber) error { return nil } +func (s *blockDB) First() types.GlobalBlockNumber { + return types.GlobalBlockNumber(s.watermark.Load()) +} + // clampPruneBoundary returns the start of the QC that covers n, or n if there is no QC covering N // (which can happen if you prune the same n twice). func (s *blockDB) clampPruneBoundary(blockHeight types.GlobalBlockNumber) (types.GlobalBlockNumber, error) { @@ -480,7 +484,7 @@ func (s *blockDB) ReadSuffix() (types.Suffix, error) { } func (s *blockDB) ReadBlockByNumber(n types.GlobalBlockNumber) (utils.Option[*types.Block], error) { - // Data below watermark should not be visible to the caller, even though it is pruned asynchronously. + // Data below watermark should not be visible to the caller, even though it is pruned asynchronously. if uint64(n) < s.watermark.Load() { return utils.None[*types.Block](), types.ErrPruned } @@ -499,8 +503,8 @@ func (s *blockDB) ReadBlockByHash(hash types.BlockHeaderHash) (utils.Option[type if err != nil { return utils.None[types.BlockWithNumber](), err } - // The number is not known until the block is resolved; - // Data below watermark should not be visible to the caller, even though it is pruned asynchronously. + // The number is not known until the block is resolved; + // Data below watermark should not be visible to the caller, even though it is pruned asynchronously. if bwn, ok := result.Get(); ok && uint64(bwn.Number) < s.watermark.Load() { return utils.None[types.BlockWithNumber](), nil } diff --git a/sei-db/ledger_db/block/memblock/mem_block_db.go b/sei-db/ledger_db/block/memblock/mem_block_db.go index 1466fb64a2..f682fc7361 100644 --- a/sei-db/ledger_db/block/memblock/mem_block_db.go +++ b/sei-db/ledger_db/block/memblock/mem_block_db.go @@ -193,6 +193,12 @@ func (s *blockDB) PruneBefore(n types.GlobalBlockNumber) error { return nil } +func (s *blockDB) First() types.GlobalBlockNumber { + s.mu.RLock() + defer s.mu.RUnlock() + return s.watermark +} + func pruneRanges[T any]( watermark types.GlobalBlockNumber, byBlock map[types.GlobalBlockNumber]T, diff --git a/sei-tendermint/autobahn/types/block_db.go b/sei-tendermint/autobahn/types/block_db.go index b88c403ed9..720ee32121 100644 --- a/sei-tendermint/autobahn/types/block_db.go +++ b/sei-tendermint/autobahn/types/block_db.go @@ -171,6 +171,10 @@ type BlockDB interface { // reclaimed — pruned entries may remain readable for a while. PruneBefore(n GlobalBlockNumber) error + // First returns the number of the oldest accessible row. + // Moved by PruneBefore. + First() GlobalBlockNumber + // Flush blocks until every Write that has returned before Flush is // called is durable on disk. Writes made concurrently with Flush // may or may not be durable when Flush returns (but are otherwise diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index a74384431d..b5e11a4ca0 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -265,7 +265,7 @@ func loadFromBlockDB(cfg *Config, blockDB types.BlockDB) (*inner, error) { } func (s *State) First() types.GlobalBlockNumber { - return max(s.blockDB.First(),s.cfg.Registry.FirstBlock()) + return max(s.blockDB.First(), s.cfg.Registry.FirstBlock()) } // Registry returns the epoch registry. diff --git a/sei-tendermint/internal/autobahn/data/state_test.go b/sei-tendermint/internal/autobahn/data/state_test.go index 0dfb7f9e6d..c3fbcc6ab0 100644 --- a/sei-tendermint/internal/autobahn/data/state_test.go +++ b/sei-tendermint/internal/autobahn/data/state_test.go @@ -751,7 +751,9 @@ func TestPushAppQCPersistsAndRecovers(t *testing.T) { require.Equal(t, gr1.Next, inner.nextAppProposal) require.Equal(t, gr1.Next, inner.nextAppQC) } - appQC, fQC, err := state2.AppQC(ctx, gr1.First) + appQC, err := state2.AppQC(ctx, gr1.First) + require.NoError(t, err) + fQC, err := state2.QC(ctx, gr1.First) require.NoError(t, err) require.Equal(t, gr1, appQC.Proposal().GlobalRange()) require.Equal(t, gr1, fQC.QC().GlobalRange()) diff --git a/sei-tendermint/internal/p2p/giga/consensus_test.go b/sei-tendermint/internal/p2p/giga/consensus_test.go index 50a7b09cf7..8d5d4169ca 100644 --- a/sei-tendermint/internal/p2p/giga/consensus_test.go +++ b/sei-tendermint/internal/p2p/giga/consensus_test.go @@ -65,7 +65,7 @@ func TestConsensusClientServer(t *testing.T) { for _, n := range nodes { t.Logf("[%v] Wait for AppHash consensus.", idx) p := wantAppProposal.OrPanic("missing app proposal") - got, _, err := n.data.AppQC(ctx, idx) + got, err := n.data.AppQC(ctx, idx) if err != nil { return fmt.Errorf("cs.avail.WaitForAppQC(): %w", err) } From 388c57ddfe46af90fe569801de7c65d7e39db40b Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Mon, 10 Aug 2026 12:34:34 +0200 Subject: [PATCH 44/61] applied comments WIP --- sei-tendermint/autobahn/types/block_db.go | 38 ++++++++++--------- sei-tendermint/internal/p2p/giga/data.go | 4 +- sei-tendermint/internal/p2p/giga/service.go | 31 +++++++-------- .../internal/p2p/giga_router_fullnode.go | 4 +- 4 files changed, 38 insertions(+), 39 deletions(-) diff --git a/sei-tendermint/autobahn/types/block_db.go b/sei-tendermint/autobahn/types/block_db.go index 720ee32121..dd0a59da7a 100644 --- a/sei-tendermint/autobahn/types/block_db.go +++ b/sei-tendermint/autobahn/types/block_db.go @@ -39,31 +39,33 @@ import ( // Writes must be ordered, and the contract is enforced (not merely // expected): // -// - Blocks must be written densely: each block's number must be exactly -// one greater than the previously written block's (the first block may -// start anywhere its covering QC allows). WriteBlock returns -// ErrBlockOutOfOrder otherwise. -// - QCs must be written contiguously — each QC's GlobalRange().First -// must equal the previous QC's GlobalRange().Next. WriteQC returns -// ErrQCNonContiguous otherwise. -// - QCs must be written before blocks. A QC covering a block must -// be written before that block is written. -// - AppQCs must be written contiguously as an exact prefix of retained QCs. -// The first AppQC starts at the retained QC floor; every AppQC's range must -// exactly match the next persisted QC range. +// - All data must be written contiguously: +// - FullCommitQC.GlobalRange().First must equal the previous FullCommitQC.GlobalRange().Next +// - Block.Number must equal the previous Block.Number + 1 +// - AppProposal.GlobalRange().First must equal the previous AppProposal.GlobalRange().Next +// - AppQC.Proposal().GlobalRange().First must equal the previous AppQC.Proposal().GlobalRange().Next +// - All data must be written in order: for every GlobalBlockNumber X, writes need to happen in order: +// - FullCommitQC covering X +// - Block X +// - AppProposal covering X +// - AppQC covering X +// - Call to PruneBefore(X) makes data before X inaccessible, however it still might be accessible after BlockDB is +// reopened, as the pruning may happen asynchronously. +// Only full rows (X such that FullCommitQC, Block, AppProposal and AppQC are in BlockDB) are eligible for pruning, +// and at least 1 full row (once written) needs to stay in BlockDB at all times. +// In particular: +// - PruneBefore is a noop until the first AppQC is written +// - If PruneBefore(X) called and the highest full row is Y, then only data in rows Date: Mon, 10 Aug 2026 13:24:23 +0200 Subject: [PATCH 45/61] fixed --- .../internal/autobahn/avail/inner.go | 7 ++- .../internal/autobahn/avail/state_test.go | 52 +++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/sei-tendermint/internal/autobahn/avail/inner.go b/sei-tendermint/internal/autobahn/avail/inner.go index a707b343b8..6cda2e1bb2 100644 --- a/sei-tendermint/internal/autobahn/avail/inner.go +++ b/sei-tendermint/internal/autobahn/avail/inner.go @@ -76,6 +76,7 @@ func newInner(ds *data.State, loaded *loadedState) (*inner, error) { // Restore persisted CommitQCs. prune() may have already pushed the // anchor's CommitQC, so skip entries below commitQCs.next. + setPersisted := false for _, qc := range loaded.commitQCs { if qc.Index() < i.roads.next { continue @@ -88,8 +89,12 @@ func newInner(ds *data.State, loaded *loadedState) (*inner, error) { return nil, fmt.Errorf("epoch not found") } i.roads.pushBack(newRoad(qc, epoch)) + setPersisted = true } - if i.roads.Len() > 0 { + // It may happen that data.State has progressed beyond avail state. + // In this case the whole persisted avail.State is invalidated and anchor.CommitQC + // is NOT stored in avail.State. We need it to get persisted before we update persistedCommitQC. + if setPersisted { i.persistedCommitQC.Store(utils.Some(i.roads.q[i.roads.next-1].commitQC)) } diff --git a/sei-tendermint/internal/autobahn/avail/state_test.go b/sei-tendermint/internal/autobahn/avail/state_test.go index c5c051df66..9943768236 100644 --- a/sei-tendermint/internal/autobahn/avail/state_test.go +++ b/sei-tendermint/internal/autobahn/avail/state_test.go @@ -62,6 +62,58 @@ func TestStateWithPersistence(t *testing.T) { } } +func TestNewStateDoesNotPublishDataAnchorAsPersistedCommitQC(t *testing.T) { + ctx := t.Context() + rng := utils.TestRng() + registry, keys := epoch.GenRegistry(rng, 3) + ds := newTestDataState(&data.Config{Registry: registry}) + qc, blocks := data.TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) + gr := qc.QC().GlobalRange() + appHash := types.GenAppHash(rng) + appProposal := types.NewAppProposal(qc.QC().Proposal(), appHash) + + require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { + runCtx, cancel := context.WithCancel(ctx) + defer cancel() + s.SpawnBgNamed("data.Run", func() error { return utils.IgnoreCancel(ds.Run(runCtx)) }) + if err := ds.PushQC(ctx, qc, blocks); err != nil { + return err + } + for n := gr.First; n < gr.Next; n++ { + if err := ds.PushAppHash(ctx, n, appHash); err != nil { + return err + } + } + if err := ds.PushAppQC(ctx, data.TestAppQC(keys, appProposal)); err != nil { + return err + } + _, err := ds.Anchor().Wait(ctx, func(anchor utils.Option[data.Anchor]) bool { + a, ok := anchor.Get() + return ok && a.CommitQC.Index() == qc.QC().Index() + }) + return err + })) + + state, err := NewState(keys[0], ds, utils.None[string]()) + require.NoError(t, err) + _, ok := state.LastCommitQC().Load().Get() + require.False(t, ok, "data anchor is not persisted avail state until avail.Run writes it") + got, err := state.CommitQC(ctx, qc.QC().Index()) + require.NoError(t, err) + require.NoError(t, utils.TestDiff(qc.QC(), got)) + + require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { + runCtx, cancel := context.WithCancel(ctx) + defer cancel() + s.SpawnBgNamed("avail.Run", func() error { return utils.IgnoreCancel(state.Run(runCtx)) }) + _, err := state.LastCommitQC().Wait(ctx, func(got utils.Option[*types.CommitQC]) bool { + gotQC, ok := got.Get() + return ok && gotQC.Index() == qc.QC().Index() + }) + return err + })) +} + func testState(t *testing.T, rng utils.Rng, stateDir utils.Option[string]) { t.Helper() ctx := t.Context() From b1da655239b0e97ec16af67c42754d958c7a5ffd Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Mon, 10 Aug 2026 17:08:37 +0200 Subject: [PATCH 46/61] anchor excluded from the avail state --- sei-tendermint/autobahn/types/proposal.go | 4 ++ .../internal/autobahn/avail/app_votes.go | 8 ++- .../internal/autobahn/avail/inner.go | 41 +++-------- .../internal/autobahn/avail/inner_test.go | 1 - .../internal/autobahn/avail/state.go | 27 ++++---- .../internal/autobahn/avail/state_test.go | 68 ++++++++++--------- 6 files changed, 71 insertions(+), 78 deletions(-) diff --git a/sei-tendermint/autobahn/types/proposal.go b/sei-tendermint/autobahn/types/proposal.go index e44825d7bc..197688ada0 100644 --- a/sei-tendermint/autobahn/types/proposal.go +++ b/sei-tendermint/autobahn/types/proposal.go @@ -68,6 +68,10 @@ type GlobalRange struct { Next GlobalBlockNumber } +func (g GlobalRange) String() string { + return fmt.Sprintf("[%v,%v)",g.First,g.Next) +} + // Len returns the number of global blocks in the range. func (g GlobalRange) Len() uint64 { return uint64(g.Next - g.First) diff --git a/sei-tendermint/internal/autobahn/avail/app_votes.go b/sei-tendermint/internal/autobahn/avail/app_votes.go index a3860af7af..b80ae9107e 100644 --- a/sei-tendermint/internal/autobahn/avail/app_votes.go +++ b/sei-tendermint/internal/autobahn/avail/app_votes.go @@ -33,13 +33,13 @@ func newRoad(commitQC *types.CommitQC, epoch *types.Epoch) *road { } // Returns qc if a new qc has been reached. -func (r *road) pushAppVote(vote *types.Signed[*types.AppVote]) { +func (r *road) pushAppVote(vote *types.Signed[*types.AppVote]) bool { if r.appQC.IsPresent() { - return + return false } k := vote.Key() if _, ok := r.appByKey[k]; ok { - return + return false } r.appByKey[k] = struct{}{} byHash, ok := r.appByHash[vote.Hash()] @@ -56,5 +56,7 @@ func (r *road) pushAppVote(vote *types.Signed[*types.AppVote]) { byHash.votes = append(byHash.votes, vote) if byHash.weight >= c.AppQuorum() { r.appQC = utils.Some(types.NewAppQC(byHash.votes)) + return true } + return false } diff --git a/sei-tendermint/internal/autobahn/avail/inner.go b/sei-tendermint/internal/autobahn/avail/inner.go index 6cda2e1bb2..2b78870ba6 100644 --- a/sei-tendermint/internal/autobahn/avail/inner.go +++ b/sei-tendermint/internal/autobahn/avail/inner.go @@ -17,7 +17,6 @@ import ( type inner struct { persistedCommitQC utils.AtomicSend[utils.Option[*types.CommitQC]] // latest persisted CommitQC roads *queue[types.RoadIndex, *road] - nextAppQC types.RoadIndex // Epoch is the current epoch for blocks votes collection. epoch *types.Epoch @@ -67,16 +66,11 @@ func newInner(ds *data.State, loaded *loadedState) (*inner, error) { // Apply the persisted prune anchor from the data.State: // avail.State can drop everything below AppQC persisted in data.State. if anchor, ok := ds.Anchor().Load().Get(); ok { - epoch, ok := ds.Registry().EpochByIndex(anchor.CommitQC.Proposal().EpochIndex()) - if !ok { - return nil, fmt.Errorf("epoch not found") - } - i.prune(epoch, anchor) + i.prune(anchor) } // Restore persisted CommitQCs. prune() may have already pushed the // anchor's CommitQC, so skip entries below commitQCs.next. - setPersisted := false for _, qc := range loaded.commitQCs { if qc.Index() < i.roads.next { continue @@ -89,12 +83,11 @@ func newInner(ds *data.State, loaded *loadedState) (*inner, error) { return nil, fmt.Errorf("epoch not found") } i.roads.pushBack(newRoad(qc, epoch)) - setPersisted = true } // It may happen that data.State has progressed beyond avail state. // In this case the whole persisted avail.State is invalidated and anchor.CommitQC // is NOT stored in avail.State. We need it to get persisted before we update persistedCommitQC. - if setPersisted { + if i.roads.Len() > 0 { i.persistedCommitQC.Store(utils.Some(i.roads.q[i.roads.next-1].commitQC)) } @@ -141,35 +134,23 @@ func (i *inner) laneQC(lane types.LaneID, n types.BlockNumber) (*types.LaneQC, b return nil, false } -func (i *inner) updateNextAppQC() bool { - updated := false - for i.nextAppQC < i.roads.next && i.roads.q[i.nextAppQC].appQC.IsPresent() { - i.nextAppQC += 1 - updated = true - } - return updated -} - // prune advances the state up to Anchor of the data state. // Returns true iff pruning occurred. -func (i *inner) prune(epoch *types.Epoch, anchor data.Anchor) { +func (i *inner) prune(anchor data.Anchor) { idx := anchor.CommitQC.Index() if idx < i.roads.first { return } - i.roads.prune(idx) - i.nextAppQC = max(idx, i.nextAppQC) - if idx == i.roads.next { - i.roads.pushBack(newRoad(anchor.CommitQC, epoch)) - } - i.roads.q[idx].appQC = utils.Some(anchor.AppQC) - i.updateNextAppQC() + i.roads.prune(idx + 1) for lane := range i.votes { lr := anchor.CommitQC.LaneRange(lane) - i.votes[lr.Lane()].prune(lr.First()) - i.blocks[lr.Lane()].prune(lr.First()) - if i.nextBlockToPersist[lr.Lane()] < lr.First() { - i.nextBlockToPersist[lr.Lane()] = lr.First() + i.votes[lr.Lane()].prune(lr.Next()) + i.blocks[lr.Lane()].prune(lr.Next()) + if i.nextBlockToPersist[lr.Lane()] < lr.Next() { + i.nextBlockToPersist[lr.Lane()] = lr.Next() } } + if i.roads.Len() == 0 { + i.persistedCommitQC.Store(utils.Some(anchor.CommitQC)) + } } diff --git a/sei-tendermint/internal/autobahn/avail/inner_test.go b/sei-tendermint/internal/autobahn/avail/inner_test.go index ea6e92895e..d040076d1f 100644 --- a/sei-tendermint/internal/autobahn/avail/inner_test.go +++ b/sei-tendermint/internal/autobahn/avail/inner_test.go @@ -28,7 +28,6 @@ func TestNewInnerFreshStart(t *testing.T) { i, err := newInner(newTestDataState(&data.Config{Registry: registry}), &loadedState{}) require.NoError(t, err) - require.Equal(t, types.RoadIndex(0), i.nextAppQC) require.Equal(t, types.RoadIndex(0), i.roads.first) require.Equal(t, types.RoadIndex(0), i.roads.next) require.NotNil(t, i.nextBlockToPersist) diff --git a/sei-tendermint/internal/autobahn/avail/state.go b/sei-tendermint/internal/autobahn/avail/state.go index 23f84c25c7..9610eceeef 100644 --- a/sei-tendermint/internal/autobahn/avail/state.go +++ b/sei-tendermint/internal/autobahn/avail/state.go @@ -110,13 +110,19 @@ func (s *State) LastCommitQC() utils.AtomicRecv[utils.Option[*types.CommitQC]] { func (s *State) appQC(ctx context.Context, idx types.RoadIndex) (*types.AppQC, error) { for inner, ctrl := range s.inner.Lock() { - if err := ctrl.WaitUntil(ctx, func() bool { return idx < inner.nextAppQC }); err != nil { - return nil, err - } - if idx < inner.roads.first { - return nil, types.ErrPruned + for { + if idx < inner.roads.first { + return nil, types.ErrPruned + } + if idx < inner.roads.next { + if appQC, ok := inner.roads.q[idx].appQC.Get(); ok { + return appQC, nil + } + } + if err := ctrl.Wait(ctx); err != nil { + return nil, err + } } - return inner.roads.q[idx].appQC.OrPanic("missing appQC"), nil } panic("unreachable") } @@ -195,8 +201,7 @@ func (s *State) PushAppVote(ctx context.Context, v *types.Signed[*types.AppVote] if idx < inner.roads.first || inner.roads.next <= idx { return nil } - inner.roads.q[idx].pushAppVote(v) - if inner.updateNextAppQC() { + if inner.roads.q[idx].pushAppVote(v) { ctrl.Updated() } } @@ -510,11 +515,7 @@ func (s *State) runEvict(ctx context.Context) error { return s.data.Anchor().Iter(ctx, func(ctx context.Context, anchor utils.Option[data.Anchor]) error { if anchor, ok := anchor.Get(); ok { for inner, ctrl := range s.inner.Lock() { - epoch, ok := s.data.Registry().EpochByIndex(anchor.CommitQC.Proposal().EpochIndex()) - if !ok { - return fmt.Errorf("epoch not found") - } - inner.prune(epoch, anchor) + inner.prune(anchor) ctrl.Updated() } } diff --git a/sei-tendermint/internal/autobahn/avail/state_test.go b/sei-tendermint/internal/autobahn/avail/state_test.go index 9943768236..ddad0bc0c2 100644 --- a/sei-tendermint/internal/autobahn/avail/state_test.go +++ b/sei-tendermint/internal/autobahn/avail/state_test.go @@ -62,51 +62,57 @@ func TestStateWithPersistence(t *testing.T) { } } -func TestNewStateDoesNotPublishDataAnchorAsPersistedCommitQC(t *testing.T) { +// Test checking that State can correctly start collecting CommitQCs starting from arbitrary anchor. +func TestAnchorResetsState(t *testing.T) { ctx := t.Context() rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 3) - ds := newTestDataState(&data.Config{Registry: registry}) - qc, blocks := data.TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) - gr := qc.QC().GlobalRange() - appHash := types.GenAppHash(rng) - appProposal := types.NewAppProposal(qc.QC().Proposal(), appHash) - + epoch := registry.LatestEpoch() require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { - runCtx, cancel := context.WithCancel(ctx) - defer cancel() - s.SpawnBgNamed("data.Run", func() error { return utils.IgnoreCancel(ds.Run(runCtx)) }) + t.Logf("data.Run()") + ds := newTestDataState(&data.Config{Registry: registry}) + s.SpawnBgNamed("data.Run", func() error { return utils.IgnoreCancel(ds.Run(ctx)) }) + + t.Logf("Push FullCommitQC, blocks, AppHash, AppQC to data") + qc, blocks := data.TestCommitQC(rng, epoch, keys, utils.None[*types.CommitQC]()) if err := ds.PushQC(ctx, qc, blocks); err != nil { return err } - for n := gr.First; n < gr.Next; n++ { - if err := ds.PushAppHash(ctx, n, appHash); err != nil { - return err - } + appHash := types.GenAppHash(rng) + if err := ds.PushAppHash(ctx, qc.QC().GlobalRange().Next-1, appHash); err != nil { + return err } - if err := ds.PushAppQC(ctx, data.TestAppQC(keys, appProposal)); err != nil { + appQC := data.TestAppQC(keys, types.NewAppProposal(qc.QC().Proposal(), appHash)) + if err := ds.PushAppQC(ctx, appQC); err != nil { return err } - _, err := ds.Anchor().Wait(ctx, func(anchor utils.Option[data.Anchor]) bool { + + t.Logf("wait for anchor to be updated") + if _, err := ds.Anchor().Wait(ctx, func(anchor utils.Option[data.Anchor]) bool { a, ok := anchor.Get() return ok && a.CommitQC.Index() == qc.QC().Index() - }) - return err - })) + }); err != nil { + return err + } - state, err := NewState(keys[0], ds, utils.None[string]()) - require.NoError(t, err) - _, ok := state.LastCommitQC().Load().Get() - require.False(t, ok, "data anchor is not persisted avail state until avail.Run writes it") - got, err := state.CommitQC(ctx, qc.QC().Index()) - require.NoError(t, err) - require.NoError(t, utils.TestDiff(qc.QC(), got)) + t.Logf("NewState() should load the anchor") + state, err := NewState(keys[0], ds, utils.None[string]()) + if err != nil { + return fmt.Errorf("NewState(): %w", err) + } + if err := utils.TestDiff(utils.Some(qc.QC()), state.LastCommitQC().Load()); err != nil { + return err + } + t.Logf("avail.Run()") + s.SpawnBgNamed("avail.Run", func() error { return utils.IgnoreCancel(state.Run(ctx)) }) - require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { - runCtx, cancel := context.WithCancel(ctx) - defer cancel() - s.SpawnBgNamed("avail.Run", func() error { return utils.IgnoreCancel(state.Run(runCtx)) }) - _, err := state.LastCommitQC().Wait(ctx, func(got utils.Option[*types.CommitQC]) bool { + t.Logf("push next CommitQC to avail") + qc, _ = data.TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc.QC())) + if err := state.PushCommitQC(ctx, qc.QC()); err != nil { + return err + } + t.Logf("wait for this CommitQC to be persisted in avail") + _, err = state.LastCommitQC().Wait(ctx, func(got utils.Option[*types.CommitQC]) bool { gotQC, ok := got.Get() return ok && gotQC.Index() == qc.QC().Index() }) From 55a22ad1086adc7078f5c92baefb2486f75db23b Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Mon, 10 Aug 2026 17:13:28 +0200 Subject: [PATCH 47/61] addressed comments --- sei-db/ledger_db/block/littblock/litt_block_db.go | 2 +- sei-tendermint/autobahn/types/proposal.go | 2 +- sei-tendermint/internal/p2p/giga/data.go | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index 25b539e7f7..6a3cdceb76 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -370,7 +370,7 @@ func (s *blockDB) Status() utils.Option[types.SuffixRange] { // ReadSuffix reads the materialized startup-recovery suffix. func (s *blockDB) ReadSuffix() (types.Suffix, error) { // Suffix is computed under lock, so that GC cannot malform it: - // locked => no new data can be appended => watermark doesn't move => GC doesn't consume data of the suffix.:w + // locked => no new data can be appended => watermark doesn't move => GC doesn't consume data of the suffix. s.mu.Lock() defer s.mu.Unlock() it, err := s.table.Iterator(true) diff --git a/sei-tendermint/autobahn/types/proposal.go b/sei-tendermint/autobahn/types/proposal.go index 197688ada0..fadb9b0a49 100644 --- a/sei-tendermint/autobahn/types/proposal.go +++ b/sei-tendermint/autobahn/types/proposal.go @@ -69,7 +69,7 @@ type GlobalRange struct { } func (g GlobalRange) String() string { - return fmt.Sprintf("[%v,%v)",g.First,g.Next) + return fmt.Sprintf("[%v,%v)", g.First, g.Next) } // Len returns the number of global blocks in the range. diff --git a/sei-tendermint/internal/p2p/giga/data.go b/sei-tendermint/internal/p2p/giga/data.go index 74b4150326..e347f208da 100644 --- a/sei-tendermint/internal/p2p/giga/data.go +++ b/sei-tendermint/internal/p2p/giga/data.go @@ -188,6 +188,7 @@ func (s *Service) serverStreamFullCommitQCs(ctx context.Context, server rpc.Serv if err != nil { if errors.Is(err, types.ErrPruned) { next = s.data.First() + continue } return fmt.Errorf("s.data.QC(): %w", err) } @@ -215,6 +216,7 @@ func (s *Service) serverStreamAppQCs(ctx context.Context, server rpc.Server[API] if err != nil { if errors.Is(err, types.ErrPruned) { next = s.data.First() + continue } return fmt.Errorf("x.validatorState().Data().AppQC(): %w", err) } From 50066c8cbcaad9003e10b1222f664a1648797c9a Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Mon, 10 Aug 2026 18:18:09 +0200 Subject: [PATCH 48/61] removed stupid shit --- .../internal/autobahn/avail/inner.go | 2 +- .../internal/autobahn/avail/state.go | 18 ++- .../internal/autobahn/avail/state_test.go | 12 +- .../autobahn/consensus/persist/blocks.go | 28 +--- .../autobahn/consensus/persist/blocks_test.go | 76 +++------- .../autobahn/consensus/persist/commitqcs.go | 11 +- .../consensus/persist/commitqcs_test.go | 142 ++++++------------ 7 files changed, 92 insertions(+), 197 deletions(-) diff --git a/sei-tendermint/internal/autobahn/avail/inner.go b/sei-tendermint/internal/autobahn/avail/inner.go index 4d4a281896..c062e55f38 100644 --- a/sei-tendermint/internal/autobahn/avail/inner.go +++ b/sei-tendermint/internal/autobahn/avail/inner.go @@ -12,7 +12,7 @@ import ( // TODO: when dynamic committee changes are supported, newly joined members // must be added to blocks, votes, and nextBlockToPersist. // Currently all four are initialized once in newInner from c.Lanes().All(). -// BlockPersister creates lane WALs lazily inside MaybePruneAndPersistLane, but the new +// BlockPersister creates lane WALs lazily inside PruneAndPersist, but the new // member must also appear in inner.blocks before the next persist cycle. type inner struct { persistedCommitQC utils.AtomicSend[utils.Option[*types.CommitQC]] // latest persisted CommitQC diff --git a/sei-tendermint/internal/autobahn/avail/state.go b/sei-tendermint/internal/autobahn/avail/state.go index 1a36c76746..9640d3656f 100644 --- a/sei-tendermint/internal/autobahn/avail/state.go +++ b/sei-tendermint/internal/autobahn/avail/state.go @@ -560,7 +560,7 @@ func (s *State) Run(ctx context.Context) error { } // runPersist is the main loop for the persist goroutine. -// 2. commitQCs.MaybePruneAndPersist and each lane's blocks.MaybePruneAndPersistLane run +// 2. commitQCs.PruneAndPersist and each lane's blocks.PruneAndPersist run // concurrently via scope.Parallel (separate WALs, no early cancellation; first error // is returned after all tasks finish). // Each path publishes (markCommitQCsPersisted / markBlockPersisted) per entry so voting @@ -572,16 +572,11 @@ func (s *State) runPersist(ctx context.Context) error { return err } - markBlock := func(p *types.Signed[*types.LaneProposal]) { - header := p.Msg().Block().Header() - s.markBlockPersisted(header.Lane(), header.BlockNumber()+1) - } - // 2. Persist commit-QCs and per-lane blocks in parallel. // Callees handle empty inputs gracefully (no-op when nothing to write/truncate). if err := scope.Parallel(func(ps scope.ParallelScope) error { ps.Spawn(func() error { - if err := s.persisters.commitQCs.Persist(batch.commitQCs.first, batch.commitQCs.tail); err != nil { + if err := s.persisters.commitQCs.PruneAndPersist(batch.commitQCs.first, batch.commitQCs.tail); err != nil { return err } if t := batch.commitQCs.tail; len(t) > 0 { @@ -591,7 +586,14 @@ func (s *State) runPersist(ctx context.Context) error { }) for lane, batch := range batch.blocks { ps.Spawn(func() error { - return s.persisters.blocks.Persist(lane, batch.first, batch.tail, utils.Some(markBlock)) + if err := s.persisters.blocks.PruneAndPersist(lane, batch.first, batch.tail); err != nil { + return err + } + if n := len(batch.tail); n > 0 { + header := batch.tail[n-1].Msg().Block().Header() + s.markBlockPersisted(header.Lane(), header.BlockNumber()+1) + } + return nil }) } return nil diff --git a/sei-tendermint/internal/autobahn/avail/state_test.go b/sei-tendermint/internal/autobahn/avail/state_test.go index 556559b54e..d2998cc015 100644 --- a/sei-tendermint/internal/autobahn/avail/state_test.go +++ b/sei-tendermint/internal/autobahn/avail/state_test.go @@ -15,10 +15,6 @@ import ( "github.com/stretchr/testify/require" ) -var ( - noBlockCB = utils.None[func(*types.Signed[*types.LaneProposal])]() -) - type byLane[T any] map[types.LaneID][]T func makeAppVotes(keys []types.SecretKey, proposal *types.AppProposal) []*types.Signed[*types.AppVote] { @@ -424,7 +420,7 @@ func TestNewStateWithPersistence(t *testing.T) { block := types.NewBlock(lane, n, parent, types.GenPayload(rng)) signed := types.Sign(keys[0], types.NewLaneProposal(block)) parent = block.Header().Hash() - require.NoError(t, bp.Persist(lane, 0, []*types.Signed[*types.LaneProposal]{signed}, noBlockCB)) + require.NoError(t, bp.PruneAndPersist(lane, 0, []*types.Signed[*types.LaneProposal]{signed})) } // Release the seeding persister's WAL locks before NewState opens the same directory. @@ -450,7 +446,7 @@ func TestNewStateWithPersistence(t *testing.T) { for i := range qcs { qcs[i] = types.BuildCommitQC(registry.LatestEpoch(), keys, prev, nil) prev = utils.Some(qcs[i]) - require.NoError(t, cp.Persist(0, []*types.CommitQC{qcs[i]})) + require.NoError(t, cp.PruneAndPersist(0, []*types.CommitQC{qcs[i]})) } // Release the seeding persister's WAL locks before NewState opens the same directory. @@ -482,9 +478,9 @@ func TestNewStateWithPersistence(t *testing.T) { cp, _, err := persist.NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) for i := range 3 { - require.NoError(t, cp.Persist(0, []*types.CommitQC{allQCs[i]})) + require.NoError(t, cp.PruneAndPersist(0, []*types.CommitQC{allQCs[i]})) } - err = cp.Persist(0, []*types.CommitQC{allQCs[5]}) + err = cp.PruneAndPersist(0, []*types.CommitQC{allQCs[5]}) require.Error(t, err) require.Contains(t, err.Error(), "out of sequence") require.NoError(t, cp.Close()) diff --git a/sei-tendermint/internal/autobahn/consensus/persist/blocks.go b/sei-tendermint/internal/autobahn/consensus/persist/blocks.go index 648e88823b..a25e3a44ba 100644 --- a/sei-tendermint/internal/autobahn/consensus/persist/blocks.go +++ b/sei-tendermint/internal/autobahn/consensus/persist/blocks.go @@ -120,7 +120,6 @@ func (lw *laneWAL) persist( lane types.LaneID, first types.BlockNumber, proposals []*types.Signed[*types.LaneProposal], - afterEach utils.Option[func(*types.Signed[*types.LaneProposal])], ) error { for s := range lw.state.Lock() { if err := s.truncateForAnchor(lane, first); err != nil { @@ -141,11 +140,6 @@ func (lw *laneWAL) persist( return err } } - if fn, ok := afterEach.Get(); ok { - for _, p := range proposals { - fn(p) - } - } return nil } panic("unreachable") @@ -168,11 +162,11 @@ func (lw *laneWAL) close() error { // // All public methods are safe for concurrent use. The lanes map is protected // by an RWMutex; each laneWAL has its own Mutex for write serialization. -// MaybePruneAndPersistLane holds the per-lane lock for the entire +// PruneAndPersist holds the per-lane lock for the entire // truncate-then-append sequence, so concurrent calls on the same lane // serialize correctly. Different lanes are fully parallel. // -// NOTE: MaybePruneAndPersistLane releases the map RLock before acquiring +// NOTE: PruneAndPersist releases the map RLock before acquiring // the per-lane lock. This is safe because lanes are only added, never // removed. If lane deletion is added in the future, the map RLock must be // held through the WAL write. @@ -292,7 +286,7 @@ func (bp *BlockPersister) getOrCreateLane(lane types.LaneID) (*laneWAL, error) { panic("unreachable") } -// MaybePruneAndPersistLane optionally truncates the lane's WAL and/or appends +// PruneAndPersist optionally truncates the lane's WAL and/or appends // new proposals, depending on which arguments are present: // // - anchor set, proposals non-empty: truncate WAL below anchor, then append (runtime path). @@ -300,27 +294,17 @@ func (bp *BlockPersister) getOrCreateLane(lane types.LaneID) (*laneWAL, error) { // - anchor empty, proposals non-empty: append only, no truncation. // - anchor empty, proposals empty: no-op. // -// afterEach, when present, is called once per appended proposal in order, after the whole batch has -// been flushed — never before, because an append is not durable until then and afterEach is what -// releases a block to the rest of consensus. It is invoked while the per-lane lock is held, so it must -// not re-enter the persister. If any append fails, afterEach is not called for the batch at all. -// No-op persister (dir=None): skips disk I/O but still invokes afterEach. +// No-op persister (dir=None): skips disk I/O. // Does not spawn goroutines — the caller schedules parallelism per lane. // // The per-lane lock is held for the entire truncate-then-append sequence, // so concurrent calls on the same lane serialize correctly. -func (bp *BlockPersister) Persist( +func (bp *BlockPersister) PruneAndPersist( lane types.LaneID, first types.BlockNumber, proposals []*types.Signed[*types.LaneProposal], - afterEach utils.Option[func(*types.Signed[*types.LaneProposal])], ) error { if _, ok := bp.dir.Get(); !ok { - if fn, ok := afterEach.Get(); ok { - for _, p := range proposals { - fn(p) - } - } return nil } @@ -328,7 +312,7 @@ func (bp *BlockPersister) Persist( if err != nil { return err } - return lw.persist(lane, first, proposals, afterEach) + return lw.persist(lane, first, proposals) } // Close shuts down all per-lane WALs, releasing the exclusive lock each one holds on its directory. diff --git a/sei-tendermint/internal/autobahn/consensus/persist/blocks_test.go b/sei-tendermint/internal/autobahn/consensus/persist/blocks_test.go index eec1b08d6f..a1fb15450c 100644 --- a/sei-tendermint/internal/autobahn/consensus/persist/blocks_test.go +++ b/sei-tendermint/internal/autobahn/consensus/persist/blocks_test.go @@ -17,8 +17,6 @@ func testSignedProposal(rng utils.Rng, key types.SecretKey, n types.BlockNumber) return types.Sign(key, types.NewLaneProposal(block)) } -var noBlockCB = utils.None[func(*types.Signed[*types.LaneProposal])]() - // liveBlocks drops blocks the prune anchor has moved past, mirroring the filter loadPersistedState // applies in the avail package. Pruning reclaims whole WAL files, so a pruned block can still be on // disk when the persister reloads; only what the anchor considers live is asserted on here. @@ -33,11 +31,10 @@ func liveBlocks(loaded []LoadedBlock, first types.BlockNumber) []LoadedBlock { func testPersistBlock(t *testing.T, bp *BlockPersister, p *types.Signed[*types.LaneProposal]) { t.Helper() - require.NoError(t, bp.Persist( + require.NoError(t, bp.PruneAndPersist( p.Msg().Block().Header().Lane(), 0, []*types.Signed[*types.LaneProposal]{p}, - noBlockCB, )) } @@ -119,8 +116,8 @@ func TestDeleteBeforeRemovesOldKeepsNew(t *testing.T) { testPersistBlock(t, bp, testSignedProposal(rng, key, i)) } - require.NoError(t, bp.Persist(lane, 3, nil, noBlockCB)) - require.NoError(t, bp.close()) + require.NoError(t, bp.PruneAndPersist(lane, 3, nil)) + require.NoError(t, bp.Close()) _, blocks, err := NewBlockPersister(utils.Some(dir)) require.NoError(t, err) @@ -149,10 +146,10 @@ func TestDeleteBeforeAndRestart(t *testing.T) { } // lane1: truncate old blocks, lane2: delete nothing (first=0), lane3: empty (no WAL). - require.NoError(t, bp.Persist(lane1, 2, nil, noBlockCB)) - require.NoError(t, bp.Persist(lane2, 0, nil, noBlockCB)) - require.NoError(t, bp.Persist(lane3, 0, nil, noBlockCB)) - require.NoError(t, bp.close()) + require.NoError(t, bp.PruneAndPersist(lane1, 2, nil)) + require.NoError(t, bp.PruneAndPersist(lane2, 0, nil)) + require.NoError(t, bp.PruneAndPersist(lane3, 0, nil)) + require.NoError(t, bp.Close()) // Restart — verify varied lane states load correctly. bp2, blocks, err := NewBlockPersister(utils.Some(dir)) @@ -192,16 +189,8 @@ func TestNoOpBlockPersister(t *testing.T) { proposals[i] = testSignedProposal(rng, key, types.BlockNumber(i)) } - // Persist and prune with first + new proposals in no-op mode. - // Verify afterEach is still invoked for every proposal. - var called int - cb := utils.Some(func(_ *types.Signed[*types.LaneProposal]) { called++ }) - require.NoError(t, bp.Persist(lane, 0, proposals[:3], cb)) - require.Equal(t, 3, called) - - called = 0 - require.NoError(t, bp.Persist(lane, 0, proposals[3:], cb)) - require.Equal(t, 2, called) + require.NoError(t, bp.PruneAndPersist(lane, 0, proposals[:3])) + require.NoError(t, bp.PruneAndPersist(lane, 0, proposals[3:])) require.NoError(t, bp.Close()) } @@ -219,7 +208,7 @@ func TestDeleteBeforeThenPersistMore(t *testing.T) { for i := range types.BlockNumber(5) { testPersistBlock(t, bp, testSignedProposal(rng, key, i)) } - require.NoError(t, bp.Persist(lane, 3, nil, noBlockCB)) + require.NoError(t, bp.PruneAndPersist(lane, 3, nil)) testPersistBlock(t, bp, testSignedProposal(rng, key, 5)) require.NoError(t, bp.Close()) @@ -246,7 +235,7 @@ func TestDeleteBeforePastAllBlocks(t *testing.T) { } // Anchor advanced past everything (nextBlockNum is 3, first=10). - require.NoError(t, bp.Persist(lane, 10, nil, noBlockCB)) + require.NoError(t, bp.PruneAndPersist(lane, 10, nil)) // Lane WAL is now empty; new writes starting from 10 should work. testPersistBlock(t, bp, testSignedProposal(rng, key, 10)) @@ -275,11 +264,11 @@ func TestDeleteBeforePastAllRejectsStaleBlock(t *testing.T) { } // Anchor advanced past everything; nextBlockNum re-anchored to 10. - require.NoError(t, bp.Persist(lane, 10, nil, noBlockCB)) + require.NoError(t, bp.PruneAndPersist(lane, 10, nil)) // Writing a stale block number (0) should be rejected. stale := testSignedProposal(rng, key, 0) - err = bp.Persist(lane, 10, []*types.Signed[*types.LaneProposal]{stale}, noBlockCB) + err = bp.PruneAndPersist(lane, 10, []*types.Signed[*types.LaneProposal]{stale}) require.Error(t, err) require.Contains(t, err.Error(), "out of sequence") @@ -302,12 +291,12 @@ func TestTruncateOnEmptyWALAdvancesCursor(t *testing.T) { } // First truncation empties the WAL (first=10 > nextBlockNum=3). - require.NoError(t, bp.Persist(lane, 10, nil, noBlockCB)) + require.NoError(t, bp.PruneAndPersist(lane, 10, nil)) // Second truncation on the already-empty WAL (first=15). // Before the fix, nextBlockNum would stay at 10 and block 15 would // be rejected as out of sequence. - require.NoError(t, bp.Persist(lane, 15, nil, noBlockCB)) + require.NoError(t, bp.PruneAndPersist(lane, 15, nil)) testPersistBlock(t, bp, testSignedProposal(rng, key, 15)) require.NoError(t, bp.Close()) @@ -383,13 +372,13 @@ func TestPersistBlockOutOfSequence(t *testing.T) { // Gap: skip block 1, try block 2. gap := testSignedProposal(rng, key, 2) - err = bp.Persist(lane, 0, []*types.Signed[*types.LaneProposal]{gap}, noBlockCB) + err = bp.PruneAndPersist(lane, 0, []*types.Signed[*types.LaneProposal]{gap}) require.Error(t, err) require.Contains(t, err.Error(), "out of sequence") // Duplicate: try block 0 again. dup := testSignedProposal(rng, key, 0) - err = bp.Persist(lane, 0, []*types.Signed[*types.LaneProposal]{dup}, noBlockCB) + err = bp.PruneAndPersist(lane, 0, []*types.Signed[*types.LaneProposal]{dup}) require.Error(t, err) require.Contains(t, err.Error(), "out of sequence") @@ -485,35 +474,6 @@ func TestPruneReclaimsSealedFiles(t *testing.T) { require.Equal(t, types.BlockNumber(total), s2.nextBlockNum) } -// TestPersistBlockInvokesAfterEachOncePerBlock covers the on-disk path: appends are flushed as a batch -// and afterEach then reports every block in it, exactly once and in order. -func TestPersistBlockInvokesAfterEachOncePerBlock(t *testing.T) { - rng := utils.TestRng() - dir := t.TempDir() - - key := types.GenSecretKey(rng) - lane := key.Public() - bp, _, err := NewBlockPersister(utils.Some(dir)) - require.NoError(t, err) - - proposals := make([]*types.Signed[*types.LaneProposal], 5) - for i := range proposals { - proposals[i] = testSignedProposal(rng, key, types.BlockNumber(i)) - } - - var seen []types.BlockNumber - cb := utils.Some(func(p *types.Signed[*types.LaneProposal]) { - seen = append(seen, p.Msg().Block().Header().BlockNumber()) - }) - require.NoError(t, bp.MaybePruneAndPersistLane(lane, utils.None[*types.CommitQC](), proposals, cb)) - require.NoError(t, bp.Close()) - - require.Equal(t, len(proposals), len(seen)) - for i := range seen { - require.Equal(t, types.BlockNumber(i), seen[i]) - } -} - func TestPersistBlockConcurrentDistinctLanes(t *testing.T) { rng := utils.TestRng() dir := t.TempDir() @@ -541,7 +501,7 @@ func TestPersistBlockConcurrentDistinctLanes(t *testing.T) { for i := range numLanes { lane := keys[i].Public() ps.Spawn(func() error { - return bp.Persist(lane, 0, proposals[i], noBlockCB) + return bp.PruneAndPersist(lane, 0, proposals[i]) }) } return nil diff --git a/sei-tendermint/internal/autobahn/consensus/persist/commitqcs.go b/sei-tendermint/internal/autobahn/consensus/persist/commitqcs.go index 73d59782c9..6d27e73391 100644 --- a/sei-tendermint/internal/autobahn/consensus/persist/commitqcs.go +++ b/sei-tendermint/internal/autobahn/consensus/persist/commitqcs.go @@ -19,10 +19,9 @@ const commitqcsWALName = "autobahn_commitqcs" // unlike the lane WALs (see blocksWALName) it has nothing to collide with. const commitqcsWALMetrics = true - // commitQCState is the mutable state protected by CommitQCPersister's mutex. type commitQCState struct { - wal utils.Option[seiwal.WAL[*types.CommitQC]] + wal utils.Option[seiwal.WAL[*types.CommitQC]] persisted types.RoadRange // Whether a QC has been appended since the last flush, so a prune that re-persists its anchor is // still made durable while a run of duplicates costs no fsync. @@ -77,7 +76,7 @@ func (s *commitQCState) deleteBefore(idx types.RoadIndex) error { } } s.persisted.First = idx - s.persisted.Next = max(s.persisted.First,s.persisted.Next) + s.persisted.Next = max(s.persisted.First, s.persisted.Next) return nil } @@ -103,7 +102,7 @@ type CommitQCPersister struct { // When stateDir is None, returns a no-op persister. // // After crash recovery with an empty WAL, LoadNext() returns 0. The caller MUST -// use MaybePruneAndPersist with the prune CommitQC in Anchor to re-establish the +// use PruneAndPersist with the prune CommitQC in Anchor to re-establish the // cursor and re-persist the anchor's CommitQC before appending more QCs. func NewCommitQCPersister(stateDir utils.Option[string]) (*CommitQCPersister, []*types.CommitQC, error) { sd, ok := stateDir.Get() @@ -140,7 +139,7 @@ func (cp *CommitQCPersister) Next() types.RoadIndex { panic("unreachable") } -// MaybePruneAndPersist optionally truncates the WAL and/or appends new +// PruneAndPersist optionally truncates the WAL and/or appends new // CommitQCs, depending on which arguments are present: // // - anchor set, commitQCs non-empty: truncate WAL below anchor, re-persist @@ -157,7 +156,7 @@ func (cp *CommitQCPersister) Next() types.RoadIndex { // need not coordinate ordering. // afterEach, when present, is called after each successful append. It is // invoked while the lock is held, so it must not re-enter the persister. -func (cp *CommitQCPersister) Persist(deleteBefore types.RoadIndex, commitQCs []*types.CommitQC) error { +func (cp *CommitQCPersister) PruneAndPersist(deleteBefore types.RoadIndex, commitQCs []*types.CommitQC) error { for s := range cp.state.Lock() { if err := s.deleteBefore(deleteBefore); err != nil { return err diff --git a/sei-tendermint/internal/autobahn/consensus/persist/commitqcs_test.go b/sei-tendermint/internal/autobahn/consensus/persist/commitqcs_test.go index 341ac8a35a..f9e867ae0c 100644 --- a/sei-tendermint/internal/autobahn/consensus/persist/commitqcs_test.go +++ b/sei-tendermint/internal/autobahn/consensus/persist/commitqcs_test.go @@ -12,9 +12,6 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/require" ) -var noQC = utils.None[*types.CommitQC]() -var noCommitQCCB = utils.None[func(*types.CommitQC)]() - // liveCommitQCs drops QCs the prune anchor has moved past, mirroring the filter loadPersistedState // applies in the avail package. Pruning reclaims whole WAL files, so a pruned QC can still be on disk // when the persister reloads; only what the anchor considers live is asserted on here. @@ -27,40 +24,25 @@ func liveCommitQCs(loaded []*types.CommitQC, first types.RoadIndex) []*types.Com return nil } -func testCommitQC( - committee *types.Committee, - keys []types.SecretKey, - prev utils.Option[*types.CommitQC], - laneQCs map[types.LaneID]*types.LaneQC, -) *types.CommitQC { - ep := types.NewEpoch(0, types.OpenRoadRange(), time.Time{}, committee, 0) - return types.BuildCommitQC(ep, keys, prev, laneQCs) -} - -func makeSequentialCommitQCs( - committee *types.Committee, - keys []types.SecretKey, - count int, -) []*types.CommitQC { +func makeSequentialCommitQCs(committee *types.Committee, keys []types.SecretKey, count int) []*types.CommitQC { var qcs []*types.CommitQC prev := utils.None[*types.CommitQC]() + ep := types.NewEpoch(0, types.OpenRoadRange(), time.Time{}, committee, 0) for range count { - qc := testCommitQC(committee, keys, prev, nil) + qc := types.BuildCommitQC(ep, keys, prev, nil) qcs = append(qcs, qc) prev = utils.Some(qc) } return qcs } -// testPersistCommitQC persists a single CommitQC via the public API. -func testPersistCommitQC(t *testing.T, cp *CommitQCPersister, qc *types.CommitQC) { - t.Helper() - require.NoError(t, cp.Persist(0, []*types.CommitQC{qc})) -} - -func testDeleteCommitQCsBefore(t *testing.T, cp *CommitQCPersister, idx types.RoadIndex) { +// clearCommitQCWAL removes all WAL files to simulate a crash between +// WAL truncation and the subsequent anchor write. +func clearCommitQCWAL(t *testing.T, dir string) { t.Helper() - require.NoError(t, cp.Persist(idx, nil)) + walDir := filepath.Join(dir, commitqcsDir) + require.NoError(t, os.RemoveAll(walDir)) + require.NoError(t, os.MkdirAll(walDir, 0700)) } func TestNewCommitQCPersisterEmptyDir(t *testing.T) { @@ -88,9 +70,7 @@ func TestPersistCommitQCAndLoad(t *testing.T) { cp, _, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) - for _, qc := range qcs { - testPersistCommitQC(t, cp, qc) - } + require.NoError(t, cp.PruneAndPersist(0, qcs)) require.Equal(t, types.RoadIndex(3), cp.Next()) require.NoError(t, cp.Close()) @@ -115,11 +95,9 @@ func TestCommitQCDeleteBeforeRemovesOldKeepsNew(t *testing.T) { qcs := makeSequentialCommitQCs(committee, keys, 5) cp, _, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) - for _, qc := range qcs { - testPersistCommitQC(t, cp, qc) - } + require.NoError(t, cp.PruneAndPersist(0, qcs)) - testDeleteCommitQCsBefore(t, cp, qcs[3].Index()) + require.NoError(t, cp.PruneAndPersist(qcs[3].Index(), nil)) require.NoError(t, cp.Close()) _, loaded, err := NewCommitQCPersister(utils.Some(dir)) @@ -140,19 +118,17 @@ func TestCommitQCDeleteBeforeZero(t *testing.T) { cp, _, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) - for _, qc := range qcs[:2] { - testPersistCommitQC(t, cp, qc) - } + require.NoError(t, cp.PruneAndPersist(0, qcs[:2])) // deleteBefore with index 0 should leave everything intact. - testDeleteCommitQCsBefore(t, cp, qcs[0].Index()) + require.NoError(t, cp.PruneAndPersist(qcs[0].Index(), nil)) require.NoError(t, cp.Close()) cp2, loaded, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) require.Equal(t, 2, len(loaded)) - testPersistCommitQC(t, cp2, qcs[2]) + require.NoError(t, cp2.PruneAndPersist(0, []*types.CommitQC{qcs[2]})) require.Equal(t, types.RoadIndex(3), cp2.Next()) require.NoError(t, cp2.Close()) } @@ -167,10 +143,10 @@ func TestCommitQCPersistDuplicateIsNoOp(t *testing.T) { cp, _, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) - testPersistCommitQC(t, cp, qcs[0]) - testPersistCommitQC(t, cp, qcs[1]) + require.NoError(t, cp.PruneAndPersist(0, []*types.CommitQC{qcs[0]})) + require.NoError(t, cp.PruneAndPersist(0, []*types.CommitQC{qcs[1]})) // Persisting qcs[0] again is a no-op (idx < next). - testPersistCommitQC(t, cp, qcs[0]) + require.NoError(t, cp.PruneAndPersist(0, []*types.CommitQC{qcs[0]})) require.Equal(t, types.RoadIndex(2), cp.Next()) require.NoError(t, cp.Close()) } @@ -185,10 +161,10 @@ func TestCommitQCPersistGapRejected(t *testing.T) { cp, _, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) - testPersistCommitQC(t, cp, qcs[0]) - testPersistCommitQC(t, cp, qcs[1]) + require.NoError(t, cp.PruneAndPersist(0, []*types.CommitQC{qcs[0]})) + require.NoError(t, cp.PruneAndPersist(0, []*types.CommitQC{qcs[1]})) // Skip qcs[2], try to persist qcs[3] — should fail because idx(3) != next(2). - err = cp.Persist(0, []*types.CommitQC{qcs[3]}) + err = cp.PruneAndPersist(0, []*types.CommitQC{qcs[3]}) require.Error(t, err) require.Contains(t, err.Error(), "out of sequence") require.NoError(t, cp.Close()) @@ -233,12 +209,12 @@ func TestNoOpCommitQCPersister(t *testing.T) { require.NoError(t, err) require.NotNil(t, cp) require.Equal(t, 0, len(loaded)) - require.NoError(t, cp.Persist(0, qcs[:5])) + require.NoError(t, cp.PruneAndPersist(0, qcs[:5])) require.Equal(t, types.RoadIndex(5), cp.Next()) // Prune with a future index. deleteBefore advances persisted.Next, // so the remaining QCs follow the new bound. - require.NoError(t, cp.Persist(8, qcs[8:])) + require.NoError(t, cp.PruneAndPersist(8, qcs[8:])) require.Equal(t, types.RoadIndex(11), cp.Next()) require.NoError(t, cp.Close()) } @@ -252,17 +228,15 @@ func TestCommitQCDeleteBeforePastAll(t *testing.T) { qcs := makeSequentialCommitQCs(committee, keys, 12) cp, _, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) - for i := range 3 { - testPersistCommitQC(t, cp, qcs[i]) - } + require.NoError(t, cp.PruneAndPersist(0, qcs[:3])) // next is 3; deleteBefore at 10 truncates the WAL and advances the // cursor to 10. - testDeleteCommitQCsBefore(t, cp, qcs[10].Index()) + require.NoError(t, cp.PruneAndPersist(qcs[10].Index(), nil)) require.Equal(t, types.RoadIndex(10), cp.Next()) // New writes starting from 10 should work. - testPersistCommitQC(t, cp, qcs[10]) - testPersistCommitQC(t, cp, qcs[11]) + require.NoError(t, cp.PruneAndPersist(0, []*types.CommitQC{qcs[10]})) + require.NoError(t, cp.PruneAndPersist(0, []*types.CommitQC{qcs[11]})) require.NoError(t, cp.Close()) // Reopen — the fast-forward left a gap, so only the entries after it are live. @@ -285,9 +259,7 @@ func TestCommitQCDeleteBeforePastAllCrashRecovery(t *testing.T) { qcs := makeSequentialCommitQCs(committee, keys, 12) cp, _, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) - for i := range 3 { - testPersistCommitQC(t, cp, qcs[i]) - } + require.NoError(t, cp.PruneAndPersist(0, qcs[:3])) require.NoError(t, cp.Close()) // Simulate crash: clear the WAL as if the prune reclaimed everything but the @@ -300,12 +272,9 @@ func TestCommitQCDeleteBeforePastAllCrashRecovery(t *testing.T) { require.Empty(t, loaded) require.Equal(t, types.RoadIndex(0), cp2.Next()) - // MaybePruneAndPersist with anchor at 10 re-establishes the cursor + // Persist with anchor at 10 re-establishes the cursor // and appends new QCs. - require.NoError(t, cp2.Persist( - qcs[10].Index(), - []*types.CommitQC{qcs[11]}, - )) + require.NoError(t, cp2.PruneAndPersist(qcs[10].Index(), []*types.CommitQC{qcs[10], qcs[11]})) require.Equal(t, types.RoadIndex(12), cp2.Next()) require.NoError(t, cp2.Close()) @@ -326,29 +295,20 @@ func TestCommitQCDeleteBeforeWithAnchorRecovers(t *testing.T) { dir := t.TempDir() qcs := makeSequentialCommitQCs(committee, keys, 5) - cp, _, err := NewCommitQCPersister(utils.Some(dir)) - require.NoError(t, err) - for _, qc := range qcs { - testPersistCommitQC(t, cp, qc) - } - require.NoError(t, cp.Close()) - // Simulate crash: clear WAL. - clearCommitQCWAL(t, dir) - - // Restart: WAL is empty. Pass the anchor QC (index 4) through deleteBefore. - cp2, loaded, err := NewCommitQCPersister(utils.Some(dir)) + // WAL is empty. Pass the anchor QC (index 4) through deleteBefore. + cp, loaded, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) require.Empty(t, loaded) - // deleteBefore advances cursor to 4, then re-persists qcs[4] via anchor. - testDeleteCommitQCsBefore(t, cp2, qcs[4].Index()) - require.Equal(t, types.RoadIndex(5), cp2.Next()) + // deleteBefore advances cursor to 4. + require.NoError(t, cp.PruneAndPersist(qcs[4].Index(), utils.Slice(qcs[4]))) + require.Equal(t, types.RoadIndex(5), cp.Next()) // Continue writing from 5. - testPersistCommitQC(t, cp2, qcs[4]) // duplicate — no-op - require.Equal(t, types.RoadIndex(5), cp2.Next()) - require.NoError(t, cp2.Close()) + require.NoError(t, cp.PruneAndPersist(0, utils.Slice(qcs[4]))) // duplicate — no-op + require.Equal(t, types.RoadIndex(5), cp.Next()) + require.NoError(t, cp.Close()) // Reopen — anchor QC should be on disk. _, loaded, err = NewCommitQCPersister(utils.Some(dir)) @@ -368,11 +328,9 @@ func TestCommitQCDeleteBeforeThenPersistMore(t *testing.T) { require.NoError(t, err) // Persist 0..4, delete before 3, then persist 5. - for i := range 5 { - testPersistCommitQC(t, cp, qcs[i]) - } - testDeleteCommitQCsBefore(t, cp, qcs[3].Index()) - testPersistCommitQC(t, cp, qcs[5]) + require.NoError(t, cp.PruneAndPersist(0, qcs[:5])) + require.NoError(t, cp.PruneAndPersist(qcs[3].Index(), nil)) + require.NoError(t, cp.PruneAndPersist(0, []*types.CommitQC{qcs[5]})) require.NoError(t, cp.Close()) _, loaded, err := NewCommitQCPersister(utils.Some(dir)) @@ -393,16 +351,14 @@ func TestCommitQCDeleteBeforeAlreadyPruned(t *testing.T) { qcs := makeSequentialCommitQCs(committee, keys, 5) cp, _, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) - for _, qc := range qcs { - testPersistCommitQC(t, cp, qc) - } + require.NoError(t, cp.PruneAndPersist(0, qcs)) // Prune up to index 3. - testDeleteCommitQCsBefore(t, cp, qcs[3].Index()) + require.NoError(t, cp.PruneAndPersist(qcs[3].Index(), nil)) // Pruning at or below the current first should be a no-op. - testDeleteCommitQCsBefore(t, cp, qcs[2].Index()) - testDeleteCommitQCsBefore(t, cp, qcs[3].Index()) + require.NoError(t, cp.PruneAndPersist(qcs[2].Index(), nil)) + require.NoError(t, cp.PruneAndPersist(qcs[3].Index(), nil)) require.NoError(t, cp.Close()) // Verify nothing extra was pruned. @@ -423,16 +379,14 @@ func TestCommitQCProgressiveDeleteBefore(t *testing.T) { qcs := makeSequentialCommitQCs(committee, keys, 8) cp, _, err := NewCommitQCPersister(utils.Some(dir)) require.NoError(t, err) - for _, qc := range qcs { - testPersistCommitQC(t, cp, qc) - } + require.NoError(t, cp.PruneAndPersist(0, qcs)) // First prune: remove 0, 1. - testDeleteCommitQCsBefore(t, cp, qcs[2].Index()) + require.NoError(t, cp.PruneAndPersist(qcs[2].Index(), nil)) require.Equal(t, types.RoadIndex(8), cp.Next()) // Second prune: remove 2, 3, 4. - testDeleteCommitQCsBefore(t, cp, qcs[5].Index()) + require.NoError(t, cp.PruneAndPersist(qcs[5].Index(), nil)) require.NoError(t, cp.Close()) // Verify indices 5, 6, 7 survive. From ff4a03c991d4195efa9771717e51b3fe5ee31bd7 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Mon, 10 Aug 2026 20:46:21 +0200 Subject: [PATCH 49/61] test fix --- .../internal/autobahn/avail/state_test.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/sei-tendermint/internal/autobahn/avail/state_test.go b/sei-tendermint/internal/autobahn/avail/state_test.go index d2998cc015..aad6588de6 100644 --- a/sei-tendermint/internal/autobahn/avail/state_test.go +++ b/sei-tendermint/internal/autobahn/avail/state_test.go @@ -177,6 +177,15 @@ func testState(t *testing.T, rng utils.Rng, stateDir utils.Option[string]) { return fmt.Errorf("state.PushCommitQC(): %w", err) } + t.Logf("Check that a CommitQC was successfully reconstructed.") + _, got, err := state.fullCommitQC(ctx, qc.Proposal().Index()) + if err != nil { + return fmt.Errorf("state.fullCommitQC(): %w", err) + } + if err := utils.TestDiff(want, qcPayloadHashes(got)); err != nil { + return fmt.Errorf("snapshot: %w", err) + } + t.Logf("Push app votes.") appHash := types.GenAppHash(rng) appProposal := types.NewAppProposal(qc.Proposal(), appHash) @@ -209,15 +218,6 @@ func testState(t *testing.T, rng utils.Rng, stateDir utils.Option[string]) { } } - t.Logf("Check that a CommitQC was successfully reconstructed.") - _, got, err := state.fullCommitQC(ctx, qc.Proposal().Index()) - if err != nil { - return fmt.Errorf("state.fullCommitQC(): %w", err) - } - if err := utils.TestDiff(want, qcPayloadHashes(got)); err != nil { - return fmt.Errorf("snapshot: %w", err) - } - t.Logf("Check that the blocks were successfully pushed to data state.") gr := got.QC().GlobalRange() for i := gr.First; i < gr.Next; i++ { From 0fc523622a188302c7ffff989e01093f094a2aef Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Mon, 10 Aug 2026 21:39:39 +0200 Subject: [PATCH 50/61] removed race condition --- sei-tendermint/internal/autobahn/avail/state.go | 5 +++-- sei-tendermint/internal/autobahn/avail/state_test.go | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/sei-tendermint/internal/autobahn/avail/state.go b/sei-tendermint/internal/autobahn/avail/state.go index 9640d3656f..55fe755e69 100644 --- a/sei-tendermint/internal/autobahn/avail/state.go +++ b/sei-tendermint/internal/autobahn/avail/state.go @@ -566,7 +566,7 @@ func (s *State) Run(ctx context.Context) error { // Each path publishes (markCommitQCsPersisted / markBlockPersisted) per entry so voting // unblocks ASAP. func (s *State) runPersist(ctx context.Context) error { - for { + for ctx.Err() == nil { batch, err := s.collectPersistBatch(ctx) if err != nil { return err @@ -591,7 +591,7 @@ func (s *State) runPersist(ctx context.Context) error { } if n := len(batch.tail); n > 0 { header := batch.tail[n-1].Msg().Block().Header() - s.markBlockPersisted(header.Lane(), header.BlockNumber()+1) + s.markBlockPersisted(lane, header.BlockNumber()+1) } return nil }) @@ -601,6 +601,7 @@ func (s *State) runPersist(ctx context.Context) error { return err } } + return ctx.Err() } type batch[I any, T any] struct { diff --git a/sei-tendermint/internal/autobahn/avail/state_test.go b/sei-tendermint/internal/autobahn/avail/state_test.go index aad6588de6..4f05e8a45f 100644 --- a/sei-tendermint/internal/autobahn/avail/state_test.go +++ b/sei-tendermint/internal/autobahn/avail/state_test.go @@ -271,9 +271,9 @@ func TestStateRestartFromPersisted(t *testing.T) { return utils.IgnoreCancel(state.Run(ctx)) }) + var prev utils.Option[*types.CommitQC] for i := range 2 { t.Logf("iteration %d", i) - prev := state.LastCommitQC().Load() for range 5 { key := keys[rng.Intn(len(keys))] @@ -315,6 +315,7 @@ func TestStateRestartFromPersisted(t *testing.T) { if _, err := state.appQC(ctx, appProposal.RoadIndex()); err != nil { return fmt.Errorf("WaitForAppQC: %w", err) } + prev = utils.Some(qc) wantAppQCIdx = appProposal.RoadIndex() } From 5785a55b788d4510f4f9ed92f6ac9e5a71c8929a Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Mon, 10 Aug 2026 22:37:06 +0200 Subject: [PATCH 51/61] fixed test --- .../internal/autobahn/avail/state_test.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/sei-tendermint/internal/autobahn/avail/state_test.go b/sei-tendermint/internal/autobahn/avail/state_test.go index 4f05e8a45f..76267051b7 100644 --- a/sei-tendermint/internal/autobahn/avail/state_test.go +++ b/sei-tendermint/internal/autobahn/avail/state_test.go @@ -44,7 +44,9 @@ func qcPayloadHashes(qc *types.FullCommitQC) byLane[types.PayloadHash] { func TestState(t *testing.T) { rng := utils.TestRng() - testState(t, rng, utils.None[string]()) + for range 5 { + testState(t, rng, utils.None[string]()) + } } // TestStateWithPersistence runs the same flow as TestState but with disk @@ -199,19 +201,16 @@ func testState(t *testing.T, rng utils.Rng, stateDir utils.Option[string]) { } } - t.Logf("Previous one should be eventually evicted") + t.Logf("Executed CommitQC should be eventually evicted") for inner, ctrl := range state.inner.Lock() { - if err := ctrl.WaitUntil(ctx, func() bool { return inner.roads.first == appProposal.RoadIndex() }); err != nil { + if err := ctrl.WaitUntil(ctx, func() bool { return inner.roads.first == appProposal.RoadIndex()+1 }); err != nil { return err } } - if _, err := state.appQC(ctx, appProposal.RoadIndex()); err != nil { - return fmt.Errorf("state.WaitForAppQC(): %w", err) - } t.Logf("Check that the executed local blocks have been pruned") for lane := range committee.Lanes().All() { - if lr := types.LaneRangeOpt(prev, lane); lr.Next() > 0 { + if lr := qc.LaneRange(lane); lr.Next() > 0 { if _, err := state.Block(ctx, lane, lr.Next()-1); !errors.Is(err, types.ErrPruned) { return fmt.Errorf("state.Block(): %w, want %v", err, types.ErrPruned) } From 84c42bfee22d4a70166e565cc543609bc4d85a86 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 11 Aug 2026 16:00:21 +0200 Subject: [PATCH 52/61] applied claude comments --- sei-db/ledger_db/block/blocksim/blocksim.go | 2 +- .../block/littblock/litt_block_db.go | 1 + sei-tendermint/autobahn/types/block_db.go | 5 +- .../internal/autobahn/avail/state.go | 4 +- .../internal/autobahn/data/state.go | 64 ++++++----- .../autobahn/data/state_recovery_test.go | 100 +---------------- .../internal/autobahn/data/state_test.go | 3 - sei-tendermint/internal/p2p/giga/avail.go | 36 +++--- sei-tendermint/internal/p2p/giga/consensus.go | 80 +++---------- sei-tendermint/internal/p2p/giga/service.go | 105 +++++++++++------- .../internal/p2p/giga_router_common.go | 16 +-- .../internal/p2p/giga_router_fullnode.go | 2 +- 12 files changed, 148 insertions(+), 270 deletions(-) diff --git a/sei-db/ledger_db/block/blocksim/blocksim.go b/sei-db/ledger_db/block/blocksim/blocksim.go index 306fa21211..a2a94aff06 100644 --- a/sei-db/ledger_db/block/blocksim/blocksim.go +++ b/sei-db/ledger_db/block/blocksim/blocksim.go @@ -204,7 +204,7 @@ func recoverResumeState( if !ok { return prev, highest, nil } - if status.NextBlock > 0 { + if status.NextBlock > status.First { highest = tmutils.Some(uint64(status.NextBlock - 1)) } if status.NextQC > 0 { diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index 6a3cdceb76..9d11ca1946 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -87,6 +87,7 @@ func NewBlockDB(config *LittBlockConfig) (types.BlockDB, error) { } suffix, err := s.ReadSuffix() if err != nil { + _ = db.Close() return nil, fmt.Errorf("ReadSuffix(): %w", err) } s.status = suffix.Status diff --git a/sei-tendermint/autobahn/types/block_db.go b/sei-tendermint/autobahn/types/block_db.go index dd0a59da7a..7f43a73ac9 100644 --- a/sei-tendermint/autobahn/types/block_db.go +++ b/sei-tendermint/autobahn/types/block_db.go @@ -7,9 +7,8 @@ import ( // BlockDB is the durable backing store for data.State. It persists the // finalized records the consensus state machine produces — finalized blocks // (indexed by GlobalBlockNumber and by header hash), FullCommitQCs (each -// covering a contiguous range of GlobalBlockNumbers), and AppQCs (each matching -// a persisted CommitQC range) — and provides the read API needed for crash -// recovery and runtime lookups. +// covering a contiguous range of GlobalBlockNumbers), AppProposals and AppQCs +// and provides the read API needed for crash recovery and runtime lookups. // // # Concurrency // diff --git a/sei-tendermint/internal/autobahn/avail/state.go b/sei-tendermint/internal/autobahn/avail/state.go index 55fe755e69..fb4ab2800d 100644 --- a/sei-tendermint/internal/autobahn/avail/state.go +++ b/sei-tendermint/internal/autobahn/avail/state.go @@ -70,7 +70,7 @@ func loadPersistedState(dir utils.Option[string]) (*loadedState, *persisters, er } cp, commitQCs, err := persist.NewCommitQCPersister(dir) if err != nil { - bp.Close() + _ = bp.Close() return nil, nil, fmt.Errorf("NewCommitQCPersister: %w", err) } pers := &persisters{blocks: bp, commitQCs: cp} @@ -88,7 +88,7 @@ func NewState(key types.SecretKey, data *data.State, stateDir utils.Option[strin } inner, err := newInner(data, loaded) if err != nil { - pers.close() + _ = pers.close() return nil, err } return &State{ diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index b5e11a4ca0..037772d447 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -1,6 +1,7 @@ package data import ( + "bytes" "context" "errors" "fmt" @@ -18,8 +19,6 @@ const blocksCacheSize = 4000 type Config struct { // Registry is the authoritative source of committee and stake information. Registry *epoch.Registry - // LastExecutedBlock is the app's committed global height if any. - LastExecutedBlock utils.Option[types.GlobalBlockNumber] } // blockEntry is a (number, block) pair collected in runPersist batches. @@ -29,9 +28,6 @@ type blockEntry struct { } type inner struct { - // Map key ranges (low end = first): - // - // Durable copies below first live in BlockDB. qcs map[types.GlobalBlockNumber]*types.FullCommitQC // [first, nextQC) blocks map[types.GlobalBlockNumber]*types.Block // [first, nextBlock) + gap-fills in [nextBlock, nextQC) appProposals map[types.GlobalBlockNumber]*types.AppProposal // [first, nextAppProposal) @@ -39,7 +35,7 @@ type inner struct { blockHashes map[types.BlockHeaderHash]types.GlobalBlockNumber // blockHashes mirrors blocks (insertBlock / setPersisted) // first is the exclusive low end of retained in-memory state: maps keep [first, next*). - // Advanced by runPersist() + // Advanced by runPersist(). Durable copies below first live in BlockDB. // // first <= nextAppQC <= nextAppProposal <= nextBlock <= nextQC first types.GlobalBlockNumber @@ -49,6 +45,8 @@ type inner struct { nextQC types.GlobalBlockNumber persisted types.SuffixRange + // Anchor represents the highest fully processed row: + // CommitQC, Blocks, AppProposal, AppQC present and persisted. anchor utils.AtomicSend[utils.Option[Anchor]] } @@ -83,7 +81,7 @@ func (i *inner) insertAppQC(registry *epoch.Registry, appQC *types.AppQC) error return nil } if gr.Next > i.nextAppProposal { - return fmt.Errorf("Missing AppProposal for this AppQC") + return fmt.Errorf("missing AppProposal for this AppQC") } if gr.First > i.nextAppQC { return fmt.Errorf("AppQC gap: expected first<=%d, got %d", i.nextAppQC, gr.First) @@ -100,7 +98,6 @@ func (i *inner) insertAppQC(registry *epoch.Registry, appQC *types.AppQC) error i.appQCs[i.nextAppQC] = appQC i.nextAppQC++ } - i.nextAppQC = gr.Next return nil } @@ -110,7 +107,7 @@ func (i *inner) insertAppProposal(appProposal *types.AppProposal) error { return nil } if gr.Next > i.nextQC { - return fmt.Errorf("Missing CommitQC for this AppProposal") + return fmt.Errorf("missing CommitQC for this AppProposal") } if gr.First > i.nextAppProposal { return fmt.Errorf("AppProposal gap: expected first<=%d, got %d", i.nextAppProposal, gr.First) @@ -183,8 +180,6 @@ type State struct { // Use memblock.NewBlockDB() for an in-memory store (testing / no persistent dir). // The caller owns blockDB and must close it after State.Run returns (nodeImpl // owns this in production); State never closes it. -// Recovery starts at cfg.LastExecutedBlock and handles a non-zero CommitQC tip -// via loadFromBlockDB (skipTo). func NewState(cfg *Config, blockDB types.BlockDB) (*State, error) { inner, err := loadFromBlockDB(cfg, blockDB) if err != nil { @@ -213,18 +208,17 @@ func loadFromBlockDB(cfg *Config, blockDB types.BlockDB) (*inner, error) { NextAppQC: firstBlock, NextBlock: firstBlock, }) - first := status.First inner := &inner{ qcs: map[types.GlobalBlockNumber]*types.FullCommitQC{}, blocks: map[types.GlobalBlockNumber]*types.Block{}, appQCs: map[types.GlobalBlockNumber]*types.AppQC{}, appProposals: map[types.GlobalBlockNumber]*types.AppProposal{}, blockHashes: map[types.BlockHeaderHash]types.GlobalBlockNumber{}, - first: first, - nextAppProposal: first, - nextAppQC: first, - nextBlock: first, - nextQC: first, + first: status.First, + nextAppProposal: status.First, + nextAppQC: status.First, + nextBlock: status.First, + nextQC: status.First, persisted: status, anchor: utils.NewAtomicSend(utils.None[Anchor]()), } @@ -409,7 +403,7 @@ func (s *State) NextBlock() types.GlobalBlockNumber { panic("unreachable") } -// NextBlock returns the index of the next block to be pushed. +// NextAppQC returns the index of the next AppQC to be pushed. func (s *State) NextAppQC() types.GlobalBlockNumber { for inner := range s.inner.Lock() { return inner.nextAppQC @@ -601,15 +595,18 @@ func (s *State) PushAppHash(ctx context.Context, n types.GlobalBlockNumber, hash if err := ctrl.WaitUntil(ctx, func() bool { return n < inner.nextBlock }); err != nil { return err } + // We only care about the AppHash of the last block of the CommitQC. p := inner.qcs[n].QC().Proposal() gr := p.GlobalRange() - if gr.First != inner.nextAppProposal { - return fmt.Errorf("unexpected app proposal : got %v, want in [%v;%v)", n, gr.First, gr.Next) - } - // We only care about the AppHash of the last block of the CommitQC. if gr.Next != n+1 { return nil } + if err := ctrl.WaitUntil(ctx, func() bool { return gr.First <= inner.nextAppProposal }); err != nil { + return err + } + if gr.Next != n+1 || n < inner.nextAppProposal { + return nil + } proposal := types.NewAppProposal(p, hash) t := time.Now() for inner.nextAppProposal < gr.Next { @@ -621,6 +618,12 @@ func (s *State) PushAppHash(ctx context.Context, n types.GlobalBlockNumber, hash inner.nextAppProposal += 1 } ctrl.Updated() + // CRITICAL: We need to persist AppHash before we return and start executing the next block, + // otherwise we lose the apphash on restart. + // TODO(gprusak): this is a temporary measure, until AppHashes are persisted outside of BlockDB. + if err := ctrl.WaitUntil(ctx, func() bool { return gr.Next <= inner.persisted.NextAppProposal }); err != nil { + return err + } } return nil } @@ -689,15 +692,14 @@ func (s *State) Anchor() utils.AtomicRecv[utils.Option[Anchor]] { } func (i *inner) nextToExecute(lane types.LaneID) types.BlockNumber { - // TODO(gprusak): decide whether 0 is a good result in this case in general. - // Empty maps (first == nextQC) only on fresh start / after skipTo with no QC. - if i.first == i.nextAppProposal { - if i.first == i.nextQC { - return 0 - } + if i.nextAppProposal < i.nextQC { return i.qcs[i.nextAppProposal].QC().LaneRange(lane).First() } - return i.qcs[i.nextAppProposal-1].QC().LaneRange(lane).Next() + if i.first < i.nextAppProposal { + return i.qcs[i.nextAppProposal-1].QC().LaneRange(lane).Next() + } + // Genesis state: i.first == i.nextQC + return 0 } // Waits until lane block n is executed, returns the next block of this lane to be executed (>n) @@ -811,7 +813,11 @@ func (s *State) runPersist(ctx context.Context) error { for inner, ctrl := range s.inner.Lock() { inner.persisted = status for inner.first < inner.persisted.First { + // Divergence detection n := inner.first + if got, want := inner.appProposals[n].AppHash(), inner.appQCs[n].Proposal().AppHash(); !bytes.Equal(got, want) { + return fmt.Errorf("AppHash divergence detected at block %v: local AppHash = %v, quorum Apphash = %v", n, got, want) + } delete(inner.blockHashes, inner.blocks[n].Header().Hash()) delete(inner.blocks, n) delete(inner.qcs, n) diff --git a/sei-tendermint/internal/autobahn/data/state_recovery_test.go b/sei-tendermint/internal/autobahn/data/state_recovery_test.go index ad6c9405c7..3ff15e75eb 100644 --- a/sei-tendermint/internal/autobahn/data/state_recovery_test.go +++ b/sei-tendermint/internal/autobahn/data/state_recovery_test.go @@ -98,83 +98,6 @@ func TestRecoveryNormal(t *testing.T) { require.Equal(t, gr2.Next, state3.NextBlock()) } -func TestRecoveryStartsAtLastExecutedBlock(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 3) - qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) - qc2, blocks2 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc1.QC())) - gr1 := qc1.QC().GlobalRange() - gr2 := qc2.QC().GlobalRange() - require.Greater(t, gr2.Len(), 2) - - db := newTestBlockDB(t, t.TempDir()) - writeToBlockDB(t, db, - []*types.FullCommitQC{qc1, qc2}, - [][]*types.Block{blocks1, blocks2}) - - lastExecuted := gr2.First - state := newTestState(t, &Config{ - Registry: registry, - LastExecutedBlock: utils.Some(lastExecuted), - }, db) - - for inner := range state.inner.Lock() { - require.Equal(t, gr1.First, inner.nextAppProposal) - } - require.Equal(t, gr2.Next, state.NextBlock()) - got, err := state.TryBlock(lastExecuted) - require.NoError(t, err) - require.Equal(t, blocks2[0].Header().Hash(), got.Header().Hash()) -} - -func TestRecoveryCapsAppTipAtLastBlockInBlockDB(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 3) - qc1, blocks1 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) - qc2, blocks2 := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.Some(qc1.QC())) - gr2 := qc2.QC().GlobalRange() - - dir := t.TempDir() - db1 := newTestBlockDB(t, dir) - writeToBlockDB(t, db1, []*types.FullCommitQC{qc1}, [][]*types.Block{blocks1}) - require.NoError(t, db1.Close()) - - db := newTestBlockDB(t, dir) - - state, err := NewState(&Config{ - Registry: registry, - LastExecutedBlock: utils.Some(gr2.First), - }, db) - require.NoError(t, err) - require.Equal(t, gr2.First, state.NextBlock()) - - require.NoError(t, state.PushQC(t.Context(), qc2, blocks2)) - got, err := state.GlobalBlock(t.Context(), gr2.First) - require.NoError(t, err) - require.Equal(t, blocks2[0].Header().Hash(), got.Header.Hash()) - - // A per-CommitQC AppProposal cannot be rebuilt from the capped mid-QC - // cursor; this test only pins the block/QC recovery cap. -} - -func TestRecoveryRejectsAppTipBeyondCrashWindow(t *testing.T) { - rng := utils.TestRng() - registry, keys := epoch.GenRegistry(rng, 3) - qc, blocks := TestCommitQC(rng, registry.LatestEpoch(), keys, utils.None[*types.CommitQC]()) - - db := newTestBlockDB(t, t.TempDir()) - writeToBlockDB(t, db, []*types.FullCommitQC{qc}, [][]*types.Block{blocks}) - dbNextBlock := db.Status().OrPanic("non-empty BlockDB status").NextBlock - lastExecuted := dbNextBlock + 1 - - state, err := NewState(&Config{ - Registry: registry, - LastExecutedBlock: utils.Some(lastExecuted), - }, db) - require.NoError(t, err) - require.Equal(t, dbNextBlock, state.NextBlock()) -} - func TestRecoveryStartsAtRegistryFloorWhenBlockDBMissingFirstCommittedBlock(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 3) @@ -185,27 +108,11 @@ func TestRecoveryStartsAtRegistryFloorWhenBlockDBMissingFirstCommittedBlock(t *t require.NoError(t, db.WriteQC(qc)) require.NoError(t, db.Flush()) - state, err := NewState(&Config{ - Registry: registry, - LastExecutedBlock: utils.Some(gr.First), - }, db) + state, err := NewState(&Config{Registry: registry}, db) require.NoError(t, err) require.Equal(t, gr.First, state.NextBlock()) } -func TestRecoveryRejectsEmptyBlockDBAfterFirstCommittedBlock(t *testing.T) { - rng := utils.TestRng() - registry, _ := epoch.GenRegistry(rng, 3) - lastExecuted := registry.FirstBlock() + 1 - - state, err := NewState(&Config{ - Registry: registry, - LastExecutedBlock: utils.Some(lastExecuted), - }, newTestBlockDB(t, t.TempDir())) - require.NoError(t, err) - require.Equal(t, registry.FirstBlock(), state.NextBlock()) -} - func TestRecoveryLeavesAppTipBelowPruneFloorUnreadable(t *testing.T) { rng := utils.TestRng() registry, keys := epoch.GenRegistry(rng, 3) @@ -219,10 +126,7 @@ func TestRecoveryLeavesAppTipBelowPruneFloorUnreadable(t *testing.T) { writeAppDataToBlockDB(t, rng, db, keys, qc1, qc2) require.NoError(t, db.PruneBefore(qc2.QC().GlobalRange().First)) - state, err := NewState(&Config{ - Registry: registry, - LastExecutedBlock: utils.Some(qc1.QC().GlobalRange().First), - }, db) + state, err := NewState(&Config{Registry: registry}, db) require.NoError(t, err) _, err = state.GlobalBlock(t.Context(), qc1.QC().GlobalRange().First) require.ErrorIs(t, err, types.ErrPruned) diff --git a/sei-tendermint/internal/autobahn/data/state_test.go b/sei-tendermint/internal/autobahn/data/state_test.go index c3fbcc6ab0..ae93a4c73f 100644 --- a/sei-tendermint/internal/autobahn/data/state_test.go +++ b/sei-tendermint/internal/autobahn/data/state_test.go @@ -369,9 +369,6 @@ func TestExecution(t *testing.T) { return fmt.Errorf("state.PushAppHash(): %w", err) } } - if err := state.PushAppHash(ctx, gr.Next-1, types.GenAppHash(rng)); err == nil { - return errors.New("PushAppHash expected to fail on duplicate proposal") - } } return nil }); err != nil { diff --git a/sei-tendermint/internal/p2p/giga/avail.go b/sei-tendermint/internal/p2p/giga/avail.go index ca951ca81d..8d4e12048f 100644 --- a/sei-tendermint/internal/p2p/giga/avail.go +++ b/sei-tendermint/internal/p2p/giga/avail.go @@ -11,7 +11,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p/rpc" ) -func (x *Service) serverStreamLaneProposals(ctx context.Context, server rpc.Server[API]) error { +func (x *validatorService) serverStreamLaneProposals(ctx context.Context, server rpc.Server[API]) error { return StreamLaneProposals.Serve(ctx, server, func(ctx context.Context, stream rpc.Stream[*pb.LaneProposal, *pb.StreamLaneProposalsReq]) error { reqRaw, err := stream.Recv(ctx) if err != nil { @@ -21,7 +21,7 @@ func (x *Service) serverStreamLaneProposals(ctx context.Context, server rpc.Serv if err != nil { return fmt.Errorf("StreamLaneProposalsReqConv.Decode(): %w", err) } - sub := x.validatorState().Avail().SubscribeLaneProposals(req.FirstBlockNumber) + sub := x.state.Avail().SubscribeLaneProposals(req.FirstBlockNumber) for { p, err := sub.Recv(ctx) if err != nil { @@ -34,14 +34,14 @@ func (x *Service) serverStreamLaneProposals(ctx context.Context, server rpc.Serv }) } -func (x *Service) serverStreamLaneVotes(ctx context.Context, server rpc.Server[API]) error { +func (x *validatorService) serverStreamLaneVotes(ctx context.Context, server rpc.Server[API]) error { return StreamLaneVotes.Serve(ctx, server, func(ctx context.Context, stream rpc.Stream[*pb.LaneVote, *pb.StreamLaneVotesReq]) error { reqRaw, err := stream.Recv(ctx) if err != nil { return err } _ = reqRaw - sub := x.validatorState().Avail().SubscribeLaneVotes() + sub := x.state.Avail().SubscribeLaneVotes() for { batch, err := sub.RecvBatch(ctx) if err != nil { @@ -56,14 +56,14 @@ func (x *Service) serverStreamLaneVotes(ctx context.Context, server rpc.Server[A }) } -func (x *Service) serverStreamAppVotes(ctx context.Context, server rpc.Server[API]) error { +func (x *validatorService) serverStreamAppVotes(ctx context.Context, server rpc.Server[API]) error { return StreamAppVotes.Serve(ctx, server, func(ctx context.Context, stream rpc.Stream[*pb.AppVote, *pb.StreamAppVotesReq]) error { reqRaw, err := stream.Recv(ctx) if err != nil { return err } _ = reqRaw - sub := x.validatorState().Avail().SubscribeAppVotes() + sub := x.state.Avail().SubscribeAppVotes() for { vote, err := sub.Recv(ctx) if err != nil { @@ -76,17 +76,17 @@ func (x *Service) serverStreamAppVotes(ctx context.Context, server rpc.Server[AP }) } -func (x *Service) serverStreamCommitQCs(ctx context.Context, server rpc.Server[API]) error { +func (x *validatorService) serverStreamCommitQCs(ctx context.Context, server rpc.Server[API]) error { return StreamCommitQCs.Serve(ctx, server, func(ctx context.Context, stream rpc.Stream[*apb.CommitQC, *pb.StreamCommitQCsReq]) error { next := types.RoadIndex(0) for { - qc, err := x.validatorState().Avail().CommitQC(ctx, next) + qc, err := x.state.Avail().CommitQC(ctx, next) if err != nil { if errors.Is(err, types.ErrPruned) { - next = x.validatorState().Avail().First() + next = x.state.Avail().First() continue } - return fmt.Errorf("x.validatorState().Avail().CommitQC(): %w", err) + return fmt.Errorf("x.state.Avail().CommitQC(): %w", err) } next = qc.Index() + 1 if err := stream.Send(ctx, types.CommitQCConv.Encode(qc)); err != nil { @@ -96,7 +96,7 @@ func (x *Service) serverStreamCommitQCs(ctx context.Context, server rpc.Server[A }) } -func (x *Service) clientStreamLaneProposals(ctx context.Context, c rpc.Client[API]) error { +func (x *validatorService) clientStreamLaneProposals(ctx context.Context, c rpc.Client[API]) error { stream, err := StreamLaneProposals.Call(ctx, c) if err != nil { return err @@ -124,13 +124,13 @@ func (x *Service) clientStreamLaneProposals(ctx context.Context, c rpc.Client[AP /*if got, want := proposal.Msg().Block().Header().Lane(), c.cfg.GetKey(); got != want { return fmt.Errorf("producer = %q, want %q", got, want) }*/ - if err := x.validatorState().Avail().PushBlock(ctx, proposal); err != nil { + if err := x.state.Avail().PushBlock(ctx, proposal); err != nil { return fmt.Errorf("s.PushLaneProposal(): %w", err) } } } -func (x *Service) clientStreamLaneVotes(ctx context.Context, c rpc.Client[API]) error { +func (x *validatorService) clientStreamLaneVotes(ctx context.Context, c rpc.Client[API]) error { stream, err := StreamLaneVotes.Call(ctx, c) if err != nil { return fmt.Errorf("client.StreamLaneVotes(): %w", err) @@ -148,13 +148,13 @@ func (x *Service) clientStreamLaneVotes(ctx context.Context, c rpc.Client[API]) if err != nil { return fmt.Errorf("LaneVoteConv.Decode(): %w", err) } - if err := x.validatorState().Avail().PushVote(ctx, vote); err != nil { + if err := x.state.Avail().PushVote(ctx, vote); err != nil { return fmt.Errorf("s.PushLaneVote(): %w", err) } } } -func (x *Service) clientStreamCommitQCs(ctx context.Context, c rpc.Client[API]) error { +func (x *validatorService) clientStreamCommitQCs(ctx context.Context, c rpc.Client[API]) error { stream, err := StreamCommitQCs.Call(ctx, c) if err != nil { return fmt.Errorf("client.StreamCommitQCs(): %w", err) @@ -172,13 +172,13 @@ func (x *Service) clientStreamCommitQCs(ctx context.Context, c rpc.Client[API]) if err != nil { return fmt.Errorf("types.CommitQCConv.Decode(): %w", err) } - if err := x.validatorState().Avail().PushCommitQC(ctx, qc); err != nil { + if err := x.state.Avail().PushCommitQC(ctx, qc); err != nil { return fmt.Errorf("s.PushCommitQC(): %w", err) } } } -func (x *Service) clientStreamAppVotes(ctx context.Context, c rpc.Client[API]) error { +func (x *validatorService) clientStreamAppVotes(ctx context.Context, c rpc.Client[API]) error { stream, err := StreamAppVotes.Call(ctx, c) if err != nil { return fmt.Errorf("client.StreamAppVotes(): %w", err) @@ -196,7 +196,7 @@ func (x *Service) clientStreamAppVotes(ctx context.Context, c rpc.Client[API]) e if err != nil { return fmt.Errorf("AppVoteConv.Decode(): %w", err) } - if err := x.validatorState().Avail().PushAppVote(ctx, vote); err != nil { + if err := x.state.Avail().PushAppVote(ctx, vote); err != nil { return fmt.Errorf("s.PushLaneVote(): %w", err) } } diff --git a/sei-tendermint/internal/p2p/giga/consensus.go b/sei-tendermint/internal/p2p/giga/consensus.go index 6d5d2bace7..8d31dbb9c3 100644 --- a/sei-tendermint/internal/p2p/giga/consensus.go +++ b/sei-tendermint/internal/p2p/giga/consensus.go @@ -3,7 +3,6 @@ package giga import ( "context" "fmt" - "time" "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" apb "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/pb" @@ -42,66 +41,21 @@ func sendUpdates[T interface { } } -const pingInterval = 10 * time.Second -const pingTimeout = 5 * time.Second - -// sendPings periodically sends Ping messages. -func (x *Service) clientPing(ctx context.Context, client rpc.Client[API]) error { - for { - if err := utils.Sleep(ctx, pingInterval); err != nil { - return err - } - if err := utils.WithTimeout(ctx, pingTimeout, func(ctx context.Context) error { - stream, err := Ping.Call(ctx, client) - if err != nil { - return fmt.Errorf("p.client.Ping(): %w", err) - } - defer stream.Close() - // TODO(gprusak): add random payload to actually verify roundtrip latency. - if err := stream.Send(ctx, &pb.PingReq{}); err != nil { - return fmt.Errorf("stream.Send(): %w", err) - } - _, err = stream.Recv(ctx) - if err != nil { - return fmt.Errorf("stream.Recv(): %w", err) - } - // - return nil - }); err != nil { - return err - } - } -} - // Run sends newest consensus messages to the peer. -func (x *Service) clientConsensus(ctx context.Context, c rpc.Client[API]) error { +func (x *validatorService) clientConsensus(ctx context.Context, c rpc.Client[API]) error { return scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { // Send updates about new consensus messages. - s.Spawn(func() error { return sendUpdates(ctx, c, x.validatorState().SubscribeProposal()) }) - s.Spawn(func() error { return sendUpdates(ctx, c, x.validatorState().SubscribePrepareVote()) }) - s.Spawn(func() error { return sendUpdates(ctx, c, x.validatorState().SubscribeCommitVote()) }) - s.Spawn(func() error { return sendUpdates(ctx, c, x.validatorState().SubscribeTimeoutVote()) }) - s.Spawn(func() error { return sendUpdates(ctx, c, x.validatorState().SubscribeTimeoutQC()) }) - return nil - }) -} - -// Ping implements pb.StreamAPIServer. -// Note that we use streaming RPC, because unary RPC apparently causes 10ms extra delay on avg (empirically tested). -func (x *Service) serverPing(ctx context.Context, server rpc.Server[API]) error { - return Ping.Serve(ctx, server, func(ctx context.Context, stream rpc.Stream[*pb.PingResp, *pb.PingReq]) error { - if _, err := stream.Recv(ctx); err != nil { - return fmt.Errorf("stream.Recv(): %w", err) - } - if err := stream.Send(ctx, &pb.PingResp{}); err != nil { - return fmt.Errorf("stream.Send(): %w", err) - } + s.Spawn(func() error { return sendUpdates(ctx, c, x.state.SubscribeProposal()) }) + s.Spawn(func() error { return sendUpdates(ctx, c, x.state.SubscribePrepareVote()) }) + s.Spawn(func() error { return sendUpdates(ctx, c, x.state.SubscribeCommitVote()) }) + s.Spawn(func() error { return sendUpdates(ctx, c, x.state.SubscribeTimeoutVote()) }) + s.Spawn(func() error { return sendUpdates(ctx, c, x.state.SubscribeTimeoutQC()) }) return nil }) } // Consensus implements pb.StreaAPIServer. -func (x *Service) serverConsensus(ctx context.Context, server rpc.Server[API]) error { +func (x *validatorService) serverConsensus(ctx context.Context, server rpc.Server[API]) error { return Consensus.Serve(ctx, server, func(ctx context.Context, stream rpc.Stream[*pb.ConsensusResp, *apb.ConsensusReq]) error { for { reqRaw, err := stream.Recv(ctx) @@ -114,24 +68,24 @@ func (x *Service) serverConsensus(ctx context.Context, server rpc.Server[API]) e } switch req := req.(type) { case *types.ConsensusReqPrepareVote: - if err := x.validatorState().PushPrepareVote(req.Signed); err != nil { - return fmt.Errorf("x.validatorState().PushPrepareVote(): %w", err) + if err := x.state.PushPrepareVote(req.Signed); err != nil { + return fmt.Errorf("x.state.PushPrepareVote(): %w", err) } case *types.ConsensusReqCommitVote: - if err := x.validatorState().PushCommitVote(req.Signed); err != nil { - return fmt.Errorf("x.validatorState().PushCommitVote(): %w", err) + if err := x.state.PushCommitVote(req.Signed); err != nil { + return fmt.Errorf("x.state.PushCommitVote(): %w", err) } case *types.FullTimeoutVote: - if err := x.validatorState().PushTimeoutVote(req); err != nil { - return fmt.Errorf("x.validatorState().PushTimeoutVote(): %w", err) + if err := x.state.PushTimeoutVote(req); err != nil { + return fmt.Errorf("x.state.PushTimeoutVote(): %w", err) } case *types.FullProposal: - if err := x.validatorState().PushProposal(ctx, req); err != nil { - return fmt.Errorf("x.validatorState().PushProposal(): %w", err) + if err := x.state.PushProposal(ctx, req); err != nil { + return fmt.Errorf("x.state.PushProposal(): %w", err) } case *types.TimeoutQC: - if err := x.validatorState().PushTimeoutQC(ctx, req); err != nil { - return fmt.Errorf("x.validatorState().PushTimeoutQC(): %w", err) + if err := x.state.PushTimeoutQC(ctx, req); err != nil { + return fmt.Errorf("x.state.PushTimeoutQC(): %w", err) } default: return fmt.Errorf("unknown consensus request type: %T", req) diff --git a/sei-tendermint/internal/p2p/giga/service.go b/sei-tendermint/internal/p2p/giga/service.go index 7005328da6..2372597712 100644 --- a/sei-tendermint/internal/p2p/giga/service.go +++ b/sei-tendermint/internal/p2p/giga/service.go @@ -3,16 +3,18 @@ package giga import ( "context" "fmt" + "time" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/consensus" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/data" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p/giga/pb" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p/rpc" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/scope" ) // Service serves the giga RPC API. NewService builds a full validator -// service (all streams); NewBlockSyncService builds a block-sync-only +// service (all streams); NewFullNodeService builds a fullnode-only // service (StreamFullCommitQCs + GetBlock) for fullnodes. state is None // on block-sync-only services; the consensus / avail handlers reach it // via validatorState() and panic if invoked outside RunServer / RunClient. @@ -22,6 +24,10 @@ type Service struct { state utils.Option[*consensus.State] } +type validatorService struct { + state *consensus.State +} + func NewService(state *consensus.State) *Service { return &Service{ getBlockReqs: make(chan req), @@ -30,80 +36,55 @@ func NewService(state *consensus.State) *Service { } } -// NewBlockSyncService constructs a Service that only serves and consumes -// block-sync streams (no consensus / avail). -func NewBlockSyncService(d *data.State) *Service { +// NewFullNodeService constructs a Service that only serves and consumes +// fullnode streams (no consensus / avail). +func NewFullNodeService(d *data.State) *Service { return &Service{ getBlockReqs: make(chan req), data: d, } } -// RunInbound dispatches an inbound peer to the right handler set. -// Non-committee peers get the block-sync subset. Committee peers get the -// full RunServer on a validator (state present); on a non-validator the -// connection is refused — committee members shouldn't be dialing -// fullnodes in any healthy configuration, and we don't want a stale -// autobahn.json entry to take down RPC nodes via a reachable panic. -func (x *Service) RunInbound(ctx context.Context, server rpc.Server[API], isCommittee bool) error { - if !isCommittee { - return x.RunFullNodeServer(ctx, server) - } - if !x.state.IsPresent() { - return fmt.Errorf("committee peer dialed a non-validator service") - } - return x.RunServer(ctx, server) -} - -// validatorState unwraps state for the validator-only handlers. Panics if -// called from a block-sync-only Service — which is structurally impossible -// because those handlers are only spawned by RunServer / RunClient. -func (x *Service) validatorState() *consensus.State { - return x.state.OrPanic("Service.state called from block-sync-only mode") -} - func (x *Service) Run(ctx context.Context) error { return x.runBlockFetcher(ctx) } -func (x *Service) RunServer(ctx context.Context, server rpc.Server[API]) error { +func (x *validatorService) RunServer(ctx context.Context, server rpc.Server[API]) error { return scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { - s.Spawn(func() error { return x.serverPing(ctx, server) }) s.Spawn(func() error { return x.serverConsensus(ctx, server) }) s.Spawn(func() error { return x.serverStreamLaneProposals(ctx, server) }) s.Spawn(func() error { return x.serverStreamLaneVotes(ctx, server) }) s.Spawn(func() error { return x.serverStreamCommitQCs(ctx, server) }) s.Spawn(func() error { return x.serverStreamAppVotes(ctx, server) }) - return x.RunFullNodeServer(ctx, server) + return nil }) } -func (x *Service) RunClient(ctx context.Context, client rpc.Client[API], getBlock bool) error { +func (x *validatorService) RunClient(ctx context.Context, client rpc.Client[API]) error { return scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { s.Spawn(func() error { return x.clientConsensus(ctx, client) }) s.Spawn(func() error { return x.clientStreamLaneProposals(ctx, client) }) s.Spawn(func() error { return x.clientStreamLaneVotes(ctx, client) }) s.Spawn(func() error { return x.clientStreamCommitQCs(ctx, client) }) s.Spawn(func() error { return x.clientStreamAppVotes(ctx, client) }) - return x.RunFullNodeClient(ctx, client, getBlock) + return nil }) } -// RunBlockSyncServer spawns only the block-sync server handlers. Used on -// both validator and fullnode inbound connections from non-committee peers. -func (x *Service) RunFullNodeServer(ctx context.Context, server rpc.Server[API]) error { +func (x *Service) RunServer(ctx context.Context, server rpc.Server[API], isCommittee bool) error { return scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { s.Spawn(func() error { return x.serverPing(ctx, server) }) s.Spawn(func() error { return x.serverStreamFullCommitQCs(ctx, server) }) s.Spawn(func() error { return x.serverGetBlock(ctx, server) }) s.Spawn(func() error { return x.serverStreamAppQCs(ctx, server) }) + if c, ok := x.state.Get(); ok && isCommittee { + return (&validatorService{c}).RunServer(ctx, server) + } return nil }) } -// RunBlockSyncClient spawns only the block-sync client handlers. Used by -// fullnodes dialing committee members. -func (x *Service) RunFullNodeClient(ctx context.Context, client rpc.Client[API], getBlock bool) error { +func (x *Service) RunClient(ctx context.Context, client rpc.Client[API], getBlock bool) error { return scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { s.Spawn(func() error { return x.clientPing(ctx, client) }) s.Spawn(func() error { return x.clientStreamFullCommitQCs(ctx, client) }) @@ -114,6 +95,54 @@ func (x *Service) RunFullNodeClient(ctx context.Context, client rpc.Client[API], if getBlock { s.Spawn(func() error { return x.clientGetBlock(ctx, client) }) } + if c, ok := x.state.Get(); ok { + return (&validatorService{c}).RunClient(ctx, client) + } + return nil + }) +} + +// Ping implements pb.StreamAPIServer. +// Note that we use streaming RPC, because unary RPC apparently causes 10ms extra delay on avg (empirically tested). +func (x *Service) serverPing(ctx context.Context, server rpc.Server[API]) error { + return Ping.Serve(ctx, server, func(ctx context.Context, stream rpc.Stream[*pb.PingResp, *pb.PingReq]) error { + if _, err := stream.Recv(ctx); err != nil { + return fmt.Errorf("stream.Recv(): %w", err) + } + if err := stream.Send(ctx, &pb.PingResp{}); err != nil { + return fmt.Errorf("stream.Send(): %w", err) + } return nil }) } + +const pingInterval = 10 * time.Second +const pingTimeout = 5 * time.Second + +// sendPings periodically sends Ping messages. +func (x *Service) clientPing(ctx context.Context, client rpc.Client[API]) error { + for { + if err := utils.Sleep(ctx, pingInterval); err != nil { + return err + } + if err := utils.WithTimeout(ctx, pingTimeout, func(ctx context.Context) error { + stream, err := Ping.Call(ctx, client) + if err != nil { + return fmt.Errorf("p.client.Ping(): %w", err) + } + defer stream.Close() + // TODO(gprusak): add random payload to actually verify roundtrip latency. + if err := stream.Send(ctx, &pb.PingReq{}); err != nil { + return fmt.Errorf("stream.Send(): %w", err) + } + _, err = stream.Recv(ctx) + if err != nil { + return fmt.Errorf("stream.Recv(): %w", err) + } + // + return nil + }); err != nil { + return err + } + } +} diff --git a/sei-tendermint/internal/p2p/giga_router_common.go b/sei-tendermint/internal/p2p/giga_router_common.go index ff8996c683..04a2f14d86 100644 --- a/sei-tendermint/internal/p2p/giga_router_common.go +++ b/sei-tendermint/internal/p2p/giga_router_common.go @@ -66,15 +66,6 @@ func BuildDataState(cfg *GigaRouterCommonConfig, blockDB atypes.BlockDB) (*data. if cfg.MaxInboundFullnodePeers < 0 || cfg.MaxInboundFullnodePeers > maxInboundFullnodePeers { return nil, fmt.Errorf("GigaRouterCommonConfig.MaxInboundFullnodePeers = %v, want 0..%v", cfg.MaxInboundFullnodePeers, maxInboundFullnodePeers) } - lastExecutedHeight := cfg.App.Info().LastBlockHeight - lastExecutedBlock := utils.None[atypes.GlobalBlockNumber]() - if lastExecutedHeight != 0 { - n, ok := utils.SafeCast[atypes.GlobalBlockNumber](lastExecutedHeight) - if !ok { - return nil, fmt.Errorf("invalid App.Info().LastBlockHeight = %v", lastExecutedHeight) - } - lastExecutedBlock = utils.Some(n) - } firstBlock := atypes.GlobalBlockNumber(cfg.GenDoc.InitialHeight) // nolint:gosec // verified to be positive. genesisWeights := map[atypes.PublicKey]uint64{} for k := range cfg.ValidatorAddrs { @@ -88,10 +79,7 @@ func BuildDataState(cfg *GigaRouterCommonConfig, blockDB atypes.BlockDB) (*data. if err != nil { return nil, fmt.Errorf("epoch.NewRegistry(): %w", err) } - ds, err := data.NewState(&data.Config{ - Registry: registry, - LastExecutedBlock: lastExecutedBlock, - }, blockDB) + ds, err := data.NewState(&data.Config{Registry: registry}, blockDB) if err != nil { return nil, fmt.Errorf("data.NewState: %w", err) } @@ -515,7 +503,7 @@ func (r *gigaRouterCommon) RunInboundConn(ctx context.Context, hConn *handshaked Global.gigaNewConnsAt("in").Add(1) Global.gigaConnsAt("in").Add(1) defer Global.gigaConnsAt("in").Add(-1) - if err := r.service.RunInbound(ctx, server, isCommittee); err != nil { + if err := r.service.RunServer(ctx, server, isCommittee); err != nil { return fmt.Errorf("inbound from %v: %w", key, err) } return nil diff --git a/sei-tendermint/internal/p2p/giga_router_fullnode.go b/sei-tendermint/internal/p2p/giga_router_fullnode.go index 386bc56a88..7a6b49608b 100644 --- a/sei-tendermint/internal/p2p/giga_router_fullnode.go +++ b/sei-tendermint/internal/p2p/giga_router_fullnode.go @@ -27,7 +27,7 @@ func NewGigaFullnodeRouter(cfg *GigaRouterCommonConfig, key NodeSecretKey, dataS cfg: cfg, key: key, data: dataState, - service: giga.NewBlockSyncService(dataState), + service: giga.NewFullNodeService(dataState), poolIn: giga.NewPool[NodePublicKey, rpc.Server[giga.API]](), poolOut: giga.NewPool[NodePublicKey, rpc.Client[giga.API]](), app: cfg.App, From 34b20ab4ed79978509b118de4d2c305c0cd1901c Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 11 Aug 2026 16:02:20 +0200 Subject: [PATCH 53/61] typo --- sei-tendermint/internal/p2p/giga/data_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sei-tendermint/internal/p2p/giga/data_test.go b/sei-tendermint/internal/p2p/giga/data_test.go index 1e90e455ac..c4ad706578 100644 --- a/sei-tendermint/internal/p2p/giga/data_test.go +++ b/sei-tendermint/internal/p2p/giga/data_test.go @@ -86,7 +86,7 @@ func (e *testEnv) Run(ctx context.Context) error { client := rpc.NewClient[API]() s.SpawnNamed("mux server", func() error { return server.Run(ctx, xConn) }) s.SpawnNamed("mux client", func() error { return client.Run(ctx, yConn) }) - s.SpawnNamed("RunServer", func() error { return x.service.RunServer(ctx, server) }) + s.SpawnNamed("RunServer", func() error { return x.service.RunServer(ctx, server, true) }) s.SpawnNamed("RunClient", func() error { return y.service.RunClient(ctx, client, true) }) } } From 19d1d792fa1e57f1b04422493405f0dc22f5c8c7 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 11 Aug 2026 16:20:11 +0200 Subject: [PATCH 54/61] buf --- sei-tendermint/internal/autobahn/autobahn.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sei-tendermint/internal/autobahn/autobahn.proto b/sei-tendermint/internal/autobahn/autobahn.proto index e4b3cfc7e8..c08b2bc15e 100644 --- a/sei-tendermint/internal/autobahn/autobahn.proto +++ b/sei-tendermint/internal/autobahn/autobahn.proto @@ -219,7 +219,7 @@ message AppProposal { reserved "global_number"; option (hashable.hashable) = true; option (wireguard.sized) = true; - + // Epoch this proposal belongs to. optional uint64 epoch_index = 4; // required // Index of the commit qc finalizing the block. From dc776b0e86b052a3937073f1734a04783b806521 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 11 Aug 2026 16:30:40 +0200 Subject: [PATCH 55/61] stricter rules for pushing AppHash --- .../internal/autobahn/data/state.go | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index 037772d447..3c0ba219ae 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -592,24 +592,20 @@ func (s *State) globalBlockByHashFromDB(hash types.BlockHeaderHash) (utils.Optio // PushAppHash marks blocks up to n as executed. func (s *State) PushAppHash(ctx context.Context, n types.GlobalBlockNumber, hash types.AppHash) error { for inner, ctrl := range s.inner.Lock() { - if err := ctrl.WaitUntil(ctx, func() bool { return n < inner.nextBlock }); err != nil { + if err := ctrl.WaitUntil(ctx, func() bool { return inner.nextAppProposal < inner.nextBlock }); err != nil { return err } - // We only care about the AppHash of the last block of the CommitQC. - p := inner.qcs[n].QC().Proposal() - gr := p.GlobalRange() - if gr.Next != n+1 { - return nil - } - if err := ctrl.WaitUntil(ctx, func() bool { return gr.First <= inner.nextAppProposal }); err != nil { - return err - } - if gr.Next != n+1 || n < inner.nextAppProposal { + p := inner.qcs[inner.nextAppProposal].QC().Proposal() + if want := p.GlobalRange().Next - 1; n > want { + // We expect the AppHashes to be pushed in order. + return fmt.Errorf("received appHash for %v, while still waiting for appHash for %v", n, want) + } else if n != want { + // We only care about the AppHash of the last block of the CommitQC. return nil } proposal := types.NewAppProposal(p, hash) t := time.Now() - for inner.nextAppProposal < gr.Next { + for inner.nextAppProposal <= n { b := inner.blocks[inner.nextAppProposal] latency := t.Sub(b.Payload().CreatedAt()).Seconds() s.metrics.Blocks.Execute.Observe(latency) @@ -621,7 +617,7 @@ func (s *State) PushAppHash(ctx context.Context, n types.GlobalBlockNumber, hash // CRITICAL: We need to persist AppHash before we return and start executing the next block, // otherwise we lose the apphash on restart. // TODO(gprusak): this is a temporary measure, until AppHashes are persisted outside of BlockDB. - if err := ctrl.WaitUntil(ctx, func() bool { return gr.Next <= inner.persisted.NextAppProposal }); err != nil { + if err := ctrl.WaitUntil(ctx, func() bool { return n <= inner.persisted.NextAppProposal }); err != nil { return err } } From 6d9af6b885262759901867618290f0575e20a0d2 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 11 Aug 2026 17:00:34 +0200 Subject: [PATCH 56/61] adapted the weakened WAL semantics --- .../internal/autobahn/avail/inner.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/sei-tendermint/internal/autobahn/avail/inner.go b/sei-tendermint/internal/autobahn/avail/inner.go index c062e55f38..5f126df1f8 100644 --- a/sei-tendermint/internal/autobahn/avail/inner.go +++ b/sei-tendermint/internal/autobahn/avail/inner.go @@ -100,23 +100,26 @@ func newInner(ds *data.State, loaded *loadedState) (*inner, error) { if !ok || len(bs) == 0 { continue } - var lastHash types.BlockHeaderHash - for j, b := range bs { + for _, b := range bs { if q.Len() >= BlocksPerLane { return nil, fmt.Errorf("lane %s: loaded %d blocks exceeds capacity %d", lane, len(bs), BlocksPerLane) } - if j > 0 { - if got := b.Proposal.Msg().Block().Header().ParentHash(); got != lastHash { - return nil, fmt.Errorf("lane %s: parent hash mismatch at block %d", lane, b.Number) - } - } - lastHash = b.Proposal.Msg().Block().Header().Hash() if b.Number < q.next { continue } if b.Number != q.next { return nil, fmt.Errorf("lane %s: non-contiguous persisted blocks: expected %d, got %d", lane, q.next, b.Number) } + // We check the parent hash only for the blocks above the anchor, because: + // * node can cast LaneVote for the block of the lane without checking the parent hash, + // in case the previous block was already (executed and) pruned from memory. + // * current WAL implementation is lazily pruning on disk, so old executed blocks might be loaded on startup. + if q.Len() > 0 { + ph := b.Proposal.Msg().Block().Header().ParentHash() + if q.q[q.next-1].Msg().Block().Header().Hash() != ph { + return nil, fmt.Errorf("lane %s: parent hash mismatch at block %d", lane, b.Number) + } + } q.pushBack(b.Proposal) } i.nextBlockToPersist[lane] = q.next From 8b8d5264aaa279ed070234e5971346ea96eeac63 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 11 Aug 2026 17:54:08 +0200 Subject: [PATCH 57/61] applied claude comments --- sei-tendermint/autobahn/types/block_db.go | 19 ++++++---------- sei-tendermint/autobahn/types/epoch.go | 2 +- .../autobahn/avail/metrics/metrics.gen.go | 22 ------------------- .../autobahn/avail/metrics/metrics.go | 17 -------------- .../internal/autobahn/data/metrics/metrics.go | 2 ++ .../internal/autobahn/data/state.go | 13 ++++++++++- sei-tendermint/internal/p2p/giga/service.go | 7 ++---- 7 files changed, 24 insertions(+), 58 deletions(-) diff --git a/sei-tendermint/autobahn/types/block_db.go b/sei-tendermint/autobahn/types/block_db.go index 7f43a73ac9..138414d435 100644 --- a/sei-tendermint/autobahn/types/block_db.go +++ b/sei-tendermint/autobahn/types/block_db.go @@ -116,25 +116,20 @@ type BlockDB interface { // must already be written: the CommitQC covering GlobalRange.First must have // the same GlobalRange. // - // AppProposals form a contiguous prefix aligned with retained CommitQCs. The - // first AppProposal must start at the retained CommitQC floor; each - // subsequent AppProposal's First must equal the previous AppProposal's Next. - // Re-writing, gaps, overlaps, mid-QC starts, and ranges that do not exactly - // match the next persisted CommitQC range are rejected. + // AppProposals form a contiguous prefix. The first AppProposal must start at the retained CommitQC floor + // each subsequent AppProposal's First must equal the previous AppProposal's Next. + // Re-writing, gaps, overlaps are rejected. // // May return before the AppProposal is on disk. See the BlockDB type doc for // the two-phase write/flush contract. WriteAppProposal(appProposal *AppProposal) error - // WriteAppQC persists an AppQC. The AppQC's proposal carries the exact - // CommitQC range it certifies. A matching CommitQC must already be written: - // the CommitQC covering GlobalRange.First must have the same GlobalRange. + // WriteAppQC persists an AppQC. A matching CommitQC must already be written. // - // AppQCs form a contiguous prefix aligned with retained CommitQCs. The first + // AppQCs form a contiguous prefix. The first // AppQC must start at the retained CommitQC floor; each subsequent AppQC's - // First must equal the previous AppQC's Next. Re-writing, gaps, overlaps, - // mid-QC starts, and ranges that do not exactly match the next persisted - // CommitQC range are rejected. + // First must equal the previous AppQC's Next. Re-writing, gaps, overlaps + // are rejected. // // May return before the AppQC is on disk. See the BlockDB type doc for the // two-phase write/flush contract. diff --git a/sei-tendermint/autobahn/types/epoch.go b/sei-tendermint/autobahn/types/epoch.go index f5d92283b0..140b43ec2b 100644 --- a/sei-tendermint/autobahn/types/epoch.go +++ b/sei-tendermint/autobahn/types/epoch.go @@ -19,7 +19,7 @@ type RoadRange struct { // Use in tests and genesis epochs where no upper bound is known yet. func OpenRoadRange() RoadRange { return RoadRange{First: 0, Next: utils.Max[RoadIndex]()} } -// Has reports whether idx falls within this range (inclusive on both ends). +// Has reports whether idx falls within this range. func (r RoadRange) Has(idx RoadIndex) bool { return r.First <= idx && idx < r.Next } // Epoch holds the complete context for a single epoch. diff --git a/sei-tendermint/internal/autobahn/avail/metrics/metrics.gen.go b/sei-tendermint/internal/autobahn/avail/metrics/metrics.gen.go index 4aa8151859..1341ba9240 100644 --- a/sei-tendermint/internal/autobahn/avail/metrics/metrics.gen.go +++ b/sei-tendermint/internal/autobahn/avail/metrics/metrics.gen.go @@ -12,9 +12,7 @@ var Global = newMetrics() func init() { prometheus.MustRegister( Global.commitRoadIndex, - Global.appRoadIndex, Global.commitGlobalBlockNumber, - Global.appGlobalBlockNumber, Global.proposalToCommitLatency, Global.commitToCommitLatency, ) @@ -28,24 +26,12 @@ func newMetrics() *metrics { Name: "commit_road_index", Help: "Road index of the highest observed commitQC.", }, nil), - appRoadIndex: tmprometheus.NewGaugeIntVec(prometheus.GaugeOpts{ - Namespace: MetricsNamespace, - Subsystem: MetricsSubsystem, - Name: "app_road_index", - Help: "Road index of the highest observed appQC.", - }, nil), commitGlobalBlockNumber: tmprometheus.NewGaugeIntVec(prometheus.GaugeOpts{ Namespace: MetricsNamespace, Subsystem: MetricsSubsystem, Name: "commit_global_block_number", Help: "Global block number of the highest observed commitQC.", }, nil), - appGlobalBlockNumber: tmprometheus.NewGaugeIntVec(prometheus.GaugeOpts{ - Namespace: MetricsNamespace, - Subsystem: MetricsSubsystem, - Name: "app_global_block_number", - Help: "Global block number of the highest observed appQC.", - }, nil), proposalToCommitLatency: tmprometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: MetricsNamespace, Subsystem: MetricsSubsystem, @@ -66,18 +52,10 @@ func (m *metrics) commitRoadIndexAt() *tmprometheus.GaugeInt { return m.commitRoadIndex.WithLabelValues() } -func (m *metrics) appRoadIndexAt() *tmprometheus.GaugeInt { - return m.appRoadIndex.WithLabelValues() -} - func (m *metrics) commitGlobalBlockNumberAt() *tmprometheus.GaugeInt { return m.commitGlobalBlockNumber.WithLabelValues() } -func (m *metrics) appGlobalBlockNumberAt() *tmprometheus.GaugeInt { - return m.appGlobalBlockNumber.WithLabelValues() -} - func (m *metrics) proposalToCommitLatencyAt() *tmprometheus.Histogram { return m.proposalToCommitLatency.WithLabelValues() } diff --git a/sei-tendermint/internal/autobahn/avail/metrics/metrics.go b/sei-tendermint/internal/autobahn/avail/metrics/metrics.go index 760ab822ac..7492041eb8 100644 --- a/sei-tendermint/internal/autobahn/avail/metrics/metrics.go +++ b/sei-tendermint/internal/autobahn/avail/metrics/metrics.go @@ -16,13 +16,9 @@ const MetricsSubsystem = "internal_autobahn_avail" type metrics struct { // Road index of the highest observed commitQC. commitRoadIndex prometheus.GaugeIntVec - // Road index of the highest observed appQC. - appRoadIndex prometheus.GaugeIntVec // Global block number of the highest observed commitQC. commitGlobalBlockNumber prometheus.GaugeIntVec - // Global block number of the highest observed appQC. - appGlobalBlockNumber prometheus.GaugeIntVec // Latency from proposal being constructed to commit being observed. proposalToCommitLatency prometheus.HistogramVec `metrics_buckets:"exp(0.01, 1.2, 35)"` @@ -40,7 +36,6 @@ func newObserved[T any]() utils.Mutex[*utils.Option[observed[T]]] { } var observedCommitQC = newObserved[*types.CommitQC]() -var observedAppQC = newObserved[*types.AppQC]() // ObserveCommitQC observes the CommitQC latency. func ObserveCommitQC(qc *types.CommitQC) { @@ -63,15 +58,3 @@ func ObserveCommitQC(qc *types.CommitQC) { *mLast = utils.Some(observed[*types.CommitQC]{now, qc}) } } - -func ObserveAppQC(qc *types.AppQC) { - now := time.Now() - for mLast := range observedAppQC.Lock() { - if last, ok := mLast.Get(); ok && last.val.Proposal().RoadIndex() >= qc.Proposal().RoadIndex() { - return - } - Global.appRoadIndexAt().Set(int64(qc.Proposal().RoadIndex())) // nolint: gosec - Global.appGlobalBlockNumberAt().Set(int64(qc.Proposal().GlobalRange().Next)) // nolint: gosec - *mLast = utils.Some(observed[*types.AppQC]{now, qc}) - } -} diff --git a/sei-tendermint/internal/autobahn/data/metrics/metrics.go b/sei-tendermint/internal/autobahn/data/metrics/metrics.go index bf230ad8fd..338d61130b 100644 --- a/sei-tendermint/internal/autobahn/data/metrics/metrics.go +++ b/sei-tendermint/internal/autobahn/data/metrics/metrics.go @@ -16,6 +16,7 @@ type metrics struct { type resourceMetrics struct { Receive *prometheus.Histogram Execute *prometheus.Histogram + Certify *prometheus.Histogram } type Metrics struct { @@ -28,6 +29,7 @@ func Get() *Metrics { return resourceMetrics{ Receive: Global.latencyAt(resource, "receive"), Execute: Global.latencyAt(resource, "execute"), + Certify: Global.latencyAt(resource, "certify"), } } return &Metrics{ diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index 3c0ba219ae..d27fbc52da 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -167,6 +167,16 @@ func (i *inner) updateNextBlock(m *metrics.Metrics) { } } +func (i *inner) observeCertify(m *metrics.Metrics, gr types.GlobalRange) { + t := time.Now() + for n := gr.First; n < gr.Next; n++ { + b := i.blocks[n] + latency := t.Sub(b.Payload().CreatedAt()).Seconds() + m.Blocks.Certify.Observe(latency) + m.Txs.Certify.ObserveWithWeight(latency, uint64(len(b.Payload().Txs()))) + } +} + // State of the chain. // Contains blocks in global order and proofs of sequencing: (CommitQC) and execution result (AppQC). type State struct { @@ -617,7 +627,7 @@ func (s *State) PushAppHash(ctx context.Context, n types.GlobalBlockNumber, hash // CRITICAL: We need to persist AppHash before we return and start executing the next block, // otherwise we lose the apphash on restart. // TODO(gprusak): this is a temporary measure, until AppHashes are persisted outside of BlockDB. - if err := ctrl.WaitUntil(ctx, func() bool { return n <= inner.persisted.NextAppProposal }); err != nil { + if err := ctrl.WaitUntil(ctx, func() bool { return n < inner.persisted.NextAppProposal }); err != nil { return err } } @@ -655,6 +665,7 @@ func (s *State) PushAppQC(ctx context.Context, appQC *types.AppQC) error { if err := inner.insertAppQC(s.cfg.Registry, appQC); err != nil { return err } + inner.observeCertify(s.metrics, gr) ctrl.Updated() return nil } diff --git a/sei-tendermint/internal/p2p/giga/service.go b/sei-tendermint/internal/p2p/giga/service.go index 2372597712..d62210247c 100644 --- a/sei-tendermint/internal/p2p/giga/service.go +++ b/sei-tendermint/internal/p2p/giga/service.go @@ -13,11 +13,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/scope" ) -// Service serves the giga RPC API. NewService builds a full validator -// service (all streams); NewFullNodeService builds a fullnode-only -// service (StreamFullCommitQCs + GetBlock) for fullnodes. state is None -// on block-sync-only services; the consensus / avail handlers reach it -// via validatorState() and panic if invoked outside RunServer / RunClient. +// Service serves the giga RPC API. type Service struct { getBlockReqs chan req data *data.State @@ -28,6 +24,7 @@ type validatorService struct { state *consensus.State } +// NewService constructs service with all streams. func NewService(state *consensus.State) *Service { return &Service{ getBlockReqs: make(chan req), From 779b0589bdf93214058f38c5a6522c7309090202 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 11 Aug 2026 18:25:36 +0200 Subject: [PATCH 58/61] added test for out-of-order PushAppHash --- .../internal/autobahn/data/state.go | 32 ++++++++-------- .../internal/autobahn/data/state_test.go | 38 +++++++++++++++++++ 2 files changed, 54 insertions(+), 16 deletions(-) diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index d27fbc52da..d0bf2a1eed 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -15,6 +15,10 @@ import ( const blocksCacheSize = 4000 +// ErrOutOfOrder is returned when PushAppHash receives an app hash past the +// next CommitQC range waiting for execution. +var ErrOutOfOrder = errors.New("out of order") + // Config is the config for the data State. type Config struct { // Registry is the authoritative source of committee and stake information. @@ -167,16 +171,6 @@ func (i *inner) updateNextBlock(m *metrics.Metrics) { } } -func (i *inner) observeCertify(m *metrics.Metrics, gr types.GlobalRange) { - t := time.Now() - for n := gr.First; n < gr.Next; n++ { - b := i.blocks[n] - latency := t.Sub(b.Payload().CreatedAt()).Seconds() - m.Blocks.Certify.Observe(latency) - m.Txs.Certify.ObserveWithWeight(latency, uint64(len(b.Payload().Txs()))) - } -} - // State of the chain. // Contains blocks in global order and proofs of sequencing: (CommitQC) and execution result (AppQC). type State struct { @@ -602,14 +596,14 @@ func (s *State) globalBlockByHashFromDB(hash types.BlockHeaderHash) (utils.Optio // PushAppHash marks blocks up to n as executed. func (s *State) PushAppHash(ctx context.Context, n types.GlobalBlockNumber, hash types.AppHash) error { for inner, ctrl := range s.inner.Lock() { - if err := ctrl.WaitUntil(ctx, func() bool { return inner.nextAppProposal < inner.nextBlock }); err != nil { + if err := ctrl.WaitUntil(ctx, func() bool { return n < inner.nextBlock }); err != nil { return err } - p := inner.qcs[inner.nextAppProposal].QC().Proposal() - if want := p.GlobalRange().Next - 1; n > want { + p := inner.qcs[n].QC().Proposal() + if inner.nextAppProposal < p.GlobalRange().First { // We expect the AppHashes to be pushed in order. - return fmt.Errorf("received appHash for %v, while still waiting for appHash for %v", n, want) - } else if n != want { + return fmt.Errorf("received appHash for %v: %w", n, ErrOutOfOrder) + } else if n != p.GlobalRange().Next-1 { // We only care about the AppHash of the last block of the CommitQC. return nil } @@ -665,7 +659,13 @@ func (s *State) PushAppQC(ctx context.Context, appQC *types.AppQC) error { if err := inner.insertAppQC(s.cfg.Registry, appQC); err != nil { return err } - inner.observeCertify(s.metrics, gr) + t := time.Now() + for n := gr.First; n < gr.Next; n++ { + b := inner.blocks[n] + latency := t.Sub(b.Payload().CreatedAt()).Seconds() + s.metrics.Blocks.Certify.Observe(latency) + s.metrics.Txs.Certify.ObserveWithWeight(latency, uint64(len(b.Payload().Txs()))) + } ctrl.Updated() return nil } diff --git a/sei-tendermint/internal/autobahn/data/state_test.go b/sei-tendermint/internal/autobahn/data/state_test.go index ae93a4c73f..ede0486d7f 100644 --- a/sei-tendermint/internal/autobahn/data/state_test.go +++ b/sei-tendermint/internal/autobahn/data/state_test.go @@ -376,6 +376,44 @@ func TestExecution(t *testing.T) { } } +func TestPushAppHashRejectsJumpOverCommitQCRange(t *testing.T) { + ctx := t.Context() + rng := utils.TestRng() + registry, keys := epoch.GenRegistry(rng, 3) + + state := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) + require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { + s.SpawnBgNamed("state.Run()", func() error { return utils.IgnoreCancel(state.Run(ctx)) }) + epoch := registry.LatestEpoch() + var qcs []*types.CommitQC + for range 3 { + var prev utils.Option[*types.CommitQC] + if len(qcs) > 0 { + prev = utils.Some(qcs[len(qcs)-1]) + } + qc, blocks := TestCommitQC(rng, epoch, keys, prev) + if err := state.PushQC(ctx, qc, blocks); err != nil { + return fmt.Errorf("PushQC(): %w", err) + } + qcs = append(qcs, qc.QC()) + } + if err := state.PushAppHash(ctx, qcs[0].GlobalRange().Next-1, types.GenAppHash(rng)); err != nil { + return fmt.Errorf("PushAppHash(qc1): %w", err) + } + if err := state.PushAppHash(ctx, qcs[2].GlobalRange().Next-1, types.GenAppHash(rng)); !errors.Is(err, ErrOutOfOrder) { + return fmt.Errorf("PushAppHash(qc3 before qc2) error = %w, want %w", err, ErrOutOfOrder) + } + + if err := state.PushAppHash(ctx, qcs[1].GlobalRange().Next-1, types.GenAppHash(rng)); err != nil { + return fmt.Errorf("PushAppHash(qc2): %w", err) + } + if err := state.PushAppHash(ctx, qcs[2].GlobalRange().Next-1, types.GenAppHash(rng)); err != nil { + return fmt.Errorf("PushAppHash(qc3): %w", err) + } + return nil + })) +} + func TestPushBlockAcceptsBlockWithQC(t *testing.T) { ctx := t.Context() rng := utils.TestRng() From d4e2c2ca2ffa17a1340199393dbc6e6cb3e29d3f Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 11 Aug 2026 18:36:22 +0200 Subject: [PATCH 59/61] more precise old apphash dropping criterion --- sei-tendermint/internal/autobahn/data/state.go | 6 +++--- sei-tendermint/internal/autobahn/data/state_test.go | 13 +++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index d0bf2a1eed..be8e674291 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -600,11 +600,11 @@ func (s *State) PushAppHash(ctx context.Context, n types.GlobalBlockNumber, hash return err } p := inner.qcs[n].QC().Proposal() - if inner.nextAppProposal < p.GlobalRange().First { + if next, first := inner.nextAppProposal, p.GlobalRange().First; next < first { // We expect the AppHashes to be pushed in order. return fmt.Errorf("received appHash for %v: %w", n, ErrOutOfOrder) - } else if n != p.GlobalRange().Next-1 { - // We only care about the AppHash of the last block of the CommitQC. + } else if next != first || n != p.GlobalRange().Next-1 { + // We only care about the AppHash of the last block of the range. return nil } proposal := types.NewAppProposal(p, hash) diff --git a/sei-tendermint/internal/autobahn/data/state_test.go b/sei-tendermint/internal/autobahn/data/state_test.go index ede0486d7f..79f4bd9faa 100644 --- a/sei-tendermint/internal/autobahn/data/state_test.go +++ b/sei-tendermint/internal/autobahn/data/state_test.go @@ -400,6 +400,12 @@ func TestPushAppHashRejectsJumpOverCommitQCRange(t *testing.T) { if err := state.PushAppHash(ctx, qcs[0].GlobalRange().Next-1, types.GenAppHash(rng)); err != nil { return fmt.Errorf("PushAppHash(qc1): %w", err) } + if qcs[2].GlobalRange().Len() < 2 { + panic("qcs[2].Len() is too small for this test") + } + if err := state.PushAppHash(ctx, qcs[2].GlobalRange().Next-2, types.GenAppHash(rng)); !errors.Is(err, ErrOutOfOrder) { + return fmt.Errorf("PushAppHash(qc3 before qc2) error = %w, want %w", err, ErrOutOfOrder) + } if err := state.PushAppHash(ctx, qcs[2].GlobalRange().Next-1, types.GenAppHash(rng)); !errors.Is(err, ErrOutOfOrder) { return fmt.Errorf("PushAppHash(qc3 before qc2) error = %w, want %w", err, ErrOutOfOrder) } @@ -410,6 +416,13 @@ func TestPushAppHashRejectsJumpOverCommitQCRange(t *testing.T) { if err := state.PushAppHash(ctx, qcs[2].GlobalRange().Next-1, types.GenAppHash(rng)); err != nil { return fmt.Errorf("PushAppHash(qc3): %w", err) } + // Inserting old stuff should be a noop. + if err := state.PushAppHash(ctx, qcs[1].GlobalRange().Next-1, types.GenAppHash(rng)); err != nil { + return fmt.Errorf("PushAppHash(qc2): %w", err) + } + if err := state.PushAppHash(ctx, qcs[2].GlobalRange().Next-2, types.GenAppHash(rng)); err != nil { + return fmt.Errorf("PushAppHash(qc2): %w", err) + } return nil })) } From 9ac7ed8c2dba5a8a13d277e1500f3f0ceb2b0a41 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 11 Aug 2026 18:50:23 +0200 Subject: [PATCH 60/61] another test --- .../internal/autobahn/data/state.go | 5 +- .../internal/autobahn/data/state_test.go | 78 ++++++++++++------- 2 files changed, 52 insertions(+), 31 deletions(-) diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index be8e674291..46c58c1659 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -599,11 +599,14 @@ func (s *State) PushAppHash(ctx context.Context, n types.GlobalBlockNumber, hash if err := ctrl.WaitUntil(ctx, func() bool { return n < inner.nextBlock }); err != nil { return err } + if n < inner.nextAppProposal { + return nil + } p := inner.qcs[n].QC().Proposal() if next, first := inner.nextAppProposal, p.GlobalRange().First; next < first { // We expect the AppHashes to be pushed in order. return fmt.Errorf("received appHash for %v: %w", n, ErrOutOfOrder) - } else if next != first || n != p.GlobalRange().Next-1 { + } else if n != p.GlobalRange().Next-1 { // We only care about the AppHash of the last block of the range. return nil } diff --git a/sei-tendermint/internal/autobahn/data/state_test.go b/sei-tendermint/internal/autobahn/data/state_test.go index 79f4bd9faa..e02097746b 100644 --- a/sei-tendermint/internal/autobahn/data/state_test.go +++ b/sei-tendermint/internal/autobahn/data/state_test.go @@ -93,17 +93,12 @@ func writeAppDataToBlockDB(t testing.TB, rng utils.Rng, db types.BlockDB, keys [ // raw goroutine so cleanup is structured. func pushAppHashesRunning(ctx context.Context, state *State, rng utils.Rng, first, next types.GlobalBlockNumber) error { return scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { - runCtx, cancel := context.WithCancel(ctx) - s.SpawnBgNamed("state.Run", func() error { - return utils.IgnoreCancel(state.Run(runCtx)) - }) + s.SpawnBgNamed("state.Run", func() error { return utils.IgnoreCancel(state.Run(ctx)) }) for n := first; n < next; n++ { if err := state.PushAppHash(ctx, n, types.GenAppHash(rng)); err != nil { - cancel() return err } } - cancel() return nil }) } @@ -504,18 +499,13 @@ func TestPushQCBeforeRunPersistsToBlockDB(t *testing.T) { require.False(t, db.Status().IsPresent(), "PushQC must not write BlockDB before Run") require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { - runCtx, cancel := context.WithCancel(ctx) - s.SpawnBgNamed("state.Run", func() error { - return utils.IgnoreCancel(state.Run(runCtx)) - }) + s.SpawnBgNamed("state.Run", func() error { return utils.IgnoreCancel(state.Run(ctx)) }) // PushAppHash waits on persisted.NextBlock, so success implies Flush. for n := gr1.First; n < gr1.Next; n++ { if err := state.PushAppHash(ctx, n, types.GenAppHash(rng)); err != nil { - cancel() return fmt.Errorf("PushAppHash(%d): %w", n, err) } } - cancel() return nil })) @@ -549,9 +539,7 @@ func TestEvictionWaitsForAppQC(t *testing.T) { state := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { - runCtx, cancel := context.WithCancel(ctx) - defer cancel() - s.SpawnBgNamed("state.Run", func() error { return utils.IgnoreCancel(state.Run(runCtx)) }) + s.SpawnBgNamed("state.Run", func() error { return utils.IgnoreCancel(state.Run(ctx)) }) if err := state.PushQC(ctx, qc1, blocks1); err != nil { return fmt.Errorf("PushQC(qc1): %w", err) @@ -654,6 +642,48 @@ func TestEvictionWaitsForPersistedAppQC(t *testing.T) { } } +func TestPushAppHashBelowAnchorSucceeds(t *testing.T) { + ctx := t.Context() + rng := utils.TestRng() + registry, keys := epoch.GenRegistry(rng, 3) + epoch := registry.LatestEpoch() + + require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { + state := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) + s.SpawnBgNamed("state.Run", func() error { return utils.IgnoreCancel(state.Run(ctx)) }) + + prev := utils.None[*types.CommitQC]() + for range 2 { + qc, blocks := TestCommitQC(rng, epoch, keys, prev) + prev = utils.Some(qc.QC()) + gr := qc.QC().GlobalRange() + if err := state.PushQC(ctx, qc, blocks); err != nil { + return fmt.Errorf("PushQC: %w", err) + } + if err := state.PushAppHash(ctx, gr.Next-1, types.GenAppHash(rng)); err != nil { + return fmt.Errorf("PushAppHash(tip): %w", err) + } + if err := pushAppQCForBlock(ctx, state, keys, gr.First); err != nil { + return fmt.Errorf("pushAppQCForBlock(%d): %w", gr.First, err) + } + } + // Wait for anchor to progress past first block. + if _, err := state.Anchor().Wait(ctx, func(anchor utils.Option[Anchor]) bool { + if anchor, ok := anchor.Get(); ok { + return registry.FirstBlock() < anchor.AppQC.Proposal().GlobalRange().First + } + return false + }); err != nil { + return fmt.Errorf("state.Anchor.Wait(): %w", err) + } + // Pushing apphash for height below the anchor should NOT expolode. + if err := state.PushAppHash(ctx, registry.FirstBlock(), types.GenAppHash(rng)); err != nil { + return fmt.Errorf("PushAppHash below anchor: %w", err) + } + return nil + })) +} + // TestNextToExecuteAfterAppEviction checks WaitUntilExecuted / nextToExecute // still work when persisted AppQC aggressively evicts through nextAppProposal // (first = persisted.First). @@ -669,11 +699,7 @@ func TestNextToExecuteAfterAppEviction(t *testing.T) { state := newTestState(t, &Config{Registry: registry}, newTestBlockDB(t, t.TempDir())) require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { - runCtx, cancel := context.WithCancel(ctx) - defer cancel() - s.SpawnBgNamed("state.Run", func() error { - return utils.IgnoreCancel(state.Run(runCtx)) - }) + s.SpawnBgNamed("state.Run", func() error { return utils.IgnoreCancel(state.Run(ctx)) }) if err := state.PushQC(ctx, qc1, blocks1); err != nil { return fmt.Errorf("PushQC(qc1): %w", err) @@ -757,11 +783,7 @@ func TestPushAppQCPersistsAndRecovers(t *testing.T) { db1 := newTestBlockDB(t, dir) state1 := newTestState(t, &Config{Registry: registry}, db1) require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { - runCtx, cancel := context.WithCancel(ctx) - defer cancel() - s.SpawnBgNamed("state.Run", func() error { - return utils.IgnoreCancel(state1.Run(runCtx)) - }) + s.SpawnBgNamed("state.Run", func() error { return utils.IgnoreCancel(state1.Run(ctx)) }) if err := state1.PushQC(ctx, qc1, blocks1); err != nil { return fmt.Errorf("PushQC(qc1): %w", err) @@ -872,11 +894,7 @@ func TestPruningWithPartialQCRange(t *testing.T) { require.NoError(t, pushAppHashesRunning(ctx, state1, rng, gr1.First, gr2.Next)) var exclusiveFloor types.GlobalBlockNumber require.NoError(t, scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { - runCtx, cancel := context.WithCancel(ctx) - defer cancel() - s.SpawnBgNamed("state.Run", func() error { - return utils.IgnoreCancel(state1.Run(runCtx)) - }) + s.SpawnBgNamed("state.Run", func() error { return utils.IgnoreCancel(state1.Run(ctx)) }) if err := pushAppQCForBlock(ctx, state1, keys, gr1.First); err != nil { return err } From bfe6137497add9e3a53e81f310e1ef2d90e409de Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 11 Aug 2026 19:26:25 +0200 Subject: [PATCH 61/61] Re-run checks