Skip to content

Commit

Permalink
v2/snapshot: add pruneSnapshotPeriodically
Browse files Browse the repository at this point in the history
pruneSnapshot: delete the nSnapshotsPrune oldest snapshots, keep the latestSnapshotsKeep snapshots
pruneSnapshotPeriodically: prune the snapshots at the start of each pruningPeriod
  • Loading branch information
Francesco4203 committed Jul 8, 2024
1 parent 265bf11 commit 38c7372
Show file tree
Hide file tree
Showing 2 changed files with 58 additions and 0 deletions.
10 changes: 10 additions & 0 deletions consensus/consortium/v2/consortium.go
Original file line number Diff line number Diff line change
Expand Up @@ -465,6 +465,7 @@ func (c *Consortium) verifyCascadingFields(chain consensus.ChainHeaderReader, he
err = c.verifyValidatorFieldsInExtraData(chain, extraData, header)
if err != nil {
return err

}

if isShillin && extraData.HasFinalityVote == 1 {
Expand Down Expand Up @@ -629,6 +630,9 @@ func (c *Consortium) snapshot(chain consensus.ChainHeaderReader, number uint64,
if err := snap.store(c.db); err != nil {
return nil, err
}
if err := snap.pruneSnapshotPeriodically(c.db, chain); err != nil {
return nil, err
}
log.Info("Stored checkpoint snapshot to disk", "number", number, "hash", hash)
figure.NewColorFigure("Welcome to DPOS", "", "green", true).Print()
break
Expand Down Expand Up @@ -683,6 +687,11 @@ func (c *Consortium) snapshot(chain consensus.ChainHeaderReader, number uint64,
}
c.recents.Add(snap.Hash, snap)

// Prune the snapshot periodically
if err := snap.pruneSnapshotPeriodically(c.db, chain); err != nil {
return nil, err
}

// If we've generated a new checkpoint snapshot, save to disk
if snap.Number%c.config.EpochV2 == 0 && len(headers) > 0 {
if err = snap.store(c.db); err != nil {
Expand Down Expand Up @@ -1832,6 +1841,7 @@ func (c *Consortium) IsPeriodBlock(chain consensus.ChainHeaderReader, header *ty
if c.isTest {
return c.testTrippPeriod
}

number := header.Number.Uint64()
if number%c.config.EpochV2 != 0 || !chain.Config().IsTripp(header.Number) {
return false
Expand Down
48 changes: 48 additions & 0 deletions consensus/consortium/v2/snapshot.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,22 @@ import (
blsCommon "github.com/ethereum/go-ethereum/crypto/bls/common"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/internal/ethapi"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/params"
lru "github.com/hashicorp/golang-lru"
)

const (
blocksPerEpoch = 200
epochsPerPeriod = 144
)

var (
latestSnapshotsKeep = blocksPerEpoch * epochsPerPeriod * 5 // 5 days
snapshotsToBePruned = epochsPerPeriod * 2 // 2 days
pruningPeriod = blocksPerEpoch * epochsPerPeriod * 1 // every 1 day
)

// Snapshot is the state of the authorization validators at a given point in time.
type Snapshot struct {
// private fields are not json.Marshalled
Expand Down Expand Up @@ -112,6 +124,42 @@ func loadSnapshot(
return snap, nil
}

// snapshot pruning
// delete the nSnapshotsPrune oldest snapshots, keep the latestSnapshotsKeep snapshots
func (s *Snapshot) pruneSnapshot(db ethdb.Database, nSnapshotPrune int, chain consensus.ChainHeaderReader) error {
log.Info("Pruning snapshots at block", "block", s.Number, "nSnapshotPrune", nSnapshotPrune)
// Get block number to start pruning
curBlockNumber := s.Number
curBlockNumber -= curBlockNumber % uint64(blocksPerEpoch) // start of the current epoch
curBlockNumber -= uint64(latestSnapshotsKeep) // start of the oldest epoch to keep

// delete nSnapshotPrune snapshots starting from curBlockNumber to the older ones
batch := db.NewBatch()
for nSnapshotPrune > 0 {
nSnapshotPrune--
header := chain.GetHeaderByNumber(curBlockNumber)
if header == nil {
// no more snapshots to prune
break
}
curHash := header.Hash()
if err := batch.Delete(append(rawdb.ConsortiumSnapshotPrefix, curHash[:]...)); err != nil {
return err
}
curBlockNumber -= uint64(blocksPerEpoch)
}
log.Info("Pruned snapshots done")
return batch.Write()
}

// periodically prune the snapshots at the start of each pruningPeriod
func (s *Snapshot) pruneSnapshotPeriodically(db ethdb.Database, chain consensus.ChainHeaderReader) error {
if s.Number%uint64(pruningPeriod) == 0 {
return s.pruneSnapshot(db, snapshotsToBePruned, chain)
}
return nil
}

// store inserts the snapshot into the database.
func (s *Snapshot) store(db ethdb.Database) error {
blob, err := json.Marshal(s)
Expand Down

0 comments on commit 38c7372

Please sign in to comment.