state: separate 'lookupFileByItsRange' (#12036)

PR for #11840

erigon-lib/state/aggregator.go

@@ -1460,7 +1460,7 @@ func (ac *AggregatorRoTx) findMergeRange(maxEndTxNum, maxSpan uint64) RangesV3 {
 			}
 			// commitment waits until storage and account are merged so it may be a bit behind (if merge was interrupted before)
 			if !dr.values.needMerge || cr.values.to < dr.values.from {
-				if mf := ac.d[kd].lookupFileByItsRange(cr.values.from, cr.values.to); mf != nil {
+				if mf := ac.d[kd].lookupDirtyFileByItsRange(cr.values.from, cr.values.to); mf != nil {
 					// file for required range exists, hold this domain from merge but allow to merge comitemnt
 					r.domain[k].values = MergeRange{}
 					ac.a.logger.Debug("findMergeRange: commitment range is different but file exists in domain, hold further merge",

erigon-lib/state/domain_committed.go

@@ -147,14 +147,34 @@ func (dt *DomainRoTx) findShortenedKey(fullKey []byte, itemGetter *seg.Reader, i
 	return nil, false
 }
 
-func (dt *DomainRoTx) lookupFileByItsRange(txFrom uint64, txTo uint64) *filesItem {
+func (dt *DomainRoTx) lookupVisibleFileByItsRange(txFrom uint64, txTo uint64) *filesItem {
 	var item *filesItem
 	for _, f := range dt.files {
 		if f.startTxNum == txFrom && f.endTxNum == txTo {
 			item = f.src
 			break
 		}
 	}
+	if item == nil || item.bindex == nil {
+		visibleFiles := ""
+		for _, f := range dt.files {
+			visibleFiles += fmt.Sprintf("%d-%d;", f.startTxNum/dt.d.aggregationStep, f.endTxNum/dt.d.aggregationStep)
+		}
+		dt.d.logger.Warn("[agg] lookupVisibleFileByItsRange: file not found",
+			"stepFrom", txFrom/dt.d.aggregationStep, "stepTo", txTo/dt.d.aggregationStep,
+			"_visible", visibleFiles, "visibleFilesCount", len(dt.files))
+
+		if item != nil && item.bindex == nil {
+			dt.d.logger.Warn("[agg] lookupVisibleFileByItsRange: file found but not indexed", "f", item.decompressor.FileName())
+		}
+
+		return nil
+	}
+	return item
+}
+
+func (dt *DomainRoTx) lookupDirtyFileByItsRange(txFrom uint64, txTo uint64) *filesItem {
+	var item *filesItem
 	if item == nil {
 		dt.d.dirtyFiles.Walk(func(files []*filesItem) bool {
 			for _, f := range files {
@@ -172,17 +192,12 @@ func (dt *DomainRoTx) lookupFileByItsRange(txFrom uint64, txTo uint64) *filesIte
 		for _, item := range dt.d.dirtyFiles.Items() {
 			fileStepsss += fmt.Sprintf("%d-%d;", item.startTxNum/dt.d.aggregationStep, item.endTxNum/dt.d.aggregationStep)
 		}
-		visibleFiles := ""
-		for _, f := range dt.files {
-			visibleFiles += fmt.Sprintf("%d-%d;", f.startTxNum/dt.d.aggregationStep, f.endTxNum/dt.d.aggregationStep)
-		}
-		dt.d.logger.Warn("[agg] lookupFileByItsRange: file not found",
+		dt.d.logger.Warn("[agg] lookupDirtyFileByItsRange: file not found",
 			"stepFrom", txFrom/dt.d.aggregationStep, "stepTo", txTo/dt.d.aggregationStep,
-			"files", fileStepsss, "_visible", visibleFiles,
-			"visibleFilesCount", len(dt.files), "filesCount", dt.d.dirtyFiles.Len())
+			"files", fileStepsss, "filesCount", dt.d.dirtyFiles.Len())
 
 		if item != nil && item.bindex == nil {
-			dt.d.logger.Warn("[agg] lookupFileByItsRange: file found but not indexed", "f", item.decompressor.FileName())
+			dt.d.logger.Warn("[agg] lookupDirtyFileByItsRange: file found but not indexed", "f", item.decompressor.FileName())
 		}
 
 		return nil
@@ -224,15 +239,15 @@ func (dt *DomainRoTx) lookupByShortenedKey(shortKey []byte, getter *seg.Reader)
 func (dt *DomainRoTx) commitmentValTransformDomain(rng MergeRange, accounts, storage *DomainRoTx, mergedAccount, mergedStorage *filesItem) (valueTransformer, error) {
 	hadToLookupStorage := mergedStorage == nil
 	if mergedStorage == nil {
-		mergedStorage = storage.lookupFileByItsRange(rng.from, rng.to)
+		mergedStorage = storage.lookupVisibleFileByItsRange(rng.from, rng.to)
 		if mergedStorage == nil {
 			// TODO may allow to merge, but storage keys will be stored as plainkeys
 			return nil, fmt.Errorf("merged v1-account.%d-%d.kv file not found", rng.from/dt.d.aggregationStep, rng.to/dt.d.aggregationStep)
 		}
 	}
 	hadToLookupAccount := mergedAccount == nil
 	if mergedAccount == nil {
-		mergedAccount = accounts.lookupFileByItsRange(rng.from, rng.to)
+		mergedAccount = accounts.lookupVisibleFileByItsRange(rng.from, rng.to)
 		if mergedAccount == nil {
 			return nil, fmt.Errorf("merged v1-account.%d-%d.kv file not found", rng.from/dt.d.aggregationStep, rng.to/dt.d.aggregationStep)
 		}
@@ -271,7 +286,7 @@ func (dt *DomainRoTx) commitmentValTransformDomain(rng MergeRange, accounts, sto
 		}
 		sig, ok := storageFileMap[keyFromTxNum][keyEndTxNum]
 		if !ok {
-			dirty := storage.lookupFileByItsRange(keyFromTxNum, keyEndTxNum)
+			dirty := storage.lookupDirtyFileByItsRange(keyFromTxNum, keyEndTxNum)
 			if dirty == nil {
 				return nil, fmt.Errorf("dirty storage file not found %d-%d", keyFromTxNum/dt.d.aggregationStep, keyEndTxNum/dt.d.aggregationStep)
 			}
@@ -284,7 +299,7 @@ func (dt *DomainRoTx) commitmentValTransformDomain(rng MergeRange, accounts, sto
 		}
 		aig, ok := accountFileMap[keyFromTxNum][keyEndTxNum]
 		if !ok {
-			dirty := accounts.lookupFileByItsRange(keyFromTxNum, keyEndTxNum)
+			dirty := accounts.lookupDirtyFileByItsRange(keyFromTxNum, keyEndTxNum)
 			if dirty == nil {
 				return nil, fmt.Errorf("dirty account file not found %d-%d", keyFromTxNum/dt.d.aggregationStep, keyEndTxNum/dt.d.aggregationStep)
 			}

erigon-lib/state/domain_shared.go

@@ -423,12 +423,12 @@ func (sd *SharedDomains) replaceShortenedKeysInBranch(prefix []byte, branch comm
 
 	sto := sd.aggTx.d[kv.StorageDomain]
 	acc := sd.aggTx.d[kv.AccountsDomain]
-	storageItem := sto.lookupFileByItsRange(fStartTxNum, fEndTxNum)
+	storageItem := sto.lookupVisibleFileByItsRange(fStartTxNum, fEndTxNum)
 	if storageItem == nil {
 		sd.logger.Crit(fmt.Sprintf("storage file of steps %d-%d not found\n", fStartTxNum/sd.aggTx.a.aggregationStep, fEndTxNum/sd.aggTx.a.aggregationStep))
 		return nil, errors.New("storage file not found")
 	}
-	accountItem := acc.lookupFileByItsRange(fStartTxNum, fEndTxNum)
+	accountItem := acc.lookupVisibleFileByItsRange(fStartTxNum, fEndTxNum)
 	if accountItem == nil {
 		sd.logger.Crit(fmt.Sprintf("storage file of steps %d-%d not found\n", fStartTxNum/sd.aggTx.a.aggregationStep, fEndTxNum/sd.aggTx.a.aggregationStep))
 		return nil, errors.New("account file not found")
@@ -650,9 +650,7 @@ func (sd *SharedDomains) SetTrace(b bool) {
 }
 
 func (sd *SharedDomains) ComputeCommitment(ctx context.Context, saveStateAfter bool, blockNum uint64, logPrefix string) (rootHash []byte, err error) {
-	sd.aggTx.RestrictSubsetFileDeletions(true)
 	rootHash, err = sd.sdCtx.ComputeCommitment(ctx, saveStateAfter, blockNum, logPrefix)
-	sd.aggTx.RestrictSubsetFileDeletions(false)
 	return
 }