Max cap search results for ots API by default + cli flag to override …

…it (#7924) Otterscan API search methods allow the user to inform the page size. This PR adds an internal max (default == 25 results) to cap the page size, regardless of what the user asks. It also adds a `--ots.search.max.pagesize` CLI args to override this max (either in erigon and rpcdaemon binaries).
erigontech · Jul 28, 2023 · d3f8b58 · d3f8b58
1 parent 7e3c6ea
commit d3f8b58
Show file tree

Hide file tree

Showing 10 changed files with 33 additions and 8 deletions.
diff --git a/cmd/rpcdaemon/cli/config.go b/cmd/rpcdaemon/cli/config.go
@@ -119,6 +119,8 @@ func RootCommand() (*cobra.Command, *httpcfg.HttpCfg) {
 	rootCmd.PersistentFlags().IntVar(&cfg.BatchLimit, utils.RpcBatchLimit.Name, utils.RpcBatchLimit.Value, utils.RpcBatchLimit.Usage)
 	rootCmd.PersistentFlags().IntVar(&cfg.ReturnDataLimit, utils.RpcReturnDataLimit.Name, utils.RpcReturnDataLimit.Value, utils.RpcReturnDataLimit.Usage)
 
+	rootCmd.PersistentFlags().Uint64Var(&cfg.OtsMaxPageSize, utils.OtsSearchMaxCapFlag.Name, utils.OtsSearchMaxCapFlag.Value, utils.OtsSearchMaxCapFlag.Usage)
+
 	if err := rootCmd.MarkPersistentFlagFilename("rpc.accessList", "json"); err != nil {
 		panic(err)
 	}

diff --git a/cmd/rpcdaemon/cli/httpcfg/http_cfg.go b/cmd/rpcdaemon/cli/httpcfg/http_cfg.go
@@ -63,4 +63,7 @@ type HttpCfg struct {
 
 	BatchLimit      int // Maximum number of requests in a batch
 	ReturnDataLimit int // Maximum number of bytes returned from calls (like eth_call)
+
+	// Ots API
+	OtsMaxPageSize uint64
 }
diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go
@@ -781,6 +781,12 @@ var (
 		Usage: "Port for sentinel",
 		Value: 7777,
 	}
+
+	OtsSearchMaxCapFlag = cli.Uint64Flag{
+		Name:  "ots.search.max.pagesize",
+		Usage: "Max allowed page size for search methods",
+		Value: 25,
+	}
 )
 
 var MetricFlags = []cli.Flag{&MetricsEnabledFlag, &MetricsHTTPFlag, &MetricsPortFlag}

diff --git a/turbo/cli/default_flags.go b/turbo/cli/default_flags.go
@@ -156,4 +156,6 @@ var DefaultFlags = []cli.Flag{
 	&utils.LightClientDiscoveryTCPPortFlag,
 	&utils.SentinelAddrFlag,
 	&utils.SentinelPortFlag,
+
+	&utils.OtsSearchMaxCapFlag,
 }
diff --git a/turbo/cli/flags.go b/turbo/cli/flags.go
@@ -405,6 +405,8 @@ func setEmbeddedRpcDaemon(ctx *cli.Context, cfg *nodecfg.Config, logger log.Logg
 		BatchLimit:           ctx.Int(utils.RpcBatchLimit.Name),
 		ReturnDataLimit:      ctx.Int(utils.RpcReturnDataLimit.Name),
 
+		OtsMaxPageSize: ctx.Uint64(utils.OtsSearchMaxCapFlag.Name),
+
 		TxPoolApiAddr: ctx.String(utils.TxpoolApiAddrFlag.Name),
 
 		StateCache: kvcache.DefaultCoherentConfig,

diff --git a/turbo/jsonrpc/daemon.go b/turbo/jsonrpc/daemon.go
@@ -32,7 +32,7 @@ func APIList(db kv.RoDB, borDb kv.RoDB, eth rpchelper.ApiBackend, txPool txpool.
 	adminImpl := NewAdminAPI(eth)
 	parityImpl := NewParityAPIImpl(base, db)
 	borImpl := NewBorAPI(base, db, borDb) // bor (consensus) specific
-	otsImpl := NewOtterscanAPI(base, db)
+	otsImpl := NewOtterscanAPI(base, db, cfg.OtsMaxPageSize)
 	gqlImpl := NewGraphQLAPI(base, db)
 
 	if cfg.GraphQLEnabled {

diff --git a/turbo/jsonrpc/otterscan_api.go b/turbo/jsonrpc/otterscan_api.go
@@ -56,13 +56,15 @@ type OtterscanAPI interface {
 
 type OtterscanAPIImpl struct {
 	*BaseAPI
-	db kv.RoDB
+	db          kv.RoDB
+	maxPageSize uint64
 }
 
-func NewOtterscanAPI(base *BaseAPI, db kv.RoDB) *OtterscanAPIImpl {
+func NewOtterscanAPI(base *BaseAPI, db kv.RoDB, maxPageSize uint64) *OtterscanAPIImpl {
 	return &OtterscanAPIImpl{
-		BaseAPI: base,
-		db:      db,
+		BaseAPI:     base,
+		db:          db,
+		maxPageSize: maxPageSize,
 	}
 }
 
@@ -172,6 +174,10 @@ func (api *OtterscanAPIImpl) GetInternalOperations(ctx context.Context, hash com
 // than the necessary to fill pageSize in the last found block, i.e., let's say you want pageSize == 25,
 // you already found 24 txs, the next block contains 4 matches, then this function will return 28 txs.
 func (api *OtterscanAPIImpl) SearchTransactionsBefore(ctx context.Context, addr common.Address, blockNum uint64, pageSize uint16) (*TransactionsWithReceipts, error) {
+	if uint64(pageSize) > api.maxPageSize {
+		return nil, fmt.Errorf("max allowed page size: %v", api.maxPageSize)
+	}
+
 	dbtx, err := api.db.BeginRo(ctx)
 	if err != nil {
 		return nil, err
@@ -347,6 +353,10 @@ func (api *OtterscanAPIImpl) searchTransactionsBeforeV3(tx kv.TemporalTx, ctx co
 // than the necessary to fill pageSize in the last found block, i.e., let's say you want pageSize == 25,
 // you already found 24 txs, the next block contains 4 matches, then this function will return 28 txs.
 func (api *OtterscanAPIImpl) SearchTransactionsAfter(ctx context.Context, addr common.Address, blockNum uint64, pageSize uint16) (*TransactionsWithReceipts, error) {
+	if uint64(pageSize) > api.maxPageSize {
+		return nil, fmt.Errorf("max allowed page size: %v", api.maxPageSize)
+	}
+
 	dbtx, err := api.db.BeginRo(ctx)
 	if err != nil {
 		return nil, err

diff --git a/turbo/jsonrpc/otterscan_contract_creator_test.go b/turbo/jsonrpc/otterscan_contract_creator_test.go
@@ -10,7 +10,7 @@ import (
 
 func TestGetContractCreator(t *testing.T) {
 	m, _, _ := rpcdaemontest.CreateTestSentry(t)
-	api := NewOtterscanAPI(newBaseApiForTest(m), m.DB)
+	api := NewOtterscanAPI(newBaseApiForTest(m), m.DB, 25)
 
 	addr := libcommon.HexToAddress("0x537e697c7ab75a26f9ecf0ce810e3154dfcaaf44")
 	expectCreator := libcommon.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7")

diff --git a/turbo/jsonrpc/otterscan_search_backward_test.go b/turbo/jsonrpc/otterscan_search_backward_test.go
@@ -148,7 +148,7 @@ func TestBackwardBlockProviderWithMultipleChunksBlockNotFound(t *testing.T) {
 
 func TestSearchTransactionsBefore(t *testing.T) {
 	m, _, _ := rpcdaemontest.CreateTestSentry(t)
-	api := NewOtterscanAPI(newBaseApiForTest(m), m.DB)
+	api := NewOtterscanAPI(newBaseApiForTest(m), m.DB, 25)
 
 	addr := libcommon.HexToAddress("0x537e697c7ab75a26f9ecf0ce810e3154dfcaaf44")
 	t.Run("small page size", func(t *testing.T) {

diff --git a/turbo/jsonrpc/otterscan_transaction_by_sender_and_nonce_test.go b/turbo/jsonrpc/otterscan_transaction_by_sender_and_nonce_test.go
@@ -12,7 +12,7 @@ import (
 func TestGetTransactionBySenderAndNonce(t *testing.T) {
 	m, _, _ := rpcdaemontest.CreateTestSentry(t)
 	agg := m.HistoryV3Components()
-	api := NewOtterscanAPI(NewBaseApi(nil, nil, m.BlockReader, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs), m.DB)
+	api := NewOtterscanAPI(NewBaseApi(nil, nil, m.BlockReader, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs), m.DB, 25)
 
 	addr := common.HexToAddress("0x537e697c7ab75a26f9ecf0ce810e3154dfcaaf44")
 	expectCreator := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7")