Dev2 (OffchainLabs#6)

* main to dev branch  (#1)

* zjubfd: add no trie options, support fastnode

* no truncate

* add prune path db

* fix destruction error

* update parallel param

* fix possible prefetch panic

* protect from nil root

* add the cache size of code

* mute unnecessary log

* Dev (OffchainLabs#4)

* add initial commit of trade

* fix root 00 issue

* refine the sync to target

* refine the sync to target

* add tg bots and more

* update gas price for test mode

* refine the order update

* refine the order update

* refine fee calculate

* update

* update

* update

* update

* update

* update

* update

* update better tg message

* add pair price track

* update test case

* add webscoket

* decouple trade module

* extract seracher as a processer

* update trade

* add executor wallets

* update

* remove duplicate ws

* add prefetch routine

* update v2 calculation method

* add test code

* add test code

* add test code

* add crypty code

* update some test log

* update some test log

* update some test log

* update some test log

* update some test log

* update some test log

* update

* support limit order

Author: unclezoro

.gitignore

@@ -30,6 +30,15 @@
 /geth*.zip.sig
 /geth*.zip.asc
 
+./trade/binance_test.go
+trade/binance_test.go
+trade/crypto_test.go
+trade/config/config.json
+trade/txhistory.md
+trade/pair_executor_test.go
+trade/network-info.md
+trade/metrics/*
+
 
 # travis
 profile.tmp
@@ -43,3 +52,4 @@ profile.cov
 .vscode
 
 tests/spec-tests/
+trade/config/config copy.json

Makefile

@@ -2,7 +2,7 @@
 # with Go source code. If you know what GOPATH is then you probably
 # don't need to bother with make.
 
-.PHONY: geth all test lint fmt clean devtools help
+.PHONY: geth searcher all test lint fmt clean devtools help
 
 GOBIN = ./build/bin
 GO ?= latest
@@ -14,6 +14,12 @@ geth:
 	@echo "Done building."
 	@echo "Run \"$(GOBIN)/geth\" to launch geth."
 
+#? searcher: Build the MEV searcher.
+searcher:
+	$(GORUN) build/ci.go install ./cmd/searcher
+	@echo "Done building."
+	@echo "Run \"$(GOBIN)/searcher\" to launch the MEV searcher."
+
 #? all: Build all packages and executables.
 all:
 	$(GORUN) build/ci.go install

accounts/abi/bind/base.go

@@ -334,10 +334,13 @@ func (c *BoundContract) createDynamicTx(opts *TransactOpts, contract *common.Add
 	gasLimit := opts.GasLimit
 	if opts.GasLimit == 0 {
 		var err error
+		// TODO 这里给一个合适的gas limit, 防止迭代次数太多
 		gasLimit, err = c.estimateGasLimit(opts, contract, input, nil, gasTipCap, gasFeeCap, value)
 		if err != nil {
 			return nil, err
 		}
+		// TODO 需要优化
+		gasLimit = gasLimit * 3 / 2
 	}
 	// create the transaction
 	nonce, err := c.getNonce(opts)
@@ -409,8 +412,10 @@ func (c *BoundContract) estimateGasLimit(opts *TransactOpts, contract *common.Ad
 			return 0, ErrNoCode
 		}
 	}
+	// Note: 这里给一个比较小的的gas limit, 防止迭代次数太多
 	msg := ethereum.CallMsg{
 		From:      opts.From,
+		Gas:       1000000,
 		To:        contract,
 		GasPrice:  gasPrice,
 		GasTipCap: gasTipCap,

cmd/geth/main.go

@@ -80,6 +80,7 @@ var (
 		utils.BlobPoolDataCapFlag,
 		utils.BlobPoolPriceBumpFlag,
 		utils.SyncModeFlag,
+		utils.TriesNoVerifyFlag,
 		utils.SyncTargetFlag,
 		utils.ExitWhenSyncedFlag,
 		utils.GCModeFlag,
@@ -256,6 +257,7 @@ func init() {
 		debug.Flags,
 		metricsFlags,
 	)
+
 	flags.AutoEnvVars(app.Flags, "GETH")
 
 	app.Before = func(ctx *cli.Context) error {

cmd/geth/snapshot.go

@@ -18,9 +18,11 @@ package main
 
 import (
 	"bytes"
+	"encoding/binary"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"math"
 	"os"
 	"slices"
 	"time"
@@ -33,6 +35,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/state/snapshot"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
@@ -63,6 +66,29 @@ two version states are available: genesis and the specific one.
 The default pruning target is the HEAD-127 state.
 
 WARNING: it's only supported in hash mode(--state.scheme=hash)".
+`,
+			},
+			{
+				Name: "insecure-prune-pathdb",
+				Usage: "Prune all pathdb state data, it will break storage for fullnode, only suitable for fast node " +
+					"who do not need trie storage at all",
+				ArgsUsage: "",
+				Action:    prunePathDB,
+				Category:  "MISCELLANEOUS COMMANDS",
+				Flags: []cli.Flag{
+					utils.DataDirFlag,
+					utils.AncientFlag,
+				},
+				Description: `
+will prune all historical trie state data except genesis block.
+All trie nodes will be deleted from the database. 
+
+It expects the genesis file as argument.
+
+WARNING: It's necessary to delete the trie clean cache after the pruning.
+If you specify another directory for the trie clean cache via "--cache.trie.journal"
+during the use of Geth, please also specify it here for correct deletion. Otherwise
+the trie clean cache with default directory will be deleted.
 `,
 			},
 			{
@@ -165,6 +191,99 @@ the expected order for the overlay tree migration.
 	}
 )
 
+// rangeCompactionThreshold is the minimal deleted entry number for
+// triggering range compaction. It's a quite arbitrary number but just
+// to avoid triggering range compaction because of small deletion.
+const rangeCompactionThreshold = 1000000
+
+func prunePathDB(ctx *cli.Context) error {
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+
+	pruneDB := utils.MakeChainDatabase(ctx, stack, false)
+	defer pruneDB.Close()
+
+	var (
+		count         int
+		size          common.StorageSize
+		pstart        = time.Now()
+		logged        = time.Now()
+		batch         = pruneDB.NewBatch()
+		iter          = pruneDB.NewIterator(nil, nil)
+		stateIDPrefix = []byte("L")
+	)
+	for iter.Next() {
+		key := iter.Key()
+		doDelete := false
+		switch {
+		case bytes.HasPrefix(key, stateIDPrefix) && len(key) == len(stateIDPrefix)+common.HashLength:
+			doDelete = true
+		case rawdb.IsAccountTrieNode(key):
+			doDelete = true
+		case rawdb.IsStorageTrieNode(key):
+			doDelete = true
+		default:
+		}
+		if doDelete {
+			count += 1
+			size += common.StorageSize(len(key) + len(iter.Value()))
+			batch.Delete(key)
+
+			var eta time.Duration // Realistically will never remain uninited
+			if done := binary.BigEndian.Uint64(key[:8]); done > 0 {
+				var (
+					left  = math.MaxUint64 - binary.BigEndian.Uint64(key[:8])
+					speed = done/uint64(time.Since(pstart)/time.Millisecond+1) + 1 // +1s to avoid division by zero
+				)
+				eta = time.Duration(left/speed) * time.Millisecond
+			}
+			if time.Since(logged) > 8*time.Second {
+				log.Info("Pruning state data", "nodes", count, "size", size,
+					"elapsed", common.PrettyDuration(time.Since(pstart)), "eta", common.PrettyDuration(eta))
+				logged = time.Now()
+			}
+			// Recreate the iterator after every batch commit in order
+			// to allow the underlying compactor to delete the entries.
+			if batch.ValueSize() >= ethdb.IdealBatchSize {
+				batch.Write()
+				batch.Reset()
+
+				iter.Release()
+				iter = pruneDB.NewIterator(nil, key)
+			}
+		}
+	}
+	if batch.ValueSize() > 0 {
+		batch.Write()
+		batch.Reset()
+	}
+	iter.Release()
+	log.Info("Pruned path db data", "nodes", count, "size", size, "elapsed", common.PrettyDuration(time.Since(pstart)))
+
+	// Start compactions, will remove the deleted data from the disk immediately.
+	// Note for small pruning, the compaction is skipped.
+	if count >= rangeCompactionThreshold {
+		cstart := time.Now()
+		for b := 0x00; b <= 0xf0; b += 0x10 {
+			var (
+				start = []byte{byte(b)}
+				end   = []byte{byte(b + 0x10)}
+			)
+			if b == 0xf0 {
+				end = nil
+			}
+			log.Info("Compacting database", "range", fmt.Sprintf("%#x-%#x", start, end), "elapsed", common.PrettyDuration(time.Since(cstart)))
+			if err := pruneDB.Compact(start, end); err != nil {
+				log.Error("Database compaction failed", "error", err)
+				return err
+			}
+		}
+		log.Info("Database compaction finished", "elapsed", common.PrettyDuration(time.Since(cstart)))
+	}
+
+	return nil
+}
+
 // Deprecation: this command should be deprecated once the hash-based
 // scheme is deprecated.
 func pruneState(ctx *cli.Context) error {
@@ -174,9 +293,6 @@ func pruneState(ctx *cli.Context) error {
 	chaindb := utils.MakeChainDatabase(ctx, stack, false)
 	defer chaindb.Close()
 
-	if rawdb.ReadStateScheme(chaindb) != rawdb.HashScheme {
-		log.Crit("Offline pruning is not required for path scheme")
-	}
 	prunerconfig := pruner.Config{
 		Datadir:   stack.ResolvePath(""),
 		BloomSize: ctx.Uint64(utils.BloomFilterSizeFlag.Name),
@@ -230,7 +346,7 @@ func verifyState(ctx *cli.Context) error {
 		NoBuild:    true,
 		AsyncBuild: false,
 	}
-	snaptree, err := snapshot.New(snapConfig, chaindb, triedb, headBlock.Root())
+	snaptree, err := snapshot.New(snapConfig, chaindb, triedb, headBlock.Root(), false)
 	if err != nil {
 		log.Error("Failed to open snapshot tree", "err", err)
 		return err
@@ -561,7 +677,7 @@ func dumpState(ctx *cli.Context) error {
 		NoBuild:    true,
 		AsyncBuild: false,
 	}
-	snaptree, err := snapshot.New(snapConfig, db, triedb, root)
+	snaptree, err := snapshot.New(snapConfig, db, triedb, root, false)
 	if err != nil {
 		return err
 	}
@@ -658,7 +774,7 @@ func snapshotExportPreimages(ctx *cli.Context) error {
 		NoBuild:    true,
 		AsyncBuild: false,
 	}
-	snaptree, err := snapshot.New(snapConfig, chaindb, triedb, root)
+	snaptree, err := snapshot.New(snapConfig, chaindb, triedb, root, false)
 	if err != nil {
 		return err
 	}

cmd/utils/flags.go

@@ -222,6 +222,14 @@ var (
 		Usage:    "Reduce key-derivation RAM & CPU usage at some expense of KDF strength",
 		Category: flags.AccountCategory,
 	}
+
+	TriesNoVerifyFlag = &cli.BoolFlag{
+		Name:     "tries-no-verify",
+		Usage:    "no merkle state root verification at all",
+		Value:    true,
+		Category: flags.FastNodeCategory,
+	}
+
 	EthRequiredBlocksFlag = &cli.StringFlag{
 		Name:     "eth.requiredblocks",
 		Usage:    "Comma separated block number-to-hash mappings to require for peering (<number>=<hash>)",
@@ -230,7 +238,7 @@ var (
 	BloomFilterSizeFlag = &cli.Uint64Flag{
 		Name:     "bloomfilter.size",
 		Usage:    "Megabytes of memory allocated to bloom-filter for pruning",
-		Value:    2048,
+		Value:    10,
 		Category: flags.EthCategory,
 	}
 	OverrideCancun = &cli.Uint64Flag{
@@ -901,7 +909,6 @@ Please note that --` + MetricsHTTPFlag.Name + ` must be set to start the server.
 		Value:    metrics.DefaultConfig.InfluxDBTags,
 		Category: flags.MetricsCategory,
 	}
-
 	MetricsEnableInfluxDBV2Flag = &cli.BoolFlag{
 		Name:     "metrics.influxdbv2",
 		Usage:    "Enable metrics export/push to an external InfluxDB v2 database",
@@ -928,6 +935,23 @@ Please note that --` + MetricsHTTPFlag.Name + ` must be set to start the server.
 		Value:    metrics.DefaultConfig.InfluxDBOrganization,
 		Category: flags.MetricsCategory,
 	}
+
+	// Searcher flag group
+	SearcherEnabledFlag = &cli.BoolFlag{
+		Name:     "searcher",
+		Usage:    "Enable MEV searcher functionality",
+		Category: flags.EthCategory,
+	}
+	SearcherConfigFlag = &cli.StringFlag{
+		Name:     "searcher.config",
+		Usage:    "Path to the searcher configuration file",
+		Category: flags.EthCategory,
+	}
+	SearcherDecryptKeyFlag = &cli.StringFlag{
+		Name:     "searcher.decryptkey-password",
+		Usage:    "Password for deriving searcher decryption key (will be securely hashed to generate a 32-byte key)",
+		Category: flags.EthCategory,
+	}
 )
 
 var (
@@ -1621,6 +1645,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 		cfg.StateScheme = rawdb.HashScheme
 		log.Warn("Forcing hash state-scheme for archive mode")
 	}
+	if ctx.IsSet(TriesNoVerifyFlag.Name) {
+		cfg.TriesNoVerify = ctx.Bool(TriesNoVerifyFlag.Name)
+	}
 	if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheTrieFlag.Name) {
 		cfg.TrieCleanCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheTrieFlag.Name) / 100
 	}