E3: Prune TotalDifficulty canonical markers (#11809)

Implements  ethereum/execution-apis#570

Author: Giulio2002

Diff for: cmd/rpcdaemon/graphql/eip-1767.graphqls.ref

@@ -154,9 +154,6 @@ type Block {
     mixHash: Bytes32!
     # Difficulty is a measure of the difficulty of mining this block.
     difficulty: BigInt!
-    # TotalDifficulty is the sum of all difficulty values up to and including
-    # this block.
-    totalDifficulty: BigInt!
     # OmmerCount is the number of ommers (AKA uncles) associated with this
     # block. If ommers are unavailable, this field will be null.
     ommerCount: Int

Diff for: cmd/rpcdaemon/graphql/geth-schema.graphqls.ref

@@ -190,9 +190,6 @@
         mixHash: Bytes32!
         # Difficulty is a measure of the difficulty of mining this block.
         difficulty: BigInt!
-        # TotalDifficulty is the sum of all difficulty values up to and including
-        # this block.
-        totalDifficulty: BigInt!
         # OmmerCount is the number of ommers (AKA uncles) associated with this
         # block. If ommers are unavailable, this field will be null.
         ommerCount: Int

Diff for: cmd/rpcdaemon/graphql/graph/generated.go

@@ -192,9 +192,6 @@ type Block {
   mixHash: Bytes32!
   # Difficulty is a measure of the difficulty of mining this block.
   difficulty: BigInt!
-  # TotalDifficulty is the sum of all difficulty values up to and including
-  # this block.
-  totalDifficulty: BigInt!
   # OmmerCount is the number of ommers (AKA uncles) associated with this
   # block. If ommers are unavailable, this field will be null.
   ommerCount: Int

Diff for: cmd/rpcdaemon/graphql/graph/model/models_gen.go

@@ -25,7 +25,6 @@
     stateRoot
     timestamp
     transactionsRoot
-    totalDifficulty
     transactions {
       createdContract {
         address

Diff for: cmd/rpcdaemon/graphql/graph/schema.graphqls

@@ -290,7 +290,6 @@
                       "    \"size\": \"0x3e1\",",
                       "    \"stateRoot\": \"0xb3f9408d80048b6f206951c4e387f8da37fb8510eccc18527865fa746c47bbc5\",",
                       "    \"timestamp\": \"0x56bff9bb\",",
-                      "    \"totalDifficulty\": \"0x6332227c16fd7c67\",",
                       "    \"transactions\": [",
                       "      \"0x730724cb08a6eb17bf6b3296359d261570d343ea7944a17a9d7287d77900db08\",",
                       "      \"0xef2ea39c20ba09553b2f3cf02380406ac766039ca56612937eed5e7f3503fb3a\",",
@@ -336,7 +335,7 @@
                   "raw": "{{HOST}}",
                   "host": ["{{HOST}}"]
                 },
-                "description": "Returns information about a block given the block's number.\r\n\r\n**Parameters**\r\n\r\nTAG - Integer block number or one of \"earliest\", \"latest\" or \"pending\"\r\n\r\nBoolean - If true it returns the full transaction objects, if false only the hashes of the transactions\r\n\r\n**Returns**\r\n\r\nObject - An object of type Block defined as:\r\n\r\nnumber: QUANTITY - The block number or null when pending\r\n\r\nhash: DATA, 32 Bytes - Hash of the block or null when pending\r\n\r\nparentHash: DATA, 32 Bytes - Hash of the parent block\r\n\r\nnonce: DATA, 8 bytes - Hash of the proof of work or null when pending\r\n\r\nsha3Uncles: DATA, 32 Bytes - SHA3 of the uncles data in the block\r\n\r\nlogsBloom: DATA, 256 Bytes - The bloom filter for the block's logs or null when pending\r\n\r\ntransactionsRoot: DATA, 32 Bytes - The root of the transaction trie of the block\r\n\r\nstateRoot: DATA, 32 Bytes - The root of the final state trie of the block\r\n\r\nreceiptsRoot: DATA, 32 Bytes - The root of the receipts trie of the block\r\n\r\nminer: DATA, 20 Bytes - The address of the beneficiary to whom the mining rewards were given\r\n\r\ndifficulty: QUANTITY - Integer of the difficulty for this block\r\n\r\ntotalDifficulty: QUANTITY - Integer of the total difficulty of the chain until this block\r\n\r\nextraData: DATA - The extra data field of this block\r\n\r\nsize: QUANTITY - Integer the size of this block in bytes\r\n\r\ngasLimit: QUANTITY - The maximum gas allowed in this block\r\n\r\ngasUsed: QUANTITY - The total used gas by all transactions in this block\r\n\r\ntimestamp: QUANTITY - The unix timestamp for when the block was collated\r\n\r\ntransactions: ARRAY - Array of transaction objects, or 32 Bytes transaction hashes depending on the last given parameter\r\n\r\nuncles: ARRAY - Array of uncle hashes\r\n\r\n"
+                "description": "Returns information about a block given the block's number.\r\n\r\n**Parameters**\r\n\r\nTAG - Integer block number or one of \"earliest\", \"latest\" or \"pending\"\r\n\r\nBoolean - If true it returns the full transaction objects, if false only the hashes of the transactions\r\n\r\n**Returns**\r\n\r\nObject - An object of type Block defined as:\r\n\r\nnumber: QUANTITY - The block number or null when pending\r\n\r\nhash: DATA, 32 Bytes - Hash of the block or null when pending\r\n\r\nparentHash: DATA, 32 Bytes - Hash of the parent block\r\n\r\nnonce: DATA, 8 bytes - Hash of the proof of work or null when pending\r\n\r\nsha3Uncles: DATA, 32 Bytes - SHA3 of the uncles data in the block\r\n\r\nlogsBloom: DATA, 256 Bytes - The bloom filter for the block's logs or null when pending\r\n\r\ntransactionsRoot: DATA, 32 Bytes - The root of the transaction trie of the block\r\n\r\nstateRoot: DATA, 32 Bytes - The root of the final state trie of the block\r\n\r\nreceiptsRoot: DATA, 32 Bytes - The root of the receipts trie of the block\r\n\r\nminer: DATA, 20 Bytes - The address of the beneficiary to whom the mining rewards were given\r\n\r\ndifficulty: QUANTITY - Integer of the difficulty for this block\r\n\r\nextraData: DATA - The extra data field of this block\r\n\r\nsize: QUANTITY - Integer the size of this block in bytes\r\n\r\ngasLimit: QUANTITY - The maximum gas allowed in this block\r\n\r\ngasUsed: QUANTITY - The total used gas by all transactions in this block\r\n\r\ntimestamp: QUANTITY - The unix timestamp for when the block was collated\r\n\r\ntransactions: ARRAY - Array of transaction objects, or 32 Bytes transaction hashes depending on the last given parameter\r\n\r\nuncles: ARRAY - Array of uncle hashes\r\n\r\n"
               },
               "response": []
             },
@@ -368,7 +367,6 @@
                       "    \"size\": \"0x3e1\",",
                       "    \"stateRoot\": \"0xb3f9408d80048b6f206951c4e387f8da37fb8510eccc18527865fa746c47bbc5\",",
                       "    \"timestamp\": \"0x56bff9bb\",",
-                      "    \"totalDifficulty\": \"0x6332227c16fd7c67\",",
                       "    \"transactions\": [",
                       "      \"0x730724cb08a6eb17bf6b3296359d261570d343ea7944a17a9d7287d77900db08\",",
                       "      \"0xef2ea39c20ba09553b2f3cf02380406ac766039ca56612937eed5e7f3503fb3a\",",
@@ -870,7 +868,6 @@
                       "    \"size\": \"0x21a\",",
                       "    \"stateRoot\": \"0x1e6e030581fd1873b4784280859cd3b3c04aa85520f08c304cf5ee63d3935add\",",
                       "    \"timestamp\": \"0x55ba4242\",",
-                      "    \"totalDifficulty\": \"0xffd003ffe\",",
                       "    \"transactionsRoot\": \"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",",
                       "    \"uncles\": []",
                       "  }",
@@ -949,7 +946,6 @@
                       "    \"size\": \"0x21a\",",
                       "    \"stateRoot\": \"0x1e6e030581fd1873b4784280859cd3b3c04aa85520f08c304cf5ee63d3935add\",",
                       "    \"timestamp\": \"0x55ba4242\",",
-                      "    \"totalDifficulty\": \"0xffd003ffe\",",
                       "    \"transactionsRoot\": \"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",",
                       "    \"uncles\": []",
                       "  }",

Diff for: cmd/rpcdaemon/graphql/graph/schema.resolvers.go

@@ -326,7 +326,6 @@ type OtsFullBlock struct {
 	StateRoot    string      `json:"stateRoot"`
 	Timestamp    string      `json:"timestamp"`
 
-	TotalDifficulty  hexutil.Big      `json:"totalDifficulty"`
 	TransactionCount uint64           `json:"transactionCount"`
 	Transactions     []OtsTransaction `json:"transactions"`
 	TxRoot           libcommon.Hash   `json:"transactionsRoot"`

Diff for: cmd/rpcdaemon/graphql/query_block.graphql

@@ -265,9 +265,6 @@ Object - An object of type Block defined as:
    * - ``QUANTITY``
      - ``difficulty``
      - Integer of the difficulty for this block
-   * - ``QUANTITY``
-     - ``totalDifficulty``
-     - Integer of the total difficulty of the chain until this block
    * - ``DATA``
      - ``extraData``
      - The extra data field of this block

Diff for: cmd/rpcdaemon/postman/RPC_Testing.json

@@ -49,7 +49,7 @@ var (
 	noMerge              = EnvBool("NO_MERGE", false)
 	discardHistory       = EnvBool("DISCARD_HISTORY", false)
 	discardCommitment    = EnvBool("DISCARD_COMMITMENT", false)
-	pruneTotalDifficulty = EnvBool("PRUNE_TOTAL_DIFFICULTY", false)
+	pruneTotalDifficulty = EnvBool("PRUNE_TOTAL_DIFFICULTY", true)
 
 	// force skipping of any non-Erigon2 .torrent files
 	DownloaderOnlyBlocks = EnvBool("DOWNLOADER_ONLY_BLOCKS", false)

Diff for: cmd/rpctest/rpctest/type.go

@@ -541,6 +541,7 @@ func pruneCanonicalMarkers(ctx context.Context, tx kv.RwTx, blockReader services
 		return err
 	}
 	defer c.Close()
+	var tdKey [40]byte
 	for k, v, err := c.First(); k != nil && err == nil; k, v, err = c.Next() {
 		blockNum := binary.BigEndian.Uint64(k)
 		if blockNum == 0 { // Do not prune genesis marker
@@ -552,6 +553,13 @@ func pruneCanonicalMarkers(ctx context.Context, tx kv.RwTx, blockReader services
 		if err := tx.Delete(kv.HeaderNumber, v); err != nil {
 			return err
 		}
+		if dbg.PruneTotalDifficulty() {
+			copy(tdKey[:], k)
+			copy(tdKey[8:], v)
+			if err := tx.Delete(kv.HeaderTD, tdKey[:]); err != nil {
+				return err
+			}
+		}
 		if err := c.DeleteCurrent(); err != nil {
 			return err
 		}

Diff for: docs/readthedocs/source/rpc/index.rst

@@ -374,30 +374,6 @@ func RPCMarshalBlockExDeprecated(block *types.Block, inclTx bool, fullTx bool, b
 	return fields, nil
 }
 
-/*
-
-// rpcMarshalHeader uses the generalized output filler, then adds the total difficulty field, which requires
-// a `PublicBlockchainAPI`.
-func (s *PublicBlockChainAPI) rpcMarshalHeader(ctx context.Context, header *types.Header) map[string]interface{} {
-	fields := RPCMarshalHeader(header)
-	fields["totalDifficulty"] = (*hexutil.Big)(s.b.GetTd(ctx, header.Hash()))
-	return fields
-}
-
-// rpcMarshalBlock uses the generalized output filler, then adds the total difficulty field, which requires
-// a `PublicBlockchainAPI`.
-func (s *PublicBlockChainAPI) rpcMarshalBlock(ctx context.Context, b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) {
-	fields, err := RPCMarshalBlock(b, inclTx, fullTx)
-	if err != nil {
-		return nil, err
-	}
-	if inclTx {
-		fields["totalDifficulty"] = (*hexutil.Big)(s.b.GetTd(ctx, b.Hash()))
-	}
-	return fields, err
-}
-*/
-
 // RPCTransaction represents a transaction that will serialize to the RPC representation of a transaction
 type RPCTransaction struct {
 	BlockHash        *libcommon.Hash    `json:"blockHash"`

Diff for: erigon-lib/common/dbg/experiments.go

@@ -197,13 +197,6 @@ func buildBlockResponse(ctx context.Context, br services.FullBlockReader, db kv.
 	}
 
 	additionalFields := make(map[string]interface{})
-	td, err := rawdb.ReadTd(db, header.Hash(), header.Number.Uint64())
-	if err != nil {
-		return nil, err
-	}
-	if td != nil {
-		additionalFields["totalDifficulty"] = (*hexutil.Big)(td)
-	}
 
 	response, err := ethapi.RPCMarshalBlockEx(block, true, fullTx, nil, common.Hash{}, additionalFields)
 

Diff for: eth/stagedsync/stage_snapshots.go

@@ -228,13 +228,6 @@ func (api *APIImpl) GetBlockByNumber(ctx context.Context, number rpc.BlockNumber
 		return nil, nil
 	}
 	additionalFields := make(map[string]interface{})
-	td, err := rawdb.ReadTd(tx, b.Hash(), b.NumberU64())
-	if err != nil {
-		return nil, err
-	}
-	if td != nil {
-		additionalFields["totalDifficulty"] = (*hexutil.Big)(td)
-	}
 
 	chainConfig, err := api.chainConfig(ctx, tx)
 	if err != nil {
@@ -289,12 +282,6 @@ func (api *APIImpl) GetBlockByHash(ctx context.Context, numberOrHash rpc.BlockNu
 	}
 	number := block.NumberU64()
 
-	td, err := rawdb.ReadTd(tx, hash, number)
-	if err != nil {
-		return nil, err
-	}
-	additionalFields["totalDifficulty"] = (*hexutil.Big)(td)
-
 	chainConfig, err := api.chainConfig(ctx, tx)
 	if err != nil {
 		return nil, err

Diff for: turbo/adapter/ethapi/api.go

@@ -24,7 +24,6 @@ import (
 	"github.com/erigontech/erigon-lib/common"
 	"github.com/erigontech/erigon-lib/log/v3"
 
-	"github.com/erigontech/erigon/core/rawdb"
 	"github.com/erigontech/erigon/core/types"
 	"github.com/erigontech/erigon/rpc"
 	"github.com/erigontech/erigon/turbo/adapter/ethapi"
@@ -51,11 +50,6 @@ func (api *APIImpl) GetUncleByBlockNumberAndIndex(ctx context.Context, number rp
 		return nil, nil // not error, see https://github.com/erigontech/erigon/issues/1645
 	}
 	additionalFields := make(map[string]interface{})
-	td, err := rawdb.ReadTd(tx, block.Hash(), blockNum)
-	if err != nil {
-		return nil, err
-	}
-	additionalFields["totalDifficulty"] = (*hexutil.Big)(td)
 
 	uncles := block.Uncles()
 	if index >= hexutil.Uint(len(uncles)) {
@@ -81,13 +75,7 @@ func (api *APIImpl) GetUncleByBlockHashAndIndex(ctx context.Context, hash common
 	if block == nil {
 		return nil, nil // not error, see https://github.com/erigontech/erigon/issues/1645
 	}
-	number := block.NumberU64()
 	additionalFields := make(map[string]interface{})
-	td, err := rawdb.ReadTd(tx, hash, number)
-	if err != nil {
-		return nil, err
-	}
-	additionalFields["totalDifficulty"] = (*hexutil.Big)(td)
 
 	uncles := block.Uncles()
 	if index >= hexutil.Uint(len(uncles)) {