geth --dev test fixture setup (ethereum#3191)

* WIP: Generate geth fixture for shanghai with clique consensus

* Initial squashed commits for getting ``geth --dev`` test fixture working:

- Quicker timeout for unmined wait-for-txn-receipt tests
- Update default fees test for geth --dev mode
- Get rid of all miner start() and stop() references
- Run integration tests with --dev flag
- Test refactoring for ``geth --dev`` test fixture setup
- Remove PoW related endpoint tests; TODO: remove / deprecate endpoints
- Support latest geth versions
- Update geth version in circleCI config + peripherally related changes
- Turn on PoS block identifier tests
- get geth --dev test suite working

* Updates towards ``geth --dev`` test fixture:

- Flaky for replace transaction
- Fix typing
- Loosen assertions on gas
- Nonce value fix
- Loosen asserts for default maxFeePerGas and maxPriorityFeePerGas tests

* ``geth --dev`` test fixture tweaks:

- Add @flaky to tests that expect mining at certain times
- This isn't ideal. Perhaps we can increase the ``dev.period`` (mining
  interval) to make these tests a bit more reliable and hopefully
  the other tests are unaffected.
- Update benchmark to run with ``geth --dev`` setup
- Put back old state of get_logs_without_logs for eth_tester

* More work towards ``geth --dev`` test fixture:

- Try with --dev.period=5
- Remove assert from test_eth_new_block_filter
- Remove sleep from test_web3_client_version
- Try without flaky
- Fixes test_eth_send_transaction_with_nonce without the need for retries.
- Retries were causing transactions to be sent with the same nonce,
  which in turn requires more gas to go through. Running the test once
  fixes this issue.
- Obtain the transaction count providing the 'pending' argument.
- Fix test_eth_send_transaction_no_max_fee

* Additional cleanup for ``geth --dev`` test fixture

* Address feedback for ``geth --dev`` test fixture

---------

Co-authored-by: Stuart Reed <[email protected]>

Author: fselmo

.circleci/config.yml

@@ -3,7 +3,7 @@ version: 2.1
 parameters:
   geth_version:
     # update default value when updating geth integration test fixture
-    default: "v1.11.6"
+    default: "v1.13.9"
     type: string
   pygeth_version:
     # update default value when updating geth integration test fixture
@@ -107,7 +107,6 @@ geth_steps: &geth_steps
           fi
           sudo ln -s /home/circleci/.py-geth/geth-<< pipeline.parameters.geth_version >>/bin/geth /usr/local/bin/geth
           geth version
-          geth makedag 0 $HOME/.ethash
     - run:
         name: run tox
         command: python -m tox -r
@@ -148,7 +147,6 @@ geth_custom_steps: &geth_custom_steps
           sudo apt-get update;
           sudo apt-get install -y build-essential;
           ./custom_geth version
-          ./custom_geth makedag 0 $HOME/.ethash
     - run:
         name: run tox
         command: python -m tox -r

docs/contributing.rst

@@ -373,13 +373,13 @@ Geth Fixtures
 
    .. code:: sh
 
-       $ python -m geth.install v1.11.6
+       $ python -m geth.install v1.13.9
 
 2. Specify the Geth binary and run the fixture creation script (from within the web3.py directory):
 
    .. code:: sh
 
-       $ GETH_BINARY=~/.py-geth/geth-v1.11.6/bin/geth python ./tests/integration/generate_fixtures/go_ethereum.py ./tests/integration/geth-1.11.6-fixture
+       $ GETH_BINARY=~/.py-geth/geth-v1.13.9/bin/geth python ./tests/integration/generate_fixtures/go_ethereum.py ./tests/integration/geth-1.13.9-fixture
 
 3. The output of this script is your fixture, a zip file, which is now stored in ``/tests/integration/``.
    Update the ``/tests/integration/go_ethereum/conftest.py`` and

newsfragments/3191.internal.rst

@@ -0,0 +1 @@
+Create test fixture for latest ``geth`` version. Run tests with ``geth`` in ``--dev`` mode.

setup.py

@@ -7,7 +7,7 @@
 extras_require = {
     "tester": [
         "eth-tester[py-evm]==v0.9.1-b.1",
-        "py-geth>=3.11.0",
+        "py-geth>=4.1.0",
     ],
     "linter": [
         "black>=22.1.0",

tests/core/mining-module/conftest.py

@@ -15,9 +15,6 @@
 from web3 import (
     constants,
 )
-from web3.exceptions import (
-    TransactionNotFound,
-)
 
 # use same coinbase value as in `web3.py/tests/integration/common.py`
 COINBASE = "0xdc544d1aa88ff8bbd2f2aec754b1f1e99e1812fd"
@@ -26,6 +23,7 @@
 KEYFILE_DATA = '{"address":"dc544d1aa88ff8bbd2f2aec754b1f1e99e1812fd","crypto":{"cipher":"aes-128-ctr","ciphertext":"52e06bc9397ea9fa2f0dae8de2b3e8116e92a2ecca9ad5ff0061d1c449704e98","cipherparams":{"iv":"aa5d0a5370ef65395c1a6607af857124"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"9fdf0764eb3645ffc184e166537f6fe70516bf0e34dc7311dea21f100f0c9263"},"mac":"4e0b51f42b865c15c485f4faefdd1f01a38637e5247f8c75ffe6a8c0eba856f6"},"id":"5a6124e0-10f1-4c1c-ae3e-d903eacb740a","version":3}'  # noqa: E501
 
 KEYFILE_PW = "web3py-test"
+KEYFILE_PW_TXT = "pw.txt"
 KEYFILE_FILENAME = "UTC--2017-08-24T19-42-47.517572178Z--dc544d1aa88ff8bbd2f2aec754b1f1e99e1812fd"  # noqa: E501
 
 RAW_TXN_ACCOUNT = "0x39EEed73fb1D3855E90Cbd42f348b3D7b340aAA6"
@@ -40,20 +38,22 @@
     "config": {
         "chainId": 131277322940537,  # the string 'web3py' as an integer
         "homesteadBlock": 0,
-        "byzantiumBlock": 0,
-        "constantinopleBlock": 0,
         "eip150Block": 0,
         "eip155Block": 0,
         "eip158Block": 0,
-        "istanbulBlock": 0,
+        "byzantiumBlock": 0,
+        "constantinopleBlock": 0,
         "petersburgBlock": 0,
+        "istanbulBlock": 0,
         "berlinBlock": 0,
         "londonBlock": 0,
         "arrowGlacierBlock": 0,
         "grayGlacierBlock": 0,
         "shanghaiTime": 0,
+        "terminalTotalDifficulty": 0,
+        "terminalTotalDifficultyPassed": True,
     },
-    "nonce": "0x0000000000000042",
+    "nonce": "0x0",
     "alloc": {
         COINBASE: {"balance": "1000000000000000000000000000"},
         UNLOCKABLE_ACCOUNT: {"balance": "1000000000000000000000000000"},
@@ -65,7 +65,7 @@
         "0000000000000000000000000000000000000005": {"balance": "1"},
         "0000000000000000000000000000000000000006": {"balance": "1"},
     },
-    "timestamp": "0x00",
+    "timestamp": "0x0",
     "parentHash": constants.HASH_ZERO,
     "extraData": "0x3535353535353535353535353535353535353535353535353535353535353535",
     "gasLimit": "0x3b9aca00",  # 1,000,000,000
@@ -172,6 +172,9 @@ def get_geth_process(
         geth_binary,
         "--datadir",
         datadir,
+        "--dev",
+        "--dev.period",
+        "1",
         "--ipcpath",
         ipc_path,
         "--nodiscover",
@@ -210,43 +213,22 @@ def mine_block(w3):
     origin_block_number = w3.eth.block_number
 
     start_time = time.time()
-    w3.geth.miner.start(1)
     while time.time() < start_time + 120:
         block_number = w3.eth.block_number
         if block_number > origin_block_number:
-            w3.geth.miner.stop()
             return block_number
         else:
             time.sleep(0.1)
     else:
         raise ValueError("No block mined during wait period")
 
 
-def mine_transaction_hash(w3, txn_hash):
-    start_time = time.time()
-    w3.geth.miner.start(1)
-    while time.time() < start_time + 120:
-        try:
-            receipt = w3.eth.get_transaction_receipt(txn_hash)
-        except TransactionNotFound:
-            continue
-        if receipt is not None:
-            w3.geth.miner.stop()
-            return receipt
-        else:
-            time.sleep(0.1)
-    else:
-        raise ValueError(
-            "Math contract deploy transaction not mined during wait period"
-        )
-
-
 def deploy_contract(w3, name, factory):
     name = name.upper()
     w3.geth.personal.unlock_account(w3.eth.coinbase, KEYFILE_PW)
     deploy_txn_hash = factory.constructor().transact({"from": w3.eth.coinbase})
     print(f"{name}_CONTRACT_DEPLOY_HASH: {deploy_txn_hash}")
-    deploy_receipt = mine_transaction_hash(w3, deploy_txn_hash)
+    deploy_receipt = w3.eth.wait_for_transaction_receipt(deploy_txn_hash)
     print(f"{name}_CONTRACT_DEPLOY_TRANSACTION_MINED")
     contract_address = deploy_receipt["contractAddress"]
     assert is_checksum_address(contract_address)