Merge pull request #1991 from ethereum/reorg-tests

Reorg tests and add finality test suite

setup.py

@@ -141,11 +141,6 @@ def ceillog2(x: uint64) -> int:
     return (x - 1).bit_length()
 '''
 PHASE0_SUNDRY_FUNCTIONS = '''
-# Monkey patch hash cache
-_hash = hash
-hash_cache: Dict[bytes, Bytes32] = {}
-
-
 def get_eth1_data(block: Eth1Block) -> Eth1Data:
     """
     A stub function return mocking Eth1Data.
@@ -156,12 +151,6 @@ def get_eth1_data(block: Eth1Block) -> Eth1Data:
         block_hash=hash_tree_root(block))
 
 
-def hash(x: bytes) -> Bytes32:  # type: ignore
-    if x not in hash_cache:
-        hash_cache[x] = Bytes32(_hash(x))
-    return hash_cache[x]
-
-
 def cache_this(key_fn, value_fn, lru_size):  # type: ignore
     cache_dict = LRU(size=lru_size)
 

tests/core/pyspec/eth2spec/test/conftest.py

@@ -59,8 +59,8 @@ def bls_default(request):
 def bls_type(request):
     bls_type = request.config.getoption("--bls-type")
     if bls_type == "py_ecc":
-        bls_utils.bls = bls_utils.py_ecc_bls
+        bls_utils.use_py_ecc()
     elif bls_type == "milagro":
-        bls_utils.bls = bls_utils.milagro_bls
+        bls_utils.use_milagro()
     else:
         raise Exception(f"unrecognized bls type: {bls_type}")

tests/core/pyspec/eth2spec/test/test_finality.py → tests/core/pyspec/eth2spec/test/phase0/finality/test_finality.py

@@ -1,4 +1,4 @@
-from eth2spec.test.context import PHASE0, spec_state_test, never_bls, with_all_phases, with_phases
+from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases, with_phases
 from eth2spec.test.helpers.state import next_epoch_via_block
 from eth2spec.test.helpers.attestations import next_epoch_with_attestations
 
@@ -30,7 +30,6 @@ def check_finality(spec,
 
 @with_phases([PHASE0])
 @spec_state_test
-@never_bls
 def test_finality_no_updates_at_genesis(spec, state):
     assert spec.get_current_epoch(state) == spec.GENESIS_EPOCH
 
@@ -54,7 +53,6 @@ def test_finality_no_updates_at_genesis(spec, state):
 
 @with_all_phases
 @spec_state_test
-@never_bls
 def test_finality_rule_4(spec, state):
     # get past first two epochs that finality does not run on
     next_epoch_via_block(spec, state)
@@ -80,7 +78,6 @@ def test_finality_rule_4(spec, state):
 
 @with_all_phases
 @spec_state_test
-@never_bls
 def test_finality_rule_1(spec, state):
     # get past first two epochs that finality does not run on
     next_epoch_via_block(spec, state)
@@ -108,7 +105,6 @@ def test_finality_rule_1(spec, state):
 
 @with_all_phases
 @spec_state_test
-@never_bls
 def test_finality_rule_2(spec, state):
     # get past first two epochs that finality does not run on
     next_epoch_via_block(spec, state)
@@ -138,7 +134,6 @@ def test_finality_rule_2(spec, state):
 
 @with_all_phases
 @spec_state_test
-@never_bls
 def test_finality_rule_3(spec, state):
     """
     Test scenario described here

tests/core/pyspec/eth2spec/utils/bls.py

@@ -14,6 +14,22 @@
 STUB_COORDINATES = _signature_to_G2(Z2_SIGNATURE)
 
 
+def use_milagro():
+    """
+    Shortcut to use Milagro as BLS library
+    """
+    global bls
+    bls = milagro_bls
+
+
+def use_py_ecc():
+    """
+    Shortcut to use Py-ecc as BLS library
+    """
+    global bls
+    bls = py_ecc_bls
+
+
 def only_with_bls(alt_return=None):
     """
     Decorator factory to make a function only run when BLS is active. Otherwise return the default.

tests/core/pyspec/eth2spec/utils/hash_function.py

@@ -1,17 +1,9 @@
 from hashlib import sha256
-from typing import Dict, Union
+from remerkleable.byte_arrays import Bytes32
+from typing import Union
 
 ZERO_BYTES32 = b'\x00' * 32
 
 
-def _hash(x: Union[bytes, bytearray, memoryview]) -> bytes:
-    return sha256(x).digest()
-
-
-hash_cache: Dict[bytes, bytes] = {}
-
-
-def hash(x: bytes) -> bytes:
-    if x in hash_cache:
-        return hash_cache[x]
-    return _hash(x)
+def hash(x: Union[bytes, bytearray, memoryview]) -> Bytes32:
+    return Bytes32(sha256(x).digest())

tests/formats/finality/README.md

@@ -0,0 +1,43 @@
+# Finality tests
+
+The aim of the tests for the finality rules.
+
+- `finality`: transitions triggered by one or more blocks.
+
+## Test case format
+
+### `meta.yaml`
+
+```yaml
+description: string    -- Optional. Description of test case, purely for debugging purposes.
+bls_setting: int       -- see general test-format spec.
+blocks_count: int      -- the number of blocks processed in this test.
+```
+
+### `pre.yaml`
+
+A YAML-encoded `BeaconState`, the state before running the block transitions.
+
+Also available as `pre.ssz`.
+
+
+### `blocks_<index>.yaml`
+
+A series of files, with `<index>` in range `[0, blocks_count)`. Blocks need to be processed in order,
+ following the main transition function (i.e. process slot and epoch transitions in between blocks as normal)
+
+Each file is a YAML-encoded `SignedBeaconBlock`.
+
+Each block is also available as `blocks_<index>.ssz`
+
+### `post.yaml`
+
+A YAML-encoded `BeaconState`, the state after applying the block transitions.
+
+Also available as `post.ssz`.
+
+
+## Condition
+
+The resulting state should match the expected `post` state, or if the `post` state is left blank,
+ the handler should reject the series of blocks as invalid.

tests/generators/bls/main.py

@@ -321,6 +321,7 @@ def cases_fn() -> Iterable[gen_typing.TestCase]:
 
 
 if __name__ == "__main__":
+    bls.use_py_ecc()  # Py-ecc is chosen instead of Milagro, since the code is better understood to be correct.
     gen_runner.run_generator("bls", [
         create_provider('sign', case01_sign),
         create_provider('verify', case02_verify),

tests/generators/epoch_processing/main.py

@@ -14,6 +14,7 @@
 from importlib import reload
 from eth2spec.config import config_util
 from eth2spec.test.context import PHASE0
+from eth2spec.utils import bls
 
 
 def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider:
@@ -22,6 +23,7 @@ def prepare_fn(configs_path: str) -> str:
         config_util.prepare_config(configs_path, config_name)
         reload(spec_phase0)
         reload(spec_phase1)
+        bls.use_milagro()
         return config_name
 
     def cases_fn() -> Iterable[gen_typing.TestCase]:

tests/generators/finality/README.md

@@ -0,0 +1,5 @@
+# Finality tests
+
+Finality tests cover regular state-transitions in a common block-list format to test finality rules.
+
+Information on the format of the tests can be found in the [finality test formats documentation](../../formats/finality/README.md).

tests/generators/finality/main.py

@@ -0,0 +1,39 @@
+from typing import Iterable
+from importlib import reload
+
+from gen_base import gen_runner, gen_typing
+from gen_from_tests.gen import generate_from_tests
+
+from eth2spec.test.context import PHASE0
+from eth2spec.test.phase0.finality import test_finality
+from eth2spec.config import config_util
+from eth2spec.phase0 import spec as spec_phase0
+from eth2spec.phase1 import spec as spec_phase1
+from eth2spec.utils import bls
+
+
+def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider:
+
+    def prepare_fn(configs_path: str) -> str:
+        config_util.prepare_config(configs_path, config_name)
+        reload(spec_phase0)
+        reload(spec_phase1)
+        bls.use_milagro()
+        return config_name
+
+    def cases_fn() -> Iterable[gen_typing.TestCase]:
+        return generate_from_tests(
+            runner_name='finality',
+            handler_name=handler_name,
+            src=tests_src,
+            fork_name=PHASE0,
+        )
+
+    return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
+
+
+if __name__ == "__main__":
+    gen_runner.run_generator("finality", [
+        create_provider('finality', test_finality, 'minimal'),
+        create_provider('finality', test_finality, 'mainnet'),
+    ])

tests/generators/finality/requirements.txt

@@ -0,0 +1,2 @@
+../../core/gen_helpers
+../../../

tests/generators/genesis/main.py

@@ -1,20 +1,22 @@
 from typing import Iterable
 
 from eth2spec.test.context import PHASE0
-from eth2spec.test.genesis import test_initialization, test_validity
+from eth2spec.test.phase0.genesis import test_initialization, test_validity
 
 from gen_base import gen_runner, gen_typing
 from gen_from_tests.gen import generate_from_tests
 from eth2spec.phase0 import spec as spec
 from importlib import reload
 from eth2spec.config import config_util
+from eth2spec.utils import bls
 
 
 def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider:
 
     def prepare_fn(configs_path: str) -> str:
         config_util.prepare_config(configs_path, config_name)
         reload(spec)
+        bls.use_milagro()
         return config_name
 
     def cases_fn() -> Iterable[gen_typing.TestCase]:

tests/generators/operations/main.py

@@ -16,6 +16,7 @@
 from eth2spec.phase0 import spec as spec_phase0
 from eth2spec.phase1 import spec as spec_phase1
 from eth2spec.test.context import PHASE0
+from eth2spec.utils import bls
 
 
 def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider:
@@ -24,6 +25,7 @@ def prepare_fn(configs_path: str) -> str:
         config_util.prepare_config(configs_path, config_name)
         reload(spec_phase0)
         reload(spec_phase1)
+        bls.use_milagro()
         return config_name
 
     def cases_fn() -> Iterable[gen_typing.TestCase]:

tests/generators/rewards/main.py

@@ -12,6 +12,7 @@
 from importlib import reload
 from eth2spec.config import config_util
 from eth2spec.test.context import PHASE0
+from eth2spec.utils import bls
 
 
 def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider:
@@ -20,6 +21,7 @@ def prepare_fn(configs_path: str) -> str:
         config_util.prepare_config(configs_path, config_name)
         reload(spec_phase0)
         reload(spec_phase1)
+        bls.use_milagro()
         return config_name
 
     def cases_fn() -> Iterable[gen_typing.TestCase]:

tests/generators/sanity/main.py

@@ -9,6 +9,7 @@
 from eth2spec.config import config_util
 from eth2spec.phase0 import spec as spec_phase0
 from eth2spec.phase1 import spec as spec_phase1
+from eth2spec.utils import bls
 
 
 def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider:
@@ -17,6 +18,7 @@ def prepare_fn(configs_path: str) -> str:
         config_util.prepare_config(configs_path, config_name)
         reload(spec_phase0)
         reload(spec_phase1)
+        bls.use_milagro()
         return config_name
 
     def cases_fn() -> Iterable[gen_typing.TestCase]: