From c99e48c60cc06a93e86cc61214012741c9b58cf5 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 6 Dec 2021 10:07:54 +1100 Subject: [PATCH 001/106] Add first efforts --- sync/optimistic.md | 105 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 sync/optimistic.md diff --git a/sync/optimistic.md b/sync/optimistic.md new file mode 100644 index 0000000000..b3220b96f0 --- /dev/null +++ b/sync/optimistic.md @@ -0,0 +1,105 @@ +# Optimistic Sync + +## Introduction + +In order to provide a syncing execution engine with a (partially-verified) view +of the head of the chain, it may be desirable for a consensus engine to import +beacon blocks without verifying the execution payloads. This partial sync is +called an *optimistic sync*. + +## Mechanisms + +To perform an optimistic sync: + +- The `execute_payload` function MUST return `True` if the execution + engine returns SYNCING or VALID. An INVALID response MUST return `False`. +- The `validate_merge_block` function MUST NOT raise an assertion if both the + `pow_block` and `pow_parent` are unknown to the execution engine. + +In addition to these changes to validation, the consensus engine MUST be able +to ascertain, after import, which blocks returned SYNCING and which returned +VALID. This document will assume consensus engines store the following sets: + +- `valid_roots: Set[Root]`: `hash_tree_root(block)` where + `block.body.execution_payload` is known to be VALID. +- `optimistic_roots: Set[Root]`: `hash_tree_root(block)` where + `block.body.execution_payload` is known to be SYNCING. + +Notably, `optimistic_roots` only includes blocks which have execution enabled +whilst `valid_roots` contains blocks *with or without* execution enabled (i.e., +all blocks). + +A consensus engine MUST be able to retrospectively (i.e., after import) modify +the status of SYNCING blocks to be either VALID or INVALID based upon responses +from an execution engine. I.e., perform the following transitions: + +- SYNCING -> VALID +- SYNCING -> INVALID + +When a block transitions from SYNCING -> VALID, all *ancestors* of the block MUST +also transition from SYNCING -> VALID. + +When a block transitions from SYNCING -> INVALID, all *descendants* of the +block MUST also transition from SYNCING -> INVALID. + +## Fork Choice + +Consensus engines MUST support removing blocks that transition from SYNCING to +INVALID. Specifically, an INVALID block MUST NOT be included in the canonical +chain and the weights from INVALID blocks MUST NOT be applied to any VALID or +SYNCING ancestors. + +## Validator assignments + +An entirely optimistically synced node is *not* a full node. It is unable to +produce blocks, since an execution engine cannot produce a payload upon an +unknown parent. It cannot faithfully attest to the head block of the chain, +since it has not fully verified that block. + +Let `head_block: BeaconBlock` be the result of calling of the fork choice +algorithm at the time of block production. Let `justified_block: BeaconBlock` +be the latest current justified ancestor ancestor of the `head_block`. + +```python +def is_optimistic(block: BeaconBlock) -> bool: + hash_tree_root(block) in optimistic_roots +``` + +```python +def latest_valid_ancestor(block: BeaconBlock) -> Optional[BeaconBlock]: + while block.parent_root != Root(): + if not is_optimistic(block): + return block + block = get_block(block.parent_root) + + return None +``` + +Let a node which returns `is_optimistic(head) == True` be an *optimistic +node*. Let a validator on an optimistic node be an *optimistic validator*. + +### Block production + +A optimistic validator MUST NOT produce a block (i.e., sign across the +`DOMAIN_BEACON_PROPOSER` domain), unless one of the follow exceptions are met: + +#### Exception 1. + +If the justified block is fully verified (i.e., `not +is_optimistic(justified_block)`, the validator MUST produce a block upon +`latest_valid_ancestor(head)`. + +### Attesting + +An optimistic validator MUST NOT participate in attestation (i.e., sign across the +`DOMAIN_BEACON_ATTESTER`, `DOMAIN_SELECTION_PROOF` or +`DOMAIN_AGGREGATE_AND_PROOF` domains), unless one of the follow exceptions are +met: + +#### Exception + +#### Exception 1. + +If a validator *does not* have an optimistic head (i.e., `not +is_optimistic(head_block)`), the node is *fully synced*. +The validator may produce an attestation. From 38fffd3e2f5d7f32d85e0b317fc5a0202534b2e9 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 13 Dec 2021 17:09:09 +1100 Subject: [PATCH 002/106] Tidy, finish duties --- sync/optimistic.md | 105 ++++++++++++++++++++++++++++++++------------- 1 file changed, 74 insertions(+), 31 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index b3220b96f0..5837a1349f 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -2,52 +2,71 @@ ## Introduction -In order to provide a syncing execution engine with a (partially-verified) view -of the head of the chain, it may be desirable for a consensus engine to import -beacon blocks without verifying the execution payloads. This partial sync is -called an *optimistic sync*. +In order to provide a syncing execution engine with a partial view of the head +of the chain, it may be desirable for a consensus engine to import beacon +blocks without verifying the execution payloads. This partial sync is called an +*optimistic sync*. ## Mechanisms To perform an optimistic sync: - The `execute_payload` function MUST return `True` if the execution - engine returns SYNCING or VALID. An INVALID response MUST return `False`. + engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. - The `validate_merge_block` function MUST NOT raise an assertion if both the `pow_block` and `pow_parent` are unknown to the execution engine. In addition to these changes to validation, the consensus engine MUST be able -to ascertain, after import, which blocks returned SYNCING and which returned -VALID. This document will assume consensus engines store the following sets: +to ascertain, after import, which blocks returned `SYNCING` and which returned +`VALID`. This document will assume that consensus engines store the following +sets: - `valid_roots: Set[Root]`: `hash_tree_root(block)` where - `block.body.execution_payload` is known to be VALID. + `block.body.execution_payload` is known to be `VALID`. - `optimistic_roots: Set[Root]`: `hash_tree_root(block)` where - `block.body.execution_payload` is known to be SYNCING. + `block.body.execution_payload` is known to be `SYNCING`. -Notably, `optimistic_roots` only includes blocks which have execution enabled -whilst `valid_roots` contains blocks *with or without* execution enabled (i.e., -all blocks). +Notably, `optimistic_roots` only includes blocks which have execution enabled. +On the other hand, `valid_roots` contains blocks *with or without* execution +enabled (i.e., all blocks). A consensus engine MUST be able to retrospectively (i.e., after import) modify -the status of SYNCING blocks to be either VALID or INVALID based upon responses +the status of `SYNCING` blocks to be either `VALID` or `INVALID` based upon responses from an execution engine. I.e., perform the following transitions: -- SYNCING -> VALID -- SYNCING -> INVALID +- `SYNCING` -> `VALID` +- `SYNCING` -> `INVALID` -When a block transitions from SYNCING -> VALID, all *ancestors* of the block MUST -also transition from SYNCING -> VALID. +When a block transitions from `SYNCING` -> `VALID`, all *ancestors* of the block MUST +also transition from `SYNCING` -> `VALID`. -When a block transitions from SYNCING -> INVALID, all *descendants* of the -block MUST also transition from SYNCING -> INVALID. +When a block transitions from `SYNCING` -> `INVALID`, all *descendants* of the +block MUST also transition from `SYNCING` -> `INVALID`. + +### Execution Engine Errors + +A consensus engine MUST NOT interpret an error or failure to respond to a +message as a `SYNCING`, `VALID` or `INVALID` response. A consensus engine MAY +queue such a message for later processing. + +## Merge Transition + +To protect against attacks during the transition from empty `ExecutionPayload` +values to those which include the terminal PoW block, a consensus engine MUST +NOT perform an optimistic sync unless the `finalized_checkpoint.root` of the head +block references a block for which +`is_execution_enabled(head_state, head_block.body) == True`. + +TODO: this restriction is very onerous, however it is the best known remedy for +the attack described in https://hackmd.io/S5ZEVhsNTqqfJirTAkBPlg I hope we can +do better. ## Fork Choice -Consensus engines MUST support removing blocks that transition from SYNCING to -INVALID. Specifically, an INVALID block MUST NOT be included in the canonical -chain and the weights from INVALID blocks MUST NOT be applied to any VALID or -SYNCING ancestors. +Consensus engines MUST support removing from fork choice blocks that transition +from `SYNCING` to `INVALID`. Specifically, a block deemed `INVALID` at any +point MUST NOT be included in the canonical chain and the weights from those +`INVALID` blocks MUST NOT be applied to any `VALID` or `SYNCING` ancestors. ## Validator assignments @@ -56,6 +75,8 @@ produce blocks, since an execution engine cannot produce a payload upon an unknown parent. It cannot faithfully attest to the head block of the chain, since it has not fully verified that block. +### Helpers + Let `head_block: BeaconBlock` be the result of calling of the fork choice algorithm at the time of block production. Let `justified_block: BeaconBlock` be the latest current justified ancestor ancestor of the `head_block`. @@ -75,8 +96,18 @@ def latest_valid_ancestor(block: BeaconBlock) -> Optional[BeaconBlock]: return None ``` -Let a node which returns `is_optimistic(head) == True` be an *optimistic -node*. Let a validator on an optimistic node be an *optimistic validator*. +```python +def recent_valid_ancestor(block: BeaconBlock) -> Optional[BeaconBlock]: + ancestor = latest_valid_ancestor(block) + + if ancestor is None or ancestor.slot + SLOTS_PER_EPOCH >= block.slot: + return None + + return ancestor +``` + +Let only a node which returns `is_optimistic(head) == True` be an *optimistic +node*. Let only a validator on an optimistic node be an *optimistic validator*. ### Block production @@ -86,9 +117,11 @@ A optimistic validator MUST NOT produce a block (i.e., sign across the #### Exception 1. If the justified block is fully verified (i.e., `not -is_optimistic(justified_block)`, the validator MUST produce a block upon +is_optimistic(justified_block)`, the validator MAY produce a block upon `latest_valid_ancestor(head)`. +If the latest valid ancestor is `None`, the validator MUST NOT produce a block. + ### Attesting An optimistic validator MUST NOT participate in attestation (i.e., sign across the @@ -96,10 +129,20 @@ An optimistic validator MUST NOT participate in attestation (i.e., sign across t `DOMAIN_AGGREGATE_AND_PROOF` domains), unless one of the follow exceptions are met: -#### Exception - #### Exception 1. -If a validator *does not* have an optimistic head (i.e., `not -is_optimistic(head_block)`), the node is *fully synced*. -The validator may produce an attestation. +If the justified block is fully verified (i.e., `not +is_optimistic(justified_block)`, the validator MAY sign across the following +domains: + +- `DOMAIN_BEACON_ATTESTER`: where `attestation.data.beacon_block_root == hash_tree_root(recent_valid_ancestor(head))`. +- `DOMAIN_AGGREGATE_AND_PROOF` and `DOMAIN_SELECTION_PROOF`: where `aggregate.message.aggregate.data.beacon_block_root == hash_tree_root(recent_valid_ancestor(head))` + +If the recent valid ancestor is `None`, the validator MUST NOT participate in +attestation. + +### Participating in Sync Committees + +An optimistic validator MUST NOT participate in sync committees (i.e., sign across the +`DOMAIN_SYNC_COMMITTEE`, `DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF` or +`DOMAIN_CONTRIBUTION_AND_PROOF` domains). From 891882307120eff346a8768aab3f3ce6d2497cd3 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 13 Dec 2021 19:02:45 +1100 Subject: [PATCH 003/106] Start adding p2p components --- sync/optimistic.md | 53 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 7 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 5837a1349f..0089abbd33 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -68,13 +68,6 @@ from `SYNCING` to `INVALID`. Specifically, a block deemed `INVALID` at any point MUST NOT be included in the canonical chain and the weights from those `INVALID` blocks MUST NOT be applied to any `VALID` or `SYNCING` ancestors. -## Validator assignments - -An entirely optimistically synced node is *not* a full node. It is unable to -produce blocks, since an execution engine cannot produce a payload upon an -unknown parent. It cannot faithfully attest to the head block of the chain, -since it has not fully verified that block. - ### Helpers Let `head_block: BeaconBlock` be the result of calling of the fork choice @@ -109,6 +102,17 @@ def recent_valid_ancestor(block: BeaconBlock) -> Optional[BeaconBlock]: Let only a node which returns `is_optimistic(head) == True` be an *optimistic node*. Let only a validator on an optimistic node be an *optimistic validator*. +When this specification only defines behaviour for an optimistic +node/validator, but *not* for the non-optimistic case, assume default +behaviours without regard for optimistic sync. + +## Validator assignments + +An entirely optimistically synced node is *not* a full node. It is unable to +produce blocks, since an execution engine cannot produce a payload upon an +unknown parent. It cannot faithfully attest to the head block of the chain, +since it has not fully verified that block. + ### Block production A optimistic validator MUST NOT produce a block (i.e., sign across the @@ -146,3 +150,38 @@ attestation. An optimistic validator MUST NOT participate in sync committees (i.e., sign across the `DOMAIN_SYNC_COMMITTEE`, `DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF` or `DOMAIN_CONTRIBUTION_AND_PROOF` domains). + +## P2P Networking + +### Topics and Messages on Gossipsub + +#### `beacon_block` + +An optimistic validator MUST subscribe to the `beacon_block` topic. Propagation +validation conditions are modified as such: + +Do not apply the existing condition: + +- [REJECT] The block's parent (defined by block.parent_root) passes validation. + +Instead, apply the new condition: + +- [REJECT] The block's parent (defined by block.parent_root) passes validation, + *and* `block.parent root not in optimistic_roots`. + +#### `beacon_aggregate_and_proof` + +An optimistic validator MUST NOT subscribe to the `beacon_aggregate_and_proof` +topic. + +#### `voluntary_exit` + +An optimistic validator MUST NOT subscribe to the `voluntary_exit` topic. + +#### `proposer_slashing` + +An optimistic validator MUST NOT subscribe to the `proposer_slashing` topic. + +#### `attester_slashing` + +An optimistic validator MUST NOT subscribe to the `attester_slashing` topic. From 7f5b7d1535fa36708472f8a2811c3a760a65dc06 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 13 Dec 2021 19:03:35 +1100 Subject: [PATCH 004/106] Remove attesting during opt sync --- sync/optimistic.md | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 0089abbd33..0b8036b210 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -130,20 +130,7 @@ If the latest valid ancestor is `None`, the validator MUST NOT produce a block. An optimistic validator MUST NOT participate in attestation (i.e., sign across the `DOMAIN_BEACON_ATTESTER`, `DOMAIN_SELECTION_PROOF` or -`DOMAIN_AGGREGATE_AND_PROOF` domains), unless one of the follow exceptions are -met: - -#### Exception 1. - -If the justified block is fully verified (i.e., `not -is_optimistic(justified_block)`, the validator MAY sign across the following -domains: - -- `DOMAIN_BEACON_ATTESTER`: where `attestation.data.beacon_block_root == hash_tree_root(recent_valid_ancestor(head))`. -- `DOMAIN_AGGREGATE_AND_PROOF` and `DOMAIN_SELECTION_PROOF`: where `aggregate.message.aggregate.data.beacon_block_root == hash_tree_root(recent_valid_ancestor(head))` - -If the recent valid ancestor is `None`, the validator MUST NOT participate in -attestation. +`DOMAIN_AGGREGATE_AND_PROOF` domains). ### Participating in Sync Committees From 1a89d167da53de19f1f02fd95c5b49f143157c50 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 13 Dec 2021 19:15:53 +1100 Subject: [PATCH 005/106] Remove recent valid ancestor --- sync/optimistic.md | 37 ++++++++++++------------------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 0b8036b210..a59c0ef876 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -76,27 +76,15 @@ be the latest current justified ancestor ancestor of the `head_block`. ```python def is_optimistic(block: BeaconBlock) -> bool: - hash_tree_root(block) in optimistic_roots + hash_tree_root(block) in optimistic_roots ``` ```python -def latest_valid_ancestor(block: BeaconBlock) -> Optional[BeaconBlock]: - while block.parent_root != Root(): - if not is_optimistic(block): - return block - block = get_block(block.parent_root) - - return None -``` - -```python -def recent_valid_ancestor(block: BeaconBlock) -> Optional[BeaconBlock]: - ancestor = latest_valid_ancestor(block) - - if ancestor is None or ancestor.slot + SLOTS_PER_EPOCH >= block.slot: - return None - - return ancestor +def latest_valid_ancestor(block: BeaconBlock) -> BeaconBlock: + while block.parent_root != Root(): + if not is_optimistic(block) or block.parent_root == Root(): + return block + block = get_block(block.parent_root) ``` Let only a node which returns `is_optimistic(head) == True` be an *optimistic @@ -113,19 +101,18 @@ produce blocks, since an execution engine cannot produce a payload upon an unknown parent. It cannot faithfully attest to the head block of the chain, since it has not fully verified that block. -### Block production +### Block Production A optimistic validator MUST NOT produce a block (i.e., sign across the -`DOMAIN_BEACON_PROPOSER` domain), unless one of the follow exceptions are met: +`DOMAIN_BEACON_PROPOSER` domain), unless one of the following exceptions are +met: -#### Exception 1. +#### Block Production Exception 1. If the justified block is fully verified (i.e., `not is_optimistic(justified_block)`, the validator MAY produce a block upon `latest_valid_ancestor(head)`. -If the latest valid ancestor is `None`, the validator MUST NOT produce a block. - ### Attesting An optimistic validator MUST NOT participate in attestation (i.e., sign across the @@ -140,11 +127,11 @@ An optimistic validator MUST NOT participate in sync committees (i.e., sign acro ## P2P Networking -### Topics and Messages on Gossipsub +### The Gossip Domain (gossipsub) #### `beacon_block` -An optimistic validator MUST subscribe to the `beacon_block` topic. Propagation +An optimistic validator MAY subscribe to the `beacon_block` topic. Propagation validation conditions are modified as such: Do not apply the existing condition: From 5a5f980fca98c54f3b83b84130d9c9644b7c7b41 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 13 Dec 2021 19:25:33 +1100 Subject: [PATCH 006/106] Add RPC responses --- sync/optimistic.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index a59c0ef876..149fdd3910 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -159,3 +159,45 @@ An optimistic validator MUST NOT subscribe to the `proposer_slashing` topic. #### `attester_slashing` An optimistic validator MUST NOT subscribe to the `attester_slashing` topic. + +#### `beacon_attestation_{subnet_id}` + +An optimistic validator MUST NOT subscribe to the +`beacon_attestation_{subnet_id}` (attestation subnets) topics. + +#### `sync_committee_contribution_and_proof` + +An optimistic validator MUST NOT subscribe to the +`sync_committee_contribution_and_proof` topic. + +#### `sync_committee_{subnet_id}` + +An optimistic validator MUST NOT subscribe to the `sync_committee_{subnet_id}` +(sync committee subnets) topics. + +### The Req/Resp Domain + +#### BeaconBlocksByRange (v1, v2) + +Consensus engines MUST NOT include any block in a response where +`is_optimistic(block) == False`. + +#### BeaconBlocksByRoot (v1, v2) + +Consensus engines MUST NOT include any block in a response where +`is_optimistic(block) == False`. + +#### Status + +An optimistic node MUST use the `latest_valid_ancestor(head)` block to form +responses, rather than the head block. Specifically, an optimistic node must +form a `Status` message as so: + +The fields are, as seen by the client at the time of sending the message: + +- `fork_digest`: As previously defined. +- `finalized_root`: `state.finalized_checkpoint.root` for the state corresponding to the latest valid ancestor block + (Note this defaults to `Root(b'\x00' * 32)` for the genesis finalized checkpoint). +- `finalized_epoch`: `state.finalized_checkpoint.epoch` for the state corresponding to the latest valid ancestor block. +- `head_root`: The `hash_tree_root` root of the current latest valid ancestor block (`BeaconBlock`). +- `head_slot`: The slot of the block corresponding to `latest_valid_ancestor(head)`. From 2c62ed3b77bf08e71f8345d35657c99a114e8c88 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 14:53:04 +1100 Subject: [PATCH 007/106] Flip bool --- sync/optimistic.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 149fdd3910..be5626bf19 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -185,7 +185,7 @@ Consensus engines MUST NOT include any block in a response where #### BeaconBlocksByRoot (v1, v2) Consensus engines MUST NOT include any block in a response where -`is_optimistic(block) == False`. +`is_optimistic(block) == True`. #### Status From 497b5f8cbb1552f694f1ccf133e68beb87837b42 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 15:44:15 +1100 Subject: [PATCH 008/106] Add EE assumptions --- sync/optimistic.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index be5626bf19..63c0600094 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -49,6 +49,15 @@ A consensus engine MUST NOT interpret an error or failure to respond to a message as a `SYNCING`, `VALID` or `INVALID` response. A consensus engine MAY queue such a message for later processing. +### Assumptions about Execution Engine Behaviour + +This specification assumes execution engines will only return `SYNCING` when +there is insufficient information available to make a `VALID` or `INVALID` +determination on the given `ExecutionPayload` (e.g., the parent payload is +unknown). Specifically, `SYNCING` responses should be fork-specific; the search +for a block on one chain MUST NOT trigger a `SYNCING` response for another +chain. + ## Merge Transition To protect against attacks during the transition from empty `ExecutionPayload` From 0ad6025b488ff690e87c270a061cd3d736de76e6 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 15:51:02 +1100 Subject: [PATCH 009/106] Remove valid roots set --- sync/optimistic.md | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 63c0600094..7092a69ee2 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -13,23 +13,16 @@ To perform an optimistic sync: - The `execute_payload` function MUST return `True` if the execution engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. -- The `validate_merge_block` function MUST NOT raise an assertion if both the - `pow_block` and `pow_parent` are unknown to the execution engine. +- The `validate_merge_block` function MUST NOT raise an assertion if both the `pow_block` and `pow_parent` are unknown to the execution engine. In addition to these changes to validation, the consensus engine MUST be able to ascertain, after import, which blocks returned `SYNCING` and which returned `VALID`. This document will assume that consensus engines store the following -sets: +set: -- `valid_roots: Set[Root]`: `hash_tree_root(block)` where - `block.body.execution_payload` is known to be `VALID`. - `optimistic_roots: Set[Root]`: `hash_tree_root(block)` where `block.body.execution_payload` is known to be `SYNCING`. -Notably, `optimistic_roots` only includes blocks which have execution enabled. -On the other hand, `valid_roots` contains blocks *with or without* execution -enabled (i.e., all blocks). - A consensus engine MUST be able to retrospectively (i.e., after import) modify the status of `SYNCING` blocks to be either `VALID` or `INVALID` based upon responses from an execution engine. I.e., perform the following transitions: From 3b67c334e4903092bdaa68bee62479f5eb000dd1 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 15:51:15 +1100 Subject: [PATCH 010/106] Add note about removal from optimistic_roots --- sync/optimistic.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 7092a69ee2..3abc5c1c65 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -36,8 +36,10 @@ also transition from `SYNCING` -> `VALID`. When a block transitions from `SYNCING` -> `INVALID`, all *descendants* of the block MUST also transition from `SYNCING` -> `INVALID`. -### Execution Engine Errors +When a node transitions from the `SYNCING` state is is removed from the set of +`optimistic_roots`. +### Execution Engine Errors A consensus engine MUST NOT interpret an error or failure to respond to a message as a `SYNCING`, `VALID` or `INVALID` response. A consensus engine MAY queue such a message for later processing. From fb520a83563570f2dddfb2cf0f9d0a1a00d424aa Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 15:51:26 +1100 Subject: [PATCH 011/106] Condense gossip topics --- sync/optimistic.md | 39 +++++++++++---------------------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 3abc5c1c65..70319503d0 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -147,37 +147,20 @@ Instead, apply the new condition: - [REJECT] The block's parent (defined by block.parent_root) passes validation, *and* `block.parent root not in optimistic_roots`. -#### `beacon_aggregate_and_proof` +#### Other Topics -An optimistic validator MUST NOT subscribe to the `beacon_aggregate_and_proof` -topic. +An optimistic node MUST NOT subscribe to the following topics: -#### `voluntary_exit` +- `beacon_aggregate_and_proof` +- `voluntary_exit` +- `proposer_slashing` +- `attester_slashing` +- `beacon_attestation_{subnet_id}` +- `sync_committee_contribution_and_proof` +- `sync_committee_{subnet_id}` -An optimistic validator MUST NOT subscribe to the `voluntary_exit` topic. - -#### `proposer_slashing` - -An optimistic validator MUST NOT subscribe to the `proposer_slashing` topic. - -#### `attester_slashing` - -An optimistic validator MUST NOT subscribe to the `attester_slashing` topic. - -#### `beacon_attestation_{subnet_id}` - -An optimistic validator MUST NOT subscribe to the -`beacon_attestation_{subnet_id}` (attestation subnets) topics. - -#### `sync_committee_contribution_and_proof` - -An optimistic validator MUST NOT subscribe to the -`sync_committee_contribution_and_proof` topic. - -#### `sync_committee_{subnet_id}` - -An optimistic validator MUST NOT subscribe to the `sync_committee_{subnet_id}` -(sync committee subnets) topics. +Once the node ceases to be optimistic, it MAY re-subscribe to the aformentioned +topics. ### The Req/Resp Domain From d1851dce21f7fb89eb65f248eca477d44f41598e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 15:53:03 +1100 Subject: [PATCH 012/106] Tidy tab formatting --- sync/optimistic.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 70319503d0..edb54ab276 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -86,8 +86,8 @@ def is_optimistic(block: BeaconBlock) -> bool: ```python def latest_valid_ancestor(block: BeaconBlock) -> BeaconBlock: while block.parent_root != Root(): - if not is_optimistic(block) or block.parent_root == Root(): - return block + if not is_optimistic(block) or block.parent_root == Root(): + return block block = get_block(block.parent_root) ``` From 4c4ffe71b4c20634ddeb238fe50e304fb1b239f1 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 15:54:33 +1100 Subject: [PATCH 013/106] Fix latest_valid_ancestor --- sync/optimistic.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index edb54ab276..ccae7d0937 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -85,7 +85,7 @@ def is_optimistic(block: BeaconBlock) -> bool: ```python def latest_valid_ancestor(block: BeaconBlock) -> BeaconBlock: - while block.parent_root != Root(): + while True: if not is_optimistic(block) or block.parent_root == Root(): return block block = get_block(block.parent_root) From 3f6e5b9c3ad5c453bcb7ef19f342801751f92c58 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 16:10:19 +1100 Subject: [PATCH 014/106] Add checkpoint sync --- sync/optimistic.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index ccae7d0937..07293bde0d 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -72,6 +72,11 @@ from `SYNCING` to `INVALID`. Specifically, a block deemed `INVALID` at any point MUST NOT be included in the canonical chain and the weights from those `INVALID` blocks MUST NOT be applied to any `VALID` or `SYNCING` ancestors. +## Checkpoint Sync (Weak Subjectivity Sync) + +A consensus engine MAY assume that the `ExecutionPayload` of a block used for +checkpoint sync is `VALID`. + ### Helpers Let `head_block: BeaconBlock` be the result of calling of the fork choice From ffc2c405c41396528eac9b8fa6ed528773655ff6 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 16:11:40 +1100 Subject: [PATCH 015/106] Add section about API --- sync/optimistic.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index 07293bde0d..c67855710e 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -193,3 +193,20 @@ The fields are, as seen by the client at the time of sending the message: - `finalized_epoch`: `state.finalized_checkpoint.epoch` for the state corresponding to the latest valid ancestor block. - `head_root`: The `hash_tree_root` root of the current latest valid ancestor block (`BeaconBlock`). - `head_slot`: The slot of the block corresponding to `latest_valid_ancestor(head)`. + +## Ethereum Beacon APIs + +Consensus engines which provide an implementation of the [Ethereum Beacon +APIs](https://github.com/ethereum/beacon-APIs) must take care to avoid +presenting optimistic blocks as fully-verified blocks. + +When information about an optimistic block is requested, the consensus engine: + +- MUST NOT return a "success"-type response (e.g., 2xx). +- MAY return an "empty"-type response (e.g., 404). +- MAY return a "beacon node is currently syncing"-type response (e.g., 503). + +When `is_optimistic(head) == True`, the consensus engine: + +- MAY substitute the head block with `latest_valid_ancestor(block)`. +- MAY return a "beacon node is currently syncing"-type response (e.g., 503). From 9d7d4d086939ccc68ef8619a3cd66e2df4fbe891 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 16:21:23 +1100 Subject: [PATCH 016/106] Remove validate_merge_block check --- sync/optimistic.md | 1 - 1 file changed, 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index c67855710e..4b0c381a79 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -13,7 +13,6 @@ To perform an optimistic sync: - The `execute_payload` function MUST return `True` if the execution engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. -- The `validate_merge_block` function MUST NOT raise an assertion if both the `pow_block` and `pow_parent` are unknown to the execution engine. In addition to these changes to validation, the consensus engine MUST be able to ascertain, after import, which blocks returned `SYNCING` and which returned From 4f1d8152d5b0d5a3eed26435bbe036c153ad5bdb Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 16:21:30 +1100 Subject: [PATCH 017/106] Add note about full verification --- sync/optimistic.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index 4b0c381a79..696a954acc 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -22,6 +22,11 @@ set: - `optimistic_roots: Set[Root]`: `hash_tree_root(block)` where `block.body.execution_payload` is known to be `SYNCING`. +Notably, blocks included in `optimistic_roots` have passed all verifications +included in `process_block` (noting the modifications to the +`execute_payload`). I.e., the blocks are fully verified but awaiting execution +of the `ExecutionPayload`. + A consensus engine MUST be able to retrospectively (i.e., after import) modify the status of `SYNCING` blocks to be either `VALID` or `INVALID` based upon responses from an execution engine. I.e., perform the following transitions: From eb32e141d29a18208b0e0f9b2447f17416934f27 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 16:22:52 +1100 Subject: [PATCH 018/106] Fix typo --- sync/optimistic.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 696a954acc..37e7d44470 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -40,7 +40,7 @@ also transition from `SYNCING` -> `VALID`. When a block transitions from `SYNCING` -> `INVALID`, all *descendants* of the block MUST also transition from `SYNCING` -> `INVALID`. -When a node transitions from the `SYNCING` state is is removed from the set of +When a node transitions from the `SYNCING` state it is removed from the set of `optimistic_roots`. ### Execution Engine Errors From 084255465da0c96e7cb94e3c2d1ac3a4a9d02652 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 16:42:35 +1100 Subject: [PATCH 019/106] Add section about re-orgs --- sync/optimistic.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index 37e7d44470..49a5b3c886 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -57,6 +57,16 @@ unknown). Specifically, `SYNCING` responses should be fork-specific; the search for a block on one chain MUST NOT trigger a `SYNCING` response for another chain. +### Re-Orgs + +The consensus engine MUST support any chain reorganisation which does *not* +affect the justified checkpoint. The consensus engine MAY support re-orgs +beyond the justified checkpoint. + +If the justified checkpoint transitions from `SYNCING` -> `INVALID`, a +consensus engine MAY choose to alert the user and force the application to +exit. + ## Merge Transition To protect against attacks during the transition from empty `ExecutionPayload` From d9a0d16cc682e88e052c50be2bf712c54ee0b643 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 16:46:31 +1100 Subject: [PATCH 020/106] Tidy --- sync/optimistic.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 49a5b3c886..fa04d5da7c 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -75,9 +75,9 @@ NOT perform an optimistic sync unless the `finalized_checkpoint.root` of the hea block references a block for which `is_execution_enabled(head_state, head_block.body) == True`. -TODO: this restriction is very onerous, however it is the best known remedy for -the attack described in https://hackmd.io/S5ZEVhsNTqqfJirTAkBPlg I hope we can -do better. +> TODO: this restriction is very onerous, however it is the best known remedy for +> the attack described in https://hackmd.io/S5ZEVhsNTqqfJirTAkBPlg I hope we can +> do better. ## Fork Choice From 538cc816814f1c95b88c7fed0078467101797578 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 15 Dec 2021 09:02:39 +1100 Subject: [PATCH 021/106] Flip bool --- sync/optimistic.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index fa04d5da7c..298d0c2769 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -186,7 +186,7 @@ topics. #### BeaconBlocksByRange (v1, v2) Consensus engines MUST NOT include any block in a response where -`is_optimistic(block) == False`. +`is_optimistic(block) == True`. #### BeaconBlocksByRoot (v1, v2) From 5c1fcaf3e7bb535113753e78f1a6c738be9dd7b3 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 19 Dec 2021 14:34:43 +1100 Subject: [PATCH 022/106] Add qualification for errors --- sync/optimistic.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 298d0c2769..e926470078 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -44,9 +44,11 @@ When a node transitions from the `SYNCING` state it is removed from the set of `optimistic_roots`. ### Execution Engine Errors + A consensus engine MUST NOT interpret an error or failure to respond to a -message as a `SYNCING`, `VALID` or `INVALID` response. A consensus engine MAY -queue such a message for later processing. +message as a `SYNCING`, `VALID` or `INVALID` response. A message which receives +and error or no response MUST NOT be permitted to modify the fork choice +`Store`. A consensus engine MAY queue such a message for later processing. ### Assumptions about Execution Engine Behaviour From 2aa4edf933a4be8178db0a49bb6a750e68223486 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 19 Dec 2021 14:44:21 +1100 Subject: [PATCH 023/106] Move helpers --- sync/optimistic.md | 75 ++++++++++++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 29 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index e926470078..d633f25ac9 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -7,14 +7,57 @@ of the chain, it may be desirable for a consensus engine to import beacon blocks without verifying the execution payloads. This partial sync is called an *optimistic sync*. +## Constants + +|Name|Value|Unit +|---|---|---| +|`SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`| `64` | slots + +## Helpers + +Let `head_block: BeaconBlock` be the result of calling of the fork choice +algorithm at the time of block production. Let `justified_block: BeaconBlock` +be the latest current justified ancestor ancestor of the `head_block`. + +```python +def is_optimistic(block: BeaconBlock) -> bool: + hash_tree_root(block) in optimistic_roots +``` + +```python +def latest_valid_ancestor(block: BeaconBlock) -> BeaconBlock: + while True: + if not is_optimistic(block) or block.parent_root == Root(): + return block + block = get_block(block.parent_root) +``` + +```python +def is_execution_block(block: BeaconBlock) -> BeaconBlock: + block.execution_payload != ExecutionPayload() +``` + +Let only a node which returns `is_optimistic(head) == True` be an *optimistic +node*. Let only a validator on an optimistic node be an *optimistic validator*. + +When this specification only defines behaviour for an optimistic +node/validator, but *not* for the non-optimistic case, assume default +behaviours without regard for optimistic sync. + ## Mechanisms -To perform an optimistic sync: +## When to optimistically import blocks + +TODO + +## How to optimistically import blocks + +To optimistically import a block: - The `execute_payload` function MUST return `True` if the execution engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. -In addition to these changes to validation, the consensus engine MUST be able +In addition to this change to validation, the consensus engine MUST be able to ascertain, after import, which blocks returned `SYNCING` and which returned `VALID`. This document will assume that consensus engines store the following set: @@ -75,7 +118,7 @@ To protect against attacks during the transition from empty `ExecutionPayload` values to those which include the terminal PoW block, a consensus engine MUST NOT perform an optimistic sync unless the `finalized_checkpoint.root` of the head block references a block for which -`is_execution_enabled(head_state, head_block.body) == True`. +`is_execution_block(head_block) == True`. > TODO: this restriction is very onerous, however it is the best known remedy for > the attack described in https://hackmd.io/S5ZEVhsNTqqfJirTAkBPlg I hope we can @@ -93,32 +136,6 @@ point MUST NOT be included in the canonical chain and the weights from those A consensus engine MAY assume that the `ExecutionPayload` of a block used for checkpoint sync is `VALID`. -### Helpers - -Let `head_block: BeaconBlock` be the result of calling of the fork choice -algorithm at the time of block production. Let `justified_block: BeaconBlock` -be the latest current justified ancestor ancestor of the `head_block`. - -```python -def is_optimistic(block: BeaconBlock) -> bool: - hash_tree_root(block) in optimistic_roots -``` - -```python -def latest_valid_ancestor(block: BeaconBlock) -> BeaconBlock: - while True: - if not is_optimistic(block) or block.parent_root == Root(): - return block - block = get_block(block.parent_root) -``` - -Let only a node which returns `is_optimistic(head) == True` be an *optimistic -node*. Let only a validator on an optimistic node be an *optimistic validator*. - -When this specification only defines behaviour for an optimistic -node/validator, but *not* for the non-optimistic case, assume default -behaviours without regard for optimistic sync. - ## Validator assignments An entirely optimistically synced node is *not* a full node. It is unable to From e49685eed5cc2c4f95b83798a6e90650f60d837b Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 19 Dec 2021 14:56:46 +1100 Subject: [PATCH 024/106] Add section for enabling opt sync --- sync/optimistic.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index d633f25ac9..ff249dcbfa 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -37,6 +37,11 @@ def is_execution_block(block: BeaconBlock) -> BeaconBlock: block.execution_payload != ExecutionPayload() ``` +```python +def should_optimistically_import_block(current_slot: Slot, block: BeaconBlock) -> bool: + block.slot + SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY <= current_slot +``` + Let only a node which returns `is_optimistic(head) == True` be an *optimistic node*. Let only a validator on an optimistic node be an *optimistic validator*. @@ -48,7 +53,13 @@ behaviours without regard for optimistic sync. ## When to optimistically import blocks -TODO +A block MUST NOT be optimistically imported, unless either of the following +conditions are met: + +1. The justified checkpoint has execution enabled. I.e., + `is_execution_block(get_block(get_state(head_block).finalized_checkpoint.root))` +1. The current slot (as per the system clock) is at least `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` ahead of + the slot of the block being imported. I.e., `should_optimistically_import_block(current_slot) == True`. ## How to optimistically import blocks From ffba24f03edf3a7a896fd96adb5d398696e17d9f Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 10:15:51 +1100 Subject: [PATCH 025/106] Add failure recovery --- sync/optimistic.md | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index ff249dcbfa..65391332c8 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -11,7 +11,10 @@ blocks without verifying the execution payloads. This partial sync is called an |Name|Value|Unit |---|---|---| -|`SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`| `64` | slots +|`SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`| `96` | slots + +*Note: the `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` must be user-configurable. See +[Failure Recovery](#failure-recovery). ## Helpers @@ -123,6 +126,38 @@ If the justified checkpoint transitions from `SYNCING` -> `INVALID`, a consensus engine MAY choose to alert the user and force the application to exit. +## Failure Recovery + +During the merge transition it is possible for an attacker to craft a +`BeaconBlock` with an execution payload that references an +eternally-unavailable `body.execution_payload.parent_hash` value. In some rare +circumstances, it is possible that an attacker can build atop such a block to +trigger justification. If an optimistic node imports this malicious chain, that +node will have a "poisoned" fork choice store, such that the node is unable to +produce a child of the head (due to the invalid chain of payloads) and the node +is unable to fork around the head (due to the justification of the malicious +chain). + +The fork choice poisoning attack is temporary for an individual node, assuming +there exists an honest chain. An honest chain which justifies a higher epoch +than the malicious chain will take precedence and revive any poisoned store +once imported. + +The `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` parameter assumes that the network +will justify a honest chain within some number of slots. With this assumption, +it is therefore "safe" to optimistically import transition blocks during the +sync process. Since there is an assumption that an honest chain with a higher +justified checkpoint exists, any fork choice poisoning will be short-lived and +resolved before that node is required to produce a block. + +However, the assumption that the honest, canonical chain will always justify +within `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` slots is dubious. Therefore, +clients MUST provide the following command line flag to assist with manual +disaster recovery: + +- `--safe_slots_to_import_optimistically`: modifies the + `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`. + ## Merge Transition To protect against attacks during the transition from empty `ExecutionPayload` From 26e934b1e1d3fd2e3f4d92cd273626eabf344d14 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 10:16:09 +1100 Subject: [PATCH 026/106] Remove merge transition section --- sync/optimistic.md | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 65391332c8..4a89e51181 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -158,18 +158,6 @@ disaster recovery: - `--safe_slots_to_import_optimistically`: modifies the `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`. -## Merge Transition - -To protect against attacks during the transition from empty `ExecutionPayload` -values to those which include the terminal PoW block, a consensus engine MUST -NOT perform an optimistic sync unless the `finalized_checkpoint.root` of the head -block references a block for which -`is_execution_block(head_block) == True`. - -> TODO: this restriction is very onerous, however it is the best known remedy for -> the attack described in https://hackmd.io/S5ZEVhsNTqqfJirTAkBPlg I hope we can -> do better. - ## Fork Choice Consensus engines MUST support removing from fork choice blocks that transition From da6cad8767c43b72ee3380b97b4ae3f3356f1255 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 10:32:45 +1100 Subject: [PATCH 027/106] Tidy --- sync/optimistic.md | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 4a89e51181..15f94f636b 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -14,7 +14,7 @@ blocks without verifying the execution payloads. This partial sync is called an |`SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`| `96` | slots *Note: the `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` must be user-configurable. See -[Failure Recovery](#failure-recovery). +[Fork Choice Poisoning](#fork-choice-poisoning).* ## Helpers @@ -64,6 +64,9 @@ conditions are met: 1. The current slot (as per the system clock) is at least `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` ahead of the slot of the block being imported. I.e., `should_optimistically_import_block(current_slot) == True`. +*See [Fork Choice Poisoning](#fork-choice-poisoning) for the motivations behind +these conditions.* + ## How to optimistically import blocks To optimistically import a block: @@ -126,7 +129,14 @@ If the justified checkpoint transitions from `SYNCING` -> `INVALID`, a consensus engine MAY choose to alert the user and force the application to exit. -## Failure Recovery +## Fork Choice + +Consensus engines MUST support removing from fork choice blocks that transition +from `SYNCING` to `INVALID`. Specifically, a block deemed `INVALID` at any +point MUST NOT be included in the canonical chain and the weights from those +`INVALID` blocks MUST NOT be applied to any `VALID` or `SYNCING` ancestors. + +### Fork Choice Poisoning During the merge transition it is possible for an attacker to craft a `BeaconBlock` with an execution payload that references an @@ -158,13 +168,6 @@ disaster recovery: - `--safe_slots_to_import_optimistically`: modifies the `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`. -## Fork Choice - -Consensus engines MUST support removing from fork choice blocks that transition -from `SYNCING` to `INVALID`. Specifically, a block deemed `INVALID` at any -point MUST NOT be included in the canonical chain and the weights from those -`INVALID` blocks MUST NOT be applied to any `VALID` or `SYNCING` ancestors. - ## Checkpoint Sync (Weak Subjectivity Sync) A consensus engine MAY assume that the `ExecutionPayload` of a block used for From 9901cb38f7f5b562d4a46dc1da78d2138a177c8b Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 10:41:50 +1100 Subject: [PATCH 028/106] Move optimsitic_roots definition --- sync/optimistic.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 15f94f636b..6b52b98a20 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -22,6 +22,10 @@ Let `head_block: BeaconBlock` be the result of calling of the fork choice algorithm at the time of block production. Let `justified_block: BeaconBlock` be the latest current justified ancestor ancestor of the `head_block`. +Let `optimistic_roots: Set[Root]` be the set of `hash_tree_root(block)` for all +optimistically imported blocks which have yet to receive an `INVALID` or +`VALID` designation from an execution engine. + ```python def is_optimistic(block: BeaconBlock) -> bool: hash_tree_root(block) in optimistic_roots @@ -74,15 +78,11 @@ To optimistically import a block: - The `execute_payload` function MUST return `True` if the execution engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. -In addition to this change to validation, the consensus engine MUST be able -to ascertain, after import, which blocks returned `SYNCING` and which returned -`VALID`. This document will assume that consensus engines store the following -set: - -- `optimistic_roots: Set[Root]`: `hash_tree_root(block)` where - `block.body.execution_payload` is known to be `SYNCING`. +In addition to this change to validation, the consensus engine MUST be able to +ascertain, after import, which blocks returned `SYNCING` (`optimistic_roots`) +and which returned `VALID`. -Notably, blocks included in `optimistic_roots` have passed all verifications +Notably, optimistically imported blocks MUST have passed all verifications included in `process_block` (noting the modifications to the `execute_payload`). I.e., the blocks are fully verified but awaiting execution of the `ExecutionPayload`. From 26431b762d92f7e4e813179d0cb11b2386f8e324 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 11:12:11 +1100 Subject: [PATCH 029/106] Tidy --- sync/optimistic.md | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 6b52b98a20..553c3e4871 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -64,7 +64,7 @@ A block MUST NOT be optimistically imported, unless either of the following conditions are met: 1. The justified checkpoint has execution enabled. I.e., - `is_execution_block(get_block(get_state(head_block).finalized_checkpoint.root))` + `is_execution_block(get_block(get_state(head_block).current_justified_checkpoint.root))` 1. The current slot (as per the system clock) is at least `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` ahead of the slot of the block being imported. I.e., `should_optimistically_import_block(current_slot) == True`. @@ -79,13 +79,11 @@ To optimistically import a block: engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. In addition to this change to validation, the consensus engine MUST be able to -ascertain, after import, which blocks returned `SYNCING` (`optimistic_roots`) -and which returned `VALID`. +ascertain, after import, which blocks returned `SYNCING` and which returned +`VALID`. -Notably, optimistically imported blocks MUST have passed all verifications -included in `process_block` (noting the modifications to the -`execute_payload`). I.e., the blocks are fully verified but awaiting execution -of the `ExecutionPayload`. +Optimistically imported blocks MUST pass all verifications included in +`process_block` (withstanding the modifications to `execute_payload`). A consensus engine MUST be able to retrospectively (i.e., after import) modify the status of `SYNCING` blocks to be either `VALID` or `INVALID` based upon responses @@ -94,8 +92,9 @@ from an execution engine. I.e., perform the following transitions: - `SYNCING` -> `VALID` - `SYNCING` -> `INVALID` -When a block transitions from `SYNCING` -> `VALID`, all *ancestors* of the block MUST -also transition from `SYNCING` -> `VALID`. +When a block transitions from `SYNCING` -> `VALID`, all *ancestors* of the +block MUST also transition from `SYNCING` -> `VALID`. Such a block is no longer +considered "optimistically imported". When a block transitions from `SYNCING` -> `INVALID`, all *descendants* of the block MUST also transition from `SYNCING` -> `INVALID`. @@ -105,10 +104,11 @@ When a node transitions from the `SYNCING` state it is removed from the set of ### Execution Engine Errors -A consensus engine MUST NOT interpret an error or failure to respond to a -message as a `SYNCING`, `VALID` or `INVALID` response. A message which receives -and error or no response MUST NOT be permitted to modify the fork choice -`Store`. A consensus engine MAY queue such a message for later processing. +When an execution engine returns an error or fails to respond to a payload +validity request some block, a consensus engine: + +- MUST NOT optimistically import the block. +- MAY queue the block for later processing. ### Assumptions about Execution Engine Behaviour @@ -140,7 +140,7 @@ point MUST NOT be included in the canonical chain and the weights from those During the merge transition it is possible for an attacker to craft a `BeaconBlock` with an execution payload that references an -eternally-unavailable `body.execution_payload.parent_hash` value. In some rare +eternally-unavailable `body.execution_payload.parent_hash` value. In rare circumstances, it is possible that an attacker can build atop such a block to trigger justification. If an optimistic node imports this malicious chain, that node will have a "poisoned" fork choice store, such that the node is unable to @@ -149,14 +149,13 @@ is unable to fork around the head (due to the justification of the malicious chain). The fork choice poisoning attack is temporary for an individual node, assuming -there exists an honest chain. An honest chain which justifies a higher epoch -than the malicious chain will take precedence and revive any poisoned store -once imported. +there exists an honest chain which justifies a higher epoch than the malicious +chain. Such an honest chain will take precedence and revive any poisoned store. The `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` parameter assumes that the network will justify a honest chain within some number of slots. With this assumption, -it is therefore "safe" to optimistically import transition blocks during the -sync process. Since there is an assumption that an honest chain with a higher +it is acceptable to optimistically import transition blocks during the sync +process. Since there is an assumption that an honest chain with a higher justified checkpoint exists, any fork choice poisoning will be short-lived and resolved before that node is required to produce a block. @@ -165,13 +164,14 @@ within `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` slots is dubious. Therefore, clients MUST provide the following command line flag to assist with manual disaster recovery: -- `--safe_slots_to_import_optimistically`: modifies the +- `--safe-slots-to-import-optimistically`: modifies the `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`. ## Checkpoint Sync (Weak Subjectivity Sync) A consensus engine MAY assume that the `ExecutionPayload` of a block used for -checkpoint sync is `VALID`. +checkpoint sync is `VALID` without providing that payload to an execution +engine. ## Validator assignments From a797ae406bbdc29f01fc9533ef230e7f613e21e2 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 14:04:33 +1100 Subject: [PATCH 030/106] Add qualification about fc store --- sync/optimistic.md | 1 + 1 file changed, 1 insertion(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index 553c3e4871..c1841d2dd6 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -108,6 +108,7 @@ When an execution engine returns an error or fails to respond to a payload validity request some block, a consensus engine: - MUST NOT optimistically import the block. +- MUST NOT apply the block to the fork choice store. - MAY queue the block for later processing. ### Assumptions about Execution Engine Behaviour From b287f65dc9e9cc4e8988825f0fe6d7776589bab5 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 16:21:11 +1100 Subject: [PATCH 031/106] Allow RPC blocks --- sync/optimistic.md | 53 +++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index c1841d2dd6..b59c5b99bd 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -240,44 +240,43 @@ topics. ### The Req/Resp Domain -#### BeaconBlocksByRange (v1, v2) +Non-faulty, optimistic nodes may send blocks which result in an INVALID +response from an execution engine. To prevent network segregation between +optimistic and non-optimistic nodes, transmission of an INVALID payload SHOULD +NOT cause a node to be down-scored or disconnected. -Consensus engines MUST NOT include any block in a response where -`is_optimistic(block) == True`. +## Ethereum Beacon APIs -#### BeaconBlocksByRoot (v1, v2) +Consensus engines which provide an implementation of the [Ethereum Beacon +APIs](https://github.com/ethereum/beacon-APIs) must take care to avoid +presenting optimistic blocks as fully-verified blocks. -Consensus engines MUST NOT include any block in a response where -`is_optimistic(block) == True`. +### Helpers -#### Status +Let the following response types be defined as any response with the +corresponding HTTP status code: -An optimistic node MUST use the `latest_valid_ancestor(head)` block to form -responses, rather than the head block. Specifically, an optimistic node must -form a `Status` message as so: +- "Success" Response: Status Codes 200-299. +- "Not Found" Response: Status Code 404. +- "Syncing" Response: Status Code 503. -The fields are, as seen by the client at the time of sending the message: +### Requests for Optimistic Blocks -- `fork_digest`: As previously defined. -- `finalized_root`: `state.finalized_checkpoint.root` for the state corresponding to the latest valid ancestor block - (Note this defaults to `Root(b'\x00' * 32)` for the genesis finalized checkpoint). -- `finalized_epoch`: `state.finalized_checkpoint.epoch` for the state corresponding to the latest valid ancestor block. -- `head_root`: The `hash_tree_root` root of the current latest valid ancestor block (`BeaconBlock`). -- `head_slot`: The slot of the block corresponding to `latest_valid_ancestor(head)`. +When information about an optimistic block is requested, the consensus engine: -## Ethereum Beacon APIs +- MUST NOT respond with success. +- MAY respond with not found. +- MAY respond with syncing. -Consensus engines which provide an implementation of the [Ethereum Beacon -APIs](https://github.com/ethereum/beacon-APIs) must take care to avoid -presenting optimistic blocks as fully-verified blocks. +### Requests for the Head -When information about an optimistic block is requested, the consensus engine: +When `is_optimistic(head) == True`, the consensus engine: -- MUST NOT return a "success"-type response (e.g., 2xx). -- MAY return an "empty"-type response (e.g., 404). -- MAY return a "beacon node is currently syncing"-type response (e.g., 503). +- MAY substitute the head block with `latest_valid_ancestor(block)`. +- MAY return syncing. + +### Requests to Validators Endpoints When `is_optimistic(head) == True`, the consensus engine: -- MAY substitute the head block with `latest_valid_ancestor(block)`. -- MAY return a "beacon node is currently syncing"-type response (e.g., 503). +MUST respon From 91cad9b19b873b66a260ae47bb20aba903d1a2aa Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 17:19:32 +1100 Subject: [PATCH 032/106] Improve gossip wording --- sync/optimistic.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index b59c5b99bd..cf9bc2d540 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -220,8 +220,14 @@ Do not apply the existing condition: Instead, apply the new condition: -- [REJECT] The block's parent (defined by block.parent_root) passes validation, - *and* `block.parent root not in optimistic_roots`. +- [IGNORE] The block's parent (defined by block.parent_root) passes validation + except the block.body.execution_payload was deemed INVALID. +- [REJECT] The block's parent (defined by block.parent_root) passes all + validation, excluding verification of the block.body.execution_payload. + +The effect of these modifications is that invalid payloads may be propagated +across the network, but only when contained inside a block that is valid in *all +other aspects*. #### Other Topics From aa9a2967aa398e9c9bddac542644462907c395cf Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 17:27:02 +1100 Subject: [PATCH 033/106] Clarify API head condition --- sync/optimistic.md | 1 + 1 file changed, 1 insertion(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index cf9bc2d540..e363781195 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -278,6 +278,7 @@ When information about an optimistic block is requested, the consensus engine: When `is_optimistic(head) == True`, the consensus engine: +- MUST NOT return `head`. - MAY substitute the head block with `latest_valid_ancestor(block)`. - MAY return syncing. From 7837dc74bff92576c868d3c4f97bb921102d5dd6 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 17:46:42 +1100 Subject: [PATCH 034/106] Tidy, add validator endpoints --- sync/optimistic.md | 53 ++++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index e363781195..fcb6d981a4 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -19,8 +19,10 @@ blocks without verifying the execution payloads. This partial sync is called an ## Helpers Let `head_block: BeaconBlock` be the result of calling of the fork choice -algorithm at the time of block production. Let `justified_block: BeaconBlock` -be the latest current justified ancestor ancestor of the `head_block`. +algorithm at the time of block production. + +Let `justified_block: BeaconBlock` be the latest current justified ancestor +ancestor of the `head_block`. Let `optimistic_roots: Set[Root]` be the set of `hash_tree_root(block)` for all optimistically imported blocks which have yet to receive an `INVALID` or @@ -99,7 +101,7 @@ considered "optimistically imported". When a block transitions from `SYNCING` -> `INVALID`, all *descendants* of the block MUST also transition from `SYNCING` -> `INVALID`. -When a node transitions from the `SYNCING` state it is removed from the set of +When a block transitions from the `SYNCING` state it is removed from the set of `optimistic_roots`. ### Execution Engine Errors @@ -116,9 +118,9 @@ validity request some block, a consensus engine: This specification assumes execution engines will only return `SYNCING` when there is insufficient information available to make a `VALID` or `INVALID` determination on the given `ExecutionPayload` (e.g., the parent payload is -unknown). Specifically, `SYNCING` responses should be fork-specific; the search -for a block on one chain MUST NOT trigger a `SYNCING` response for another -chain. +unknown). Specifically, `SYNCING` responses should be fork-specific, in that +the search for a block on one chain MUST NOT trigger a `SYNCING` response for +another chain. ### Re-Orgs @@ -132,7 +134,7 @@ exit. ## Fork Choice -Consensus engines MUST support removing from fork choice blocks that transition +Consensus engines MUST support removing blocks from fork choice that transition from `SYNCING` to `INVALID`. Specifically, a block deemed `INVALID` at any point MUST NOT be included in the canonical chain and the weights from those `INVALID` blocks MUST NOT be applied to any `VALID` or `SYNCING` ancestors. @@ -141,17 +143,18 @@ point MUST NOT be included in the canonical chain and the weights from those During the merge transition it is possible for an attacker to craft a `BeaconBlock` with an execution payload that references an -eternally-unavailable `body.execution_payload.parent_hash` value. In rare -circumstances, it is possible that an attacker can build atop such a block to -trigger justification. If an optimistic node imports this malicious chain, that -node will have a "poisoned" fork choice store, such that the node is unable to -produce a child of the head (due to the invalid chain of payloads) and the node -is unable to fork around the head (due to the justification of the malicious +eternally-unavailable `body.execution_payload.parent_hash` (i.e., the parent +hash is random bytes). In rare circumstances, it is possible that an attacker +can build atop such a block to trigger justification. If an optimistic node +imports this malicious chain, that node will have a "poisoned" fork choice +store, such that the node is unable to produce a block that descends from the +head (due to the invalid chain of payloads) and the node is unable to produce a +block that forks around the head (due to the justification of the malicious chain). -The fork choice poisoning attack is temporary for an individual node, assuming -there exists an honest chain which justifies a higher epoch than the malicious -chain. Such an honest chain will take precedence and revive any poisoned store. +The fork choice poisoning attack is temporary for an individual node when that +an honest chain exists which justifies a higher epoch than the malicious chain. +Such an honest chain will take precedence and revive any poisoned store. The `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` parameter assumes that the network will justify a honest chain within some number of slots. With this assumption, @@ -170,9 +173,9 @@ disaster recovery: ## Checkpoint Sync (Weak Subjectivity Sync) -A consensus engine MAY assume that the `ExecutionPayload` of a block used for -checkpoint sync is `VALID` without providing that payload to an execution -engine. +A consensus engine MAY assume that the `ExecutionPayload` of a block used as an +anchor for checkpoint sync is `VALID` without necessarily providing that +payload to an execution engine. ## Validator assignments @@ -220,8 +223,7 @@ Do not apply the existing condition: Instead, apply the new condition: -- [IGNORE] The block's parent (defined by block.parent_root) passes validation - except the block.body.execution_payload was deemed INVALID. +- [IGNORE] The block's parent was imported optimistically. - [REJECT] The block's parent (defined by block.parent_root) passes all validation, excluding verification of the block.body.execution_payload. @@ -274,16 +276,17 @@ When information about an optimistic block is requested, the consensus engine: - MAY respond with not found. - MAY respond with syncing. -### Requests for the Head +### Requests for an Optimistic Head When `is_optimistic(head) == True`, the consensus engine: -- MUST NOT return `head`. +- MUST NOT return an optimistic `head`. - MAY substitute the head block with `latest_valid_ancestor(block)`. - MAY return syncing. ### Requests to Validators Endpoints -When `is_optimistic(head) == True`, the consensus engine: +When `is_optimistic(head) == True`, the consensus engine MUST return syncing to +all endpoints which match the following pattern: -MUST respon +- `eth/*/validator/*` From 451ae293ce7cc9ff4ee9230dfb932698d979e884 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 17:55:10 +1100 Subject: [PATCH 035/106] Specify no invalid parents --- sync/optimistic.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index fcb6d981a4..515e340cc5 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -79,6 +79,9 @@ To optimistically import a block: - The `execute_payload` function MUST return `True` if the execution engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. +- The `validate_merge_block` function MUST NOT raise an assertion if both the +`pow_block` and `pow_parent` are unknown to the execution engine. +- The parent of the block MUST NOT have an INVALID execution payload. In addition to this change to validation, the consensus engine MUST be able to ascertain, after import, which blocks returned `SYNCING` and which returned From 9421bf3dfd7ea5a4658e419eca66c63f75a388fc Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 18:32:47 +1100 Subject: [PATCH 036/106] Tidy --- sync/optimistic.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 515e340cc5..665bea63d3 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -43,7 +43,7 @@ def latest_valid_ancestor(block: BeaconBlock) -> BeaconBlock: ```python def is_execution_block(block: BeaconBlock) -> BeaconBlock: - block.execution_payload != ExecutionPayload() + block.body.execution_payload != ExecutionPayload() ``` ```python @@ -110,7 +110,7 @@ When a block transitions from the `SYNCING` state it is removed from the set of ### Execution Engine Errors When an execution engine returns an error or fails to respond to a payload -validity request some block, a consensus engine: +validity request for some block, a consensus engine: - MUST NOT optimistically import the block. - MUST NOT apply the block to the fork choice store. @@ -155,9 +155,10 @@ head (due to the invalid chain of payloads) and the node is unable to produce a block that forks around the head (due to the justification of the malicious chain). -The fork choice poisoning attack is temporary for an individual node when that -an honest chain exists which justifies a higher epoch than the malicious chain. -Such an honest chain will take precedence and revive any poisoned store. +If honest chain exists which justifies a higher epoch than the malicious chain, +that chain will take precedence and revive any poisoned store. Therefore, the +poisoning attack is temporary if >= 2/3rds of the network is honest and +non-faulty. The `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` parameter assumes that the network will justify a honest chain within some number of slots. With this assumption, From ff50bfe6e8695a6fbe92b6964ba454b6e225e562 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 18:32:54 +1100 Subject: [PATCH 037/106] Remove block production exception --- sync/optimistic.md | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 665bea63d3..ac805be0e9 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -183,22 +183,15 @@ payload to an execution engine. ## Validator assignments -An entirely optimistically synced node is *not* a full node. It is unable to -produce blocks, since an execution engine cannot produce a payload upon an -unknown parent. It cannot faithfully attest to the head block of the chain, -since it has not fully verified that block. +An optimistic node is *not* a full node. It is unable to produce blocks, since +an execution engine cannot produce a payload upon an unknown parent. It cannot +faithfully attest to the head block of the chain, since it has not fully +verified that block. ### Block Production A optimistic validator MUST NOT produce a block (i.e., sign across the -`DOMAIN_BEACON_PROPOSER` domain), unless one of the following exceptions are -met: - -#### Block Production Exception 1. - -If the justified block is fully verified (i.e., `not -is_optimistic(justified_block)`, the validator MAY produce a block upon -`latest_valid_ancestor(head)`. +`DOMAIN_BEACON_PROPOSER` domain). ### Attesting From e696d1103c4efec9d6d8432953fcee0c06ae5cac Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 18:36:03 +1100 Subject: [PATCH 038/106] Update gossip conditions --- sync/optimistic.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index ac805be0e9..e45d6bbb29 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -220,9 +220,10 @@ Do not apply the existing condition: Instead, apply the new condition: -- [IGNORE] The block's parent was imported optimistically. - [REJECT] The block's parent (defined by block.parent_root) passes all validation, excluding verification of the block.body.execution_payload. +- [IGNORE] The block's parent (defined by block.parent_root) passes all + validation, including verification of the block.body.execution_payload. The effect of these modifications is that invalid payloads may be propagated across the network, but only when contained inside a block that is valid in *all From 941531c8784a960c5bbb56a417658fcbc11150d7 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 22 Dec 2021 09:24:14 +1100 Subject: [PATCH 039/106] Update sync/optimistic.md Co-authored-by: terence tsao --- sync/optimistic.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index e45d6bbb29..abf352d070 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -68,7 +68,7 @@ conditions are met: 1. The justified checkpoint has execution enabled. I.e., `is_execution_block(get_block(get_state(head_block).current_justified_checkpoint.root))` 1. The current slot (as per the system clock) is at least `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` ahead of - the slot of the block being imported. I.e., `should_optimistically_import_block(current_slot) == True`. + the slot of the block being imported. I.e., `should_optimistically_import_block(current_slot, block) == True`. *See [Fork Choice Poisoning](#fork-choice-poisoning) for the motivations behind these conditions.* From 12293c999aa3ead27934d01e0887c6d48cd6693f Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 21 Dec 2021 16:49:43 +1100 Subject: [PATCH 040/106] Bump safe slots --- sync/optimistic.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index abf352d070..b67cf85d4e 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -11,7 +11,7 @@ blocks without verifying the execution payloads. This partial sync is called an |Name|Value|Unit |---|---|---| -|`SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`| `96` | slots +|`SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`| `128` | slots *Note: the `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` must be user-configurable. See [Fork Choice Poisoning](#fork-choice-poisoning).* From 50f526e418cb4115937271b49371d09502fbb820 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 22 Dec 2021 09:32:33 +1100 Subject: [PATCH 041/106] Fix typo --- sync/optimistic.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index b67cf85d4e..33ae71e9a7 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -241,8 +241,8 @@ An optimistic node MUST NOT subscribe to the following topics: - `sync_committee_contribution_and_proof` - `sync_committee_{subnet_id}` -Once the node ceases to be optimistic, it MAY re-subscribe to the aformentioned -topics. +Once the node ceases to be optimistic, it MAY re-subscribe to the +aforementioned topics. ### The Req/Resp Domain From 46bc2067402057590285f6253729ca0275d6a90f Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Wed, 29 Dec 2021 12:40:16 +0100 Subject: [PATCH 042/106] Add `CONFIG_NAME` to configs Runtime configurations apply to a certain network and the name of that network is useful for humans such that they can talk about it. Some of the existing configs already include a `CONFIG_NAME` toggle - might as well add it here as well and avoid some confusion - this name above all becomes useful in the beacon API. By extension, the `CONFIG_NAME` config will appear in the beacon api as a result of being defined here. --- configs/mainnet.yaml | 7 +++++++ configs/minimal.yaml | 7 +++++++ setup.py | 2 +- tests/core/pyspec/eth2spec/config/config_util.py | 2 +- 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 6c6af62828..f123f4f818 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -3,6 +3,13 @@ # Extends the mainnet preset PRESET_BASE: 'mainnet' +# Free-form short name of the network that this configuration applies to - known +# canonical network names include: +# * 'mainnet' - there can be only one +# * 'prater' - testnet +# Must match the regex: [a-z0-9\-] +CONFIG_NAME: 'mainnet' + # Transition # --------------------------------------------------------------- # TBD, 2**256-2**10 is a placeholder diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 4e48c470bd..61deb0be43 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -3,6 +3,13 @@ # Extends the minimal preset PRESET_BASE: 'minimal' +# Free-form short name of the network that this configuration applies to - known +# canonical network names include: +# * 'mainnet' - there can be only one +# * 'prater' - testnet +# Must match the regex: [a-z0-9\-] +CONFIG_NAME: 'minimal' + # Transition # --------------------------------------------------------------- # TBD, 2**256-2**10 is a placeholder diff --git a/setup.py b/setup.py index 81a629531a..aa4157b292 100644 --- a/setup.py +++ b/setup.py @@ -752,7 +752,7 @@ def parse_config_vars(conf: Dict[str, str]) -> Dict[str, str]: """ out: Dict[str, str] = dict() for k, v in conf.items(): - if isinstance(v, str) and (v.startswith("0x") or k == 'PRESET_BASE'): + if isinstance(v, str) and (v.startswith("0x") or k == 'PRESET_BASE' or k == 'CONFIG_NAME'): # Represent byte data with string, to avoid misinterpretation as big-endian int. # Everything is either byte data or an integer, with PRESET_BASE as one exception. out[k] = f"'{v}'" diff --git a/tests/core/pyspec/eth2spec/config/config_util.py b/tests/core/pyspec/eth2spec/config/config_util.py index 0d06428ea1..8fe2d8344a 100644 --- a/tests/core/pyspec/eth2spec/config/config_util.py +++ b/tests/core/pyspec/eth2spec/config/config_util.py @@ -14,7 +14,7 @@ def parse_config_vars(conf: Dict[str, Any]) -> Dict[str, Any]: out[k] = [int(item) if item.isdigit() else item for item in v] elif isinstance(v, str) and v.startswith("0x"): out[k] = bytes.fromhex(v[2:]) - elif k != 'PRESET_BASE': + elif k != 'PRESET_BASE' and k != 'CONFIG_NAME': out[k] = int(v) else: out[k] = v From af4725d08a2036767d6e3f208fddcc793338fc52 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 3 Jan 2022 07:51:35 -0700 Subject: [PATCH 043/106] add some non-empty extra_data tests --- .../test_process_execution_payload.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py index b8c5f55c3c..16c7598396 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py @@ -214,3 +214,35 @@ def test_bad_timestamp_regular_payload(spec, state): execution_payload.timestamp = execution_payload.timestamp + 1 yield from run_execution_payload_processing(spec, state, execution_payload, valid=False) + + +@with_bellatrix_and_later +@spec_state_test +def test_non_empty_extra_data_first_payload(spec, state): + # pre-state + state = build_state_with_incomplete_transition(spec, state) + next_slot(spec, state) + + # execution payload + execution_payload = build_empty_execution_payload(spec, state) + execution_payload.extra_data = b'\x45' * 12 + + yield from run_execution_payload_processing(spec, state, execution_payload) + + assert state.latest_execution_payload_header.extra_data == execution_payload.extra_data + + +@with_bellatrix_and_later +@spec_state_test +def test_non_empty_extra_data_regular_payload(spec, state): + # pre-state + state = build_state_with_complete_transition(spec, state) + next_slot(spec, state) + + # execution payload + execution_payload = build_empty_execution_payload(spec, state) + execution_payload.extra_data = b'\x45' * 12 + + yield from run_execution_payload_processing(spec, state, execution_payload) + + assert state.latest_execution_payload_header.extra_data == execution_payload.extra_data From b220655d243eb36cac0d01a05324e0268b15d0e2 Mon Sep 17 00:00:00 2001 From: Antonio Sanso Date: Thu, 6 Jan 2022 13:08:47 +0100 Subject: [PATCH 044/106] Add 3 new invalid test cases --- tests/generators/ssz_generic/ssz_container.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/generators/ssz_generic/ssz_container.py b/tests/generators/ssz_generic/ssz_container.py index 8f3a59ce3b..9c1d9d0cb6 100644 --- a/tests/generators/ssz_generic/ssz_container.py +++ b/tests/generators/ssz_generic/ssz_container.py @@ -105,7 +105,7 @@ def invalid_cases(): RandomizationMode.mode_one_count, RandomizationMode.mode_max_count]: if len(offsets) != 0: - for offset_index in offsets: + for index,offset_index in enumerate(offsets): yield f'{name}_{mode.to_name()}_offset_{offset_index}_plus_one', \ invalid_test_case(lambda: mod_offset( b=serialize(container_case_fn(rng, mode, typ)), @@ -118,3 +118,20 @@ def invalid_cases(): offset_index=offset_index, change=lambda x: 0 )) + if index == 0: + yield f'{name}_{mode.to_name()}_first offset_{offset_index}_minus_one', \ + invalid_test_case(lambda: mod_offset( + b=serialize(container_case_fn(rng, mode, typ)), + offset_index=offset_index, + change=lambda x: x - 1 + )) + if mode == RandomizationMode.mode_max_count: + serialized = serialize(container_case_fn(rng, mode, typ)) + serialized = serialized + serialized[0:2] + yield f'{name}_{mode.to_name()}_last offset_{offset_index}_overflow', \ + invalid_test_case(lambda:serialized) + if mode == RandomizationMode.mode_one_count: + serialized = serialize(container_case_fn(rng, mode, typ)) + serialized = serialized + serialized[0:1] + yield f'{name}_{mode.to_name()}_last offset_{offset_index}_wrong_byte_length', \ + invalid_test_case(lambda:serialized) From 7ccd528cd15a7310e95aa07fdbc0c7971f163e89 Mon Sep 17 00:00:00 2001 From: Antonio Sanso Date: Thu, 6 Jan 2022 13:26:00 +0100 Subject: [PATCH 045/106] fixing lint --- tests/generators/ssz_generic/ssz_container.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/generators/ssz_generic/ssz_container.py b/tests/generators/ssz_generic/ssz_container.py index 9c1d9d0cb6..8c8ed0b467 100644 --- a/tests/generators/ssz_generic/ssz_container.py +++ b/tests/generators/ssz_generic/ssz_container.py @@ -105,7 +105,7 @@ def invalid_cases(): RandomizationMode.mode_one_count, RandomizationMode.mode_max_count]: if len(offsets) != 0: - for index,offset_index in enumerate(offsets): + for index, offset_index in enumerate(offsets): yield f'{name}_{mode.to_name()}_offset_{offset_index}_plus_one', \ invalid_test_case(lambda: mod_offset( b=serialize(container_case_fn(rng, mode, typ)), @@ -120,7 +120,7 @@ def invalid_cases(): )) if index == 0: yield f'{name}_{mode.to_name()}_first offset_{offset_index}_minus_one', \ - invalid_test_case(lambda: mod_offset( + invalid_test_case(lambda: mod_offset( b=serialize(container_case_fn(rng, mode, typ)), offset_index=offset_index, change=lambda x: x - 1 @@ -129,9 +129,9 @@ def invalid_cases(): serialized = serialize(container_case_fn(rng, mode, typ)) serialized = serialized + serialized[0:2] yield f'{name}_{mode.to_name()}_last offset_{offset_index}_overflow', \ - invalid_test_case(lambda:serialized) + invalid_test_case(lambda: serialized) if mode == RandomizationMode.mode_one_count: serialized = serialize(container_case_fn(rng, mode, typ)) serialized = serialized + serialized[0:1] yield f'{name}_{mode.to_name()}_last offset_{offset_index}_wrong_byte_length', \ - invalid_test_case(lambda:serialized) + invalid_test_case(lambda: serialized) From b5908cf2948146c15b327a329c9303b4a0cd0d3c Mon Sep 17 00:00:00 2001 From: Antonio Sanso Date: Thu, 6 Jan 2022 13:28:19 +0100 Subject: [PATCH 046/106] fixing lint --- tests/generators/ssz_generic/ssz_container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generators/ssz_generic/ssz_container.py b/tests/generators/ssz_generic/ssz_container.py index 8c8ed0b467..4cff3ed4a2 100644 --- a/tests/generators/ssz_generic/ssz_container.py +++ b/tests/generators/ssz_generic/ssz_container.py @@ -124,7 +124,7 @@ def invalid_cases(): b=serialize(container_case_fn(rng, mode, typ)), offset_index=offset_index, change=lambda x: x - 1 - )) + )) if mode == RandomizationMode.mode_max_count: serialized = serialize(container_case_fn(rng, mode, typ)) serialized = serialized + serialized[0:2] From db16f590cbd32f15b1531a29541503ab5fbb1a76 Mon Sep 17 00:00:00 2001 From: Antonio Sanso Date: Thu, 6 Jan 2022 13:29:57 +0100 Subject: [PATCH 047/106] fixing lint --- tests/generators/ssz_generic/ssz_container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generators/ssz_generic/ssz_container.py b/tests/generators/ssz_generic/ssz_container.py index 4cff3ed4a2..59f53e66f5 100644 --- a/tests/generators/ssz_generic/ssz_container.py +++ b/tests/generators/ssz_generic/ssz_container.py @@ -124,7 +124,7 @@ def invalid_cases(): b=serialize(container_case_fn(rng, mode, typ)), offset_index=offset_index, change=lambda x: x - 1 - )) + )) if mode == RandomizationMode.mode_max_count: serialized = serialize(container_case_fn(rng, mode, typ)) serialized = serialized + serialized[0:2] From 6f64d46bc470fbe8ba32d44b3c0fa3b3d45079cd Mon Sep 17 00:00:00 2001 From: Antonio Sanso Date: Thu, 6 Jan 2022 19:06:42 +0100 Subject: [PATCH 048/106] Update tests/generators/ssz_generic/ssz_container.py Co-authored-by: Alex Stokes --- tests/generators/ssz_generic/ssz_container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generators/ssz_generic/ssz_container.py b/tests/generators/ssz_generic/ssz_container.py index 59f53e66f5..2498d392b4 100644 --- a/tests/generators/ssz_generic/ssz_container.py +++ b/tests/generators/ssz_generic/ssz_container.py @@ -119,7 +119,7 @@ def invalid_cases(): change=lambda x: 0 )) if index == 0: - yield f'{name}_{mode.to_name()}_first offset_{offset_index}_minus_one', \ + yield f'{name}_{mode.to_name()}_offset_{offset_index}_minus_one', \ invalid_test_case(lambda: mod_offset( b=serialize(container_case_fn(rng, mode, typ)), offset_index=offset_index, From 22719f2748f932e3d37a05026a088f26ae18c22c Mon Sep 17 00:00:00 2001 From: Antonio Sanso Date: Thu, 6 Jan 2022 19:06:50 +0100 Subject: [PATCH 049/106] Update tests/generators/ssz_generic/ssz_container.py Co-authored-by: Alex Stokes --- tests/generators/ssz_generic/ssz_container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generators/ssz_generic/ssz_container.py b/tests/generators/ssz_generic/ssz_container.py index 2498d392b4..818361fda3 100644 --- a/tests/generators/ssz_generic/ssz_container.py +++ b/tests/generators/ssz_generic/ssz_container.py @@ -127,7 +127,7 @@ def invalid_cases(): )) if mode == RandomizationMode.mode_max_count: serialized = serialize(container_case_fn(rng, mode, typ)) - serialized = serialized + serialized[0:2] + serialized = serialized + serialized[:2] yield f'{name}_{mode.to_name()}_last offset_{offset_index}_overflow', \ invalid_test_case(lambda: serialized) if mode == RandomizationMode.mode_one_count: From 7df0d4d70e60ff10d214578ec555b341d3b78f9d Mon Sep 17 00:00:00 2001 From: Antonio Sanso Date: Thu, 6 Jan 2022 19:07:23 +0100 Subject: [PATCH 050/106] Update tests/generators/ssz_generic/ssz_container.py Co-authored-by: Alex Stokes --- tests/generators/ssz_generic/ssz_container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generators/ssz_generic/ssz_container.py b/tests/generators/ssz_generic/ssz_container.py index 818361fda3..70e765c0c5 100644 --- a/tests/generators/ssz_generic/ssz_container.py +++ b/tests/generators/ssz_generic/ssz_container.py @@ -128,7 +128,7 @@ def invalid_cases(): if mode == RandomizationMode.mode_max_count: serialized = serialize(container_case_fn(rng, mode, typ)) serialized = serialized + serialized[:2] - yield f'{name}_{mode.to_name()}_last offset_{offset_index}_overflow', \ + yield f'{name}_{mode.to_name()}_last_offset_{offset_index}_overflow', \ invalid_test_case(lambda: serialized) if mode == RandomizationMode.mode_one_count: serialized = serialize(container_case_fn(rng, mode, typ)) From 69a2fb4af09cb2c97d3a33c713917fa2512de4f7 Mon Sep 17 00:00:00 2001 From: Antonio Sanso Date: Thu, 6 Jan 2022 19:07:31 +0100 Subject: [PATCH 051/106] Update tests/generators/ssz_generic/ssz_container.py Co-authored-by: Alex Stokes --- tests/generators/ssz_generic/ssz_container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generators/ssz_generic/ssz_container.py b/tests/generators/ssz_generic/ssz_container.py index 70e765c0c5..72b1729a4f 100644 --- a/tests/generators/ssz_generic/ssz_container.py +++ b/tests/generators/ssz_generic/ssz_container.py @@ -132,6 +132,6 @@ def invalid_cases(): invalid_test_case(lambda: serialized) if mode == RandomizationMode.mode_one_count: serialized = serialize(container_case_fn(rng, mode, typ)) - serialized = serialized + serialized[0:1] + serialized = serialized + serialized[:1] yield f'{name}_{mode.to_name()}_last offset_{offset_index}_wrong_byte_length', \ invalid_test_case(lambda: serialized) From e56dddd6ecd3b507721195f06782916b4dd8a5ae Mon Sep 17 00:00:00 2001 From: Antonio Sanso Date: Thu, 6 Jan 2022 19:07:48 +0100 Subject: [PATCH 052/106] Update tests/generators/ssz_generic/ssz_container.py Co-authored-by: Alex Stokes --- tests/generators/ssz_generic/ssz_container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generators/ssz_generic/ssz_container.py b/tests/generators/ssz_generic/ssz_container.py index 72b1729a4f..18fb3c09f8 100644 --- a/tests/generators/ssz_generic/ssz_container.py +++ b/tests/generators/ssz_generic/ssz_container.py @@ -133,5 +133,5 @@ def invalid_cases(): if mode == RandomizationMode.mode_one_count: serialized = serialize(container_case_fn(rng, mode, typ)) serialized = serialized + serialized[:1] - yield f'{name}_{mode.to_name()}_last offset_{offset_index}_wrong_byte_length', \ + yield f'{name}_{mode.to_name()}_last_offset_{offset_index}_wrong_byte_length', \ invalid_test_case(lambda: serialized) From d74cb5c9ec416ac03222f11cf27f9a72285dc0c7 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 6 Jan 2022 15:34:59 -0800 Subject: [PATCH 053/106] remove unnecessary conditional --- tests/generators/ssz_generic/ssz_container.py | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/tests/generators/ssz_generic/ssz_container.py b/tests/generators/ssz_generic/ssz_container.py index 8f3a59ce3b..088a8ca543 100644 --- a/tests/generators/ssz_generic/ssz_container.py +++ b/tests/generators/ssz_generic/ssz_container.py @@ -104,17 +104,16 @@ def invalid_cases(): RandomizationMode.mode_nil_count, RandomizationMode.mode_one_count, RandomizationMode.mode_max_count]: - if len(offsets) != 0: - for offset_index in offsets: - yield f'{name}_{mode.to_name()}_offset_{offset_index}_plus_one', \ - invalid_test_case(lambda: mod_offset( - b=serialize(container_case_fn(rng, mode, typ)), - offset_index=offset_index, - change=lambda x: x + 1 - )) - yield f'{name}_{mode.to_name()}_offset_{offset_index}_zeroed', \ - invalid_test_case(lambda: mod_offset( - b=serialize(container_case_fn(rng, mode, typ)), - offset_index=offset_index, - change=lambda x: 0 - )) + for offset_index in offsets: + yield f'{name}_{mode.to_name()}_offset_{offset_index}_plus_one', \ + invalid_test_case(lambda: mod_offset( + b=serialize(container_case_fn(rng, mode, typ)), + offset_index=offset_index, + change=lambda x: x + 1 + )) + yield f'{name}_{mode.to_name()}_offset_{offset_index}_zeroed', \ + invalid_test_case(lambda: mod_offset( + b=serialize(container_case_fn(rng, mode, typ)), + offset_index=offset_index, + change=lambda x: 0 + )) From fe605894e87e85ded911d62cb67ca1de88340d18 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 10 Jan 2022 17:07:13 +1100 Subject: [PATCH 054/106] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 738d43a058..86079a4c53 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ The current features are: * [Honest Validator guide changes](specs/altair/validator.md) * [P2P Networking](specs/altair/p2p-interface.md) -### Bellatrix (as known as The Merge) +### Bellatrix (also known as The Merge) The Bellatrix protocol upgrade is still actively in development. The exact specification has not been formally accepted as final and details are still subject to change. From 9e619f88b81041cb2b38887789b0fa830a5e71e6 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 11 Jan 2022 16:42:20 +1100 Subject: [PATCH 055/106] Apply suggestions from code review Co-authored-by: Hsiao-Wei Wang Co-authored-by: Mikhail Kalinin Co-authored-by: terence tsao --- sync/optimistic.md | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 33ae71e9a7..964413e253 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -29,29 +29,29 @@ optimistically imported blocks which have yet to receive an `INVALID` or `VALID` designation from an execution engine. ```python -def is_optimistic(block: BeaconBlock) -> bool: - hash_tree_root(block) in optimistic_roots +def is_optimistic(block: BeaconBlock, optimistic_roots: Set[Root]) -> bool: + return hash_tree_root(block) in optimistic_roots ``` ```python -def latest_valid_ancestor(block: BeaconBlock) -> BeaconBlock: +def latest_verified_ancestor(block: BeaconBlock) -> BeaconBlock: while True: if not is_optimistic(block) or block.parent_root == Root(): - return block + return block block = get_block(block.parent_root) ``` ```python def is_execution_block(block: BeaconBlock) -> BeaconBlock: - block.body.execution_payload != ExecutionPayload() + return block.body.execution_payload != ExecutionPayload() ``` ```python def should_optimistically_import_block(current_slot: Slot, block: BeaconBlock) -> bool: - block.slot + SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY <= current_slot + return block.slot + SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY <= current_slot ``` -Let only a node which returns `is_optimistic(head) == True` be an *optimistic +Let only a node which returns `is_optimistic(head) is True` be an *optimistic node*. Let only a validator on an optimistic node be an *optimistic validator*. When this specification only defines behaviour for an optimistic @@ -60,7 +60,7 @@ behaviours without regard for optimistic sync. ## Mechanisms -## When to optimistically import blocks +### When to optimistically import blocks A block MUST NOT be optimistically imported, unless either of the following conditions are met: @@ -73,13 +73,13 @@ conditions are met: *See [Fork Choice Poisoning](#fork-choice-poisoning) for the motivations behind these conditions.* -## How to optimistically import blocks +### How to optimistically import blocks To optimistically import a block: -- The `execute_payload` function MUST return `True` if the execution +- The [`execute_payload`](../specs/merge/beacon-chain.md#execute_payload) function MUST return `True` if the execution engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. -- The `validate_merge_block` function MUST NOT raise an assertion if both the +- The [`validate_merge_block`](../specs/merge/fork-choice.md#validate_merge_block) function MUST NOT raise an assertion if both the `pow_block` and `pow_parent` are unknown to the execution engine. - The parent of the block MUST NOT have an INVALID execution payload. @@ -190,7 +190,7 @@ verified that block. ### Block Production -A optimistic validator MUST NOT produce a block (i.e., sign across the +An optimistic validator MUST NOT produce a block (i.e., sign across the `DOMAIN_BEACON_PROPOSER` domain). ### Attesting @@ -216,14 +216,14 @@ validation conditions are modified as such: Do not apply the existing condition: -- [REJECT] The block's parent (defined by block.parent_root) passes validation. +- [REJECT] The block's parent (defined by `block.parent_root`) passes validation. Instead, apply the new condition: -- [REJECT] The block's parent (defined by block.parent_root) passes all - validation, excluding verification of the block.body.execution_payload. -- [IGNORE] The block's parent (defined by block.parent_root) passes all - validation, including verification of the block.body.execution_payload. +- [REJECT] The block's parent (defined by `block.parent_root`) passes all + validation, excluding verification of the `block.body.execution_payload`. +- [IGNORE] The block's parent (defined by `block.parent_root`) passes all + validation, including verification of the `block.body.execution_payload`. The effect of these modifications is that invalid payloads may be propagated across the network, but only when contained inside a block that is valid in *all @@ -276,7 +276,7 @@ When information about an optimistic block is requested, the consensus engine: ### Requests for an Optimistic Head -When `is_optimistic(head) == True`, the consensus engine: +When `is_optimistic(head) is True`, the consensus engine: - MUST NOT return an optimistic `head`. - MAY substitute the head block with `latest_valid_ancestor(block)`. @@ -284,7 +284,7 @@ When `is_optimistic(head) == True`, the consensus engine: ### Requests to Validators Endpoints -When `is_optimistic(head) == True`, the consensus engine MUST return syncing to +When `is_optimistic(head) is True`, the consensus engine MUST return syncing to all endpoints which match the following pattern: - `eth/*/validator/*` From 6eba269e9ba772f40451716290e9e030fdd6a34f Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 11 Jan 2022 19:02:40 +1100 Subject: [PATCH 056/106] Update sync/optimistic.md Co-authored-by: Raul Jordan --- sync/optimistic.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 964413e253..3b96ae9a82 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -62,7 +62,7 @@ behaviours without regard for optimistic sync. ### When to optimistically import blocks -A block MUST NOT be optimistically imported, unless either of the following +A block MAY be optimistically imported when either of the following conditions are met: 1. The justified checkpoint has execution enabled. I.e., From 890a020f25a39a1da972cc6e56ab70e9c25c7c02 Mon Sep 17 00:00:00 2001 From: Fredrik Svantes Date: Tue, 11 Jan 2022 10:31:57 +0100 Subject: [PATCH 057/106] Updating URL for eth-clients Was renamed from https://github.com/eth2-clients to https://github.com/eth-clients --- configs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/README.md b/configs/README.md index f470b932d0..6ef081e4c4 100644 --- a/configs/README.md +++ b/configs/README.md @@ -9,7 +9,7 @@ Standard configs: - [`minimal.yaml`](./minimal.yaml): Minimal configuration, used in spec-testing along with the [`minimal`](../presets/minimal) preset. Not all network configurations are in scope for the specification, -see [`github.com/eth2-clients/eth2-networks`](https://github.com/eth2-clients/eth2-networks) for common networks, +see [`github.com/eth-clients/eth2-networks`](https://github.com/eth-clients/eth2-networks) for common networks, and additional testnet assets. ## Forking From 48e19b15ae92428734a671b1f7d8b902b8562e77 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Tue, 11 Jan 2022 11:22:39 +0100 Subject: [PATCH 058/106] Rename `sync_committee_aggregate` > `sync_aggregate` This renames the `sync_committee_aggregate` field of `LightClientUpdate` to `sync_aggregate` for consistency with the terminology in the rest of the spec. --- specs/altair/sync-protocol.md | 10 +++++----- .../test/altair/unittests/test_sync_protocol.py | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index c8c7c3d4dc..38102e00fe 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -69,7 +69,7 @@ class LightClientUpdate(Container): finalized_header: BeaconBlockHeader finality_branch: Vector[Bytes32, floorlog2(FINALIZED_ROOT_INDEX)] # Sync committee aggregate signature - sync_committee_aggregate: SyncAggregate + sync_aggregate: SyncAggregate # Fork version for the aggregate signature fork_version: Version ``` @@ -187,8 +187,8 @@ def validate_light_client_update(store: LightClientStore, index=get_subtree_index(NEXT_SYNC_COMMITTEE_INDEX), root=active_header.state_root, ) - - sync_aggregate = update.sync_committee_aggregate + + sync_aggregate = update.sync_aggregate # Verify sync committee has sufficient participants assert sum(sync_aggregate.sync_committee_bits) >= MIN_SYNC_COMMITTEE_PARTICIPANTS @@ -225,12 +225,12 @@ def process_light_client_update(store: LightClientStore, genesis_validators_root: Root) -> None: validate_light_client_update(store, update, current_slot, genesis_validators_root) - sync_committee_bits = update.sync_committee_aggregate.sync_committee_bits + sync_committee_bits = update.sync_aggregate.sync_committee_bits # Update the best update in case we have to force-update to it if the timeout elapses if ( store.best_valid_update is None - or sum(sync_committee_bits) > sum(store.best_valid_update.sync_committee_aggregate.sync_committee_bits) + or sum(sync_committee_bits) > sum(store.best_valid_update.sync_aggregate.sync_committee_bits) ): store.best_valid_update = update diff --git a/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py b/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py index 30444c4ce4..120004654c 100644 --- a/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py +++ b/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py @@ -58,7 +58,7 @@ def test_process_light_client_update_not_timeout(spec, state): block_header.slot, committee, ) - sync_committee_aggregate = spec.SyncAggregate( + sync_aggregate = spec.SyncAggregate( sync_committee_bits=sync_committee_bits, sync_committee_signature=sync_committee_signature, ) @@ -76,7 +76,7 @@ def test_process_light_client_update_not_timeout(spec, state): next_sync_committee_branch=next_sync_committee_branch, finalized_header=finality_header, finality_branch=finality_branch, - sync_committee_aggregate=sync_committee_aggregate, + sync_aggregate=sync_aggregate, fork_version=state.fork.current_version, ) @@ -123,7 +123,7 @@ def test_process_light_client_update_timeout(spec, state): committee, block_root=spec.Root(block_header.hash_tree_root()), ) - sync_committee_aggregate = spec.SyncAggregate( + sync_aggregate = spec.SyncAggregate( sync_committee_bits=sync_committee_bits, sync_committee_signature=sync_committee_signature, ) @@ -140,7 +140,7 @@ def test_process_light_client_update_timeout(spec, state): next_sync_committee_branch=next_sync_committee_branch, finalized_header=finality_header, finality_branch=finality_branch, - sync_committee_aggregate=sync_committee_aggregate, + sync_aggregate=sync_aggregate, fork_version=state.fork.current_version, ) @@ -201,7 +201,7 @@ def test_process_light_client_update_finality_updated(spec, state): committee, block_root=spec.Root(block_header.hash_tree_root()), ) - sync_committee_aggregate = spec.SyncAggregate( + sync_aggregate = spec.SyncAggregate( sync_committee_bits=sync_committee_bits, sync_committee_signature=sync_committee_signature, ) @@ -212,7 +212,7 @@ def test_process_light_client_update_finality_updated(spec, state): next_sync_committee_branch=next_sync_committee_branch, finalized_header=finalized_block_header, finality_branch=finality_branch, - sync_committee_aggregate=sync_committee_aggregate, + sync_aggregate=sync_aggregate, fork_version=state.fork.current_version, ) From 212eb00fc15736d58d0b7a5ada44060e56a8093b Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Tue, 11 Jan 2022 12:07:49 +0100 Subject: [PATCH 059/106] Document light client constants This adds documentation about the unit and actual value of light client specific constants, consistently with the rest of the spec. --- specs/altair/beacon-chain.md | 2 +- specs/altair/sync-protocol.md | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index 3c7177a1b4..2e897deb7d 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -126,7 +126,7 @@ This patch updates a few configuration values to move penalty parameters closer | Name | Value | Unit | Duration | | - | - | - | - | -| `SYNC_COMMITTEE_SIZE` | `uint64(2**9)` (= 512) | Validators | | +| `SYNC_COMMITTEE_SIZE` | `uint64(2**9)` (= 512) | validators | | | `EPOCHS_PER_SYNC_COMMITTEE_PERIOD` | `uint64(2**8)` (= 256) | epochs | ~27 hours | ## Configuration diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index c8c7c3d4dc..1baa830f87 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -42,17 +42,17 @@ uses sync committees introduced in [this beacon chain extension](./beacon-chain. | Name | Value | | - | - | -| `FINALIZED_ROOT_INDEX` | `get_generalized_index(BeaconState, 'finalized_checkpoint', 'root')` | -| `NEXT_SYNC_COMMITTEE_INDEX` | `get_generalized_index(BeaconState, 'next_sync_committee')` | +| `FINALIZED_ROOT_INDEX` | `get_generalized_index(BeaconState, 'finalized_checkpoint', 'root')` (= 105) | +| `NEXT_SYNC_COMMITTEE_INDEX` | `get_generalized_index(BeaconState, 'next_sync_committee')` (= 55) | ## Preset ### Misc -| Name | Value | Notes | -| - | - | - | -| `MIN_SYNC_COMMITTEE_PARTICIPANTS` | `1` | | -| `UPDATE_TIMEOUT` | `SLOTS_PER_EPOCH * EPOCHS_PER_SYNC_COMMITTEE_PERIOD` | ~27.3 hours | +| Name | Value | Unit | Duration | +| - | - | - | - | +| `MIN_SYNC_COMMITTEE_PARTICIPANTS` | `1` | validators | +| `UPDATE_TIMEOUT` | `SLOTS_PER_EPOCH * EPOCHS_PER_SYNC_COMMITTEE_PERIOD` | epochs | ~27.3 hours | ## Containers From 79c456b9f400d029af0883774c6e7feef5188f72 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Tue, 11 Jan 2022 11:48:22 +0100 Subject: [PATCH 060/106] Fix light client docs to match function signature In the light client docs a mentioning of a function trigger is lacking the `genesis_validators_root` argument. This patch adds that argument to the documentation to match the real function signature. It also slightly improves the grammar. --- specs/altair/sync-protocol.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index c8c7c3d4dc..0cabf51b88 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -127,7 +127,7 @@ def get_safety_threshold(store: LightClientStore) -> uint64: ## Light client state updates -A light client maintains its state in a `store` object of type `LightClientStore` and receives `update` objects of type `LightClientUpdate`. Every `update` triggers `process_light_client_update(store, update, current_slot)` where `current_slot` is the current slot based on some local clock. `process_slot_for_light_client_store` is processed every time the current slot increments. +A light client maintains its state in a `store` object of type `LightClientStore` and receives `update` objects of type `LightClientUpdate`. Every `update` triggers `process_light_client_update(store, update, current_slot, genesis_validators_root)` where `current_slot` is the current slot based on a local clock. `process_slot_for_light_client_store` is triggered every time the current slot increments. #### `process_slot_for_light_client_store` From 7e2461e8056a709f0c20177f1af768cf6ca94376 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Tue, 11 Jan 2022 20:34:06 -0800 Subject: [PATCH 061/106] Remove client setting --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 86079a4c53..fb58edf8d1 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,6 @@ The Bellatrix protocol upgrade is still actively in development. The exact speci * [Bellatrix fork](specs/bellatrix/fork.md) * [Fork Choice changes](specs/bellatrix/fork-choice.md) * [Validator additions](specs/bellatrix/validator.md) - * [Client settings](specs/bellatrix/client-settings.md) * [P2P Networking](specs/bellatrix/p2p-interface.md) ### Sharding From 6c13e2ecdbe2fbb2262e5570788dec1664d90803 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 12 Jan 2022 15:54:11 +1100 Subject: [PATCH 062/106] Expand `should_optimistically_import_block` --- sync/optimistic.md | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 3b96ae9a82..6137579cb8 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -29,14 +29,23 @@ optimistically imported blocks which have yet to receive an `INVALID` or `VALID` designation from an execution engine. ```python -def is_optimistic(block: BeaconBlock, optimistic_roots: Set[Root]) -> bool: - return hash_tree_root(block) in optimistic_roots +@dataclass +class Store(object): + optimistic_roots: Set[Root] + head_block_root: Root + blocks: Dict[Root, BeaconBlock] + block_states: Dict[Root, BeaconState] ``` ```python -def latest_verified_ancestor(block: BeaconBlock) -> BeaconBlock: +def is_optimistic(store: Store, block: BeaconBlock) -> bool: + return hash_tree_root(block) in store.optimistic_roots +``` + +```python +def latest_verified_ancestor(store: Store, block: BeaconBlock) -> BeaconBlock: while True: - if not is_optimistic(block) or block.parent_root == Root(): + if not is_optimistic(store, block) or block.parent_root == Root(): return block block = get_block(block.parent_root) ``` @@ -47,8 +56,11 @@ def is_execution_block(block: BeaconBlock) -> BeaconBlock: ``` ```python -def should_optimistically_import_block(current_slot: Slot, block: BeaconBlock) -> bool: - return block.slot + SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY <= current_slot +def should_optimistically_import_block(store: Store, current_slot: Slot, block: BeaconBlock) -> bool: + justified_root = store.block_states[store.head_block_root].current_justified_checkpoint.root + justifed_is_verified = is_execution_block(store.blocks[justified_root]) + block_is_deep = block.slot + SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY <= current_slot + return justified_is_verified or block_is_deep ``` Let only a node which returns `is_optimistic(head) is True` be an *optimistic @@ -105,7 +117,7 @@ When a block transitions from `SYNCING` -> `INVALID`, all *descendants* of the block MUST also transition from `SYNCING` -> `INVALID`. When a block transitions from the `SYNCING` state it is removed from the set of -`optimistic_roots`. +`store.optimistic_roots`. ### Execution Engine Errors From 2ce2aac0c89eac7ff6098cc2d0a4ad7f700f1d06 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 12 Jan 2022 15:57:27 +1100 Subject: [PATCH 063/106] Address "yet to" paragraph --- sync/optimistic.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index 6137579cb8..f3f70ee2fc 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -28,6 +28,10 @@ Let `optimistic_roots: Set[Root]` be the set of `hash_tree_root(block)` for all optimistically imported blocks which have yet to receive an `INVALID` or `VALID` designation from an execution engine. +Let `optimistic_roots: Set[Root]` be the set of `hash_tree_root(block)` for all +optimistically imported blocks which have only received a `SYNCING` designation +from an execution engine (i.e., they are not known to be `INVALID` or `VALID`). + ```python @dataclass class Store(object): From 736f3cec3f1d5c5a167c8d74b33a4d73b3f6b07b Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 12 Jan 2022 16:09:24 +1100 Subject: [PATCH 064/106] Remove `justified_block` --- sync/optimistic.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index f3f70ee2fc..c511e9781c 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -21,9 +21,6 @@ blocks without verifying the execution payloads. This partial sync is called an Let `head_block: BeaconBlock` be the result of calling of the fork choice algorithm at the time of block production. -Let `justified_block: BeaconBlock` be the latest current justified ancestor -ancestor of the `head_block`. - Let `optimistic_roots: Set[Root]` be the set of `hash_tree_root(block)` for all optimistically imported blocks which have yet to receive an `INVALID` or `VALID` designation from an execution engine. From 55d92cee34842be718d4580619fdd16812a5f9d2 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 12 Jan 2022 16:10:59 +1100 Subject: [PATCH 065/106] Fix typo --- sync/optimistic.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index c511e9781c..959fa9ffb8 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -168,10 +168,10 @@ head (due to the invalid chain of payloads) and the node is unable to produce a block that forks around the head (due to the justification of the malicious chain). -If honest chain exists which justifies a higher epoch than the malicious chain, -that chain will take precedence and revive any poisoned store. Therefore, the -poisoning attack is temporary if >= 2/3rds of the network is honest and -non-faulty. +If an honest chain exists which justifies a higher epoch than the malicious +chain, that chain will take precedence and revive any poisoned store. +Therefore, the poisoning attack is temporary if >= 2/3rds of the network is +honest and non-faulty. The `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` parameter assumes that the network will justify a honest chain within some number of slots. With this assumption, From 1228e01883685ae90aeef8b05676879ed3d8bb13 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 12 Jan 2022 16:18:53 +1100 Subject: [PATCH 066/106] Update p2p-networking --- specs/merge/p2p-interface.md | 17 +++++++++++++ sync/optimistic.md | 46 ------------------------------------ 2 files changed, 17 insertions(+), 46 deletions(-) diff --git a/specs/merge/p2p-interface.md b/specs/merge/p2p-interface.md index 0ab3d08258..9646eae4dc 100644 --- a/specs/merge/p2p-interface.md +++ b/specs/merge/p2p-interface.md @@ -85,6 +85,10 @@ The *type* of the payload of this topic changes to the (modified) `SignedBeaconB Specifically, this type changes with the addition of `execution_payload` to the inner `BeaconBlockBody`. See the Merge [state transition document](./beacon-chain.md#beaconblockbody) for further details. +Blocks with execution enabled will be permitted to propagate regardless of the +validity of the execution payload. This prevents network segregation between +[optimistic](/sync/optimistic.md) and non-optimistic nodes. + In addition to the gossip validations for this topic from prior specifications, the following validations MUST pass before forwarding the `signed_beacon_block` on the network. Alias `block = signed_beacon_block.message`, `execution_payload = block.body.execution_payload`. @@ -92,6 +96,13 @@ Alias `block = signed_beacon_block.message`, `execution_payload = block.body.exe then validate the following: - _[REJECT]_ The block's execution payload timestamp is correct with respect to the slot -- i.e. `execution_payload.timestamp == compute_timestamp_at_slot(state, block.slot)`. + - [REJECT] The block's parent (defined by `block.parent_root`) passes all + validation, excluding verification of the `block.body.execution_payload`. + - [IGNORE] The block's parent (defined by `block.parent_root`) passes all + validation, including verification of the `block.body.execution_payload`. + +The following gossip validation from prior specifications MUST NOT be applied if the execution is enabled for the block -- i.e. `is_execution_enabled(state, block.body)`: + - [REJECT] The block's parent (defined by `block.parent_root`) passes validation. ### Transitioning the gossip @@ -100,6 +111,12 @@ details on how to handle transitioning gossip topics for the Merge. ## The Req/Resp domain +Non-faulty, [optimistic](/sync/optimistic.md) nodes may send blocks which +result in an INVALID response from an execution engine. To prevent network +segregation between optimistic and non-optimistic nodes, transmission of an +INVALID payload via the Req/Resp domain SHOULD NOT cause a node to be +down-scored or disconnected. + ### Messages #### BeaconBlocksByRange v2 diff --git a/sync/optimistic.md b/sync/optimistic.md index 959fa9ffb8..84175d06cf 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -218,52 +218,6 @@ An optimistic validator MUST NOT participate in sync committees (i.e., sign acro `DOMAIN_SYNC_COMMITTEE`, `DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF` or `DOMAIN_CONTRIBUTION_AND_PROOF` domains). -## P2P Networking - -### The Gossip Domain (gossipsub) - -#### `beacon_block` - -An optimistic validator MAY subscribe to the `beacon_block` topic. Propagation -validation conditions are modified as such: - -Do not apply the existing condition: - -- [REJECT] The block's parent (defined by `block.parent_root`) passes validation. - -Instead, apply the new condition: - -- [REJECT] The block's parent (defined by `block.parent_root`) passes all - validation, excluding verification of the `block.body.execution_payload`. -- [IGNORE] The block's parent (defined by `block.parent_root`) passes all - validation, including verification of the `block.body.execution_payload`. - -The effect of these modifications is that invalid payloads may be propagated -across the network, but only when contained inside a block that is valid in *all -other aspects*. - -#### Other Topics - -An optimistic node MUST NOT subscribe to the following topics: - -- `beacon_aggregate_and_proof` -- `voluntary_exit` -- `proposer_slashing` -- `attester_slashing` -- `beacon_attestation_{subnet_id}` -- `sync_committee_contribution_and_proof` -- `sync_committee_{subnet_id}` - -Once the node ceases to be optimistic, it MAY re-subscribe to the -aforementioned topics. - -### The Req/Resp Domain - -Non-faulty, optimistic nodes may send blocks which result in an INVALID -response from an execution engine. To prevent network segregation between -optimistic and non-optimistic nodes, transmission of an INVALID payload SHOULD -NOT cause a node to be down-scored or disconnected. - ## Ethereum Beacon APIs Consensus engines which provide an implementation of the [Ethereum Beacon From 60eab25774e6282c2531b02921b4172780e8fb04 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 12 Jan 2022 16:39:43 +1100 Subject: [PATCH 067/106] Add section about merge block --- sync/optimistic.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 84175d06cf..d908803100 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -1,5 +1,7 @@ # Optimistic Sync +[`validate_merge_block`]: ../specs/merge/fork-choice.md#validate_merge_block + ## Introduction In order to provide a syncing execution engine with a partial view of the head @@ -92,7 +94,7 @@ To optimistically import a block: - The [`execute_payload`](../specs/merge/beacon-chain.md#execute_payload) function MUST return `True` if the execution engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. -- The [`validate_merge_block`](../specs/merge/fork-choice.md#validate_merge_block) function MUST NOT raise an assertion if both the +- The [`validate_merge_block`][] function MUST NOT raise an assertion if both the `pow_block` and `pow_parent` are unknown to the execution engine. - The parent of the block MUST NOT have an INVALID execution payload. @@ -120,6 +122,13 @@ block MUST also transition from `SYNCING` -> `INVALID`. When a block transitions from the `SYNCING` state it is removed from the set of `store.optimistic_roots`. +When a "merge block" (i.e. a block which enables execution) is declared to be +`VALID` by an execution engine (either directly or indirectly), the full +[`validate_merge_block`][] MUST be run against the merge block. If the block +fails [`validate_merge_block`][], the merge block MUST be treated the same as +an `INVALID` block (i.e., it and all its descendants are invalidated and +removed from the block tree). + ### Execution Engine Errors When an execution engine returns an error or fails to respond to a payload From 0ae80d93826c1dcf0051ebd7afc49febe46d4bb5 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 12 Jan 2022 16:41:24 +1100 Subject: [PATCH 068/106] Fix typo --- sync/optimistic.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index d908803100..05ab5076fc 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -255,7 +255,7 @@ When information about an optimistic block is requested, the consensus engine: When `is_optimistic(head) is True`, the consensus engine: - MUST NOT return an optimistic `head`. -- MAY substitute the head block with `latest_valid_ancestor(block)`. +- MAY substitute the head block with `latest_verified_ancestor(block)`. - MAY return syncing. ### Requests to Validators Endpoints From de1a6caa5dec08fa6358f2e2506e5c4e50548814 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 12 Jan 2022 16:43:36 +1100 Subject: [PATCH 069/106] Add comment about INVALID block --- sync/optimistic.md | 1 + 1 file changed, 1 insertion(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index 05ab5076fc..72037dc3a5 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -47,6 +47,7 @@ def is_optimistic(store: Store, block: BeaconBlock) -> bool: ```python def latest_verified_ancestor(store: Store, block: BeaconBlock) -> BeaconBlock: + # It is assumed that the `block` parameter is never an INVALID block. while True: if not is_optimistic(store, block) or block.parent_root == Root(): return block From ad7e92433a05b0fd6c2b74fddae3c5cf154b1533 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 12 Jan 2022 16:49:08 +1100 Subject: [PATCH 070/106] Update links to bellatrix --- sync/optimistic.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 72037dc3a5..656a328b12 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -1,6 +1,6 @@ # Optimistic Sync -[`validate_merge_block`]: ../specs/merge/fork-choice.md#validate_merge_block +[`validate_merge_block`]: ../specs/bellatrix/fork-choice.md#validate_merge_block ## Introduction @@ -93,7 +93,7 @@ these conditions.* To optimistically import a block: -- The [`execute_payload`](../specs/merge/beacon-chain.md#execute_payload) function MUST return `True` if the execution +- The [`execute_payload`](../specs/bellatrix/beacon-chain.md#execute_payload) function MUST return `True` if the execution engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. - The [`validate_merge_block`][] function MUST NOT raise an assertion if both the `pow_block` and `pow_parent` are unknown to the execution engine. From 90fb7f68719b9bae2889c505e892114ee22540e8 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 12 Jan 2022 19:04:43 +1100 Subject: [PATCH 071/106] Add rationale --- specs/bellatrix/p2p-interface.md | 23 +++++++++++ sync/optimistic.md | 70 ++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) diff --git a/specs/bellatrix/p2p-interface.md b/specs/bellatrix/p2p-interface.md index 75c70a18a5..5a260d982b 100644 --- a/specs/bellatrix/p2p-interface.md +++ b/specs/bellatrix/p2p-interface.md @@ -198,3 +198,26 @@ valid block sizes in the range of gas limits expected in the medium term. As with both gossip and req/rsp maximum values, type-specific limits should always by simultaneously respected. + +### Why allow invalid payloads on the P2P network? + +The specification allows blocks with invalid payloads to propagate across +gossip and via RPC calls. The reasoning for this is as follows: + +1. Optimistic nodes must listen to block gossip to obtain a view of the head of + the chain. +2. Therefore, optimistic nodes must propagate gossip blocks. Otherwise, they'd + be censoring. +3. If optimistic nodes will propose blocks via gossip, then they must respond + to requests for the parent via RPC. +4. Therefore, optimistic nodes must send optimistic blocks via RPC. + +So, to prevent network segregation from optimistic nodes accidentally sending +invalid payloads, nodes should never downscore/disconnect nodes due to invalid +payloads. This does open the network to some DoS attacks from invalid execution +payloads, but the scope of actors is limited to validators who can put those +payloads in valid (and slashable) beacon blocks. Therefore, it is argued that +the DoS risk introduced in tolerable. + +More complicated schemes are possible that could restrict invalid payloads from +RPC. However, it's not clear that complexity is warranted. diff --git a/sync/optimistic.md b/sync/optimistic.md index 656a328b12..6793b972d8 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -265,3 +265,73 @@ When `is_optimistic(head) is True`, the consensus engine MUST return syncing to all endpoints which match the following pattern: - `eth/*/validator/*` + +## Design Decision Rationale + +### Why `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`? + +Nodes can only import an optimistic block if their justified checkpoint is +verified or the block is older than `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`. + +These restraints are applied in order to mitigate an attack where a block which +enables execution (a *transition block*) can reference a junk parent hash. This +makes it impossible for honest nodes to build atop that block. If an attacker +exploits a nuance in fork choice `filter_block_tree`, they can, in some rare +cases, produce a junk block that out-competes all locally produced blocks for +the head. This prevents a node from producing a chain of blocks, therefore +breaking liveness. + +Thankfully, if 2/3rds of validators are not poisoned, they can justify an +honest chain which will un-poison all other nodes. + +Notably, this attack only exists for optimistic nodes. Nodes which fully verify +the transition block will reject a block with a junk parent hash. + +Given all of this, we can say two things: + +1. **BNs which are following the head during the transition shouldn't + optimistically import the transition block.** If 1/3rd of validators + optimistically import the poison block, there will be no remaining nodes to + justify an honest chain. +2. **BNs which are syncing can optimistically import transition blocks.** In + this case a justified chain already exists blocks. The poison block would be + quickly reverted and would have no affect on liveness. + +Astute readers will notice that (2) contains a glaring assumption about network +liveness. This is necessary because a node cannot feasibly ascertain that the +transition block is justified without importing that block and risking +poisoning. Therefore, we use `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` to say +something along the lines of: *"if the transition block is sufficiently old +enough, then we can just assume that block is honest or there exists an honest +justified chain to out-compete it."* + +Note the use of "feasibly" in the previous paragraph. One can imagine +mechanisms to check that a block is justified before importing it. For example, +just keep processing blocks without adding them to fork choice. However, there +are still edge-cases here (e.g., when to halt and declare there was no +justification?) and how to mitigate implemenation complexity. At this point, +it's important to reflect on the attack and how likely it is to happen. It +requires some rather contrived circumstances and it seems very unlikley to +occur. Therefore, we need to consider if adding complexity to avoid an +unlikely attack increases or decreases our total risk. Presently, it appears +that `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` sits in a sweet spot for this +trade-off. + +### Transitioning from VALID -> INVALID or INVALID -> VALID + +These operations are purposefully omitted. It is outside of the scope of the +specification since it's only possible with a faulty EE. + +Such a scenario requires manual intervention. + +## What about Light Clients? + +An alternative to optimistic sync is to run a light client inside/alongside +beacon nodes that mitigates the need for optimistic sync by providing +tip-of-chain blocks to the execution engine. However, light clients comes with +their own set of complexities. Relying on light clients may also restrict nodes +from syncing from genesis, if they so desire. + +A notable thing about optimistic sync is that it's *optional*. Should an +implementation decide to go the light-client route, then they can just ignore +optimistic sync all together. From 6d73b0a4ac30a8fae1ba909d1ebf628234cb41ca Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 13 Jan 2022 06:52:07 +1100 Subject: [PATCH 072/106] Add poisoning prevention --- sync/optimistic.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 6793b972d8..217b8bfd18 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -179,9 +179,10 @@ block that forks around the head (due to the justification of the malicious chain). If an honest chain exists which justifies a higher epoch than the malicious -chain, that chain will take precedence and revive any poisoned store. -Therefore, the poisoning attack is temporary if >= 2/3rds of the network is -honest and non-faulty. +chain, that chain will take precedence and revive any poisoned store. Such a +chain, if imported before the malicious chain, will prevent the store from +being poisoned. Therefore, the poisoning attack is temporary if >= 2/3rds of +the network is honest and non-faulty. The `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` parameter assumes that the network will justify a honest chain within some number of slots. With this assumption, From 0c2e416a6e9cc9a7fb8c8e30b07ea0ece77832ff Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 13 Jan 2022 06:56:32 +1100 Subject: [PATCH 073/106] Run doctoc --- specs/bellatrix/p2p-interface.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/bellatrix/p2p-interface.md b/specs/bellatrix/p2p-interface.md index 5a260d982b..964a77464a 100644 --- a/specs/bellatrix/p2p-interface.md +++ b/specs/bellatrix/p2p-interface.md @@ -29,6 +29,7 @@ Readers should understand the Phase 0 and Altair documents and use them as a bas - [Why was the max gossip message size increased at Bellatrix?](#why-was-the-max-gossip-message-size-increased-at-bellatrix) - [Req/Resp](#reqresp) - [Why was the max chunk response size increased at Bellatrix?](#why-was-the-max-chunk-response-size-increased-at-bellatrix) + - [Why allow invalid payloads on the P2P network?](#why-allow-invalid-payloads-on-the-p2p-network) From 6d72038f12583de772e6972f4f9b834b2508353e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 13 Jan 2022 06:56:38 +1100 Subject: [PATCH 074/106] Fix spelling mistakes --- sync/optimistic.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 217b8bfd18..1b413ea719 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -310,9 +310,9 @@ Note the use of "feasibly" in the previous paragraph. One can imagine mechanisms to check that a block is justified before importing it. For example, just keep processing blocks without adding them to fork choice. However, there are still edge-cases here (e.g., when to halt and declare there was no -justification?) and how to mitigate implemenation complexity. At this point, +justification?) and how to mitigate implementation complexity. At this point, it's important to reflect on the attack and how likely it is to happen. It -requires some rather contrived circumstances and it seems very unlikley to +requires some rather contrived circumstances and it seems very unlikely to occur. Therefore, we need to consider if adding complexity to avoid an unlikely attack increases or decreases our total risk. Presently, it appears that `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` sits in a sweet spot for this From 18c32e0a5e10e4b2c7170139ea1a7ba1acdf08a3 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 13 Jan 2022 07:00:28 +1100 Subject: [PATCH 075/106] Fix indents --- sync/optimistic.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 1b413ea719..826396a416 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -61,8 +61,8 @@ def is_execution_block(block: BeaconBlock) -> BeaconBlock: ```python def should_optimistically_import_block(store: Store, current_slot: Slot, block: BeaconBlock) -> bool: - justified_root = store.block_states[store.head_block_root].current_justified_checkpoint.root - justifed_is_verified = is_execution_block(store.blocks[justified_root]) + justified_root = store.block_states[store.head_block_root].current_justified_checkpoint.root + justifed_is_verified = is_execution_block(store.blocks[justified_root]) block_is_deep = block.slot + SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY <= current_slot return justified_is_verified or block_is_deep ``` From 6af3d4cbfc5b2e075e0441172277d08736e45b72 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 13 Jan 2022 07:13:10 +1100 Subject: [PATCH 076/106] Fix comment indents --- sync/optimistic.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 826396a416..ab539a64d0 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -47,7 +47,7 @@ def is_optimistic(store: Store, block: BeaconBlock) -> bool: ```python def latest_verified_ancestor(store: Store, block: BeaconBlock) -> BeaconBlock: - # It is assumed that the `block` parameter is never an INVALID block. + # It is assumed that the `block` parameter is never an INVALID block. while True: if not is_optimistic(store, block) or block.parent_root == Root(): return block From b7c332f0ee836da702c7a9e74afa8b8c285f8a20 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 13 Jan 2022 07:17:43 +1100 Subject: [PATCH 077/106] Tidy --- sync/optimistic.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index ab539a64d0..31f182ac48 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -20,7 +20,7 @@ blocks without verifying the execution payloads. This partial sync is called an ## Helpers -Let `head_block: BeaconBlock` be the result of calling of the fork choice +Let `head: BeaconBlock` be the result of calling of the fork choice algorithm at the time of block production. Let `optimistic_roots: Set[Root]` be the set of `hash_tree_root(block)` for all @@ -51,7 +51,7 @@ def latest_verified_ancestor(store: Store, block: BeaconBlock) -> BeaconBlock: while True: if not is_optimistic(store, block) or block.parent_root == Root(): return block - block = get_block(block.parent_root) + block = store.blocks[block.parent_root] ``` ```python @@ -67,7 +67,7 @@ def should_optimistically_import_block(store: Store, current_slot: Slot, block: return justified_is_verified or block_is_deep ``` -Let only a node which returns `is_optimistic(head) is True` be an *optimistic +Let only a node which returns `is_optimistic(store, head) is True` be an *optimistic node*. Let only a validator on an optimistic node be an *optimistic validator*. When this specification only defines behaviour for an optimistic @@ -254,7 +254,7 @@ When information about an optimistic block is requested, the consensus engine: ### Requests for an Optimistic Head -When `is_optimistic(head) is True`, the consensus engine: +When `is_optimistic(store, head) is True`, the consensus engine: - MUST NOT return an optimistic `head`. - MAY substitute the head block with `latest_verified_ancestor(block)`. @@ -262,7 +262,7 @@ When `is_optimistic(head) is True`, the consensus engine: ### Requests to Validators Endpoints -When `is_optimistic(head) is True`, the consensus engine MUST return syncing to +When `is_optimistic(store, head) is True`, the consensus engine MUST return syncing to all endpoints which match the following pattern: - `eth/*/validator/*` From 8522f27b8ecfcffd067f873c957f824808ab53e9 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 13 Jan 2022 07:17:57 +1100 Subject: [PATCH 078/106] Modify "when" section --- sync/optimistic.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 31f182ac48..5d7f0d1f40 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -78,13 +78,14 @@ behaviours without regard for optimistic sync. ### When to optimistically import blocks -A block MAY be optimistically imported when either of the following -conditions are met: - -1. The justified checkpoint has execution enabled. I.e., - `is_execution_block(get_block(get_state(head_block).current_justified_checkpoint.root))` -1. The current slot (as per the system clock) is at least `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` ahead of - the slot of the block being imported. I.e., `should_optimistically_import_block(current_slot, block) == True`. +A block MAY be optimistically imported when +`should_optimistically_import_block(store, current_slot, block)` returns +`True`. This ensures that blocks are only optimistically imported if either: + +1. The justified checkpoint has execution enabled. +1. The current slot (as per the system clock) is at least + `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` ahead of the slot of the block being + imported. *See [Fork Choice Poisoning](#fork-choice-poisoning) for the motivations behind these conditions.* From 856eea4986f9b20d98132fc0d91898d68b31aac3 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 13 Jan 2022 07:23:12 +1100 Subject: [PATCH 079/106] Describe all fields in store --- sync/optimistic.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 5d7f0d1f40..cb45b70365 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -21,11 +21,12 @@ blocks without verifying the execution payloads. This partial sync is called an ## Helpers Let `head: BeaconBlock` be the result of calling of the fork choice -algorithm at the time of block production. +algorithm at the time of block production. Let `head_block_root: Root` be the +root of that block. -Let `optimistic_roots: Set[Root]` be the set of `hash_tree_root(block)` for all -optimistically imported blocks which have yet to receive an `INVALID` or -`VALID` designation from an execution engine. +Let `blocks: Dict[Root, BeaconBlock]` and `block_states: Dict[Root, +BeaconState]` be the blocks (and accompanying states) that have been verified +either completely or optimistically. Let `optimistic_roots: Set[Root]` be the set of `hash_tree_root(block)` for all optimistically imported blocks which have only received a `SYNCING` designation From 7255faf41b0e1aa3dc1bd12a823a98b459a45a70 Mon Sep 17 00:00:00 2001 From: Antonio Sanso Date: Thu, 13 Jan 2022 13:40:48 +0100 Subject: [PATCH 080/106] Update ssz_container.py --- tests/generators/ssz_generic/ssz_container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generators/ssz_generic/ssz_container.py b/tests/generators/ssz_generic/ssz_container.py index b933995229..1b30d687ac 100644 --- a/tests/generators/ssz_generic/ssz_container.py +++ b/tests/generators/ssz_generic/ssz_container.py @@ -104,7 +104,7 @@ def invalid_cases(): RandomizationMode.mode_nil_count, RandomizationMode.mode_one_count, RandomizationMode.mode_max_count]: - for offset_index in offsets: + for index, offset_index in enumerate(offsets): yield f'{name}_{mode.to_name()}_offset_{offset_index}_plus_one', \ invalid_test_case(lambda: mod_offset( b=serialize(container_case_fn(rng, mode, typ)), From fd27d938986ed79017c62f9ddb842f147c56234e Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 13 Jan 2022 11:20:30 -0800 Subject: [PATCH 081/106] add `pylint` to catch unused args --- Makefile | 2 ++ setup.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ca0096fb9e..ec3302e27b 100644 --- a/Makefile +++ b/Makefile @@ -130,9 +130,11 @@ codespell: codespell . --skip ./.git -I .codespell-whitelist # TODO: add future protocol upgrade patch packages to linting. +# NOTE: we use `pylint` just for catching unused arguments in spec code lint: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ flake8 --config $(LINTER_CONFIG_FILE) ./eth2spec \ + && pylint --disable=all --enable unused-argument ./eth2spec/phase0 ./eth2spec/altair ./eth2spec/bellatrix \ && mypy --config-file $(LINTER_CONFIG_FILE) -p eth2spec.phase0 -p eth2spec.altair -p eth2spec.bellatrix lint_generators: pyspec diff --git a/setup.py b/setup.py index aa4157b292..625b6f5724 100644 --- a/setup.py +++ b/setup.py @@ -1008,7 +1008,7 @@ def run(self): python_requires=">=3.8, <4", extras_require={ "test": ["pytest>=4.4", "pytest-cov", "pytest-xdist"], - "lint": ["flake8==3.7.7", "mypy==0.812"], + "lint": ["flake8==3.7.7", "mypy==0.812", "pylint==2.12.2"], "generator": ["python-snappy==0.5.4"], }, install_requires=[ From 2b45496fe48fa75450ad29a05bdd48866f86528a Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 13 Jan 2022 11:31:27 -0800 Subject: [PATCH 082/106] clean up unused argument from `phase0` --- specs/altair/beacon-chain.md | 2 +- specs/phase0/beacon-chain.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index 2e897deb7d..a14ddb4f61 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -521,7 +521,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: signing_root = compute_signing_root(deposit_message, domain) # Initialize validator if the deposit signature is valid if bls.Verify(pubkey, signing_root, deposit.data.signature): - state.validators.append(get_validator_from_deposit(state, deposit)) + state.validators.append(get_validator_from_deposit(deposit)) state.balances.append(amount) state.previous_epoch_participation.append(ParticipationFlags(0b0000_0000)) state.current_epoch_participation.append(ParticipationFlags(0b0000_0000)) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 7172307c60..618f396374 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1832,7 +1832,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: ##### Deposits ```python -def get_validator_from_deposit(state: BeaconState, deposit: Deposit) -> Validator: +def get_validator_from_deposit(deposit: Deposit) -> Validator: amount = deposit.data.amount effective_balance = min(amount - amount % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) @@ -1877,7 +1877,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: return # Add validator and balance entries - state.validators.append(get_validator_from_deposit(state, deposit)) + state.validators.append(get_validator_from_deposit(deposit)) state.balances.append(amount) else: # Increase balance by deposit amount From 10603b71c073c43b6aa85ff0983eca0bc315708c Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 13 Jan 2022 11:31:45 -0800 Subject: [PATCH 083/106] indicate unused argument in utility function --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 625b6f5724..f119514151 100644 --- a/setup.py +++ b/setup.py @@ -514,7 +514,7 @@ def get_pow_block(hash: Bytes32) -> Optional[PowBlock]: return PowBlock(block_hash=hash, parent_hash=Bytes32(), total_difficulty=uint256(0)) -def get_execution_state(execution_state_root: Bytes32) -> ExecutionState: +def get_execution_state(_execution_state_root: Bytes32) -> ExecutionState: pass From fd89b72c6b53e7191870ea870550a90a63dc371f Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Mon, 17 Jan 2022 12:56:51 +0100 Subject: [PATCH 084/106] `doctoc` on `custody_game` specs This updates the TOC in `custody_game/beacon-chain.md` using `make doctoc specs`. --- specs/custody_game/beacon-chain.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/custody_game/beacon-chain.md b/specs/custody_game/beacon-chain.md index 6f9f61cf92..fd00deddaf 100644 --- a/specs/custody_game/beacon-chain.md +++ b/specs/custody_game/beacon-chain.md @@ -11,7 +11,8 @@ - [Introduction](#introduction) - [Constants](#constants) - [Misc](#misc) -- [Configuration](#configuration) + - [Domain types](#domain-types) +- [Preset](#preset) - [Time parameters](#time-parameters) - [Max operations per block](#max-operations-per-block) - [Size parameters](#size-parameters) From 671c4f0e3a87cb52e90d57699df1f54942378f4d Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 17 Jan 2022 11:05:01 -0700 Subject: [PATCH 085/106] receipt_root -> receipts_root --- specs/bellatrix/beacon-chain.md | 6 +++--- .../core/pyspec/eth2spec/test/helpers/execution_payload.py | 4 ++-- tests/core/pyspec/eth2spec/test/helpers/genesis.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/bellatrix/beacon-chain.md b/specs/bellatrix/beacon-chain.md index 738f03556d..ea737e233a 100644 --- a/specs/bellatrix/beacon-chain.md +++ b/specs/bellatrix/beacon-chain.md @@ -169,7 +169,7 @@ class ExecutionPayload(Container): parent_hash: Hash32 fee_recipient: ExecutionAddress # 'beneficiary' in the yellow paper state_root: Bytes32 - receipt_root: Bytes32 # 'receipts root' in the yellow paper + receipts_root: Bytes32 # 'receipts root' in the yellow paper logs_bloom: ByteVector[BYTES_PER_LOGS_BLOOM] random: Bytes32 # 'difficulty' in the yellow paper block_number: uint64 # 'number' in the yellow paper @@ -191,7 +191,7 @@ class ExecutionPayloadHeader(Container): parent_hash: Hash32 fee_recipient: ExecutionAddress state_root: Bytes32 - receipt_root: Bytes32 + receipts_root: Bytes32 logs_bloom: ByteVector[BYTES_PER_LOGS_BLOOM] random: Bytes32 block_number: uint64 @@ -359,7 +359,7 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe parent_hash=payload.parent_hash, fee_recipient=payload.fee_recipient, state_root=payload.state_root, - receipt_root=payload.receipt_root, + receipts_root=payload.receipts_root, logs_bloom=payload.logs_bloom, random=payload.random, block_number=payload.block_number, diff --git a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py index 9c9663584b..49cec884a5 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py @@ -13,7 +13,7 @@ def build_empty_execution_payload(spec, state, randao_mix=None): parent_hash=latest.block_hash, fee_recipient=spec.ExecutionAddress(), state_root=latest.state_root, # no changes to the state - receipt_root=b"no receipts here" + b"\x00" * 16, # TODO: root of empty MPT may be better. + receipts_root=b"no receipts here" + b"\x00" * 16, # TODO: root of empty MPT may be better. logs_bloom=spec.ByteVector[spec.BYTES_PER_LOGS_BLOOM](), # TODO: zeroed logs bloom for empty logs ok? block_number=latest.block_number + 1, random=randao_mix, @@ -36,7 +36,7 @@ def get_execution_payload_header(spec, execution_payload): parent_hash=execution_payload.parent_hash, fee_recipient=execution_payload.fee_recipient, state_root=execution_payload.state_root, - receipt_root=execution_payload.receipt_root, + receipts_root=execution_payload.receipts_root, logs_bloom=execution_payload.logs_bloom, random=execution_payload.random, block_number=execution_payload.block_number, diff --git a/tests/core/pyspec/eth2spec/test/helpers/genesis.py b/tests/core/pyspec/eth2spec/test/helpers/genesis.py index 5ebe46c0e2..0539a5d74c 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/genesis.py +++ b/tests/core/pyspec/eth2spec/test/helpers/genesis.py @@ -28,7 +28,7 @@ def get_sample_genesis_execution_payload_header(spec, parent_hash=b'\x30' * 32, fee_recipient=b'\x42' * 20, state_root=b'\x20' * 32, - receipt_root=b'\x20' * 32, + receipts_root=b'\x20' * 32, logs_bloom=b'\x35' * spec.BYTES_PER_LOGS_BLOOM, random=eth1_block_hash, block_number=0, From 15ef2f30d2dfd7a19f30d99df76c646ce07085ac Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 18 Jan 2022 09:30:01 +1100 Subject: [PATCH 086/106] Apply suggestions from @djrtwo review Co-authored-by: Danny Ryan --- specs/bellatrix/p2p-interface.md | 6 +++--- sync/optimistic.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/bellatrix/p2p-interface.md b/specs/bellatrix/p2p-interface.md index 964a77464a..2f07068fec 100644 --- a/specs/bellatrix/p2p-interface.md +++ b/specs/bellatrix/p2p-interface.md @@ -202,7 +202,7 @@ always by simultaneously respected. ### Why allow invalid payloads on the P2P network? -The specification allows blocks with invalid payloads to propagate across +The specification allows blocks with invalid execution payloads to propagate across gossip and via RPC calls. The reasoning for this is as follows: 1. Optimistic nodes must listen to block gossip to obtain a view of the head of @@ -213,8 +213,8 @@ gossip and via RPC calls. The reasoning for this is as follows: to requests for the parent via RPC. 4. Therefore, optimistic nodes must send optimistic blocks via RPC. -So, to prevent network segregation from optimistic nodes accidentally sending -invalid payloads, nodes should never downscore/disconnect nodes due to invalid +So, to prevent network segregation from optimistic nodes inadvertently sending +invalid execution payloads, nodes should never downscore/disconnect nodes due to such invalid payloads. This does open the network to some DoS attacks from invalid execution payloads, but the scope of actors is limited to validators who can put those payloads in valid (and slashable) beacon blocks. Therefore, it is argued that diff --git a/sync/optimistic.md b/sync/optimistic.md index cb45b70365..88dc284441 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -116,16 +116,16 @@ from an execution engine. I.e., perform the following transitions: - `SYNCING` -> `INVALID` When a block transitions from `SYNCING` -> `VALID`, all *ancestors* of the -block MUST also transition from `SYNCING` -> `VALID`. Such a block is no longer +block MUST also transition from `SYNCING` -> `VALID`. Such a block and any previously `SYNCING` ancestors are no longer considered "optimistically imported". When a block transitions from `SYNCING` -> `INVALID`, all *descendants* of the block MUST also transition from `SYNCING` -> `INVALID`. -When a block transitions from the `SYNCING` state it is removed from the set of +When a block transitions from the `SYNCING` state, it is removed from the set of `store.optimistic_roots`. -When a "merge block" (i.e. a block which enables execution) is declared to be +When a "merge block" (i.e. the first block which enables execution in a chain) is declared to be `VALID` by an execution engine (either directly or indirectly), the full [`validate_merge_block`][] MUST be run against the merge block. If the block fails [`validate_merge_block`][], the merge block MUST be treated the same as From b1ec9bcfbc395549f3871a4d80960a5ddd27322d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 18 Jan 2022 11:12:55 +1100 Subject: [PATCH 087/106] Update gossip conditions --- specs/bellatrix/p2p-interface.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/specs/bellatrix/p2p-interface.md b/specs/bellatrix/p2p-interface.md index 2f07068fec..1d790bce72 100644 --- a/specs/bellatrix/p2p-interface.md +++ b/specs/bellatrix/p2p-interface.md @@ -95,12 +95,14 @@ the following validations MUST pass before forwarding the `signed_beacon_block` Alias `block = signed_beacon_block.message`, `execution_payload = block.body.execution_payload`. - If the execution is enabled for the block -- i.e. `is_execution_enabled(state, block.body)` then validate the following: - - _[REJECT]_ The block's execution payload timestamp is correct with respect to the slot - -- i.e. `execution_payload.timestamp == compute_timestamp_at_slot(state, block.slot)`. - - [REJECT] The block's parent (defined by `block.parent_root`) passes all - validation, excluding verification of the `block.body.execution_payload`. - - [IGNORE] The block's parent (defined by `block.parent_root`) passes all - validation, including verification of the `block.body.execution_payload`. + - _[REJECT]_ The block's execution payload timestamp is correct with respect to the slot + -- i.e. `execution_payload.timestamp == compute_timestamp_at_slot(state, block.slot)`. + - If `exection_payload` verification of block's parent by an execution node is *not* complete: + - [REJECT] The block's parent (defined by `block.parent_root`) passes all + validation (excluding execution node verification of the `block.body.execution_payload`). + - otherwise: + - [IGNORE] The block's parent (defined by `block.parent_root`) passes all + validation (including execution node verification of the `block.body.execution_payload`). The following gossip validation from prior specifications MUST NOT be applied if the execution is enabled for the block -- i.e. `is_execution_enabled(state, block.body)`: - [REJECT] The block's parent (defined by `block.parent_root`) passes validation. From 6225236a52df6017f139127deec806a069cbabad Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 18 Jan 2022 11:20:52 +1100 Subject: [PATCH 088/106] Specify about EL/CL scoring rules --- specs/bellatrix/p2p-interface.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/bellatrix/p2p-interface.md b/specs/bellatrix/p2p-interface.md index 1d790bce72..4f3a709907 100644 --- a/specs/bellatrix/p2p-interface.md +++ b/specs/bellatrix/p2p-interface.md @@ -117,8 +117,10 @@ details on how to handle transitioning gossip topics for Bellatrix. Non-faulty, [optimistic](/sync/optimistic.md) nodes may send blocks which result in an INVALID response from an execution engine. To prevent network segregation between optimistic and non-optimistic nodes, transmission of an -INVALID payload via the Req/Resp domain SHOULD NOT cause a node to be -down-scored or disconnected. +INVALID execution payload via the Req/Resp domain SHOULD NOT cause a node to be +down-scored or disconnected. Transmission of a block which is invalid due to +any consensus layer rules (i.e., *not* execution layer rules) MAY result in +down-scoring or disconnection. ### Messages From 092f3e0b167d6ad65d7e83ea75956c276206700a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 18 Jan 2022 11:22:07 +1100 Subject: [PATCH 089/106] Propose -> Propagate --- specs/bellatrix/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/bellatrix/p2p-interface.md b/specs/bellatrix/p2p-interface.md index 4f3a709907..60a9be7746 100644 --- a/specs/bellatrix/p2p-interface.md +++ b/specs/bellatrix/p2p-interface.md @@ -213,7 +213,7 @@ gossip and via RPC calls. The reasoning for this is as follows: the chain. 2. Therefore, optimistic nodes must propagate gossip blocks. Otherwise, they'd be censoring. -3. If optimistic nodes will propose blocks via gossip, then they must respond +3. If optimistic nodes will propagate blocks via gossip, then they must respond to requests for the parent via RPC. 4. Therefore, optimistic nodes must send optimistic blocks via RPC. From 52caba6eae98169549b78b3e5f5219f79c0db740 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 18 Jan 2022 11:28:08 +1100 Subject: [PATCH 090/106] Add section about backwards compat --- sync/optimistic.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index 88dc284441..d794269a28 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -9,6 +9,13 @@ of the chain, it may be desirable for a consensus engine to import beacon blocks without verifying the execution payloads. This partial sync is called an *optimistic sync*. +Optimistic sync is designed to be opt-in and backwards compatible (i.e., +non-optimistic nodes can tolerate optimistic nodes on the network and vice +versa). Optimistic sync is not a fundamental requirement for consensus nodes. +Rather, it's a stop-gap measure to allow execution nodes to sync via +established methods until future Ethereum roadmap items are implemented (e.g., +statelessness). + ## Constants |Name|Value|Unit From b50bee16f6ca45ed535c608f556983622cb2c567 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 18 Jan 2022 11:30:28 +1100 Subject: [PATCH 091/106] Rename Store -> OptimisticStore --- sync/optimistic.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index d794269a28..3aaa2688be 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -41,7 +41,7 @@ from an execution engine (i.e., they are not known to be `INVALID` or `VALID`). ```python @dataclass -class Store(object): +class OptimisticStore(object): optimistic_roots: Set[Root] head_block_root: Root blocks: Dict[Root, BeaconBlock] @@ -49,17 +49,17 @@ class Store(object): ``` ```python -def is_optimistic(store: Store, block: BeaconBlock) -> bool: - return hash_tree_root(block) in store.optimistic_roots +def is_optimistic(opt_store: OptimisticStore, block: BeaconBlock) -> bool: + return hash_tree_root(block) in opt_store.optimistic_roots ``` ```python -def latest_verified_ancestor(store: Store, block: BeaconBlock) -> BeaconBlock: +def latest_verified_ancestor(opt_store: OptimisticStore, block: BeaconBlock) -> BeaconBlock: # It is assumed that the `block` parameter is never an INVALID block. while True: - if not is_optimistic(store, block) or block.parent_root == Root(): + if not is_optimistic(opt_store, block) or block.parent_root == Root(): return block - block = store.blocks[block.parent_root] + block = opt_store.blocks[block.parent_root] ``` ```python @@ -68,14 +68,14 @@ def is_execution_block(block: BeaconBlock) -> BeaconBlock: ``` ```python -def should_optimistically_import_block(store: Store, current_slot: Slot, block: BeaconBlock) -> bool: - justified_root = store.block_states[store.head_block_root].current_justified_checkpoint.root - justifed_is_verified = is_execution_block(store.blocks[justified_root]) +def should_optimistically_import_block(opt_store: OptimisticStore, current_slot: Slot, block: BeaconBlock) -> bool: + justified_root = opt_store.block_states[opt_store.head_block_root].current_justified_checkpoint.root + justifed_is_verified = is_execution_block(opt_store.blocks[justified_root]) block_is_deep = block.slot + SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY <= current_slot return justified_is_verified or block_is_deep ``` -Let only a node which returns `is_optimistic(store, head) is True` be an *optimistic +Let only a node which returns `is_optimistic(opt_store, head) is True` be an *optimistic node*. Let only a validator on an optimistic node be an *optimistic validator*. When this specification only defines behaviour for an optimistic @@ -87,7 +87,7 @@ behaviours without regard for optimistic sync. ### When to optimistically import blocks A block MAY be optimistically imported when -`should_optimistically_import_block(store, current_slot, block)` returns +`should_optimistically_import_block(opt_store, current_slot, block)` returns `True`. This ensures that blocks are only optimistically imported if either: 1. The justified checkpoint has execution enabled. @@ -130,7 +130,7 @@ When a block transitions from `SYNCING` -> `INVALID`, all *descendants* of the block MUST also transition from `SYNCING` -> `INVALID`. When a block transitions from the `SYNCING` state, it is removed from the set of -`store.optimistic_roots`. +`opt_store.optimistic_roots`. When a "merge block" (i.e. the first block which enables execution in a chain) is declared to be `VALID` by an execution engine (either directly or indirectly), the full @@ -263,7 +263,7 @@ When information about an optimistic block is requested, the consensus engine: ### Requests for an Optimistic Head -When `is_optimistic(store, head) is True`, the consensus engine: +When `is_optimistic(opt_store, head) is True`, the consensus engine: - MUST NOT return an optimistic `head`. - MAY substitute the head block with `latest_verified_ancestor(block)`. @@ -271,7 +271,7 @@ When `is_optimistic(store, head) is True`, the consensus engine: ### Requests to Validators Endpoints -When `is_optimistic(store, head) is True`, the consensus engine MUST return syncing to +When `is_optimistic(opt_store, head) is True`, the consensus engine MUST return syncing to all endpoints which match the following pattern: - `eth/*/validator/*` From 4ccd38b54e3a2a19b159ff4c1383bd2f820156c9 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 18 Jan 2022 11:32:47 +1100 Subject: [PATCH 092/106] Tidy tracking note --- sync/optimistic.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 3aaa2688be..dd9d428d48 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -108,9 +108,8 @@ To optimistically import a block: `pow_block` and `pow_parent` are unknown to the execution engine. - The parent of the block MUST NOT have an INVALID execution payload. -In addition to this change to validation, the consensus engine MUST be able to -ascertain, after import, which blocks returned `SYNCING` and which returned -`VALID`. +In addition to this change in validation, the consensus engine MUST track which +blocks returned `SYNCING` and which returned `VALID` for subsequent processing. Optimistically imported blocks MUST pass all verifications included in `process_block` (withstanding the modifications to `execute_payload`). From f4a125c89aaf1523322ff175ba32c169e0dbf493 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 18 Jan 2022 11:36:36 +1100 Subject: [PATCH 093/106] Remove reorg section --- sync/optimistic.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index dd9d428d48..67f2e3ee51 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -159,8 +159,7 @@ another chain. ### Re-Orgs The consensus engine MUST support any chain reorganisation which does *not* -affect the justified checkpoint. The consensus engine MAY support re-orgs -beyond the justified checkpoint. +affect the justified checkpoint. If the justified checkpoint transitions from `SYNCING` -> `INVALID`, a consensus engine MAY choose to alert the user and force the application to From 0ec61bd195a308a81e2bf221346394284801c42a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 18 Jan 2022 12:21:34 +1100 Subject: [PATCH 094/106] Clarify liveness --- sync/optimistic.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 67f2e3ee51..66ec8c3127 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -293,7 +293,9 @@ Thankfully, if 2/3rds of validators are not poisoned, they can justify an honest chain which will un-poison all other nodes. Notably, this attack only exists for optimistic nodes. Nodes which fully verify -the transition block will reject a block with a junk parent hash. +the transition block will reject a block with a junk parent hash. Therefore, +liveness is unaffected if a vast majority of nodes have fully synced execution +and consensus clients before and during the transition. Given all of this, we can say two things: From bfe4172584b35a881020b01e7f781ad0f91e8011 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 18 Jan 2022 12:26:30 +1100 Subject: [PATCH 095/106] Define `current_slot` --- sync/optimistic.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index 66ec8c3127..a66408b93e 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -39,6 +39,9 @@ Let `optimistic_roots: Set[Root]` be the set of `hash_tree_root(block)` for all optimistically imported blocks which have only received a `SYNCING` designation from an execution engine (i.e., they are not known to be `INVALID` or `VALID`). +Let `current_slot: Slot` be `(time - genesis_time) // SECONDS_PER_SLOT` where +`time` is the UNIX time according to the local system clock. + ```python @dataclass class OptimisticStore(object): From 24947be7c3446c142aa767fdbbb57b412a008b86 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 18 Jan 2022 12:29:20 +1100 Subject: [PATCH 096/106] Rename should_optimistically_import... --- sync/optimistic.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index a66408b93e..257574a62f 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -71,7 +71,7 @@ def is_execution_block(block: BeaconBlock) -> BeaconBlock: ``` ```python -def should_optimistically_import_block(opt_store: OptimisticStore, current_slot: Slot, block: BeaconBlock) -> bool: +def is_optimistic_candidate_block(opt_store: OptimisticStore, current_slot: Slot, block: BeaconBlock) -> bool: justified_root = opt_store.block_states[opt_store.head_block_root].current_justified_checkpoint.root justifed_is_verified = is_execution_block(opt_store.blocks[justified_root]) block_is_deep = block.slot + SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY <= current_slot @@ -90,7 +90,7 @@ behaviours without regard for optimistic sync. ### When to optimistically import blocks A block MAY be optimistically imported when -`should_optimistically_import_block(opt_store, current_slot, block)` returns +`is_optimistic_candidate_block(opt_store, current_slot, block)` returns `True`. This ensures that blocks are only optimistically imported if either: 1. The justified checkpoint has execution enabled. From be4319a048926aeecb53d707488612b971bccde7 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 18 Jan 2022 12:31:36 +1100 Subject: [PATCH 097/106] Rename `justified_is_verified` --- sync/optimistic.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 257574a62f..44d1fb6169 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -73,9 +73,9 @@ def is_execution_block(block: BeaconBlock) -> BeaconBlock: ```python def is_optimistic_candidate_block(opt_store: OptimisticStore, current_slot: Slot, block: BeaconBlock) -> bool: justified_root = opt_store.block_states[opt_store.head_block_root].current_justified_checkpoint.root - justifed_is_verified = is_execution_block(opt_store.blocks[justified_root]) + justifed_is_execution_block = is_execution_block(opt_store.blocks[justified_root]) block_is_deep = block.slot + SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY <= current_slot - return justified_is_verified or block_is_deep + return justifed_is_execution_block or block_is_deep ``` Let only a node which returns `is_optimistic(opt_store, head) is True` be an *optimistic From 50b236e5ebbae2eecf60b867442a3df83a88d056 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 18 Jan 2022 14:55:10 +1100 Subject: [PATCH 098/106] Address TERMINAL_BLOCK_HASH --- sync/optimistic.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index 44d1fb6169..01765112cd 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -109,6 +109,7 @@ To optimistically import a block: engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. - The [`validate_merge_block`][] function MUST NOT raise an assertion if both the `pow_block` and `pow_parent` are unknown to the execution engine. + - All other assertions in [`validate_merge_block`][] (e.g., `TERMINAL_BLOCK_HASH`) MUST prevent an optimistic import. - The parent of the block MUST NOT have an INVALID execution payload. In addition to this change in validation, the consensus engine MUST track which @@ -348,3 +349,13 @@ from syncing from genesis, if they so desire. A notable thing about optimistic sync is that it's *optional*. Should an implementation decide to go the light-client route, then they can just ignore optimistic sync all together. + +## What if `TERMINAL_BLOCK_HASH` is used? + +If the terminal block hash override is used (i.e., `TERMINAL_BLOCK_HASH != +Hash32()`), the [`validate_merge_block`][] function will deterministically +return `True` or `False`. Whilst it's not *technically* required +retrospectively call [`validate_merge_block`][] on a transition block that +matches `TERMINAL_BLOCK_HASH` after an optimistic sync, doing so will have no +effect. For simplicity, the optimistic sync specification does not define +edge-case behaviour for when `TERMINAL_BLOCK_HASH` is used. From f77fc055f8a34e645d9ad183245952bc2c37540d Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 18 Jan 2022 07:19:00 -0700 Subject: [PATCH 099/106] Update specs/bellatrix/beacon-chain.md Co-authored-by: Hsiao-Wei Wang --- specs/bellatrix/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/bellatrix/beacon-chain.md b/specs/bellatrix/beacon-chain.md index ea737e233a..c7bd7af693 100644 --- a/specs/bellatrix/beacon-chain.md +++ b/specs/bellatrix/beacon-chain.md @@ -169,7 +169,7 @@ class ExecutionPayload(Container): parent_hash: Hash32 fee_recipient: ExecutionAddress # 'beneficiary' in the yellow paper state_root: Bytes32 - receipts_root: Bytes32 # 'receipts root' in the yellow paper + receipts_root: Bytes32 logs_bloom: ByteVector[BYTES_PER_LOGS_BLOOM] random: Bytes32 # 'difficulty' in the yellow paper block_number: uint64 # 'number' in the yellow paper From a5b3c91f250a8d6a057c3f4021c41199d41e4c21 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 20 Jan 2022 09:25:02 -0700 Subject: [PATCH 100/106] build opimistic sync file and fix a minor lint/typing issue --- setup.py | 1 + sync/optimistic.md | 25 ++++++++++++++++--------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index aa4157b292..888c16142e 100644 --- a/setup.py +++ b/setup.py @@ -868,6 +868,7 @@ def finalize_options(self): specs/bellatrix/fork.md specs/bellatrix/fork-choice.md specs/bellatrix/validator.md + sync/optimistic.md """ if len(self.md_doc_paths) == 0: raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork) diff --git a/sync/optimistic.md b/sync/optimistic.md index 01765112cd..80e9de3e90 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -1,7 +1,8 @@ # Optimistic Sync -[`validate_merge_block`]: ../specs/bellatrix/fork-choice.md#validate_merge_block + +../specs/bellatrix/fork-choice.md#validate_merge_block ## Introduction In order to provide a syncing execution engine with a partial view of the head @@ -66,7 +67,7 @@ def latest_verified_ancestor(opt_store: OptimisticStore, block: BeaconBlock) -> ``` ```python -def is_execution_block(block: BeaconBlock) -> BeaconBlock: +def is_execution_block(block: BeaconBlock) -> bool: return block.body.execution_payload != ExecutionPayload() ``` @@ -106,10 +107,12 @@ these conditions.* To optimistically import a block: - The [`execute_payload`](../specs/bellatrix/beacon-chain.md#execute_payload) function MUST return `True` if the execution - engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. -- The [`validate_merge_block`][] function MUST NOT raise an assertion if both the + engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. +- The [`validate_merge_block`](../specs/bellatrix/fork-choice.md#validate_merge_block) + function MUST NOT raise an assertion if both the `pow_block` and `pow_parent` are unknown to the execution engine. - - All other assertions in [`validate_merge_block`][] (e.g., `TERMINAL_BLOCK_HASH`) MUST prevent an optimistic import. + - All other assertions in [`validate_merge_block`](../specs/bellatrix/fork-choice.md#validate_merge_block) + (e.g., `TERMINAL_BLOCK_HASH`) MUST prevent an optimistic import. - The parent of the block MUST NOT have an INVALID execution payload. In addition to this change in validation, the consensus engine MUST track which @@ -137,8 +140,10 @@ When a block transitions from the `SYNCING` state, it is removed from the set of When a "merge block" (i.e. the first block which enables execution in a chain) is declared to be `VALID` by an execution engine (either directly or indirectly), the full -[`validate_merge_block`][] MUST be run against the merge block. If the block -fails [`validate_merge_block`][], the merge block MUST be treated the same as +[`validate_merge_block`](../specs/bellatrix/fork-choice.md#validate_merge_block) +MUST be run against the merge block. If the block +fails [`validate_merge_block`](../specs/bellatrix/fork-choice.md#validate_merge_block), +the merge block MUST be treated the same as an `INVALID` block (i.e., it and all its descendants are invalidated and removed from the block tree). @@ -353,9 +358,11 @@ optimistic sync all together. ## What if `TERMINAL_BLOCK_HASH` is used? If the terminal block hash override is used (i.e., `TERMINAL_BLOCK_HASH != -Hash32()`), the [`validate_merge_block`][] function will deterministically +Hash32()`), the [`validate_merge_block`](../specs/bellatrix/fork-choice.md#validate_merge_block) +function will deterministically return `True` or `False`. Whilst it's not *technically* required -retrospectively call [`validate_merge_block`][] on a transition block that +retrospectively call [`validate_merge_block`](../specs/bellatrix/fork-choice.md#validate_merge_block) +on a transition block that matches `TERMINAL_BLOCK_HASH` after an optimistic sync, doing so will have no effect. For simplicity, the optimistic sync specification does not define edge-case behaviour for when `TERMINAL_BLOCK_HASH` is used. From 182e4496c3f4a05e20bd422d0f02df7bcbefea4a Mon Sep 17 00:00:00 2001 From: Dustin Brody Date: Fri, 21 Jan 2022 19:01:56 +0000 Subject: [PATCH 101/106] use INTERVALS_PER_SLOT in sync committee specs --- specs/altair/validator.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/altair/validator.md b/specs/altair/validator.md index 461c9a70db..c59aa29f71 100644 --- a/specs/altair/validator.md +++ b/specs/altair/validator.md @@ -268,7 +268,7 @@ This process occurs each slot. If a validator is in the current sync committee (i.e. `is_assigned_to_sync_committee()` above returns `True`), then for every `slot` in the current sync committee period, the validator should prepare a `SyncCommitteeMessage` for the previous slot (`slot - 1`) according to the logic in `get_sync_committee_message` as soon as they have determined the head block of `slot - 1`. This means that when assigned to `slot` a `SyncCommitteeMessage` is prepared and broadcast in `slot-1 ` instead of `slot`. This logic is triggered upon the same conditions as when producing an attestation. -Meaning, a sync committee member should produce and broadcast a `SyncCommitteeMessage` either when (a) the validator has received a valid block from the expected block proposer for the current `slot` or (b) one-third of the slot has transpired (`SECONDS_PER_SLOT / 3` seconds after the start of the slot) -- whichever comes first. +Meaning, a sync committee member should produce and broadcast a `SyncCommitteeMessage` either when (a) the validator has received a valid block from the expected block proposer for the current `slot` or (b) one-third of the slot has transpired (`SECONDS_PER_SLOT / INTERVALS_PER_SLOT` seconds after the start of the slot) -- whichever comes first. `get_sync_committee_message(state, block_root, validator_index, privkey)` assumes the parameter `state` is the head state corresponding to processing the block up to the current slot as determined by the fork choice (including any empty slots up to the current slot processed with `process_slots` on top of the latest block), `block_root` is the root of the head block, `validator_index` is the index of the validator in the registry `state.validators` controlled by `privkey`, and `privkey` is the BLS private key for the validator. @@ -385,7 +385,7 @@ The collection of input signatures should include one signature per validator wh ##### Broadcast sync committee contribution -If the validator is selected to aggregate (`is_sync_committee_aggregator()`), then they broadcast their best aggregate as a `SignedContributionAndProof` to the global aggregate channel (`sync_committee_contribution_and_proof` topic) two-thirds of the way through the `slot`-that is, `SECONDS_PER_SLOT * 2 / 3` seconds after the start of `slot`. +If the validator is selected to aggregate (`is_sync_committee_aggregator()`), then they broadcast their best aggregate as a `SignedContributionAndProof` to the global aggregate channel (`sync_committee_contribution_and_proof` topic) two-thirds of the way through the `slot`-that is, `SECONDS_PER_SLOT * 2 / INTERVALS_PER_SLOT` seconds after the start of `slot`. Selection proofs are provided in `ContributionAndProof` to prove to the gossip channel that the validator has been selected as an aggregator. From eb4497fae71ea350530a8370601e54e603a150b0 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Thu, 27 Jan 2022 15:24:53 +0600 Subject: [PATCH 102/106] Bellatrix: Rename execute_payload to notify_new_payload --- setup.py | 2 +- specs/bellatrix/beacon-chain.md | 12 ++++++------ sync/optimistic.md | 6 +++--- .../test_process_execution_payload.py | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/setup.py b/setup.py index ac60eb3b75..f826968f21 100644 --- a/setup.py +++ b/setup.py @@ -524,7 +524,7 @@ def get_pow_chain_head() -> PowBlock: class NoopExecutionEngine(ExecutionEngine): - def execute_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: + def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: return True def notify_forkchoice_updated(self: ExecutionEngine, diff --git a/specs/bellatrix/beacon-chain.md b/specs/bellatrix/beacon-chain.md index c7bd7af693..63bad7f083 100644 --- a/specs/bellatrix/beacon-chain.md +++ b/specs/bellatrix/beacon-chain.md @@ -35,7 +35,7 @@ - [Modified `slash_validator`](#modified-slash_validator) - [Beacon chain state transition function](#beacon-chain-state-transition-function) - [Execution engine](#execution-engine) - - [`execute_payload`](#execute_payload) + - [`notify_new_payload`](#notify_new_payload) - [Block processing](#block-processing) - [Execution payload](#execution-payload) - [`process_execution_payload`](#process_execution_payload) @@ -307,17 +307,17 @@ def slash_validator(state: BeaconState, The implementation-dependent `ExecutionEngine` protocol encapsulates the execution sub-system logic via: * a state object `self.execution_state` of type `ExecutionState` -* a state transition function `self.execute_payload` which applies changes to the `self.execution_state` +* a notification function `self.notify_new_payload` which may apply changes to the `self.execution_state` -*Note*: `execute_payload` is a function accessed through the `EXECUTION_ENGINE` module which instantiates the `ExecutionEngine` protocol. +*Note*: `notify_new_payload` is a function accessed through the `EXECUTION_ENGINE` module which instantiates the `ExecutionEngine` protocol. The body of this function is implementation dependent. The Engine API may be used to implement this and similarly defined functions via an external execution engine. -#### `execute_payload` +#### `notify_new_payload` ```python -def execute_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: +def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: """ Return ``True`` if and only if ``execution_payload`` is valid with respect to ``self.execution_state``. """ @@ -353,7 +353,7 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe # Verify timestamp assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) # Verify the execution payload is valid - assert execution_engine.execute_payload(payload) + assert execution_engine.notify_new_payload(payload) # Cache execution payload header state.latest_execution_payload_header = ExecutionPayloadHeader( parent_hash=payload.parent_hash, diff --git a/sync/optimistic.md b/sync/optimistic.md index 80e9de3e90..abd345b5c0 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -106,8 +106,8 @@ these conditions.* To optimistically import a block: -- The [`execute_payload`](../specs/bellatrix/beacon-chain.md#execute_payload) function MUST return `True` if the execution - engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. +- The [`notify_new_payload`](../specs/bellatrix/beacon-chain.md#notify_new_payload) function MUST return `True` if the execution + engine returns `SYNCING`, `VALID`, or `ACCEPTED`. An `INVALID` or `INVALID_BLOCK_HASH` response MUST return `False`. - The [`validate_merge_block`](../specs/bellatrix/fork-choice.md#validate_merge_block) function MUST NOT raise an assertion if both the `pow_block` and `pow_parent` are unknown to the execution engine. @@ -119,7 +119,7 @@ In addition to this change in validation, the consensus engine MUST track which blocks returned `SYNCING` and which returned `VALID` for subsequent processing. Optimistically imported blocks MUST pass all verifications included in -`process_block` (withstanding the modifications to `execute_payload`). +`process_block` (withstanding the modifications to `notify_new_payload`). A consensus engine MUST be able to retrospectively (i.e., after import) modify the status of `SYNCING` blocks to be either `VALID` or `INVALID` based upon responses diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py index 16c7598396..a24e3e199d 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py @@ -25,7 +25,7 @@ def run_execution_payload_processing(spec, state, execution_payload, valid=True, called_new_block = False class TestEngine(spec.NoopExecutionEngine): - def execute_payload(self, payload) -> bool: + def notify_new_payload(self, payload) -> bool: nonlocal called_new_block, execution_valid called_new_block = True assert payload == execution_payload From 020d72c7564bae26b7127ef9dc45bb7ace2c4682 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 28 Jan 2022 08:43:18 +1100 Subject: [PATCH 103/106] Remove link fragment --- sync/optimistic.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index abd345b5c0..5f07548c31 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -1,8 +1,5 @@ # Optimistic Sync - - -../specs/bellatrix/fork-choice.md#validate_merge_block ## Introduction In order to provide a syncing execution engine with a partial view of the head From cbbe6e744a626a9f4948ef2f8e8df54a03b289d6 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 28 Jan 2022 09:45:08 +1100 Subject: [PATCH 104/106] Add aliases --- sync/optimistic.md | 57 +++++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 5f07548c31..b44384d347 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -25,6 +25,17 @@ statelessness). ## Helpers +For brevity, we define two aliases for values of the `status` field on +`PayloadStatusV1`: + +- Alias `NOT_VALIDATED` to: + - `SYNCING` + - `ACCEPTED` +- Alias `INVALIDATED` to: + - `INVALID` + - `INVALID_BLOCK_HASH` + - `INVALID_TERMINAL_BLOCK` + Let `head: BeaconBlock` be the result of calling of the fork choice algorithm at the time of block production. Let `head_block_root: Root` be the root of that block. @@ -34,8 +45,8 @@ BeaconState]` be the blocks (and accompanying states) that have been verified either completely or optimistically. Let `optimistic_roots: Set[Root]` be the set of `hash_tree_root(block)` for all -optimistically imported blocks which have only received a `SYNCING` designation -from an execution engine (i.e., they are not known to be `INVALID` or `VALID`). +optimistically imported blocks which have only received a `NOT_VALIDATED` designation +from an execution engine (i.e., they are not known to be `INVALIDATED` or `VALID`). Let `current_slot: Slot` be `(time - genesis_time) // SECONDS_PER_SLOT` where `time` is the UNIX time according to the local system clock. @@ -56,7 +67,7 @@ def is_optimistic(opt_store: OptimisticStore, block: BeaconBlock) -> bool: ```python def latest_verified_ancestor(opt_store: OptimisticStore, block: BeaconBlock) -> BeaconBlock: - # It is assumed that the `block` parameter is never an INVALID block. + # It is assumed that the `block` parameter is never an INVALIDATED block. while True: if not is_optimistic(opt_store, block) or block.parent_root == Root(): return block @@ -104,35 +115,35 @@ these conditions.* To optimistically import a block: - The [`notify_new_payload`](../specs/bellatrix/beacon-chain.md#notify_new_payload) function MUST return `True` if the execution - engine returns `SYNCING`, `VALID`, or `ACCEPTED`. An `INVALID` or `INVALID_BLOCK_HASH` response MUST return `False`. + engine returns `NOT_VALIDATED` or `VALID`. An `INVALIDATED` response MUST return `False`. - The [`validate_merge_block`](../specs/bellatrix/fork-choice.md#validate_merge_block) function MUST NOT raise an assertion if both the `pow_block` and `pow_parent` are unknown to the execution engine. - All other assertions in [`validate_merge_block`](../specs/bellatrix/fork-choice.md#validate_merge_block) (e.g., `TERMINAL_BLOCK_HASH`) MUST prevent an optimistic import. -- The parent of the block MUST NOT have an INVALID execution payload. +- The parent of the block MUST NOT have an `INVALIDATED` execution payload. In addition to this change in validation, the consensus engine MUST track which -blocks returned `SYNCING` and which returned `VALID` for subsequent processing. +blocks returned `NOT_VALIDATED` and which returned `VALID` for subsequent processing. Optimistically imported blocks MUST pass all verifications included in `process_block` (withstanding the modifications to `notify_new_payload`). A consensus engine MUST be able to retrospectively (i.e., after import) modify -the status of `SYNCING` blocks to be either `VALID` or `INVALID` based upon responses +the status of `NOT_VALIDATED` blocks to be either `VALID` or `INVALIDATED` based upon responses from an execution engine. I.e., perform the following transitions: -- `SYNCING` -> `VALID` -- `SYNCING` -> `INVALID` +- `NOT_VALIDATED` -> `VALID` +- `NOT_VALIDATED` -> `INVALIDATED` -When a block transitions from `SYNCING` -> `VALID`, all *ancestors* of the -block MUST also transition from `SYNCING` -> `VALID`. Such a block and any previously `SYNCING` ancestors are no longer +When a block transitions from `NOT_VALIDATED` -> `VALID`, all *ancestors* of the +block MUST also transition from `NOT_VALIDATED` -> `VALID`. Such a block and any previously `NOT_VALIDATED` ancestors are no longer considered "optimistically imported". -When a block transitions from `SYNCING` -> `INVALID`, all *descendants* of the -block MUST also transition from `SYNCING` -> `INVALID`. +When a block transitions from `NOT_VALIDATED` -> `INVALIDATED`, all *descendants* of the +block MUST also transition from `NOT_VALIDATED` -> `INVALIDATED`. -When a block transitions from the `SYNCING` state, it is removed from the set of +When a block transitions from the `NOT_VALIDATED` state, it is removed from the set of `opt_store.optimistic_roots`. When a "merge block" (i.e. the first block which enables execution in a chain) is declared to be @@ -141,7 +152,7 @@ When a "merge block" (i.e. the first block which enables execution in a chain) i MUST be run against the merge block. If the block fails [`validate_merge_block`](../specs/bellatrix/fork-choice.md#validate_merge_block), the merge block MUST be treated the same as -an `INVALID` block (i.e., it and all its descendants are invalidated and +an `INVALIDATED` block (i.e., it and all its descendants are invalidated and removed from the block tree). ### Execution Engine Errors @@ -155,11 +166,11 @@ validity request for some block, a consensus engine: ### Assumptions about Execution Engine Behaviour -This specification assumes execution engines will only return `SYNCING` when -there is insufficient information available to make a `VALID` or `INVALID` +This specification assumes execution engines will only return `NOT_VALIDATED` when +there is insufficient information available to make a `VALID` or `INVALIDATED` determination on the given `ExecutionPayload` (e.g., the parent payload is -unknown). Specifically, `SYNCING` responses should be fork-specific, in that -the search for a block on one chain MUST NOT trigger a `SYNCING` response for +unknown). Specifically, `NOT_VALIDATED` responses should be fork-specific, in that +the search for a block on one chain MUST NOT trigger a `NOT_VALIDATED` response for another chain. ### Re-Orgs @@ -167,16 +178,16 @@ another chain. The consensus engine MUST support any chain reorganisation which does *not* affect the justified checkpoint. -If the justified checkpoint transitions from `SYNCING` -> `INVALID`, a +If the justified checkpoint transitions from `NOT_VALIDATED` -> `INVALIDATED`, a consensus engine MAY choose to alert the user and force the application to exit. ## Fork Choice Consensus engines MUST support removing blocks from fork choice that transition -from `SYNCING` to `INVALID`. Specifically, a block deemed `INVALID` at any +from `NOT_VALIDATED` to `INVALIDATED`. Specifically, a block deemed `INVALIDATED` at any point MUST NOT be included in the canonical chain and the weights from those -`INVALID` blocks MUST NOT be applied to any `VALID` or `SYNCING` ancestors. +`INVALIDATED` blocks MUST NOT be applied to any `VALID` or `NOT_VALIDATED` ancestors. ### Fork Choice Poisoning @@ -333,7 +344,7 @@ unlikely attack increases or decreases our total risk. Presently, it appears that `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` sits in a sweet spot for this trade-off. -### Transitioning from VALID -> INVALID or INVALID -> VALID +### Transitioning from VALID -> INVALIDATED or INVALIDATED -> VALID These operations are purposefully omitted. It is outside of the scope of the specification since it's only possible with a faulty EE. From 129d9e28b475d1624ed979432bcacb781494eba3 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 27 Jan 2022 19:08:59 -0800 Subject: [PATCH 105/106] add randomized tests for bellatrix --- .../test/bellatrix/random/__init__.py | 0 .../test/bellatrix/random/test_random.py | 438 ++++++++++++++++++ .../test/utils/randomized_block_tests.py | 20 +- tests/generators/random/Makefile | 2 + tests/generators/random/generate.py | 11 +- tests/generators/random/main.py | 6 +- 6 files changed, 473 insertions(+), 4 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/bellatrix/random/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/bellatrix/random/test_random.py diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/random/__init__.py b/tests/core/pyspec/eth2spec/test/bellatrix/random/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/random/test_random.py b/tests/core/pyspec/eth2spec/test/bellatrix/random/test_random.py new file mode 100644 index 0000000000..2dccaf949d --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/bellatrix/random/test_random.py @@ -0,0 +1,438 @@ +""" +This module is generated from the ``random`` test generator. +Please do not edit this file manually. +See the README for that generator for more information. +""" + +from eth2spec.test.helpers.constants import BELLATRIX +from eth2spec.test.context import ( + misc_balances_in_default_range_with_many_validators, + with_phases, + zero_activation_threshold, + only_generator, +) +from eth2spec.test.context import ( + always_bls, + spec_test, + with_custom_state, + single_phase, +) +from eth2spec.test.utils.randomized_block_tests import ( + run_generated_randomized_test, +) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_0(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_1(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_2(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_3(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_4(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_5(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_6(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_7(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_8(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_9(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_10(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_11(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_12(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_13(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_14(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_15(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) diff --git a/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py b/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py index 02a5464f70..386b4c8b23 100644 --- a/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py +++ b/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py @@ -55,12 +55,22 @@ def randomize_state(spec, state, stats, exit_fraction=0.1, slash_fraction=0.1): return scenario_state -def randomize_state_altair(spec, state, stats): - scenario_state = randomize_state(spec, state, stats, exit_fraction=0.1, slash_fraction=0.1) +def randomize_state_altair(spec, state, stats, exit_fraction=0.1, slash_fraction=0.1): + scenario_state = randomize_state(spec, state, stats, exit_fraction=exit_fraction, slash_fraction=slash_fraction) randomize_inactivity_scores(spec, state) return scenario_state +def randomize_state_bellatrix(spec, state, stats, exit_fraction=0.1, slash_fraction=0.1): + scenario_state = randomize_state_altair(spec, + state, + stats, + exit_fraction=exit_fraction, + slash_fraction=slash_fraction) + # TODO: randomize execution payload, merge status, etc. + return scenario_state + + # epochs def epochs_until_leak(spec): @@ -179,6 +189,12 @@ def random_block_altair_with_cycling_sync_committee_participation(spec, return block +def random_block_bellatrix(spec, state, signed_blocks, scenario_state): + block = random_block_altair_with_cycling_sync_committee_participation(spec, state, signed_blocks, scenario_state) + # TODO: return randomized execution payload + return block + + # validations def no_op_validation(_spec, _state): diff --git a/tests/generators/random/Makefile b/tests/generators/random/Makefile index 1b518bfde7..607d69a470 100644 --- a/tests/generators/random/Makefile +++ b/tests/generators/random/Makefile @@ -4,5 +4,7 @@ all: pip3 install -r requirements.txt rm -f ../../core/pyspec/eth2spec/test/phase0/random/test_random.py rm -f ../../core/pyspec/eth2spec/test/altair/random/test_random.py + rm -f ../../core/pyspec/eth2spec/test/bellatrix/random/test_random.py python3 generate.py phase0 > ../../core/pyspec/eth2spec/test/phase0/random/test_random.py python3 generate.py altair > ../../core/pyspec/eth2spec/test/altair/random/test_random.py + python3 generate.py bellatrix > ../../core/pyspec/eth2spec/test/bellatrix/random/test_random.py diff --git a/tests/generators/random/generate.py b/tests/generators/random/generate.py index b880cb12ba..f96f05a754 100644 --- a/tests/generators/random/generate.py +++ b/tests/generators/random/generate.py @@ -19,8 +19,10 @@ no_op_validation, randomize_state, randomize_state_altair, + randomize_state_bellatrix, random_block, random_block_altair_with_cycling_sync_committee_participation, + random_block_bellatrix, last_slot_in_epoch, random_slot_in_epoch, penultimate_slot_in_epoch, @@ -30,7 +32,7 @@ transition_to_leaking, transition_without_leak, ) -from eth2spec.test.helpers.constants import PHASE0, ALTAIR +from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX # Ensure this many blocks are present in *each* randomized scenario @@ -254,5 +256,12 @@ def run_generate_tests_to_std_out(phase, state_randomizer, block_randomizer): state_randomizer=randomize_state_altair, block_randomizer=random_block_altair_with_cycling_sync_committee_participation, ) + if BELLATRIX in sys.argv: + did_generate = True + run_generate_tests_to_std_out( + BELLATRIX, + state_randomizer=randomize_state_bellatrix, + block_randomizer=random_block_bellatrix, + ) if not did_generate: warnings.warn("no phase given for test generation") diff --git a/tests/generators/random/main.py b/tests/generators/random/main.py index f6f1b18476..1791da83b9 100644 --- a/tests/generators/random/main.py +++ b/tests/generators/random/main.py @@ -1,4 +1,4 @@ -from eth2spec.test.helpers.constants import PHASE0, ALTAIR +from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators @@ -9,10 +9,14 @@ altair_mods = {key: 'eth2spec.test.altair.random.test_' + key for key in [ 'random', ]} + bellatrix_mods = {key: 'eth2spec.test.bellatrix.random.test_' + key for key in [ + 'random', + ]} all_mods = { PHASE0: phase_0_mods, ALTAIR: altair_mods, + BELLATRIX: bellatrix_mods, } run_state_test_generators(runner_name="random", all_mods=all_mods) From 67fcbf22ec7b478312d590f9b620b2126343a4a1 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 28 Jan 2022 06:44:04 -0700 Subject: [PATCH 106/106] bump version --- tests/core/pyspec/eth2spec/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index db15278970..a5e4282993 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -1.1.8 \ No newline at end of file +1.1.9 \ No newline at end of file