feat: add hetzner, remote roles, remote checkpointz

.github/workflows/lint-ansible.yaml

@@ -16,7 +16,7 @@ on:
       - '.github/workflows/lint-ansible.yaml'
 
 jobs:
-  job:
+  ansible-lint:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

.github/workflows/lint-helm.yaml

@@ -1,4 +1,5 @@
 name: Helm lint
+
 on:
   pull_request:
     paths:
@@ -16,11 +17,26 @@ on:
       - '.github/workflows/lint-helm.yaml'
 
 jobs:
-  job:
+  helm-lint:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/setup
+      - name: Check if kubernetes/*/* is not empty
+        id: check-kubernetes-directory
+        run: |
+          if [ "$(ls -A kubernetes/*/* 2>/dev/null)" ]; then
+            echo "Kubernetes directory is not empty."
+            echo "::set-output name=not_empty::true"
+          else
+            echo "Kubernetes directory is empty."
+            echo "::set-output name=not_empty::false"
+          fi
+
       - name: Lint
-        run: >
-          helm lint kubernetes/*/*
+        run: |
+          if ${{ steps.check-kubernetes-directory.outputs.not_empty }}; then
+            helm lint kubernetes/*/*
+          else
+            echo "Skipping Helm lint because kubernetes/*/* is empty."
+          fi

.github/workflows/lint-terraform.yaml

@@ -16,7 +16,7 @@ on:
       - '.github/workflows/lint-terraform.yaml'
 
 jobs:
-  job:
+  terraform-lint:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

README.md

@@ -25,6 +25,9 @@ You can then use [`./setup.sh`](./asdf-setup.sh) to install all dependencies.
 
 ## Terraform
 From [`./terraform/devnet-0/`](./terraform/devnet-0/)
+
+Make sure you select either hetzner or digitalocean (default is digitialocean), if you want to use hetzner rename digitalocean.tf to digitalocean.tf.disabled and rename hetzner.tf.disabled to hetzner.tf and vice versa.
+
 ```shell
 terraform init
 terraform apply

ansible/ansible.cfg

@@ -1,4 +1,5 @@
 [defaults]
+ansible_managed      = Ansible managed. Don't change this file manually. Template info: {{{{ (template_fullpath | replace(playbook_dir,'')) }}}}
 stdout_callback      = yaml
 inventory            = inventories/devnet-0/inventory.ini
 roles_path           = vendor/roles/:roles # First path in roles_path and collections_path will be used by ansible-galaxy to install dependencies.

ansible/cleanup_ethereum.yaml

@@ -1,4 +1,4 @@
-- hosts: all
+- hosts: all, !localhost
   serial: "{{ batch_count | default('100%') }}"
   become: true
   tasks:
@@ -49,7 +49,7 @@
             - "ethereum-metrics-exporter"
             - "snooper-engine"
             - "snooper-rpc"
-            #- "cl-bootnode"
+            - "cl-bootnode"
         - name: Remove data dirs
           file:
             state: absent
@@ -75,3 +75,4 @@
             # Config files
             - /data/ethereum-network-config
             - /data/execution-auth.secret
+            - /data/cl-bootnode

ansible/inventories/devnet-0/group_vars/all/all.yaml

@@ -21,7 +21,11 @@ ethereum_node_beacon_hostname: "bn.{{ server_fqdn }}"
 ethstats_url: "ethstats.{{ ethereum_network_name }}.ethpandaops.io"
 ethstats_secret: "{{ secret_ethstats }}"
 
-
+### Checkpoint sync specific for Ansible & Kubernetes
+checkpoint_sync_node: teku-nethermind-1
+checkpoint_sync_enabled: false
+checkpoint_sync_url: https://{{ secret_nginx_shared_basic_auth.name }}:{{ secret_nginx_shared_basic_auth.password }}@bn.{{ checkpoint_sync_node }}.srv.{{ ethereum_network_name }}.ethpandaops.io
+checkpoint_sync_url_kube: https://<path:/secrets/services/services.enc.yaml#ethereum | jsonPath {.testnets.dencun-devnets.node_ingress.combined}>@bn.{{ checkpoint_sync_node }}.srv.{{ ethereum_network_name }}.ethpandaops.io
 #  ██████╗░░█████╗░██╗░░░░░███████╗  ██╗░░░██╗░█████╗░██████╗░░██████╗
 #  ██╔══██╗██╔══██╗██║░░░░░██╔════╝  ██║░░░██║██╔══██╗██╔══██╗██╔════╝
 #  ██████╔╝██║░░██║██║░░░░░█████╗░░  ╚██╗░██╔╝███████║██████╔╝╚█████╗░
@@ -30,7 +34,7 @@ ethstats_secret: "{{ secret_ethstats }}"
 #  ╚═╝░░╚═╝░╚════╝░╚══════╝╚══════╝  ░░░╚═╝░░░╚═╝░░╚═╝╚═╝░░╚═╝╚═════╝░
 
 # role: ethpandaops.general.ethereum_genesis
-ethereum_genesis_generator_container_image: "ethpandaops/ethereum-genesis-generator:1.3.4"
+ethereum_genesis_generator_container_image: "ethpandaops/ethereum-genesis-generator:2.0.0-rc.7"
 ethereum_genesis_generator_version: "{{ ethereum_genesis_generator_container_image.split(':')[-1] }}"
 ethereum_genesis_network_seed: "{{ ansible_inventory_sources[0] }}"
 ethereum_genesis_chain_id: "70{{ 99999999 | random(start=10000000, seed=ethereum_genesis_network_seed) }}"
@@ -56,7 +60,6 @@ ethereum_genesis_generator_config_files:
     export ALTAIR_FORK_VERSION="0x20{{ ethereum_genesis_fork_version_suffix }}"
     export BELLATRIX_FORK_VERSION="0x30{{ ethereum_genesis_fork_version_suffix }}"
     export CAPELLA_FORK_VERSION="0x40{{ ethereum_genesis_fork_version_suffix }}"
-    export CAPELLA_FORK_EPOCH="1"
     export DENEB_FORK_VERSION="0x50{{ ethereum_genesis_fork_version_suffix }}"
     export DENEB_FORK_EPOCH="2"
     export WITHDRAWAL_TYPE="0x00"

ansible/inventories/devnet-0/group_vars/all/images.yaml

@@ -1,30 +1,33 @@
 default_ethereum_client_images:
-  # Consensus layer clients
-  prysm: gcr.io/prysmaticlabs/prysm/beacon-chain:v4.0.7
-  prysm_validator: gcr.io/prysmaticlabs/prysm/validator:v4.0.7
-  teku: consensys/teku:23.6.2-amd64
-  lighthouse: sigp/lighthouse:v4.3.0
-  nimbus: statusim/nimbus-eth2:multiarch-v23.7.0
-  lodestar: chainsafe/lodestar:v1.9.2
-  # Execution layer clients
-  geth: ethereum/client-go:v1.12.0
-  nethermind: nethermind/nethermind:1.20.1
-  besu: hyperledger/besu:23.4.4-amd64
-  erigon: ethpandaops/erigon:latest
-  ethereumjs: g11tech/ethereumjs:latest
+### Consensus layer clients
+  lighthouse: sigp/lighthouse:latest
+  lodestar: chainsafe/lodestar:latest
+  nimbus: statusim/nimbus-eth2:multiarch-latest
+  prysm: ethpandaops/prysm:develop
+  prysm_validator: ethpandaops/prysm-validator:develop
+  teku: consensys/teku:latest
+### Execution layer clients
+  besu: hyperledger/besu:latest
+  geth: ethereum/client-go:stable
+  erigon: thorax/erigon:stable
+  ethereumjs: ethpandaops/ethereumjs:master
+  nethermind: nethermindeth/nethermind:cancun
+  reth: ethpandaops/reth:main
+
 
 default_tooling_images:
-  xatu_sentry: ethpandaops/xatu:0.0.60
-  ethereum_metrics_exporter: ethpandaops/ethereum-metrics-exporter:0.21.0
-  tx_fuzz: ethpandaops/tx-fuzz:1.1.0
+  xatu_sentry: ethpandaops/xatu:latest
+  ethereum_metrics_exporter: ethpandaops/ethereum-metrics-exporter:latest
+  tx_fuzz: ethpandaops/tx-fuzz:latest
   forkmon: skylenet/nodemonitor:darkmode
-  forky: ethpandaops/forky:0.0.34-debian
+  forky: ethpandaops/forky:latest
   fauceth: skylenet/fauceth:fix_fee_estimation
   ethstats: skylenet/ethstats-server:poa-no-geoip
   homepage: ethpandaops/ethereum-testnet-homepage:latest
-  checkpointz: ethpandaops/checkpointz:0.15.0
+  checkpointz: ethpandaops/checkpointz:0.0.10-dencun-debian
   blockscout: blockscout/blockscout:master
-  beacon_metrics_gazer: dapplion/beacon-metrics-gazer:v0.1.3
+  beacon_metrics_gazer: dapplion/beacon-metrics-gazer:latest
   eth_fauceth: chainflag/eth-faucet:latest
   blobscan: blossomlabs/blobscan:latest
   blobscan_indexer: blossomlabs/blobscan-indexer:latest
+  light_beaconchain_explorer: pk910/light-beaconchain-explorer:latest

ansible/inventories/devnet-0/group_vars/bootnode.sops.yaml

@@ -1,12 +1,12 @@
-cl_bootnode_privkey: ENC[AES256_GCM,data:1rqR,iv:i8pJEXj5piOEWNOwNuZWpjdf8S87Y6aO+o4r4f0kIts=,tag:U9eF9h0uyBs+LihMHXpTMw==,type:str]
+cl_bootnode_privkey: ENC[AES256_GCM,data:YWNv97QCL5pCy8lowEb5jJtFJ6fXjamE5bB+takjWNbQhwxxHkprENN0jcuUtNfuSfQFRYymR+fI0lgtnQq4zw==,iv:iEHtpLwDBPbJ3Ny/ZZLf+jX2nWunP21VhgU5D6fiSyQ=,tag:yEqNeQKvgkgC8oCH/4schg==,type:str]
 sops:
     kms: []
     gcp_kms: []
     azure_kv: []
     hc_vault: []
     age: []
-    lastmodified: "2023-05-04T09:46:57Z"
-    mac: ENC[AES256_GCM,data:FTssOZs2fDxEhsdwh3pcPRFwNIgK7oFZAer94B9Wt5ZkThm/ofcKWfl3vCZbx2LtI3hYIS2AdW9/LqalBYvClKqW0TKYfTJkb+8wTeflk9uR/dwMwelbGPV5LCeKSS+AxCCTsBGm3gKtXk15iThxx6srNehlzedfx0FmYFqbWds=,iv:mUHdSt8zaYlzJaJrlQ43/uaoZdDYo2J8YU+p2U0zyxA=,tag:i1PnDiazrqehQ7elrU/zeg==,type:str]
+    lastmodified: "2023-09-01T10:51:26Z"
+    mac: ENC[AES256_GCM,data:KfZGx1QfBJiqKl2FSNvtPvoeHkV167h8K3WaZ/H+9Q9WlwLVPBMwzGasbM/0GI+HPcLd5u11r6szAaKKeYquImP/XGihTP8vwwinzXr7mtp2zwEJb4AQqKy8yklY7uSMHkIjNB6kRRcy0QYjrzUeoKlS8U5LQJ/qgN1yqJJJp4s=,iv:YhQU1f+xjL3OCWe8FRjQI56wzEgkk5yFEb0sOQa05IM=,tag:5Yn5FU99kFRrhiIg+iT13Q==,type:str]
     pgp:
         - created_at: "2023-01-24T13:12:13Z"
           enc: |-

ansible/inventories/devnet-0/group_vars/bootnode.yaml

@@ -48,7 +48,6 @@ geth_container_command_extra_args:
   - --http.vhosts=*
   - --networkid={{ ethereum_network_id }}
   - --syncmode=full
-  - --bootnodes={{ ethereum_el_bootnodes | join(',') }}
   - --ethstats={{ inventory_hostname }}:{{ ethstats_secret }}@{{ ethstats_url }}
 geth_init_custom_network: true
 geth_init_custom_network_genesis_file: "{{ eth_testnet_config_dir }}/genesis.json"

ansible/inventories/devnet-0/group_vars/ethereumjs.yaml

@@ -29,4 +29,3 @@ ethereumjs_container_command_extra_args:
   - --maxPeers=75
   - --isSingleNode=true
   - --logLevel=debug
-  #- --trustedSetup=/network-config/trusted_setup.txt

ansible/inventories/devnet-0/group_vars/lighthouse.yaml

@@ -42,12 +42,12 @@ lighthouse_container_command_extra_args:
   - --testnet-dir=/network-config
   - --boot-nodes={{ ethereum_cl_bootnodes | join(',') }}
   - --debug-level=debug
-  #- --checkpoint-sync-url={{ checkpoint_sync_url }}
 lighthouse_validator_container_volumes:
   - "{{ lighthouse_validator_datadir }}:/validator-data"
   - "{{ eth_testnet_config_dir }}:/network-config:ro"
 lighthouse_validator_container_command_extra_args:
   - --testnet-dir=/network-config
   - --graffiti={{ ethereum_node_cl }}/{{ ethereum_node_el }}
 lighthouse_validator_datadir: /data/lighthouse-validator
-lighthouse_container_pull: true
+lighthouse_container_pull: true
+lighthouse_checkpoint_sync_enabled: "{{ checkpoint_sync_enabled }}" 

ansible/inventories/devnet-0/group_vars/lodestar.yaml

@@ -42,13 +42,12 @@ lodestar_container_command_extra_args:
   - --rest.namespace="*"
   - --network.connectToDiscv5Bootnodes
   - --nat=true
-  #- --chain.trustedSetup=/network-config/trusted_setup.txt
-  #- --checkpointSyncUrl={{ checkpoint_sync_url }}
 lodestar_validator_container_volumes:
   - "{{ lodestar_validator_datadir }}:/validator-data"
   - "{{ eth_testnet_config_dir }}:/network-config:ro"
 lodestar_validator_container_command_extra_args:
   - --paramsFile=/network-config/config.yaml
   - --graffiti={{ ethereum_node_cl }}/{{ ethereum_node_el }}
 lodestar_validator_datadir: /data/lodestar-validator
-lodestar_container_pull: true
+lodestar_container_pull: true
+lodestar_checkpoint_sync_enabled: "{{ checkpoint_sync_enabled }}" 

ansible/inventories/devnet-0/group_vars/nethermind.yaml

@@ -21,6 +21,8 @@ nethermind_container_env:
   VIRTUAL_HOST: "{{ ethereum_node_rcp_hostname }}"
   VIRTUAL_PORT: "{{ ethereum_node_el_ports_http_rpc | string }}"
   LETSENCRYPT_HOST: "{{ ethereum_node_rcp_hostname }}"
+nethermind_container_entrypoint:
+  - /nethermind/nethermind
 nethermind_container_volumes:
   - "{{ nethermind_datadir }}:/data"
   - "{{ nethermind_auth_jwt_path }}:/execution-auth.jwt:ro"
@@ -37,4 +39,4 @@ nethermind_container_command_extra_args:
   - --EthStats.Secret={{ ethstats_secret }}
   - --EthStats.Server=wss://{{ ethstats_url }}/api/
   - --log=DEBUG
-nethermind_container_pull: true
+nethermind_container_pull: true

ansible/inventories/devnet-0/group_vars/nimbus.yaml

@@ -22,14 +22,9 @@ validator_bls_sync_files:
 ethereum_node_cl: nimbus
 
 # Checkpointsync 
-nimbus_checkpoint_sync_enabled: false 
-nimbus_checkpoint_container_command_extra_args:
-  - --trusted-node-url={{ checkpoint_sync_url }}
-  - --network=/network-config
-  - --backfill=false
+nimbus_checkpoint_sync_enabled: "{{ checkpoint_sync_enabled }}" 
 nimbus_checkpoint_autoremove_enabled: true
 
-
 # role: ethpandaops.general.nimbus
 nimbus_container_name: beacon
 nimbus_container_image: "{{ default_ethereum_client_images.nimbus }}"

ansible/inventories/devnet-0/group_vars/prysm.yaml

@@ -49,8 +49,6 @@ prysm_container_command_extra_simple_args:
   - --enable-debug-rpc-endpoints
   - --verbosity=debug
   - --subscribe-all-subnets
-  # - --checkpoint-sync-url={{ checkpoint_sync_url }}
-  # - --genesis-beacon-api-url={{ checkpoint_sync_url }}
 prysm_container_command_extra_bootnode_args: >-
   {{ ethereum_cl_bootnodes | map('regex_replace', '^', '--bootstrap-node=') | list }}
 
@@ -64,3 +62,4 @@ prysm_validator_container_command_extra_args:
   - --chain-config-file=/network-config/config.yaml
   - --graffiti={{ ethereum_node_cl }}/{{ ethereum_node_el }}
 prysm_validator_datadir: /data/prysm-validator
+prysm_checkpoint_sync_enabled: "{{ checkpoint_sync_enabled }}"

ansible/inventories/devnet-0/group_vars/reth.yaml

@@ -1,6 +1,6 @@
 # role: ethpandaops.general.bootstrap
 bootstrap_default_user_authorized_keys_github_team_el:
-#- gakonst
+  # - gakonst
 
 # role: ethpandaops.general.ethereum_node
 ethereum_node_el: reth
@@ -20,4 +20,3 @@ reth_container_command_extra_args:
   - --chain=/network-config/genesis.json
   - --bootnodes={{ ethereum_el_bootnodes | join(',') }}
   - --db.log-level=debug
-  #- --trustedSetup=/network-config/trusted_setup.txt

ansible/inventories/devnet-0/group_vars/teku.yaml

@@ -43,10 +43,9 @@ teku_container_command_extra_args:
   - --logging=info
   - --Xlog-include-p2p-warnings-enabled
   - --metrics-block-timing-tracking-enabled
-  # - --initial-state={{ checkpoint_sync_url }} # Need to remove genesis.ssz if using this
 teku_validator_container_volumes:
   - "{{ teku_validator_datadir }}:/validator-data"
   - "{{ eth_testnet_config_dir }}:/network-config:ro"
 teku_validator_datadir: /data/teku-validator
-
 teku_container_pull: true
+teku_checkpoint_sync_enabled: "{{ checkpoint_sync_enabled }}"

ansible/playbook.yaml

@@ -19,7 +19,6 @@
     - role: geerlingguy.pip
       pip_install_packages:
         - name: docker
-        - name: docker-compose
       tags: [init-server, pip]
     - role: ethpandaops.general.docker_cleanup
       tags: [init-server, docker_cleanup]
@@ -29,35 +28,37 @@
       tags: [init-server, node_exporter]
     - role: ethpandaops.general.prometheus
       tags: [init-server, prometheus]
+    - role: ethpandaops.general.vector
+      tags: [init-server, vector]
 
 - hosts: bootnode
   become: true
   roles:
     - role: ethpandaops.general.cl_bootnode
       tags: [cl_bootnode]
-    - role: eth_testnet_config
-      tags: [ethereum, eth_testnet_config]
+    - role: ethpandaops.general.ethereum_testnet_config
+      tags: [ethereum, eth_testnet_config, ethereum_testnet_config]
     - role: ethpandaops.general.ethereum_node
       tags: [ethereum, ethereum_node]
     - role: ethpandaops.general.docker_nginx_proxy
       tags: [docker_nginx_proxy]
-    - role: gen_basic_auth_nginx
+    - role: ethpandaops.general.generate_basic_auth_nginx
       tags: [docker_nginx_proxy]
 
 - hosts: ethereum_node
   serial: "{{ batch_count | default('100%') }}"
   become: true
   roles:
-    - role: eth_testnet_config
-      tags: [ethereum, eth_testnet_config]
-    - role: validator_keys
+    - role: ethpandaops.general.ethereum_testnet_config
+      tags: [ethereum, eth_testnet_config, ethereum_testnet_config]
+    - role: ethpandaops.general.validator_keys
       when: ethereum_node_cl_validator_enabled == true
       tags: [ethereum, validator_keys]
     - role: ethpandaops.general.ethereum_node
       tags: [ethereum, ethereum_node]
     - role: ethpandaops.general.docker_nginx_proxy
       tags: [docker_nginx_proxy]
-    - role: gen_basic_auth_nginx
+    - role: ethpandaops.general.generate_basic_auth_nginx
       tags: [docker_nginx_proxy]
   post_tasks:
     - name: Wait between runs
@@ -74,13 +75,18 @@
 - hosts: bootnode
   become: true
   roles:
-    - role: eth_inventory_web
-      tags: [eth_inventory_web]
+    - role: ethpandaops.general.ethereum_inventory_web
+      tags: [eth_inventory_web, ethereum_inventory_web]
 
+- hosts: beaconlight
+  become: true
+  roles:
+    - role: ethpandaops.general.beaconlight_explorer
+      tags: [beaconlight_explorer]
 
 - hosts: localhost
   roles:
-    - role: eth_post_network_setup
-      tags: [eth_post_network_setup]
-    - role: gen_kubernetes_config
-      tags: [gen_kubernetes_config]
+    - role: ethpandaops.general.ethereum_post_network_setup
+      tags: [eth_post_network_setup, ethereum_post_network_setup]
+    - role: ethpandaops.general.generate_kubernetes_config
+      tags: [gen_kubernetes_config, generate_kubernetes_config]