devnet_release.yml

name: Devnet Release Scenarios

on:
  schedule:
    - cron: '0 */2 * * *'
  workflow_dispatch:

jobs:
  Run:
    strategy:
      fail-fast: false
      matrix:
        test:
        - FourValidatorsReconnect
        - MultipleValidatorsDown
        - FourValidatorsReconnectRmDatabase
        - FourValidatorsReconnectSpammer
        - MacroBlockProduction
        - Validators90sDownFullNodes
        - LowBlockProducerTimeout
        - FiveValidatorsWithSpammer
        - ValidatorsStress

        include:
        - test: FourValidatorsReconnect
          devnet_args: -t .github/devnet_topologies/four_validators.toml -R
        - test: MultipleValidatorsDown
          devnet_args: -t .github/devnet_topologies/four_validators.toml -k 2 -R -as
        - test: FourValidatorsReconnectRmDatabase
          devnet_args: -t .github/devnet_topologies/four_validators.toml -d -R
        - test: FourValidatorsReconnectSpammer
          devnet_args: -t .github/devnet_topologies/four_validators_spammer_3.toml -R
        - test: MacroBlockProduction
          pre: "sed -i 's/BLOCKS_PER_BATCH: u32 = 32;/BLOCKS_PER_BATCH: u32 = 2;/g' primitives/src/policy.rs"
          devnet_args: -t .github/devnet_topologies/four_validators_spammer_2.toml -k 0 -R
        - test: Validators90sDownFullNodes
          devnet_args: -t .github/devnet_topologies/four_validators_spammer_3.toml -dt 90 -R
        - test: LowBlockProducerTimeout
          pre: "sed -i 's/MIN_PRODUCER_TIMEOUT: u64 = 4 * 1000;/MIN_PRODUCER_TIMEOUT: u64 = 2 * 1000;/g' primitives/src/policy.rs"
          devnet_args: -t .github/devnet_topologies/four_validators_spammer_2.toml -k 0 -R
        - test: FiveValidatorsWithSpammer
          devnet_args: -t .github/devnet_topologies/five_validators_multi_spammer.toml -R -k 0
        - test: ValidatorsStress
          devnet_args: -t .github/devnet_topologies/validators_stress.toml -R -k 2 -as -dt 30

    runs-on: ubuntu-22.04

    steps:
    - name: Free Disk Space (Ubuntu)
      uses: jlumbroso/free-disk-space@main
      with:
        tool-cache: true
        large-packages: false
    - uses: actions/checkout@v4
    - uses: actions/setup-python@v5
      with:
        python-version: '3.8'
    - name: Install python dependencies
      run: pip install scripts/devnet
    - name: Set up Rust toolchain
      uses: dtolnay/rust-toolchain@stable
    - uses: Swatinem/rust-cache@v2
    - name: Optionally patch the source
      run: ${{ matrix.pre }}
    - name: Build the code
      run: cargo build --release
    - name: Retrieve initial timestamp
      id: initial_ts
      run: |
         echo "INITIAL_TS=$(date +%s%N | cut -b1-13)" >> $GITHUB_OUTPUT
    - name: Execute the test
      run: |
          python3 scripts/devnet/devnet.py --run-environment ci ${{ matrix.devnet_args }}
      env:
        NIMIQ_LOKI_URL: ${{ secrets.LOKI_URL }}
        NIMIQ_LOKI_EXTRA_FIELDS: nimiq_ci_name=${{ github.workflow }}:nimiq_ci_run=${{ github.run_number }}:nimiq_test=${{ matrix.test }}
    - name: Redact Loki URL
      if: always()
      run: sed -i 's/url = "[^"]*"/url = "<redacted>"/g' temp-logs/*/conf/*/*.toml
    - name: Archive test results
      if: always()
      uses: actions/upload-artifact@v4
      with:
          name: ${{ matrix.test }}-logs
          path: |
            temp-logs/
    - name: Retrieve failure reason
      if: always()
      run: |
          if [ -f temp-state/*/RESULT.TXT ]; then
            echo "FAIL_REASON=$(cat temp-state/*/RESULT.TXT)" >> $GITHUB_OUTPUT
          else
            echo "FAIL_REASON=other" >> $GITHUB_OUTPUT
          fi
      id: reason
    - name: Retrieve final timestamp and Run ID
      if: always()
      id: final_ts_run_id
      run: |
         echo "FINAL_TS=$(date +%s%N | cut -b1-13)" >> $GITHUB_OUTPUT
         echo "RUN_ID=$(cat temp-logs/*/conf/run_id.info)" >> $GITHUB_OUTPUT
    - name: Build Grafana dashboard link
      if: always()
      id: grafana_url
      run: |
         echo "GRAFANA_URL=http://localhost:3001/d/YbjdvlU7z/nimiq-test?orgId=1&var-env=ci&var-run_id=${{steps.final_ts_run_id.outputs.RUN_ID}}&from=${{steps.initial_ts.outputs.INITIAL_TS}}&to=${{steps.final_ts_run_id.outputs.FINAL_TS}}" >> $GITHUB_OUTPUT
    - name: Report potential deadlocks to slack
      if: always() && contains(steps.reason.outputs.FAIL_REASON, 'DEADLOCK')
      uses: ravsamhq/notify-slack-action@v2
      with:
          status: ${{ job.status }}
          notification_title: 'Potential deadlock detected in ${{ matrix.test }}'
          footer: '<{run_url}|View Run>'
      env:
          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
#    - name: Report long lock hold times
#      if: always() && contains(steps.reason.outputs.FAIL_REASON, 'LONG_LOCK_HOLD_TIME')
#      uses: ravsamhq/notify-slack-action@v2
#      with:
#          status: ${{ job.status }}
#          notification_title: 'Long lock hold time in ${{ matrix.test }}'
#          footer: '<{run_url}|View Run>'
#      env:
#          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
#    - name: Report slow lock acquisitions
#      if: always() && contains(steps.reason.outputs.FAIL_REASON, 'SLOW_LOCK_ACQUISITION')
#      uses: ravsamhq/notify-slack-action@v2
#      with:
#          status: ${{ job.status }}
#          notification_title: 'Slow lock acquisition detected in ${{ matrix.test }}'
#          footer: '<{run_url}|View Run>'
#      env:
#          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
    - name: Report Status to Slack
      if: always() && github.ref == 'refs/heads/albatross'
      uses: ravsamhq/notify-slack-action@v2
      with:
          status: ${{ job.status }}
          notify_when: 'failure'
          notification_title: '${{ matrix.test }} failed because of ${{ steps.reason.outputs.FAIL_REASON }}'
          footer: '<{run_url}|View Run> | <${{steps.grafana_url.outputs.GRAFANA_URL}}|Grafana dashboard>'
      env:
          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}