deploy.yml

---
# yamllint disable rule:line-length

.setup: &setup_script
  - eval $(ssh-agent -s)
  - echo "$SSH_PRIVATE_KEY" | ssh-add -
  - mkdir -p ~/.ssh
  - chmod 700 ~/.ssh
  - echo "$SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
  - chmod 644 ~/.ssh/known_hosts
  - set -exo pipefail
  - export TARGET=${TARGET:-"openqa.suse.de"}

.defaults:
  image: registry.opensuse.org/home/okurz/container/containers/tumbleweed:salt-minion-git-core-ssh
  timeout: 2h
  rules:
    - if: $CI_PROJECT_NAMESPACE != "openqa"
      when: never
    - if: >-
          $CI_PIPELINE_SOURCE == "push" &&
          $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH
  artifacts:
    when: always
    paths:
      - salt_*.log
    expire_in: 1 week

clone:
  extends: .defaults
  stage: deploy
  parallel:
    matrix:
      - TOKEN: ${STATES_CLONE_TOKEN}
        REPO: states
        DIR: salt
      - TOKEN: ${PILLARS_CLONE_TOKEN}
        REPO: pillars
        DIR: pillar
  script:
    - *setup_script
    - ssh $TARGET "(
       set -x; cd /srv &&
       mkdir -p ${DIR:-$REPO} &&
       chown root:salt ${DIR:-$REPO} &&
       cd ${DIR:-$REPO} &&
       git status || git clone --depth 3 https://osd_deployment_ci:$TOKEN@gitlab.suse.de/openqa/salt-$REPO-openqa.git . &&
       git fetch origin &&
       git reset --hard origin/master &&
       git checkout -f master &&
       git pull
       )"

refresh:
  extends: .defaults
  stage: deploy
  needs:
    - clone
  script:
    - *setup_script
    - ssh $TARGET "echo 'Currently the following minions are down:'; diff <(sudo salt --timeout 1 '*' test.ping --out json 2>/dev/null | jq 'keys[]' | sort) <(sudo salt --timeout 1 --hide-timeout '*' test.ping --out json 2>/dev/null | jq 'keys[]' | sort); echo '==================='" | tee salt_ping.log
    # exit code of compound commands are not reliable hence looking for
    # Traceback error in output
    # https://progress.opensuse.org/issues/137984
    # https://github.com/saltstack/salt/issues/42814
    - timeout -v --kill-after=10m 90m ssh $TARGET "set -eo pipefail; rm -f salt_syncupdate.log; salt --log-file=salt_syncupdate.log --log-file-level=debug --state-output=mixed --hide-timeout \* saltutil.sync_grains,saltutil.refresh_grains,saltutil.refresh_pillar,mine.update ,,, | (! ack --passthru --color Traceback)"

deploy:
  extends: .defaults
  stage: deploy
  needs:
    - refresh
  script:
    - *setup_script
    - timeout -v --kill-after=10m 90m ssh $TARGET "set -eo pipefail; rm -f salt_highstate.log; salt --log-file=salt_highstate.log --log-file-level=debug --state-output=changes --hide-timeout \* state.highstate queue=True | awk '/Result..Clean[- ]*Started/ {print > \"salt_profiling.log\"; next} 1'"
  after_script:
    - *setup_script
    - scp $TARGET:salt_\*.log .

telegraf:
  extends: .defaults
  stage: post-deploy
  script:
    - *setup_script
    - ssh $TARGET "salt --hide-timeout \* cmd.run 'sudo timeout -v --kill-after=1m 5m telegraf --test --config /etc/telegraf/telegraf.conf --config-directory /etc/telegraf/telegraf.d/ 2>&1 | grep \" E\! \" && echo \"telegraf errors\" || echo \"telegraf is fine\"'" | tee salt_post_deploy_checks.log; ! grep ' E! ' salt_post_deploy_checks.log
  # telegraf is very prone for network issues so we really just want to
  # run it if a MR changes files affecting its configuration.
  # https://progress.opensuse.org/issues/167051
  rules:
    # extends does not deep-merge lists so we have to add our additional "changes" here
    # also see https://docs.gitlab.com/ee/ci/yaml/yaml_optimization.html#merge-details for details
    - if: $CI_PROJECT_NAMESPACE != "openqa"
      when: never
    - if: >-
          $CI_PIPELINE_SOURCE == "push" &&
          $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH
      changes:
        - "monitoring/telegraf/*"