risedev.yml

# `src/risedevtool/risedev-schema.json` defines the schema for risedev.yml
#
# You can add the following section to `.vscode/settings.json` to get hover support in VS Code:
#
# ```
#     "yaml.schemas": {
#         "src/risedevtool/risedev-schema.json": "risedev.yml"
#     }
# ```

risedev:
  #########################################
  ### Configurations used by developers ###
  #########################################

  # The default configuration will start 1 compute node, 1 meta node and 1 frontend.
  default:
    config-path: src/config/default.toml
    steps:
      # If you want to use the local s3 storage, enable the following line
      # - use: minio

      # If you want to use aws-s3, configure AK and SK in env var and enable the following lines:
      # - use: aws-s3
      #   bucket: test-bucket
      # If you want to use other s3 compatible object store, open this flag:
      #   s3-compatible: false

      # if you want to enable etcd backend, uncomment the following lines.
      # - use: etcd
      #   unsafe-no-fsync: true
      - use: meta-node
      - use: compute-node
      - use: frontend

      # If you want to create CDC source, uncomment the following line
      # - use: connector-node

      # If you want to enable compactor, uncomment the following line, and enable either minio or aws-s3 as well.
      # - use: compactor

      # If you want to enable metrics, uncomment those two lines.
      # - use: prometheus
      # - use: grafana

      # If you want to enable tracing, uncomment the following line.
      # - use: jaeger

      # If you want to create source from Kafka, uncomment the following lines
      # Note that kafka depends on zookeeper, so zookeeper must be started beforehand.
      # - use: zookeeper
      #   persist-data: true
      # - use: kafka
      #   persist-data: true

  default-v6:
    steps:
      - use: meta-node
        address: "[::1]"
        listen-address: "[::]"
      - use: compute-node
        address: "[::1]"
        listen-address: "[::]"
      - use: frontend
        address: "[::1]"
        listen-address: "[::]"

  # The minimum config to use with risectl.
  for-ctl:
    config-path: src/config/default.toml
    steps:
      - use: minio
      - use: meta-node
      - use: compute-node
      - use: frontend
      - use: compactor

  # `dev-compute-node` have the same settings as default except the the compute node will be started by user.
  dev-compute-node:
    steps:
      - use: meta-node
      - use: compute-node
        user-managed: true
      - use: frontend

  dev-frontend:
    steps:
      - use: meta-node
      - use: compute-node
      - use: frontend
        user-managed: true

  dev-meta:
    steps:
      - use: meta-node
        user-managed: true
      - use: compute-node
      - use: frontend

  full:
    steps:
      - use: minio
      - use: etcd
      - use: meta-node
      - use: compute-node
      - use: frontend
      - use: compactor
      - use: prometheus
      - use: grafana
      - use: zookeeper
        persist-data: true
      - use: kafka
        persist-data: true

  full-compaction-test:
    config-path: src/config/compaction-test.toml
    steps:
      - use: aws-s3
        bucket: some-bucket # configure a valid bucket name
      - use: etcd
      - use: meta-node
      - use: compute-node
      - use: frontend
      - use: compactor
      - use: prometheus
      - use: grafana
      - use: zookeeper
        persist-data: true
      - use: kafka
        persist-data: true

  full-benchmark:
    steps:
      - use: minio
      - use: etcd
      - use: meta-node
      - use: compute-node
      - use: frontend
      - use: compactor
      - use: prometheus
        remote-write: true
        remote-write-region: "ap-southeast-1"
        remote-write-url: "https://aps-workspaces.ap-southeast-1.amazonaws.com/workspaces/ws-f3841dad-6a5c-420f-8f62-8f66487f512a/api/v1/remote_write"
      - use: grafana
      - use: zookeeper
        persist-data: true
      - use: kafka
        persist-data: true

  3etcd-3meta-1cn-1fe:
    config-path: src/config/default.toml
    steps:
      - use: etcd
        unsafe-no-fsync: true
        port: 2388
        peer-port: 2389
        exporter-port: 2379
      - use: etcd
        unsafe-no-fsync: true
        port: 12388
        peer-port: 12389
        exporter-port: 12379
      - use: etcd
        unsafe-no-fsync: true
        port: 22388
        peer-port: 22389
        exporter-port: 22379
      - use: meta-node
        port: 5690
        dashboard-port: 5691
        exporter-port: 1250
      - use: meta-node
        port: 15690
        dashboard-port: 15691
        exporter-port: 11250
      - use: meta-node
        port: 25690
        dashboard-port: 25691
        exporter-port: 21250
      - use: compute-node
      - use: frontend

  ##########################################
  ### Configuration used for ./risedev p ###
  ##########################################

  playground:
    config-path: src/config/playground.toml
    steps:
      - use: meta-node
      - use: compute-node
      - use: frontend

  playground-3cn:
    config-path: src/config/playground.toml
    steps:
      - use: meta-node
      - use: compute-node
        port: 5687
        exporter-port: 1222
      - use: compute-node
        port: 5688
        exporter-port: 1223
      - use: compute-node
        port: 5689
        exporter-port: 1224
      - use: frontend
      - use: compactor

  docker-playground:
    config-path: src/config/default.toml
    steps:
      - use: meta-node
        listen-address: "0.0.0.0"
        address: "127.0.0.1"
      - use: compute-node
        listen-address: "0.0.0.0"
        address: "127.0.0.1"
      - use: frontend
        listen-address: "0.0.0.0"
        address: "127.0.0.1"

  # The profile for the online playground
  online-docker-playground:
    config-path: src/config/playground.toml
    steps:
      - use: meta-node
        listen-address: "0.0.0.0"
        address: "127.0.0.1"
      - use: compute-node
        listen-address: "0.0.0.0"
        address: "127.0.0.1"
      - use: frontend
        listen-address: "0.0.0.0"
        address: "127.0.0.1"

  ######################################
  ### Configurations used in Compose ###
  ######################################

  compose:
    steps:
      - use: minio
        id: minio-0
        address: ${id}
        listen-address: "0.0.0.0"
        console-address: "0.0.0.0"

      - use: meta-node
        # Id must starts with `meta-node`, therefore to be picked up by other
        # components.
        id: meta-node-0

        # Advertise address can be `id`, so as to use docker's DNS. If running
        # in host network mode, we should use IP directly in this field.
        address: ${id}

        listen-address: "0.0.0.0"

      - use: compute-node
        id: compute-node-0
        listen-address: "0.0.0.0"
        address: ${id}

      - use: frontend
        id: frontend-node-0
        listen-address: "0.0.0.0"
        address: ${id}

      - use: compactor
        id: compactor-0
        listen-address: "0.0.0.0"
        address: ${id}

      - use: redpanda

      - use: prometheus
        id: prometheus-0
        listen-address: "0.0.0.0"
        address: ${id}

      - use: grafana
        listen-address: "0.0.0.0"
        address: ${id}
        id: grafana-0

      - use: etcd
        listen-address: "0.0.0.0"
        address: ${id}
        id: etcd-0

  # special config for deployment, see related PR for more information
  compose-3node-deploy:
    steps:
      # - use: minio
      #   id: minio-0
      #   address: ${dns-host:rw-source-0}
      #   listen-address: "0.0.0.0"
      #   console-address: "0.0.0.0"

      - use: aws-s3
        bucket: ${terraform:s3-bucket}

      # Not enabled by default as all previous benchmarks are not done with etcd.
      # Also we currently only support node-level docker volume tear down.
      # - use: etcd
      #   listen-address: "0.0.0.0"
      #   address: ${dns-host:rw-meta-0}
      #   id: etcd-0

      - use: meta-node
        # Id must starts with `meta-node`, therefore to be picked up by other
        # components.
        id: meta-node-0

        # Advertise address can be `id`, so as to use docker's DNS. If running
        # in host network mode, we should use IP directly in this field.
        address: ${dns-host:rw-meta-0}
        listen-address: "0.0.0.0"

      - use: compute-node
        id: compute-node-0
        listen-address: "0.0.0.0"
        address: ${dns-host:rw-compute-0}
        async-stack-trace: verbose
        enable-tiered-cache: true

      - use: compute-node
        id: compute-node-1
        listen-address: "0.0.0.0"
        address: ${dns-host:rw-compute-1}
        async-stack-trace: verbose
        enable-tiered-cache: true

      - use: compute-node
        id: compute-node-2
        listen-address: "0.0.0.0"
        address: ${dns-host:rw-compute-2}
        async-stack-trace: verbose
        enable-tiered-cache: true

      - use: frontend
        id: frontend-node-0
        listen-address: "0.0.0.0"
        address: ${dns-host:rw-meta-0}

      - use: compactor
        id: compactor-0
        listen-address: "0.0.0.0"
        address: ${dns-host:rw-source-0}
        max-concurrent-task-number: 15
        compaction-worker-threads-number: 15

      - use: redpanda
        address: ${dns-host:rw-source-0}

      - use: prometheus
        id: prometheus-0
        listen-address: "0.0.0.0"
        address: ${dns-host:rw-meta-0}

      - use: grafana
        listen-address: "0.0.0.0"
        address: ${dns-host:rw-meta-0}
        id: grafana-0

  #################################
  ### Configurations used on CI ###
  #################################

  ci-1cn-1fe:
    config-path: src/config/ci.toml
    steps:
      - use: minio
      - use: etcd
        unsafe-no-fsync: true
      - use: meta-node
      - use: compute-node
        enable-tiered-cache: true
      - use: frontend
      - use: compactor

  ci-3cn-1fe:
    config-path: src/config/ci.toml
    steps:
      - use: minio
      - use: etcd
        unsafe-no-fsync: true
      - use: meta-node
      - use: compute-node
        port: 5687
        exporter-port: 1222
        enable-tiered-cache: true
      - use: compute-node
        port: 5688
        exporter-port: 1223
        enable-tiered-cache: true
      - use: compute-node
        port: 5689
        exporter-port: 1224
        enable-tiered-cache: true
      - use: frontend
      - use: compactor

  ci-3cn-3fe:
    config-path: src/config/ci.toml
    steps:
      - use: minio
      - use: etcd
        unsafe-no-fsync: true
      - use: meta-node
      - use: compute-node
        port: 5687
        exporter-port: 1222
        enable-tiered-cache: true
      - use: compute-node
        port: 5688
        exporter-port: 1223
        enable-tiered-cache: true
      - use: compute-node
        port: 5689
        exporter-port: 1224
        enable-tiered-cache: true
      - use: frontend
        port: 4565
        exporter-port: 2222
        health-check-port: 6786
      - use: frontend
        port: 4566
        exporter-port: 2223
        health-check-port: 6787
      - use: frontend
        port: 4567
        exporter-port: 2224
        health-check-port: 6788
      - use: compactor

  ci-3cn-3fe-in-memory:
    config-path: src/config/ci.toml
    steps:
      - use: etcd
        unsafe-no-fsync: true
      - use: meta-node
      - use: compute-node
        port: 5687
        exporter-port: 1222
        enable-in-memory-kv-state-backend: true
      - use: compute-node
        port: 5688
        exporter-port: 1223
        enable-in-memory-kv-state-backend: true
      - use: compute-node
        port: 5689
        exporter-port: 1224
        enable-in-memory-kv-state-backend: true
      - use: frontend
        port: 4565
        exporter-port: 2222
        health-check-port: 6786
      - use: frontend
        port: 4566
        exporter-port: 2223
        health-check-port: 6787
      - use: frontend
        port: 4567
        exporter-port: 2224
        health-check-port: 6788

  ci-pubsub:
    config-path: src/config/ci.toml
    steps:
      - use: minio
      - use: etcd
        unsafe-no-fsync: true
      - use: meta-node
      - use: compute-node
        enable-tiered-cache: true
      - use: frontend
      - use: compactor
      - use: pubsub
        persist-data: true

  ci-kafka:
    config-path: src/config/ci.toml
    steps:
      - use: minio
      - use: etcd
        unsafe-no-fsync: true
      - use: meta-node
      - use: compute-node
        enable-tiered-cache: true
      - use: frontend
      - use: compactor
      - use: zookeeper
        persist-data: true
      - use: kafka
        persist-data: true

  ci-kafka-plus-pubsub:
    config-path: src/config/ci.toml
    steps:
      - use: minio
      - use: etcd
        unsafe-no-fsync: true
      - use: meta-node
      - use: compute-node
        enable-tiered-cache: true
      - use: frontend
      - use: compactor
      - use: zookeeper
        persist-data: true
      - use: kafka
        persist-data: true
      - use: pubsub
        persist-data: true

  ci-redis:
    config-path: src/config/ci.toml
    steps:
      - use: minio
      - use: etcd
        unsafe-no-fsync: true
      - use: meta-node
      - use: compute-node
        enable-tiered-cache: true
      - use: frontend
      - use: compactor
      - use: redis

  ci-compaction-test:
    config-path: src/config/ci-compaction-test.toml
    steps:
      - use: minio
      - use: etcd
        unsafe-no-fsync: true
      - use: meta-node
      - use: compute-node
        enable-tiered-cache: true
      - use: frontend
      - use: compactor

  ci-1cn-1fe-with-recovery:
    config-path: src/config/ci-recovery.toml
    steps:
      - use: minio
      - use: etcd
      - use: meta-node
      - use: compute-node
        enable-tiered-cache: true
      - use: frontend
      - use: compactor

compose:
  risingwave: "ghcr.io/risingwavelabs/risingwave:latest"
  prometheus: "prom/prometheus:latest"
  minio: "quay.io/minio/minio:latest"
  redpanda: "docker.vectorized.io/vectorized/redpanda:latest"
  grafana: "grafana/grafana-oss:latest"
  etcd: "quay.io/coreos/etcd:latest"

# The `use` field specified in the above `risedev` section will refer to the templates below.
template:
  minio:
    # Advertise address of MinIO s3 endpoint
    address: "127.0.0.1"

    # Advertise port of MinIO s3 endpoint
    port: 9301

    # Listen address of MinIO endpoint
    listen-address: ${address}

    # Console address of MinIO s3 endpoint
    console-address: "127.0.0.1"

    # Console port of MinIO s3 endpoint
    console-port: 9400

    # Root username (can be used to login to MinIO console)
    root-user: hummockadmin

    # Root password (can be used to login to MinIO console)
    root-password: hummockadmin

    # Bucket name to store hummock information
    hummock-bucket: hummock001

    # Id of this instance
    id: minio

    # Prometheus nodes used by this MinIO
    provide-prometheus: "prometheus*"

  etcd:
    # Id of this instance
    id: etcd-${port}

    # Advertise address of the single-node etcd.
    address: "127.0.0.1"

    # Listen port of the single-node etcd.
    port: 2388

    # Listen address
    listen-address: ${address}

    # Peer listen port of the single-node etcd.
    peer-port: 2389

    # Prometheus exporter listen port
    exporter-port: 2379

    # Whether to enable fsync (NEVER SET TO TRUE IN PRODUCTION ENVIRONMENT!)
    unsafe-no-fsync: false

    # Other etcd nodes
    provide-etcd: "etcd*"

  compute-node:
    # Compute-node advertise address
    address: "127.0.0.1"

    # Listen address
    listen-address: ${address}

    # Compute-node listen port
    port: 5688

    # Prometheus exporter listen port
    exporter-port: 1222

    # Id of this instance
    id: compute-node-${port}

    # Whether to enable async stack trace for this compute node, `off`, `on`, or `verbose`.
    # Considering the performance, `verbose` mode only effect under `release` profile with `debug_assertions` off.
    async-stack-trace: on

    # If `enable-tiered-cache` is true, hummock will use data directory as file cache.
    enable-tiered-cache: false

    # RPC endpoint for connector node
    connector-rpc-endpoint: "127.0.0.1:60061"

    # Minio instances used by this compute node
    provide-minio: "minio*"

    # AWS s3 bucket used by this compute node
    provide-aws-s3: "aws-s3*"

    # Meta-nodes used by this compute node
    provide-meta-node: "meta-node*"

    # Jaeger used by this compute node
    provide-jaeger: "jaeger*"

    # Sanity check: should use shared storage if there're multiple compute nodes
    provide-compute-node: "compute-node*"

    # Sanity check: should start at lease one compactor if using shared object store
    provide-compactor: "compactor*"

    # If `user-managed` is true, this service will be started by user with the above config
    user-managed: false

    # Whether to enable in-memory pure KV state backend
    enable-in-memory-kv-state-backend: false

  meta-node:
    # Meta-node advertise address
    address: "127.0.0.1"

    # Meta-node listen port
    port: 5690

    # Listen address
    listen-address: ${address}

    # Dashboard listen port
    dashboard-port: 5691

    # Prometheus exporter listen port
    exporter-port: 1250

    # Id of this instance
    id: meta-node-${port}

    # If `user-managed` is true, this service will be started by user with the above config
    user-managed: false

    # Etcd backend config
    provide-etcd-backend: "etcd*"

    # Prometheus nodes used by dashboard service
    provide-prometheus: "prometheus*"

  prometheus:
    # Advertise address of Prometheus
    address: "127.0.0.1"

    # Listen port of Prometheus
    port: 9500

    # Listen address
    listen-address: ${address}

    # Id of this instance
    id: prometheus

    # If `remote_write` is true, this Prometheus instance will push metrics to remote instance
    remote-write: false

    # AWS region of remote write
    remote-write-region: ""

    # Remote write url of this instance
    remote-write-url: ""

    # Compute-nodes used by this Prometheus instance
    provide-compute-node: "compute-node*"

    # Meta-nodes used by this Prometheus instance
    provide-meta-node: "meta-node*"

    # Minio instances used by this Prometheus instance
    provide-minio: "minio*"

    # Compactors used by this Prometheus instance
    provide-compactor: "compactor*"

    # Etcd used by this Prometheus instance
    provide-etcd: "etcd*"

    # Redpanda used by this Prometheus instance
    provide-redpanda: "redpanda*"

    # Frontend used by this Prometheus instance
    provide-frontend: "frontend*"

  frontend:
    # Advertise address of frontend
    address: "127.0.0.1"

    # Listen port of frontend
    port: 4566

    # Listen address
    listen-address: ${address}

    # Prometheus exporter listen port
    exporter-port: 2222

    # Health check listen port
    health-check-port: 6786

    # Id of this instance
    id: frontend-${port}

    # Meta-nodes used by this frontend instance
    provide-meta-node: "meta-node*"

    # If `user-managed` is true, this service will be started by user with the above config
    user-managed: false

  compactor:
    # Compactor advertise address
    address: "127.0.0.1"

    # Compactor listen port
    port: 6660

    # Listen address
    listen-address: ${address}

    # Prometheus exporter listen port
    exporter-port: 1260

    # Id of this instance
    id: compactor-${port}

    # Minio instances used by this compute node
    provide-minio: "minio*"

    # AWS s3 bucket used by this compute node
    provide-aws-s3: "aws-s3*"

    # Meta-nodes used by this compute node
    provide-meta-node: "meta-node*"

    # If `user-managed` is true, this service will be started by user with the above config
    user-managed: false

    # Hint used by meta node
    max-concurrent-task-number: 15

  connector-node:
    # Connector node advertise address
    address: "127.0.0.1"

    # Connector node listen port
    port: 60061

    # Id of this instance
    id: connector-${port}

  grafana:
    # Listen address of Grafana
    listen-address: ${address}

    # Advertise address of Grafana
    address: "127.0.0.1"

    # Listen port of Grafana
    port: 3001

    # Id of this instance
    id: grafana

    # Prometheus used by this Grafana instance
    provide-prometheus: "prometheus*"

  jaeger:
    # Id of this instance
    id: jaeger

    # Dashboard listen address of Jaeger
    dashboard-address: "127.0.0.1"

    # Dashboard listen port of Jaeger
    dashboard-port: 16680

    # Jaeger has a lot of ports open, and we don't want to make this config more complex.
    # So we keep the default value of jaeger instead of making it part of RiseDev config.

  # aws-s3 is a placeholder service to provide configurations
  aws-s3:
    # Id to be picked-up by services
    id: aws-s3

    # The bucket to be used for AWS S3
    bucket: test-bucket

    # access key, secret key and region should be set in aws config (either by env var or .aws/config)

    # used to support other s3 compatible object store.
    s3-compatible: false

  # Apache Kafka service
  kafka:
    # Id to be picked-up by services
    id: kafka-${port}

    # Advertise address of Kafka
    address: "127.0.0.1"

    # Listen port of Kafka
    port: 29092

    # Listen address
    listen-address: ${address}

    # ZooKeeper used by this Kafka instance
    provide-zookeeper: "zookeeper*"

    # If set to true, data will be persisted at data/{id}.
    persist-data: true

    # Kafka broker id. If there are multiple instances of Kafka, we will need to set.
    broker-id: 0

  # Google pubsub emulator service
  pubsub:
    id: pubsub-${port}

    address: "127.0.0.1"

    port: 5980

    persist-data: true

  # Apache ZooKeeper service
  zookeeper:
    # Id to be picked-up by services
    id: zookeeper-${port}

    # Advertise address of ZooKeeper
    address: "127.0.0.1"

    # Listen address
    listen-address: ${address}

    # Listen port of ZooKeeper
    port: 2181

    # If set to true, data will be persisted at data/{id}.
    persist-data: true

  # Only supported in RiseDev compose
  redpanda:
    # Id to be picked-up by services
    id: redpanda

    # Port used inside docker-compose cluster (e.g. create MV)
    internal-port: 29092

    # Port used on host (e.g. import data, connecting using kafkacat)
    outside-port: 9092

    # Connect address
    address: ${id}

    # Number of CPUs to use
    cpus: 8

    # Memory limit for Redpanda
    memory: 16G

  # redis service
  redis:
    # Id to be picked-up by services
    id: redis

    # listen port of redis
    port: 6379

    # address of redis
    address: "127.0.0.1"