docker-compose.yml

version: '3.3'

services:
  # Primary services
  app:
    build:
      context: .
      dockerfile: docker/Dockerfile
    env_file: .env
    image: pemi-app
    container_name: pemi-app
    command: jupyter lab . --ip=0.0.0.0 --allow-root --no-browser
    depends_on:
      - db
    networks:
      - services
    volumes:
      - ./.data/services/ap/bash_history:/root/.bash_history
      - .:/app
      - ./.data/spark:/tmp/data
    ports:
      - "${HOST_JUPYTER_PORT}:8888"

  # Upstream services
  db:
    image: postgres:9.6
    container_name: pemi-db
    hostname: pemi-db
    env_file: .env
    volumes:
      - ./.data/db:/var/lib/postgresql/data
    ports:
      - "${HOST_POSTGRES_PORT}:5432"
    networks:
      - services

  spark-master:
    image: pemi-spark
    container_name: pemi-spark-master
    build:
      context: .
      dockerfile: docker/Dockerfile.spark
    command: bin/spark-class org.apache.spark.deploy.master.Master -h spark-master
    hostname: spark-master
    environment:
      MASTER: spark://spark-master:7077
      SPARK_CONF_DIR: /conf
      SPARK_PUBLIC_DNS: 127.0.0.1
    expose:
      - 7001
      - 7002
      - 7003
      - 7004
      - 7005
      - 7006
      - 7077
      - 6066
    ports:
      - 4040:4040
      - 6066:6066
      - 7077:7077
      - 8080:8080
    volumes:
      - ./docker/spark/conf/spark-master:/conf
      - ./.data/spark:/tmp/data
    networks:
      - services


  spark-worker-1:
    image: pemi-spark
    container_name: pemi-spark-worker-1
    command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077
    hostname: spark-worker-1
    environment:
      SPARK_CONF_DIR: /conf
      SPARK_PUBLIC_DNS: 127.0.0.1
      SPARK_WORKER_CORES: 2
      SPARK_WORKER_MEMORY: 2g
      SPARK_WORKER_PORT: 8881
      SPARK_WORKER_WEBUI_PORT: 8081
    links:
      - spark-master
    expose:
      - 7012
      - 7013
      - 7014
      - 7015
      - 7016
      - 8881
    ports:
      - 8081:8081
    volumes:
      - ./docker/spark/conf/spark-worker-1:/conf
      - ./.data/spark:/tmp/data
    networks:
      - services

  spark-worker-2:
    image: pemi-spark
    container_name: pemi-spark-worker-2
    command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077
    hostname: spark-worker-2
    environment:
      SPARK_CONF_DIR: /conf
      SPARK_PUBLIC_DNS: 127.0.0.1
      SPARK_WORKER_CORES: 2
      SPARK_WORKER_MEMORY: 2g
      SPARK_WORKER_PORT: 8882
      SPARK_WORKER_WEBUI_PORT: 8082
    links:
      - spark-master
    expose:
      - 7022
      - 7023
      - 7024
      - 7025
      - 7026
      - 8882
    ports:
      - 8082:8082
    volumes:
      - ./docker/spark/conf/spark-worker-2:/conf
      - ./.data/spark:/tmp/data
    networks:
      - services

networks:
  services: