docker-compose.yaml

x-airflow-common: &airflow-common
  build:
    context: ./docker/airflow
    dockerfile: Dockerfile
  environment: &airflow-common-env
    AIRFLOW__CORE__EXECUTOR: LocalExecutor
    AIRFLOW_CONN_SPARK_DEFAULT: '{"conn_type": "spark", "host": "spark://spark-master", "port": 7077}'
    AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow
    AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW__CORE__FERNET_KEY}
    AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "true"
    AIRFLOW__CORE__LOAD_EXAMPLES: "false"
    AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "true"
    # GCS / BigQuery auth
    GOOGLE_APPLICATION_CREDENTIALS: /opt/airflow/secrets/gcp-key.json
    GCP_PROJECT_ID: ${GCP_PROJECT_ID}
    GCS_BUCKET: ${GCS_BUCKET}
    GCP_DATASET: ${GCP_DATASET}
    GCP_LOCATION: ${GCP_LOCATION:-EU}
    DBT_PROJECT_DIR: /opt/airflow/dbt
  volumes:
    - ./dags:/opt/airflow/dags
    - ./logs:/opt/airflow/logs
    - ./plugins:/opt/airflow/plugins
    - ./spark/jobs:/opt/airflow/spark_jobs
    - ./spark/jobs:/opt/airflow/dags/jobs 
    - ./dbt:/opt/airflow/dbt                     # dbt project — run via BashOperator in DAGs
    - ./secrets/gcp-key.json:/opt/airflow/secrets/gcp-key.json:ro
    - ./secrets/gcp-key.json:/opt/spark/secrets/gcp-key.json:ro
  networks:
    - data_net
  depends_on:
    postgres:
      condition: service_healthy

services:

  # ─────────────────────────────────────────────
  # Metadata DB for Airflow
  # ─────────────────────────────────────────────
  postgres:
    image: postgres:15
    container_name: de_postgres
    environment:
      POSTGRES_USER: airflow
      POSTGRES_PASSWORD: airflow
      POSTGRES_DB: airflow
    volumes:
      - postgres_data:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD", "pg_isready", "-U", "airflow"]
      interval: 10s
      retries: 5
      start_period: 5s
    networks:
      - data_net

  # ─────────────────────────────────────────────
  # Terraform — Infrastructure Provisioning
  # ─────────────────────────────────────────────
  terraform:
    image: hashicorp/terraform:latest
    container_name: de_terraform
    working_dir: /workspace
    volumes:
      - ./terraform:/workspace
      - ./secrets/gcp-key.json:/workspace/secrets/gcp-key.json:ro
    environment:
      - TF_VAR_project=${GCP_PROJECT_ID}
      - TF_VAR_gcs_bucket_name=${GCS_BUCKET}
      - TF_VAR_bq_dataset_name=${GCP_DATASET}
      - TF_VAR_region=${GCP_REGION:-europe-west1}
      - TF_VAR_location=${GCP_LOCATION:-EU}
    entrypoint: [""]
    command: ["/bin/sh", "-c", "terraform init && terraform apply -auto-approve"]

  # ─────────────────────────────────────────────
  # Airflow — DB init (runs once)
  # ─────────────────────────────────────────────
  airflow-init:
    <<: *airflow-common
    container_name: de_airflow_init
    entrypoint: /bin/bash
    command:
      - -c
      - |
        airflow db migrate
        airflow users create \
          --username admin \
          --password admin \
          --firstname Admin \
          --lastname User \
          --role Admin \
          --email admin@example.com
    restart: "no"
    depends_on:
      terraform:
        condition: service_completed_successfully

  # ─────────────────────────────────────────────
  # Airflow — Scheduler
  # ─────────────────────────────────────────────
  airflow-scheduler:
    <<: *airflow-common
    container_name: de_airflow_scheduler
    command: scheduler
    restart: unless-stopped
    depends_on:
      airflow-init:
        condition: service_completed_successfully

  # ─────────────────────────────────────────────
  # Airflow — Webserver  (http://localhost:8080)
  # ─────────────────────────────────────────────
  airflow-webserver:
    <<: *airflow-common
    container_name: de_airflow_webserver
    command: webserver
    ports:
      - "8080:8080"
    restart: unless-stopped
    depends_on:
      airflow-init:
        condition: service_completed_successfully
    healthcheck:
      test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 30s

  # ─────────────────────────────────────────────
  # Spark — Master  (UI: http://localhost:8090)
  # ─────────────────────────────────────────────
  spark-master:
    build:
      context: ./docker/spark
      dockerfile: Dockerfile
    container_name: de_spark_master
    entrypoint: /opt/spark/bin/spark-class        # ← apache/spark path
    command: org.apache.spark.deploy.master.Master # ← java class, not "master"
    restart: unless-stopped
    environment:
      - SPARK_MASTER_HOST=spark-master
      - SPARK_NO_DAEMONIZE=true
      - GOOGLE_APPLICATION_CREDENTIALS=/opt/spark/secrets/gcp-key.json
      - GCP_PROJECT_ID=${GCP_PROJECT_ID}
    ports:
      - "8090:8080"
      - "7077:7077"
    volumes:
      - ./spark/jobs:/opt/spark/jobs
      - ./secrets/gcp-key.json:/opt/spark/secrets/gcp-key.json:ro
    networks:
      - data_net

  spark-worker:
    build:
      context: ./docker/spark
      dockerfile: Dockerfile
    container_name: de_spark_worker
    entrypoint: /opt/spark/bin/spark-class                            # ← apache/spark path
    command: org.apache.spark.deploy.worker.Worker spark://spark-master:7077
    restart: unless-stopped
    environment:
      - SPARK_WORKER_MEMORY=2G
      - SPARK_WORKER_CORES=2
      - SPARK_NO_DAEMONIZE=true
      - GOOGLE_APPLICATION_CREDENTIALS=/opt/spark/secrets/gcp-key.json
    volumes:
      - ./spark/jobs:/opt/spark/jobs
      - ./secrets/gcp-key.json:/opt/spark/secrets/gcp-key.json:ro
    depends_on:
      - spark-master
    networks:
      - data_net


networks:
  data_net:
    driver: bridge

volumes:
  postgres_data: