docker-compose.yml for airflow

Apache Airflow 기반의 데이터 파이프라인 책을 보다가, 오타가 있어 수정을 하고,

리스트 2.10 도커에서 Airflow 실행하기

$ docker run -it \
    -p 8080:8080 \
    -v ${PWD}/dags/:/opt/airflow/dags/ \
    --entrypoint=/bin/bash \
    --name airflow \
    apache/airflow:2.5.3-python3.8 \
    -c '( \
        airflow db init && \
        airflow users create \
            --username admin \
            --password admin \
            --firstname Anonymous \
            --lastname Admin \
            --role Admin \
            --email admin@example.org \
    ); \
    airflow webserver & \
    airflow scheduler \
    '

책의 github repository를 방문해 보게되었습니다.

책 repository에서 한번에 실행이 잘 안되어, 책에 적혀 있는 명령어와 github repository를 조합하여, 한번에 실행 가능하게 작성해 보았습니다.

version: '3.7'

# ====================================== AIRFLOW ENVIRONMENT VARIABLES =======================================
x-environment: &airflow_environment
  AIRFLOW__CORE__EXECUTOR: LocalExecutor
  AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS: False
  AIRFLOW__CORE__LOAD_EXAMPLES: False
  AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql://airflow:airflow@postgres:5432/airflow
  AIRFLOW__CORE__STORE_DAG_CODE: True
  AIRFLOW__CORE__STORE_SERIALIZED_DAGS: True
  AIRFLOW__WEBSERVER__EXPOSE_CONFIG: True
# ====================================== /AIRFLOW ENVIRONMENT VARIABLES ======================================

services:
  postgres:
    image: postgres:13.10-alpine
    environment:
      - POSTGRES_USER=airflow
      - POSTGRES_PASSWORD=airflow
      - POSTGRES_DB=airflow
    ports:
      - "5432:5432"
    volumes:
      - ./postgres/:/var/lib/postgresql/data/

  airflow:
    image: apache/airflow:2.5.3-python3.8
    depends_on:
      - postgres
    ports:
      - "8080:8080"
    environment:
      <<: *airflow_environment
    volumes:
      - ./logs:/opt/airflow/logs
      - ./dags:/opt/airflow/dags
    command:
      - bash
      - -c
      - | # https://yaml-multiline.info/
        airflow db init && 
        airflow users create --username admin --password admin --firstname Anonymous --lastname Admin --role Admin --email admin@example.org;
        airflow webserver &
        airflow scheduler;

https://github.com/youngkiu/airflow_docker

docker-compose up 실행 후, 모습입니다.