From 5c069c7addccd1f91e9b0dfc32ac060d22628f26 Mon Sep 17 00:00:00 2001 From: Barbara Hui Date: Fri, 28 Jul 2023 14:54:42 -0700 Subject: [PATCH 1/7] Modify env var names and add to .env.localrunner file --- docker/config/.env.localrunner | 5 +++++ docker/docker-compose-local.yml | 8 ++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docker/config/.env.localrunner b/docker/config/.env.localrunner index 04859fb09..91ca7971f 100644 --- a/docker/config/.env.localrunner +++ b/docker/config/.env.localrunner @@ -9,3 +9,8 @@ DEFAULT_PASSWORD="test" S3_DAGS_PATH="" S3_PLUGINS_PATH="" S3_REQUIREMENTS_PATH="" + +DAGS_HOME="" +PLUGINS_HOME="" +REQS_HOME="" +STARTUP_HOME="" diff --git a/docker/docker-compose-local.yml b/docker/docker-compose-local.yml index 8699cc6f6..2f2d9317c 100644 --- a/docker/docker-compose-local.yml +++ b/docker/docker-compose-local.yml @@ -26,10 +26,10 @@ services: max-size: 10m max-file: "3" volumes: - - "${PWD}/dags:/usr/local/airflow/dags" - - "${PWD}/plugins:/usr/local/airflow/plugins" - - "${PWD}/requirements:/usr/local/airflow/requirements" - - "${PWD}/startup_script:/usr/local/airflow/startup" + - "${DAGS_HOME}:/usr/local/airflow/dags" + - "${PLUGINS_HOME}:/usr/local/airflow/plugins" + - "${REQS_HOME}:/usr/local/airflow/requirements" + - "${STARTUP_HOME}:/usr/local/airflow/startup" ports: - "8080:8080" command: local-runner From 589788a148c7d248aa0ea7ed7475c63a7f9150b4 Mon Sep 17 00:00:00 2001 From: Barbara Hui Date: Fri, 28 Jul 2023 14:36:33 -0700 Subject: [PATCH 2/7] Add pull_upstream github workflow --- .github/workflows/pull_upstream.yml | 42 +++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/pull_upstream.yml diff --git a/.github/workflows/pull_upstream.yml b/.github/workflows/pull_upstream.yml new file mode 100644 index 000000000..6d1d0f307 --- /dev/null +++ b/.github/workflows/pull_upstream.yml @@ -0,0 +1,42 @@ +name: Sync Upstream + +# https://github.com/marketplace/actions/sync-and-merge-upstream-repository-with-your-current-repository +env: + UPSTREAM_URL: "https://github.com/aws/aws-mwaa-local-runner.git" + WORKFLOW_TOKEN: ${{ secrets.WORKFLOW_TOKEN }} + UPSTREAM_BRANCH: "v2.5.1" + # Optional, defaults to UPSTREAM_BRANCH + DOWNSTREAM_BRANCH: "" + # Optional fetch arguments + FETCH_ARGS: "" + # Optional merge arguments + MERGE_ARGS: "" + # Optional push arguments + PUSH_ARGS: "" + # Optional toggle to spawn time logs (keeps action active) + SPAWN_LOGS: "false" # "true" or "false" + +# This runs every day on 1201 UTC +on: + schedule: + - cron: '1 12 * * *' + # Allows manual workflow run (must be in default branch to work) + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: GitHub Sync to Upstream Repository + uses: dabreadman/sync-upstream-repo@v1.3.0 + with: + upstream_repo: ${{ env.UPSTREAM_URL }} + upstream_branch: ${{ env.UPSTREAM_BRANCH }} + downstream_branch: ${{ env.DOWNSTREAM_BRANCH }} + token: ${{ env.WORKFLOW_TOKEN }} + fetch_args: ${{ env.FETCH_ARGS }} + merge_args: ${{ env.MERGE_ARGS }} + push_args: ${{ env.PUSH_ARGS }} + spawn_logs: ${{ env.SPAWN_LOGS }} + + From 2c9181f13191424fed897e45053e0bb96cefa6ed Mon Sep 17 00:00:00 2001 From: Barbara Hui Date: Fri, 4 Aug 2023 15:19:12 -0700 Subject: [PATCH 3/7] Add .env file for env vars used in docker-compose files. Remove these vars from .env.localrunner --- docker/.env | 4 ++++ docker/config/.env.localrunner | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 docker/.env diff --git a/docker/.env b/docker/.env new file mode 100644 index 000000000..d6283d7f7 --- /dev/null +++ b/docker/.env @@ -0,0 +1,4 @@ +DAGS_HOME="/Users/bhui/dev/rikolti/airflow/dags" +PLUGINS_HOME="/Users/bhui/dev/rikolti/airflow/plugins" +REQS_HOME="/Users/bhui/dev/rikolti/airflow" +STARTUP_HOME="/Users/bhui/dev/rikolti/airflow" diff --git a/docker/config/.env.localrunner b/docker/config/.env.localrunner index 91ca7971f..b7cc4fb86 100644 --- a/docker/config/.env.localrunner +++ b/docker/config/.env.localrunner @@ -10,7 +10,3 @@ S3_DAGS_PATH="" S3_PLUGINS_PATH="" S3_REQUIREMENTS_PATH="" -DAGS_HOME="" -PLUGINS_HOME="" -REQS_HOME="" -STARTUP_HOME="" From b1304159d9b1387095535e947ee4e1b31bd62aa8 Mon Sep 17 00:00:00 2001 From: Barbara Hui Date: Fri, 4 Aug 2023 15:21:32 -0700 Subject: [PATCH 4/7] Remove Barbara-specific values from .env file --- docker/.env | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docker/.env b/docker/.env index d6283d7f7..aa5958a5f 100644 --- a/docker/.env +++ b/docker/.env @@ -1,4 +1,5 @@ -DAGS_HOME="/Users/bhui/dev/rikolti/airflow/dags" -PLUGINS_HOME="/Users/bhui/dev/rikolti/airflow/plugins" -REQS_HOME="/Users/bhui/dev/rikolti/airflow" -STARTUP_HOME="/Users/bhui/dev/rikolti/airflow" +# Environment variables set in this file will be used in the docker-compose files stored in the same directory +DAGS_HOME="" +PLUGINS_HOME="" +REQS_HOME="" +STARTUP_HOME="" From ada03597e60b54e5defa0c4f8690db66bf99824e Mon Sep 17 00:00:00 2001 From: Amy Wieliczka Date: Fri, 18 Aug 2023 14:45:06 -0500 Subject: [PATCH 5/7] Enable tasks to write to localhost; allow parameters --- docker/.env | 1 + docker/config/airflow.cfg | 2 +- docker/docker-compose-local.yml | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/.env b/docker/.env index aa5958a5f..adf129dc7 100644 --- a/docker/.env +++ b/docker/.env @@ -3,3 +3,4 @@ DAGS_HOME="" PLUGINS_HOME="" REQS_HOME="" STARTUP_HOME="" +RIKOLTI_DATA_HOME="" \ No newline at end of file diff --git a/docker/config/airflow.cfg b/docker/config/airflow.cfg index b10aae89d..43932e88e 100644 --- a/docker/config/airflow.cfg +++ b/docker/config/airflow.cfg @@ -165,7 +165,7 @@ killed_task_cleanup_time = 60 # Whether to override params with dag_run.conf. If you pass some key-value pairs # through ``airflow dags backfill -c`` or # ``airflow dags trigger -c``, the key-value pairs will override the existing ones in params. -dag_run_conf_overrides_params = False +dag_run_conf_overrides_params = True # When discovering DAGs, ignore any files that don't contain the strings ``DAG`` and ``airflow``. dag_discovery_safe_mode = True diff --git a/docker/docker-compose-local.yml b/docker/docker-compose-local.yml index 2f2d9317c..95d0950f4 100644 --- a/docker/docker-compose-local.yml +++ b/docker/docker-compose-local.yml @@ -30,6 +30,7 @@ services: - "${PLUGINS_HOME}:/usr/local/airflow/plugins" - "${REQS_HOME}:/usr/local/airflow/requirements" - "${STARTUP_HOME}:/usr/local/airflow/startup" + - "${RIKOLTI_DATA_HOME}:/usr/local/airflow/rikolti_data" ports: - "8080:8080" command: local-runner From 32e269b08134fa2b17b061b91c539202273fe3d2 Mon Sep 17 00:00:00 2001 From: Amy Wieliczka Date: Mon, 21 Aug 2023 14:22:31 -0700 Subject: [PATCH 6/7] Mount the contents of the rikolti repo to a dags/rikolti/ folder --- docker/docker-compose-local.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose-local.yml b/docker/docker-compose-local.yml index 95d0950f4..1080f91e4 100644 --- a/docker/docker-compose-local.yml +++ b/docker/docker-compose-local.yml @@ -26,7 +26,7 @@ services: max-size: 10m max-file: "3" volumes: - - "${DAGS_HOME}:/usr/local/airflow/dags" + - "${DAGS_HOME}:/usr/local/airflow/dags/rikolti" - "${PLUGINS_HOME}:/usr/local/airflow/plugins" - "${REQS_HOME}:/usr/local/airflow/requirements" - "${STARTUP_HOME}:/usr/local/airflow/startup" From 4bf5b692309b0e0fd3d78ef95cb1e0cb27f7f204 Mon Sep 17 00:00:00 2001 From: amy wieliczka Date: Fri, 29 Sep 2023 15:45:17 -0700 Subject: [PATCH 7/7] Feat: enable DockerOperator usage (#4) * mount docker.sock & set perms; required for DockerOperator usage :socks: * Move docker/.env into docker/.env.example --- .gitignore | 1 + docker/{.env => .env.example} | 3 ++- docker/docker-compose-local.yml | 1 + docker/script/run-startup.sh | 6 +++++- 4 files changed, 9 insertions(+), 2 deletions(-) rename docker/{.env => .env.example} (82%) diff --git a/.gitignore b/.gitignore index 849b35b33..88ddd02f3 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ dags/**/*.pyc /logs /db-data .DS_Store +/docker/.env \ No newline at end of file diff --git a/docker/.env b/docker/.env.example similarity index 82% rename from docker/.env rename to docker/.env.example index adf129dc7..a6b312cf0 100644 --- a/docker/.env +++ b/docker/.env.example @@ -3,4 +3,5 @@ DAGS_HOME="" PLUGINS_HOME="" REQS_HOME="" STARTUP_HOME="" -RIKOLTI_DATA_HOME="" \ No newline at end of file +RIKOLTI_DATA_HOME="" +DOCKER_SOCKET="" \ No newline at end of file diff --git a/docker/docker-compose-local.yml b/docker/docker-compose-local.yml index 1080f91e4..eb55a8b52 100644 --- a/docker/docker-compose-local.yml +++ b/docker/docker-compose-local.yml @@ -31,6 +31,7 @@ services: - "${REQS_HOME}:/usr/local/airflow/requirements" - "${STARTUP_HOME}:/usr/local/airflow/startup" - "${RIKOLTI_DATA_HOME}:/usr/local/airflow/rikolti_data" + - "${DOCKER_SOCKET}:/var/run/docker.sock" ports: - "8080:8080" command: local-runner diff --git a/docker/script/run-startup.sh b/docker/script/run-startup.sh index eb1e8013f..083dd0546 100644 --- a/docker/script/run-startup.sh +++ b/docker/script/run-startup.sh @@ -1,4 +1,8 @@ #!/usr/bin/env bash source "$AIRFLOW_HOME/startup/startup.sh" -declare -p | grep -v '^declare \-[aAilnrtux]*r ' > stored_env \ No newline at end of file +declare -p | grep -v '^declare \-[aAilnrtux]*r ' > stored_env + +# allows the airflow user on the airflow docker container to access +# the host's docker.sock. This is needed for the DockerOperator to work. +sudo chmod 666 /var/run/docker.sock \ No newline at end of file