Airflow Dagster - sgml/signature GitHub Wiki
task_scheduling_comparison:
github_actions:
primary_use_case: "CI/CD for GitHub repositories"
workflow_definition: "YAML-based workflows"
scheduling_mechanism: "Built-in cron scheduling (`schedule`)"
cost_efficiency: "Free for public repos, low-cost tiers"
integration_depth: "Deep GitHub integration"
error_handling: "Logs in GitHub UI"
scalability: "Optimized for CI/CD pipelines"
best_for_cicd: true
best_for_event_processing: false
webhooks_support: "✅ [GitHub Webhooks](https://docs.github.com/en/developers/webhooks-and-events/webhooks)"
license_type: "Open-source (MIT)"
vendor_lock_in: "High—GitHub Actions is tightly coupled with GitHub repositories"
localhost_emulation_complexity: "65%—Limited support for localhost execution, requires container-based emulation"
localhost_dockerization: "Supports local Docker execution, but GitHub-hosted runners restrict privileged container operations."
advantages_over_airflow: "Better suited for CI/CD workflows, simpler YAML-based configuration, native GitHub integration"
advantages_over_step_functions_for_etl: "Limited—Not optimized for ETL workflows, lacks built-in data processing capabilities"
circleci:
primary_use_case: "CI/CD pipelines for multiple VCS"
workflow_definition: "YAML-based workflows"
scheduling_mechanism: "Cron scheduling via `triggers`"
cost_efficiency: "Free-tier usage available"
integration_depth: "GitHub, Bitbucket, GitLab"
error_handling: "Debugging via CircleCI UI"
scalability: "High concurrency for CI/CD"
best_for_cicd: true
best_for_event_processing: false
webhooks_support: "✅ [CircleCI Webhooks](https://circleci.com/docs/webhooks)"
license_type: "Proprietary (Paid Plans)"
vendor_lock_in: "Moderate—Supports multiple VCS providers, but proprietary"
localhost_emulation_complexity: "75%—Docker-based local execution available, but cloud-native steps may differ"
localhost_dockerization: "Native support for local Docker execution via `circleci local execute`."
advantages_over_airflow: "Optimized for CI/CD pipelines, easier debugging, better Git integration"
advantages_over_step_functions_for_etl: "Limited—Not designed for complex ETL workflows, lacks built-in data transformation"
sqs:
primary_use_case: "Decoupled event-driven tasks"
workflow_definition: "Queue-based task handling"
scheduling_mechanism: "Event-driven execution"
cost_efficiency: "Pay-per-request pricing model"
integration_depth: "AWS services integration"
error_handling: "Built-in DLQ (Dead Letter Queue)"
scalability: "High-throughput message queuing"
best_for_cicd: false
best_for_event_processing: true
webhooks_support: "❌ No direct webhook support"
license_type: "Proprietary (AWS-managed)"
vendor_lock_in: "High—Deeply integrated into AWS ecosystem"
localhost_emulation_complexity: "40%—Partial emulation via LocalStack, lacks full AWS networking"
localhost_dockerization: "Not applicable—SQS does not require local Docker execution."
advantages_over_airflow: "Simpler event-driven execution, lower operational overhead, built-in retry mechanisms"
advantages_over_step_functions_for_etl: "Moderate—Can queue messages for event-driven ETL but lacks full orchestration capabilities"
dagster:
primary_use_case: "Data pipeline orchestration"
workflow_definition: "Python-native DAGs"
scheduling_mechanism: "Cron-based scheduling (`ScheduleDefinition`)"
cost_efficiency: "Open-source, self-hosted or managed"
integration_depth: "Native support for dbt, Snowflake, AWS"
error_handling: "Asset-centric lineage tracking"
scalability: "Modular, scalable data workflows"
best_for_cicd: false
best_for_event_processing: true
webhooks_support: "✅ [Dagster Webhooks](https://docs.dagster.io/concepts/partitions-schedules-sensors/sensors)"
license_type: "Open-source (Apache 2.0)"
vendor_lock_in: "Low—Self-hosted and open-source, flexible integrations"
localhost_emulation_complexity: "90%—Can fully run locally, minor cloud API differences"
localhost_dockerization: "Fully supports local Docker-based execution for development environments."
advantages_over_airflow: "Better asset-based lineage tracking, simpler Python-native DAGs, more modular execution"
advantages_over_step_functions_for_etl: "Limited—Focuses on batch processing rather than real-time event-driven ETL"
aws_glue:
primary_use_case: "ETL (Extract, Transform, Load)"
workflow_definition: "Glue Studio visual workflows"
scheduling_mechanism: "EventBridge-based scheduling"
cost_efficiency: "Pay-per-use pricing model"
integration_depth: "Deep AWS ecosystem integration"
error_handling: "Automatic retries & logging"
scalability: "Serverless, scalable ETL"
best_for_cicd: false
best_for_event_processing: true
webhooks_support: "❌ No direct webhook support"
license_type: "Proprietary (AWS-managed)"
vendor_lock_in: "High—Fully dependent on AWS infrastructure"
localhost_emulation_complexity: "30%—AWS Glue jobs require cloud execution, limited local testing"
localhost_dockerization: "Limited—Glue jobs require AWS-managed infrastructure and cannot run directly via local Docker."
advantages_over_airflow: "Fully managed ETL, serverless execution, deeper AWS integration"
advantages_over_step_functions_for_etl: "High—Simpler ETL orchestration, built-in data transformations, deeper integration with AWS-native ETL tools."
- https://github.com/puckel/docker-airflow/blob/master/script/entrypoint.sh
- https://github.com/xnuinside/airflow_in_docker_compose
- https://medium.com/@xnuinside/quick-guide-how-to-run-apache-airflow-cluster-in-docker-compose-615eb8abd67a
- https://towardsdatascience.com/getting-started-with-airflow-using-docker-cd8b44dbff98
- https://peterlyons.com/problog/2017/12/how-to-use-docker-to-run-local-databases/
- https://cwiki.apache.org/confluence/display/AIRFLOW/Airflow+Release+Planning+and+Supported+Release+Lifetime
- https://github.com/runodp/dagster-odp/blob/main/docs/concepts/concepts.md
- https://docs.dagster.io/deployment/guides/service
- https://www.youtube.com/watch?v=pyRwQByuFfc
- https://stackoverflow.com/questions/52054427/how-to-integrate-apache-airflow-with-slack
- https://airflow.apache.org/docs/stable/_api/airflow/operators/slack_operator/index.html
+-------------------+ +-------------------+
| | | |
| Apache Airflow | | Slack |
| | | |
+-------------------+ +-------------------+
| |
| |
| |
v v
+-------------------+ +-------------------+
| | | |
| DAG Execution | ----> | Slack Channel |
| | | |
+-------------------+ +-------------------+
- https://stlong0521.github.io/20161023%20-%20Airflow.html
- https://6chaoran.github.io/data-story/data-engineering/airflow-set-up/
- https://stlong0521.github.io/20161023%20-%20Airflow.html
- https://stackoverflow.com/questions/39862590/airflow-does-not-trigger-concurrent-dags-with-localexecutor
- https://stackoverflow.com/questions/55875683/apache-airflow-how-many-tasks-in-a-dag-is-too-many
- https://github.com/puckel/docker-airflow/issues/446
- https://airflow.readthedocs.io/en/stable/tutorial.html?highlight=example#example-pipeline-definition
- https://blog.freetrade.io/how-we-simplified-our-data-pipeline-54f377fad3c
- https://stackoverflow.com/questions/52389105/how-operator-defines-task-dependencies-in-airflow
- https://www.astronomer.io/guides/managing-dependencies/
- https://stackoverflow.com/questions/49856870/airflow-how-to-run-a-task-on-multiple-workers
- http://engineering.pmc.com/2017/03/playing-around-with-apache-airflow-bigquery-62/
- https://stackoverflow.com/questions/38200666/airflow-parallelism
- https://www.astronomer.io/guides/airflow-scaling-workers/
- https://html.developreference.com/article/10279398/Apache+Airflow%3A+run+all+parallel+tasks+in+single+DAG+run
- https://stackoverflow.com/questions/38407314/airflow-too-many-connections-as-a-default
- https://medium.com/leboncoin-engineering-blog/data-traffic-control-with-apache-airflow-ab8fd3fc8638
- https://stackoverflow.com/questions/57761998/problem-updating-the-connections-in-airflow-programatically/57765115#57765115
- https://jsatt.com/blog/class-based-celery-tasks/
- https://airflow.readthedocs.io/en/stable/_modules/airflow/executors/celery_executor.html
- https://news.ycombinator.com/item?id=11868774
- https://docs.celeryproject.org/en/stable/userguide/tasks.html
- https://stackoverflow.com/questions/18122458/celery-task-decorator-throws-typeerror-module-object-is-not-callable
- https://docs.celeryproject.org/en/2.1-archived/reference/celery.task.html#celery.task.execute_remote
- https://medium.com/@ZymergenTechBlog/building-a-parallel-task-api-with -celery-dbae5ced4e28
- https://stackoverflow.com/questions/59956806/enable-pyspark-on-airflow-for-docker
- https://towardsdatascience.com/getting-started-with-airflow-locally-and-remotely-d068df7fcb4
- https://medium.com/@xnuinside/install-python-dependencies-to-docker-compose-cluster-without-re-build-images-8c63a431e11c
- https://stackoverflow.com/questions/57433501/where-does-airflow-dags-get-its-dependencies
- https://news.ycombinator.com/item?id=18612590
- https://sourcery.ai/blog/python-docker/
- https://stackoverflow.com/questions/59412917/errno-13-permission-denied-when-airflow-tries-to-write-to-logs
- https://eng.lyft.com/securing-apache-airflow-ui-with-dag-level-access-a7bc649a2821
- https://community.bitnami.com/t/permission-issues-running-dags-with-local-package/76175/6
- https://stackoverflow.com/questions/62499661/airflow-dockeroperator-fails-with-permission-denied-error
- https://airflow.apache.org/docs/stable/security.html
- https://www.digitalocean.com/community/questions/how-to-fix-docker-got-permission-denied-while-trying-to-connect-to-the-docker-daemon-socket
- https://airflow.readthedocs.io/en/latest/howto/add-new-role.html
- https://wecode.wepay.com/posts/improving-airflow-ui-security
- https://stackoverflow.com/questions/53386012/how-to-use-the-user-role-in-airflow-1-10-rbac
- https://airflow.apache.org/docs/stable/security.html#default-roles
- https://cwiki.apache.org/confluence/display/AIRFLOW/Airflow+Impersonation+Design+Doc
- https://airflow.apache.org/docs/stable/concepts.html#concepts-xcom
- https://github.com/apache/airflow/blob/master/airflow/example_dags/example_xcom.py
- https://stackoverflow.com/questions/60282738/airflow-communicate-between-task-without-xcom
- https://stackoverflow.com/questions/46059161/airflow-how-to-pass-xcom-variable-into-python-function
- https://stackoverflow.com/questions/46707132/how-to-delete-xcom-objects-once-the-dag-finishes-its-run-in-airflow
- https://stackoverflow.com/questions/57686514/how-to-cancel-the-xcoms-display-in-airflow-ui?noredirect=1&lq=1
- https://stackoverflow.com/questions/58792721/global-variables-in-airflow
- https://ralsina.me/weblog/posts/airflow-by-example.html
- https://stackoverflow.com/questions/54045048/with-code-how-do-you-update-an-airflow-variable
- https://github.com/apache/airflow/blob/master/tests/models/test_variable.py
- https://github.com/irvifa/google-cloud-platform-notes/wiki/dynamic-dags
- https://github.com/aktechthoughts/airflow-examples/blob/master/airflow/cli/commands/variable_command.py
- https://airflow.readthedocs.io/en/latest/operators-and-hooks-ref.html
- https://github.com/cprakti/airflow_tutorial/blob/master/README.md
- https://github.com/apache/airflow/blob/master/tests/operators/test_python.py
- https://stackoverflow.com/questions/50708226/airflow-macros-in-python-operator
- https://www.astronomer.io/guides/dynamically-generating-dags/
- https://github.com/apache/airflow/blob/master/scripts/perf/dags/perf_dag_2.py
- https://github.com/apache/airflow/blob/master/scripts/perf/dags/perf_dag_1.py
- https://stackoverflow.com/questions/44718584/airflow-cannot-run-dag-because-of-upstream-tasks-been-failed
- https://issues.apache.org/jira/browse/AIRFLOW-3237
- https://medium.com/snaptravel/airflow-part-2-lessons-learned-793fa3c0841e
- https://bcb.github.io/airflow/run-dag-and-watch-logs
- https://tech.marksblogg.com/install-and-configure-apache-airflow.html
- https://stackoverflow.com/questions/43548744/removing-airflow-task-logs
- https://github.com/apache/airflow/blob/master/tests/utils/test_log_handlers.py
- https://github.com/puckel/docker-airflow/issues/44
- https://github.com/puckel/docker-airflow/issues/307
- https://stackoverflow.com/questions/59412917/errno-13-permission-denied-when-airflow-tries-to-write-to-logs
- https://stackoverflow.com/questions/55588755/airflow-failed-to-fetch-log-file-from-worker-404-client-error-not-found-for
- https://www.astronomer.io/guides/logging/
- https://airflow.readthedocs.io/en/stable/howto/write-logs.html
- https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/hooks/s3/index.html
- https://stackoverflow.com/questions/44780736/setting-up-s3-for-logs-in-airflow
- https://airflow.apache.org/docs/apache-airflow-providers/index.html
- https://docs.aws.amazon.com/mwaa/latest/userguide/amazon-mwaa-user-guide.pdf
- https://airflow.readthedocs.io/en/latest/_api/airflow/models/index.html
- https://airflow.apache.org/docs/1.10.3/_modules/airflow/models.html
- https://airflow.readthedocs.io/en/latest/_api/airflow/exceptions/index.html
- https://airflow.apache.org/docs/1.10.3/_modules/airflow/exceptions.html
- https://github.com/apache/airflow/blob/master/tests/models/test_dag.py
- https://github.com/adaptaqatestaccount/airfloe-test/blob/master/tests/test_core.py
- https://github.com/apache/airflow/blob/master/airflow/config_templates/default_test.cfg
- https://opensource.creativecommons.org/blog/entries/apache-airflow-testing-with-pytest/
- https://stackoverflow.com/questions/51047050/test-dag-run-for-airflow-1-9-in-unittest
- https://opensource.creativecommons.org/blog/entries/apache-airflow-testing-with-pytest/
- https://dev.to/digitaldisorder/how-to-develop-apache-airflow-dags-in-docker-compose-95m
- https://itnext.io/testing-airflow-data-pipelines-with-catcher-end-to-end-93cec5c881ad
- https://medium.com/slalom-technology/automated-airflow-testing-a29b6426148b
- https://airflow.apache.org/docs/stable/_api/airflow/executors/local_executor/index.html
- https://cwiki.apache.org/confluence/display/INCUBATOR/AirflowProposal
- https://airflow.apache.org/docs/stable/start.html
- https://airflow-tutorial.readthedocs.io/en/latest/first-airflow.html
- https://www.qubole.com/tech-blog/understand-apache-airflows-modular-architecture/
- https://technofob.com/2019/05/30/get-started-developing-workflows-with-apache-airflow/
- https://artwr.github.io/airflow-workshop-dataengconf-sf-2017/anatomy_of_a_dag.html
- https://big-data-demystified.ninja/2020/05/26/why-choose-airflow-over-a-legacy-etl-or-a-modern-elt-tool-for-data-ingestion-why-not-choose-airflow/
- https://www.solita.fi/en/blogs/apache-airflow-why-everyone-working-on-data-domain-should-be-interested-of-it/
- https://news.ycombinator.com/item?id=17867987
- https://www.linkedin.com/pulse/building-data-pipeline-airflow-mehmet-vergili/
- https://medium.com/@dustinstansbury/understanding-apache-airflows-key-concepts-a96efed52b1a
- https://medium.com/flyr-labs-blog/airflow-a5a78238bfbe
- https://medium.com/@grahamplace/airflow-zero-to-production-aa145ea72fe9
- http://michal.karzynski.pl/blog/2017/03/19/developing-workflows-with-apache-airflow/
- https://issues.apache.org/jira/browse/AIRFLOW-57
- https://engblog.nextdoor.com/how-to-trick-airflow-to-reduce-wasteful-processing-b1744f4aaece
- https://confusedcoders.com/data-engineering/airflow/how-to-run-incremental-and-range-run-scripts-in-airflow
- https://cwiki.apache.org/confluence/display/AIRFLOW/Scheduler+Basics
- https://cwiki.apache.org/confluence/display/AIRFLOW/Event+Based+Scheduling
- https://github.com/apache/airflow/blob/master/tests/jobs/test_scheduler_job.py
- https://airflow.readthedocs.io/en/stable/scheduler.html
- https://stackoverflow.com/questions/45477935/airflow-log-file-isnt-local-unsupported-remote-log-location
- https://stackoverflow.com/questions/62476844/airflow-how-to-run-webserver-and-scheduler-together-from-a-docker-image
- https://airflow.apache.org/docs/stable/howto/check-health.html
- https://hub.docker.com/r/bitnami/airflow-scheduler/
- https://github.com/apache/airflow/pull/1559
- https://forum.astronomer.io/t/airflow-start-date-concepts/393
- https://stackoverflow.com/questions/53783626/does-apache-airflow-1-10-scheduler-support-running-2-dags-in-different-dst-awar
- https://livebook.manning.com/book/data-pipelines-with-apache-airflow/chapter-3/v-1/
- https://tech.marksblogg.com/install-and-configure-apache-airflow.html
- https://eng.lyft.com/running-apache-airflow-at-lyft-6e53bb8fccff
- https://www.waitingforcode.com/apache-airflow/apache-airflow-sequential-execution/read
- https://airflow.apache.org/docs/1.10.2/cli.html
- https://airflow.readthedocs.io/en/stable/usage-cli.html?highlight=example#display-dags-structure
- https://stackoverflow.com/questions/58991074/airflow-initdb-slot-pool-does-not-exists
- https://github.com/apache/airflow/tree/master/airflow/cli
- http://michael-harmon.com/blog/AirflowETL.html
- https://dev.framing.life/data/airflow-aws-lessons/
- https://stackoverflow.com/questions/48378712/efficient-way-to-deploy-dag-files-on-airflow
- https://stackoverflow.com/questions/42403755/running-job-on-airflow-based-on-webrequest
- https://donagh.io/configuring-airflow-dags-with-yaml/
- https://github.com/apache/airflow/blob/master/airflow/config_templates/default_airflow.cfg
- https://github.com/apache/airflow/blob/master/airflow/config_templates/airflow_local_settings.py
- https://stlong0521.github.io/20161023%20-%20Airflow.html
- https://gist.github.com/rosiehoyem/9e111067fe4373eb701daf9e7abcc423
- https://stackoverflow.com/questions/43410836/how-to-remove-default-example-dags-in-airflow
- https://airflow.apache.org/docs/stable/howto/set-config.html
- https://stackoverflow.com/questions/48825335/dynamic-task-definition-in-airflow
- https://github.com/InesDon/avg_rep_airflow/blob/master/dags/api/insert_into_postgres.py
- https://github.com/InesDon/avg_rep_airflow/blob/master/dags/api/insert_into_postgres.py
- https://github.com/search?l=Python&p=2&q=%22return+pythonoperator%22+range&type=Code
- https://github.com/search?l=Python&q=airflow+%22set_upstream%22+dynamic+provide_context%3D%27True%27%22+%22op_kwargs%22+kwargs&type=Code
- https://codereview.stackexchange.com/questions/220907/dynamic-operators-within-dags-airflow
- https://github.com/jonathasrocha/desafio-stones/blob/master/tmdb_etl.py
- https://github.com/molei20021/airflowdag/blob/a8108eb7e32106d1d145a3a9d7410e5ce6ff7476/dynamictask.py
- https://github.com/nuebar/fantasy-football-app/blob/master/airflow/dags/dags.py
- https://www.data-essential.com/create-dynamic-workflow-in-apache-airflow/
- Set an empty array as a global
- Populate it using a method not bound to a task
- Create a method that returns a PythonOperator
- Call that method while looping through the populated array
- https://airflow.apache.org/docs/stable/_modules/airflow/example_dags/
- https://github.com/dimk00z/airflow_project/blob/master/air_project_parallels_dag.py
- https://github.com/airflow-plugins/Example-Airflow-DAGs
- https://github.com/hubmapconsortium/ingest-pipeline/blob/master/src/ingest-pipeline/airflow/dags/devtest_step2.py
- https://github.com/Loganjaden/airflow/blob/master/airflow_DAG/old_back_active_adv_action_user_action.py
- https://gist.github.com/adamhaney/916a21b0adcabef4038c38e3ac96a36f
- https://www.slideshare.net/tedmiston/momentum-dev-con-airflow-talk-final-version-pdf
- https://stackoverflow.com/questions/55069155/airflow-error-handling-task-trigger-rule-for-triggering-directly-linked-task-fai
- https://www.astronomer.io/guides/error-notifications-in-airflow/
- https://blog.twitter.com/engineering/en_us/topics/insights/2018/ml-workflows.html
- https://airflow.apache.org/docs/stable/executor/debug.html
- https://big-data-demystified.ninja/2020/04/03/airflow-use-case-improving-success-rate-of-api-calls/
- https://airflow.readthedocs.io/en/stable/_modules/airflow/models/dag.html#DAG.cli
- https://cwiki.apache.org/confluence/display/AIRFLOW/Common+Pitfalls
- https://gtoonstra.github.io/etl-with-airflow/gotchas.html
- https://forum.astronomer.io/t/errno-13-permission-denied-usr-local-airflow-logs-scheduler/410/3
- https://github.com/puckel/docker-airflow/issues/44
- https://github.com/bitnami/bitnami-docker-airflow-scheduler/issues/2
- https://stackoverflow.com/questions/49783850/remove-airflow-scheduler-logs
- https://github.com/puckel/docker-airflow/issues/94
- https://medium.com/@hafizbadrie/airflow-when-your-dag-is-far-behind-the-schedule-ea11bf02e44c
- https://github.com/puckel/docker-airflow/issues/94
- https://github.com/puckel/docker-airflow/issues/233
- https://issues.apache.org/jira/browse/AIRFLOW-5506
- https://stackoverflow.com/questions/45853013/airflow-tasks-get-stuck-at-queued-status-and-never-gets-running
- https://airflow.apache.org/docs/stable/faq.html#why-isn-t-my-task-getting-scheduled
- https://stackoverflow.com/questions/49021055/airflow-1-9-0-is-queuing-but-not-launching-tasks/49047832#49047832
- https://github.com/steinsag/apache-airflow-test
- https://github.com/feryandi/freeflow
- https://github.com/dnvriend/airflow-test
- https://airflow.apache.org/blog/
- https://blog.clairvoyantsoft.com/tagged/airflow
- https://airflowsummit.org/blog/tags/general/
volumes:
airflowlogs:/usr/local/airflow/log #Add a named volume to the webserver and scheduler section of docker-compose map to fix Errno 13
- https://cwiki.apache.org/confluence/display/AIRFLOW/Streamlined+%28Functional%29+Airflow
- https://github.com/ziwon/airflow-boilerplate
- https://danvatterott.com/blog/2018/08/29/custom-email-alerts-in-airflow/
- https://stackoverflow.com/questions/51829200/how-to-set-up-airflow-send-email
- https://buildmedia.readthedocs.org/media/pdf/airflow/1.10.0/airflow.pdf
- https://airflow.apache.org/docs/1.10.3/cli.html
- https://airflow.apache.org/docs/1.10.3/faq.html
- https://medium.com/analytics-and-data/10-benefits-to-using-airflow-33d312537bae
- https://github.com/jghoman/awesome-apache-airflow
- https://gtoonstra.github.io/etl-with-airflow/principles.html
- https://www.reddit.com/r/dataengineering/comments/epzhs6/opinions_of_apache_airflow/
- "Maxime Beauchemin" title:airflow slideshare