Configuring the DLT Pipeline - pathfinder-analytics-uk/dab_project GitHub Wiki

Project Code

resources/citibike_etl_pipeline.dlt.yml

resources:
  pipelines:
    pipeline_citibike_etl_pipeline:
      name: citibike_etl_pipeline
      configuration:
        catalog: "${var.catalog}"
      libraries:
        - notebook:
            path: ../citibike_etl/dlt/01_bronze/01_bronze_citibike.ipynb
        - notebook:
            path: ../citibike_etl/dlt/02_silver/02_silver_citibike.ipynb
        - notebook:
            path: ../citibike_etl/dlt/03_gold/03_gold_citibike_daily_ride_summary.ipynb
        - notebook:
            path: ../citibike_etl/dlt/03_gold/03_gold_citibike_daily_station_performance.ipynb
      schema: default
      development: "${var.dev_flag}"
      photon: true
      catalog: "${var.catalog}"
      serverless: true

databricks.yml

# This is a Databricks asset bundle definition for dab_project.
# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation.
bundle:
  name: dab_project

include:
  - resources/*.yml

variables:
  catalog:
    default: "citibike_dev"
  dev_flag:
    default: true

targets:
  dev:
    # The default target uses 'mode: development' to create a development copy.
    # - Deployed resources get prefixed with '[dev my_user_name]'
    # - Any job schedules and triggers are paused by default.
    # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html.
    mode: development
    default: true
    workspace:
      host: your-workspace-url # replace with your Workspace URL

  test:
    mode: production
    variables:
      catalog:
        default: "citibike_test"
      dev_flag:
        default: false
    presets:
      name_prefix: '[testing] '
    workspace:
      host: your-workspace-url # replace with your Workspace URL
      root_path: /Workspace/Shared/.bundle/${bundle.name}/${bundle.target}
    permissions:
      - user_name: your-user-name # update with your user name
        level: CAN_MANAGE
    run_as:
      user_name: your-user-name # update with your user name

  prod:
    mode: production
    variables:
      catalog:
        default: "citibike_prod"
      dev_flag:
        default: false
    workspace:
      host: your-workspace-url # replace with your Workspace URL
      root_path: /Workspace/Shared/.bundle/${bundle.name}/${bundle.target}
    permissions:
      - user_name: your-user-name # update with your user name
        level: CAN_MANAGE
    run_as:
      user_name: your-user-name # update with your user name