Azure Synapse Analytics Connector - ja-guzzle/guzzle_docs GitHub Wiki
create logical endpoint file at /guzzle/conf/default/logical-endpoints/lo_azure_synapse_analytics.yml :
version: 1
technology: azure-synapse-analytics
create physical endpoint at /guzzle/conf/instance/physical-endpoints/ph_azure_synapse_analytics.yml :
for blob storage account:
version: 1
technology: "azure-synapse-analytics"
url: "jdbc:sqlserver://<the-rest-of-the-connection-string>"
database: "<database name>"
username: "<database username>"
password: "<database password>"
tempDir: "wasbs://<your-container-name>@<your-storage-account-name>.blob.core.windows.net/<your-directory-name>"
storageAccountName: "<your-storage-account-name>"
storageAccountAccessKey: "<storage-account-access-key>"
forwardSparkAzureStorageCredentials: true
for Adls Gen2 storage account:
version: 1
technology: "azure-synapse-analytics"
url: "jdbc:sqlserver://<the-rest-of-the-connection-string>"
database: "<database name>"
username: "<database username>"
password: "<database password>"
tempDir: "abfss://<your-container-name>@<your-storage-account-name>.dfs.core.windows.net/<your-directory-name>"
storageAccountName: "<your-storage-account-name>"
storageAccountAccessKey: "<storage-account-access-key>"
useAzureMSI: true
create environment mapping using guzzle ui or add endpoint mapping entry in environment file at /guzzle/conf/instance/environments/-endpoints.yml location :
endpoint_mapping:
lo_azure_synapse_analytics:
physical_endpoint: "ph_azure_synapse_analytics"
use azure synapse connector in source section:
source:
endpoint: lo_azure_synapse_analytics
properties:
table: flights // you can use query as well, query: select * from flights
use azure synapse analytics connector in target or reject section:
target:
endpoint: lo_azure_synapse_analytics
properties:
table: flights
format_properties:
write_semantics: polybase
max_str_length: 4000
operation: [append | overwrite]
reject:
endpoint: lo_azure_synapse_analytics
properties:
table: flights
format_properties:
write_semantics: polybase
max_str_length: 4000
failure_threshold: 50
azure synapse analytics connector ingestion job config example:
version: 1
job:
description: 'azure synapse analytics connector example'
tags:
- azure_synapse_analytics
type: ingestion
source:
endpoint: lo_delta
properties:
table: dim_lease
schema:
strict_schema_check: true
schema_derivation_strategy: source
discard: true
target:
endpoint: lo_azure_synapse_analytics
properties:
table: dim_lease
format_properties:
write_semantics: polybase
max_str_length: 4000
pre_sql:
- truncate table dim_lease
reject:
endpoint: lo_azure_synapse_analytics
properties:
table: reject_dim_lease
format_properties:
write_semantics: polybase
max_str_length: 4000
pre_sql:
- truncate table reject_dim_lease
failure_threshold: 50