Any leads/posts for Databricks CI/CD integration with Bitbucket pipeline. I am facing the below error while I creation my CICD pipeline
pipelines:
branches:
master:
- step:
name: Deploy Databricks Changes
image: docker:19.03.12
services:
- docker
script:
# Update and install required packages
- apk add --no-cache py-pip expect git
- pip install --upgrade pip
- pip install databricks-cli
- databricks --version
# Create Databricks CLI configuration file for Source Workspace
- echo -e "[DEFAULT]\nhost = $DB_HOST_SOURCE\ntoken = $DB_TOKEN_QA" > ~/.databrickscfg_source
# Create Databricks CLI configuration file for Destination Workspace
- echo -e "[DEFAULT]\nhost = $DB_HOST_DESTINATION\ntoken = $DB_TOKEN_PROD" > ~/.databrickscfg_destination
# Display the contents of the configuration files for verification
- cat ~/.databrickscfg_source
- cat ~/.databrickscfg_destination
# Configure Databricks CLI for source
- |
expect -c "
spawn /usr/local/bin/databricks configure --host $DB_HOST_SOURCE --token $DB_TOKEN_QA
expect eof
"
# Configure Databricks CLI for destination
- |
expect -c "
spawn /usr/local/bin/databricks configure --host $DB_HOST_DESTINATION --token $DB_TOKEN_PROD
expect eof
"
# Clone the latest code from Master branch using App Password
- git clone --depth 1 https://<>username:<token>@bitbucket.org/ttec-digital-ip/insights_databricks_qa.git /tmp/source_workspace_export/
# Configure Databricks CLI with token for destination using expect
- |
expect -c "
spawn /usr/local/bin/databricks configure --profile destination
expect \"Databricks Host (should begin with https://): \"
send \"$DB_HOST_DESTINATION\r\"
expect \"Token: \"
send \"$DB_TOKEN_PROD\r\"
expect eof
"
# Copy files from Local File System to Destination Workspace
- databricks fs cp -r /tmp/source_workspace_export /Workspace/Shared/ --profile destination
Error: InvalidConfigurationError: You haven't configured the CLI yet for the profile destination! Please configure by entering `/usr/bin/databricks configure --profile destination`
