cancel
Showing results for 
Search instead for 
Did you mean: 
Administration & Architecture
Explore discussions on Databricks administration, deployment strategies, and architectural best practices. Connect with administrators and architects to optimize your Databricks environment for performance, scalability, and security.
cancel
Showing results for 
Search instead for 
Did you mean: 

Unable to create a databricks workspace

sjs
New Contributor II

I am unable to create a databricks workspace with vnet injection. I get this error:

│ {
│   "status": "Failed",
│   "error": {
│     "code": "InternalServerError",
│     "message": "INTERNAL_ERROR: Unexpected error: Cannot call getCertifiedMetastoreForRegion: metastore certification is not enabled."
│   }
│ }

 I get the same error from Azure portal and terraform.

Template from Azure portal:

{
    "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
    "contentVersion": "1.0.0.0",
    "parameters": {
        "location": {
            "type": "String"
        },
        "workspaceName": {
            "type": "String"
        },
        "tier": {
            "defaultValue": "premium",
            "type": "String"
        },
        "tagValues": {
            "type": "Object"
        },
        "managedResourceGroupName": {
            "defaultValue": "",
            "type": "String"
        },
        "enableNoPublicIp": {
            "type": "Bool"
        }
    },
    "variables": {
        "managedResourceGroupName": "[if(not(empty(parameters('managedResourceGroupName'))), parameters('managedResourceGroupName'), concat('databricks-rg-', parameters('workspaceName'), '-', uniqueString(parameters('workspaceName'), resourceGroup().id)))]",
        "trimmedMRGName": "[substring(variables('managedResourceGroupName'), 0, min(length(variables('managedResourceGroupName')), 90))]",
        "managedResourceGroupId": "[concat(subscription().id, '/resourceGroups/', variables('trimmedMRGName'))]"
    },
    "resources": [
        {
            "type": "Microsoft.Databricks/workspaces",
            "apiVersion": "2024-05-01",
            "name": "[parameters('workspaceName')]",
            "location": "[parameters('location')]",
            "dependsOn": [],
            "tags": "[parameters('tagValues')]",
            "sku": {
                "name": "[parameters('tier')]"
            },
            "properties": {
                "ManagedResourceGroupId": "[variables('managedResourceGroupId')]",
                "parameters": {
                    "enableNoPublicIp": {
                        "value": "[parameters('enableNoPublicIp')]"
                    }
                },
                "defaultCatalog": {
                    "initialType": "UnityCatalog",
                    "initialName": ""
                }
            }
        }
    ]
}


Terraform config:

variable "databricks_location" {
  description = "The location for the Databricks workspace"
  type        = string
  default     = "northeurope"
}

resource "azurerm_resource_group" "databricks_rg" {
  name     = "rg-databricks-${terraform.workspace}"
  location = var.databricks_location
  tags     = local.ai_tags
}

resource "azurerm_virtual_network" "databricks_vnet" {
  name                = "vnet-databricks-${terraform.workspace}"
  address_space       = ["10.2.0.0/16"]
  location            = azurerm_resource_group.databricks_rg.location
  resource_group_name = azurerm_resource_group.databricks_rg.name
  tags                = local.ai_tags
}

resource "azurerm_subnet" "databricks_public" {
  name                 = "snet-databricks-public-${terraform.workspace}"
  resource_group_name  = azurerm_resource_group.databricks_rg.name
  virtual_network_name = azurerm_virtual_network.databricks_vnet.name
  address_prefixes     = ["10.2.1.0/24"]

  delegation {
    name = "databricks-del-public"
    service_delegation {
      name    = "Microsoft.Databricks/workspaces"
      actions = ["Microsoft.Network/virtualNetworks/subnets/join/action", "Microsoft.Network/virtualNetworks/subnets/prepareNetworkPolicies/action", "Microsoft.Network/virtualNetworks/subnets/unprepareNetworkPolicies/action"]
    }
  }
}

resource "azurerm_subnet" "databricks_private" {
  name                 = "snet-databricks-private-${terraform.workspace}"
  resource_group_name  = azurerm_resource_group.databricks_rg.name
  virtual_network_name = azurerm_virtual_network.databricks_vnet.name
  address_prefixes     = ["10.2.2.0/24"]

  delegation {
    name = "databricks-del-private"
    service_delegation {
      name    = "Microsoft.Databricks/workspaces"
      actions = ["Microsoft.Network/virtualNetworks/subnets/join/action", "Microsoft.Network/virtualNetworks/subnets/prepareNetworkPolicies/action", "Microsoft.Network/virtualNetworks/subnets/unprepareNetworkPolicies/action"]
    }
  }
}

resource "azurerm_subnet" "databricks_pe" {
  name                 = "snet-databricks-pe-${terraform.workspace}"
  resource_group_name  = azurerm_resource_group.databricks_rg.name
  virtual_network_name = azurerm_virtual_network.databricks_vnet.name
  address_prefixes     = ["10.2.3.0/24"]

  private_endpoint_network_policies_enabled = true
}

resource "azurerm_network_security_group" "databricks_nsg" {
  name                = "nsg-databricks-${terraform.workspace}"
  location            = azurerm_resource_group.databricks_rg.location
  resource_group_name = azurerm_resource_group.databricks_rg.name
  tags                = local.ai_tags
}

resource "azurerm_subnet_network_security_group_association" "databricks_private_nsg" {
  subnet_id                 = azurerm_subnet.databricks_private.id
  network_security_group_id = azurerm_network_security_group.databricks_nsg.id
}

resource "azurerm_subnet_network_security_group_association" "databricks_public_nsg" {
  subnet_id                 = azurerm_subnet.databricks_public.id
  network_security_group_id = azurerm_network_security_group.databricks_nsg.id
}

resource "azurerm_databricks_workspace" "this" {
  name                        = "dbw-${terraform.workspace}"
  resource_group_name         = azurerm_resource_group.databricks_rg.name
  location                    = azurerm_resource_group.databricks_rg.location
  sku                         = "premium"
  managed_resource_group_name = "rg-databricks-managed-${terraform.workspace}"

  public_network_access_enabled         = true       # Changed from false to true
  network_security_group_rules_required = "AllRules" # Changed from "NoAzureDatabricksRules" to "AllRules"

  custom_parameters {
    no_public_ip        = false # Changed from true to false
    public_subnet_name  = azurerm_subnet.databricks_public.name
    private_subnet_name = azurerm_subnet.databricks_private.name
    virtual_network_id  = azurerm_virtual_network.databricks_vnet.id

    public_subnet_network_security_group_association_id  = azurerm_subnet_network_security_group_association.databricks_public_nsg.id
    private_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.databricks_private_nsg.id
  }

  tags = local.ai_tags

  depends_on = [
    azurerm_subnet_network_security_group_association.databricks_public_nsg,
    azurerm_subnet_network_security_group_association.databricks_private_nsg
  ]
}

resource "azurerm_private_dns_zone" "databricks_dns" {
  name                = "privatelink.azuredatabricks.net"
  resource_group_name = azurerm_resource_group.databricks_rg.name
}

resource "azurerm_private_dns_zone_virtual_network_link" "databricks_dns_link" {
  name                  = "databricks-dns-link"
  resource_group_name   = azurerm_resource_group.databricks_rg.name
  private_dns_zone_name = azurerm_private_dns_zone.databricks_dns.name
  virtual_network_id    = azurerm_virtual_network.databricks_vnet.id
}

resource "azurerm_private_endpoint" "databricks_pe" {
  name                = "pe-databricks-${terraform.workspace}"
  location            = azurerm_resource_group.databricks_rg.location
  resource_group_name = azurerm_resource_group.databricks_rg.name
  subnet_id           = azurerm_subnet.databricks_pe.id

  private_service_connection {
    name                           = "psc-databricks-${terraform.workspace}"
    is_manual_connection           = false
    private_connection_resource_id = azurerm_databricks_workspace.this.id
    subresource_names              = ["databricks_ui_api"]
  }

  private_dns_zone_group {
    name                 = "privatelink-databricks-${terraform.workspace}"
    private_dns_zone_ids = [azurerm_private_dns_zone.databricks_dns.id]
  }

  depends_on = [azurerm_databricks_workspace.this]
}

resource "azurerm_virtual_network_peering" "databricks_to_ai" {
  name                      = "peer-databricks-to-ai"
  resource_group_name       = azurerm_resource_group.databricks_rg.name
  virtual_network_name      = azurerm_virtual_network.databricks_vnet.name
  remote_virtual_network_id = azurerm_virtual_network.local.id
}

resource "azurerm_virtual_network_peering" "ai_to_databricks" {
  name                      = "peer-ai-to-databricks"
  resource_group_name       = azurerm_resource_group.this.name
  virtual_network_name      = azurerm_virtual_network.local.name
  remote_virtual_network_id = azurerm_virtual_network.databricks_vnet.id
}



1 REPLY 1

sjs
New Contributor II

The issue resolved itself when I tried to create a new resource group, dedicated to just Databricks. 
I don't know why that worked. If anyone know what went wrong, I would appreciate feedback!

Connect with Databricks Users in Your Area

Join a Regional User Group to connect with local Databricks users. Events will be happening in your city, and you won’t want to miss the chance to attend and share knowledge.

If there isn’t a group near you, start one and help create a community that brings people together.

Request a New Group