Options
- Mark as New
- Bookmark
- Subscribe
- Mute
- Subscribe to RSS Feed
- Permalink
- Report Inappropriate Content
11-25-2024 11:04 PM
Also I have give all permissions for that user and I am trying it from my spark engine.
.config("spark.jars.packages", "org.apache.hadoop:hadoop-aws:3.3.4,org.apache.hadoop:hadoop-common:3.3.4,io.delta:delta-spark_2.12:3.2.1,io.unitycatalog:unitycatalog-spark_2.12:0.2.0") \
.config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \
.config("spark.sql.catalog.spark_catalog", "io.unitycatalog.spark.UCSingleCatalog") \
.config("spark.hadoop.fs.s3.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem") \
.config("spark.sql.catalog.datatest", "io.unitycatalog.spark.UCSingleCatalog") \
.config("spark.sql.adaptive.enabled", "true") \
.config("spark.sql.constraintPropagation.enabled", "false") \
.config("spark.sql.catalog.datatest.uri", "https://adb-34xxx827.7.azuredatabricks.net/api/2.1/unity-catalog") \
.config("spark.sql.catalog.datatest.token", "dapi0xx4b6d85cdf8") \
.config("spark.sql.defaultCatalog", "datatest") \
.config("fs.azure.account.key.dataxxcdixon.dfs.core.windows.net", "DuHZxxLGnyOSxxxxezyzoC+AStvF46+w==") \
.config("spark.hadoop.fs.azure.account.auth.type.dataxxdxixon.dfs.core.windows.net", "OAuth") \
.config("spark.hadoop.fs.azure.account.oauth.provider.type.databxxdixon.dfs.core.windows.net", "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider") \
.config("spark.hadoop.fs.azure.account.oauth2.client.id.datxxcdixon.dfs.core.windows.net", "xxx-xx-xx-xx-xx") \
.config("spark.hadoop.fs.azure.account.oauth2.client.secret.daxdixon.dfs.core.windows.net", "dapi0xxxxx85cdf8") \
.config("spark.hadoop.fs.azure.account.oauth2.client.endpoint.daxxdixon.dfs.core.windows.net", "https://login.microsoftonline.com/fxxxx0/oauth2/token") \
.getOrCreate()
