I am using Spark 3.1.2 (listed 3.1.1 in initial post, deployment originally listed 3.1.1) initializing with: spark-shell spark-shell -target:jvm-1.8

I am bringing in the following packages (some used and unused at the moment):

--conf "spark.driver.extraClassPath=~/jars/SparkJDBC42.jar" \

--conf "spark.jars.repositories=path/to/repos" \

--packages "com.microsoft.sqlserver:mssql-jdbc:6.4.0.jre8,net.sourceforge.jtds:jtds:1.3.1,com.oracle.database.jdbc:ojdbc8:21.1.0.0,com.databricks:spark-avro_2.10:4.0.0"

Following are the de-identified setting that come implicitly from my /opt/spark/conf/spark-default.conf:

spark.master=k8s://https://kubernetes.default.svc.cluster.local:443

spark.app.name=sp-worker

spark.eventLog.enabled true

spark.eventLog.dir file:///spark/logs

spark.kubernetes.driver.limit.cores=2

spark.kubernetes.executor.limit.cores=4

spark.executor.instances=2

spark.executor.memory=4G

spark.kubernetes.executor.limit.cores=4

spark.kubernetes.executor.request.cores=250m

spark.kubernetes.driver.label.app=spark-jlab

spark.kubernetes.executor.label.app=spark-jlab

spark.kubernetes.executor.label.deployment=jlab

spark.kubernetes.local.dirs.tmpfs=true

spark.kubernetes.container.image.pullPolicy=IfNotPresent 

spark.kubernetes.container.image=docker.someurl.com/path/to/image/repo

# Home for Python libraries

spark.kubernetes.driver.volumes.persistentVolumeClaim.home.mount.path=/home

spark.kubernetes.driver.volumes.persistentVolumeClaim.home.mount.readOnly=false

spark.kubernetes.driver.volumes.persistentVolumeClaim.home.options.claimName=jlab

spark.kubernetes.executor.volumes.persistentVolumeClaim.home.mount.path=/home

spark.kubernetes.executor.volumes.persistentVolumeClaim.home.mount.readOnly=false

spark.kubernetes.executor.volumes.persistentVolumeClaim.home.options.claimName=jlab

# #Logs

spark.kubernetes.driver.volumes.persistentVolumeClaim.logs.mount.path=/logs

spark.kubernetes.driver.volumes.persistentVolumeClaim.logs.mount.readOnly=false

spark.kubernetes.driver.volumes.persistentVolumeClaim.logs.options.claimName=k8s-sparkhistorylogs

spark.kubernetes.executor.volumes.persistentVolumeClaim.logs.mount.path=/logs

spark.kubernetes.executor.volumes.persistentVolumeClaim.logs.mount.readOnly=false

spark.kubernetes.executor.volumes.persistentVolumeClaim.logs.options.claimName=k8s-sparkhistorylogs

spark.kubernetes.namespace=namespace-00000

spark.kubernetes.authenticate.driver.serviceAccountName=svcactnm-00000

spark.kubernetes.authenticate.oauthToken=token

spark.kubernetes.authenticate.caCertFile=/opt/osficerts/ctc.crt

spark.ui.xContentTypeOptions.enabled=true

spark.kubernetes.pyspark.pythonVersion=3