Hi.
I am trying to run a Spark Job in Databricks (Azure) using the JAR type.
I can't figure out why the job fails to run by not finding the SparkContext.
Databricks Runtime: 14.3 LTS (includes Apache Spark 3.5.0, Scala 2.12)
Error message:
java.lang.NoClassDefFoundError: org/apache/spark/SparkContext$
Caused by: java.lang.ClassNotFoundException: org.apache.spark.SparkContext$
java.lang.NoClassDefFoundError: org/apache/spark/SparkContext$
com.cognira.crsp.Main$.delayedEndpoint$com$cognira$crsp$Main$1(Main.scala:14)
com.cognira.crsp.Main$delayedInit$body.apply(Main.scala:8)
scala.Function0.apply$mcV$sp(Function0.scala:39)
scala.Function0.apply$mcV$sp$(Function0.scala:39)
scala.runtime.AbstractFunction0.apply$mcV$sp(AbstractFunction0.scala:17)
scala.App.$anonfun$main$1$adapted(App.scala:80)
scala.collection.immutable.List.foreach(List.scala:431)
scala.App.main(App.scala:80)
scala.App.main$(App.scala:78)
com.cognira.crsp.Main$.main(Main.scala:8)
ammonite.$sess.cmd4$Helper.<init>(cmd4.sc:1)
ammonite.$sess.cmd4$.<init>(cmd4.sc:7)
ammonite.$sess.cmd4$.<clinit>(cmd4.sc:-1)
java.lang.ClassNotFoundException: org.apache.spark.SparkContext$
java.net.URLClassLoader.findClass(URLClassLoader.java:387)
java.lang.ClassLoader.loadClass(ClassLoader.java:419)
sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
java.lang.ClassLoader.loadClass(ClassLoader.java:352)
com.cognira.crsp.Main$.delayedEndpoint$com$cognira$crsp$Main$1(Main.scala:14)
com.cognira.crsp.Main$delayedInit$body.apply(Main.scala:8)
scala.Function0.apply$mcV$sp(Function0.scala:39)
scala.Function0.apply$mcV$sp$(Function0.scala:39)
scala.runtime.AbstractFunction0.apply$mcV$sp(AbstractFunction0.scala:17)
scala.App.$anonfun$main$1$adapted(App.scala:80)
scala.collection.immutable.List.foreach(List.scala:431)
scala.App.main(App.scala:80)
scala.App.main$(App.scala:78)
com.cognira.crsp.Main$.main(Main.scala:8)
ammonite.$sess.cmd4$Helper.<init>(cmd4.sc:1)
ammonite.$sess.cmd4$.<init>(cmd4.sc:7)
ammonite.$sess.cmd4$.<clinit>(cmd4.sc:-1)
My dependencies:
libraryDependencies ++= Seq(
"org.apache.hadoop" % "hadoop-azure" % "3.3.4",
"org.apache.spark" %% "spark-core" % "3.5.0" % "provided",
"org.apache.spark" %% "spark-sql" % "3.5.0" % "provided",
)
libraryDependencies ++= Seq(
"com.databricks" % "databricks-dbutils-scala_2.12" % "0.1.4"
)
.map(_.excludeAll(
ExclusionRule("com.fasterxml.jackson.core", "jackson-core"),
ExclusionRule("com.fasterxml.jackson.core", "jackson-databind"),
))
Main snippet (REDACTED):
object Main extends App {
val dbutils = DBUtils.getDBUtils()
val spark = SparkSession.builder.getOrCreate()
val sc = SparkContext.getOrCreate() // FAILS HERE !
}