Hello,

I have a quick question. If my source code call pysark collect() or any method related to rdd methods, then pytest on my local PC will report the following error. My local machine doesn't have any specific setting for pyspark and I used findspark package. If you know the solution it will be greatly appreciated. Thanks. 

src\tests\test_calculate_psi_for_each_column.py:14: in <module>
spark = spu.create_spark_obj()
src\prototype\supplementary_utils.py:959: in create_spark_obj
spark = SparkSession.builder.appName("ABC").getOrCreate()
venv\lib\site-packages\pyspark\sql\session.py:269: in getOrCreate
sc = SparkContext.getOrCreate(sparkConf)
venv\lib\site-packages\pyspark\context.py:483: in getOrCreate
SparkContext(conf=conf or SparkConf())
venv\lib\site-packages\pyspark\context.py:197: in __init__
self._do_init(
venv\lib\site-packages\pyspark\context.py:282: in _do_init
self._jsc = jsc or self._initialize_context(self._conf._jconf)
venv\lib\site-packages\pyspark\context.py:402: in _initialize_context
return self._jvm.JavaSparkContext(jconf)
venv\lib\site-packages\py4j\java_gateway.py:1585: in __call__
return_value = get_return_value(
venv\lib\site-packages\py4j\protocol.py:326: in get_return_value
raise Py4JJavaError(
E py4j.protocol.Py4JJavaError: An error occurred while calling None.org.apache.spark.api.java.JavaSparkContext.
E : java.lang.ExceptionInInitializerError
E at org.apache.spark.unsafe.array.ByteArrayMethods.<clinit>(ByteArrayMethods.java:56)
E at org.apache.spark.memory.MemoryManager.defaultPageSizeBytes$lzycompute(MemoryManager.scala:264)
E at org.apache.spark.memory.MemoryManager.defaultPageSizeBytes(MemoryManager.scala:254)
E at org.apache.spark.memory.MemoryManager.$anonfun$pageSizeBytes$1(MemoryManager.scala:273)
E at scala.runtime.java8.JFunction0$mcJ$sp.apply(JFunction0$mcJ$sp.java:23)
E at scala.Option.getOrElse(Option.scala:189)
E at org.apache.spark.memory.MemoryManager.<init>(MemoryManager.scala:273)
E at org.apache.spark.memory.UnifiedMemoryManager.<init>(UnifiedMemoryManager.scala:58)
E at org.apache.spark.memory.UnifiedMemoryManager$.apply(UnifiedMemoryManager.scala:207)
E at org.apache.spark.SparkEnv$.create(SparkEnv.scala:320)
E at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194)
E at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:279)
E at org.apache.spark.SparkContext.<init>(SparkContext.scala:464)
E at org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58)
E at java.base/jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(DirectConstructorHandleAccessor.java:62)
E at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:502)
E at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:486)
E at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:247)
E at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
E at py4j.Gateway.invoke(Gateway.java:238)
E at py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:80)
E at py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:69)
E at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
E at py4j.ClientServerConnection.run(ClientServerConnection.java:106)
E at java.base/java.lang.Thread.run(Thread.java:1570)
E Caused by: java.lang.IllegalStateException: java.lang.NoSuchMethodException: java.nio.DirectByteBuffer.<init>(long,int)
E at org.apache.spark.unsafe.Platform.<clinit>(Platform.java:113)
E ... 25 more
E Caused by: java.lang.NoSuchMethodException: java.nio.DirectByteBuffer.<init>(long,int)
E at java.base/java.lang.Class.getConstructor0(Class.java:3784)
E at java.base/java.lang.Class.getDeclaredConstructor(Class.java:2955)
E at org.apache.spark.unsafe.Platform.<clinit>(Platform.java:71)
E ... 25 more