Ugly, but this seems to work for now

import sys
import os
import shutil
from ray.util.spark import setup_ray_cluster, shutdown_ray_cluster
 
shutil.copy(
    "/local_disk0/.ephemeral_nfs/cluster_libraries/python/bin/ray",
    os.path.dirname(sys.executable),
)
 
setup_ray_cluster(
  num_worker_nodes=4,
  num_cpus_per_node=8,
  collect_log_to_path="/dbfs/ray/logs"
)