Hi @m_koch_unify,

Thank you for posting the question in the Databricks community.

This issue may occur due to the package 'cusparse' that is not found. As Databricks removed several uncommon packages since MLR >= 8.1 and cusparse is one of them. If you need cusparse library, you can manually install it via the init script below:

#!/bin/bash

set -e

# The root directory which has the necessary code to initialize the cluster during cluster init.  This should ideally
# be a user directory that was synced over according to the README.md.
DS_PROJECTS_ROOT=${DS_PROJECTS_ROOT:-/dbfs/data-mle/llm/ds-projects}

# This points to the requirements that should be installed during cluster init.  The requirements we install may change
# depending on the notebook.
export REQUIREMENTS_NAME=${REQUIREMENTS_NAME:-deepspeed.in}
export REQUIREMENTS_PATH=${REQUIREMENTS_PATH:-$DS_PROJECTS_ROOT/app/llm/requirements/$REQUIREMENTS_NAME}

# This points to the path where the Python environment should be cached in DBFS after pip install complets.  The
# default path here should generally be fine.
export PYTHON_ENV_CACHE=${PYTHON_ENV_CACHE:-$DS_PROJECTS_ROOT/python-env-cache-$DATABRICKS_RUNTIME_VERSION}

echo Root dir: $DS_PROJECTS_ROOT
echo Requirements path: $REQUIREMENTS_PATH
echo Python env cache: $PYTHON_ENV_CACHE

if [ -f "/usr/local/cuda/bin/nvcc" ]; then

    # Needed by deepspeed for distributed training.
    apt-get install -y pdsh expect

    # When developing CUDA applications, it is often necessary to include the CUDA headers in your source code.
    # The CUDA headers are a collection of C/C++ header files that define the interfaces to the CUDA libraries and runtime.
    # The header files provide declarations for functions, constants, and data types that are used in CUDA programming.
    # By including these headers in your source code, you can access the CUDA API and use the functions and data types
    # defined in the headers.

    # Databricks Runtime 13.x uses CUDA 11.7
    if /usr/local/cuda/bin/nvcc -V | grep -q cuda_11.7; then
        echo Found CUDA 11.7
        # TODO should these be dev version as with 11.3?
        export libcusparse=libcusparse-11-7_11.7.4.91-1_amd64.deb
        export libcublas=libcublas-11-7_11.10.3.66-1_amd64.deb
        export libcusolver=libcusolver-11-7_11.4.0.1-1_amd64.deb
        export libcurand=libcurand-11-7_10.2.10.91-1_amd64.deb

    # Databricks Runtime 12.x uses CUDA 11.3
    elif /usr/local/cuda/bin/nvcc -V | grep -q cuda_11.3; then
        echo Found CUDA 11.3
        export libcusparse=libcusparse-dev-11-3_11.5.0.58-1_amd64.deb
        export libcublas=libcublas-dev-11-3_11.5.1.109-1_amd64.deb
        export libcusolver=libcusolver-dev-11-3_11.1.2.109-1_amd64.deb
        export libcurand=libcurand-dev-11-3_10.2.4.109-1_amd64.deb

    else
        /usr/local/cuda/bin/nvcc -V
        echo Unsupported cuda version
        exit 1
    fi

    # includes the headers and static libraries for the cuSPARSE library, used for sparse matrix operations.
    wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/$libcusparse \
        -O /tmp/libcusparse.deb && \
        dpkg -i /tmp/libcusparse.deb

    # includes the headers and static libraries for the cuBLAS library, used for linear algebra operations.
    wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/$libcublas \
        -O /tmp/libcublas.deb && \
        dpkg -i /tmp/libcublas.deb

    # includes the headers and static libraries for the cuSOLVER library, used for solving dense and sparse linear systems.
    wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/$libcusolver \
        -O /tmp/libcusolver.deb && \
        dpkg -i /tmp/libcusolver.deb

    # includes the headers and static libraries for the cuRAND library, used for random number generation.
    wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/$libcurand \
        -O /tmp/libcurand.deb && \
        dpkg -i /tmp/libcurand.deb

    # Create an environment variables file that deepspeed will apply on the worker nodes.
    # The PATH will not include the /databricks/python3/bin directory, which is where we have all the Python
    # binaries we use will be installed.  For example, without updating the path "ninja" won't be found and
    # pytorch will have a compilation failure.
    echo "PATH=/databricks/python3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" >> /root/.deepspeed_env
fi

# Save the cluster ID and workspace ID to a file which we can conveniently use later when starting the service to
# set environment variables it requires.
echo "export DB_CLUSTER_ID=\"$DB_CLUSTER_ID\"" >> /root/.envinfo
echo "export DB_CLUSTER_NAME=\"$DB_CLUSTER_NAME\"" >> /root/.envinfo
echo "export WORKSPACE_ID=2548836972759138" >> /root/.envinfo
echo "export REQUIREMENTS_NAME=\"$REQUIREMENTS_NAME\"" >> /root/.envinfo
echo "export REQUIREMENTS_PATH=\"$REQUIREMENTS_PATH\"" >> /root/.envinfo
echo "export PYTHON_ENV_CACHE=\"$PYTHON_ENV_CACHE\"" >> /root/.envinfo
echo "export DS_PROJECTS_ROOT=\"$DS_PROJECTS_ROOT\"" >> /root/.envinfo

# This is similar to the init scripts under deploy/resources except it uses paths under dbfs/data-mle/llm,
# or a root configurd with DS_PROJECTS_ROOT.
# This enables us to ship more up-to-date requirements and cache independently from other code in logfood.

/databricks/python/bin/python $DS_PROJECTS_ROOT/deploy/resources/pip_installs.py

if [ -d "/databricks/python3-restored" ]; then
   rm -rf  /databricks/python3
   echo "Replacing /databricks/python3"
   mv /databricks/python3-restored /databricks/python3
fi