Use the below code snippet to forcefully drop the table:
package org.apache.spark.sql.hive {
import org.apache.spark.sql.hive.HiveUtils
import org.apache.spark.SparkContext
object utils {
def dropTable(sc: SparkContext, dbName: String, tableName: String, ignoreIfNotExists: Boolean, purge: Boolean): Unit = {
HiveUtils
.newClientForMetadata(sc.getConf, sc.hadoopConfiguration)
.dropTable(dbName, tableName, ignoreIfNotExists, false)
}
}
}
import org.apache.spark.sql.hive.utils
utils.dropTable(sc, "default", "my_table", true, true)