Hi,

I have created a python wheel with the following code. And the package name is rule_engine

"""

The entry point of the Python Wheel

"""

import sys

from pyspark.sql.functions import expr, col

def get_rules(tag):

 """

  loads data quality rules from a table

  :param tag: tag to match

  :return: dictionary of rules that matched the tag

 """

 rules = {}

 df = spark.read.table("rules")

 for row in df.filter(col("tag") == tag).collect():

  rules[row['name']] = row['constraint']

 return rules

def get_quarantine_rules(tag):

 """

  loads data quality rules from a table

  :param tag: tag to match

  :return: dictionary of rules that matched the tag

 """

 all_rules_in_tags=get_rules(tag)

 qurantine_rule="NOT({0})".format(" AND ".join(all_rules_in_tags.values()))

 return qurantine_rule

Now after I install it into Databricks Cluster and then import it so I can call the function defined into it.

import rule_engine

rule_dict=rule_engine.get_quarantine_rules("maintained")

It throws this error:

NameError Traceback (most recent call last)

<command-502204870200978> in <cell line: 2>()

1 import rule_engine

----> 2 rule_dict=rule_engine.get_quarantine_rules("maintained")

/local_disk0/.ephemeral_nfs/cluster_libraries/python/lib/python3.9/site-packages/rule_engine/functions.py in get_quarantine_rules(tag)

27 :return: dictionary of rules that matched the tag

28 """

---> 29 all_rules_in_tags=get_rules(tag)

30 qurantine_rule="NOT({0})".format(" AND ".join(all_rules_in_tags.values()))

31 return qurantine_rule

/local_disk0/.ephemeral_nfs/cluster_libraries/python/lib/python3.9/site-packages/rule_engine/functions.py in get_rules(tag)

15 """

16 rules = {}

---> 17 df = spark.read.table("rules")

18 for row in df.filter(col("tag") == tag).collect():

19 rules[row['name']] = row['constraint']

NameError: name 'spark' is not defined

Regards

Rajaniesh