This is more code and details:

space = {
      "pool_1": hp.choice('pool_1',np.arange(2, 5,1, dtype=int)),
      "conv_1": hp.choice('conv_1', np.arange(16, 128, 16,dtype=int)),
      "conv_1b": hp.choice('conv_1b', np.arange(2, 5, 1,dtype=int)),
 
      "pool_2": hp.choice('pool_2',np.arange(2, 5,1, dtype=int)), 
      "reg_2" : hp.choice('reg_2', np.arange(0.00005, 0.01, 0.00001, dtype=float)), 
      "conv_2": hp.choice("conv_2", np.arange(16, 128, 16, dtype=int)),
      "conv_2b": hp.choice("conv_2b", np.arange(2, 5, 1, dtype=int)),
 
      "pool_3": hp.choice('pool_3', np.arange(2, 5, 1, dtype=int)),  
      "reg_3" : hp.choice('reg_3', np.arange(0.00005, 0.01, 0.00001, dtype=float)), 
      "conv_3": hp.choice("conv_3", np.arange(16, 128, 16, dtype=int)),
      "conv_3b": hp.choice("conv_3b", np.arange(2, 5, 1, dtype=int)),
 
      "pool_4" : hp.choice('pool_4', np.arange(2, 5, 1, dtype=int)),   
      "reg_4" : hp.choice('reg_4', np.arange(0.00005, 0.01, 0.00001, dtype=float)),
      "conv_4": hp.choice("conv_4", np.arange(16, 128, 16, dtype=int)),
      "conv_4b": hp.choice("conv_4b",np.arange(2, 5, 1, dtype=int)),
      "drop_4": hp.choice('drop_4', np.arange(0.00005, 0.01, 0.00001, dtype=float)) 
    }

def model_builder(params,dense_size): #CNN builder function
    model = Sequential()
    model.add(Conv2D(int(params['conv_1']), (int(params['conv_1b']), int(params['conv_1b'])), activation='relu', input_shape=(150, 150, 3)))
    model.add(MaxPooling2D(int(params['pool_1']), int(params['pool_1'])))
    
    model.add(Conv2D(int(params['conv_2']), (int(params['conv_2b']), int(params['conv_2b'])), activation='relu', kernel_regularizer=L1L2(float(params['reg_2']), float(params['reg_2']))))
    model.add(MaxPooling2D(int(params['pool_2']), int(params['pool_2'])))
    
    model.add(Conv2D(int(params['conv_3']), (int(params['conv_3b']), int(params['conv_3b'])), activation='relu', kernel_regularizer=L1L2(float(params['reg_3']), float(params['reg_3']))))
    model.add(MaxPooling2D(int(params['pool_3']), int(params['pool_3'])))
    
    model.add(Conv2D(int(params['conv_4']), (int(params['conv_4b']), int(params['conv_4b'])), activation='relu', kernel_regularizer=L1L2(float(params['reg_4']), float(params['reg_4']))))
    model.add(MaxPooling2D(int(params['pool_4']), int(params['pool_4'])))
    
    model.add(Dropout(float(params['drop_4'])))
    
    model.add(Flatten())
    model.add(Dense(512, activation='relu'))
    model.add(Dense(dense_size, activation='softmax'))
 
    return model

def CNN_HOF(params): #Hyperopt objective function
    mlflow.tensorflow.autolog()
    model = model_builder(params,dense_size)
    model.compile(loss="categorical_crossentropy",
                optimizer=Adam(),
                metrics=["accuracy"])
 
    history = model.fit(train_generator,
                        steps_per_epoch=train_step,
                        epochs=tuner_epochs,
                        validation_data=valid_generator,
                        validation_steps=valid_step,
                        verbose=2)
  # Evaluate the model
    score = model.evaluate(test_generator, steps=1, verbose=0)
    obj_metric = score[0]
    return {"loss": obj_metric, "status": STATUS_OK}
 
 
spark_trials = SparkTrials(parallelism=4)
...
with mlflow.start_run(run_name=model_name+"_Tuning"):
        best_hyperparam = fmin(fn=CNN_HOF,
                                 space=space,
                                 algo=tpe.suggest,
                                 max_evals=tuner_max_evals,
                                 early_stop_fn=no_progress_loss(10),
                                    trials=spark_trials)

This is the complete error message:

TypeError Traceback (most recent call last)

<command-2155252138731800> in <module>

1 if tuning:

----> 2 Hyperparameter_tuning(model_name)

<command-3238776031025884> in Hyperparameter_tuning(model_name)

2 with mlflow.start_run(run_name=model_name+"_Tuning"):

3 # mlflow.tensorflow.autolog()

----> 4 best_hyperparam = fmin(fn=CNN_HOF,

5 space=space,

6 algo=tpe.suggest,

/databricks/.python_edge_libs/hyperopt/fmin.py in fmin(fn, space, algo, max_evals, timeout, loss_threshold, trials, rstate, allow_trials_fmin, pass_expr_memo_ctrl, catch_eval_exceptions, verbose, return_argmin, points_to_evaluate, max_queue_len, show_progressbar, early_stop_fn, trials_save_file)

563

564 if allow_trials_fmin and hasattr(trials, "fmin"):

--> 565 return trials.fmin(

566 fn,

567 space,

/databricks/.python_edge_libs/hyperopt/instrumentation.py in instrumented(func, self, args, kwargs)

25 )

26 try:

---> 27 return_val = func(*args, **kwargs)

28 except Exception as exc:

29 error_string = "{} with message: {}".format(type(exc).__name__, str(exc))

/databricks/.python_edge_libs/hyperopt/spark.py in fmin(self, fn, space, algo, max_evals, timeout, loss_threshold, max_queue_len, rstate, verbose, pass_expr_memo_ctrl, catch_eval_exceptions, return_argmin, show_progressbar, early_stop_fn, trials_save_file)

311 except BaseException as e:

312 logger.debug("fmin thread exits with an exception raised.")

--> 313 raise e

314 else:

315 logger.debug("fmin thread exits normally.")

/databricks/.python_edge_libs/hyperopt/spark.py in fmin(self, fn, space, algo, max_evals, timeout, loss_threshold, max_queue_len, rstate, verbose, pass_expr_memo_ctrl, catch_eval_exceptions, return_argmin, show_progressbar, early_stop_fn, trials_save_file)

283 )

284

--> 285 res = fmin(

286 fn,

287 space,

/databricks/.python_edge_libs/hyperopt/fmin.py in fmin(fn, space, algo, max_evals, timeout, loss_threshold, trials, rstate, allow_trials_fmin, pass_expr_memo_ctrl, catch_eval_exceptions, verbose, return_argmin, points_to_evaluate, max_queue_len, show_progressbar, early_stop_fn, trials_save_file)

592 domain = base.Domain(fn, space, pass_expr_memo_ctrl=pass_expr_memo_ctrl)

593

--> 594 rval = FMinIter(

595 algo,

596 domain,

/databricks/.python_edge_libs/hyperopt/fmin.py in __init__(self, algo, domain, trials, rstate, asynchronous, max_queue_len, poll_interval_secs, max_evals, timeout, loss_threshold, verbose, show_progressbar, early_stop_fn, trials_save_file)

180 )

181 else:

--> 182 raise e

183 trials.attachments["FMinIter_Domain"] = msg

184

/databricks/.python_edge_libs/hyperopt/fmin.py in __init__(self, algo, domain, trials, rstate, asynchronous, max_queue_len, poll_interval_secs, max_evals, timeout, loss_threshold, verbose, show_progressbar, early_stop_fn, trials_save_file)

163 logger.warning("over-writing old domain trials attachment")

164 try:

--> 165 msg = pickler.dumps(domain)

166 except TypeError as e:

167 if "cannot pickle '_thread.RLock' object" in str(e):

/databricks/python/lib/python3.8/site-packages/cloudpickle/cloudpickle_fast.py in dumps(obj, protocol, buffer_callback)

71 file, protocol=protocol, buffer_callback=buffer_callback

72 )

---> 73 cp.dump(obj)

74 return file.getvalue()

75

/databricks/python/lib/python3.8/site-packages/cloudpickle/cloudpickle_fast.py in dump(self, obj)

561 def dump(self, obj):

562 try:

--> 563 return Pickler.dump(self, obj)

564 except RuntimeError as e:

565 if "recursion" in e.args[0]:

TypeError: cannot pickle '_thread.lock' object