Documentation, test and bugfix (#556)

* add bs restore test

* use default metric when not provided

* update documentation

* remove print

* period

* remove bs restore test

* Update website/docs/Use-Cases/Task-Oriented-AutoML.md

flaml/tune/tune.py

@@ -371,7 +371,7 @@ def run(
         )
     else:
         if metric is None or mode is None:
-            metric = metric or search_alg.metric
+            metric = metric or search_alg.metric or DEFAULT_METRIC
             mode = mode or search_alg.mode
         if ray_import:
             from ray.tune.suggest import ConcurrencyLimiter

test/tune/test_restore.py

@@ -19,7 +19,7 @@ class AbstractWarmStartTest:
         # ray.shutdown()
 
     def set_basic_conf(self):
-        raise NotImplementedError()
+        raise NotImplementedError
 
     def run_part_from_scratch(self):
         np.random.seed(162)
@@ -36,7 +36,6 @@ class AbstractWarmStartTest:
         search_alg2, cost = self.set_basic_conf()
         search_alg2 = ConcurrencyLimiter(search_alg2, 1)
         search_alg2.restore(checkpoint_path)
-        np.random.set_state(random_state)
         return tune.run(cost, num_samples=5, search_alg=search_alg2, verbose=0)
 
     def run_full(self):
@@ -82,16 +81,16 @@ class CFOWarmStartTest(AbstractWarmStartTest, unittest.TestCase):
         return search_alg, cost
 
 
-# # # Not doing test for BS because of problems with random seed in OptunaSearch
 # class BlendsearchWarmStartTest(AbstractWarmStartTest, unittest.TestCase):
 #     def set_basic_conf(self):
+#         from flaml import BlendSearch
 #         space = {
 #             "height": tune.uniform(-100, 100),
 #             "width": tune.randint(0, 100),
 #         }
 
 #         def cost(param):
-#             tune.report(loss=(param["height"] - 14)**2 - abs(param["width"] - 3))
+#             tune.report(loss=(param["height"] - 14) ** 2 - abs(param["width"] - 3))
 
 #         search_alg = BlendSearch(
 #             space=space,

website/docs/Use-Cases/Task-Oriented-AutoML.md

@@ -357,7 +357,8 @@ automl2 = AutoML()
 automl2.fit(X_train, y_train, time_budget=7200, starting_points=automl1.best_config_per_estimator)
 ```
 
-`starting_points` is a dictionary. The keys are the estimator names. If you do not need to specify starting points for an estimator, exclude its name from the dictionary. The value for each key can be either a dictionary of a list of dictionaries, corresponding to one hyperparameter configuration, or multiple hyperparameter configurations, respectively.
+`starting_points` is a dictionary or a str to specify the starting hyperparameter config. (1) When it is a dictionary, the keys are the estimator names. If you do not need to specify starting points for an estimator, exclude its name from the dictionary. The value for each key can be either a dictionary of a list of dictionaries, corresponding to one hyperparameter configuration, or multiple hyperparameter configurations, respectively. (2) When it is a str: if "data", use data-dependent defaults; if "data:path", use data-dependent defaults which are stored at path; if "static", use data-independent defaults. Please find more details about data-dependent defaults in [zero shot AutoML](https://microsoft.github.io/FLAML/docs/Use-Cases/Zero-Shot-AutoML#combine-zero-shot-automl-and-hyperparameter-tuning).
+
 
 ### Log the trials
 

website/docs/Use-Cases/Tune-User-Defined-Function.md

@@ -43,7 +43,7 @@ def evaluate_config(config: dict):
     # we can return a single float as a score on the input config:
     # return score
     # or, we can return a dictionary that maps metric name to metric value:
-    return {"score": score, "evaluation_cost": faked_evaluation_cost, "constraint_metric": x * y}
+    return {"score": score, "evaluation_cost": faked_evaluation_cost, "constraint_metric": config["x"] * config["y"]}
 ```
 
 When the evaluation function returns a dictionary of metrics, you need to specify the name of the metric to optimize via the argument `metric` (this can be skipped when the function is just returning a scalar). In addition, you need to specify a mode of your optimization/tuning task (maximization or minimization) via the argument `mode` by choosing from "min" or "max".
@@ -403,7 +403,7 @@ analysis = tune.run(
 Related arguments:
 
 - `points_to_evaluate`: A list of initial hyperparameter configurations to run first.
-- `evaluated_rewards`: If you have previously evaluated the parameters passed in as `points_to_evaluate` , you can avoid re-running those trials by passing in the reward attributes as a list so the optimizer can be told the results without needing to re-compute the trial. Must be the same length as `points_to_evaluate`.
+- `evaluated_rewards`: If you have previously evaluated the parameters passed in as `points_to_evaluate` , you can avoid re-running those trials by passing in the reward attributes as a list so the optimizer can be told the results without needing to re-compute the trial. Must be the same length or shorter length than `points_to_evaluate`.
 
 If you are aware of some good hyperparameter configurations, you are encouraged to provide them via `points_to_evaluate`. The search algorithm will try them first and use them to bootstrap the search.
 
@@ -425,6 +425,8 @@ config_search_space = {
 points_to_evaluate = [
     {"b": .99, "a": 3},
     {"b": .99, "a": 2},
+    {"b": .80, "a": 3},
+    {"b": .80, "a": 2},
 ]
 evaluated_rewards = [3.99, 2.99]