Add initial version of auto estimator

pyzoo/test/zoo/orca/automl/autoestimator/test_autoestimator_keras.py

@@ -82,6 +82,21 @@ def test_fit(self):
         best_model = auto_est.get_best_model()
         assert "hidden_size" in best_model.config
 
+    def test_fit_multiple_times(self):
+        auto_est = AutoEstimator.from_keras(model_creator=model_creator,
+                                            logs_dir="/tmp/zoo_automl_logs",
+                                            resources_per_trial={"cpu": 2},
+                                            name="test_fit")
+        data = get_train_val_data()
+        auto_est.fit(data,
+                     recipe=LinearRecipe(),
+                     metric="mse")
+        with pytest.raises(RuntimeError) as excinfo:
+            auto_est.fit(data,
+                         recipe=LinearRecipe(),
+                         metric="mse")
+        assert "multiple times" in str(excinfo)
+
 
 if __name__ == "__main__":
     pytest.main([__file__])

pyzoo/test/zoo/orca/automl/autoestimator/test_autoestimator_pytorch.py

@@ -146,20 +146,6 @@ def test_fit_optimizer_name(self):
         best_model = auto_est.get_best_model()
         assert best_model.optimizer.__class__.__name__ == "SGD"
 
-    def test_fit_optimizer_class(self):
-        auto_est = AutoEstimator.from_torch(model_creator=model_creator,
-                                            optimizer=torch.optim.SGD,
-                                            loss=nn.BCELoss(),
-                                            logs_dir="/tmp/zoo_automl_logs",
-                                            resources_per_trial={"cpu": 2},
-                                            name="test_fit")
-        data = get_train_val_data()
-        auto_est.fit(data,
-                     recipe=LinearRecipe(),
-                     metric="accuracy")
-        best_model = auto_est.get_best_model()
-        assert best_model.optimizer.__class__.__name__ == "SGD"
-
     def test_fit_invalid_optimizer_name(self):
         invalid_optimizer_name = "ADAM"
         with pytest.raises(ValueError) as excinfo:
@@ -182,6 +168,23 @@ def test_fit_invalid_loss_name(self):
                                                 name="test_fit")
         assert "valid torch loss name" in str(excinfo)
 
+    def test_fit_multiple_times(self):
+        auto_est = AutoEstimator.from_torch(model_creator=model_creator,
+                                            optimizer="SGD",
+                                            loss="BCELoss",
+                                            logs_dir="/tmp/zoo_automl_logs",
+                                            resources_per_trial={"cpu": 2},
+                                            name="test_fit")
+        data = get_train_val_data()
+        auto_est.fit(data,
+                     recipe=LinearRecipe(),
+                     metric="accuracy")
+        with pytest.raises(RuntimeError) as excinfo:
+            auto_est.fit(data,
+                         recipe=LinearRecipe(),
+                         metric="accuracy")
+        assert "multiple times" in str(excinfo)
+
 
 if __name__ == "__main__":
     pytest.main([__file__])

pyzoo/zoo/automl/model/base_keras_model.py

@@ -143,7 +143,7 @@ def restore(self, checkpoint_file, **config):
         self.load_state_dict(state_dict)
 
     def _get_required_parameters(self):
-        return {}
+        return set()
 
     def _get_optional_parameters(self):
         return {"batch_size"}

pyzoo/zoo/orca/automl/auto_estimator.py

@@ -19,6 +19,7 @@ class AutoEstimator:
     def __init__(self, model_builder, searcher):
         self.model_builder = model_builder
         self.searcher = searcher
+        self._fitted = False
 
     @staticmethod
     def from_torch(*,
@@ -94,7 +95,8 @@ def fit(self,
             scheduler=None,
             scheduler_params=None,
             ):
-
+        if self._fitted:
+            raise RuntimeError("auto_estimator.fit is not expected to be called multiple times.")
         self.searcher.compile(data=data,
                               model_create_func=self.model_builder,
                               recipe=recipe,
@@ -103,8 +105,8 @@ def fit(self,
                               search_alg_params=search_alg_params,
                               scheduler=scheduler,
                               scheduler_params=scheduler_params)
-        analysis = self.searcher.run()
-        return analysis
+        self.searcher.run()
+        self._fitted = True
 
     def get_best_model(self):
         best_trial = self.searcher.get_best_trials(k=1)[0]

pyzoo/zoo/orca/automl/pytorch_utils.py

@@ -47,8 +47,8 @@ def validate_pytorch_optim(optim):
             return getattr(torch.optim, optim)
         raise ValueError(f'Must provide a valid torch optimizer name among {PYTORCH_OPTIM_NAMES}')
 
-    if isinstance(optim, types.FunctionType) or issubclass(optim, torch.optim.Optimizer):
+    if isinstance(optim, types.FunctionType):
         return optim
 
-    raise ValueError("Must provide a valid pytorch optimizer name or a pytorch optimizer class"
+    raise ValueError("Must provide a valid pytorch optimizer name "
                      "or a pytorch optimizer creator function.")