From 05bd12d1f05046eb106e2578b543da2a157d702e Mon Sep 17 00:00:00 2001 From: shuki Date: Sun, 27 Jan 2019 17:48:33 +0100 Subject: [PATCH 01/16] WIP start on rework data-structure --- cave/analyzer/base_analyzer.py | 2 +- cave/cavefacade.py | 319 +++++++++++++++----------------- cave/reader/configurator_run.py | 258 +++++++++++++++++++------- cave/utils/hpbandster2smac.py | 10 +- doc/dev_guide.rst | 28 +++ 5 files changed, 373 insertions(+), 244 deletions(-) create mode 100644 doc/dev_guide.rst diff --git a/cave/analyzer/base_analyzer.py b/cave/analyzer/base_analyzer.py index c4a19809..118af0bb 100644 --- a/cave/analyzer/base_analyzer.py +++ b/cave/analyzer/base_analyzer.py @@ -3,7 +3,7 @@ class BaseAnalyzer(object): def __init__(self, *args, **kwargs): - self.plots = [] + pass def get_static_plots(self) -> List[str]: """ Returns plot-paths, if any are available diff --git a/cave/cavefacade.py b/cave/cavefacade.py index 1587ad0f..3a122a0d 100644 --- a/cave/cavefacade.py +++ b/cave/cavefacade.py @@ -177,6 +177,7 @@ def __init__(self, verbose_level: str from [OFF, INFO, DEBUG, DEV_DEBUG and WARNING] """ + # Administrative self.logger = logging.getLogger(self.__module__ + '.' + self.__class__.__name__) self.output_dir = output_dir self.set_verbosity(verbose_level.upper()) @@ -196,6 +197,7 @@ def __init__(self, self.seed = seed self.rng = np.random.RandomState(seed) self.use_budgets = use_budgets + self.folders = folders self.ta_exec_dir = ta_exec_dir self.file_format = file_format self.validation_format = validation_format @@ -203,15 +205,11 @@ def __init__(self, self.pimp_max_samples = pimp_max_samples self.fanova_pairwise = fanova_pairwise - # To be set during execution (used for dependencies of analysis-methods) - self.param_imp = OrderedDict() - self.feature_imp = OrderedDict() - self.evaluators = [] - self.validator = None - - self.feature_names = None + # If only one ta_exec_dir, need to extend so it's same length as folders + self.ta_exec_dir = self.ta_exec_dir.extend([self.ta_exec_dir[0] for i in range(len(self.folders) - len(self.ta_exec_dir))]) - self.bohb_result = None # only relevant for bohb_result + # To be set during execution (used for dependencies of analysis-methods) + # TODO this should go into ConfiguratorRun # Create output_dir if necessary self.logger.info("Saving results to '%s'", self.output_dir) @@ -219,53 +217,14 @@ def __init__(self, self.logger.debug("Output-dir '%s' does not exist, creating", self.output_dir) os.makedirs(output_dir) - if file_format == 'BOHB': - if len(folders) != 1: - raise ValueError("For file format BOHB you can only specify one folder.") - self.use_budgets = True - self.bohb_result, folders = HpBandSter2SMAC().convert(folders[0]) - if "DEBUG" in self.verbose_level: - for f in folders: - debug_f = os.path.join(output_dir, 'debug', os.path.basename(f)) - shutil.rmtree(debug_f, ignore_errors=True) - shutil.copytree(f, debug_f) - - file_format = 'SMAC3' - # Save all relevant configurator-runs in a list self.logger.debug("Folders: %s; ta-exec-dirs: %s", str(folders), str(ta_exec_dir)) - self.runs = [] - if len(ta_exec_dir) < len(folders): - for i in range(len(folders) - len(ta_exec_dir)): - ta_exec_dir.append(ta_exec_dir[0]) - for ta_exec_dir, folder in zip(ta_exec_dir, folders): - try: - self.logger.debug("Collecting data from %s.", folder) - self.runs.append(ConfiguratorRun(folder, ta_exec_dir, file_format=file_format, - validation_format=validation_format)) - except Exception as err: - self.logger.warning("Folder %s could with ta_exec_dir %s not be loaded, failed with error message: %s", - folder, ta_exec_dir, err) - self.logger.exception(err) - continue + self.use_budgets = file_format == 'BOHB' + self.runs = RunsContainer(ta_exec_dirs, folders) if not self.runs: raise ValueError("None of the specified folders could be loaded.") - self.folder_to_run = {os.path.basename(run.folder) : run for run in self.runs} # Use scenario of first run for general purposes (expecting they are all the same anyway! - self.scenario = self.runs[0].solver.scenario - scenario_sanity_check(self.scenario, self.logger) - self.feature_names = self._get_feature_names() - self.default = self.scenario.cs.get_default_configuration() - - # All runs that have been actually explored during optimization - self.global_original_rh = None - # All original runs + validated runs if available - self.global_validated_rh = None - # All validated runs + EPM-estimated for def and inc on all insts - self.global_epm_rh = None - self.pimp = None - self.model = None if self.use_budgets: self._init_helper_budgets() @@ -287,6 +246,7 @@ def __init__(self, self.builder = HTMLBuilder(self.output_dir, "CAVE", logo_fn=logo_fn, logo_custom=custom_logo==logo_fn) self.website = OrderedDict([]) + def _init_helper_budgets(self): """ Each run gets it's own CAVE-instance. This way, we can simply use the individual objects (runhistories, @@ -311,108 +271,6 @@ def _init_helper_budgets(self): verbose_level='OFF') self.incumbent = self.runs[-1].incumbent - def _init_helper_no_budgets(self): - """ - No budgets means using global, aggregated runhistories to analyze the Configurator's behaviour. - Also it creates an EPM using all available information, since all runs are "equal". - """ - self.global_original_rh = RunHistory(average_cost) - self.global_validated_rh = RunHistory(average_cost) - self.global_epm_rh = RunHistory(average_cost) - self.logger.debug("Update original rh with all available rhs!") - for run in self.runs: - self.global_original_rh.update(run.original_runhistory, origin=DataOrigin.INTERNAL) - self.global_validated_rh.update(run.original_runhistory, origin=DataOrigin.INTERNAL) - if run.validated_runhistory: - self.global_validated_rh.update(run.validated_runhistory, origin=DataOrigin.EXTERNAL_SAME_INSTANCES) - - self._init_pimp_and_validator(self.global_validated_rh) - - # Estimate missing costs for [def, inc1, inc2, ...] - self._validate_default_and_incumbents(self.validation_method, self.ta_exec_dir) - self.global_epm_rh.update(self.global_validated_rh) - - for rh_name, rh in [("original", self.global_original_rh), - ("validated", self.global_validated_rh), - ("epm", self.global_epm_rh)]: - self.logger.debug('Combined number of RunHistory data points for %s runhistory: %d ' - '# Configurations: %d. # Configurator runs: %d', - rh_name, len(rh.data), len(rh.get_all_configs()), len(self.runs)) - - # Sort runs (best first) - self.runs = sorted(self.runs, key=lambda run: self.global_epm_rh.get_cost(run.solver.incumbent)) - self.best_run = self.runs[0] - - self.incumbent = self.pimp.incumbent = self.best_run.solver.incumbent - self.logger.debug("Overall best run: %s, with incumbent: %s", self.best_run.folder, self.incumbent) - - def _init_pimp_and_validator(self, rh, alternative_output_dir=None): - """Create ParameterImportance-object and use it's trained model for validation and further predictions - We pass validated runhistory, so that the returned model will be based on as much information as possible - - Parameters - ---------- - rh: RunHistory - runhistory used to build EPM - alternative_output_dir: str - e.g. for budgets we want pimp to use an alternative output-dir (subfolders per budget) - """ - self.logger.debug("Using '%s' as output for pimp", alternative_output_dir if alternative_output_dir else - self.output_dir) - self.pimp = Importance(scenario=copy.deepcopy(self.scenario), - runhistory=rh, - incumbent=self.default, # Inject correct incumbent later - parameters_to_evaluate=4, - save_folder=alternative_output_dir if alternative_output_dir else self.output_dir, - seed=self.rng.randint(1, 100000), - max_sample_size=self.pimp_max_samples, - fANOVA_pairwise=self.fanova_pairwise, - preprocess=False, - verbose=self.verbose_level != 'OFF', # disable progressbars - ) - self.model = self.pimp.model - - # Validator (initialize without trajectory) - self.validator = Validator(self.scenario, None, None) - self.validator.epm = self.model - - @timing - def _validate_default_and_incumbents(self, method, ta_exec_dir): - """Validate default and incumbent configurations on all instances possible. - Either use validation (physically execute the target algorithm) or EPM-estimate and update according runhistory - (validation -> self.global_validated_rh; epm -> self.global_epm_rh). - - Parameters - ---------- - method: str - epm or validation - ta_exec_dir: str - path from where the target algorithm can be executed as found in scenario (only used for actual validation) - """ - for run in self.runs: - self.logger.debug("Validating %s using %s!", run.folder, method) - self.validator.traj = run.traj - if method == "validation": - with _changedir(ta_exec_dir): - # TODO determine # repetitions - new_rh = self.validator.validate('def+inc', 'train+test', 1, -1, runhistory=self.global_validated_rh) - self.global_validated_rh.update(new_rh) - elif method == "epm": - # Only do test-instances if features for test-instances are available - instance_mode = 'train+test' - if (any([i not in self.scenario.feature_dict for i in self.scenario.test_insts]) and - any([i in self.scenario.feature_dict for i in self.scenario.train_insts])): # noqa - self.logger.debug("No features provided for test-instances (but for train!). " - "Cannot validate on \"epm\".") - self.logger.warning("Features detected for train-instances, but not for test-instances. This is " - "unintended usage and may lead to errors for some analysis-methods.") - instance_mode = 'train' - - new_rh = self.validator.validate_epm('def+inc', instance_mode, 1, runhistory=self.global_validated_rh) - self.global_epm_rh.update(new_rh) - else: - raise ValueError("Missing data method illegal (%s)", method) - self.validator.traj = None # Avoid usage-mistakes @timing def analyze(self, @@ -479,7 +337,6 @@ def analyze(self, % (num_configs, num_params)) param_importance = [] - # Start analysis headings = ["Meta Data", "Best Configuration", @@ -1026,7 +883,7 @@ def budget_correlation(self, cave): def print_budgets(self): """If the analyzed configurator uses budgets, print a list of available budgets.""" if self.use_budgets: - print(list(self.folder_to_run.keys())) + print(self.runs.get_budgets()) else: raise NotImplementedError("This CAVE instance does not seem to use budgets.") @@ -1037,26 +894,6 @@ def _get_tooltip(self, f): tooltip = tooltip.replace("\n", " ") return tooltip - def _get_feature_names(self): - if not self.scenario.feature_dict: - self.logger.info("No features available. Skipping feature analysis.") - return - feat_fn = self.scenario.feature_fn - if not self.scenario.feature_names: - self.logger.debug("`scenario.feature_names` is not set. Loading from '%s'", feat_fn) - with _changedir(self.ta_exec_dir if self.ta_exec_dir else '.'): - if not feat_fn or not os.path.exists(feat_fn): - self.logger.warning("Feature names are missing. Either provide valid feature_file in scenario " - "(currently %s) or set `scenario.feature_names` manually." % feat_fn) - self.logger.error("Skipping Feature Analysis.") - return - else: - # Feature names are contained in feature-file and retrieved - feat_names = InputReader().read_instance_features_file(feat_fn)[0] - else: - feat_names = copy.deepcopy(self.scenario.feature_names) - return feat_names - def _build_website(self): self.builder.generate_html(self.website) @@ -1104,3 +941,139 @@ def set_verbosity(self, level): fh.setLevel(logging.DEBUG) fh.setFormatter(formatter) logging.getLogger().addHandler(fh) + + +class RunsContainer(object): + + def __init__(self, ta_exec_dirs, folders): + """ Reads in runs. There will be `n_budgets * m_parallel_execution` runs in CAVE. + + Parameters + ---------- + ta_exec_dirs: List[str] + list of execution directories for target-algorithms (to find filepaths, etc.). If you're not sure, just set + to current working directory. + folders: List[str] + list of folders to read in + output_dir: str + path for intermediate results of conversion + file_format, validation_format: str, str + formats for data (from [SMAC3, SMAC2, BOHB, CSV(, NONE)]), where NONE is only valid for validation + """ + run = [] + + self.budgets2folders = {} + self.folders2budgets = {} + if self.use_budgets: + # Convert into SMAC-format + for f in folders: + if not f in folders2budgets: + self.folders2budgets[f] = {} + self.bohb_result, sub_folders, budgets = HpBandSter2SMAC().convert(f, os.path.join(output_dir, 'data')) + for sf, b in zip(sub_folders, budgets): + self.logger.debug("Collecting data from %s after converting to %s for budget %s.", f, sf, b) + cr = ConfiguratorRun(sf, + '.', + file_format=self.file_format, + validation_format=self.validation_format, + budget=b) + if not b in budgets2folders: + budgets2folders[b] = {} + self.budgets2folders[b][sf] = cr + self.folders2budgets[sf][b] = cr + + runs.append(cr) + # Add aggregated runs + for b, f in budgets2folders.items(): + runs.append('aggregate_budget_{}'.format(b), aggregate_configurator_runs(list(f.values()))) + for f, b in budget + else: + for ta_exec_dir, folder in zip(ta_exec_dirs, folders): + self.logger.debug("Collecting data from %s.", folder) + cr = ConfiguratorRun(f, + ta_exec_dir, + file_format=self.file_format, + validation_format=self.validation_format, + )) + runs.append(cr) + self.folder2run[f] = cr + + + def __getitem__(self, key): + """ Return highest budget for given folder. """ + if self.use_budgets: + return self.folders2budgets[key][self.get_highest_budget()] + else: + return self.folder2run[key] + + def get_highest_budget(self): + return max(self.budgets2folders.keys()) if self.use_budgets else None + + def get_budgets(self): + return list(self.budgets2folders.keys()) + + def get_runs_for_budget(self, b): + return list(self.budgets2folders.values()) + + def get_folders(self): + if self.use_budgets: + return list(self.folders2budgets.keys()) + else: + return list(self.folder2run.keys()) + + def get_runs_for_folder(self, f) + if self.use_budgets: + return list(self.folders2budgets[f].values()) + else: + return list(self.folder2run.values()) + + def get_aggregated(self, keep_budgets=True, keep_folders=False): + """ Collapse data-structure along a given "axis". + + Returns + ------- + aggregated_runs: either ConfiguratorRun or Dict(str->ConfiguratorRun) + run(s) with aggregated data + """ + if not self.use_budgets: + keep_budgets = False + + if (not keep_budgets) and (not keep_folders): + return self._aggregate(self.runs) + elif keep_budgets: + return {b : self._aggregate(self.get_runs_for_budget(b)) for b in self.get_budgets()} + elif keep_folders: + return {f : self._aggregate(self.get_runs_for_folder(f)) for f in self.get_folders()} + else: + return self.runs + + def _aggregate(self, runs): + """ + """ + orig_rh, vali_rh = RunHistory(average_cost), RunHistory(average_cost) + for run in runs: + self.orig_rh.update(run.original_runhistory, origin=DataOrigin.INTERNAL) + self.vali_rh.update(run.original_runhistory, origin=DataOrigin.INTERNAL) + if run.validated_runhistory: + self.vali_rh.update(run.validated_runhistory, origin=DataOrigin.EXTERNAL_SAME_INSTANCES) + + self._init_pimp_and_validator(self.global_validated_rh) + + # Estimate missing costs for [def, inc1, inc2, ...] + self._validate_default_and_incumbents(self.validation_method, self.ta_exec_dir) + self.global_epm_rh.update(self.global_validated_rh) + + for rh_name, rh in [("original", self.global_original_rh), + ("validated", self.global_validated_rh), + ("epm", self.global_epm_rh)]: + self.logger.debug('Combined number of RunHistory data points for %s runhistory: %d ' + '# Configurations: %d. # Configurator runs: %d', + rh_name, len(rh.data), len(rh.get_all_configs()), len(self.runs)) + + # Sort runs (best first) + self.runs = sorted(self.runs, key=lambda run: self.global_epm_rh.get_cost(run.solver.incumbent)) + self.best_run = self.runs[0] + + self.incumbent = self.pimp.incumbent = self.best_run.solver.incumbent + self.logger.debug("Overall best run: %s, with incumbent: %s", self.best_run.folder, self.incumbent) + diff --git a/cave/reader/configurator_run.py b/cave/reader/configurator_run.py index a65d3d65..50b46fb5 100644 --- a/cave/reader/configurator_run.py +++ b/cave/reader/configurator_run.py @@ -8,6 +8,20 @@ from cave.reader.smac2_reader import SMAC2Reader from cave.reader.csv_reader import CSVReader +def get_reader(name): + """ Returns an appropriate reader for the specified format. """ + if name == 'SMAC3': + return SMAC3Reader(folder, ta_exec_dir) + elif name == 'BOHB': + self.logger.debug("File format is BOHB, assmuming data was converted to SMAC3-format using " + "HpBandSter2SMAC from cave.utils.converter.hpbandster2smac.") + return SMAC3Reader(folder, ta_exec_dir) + elif name == 'SMAC2': + return SMAC2Reader(folder, ta_exec_dir) + elif name == 'CSV': + return CSVReader(folder, ta_exec_dir) + else: + raise ValueError("%s not supported as file-format" % name) class ConfiguratorRun(SMAC): """ @@ -17,90 +31,115 @@ class ConfiguratorRun(SMAC): trajectory and handling original/validated data appropriately. """ def __init__(self, - folder: str, - ta_exec_dir: str, - file_format: str='SMAC3', - validation_format: str='NONE'): + scenario, + original_runhistory, + validated_runhistory, + trajectory, + folder, + ta_exec_dir, + file_format, + validation_format, + budget=None, + ): + self.scenario = scenario + self.original_runhistory = original_runhistory + self.validated_runhistory = validated_runhistory + self.trajectory = trajectory + self.path_to_folder = path_to_folder + self.ta_exec_dir = ta_exec_dir + self.file_format = file_format + self.validation_format = validation_format + self.budget = budget + + self.default = self.scenario.cs.get_default_configuration() + self.incumbent = self.trajectory[-1]['incumbent'] if self.trajectory else None + self.feature_names = self._get_feature_names() + + # Create combined runhistory to collect all "real" runs + self.combined_runhistory = RunHistory(average_cost) + self.combined_runhistory.update(self.original_runhistory, origin=DataOrigin.INTERNAL) + if self.validated_runhistory: + self.combined_runhistory.update(self.validated_runhistory, origin=DataOrigin.EXTERNAL_SAME_INSTANCES) + + # Create runhistory with estimated runs (create Importance-object of pimp and use epm-model for validation) + self.epm_runhistory = RunHistory(average_cost) + self.epm_runhistory.update(self.combined_runhistory) + self._init_pimp_and_validator() + + + # Set during execution, to share information between Analyzers + self.share_information = {'parameter_importance' : OrderedDict(), + 'feature_importance' : OrderedDict(), + 'evaluators' : [], + 'validator' : None} + + # Initialize SMAC-object + super().__init__(scenario=self.scen, runhistory=self.combined_runhistory) # restore_incumbent=incumbent) + # TODO use restore, delete next line + self.solver.incumbent = self.incumbent + + @classmethod + def from_folder(cls, + folder: str, + ta_exec_dir: str, + file_format: str='SMAC3', + validation_format: str='NONE', + budget=None, + ): """Initialize scenario, runhistory and incumbent from folder, execute - init-method of SMAC facade (so you could simply use SMAC-instances instead) + init-method of SMAC facade (so you could simply use SMAC-instances instead). Parameters ---------- folder: string - output-dir of this run + output-dir of this run -> this is also the 'id' for a single run in parallel optimization ta_exec_dir: string - if the execution directory for the SMAC-run differs from the cwd, - there might be problems loading instance-, feature- or PCS-files - in the scenario-object. since instance- and PCS-files are necessary, - specify the path to the execution-dir of SMAC here + if the execution directory for the SMAC-run differs from the cwd, there might be problems loading instance-, + feature- or PCS-files in the scenario-object. since instance- and PCS-files are necessary, specify the path + to the execution-dir of SMAC here file_format: string from [SMAC2, SMAC3, CSV] validation_format: string from [SMAC2, SMAC3, CSV, NONE], in which format to look for validated data + budget: int + budget for this run-instance (only for budgeted optimization!) """ self.logger = logging.getLogger("cave.ConfiguratorRun.{}".format(folder)) - self.cave = None # Set if we analyze configurators that use budgets + self.logger.debug("Loading from \'%s\' with ta_exec_dir \'%s\' with file-format '%s' and validation-format %s. " + "Budget (if present): %s", folder, ta_exec_dir, file_format, validation_format, budget) - self.folder = folder - self.ta_exec_dir = ta_exec_dir - self.file_format = file_format - self.validation_format = validation_format + self.validation_format = validation_format if validation_format != 'NONE' else None - self.logger.debug("Loading from \'%s\' with ta_exec_dir \'%s\'.", - folder, ta_exec_dir) - if validation_format == 'NONE': - validation_format = None - - def get_reader(name): - if name == 'SMAC3': - return SMAC3Reader(folder, ta_exec_dir) - elif name == 'BOHB': - self.logger.debug("File format is BOHB, assmuming data was converted to SMAC3-format using " - "HpBandSter2SMAC from cave.utils.converter.hpbandster2smac.") - return SMAC3Reader(folder, ta_exec_dir) - elif name == 'SMAC2': - return SMAC2Reader(folder, ta_exec_dir) - elif name == 'CSV': - return CSVReader(folder, ta_exec_dir) - else: - raise ValueError("%s not supported as file-format" % name) - self.reader = get_reader(file_format) - - self.scen = self.reader.get_scenario() - self.original_runhistory = self.reader.get_runhistory(self.scen.cs) - self.validated_runhistory = None - - self.traj = self.reader.get_trajectory(cs=self.scen.cs) - self.default = self.scen.cs.get_default_configuration() - self.incumbent = self.traj[-1]['incumbent'] - self.train_inst = self.scen.train_insts - self.test_inst = self.scen.test_insts - self._check_rh_for_inc_and_def(self.original_runhistory, 'original runhistory') + #### Read in data (scenario, runhistory & trajectory) + reader = get_reader(file_format) - if validation_format: - self.logger.debug('Using format %s for validation', validation_format) - reader = get_reader(validation_format) - reader.scen = self.scen - self.validated_runhistory = reader.get_validated_runhistory(self.scen.cs) - self._check_rh_for_inc_and_def(self.validated_runhistory, 'validated runhistory') - self.logger.info("Found validated runhistory for \"%s\" and using " - "it for evaluation. #configs in validated rh: %d", - self.folder, len(self.validated_runhistory.config_ids)) + scenario = self.reader.get_scenario() + scenario_sanity_check(scenario, self.logger) + original_runhistory = reader.get_runhistory(scenario.cs) + validated_runhistory = None - self.combined_runhistory = RunHistory(average_cost) - self.combined_runhistory.update(self.original_runhistory, - origin=DataOrigin.INTERNAL) - if self.validated_runhistory: - self.combined_runhistory.update(self.validated_runhistory, - origin=DataOrigin.EXTERNAL_SAME_INSTANCES) + trajectory = reader.get_trajectory(cs=scenario.cs) - self.epm_runhistory = RunHistory(average_cost) - self.epm_runhistory.update(self.combined_runhistory) + if validation_format: + reader = get_reader(validation_format) + reader.scen = scenario + validated_runhistory = reader.get_validated_runhistory(scenario.cs) + self.logger.info("Found validated runhistory for \"%s\" and using it for evaluation. #configs in " + "validated rh: %d", folder, len(validated_runhistory.config_ids)) - # Initialize SMAC-object - super().__init__(scenario=self.scen, runhistory=self.combined_runhistory) # restore_incumbent=incumbent) - # TODO use restore, delete next line - self.solver.incumbent = self.incumbent + self.__init__( + scenario, + original_runhistory, + validated_runhistory, + combined_runhistory, + epm_runhistory, + trajectory, + folder, + ta_exec_dir, + file_format, + validation_format, + budget=None, + ) def get_incumbent(self): return self.solver.incumbent @@ -136,3 +175,90 @@ def _check_rh_for_inc_and_def(self, rh, name=''): i_name, c_name, self.folder) return_value = False return return_value + + def _init_pimp_and_validator(self, alternative_output_dir=None): + """Create ParameterImportance-object and use it's trained model for validation and further predictions We pass a + combined (original + validated) runhistory, so that the returned model will be based on as much information as + possible + + Parameters + ---------- + alternative_output_dir: str + e.g. for budgets we want pimp to use an alternative output-dir (subfolders per budget) + """ + self.logger.debug("Using '%s' as output for pimp", alternative_output_dir if alternative_output_dir else + self.output_dir) + self.pimp = Importance(scenario=copy.deepcopy(self.scenario), + runhistory=self.combined_runhistory, + incumbent=self.incumbent if self.incumbent else self.default, + parameters_to_evaluate=4, + save_folder=alternative_output_dir if alternative_output_dir else self.output_dir, + seed=self.rng.randint(1, 100000), + max_sample_size=self.pimp_max_samples, + fANOVA_pairwise=self.fanova_pairwise, + preprocess=False, + verbose=self.verbose_level != 'OFF', # disable progressbars + ) + self.model = self.pimp.model + + # Validator (initialize without trajectory) + self.validator = Validator(self.scenario, None, None) + self.validator.epm = self.model + + @timing + def _validate_default_and_incumbents(self, method, ta_exec_dir): + """Validate default and incumbent configurations on all instances possible. + Either use validation (physically execute the target algorithm) or EPM-estimate and update according runhistory + (validation -> self.global_validated_rh; epm -> self.global_epm_rh). + + Parameters + ---------- + method: str + epm or validation + ta_exec_dir: str + path from where the target algorithm can be executed as found in scenario (only used for actual validation) + """ + self.logger.debug("Validating %s using %s!", self.folder, method) + self.validator.traj = self.trajectory + if method == "validation": + with _changedir(ta_exec_dir): + # TODO determine # repetitions + new_rh = self.validator.validate('def+inc', 'train+test', 1, -1, runhistory=self.combined_runhistory) + self.validated_runhistory.update(new_rh) + self.combined_runhistory_rh.update(new_rh) + elif method == "epm": + # Only do test-instances if features for test-instances are available + instance_mode = 'train+test' + if (any([i not in self.scenario.feature_dict for i in self.scenario.test_insts]) and + any([i in self.scenario.feature_dict for i in self.scenario.train_insts])): # noqa + self.logger.debug("No features provided for test-instances (but for train!). Cannot validate on \"epm\".") + self.logger.warning("Features detected for train-instances, but not for test-instances. This is " + "unintended usage and may lead to errors for some analysis-methods.") + instance_mode = 'train' + + new_rh = self.validator.validate_epm('def+inc', instance_mode, 1, runhistory=self.combined_runhistory) + self.epm_runhistory.update(new_rh) + else: + raise ValueError("Missing data method illegal (%s)", method) + self.validator.traj = None # Avoid usage-mistakes + + def _get_feature_names(self): + if not self.scenario.feature_dict: + self.logger.info("No features available. Skipping feature analysis.") + return + feat_fn = self.scenario.feature_fn + if not self.scenario.feature_names: + self.logger.debug("`scenario.feature_names` is not set. Loading from '%s'", feat_fn) + with _changedir(self.ta_exec_dir if self.ta_exec_dir else '.'): + if not feat_fn or not os.path.exists(feat_fn): + self.logger.warning("Feature names are missing. Either provide valid feature_file in scenario " + "(currently %s) or set `scenario.feature_names` manually." % feat_fn) + self.logger.error("Skipping Feature Analysis.") + return + else: + # Feature names are contained in feature-file and retrieved + feat_names = InputReader().read_instance_features_file(feat_fn)[0] + else: + feat_names = copy.deepcopy(self.scenario.feature_names) + return feat_names + diff --git a/cave/utils/hpbandster2smac.py b/cave/utils/hpbandster2smac.py index 668c611c..7a999839 100644 --- a/cave/utils/hpbandster2smac.py +++ b/cave/utils/hpbandster2smac.py @@ -22,7 +22,7 @@ class HpBandSter2SMAC(object): def __init__(self): self.logger = logging.getLogger(self.__module__ + '.' + self.__class__.__name__) - def convert(self, folder): + def convert(self, folder, output_dir=None): try: from hpbandster.core.result import Result as HPBResult from hpbandster.core.result import logged_results_to_HBS_result @@ -35,9 +35,11 @@ def convert(self, folder): cs, backup_cs = self.load_configspace(folder) # Using temporary files for the intermediate smac-result-like format - tmp_dir = tempfile.mkdtemp() - paths = list(self.hpbandster2smac(result, cs, backup_cs, tmp_dir).values()) - return result, paths + if not output_dir: + output_dir = tempfile.mkdtemp() + budgets, paths = zip(*self.hpbandster2smac(result, cs, backup_cs, output_dir).items()) + + return result, paths, budgets def load_configspace(self, folder): """Will try to load the configspace. If it's a pcs-file, backup_cs will be a list containing all possible diff --git a/doc/dev_guide.rst b/doc/dev_guide.rst new file mode 100644 index 00000000..f18eaf71 --- /dev/null +++ b/doc/dev_guide.rst @@ -0,0 +1,28 @@ +Developer's Guide +----------------- +CAVE aims to be modular and easily extendable. This section summarizes the most important concepts behind the +architecture. + +Custom Analyzers +~~~~~~~~~~~~~~~~ +To write a custom analyzer, you need to inherit from `the BaseAnalyzer `_ + + +While it's possible to generate a HTML-report via the commandline on a +given result-folder, CAVE may also run in interactive mode, running `individual analysis-methods `_ on demand. We provide a +few examples to demonstrate this. +Make sure you followed the `installation details `_ before starting. + +Analyse existing results via the commandline +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +There are example toy results of all supported file formats in the folder `examples +`_ on the github-repo. +Run + +.. code-block:: bash + + cave --folders examples/smac3/example_output/* --ta_exec_dir examples/smac3/ --output CAVE_OUTPUT + +to start the example (assuming you cloned the GitHub-repository in which the example is included). +By default, CAVE will execute all parts of the analysis. To disable certain (timeconsuming) parts From dd33e9bd99e29839af681c07c444c463afc5c3ce Mon Sep 17 00:00:00 2001 From: shuki Date: Fri, 1 Mar 2019 19:24:11 +0100 Subject: [PATCH 02/16] UPDATE version to 1.1.6.dev --- cave/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cave/__version__.py b/cave/__version__.py index 9b102be7..d62e38af 100644 --- a/cave/__version__.py +++ b/cave/__version__.py @@ -1 +1 @@ -__version__ = "1.1.5" +__version__ = "1.1.6.dev" From 961ccf03dfc319ab838d9639fe2f09efe287d214 Mon Sep 17 00:00:00 2001 From: shuki Date: Fri, 1 Mar 2019 21:42:15 +0100 Subject: [PATCH 03/16] UPDATE gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 5f28de84..6c09a451 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.swp *.sh data/ +output/ results/ vespy/ _run_number.txt From 68baf01c613d2c851c04e392ead5416f8c617c7e Mon Sep 17 00:00:00 2001 From: shukon Date: Thu, 14 Mar 2019 16:34:03 +0100 Subject: [PATCH 04/16] REVERT merge --- cave/analyzer/base_analyzer.py | 2 +- cave/cavefacade.py | 305 +++++++++++++++++--------------- cave/reader/configurator_run.py | 258 +++++++-------------------- cave/utils/hpbandster2smac.py | 2 +- doc/dev_guide.rst | 28 --- 5 files changed, 231 insertions(+), 364 deletions(-) delete mode 100644 doc/dev_guide.rst diff --git a/cave/analyzer/base_analyzer.py b/cave/analyzer/base_analyzer.py index 118af0bb..c4a19809 100644 --- a/cave/analyzer/base_analyzer.py +++ b/cave/analyzer/base_analyzer.py @@ -3,7 +3,7 @@ class BaseAnalyzer(object): def __init__(self, *args, **kwargs): - pass + self.plots = [] def get_static_plots(self) -> List[str]: """ Returns plot-paths, if any are available diff --git a/cave/cavefacade.py b/cave/cavefacade.py index 083c7161..f7644eb5 100644 --- a/cave/cavefacade.py +++ b/cave/cavefacade.py @@ -179,7 +179,6 @@ def __init__(self, verbose_level: str from [OFF, INFO, DEBUG, DEV_DEBUG and WARNING] """ - # Administrative self.logger = logging.getLogger(self.__module__ + '.' + self.__class__.__name__) self.output_dir = output_dir self.output_dir_created = False @@ -200,7 +199,6 @@ def __init__(self, self.seed = seed self.rng = np.random.RandomState(seed) self.use_budgets = use_budgets - self.folders = folders self.ta_exec_dir = ta_exec_dir self.file_format = file_format self.validation_format = validation_format @@ -209,8 +207,16 @@ def __init__(self, self.fanova_pairwise = fanova_pairwise self.pc_sort_by = pc_sort_by - # If only one ta_exec_dir, need to extend so it's same length as folders - self.ta_exec_dir = self.ta_exec_dir.extend([self.ta_exec_dir[0] for i in range(len(self.folders) - len(self.ta_exec_dir))]) + # To be set during execution (used for dependencies of analysis-methods) + self.param_imp = OrderedDict() + self.feature_imp = OrderedDict() + self.evaluators = [] + self.validator = None + + self.feature_names = None + + self.num_bohb_results = 0 + self.bohb_result = None # only relevant for bohb_result # Create output_dir if necessary self._create_outputdir(self.output_dir) @@ -229,12 +235,38 @@ def __init__(self, # Save all relevant configurator-runs in a list self.logger.debug("Folders: %s; ta-exec-dirs: %s", str(folders), str(ta_exec_dir)) - self.use_budgets = file_format == 'BOHB' - self.runs = RunsContainer(ta_exec_dirs, folders) + self.runs = [] + if len(ta_exec_dir) < len(folders): + for i in range(len(folders) - len(ta_exec_dir)): + ta_exec_dir.append(ta_exec_dir[0]) + for ta_exec_dir, folder in zip(ta_exec_dir, folders): + try: + self.logger.debug("Collecting data from %s.", folder) + self.runs.append(ConfiguratorRun(folder, ta_exec_dir, file_format=file_format, + validation_format=validation_format)) + except Exception as err: + self.logger.warning("Folder %s could with ta_exec_dir %s not be loaded, failed with error message: %s", + folder, ta_exec_dir, err) + self.logger.exception(err) + continue if not self.runs: raise ValueError("None of the specified folders could be loaded.") + self.folder_to_run = {os.path.basename(run.folder) : run for run in self.runs} # Use scenario of first run for general purposes (expecting they are all the same anyway! + self.scenario = self.runs[0].solver.scenario + scenario_sanity_check(self.scenario, self.logger) + self.feature_names = self._get_feature_names() + self.default = self.scenario.cs.get_default_configuration() + + # All runs that have been actually explored during optimization + self.global_original_rh = None + # All original runs + validated runs if available + self.global_validated_rh = None + # All validated runs + EPM-estimated for def and inc on all insts + self.global_epm_rh = None + self.pimp = None + self.model = None if self.use_budgets: self._init_helper_budgets() @@ -256,7 +288,6 @@ def __init__(self, self.builder = HTMLBuilder(self.output_dir, "CAVE", logo_fn=logo_fn, logo_custom=custom_logo==logo_fn) self.website = OrderedDict([]) - def _init_helper_budgets(self): """ Each run gets it's own CAVE-instance. This way, we can simply use the individual objects (runhistories, @@ -281,6 +312,109 @@ def _init_helper_budgets(self): verbose_level='OFF') self.incumbent = self.runs[-1].incumbent + def _init_helper_no_budgets(self): + """ + No budgets means using global, aggregated runhistories to analyze the Configurator's behaviour. + Also it creates an EPM using all available information, since all runs are "equal". + """ + self.global_original_rh = RunHistory(average_cost) + self.global_validated_rh = RunHistory(average_cost) + self.global_epm_rh = RunHistory(average_cost) + self.logger.debug("Update original rh with all available rhs!") + for run in self.runs: + self.global_original_rh.update(run.original_runhistory, origin=DataOrigin.INTERNAL) + self.global_validated_rh.update(run.original_runhistory, origin=DataOrigin.INTERNAL) + if run.validated_runhistory: + self.global_validated_rh.update(run.validated_runhistory, origin=DataOrigin.EXTERNAL_SAME_INSTANCES) + + self._init_pimp_and_validator(self.global_validated_rh) + + # Estimate missing costs for [def, inc1, inc2, ...] + self._validate_default_and_incumbents(self.validation_method, self.ta_exec_dir) + self.global_epm_rh.update(self.global_validated_rh) + + for rh_name, rh in [("original", self.global_original_rh), + ("validated", self.global_validated_rh), + ("epm", self.global_epm_rh)]: + self.logger.debug('Combined number of RunHistory data points for %s runhistory: %d ' + '# Configurations: %d. # Configurator runs: %d', + rh_name, len(rh.data), len(rh.get_all_configs()), len(self.runs)) + + # Sort runs (best first) + self.runs = sorted(self.runs, key=lambda run: self.global_epm_rh.get_cost(run.solver.incumbent)) + self.best_run = self.runs[0] + + self.incumbent = self.pimp.incumbent = self.best_run.solver.incumbent + self.logger.debug("Overall best run: %s, with incumbent: %s", self.best_run.folder, self.incumbent) + + def _init_pimp_and_validator(self, rh, pimp_output_dir=None): + """Create ParameterImportance-object and use it's trained model for validation and further predictions + We pass validated runhistory, so that the returned model will be based on as much information as possible + + Parameters + ---------- + rh: RunHistory + runhistory used to build EPM + alternative_output_dir: str + e.g. for budgets we want pimp to use an alternative output-dir (subfolders per budget) + """ + if not pimp_output_dir: + pimp_output_dir = os.path.join(self.output_dir, 'content') + self.logger.debug("Using '%s' as output for pimp", pimp_output_dir) + self.pimp = Importance(scenario=copy.deepcopy(self.scenario), + runhistory=rh, + incumbent=self.default, # Inject correct incumbent later + save_folder=pimp_output_dir, + seed=self.rng.randint(1, 100000), + max_sample_size=self.pimp_max_samples, + fANOVA_pairwise=self.fanova_pairwise, + preprocess=False, + verbose=self.verbose_level != 'OFF', # disable progressbars + ) + self.model = self.pimp.model + + # Validator (initialize without trajectory) + self.validator = Validator(self.scenario, None, None) + self.validator.epm = self.model + + @timing + def _validate_default_and_incumbents(self, method, ta_exec_dir): + """Validate default and incumbent configurations on all instances possible. + Either use validation (physically execute the target algorithm) or EPM-estimate and update according runhistory + (validation -> self.global_validated_rh; epm -> self.global_epm_rh). + + Parameters + ---------- + method: str + epm or validation + ta_exec_dir: str + path from where the target algorithm can be executed as found in scenario (only used for actual validation) + """ + for run in self.runs: + self.logger.debug("Validating %s using %s!", run.folder, method) + self.validator.traj = run.traj + if method == "validation": + with _changedir(ta_exec_dir): + # TODO determine # repetitions + new_rh = self.validator.validate('def+inc', 'train+test', 1, -1, runhistory=self.global_validated_rh) + self.global_validated_rh.update(new_rh) + elif method == "epm": + # Only do test-instances if features for test-instances are available + instance_mode = 'train+test' + if (any([i not in self.scenario.feature_dict for i in self.scenario.test_insts]) and + any([i in self.scenario.feature_dict for i in self.scenario.train_insts])): # noqa + self.logger.debug("No features provided for test-instances (but for train!). " + "Cannot validate on \"epm\".") + self.logger.warning("Features detected for train-instances, but not for test-instances. This is " + "unintended usage and may lead to errors for some analysis-methods.") + instance_mode = 'train' + + new_rh = self.validator.validate_epm('def+inc', instance_mode, 1, runhistory=self.global_validated_rh) + self.global_epm_rh.update(new_rh) + else: + raise ValueError("Missing data method illegal (%s)", method) + self.validator.traj = None # Avoid usage-mistakes + @timing def analyze(self, performance=True, @@ -346,6 +480,7 @@ def analyze(self, % (num_configs, num_params)) param_importance = [] + # Start analysis headings = ["Meta Data", "Best Configuration", @@ -894,7 +1029,7 @@ def budget_correlation(self, cave): def print_budgets(self): """If the analyzed configurator uses budgets, print a list of available budgets.""" if self.use_budgets: - print(self.runs.get_budgets()) + print(list(self.folder_to_run.keys())) else: raise NotImplementedError("This CAVE instance does not seem to use budgets.") @@ -905,6 +1040,26 @@ def _get_tooltip(self, f): tooltip = tooltip.replace("\n", " ") return tooltip + def _get_feature_names(self): + if not self.scenario.feature_dict: + self.logger.info("No features available. Skipping feature analysis.") + return + feat_fn = self.scenario.feature_fn + if not self.scenario.feature_names: + self.logger.debug("`scenario.feature_names` is not set. Loading from '%s'", feat_fn) + with _changedir(self.ta_exec_dir if self.ta_exec_dir else '.'): + if not feat_fn or not os.path.exists(feat_fn): + self.logger.warning("Feature names are missing. Either provide valid feature_file in scenario " + "(currently %s) or set `scenario.feature_names` manually." % feat_fn) + self.logger.error("Skipping Feature Analysis.") + return + else: + # Feature names are contained in feature-file and retrieved + feat_names = InputReader().read_instance_features_file(feat_fn)[0] + else: + feat_names = copy.deepcopy(self.scenario.feature_names) + return feat_names + def _build_website(self): self.builder.generate_html(self.website) @@ -976,137 +1131,3 @@ def _create_outputdir(self, output_dir): os.path.join(self.output_dir, '.OLD.zip')) self.output_dir_created = True - -class RunsContainer(object): - - def __init__(self, ta_exec_dirs, folders): - """ Reads in runs. There will be `n_budgets * m_parallel_execution` runs in CAVE. - - Parameters - ---------- - ta_exec_dirs: List[str] - list of execution directories for target-algorithms (to find filepaths, etc.). If you're not sure, just set - to current working directory. - folders: List[str] - list of folders to read in - output_dir: str - path for intermediate results of conversion - file_format, validation_format: str, str - formats for data (from [SMAC3, SMAC2, BOHB, CSV(, NONE)]), where NONE is only valid for validation - """ - run = [] - - self.budgets2folders = {} - self.folders2budgets = {} - if self.use_budgets: - # Convert into SMAC-format - for f in folders: - if not f in folders2budgets: - self.folders2budgets[f] = {} - self.bohb_result, sub_folders, budgets = HpBandSter2SMAC().convert(f, os.path.join(output_dir, 'data')) - for sf, b in zip(sub_folders, budgets): - self.logger.debug("Collecting data from %s after converting to %s for budget %s.", f, sf, b) - cr = ConfiguratorRun(sf, - '.', - file_format=self.file_format, - validation_format=self.validation_format, - budget=b) - if not b in budgets2folders: - budgets2folders[b] = {} - self.budgets2folders[b][sf] = cr - self.folders2budgets[sf][b] = cr - - runs.append(cr) - # Add aggregated runs - for b, f in budgets2folders.items(): - runs.append('aggregate_budget_{}'.format(b), aggregate_configurator_runs(list(f.values()))) - for f, b in budget - else: - for ta_exec_dir, folder in zip(ta_exec_dirs, folders): - self.logger.debug("Collecting data from %s.", folder) - cr = ConfiguratorRun(f, - ta_exec_dir, - file_format=self.file_format, - validation_format=self.validation_format, - )) - runs.append(cr) - self.folder2run[f] = cr - - - def __getitem__(self, key): - """ Return highest budget for given folder. """ - if self.use_budgets: - return self.folders2budgets[key][self.get_highest_budget()] - else: - return self.folder2run[key] - - def get_highest_budget(self): - return max(self.budgets2folders.keys()) if self.use_budgets else None - - def get_budgets(self): - return list(self.budgets2folders.keys()) - - def get_runs_for_budget(self, b): - return list(self.budgets2folders.values()) - - def get_folders(self): - if self.use_budgets: - return list(self.folders2budgets.keys()) - else: - return list(self.folder2run.keys()) - - def get_runs_for_folder(self, f) - if self.use_budgets: - return list(self.folders2budgets[f].values()) - else: - return list(self.folder2run.values()) - - def get_aggregated(self, keep_budgets=True, keep_folders=False): - """ Collapse data-structure along a given "axis". - - Returns - ------- - aggregated_runs: either ConfiguratorRun or Dict(str->ConfiguratorRun) - run(s) with aggregated data - """ - if not self.use_budgets: - keep_budgets = False - - if (not keep_budgets) and (not keep_folders): - return self._aggregate(self.runs) - elif keep_budgets: - return {b : self._aggregate(self.get_runs_for_budget(b)) for b in self.get_budgets()} - elif keep_folders: - return {f : self._aggregate(self.get_runs_for_folder(f)) for f in self.get_folders()} - else: - return self.runs - - def _aggregate(self, runs): - """ - """ - orig_rh, vali_rh = RunHistory(average_cost), RunHistory(average_cost) - for run in runs: - self.orig_rh.update(run.original_runhistory, origin=DataOrigin.INTERNAL) - self.vali_rh.update(run.original_runhistory, origin=DataOrigin.INTERNAL) - if run.validated_runhistory: - self.vali_rh.update(run.validated_runhistory, origin=DataOrigin.EXTERNAL_SAME_INSTANCES) - - self._init_pimp_and_validator(self.global_validated_rh) - - # Estimate missing costs for [def, inc1, inc2, ...] - self._validate_default_and_incumbents(self.validation_method, self.ta_exec_dir) - self.global_epm_rh.update(self.global_validated_rh) - - for rh_name, rh in [("original", self.global_original_rh), - ("validated", self.global_validated_rh), - ("epm", self.global_epm_rh)]: - self.logger.debug('Combined number of RunHistory data points for %s runhistory: %d ' - '# Configurations: %d. # Configurator runs: %d', - rh_name, len(rh.data), len(rh.get_all_configs()), len(self.runs)) - - # Sort runs (best first) - self.runs = sorted(self.runs, key=lambda run: self.global_epm_rh.get_cost(run.solver.incumbent)) - self.best_run = self.runs[0] - - self.incumbent = self.pimp.incumbent = self.best_run.solver.incumbent - self.logger.debug("Overall best run: %s, with incumbent: %s", self.best_run.folder, self.incumbent) \ No newline at end of file diff --git a/cave/reader/configurator_run.py b/cave/reader/configurator_run.py index c4a0abc6..c2c3220f 100644 --- a/cave/reader/configurator_run.py +++ b/cave/reader/configurator_run.py @@ -8,20 +8,6 @@ from cave.reader.smac2_reader import SMAC2Reader from cave.reader.csv_reader import CSVReader -def get_reader(name): - """ Returns an appropriate reader for the specified format. """ - if name == 'SMAC3': - return SMAC3Reader(folder, ta_exec_dir) - elif name == 'BOHB': - self.logger.debug("File format is BOHB, assmuming data was converted to SMAC3-format using " - "HpBandSter2SMAC from cave.utils.converter.hpbandster2smac.") - return SMAC3Reader(folder, ta_exec_dir) - elif name == 'SMAC2': - return SMAC2Reader(folder, ta_exec_dir) - elif name == 'CSV': - return CSVReader(folder, ta_exec_dir) - else: - raise ValueError("%s not supported as file-format" % name) class ConfiguratorRun(SMAC): """ @@ -31,115 +17,90 @@ class ConfiguratorRun(SMAC): trajectory and handling original/validated data appropriately. """ def __init__(self, - scenario, - original_runhistory, - validated_runhistory, - trajectory, - folder, - ta_exec_dir, - file_format, - validation_format, - budget=None, - ): - self.scenario = scenario - self.original_runhistory = original_runhistory - self.validated_runhistory = validated_runhistory - self.trajectory = trajectory - self.path_to_folder = path_to_folder - self.ta_exec_dir = ta_exec_dir - self.file_format = file_format - self.validation_format = validation_format - self.budget = budget - - self.default = self.scenario.cs.get_default_configuration() - self.incumbent = self.trajectory[-1]['incumbent'] if self.trajectory else None - self.feature_names = self._get_feature_names() - - # Create combined runhistory to collect all "real" runs - self.combined_runhistory = RunHistory(average_cost) - self.combined_runhistory.update(self.original_runhistory, origin=DataOrigin.INTERNAL) - if self.validated_runhistory: - self.combined_runhistory.update(self.validated_runhistory, origin=DataOrigin.EXTERNAL_SAME_INSTANCES) - - # Create runhistory with estimated runs (create Importance-object of pimp and use epm-model for validation) - self.epm_runhistory = RunHistory(average_cost) - self.epm_runhistory.update(self.combined_runhistory) - self._init_pimp_and_validator() - - - # Set during execution, to share information between Analyzers - self.share_information = {'parameter_importance' : OrderedDict(), - 'feature_importance' : OrderedDict(), - 'evaluators' : [], - 'validator' : None} - - # Initialize SMAC-object - super().__init__(scenario=self.scen, runhistory=self.combined_runhistory) # restore_incumbent=incumbent) - # TODO use restore, delete next line - self.solver.incumbent = self.incumbent - - @classmethod - def from_folder(cls, - folder: str, - ta_exec_dir: str, - file_format: str='SMAC3', - validation_format: str='NONE', - budget=None, - ): + folder: str, + ta_exec_dir: str, + file_format: str='SMAC3', + validation_format: str='NONE'): """Initialize scenario, runhistory and incumbent from folder, execute - init-method of SMAC facade (so you could simply use SMAC-instances instead). + init-method of SMAC facade (so you could simply use SMAC-instances instead) Parameters ---------- folder: string - output-dir of this run -> this is also the 'id' for a single run in parallel optimization + output-dir of this run ta_exec_dir: string - if the execution directory for the SMAC-run differs from the cwd, there might be problems loading instance-, - feature- or PCS-files in the scenario-object. since instance- and PCS-files are necessary, specify the path - to the execution-dir of SMAC here + if the execution directory for the SMAC-run differs from the cwd, + there might be problems loading instance-, feature- or PCS-files + in the scenario-object. since instance- and PCS-files are necessary, + specify the path to the execution-dir of SMAC here file_format: string from [SMAC2, SMAC3, BOHB, CSV] validation_format: string from [SMAC2, SMAC3, CSV, NONE], in which format to look for validated data - budget: int - budget for this run-instance (only for budgeted optimization!) """ self.logger = logging.getLogger("cave.ConfiguratorRun.{}".format(folder)) - self.logger.debug("Loading from \'%s\' with ta_exec_dir \'%s\' with file-format '%s' and validation-format %s. " - "Budget (if present): %s", folder, ta_exec_dir, file_format, validation_format, budget) + self.cave = None # Set if we analyze configurators that use budgets - self.validation_format = validation_format if validation_format != 'NONE' else None - - #### Read in data (scenario, runhistory & trajectory) - reader = get_reader(file_format) - - scenario = self.reader.get_scenario() - scenario_sanity_check(scenario, self.logger) - original_runhistory = reader.get_runhistory(scenario.cs) - validated_runhistory = None + self.folder = folder + self.ta_exec_dir = ta_exec_dir + self.file_format = file_format + self.validation_format = validation_format - trajectory = reader.get_trajectory(cs=scenario.cs) + self.logger.debug("Loading from \'%s\' with ta_exec_dir \'%s\'.", + folder, ta_exec_dir) + if validation_format == 'NONE': + validation_format = None + + def get_reader(name): + if name == 'SMAC3': + return SMAC3Reader(folder, ta_exec_dir) + elif name == 'BOHB': + self.logger.debug("File format is BOHB, assmuming data was converted to SMAC3-format using " + "HpBandSter2SMAC from cave.utils.converter.hpbandster2smac.") + return SMAC3Reader(folder, ta_exec_dir) + elif name == 'SMAC2': + return SMAC2Reader(folder, ta_exec_dir) + elif name == 'CSV': + return CSVReader(folder, ta_exec_dir) + else: + raise ValueError("%s not supported as file-format" % name) + self.reader = get_reader(file_format) + + self.scen = self.reader.get_scenario() + self.original_runhistory = self.reader.get_runhistory(self.scen.cs) + self.validated_runhistory = None + + self.traj = self.reader.get_trajectory(cs=self.scen.cs) + self.default = self.scen.cs.get_default_configuration() + self.incumbent = self.traj[-1]['incumbent'] + self.train_inst = self.scen.train_insts + self.test_inst = self.scen.test_insts + self._check_rh_for_inc_and_def(self.original_runhistory, 'original runhistory') if validation_format: + self.logger.debug('Using format %s for validation', validation_format) reader = get_reader(validation_format) - reader.scen = scenario - validated_runhistory = reader.get_validated_runhistory(scenario.cs) - self.logger.info("Found validated runhistory for \"%s\" and using it for evaluation. #configs in " - "validated rh: %d", folder, len(validated_runhistory.config_ids)) + reader.scen = self.scen + self.validated_runhistory = reader.get_validated_runhistory(self.scen.cs) + self._check_rh_for_inc_and_def(self.validated_runhistory, 'validated runhistory') + self.logger.info("Found validated runhistory for \"%s\" and using " + "it for evaluation. #configs in validated rh: %d", + self.folder, len(self.validated_runhistory.config_ids)) - self.__init__( - scenario, - original_runhistory, - validated_runhistory, - combined_runhistory, - epm_runhistory, - trajectory, - folder, - ta_exec_dir, - file_format, - validation_format, - budget=None, - ) + self.combined_runhistory = RunHistory(average_cost) + self.combined_runhistory.update(self.original_runhistory, + origin=DataOrigin.INTERNAL) + if self.validated_runhistory: + self.combined_runhistory.update(self.validated_runhistory, + origin=DataOrigin.EXTERNAL_SAME_INSTANCES) + + self.epm_runhistory = RunHistory(average_cost) + self.epm_runhistory.update(self.combined_runhistory) + + # Initialize SMAC-object + super().__init__(scenario=self.scen, runhistory=self.combined_runhistory) # restore_incumbent=incumbent) + # TODO use restore, delete next line + self.solver.incumbent = self.incumbent def get_incumbent(self): return self.solver.incumbent @@ -175,90 +136,3 @@ def _check_rh_for_inc_and_def(self, rh, name=''): i_name, c_name, self.folder) return_value = False return return_value - - def _init_pimp_and_validator(self, alternative_output_dir=None): - """Create ParameterImportance-object and use it's trained model for validation and further predictions We pass a - combined (original + validated) runhistory, so that the returned model will be based on as much information as - possible - - Parameters - ---------- - alternative_output_dir: str - e.g. for budgets we want pimp to use an alternative output-dir (subfolders per budget) - """ - self.logger.debug("Using '%s' as output for pimp", alternative_output_dir if alternative_output_dir else - self.output_dir) - self.pimp = Importance(scenario=copy.deepcopy(self.scenario), - runhistory=self.combined_runhistory, - incumbent=self.incumbent if self.incumbent else self.default, - parameters_to_evaluate=4, - save_folder=alternative_output_dir if alternative_output_dir else self.output_dir, - seed=self.rng.randint(1, 100000), - max_sample_size=self.pimp_max_samples, - fANOVA_pairwise=self.fanova_pairwise, - preprocess=False, - verbose=self.verbose_level != 'OFF', # disable progressbars - ) - self.model = self.pimp.model - - # Validator (initialize without trajectory) - self.validator = Validator(self.scenario, None, None) - self.validator.epm = self.model - - @timing - def _validate_default_and_incumbents(self, method, ta_exec_dir): - """Validate default and incumbent configurations on all instances possible. - Either use validation (physically execute the target algorithm) or EPM-estimate and update according runhistory - (validation -> self.global_validated_rh; epm -> self.global_epm_rh). - - Parameters - ---------- - method: str - epm or validation - ta_exec_dir: str - path from where the target algorithm can be executed as found in scenario (only used for actual validation) - """ - self.logger.debug("Validating %s using %s!", self.folder, method) - self.validator.traj = self.trajectory - if method == "validation": - with _changedir(ta_exec_dir): - # TODO determine # repetitions - new_rh = self.validator.validate('def+inc', 'train+test', 1, -1, runhistory=self.combined_runhistory) - self.validated_runhistory.update(new_rh) - self.combined_runhistory_rh.update(new_rh) - elif method == "epm": - # Only do test-instances if features for test-instances are available - instance_mode = 'train+test' - if (any([i not in self.scenario.feature_dict for i in self.scenario.test_insts]) and - any([i in self.scenario.feature_dict for i in self.scenario.train_insts])): # noqa - self.logger.debug("No features provided for test-instances (but for train!). Cannot validate on \"epm\".") - self.logger.warning("Features detected for train-instances, but not for test-instances. This is " - "unintended usage and may lead to errors for some analysis-methods.") - instance_mode = 'train' - - new_rh = self.validator.validate_epm('def+inc', instance_mode, 1, runhistory=self.combined_runhistory) - self.epm_runhistory.update(new_rh) - else: - raise ValueError("Missing data method illegal (%s)", method) - self.validator.traj = None # Avoid usage-mistakes - - def _get_feature_names(self): - if not self.scenario.feature_dict: - self.logger.info("No features available. Skipping feature analysis.") - return - feat_fn = self.scenario.feature_fn - if not self.scenario.feature_names: - self.logger.debug("`scenario.feature_names` is not set. Loading from '%s'", feat_fn) - with _changedir(self.ta_exec_dir if self.ta_exec_dir else '.'): - if not feat_fn or not os.path.exists(feat_fn): - self.logger.warning("Feature names are missing. Either provide valid feature_file in scenario " - "(currently %s) or set `scenario.feature_names` manually." % feat_fn) - self.logger.error("Skipping Feature Analysis.") - return - else: - # Feature names are contained in feature-file and retrieved - feat_names = InputReader().read_instance_features_file(feat_fn)[0] - else: - feat_names = copy.deepcopy(self.scenario.feature_names) - return feat_names - diff --git a/cave/utils/hpbandster2smac.py b/cave/utils/hpbandster2smac.py index 8f6f5878..7859700e 100644 --- a/cave/utils/hpbandster2smac.py +++ b/cave/utils/hpbandster2smac.py @@ -55,8 +55,8 @@ def convert(self, folders, output_dir=None): # Using temporary files for the intermediate smac-result-like format if not output_dir: + self.logger.debug("New outputdir") output_dir = tempfile.mkdtemp() - self.logger.debug("New output-dir: %s", output_dir) budgets, paths = zip(*self.hpbandster2smac(folder2result, cs, backup_cs, output_dir).items()) return list(folder2result.values())[0], paths, budgets diff --git a/doc/dev_guide.rst b/doc/dev_guide.rst deleted file mode 100644 index f18eaf71..00000000 --- a/doc/dev_guide.rst +++ /dev/null @@ -1,28 +0,0 @@ -Developer's Guide ------------------ -CAVE aims to be modular and easily extendable. This section summarizes the most important concepts behind the -architecture. - -Custom Analyzers -~~~~~~~~~~~~~~~~ -To write a custom analyzer, you need to inherit from `the BaseAnalyzer `_ - - -While it's possible to generate a HTML-report via the commandline on a -given result-folder, CAVE may also run in interactive mode, running `individual analysis-methods `_ on demand. We provide a -few examples to demonstrate this. -Make sure you followed the `installation details `_ before starting. - -Analyse existing results via the commandline -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -There are example toy results of all supported file formats in the folder `examples -`_ on the github-repo. -Run - -.. code-block:: bash - - cave --folders examples/smac3/example_output/* --ta_exec_dir examples/smac3/ --output CAVE_OUTPUT - -to start the example (assuming you cloned the GitHub-repository in which the example is included). -By default, CAVE will execute all parts of the analysis. To disable certain (timeconsuming) parts From 1735007aa300925799dfb1c3b5aad7ca23a1109f Mon Sep 17 00:00:00 2001 From: shukon Date: Thu, 14 Mar 2019 22:44:10 +0100 Subject: [PATCH 05/16] FIX std over bohb plots --- cave/analyzer/cost_over_time.py | 66 +++++++++++++++++++++------------ cave/cavefacade.py | 10 ++--- cave/utils/hpbandster2smac.py | 2 +- 3 files changed, 48 insertions(+), 30 deletions(-) diff --git a/cave/analyzer/cost_over_time.py b/cave/analyzer/cost_over_time.py index 0da81bdc..59c7f40d 100644 --- a/cave/analyzer/cost_over_time.py +++ b/cave/analyzer/cost_over_time.py @@ -41,7 +41,7 @@ def __init__(self, rh: RunHistory, runs: List[ConfiguratorRun], block_epm: bool=False, - bohb_result=None, + bohb_results=None, average_over_runs: bool=True, output_fn: str="performance_over_time.png", validator: Union[None, Validator]=None): @@ -74,7 +74,7 @@ def __init__(self, self.output_dir = output_dir self.rh = rh self.runs = runs - self.bohb_result = bohb_result + self.bohb_results = bohb_results self.block_epm = block_epm self.average_over_runs = average_over_runs self.output_fn =output_fn @@ -223,26 +223,43 @@ def _get_all_runs(self, validator, runs, rh): return lines def _get_bohb_avg(self, validator, runs, rh): - if len(runs) > 1 and self.bohb_result: - # Add bohb-specific line - # Get collective rh - rh_bohb = RunHistory(average_cost) - for run in runs: - rh_bohb.update(run.combined_runhistory) - #self.logger.debug(rh_bohb.data) - # Get collective trajectory - traj = HpBandSter2SMAC().get_trajectory({'' : self.bohb_result}, '', self.scenario, rh_bohb) - #self.logger.debug(traj) - mean, time, configs = [], [], [] - traj_dict = self.bohb_result.get_incumbent_trajectory() - - mean, _, time, configs = self._get_mean_var_time(validator, traj, False, rh_bohb) - - configs, time, budget, mean = traj_dict['config_ids'], traj_dict['times_finished'], traj_dict['budgets'], traj_dict['losses'] - time_double = [t for sub in zip(time, time) for t in sub][1:] - mean_double = [t for sub in zip(mean, mean) for t in sub][:-1] - configs_double = [c for sub in zip(configs, configs) for c in sub][:-1] - return Line('all_budgets', time_double, mean_double, mean_double, mean_double, configs_double) + if len(runs) > 1 and self.bohb_results: + data = {} + for idx, bohb_result in enumerate(self.bohb_results): + data[idx] = {'costs' : [], 'times' : []} + traj_dict = bohb_result.get_incumbent_trajectory() + data[idx]['costs'] = traj_dict['losses'] + data[idx]['times'] = traj_dict['times_finished'] + + # Average over parallel bohb iterations to get final values + f_time, f_config, f_mean, f_std = [], [], [], [] + + pointer = {idx : {'cost' : data[idx]['costs'][0], + 'time' : 0} for idx in list(data.keys())} + + f_time.append(min([values['time'] for values in pointer.values()]) / 100) + costs = [values['cost'] for values in pointer.values()] + f_mean.append(np.mean(costs)) + f_std.append(np.std(costs)) + + while (len(data) > 0): + next_idx = min({idx : data[idx]['times'][0] for idx in data.keys()}.items(), key=lambda x: x[1])[0] + pointer[next_idx] = {'cost' : data[next_idx]['costs'].pop(0), + 'time' : data[next_idx]['times'].pop(0)} + f_time.append(pointer[next_idx]['time']) + f_mean.append(np.mean([values['cost'] for values in pointer.values()])) + f_std.append(np.std([values['cost'] for values in pointer.values()])) + + if len(data[next_idx]['times']) == 0: + data.pop(next_idx) + + time_double = [t for sub in zip(f_time, f_time) for t in sub][1:] + mean_double = [t for sub in zip(f_mean, f_mean) for t in sub][:-1] + std_double = [t for sub in zip(f_std, f_std) for t in sub][:-1] + configs_double = ['N/A' for _ in time_double] + return Line('all_budgets', time_double, mean_double, + [x + y for x, y in zip(mean_double, std_double)], + [x - y for x, y in zip(mean_double, std_double)], configs_double) def plot(self): """ @@ -254,8 +271,9 @@ def plot(self): lines = [] # Get plotting data and create CDS - if self.bohb_result: + if self.bohb_results: lines.append(self._get_bohb_avg(validator, runs, rh)) + self.logger.debug(lines[-1]) else: lines.append(self._get_avg(validator, runs, rh)) lines.extend(self._get_all_runs(validator, runs, rh)) @@ -304,7 +322,7 @@ def plot(self): # Add to legend legend_it.append((name, renderers[-1])) - if name == 'average': + if name in ['average', 'all_budgets']: # Fill area (uncertainty) # Defined as sequence of coordinates, so for step-effect double and arange accordingly ([(t0, v0), (t1, v0), (t1, v1), ... (tn, vn-1)]) band_x = np.append(line.time, line.time[::-1]) diff --git a/cave/cavefacade.py b/cave/cavefacade.py index f7644eb5..0ed56fe9 100644 --- a/cave/cavefacade.py +++ b/cave/cavefacade.py @@ -216,7 +216,7 @@ def __init__(self, self.feature_names = None self.num_bohb_results = 0 - self.bohb_result = None # only relevant for bohb_result + self.bohb_results = None # only relevant for bohb_result # Create output_dir if necessary self._create_outputdir(self.output_dir) @@ -224,7 +224,7 @@ def __init__(self, if file_format == 'BOHB': self.use_budgets = True self.num_bohb_results = len(folders) - self.bohb_result, folders, budgets = HpBandSter2SMAC().convert(folders, output_dir) + self.bohb_results, folders, budgets = HpBandSter2SMAC().convert(folders, output_dir) if "DEBUG" in self.verbose_level: for f in folders: debug_f = os.path.join(output_dir, 'debug', os.path.basename(f)) @@ -500,7 +500,7 @@ def analyze(self, # TODO: Currently, the code below is configured for bohb... if we extend to other budget-driven configurators, review! # Perform analysis for each run - if self.bohb_result: + if self.bohb_results: self.website["Budget Correlation"] = OrderedDict() self.budget_correlation(d=self.website["Budget Correlation"]) self.bohb_learning_curves(d=self.website) @@ -722,7 +722,7 @@ def cost_over_time(self, cave): cave.global_validated_rh, self.runs, block_epm=self.use_budgets, # blocking epms if bohb is analyzed - bohb_result=self.bohb_result, + bohb_results=self.bohb_results, validator=cave.validator) @_analyzer_type @@ -1000,7 +1000,7 @@ def bohb_learning_curves(self, cave): iteration and the stage is the index of the budget in which the configuration was first sampled (should be 0). The third index is just a sequential enumeration. This id can be interpreted as a nested index-identifier. """ - return BohbLearningCurves(self.scenario.cs.get_hyperparameter_names(), result_object=self.bohb_result) + return BohbLearningCurves(self.scenario.cs.get_hyperparameter_names(), result_object=self.bohb_results[0]) @_analyzer_type def bohb_incumbents_per_budget(self, cave): diff --git a/cave/utils/hpbandster2smac.py b/cave/utils/hpbandster2smac.py index 7859700e..975be798 100644 --- a/cave/utils/hpbandster2smac.py +++ b/cave/utils/hpbandster2smac.py @@ -59,7 +59,7 @@ def convert(self, folders, output_dir=None): output_dir = tempfile.mkdtemp() budgets, paths = zip(*self.hpbandster2smac(folder2result, cs, backup_cs, output_dir).items()) - return list(folder2result.values())[0], paths, budgets + return list(folder2result.values()), paths, budgets def load_configspace(self, folder): """Will try to load the configspace. If it's a pcs-file, backup_cs will be a list containing all possible From bc19da99f41acb83d7227c27386bd1e6198da83a Mon Sep 17 00:00:00 2001 From: shukon Date: Fri, 15 Mar 2019 10:42:22 +0100 Subject: [PATCH 06/16] FIX output-dir recreation --- cave/analyzer/cost_over_time.py | 11 +++-------- cave/cavefacade.py | 17 +++++++++-------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/cave/analyzer/cost_over_time.py b/cave/analyzer/cost_over_time.py index 59c7f40d..26245fe2 100644 --- a/cave/analyzer/cost_over_time.py +++ b/cave/analyzer/cost_over_time.py @@ -234,21 +234,16 @@ def _get_bohb_avg(self, validator, runs, rh): # Average over parallel bohb iterations to get final values f_time, f_config, f_mean, f_std = [], [], [], [] - pointer = {idx : {'cost' : data[idx]['costs'][0], + pointer = {idx : {'cost' : np.nan, 'time' : 0} for idx in list(data.keys())} - f_time.append(min([values['time'] for values in pointer.values()]) / 100) - costs = [values['cost'] for values in pointer.values()] - f_mean.append(np.mean(costs)) - f_std.append(np.std(costs)) - while (len(data) > 0): next_idx = min({idx : data[idx]['times'][0] for idx in data.keys()}.items(), key=lambda x: x[1])[0] pointer[next_idx] = {'cost' : data[next_idx]['costs'].pop(0), 'time' : data[next_idx]['times'].pop(0)} f_time.append(pointer[next_idx]['time']) - f_mean.append(np.mean([values['cost'] for values in pointer.values()])) - f_std.append(np.std([values['cost'] for values in pointer.values()])) + f_mean.append(np.nanmean([values['cost'] for values in pointer.values()])) + f_std.append(np.nanstd([values['cost'] for values in pointer.values()])) if len(data[next_idx]['times']) == 0: data.pop(next_idx) diff --git a/cave/cavefacade.py b/cave/cavefacade.py index 0ed56fe9..d188654d 100644 --- a/cave/cavefacade.py +++ b/cave/cavefacade.py @@ -182,7 +182,7 @@ def __init__(self, self.logger = logging.getLogger(self.__module__ + '.' + self.__class__.__name__) self.output_dir = output_dir self.output_dir_created = False - self.set_verbosity(verbose_level.upper(), os.path.join(self.output_dir, "debug")) + self.set_verbosity(verbose_level.upper()) self.logger.debug("Running CAVE version %s", v) self.show_jupyter = show_jupyter if self.show_jupyter: @@ -1063,7 +1063,7 @@ def _get_feature_names(self): def _build_website(self): self.builder.generate_html(self.website) - def set_verbosity(self, level, output_dir): + def set_verbosity(self, level): # Log to stream (console) logging.getLogger().setLevel(logging.DEBUG) formatter = logging.Formatter('%(levelname)s:%(name)s:%(message)s') @@ -1100,9 +1100,11 @@ def set_verbosity(self, level, output_dir): logging.getLogger().addHandler(stdout_handler) # Log to file is always debug - logging.getLogger('cave.settings').debug("Output-file for debug-log: '%s'", os.path.join(output_dir, "debug.log")) - self._create_outputdir(output_dir) - fh = logging.FileHandler(os.path.join(output_dir, "debug.log"), "w") + debug_path = os.path.join(self.output_dir, "debug", "debug.log") + logging.getLogger('cave.settings').debug("Output-file for debug-log: '%s'", debug_path) + self._create_outputdir(self.output_dir) + os.makedirs(os.path.split(debug_path)[0]) + fh = logging.FileHandler(debug_path, "w") fh.setLevel(logging.DEBUG) fh.setFormatter(formatter) logging.getLogger().addHandler(fh) @@ -1122,11 +1124,10 @@ def _create_outputdir(self, output_dir): self.logger.debug("Output-dir '%s' does not exist, creating", output_dir) os.makedirs(output_dir) else: - archive_path = os.path.join(tempfile.mkdtemp(), '.OLD') - shutil.make_archive(archive_path, 'zip', output_dir) + archive_path = shutil.make_archive(os.path.join(tempfile.mkdtemp(), '.OLD'), 'zip', output_dir) shutil.rmtree(output_dir) os.makedirs(output_dir) - shutil.move(archive_path + '.zip', output_dir) + shutil.move(archive_path, output_dir) self.logger.debug("Output-dir '%s' exists, moving old content to '%s'", self.output_dir, os.path.join(self.output_dir, '.OLD.zip')) From ce678509cbf0e5032d73b195f754cc877198880f Mon Sep 17 00:00:00 2001 From: shukon Date: Sat, 16 Mar 2019 19:57:10 +0100 Subject: [PATCH 07/16] FIX single budgets (add std's and use bohb-trajs) --- cave/analyzer/cost_over_time.py | 166 +++++++++++++++++++++++--------- cave/cavefacade.py | 5 +- 2 files changed, 124 insertions(+), 47 deletions(-) diff --git a/cave/analyzer/cost_over_time.py b/cave/analyzer/cost_over_time.py index 26245fe2..22647329 100644 --- a/cave/analyzer/cost_over_time.py +++ b/cave/analyzer/cost_over_time.py @@ -222,39 +222,118 @@ def _get_all_runs(self, validator, runs, rh): lines.append(Line(os.path.basename(run.folder), time_double, mean_double, mean_double, mean_double, configs)) return lines - def _get_bohb_avg(self, validator, runs, rh): - if len(runs) > 1 and self.bohb_results: - data = {} - for idx, bohb_result in enumerate(self.bohb_results): - data[idx] = {'costs' : [], 'times' : []} - traj_dict = bohb_result.get_incumbent_trajectory() - data[idx]['costs'] = traj_dict['losses'] - data[idx]['times'] = traj_dict['times_finished'] - - # Average over parallel bohb iterations to get final values - f_time, f_config, f_mean, f_std = [], [], [], [] - - pointer = {idx : {'cost' : np.nan, - 'time' : 0} for idx in list(data.keys())} - - while (len(data) > 0): - next_idx = min({idx : data[idx]['times'][0] for idx in data.keys()}.items(), key=lambda x: x[1])[0] - pointer[next_idx] = {'cost' : data[next_idx]['costs'].pop(0), - 'time' : data[next_idx]['times'].pop(0)} - f_time.append(pointer[next_idx]['time']) - f_mean.append(np.nanmean([values['cost'] for values in pointer.values()])) - f_std.append(np.nanstd([values['cost'] for values in pointer.values()])) - - if len(data[next_idx]['times']) == 0: - data.pop(next_idx) - - time_double = [t for sub in zip(f_time, f_time) for t in sub][1:] - mean_double = [t for sub in zip(f_mean, f_mean) for t in sub][:-1] - std_double = [t for sub in zip(f_std, f_std) for t in sub][:-1] - configs_double = ['N/A' for _ in time_double] - return Line('all_budgets', time_double, mean_double, - [x + y for x, y in zip(mean_double, std_double)], - [x - y for x, y in zip(mean_double, std_double)], configs_double) + def _get_bohb_line(self, validator, runs, rh, budget=None): + label = 'budget ' + str(int(budget) if (budget).is_integer() else budget) if budget else 'all budgets' + if budget is None: + budgets = self.bohb_results[0].HB_config['budgets'] + else: + budgets = [budget] + + data = {} + for idx, bohb_result in enumerate(self.bohb_results): + data[idx] = {'costs' : [], 'times' : []} + traj_dict = self.get_incumbent_trajectory(bohb_result, budgets) + data[idx]['costs'] = traj_dict['losses'] + data[idx]['times'] = traj_dict['times_finished'] + + # Average over parallel bohb iterations to get final values + f_time, f_config, f_mean, f_std = [], [], [], [] + + pointer = {idx : {'cost' : np.nan, + 'time' : 0} for idx in list(data.keys())} + + while (len(data) > 0): + next_idx = min({idx : data[idx]['times'][0] for idx in data.keys()}.items(), key=lambda x: x[1])[0] + pointer[next_idx] = {'cost' : data[next_idx]['costs'].pop(0), + 'time' : data[next_idx]['times'].pop(0)} + f_time.append(pointer[next_idx]['time']) + f_mean.append(np.nanmean([values['cost'] for values in pointer.values()])) + f_std.append(np.nanstd([values['cost'] for values in pointer.values()])) + + if len(data[next_idx]['times']) == 0: + data.pop(next_idx) + + time_double = [t for sub in zip(f_time, f_time) for t in sub][1:] + mean_double = [t for sub in zip(f_mean, f_mean) for t in sub][:-1] + std_double = [t for sub in zip(f_std, f_std) for t in sub][:-1] + configs_double = ['N/A' for _ in time_double] + return Line(str(label), time_double, mean_double, + [x + y for x, y in zip(mean_double, std_double)], + [x - y for x, y in zip(mean_double, std_double)], configs_double) + + def get_incumbent_trajectory(self, result, budgets, bigger_is_better=True, non_decreasing_budget=True): + """ + Returns the best configurations over time + + !! Copied from hpbandster and modified to enable getting trajectories for individual budgets !! + + Parameters + ---------- + result: + result + budgets: List[budgets] + budgets to be considered + bigger_is_better:bool + flag whether an evaluation on a larger budget is always considered better. + If True, the incumbent might increase for the first evaluations on a bigger budget + non_decreasing_budget: bool + flag whether the budget of a new incumbent should be at least as big as the one for + the current incumbent. + Returns + ------- + dict: + dictionary with all the config IDs, the times the runs + finished, their respective budgets, and corresponding losses + """ + all_runs = result.get_all_runs(only_largest_budget=False) + + all_runs = list(filter(lambda r: r.budget in budgets, all_runs)) + + all_runs.sort(key=lambda r: r.time_stamps['finished']) + + return_dict = { 'config_ids' : [], + 'times_finished': [], + 'budgets' : [], + 'losses' : [], + } + + current_incumbent = float('inf') + incumbent_budget = min(budgets) + + for r in all_runs: + if r.loss is None: continue + + new_incumbent = False + + if bigger_is_better and r.budget > incumbent_budget: + new_incumbent = True + + if r.loss < current_incumbent: + new_incumbent = True + + if non_decreasing_budget and r.budget < incumbent_budget: + new_incumbent = False + + if new_incumbent: + current_incumbent = r.loss + incumbent_budget = r.budget + + return_dict['config_ids'].append(r.config_id) + return_dict['times_finished'].append(r.time_stamps['finished']) + return_dict['budgets'].append(r.budget) + return_dict['losses'].append(r.loss) + + if current_incumbent != r.loss: + r = all_runs[-1] + + return_dict['config_ids'].append(return_dict['config_ids'][-1]) + return_dict['times_finished'].append(r.time_stamps['finished']) + return_dict['budgets'].append(return_dict['budgets'][-1]) + return_dict['losses'].append(return_dict['losses'][-1]) + + + return (return_dict) + def plot(self): """ @@ -267,11 +346,12 @@ def plot(self): # Get plotting data and create CDS if self.bohb_results: - lines.append(self._get_bohb_avg(validator, runs, rh)) - self.logger.debug(lines[-1]) + lines.append(self._get_bohb_line(validator, runs, rh)) + for b in self.bohb_results[0].HB_config['budgets']: + lines.append(self._get_bohb_line(validator, runs, rh, b)) else: lines.append(self._get_avg(validator, runs, rh)) - lines.extend(self._get_all_runs(validator, runs, rh)) + lines.extend(self._get_all_runs(validator, runs, rh)) data = {'name' : [], 'time' : [], 'mean' : [], 'upper' : [], 'lower' : []} hp_names = self.scenario.cs.get_hyperparameter_names() @@ -288,7 +368,6 @@ def plot(self): data[p].append(c[p] if (c and p in c) else 'inactive') source = ColumnDataSource(data=data) - # Create plot x_range = Range1d(min(source.data['time']), max(source.data['time'])) @@ -301,7 +380,6 @@ def plot(self): y_axis_label=y_label, title="Cost over time") - colors = itertools.cycle(Dark2_5) renderers = [] legend_it = [] @@ -310,19 +388,20 @@ def plot(self): name = line.name view = CDSView(source=source, filters=[GroupFilter(column_name='name', group=str(name))]) renderers.append([p.line('time', 'mean', - source=source, view=view, - line_color=color, - visible=True)]) + source=source, view=view, + line_color=color, + visible=True if line.name in ['average', 'all budgets'] else False)]) # Add to legend legend_it.append((name, renderers[-1])) - if name in ['average', 'all_budgets']: + if name in ['average', 'all budgets'] or 'budget' in name: # Fill area (uncertainty) # Defined as sequence of coordinates, so for step-effect double and arange accordingly ([(t0, v0), (t1, v0), (t1, v1), ... (tn, vn-1)]) band_x = np.append(line.time, line.time[::-1]) band_y = np.append(line.lower, line.upper[::-1]) - renderers[-1].extend([p.patch(band_x, band_y, color='#7570B3', fill_alpha=0.2)]) + renderers[-1].extend([p.patch(band_x, band_y, color='#7570B3', fill_alpha=0.2, + visible=True if line.name in ['average', 'all budgets'] else False)]) # Tooltips tooltips = [("estimated performance", "@mean"), @@ -346,6 +425,7 @@ def plot(self): # Wrap renderers in nested lists for checkbox-code checkbox, select_all, select_none = get_checkbox(renderers, [l[0] for l in legend_it]) + checkbox.active = [0] # Tilt tick labels and configure axis labels p.xaxis.major_label_orientation = 3/4 diff --git a/cave/cavefacade.py b/cave/cavefacade.py index d188654d..ed896741 100644 --- a/cave/cavefacade.py +++ b/cave/cavefacade.py @@ -1086,10 +1086,7 @@ def set_verbosity(self, level): "LPI", # Other (mostly bokeh) "PIL.PngImagePlugin", - "matplotlib.font_manager", - "matplotlib.ticker", - "matplotlib.axes", - "matplotlib.colorbar", + "matplotlib", "urllib3.connectionpool", "selenium.webdriver.remote.remote_connection"] for logger in disable_loggers: From 8e09a02bb8960681f5d30a2a13f1a736e2eaee6e Mon Sep 17 00:00:00 2001 From: shukon Date: Sat, 16 Mar 2019 21:06:43 +0100 Subject: [PATCH 08/16] MAINT budgets as int if they are integers --- cave/analyzer/budget_correlation.py | 2 +- cave/analyzer/overview_table.py | 13 +++++++++++-- cave/cavefacade.py | 2 +- cave/html/html_builder.py | 2 +- cave/plot/configurator_footprint.py | 1 + cave/utils/hpbandster2smac.py | 3 ++- 6 files changed, 17 insertions(+), 6 deletions(-) diff --git a/cave/analyzer/budget_correlation.py b/cave/analyzer/budget_correlation.py index 7a447c29..0a7cc963 100644 --- a/cave/analyzer/budget_correlation.py +++ b/cave/analyzer/budget_correlation.py @@ -59,7 +59,7 @@ def _get_table(self, runs): else: table[-1].append("{:.2f} ({} samples)".format(rho, len(costs[0]))) - budget_names = [os.path.basename(run.folder) for run in runs] + budget_names = [os.path.basename(run.folder).replace('_', ' ') for run in runs] # TODO return DataFrame(data=table, columns=budget_names, index=budget_names) def plot(self): diff --git a/cave/analyzer/overview_table.py b/cave/analyzer/overview_table.py index cf98d6a4..27b9ffa2 100644 --- a/cave/analyzer/overview_table.py +++ b/cave/analyzer/overview_table.py @@ -11,6 +11,7 @@ from cave.utils.bokeh_routines import array_to_bokeh_table class OverviewTable(BaseAnalyzer): + def __init__(self, runs, bohb_parallel, output_dir): """ Create overview-table. @@ -48,7 +49,15 @@ def run(self): return html_table_general, html_table_specific def _general_dict(self, scenario, bohb_parallel=False): - """ Generate the meta-information that holds for all runs (scenario info etc) """ + """ Generate the meta-information that holds for all runs (scenario info etc) + + Parameters + ---------- + scenario: smac.Scenario + scenario file to get information from + bohb_parallel: Union[False, int] + if set, defines number of parallel runs + """ # general stores information that holds for all runs, runspec holds information on a run-basis general = OrderedDict() @@ -90,7 +99,7 @@ def _runspec_dict(self, runs): runspec = OrderedDict() for run in runs: - name = os.path.basename(run.folder) # TODO this should be changed with multiple BOHB-folder suppor (no basename should be necessary) + name = os.path.basename(run.folder).replace('_', ' ') # TODO this should be changed with multiple BOHB-folder suppor (no basename should be necessary) runspec[name] = self._stats_for_run(run.original_runhistory, run.scenario, run.incumbent) diff --git a/cave/cavefacade.py b/cave/cavefacade.py index ed896741..01326839 100644 --- a/cave/cavefacade.py +++ b/cave/cavefacade.py @@ -1009,7 +1009,7 @@ def bohb_incumbents_per_budget(self, cave): budget). """ return BohbIncumbentsPerBudget([b.incumbent for b in self.runs], - [b.folder for b in self.runs], + [b.folder.replace('_', ' ') for b in self.runs], [b.epm_runhistory for b in self.runs]) @_analyzer_type diff --git a/cave/html/html_builder.py b/cave/html/html_builder.py index 014f06bd..1974b5b9 100644 --- a/cave/html/html_builder.py +++ b/cave/html/html_builder.py @@ -224,7 +224,7 @@ def add_layer(self, layer_name, data_dict: OrderedDict, is_tab: bool=False): if use_tabs: div += "
\n" - tabs_names = [k for k, v in data_dict.items() if isinstance(v, dict)] + tabs_names = [k.replace('_', ' ') for k, v in data_dict.items() if isinstance(v, dict)] rnd_prefix = str(random.randn()) default_open_id = "defaultOpen" + self.get_unique_id() div += "