diff --git a/n3fit/runcards/hyperopt_studies/restricted_search_space_renew_hyperopt.yml b/n3fit/runcards/hyperopt_studies/restricted_search_space_renew_hyperopt.yml new file mode 100644 index 0000000000..02b34d7cce --- /dev/null +++ b/n3fit/runcards/hyperopt_studies/restricted_search_space_renew_hyperopt.yml @@ -0,0 +1,284 @@ +# +# Configuration file for n3fit +# +############################################################ +description: "NNPDF4.0 NNLO reduced set for hyperoptimization" +############################################################ +# frac: training fraction +dataset_inputs: +- {dataset: NMC_NC_NOTFIXED_DW_EM-F2, frac: 0.75, variant: legacy} +- {dataset: NMC_NC_NOTFIXED_P_EM-SIGMARED, frac: 0.75, variant: legacy} +- {dataset: SLAC_NC_NOTFIXED_P_DW_EM-F2, frac: 0.75, variant: legacy} +- {dataset: SLAC_NC_NOTFIXED_D_DW_EM-F2, frac: 0.75, variant: legacy} +- {dataset: BCDMS_NC_NOTFIXED_P_DW_EM-F2, frac: 0.75, variant: legacy} +- {dataset: BCDMS_NC_NOTFIXED_D_DW_EM-F2, frac: 0.75, variant: legacy} +- {dataset: CHORUS_CC_NOTFIXED_PB_DW_NU-SIGMARED, frac: 0.75, variant: legacy} +- {dataset: CHORUS_CC_NOTFIXED_PB_DW_NB-SIGMARED, frac: 0.75, variant: legacy} +- {dataset: NUTEV_CC_NOTFIXED_FE_DW_NU-SIGMARED, cfac: [MAS], frac: 0.75, variant: legacy} +- {dataset: NUTEV_CC_NOTFIXED_FE_DW_NB-SIGMARED, cfac: [MAS], frac: 0.75, variant: legacy} +- {dataset: HERA_NC_318GEV_EM-SIGMARED, frac: 0.75, variant: legacy} +- {dataset: HERA_NC_225GEV_EP-SIGMARED, frac: 0.75, variant: legacy} +- {dataset: HERA_NC_251GEV_EP-SIGMARED, frac: 0.75, variant: legacy} +- {dataset: HERA_NC_300GEV_EP-SIGMARED, frac: 0.75, variant: legacy} +- {dataset: HERA_NC_318GEV_EP-SIGMARED, frac: 0.75, variant: legacy} +- {dataset: HERA_CC_318GEV_EM-SIGMARED, frac: 0.75, variant: legacy} +- {dataset: HERA_CC_318GEV_EP-SIGMARED, frac: 0.75, variant: legacy} +- {dataset: HERA_NC_318GEV_EAVG_CHARM-SIGMARED, frac: 0.75, variant: legacy} +- {dataset: HERA_NC_318GEV_EAVG_BOTTOM-SIGMARED, frac: 0.75, variant: legacy} +- {dataset: DYE866_Z0_800GEV_DW_RATIO_PDXSECRATIO, frac: 0.75, variant: legacy} +- {dataset: DYE866_Z0_800GEV_PXSEC, frac: 0.75, cfac: [], variant: legacy} +- {dataset: DYE605_Z0_38P8GEV_DW_PXSEC, frac: 0.75, cfac: [], variant: legacy} +- {dataset: DYE906_Z0_120GEV_DW_PDXSECRATIO, frac: 0.75, cfac: [ACC], variant: legacy} +- {dataset: CDF_Z0_1P96TEV_ZRAP, frac: 0.75, cfac: [], variant: legacy} +- {dataset: D0_Z0_1P96TEV_ZRAP, frac: 0.75, cfac: [], variant: legacy} +- {dataset: D0_WPWM_1P96TEV_ASY, frac: 0.75, cfac: [], variant: legacy} +- {dataset: ATLAS_WPWM_7TEV_36PB_ETA, frac: 0.75, cfac: [], variant: legacy} +- {dataset: ATLAS_Z0_7TEV_36PB_ETA, frac: 0.75, cfac: [], variant: legacy} +- {dataset: ATLAS_Z0_7TEV_49FB_HIMASS, frac: 0.75, cfac: [], variant: legacy} +- {dataset: ATLAS_Z0_7TEV_LOMASS_M, frac: 0.75, cfac: [], variant: legacy} +- {dataset: ATLAS_WPWM_7TEV_46FB_CC-ETA, frac: 0.75, cfac: [], variant: legacy} +- {dataset: ATLAS_Z0_7TEV_46FB_CC-Y, frac: 0.75, cfac: [], variant: legacy} +- {dataset: ATLAS_Z0_7TEV_46FB_CF-Y, frac: 0.75, cfac: [], variant: legacy} +- {dataset: ATLAS_Z0_8TEV_HIMASS_M-Y, frac: 0.75, cfac: [], variant: legacy} +- {dataset: ATLAS_Z0_8TEV_LOWMASS_M-Y, frac: 0.75, cfac: [], variant: legacy} +- {dataset: ATLAS_Z0_13TEV_TOT, frac: 0.75, cfac: [NRM], variant: legacy} +- {dataset: ATLAS_WPWM_13TEV_TOT, frac: 0.75, cfac: [NRM], variant: legacy} +- {dataset: ATLAS_WJ_JET_8TEV_WP-PT, frac: 0.75, cfac: [], variant: legacy} +- {dataset: ATLAS_WJ_JET_8TEV_WM-PT, frac: 0.75, cfac: [], variant: legacy} +- {dataset: ATLAS_Z0J_8TEV_PT-M, frac: 0.75, cfac: [], variant: legacy_10} +- {dataset: ATLAS_Z0J_8TEV_PT-Y, frac: 0.75, cfac: [], variant: legacy_10} +- {dataset: ATLAS_TTBAR_7TEV_TOT_X-SEC, frac: 0.75, cfac: [], variant: legacy} +- {dataset: ATLAS_TTBAR_8TEV_TOT_X-SEC, frac: 0.75, cfac: [], variant: legacy} +- {dataset: ATLAS_TTBAR_13TEV_TOT_X-SEC, frac: 0.75, cfac: [], variant: legacy} +- {dataset: ATLAS_TTBAR_8TEV_LJ_DIF_YT-NORM, frac: 0.75, cfac: [], variant: legacy} +- {dataset: ATLAS_TTBAR_8TEV_LJ_DIF_YTTBAR-NORM, frac: 0.75, cfac: [], variant: legacy} +- {dataset: ATLAS_TTBAR_8TEV_2L_DIF_YTTBAR-NORM, frac: 0.75, cfac: [], variant: legacy} +- {dataset: ATLAS_1JET_8TEV_R06_PTY, frac: 0.75, cfac: [], variant: legacy_decorrelated} +- {dataset: ATLAS_2JET_7TEV_R06_M12Y, frac: 0.75, cfac: [], variant: legacy} +- {dataset: ATLAS_PH_13TEV_XSEC, frac: 0.75, cfac: [EWK], variant: legacy} +- {dataset: ATLAS_SINGLETOP_7TEV_TCHANNEL-XSEC, frac: 0.75, cfac: [], variant: legacy} +- {dataset: ATLAS_SINGLETOP_13TEV_TCHANNEL-XSEC, frac: 0.75, cfac: [], variant: legacy} +- {dataset: ATLAS_SINGLETOP_7TEV_T-Y-NORM, frac: 0.75, cfac: [], variant: legacy} +- {dataset: ATLAS_SINGLETOP_7TEV_TBAR-Y-NORM, frac: 0.75, cfac: [], variant: legacy} +- {dataset: ATLAS_SINGLETOP_8TEV_T-RAP-NORM, frac: 0.75, cfac: [], variant: legacy} +- {dataset: ATLAS_SINGLETOP_8TEV_TBAR-RAP-NORM, frac: 0.75, cfac: [], variant: legacy} +- {dataset: CMS_WPWM_7TEV_ELECTRON_ASY, frac: 0.75, cfac: []} +- {dataset: CMS_WPWM_7TEV_MUON_ASY, frac: 0.75, cfac: [], variant: legacy} +- {dataset: CMS_Z0_7TEV_DIMUON_2D, frac: 0.75, cfac: []} +- {dataset: CMS_WPWM_8TEV_MUON_Y, frac: 0.75, cfac: [], variant: legacy} +- {dataset: CMS_Z0J_8TEV_PT-Y, frac: 0.75, cfac: [NRM], variant: legacy_10} +- {dataset: CMS_2JET_7TEV_M12Y, frac: 0.75, cfac: []} +- {dataset: CMS_1JET_8TEV_PTY, frac: 0.75, cfac: [], variant: legacy} +- {dataset: CMS_TTBAR_7TEV_TOT_X-SEC, frac: 0.75, cfac: [], variant: legacy} +- {dataset: CMS_TTBAR_8TEV_TOT_X-SEC, frac: 0.75, cfac: [], variant: legacy} +- {dataset: CMS_TTBAR_13TEV_TOT_X-SEC, frac: 0.75, cfac: [], variant: legacy} +- {dataset: CMS_TTBAR_8TEV_LJ_DIF_YTTBAR-NORM, frac: 0.75, cfac: [], variant: legacy} +- {dataset: CMS_TTBAR_5TEV_TOT_X-SEC, frac: 0.75, cfac: [], variant: legacy} +- {dataset: CMS_TTBAR_8TEV_2L_DIF_MTTBAR-YT-NORM, frac: 0.75, cfac: [], variant: legacy} +- {dataset: CMS_TTBAR_13TEV_2L_DIF_YT, frac: 0.75, cfac: [], variant: legacy} +- {dataset: CMS_TTBAR_13TEV_LJ_2016_DIF_YTTBAR, frac: 0.75, cfac: [], variant: legacy} +- {dataset: CMS_SINGLETOP_7TEV_TCHANNEL-XSEC, frac: 0.75, cfac: [], variant: legacy} +- {dataset: CMS_SINGLETOP_8TEV_TCHANNEL-XSEC, frac: 0.75, cfac: [], variant: legacy} +- {dataset: CMS_SINGLETOP_13TEV_TCHANNEL-XSEC, frac: 0.75, cfac: [], variant: legacy} +- {dataset: LHCB_Z0_7TEV_DIELECTRON_Y, frac: 0.75, cfac: []} +- {dataset: LHCB_Z0_8TEV_DIELECTRON_Y, frac: 0.75, cfac: []} +- {dataset: LHCB_WPWM_7TEV_MUON_Y, frac: 0.75, cfac: [NRM]} +- {dataset: LHCB_Z0_7TEV_MUON_Y, frac: 0.75, cfac: [NRM]} +- {dataset: LHCB_WPWM_8TEV_MUON_Y, frac: 0.75, cfac: [NRM]} +- {dataset: LHCB_Z0_8TEV_MUON_Y, frac: 0.75, cfac: [NRM]} +- {dataset: LHCB_Z0_13TEV_DIMUON-Y, frac: 0.75, cfac: []} +- {dataset: LHCB_Z0_13TEV_DIELECTRON-Y, frac: 0.75, cfac: []} + +############################################################ +datacuts: + t0pdfset: NNPDF40_nnlo_as_01180_qcd # PDF set to generate t0 covmat + q2min: 3.49 # Q2 minimum + w2min: 12.5 # W2 minimum + +############################################################ +theory: + theoryid: 700 # database id + +sampling: + separate_multiplicative: false + +hyperscan_config: + architecture: + n_layers: [2] + min_units: 10 + max_units: 45 + optimizer: + - optimizer_name: 'Nadam' + learning_rate: + sampling: log + min: 1e-4 + max: 1e-2 + clipnorm: + sampling: log + min: 1e-7 + max: 1e-4 + - optimizer_name: 'Adam' + learning_rate: + sampling: log + min: 1e-4 + max: 1e-2 + clipnorm: + sampling: log + min: 1e-7 + max: 1e-4 + +kfold: + loss_type: chi2 + replica_statistic: average + fold_statistic: average + penalties: + - saturation + - patience + - integrability + threshold: 10 + partitions: + - datasets: + # DIS + - HERA_CC_318GEV_EM-SIGMARED + - HERA_NC_225GEV_EP-SIGMARED + #- HERA_NC_318GEV_EAVG_BOTTOM-SIGMARED + - HERA_NC_318GEV_EAVG_CHARM-SIGMARED + - NMC_NC_NOTFIXED_P_EM-SIGMARED + - NUTEV_CC_NOTFIXED_FE_DW_NB-SIGMARED + # EWK + - LHCB_Z0_8TEV_DIELECTRON_Y + - CMS_WPWM_7TEV_ELECTRON_ASY + - ATLAS_Z0J_8TEV_PT-M + - D0_WPWM_1P96TEV_ASY + - DYE866_Z0_800GEV_PXSEC + # JETS+TOP + - ATLAS_PH_13TEV_XSEC + - ATLAS_2JET_7TEV_R06_M12Y + - ATLAS_SINGLETOP_8TEV_TBAR-RAP-NORM + - CMS_TTBAR_7TEV_TOT_X-SEC # is this right or should it be one of the others? + - CMS_SINGLETOP_7TEV_TCHANNEL-XSEC + - datasets: + # DIS + - CHORUS_CC_NOTFIXED_PB_DW_NU-SIGMARED + - HERA_NC_318GEV_EP-SIGMARED + - BCDMS_NC_NOTFIXED_P_DW_EM-F2 + # EWK + - LHCB_Z0_7TEV_DIELECTRON_Y + - ATLAS_Z0_7TEV_36PB_ETA + - CMS_Z0J_8TEV_PT-Y + - DYE605_Z0_38P8GEV_DW_PXSEC + - CMS_Z0_7TEV_DIMUON_2D + # JET+TOP + - CMS_2JET_7TEV_M12Y + - ATLAS_SINGLETOP_7TEV_TBAR-Y-NORM + - ATLAS_SINGLETOP_7TEV_TCHANNEL-XSEC + - CMS_TTBAR_8TEV_LJ_DIF_YTTBAR-NORM + - CMS_SINGLETOP_8TEV_TCHANNEL-XSEC + - datasets: + # DIS + - HERA_CC_318GEV_EP-SIGMARED + - HERA_NC_251GEV_EP-SIGMARED + - NMC_NC_NOTFIXED_DW_EM-F2 + - NUTEV_CC_NOTFIXED_FE_DW_NU-SIGMARED + # EWK + - LHCB_WPWM_7TEV_MUON_Y + - LHCB_Z0_13TEV_DIELECTRON-Y + - ATLAS_Z0_7TEV_46FB_CC-Y + - ATLAS_WJ_JET_8TEV_WP-PT + - ATLAS_Z0_7TEV_49FB_HIMASS + - CMS_WPWM_7TEV_MUON_ASY + - DYE866_Z0_800GEV_DW_RATIO_PDXSECRATIO + - CDF_Z0_1P96TEV_ZRAP + # JET+TOP + - ATLAS_TTBAR_7TEV_TOT_X-SEC + - ATLAS_SINGLETOP_8TEV_T-RAP-NORM + - CMS_TTBAR_5TEV_TOT_X-SEC + - CMS_TTBAR_8TEV_2L_DIF_MTTBAR-YT-NORM + - datasets: + # DIS + - CHORUS_CC_NOTFIXED_PB_DW_NB-SIGMARED + - HERA_NC_300GEV_EP-SIGMARED + # EWK + - LHCB_WPWM_8TEV_MUON_Y + - LHCB_Z0_13TEV_DIMUON-Y + - ATLAS_Z0_7TEV_46FB_CF-Y + - ATLAS_WJ_JET_8TEV_WM-PT + - ATLAS_Z0_7TEV_LOMASS_M + - ATLAS_Z0J_8TEV_PT-Y + - CMS_WPWM_8TEV_MUON_Y + - D0_Z0_1P96TEV_ZRAP + # JET+TOP + - CMS_1JET_8TEV_PTY + - ATLAS_SINGLETOP_7TEV_T-Y-NORM + - ATLAS_SINGLETOP_13TEV_TCHANNEL-XSEC + - CMS_SINGLETOP_13TEV_TCHANNEL-XSEC + +############################################################ +trvlseed: 2182363835 +nnseed: 4044040809 +mcseed: 1977428487 +genrep: true # true = generate MC replicas, false = use real data + +# Baseline parameters from table 9 of https://doi.org/10.1140/epjc/s10052-022-10328-7 +# These are used for parameters that are not included in the hyperoptimization +parameters: # This defines the parameter dictionary that is passed to the Model Trainer + nodes_per_layer: [25, 20, 8] + activation_per_layer: [tanh, tanh, linear] + initializer: glorot_normal + optimizer: + clipnorm: 6.073e-6 + learning_rate: 2.621e-3 + optimizer_name: Nadam + epochs: 17000 + positivity: + initial: 184.8 + multiplier: + integrability: + initial: 10 + multiplier: + stopping_patience: 0.1 + layer_type: dense + dropout: 0.0 + threshold_chi2: 3.5 + +fitting: + savepseudodata: false + fitbasis: EVOL + basis: + - {fl: sng, trainable: false, smallx: [1.091, 1.119], largex: [1.471, 3.021]} + - {fl: g, trainable: false, smallx: [0.7795, 1.095], largex: [2.742, 5.547]} + - {fl: v, trainable: false, smallx: [0.472, 0.7576], largex: [1.571, 3.559]} + - {fl: v3, trainable: false, smallx: [0.07483, 0.4501], largex: [1.714, 3.467]} + - {fl: v8, trainable: false, smallx: [0.5731, 0.779], largex: [1.555, 3.465]} + - {fl: t3, trainable: false, smallx: [-0.5498, 1.0], largex: [1.778, 3.5]} + - {fl: t8, trainable: false, smallx: [0.5469, 0.857], largex: [1.555, 3.391]} + - {fl: t15, trainable: false, smallx: [1.081, 1.142], largex: [1.491, 3.092]} + +############################################################ +positivity: + posdatasets: + - {dataset: NNPDF_POS_2P24GEV_F2U, maxlambda: 1e6} # Positivity Lagrange Multiplier + - {dataset: NNPDF_POS_2P24GEV_F2D, maxlambda: 1e6} + - {dataset: NNPDF_POS_2P24GEV_F2S, maxlambda: 1e6} + - {dataset: NNPDF_POS_2P24GEV_FLL-19PTS, maxlambda: 1e6} + - {dataset: NNPDF_POS_2P24GEV_DYU, maxlambda: 1e10} + - {dataset: NNPDF_POS_2P24GEV_DYD, maxlambda: 1e10} + - {dataset: NNPDF_POS_2P24GEV_DYS, maxlambda: 1e10} + - {dataset: NNPDF_POS_2P24GEV_F2C-17PTS, maxlambda: 1e6} + - {dataset: NNPDF_POS_2P24GEV_XUQ, maxlambda: 1e6} # Positivity of MSbar PDFs + - {dataset: NNPDF_POS_2P24GEV_XUB, maxlambda: 1e6} + - {dataset: NNPDF_POS_2P24GEV_XDQ, maxlambda: 1e6} + - {dataset: NNPDF_POS_2P24GEV_XDB, maxlambda: 1e6} + - {dataset: NNPDF_POS_2P24GEV_XSQ, maxlambda: 1e6} + - {dataset: NNPDF_POS_2P24GEV_XSB, maxlambda: 1e6} + - {dataset: NNPDF_POS_2P24GEV_XGL, maxlambda: 1e6} + +############################################################ +integrability: + integdatasets: + - {dataset: NNPDF_INTEG_3GEV_XT8, maxlambda: 1e2} + - {dataset: NNPDF_INTEG_3GEV_XT3, maxlambda: 1e2} + +############################################################ +debug: false +parallel_models: true diff --git a/n3fit/src/evolven3fit/cli.py b/n3fit/src/evolven3fit/cli.py index 1996e0513c..3943ba6319 100644 --- a/n3fit/src/evolven3fit/cli.py +++ b/n3fit/src/evolven3fit/cli.py @@ -2,7 +2,7 @@ def cli_evolven3fit( - configuration_folder, q_fin, q_points, op_card_info, theory_card_info, dump, load, force + configuration_folder, q_fin, q_points, op_card_info, theory_card_info, force, load, dump ): """Evolves the fitted PDFs. diff --git a/n3fit/src/evolven3fit/evolve.py b/n3fit/src/evolven3fit/evolve.py index 05db89c4f4..087d3fcd70 100644 --- a/n3fit/src/evolven3fit/evolve.py +++ b/n3fit/src/evolven3fit/evolve.py @@ -10,7 +10,6 @@ import eko from eko import basis_rotation, runner from reportengine.compat import yaml -from validphys.loader import Loader from . import eko_utils, utils @@ -25,7 +24,7 @@ def evolve_fit( - fit_folder, q_fin, q_points, op_card_dict, theory_card_dict, force, eko_path=None, dump_eko=None + fit_folder, q_fin, q_points, op_card_dict, theory_card_dict, force, eko_path, dump_eko=None ): """ Evolves all the fitted replica in fit_folder/nnfit @@ -63,13 +62,12 @@ def evolve_fit( stdout_log = logging.StreamHandler(sys.stdout) for log in [log_file, stdout_log]: log.setFormatter(LOGGING_SETTINGS["formatter"]) - + # The log file will get everything log_file.setLevel(LOGGING_SETTINGS["level"]) # While the terminal only up to info stdout_log.setLevel(logging.INFO) - for logger in (_logger, *[logging.getLogger("eko")]): logger.handlers = [] logger.setLevel(LOGGING_SETTINGS["level"]) @@ -84,17 +82,17 @@ def evolve_fit( if eko_path is not None: eko_path = pathlib.Path(eko_path) _logger.info(f"Loading eko from : {eko_path}") - else: - try: - _logger.info(f"Loading eko from theory {theoryID}") - eko_path = (Loader().check_theoryID(theoryID).path) / "eko.tar" - except FileNotFoundError: - _logger.warning(f"eko not found in theory {theoryID}, we will construct it") + + if eko_path is None or not eko_path.exists(): + if dump_eko is not None: + _logger.warning(f"Trying to construct the eko at {dump_eko}") theory, op = eko_utils.construct_eko_cards( theoryID, q_fin, q_points, x_grid, op_card_dict, theory_card_dict ) runner.solve(theory, op, dump_eko) eko_path = dump_eko + else: + raise ValueError(f"dump_eko not provided and {eko_path=} not found") with eko.EKO.edit(eko_path) as eko_op: x_grid_obj = eko.interpolation.XGrid(x_grid) diff --git a/n3fit/src/n3fit/checks.py b/n3fit/src/n3fit/checks.py index 607d838e55..3ff404ffe5 100644 --- a/n3fit/src/n3fit/checks.py +++ b/n3fit/src/n3fit/checks.py @@ -9,6 +9,7 @@ from n3fit.hyper_optimization import penalties as penalties_module from n3fit.hyper_optimization.rewards import IMPLEMENTED_LOSSES, IMPLEMENTED_STATS from reportengine.checks import CheckError, make_argcheck +from validphys.loader import FallbackLoader from validphys.pdfbases import check_basis log = logging.getLogger(__name__) @@ -485,3 +486,14 @@ def check_polarized_configs(fitting, fitbasis, positivity_bound): ) if fitting.get("sum_rules", True) and fitting.get("sum_rules") != "TSR": raise CheckError("The 'sum_rules' key needs to be 'TSR' for polarised PDF fits.") + + +@make_argcheck +def check_eko_exists(theoryid): + """Check that an eko for this theory exists. + Since there might still be theories without an associated eko, + this function raises a logger' error instead of an Exception.""" + try: + _ = FallbackLoader().check_eko(theoryid.id) + except FileNotFoundError: + log.error(f"No eko found for {theoryid}") diff --git a/n3fit/src/n3fit/n3fit_checks_provider.py b/n3fit/src/n3fit/n3fit_checks_provider.py index 79a19bdb05..a06b3f0efd 100644 --- a/n3fit/src/n3fit/n3fit_checks_provider.py +++ b/n3fit/src/n3fit/n3fit_checks_provider.py @@ -32,3 +32,8 @@ def n3fit_checks_action( double_precision=False, ): return + + +@n3fit.checks.check_eko_exists +def evolven3fit_checks_action(theoryid): + return diff --git a/n3fit/src/n3fit/scripts/evolven3fit.py b/n3fit/src/n3fit/scripts/evolven3fit.py index 7d826182c4..a304e188ec 100644 --- a/n3fit/src/n3fit/scripts/evolven3fit.py +++ b/n3fit/src/n3fit/scripts/evolven3fit.py @@ -7,12 +7,12 @@ import pathlib import sys -from evolven3fit import cli, eko_utils, evolve +from evolven3fit import cli, eko_utils, evolve, utils import numpy as np from eko.runner.managed import solve from n3fit.io.writer import XGRID -from validphys.loader import FallbackLoader +from validphys.loader import FallbackLoader, Loader _logger = logging.getLogger(__name__) @@ -115,6 +115,7 @@ def main(): "-p", "--q-points", type=int, default=None, help="Number of q points for the evolution" ) parser.add_argument("-n", "--n-cores", type=int, default=1, help="Number of cores to be used") + parser.add_argument("--no-net", action="store_true", help="Emulates validphys' --no-net") parser.add_argument( "-e", "--ev-op-iterations", @@ -146,21 +147,36 @@ def main(): if args.use_fhmruvv: theory_card_info["use_fhmruvv"] = args.use_fhmruvv + if args.no_net: + loader = Loader() + else: + loader = FallbackLoader() + if args.actions == "evolve": + + if args.load is None: + fit_folder = pathlib.Path(args.configuration_folder) + theoryID = utils.get_theoryID_from_runcard(fit_folder) + + _logger.info(f"Loading eko from theory {theoryID}") + eko_path = loader.check_eko(theoryID) + else: + eko_path = args.load + cli.cli_evolven3fit( - args.configuration_folder, + fit_folder, args.q_fin, args.q_points, op_card_info, theory_card_info, - args.dump, - args.load, args.force, + eko_path, + None, ) else: # If we are in the business of producing an eko, do some checks before starting: # 1. load the nnpdf theory early to check for inconsistent options and theory problems - nnpdf_theory = FallbackLoader().check_theoryID(args.theoryID).get_description() + nnpdf_theory = loader.check_theoryID(args.theoryID).get_description() if nnpdf_theory.get("ModEv") == "TRN" and args.ev_op_iterations is not None: raise ValueError("ev_op_iterations is not accepted with ModEv=TRN solution") diff --git a/n3fit/src/n3fit/scripts/vp_setupfit.py b/n3fit/src/n3fit/scripts/vp_setupfit.py index 3fe9ae8b47..c56b716fce 100644 --- a/n3fit/src/n3fit/scripts/vp_setupfit.py +++ b/n3fit/src/n3fit/scripts/vp_setupfit.py @@ -38,7 +38,13 @@ from validphys.app import App from validphys.config import Config, ConfigError, Environment, EnvironmentError_ -SETUPFIT_FIXED_CONFIG = dict(actions_=['datacuts check_t0pdfset', 'theory check_positivity']) +SETUPFIT_FIXED_CONFIG = dict( + actions_=[ + 'datacuts check_t0pdfset', + 'theory check_positivity', + 'theory evolven3fit_checks_action', + ] +) SETUPFIT_PROVIDERS = [ 'n3fit.n3fit_checks_provider', @@ -159,6 +165,7 @@ def from_yaml(cls, o, *args, **kwargs): SETUPFIT_FIXED_CONFIG['actions_'].append('positivity_bound check_unpolarized_bc') for k, v in SETUPFIT_DEFAULTS.items(): file_content.setdefault(k, v) + file_content.update(SETUPFIT_FIXED_CONFIG) return cls(file_content, *args, **kwargs) diff --git a/nnpdf_data/nnpdf_data/new_commondata/NNPDF_INTEG_3GEV/kinematics_XT3.yaml b/nnpdf_data/nnpdf_data/new_commondata/NNPDF_INTEG_3GEV/kinematics.yaml similarity index 61% rename from nnpdf_data/nnpdf_data/new_commondata/NNPDF_INTEG_3GEV/kinematics_XT3.yaml rename to nnpdf_data/nnpdf_data/new_commondata/NNPDF_INTEG_3GEV/kinematics.yaml index bffc3832de..46c443ada2 100644 --- a/nnpdf_data/nnpdf_data/new_commondata/NNPDF_INTEG_3GEV/kinematics_XT3.yaml +++ b/nnpdf_data/nnpdf_data/new_commondata/NNPDF_INTEG_3GEV/kinematics.yaml @@ -1,13 +1,9 @@ bins: -- k1: +- x: min: null mid: 1.0e-09 max: null - k2: + Q2: min: null mid: 2.7225 max: null - k3: - min: null - mid: 0.0 - max: null diff --git a/nnpdf_data/nnpdf_data/new_commondata/NNPDF_INTEG_3GEV/kinematics_XT8.yaml b/nnpdf_data/nnpdf_data/new_commondata/NNPDF_INTEG_3GEV/kinematics_XT8.yaml deleted file mode 100644 index bffc3832de..0000000000 --- a/nnpdf_data/nnpdf_data/new_commondata/NNPDF_INTEG_3GEV/kinematics_XT8.yaml +++ /dev/null @@ -1,13 +0,0 @@ -bins: -- k1: - min: null - mid: 1.0e-09 - max: null - k2: - min: null - mid: 2.7225 - max: null - k3: - min: null - mid: 0.0 - max: null diff --git a/nnpdf_data/nnpdf_data/new_commondata/NNPDF_INTEG_3GEV/metadata.yaml b/nnpdf_data/nnpdf_data/new_commondata/NNPDF_INTEG_3GEV/metadata.yaml index c0a940203a..fee2946f1e 100644 --- a/nnpdf_data/nnpdf_data/new_commondata/NNPDF_INTEG_3GEV/metadata.yaml +++ b/nnpdf_data/nnpdf_data/new_commondata/NNPDF_INTEG_3GEV/metadata.yaml @@ -14,37 +14,30 @@ hepdata: implemented_observables: - observable_name: XT3 observable: - description: Deep Inelastic Scattering + description: The value of the T3 PDF at Q=Q0, used to ensure integrability label: 'integrability dataset: $xT_3$ PDF, Q=Q_0' units: '' - process_type: DIS_XT3 + process_type: INTEG_XT3 tables: [] npoints: [] ndata: 1 plotting: - kinematics_override: dis_sqrt_scale - theory_reference: Bertone:2013vaa dataset_label: 'integrability dataset: $xT_3$ PDF, Q=Q_0' - plot_x: k1 + plot_x: x kinematic_coverage: - - k1 - - k2 - - k3 + - x + - Q2 kinematics: variables: - k1: - description: Variable k1 - label: k1 + x: + description: Bjorken x + label: $x$ units: '' - k2: - description: Variable k2 - label: k2 + Q2: + description: Factorization scale [GeV2] + label: Q2 units: '' - k3: - description: Variable k3 - label: k3 - units: '' - file: kinematics_XT3.yaml + file: kinematics.yaml theory: conversion_factor: 1.0 operation: 'null' @@ -54,37 +47,30 @@ implemented_observables: ported_from: INTEGXT3 - observable_name: XT8 observable: - description: Deep Inelastic Scattering + description: The value of the T8 PDF at Q=Q0, used to ensure integrability label: 'integrability dataset: $xT_8$ PDF, Q=Q_0' units: '' - process_type: DIS_XT8 + process_type: INTEG_XT8 tables: [] npoints: [] ndata: 1 plotting: - kinematics_override: dis_sqrt_scale - theory_reference: Bertone:2013vaa dataset_label: 'integrability dataset: $xT_8$ PDF, Q=Q_0' - plot_x: k1 + plot_x: x kinematic_coverage: - - k1 - - k2 - - k3 + - x + - Q2 kinematics: variables: - k1: - description: Variable k1 - label: k1 - units: '' - k2: - description: Variable k2 - label: k2 + x: + description: Bjorken x + label: $x$ units: '' - k3: - description: Variable k3 - label: k3 + Q2: + description: Factorization scale [GeV2] + label: Q2 units: '' - file: kinematics_XT8.yaml + file: kinematics.yaml theory: conversion_factor: 1.0 operation: 'null' @@ -92,3 +78,102 @@ implemented_observables: - - INTEGXT8 data_uncertainties: [] ported_from: INTEGXT8 +- observable_name: XV + observable: + description: The value of the V PDF at Q=Q0, used to ensure integrability + label: 'integrability dataset: $xV$ PDF, Q=Q_0' + units: '' + process_type: INTEG_XV + tables: [] + npoints: [] + ndata: 1 + plotting: + dataset_label: 'integrability dataset: $xV$ PDF, Q=Q_0' + plot_x: x + kinematic_coverage: + - x + - Q2 + kinematics: + variables: + x: + description: Bjorken x + label: $x$ + units: '' + Q2: + description: Factorization scale [GeV2] + label: Q2 + units: '' + file: kinematics.yaml + theory: + conversion_factor: 1.0 + operation: 'null' + FK_tables: + - - NNPDF_INTEG_XV_40 + data_uncertainties: [] + ported_from: NNPDF_INTEG_XV_40 +- observable_name: XV3 + observable: + description: The value of the V3 PDF at Q=Q0, used to ensure integrability + label: 'integrability dataset: $xV_3$ PDF, Q=Q_0' + units: '' + process_type: INTEG_XV3 + tables: [] + npoints: [] + ndata: 1 + plotting: + dataset_label: 'integrability dataset: $xV_3$ PDF, Q=Q_0' + plot_x: x + kinematic_coverage: + - x + - Q2 + kinematics: + variables: + x: + description: Bjorken x + label: $x$ + units: '' + Q2: + description: Factorization scale [GeV2] + label: Q2 + units: '' + file: kinematics.yaml + theory: + conversion_factor: 1.0 + operation: 'null' + FK_tables: + - - NNPDF_INTEG_XV3_40 + data_uncertainties: [] + ported_from: NNPDF_INTEG_XV3_40 +- observable_name: XV8 + observable: + description: The value of the V8 PDF at Q=Q0, used to ensure integrability + label: 'integrability dataset: $xV_8$ PDF, Q=Q_0' + units: '' + process_type: INTEG_XV8 + tables: [] + npoints: [] + ndata: 1 + plotting: + dataset_label: 'integrability dataset: $xV_8$ PDF, Q=Q_0' + plot_x: x + kinematic_coverage: + - x + - Q2 + kinematics: + variables: + x: + description: Bjorken x + label: $x$ + units: '' + Q2: + description: Factorization scale [GeV2] + label: Q2 + units: '' + file: kinematics.yaml + theory: + conversion_factor: 1.0 + operation: 'null' + FK_tables: + - - NNPDF_INTEG_XV8_40 + data_uncertainties: [] + ported_from: NNPDF_INTEG_XV8_40 diff --git a/nnpdf_data/nnpdf_data/new_commondata/dataset_names.yml b/nnpdf_data/nnpdf_data/new_commondata/dataset_names.yml index 97fa245ced..5beea7b662 100644 --- a/nnpdf_data/nnpdf_data/new_commondata/dataset_names.yml +++ b/nnpdf_data/nnpdf_data/new_commondata/dataset_names.yml @@ -346,3 +346,9 @@ INTEGXT3: dataset: NNPDF_INTEG_3GEV_XT3 INTEGXT8: dataset: NNPDF_INTEG_3GEV_XT8 +INTEGXV: + dataset: NNPDF_INTEG_3GEV_XV +INTEGXV3: + dataset: NNPDF_INTEG_3GEV_XV3 +INTEGXV8: + dataset: NNPDF_INTEG_3GEV_XV8 diff --git a/validphys2/src/validphys/loader.py b/validphys2/src/validphys/loader.py index 88017eed68..4e8e54b429 100644 --- a/validphys2/src/validphys/loader.py +++ b/validphys2/src/validphys/loader.py @@ -78,6 +78,10 @@ class TheoryNotFound(LoadFailedError): pass +class EkoNotFound(LoadFailedError): + pass + + class TheoryMetadataNotFound(LoadFailedError): pass @@ -301,6 +305,14 @@ def available_theories(self): for folder in self._theories_path.glob(theory_token + '*') } + @property + @functools.lru_cache() + def available_ekos(self): + """Return a string token for each of the available theories""" + return { + eko_path.parent.name.split("_")[1] for eko_path in self._theories_path.glob("*/eko.tar") + } + @property @functools.lru_cache() def _available_old_datasets(self): @@ -510,6 +522,15 @@ def check_theoryID(self, theoryID): ) return TheoryIDSpec(theoryID, theopath, self.theorydb_folder) + @functools.lru_cache() + def check_eko(self, theoryID): + """Check the eko (and the parent theory) both exists and returns the path to it""" + theory = self.check_theoryID(theoryID) + eko_path = theory.path / "eko.tar" + if not eko_path.exists(): + raise EkoNotFound(f"Could not find eko {eko_path} in theory: {theoryID}") + return eko_path + @property def theorydb_folder(self): """Checks theory db file exists and returns path to it""" @@ -900,7 +921,7 @@ def _download_and_show(response, stream): sys.stdout.write('\n') -def download_file(url, stream_or_path, make_parents=False): +def download_file(url, stream_or_path, make_parents=False, delete_on_failure=False): """Download a file and show a progress bar if the INFO log level is enabled. If ``make_parents`` is ``True`` ``stream_or_path`` is path-like, all the parent folders will @@ -929,7 +950,7 @@ def download_file(url, stream_or_path, make_parents=False): p.parent.mkdir(exist_ok=True, parents=True) download_target = tempfile.NamedTemporaryFile( - delete=False, dir=p.parent, prefix=p.name, suffix='.part' + delete=delete_on_failure, dir=p.parent, prefix=p.name, suffix='.part' ) with download_target as f: @@ -1026,6 +1047,16 @@ def theory_urls(self): def theory_index(self): return self.nnprofile['theory_index'] + @property + @_key_or_loader_error + def eko_index(self): + return self.nnprofile['eko_index'] + + @property + @_key_or_loader_error + def eko_urls(self): + return self.nnprofile['eko_urls'] + @property @_key_or_loader_error def nnpdf_pdfs_urls(self): @@ -1091,6 +1122,13 @@ def remote_theories(self): rt = self.remote_files(self.theory_urls, self.theory_index, thing="theories") return {k[len(token) :]: v for k, v in rt.items()} + @property + @functools.lru_cache() + def remote_ekos(self): + token = 'eko_' + rt = self.remote_files(self.eko_urls, self.eko_index, thing="ekos") + return {k[len(token) :]: v for k, v in rt.items()} + @property @functools.lru_cache() def remote_nnpdf_pdfs(self): @@ -1121,6 +1159,10 @@ def downloadable_hyperscans(self): def downloadable_theories(self): return list(self.remote_theories) + @property + def downloadable_ekos(self): + return list(self.remote_ekos) + @property def lhapdf_pdfs(self): return lhaindex.expand_index_names('*') @@ -1293,6 +1335,17 @@ def download_theoryID(self, thid): raise TheoryNotFound("Theory %s not available." % thid) download_and_extract(remote[thid], self._theories_path, target_name=f"theory_{thid}") + def download_eko(self, thid): + """Download the EKO for a given theory ID""" + thid = str(thid) + remote = self.remote_ekos + if thid not in remote: + raise EkoNotFound(f"EKO for TheoryID {thid} is not available in the remote server") + # Check that we have the theory we need + theory = self.check_theoryID(thid) + target_path = theory.path / "eko.tar" + download_file(remote[thid], target_path, delete_on_failure=True) + def download_vp_output_file(self, filename, **kwargs): try: root_url = self.nnprofile['reports_root_url'] diff --git a/validphys2/src/validphys/nnprofile_default.yaml b/validphys2/src/validphys/nnprofile_default.yaml index 058273497b..e1c6c7414d 100644 --- a/validphys2/src/validphys/nnprofile_default.yaml +++ b/validphys2/src/validphys/nnprofile_default.yaml @@ -3,7 +3,7 @@ # The location of a custom profile can be given with the `NNPDF_PROFILE_PATH` enviroment variable # otherwise by default ${XDG_CONFIG_HOME}/.config/NNPDF/nnprofile.yaml will be read # which in most systems defaults to `~/.config/NNPDF/nnprofile.yaml` -# +# # # The following defines where NNPDF resources will be stored # The directories for results / theories / hyperscan / validphys are declared as @@ -50,6 +50,10 @@ theory_urls: theory_index: 'theorydata.json' +eko_urls: + - 'https://nnpdf.web.cern.ch/nnpdf/ekos/' +eko_index: 'ekodata.json' + lhapdf_urls: - 'http://lhapdfsets.web.cern.ch/lhapdfsets/current/' nnpdf_pdfs_urls: diff --git a/validphys2/src/validphys/scripts/vp_list.py b/validphys2/src/validphys/scripts/vp_list.py index ae0340005b..5653053c0e 100644 --- a/validphys2/src/validphys/scripts/vp_list.py +++ b/validphys2/src/validphys/scripts/vp_list.py @@ -4,6 +4,7 @@ Script which lists available resources locally and remotely """ + import argparse import fnmatch from functools import partial @@ -11,7 +12,6 @@ import re from reportengine import colors - from validphys.loader import FallbackLoader as L log = logging.getLogger() @@ -59,11 +59,9 @@ def main(command_line=None): attrs = dir(L) - available = [ - attr.lstrip(LOCAL_TOKEN) for attr in attrs if attr.startswith(LOCAL_TOKEN) - ] + available = [attr.removeprefix(LOCAL_TOKEN) for attr in attrs if attr.startswith(LOCAL_TOKEN)] downloadable = [ - attr.lstrip(REMOTE_TOKEN) for attr in attrs if attr.startswith(REMOTE_TOKEN) + attr.removeprefix(REMOTE_TOKEN) for attr in attrs if attr.startswith(REMOTE_TOKEN) ] # set metavar and print choices in help string - otherwise looks ugly. parser.add_argument( @@ -110,13 +108,9 @@ def main(command_line=None): "--regex", type=str, default=None, - help=( - "Filter search using regular expression, only list resources which " - "match pattern." - ), + help="Filter search using regular expression, only list resources which match pattern.", ) - args = parser.parse_args(command_line) results_filter = _get_filter(glob_pattern=args.glob, re_pattern=args.regex) # sane ordering is quite expensive and only really required with theories.