diff --git a/mpisppy/generic_cylinders.py b/mpisppy/generic_cylinders.py index 7b834d508..a39752533 100644 --- a/mpisppy/generic_cylinders.py +++ b/mpisppy/generic_cylinders.py @@ -10,9 +10,7 @@ import sys import os -import json import copy -import shutil import numpy as np import pyomo.environ as pyo import pyomo.common.config as pyofig @@ -23,26 +21,14 @@ import mpisppy.utils.config as config import mpisppy.utils.sputils as sputils -from mpisppy.convergers.norm_rho_converger import NormRhoConverger -from mpisppy.convergers.primal_dual_converger import PrimalDualConverger - from mpisppy.extensions.extension import MultiExtension, Extension -from mpisppy.extensions.norm_rho_updater import NormRhoUpdater -from mpisppy.extensions.primal_dual_rho import PrimalDualRho -from mpisppy.extensions.grad_rho import GradRho -from mpisppy.extensions.scenario_lp_mps_files import Scenario_lp_mps_files - -from mpisppy.utils.wxbarwriter import WXBarWriter -from mpisppy.utils.wxbarreader import WXBarReader - -import mpisppy.utils.pickle_bundle as pickle_bundle -import mpisppy.utils.proper_bundler as proper_bundler import mpisppy.utils.solver_spec as solver_spec from mpisppy import global_toc from mpisppy import MPI + def _parse_args(m): # m is the model file module cfg = config.Config() @@ -153,11 +139,13 @@ def _do_decomp(module, cfg, scenario_creator, scenario_creator_kwargs, scenario_ raise RuntimeError("No rho_setter so a default must be specified via --default-rho") if cfg.use_norm_rho_converger: + from mpisppy.convergers.norm_rho_converger import NormRhoConverger if not cfg.use_norm_rho_updater: raise RuntimeError("--use-norm-rho-converger requires --use-norm-rho-updater") else: ph_converger = NormRhoConverger elif cfg.primal_dual_converger: + from mpisppy.convergers.primal_dual_converger import PrimalDualConverger ph_converger = PrimalDualConverger else: ph_converger = None @@ -238,21 +226,26 @@ def _do_decomp(module, cfg, scenario_creator, scenario_creator_kwargs, scenario_ vanilla.add_integer_relax_then_enforce(hub_dict, cfg) if cfg.grad_rho: + from mpisppy.extensions.grad_rho import GradRho ext_classes.append(GradRho) hub_dict['opt_kwargs']['options']['grad_rho_options'] = {'cfg': cfg} if cfg.write_scenario_lp_mps_files_dir is not None: + from mpisppy.extensions.scenario_lp_mps_files import Scenario_lp_mps_files ext_classes.append(Scenario_lp_mps_files) hub_dict['opt_kwargs']['options']["write_lp_mps_extension_options"]\ = {"write_scenario_lp_mps_files_dir": cfg.write_scenario_lp_mps_files_dir} if cfg.W_and_xbar_reader: + from mpisppy.utils.wxbarreader import WXBarReader ext_classes.append(WXBarReader) if cfg.W_and_xbar_writer: + from mpisppy.utils.wxbarwriter import WXBarWriter ext_classes.append(WXBarWriter) if cfg.user_defined_extensions is not None: + import json for ext_name in cfg.user_defined_extensions: module = sputils.module_name_to_module(ext_name) ext_classes = [] @@ -300,10 +293,12 @@ def _do_decomp(module, cfg, scenario_creator, scenario_creator_kwargs, scenario_ # norm rho adaptive rho (not the gradient version) if cfg.use_norm_rho_updater: + from mpisppy.extensions.norm_rho_updater import NormRhoUpdater vanilla.extension_adder(hub_dict, NormRhoUpdater) hub_dict['opt_kwargs']['options']['norm_rho_options'] = {'verbose': cfg.verbose} if cfg.use_primal_dual_rho_updater: + from mpisppy.extensions.primal_dual_rho import PrimalDualRho vanilla.extension_adder(hub_dict, PrimalDualRho) hub_dict['opt_kwargs']['options']['primal_dual_rho_options'] = { 'verbose': cfg.verbose, @@ -463,6 +458,8 @@ def _write_scenarios(module, scenario_creator_kwargs, scenario_denouement, comm): + import mpisppy.utils.pickle_bundle as pickle_bundle + import shutil assert hasattr(cfg, "num_scens") ScenCount = cfg.num_scens @@ -492,6 +489,7 @@ def _write_scenarios(module, #========== def _read_pickled_scenario(sname, cfg): + import mpisppy.utils.pickle_bundle as pickle_bundle fname = os.path.join(cfg.unpickle_scenarios_dir, sname+".pkl") scen = pickle_bundle.dill_unpickle(fname) return scen @@ -503,6 +501,8 @@ def _write_bundles(module, scenario_creator, scenario_creator_kwargs, comm): + import mpisppy.utils.pickle_bundle as pickle_bundle + import shutil assert hasattr(cfg, "num_scens") ScenCount = cfg.num_scens bsize = int(cfg.scenarios_per_bundle) @@ -644,6 +644,7 @@ def _proper_bundles(cfg): bundle_wrapper = None # the default if _proper_bundles(cfg): + import mpisppy.utils.proper_bundler as proper_bundler bundle_wrapper = proper_bundler.ProperBundler(module) bundle_wrapper.set_bunBFs(cfg) scenario_creator = bundle_wrapper.scenario_creator @@ -669,7 +670,6 @@ def _proper_bundles(cfg): scenario_creator_kwargs, global_comm) elif cfg.pickle_scenarios_dir is not None: - import mpisppy.utils.pickle_bundle as pickle_bundle global_comm = MPI.COMM_WORLD _write_scenarios(module, cfg, diff --git a/mpisppy/utils/cfg_vanilla.py b/mpisppy/utils/cfg_vanilla.py index cd198aee3..042e4877d 100644 --- a/mpisppy/utils/cfg_vanilla.py +++ b/mpisppy/utils/cfg_vanilla.py @@ -15,43 +15,7 @@ import json # Hub and spoke SPBase classes -from mpisppy.phbase import PHBase -from mpisppy.opt.ph import PH -from mpisppy.opt.aph import APH -from mpisppy.opt.lshaped import LShapedMethod -from mpisppy.opt.subgradient import Subgradient -from mpisppy.opt.fwph import FWPH -from mpisppy.utils.xhat_eval import Xhat_Eval import mpisppy.utils.sputils as sputils -from mpisppy.cylinders.fwph_spoke import FrankWolfeOuterBound -from mpisppy.cylinders.lagrangian_bounder import LagrangianOuterBound -from mpisppy.cylinders.lagranger_bounder import LagrangerOuterBound -from mpisppy.cylinders.subgradient_bounder import SubgradientOuterBound -from mpisppy.cylinders.xhatlooper_bounder import XhatLooperInnerBound -from mpisppy.cylinders.xhatxbar_bounder import XhatXbarInnerBound -from mpisppy.cylinders.xhatspecific_bounder import XhatSpecificInnerBound -from mpisppy.cylinders.xhatshufflelooper_bounder import XhatShuffleInnerBound -from mpisppy.cylinders.lshaped_bounder import XhatLShapedInnerBound -from mpisppy.cylinders.slam_heuristic import SlamMaxHeuristic, SlamMinHeuristic -from mpisppy.cylinders.cross_scen_spoke import CrossScenarioCutSpoke -from mpisppy.cylinders.reduced_costs_spoke import ReducedCostsSpoke -from mpisppy.cylinders.relaxed_ph_spoke import RelaxedPHSpoke -from mpisppy.cylinders.ph_dual_spoke import PHDualSpoke -from mpisppy.cylinders.hub import PHPrimalHub, PHHub, SubgradientHub, APHHub, FWPHHub -from mpisppy.extensions.extension import MultiExtension -from mpisppy.extensions.fixer import Fixer -from mpisppy.extensions.mipgapper import Gapper -from mpisppy.extensions.integer_relax_then_enforce import IntegerRelaxThenEnforce -from mpisppy.extensions.cross_scen_extension import CrossScenarioExtension -from mpisppy.extensions.reduced_costs_fixer import ReducedCostsFixer -from mpisppy.extensions.reduced_costs_rho import ReducedCostsRho -from mpisppy.extensions.relaxed_ph_fixer import RelaxedPHFixer -from mpisppy.extensions.sep_rho import SepRho -from mpisppy.extensions.grad_rho import GradRho -from mpisppy.extensions.coeff_rho import CoeffRho -from mpisppy.extensions.sensi_rho import SensiRho -from mpisppy.utils.wxbarreader import WXBarReader -from mpisppy.utils.wxbarwriter import WXBarWriter def _hasit(cfg, argname): # aside: Config objects act like a dict or an object TBD: so why the and? @@ -128,6 +92,8 @@ def ph_hub( variable_probability=None, all_nodenames=None, ): + from mpisppy.opt.ph import PH + from mpisppy.cylinders.hub import PHHub shoptions = shared_options(cfg) options = copy.deepcopy(shoptions) options["convthresh"] = cfg.intra_hub_conv_thresh @@ -177,6 +143,7 @@ def ph_primal_hub( variable_probability=None, all_nodenames=None, ): + from mpisppy.cylinders.hub import PHPrimalHub hub_dict = ph_hub( cfg, scenario_creator, @@ -205,6 +172,8 @@ def aph_hub(cfg, variable_probability=None, all_nodenames=None, ): + from mpisppy.opt.aph import APH + from mpisppy.cylinders.hub import APHHub # TBD: March 2023: multiple extensions needs work hub_dict = ph_hub(cfg, scenario_creator, @@ -241,6 +210,8 @@ def subgradient_hub(cfg, variable_probability=None, all_nodenames=None, ): + from mpisppy.opt.subgradient import Subgradient + from mpisppy.cylinders.hub import SubgradientHub shoptions = shared_options(cfg) options = copy.deepcopy(shoptions) options["convthresh"] = cfg.intra_hub_conv_thresh @@ -283,6 +254,8 @@ def fwph_hub(cfg, variable_probability=None, all_nodenames=None, ): + from mpisppy.opt.fwph import FWPH + from mpisppy.cylinders.hub import FWPHHub shoptions = shared_options(cfg) options = copy.deepcopy(shoptions) options["convthresh"] = cfg.intra_hub_conv_thresh @@ -316,6 +289,7 @@ def fwph_hub(cfg, return hub_dict def extension_adder(hub_dict,ext_class): + from mpisppy.extensions.extension import MultiExtension # TBD March 2023: this is not really good enough if "extensions" not in hub_dict["opt_kwargs"] or \ hub_dict["opt_kwargs"]["extensions"] is None: @@ -335,6 +309,7 @@ def extension_adder(hub_dict,ext_class): return hub_dict def add_gapper(hub_dict, cfg, name=None): + from mpisppy.extensions.mipgapper import Gapper hub_dict = extension_adder(hub_dict, Gapper) if name is None and cfg.mipgaps_json is not None: with open(cfg.mipgaps_json) as fin: @@ -357,6 +332,7 @@ def add_fixer(hub_dict, cfg, module, ): + from mpisppy.extensions.fixer import Fixer hub_dict = extension_adder(hub_dict,Fixer) hub_dict["opt_kwargs"]["options"]["fixeroptions"] = {"verbose":cfg.verbose, "boundtol": cfg.fixer_tol, @@ -366,27 +342,33 @@ def add_fixer(hub_dict, def add_integer_relax_then_enforce(hub_dict, cfg, ): + from mpisppy.extensions.integer_relax_then_enforce import IntegerRelaxThenEnforce hub_dict = extension_adder(hub_dict,IntegerRelaxThenEnforce) hub_dict["opt_kwargs"]["options"]["integer_relax_then_enforce_options"] = {"ratio":cfg.integer_relax_then_enforce_ratio} return hub_dict def add_reduced_costs_rho(hub_dict, cfg): + from mpisppy.extensions.reduced_costs_rho import ReducedCostsRho hub_dict = extension_adder(hub_dict,ReducedCostsRho) hub_dict["opt_kwargs"]["options"]["reduced_costs_rho_options"] = {"multiplier" : cfg.reduced_costs_rho_multiplier, "cfg": cfg} def add_sep_rho(hub_dict, cfg): + from mpisppy.extensions.sep_rho import SepRho hub_dict = extension_adder(hub_dict,SepRho) hub_dict["opt_kwargs"]["options"]["sep_rho_options"] = {"multiplier" : cfg.sep_rho_multiplier, "cfg": cfg} def add_grad_rho(hub_dict, cfg): + from mpisppy.extensions.grad_rho import GradRho hub_dict = extension_adder(hub_dict,GradRho) hub_dict['opt_kwargs']['options']['grad_rho_options'] = {'cfg': cfg} def add_coeff_rho(hub_dict, cfg): + from mpisppy.extensions.coeff_rho import CoeffRho hub_dict = extension_adder(hub_dict,CoeffRho) hub_dict["opt_kwargs"]["options"]["coeff_rho_options"] = {"multiplier" : cfg.coeff_rho_multiplier} def add_sensi_rho(hub_dict, cfg): + from mpisppy.extensions.sensi_rho import SensiRho hub_dict = extension_adder(hub_dict,SensiRho) hub_dict["opt_kwargs"]["options"]["sensi_rho_options"] = {"multiplier" : cfg.sensi_rho_multiplier, "cfg": cfg} @@ -394,6 +376,7 @@ def add_cross_scenario_cuts(hub_dict, cfg, ): #WARNING: Do not use without a cross_scenario_cuts spoke + from mpisppy.extensions.cross_scen_extension import CrossScenarioExtension hub_dict = extension_adder(hub_dict, CrossScenarioExtension) hub_dict["opt_kwargs"]["options"]["cross_scen_options"]\ = {"check_bound_improve_iterations" : cfg.cross_scenario_iter_cnt} @@ -403,6 +386,7 @@ def add_reduced_costs_fixer(hub_dict, cfg, ): #WARNING: Do not use without a reduced_costs_spoke spoke + from mpisppy.extensions.reduced_costs_fixer import ReducedCostsFixer hub_dict = extension_adder(hub_dict, ReducedCostsFixer) hub_dict["opt_kwargs"]["options"]["rc_options"] = { @@ -422,6 +406,7 @@ def add_relaxed_ph_fixer(hub_dict, cfg, ): #WARNING: Do not use without a reduced_costs_spoke spoke + from mpisppy.extensions.relaxed_ph_fixer import RelaxedPHFixer hub_dict = extension_adder(hub_dict, RelaxedPHFixer) hub_dict["opt_kwargs"]["options"]["relaxed_ph_fixer_options"] = { @@ -439,6 +424,7 @@ def add_wxbar_read_write(hub_dict, cfg): but are 'loose' in the hub options dict """ if _hasit(cfg, 'init_W_fname') or _hasit(cfg, 'init_Xbar_fname'): + from mpisppy.utils.wxbarreader import WXBarReader hub_dict = extension_adder(hub_dict, WXBarReader) hub_dict["opt_kwargs"]["options"].update( {"init_W_fname" : cfg.init_W_fname, @@ -446,6 +432,7 @@ def add_wxbar_read_write(hub_dict, cfg): "init_separate_W_files" : cfg.init_separate_W_files }) if _hasit(cfg, 'W_fname') or _hasit(cfg, 'Xbar_fname'): + from mpisppy.utils.wxbarwriter import WXBarWriter hub_dict = extension_adder(hub_dict, WXBarWriter) hub_dict["opt_kwargs"]["options"].update( {"W_fname" : cfg.W_fname, @@ -544,6 +531,8 @@ def fwph_spoke( all_nodenames=None, rho_setter=None, ): + from mpisppy.opt.fwph import FWPH + from mpisppy.cylinders.fwph_spoke import FrankWolfeOuterBound shoptions = shared_options(cfg) options = copy.deepcopy(shoptions) @@ -578,6 +567,7 @@ def _PHBase_spoke_foundation( ph_extensions=None, extension_kwargs=None, ): + from mpisppy.phbase import PHBase # only the shared options shoptions = shared_options(cfg) my_options = copy.deepcopy(shoptions) # extra safe... @@ -615,6 +605,7 @@ def _Xhat_Eval_spoke_foundation( ph_extensions=None, extension_kwargs=None, ): + from mpisppy.utils.xhat_eval import Xhat_Eval spoke_dict = _PHBase_spoke_foundation( spoke_class, cfg, @@ -642,6 +633,7 @@ def lagrangian_spoke( ph_extensions=None, extension_kwargs=None, ): + from mpisppy.cylinders.lagrangian_bounder import LagrangianOuterBound lagrangian_spoke = _PHBase_spoke_foundation( LagrangianOuterBound, cfg, @@ -676,6 +668,7 @@ def reduced_costs_spoke( ph_extensions=None, extension_kwargs=None, ): + from mpisppy.cylinders.reduced_costs_spoke import ReducedCostsSpoke rc_spoke = _PHBase_spoke_foundation( ReducedCostsSpoke, cfg, @@ -707,6 +700,7 @@ def lagranger_spoke( ph_extensions=None, extension_kwargs=None, ): + from mpisppy.cylinders.lagranger_bounder import LagrangerOuterBound lagranger_spoke = _PHBase_spoke_foundation( LagrangerOuterBound, cfg, @@ -744,6 +738,8 @@ def subgradient_spoke( ph_extensions=None, extension_kwargs=None, ): + from mpisppy.opt.subgradient import Subgradient + from mpisppy.cylinders.subgradient_bounder import SubgradientOuterBound subgradient_spoke = _PHBase_spoke_foundation( SubgradientOuterBound, cfg, @@ -790,6 +786,7 @@ def ph_dual_spoke( ph_extensions=None, extension_kwargs=None, ): + from mpisppy.cylinders.ph_dual_spoke import PHDualSpoke ph_dual_spoke = _PHBase_spoke_foundation( PHDualSpoke, cfg, @@ -828,6 +825,7 @@ def relaxed_ph_spoke( ph_extensions=None, extension_kwargs=None, ): + from mpisppy.cylinders.relaxed_ph_spoke import RelaxedPHSpoke relaxed_ph_spoke = _PHBase_spoke_foundation( RelaxedPHSpoke, cfg, @@ -865,6 +863,7 @@ def xhatlooper_spoke( extension_kwargs=None, ): + from mpisppy.cylinders.xhatlooper_bounder import XhatLooperInnerBound xhatlooper_dict = _Xhat_Eval_spoke_foundation( XhatLooperInnerBound, cfg, @@ -898,6 +897,7 @@ def xhatxbar_spoke( extension_kwargs=None, all_nodenames=None, ): + from mpisppy.cylinders.xhatxbar_bounder import XhatXbarInnerBound xhatxbar_dict = _Xhat_Eval_spoke_foundation( XhatXbarInnerBound, cfg, @@ -933,6 +933,7 @@ def xhatshuffle_spoke( extension_kwargs=None, ): + from mpisppy.cylinders.xhatshufflelooper_bounder import XhatShuffleInnerBound xhatshuffle_dict = _Xhat_Eval_spoke_foundation( XhatShuffleInnerBound, cfg, @@ -970,6 +971,7 @@ def xhatspecific_spoke( extension_kwargs=None, ): + from mpisppy.cylinders.xhatspecific_bounder import XhatSpecificInnerBound xhatspecific_dict = _Xhat_Eval_spoke_foundation( XhatSpecificInnerBound, cfg, @@ -993,6 +995,7 @@ def xhatlshaped_spoke( extension_kwargs=None, ): + from mpisppy.cylinders.lshaped_bounder import XhatLShapedInnerBound xhatlshaped_dict = _Xhat_Eval_spoke_foundation( XhatLShapedInnerBound, cfg, @@ -1016,6 +1019,7 @@ def slammax_spoke( ph_extensions=None, ): + from mpisppy.cylinders.slam_heuristic import SlamMaxHeuristic slammax_dict = _Xhat_Eval_spoke_foundation( SlamMaxHeuristic, cfg, @@ -1036,6 +1040,7 @@ def slammin_spoke( scenario_creator_kwargs=None, ph_extensions=None, ): + from mpisppy.cylinders.slam_heuristic import SlamMinHeuristic slammin_dict = _Xhat_Eval_spoke_foundation( SlamMinHeuristic, cfg, @@ -1058,6 +1063,8 @@ def cross_scenario_cuts_spoke( all_nodenames=None, ): + from mpisppy.opt.lshaped import LShapedMethod + from mpisppy.cylinders.cross_scen_spoke import CrossScenarioCutSpoke if _hasit(cfg, "max_solver_threads"): sp_solver_options = {"threads":cfg.max_solver_threads} else: