From 9dc3a6e32715f56dbcbb6bec47952b4eaf2bfd15 Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Fri, 7 Mar 2025 11:17:55 -0700 Subject: [PATCH 01/36] towards generic communicators --- mpisppy/cylinders/cross_scen_spoke.py | 2 - mpisppy/cylinders/hub.py | 30 +++++----- mpisppy/cylinders/spcommunicator.py | 26 +++++++-- mpisppy/cylinders/spoke.py | 4 +- mpisppy/extensions/cross_scen_extension.py | 6 +- mpisppy/extensions/reduced_costs_fixer.py | 6 +- mpisppy/spin_the_wheel.py | 64 ++++++++++++---------- 7 files changed, 81 insertions(+), 57 deletions(-) diff --git a/mpisppy/cylinders/cross_scen_spoke.py b/mpisppy/cylinders/cross_scen_spoke.py index e4c5109b0..964a48cdf 100644 --- a/mpisppy/cylinders/cross_scen_spoke.py +++ b/mpisppy/cylinders/cross_scen_spoke.py @@ -17,8 +17,6 @@ import mpisppy.cylinders.spoke as spoke class CrossScenarioCutSpoke(spoke.Spoke): - def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, options=None): - super().__init__(spbase_object, fullcomm, strata_comm, cylinder_comm, options=options) def register_send_fields(self) -> None: diff --git a/mpisppy/cylinders/hub.py b/mpisppy/cylinders/hub.py index 2d7c3dd22..a82566508 100644 --- a/mpisppy/cylinders/hub.py +++ b/mpisppy/cylinders/hub.py @@ -32,10 +32,8 @@ class Hub(SPCommunicator): _hub_algo_best_bound_provider = False - def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, spokes, options=None): - super().__init__(spbase_object, fullcomm, strata_comm, cylinder_comm, options=options) - assert len(spokes) == self.n_spokes - self.spokes = spokes # List of dicts + def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, communicators, options=None): + super().__init__(spbase_object, fullcomm, strata_comm, cylinder_comm, communicators, options=options) logger.debug(f"Built the hub object on global rank {fullcomm.Get_rank()}") # for logging self.print_init = True @@ -352,20 +350,22 @@ def initialize_spoke_indices(self): self.outerbound_spoke_chars = dict() self.innerbound_spoke_chars = dict() - for (i, spoke) in enumerate(self.spokes): - spoke_class = spoke["spoke_class"] + for (i, spoke) in enumerate(self.communicators): + if i == self.strata_rank: + continue + spoke_class = spoke["spcomm_class"] if hasattr(spoke_class, "converger_spoke_types"): for cst in spoke_class.converger_spoke_types: if cst == ConvergerSpokeType.OUTER_BOUND: - self.outerbound_spoke_indices.add(i + 1) - self.outerbound_spoke_chars[i+1] = spoke_class.converger_spoke_char + self.outerbound_spoke_indices.add(i) + self.outerbound_spoke_chars[i] = spoke_class.converger_spoke_char elif cst == ConvergerSpokeType.INNER_BOUND: - self.innerbound_spoke_indices.add(i + 1) - self.innerbound_spoke_chars[i+1] = spoke_class.converger_spoke_char + self.innerbound_spoke_indices.add(i) + self.innerbound_spoke_chars[i] = spoke_class.converger_spoke_char elif cst == ConvergerSpokeType.W_GETTER: - self.w_spoke_indices.add(i + 1) + self.w_spoke_indices.add(i) elif cst == ConvergerSpokeType.NONANT_GETTER: - self.nonant_spoke_indices.add(i + 1) + self.nonant_spoke_indices.add(i) else: raise RuntimeError(f"Unrecognized converger_spoke_type {cst}") @@ -396,8 +396,10 @@ def register_send_fields(self): self.shutdown = self.register_send_field(Field.SHUTDOWN, 1) required_fields = set() - for spoke in self.spokes: - spoke_class = spoke["spoke_class"] + for i, spoke in enumerate(self.communicators): + if i == self.strata_rank: + continue + spoke_class = spoke["spcomm_class"] if hasattr(spoke_class, "converger_spoke_types"): for cst in spoke_class.converger_spoke_types: if cst == ConvergerSpokeType.W_GETTER: diff --git a/mpisppy/cylinders/spcommunicator.py b/mpisppy/cylinders/spcommunicator.py index 19cb35d82..2f6d07401 100644 --- a/mpisppy/cylinders/spcommunicator.py +++ b/mpisppy/cylinders/spcommunicator.py @@ -108,15 +108,24 @@ def _pull_id(self) -> int: class SPCommunicator: - """ Notes: TODO + """ Base class for communicator objects. Each communicator object should register + as a class attribute what Field attributes it provides in its buffer + or expects to receive from another SPCommunicator object. Additionally, optional + receive attributes can be specified, for which the SPCommunicator will still work + but can read additional attributes. """ + send_fields = () + receive_fields = () + optional_receive_fields = () - def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, options=None): + def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, communicators, options=None): # flag for if the windows have been constructed self._windows_constructed = False self.fullcomm = fullcomm self.strata_comm = strata_comm self.cylinder_comm = cylinder_comm + self.communicators = communicators + assert len(communicators) == strata_comm.Get_size() self.global_rank = fullcomm.Get_rank() self.strata_rank = strata_comm.Get_rank() self.cylinder_rank = cylinder_comm.Get_rank() @@ -132,6 +141,11 @@ def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, options= self._locals = dict() self._sends = dict() + # setup FieldLengths which calculates + # the length of each buffer type based + # on the problem data + # self._field_lengths = FieldLengths(self.opt) + # attach the SPCommunicator to # the SPBase object self.opt.spcomm = self @@ -168,8 +182,10 @@ def _build_window_spec(self) -> dict[Field, int]: ## End for return window_spec - def register_recv_field(self, field: Field, origin: int, length: int) -> RecvArray: + def register_recv_field(self, field: Field, origin: int, length: int = -1) -> RecvArray: key = self._make_key(field, origin) + if length == -1: + length = self._field_lengths[field] if key in self._locals: my_fa = self._locals[key] assert(length + 1 == np.size(my_fa.array())) @@ -179,8 +195,10 @@ def register_recv_field(self, field: Field, origin: int, length: int) -> RecvArr ## End if return my_fa - def register_send_field(self, field: Field, length: int) -> SendArray: + def register_send_field(self, field: Field, length: int = -1) -> SendArray: assert field not in self._sends, "Field {} is already registered".format(field) + if length == -1: + length = self._field_lengths[field] # if field in self._sends: # my_fa = self._sends[field] # assert(length + 1 == np.size(my_fa.array())) diff --git a/mpisppy/cylinders/spoke.py b/mpisppy/cylinders/spoke.py index 6d0de08ba..bf27f5a1e 100644 --- a/mpisppy/cylinders/spoke.py +++ b/mpisppy/cylinders/spoke.py @@ -26,9 +26,9 @@ class ConvergerSpokeType(enum.Enum): NONANT_GETTER = 4 class Spoke(SPCommunicator): - def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, options=None): + def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, communicators, options=None): - super().__init__(spbase_object, fullcomm, strata_comm, cylinder_comm, options) + super().__init__(spbase_object, fullcomm, strata_comm, cylinder_comm, communicators, options) self.last_call_to_got_kill_signal = time.time() diff --git a/mpisppy/extensions/cross_scen_extension.py b/mpisppy/extensions/cross_scen_extension.py index 1026e58e5..6142e37d0 100644 --- a/mpisppy/extensions/cross_scen_extension.py +++ b/mpisppy/extensions/cross_scen_extension.py @@ -288,9 +288,9 @@ def setup_hub(self): self.new_cuts = False def initialize_spoke_indices(self): - for (i, spoke) in enumerate(self.opt.spcomm.spokes): - if spoke["spoke_class"] == CrossScenarioCutSpoke: - self.cut_gen_spoke_index = i + 1 + for (i, spoke) in enumerate(self.opt.spcomm.communicators): + if spoke["spcomm_class"] == CrossScenarioCutSpoke: + self.cut_gen_spoke_index = i ## End if ## End for diff --git a/mpisppy/extensions/reduced_costs_fixer.py b/mpisppy/extensions/reduced_costs_fixer.py index 47d9b7bb7..46292c1b6 100644 --- a/mpisppy/extensions/reduced_costs_fixer.py +++ b/mpisppy/extensions/reduced_costs_fixer.py @@ -94,9 +94,9 @@ def post_iter0_after_sync(self): self.fix_fraction_target = self._fix_fraction_target_iterK def initialize_spoke_indices(self): - for (i, spoke) in enumerate(self.opt.spcomm.spokes): - if spoke["spoke_class"] == ReducedCostsSpoke: - self.reduced_costs_spoke_index = i + 1 + for (i, spoke) in enumerate(self.opt.spcomm.communicators): + if spoke["spcomm_class"] == ReducedCostsSpoke: + self.reduced_costs_spoke_index = i ## End if ## End for diff --git a/mpisppy/spin_the_wheel.py b/mpisppy/spin_the_wheel.py index c37d86e5f..33ce62a90 100644 --- a/mpisppy/spin_the_wheel.py +++ b/mpisppy/spin_the_wheel.py @@ -25,7 +25,7 @@ def __init__(self, hub_dict, list_of_spoke_dict): Returns: spcomm (Hub or Spoke object): the object that did the work (windowless) - opt_dict (dict): the dictionary that controlled creation for this rank + spcomm_dict (dict): the dictionary that controlled creation for this rank NOTE: the return is after termination; the objects are provided for query. @@ -84,29 +84,39 @@ def run(self, comm_world=None): if comm_world is None: comm_world = MPI.COMM_WORLD - n_spokes = len(list_of_spoke_dict) + + _key_conversion = { + "hub_class" : "spcomm_class", + "hub_kwargs" : "spcomm_kwargs", + "spoke_class" : "spcomm_class", + "spoke_kwargs" : "spcomm_kwargs", + } + + # Put the hub at the beginning so its strata_rank is 0 + communicator_list = [hub_dict] + list_of_spoke_dict + + # TODO: we should change the API upstream eventually + for d in communicator_list: + for oldk, newk in _key_conversion.items(): + if oldk in d: + d[newk] = d.pop(oldk) + + n_spcomms = len(communicator_list) # Create the necessary communicators fullcomm = comm_world - strata_comm, cylinder_comm = _make_comms(n_spokes, fullcomm=fullcomm) + strata_comm, cylinder_comm = _make_comms(n_spcomms, fullcomm=fullcomm) strata_rank = strata_comm.Get_rank() cylinder_rank = cylinder_comm.Get_rank() global_rank = fullcomm.Get_rank() + spcomm_dict = communicator_list[strata_rank] + # Assign hub/spokes to individual ranks - if strata_rank == 0: # This rank is a hub - sp_class = hub_dict["hub_class"] - sp_kwargs = hub_dict["hub_kwargs"] - opt_class = hub_dict["opt_class"] - opt_kwargs = hub_dict["opt_kwargs"] - opt_dict = hub_dict - else: # This rank is a spoke - spoke_dict = list_of_spoke_dict[strata_rank - 1] - sp_class = spoke_dict["spoke_class"] - sp_kwargs = spoke_dict["spoke_kwargs"] - opt_class = spoke_dict["opt_class"] - opt_kwargs = spoke_dict["opt_kwargs"] - opt_dict = spoke_dict + sp_class = spcomm_dict["spcomm_class"] + sp_kwargs = spcomm_dict["spcomm_kwargs"] + opt_class = spcomm_dict["opt_class"] + opt_kwargs = spcomm_dict["opt_kwargs"] # Create the appropriate opt object locally opt_kwargs["mpicomm"] = cylinder_comm @@ -114,11 +124,8 @@ def run(self, comm_world=None): # Create the SPCommunicator object (hub/spoke) with # the appropriate SPBase object attached - if strata_rank == 0: # Hub - spcomm = sp_class(opt, fullcomm, strata_comm, cylinder_comm, - list_of_spoke_dict, **sp_kwargs) - else: # Spokes - spcomm = sp_class(opt, fullcomm, strata_comm, cylinder_comm, **sp_kwargs) + spcomm = sp_class(opt, fullcomm, strata_comm, cylinder_comm, + communicator_list, **sp_kwargs) # Create the windows, run main(), destroy the windows spcomm.make_windows() @@ -149,7 +156,7 @@ def run(self, comm_world=None): global_toc("Windows freed") self.spcomm = spcomm - self.opt_dict = opt_dict + self.spcomm_dict = spcomm_dict self.global_rank = global_rank self.strata_rank = strata_rank self.cylinder_rank = cylinder_rank @@ -166,7 +173,7 @@ def run(self, comm_world=None): def on_hub(self): if not self._ran: raise RuntimeError("Need to call WheelSpinner.run() before finding out.") - return ("hub_class" in self.opt_dict) + return self.strata_rank == 0 def write_first_stage_solution(self, solution_file_name, first_stage_solution_writer=first_stage_nonant_writer): @@ -221,22 +228,21 @@ def _determine_innerbound_winner(self): best_strata_rank = self.spcomm.fullcomm.bcast(best_strata_rank, root=0) return (self.spcomm.strata_rank == best_strata_rank) -def _make_comms(n_spokes, fullcomm=None): +def _make_comms(n_spcomms, fullcomm=None): """ Create the strata_comm and cylinder_comm for hub/spoke style runs """ if not haveMPI: raise RuntimeError("make_comms called, but cannot import mpi4py") # Ensure that the proper number of processes have been invoked - nsp1 = n_spokes + 1 # Add 1 for the hub if fullcomm is None: fullcomm = MPI.COMM_WORLD n_proc = fullcomm.Get_size() - if n_proc % nsp1 != 0: - raise RuntimeError(f"Need a multiple of {nsp1} processes (got {n_proc})") + if n_proc % n_spcomms != 0: + raise RuntimeError(f"Need a multiple of {n_spcomms} processes (got {n_proc})") # Create the strata_comm and cylinder_comm # Cryptic comment: intra is vertical, inter is around the hub global_rank = fullcomm.Get_rank() - strata_comm = fullcomm.Split(key=global_rank, color=global_rank // nsp1) - cylinder_comm = fullcomm.Split(key=global_rank, color=global_rank % nsp1) + strata_comm = fullcomm.Split(key=global_rank, color=global_rank // n_spcomms) + cylinder_comm = fullcomm.Split(key=global_rank, color=global_rank % n_spcomms) return strata_comm, cylinder_comm From da2e00b3ebc55b46bbe16cf48f50f5b95c5863f5 Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Fri, 7 Mar 2025 11:16:23 -0700 Subject: [PATCH 02/36] work towards automatic field length computation --- mpisppy/cylinders/spwindow.py | 47 +++++++++++++++++++++++++++++++++++ mpisppy/spbase.py | 3 +++ 2 files changed, 50 insertions(+) diff --git a/mpisppy/cylinders/spwindow.py b/mpisppy/cylinders/spwindow.py index 48573c925..e44eb80c4 100644 --- a/mpisppy/cylinders/spwindow.py +++ b/mpisppy/cylinders/spwindow.py @@ -14,6 +14,8 @@ import enum +import pyomo.environ as pyo + class Field(enum.IntEnum): SHUTDOWN=-1000 NONANT=1 @@ -27,6 +29,51 @@ class Field(enum.IntEnum): CROSS_SCENARIO_COST=400 WHOLE=1_000_000 + +_field_length_components = pyo.ConcreteModel() +_field_length_components.local_nonant_length = pyo.Param(mutable=True) +_field_length_components.local_scenario_length = pyo.Param(mutable=True) +_field_length_components.total_number_nonants = pyo.Param(mutable=True) +_field_length_components.total_number_scenarios = pyo.Param(mutable=True) + +_field_lengths = { + Field.SHUTDOWN : 1, + Field.NONANT : _field_length_components.local_nonant_length, + Field.DUALS : _field_length_components.local_nonant_length, + Field.OBJECTIVE_BOUNDS : 2, + Field.OBJECTIVE_INNER_BOUND : 1, + Field.OBJECTIVE_OUTER_BOUND : 1, + Field.EXPECTED_REDUCED_COST : _field_length_components.total_number_nonants, + Field.SCENARIO_REDUCED_COST : _field_length_components.local_nonant_length, + Field.CROSS_SCENARIO_CUT : _field_length_components.local_nonant_length, + Field.CROSS_SCENARIO_COST : _field_length_components.total_number_scenarios * _field_length_components.total_number_scenarios, +} + + +class FieldLengths: + def __init__(self, opt): + number_nonants = ( + sum( + len(s._mpisppy_data.nonant_indices) + for s in opt.local_scenarios.values() + ) + ) + + _field_length_components.local_nonant_length.value = number_nonants + _field_length_components.local_scenario_length.value = len(opt.local_scenarios) + _field_length_components.total_number_nonants.value = opt.nonant_length + _field_length_components.total_number_scenarios.value = len(opt.local_scenarios) + + self._field_lengths = {k : pyo.value(v) for k, v in _field_lengths.items()} + + # reset the _field_length_components + for p in _field_length_components.component_data_objects(): + p.clear() + + def __getitem__(self, field: Field): + return self._field_lengths[field] + + class SPWindow: def __init__(self, my_fields: dict, strata_comm: MPI.Comm, field_order=None): diff --git a/mpisppy/spbase.py b/mpisppy/spbase.py index 11479e824..bfd7dede1 100644 --- a/mpisppy/spbase.py +++ b/mpisppy/spbase.py @@ -319,6 +319,9 @@ def _attach_nonant_indices(self): scenario._mpisppy_data.nonant_indices = _nonant_indices scenario._mpisppy_data.all_surrogate_nonants = _all_surrogate_nonants self.nonant_length = len(_nonant_indices) + # sanity check the nonant length + for s in self.local_scenarios.values(): + assert self.nonant_length == len(s._mpisppy_data.nonant_indices) def _attach_nlens(self): From ad1f6b95a2a2fd68d79a44545c28579597a6c11b Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Fri, 7 Mar 2025 13:29:30 -0700 Subject: [PATCH 03/36] automatically register send fields based on class attributes --- mpisppy/cylinders/cross_scen_spoke.py | 13 +-- mpisppy/cylinders/hub.py | 88 +++++++------------ mpisppy/cylinders/reduced_costs_spoke.py | 15 ++-- mpisppy/cylinders/spcommunicator.py | 36 ++++---- mpisppy/cylinders/spoke.py | 55 +++++++++--- mpisppy/cylinders/spwindow.py | 6 +- .../cylinders/xhatshufflelooper_bounder.py | 2 +- 7 files changed, 114 insertions(+), 101 deletions(-) diff --git a/mpisppy/cylinders/cross_scen_spoke.py b/mpisppy/cylinders/cross_scen_spoke.py index 964a48cdf..0347d332f 100644 --- a/mpisppy/cylinders/cross_scen_spoke.py +++ b/mpisppy/cylinders/cross_scen_spoke.py @@ -11,12 +11,16 @@ from mpisppy import MPI from mpisppy.utils.lshaped_cuts import LShapedCutGenerator from mpisppy.cylinders.spwindow import Field +from mpisppy.cylinders.spoke import Spoke import numpy as np import pyomo.environ as pyo -import mpisppy.cylinders.spoke as spoke -class CrossScenarioCutSpoke(spoke.Spoke): +class CrossScenarioCutSpoke(Spoke): + + send_fields = (*Spoke.send_fields, Field.CROSS_SCENARIO_CUT) + receive_fields = (*Spoke.receive_fields, Field.NONANT, Field.CROSS_SCENARIO_COST) + optional_receive_fields = (*Spoke.optional_receive_fields, ) def register_send_fields(self) -> None: @@ -35,15 +39,13 @@ def register_send_fields(self) -> None: (self.nonant_per_scen, remainder) = divmod(vbuflen, local_scen_count) assert(remainder == 0) - ## the _locals will also have the kill signal self.all_nonant_len = vbuflen self.all_eta_len = nscen*local_scen_count self.all_nonants = self.register_recv_field(Field.NONANT, 0, vbuflen) self.all_etas = self.register_recv_field(Field.CROSS_SCENARIO_COST, 0, nscen * nscen) - self.all_coefs = self.register_send_field(Field.CROSS_SCENARIO_CUT, - nscen*(self.nonant_per_scen + 1 + 1)) + self.all_coefs = self.send_buffers[Field.CROSS_SCENARIO_CUT] return @@ -301,7 +303,6 @@ def main(self): # main loop while not (self.got_kill_signal()): - # if self._new_locals: if self.all_nonants.is_new() and self.all_etas.is_new(): self.make_cut() ## End if diff --git a/mpisppy/cylinders/hub.py b/mpisppy/cylinders/hub.py index a82566508..993518d89 100644 --- a/mpisppy/cylinders/hub.py +++ b/mpisppy/cylinders/hub.py @@ -30,6 +30,10 @@ class Hub(SPCommunicator): + send_fields = (*SPCommunicator.send_fields, Field.SHUTDOWN, Field.BEST_OBJECTIVE_BOUNDS,) + receive_fields = (*SPCommunicator.receive_fields, ) + optional_receive_fields = (*SPCommunicator.optional_receive_fields, Field.OBJECTIVE_INNER_BOUND, Field.OBJECTIVE_OUTER_BOUND, ) + _hub_algo_best_bound_provider = False def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, communicators, options=None): @@ -85,7 +89,7 @@ def register_extension_recv_field(self, field: Field, strata_rank: int, buf_len: to the extension sync_with_spokes function. """ key = self._make_key(field, strata_rank) - if key not in self._locals: + if key not in self.receive_buffers: # if it is not already registered, we need to update the local buffer self.extension_recv.add(key) ## End if @@ -103,7 +107,7 @@ def register_extension_send_field(self, field: Field, buf_len: int) -> SendArray return self.register_send_field(field, buf_len) def is_send_field_registered(self, field: Field) -> bool: - return field in self._sends + return field in self.send_buffers def extension_send_field(self, field: Field, buf: SendArray): """ @@ -117,7 +121,7 @@ def sync_extension_fields(self): Update all registered extension fields. Safe to call even when there are no extension fields. """ for key in self.extension_recv: - ext_buf = self._locals[key] + ext_buf = self.receive_buffers[key] (field, srank) = self._split_key(key) ext_buf._is_new = self.hub_from_spoke(ext_buf, srank, field) ## End for @@ -233,7 +237,7 @@ def receive_innerbounds(self): logging.debug("Hub is trying to receive from InnerBounds") for idx in self.innerbound_spoke_indices: key = self._make_key(Field.OBJECTIVE_INNER_BOUND, idx) - recv_buf = self._locals[key] + recv_buf = self.receive_buffers[key] is_new = self.hub_from_spoke(recv_buf, idx, Field.OBJECTIVE_INNER_BOUND) if is_new: bound = recv_buf[0] @@ -249,7 +253,7 @@ def receive_outerbounds(self): logging.debug("Hub is trying to receive from OuterBounds") for idx in self.outerbound_spoke_indices: key = self._make_key(Field.OBJECTIVE_OUTER_BOUND, idx) - recv_buf = self._locals[key] + recv_buf = self.receive_buffers[key] is_new = self.hub_from_spoke(recv_buf, idx, Field.OBJECTIVE_OUTER_BOUND) if is_new: bound = recv_buf[0] @@ -320,18 +324,18 @@ def initialize_inner_bound_buffers(self): def _populate_boundsout_cache(self, buf): """ Populate a given buffer with the current bounds """ - buf[-3] = self.BestOuterBound - buf[-2] = self.BestInnerBound + buf[0] = self.BestOuterBound + buf[1] = self.BestInnerBound def send_boundsout(self): """ Send bounds to the appropriate spokes This is called only for spokes which are bounds only. w and nonant spokes are passed bounds through the w and nonant buffers """ - my_bounds = self.boundsout_send_buffer + my_bounds = self.send_buffers[Field.BEST_OBJECTIVE_BOUNDS] self._populate_boundsout_cache(my_bounds.array()) logging.debug("hub is sending bounds={}".format(my_bounds)) - self.hub_to_spoke(my_bounds, Field.OBJECTIVE_BOUNDS) + self.hub_to_spoke(my_bounds, Field.BEST_OBJECTIVE_BOUNDS) return def initialize_spoke_indices(self): @@ -392,45 +396,7 @@ def initialize_spoke_indices(self): def register_send_fields(self): - - self.shutdown = self.register_send_field(Field.SHUTDOWN, 1) - - required_fields = set() - for i, spoke in enumerate(self.communicators): - if i == self.strata_rank: - continue - spoke_class = spoke["spcomm_class"] - if hasattr(spoke_class, "converger_spoke_types"): - for cst in spoke_class.converger_spoke_types: - if cst == ConvergerSpokeType.W_GETTER: - required_fields.add(Field.DUALS) - elif cst == ConvergerSpokeType.NONANT_GETTER: - required_fields.add(Field.NONANT) - elif cst == ConvergerSpokeType.INNER_BOUND or cst == ConvergerSpokeType.OUTER_BOUND: - required_fields.add(Field.OBJECTIVE_BOUNDS) - else: - pass # Intentional no-op - ## End if - ## End for - else: - # Intentional no-op. Non-converger spokes need to register any needed - # fields separately. See the functions `register_extension_recv_field` - # and `register_extension_send_field`. - pass - ## End if - ## End for - - n_nonants = 0 - for s in self.opt.local_scenarios.values(): - n_nonants += len(s._mpisppy_data.nonant_indices) - ## End for - - if Field.DUALS in required_fields: - self.w_send_buffer = self.register_send_field(Field.DUALS, n_nonants) - if Field.NONANT in required_fields: - self.nonant_send_buffer = self.register_send_field(Field.NONANT, n_nonants) - if Field.OBJECTIVE_BOUNDS in required_fields: - self.boundsout_send_buffer = self.register_send_field(Field.OBJECTIVE_BOUNDS, 2) + super().register_send_fields() # Not all opt classes may have extensions if getattr(self.opt, "extensions", None) is not None: @@ -439,7 +405,6 @@ def register_send_fields(self): return - def hub_to_spoke(self, buf: SendArray, field: Field): """ Put the specified values into the specified locally-owned buffer for the spoke to pick up. @@ -534,13 +499,17 @@ def send_terminate(self): buffer, so every spoke will see it simultaneously. processes (don't need to call them one at a time). """ - shutdown = self.shutdown - shutdown[0] = 1.0 - self.hub_to_spoke(shutdown, Field.SHUTDOWN) + self.send_buffers[Field.SHUTDOWN][0] = 1.0 + self.hub_to_spoke(self.send_buffers[Field.SHUTDOWN], Field.SHUTDOWN) return class PHHub(Hub): + + send_fields = (*Hub.send_fields, Field.NONANT, Field.DUALS) + receive_fields = (*Hub.receive_fields,) + optional_receive_fields = (*Hub.optional_receive_fields,) + def setup_hub(self): """ Must be called after make_windows(), so that the hub knows the sizes of all the spokes windows @@ -673,8 +642,7 @@ def send_nonants(self): """ self.opt._save_nonants() ci = 0 ## index to self.nonant_send_buffer - # my_nonants = self._sends[Field.NONANT] - nonant_send_buffer = self.nonant_send_buffer + nonant_send_buffer = self.send_buffers[Field.NONANT] for k, s in self.opt.local_scenarios.items(): for xvar in s._mpisppy_data.nonant_indices.values(): nonant_send_buffer[ci] = xvar._value @@ -690,7 +658,7 @@ def send_ws(self): """ Send dual weights to the appropriate spokes """ # NOTE: my_ws.array() and self.w_send_buffer should be the same array. - my_ws = self._sends[Field.DUALS] + my_ws = self.send_buffers[Field.DUALS] self.opt._populate_W_cache(my_ws.array(), padding=1) logging.debug("hub is sending Ws={}".format(my_ws.array())) @@ -701,6 +669,10 @@ def send_ws(self): class LShapedHub(Hub): + send_fields = (*Hub.send_fields, Field.NONANT,) + receive_fields = (*Hub.receive_fields,) + optional_receive_fields = (*Hub.optional_receive_fields,) + def setup_hub(self): """ Must be called after make_windows(), so that the hub knows the sizes of all the spokes windows @@ -781,7 +753,7 @@ def send_nonants(self): TODO: Will likely fail with bundling """ ci = 0 ## index to self.nonant_send_buffer - nonant_send_buffer = self.nonant_send_buffer + nonant_send_buffer = self.send_buffers[Field.NONANT] for k, s in self.opt.local_scenarios.items(): nonant_to_root_var_map = s._mpisppy_model.subproblem_to_root_vars_map for xvar in s._mpisppy_data.nonant_indices.values(): @@ -797,6 +769,8 @@ def send_nonants(self): class SubgradientHub(PHHub): + # send / receive fields are same as PHHub + _hub_algo_best_bound_provider = True def main(self): @@ -806,6 +780,8 @@ def main(self): class APHHub(PHHub): + # send / receive fields are same as PHHub + def main(self): """ SPComm gets attached by self.__init___; holding APH harmless """ logger.critical("aph debug main in hub.py") diff --git a/mpisppy/cylinders/reduced_costs_spoke.py b/mpisppy/cylinders/reduced_costs_spoke.py index cb96c909b..f2a643db7 100644 --- a/mpisppy/cylinders/reduced_costs_spoke.py +++ b/mpisppy/cylinders/reduced_costs_spoke.py @@ -15,6 +15,10 @@ class ReducedCostsSpoke(LagrangianOuterBound): + send_fields = (*LagrangianOuterBound.send_fields, Field.EXPECTED_REDUCED_COST, Field.SCENARIO_REDUCED_COST ,) + receive_fields = (*LagrangianOuterBound.receive_fields,) + optional_receive_fields = (*LagrangianOuterBound.optional_receive_fields,) + converger_spoke_char = 'R' def __init__(self, *args, **kwargs): @@ -54,28 +58,25 @@ def register_send_fields(self) -> None: scenario_buffer_len += len(s._mpisppy_data.nonant_indices) self._scenario_rc_buffer = np.zeros(scenario_buffer_len) - self.register_send_field(Field.EXPECTED_REDUCED_COST, self.nonant_length) - self.register_send_field(Field.SCENARIO_REDUCED_COST, scenario_buffer_len) - return @property def rc_global(self): - return self._sends[Field.EXPECTED_REDUCED_COST].value_array() + return self.send_buffers[Field.EXPECTED_REDUCED_COST].value_array() @rc_global.setter def rc_global(self, vals): - arr = self._sends[Field.EXPECTED_REDUCED_COST].value_array() + arr = self.send_buffers[Field.EXPECTED_REDUCED_COST].value_array() arr[:] = vals return @property def rc_scenario(self): - return self._sends[Field.SCENARIO_REDUCED_COST].value_array() + return self.send_buffers[Field.SCENARIO_REDUCED_COST].value_array() @rc_scenario.setter def rc_scenario(self, vals): - arr = self._sends[Field.SCENARIO_REDUCED_COST].value_array() + arr = self.send_buffers[Field.SCENARIO_REDUCED_COST].value_array() arr[:] = vals return diff --git a/mpisppy/cylinders/spcommunicator.py b/mpisppy/cylinders/spcommunicator.py index 2f6d07401..db0e2a241 100644 --- a/mpisppy/cylinders/spcommunicator.py +++ b/mpisppy/cylinders/spcommunicator.py @@ -23,7 +23,7 @@ import abc import time -from mpisppy.cylinders.spwindow import Field, SPWindow +from mpisppy.cylinders.spwindow import Field, FieldLengths, SPWindow def communicator_array(size): arr = np.empty(size+1, dtype='d') @@ -138,26 +138,26 @@ def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, communic self.options = options # Common fields for spokes and hubs - self._locals = dict() - self._sends = dict() + self.receive_buffers = dict() + self.send_buffers = dict() # setup FieldLengths which calculates # the length of each buffer type based # on the problem data - # self._field_lengths = FieldLengths(self.opt) + self._field_lengths = FieldLengths(self.opt) # attach the SPCommunicator to # the SPBase object self.opt.spcomm = self - # self.register_send_fields() + self.register_send_fields() return def _make_key(self, field: Field, origin: int): """ Given a field and an origin (i.e. a strata_rank), generate a key for indexing - into the self._locals dictionary and getting the corresponding RecvArray. + into the self.receive_buffers dictionary and getting the corresponding RecvArray. Undone by `_split_key`. Currently, the key is simply a Tuple[field, origin]. """ @@ -175,9 +175,8 @@ def _split_key(self, key) -> tuple[Field, int]: def _build_window_spec(self) -> dict[Field, int]: """ Build dict with fields and lengths needed for local MPI window """ - self.register_send_fields() window_spec = dict() - for (field,buf) in self._sends.items(): + for (field,buf) in self.send_buffers.items(): window_spec[field] = np.size(buf.array()) ## End for return window_spec @@ -186,28 +185,28 @@ def register_recv_field(self, field: Field, origin: int, length: int = -1) -> Re key = self._make_key(field, origin) if length == -1: length = self._field_lengths[field] - if key in self._locals: - my_fa = self._locals[key] + if key in self.receive_buffers: + my_fa = self.receive_buffers[key] assert(length + 1 == np.size(my_fa.array())) else: my_fa = RecvArray(length) - self._locals[key] = my_fa + self.receive_buffers[key] = my_fa ## End if return my_fa def register_send_field(self, field: Field, length: int = -1) -> SendArray: - assert field not in self._sends, "Field {} is already registered".format(field) + assert field not in self.send_buffers, "Field {} is already registered".format(field) if length == -1: length = self._field_lengths[field] - # if field in self._sends: - # my_fa = self._sends[field] + # if field in self.send_buffers: + # my_fa = self.send_buffers[field] # assert(length + 1 == np.size(my_fa.array())) # else: # my_fa = SendArray(length) - # self._sends[field] = my_fa + # self.send_buffers[field] = my_fa # ## End if else my_fa = SendArray(length) - self._sends[field] = my_fa + self.send_buffers[field] = my_fa return my_fa @abc.abstractmethod @@ -259,6 +258,7 @@ def make_windows(self) -> None: return - @abc.abstractmethod def register_send_fields(self) -> None: - pass + self.send_buffers = {} + for field in self.send_fields: + self.send_buffers[field] = self.register_send_field(field) diff --git a/mpisppy/cylinders/spoke.py b/mpisppy/cylinders/spoke.py index bf27f5a1e..39987b1dd 100644 --- a/mpisppy/cylinders/spoke.py +++ b/mpisppy/cylinders/spoke.py @@ -26,6 +26,11 @@ class ConvergerSpokeType(enum.Enum): NONANT_GETTER = 4 class Spoke(SPCommunicator): + + send_fields = (*SPCommunicator.send_fields, ) + receive_fields = (*SPCommunicator.receive_fields, Field.SHUTDOWN, ) + optional_receive_fields = (*SPCommunicator.optional_receive_fields, ) + def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, communicators, options=None): super().__init__(spbase_object, fullcomm, strata_comm, cylinder_comm, communicators, options) @@ -96,7 +101,7 @@ def _spoke_from_hub(self, return False def _got_kill_signal(self): - shutdown_buf = self._locals[self._make_key(Field.SHUTDOWN, 0)] + shutdown_buf = self.receive_buffers[self._make_key(Field.SHUTDOWN, 0)] if shutdown_buf.is_new(): shutdown = (self.shutdown[0] == 1.0) else: @@ -108,7 +113,7 @@ def got_kill_signal(self): """ Spoke should call this method at least every iteration to see if the Hub terminated """ - self.update_locals() + self.updatereceive_buffers() return self._got_kill_signal() @abc.abstractmethod @@ -121,8 +126,8 @@ def main(self): """ pass - def update_locals(self): - for (key, recv_buf) in self._locals.items(): + def updatereceive_buffers(self): + for (key, recv_buf) in self.receive_buffers.items(): field, rank = self._split_key(key) # The below code will need to be updated for spoke to spoke communication assert(rank == 0) @@ -134,6 +139,11 @@ def update_locals(self): class _BoundSpoke(Spoke): """ A base class for bound spokes """ + + send_fields = (*Spoke.send_fields, ) + receive_fields = (*Spoke.receive_fields, Field.BEST_OBJECTIVE_BOUNDS) + optional_receive_fields = (*Spoke.optional_receive_fields, ) + def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, options=None): super().__init__(spbase_object, fullcomm, strata_comm, cylinder_comm, options) if self.cylinder_rank == 0 and \ @@ -155,8 +165,9 @@ def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, options= def register_send_fields(self) -> None: - self._bound = self.register_send_field(self.bound_type(), 1) - self._hub_bounds = self.register_recv_field(Field.OBJECTIVE_BOUNDS, 0, 2) + super().register_send_fields() + self._bound = self.send_buffers[self.bound_type()] + self._hub_bounds = self.register_recv_field(Field.BEST_OBJECTIVE_BOUNDS, 0, 2) return @abc.abstractmethod @@ -219,6 +230,11 @@ class InnerBoundSpoke(_BoundSpoke): """ For Spokes that provide an inner bound through self.bound to the Hub, and do not need information from the main PH OPT hub. """ + + send_fields = (*_BoundSpoke.send_fields, Field.OBJECTIVE_INNER_BOUND, ) + receive_fields = (*_BoundSpoke.receive_fields, ) + optional_receive_fields = (*_BoundSpoke.optional_receive_fields,) + converger_spoke_types = (ConvergerSpokeType.INNER_BOUND,) converger_spoke_char = 'I' @@ -230,6 +246,11 @@ class OuterBoundSpoke(_BoundSpoke): """ For Spokes that provide an outer bound through self.bound to the Hub, and do not need information from the main PH OPT hub. """ + + send_fields = (*_BoundSpoke.send_fields, Field.OBJECTIVE_OUTER_BOUND, ) + receive_fields = (*_BoundSpoke.receive_fields, ) + optional_receive_fields = (*_BoundSpoke.optional_receive_fields,) + converger_spoke_types = (ConvergerSpokeType.OUTER_BOUND,) converger_spoke_char = 'O' @@ -249,7 +270,7 @@ def nonant_len_type(self) -> Field: def localWs(self): """Returns the local copy of the weights""" key = self._make_key(Field.DUALS, 0) - return self._locals[key].value_array() + return self.receive_buffers[key].value_array() @property def new_Ws(self): @@ -258,7 +279,7 @@ def new_Ws(self): the last call to got_kill_signal """ key = self._make_key(Field.DUALS, 0) - return self._locals[key].is_new() + return self.receive_buffers[key].is_new() class OuterBoundWSpoke(_BoundWSpoke): @@ -269,6 +290,10 @@ class OuterBoundWSpoke(_BoundWSpoke): the main PH OPT hub. """ + send_fields = (*_BoundWSpoke.send_fields, Field.OBJECTIVE_OUTER_BOUND, ) + receive_fields = (*_BoundWSpoke.receive_fields, Field.DUALS) + optional_receive_fields = (*_BoundWSpoke.optional_receive_fields,) + converger_spoke_types = ( ConvergerSpokeType.OUTER_BOUND, ConvergerSpokeType.W_GETTER, @@ -291,7 +316,7 @@ def nonant_len_type(self) -> Field: def localnonants(self): """Returns the local copy of the nonants""" key = self._make_key(Field.NONANT, 0) - return self._locals[key].value_array() + return self.receive_buffers[key].value_array() @property def new_nonants(self): @@ -299,7 +324,7 @@ def new_nonants(self): the nonants has been updated since the last call to got_kill_signal""" key = self._make_key(Field.NONANT, 0) - return self._locals[key].is_new() + return self.receive_buffers[key].is_new() class InnerBoundNonantSpoke(_BoundNonantSpoke): @@ -311,6 +336,11 @@ class InnerBoundNonantSpoke(_BoundNonantSpoke): Includes some helpful methods for saving and restoring results """ + + send_fields = (*_BoundNonantSpoke.send_fields, Field.OBJECTIVE_INNER_BOUND, ) + receive_fields = (*_BoundNonantSpoke.receive_fields, Field.NONANT) + optional_receive_fields = (*_BoundNonantSpoke.optional_receive_fields,) + converger_spoke_types = ( ConvergerSpokeType.INNER_BOUND, ConvergerSpokeType.NONANT_GETTER, @@ -355,6 +385,11 @@ class OuterBoundNonantSpoke(_BoundNonantSpoke): and receive the nonants from the main OPT hub. """ + + send_fields = (*_BoundNonantSpoke.send_fields, Field.OBJECTIVE_OUTER_BOUND, ) + receive_fields = (*_BoundNonantSpoke.receive_fields, Field.NONANT) + optional_receive_fields = (*_BoundNonantSpoke.optional_receive_fields,) + converger_spoke_types = ( ConvergerSpokeType.OUTER_BOUND, ConvergerSpokeType.NONANT_GETTER, diff --git a/mpisppy/cylinders/spwindow.py b/mpisppy/cylinders/spwindow.py index e44eb80c4..1416329bf 100644 --- a/mpisppy/cylinders/spwindow.py +++ b/mpisppy/cylinders/spwindow.py @@ -20,7 +20,7 @@ class Field(enum.IntEnum): SHUTDOWN=-1000 NONANT=1 DUALS=2 - OBJECTIVE_BOUNDS=100 # Both inner and outer bounds from the hub. Layout: [OUTER INNER ID] + BEST_OBJECTIVE_BOUNDS=100 # Both inner and outer bounds from the hub. Layout: [OUTER INNER ID] OBJECTIVE_INNER_BOUND=101 OBJECTIVE_OUTER_BOUND=102 EXPECTED_REDUCED_COST=200 @@ -40,12 +40,12 @@ class Field(enum.IntEnum): Field.SHUTDOWN : 1, Field.NONANT : _field_length_components.local_nonant_length, Field.DUALS : _field_length_components.local_nonant_length, - Field.OBJECTIVE_BOUNDS : 2, + Field.BEST_OBJECTIVE_BOUNDS : 2, Field.OBJECTIVE_INNER_BOUND : 1, Field.OBJECTIVE_OUTER_BOUND : 1, Field.EXPECTED_REDUCED_COST : _field_length_components.total_number_nonants, Field.SCENARIO_REDUCED_COST : _field_length_components.local_nonant_length, - Field.CROSS_SCENARIO_CUT : _field_length_components.local_nonant_length, + Field.CROSS_SCENARIO_CUT : _field_length_components.total_number_scenarios * (_field_length_components.total_number_nonants + 1 + 1), Field.CROSS_SCENARIO_COST : _field_length_components.total_number_scenarios * _field_length_components.total_number_scenarios, } diff --git a/mpisppy/cylinders/xhatshufflelooper_bounder.py b/mpisppy/cylinders/xhatshufflelooper_bounder.py index 462f07328..37b97cfbc 100644 --- a/mpisppy/cylinders/xhatshufflelooper_bounder.py +++ b/mpisppy/cylinders/xhatshufflelooper_bounder.py @@ -103,7 +103,7 @@ def _vb(msg): # if self.get_serial_number() == 0: # continue - if self._locals[self._make_key(Field.NONANT, 0)].id() == 0: + if self.receive_buffers[self._make_key(Field.NONANT, 0)].id() == 0: continue if (xh_iter-1) % 100 == 0: From dd91f148852b41d261b7b0fc68f5b7f826df47e0 Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Fri, 7 Mar 2025 13:54:21 -0700 Subject: [PATCH 04/36] remove has_*_spokes --- mpisppy/cylinders/hub.py | 78 +++++++++++++--------------------------- 1 file changed, 25 insertions(+), 53 deletions(-) diff --git a/mpisppy/cylinders/hub.py b/mpisppy/cylinders/hub.py index 993518d89..cf3e84e6c 100644 --- a/mpisppy/cylinders/hub.py +++ b/mpisppy/cylinders/hub.py @@ -219,10 +219,8 @@ def determine_termination(self): return abs_gap_satisfied or rel_gap_satisfied or max_stalled_satisfied def hub_finalize(self): - if self.has_outerbound_spokes: - self.receive_outerbounds() - if self.has_innerbound_spokes: - self.receive_innerbounds() + self.receive_outerbounds() + self.receive_innerbounds() if self.global_rank == 0: self.print_init = True @@ -382,12 +380,6 @@ def initialize_spoke_indices(self): (self.outerbound_spoke_indices | self.innerbound_spoke_indices) - \ (self.w_spoke_indices | self.nonant_spoke_indices) - self.has_outerbound_spokes = len(self.outerbound_spoke_indices) > 0 - self.has_innerbound_spokes = len(self.innerbound_spoke_indices) > 0 - self.has_nonant_spokes = len(self.nonant_spoke_indices) > 0 - self.has_w_spokes = len(self.w_spoke_indices) > 0 - self.has_bounds_only_spokes = len(self.bounds_only_indices) > 0 - # Not all opt classes may have extensions if getattr(self.opt, "extensions", None) is not None: self.opt.extobject.initialize_spoke_indices() @@ -522,10 +514,8 @@ def setup_hub(self): self.initialize_spoke_indices() self.initialize_bound_values() - if self.has_outerbound_spokes: - self.initialize_outer_bound_buffers() - if self.has_innerbound_spokes: - self.initialize_inner_bound_buffers() + self.initialize_outer_bound_buffers() + self.initialize_inner_bound_buffers() ## Do some checking for things we currently don't support if len(self.outerbound_spoke_indices & self.innerbound_spoke_indices) > 0: @@ -539,13 +529,13 @@ def setup_hub(self): ) ## Generate some warnings if nothing is giving bounds - if not self.has_outerbound_spokes: + if not self.outerbound_spoke_indices: logger.warn( "No OuterBound Spokes defined, this converger " "will not cause the hub to terminate" ) - if not self.has_innerbound_spokes: + if not self.innerbound_spoke_indices: logger.warn( "No InnerBound Spokes defined, this converger " "will not cause the hub to terminate" @@ -557,16 +547,11 @@ def sync(self): """ Manages communication with Spokes """ - if self.has_w_spokes: - self.send_ws() - if self.has_nonant_spokes: - self.send_nonants() - if self.has_bounds_only_spokes: - self.send_boundsout() - if self.has_outerbound_spokes: - self.receive_outerbounds() - if self.has_innerbound_spokes: - self.receive_innerbounds() + self.send_ws() + self.send_nonants() + self.send_boundsout() + self.receive_outerbounds() + self.receive_innerbounds() if self.opt.extensions is not None: self.sync_extension_fields() self.opt.extobject.sync_with_spokes() @@ -575,30 +560,25 @@ def sync_with_spokes(self): self.sync() def sync_bounds(self): - if self.has_outerbound_spokes: - self.receive_outerbounds() - if self.has_innerbound_spokes: - self.receive_innerbounds() - if self.has_bounds_only_spokes: - self.send_boundsout() + self.receive_outerbounds() + self.receive_innerbounds() + self.send_boundsout() def sync_extensions(self): if self.opt.extensions is not None: self.opt.extobject.sync_with_spokes() def sync_nonants(self): - if self.has_nonant_spokes: - self.send_nonants() + self.send_nonants() def sync_Ws(self): - if self.has_w_spokes: - self.send_ws() + self.send_ws() def is_converged(self): if self.opt.best_bound_obj_val is not None: self.BestOuterBound = self.OuterBoundUpdate(self.opt.best_bound_obj_val) - if not self.has_innerbound_spokes: + if not self.innerbound_spoke_indices: if self.opt._PHIter == 1: logger.warning( "PHHub cannot compute convergence without " @@ -611,7 +591,7 @@ def is_converged(self): return False - if not self.has_outerbound_spokes: + if not self.outerbound_spoke_indices: if self.opt._PHIter == 1 and not self._hub_algo_best_bound_provider: global_toc( "Without outer bound spokes, no progress " @@ -638,9 +618,7 @@ def finalize(self): def send_nonants(self): """ Gather nonants and send them to the appropriate spokes - TODO: Will likely fail with bundling """ - self.opt._save_nonants() ci = 0 ## index to self.nonant_send_buffer nonant_send_buffer = self.send_buffers[Field.NONANT] for k, s in self.opt.local_scenarios.items(): @@ -685,17 +663,13 @@ def setup_hub(self): self.initialize_spoke_indices() self.initialize_bound_values() - if self.has_outerbound_spokes: - self.initialize_outer_bound_buffers() - if self.has_innerbound_spokes: - self.initialize_inner_bound_buffers() + self.initialize_outer_bound_buffers() + self.initialize_inner_bound_buffers() ## Do some checking for things we currently ## do not support - if self.has_w_spokes: + if self.w_spoke_indices: raise RuntimeError("LShaped hub does not compute dual weights (Ws)") - # if self.has_nonant_spokes: - # self.initialize_nonants() if len(self.outerbound_spoke_indices & self.innerbound_spoke_indices) > 0: raise RuntimeError( "A Spoke providing both inner and outer " @@ -703,7 +677,7 @@ def setup_hub(self): ) ## Generate some warnings if nothing is giving bounds - if not self.has_innerbound_spokes: + if not self.innerbound_spoke_indices: logger.warn( "No InnerBound Spokes defined, this converger " "will not cause the hub to terminate" @@ -713,12 +687,10 @@ def sync(self, send_nonants=True): """ Manages communication with Bound Spokes """ - if send_nonants and self.has_nonant_spokes: + if send_nonants: self.send_nonants() - if self.has_outerbound_spokes: - self.receive_outerbounds() - if self.has_innerbound_spokes: - self.receive_innerbounds() + self.receive_outerbounds() + self.receive_innerbounds() # in case LShaped ever gets extensions if getattr(self.opt, "extensions", None) is not None: self.sync_extension_fields() From d89676c6596ef79d1b423a488fb4aacbb8a9a069 Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Fri, 7 Mar 2025 15:19:29 -0700 Subject: [PATCH 05/36] more generic receive logic --- mpisppy/cylinders/hub.py | 134 +++------------------ mpisppy/cylinders/spcommunicator.py | 27 ++++- mpisppy/extensions/cross_scen_extension.py | 35 ++---- mpisppy/extensions/extension.py | 26 ++-- mpisppy/extensions/reduced_costs_fixer.py | 39 +++--- 5 files changed, 88 insertions(+), 173 deletions(-) diff --git a/mpisppy/cylinders/hub.py b/mpisppy/cylinders/hub.py index cf3e84e6c..866347754 100644 --- a/mpisppy/cylinders/hub.py +++ b/mpisppy/cylinders/hub.py @@ -16,7 +16,6 @@ from mpisppy import MPI from mpisppy.cylinders.spcommunicator import RecvArray, SendArray, SPCommunicator from math import inf -from mpisppy.cylinders.spoke import ConvergerSpokeType from mpisppy import global_toc @@ -51,6 +50,8 @@ def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, communic self.extension_recv = set() + self.initialize_bound_values() + return @abc.abstractmethod @@ -233,14 +234,12 @@ def receive_innerbounds(self): (but should be harmless to call if there are none) """ logging.debug("Hub is trying to receive from InnerBounds") - for idx in self.innerbound_spoke_indices: - key = self._make_key(Field.OBJECTIVE_INNER_BOUND, idx) - recv_buf = self.receive_buffers[key] + for idx, cls, recv_buf in self.receive_field_spcomms[Field.OBJECTIVE_INNER_BOUND]: is_new = self.hub_from_spoke(recv_buf, idx, Field.OBJECTIVE_INNER_BOUND) if is_new: bound = recv_buf[0] logging.debug("!! new InnerBound to opt {}".format(bound)) - self.BestInnerBound = self.InnerBoundUpdate(bound, idx) + self.BestInnerBound = self.InnerBoundUpdate(bound, cls, idx) logging.debug("ph back from InnerBounds") def receive_outerbounds(self): @@ -249,37 +248,35 @@ def receive_outerbounds(self): (but should be harmless to call if there are none) """ logging.debug("Hub is trying to receive from OuterBounds") - for idx in self.outerbound_spoke_indices: - key = self._make_key(Field.OBJECTIVE_OUTER_BOUND, idx) - recv_buf = self.receive_buffers[key] + for idx, cls, recv_buf in self.receive_field_spcomms[Field.OBJECTIVE_OUTER_BOUND]: is_new = self.hub_from_spoke(recv_buf, idx, Field.OBJECTIVE_OUTER_BOUND) if is_new: bound = recv_buf[0] logging.debug("!! new OuterBound to opt {}".format(bound)) - self.BestOuterBound = self.OuterBoundUpdate(bound, idx) + self.BestOuterBound = self.OuterBoundUpdate(bound, cls, idx) logging.debug("ph back from OuterBounds") - def OuterBoundUpdate(self, new_bound, idx=None, char='*'): + def OuterBoundUpdate(self, new_bound, cls=None, idx=None, char='*'): current_bound = self.BestOuterBound if self._outer_bound_update(new_bound, current_bound): - if idx is None: + if cls is None: self.latest_ob_char = char self.last_ob_idx = 0 else: - self.latest_ob_char = self.outerbound_spoke_chars[idx] + self.latest_ib_char = cls.converger_spoke_char self.last_ob_idx = idx return new_bound else: return current_bound - def InnerBoundUpdate(self, new_bound, idx=None, char='*'): + def InnerBoundUpdate(self, new_bound, cls=None, idx=None, char='*'): current_bound = self.BestInnerBound if self._inner_bound_update(new_bound, current_bound): - if idx is None: + if cls is None: self.latest_ib_char = char self.last_ib_idx = 0 else: - self.latest_ib_char = self.innerbound_spoke_chars[idx] + self.latest_ib_char = cls.converger_spoke_char self.last_ib_idx = idx return new_bound else: @@ -297,28 +294,6 @@ def initialize_bound_values(self): self._inner_bound_update = lambda new, old : (new > old) self._outer_bound_update = lambda new, old : (new < old) - def initialize_outer_bound_buffers(self): - """ Initialize outer bound receive buffers - """ - self.outerbound_receive_buffers = dict() - for idx in self.outerbound_spoke_indices: - self.outerbound_receive_buffers[idx] = self.register_recv_field( - Field.OBJECTIVE_OUTER_BOUND, idx, 1, - ) - ## End for - return - - def initialize_inner_bound_buffers(self): - """ Initialize inner bound receive buffers - """ - self.innerbound_receive_buffers = dict() - for idx in self.innerbound_spoke_indices: - self.innerbound_receive_buffers[idx] = self.register_recv_field( - Field.OBJECTIVE_INNER_BOUND, idx, 1 - ) - ## End for - return - def _populate_boundsout_cache(self, buf): """ Populate a given buffer with the current bounds """ @@ -327,8 +302,6 @@ def _populate_boundsout_cache(self, buf): def send_boundsout(self): """ Send bounds to the appropriate spokes - This is called only for spokes which are bounds only. - w and nonant spokes are passed bounds through the w and nonant buffers """ my_bounds = self.send_buffers[Field.BEST_OBJECTIVE_BOUNDS] self._populate_boundsout_cache(my_bounds.array()) @@ -336,7 +309,7 @@ def send_boundsout(self): self.hub_to_spoke(my_bounds, Field.BEST_OBJECTIVE_BOUNDS) return - def initialize_spoke_indices(self): + def register_receive_fields(self): """ Figure out what types of spokes we have, and sort them into the appropriate classes. @@ -344,45 +317,11 @@ def initialize_spoke_indices(self): Some spokes may be multiple types (e.g. outerbound and nonant), though not all combinations are supported. """ - self.outerbound_spoke_indices = set() - self.innerbound_spoke_indices = set() - self.nonant_spoke_indices = set() - self.w_spoke_indices = set() - - self.outerbound_spoke_chars = dict() - self.innerbound_spoke_chars = dict() - - for (i, spoke) in enumerate(self.communicators): - if i == self.strata_rank: - continue - spoke_class = spoke["spcomm_class"] - if hasattr(spoke_class, "converger_spoke_types"): - for cst in spoke_class.converger_spoke_types: - if cst == ConvergerSpokeType.OUTER_BOUND: - self.outerbound_spoke_indices.add(i) - self.outerbound_spoke_chars[i] = spoke_class.converger_spoke_char - elif cst == ConvergerSpokeType.INNER_BOUND: - self.innerbound_spoke_indices.add(i) - self.innerbound_spoke_chars[i] = spoke_class.converger_spoke_char - elif cst == ConvergerSpokeType.W_GETTER: - self.w_spoke_indices.add(i) - elif cst == ConvergerSpokeType.NONANT_GETTER: - self.nonant_spoke_indices.add(i) - else: - raise RuntimeError(f"Unrecognized converger_spoke_type {cst}") - - else: ##this isn't necessarily wrong, i.e., cut generators - logger.debug(f"Spoke class {spoke_class} not recognized by hub") - - # all _BoundSpoke spokes get hub bounds so we determine which spokes - # are "bounds only" - self.bounds_only_indices = \ - (self.outerbound_spoke_indices | self.innerbound_spoke_indices) - \ - (self.w_spoke_indices | self.nonant_spoke_indices) + super().register_receive_fields() # Not all opt classes may have extensions if getattr(self.opt, "extensions", None) is not None: - self.opt.extobject.initialize_spoke_indices() + self.opt.extobject.register_receive_fields() return @@ -511,31 +450,14 @@ def setup_hub(self): "Cannot call setup_hub before memory windows are constructed" ) - self.initialize_spoke_indices() - self.initialize_bound_values() - - self.initialize_outer_bound_buffers() - self.initialize_inner_bound_buffers() - - ## Do some checking for things we currently don't support - if len(self.outerbound_spoke_indices & self.innerbound_spoke_indices) > 0: - raise RuntimeError( - "A Spoke providing both inner and outer " - "bounds is currently unsupported" - ) - if len(self.w_spoke_indices & self.nonant_spoke_indices) > 0: - raise RuntimeError( - "A Spoke needing both Ws and nonants is currently unsupported" - ) - ## Generate some warnings if nothing is giving bounds - if not self.outerbound_spoke_indices: + if not self.receive_field_spcomms[Field.OBJECTIVE_OUTER_BOUND]: logger.warn( "No OuterBound Spokes defined, this converger " "will not cause the hub to terminate" ) - if not self.innerbound_spoke_indices: + if not self.receive_field_spcomms[Field.OBJECTIVE_INNER_BOUND]: logger.warn( "No InnerBound Spokes defined, this converger " "will not cause the hub to terminate" @@ -578,7 +500,7 @@ def is_converged(self): if self.opt.best_bound_obj_val is not None: self.BestOuterBound = self.OuterBoundUpdate(self.opt.best_bound_obj_val) - if not self.innerbound_spoke_indices: + if not self.receive_field_spcomms[Field.OBJECTIVE_INNER_BOUND]: if self.opt._PHIter == 1: logger.warning( "PHHub cannot compute convergence without " @@ -591,7 +513,7 @@ def is_converged(self): return False - if not self.outerbound_spoke_indices: + if not self.receive_field_spcomms[Field.OBJECTIVE_OUTER_BOUND]: if self.opt._PHIter == 1 and not self._hub_algo_best_bound_provider: global_toc( "Without outer bound spokes, no progress " @@ -660,24 +582,8 @@ def setup_hub(self): "Cannot call setup_hub before memory windows are constructed" ) - self.initialize_spoke_indices() - self.initialize_bound_values() - - self.initialize_outer_bound_buffers() - self.initialize_inner_bound_buffers() - - ## Do some checking for things we currently - ## do not support - if self.w_spoke_indices: - raise RuntimeError("LShaped hub does not compute dual weights (Ws)") - if len(self.outerbound_spoke_indices & self.innerbound_spoke_indices) > 0: - raise RuntimeError( - "A Spoke providing both inner and outer " - "bounds is currently unsupported" - ) - ## Generate some warnings if nothing is giving bounds - if not self.innerbound_spoke_indices: + if not self.receive_field_spcomms[Field.OBJECTIVE_INNER_BOUND]: logger.warn( "No InnerBound Spokes defined, this converger " "will not cause the hub to terminate" diff --git a/mpisppy/cylinders/spcommunicator.py b/mpisppy/cylinders/spcommunicator.py index db0e2a241..b0dc23d7c 100644 --- a/mpisppy/cylinders/spcommunicator.py +++ b/mpisppy/cylinders/spcommunicator.py @@ -22,6 +22,7 @@ import numpy as np import abc import time +import itertools from mpisppy.cylinders.spwindow import Field, FieldLengths, SPWindow @@ -138,8 +139,10 @@ def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, communic self.options = options # Common fields for spokes and hubs - self.receive_buffers = dict() - self.send_buffers = dict() + self.receive_buffers = {} + self.send_buffers = {} + # key: Field, value: list of (strata_rank, SPComm) with that Field + self.receive_field_spcomms = {} # setup FieldLengths which calculates # the length of each buffer type based @@ -151,6 +154,11 @@ def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, communic self.opt.spcomm = self self.register_send_fields() + # TODO: here we can have a dynamic exchange of the send fields + # so we can do error checking (all-to-all in send fields) + self.register_receive_fields() + + # TODO: check that we have something in receive_field_spcomms?? return @@ -259,6 +267,17 @@ def make_windows(self) -> None: return def register_send_fields(self) -> None: - self.send_buffers = {} for field in self.send_fields: - self.send_buffers[field] = self.register_send_field(field) + self.register_send_field(field) + + def register_receive_fields(self) -> None: + # TODO: make receive_fields dynamic? Make the callback handles this??? + for field in itertools.chain(self.receive_fields, self.optional_receive_fields): + self.receive_field_spcomms[field] = [] + for strata_rank, comm in enumerate(self.communicators): + if strata_rank == self.strata_rank: + continue + cls = comm["spcomm_class"] + if field in cls.send_fields: + buff = self.register_recv_field(field, strata_rank) + self.receive_field_spcomms[field].append((strata_rank, cls, buff)) diff --git a/mpisppy/extensions/cross_scen_extension.py b/mpisppy/extensions/cross_scen_extension.py index 6142e37d0..4f6955e50 100644 --- a/mpisppy/extensions/cross_scen_extension.py +++ b/mpisppy/extensions/cross_scen_extension.py @@ -127,12 +127,6 @@ def _check_bound(self): cached_ph_obj[k].activate() def get_from_cross_cuts(self): - # spcomm = self.opt.spcomm - # idx = self.cut_gen_spoke_index - # receive_buffer = np.empty(spcomm.remote_lengths[idx - 1] + 1, dtype="d") # Must be doubles - # is_new = spcomm.hub_from_spoke(receive_buffer, idx) - # if is_new: - # self.make_cuts(receive_buffer) if self.cuts.is_new(): self.make_cuts(self.cuts.array()) @@ -275,8 +269,6 @@ def register_send_fields(self): return def setup_hub(self): - # idx = self.cut_gen_spoke_index - # self.all_nonants_and_etas = np.zeros(self.opt.spcomm.local_lengths[idx - 1] + 1) self.nonant_len = self.opt.nonant_length @@ -287,22 +279,17 @@ def setup_hub(self): # helping the extension track cuts self.new_cuts = False - def initialize_spoke_indices(self): - for (i, spoke) in enumerate(self.opt.spcomm.communicators): - if spoke["spcomm_class"] == CrossScenarioCutSpoke: - self.cut_gen_spoke_index = i - ## End if - ## End for - - if hasattr(self, "cut_gen_spoke_index"): - spcomm = self.opt.spcomm - nscen = len(self.opt.all_scenario_names) - self.cuts = spcomm.register_extension_recv_field( - Field.CROSS_SCENARIO_CUT, - self.cut_gen_spoke_index, - nscen*(self.opt.nonant_length + 1 + 1) - ) - ## End if + def register_receive_fields(self): + spcomm = self.opt.spcomm + spcomms_cross_scenario_cut = spcomm.receive_field_spcomms[Field.CROSS_SCENARIO_CUT] + assert len(spcomms_cross_scenario_cut) == 1 + index, cls = spcomms_cross_scenario_cut[0] + assert cls is CrossScenarioCutSpoke + + self.cuts = spcomm.register_extension_recv_field( + Field.CROSS_SCENARIO_CUT, + index, + ) def sync_with_spokes(self): self.send_to_cross_cuts() diff --git a/mpisppy/extensions/extension.py b/mpisppy/extensions/extension.py index 3d6d7a26a..493ddcddd 100644 --- a/mpisppy/extensions/extension.py +++ b/mpisppy/extensions/extension.py @@ -21,14 +21,6 @@ class Extension: def __init__(self, spopt_object): self.opt = spopt_object - def register_send_fields(self): - ''' - Method called by the Hub SPCommunicator to get any fields that the extension - will make available to spokes. Use hub function `register_extension_send_field` - to register a field. - ''' - return - def setup_hub(self): ''' Method called when the Hub SPCommunicator is set up (if used) @@ -39,7 +31,15 @@ def setup_hub(self): ''' pass - def initialize_spoke_indices(self): + def register_send_fields(self): + ''' + Method called by the Hub SPCommunicator to get any fields that the extension + will make available to spokes. Use hub function `register_extension_send_field` + to register a field. + ''' + return + + def register_receive_fields(self): ''' Method called when the Hub SPCommunicator initializes its spoke indices @@ -176,9 +176,13 @@ def setup_hub(self): for lobject in self.extdict.values(): lobject.setup_hub() - def initialize_spoke_indices(self): + def register_send_fields(self): + for lobject in self.extdict.values(): + lobject.register_send_fields() + + def register_receive_fields(self): for lobject in self.extdict.values(): - lobject.initialize_spoke_indices() + lobject.register_receive_fields() def sync_with_spokes(self): for lobject in self.extdict.values(): diff --git a/mpisppy/extensions/reduced_costs_fixer.py b/mpisppy/extensions/reduced_costs_fixer.py index 46292c1b6..1d8f41e4f 100644 --- a/mpisppy/extensions/reduced_costs_fixer.py +++ b/mpisppy/extensions/reduced_costs_fixer.py @@ -85,7 +85,7 @@ def iter0_post_solver_creation(self): if self.opt.cylinder_rank == 0 and self.verbose: print("Fixing based on reduced costs prior to iteration 0!") if self.reduced_cost_buf.id() == 0: - while not self.opt.spcomm.hub_from_spoke(self.opt.spcomm.outerbound_receive_buffers[self.reduced_costs_spoke_index], self.reduced_costs_spoke_index): + while not self.opt.spcomm.hub_from_spoke(self.outer_bound_buf, self.reduced_costs_spoke_index, Field.EXPECTED_REDUCED_COST): continue self.sync_with_spokes(pre_iter0 = True) self.fix_fraction_target = self._fix_fraction_target_iter0 @@ -93,25 +93,23 @@ def iter0_post_solver_creation(self): def post_iter0_after_sync(self): self.fix_fraction_target = self._fix_fraction_target_iterK - def initialize_spoke_indices(self): - for (i, spoke) in enumerate(self.opt.spcomm.communicators): - if spoke["spcomm_class"] == ReducedCostsSpoke: - self.reduced_costs_spoke_index = i - ## End if - ## End for - - if hasattr(self, "reduced_costs_spoke_index"): - spcomm = self.opt.spcomm - self.reduced_cost_buf = spcomm.register_extension_recv_field( - Field.EXPECTED_REDUCED_COST, - self.reduced_costs_spoke_index, - self.opt.nonant_length, - ) - self.outer_bound_buf = spcomm.register_extension_recv_field( - Field.OBJECTIVE_OUTER_BOUND, - self.reduced_costs_spoke_index, - 1, - ) + def register_receive_fields(self): + spcomm = self.opt.spcomm + spcomms_expected_reduced_cost = spcomm.receive_field_spcomms[Field.EXPECTED_REDUCED_COST] + assert len(spcomms_expected_reduced_cost) == 1 + index, cls = spcomms_expected_reduced_cost[0] + assert cls is ReducedCostsSpoke + + self.reduced_costs_spoke_index = index + + self.reduced_cost_buf = spcomm.register_extension_recv_field( + Field.EXPECTED_REDUCED_COST, + self.reduced_costs_spoke_index, + ) + self.outer_bound_buf = spcomm.register_extension_recv_field( + Field.OBJECTIVE_OUTER_BOUND, + self.reduced_costs_spoke_index, + ) ## End if return @@ -126,6 +124,7 @@ def sync_with_spokes(self, pre_iter0 = False): # make sure we set the bound we compute prior to iteration 0 self.opt.spcomm.BestOuterBound = self.opt.spcomm.OuterBoundUpdate( self._best_outer_bound, + cls=ReducedCostsSpoke, idx=self.reduced_costs_spoke_index, ) if not pre_iter0 and self._use_rc_bt: From f5baab56fecda078c4f8fc858fac37a5a50e9d24 Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Fri, 7 Mar 2025 15:33:23 -0700 Subject: [PATCH 06/36] remove converger_spoke_types --- mpisppy/cylinders/spoke.py | 20 -------------------- mpisppy/extensions/phtracker.py | 6 +++--- 2 files changed, 3 insertions(+), 23 deletions(-) diff --git a/mpisppy/cylinders/spoke.py b/mpisppy/cylinders/spoke.py index 39987b1dd..958b5c84f 100644 --- a/mpisppy/cylinders/spoke.py +++ b/mpisppy/cylinders/spoke.py @@ -19,12 +19,6 @@ from mpisppy.cylinders.spwindow import Field -class ConvergerSpokeType(enum.Enum): - OUTER_BOUND = 1 - INNER_BOUND = 2 - W_GETTER = 3 - NONANT_GETTER = 4 - class Spoke(SPCommunicator): send_fields = (*SPCommunicator.send_fields, ) @@ -235,7 +229,6 @@ class InnerBoundSpoke(_BoundSpoke): receive_fields = (*_BoundSpoke.receive_fields, ) optional_receive_fields = (*_BoundSpoke.optional_receive_fields,) - converger_spoke_types = (ConvergerSpokeType.INNER_BOUND,) converger_spoke_char = 'I' def bound_type(self) -> Field: @@ -251,7 +244,6 @@ class OuterBoundSpoke(_BoundSpoke): receive_fields = (*_BoundSpoke.receive_fields, ) optional_receive_fields = (*_BoundSpoke.optional_receive_fields,) - converger_spoke_types = (ConvergerSpokeType.OUTER_BOUND,) converger_spoke_char = 'O' def bound_type(self) -> Field: @@ -294,10 +286,6 @@ class OuterBoundWSpoke(_BoundWSpoke): receive_fields = (*_BoundWSpoke.receive_fields, Field.DUALS) optional_receive_fields = (*_BoundWSpoke.optional_receive_fields,) - converger_spoke_types = ( - ConvergerSpokeType.OUTER_BOUND, - ConvergerSpokeType.W_GETTER, - ) converger_spoke_char = 'O' def bound_type(self) -> Field: @@ -341,10 +329,6 @@ class InnerBoundNonantSpoke(_BoundNonantSpoke): receive_fields = (*_BoundNonantSpoke.receive_fields, Field.NONANT) optional_receive_fields = (*_BoundNonantSpoke.optional_receive_fields,) - converger_spoke_types = ( - ConvergerSpokeType.INNER_BOUND, - ConvergerSpokeType.NONANT_GETTER, - ) converger_spoke_char = 'I' def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, options=None): @@ -390,10 +374,6 @@ class OuterBoundNonantSpoke(_BoundNonantSpoke): receive_fields = (*_BoundNonantSpoke.receive_fields, Field.NONANT) optional_receive_fields = (*_BoundNonantSpoke.optional_receive_fields,) - converger_spoke_types = ( - ConvergerSpokeType.OUTER_BOUND, - ConvergerSpokeType.NONANT_GETTER, - ) converger_spoke_char = 'A' # probably Lagrangian def bound_type(self) -> Field: diff --git a/mpisppy/extensions/phtracker.py b/mpisppy/extensions/phtracker.py index c95ab4624..9c800bbbe 100644 --- a/mpisppy/extensions/phtracker.py +++ b/mpisppy/extensions/phtracker.py @@ -15,7 +15,7 @@ import os import pandas as pd from mpisppy.extensions.extension import Extension -from mpisppy.cylinders.spoke import ConvergerSpokeType +from mpisppy.cylinders.spwindow import Field from mpisppy.cylinders.spoke import Spoke from mpisppy.cylinders.reduced_costs_spoke import ReducedCostsSpoke @@ -365,11 +365,11 @@ def add_gaps(self, final=False): # compute spoke gap relative to hub bound so that negative gap means # the spoke is worse than the hub - if ConvergerSpokeType.OUTER_BOUND in self.spcomm.converger_spoke_types: + if Field.OBJECTIVE_OUTER_BOUND in self.spcomm.send_fields: spoke_abs_gap = spoke_bound - hub_outer_bound \ if self.opt.is_minimizing else hub_outer_bound - spoke_bound spoke_rel_gap = self._compute_rel_gap(spoke_abs_gap, hub_outer_bound) - elif ConvergerSpokeType.INNER_BOUND in self.spcomm.converger_spoke_types: + elif Field.OBJECTIVE_INNER_BOUND in self.spcomm.send_fields: spoke_abs_gap = hub_inner_bound - spoke_bound \ if self.opt.is_minimizing else spoke_bound - hub_inner_bound spoke_rel_gap = self._compute_rel_gap(spoke_abs_gap, hub_inner_bound) From 3d7b47e76553b94858b018f71f028747b63d426e Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Fri, 7 Mar 2025 16:05:47 -0700 Subject: [PATCH 07/36] communicators exchange send data --- mpisppy/cylinders/spcommunicator.py | 18 ++++++++++++++++-- mpisppy/cylinders/spoke.py | 1 - mpisppy/extensions/cross_scen_extension.py | 8 +++----- mpisppy/extensions/reduced_costs_fixer.py | 7 +++---- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/mpisppy/cylinders/spcommunicator.py b/mpisppy/cylinders/spcommunicator.py index b0dc23d7c..b11e7f8e2 100644 --- a/mpisppy/cylinders/spcommunicator.py +++ b/mpisppy/cylinders/spcommunicator.py @@ -154,6 +154,8 @@ def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, communic self.opt.spcomm = self self.register_send_fields() + + self._exchange_send_fields() # TODO: here we can have a dynamic exchange of the send fields # so we can do error checking (all-to-all in send fields) self.register_receive_fields() @@ -189,6 +191,19 @@ def _build_window_spec(self) -> dict[Field, int]: ## End for return window_spec + def _exchange_send_fields(self) -> None: + """ Do an all-to-all so we know what the other communicators are sending """ + self.send_fields_by_rank = self.strata_comm.allgather(tuple(self.send_buffers.keys())) + + self.available_receive_fields = {} + for rank, fields in enumerate(self.send_fields_by_rank): + if rank == self.strata_rank: + continue + for f in fields: + if f not in self.available_receive_fields: + self.available_receive_fields[f] = [] + self.available_receive_fields[f].append(rank) + def register_recv_field(self, field: Field, origin: int, length: int = -1) -> RecvArray: key = self._make_key(field, origin) if length == -1: @@ -271,13 +286,12 @@ def register_send_fields(self) -> None: self.register_send_field(field) def register_receive_fields(self) -> None: - # TODO: make receive_fields dynamic? Make the callback handles this??? for field in itertools.chain(self.receive_fields, self.optional_receive_fields): self.receive_field_spcomms[field] = [] for strata_rank, comm in enumerate(self.communicators): if strata_rank == self.strata_rank: continue cls = comm["spcomm_class"] - if field in cls.send_fields: + if field in self.send_fields_by_rank[strata_rank]: buff = self.register_recv_field(field, strata_rank) self.receive_field_spcomms[field].append((strata_rank, cls, buff)) diff --git a/mpisppy/cylinders/spoke.py b/mpisppy/cylinders/spoke.py index 958b5c84f..95a8e9bac 100644 --- a/mpisppy/cylinders/spoke.py +++ b/mpisppy/cylinders/spoke.py @@ -9,7 +9,6 @@ import numpy as np import abc -import enum import time import os import math diff --git a/mpisppy/extensions/cross_scen_extension.py b/mpisppy/extensions/cross_scen_extension.py index 4f6955e50..49f21f461 100644 --- a/mpisppy/extensions/cross_scen_extension.py +++ b/mpisppy/extensions/cross_scen_extension.py @@ -10,7 +10,6 @@ from mpisppy.utils.sputils import find_active_objective from pyomo.repn.standard_repn import generate_standard_repn from pyomo.core.expr.numeric_expr import LinearExpression -from mpisppy.cylinders.cross_scen_spoke import CrossScenarioCutSpoke from mpisppy.cylinders.spwindow import Field import pyomo.environ as pyo @@ -281,10 +280,9 @@ def setup_hub(self): def register_receive_fields(self): spcomm = self.opt.spcomm - spcomms_cross_scenario_cut = spcomm.receive_field_spcomms[Field.CROSS_SCENARIO_CUT] - assert len(spcomms_cross_scenario_cut) == 1 - index, cls = spcomms_cross_scenario_cut[0] - assert cls is CrossScenarioCutSpoke + cross_scenario_cut_ranks = spcomm.available_receive_fields[Field.CROSS_SCENARIO_CUT] + assert len(cross_scenario_cut_ranks) == 1 + index = cross_scenario_cut_ranks[0] self.cuts = spcomm.register_extension_recv_field( Field.CROSS_SCENARIO_CUT, diff --git a/mpisppy/extensions/reduced_costs_fixer.py b/mpisppy/extensions/reduced_costs_fixer.py index 1d8f41e4f..73c449a16 100644 --- a/mpisppy/extensions/reduced_costs_fixer.py +++ b/mpisppy/extensions/reduced_costs_fixer.py @@ -95,10 +95,9 @@ def post_iter0_after_sync(self): def register_receive_fields(self): spcomm = self.opt.spcomm - spcomms_expected_reduced_cost = spcomm.receive_field_spcomms[Field.EXPECTED_REDUCED_COST] - assert len(spcomms_expected_reduced_cost) == 1 - index, cls = spcomms_expected_reduced_cost[0] - assert cls is ReducedCostsSpoke + expected_reduced_cost_ranks = spcomm.available_receive_fields[Field.EXPECTED_REDUCED_COST] + assert len(expected_reduced_cost_ranks) == 1 + index = expected_reduced_cost_ranks[0] self.reduced_costs_spoke_index = index From d846615ecbf051344feb21800a0d718833266848 Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Fri, 7 Mar 2025 16:09:03 -0700 Subject: [PATCH 08/36] remove optional_recieve_fields --- mpisppy/cylinders/cross_scen_spoke.py | 1 - mpisppy/cylinders/hub.py | 5 +---- mpisppy/cylinders/reduced_costs_spoke.py | 1 - mpisppy/cylinders/spcommunicator.py | 7 ++----- mpisppy/cylinders/spoke.py | 7 ------- 5 files changed, 3 insertions(+), 18 deletions(-) diff --git a/mpisppy/cylinders/cross_scen_spoke.py b/mpisppy/cylinders/cross_scen_spoke.py index 0347d332f..33270b96e 100644 --- a/mpisppy/cylinders/cross_scen_spoke.py +++ b/mpisppy/cylinders/cross_scen_spoke.py @@ -20,7 +20,6 @@ class CrossScenarioCutSpoke(Spoke): send_fields = (*Spoke.send_fields, Field.CROSS_SCENARIO_CUT) receive_fields = (*Spoke.receive_fields, Field.NONANT, Field.CROSS_SCENARIO_COST) - optional_receive_fields = (*Spoke.optional_receive_fields, ) def register_send_fields(self) -> None: diff --git a/mpisppy/cylinders/hub.py b/mpisppy/cylinders/hub.py index 866347754..03a0c6230 100644 --- a/mpisppy/cylinders/hub.py +++ b/mpisppy/cylinders/hub.py @@ -30,8 +30,7 @@ class Hub(SPCommunicator): send_fields = (*SPCommunicator.send_fields, Field.SHUTDOWN, Field.BEST_OBJECTIVE_BOUNDS,) - receive_fields = (*SPCommunicator.receive_fields, ) - optional_receive_fields = (*SPCommunicator.optional_receive_fields, Field.OBJECTIVE_INNER_BOUND, Field.OBJECTIVE_OUTER_BOUND, ) + receive_fields = (*SPCommunicator.receive_fields, Field.OBJECTIVE_INNER_BOUND, Field.OBJECTIVE_OUTER_BOUND, ) _hub_algo_best_bound_provider = False @@ -439,7 +438,6 @@ class PHHub(Hub): send_fields = (*Hub.send_fields, Field.NONANT, Field.DUALS) receive_fields = (*Hub.receive_fields,) - optional_receive_fields = (*Hub.optional_receive_fields,) def setup_hub(self): """ Must be called after make_windows(), so that @@ -571,7 +569,6 @@ class LShapedHub(Hub): send_fields = (*Hub.send_fields, Field.NONANT,) receive_fields = (*Hub.receive_fields,) - optional_receive_fields = (*Hub.optional_receive_fields,) def setup_hub(self): """ Must be called after make_windows(), so that diff --git a/mpisppy/cylinders/reduced_costs_spoke.py b/mpisppy/cylinders/reduced_costs_spoke.py index f2a643db7..7d95811df 100644 --- a/mpisppy/cylinders/reduced_costs_spoke.py +++ b/mpisppy/cylinders/reduced_costs_spoke.py @@ -17,7 +17,6 @@ class ReducedCostsSpoke(LagrangianOuterBound): send_fields = (*LagrangianOuterBound.send_fields, Field.EXPECTED_REDUCED_COST, Field.SCENARIO_REDUCED_COST ,) receive_fields = (*LagrangianOuterBound.receive_fields,) - optional_receive_fields = (*LagrangianOuterBound.optional_receive_fields,) converger_spoke_char = 'R' diff --git a/mpisppy/cylinders/spcommunicator.py b/mpisppy/cylinders/spcommunicator.py index b11e7f8e2..d175e5a1b 100644 --- a/mpisppy/cylinders/spcommunicator.py +++ b/mpisppy/cylinders/spcommunicator.py @@ -111,13 +111,10 @@ def _pull_id(self) -> int: class SPCommunicator: """ Base class for communicator objects. Each communicator object should register as a class attribute what Field attributes it provides in its buffer - or expects to receive from another SPCommunicator object. Additionally, optional - receive attributes can be specified, for which the SPCommunicator will still work - but can read additional attributes. + or expects to receive from another SPCommunicator object. """ send_fields = () receive_fields = () - optional_receive_fields = () def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, communicators, options=None): # flag for if the windows have been constructed @@ -286,7 +283,7 @@ def register_send_fields(self) -> None: self.register_send_field(field) def register_receive_fields(self) -> None: - for field in itertools.chain(self.receive_fields, self.optional_receive_fields): + for field in self.receive_fields: self.receive_field_spcomms[field] = [] for strata_rank, comm in enumerate(self.communicators): if strata_rank == self.strata_rank: diff --git a/mpisppy/cylinders/spoke.py b/mpisppy/cylinders/spoke.py index 95a8e9bac..05eacd0b7 100644 --- a/mpisppy/cylinders/spoke.py +++ b/mpisppy/cylinders/spoke.py @@ -22,7 +22,6 @@ class Spoke(SPCommunicator): send_fields = (*SPCommunicator.send_fields, ) receive_fields = (*SPCommunicator.receive_fields, Field.SHUTDOWN, ) - optional_receive_fields = (*SPCommunicator.optional_receive_fields, ) def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, communicators, options=None): @@ -135,7 +134,6 @@ class _BoundSpoke(Spoke): send_fields = (*Spoke.send_fields, ) receive_fields = (*Spoke.receive_fields, Field.BEST_OBJECTIVE_BOUNDS) - optional_receive_fields = (*Spoke.optional_receive_fields, ) def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, options=None): super().__init__(spbase_object, fullcomm, strata_comm, cylinder_comm, options) @@ -226,7 +224,6 @@ class InnerBoundSpoke(_BoundSpoke): send_fields = (*_BoundSpoke.send_fields, Field.OBJECTIVE_INNER_BOUND, ) receive_fields = (*_BoundSpoke.receive_fields, ) - optional_receive_fields = (*_BoundSpoke.optional_receive_fields,) converger_spoke_char = 'I' @@ -241,7 +238,6 @@ class OuterBoundSpoke(_BoundSpoke): send_fields = (*_BoundSpoke.send_fields, Field.OBJECTIVE_OUTER_BOUND, ) receive_fields = (*_BoundSpoke.receive_fields, ) - optional_receive_fields = (*_BoundSpoke.optional_receive_fields,) converger_spoke_char = 'O' @@ -283,7 +279,6 @@ class OuterBoundWSpoke(_BoundWSpoke): send_fields = (*_BoundWSpoke.send_fields, Field.OBJECTIVE_OUTER_BOUND, ) receive_fields = (*_BoundWSpoke.receive_fields, Field.DUALS) - optional_receive_fields = (*_BoundWSpoke.optional_receive_fields,) converger_spoke_char = 'O' @@ -326,7 +321,6 @@ class InnerBoundNonantSpoke(_BoundNonantSpoke): send_fields = (*_BoundNonantSpoke.send_fields, Field.OBJECTIVE_INNER_BOUND, ) receive_fields = (*_BoundNonantSpoke.receive_fields, Field.NONANT) - optional_receive_fields = (*_BoundNonantSpoke.optional_receive_fields,) converger_spoke_char = 'I' @@ -371,7 +365,6 @@ class OuterBoundNonantSpoke(_BoundNonantSpoke): send_fields = (*_BoundNonantSpoke.send_fields, Field.OBJECTIVE_OUTER_BOUND, ) receive_fields = (*_BoundNonantSpoke.receive_fields, Field.NONANT) - optional_receive_fields = (*_BoundNonantSpoke.optional_receive_fields,) converger_spoke_char = 'A' # probably Lagrangian From c547777a479ff1f6dd509f95988249aabf085465 Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Fri, 7 Mar 2025 16:17:46 -0700 Subject: [PATCH 09/36] fix API for register_extension_recv_field --- mpisppy/cylinders/hub.py | 2 +- mpisppy/cylinders/spcommunicator.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mpisppy/cylinders/hub.py b/mpisppy/cylinders/hub.py index 03a0c6230..766059faa 100644 --- a/mpisppy/cylinders/hub.py +++ b/mpisppy/cylinders/hub.py @@ -82,7 +82,7 @@ def main(self): pass - def register_extension_recv_field(self, field: Field, strata_rank: int, buf_len: int) -> RecvArray: + def register_extension_recv_field(self, field: Field, strata_rank: int, buf_len: int = -1) -> RecvArray: """ Register an extensions interest in the given field from the given spoke. The hub is then responsible for updating this field into a local buffer prior to the call diff --git a/mpisppy/cylinders/spcommunicator.py b/mpisppy/cylinders/spcommunicator.py index d175e5a1b..646524fba 100644 --- a/mpisppy/cylinders/spcommunicator.py +++ b/mpisppy/cylinders/spcommunicator.py @@ -43,6 +43,7 @@ class FieldArray: """ def __init__(self, length: int): + self._length = length self._array = communicator_array(length) self._id = 0 return From 4d3b66884d01896daf06f5d1d863d23dabefa210 Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Fri, 7 Mar 2025 16:42:45 -0700 Subject: [PATCH 10/36] unifying register_receive_fields --- mpisppy/cylinders/cross_scen_spoke.py | 7 +++++-- mpisppy/cylinders/hub.py | 6 ++++-- mpisppy/cylinders/spcommunicator.py | 23 +++++++++++++++++++---- mpisppy/cylinders/spoke.py | 24 +++++------------------- 4 files changed, 33 insertions(+), 27 deletions(-) diff --git a/mpisppy/cylinders/cross_scen_spoke.py b/mpisppy/cylinders/cross_scen_spoke.py index 33270b96e..d6297da2f 100644 --- a/mpisppy/cylinders/cross_scen_spoke.py +++ b/mpisppy/cylinders/cross_scen_spoke.py @@ -41,13 +41,16 @@ def register_send_fields(self) -> None: self.all_nonant_len = vbuflen self.all_eta_len = nscen*local_scen_count - self.all_nonants = self.register_recv_field(Field.NONANT, 0, vbuflen) - self.all_etas = self.register_recv_field(Field.CROSS_SCENARIO_COST, 0, nscen * nscen) self.all_coefs = self.send_buffers[Field.CROSS_SCENARIO_CUT] return + def register_receive_fields(self): + super().register_receive_fields() + self.all_nonants = self.register_recv_field(Field.NONANT, 0) + self.all_etas = self.register_recv_field(Field.CROSS_SCENARIO_COST, 0) + def prep_cs_cuts(self): # create a map scenario -> index, this index is used for various lists containing scenario dependent info. self.scenario_to_index = { scen : indx for indx, scen in enumerate(self.opt.all_scenario_names) } diff --git a/mpisppy/cylinders/hub.py b/mpisppy/cylinders/hub.py index 766059faa..e6bae0a17 100644 --- a/mpisppy/cylinders/hub.py +++ b/mpisppy/cylinders/hub.py @@ -35,7 +35,11 @@ class Hub(SPCommunicator): _hub_algo_best_bound_provider = False def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, communicators, options=None): + # The extensions will be registered in SPCommunicator.__init__ + self.extension_recv = set() + super().__init__(spbase_object, fullcomm, strata_comm, cylinder_comm, communicators, options=options) + logger.debug(f"Built the hub object on global rank {fullcomm.Get_rank()}") # for logging self.print_init = True @@ -47,8 +51,6 @@ def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, communic self.stalled_iter_cnt = 0 self.last_gap = float('inf') # abs_gap tracker - self.extension_recv = set() - self.initialize_bound_values() return diff --git a/mpisppy/cylinders/spcommunicator.py b/mpisppy/cylinders/spcommunicator.py index 646524fba..986ed55d6 100644 --- a/mpisppy/cylinders/spcommunicator.py +++ b/mpisppy/cylinders/spcommunicator.py @@ -22,7 +22,6 @@ import numpy as np import abc import time -import itertools from mpisppy.cylinders.spwindow import Field, FieldLengths, SPWindow @@ -191,18 +190,26 @@ def _build_window_spec(self) -> dict[Field, int]: def _exchange_send_fields(self) -> None: """ Do an all-to-all so we know what the other communicators are sending """ - self.send_fields_by_rank = self.strata_comm.allgather(tuple(self.send_buffers.keys())) + send_buffers = tuple((k, buff._length) for k, buff in self.send_buffers.items()) + self.send_fields_lengths_by_rank = self.strata_comm.allgather(send_buffers) + + self.send_fields_by_rank = {} self.available_receive_fields = {} - for rank, fields in enumerate(self.send_fields_by_rank): + for rank, fields_lengths in enumerate(self.send_fields_lengths_by_rank): if rank == self.strata_rank: continue - for f in fields: + self.send_fields_by_rank[rank] = [] + for f, length in fields_lengths: if f not in self.available_receive_fields: self.available_receive_fields[f] = [] self.available_receive_fields[f].append(rank) + self.send_fields_by_rank[rank].append(f) + + # print(f"{self.__class__.__name__}: {self.available_receive_fields=}") def register_recv_field(self, field: Field, origin: int, length: int = -1) -> RecvArray: + # print(f"{self.__class__.__name__}.register_recv_field, {field=}, {origin=}") key = self._make_key(field, origin) if length == -1: length = self._field_lengths[field] @@ -210,6 +217,13 @@ def register_recv_field(self, field: Field, origin: int, length: int = -1) -> Re my_fa = self.receive_buffers[key] assert(length + 1 == np.size(my_fa.array())) else: + available_fields_from_origin = self.send_fields_lengths_by_rank[origin] + for _field, _length in available_fields_from_origin: + if field == _field: + assert length == _length + break + else: # couldn't find field! + raise RuntimeError(f"Couldn't find {field=} from {origin=}") my_fa = RecvArray(length) self.receive_buffers[key] = my_fa ## End if @@ -284,6 +298,7 @@ def register_send_fields(self) -> None: self.register_send_field(field) def register_receive_fields(self) -> None: + # print(f"{self.__class__.__name__}: {self.receive_fields=}") for field in self.receive_fields: self.receive_field_spcomms[field] = [] for strata_rank, comm in enumerate(self.communicators): diff --git a/mpisppy/cylinders/spoke.py b/mpisppy/cylinders/spoke.py index 05eacd0b7..86f640edd 100644 --- a/mpisppy/cylinders/spoke.py +++ b/mpisppy/cylinders/spoke.py @@ -29,10 +29,6 @@ def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, communic self.last_call_to_got_kill_signal = time.time() - # All spokes need the SHUTDOWN field to know when to terminate. Just - # register that here. - self.shutdown = self.register_recv_field(Field.SHUTDOWN, 0, 1) - return def spoke_to_hub(self, buf: SendArray, field: Field): @@ -95,7 +91,7 @@ def _spoke_from_hub(self, def _got_kill_signal(self): shutdown_buf = self.receive_buffers[self._make_key(Field.SHUTDOWN, 0)] if shutdown_buf.is_new(): - shutdown = (self.shutdown[0] == 1.0) + shutdown = (shutdown_buf[0] == 1.0) else: shutdown = False ## End if @@ -158,9 +154,12 @@ def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, options= def register_send_fields(self) -> None: super().register_send_fields() self._bound = self.send_buffers[self.bound_type()] - self._hub_bounds = self.register_recv_field(Field.BEST_OBJECTIVE_BOUNDS, 0, 2) return + def register_receive_fields(self) -> None: + super().register_receive_fields() + self._hub_bounds = self.register_recv_field(Field.BEST_OBJECTIVE_BOUNDS, 0, 2) + @abc.abstractmethod def bound_type(self) -> Field: pass @@ -203,19 +202,6 @@ def nonant_len_type(self) -> Field: # TODO: Make this a static method? pass - def register_send_fields(self) -> None: - - super().register_send_fields() - - vbuflen = 0 - for s in self.opt.local_scenarios.values(): - vbuflen += len(s._mpisppy_data.nonant_indices) - ## End for - - self.register_recv_field(self.nonant_len_type(), 0, vbuflen) - - return - class InnerBoundSpoke(_BoundSpoke): """ For Spokes that provide an inner bound through self.bound to the From f2ff3313c9f9962b4350ca43d5f9e58464ecbe39 Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Mon, 10 Mar 2025 10:36:03 -0600 Subject: [PATCH 11/36] better validation; code cleanup --- mpisppy/cylinders/hub.py | 16 ----- mpisppy/cylinders/spcommunicator.py | 75 ++++++++++------------ mpisppy/cylinders/spoke.py | 4 +- mpisppy/cylinders/spwindow.py | 1 - mpisppy/extensions/cross_scen_extension.py | 2 +- mpisppy/extensions/reduced_costs_fixer.py | 2 +- mpisppy/spin_the_wheel.py | 6 +- 7 files changed, 40 insertions(+), 66 deletions(-) diff --git a/mpisppy/cylinders/hub.py b/mpisppy/cylinders/hub.py index e6bae0a17..db318f228 100644 --- a/mpisppy/cylinders/hub.py +++ b/mpisppy/cylinders/hub.py @@ -442,14 +442,6 @@ class PHHub(Hub): receive_fields = (*Hub.receive_fields,) def setup_hub(self): - """ Must be called after make_windows(), so that - the hub knows the sizes of all the spokes windows - """ - if not self._windows_constructed: - raise RuntimeError( - "Cannot call setup_hub before memory windows are constructed" - ) - ## Generate some warnings if nothing is giving bounds if not self.receive_field_spcomms[Field.OBJECTIVE_OUTER_BOUND]: logger.warn( @@ -573,14 +565,6 @@ class LShapedHub(Hub): receive_fields = (*Hub.receive_fields,) def setup_hub(self): - """ Must be called after make_windows(), so that - the hub knows the sizes of all the spokes windows - """ - if not self._windows_constructed: - raise RuntimeError( - "Cannot call setup_hub before memory windows are constructed" - ) - ## Generate some warnings if nothing is giving bounds if not self.receive_field_spcomms[Field.OBJECTIVE_INNER_BOUND]: logger.warn( diff --git a/mpisppy/cylinders/spcommunicator.py b/mpisppy/cylinders/spcommunicator.py index 986ed55d6..e81a35d4f 100644 --- a/mpisppy/cylinders/spcommunicator.py +++ b/mpisppy/cylinders/spcommunicator.py @@ -42,7 +42,6 @@ class FieldArray: """ def __init__(self, length: int): - self._length = length self._array = communicator_array(length) self._id = 0 return @@ -117,8 +116,6 @@ class SPCommunicator: receive_fields = () def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, communicators, options=None): - # flag for if the windows have been constructed - self._windows_constructed = False self.fullcomm = fullcomm self.strata_comm = strata_comm self.cylinder_comm = cylinder_comm @@ -152,9 +149,9 @@ def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, communic self.register_send_fields() - self._exchange_send_fields() - # TODO: here we can have a dynamic exchange of the send fields - # so we can do error checking (all-to-all in send fields) + self._make_windows() + self._create_field_rank_mappings() + self.register_receive_fields() # TODO: check that we have something in receive_field_spcomms?? @@ -188,25 +185,37 @@ def _build_window_spec(self) -> dict[Field, int]: ## End for return window_spec - def _exchange_send_fields(self) -> None: - """ Do an all-to-all so we know what the other communicators are sending """ - send_buffers = tuple((k, buff._length) for k, buff in self.send_buffers.items()) - self.send_fields_lengths_by_rank = self.strata_comm.allgather(send_buffers) - - self.send_fields_by_rank = {} + def _create_field_rank_mappings(self) -> None: + self.fields_to_ranks = {} + self.ranks_to_fields = {} - self.available_receive_fields = {} - for rank, fields_lengths in enumerate(self.send_fields_lengths_by_rank): + for rank, buffer_layout in enumerate(self.window.strata_buffer_layouts): if rank == self.strata_rank: continue - self.send_fields_by_rank[rank] = [] - for f, length in fields_lengths: - if f not in self.available_receive_fields: - self.available_receive_fields[f] = [] - self.available_receive_fields[f].append(rank) - self.send_fields_by_rank[rank].append(f) - - # print(f"{self.__class__.__name__}: {self.available_receive_fields=}") + self.ranks_to_fields[rank] = [] + for field in buffer_layout: + if field not in self.fields_to_ranks: + self.fields_to_ranks[field] = [] + self.fields_to_ranks[field].append(rank) + self.ranks_to_fields[rank].append(field) + + # print(f"{self.__class__.__name__}: {self.fields_to_ranks=}, {self.ranks_to_fields=}") + + def _validate_recv_field(self, field: Field, origin: int, length: int): + remote_buffer_layout = self.window.strata_buffer_layouts[origin] + if field not in remote_buffer_layout: + raise RuntimeError(f"{self.__class__.__name__} on local {self.strata_rank=} " + f"could not find {field=} on remote rank {origin} with " + f"class {self.communicators[origin]['spcomm_class']}." + ) + _, remote_length = remote_buffer_layout[field] + if (length + 1) != remote_length: + raise RuntimeError(f"{self.__class__.__name__} on local {self.strata_rank=} " + f"{field=} has length {length} on local " + f"{self.strata_rank=} and length {remote_length} " + f"on remote rank {origin} with class " + f"{self.communicators[origin]['spcomm_class']}." + ) def register_recv_field(self, field: Field, origin: int, length: int = -1) -> RecvArray: # print(f"{self.__class__.__name__}.register_recv_field, {field=}, {origin=}") @@ -217,13 +226,7 @@ def register_recv_field(self, field: Field, origin: int, length: int = -1) -> Re my_fa = self.receive_buffers[key] assert(length + 1 == np.size(my_fa.array())) else: - available_fields_from_origin = self.send_fields_lengths_by_rank[origin] - for _field, _length in available_fields_from_origin: - if field == _field: - assert length == _length - break - else: # couldn't find field! - raise RuntimeError(f"Couldn't find {field=} from {origin=}") + self._validate_recv_field(field, origin, length) my_fa = RecvArray(length) self.receive_buffers[key] = my_fa ## End if @@ -276,20 +279,10 @@ def hub_finalize(self): def allreduce_or(self, val): return self.opt.allreduce_or(val) - def free_windows(self): - """ - """ - if self._windows_constructed: - self.window.free() - self._windows_constructed = False - - def make_windows(self) -> None: - if self._windows_constructed: - return + def _make_windows(self) -> None: window_spec = self._build_window_spec() self.window = SPWindow(window_spec, self.strata_comm) - self._windows_constructed = True return @@ -305,6 +298,6 @@ def register_receive_fields(self) -> None: if strata_rank == self.strata_rank: continue cls = comm["spcomm_class"] - if field in self.send_fields_by_rank[strata_rank]: + if field in self.ranks_to_fields[strata_rank]: buff = self.register_recv_field(field, strata_rank) self.receive_field_spcomms[field].append((strata_rank, cls, buff)) diff --git a/mpisppy/cylinders/spoke.py b/mpisppy/cylinders/spoke.py index 86f640edd..191f3c9b3 100644 --- a/mpisppy/cylinders/spoke.py +++ b/mpisppy/cylinders/spoke.py @@ -101,7 +101,7 @@ def got_kill_signal(self): """ Spoke should call this method at least every iteration to see if the Hub terminated """ - self.updatereceive_buffers() + self.update_receive_buffers() return self._got_kill_signal() @abc.abstractmethod @@ -114,7 +114,7 @@ def main(self): """ pass - def updatereceive_buffers(self): + def update_receive_buffers(self): for (key, recv_buf) in self.receive_buffers.items(): field, rank = self._split_key(key) # The below code will need to be updated for spoke to spoke communication diff --git a/mpisppy/cylinders/spwindow.py b/mpisppy/cylinders/spwindow.py index 1416329bf..a40a59c44 100644 --- a/mpisppy/cylinders/spwindow.py +++ b/mpisppy/cylinders/spwindow.py @@ -125,7 +125,6 @@ def __init__(self, my_fields: dict, strata_comm: MPI.Comm, field_order=None): return - def free(self): if self.window_constructed: diff --git a/mpisppy/extensions/cross_scen_extension.py b/mpisppy/extensions/cross_scen_extension.py index 49f21f461..480f9b6e5 100644 --- a/mpisppy/extensions/cross_scen_extension.py +++ b/mpisppy/extensions/cross_scen_extension.py @@ -280,7 +280,7 @@ def setup_hub(self): def register_receive_fields(self): spcomm = self.opt.spcomm - cross_scenario_cut_ranks = spcomm.available_receive_fields[Field.CROSS_SCENARIO_CUT] + cross_scenario_cut_ranks = spcomm.fields_to_ranks[Field.CROSS_SCENARIO_CUT] assert len(cross_scenario_cut_ranks) == 1 index = cross_scenario_cut_ranks[0] diff --git a/mpisppy/extensions/reduced_costs_fixer.py b/mpisppy/extensions/reduced_costs_fixer.py index 73c449a16..c31726b45 100644 --- a/mpisppy/extensions/reduced_costs_fixer.py +++ b/mpisppy/extensions/reduced_costs_fixer.py @@ -95,7 +95,7 @@ def post_iter0_after_sync(self): def register_receive_fields(self): spcomm = self.opt.spcomm - expected_reduced_cost_ranks = spcomm.available_receive_fields[Field.EXPECTED_REDUCED_COST] + expected_reduced_cost_ranks = spcomm.fields_to_ranks[Field.EXPECTED_REDUCED_COST] assert len(expected_reduced_cost_ranks) == 1 index = expected_reduced_cost_ranks[0] diff --git a/mpisppy/spin_the_wheel.py b/mpisppy/spin_the_wheel.py index 33ce62a90..24915db4b 100644 --- a/mpisppy/spin_the_wheel.py +++ b/mpisppy/spin_the_wheel.py @@ -127,8 +127,7 @@ def run(self, comm_world=None): spcomm = sp_class(opt, fullcomm, strata_comm, cylinder_comm, communicator_list, **sp_kwargs) - # Create the windows, run main(), destroy the windows - spcomm.make_windows() + # Run main() if strata_rank == 0: spcomm.setup_hub() @@ -151,9 +150,8 @@ def run(self, comm_world=None): spcomm.hub_finalize() fullcomm.Barrier() + global_toc("Finalize Complete") - spcomm.free_windows() - global_toc("Windows freed") self.spcomm = spcomm self.spcomm_dict = spcomm_dict From 36b1a0ec0101523b882808143b0050b45aa47c78 Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Mon, 10 Mar 2025 13:10:23 -0600 Subject: [PATCH 12/36] remove *_to_* methods; nothing was really pushed anyways... --- mpisppy/cylinders/cross_scen_spoke.py | 4 +-- mpisppy/cylinders/hub.py | 46 ++++----------------------- mpisppy/cylinders/spcommunicator.py | 28 ++++++++++------ mpisppy/cylinders/spoke.py | 17 +--------- 4 files changed, 27 insertions(+), 68 deletions(-) diff --git a/mpisppy/cylinders/cross_scen_spoke.py b/mpisppy/cylinders/cross_scen_spoke.py index d6297da2f..2fe006b09 100644 --- a/mpisppy/cylinders/cross_scen_spoke.py +++ b/mpisppy/cylinders/cross_scen_spoke.py @@ -137,7 +137,7 @@ def make_eta_lb_cut(self): ## this cut -- [ LB, -1, *0s ], i.e., -1*\eta + LB <= 0 all_coefs[row_len*idx] = self._eta_lb_array[idx] all_coefs[row_len*idx+1] = -1 - self.spoke_to_hub(all_coefs, Field.CROSS_SCENARIO_CUT) + self.put_send_buffer(all_coefs, Field.CROSS_SCENARIO_CUT) def make_cut(self): @@ -295,7 +295,7 @@ def make_cut(self): all_coefs[row_len*idx:row_len*(idx+1)] = coef_dict[k] elif feas_cuts: all_coefs[row_len*idx:row_len*(idx+1)] = feas_cuts.pop() - self.spoke_to_hub(all_coefs, Field.CROSS_SCENARIO_CUT) + self.put_send_buffer(all_coefs, Field.CROSS_SCENARIO_CUT) def main(self): # call main cut generation routine diff --git a/mpisppy/cylinders/hub.py b/mpisppy/cylinders/hub.py index db318f228..50d0e77e6 100644 --- a/mpisppy/cylinders/hub.py +++ b/mpisppy/cylinders/hub.py @@ -116,7 +116,7 @@ def extension_send_field(self, field: Field, buf: SendArray): Send the data in the SendArray `buf` which stores the Field `field`. This will make the data available to the spokes in this strata. """ - return self.hub_to_spoke(buf, field) + return self.put_send_buffer(buf, field) def sync_extension_fields(self): """ @@ -307,7 +307,7 @@ def send_boundsout(self): my_bounds = self.send_buffers[Field.BEST_OBJECTIVE_BOUNDS] self._populate_boundsout_cache(my_bounds.array()) logging.debug("hub is sending bounds={}".format(my_bounds)) - self.hub_to_spoke(my_bounds, Field.BEST_OBJECTIVE_BOUNDS) + self.put_send_buffer(my_bounds, Field.BEST_OBJECTIVE_BOUNDS) return def register_receive_fields(self): @@ -336,38 +336,6 @@ def register_send_fields(self): return - - def hub_to_spoke(self, buf: SendArray, field: Field): - """ Put the specified values into the specified locally-owned buffer - for the spoke to pick up. - - Notes: - This automatically updates handles the write id. - """ - return self._hub_to_spoke(buf.array(), field, buf._next_write_id()) - - - def _hub_to_spoke(self, values: np.typing.NDArray, field: Field, write_id: int): - """ Put the specified values into the specified locally-owned buffer - for the spoke to pick up. - - Notes: - This automatically does the -1 indexing - - This assumes that values contains a slot at the end for the - write_id - """ - - if not isinstance(self.opt, APH): - self.cylinder_comm.Barrier() - ## End if - - values[-1] = write_id - self.window.put(values, field) - - return - - def hub_from_spoke(self, buf: RecvArray, spoke_num: int, @@ -432,7 +400,7 @@ def send_terminate(self): processes (don't need to call them one at a time). """ self.send_buffers[Field.SHUTDOWN][0] = 1.0 - self.hub_to_spoke(self.send_buffers[Field.SHUTDOWN], Field.SHUTDOWN) + self.put_send_buffer(self.send_buffers[Field.SHUTDOWN], Field.SHUTDOWN) return @@ -541,8 +509,7 @@ def send_nonants(self): ci += 1 logging.debug("hub is sending X nonants={}".format(nonant_send_buffer)) - # self.hub_to_spoke(nonant_send_buffer.array(), Field.NONANT, nonant_send_buffer.next_write_id()) - self.hub_to_spoke(nonant_send_buffer, Field.NONANT) + self.put_send_buffer(nonant_send_buffer, Field.NONANT) return @@ -554,8 +521,7 @@ def send_ws(self): self.opt._populate_W_cache(my_ws.array(), padding=1) logging.debug("hub is sending Ws={}".format(my_ws.array())) - # self.hub_to_spoke(my_ws.array(), Field.DUALS, my_ws.next_write_id()) - self.hub_to_spoke(my_ws, Field.DUALS) + self.put_send_buffer(my_ws, Field.DUALS) return @@ -623,7 +589,7 @@ def send_nonants(self): ci += 1 logging.debug("hub is sending X nonants={}".format(nonant_send_buffer)) - self.hub_to_spoke(nonant_send_buffer, Field.NONANT) + self.put_send_buffer(nonant_send_buffer, Field.NONANT) return diff --git a/mpisppy/cylinders/spcommunicator.py b/mpisppy/cylinders/spcommunicator.py index e81a35d4f..d458bf161 100644 --- a/mpisppy/cylinders/spcommunicator.py +++ b/mpisppy/cylinders/spcommunicator.py @@ -82,9 +82,11 @@ def __setitem__(self, key, value): def _next_write_id(self) -> int: """ - Updates the internal id field to the next write id and returns that id + Updates the internal id field to the next write id, uses that id in the field data, + and returns that id """ self._id += 1 + self._array[-1] = self._id return self._id @@ -154,8 +156,6 @@ def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, communic self.register_receive_fields() - # TODO: check that we have something in receive_field_spcomms?? - return def _make_key(self, field: Field, origin: int): @@ -236,13 +236,6 @@ def register_send_field(self, field: Field, length: int = -1) -> SendArray: assert field not in self.send_buffers, "Field {} is already registered".format(field) if length == -1: length = self._field_lengths[field] - # if field in self.send_buffers: - # my_fa = self.send_buffers[field] - # assert(length + 1 == np.size(my_fa.array())) - # else: - # my_fa = SendArray(length) - # self.send_buffers[field] = my_fa - # ## End if else my_fa = SendArray(length) self.send_buffers[field] = my_fa return my_fa @@ -293,6 +286,10 @@ def register_send_fields(self) -> None: def register_receive_fields(self) -> None: # print(f"{self.__class__.__name__}: {self.receive_fields=}") for field in self.receive_fields: + # NOTE: If this list is empty after this method, it is up + # to the caller to raise an error. Sometimes optional + # receive fields are perfectly sensible, and sometimes + # they are nonsensical. self.receive_field_spcomms[field] = [] for strata_rank, comm in enumerate(self.communicators): if strata_rank == self.strata_rank: @@ -301,3 +298,14 @@ def register_receive_fields(self) -> None: if field in self.ranks_to_fields[strata_rank]: buff = self.register_recv_field(field, strata_rank) self.receive_field_spcomms[field].append((strata_rank, cls, buff)) + + def put_send_buffer(self, buf: SendArray, field: Field): + """ Put the specified values into the specified locally-owned buffer + for the another cylinder to pick up. + + Notes: + This automatically updates handles the write id. + """ + buf._next_write_id() + self.window.put(buf.array(), field) + return diff --git a/mpisppy/cylinders/spoke.py b/mpisppy/cylinders/spoke.py index 191f3c9b3..6355cf394 100644 --- a/mpisppy/cylinders/spoke.py +++ b/mpisppy/cylinders/spoke.py @@ -31,21 +31,6 @@ def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, communic return - def spoke_to_hub(self, buf: SendArray, field: Field): - """ Put the specified values into the locally-owned buffer for the hub - to pick up. - - Notes: - Automatically handles write id updating and setting - """ - return self._spoke_to_hub(buf.array(), field, buf._next_write_id()) - - def _spoke_to_hub(self, values: np.typing.NDArray, field: Field, write_id: int): - self.cylinder_comm.Barrier() - values[-1] = write_id - self.window.put(values, field) - return - def spoke_from_hub(self, buf: RecvArray, field: Field, @@ -172,7 +157,7 @@ def bound(self): def bound(self, value): self._append_trace(value) self._bound[0] = value - self.spoke_to_hub(self._bound, self.bound_type()) + self.put_send_buffer(self._bound, self.bound_type()) return @property From 8675a8223420092c8b445ee22e731c7410b55a84 Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Mon, 10 Mar 2025 14:33:08 -0600 Subject: [PATCH 13/36] add coverage for RC rho --- examples/run_all.py | 2 +- examples/sslp/sslp_cylinders.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/examples/run_all.py b/examples/run_all.py index 42b91b724..55bc64c8b 100644 --- a/examples/run_all.py +++ b/examples/run_all.py @@ -280,7 +280,7 @@ def do_one_mmw(dirname, runefstring, npyfile, mmwargstring): "--integer-relax-then-enforce " "--integer-relax-then-enforce-ratio=0.95 " "--lagrangian " - "--max-iterations=100 --default-rho=1 " + "--max-iterations=100 --default-rho=1 --reduced-costs-rho " "--reduced-costs --rc-fixer --xhatshuffle " "--linearize-proximal-terms " "--rel-gap=0.0 --surrogate-nonant " diff --git a/examples/sslp/sslp_cylinders.py b/examples/sslp/sslp_cylinders.py index bf7623886..d75958e01 100644 --- a/examples/sslp/sslp_cylinders.py +++ b/examples/sslp/sslp_cylinders.py @@ -36,6 +36,7 @@ def _parse_args(): cfg.xhatshuffle_args() cfg.subgradient_bounder_args() cfg.reduced_costs_args() + cfg.reduced_costs_rho_args() cfg.coeff_rho_args() cfg.integer_relax_then_enforce_args() cfg.parse_command_line("sslp_cylinders") @@ -96,6 +97,9 @@ def main(): if cfg.coeff_rho: vanilla.add_coeff_rho(hub_dict, cfg) + if cfg.reduced_costs_rho: + vanilla.add_reduced_costs_rho(hub_dict, cfg) + if cfg.integer_relax_then_enforce: vanilla.add_integer_relax_then_enforce(hub_dict, cfg) From c4b4efc74b61edfd4ff5617ce688304dccd1870a Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Mon, 10 Mar 2025 14:34:11 -0600 Subject: [PATCH 14/36] update RC rho for PR #476 --- mpisppy/extensions/reduced_costs_fixer.py | 2 +- mpisppy/extensions/reduced_costs_rho.py | 33 +++++++++++------------ 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/mpisppy/extensions/reduced_costs_fixer.py b/mpisppy/extensions/reduced_costs_fixer.py index c31726b45..4c6e13c1c 100644 --- a/mpisppy/extensions/reduced_costs_fixer.py +++ b/mpisppy/extensions/reduced_costs_fixer.py @@ -85,7 +85,7 @@ def iter0_post_solver_creation(self): if self.opt.cylinder_rank == 0 and self.verbose: print("Fixing based on reduced costs prior to iteration 0!") if self.reduced_cost_buf.id() == 0: - while not self.opt.spcomm.hub_from_spoke(self.outer_bound_buf, self.reduced_costs_spoke_index, Field.EXPECTED_REDUCED_COST): + while not self.opt.spcomm.get_receive_buffer(self.outer_bound_buf, Field.EXPECTED_REDUCED_COST, self.reduced_costs_spoke_index): continue self.sync_with_spokes(pre_iter0 = True) self.fix_fraction_target = self._fix_fraction_target_iter0 diff --git a/mpisppy/extensions/reduced_costs_rho.py b/mpisppy/extensions/reduced_costs_rho.py index 8200cfdb1..6cdd32731 100644 --- a/mpisppy/extensions/reduced_costs_rho.py +++ b/mpisppy/extensions/reduced_costs_rho.py @@ -10,7 +10,8 @@ import numpy as np from mpisppy import global_toc from mpisppy.extensions.sensi_rho import _SensiRhoBase -from mpisppy.cylinders.reduced_costs_spoke import ReducedCostsSpoke + +from mpisppy.cylinders.spwindow import Field class ReducedCostsRho(_SensiRhoBase): """ @@ -41,24 +42,20 @@ def __init__(self, ph, comm=None): self._last_serial_number = -1 self.reduced_costs_spoke_index = None - def initialize_spoke_indices(self): - for (i, spoke) in enumerate(self.opt.spcomm.spokes): - if spoke["spoke_class"] == ReducedCostsSpoke: - self.reduced_costs_spoke_index = i + 1 - if self.reduced_costs_spoke_index is None: - raise RuntimeError("ReducedCostsRho requires a ReducedCostsSpoke for calculations") - - def _get_serial_number(self): - return int(round(self.opt.spcomm.outerbound_receive_buffers[self.reduced_costs_spoke_index][-1])) + def register_receive_fields(self): + spcomm = self.opt.spcomm + reduced_cost_ranks = spcomm.fields_to_ranks[Field.SCENARIO_REDUCED_COST] + assert len(reduced_cost_ranks) == 1 + self.reduced_costs_spoke_index = reduced_cost_ranks[0] - def _get_reduced_costs_from_spoke(self): - return self.opt.spcomm.outerbound_receive_buffers[self.reduced_costs_spoke_index][1+self.nonant_length:1+self.nonant_length+len(self._scenario_rc_buffer)] + self.scenario_reduced_cost_buf = spcomm.register_extension_recv_field( + Field.SCENARIO_REDUCED_COST, + self.reduced_costs_spoke_index, + ) def sync_with_spokes(self): - serial_number = self._get_serial_number() - if serial_number > self._last_serial_number: - self._last_serial_number = serial_number - self._scenario_rc_buffer[:] = self._get_reduced_costs_from_spoke() + if self.scenario_reduced_cost_buf.is_new(): + self._scenario_rc_buffer[:] = self.scenario_reduced_cost_buf.value_array() # print(f"In ReducedCostsRho; {self._scenario_rc_buffer=}") else: if self.opt.cylinder_rank == 0 and self.verbose: @@ -89,8 +86,8 @@ def post_iter0_after_sync(self): global_toc("Using reduced cost rho setter") self.update_caches() # wait until the spoke has data - if self._get_serial_number() == 0: - while not self.ph.spcomm.hub_from_spoke(self.opt.spcomm.outerbound_receive_buffers[self.reduced_costs_spoke_index], self.reduced_costs_spoke_index): + if self.scenario_reduced_cost_buf.id() == 0: + while not self.ph.spcomm.get_receive_buffer(self.scenario_reduced_cost_buf, Field.SCENARIO_REDUCED_COST, self.reduced_costs_spoke_index): continue self.sync_with_spokes() self.compute_and_update_rho() From 4e2085e4af3b5ab30a21daca5841b7e928263878 Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Mon, 10 Mar 2025 14:42:20 -0600 Subject: [PATCH 15/36] remove hub_from_spoke --- mpisppy/cylinders/hub.py | 81 +++++------------------------ mpisppy/cylinders/spcommunicator.py | 60 +++++++++++++++++++++ mpisppy/cylinders/spoke.py | 2 +- 3 files changed, 74 insertions(+), 69 deletions(-) diff --git a/mpisppy/cylinders/hub.py b/mpisppy/cylinders/hub.py index 50d0e77e6..c774a8c9c 100644 --- a/mpisppy/cylinders/hub.py +++ b/mpisppy/cylinders/hub.py @@ -7,13 +7,10 @@ # full copyright and license information. ############################################################################### -import numpy as np import abc import logging import mpisppy.log -from mpisppy.opt.aph import APH -from mpisppy import MPI from mpisppy.cylinders.spcommunicator import RecvArray, SendArray, SPCommunicator from math import inf @@ -83,7 +80,6 @@ def current_iteration(self): def main(self): pass - def register_extension_recv_field(self, field: Field, strata_rank: int, buf_len: int = -1) -> RecvArray: """ Register an extensions interest in the given field from the given spoke. The hub @@ -125,7 +121,7 @@ def sync_extension_fields(self): for key in self.extension_recv: ext_buf = self.receive_buffers[key] (field, srank) = self._split_key(key) - ext_buf._is_new = self.hub_from_spoke(ext_buf, srank, field) + ext_buf._is_new = self.get_receive_buffer(ext_buf, field, srank) ## End for return @@ -133,7 +129,6 @@ def clear_latest_chars(self): self.latest_ib_char = None self.latest_ob_char = None - def compute_gaps(self): """ Compute the current absolute and relative gaps, using the current self.BestInnerBound and self.BestOuterBound @@ -157,7 +152,6 @@ def compute_gaps(self): rel_gap = float("inf") return abs_gap, rel_gap - def get_update_string(self): if self.latest_ib_char is None and \ self.latest_ob_char is None: @@ -236,7 +230,7 @@ def receive_innerbounds(self): """ logging.debug("Hub is trying to receive from InnerBounds") for idx, cls, recv_buf in self.receive_field_spcomms[Field.OBJECTIVE_INNER_BOUND]: - is_new = self.hub_from_spoke(recv_buf, idx, Field.OBJECTIVE_INNER_BOUND) + is_new = self.get_receive_buffer(recv_buf, Field.OBJECTIVE_INNER_BOUND, idx) if is_new: bound = recv_buf[0] logging.debug("!! new InnerBound to opt {}".format(bound)) @@ -250,7 +244,7 @@ def receive_outerbounds(self): """ logging.debug("Hub is trying to receive from OuterBounds") for idx, cls, recv_buf in self.receive_field_spcomms[Field.OBJECTIVE_OUTER_BOUND]: - is_new = self.hub_from_spoke(recv_buf, idx, Field.OBJECTIVE_OUTER_BOUND) + is_new = self.get_receive_buffer(recv_buf, Field.OBJECTIVE_OUTER_BOUND, idx) if is_new: bound = recv_buf[0] logging.debug("!! new OuterBound to opt {}".format(bound)) @@ -264,7 +258,7 @@ def OuterBoundUpdate(self, new_bound, cls=None, idx=None, char='*'): self.latest_ob_char = char self.last_ob_idx = 0 else: - self.latest_ib_char = cls.converger_spoke_char + self.latest_ob_char = cls.converger_spoke_char self.last_ob_idx = idx return new_bound else: @@ -326,7 +320,6 @@ def register_receive_fields(self): return - def register_send_fields(self): super().register_send_fields() @@ -336,63 +329,6 @@ def register_send_fields(self): return - def hub_from_spoke(self, - buf: RecvArray, - spoke_num: int, - field: Field, - ): - """ spoke_num is the rank in the strata_comm, so it is 1-based not 0-based - - Returns: - is_new (bool): Indicates whether the "gotten" values are new, - based on the write_id. - """ - buf._is_new = self._hub_from_spoke(buf.array(), spoke_num, field, buf.id()) - if buf.is_new(): - buf._pull_id() - return buf.is_new() - - def _hub_from_spoke(self, - values: np.typing.NDArray, - spoke_num: int, - field: Field, - last_write_id: int, - ): - """ spoke_num is the rank in the strata_comm, so it is 1-based not 0-based - - Returns: - is_new (bool): Indicates whether the "gotten" values are new, - based on the write_id. - """ - # so the window in each rank gets read at approximately the same time, - # and so has the same write_id - if not isinstance(self.opt, APH): - self.cylinder_comm.Barrier() - ## End if - self.window.get(values, spoke_num, field) - - if isinstance(self.opt, APH): - # # reverting part of changes from Ben getting rid of spoke sleep DLW jan 2023 - if values[-1] > last_write_id: - return True - else: - new_id = int(values[-1]) - local_val = np.array((new_id,), 'i') - sum_ids = np.zeros(1, 'i') - self.cylinder_comm.Allreduce((local_val, MPI.INT), - (sum_ids, MPI.INT), - op=MPI.SUM) - if new_id != sum_ids[0] / self.cylinder_comm.size: - return False - ## End if - if new_id > last_write_id or new_id < 0: - return True - ## End if - ## End if - - return False - - def send_terminate(self): """ Send an array of zeros with a -1 appended to the end to indicate termination. This function puts to the local @@ -614,6 +550,15 @@ def main(self): logger.critical("aph debug main in hub.py") self.opt.APH_main(spcomm=self, finalize=False) + # overwrite the default behavior of this method for APH + def get_receive_buffer(self, + buf: RecvArray, + field: Field, + origin: int = -1, + synchronize: bool = False, + ): + return super().get_receive_buffer(buf, field, origin, synchronize) + def finalize(self): """ does PH.post_loops, returns Eobj """ # NOTE: APH_main does NOT pass in extensions diff --git a/mpisppy/cylinders/spcommunicator.py b/mpisppy/cylinders/spcommunicator.py index d458bf161..d11a7bb7f 100644 --- a/mpisppy/cylinders/spcommunicator.py +++ b/mpisppy/cylinders/spcommunicator.py @@ -23,6 +23,7 @@ import abc import time +from mpisppy import MPI from mpisppy.cylinders.spwindow import Field, FieldLengths, SPWindow def communicator_array(size): @@ -309,3 +310,62 @@ def put_send_buffer(self, buf: SendArray, field: Field): buf._next_write_id() self.window.put(buf.array(), field) return + + def get_receive_buffer(self, + buf: RecvArray, + field: Field, + origin: int = -1, + synchronize: bool = True, + ): + """ Gets the specified values from another cylinder and copies them into + the specified locally-owned buffer. Updates the write_id in the locally- + owned buffer, if appropriate. + + Args: + buf (RecvArray) : Buffer to put the data in + field (Field) : The source field + origin (:obj:`int`, optional) : The rank on strata_comm to get the data. + If not provided (or -1), will attempt to infer a unique origin. If + no unique origin is found, will raise an error. Default: -1. + synchronize (:obj:`bool`, optional) : If True, will only report + updated data if the write_ids are the same across the cylinder_comm + are identical. Default: True. + + Returns: + is_new (bool): Indicates whether the "gotten" values are new, + based on the write_id. + """ + if not synchronize: + self.cylinder_comm.Barrier() + + if origin == -1: + origin = self.fields_to_ranks[field][0] + if len(self.fields_to_ranks[field]) > 1: + raise RuntimeError(f"Non-unique origin for {field=}. Possible " + f"origins are {self.fields_to_ranks[field]=}.") + + last_id = buf.id() + + self.window.get(buf.array(), origin, field) + + new_id = int(buf.array()[-1]) + if synchronize: + local_val = np.array((new_id,), 'i') + sum_ids = np.zeros(1, 'i') + self.cylinder_comm.Allreduce((local_val, MPI.INT), + (sum_ids, MPI.INT), + op=MPI.SUM) + if new_id != sum_ids[0] / self.cylinder_comm.size: + buf._is_new = False + return False + + else: + if new_id <= last_id: + buf._is_new = False + return False + + # in either case, now we have new data + buf._is_new = True + buf._pull_id() + + return True diff --git a/mpisppy/cylinders/spoke.py b/mpisppy/cylinders/spoke.py index 6355cf394..bd637c09c 100644 --- a/mpisppy/cylinders/spoke.py +++ b/mpisppy/cylinders/spoke.py @@ -14,7 +14,7 @@ import math from mpisppy import MPI -from mpisppy.cylinders.spcommunicator import RecvArray, SendArray, SPCommunicator +from mpisppy.cylinders.spcommunicator import RecvArray, SPCommunicator from mpisppy.cylinders.spwindow import Field From 091a2c83b74ef6c75a579051f95374d7956637be Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Mon, 10 Mar 2025 15:07:14 -0600 Subject: [PATCH 16/36] remove spoke_from_hub --- mpisppy/cylinders/spoke.py | 49 ++------------------------------------ 1 file changed, 2 insertions(+), 47 deletions(-) diff --git a/mpisppy/cylinders/spoke.py b/mpisppy/cylinders/spoke.py index bd637c09c..8dbfc5d94 100644 --- a/mpisppy/cylinders/spoke.py +++ b/mpisppy/cylinders/spoke.py @@ -7,14 +7,12 @@ # full copyright and license information. ############################################################################### -import numpy as np import abc import time import os import math -from mpisppy import MPI -from mpisppy.cylinders.spcommunicator import RecvArray, SPCommunicator +from mpisppy.cylinders.spcommunicator import SPCommunicator from mpisppy.cylinders.spwindow import Field @@ -31,48 +29,6 @@ def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, communic return - def spoke_from_hub(self, - buf: RecvArray, - field: Field, - ): - buf._is_new = self._spoke_from_hub(buf.array(), field, buf.id()) - if buf.is_new(): - buf._pull_id() - return buf.is_new() - - def _spoke_from_hub(self, - values: np.typing.NDArray, - field: Field, - last_write_id: int - ): - """ - """ - - self.cylinder_comm.Barrier() - self.window.get(values, 0, field) - - # On rare occasions a NaN is seen... - new_id = int(values[-1]) if not math.isnan(values[-1]) else 0 - local_val = np.array((new_id,-new_id), 'i') - max_min_ids = np.zeros(2, 'i') - self.cylinder_comm.Allreduce((local_val, MPI.INT), - (max_min_ids, MPI.INT), - op=MPI.MAX) - - max_id = max_min_ids[0] - min_id = -max_min_ids[1] - # NOTE: we only proceed if all the ranks agree - # on the ID - if max_id != min_id: - return False - - assert max_id == min_id == new_id - - if new_id > last_write_id or new_id < 0: - return True - - return False - def _got_kill_signal(self): shutdown_buf = self.receive_buffers[self._make_key(Field.SHUTDOWN, 0)] if shutdown_buf.is_new(): @@ -103,8 +59,7 @@ def update_receive_buffers(self): for (key, recv_buf) in self.receive_buffers.items(): field, rank = self._split_key(key) # The below code will need to be updated for spoke to spoke communication - assert(rank == 0) - self.spoke_from_hub(recv_buf, field) + self.get_receive_buffer(recv_buf, field, rank) ## End for return From 2769a540bbc35908af192e10bc2834c5a42ed841 Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Mon, 10 Mar 2025 16:04:13 -0600 Subject: [PATCH 17/36] fixing a few bugs --- mpisppy/cylinders/hub.py | 1 + mpisppy/cylinders/reduced_costs_spoke.py | 9 ++++++ mpisppy/cylinders/spcommunicator.py | 35 +++++++++++------------ mpisppy/extensions/reduced_costs_fixer.py | 5 ++-- 4 files changed, 29 insertions(+), 21 deletions(-) diff --git a/mpisppy/cylinders/hub.py b/mpisppy/cylinders/hub.py index c774a8c9c..c634d194a 100644 --- a/mpisppy/cylinders/hub.py +++ b/mpisppy/cylinders/hub.py @@ -384,6 +384,7 @@ def sync_bounds(self): def sync_extensions(self): if self.opt.extensions is not None: + self.sync_extension_fields() self.opt.extobject.sync_with_spokes() def sync_nonants(self): diff --git a/mpisppy/cylinders/reduced_costs_spoke.py b/mpisppy/cylinders/reduced_costs_spoke.py index 7d95811df..178ccddc1 100644 --- a/mpisppy/cylinders/reduced_costs_spoke.py +++ b/mpisppy/cylinders/reduced_costs_spoke.py @@ -167,6 +167,15 @@ def extract_and_store_reduced_costs(self): self.cylinder_comm.Allreduce(rc, rcg, op=MPI.SUM) self.rc_global = rcg + self.put_send_buffer( + self.send_buffers[Field.EXPECTED_REDUCED_COST], + Field.EXPECTED_REDUCED_COST, + ) + self.put_send_buffer( + self.send_buffers[Field.SCENARIO_REDUCED_COST], + Field.SCENARIO_REDUCED_COST, + ) + def main(self): # need the solution for ReducedCostsSpoke super().main(need_solution=True) diff --git a/mpisppy/cylinders/spcommunicator.py b/mpisppy/cylinders/spcommunicator.py index d11a7bb7f..d708a5d46 100644 --- a/mpisppy/cylinders/spcommunicator.py +++ b/mpisppy/cylinders/spcommunicator.py @@ -220,6 +220,13 @@ def _validate_recv_field(self, field: Field, origin: int, length: int): def register_recv_field(self, field: Field, origin: int, length: int = -1) -> RecvArray: # print(f"{self.__class__.__name__}.register_recv_field, {field=}, {origin=}") + # TODO: better handle this case... + if origin == -1: + origin = self.fields_to_ranks[field][0] + if len(self.fields_to_ranks[field]) > 1: + raise RuntimeError(f"Non-unique origin for {field=}. Possible " + f"origins are {self.fields_to_ranks[field]=}.") + key = self._make_key(field, origin) if length == -1: length = self._field_lengths[field] @@ -314,7 +321,7 @@ def put_send_buffer(self, buf: SendArray, field: Field): def get_receive_buffer(self, buf: RecvArray, field: Field, - origin: int = -1, + origin: int, synchronize: bool = True, ): """ Gets the specified values from another cylinder and copies them into @@ -324,9 +331,7 @@ def get_receive_buffer(self, Args: buf (RecvArray) : Buffer to put the data in field (Field) : The source field - origin (:obj:`int`, optional) : The rank on strata_comm to get the data. - If not provided (or -1), will attempt to infer a unique origin. If - no unique origin is found, will raise an error. Default: -1. + origin (int) : The rank on strata_comm to get the data. synchronize (:obj:`bool`, optional) : If True, will only report updated data if the write_ids are the same across the cylinder_comm are identical. Default: True. @@ -338,17 +343,12 @@ def get_receive_buffer(self, if not synchronize: self.cylinder_comm.Barrier() - if origin == -1: - origin = self.fields_to_ranks[field][0] - if len(self.fields_to_ranks[field]) > 1: - raise RuntimeError(f"Non-unique origin for {field=}. Possible " - f"origins are {self.fields_to_ranks[field]=}.") - last_id = buf.id() self.window.get(buf.array(), origin, field) new_id = int(buf.array()[-1]) + if synchronize: local_val = np.array((new_id,), 'i') sum_ids = np.zeros(1, 'i') @@ -359,13 +359,10 @@ def get_receive_buffer(self, buf._is_new = False return False + if new_id > last_id: + buf._is_new = True + buf._pull_id() + return True else: - if new_id <= last_id: - buf._is_new = False - return False - - # in either case, now we have new data - buf._is_new = True - buf._pull_id() - - return True + buf._is_new = False + return False diff --git a/mpisppy/extensions/reduced_costs_fixer.py b/mpisppy/extensions/reduced_costs_fixer.py index 4c6e13c1c..42d948d59 100644 --- a/mpisppy/extensions/reduced_costs_fixer.py +++ b/mpisppy/extensions/reduced_costs_fixer.py @@ -85,7 +85,7 @@ def iter0_post_solver_creation(self): if self.opt.cylinder_rank == 0 and self.verbose: print("Fixing based on reduced costs prior to iteration 0!") if self.reduced_cost_buf.id() == 0: - while not self.opt.spcomm.get_receive_buffer(self.outer_bound_buf, Field.EXPECTED_REDUCED_COST, self.reduced_costs_spoke_index): + while not self.opt.spcomm.get_receive_buffer(self.outer_bound_buf, Field.OBJECTIVE_OUTER_BOUND, self.reduced_costs_spoke_index): continue self.sync_with_spokes(pre_iter0 = True) self.fix_fraction_target = self._fix_fraction_target_iter0 @@ -114,7 +114,8 @@ def register_receive_fields(self): return def sync_with_spokes(self, pre_iter0 = False): - # TODO: Not sure the second part of this if clause is necessary... + # TODO: if we calculate the new bounds in the spoke we don't need to check if the buffers + # have the same ID if self.reduced_cost_buf.is_new() and self.reduced_cost_buf.id() == self.outer_bound_buf.id(): reduced_costs = self.reduced_cost_buf.value_array() this_outer_bound = self.outer_bound_buf.value_array()[0] From a94c2dd984cd9ac0f22fdbbfdf12bd3080676f26 Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Mon, 10 Mar 2025 20:53:07 -0600 Subject: [PATCH 18/36] extensions have to do nearly everything anyways; might as well be explicit about it --- mpisppy/cylinders/hub.py | 53 +--------------------- mpisppy/cylinders/spcommunicator.py | 3 ++ mpisppy/extensions/coeff_rho.py | 4 +- mpisppy/extensions/cross_scen_extension.py | 19 +++++--- mpisppy/extensions/extension.py | 3 +- mpisppy/extensions/reduced_costs_fixer.py | 14 +++++- mpisppy/extensions/reduced_costs_rho.py | 7 ++- mpisppy/extensions/sensi_rho.py | 3 +- 8 files changed, 38 insertions(+), 68 deletions(-) diff --git a/mpisppy/cylinders/hub.py b/mpisppy/cylinders/hub.py index c634d194a..d6199b2c4 100644 --- a/mpisppy/cylinders/hub.py +++ b/mpisppy/cylinders/hub.py @@ -11,7 +11,7 @@ import logging import mpisppy.log -from mpisppy.cylinders.spcommunicator import RecvArray, SendArray, SPCommunicator +from mpisppy.cylinders.spcommunicator import RecvArray, SPCommunicator from math import inf from mpisppy import global_toc @@ -32,9 +32,6 @@ class Hub(SPCommunicator): _hub_algo_best_bound_provider = False def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, communicators, options=None): - # The extensions will be registered in SPCommunicator.__init__ - self.extension_recv = set() - super().__init__(spbase_object, fullcomm, strata_comm, cylinder_comm, communicators, options=options) logger.debug(f"Built the hub object on global rank {fullcomm.Get_rank()}") @@ -80,51 +77,6 @@ def current_iteration(self): def main(self): pass - def register_extension_recv_field(self, field: Field, strata_rank: int, buf_len: int = -1) -> RecvArray: - """ - Register an extensions interest in the given field from the given spoke. The hub - is then responsible for updating this field into a local buffer prior to the call - to the extension sync_with_spokes function. - """ - key = self._make_key(field, strata_rank) - if key not in self.receive_buffers: - # if it is not already registered, we need to update the local buffer - self.extension_recv.add(key) - ## End if - ra = self.register_recv_field(field, strata_rank, buf_len) - return ra - - def register_extension_send_field(self, field: Field, buf_len: int) -> SendArray: - """ - Register a field with the hub that an extension will be making available to spokes. Returns a - buffer that is usable for sending the desired values. The extension is responsible for calling - the hub publish_extension_field when ready to send the values. Returns a SendArray to use - to publish values to spokes. Meant to be called within the extension function - `register_send_fields`. - """ - return self.register_send_field(field, buf_len) - - def is_send_field_registered(self, field: Field) -> bool: - return field in self.send_buffers - - def extension_send_field(self, field: Field, buf: SendArray): - """ - Send the data in the SendArray `buf` which stores the Field `field`. This will make - the data available to the spokes in this strata. - """ - return self.put_send_buffer(buf, field) - - def sync_extension_fields(self): - """ - Update all registered extension fields. Safe to call even when there are no extension fields. - """ - for key in self.extension_recv: - ext_buf = self.receive_buffers[key] - (field, srank) = self._split_key(key) - ext_buf._is_new = self.get_receive_buffer(ext_buf, field, srank) - ## End for - return - def clear_latest_chars(self): self.latest_ib_char = None self.latest_ob_char = None @@ -371,7 +323,6 @@ def sync(self): self.receive_outerbounds() self.receive_innerbounds() if self.opt.extensions is not None: - self.sync_extension_fields() self.opt.extobject.sync_with_spokes() def sync_with_spokes(self): @@ -384,7 +335,6 @@ def sync_bounds(self): def sync_extensions(self): if self.opt.extensions is not None: - self.sync_extension_fields() self.opt.extobject.sync_with_spokes() def sync_nonants(self): @@ -485,7 +435,6 @@ def sync(self, send_nonants=True): self.receive_innerbounds() # in case LShaped ever gets extensions if getattr(self.opt, "extensions", None) is not None: - self.sync_extension_fields() self.opt.extobject.sync_with_spokes() def is_converged(self): diff --git a/mpisppy/cylinders/spcommunicator.py b/mpisppy/cylinders/spcommunicator.py index d708a5d46..0c08b76b7 100644 --- a/mpisppy/cylinders/spcommunicator.py +++ b/mpisppy/cylinders/spcommunicator.py @@ -287,6 +287,9 @@ def _make_windows(self) -> None: return + def is_send_field_registered(self, field: Field) -> bool: + return field in self.send_buffers + def register_send_fields(self) -> None: for field in self.send_fields: self.register_send_field(field) diff --git a/mpisppy/extensions/coeff_rho.py b/mpisppy/extensions/coeff_rho.py index 3aa687923..eecf82730 100644 --- a/mpisppy/extensions/coeff_rho.py +++ b/mpisppy/extensions/coeff_rho.py @@ -7,6 +7,7 @@ # full copyright and license information. ############################################################################### +from mpisppy import global_toc import mpisppy.extensions.extension from mpisppy.utils.sputils import nonant_cost_coeffs @@ -36,5 +37,4 @@ def post_iter0(self): # nv = s._mpisppy_data.nonant_indices[ndn_i] # var_data object # print(ndn_i,nv.getname(),cc[ndn_i],rho._value) - if self.ph.cylinder_rank == 0: - print("Rho values updated by CoeffRho Extension") + global_toc("Rho values updated by CoeffRho Extension", self.ph.cylinder_rank == 0) diff --git a/mpisppy/extensions/cross_scen_extension.py b/mpisppy/extensions/cross_scen_extension.py index 480f9b6e5..6ccd625a7 100644 --- a/mpisppy/extensions/cross_scen_extension.py +++ b/mpisppy/extensions/cross_scen_extension.py @@ -126,6 +126,11 @@ def _check_bound(self): cached_ph_obj[k].activate() def get_from_cross_cuts(self): + self.opt.spcomm.get_receive_buffer( + self.cuts, + Field.CROSS_SCENARIO_CUT, + self.cross_scenario_index, + ) if self.cuts.is_new(): self.make_cuts(self.cuts.array()) @@ -144,7 +149,7 @@ def send_to_cross_cuts(self): ## End for ## End for - self.opt.spcomm.extension_send_field(Field.NONANT, all_nonants) + self.opt.spcomm.put_send_buffer(all_nonants, Field.NONANT) ## End if @@ -156,7 +161,7 @@ def send_to_cross_cuts(self): all_etas[ci] = s._mpisppy_model.eta[sn]._value ci += 1 - self.opt.spcomm.extension_send_field(Field.CROSS_SCENARIO_COST, all_etas) + self.opt.spcomm.put_send_buffer(all_etas, Field.CROSS_SCENARIO_COST) return @@ -255,13 +260,13 @@ def register_send_fields(self): if spcomm.is_send_field_registered(Field.NONANT): self.send_nonants = False else: - self.all_nonants = spcomm.register_extension_send_field( + self.all_nonants = spcomm.register_send_field( Field.NONANT, local_scen_count * self.opt.nonant_length ) self.send_nonants = True ## End if-else - self.all_etas = spcomm.register_extension_send_field( + self.all_etas = spcomm.register_send_field( Field.CROSS_SCENARIO_COST, nscen * nscen, ) @@ -282,11 +287,11 @@ def register_receive_fields(self): spcomm = self.opt.spcomm cross_scenario_cut_ranks = spcomm.fields_to_ranks[Field.CROSS_SCENARIO_CUT] assert len(cross_scenario_cut_ranks) == 1 - index = cross_scenario_cut_ranks[0] + self.cross_scenario_index = cross_scenario_cut_ranks[0] - self.cuts = spcomm.register_extension_recv_field( + self.cuts = spcomm.register_recv_field( Field.CROSS_SCENARIO_CUT, - index, + self.cross_scenario_index, ) def sync_with_spokes(self): diff --git a/mpisppy/extensions/extension.py b/mpisppy/extensions/extension.py index 493ddcddd..5a372f765 100644 --- a/mpisppy/extensions/extension.py +++ b/mpisppy/extensions/extension.py @@ -34,8 +34,7 @@ def setup_hub(self): def register_send_fields(self): ''' Method called by the Hub SPCommunicator to get any fields that the extension - will make available to spokes. Use hub function `register_extension_send_field` - to register a field. + will make available to spokes. ''' return diff --git a/mpisppy/extensions/reduced_costs_fixer.py b/mpisppy/extensions/reduced_costs_fixer.py index 42d948d59..eb6f62a23 100644 --- a/mpisppy/extensions/reduced_costs_fixer.py +++ b/mpisppy/extensions/reduced_costs_fixer.py @@ -101,11 +101,11 @@ def register_receive_fields(self): self.reduced_costs_spoke_index = index - self.reduced_cost_buf = spcomm.register_extension_recv_field( + self.reduced_cost_buf = spcomm.register_recv_field( Field.EXPECTED_REDUCED_COST, self.reduced_costs_spoke_index, ) - self.outer_bound_buf = spcomm.register_extension_recv_field( + self.outer_bound_buf = spcomm.register_recv_field( Field.OBJECTIVE_OUTER_BOUND, self.reduced_costs_spoke_index, ) @@ -116,6 +116,16 @@ def register_receive_fields(self): def sync_with_spokes(self, pre_iter0 = False): # TODO: if we calculate the new bounds in the spoke we don't need to check if the buffers # have the same ID + self.opt.spcomm.get_receive_buffer( + self.reduced_cost_buf, + Field.EXPECTED_REDUCED_COST, + self.reduced_costs_spoke_index, + ) + self.opt.spcomm.get_receive_buffer( + self.outer_bound_buf, + Field.OBJECTIVE_OUTER_BOUND, + self.reduced_costs_spoke_index, + ) if self.reduced_cost_buf.is_new() and self.reduced_cost_buf.id() == self.outer_bound_buf.id(): reduced_costs = self.reduced_cost_buf.value_array() this_outer_bound = self.outer_bound_buf.value_array()[0] diff --git a/mpisppy/extensions/reduced_costs_rho.py b/mpisppy/extensions/reduced_costs_rho.py index 6cdd32731..0c7639788 100644 --- a/mpisppy/extensions/reduced_costs_rho.py +++ b/mpisppy/extensions/reduced_costs_rho.py @@ -48,12 +48,17 @@ def register_receive_fields(self): assert len(reduced_cost_ranks) == 1 self.reduced_costs_spoke_index = reduced_cost_ranks[0] - self.scenario_reduced_cost_buf = spcomm.register_extension_recv_field( + self.scenario_reduced_cost_buf = spcomm.register_recv_field( Field.SCENARIO_REDUCED_COST, self.reduced_costs_spoke_index, ) def sync_with_spokes(self): + self.opt.spcomm.get_receive_buffer( + self.scenario_reduced_cost_buf, + Field.SCENARIO_REDUCED_COST, + self.reduced_costs_spoke_index, + ) if self.scenario_reduced_cost_buf.is_new(): self._scenario_rc_buffer[:] = self.scenario_reduced_cost_buf.value_array() # print(f"In ReducedCostsRho; {self._scenario_rc_buffer=}") diff --git a/mpisppy/extensions/sensi_rho.py b/mpisppy/extensions/sensi_rho.py index ae6374efc..72d1a22da 100644 --- a/mpisppy/extensions/sensi_rho.py +++ b/mpisppy/extensions/sensi_rho.py @@ -54,8 +54,7 @@ def compute_and_update_rho(self): # nv = s._mpisppy_data.nonant_indices[ndn_i] # var_data object # print(f"{s.name=}, {nv.name=}, {rho.value=}") - if ph.cylinder_rank == 0: - print(f"Rho values updated by {self.__class__.__name__} Extension") + global_toc(f"Rho values updated by {self.__class__.__name__} Extension", ph.cylinder_rank == 0) def miditer(self): self.update_caches() From 3948b2f0a1f8ab3fee4c54d715f7a5d83568ba55 Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Mon, 10 Mar 2025 21:18:18 -0600 Subject: [PATCH 19/36] spokes can get DUALS and NONANTs from whereever, as long as the provider is unique --- mpisppy/cylinders/spoke.py | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/mpisppy/cylinders/spoke.py b/mpisppy/cylinders/spoke.py index 8dbfc5d94..be107841c 100644 --- a/mpisppy/cylinders/spoke.py +++ b/mpisppy/cylinders/spoke.py @@ -21,22 +21,9 @@ class Spoke(SPCommunicator): send_fields = (*SPCommunicator.send_fields, ) receive_fields = (*SPCommunicator.receive_fields, Field.SHUTDOWN, ) - def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, communicators, options=None): - - super().__init__(spbase_object, fullcomm, strata_comm, cylinder_comm, communicators, options) - - self.last_call_to_got_kill_signal = time.time() - - return - def _got_kill_signal(self): shutdown_buf = self.receive_buffers[self._make_key(Field.SHUTDOWN, 0)] - if shutdown_buf.is_new(): - shutdown = (shutdown_buf[0] == 1.0) - else: - shutdown = False - ## End if - return shutdown + return (shutdown_buf.is_new() and shutdown_buf[0] == 1.0) def got_kill_signal(self): """ Spoke should call this method at least every iteration @@ -58,7 +45,6 @@ def main(self): def update_receive_buffers(self): for (key, recv_buf) in self.receive_buffers.items(): field, rank = self._split_key(key) - # The below code will need to be updated for spoke to spoke communication self.get_receive_buffer(recv_buf, field, rank) ## End for return @@ -142,6 +128,15 @@ def nonant_len_type(self) -> Field: # TODO: Make this a static method? pass + def register_receive_fields(self) -> None: + super().register_receive_fields() + nonant_len_ranks = self.fields_to_ranks[self.nonant_len_type()] + if len(nonant_len_ranks) > 1: + raise RuntimeError( + f"More than one cylinder to select from for {self.nonant_len_type()}!" + ) + self._receive_rank = nonant_len_ranks[0] + class InnerBoundSpoke(_BoundSpoke): """ For Spokes that provide an inner bound through self.bound to the @@ -182,7 +177,7 @@ def nonant_len_type(self) -> Field: @property def localWs(self): """Returns the local copy of the weights""" - key = self._make_key(Field.DUALS, 0) + key = self._make_key(Field.DUALS, self._receive_rank) return self.receive_buffers[key].value_array() @property @@ -191,7 +186,7 @@ def new_Ws(self): the weights has been updated since the last call to got_kill_signal """ - key = self._make_key(Field.DUALS, 0) + key = self._make_key(Field.DUALS, self._receive_rank) return self.receive_buffers[key].is_new() @@ -223,7 +218,7 @@ def nonant_len_type(self) -> Field: @property def localnonants(self): """Returns the local copy of the nonants""" - key = self._make_key(Field.NONANT, 0) + key = self._make_key(Field.NONANT, self._receive_rank) return self.receive_buffers[key].value_array() @property @@ -231,7 +226,7 @@ def new_nonants(self): """Returns True if the local copy of the nonants has been updated since the last call to got_kill_signal""" - key = self._make_key(Field.NONANT, 0) + key = self._make_key(Field.NONANT, self._receive_rank) return self.receive_buffers[key].is_new() From 62a3da932a08f00d68d7f6e4692d215724a4b96f Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Wed, 26 Mar 2025 10:16:15 -0400 Subject: [PATCH 20/36] cleaning up windows; errata --- mpisppy/cylinders/cross_scen_spoke.py | 1 + mpisppy/cylinders/reduced_costs_spoke.py | 2 +- mpisppy/cylinders/spcommunicator.py | 11 ++--------- mpisppy/cylinders/spwindow.py | 19 +++---------------- mpisppy/extensions/reduced_costs_fixer.py | 8 +++++--- 5 files changed, 12 insertions(+), 29 deletions(-) diff --git a/mpisppy/cylinders/cross_scen_spoke.py b/mpisppy/cylinders/cross_scen_spoke.py index 2fe006b09..04c29caf1 100644 --- a/mpisppy/cylinders/cross_scen_spoke.py +++ b/mpisppy/cylinders/cross_scen_spoke.py @@ -48,6 +48,7 @@ def register_send_fields(self) -> None: def register_receive_fields(self): super().register_receive_fields() + # TODO: look up rank self.all_nonants = self.register_recv_field(Field.NONANT, 0) self.all_etas = self.register_recv_field(Field.CROSS_SCENARIO_COST, 0) diff --git a/mpisppy/cylinders/reduced_costs_spoke.py b/mpisppy/cylinders/reduced_costs_spoke.py index 178ccddc1..c77e2f76b 100644 --- a/mpisppy/cylinders/reduced_costs_spoke.py +++ b/mpisppy/cylinders/reduced_costs_spoke.py @@ -15,7 +15,7 @@ class ReducedCostsSpoke(LagrangianOuterBound): - send_fields = (*LagrangianOuterBound.send_fields, Field.EXPECTED_REDUCED_COST, Field.SCENARIO_REDUCED_COST ,) + send_fields = (*LagrangianOuterBound.send_fields, Field.EXPECTED_REDUCED_COST, Field.SCENARIO_REDUCED_COST,) receive_fields = (*LagrangianOuterBound.receive_fields,) converger_spoke_char = 'R' diff --git a/mpisppy/cylinders/spcommunicator.py b/mpisppy/cylinders/spcommunicator.py index 0c08b76b7..92ff199d3 100644 --- a/mpisppy/cylinders/spcommunicator.py +++ b/mpisppy/cylinders/spcommunicator.py @@ -83,8 +83,8 @@ def __setitem__(self, key, value): def _next_write_id(self) -> int: """ - Updates the internal id field to the next write id, uses that id in the field data, - and returns that id + Updates the internal id field to the next write id, sets that id in the + field data array, and returns that id """ self._id += 1 self._array[-1] = self._id @@ -220,13 +220,6 @@ def _validate_recv_field(self, field: Field, origin: int, length: int): def register_recv_field(self, field: Field, origin: int, length: int = -1) -> RecvArray: # print(f"{self.__class__.__name__}.register_recv_field, {field=}, {origin=}") - # TODO: better handle this case... - if origin == -1: - origin = self.fields_to_ranks[field][0] - if len(self.fields_to_ranks[field]) > 1: - raise RuntimeError(f"Non-unique origin for {field=}. Possible " - f"origins are {self.fields_to_ranks[field]=}.") - key = self._make_key(field, origin) if length == -1: length = self._field_lengths[field] diff --git a/mpisppy/cylinders/spwindow.py b/mpisppy/cylinders/spwindow.py index a40a59c44..5b66d1cae 100644 --- a/mpisppy/cylinders/spwindow.py +++ b/mpisppy/cylinders/spwindow.py @@ -13,6 +13,7 @@ import numpy.typing as nptyping import enum +import weakref import pyomo.environ as pyo @@ -110,6 +111,8 @@ def __init__(self, my_fields: dict, strata_comm: MPI.Comm, field_order=None): self.buffer_length = total_buffer_length self.window = MPI.Win.Allocate(window_size_bytes, MPI.DOUBLE.size, comm=strata_comm) + # ensure the memory allocated for the window is freed + self._window_finalizer = weakref.finalize(self, self.window.free) self.buff = np.ndarray(dtype="d", shape=(total_buffer_length,), buffer=self.window.tomemory()) self.buff[:] = np.nan @@ -121,22 +124,6 @@ def __init__(self, my_fields: dict, strata_comm: MPI.Comm, field_order=None): self.strata_buffer_layouts = strata_comm.allgather(self.buffer_layout) - self.window_constructed = True - - return - - def free(self): - - if self.window_constructed: - self.window.Free() - self.buff = None - self.buffer_layout = None - self.buffer_length = 0 - self.window = None - self.strata_buffer_layouts = None - self.window_constructed = False - ## End if - return #### Functions #### diff --git a/mpisppy/extensions/reduced_costs_fixer.py b/mpisppy/extensions/reduced_costs_fixer.py index eb6f62a23..2724f84c8 100644 --- a/mpisppy/extensions/reduced_costs_fixer.py +++ b/mpisppy/extensions/reduced_costs_fixer.py @@ -109,13 +109,15 @@ def register_receive_fields(self): Field.OBJECTIVE_OUTER_BOUND, self.reduced_costs_spoke_index, ) - ## End if return def sync_with_spokes(self, pre_iter0 = False): - # TODO: if we calculate the new bounds in the spoke we don't need to check if the buffers - # have the same ID + # TODO: If we calculate the new bounds in the spoke we don't need to + # check if the buffers have the same ID. + # NOTE: If we do this, then the heuristic reduced cost fixing might fix + # different variables in different subproblems. But this might be + # fine. self.opt.spcomm.get_receive_buffer( self.reduced_cost_buf, Field.EXPECTED_REDUCED_COST, From a549c7dae4e10a937e40247c27111f94af0bddfa Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Wed, 26 Mar 2025 11:06:12 -0400 Subject: [PATCH 21/36] Use `Free` for older version of mpi4py --- mpisppy/cylinders/spwindow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mpisppy/cylinders/spwindow.py b/mpisppy/cylinders/spwindow.py index 5b66d1cae..62cf74271 100644 --- a/mpisppy/cylinders/spwindow.py +++ b/mpisppy/cylinders/spwindow.py @@ -112,7 +112,7 @@ def __init__(self, my_fields: dict, strata_comm: MPI.Comm, field_order=None): self.buffer_length = total_buffer_length self.window = MPI.Win.Allocate(window_size_bytes, MPI.DOUBLE.size, comm=strata_comm) # ensure the memory allocated for the window is freed - self._window_finalizer = weakref.finalize(self, self.window.free) + self._window_finalizer = weakref.finalize(self, self.window.Free) self.buff = np.ndarray(dtype="d", shape=(total_buffer_length,), buffer=self.window.tomemory()) self.buff[:] = np.nan From 1a0ceb8bdb975c24c8167353485b74373ee73106 Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Wed, 26 Mar 2025 10:59:18 -0600 Subject: [PATCH 22/36] bring back explicit window creation / distruction --- mpisppy/cylinders/spcommunicator.py | 36 ++++++++++++++++++++++------- mpisppy/cylinders/spwindow.py | 13 +++++++++-- mpisppy/spin_the_wheel.py | 3 +++ 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/mpisppy/cylinders/spcommunicator.py b/mpisppy/cylinders/spcommunicator.py index 92ff199d3..d8275322e 100644 --- a/mpisppy/cylinders/spcommunicator.py +++ b/mpisppy/cylinders/spcommunicator.py @@ -146,17 +146,12 @@ def __init__(self, spbase_object, fullcomm, strata_comm, cylinder_comm, communic # on the problem data self._field_lengths = FieldLengths(self.opt) + self.window = None + # attach the SPCommunicator to # the SPBase object self.opt.spcomm = self - self.register_send_fields() - - self._make_windows() - self._create_field_rank_mappings() - - self.register_receive_fields() - return def _make_key(self, field: Field, origin: int): @@ -273,13 +268,38 @@ def hub_finalize(self): def allreduce_or(self, val): return self.opt.allreduce_or(val) - def _make_windows(self) -> None: + def make_windows(self) -> None: + """ Make MPI windows: blocking call for all ranks in `strata_comm`. + """ + + if self.window is not None: + return + + self.register_send_fields() window_spec = self._build_window_spec() self.window = SPWindow(window_spec, self.strata_comm) + self._create_field_rank_mappings() + self.register_receive_fields() + return + def free_windows(self) -> None: + """ Free MPI windows: blocking call for all ranks in `strata_comm`. + """ + + if self.window is None: + return + + self.receive_buffers = {} + self.send_buffers = {} + self.receive_field_spcomms = {} + + self.window.free() + + self.window = None + def is_send_field_registered(self, field: Field) -> bool: return field in self.send_buffers diff --git a/mpisppy/cylinders/spwindow.py b/mpisppy/cylinders/spwindow.py index 62cf74271..b5e288778 100644 --- a/mpisppy/cylinders/spwindow.py +++ b/mpisppy/cylinders/spwindow.py @@ -13,7 +13,6 @@ import numpy.typing as nptyping import enum -import weakref import pyomo.environ as pyo @@ -112,7 +111,6 @@ def __init__(self, my_fields: dict, strata_comm: MPI.Comm, field_order=None): self.buffer_length = total_buffer_length self.window = MPI.Win.Allocate(window_size_bytes, MPI.DOUBLE.size, comm=strata_comm) # ensure the memory allocated for the window is freed - self._window_finalizer = weakref.finalize(self, self.window.Free) self.buff = np.ndarray(dtype="d", shape=(total_buffer_length,), buffer=self.window.tomemory()) self.buff[:] = np.nan @@ -126,6 +124,17 @@ def __init__(self, my_fields: dict, strata_comm: MPI.Comm, field_order=None): return + def free(self): + if self.window is not None: + self.window.Free() + self.buff = None + self.buffer_layout = None + self.buffer_length = 0 + self.window = None + self.strata_buffer_layouts = None + self.window = None + return + #### Functions #### def get(self, dest: nptyping.ArrayLike, strata_rank: int, field: Field): diff --git a/mpisppy/spin_the_wheel.py b/mpisppy/spin_the_wheel.py index 24915db4b..6d7562ee1 100644 --- a/mpisppy/spin_the_wheel.py +++ b/mpisppy/spin_the_wheel.py @@ -127,6 +127,8 @@ def run(self, comm_world=None): spcomm = sp_class(opt, fullcomm, strata_comm, cylinder_comm, communicator_list, **sp_kwargs) + spcomm.make_windows() + # Run main() if strata_rank == 0: spcomm.setup_hub() @@ -148,6 +150,7 @@ def run(self, comm_world=None): ## give the hub the chance to catch new values spcomm.hub_finalize() + spcomm.free_windows() fullcomm.Barrier() global_toc("Finalize Complete") From 5fdb1f04932800717c22ea8ee60a3599390145b0 Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Thu, 27 Mar 2025 09:51:42 -0600 Subject: [PATCH 23/36] look up nonants/cost in cross scenario spoke --- mpisppy/cylinders/cross_scen_spoke.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/mpisppy/cylinders/cross_scen_spoke.py b/mpisppy/cylinders/cross_scen_spoke.py index 04c29caf1..acf98651a 100644 --- a/mpisppy/cylinders/cross_scen_spoke.py +++ b/mpisppy/cylinders/cross_scen_spoke.py @@ -48,9 +48,17 @@ def register_send_fields(self) -> None: def register_receive_fields(self): super().register_receive_fields() - # TODO: look up rank - self.all_nonants = self.register_recv_field(Field.NONANT, 0) - self.all_etas = self.register_recv_field(Field.CROSS_SCENARIO_COST, 0) + + nonant_ranks = self.opt.spcomm.fields_to_ranks[Field.NONANT] + cs_cost_ranks = self.opt.spcomm.fields_to_ranks[Field.CROSS_SCENARIO_COST] + + assert len(nonant_ranks) == 1 + assert len(cs_cost_ranks) == 1 + assert nonant_ranks[0] == cs_cost_ranks[0] + source_rank = nonant_ranks[0] + + self.all_nonants = self.register_recv_field(Field.NONANT, source_rank) + self.all_etas = self.register_recv_field(Field.CROSS_SCENARIO_COST, source_rank) def prep_cs_cuts(self): # create a map scenario -> index, this index is used for various lists containing scenario dependent info. From 821bc31807bf643fcc80e4ef86de30854181f1af Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Mon, 31 Mar 2025 10:37:32 -0600 Subject: [PATCH 24/36] remove implicit field update for spokes: spokes must now explicitly update their fields --- mpisppy/cylinders/hub.py | 2 +- mpisppy/cylinders/lagrangian_bounder.py | 2 +- mpisppy/cylinders/lshaped_bounder.py | 2 +- mpisppy/cylinders/slam_heuristic.py | 2 +- mpisppy/cylinders/spoke.py | 59 +++++++++---------- mpisppy/cylinders/xhatlooper_bounder.py | 2 +- .../cylinders/xhatshufflelooper_bounder.py | 18 +++--- mpisppy/cylinders/xhatspecific_bounder.py | 2 +- mpisppy/cylinders/xhatxbar_bounder.py | 2 +- mpisppy/extensions/phtracker.py | 1 + 10 files changed, 43 insertions(+), 49 deletions(-) diff --git a/mpisppy/cylinders/hub.py b/mpisppy/cylinders/hub.py index d6199b2c4..22d3b3fa9 100644 --- a/mpisppy/cylinders/hub.py +++ b/mpisppy/cylinders/hub.py @@ -504,7 +504,7 @@ def main(self): def get_receive_buffer(self, buf: RecvArray, field: Field, - origin: int = -1, + origin: int, synchronize: bool = False, ): return super().get_receive_buffer(buf, field, origin, synchronize) diff --git a/mpisppy/cylinders/lagrangian_bounder.py b/mpisppy/cylinders/lagrangian_bounder.py index 11912305d..dc1548f0e 100644 --- a/mpisppy/cylinders/lagrangian_bounder.py +++ b/mpisppy/cylinders/lagrangian_bounder.py @@ -78,7 +78,7 @@ def main(self, need_solution=False): self.opt.extobject.post_iter0_after_sync() while not self.got_kill_signal(): - if self.new_Ws: + if self.update_Ws(): if extensions: self.opt.extobject.miditer() bound = self._set_weights_and_solve(need_solution=need_solution) diff --git a/mpisppy/cylinders/lshaped_bounder.py b/mpisppy/cylinders/lshaped_bounder.py index 550ea5227..55f5625f7 100644 --- a/mpisppy/cylinders/lshaped_bounder.py +++ b/mpisppy/cylinders/lshaped_bounder.py @@ -61,7 +61,7 @@ def main(self): #xh_iter = 1 while not self.got_kill_signal(): - if self.new_nonants: + if self.update_nonants(): self.opt._put_nonant_cache(self.localnonants) self.opt._restore_nonants() diff --git a/mpisppy/cylinders/slam_heuristic.py b/mpisppy/cylinders/slam_heuristic.py index 66e94bdfb..b2977b829 100644 --- a/mpisppy/cylinders/slam_heuristic.py +++ b/mpisppy/cylinders/slam_heuristic.py @@ -76,7 +76,7 @@ def main(self): logger.debug(f' {self.__class__.__name__} loop iter={slam_iter} on rank {self.global_rank}') logger.debug(f' {self.__class__.__name__} got from opt on rank {self.global_rank}') - if self.new_nonants: + if self.update_nonants(): local_candidate = self.extract_local_candidate_soln() diff --git a/mpisppy/cylinders/spoke.py b/mpisppy/cylinders/spoke.py index be107841c..a426a6fd4 100644 --- a/mpisppy/cylinders/spoke.py +++ b/mpisppy/cylinders/spoke.py @@ -21,16 +21,13 @@ class Spoke(SPCommunicator): send_fields = (*SPCommunicator.send_fields, ) receive_fields = (*SPCommunicator.receive_fields, Field.SHUTDOWN, ) - def _got_kill_signal(self): - shutdown_buf = self.receive_buffers[self._make_key(Field.SHUTDOWN, 0)] - return (shutdown_buf.is_new() and shutdown_buf[0] == 1.0) - def got_kill_signal(self): """ Spoke should call this method at least every iteration to see if the Hub terminated """ - self.update_receive_buffers() - return self._got_kill_signal() + shutdown_buf = self.receive_buffers[self._make_key(Field.SHUTDOWN, 0)] + self.get_receive_buffer(shutdown_buf, Field.SHUTDOWN, 0) + return (shutdown_buf.is_new() and shutdown_buf[0] == 1.0) @abc.abstractmethod def main(self): @@ -42,13 +39,6 @@ def main(self): """ pass - def update_receive_buffers(self): - for (key, recv_buf) in self.receive_buffers.items(): - field, rank = self._split_key(key) - self.get_receive_buffer(recv_buf, field, rank) - ## End for - return - class _BoundSpoke(Spoke): """ A base class for bound spokes @@ -101,6 +91,10 @@ def bound(self, value): self.put_send_buffer(self._bound, self.bound_type()) return + def update_hub_bounds(self) -> None: + """ get new hub inner / outer bounds from the hub """ + return self.get_receive_buffer(self._hub_bounds, Field.BEST_OBJECTIVE_BOUNDS, 0) + @property def hub_inner_bound(self): """Returns the local copy of the inner bound from the hub""" @@ -135,7 +129,13 @@ def register_receive_fields(self) -> None: raise RuntimeError( f"More than one cylinder to select from for {self.nonant_len_type()}!" ) - self._receive_rank = nonant_len_ranks[0] + key = self._make_key(self.nonant_len_type(), nonant_len_ranks[0]) + self._nonant_len_receive_buffer = self.receive_buffers[key] + self._nonant_len_receive_rank = nonant_len_ranks[0] + + def _update_nonant_len_buffer(self) -> bool: + """ get new data from the hub """ + return self.get_receive_buffer(self._nonant_len_receive_buffer, self.nonant_len_type(), self._nonant_len_receive_rank) class InnerBoundSpoke(_BoundSpoke): @@ -177,17 +177,14 @@ def nonant_len_type(self) -> Field: @property def localWs(self): """Returns the local copy of the weights""" - key = self._make_key(Field.DUALS, self._receive_rank) - return self.receive_buffers[key].value_array() + return self._nonant_len_receive_buffer.value_array() - @property - def new_Ws(self): - """ Returns True if the local copy of - the weights has been updated since - the last call to got_kill_signal + def update_Ws(self) -> bool: + """ Check for new Ws from the source. + Returns True if the Ws are new. False otherwise. + Puts the result in `localWs`. """ - key = self._make_key(Field.DUALS, self._receive_rank) - return self.receive_buffers[key].is_new() + return self._update_nonant_len_buffer() class OuterBoundWSpoke(_BoundWSpoke): @@ -218,16 +215,14 @@ def nonant_len_type(self) -> Field: @property def localnonants(self): """Returns the local copy of the nonants""" - key = self._make_key(Field.NONANT, self._receive_rank) - return self.receive_buffers[key].value_array() + return self._nonant_len_receive_buffer.value_array() - @property - def new_nonants(self): - """Returns True if the local copy of - the nonants has been updated since - the last call to got_kill_signal""" - key = self._make_key(Field.NONANT, self._receive_rank) - return self.receive_buffers[key].is_new() + def update_nonants(self) -> bool: + """ Check for new nonants from the source. + Returns True if the nonants are new. False otherwise. + Puts the result in `localnonants`. + """ + return self._update_nonant_len_buffer() class InnerBoundNonantSpoke(_BoundNonantSpoke): diff --git a/mpisppy/cylinders/xhatlooper_bounder.py b/mpisppy/cylinders/xhatlooper_bounder.py index e4be56470..dc3c2fad5 100644 --- a/mpisppy/cylinders/xhatlooper_bounder.py +++ b/mpisppy/cylinders/xhatlooper_bounder.py @@ -39,7 +39,7 @@ def main(self): logger.debug(f' Xhatlooper loop iter={xh_iter} on rank {self.global_rank}') logger.debug(f' Xhatlooper got from opt on rank {self.global_rank}') - if self.new_nonants: + if self.update_nonants(): logger.debug(f' *Xhatlooper loop iter={xh_iter}') logger.debug(f' *got a new one! on rank {self.global_rank}') logger.debug(f' *localnonants={str(self.localnonants)}') diff --git a/mpisppy/cylinders/xhatshufflelooper_bounder.py b/mpisppy/cylinders/xhatshufflelooper_bounder.py index 37b97cfbc..9c29074fa 100644 --- a/mpisppy/cylinders/xhatshufflelooper_bounder.py +++ b/mpisppy/cylinders/xhatshufflelooper_bounder.py @@ -13,7 +13,6 @@ from mpisppy.extensions.xhatbase import XhatBase from mpisppy.cylinders.xhatbase import XhatInnerBoundBase -from mpisppy.cylinders.spwindow import Field # Could also pass, e.g., sys.stdout instead of a filename mpisppy.log.setup_logger("mpisppy.cylinders.xhatshufflelooper_bounder", @@ -98,21 +97,20 @@ def _vb(msg): xh_iter = 1 while not self.got_kill_signal(): - # When there is no iter0, the serial number must be checked. # (unrelated: uncomment the next line to see the source of delay getting an xhat) - # if self.get_serial_number() == 0: - # continue - - if self.receive_buffers[self._make_key(Field.NONANT, 0)].id() == 0: - continue - if (xh_iter-1) % 100 == 0: logger.debug(f' Xhatshuffle loop iter={xh_iter} on rank {self.global_rank}') logger.debug(f' Xhatshuffle got from opt on rank {self.global_rank}') - if self.new_nonants: + new_nonants = self.update_nonants() + + # When there is no iter0, the serial number must be checked. + if self._nonant_len_receive_buffer.id() == 0: + continue + + if new_nonants: # similar to above, not all ranks will agree on - # when there are new_nonants (in the same loop) + # when there are new nonants (in the same loop) logger.debug(f' *Xhatshuffle loop iter={xh_iter}') logger.debug(f' *got a new one! on rank {self.global_rank}') logger.debug(f' *localnonants={str(self.localnonants)}') diff --git a/mpisppy/cylinders/xhatspecific_bounder.py b/mpisppy/cylinders/xhatspecific_bounder.py index 7fed91d75..083910409 100644 --- a/mpisppy/cylinders/xhatspecific_bounder.py +++ b/mpisppy/cylinders/xhatspecific_bounder.py @@ -50,7 +50,7 @@ def main(self): logging.debug(' IB got from opt on global rank {}'.\ format(global_rank)) - if (self.new_nonants): + if self.update_nonants(): logging.debug(' and its new! on global rank {}'.\ format(global_rank)) logging.debug(' localnonants={}'.format(str(self.localnonants))) diff --git a/mpisppy/cylinders/xhatxbar_bounder.py b/mpisppy/cylinders/xhatxbar_bounder.py index 39f3716f7..33c8bfa73 100644 --- a/mpisppy/cylinders/xhatxbar_bounder.py +++ b/mpisppy/cylinders/xhatxbar_bounder.py @@ -69,7 +69,7 @@ def main(self): logging.debug(' IB got from opt on global rank {}'.\ format(global_rank)) - if (self.new_nonants): + if self.update_nonants(): logging.debug(' and its new! on global rank {}'.\ format(global_rank)) logging.debug(' localnonants={}'.format(str(self.localnonants))) diff --git a/mpisppy/extensions/phtracker.py b/mpisppy/extensions/phtracker.py index 9c800bbbe..c1862cd1b 100644 --- a/mpisppy/extensions/phtracker.py +++ b/mpisppy/extensions/phtracker.py @@ -286,6 +286,7 @@ def _add_data_and_write(self, track_var, data, gather=True, final=False): def _get_bounds(self): spoke_bound = None if isinstance(self.spcomm, Spoke): + self.spcomm.update_hub_bounds() hub_inner_bound = self.spcomm.hub_inner_bound hub_outer_bound = self.spcomm.hub_outer_bound spoke_bound = self.spcomm.bound From f270c3f4fa5955f67b02bc19c729c7823c08ef9a Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Mon, 31 Mar 2025 12:37:44 -0600 Subject: [PATCH 25/36] terminate even if signal is stale; faster termination for shuffle bounder --- mpisppy/cylinders/spoke.py | 2 +- mpisppy/cylinders/xhatshufflelooper_bounder.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/mpisppy/cylinders/spoke.py b/mpisppy/cylinders/spoke.py index a426a6fd4..b1849e952 100644 --- a/mpisppy/cylinders/spoke.py +++ b/mpisppy/cylinders/spoke.py @@ -27,7 +27,7 @@ def got_kill_signal(self): """ shutdown_buf = self.receive_buffers[self._make_key(Field.SHUTDOWN, 0)] self.get_receive_buffer(shutdown_buf, Field.SHUTDOWN, 0) - return (shutdown_buf.is_new() and shutdown_buf[0] == 1.0) + return shutdown_buf[0] == 1.0 @abc.abstractmethod def main(self): diff --git a/mpisppy/cylinders/xhatshufflelooper_bounder.py b/mpisppy/cylinders/xhatshufflelooper_bounder.py index 9c29074fa..7f5a7f365 100644 --- a/mpisppy/cylinders/xhatshufflelooper_bounder.py +++ b/mpisppy/cylinders/xhatshufflelooper_bounder.py @@ -136,6 +136,10 @@ def _vb(msg): _vb(f" Updating best to {next_scendict}") scenario_cycler.best = next_scendict["ROOT"] + if self.got_kill_signal(): + # time to go; don't solve next + return + next_scendict = scenario_cycler.get_next() if next_scendict is not None: _vb(f" Trying next {next_scendict}") From 1ca8ff9271fb53ea146677733db762df70a21f83 Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Mon, 31 Mar 2025 15:12:55 -0600 Subject: [PATCH 26/36] re-do termination synchronization --- mpisppy/cylinders/spoke.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mpisppy/cylinders/spoke.py b/mpisppy/cylinders/spoke.py index b1849e952..de172548c 100644 --- a/mpisppy/cylinders/spoke.py +++ b/mpisppy/cylinders/spoke.py @@ -26,8 +26,8 @@ def got_kill_signal(self): to see if the Hub terminated """ shutdown_buf = self.receive_buffers[self._make_key(Field.SHUTDOWN, 0)] - self.get_receive_buffer(shutdown_buf, Field.SHUTDOWN, 0) - return shutdown_buf[0] == 1.0 + self.get_receive_buffer(shutdown_buf, Field.SHUTDOWN, 0, synchronize=False) + return self.allreduce_or(shutdown_buf[0] == 1.0) @abc.abstractmethod def main(self): From 252166c44d4ef16ac42b955a2ee06b5133819ebc Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Tue, 1 Apr 2025 12:09:25 -0600 Subject: [PATCH 27/36] refactor bounds sync to base class --- mpisppy/cylinders/hub.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/mpisppy/cylinders/hub.py b/mpisppy/cylinders/hub.py index 22d3b3fa9..39a29ea77 100644 --- a/mpisppy/cylinders/hub.py +++ b/mpisppy/cylinders/hub.py @@ -291,6 +291,11 @@ def send_terminate(self): self.put_send_buffer(self.send_buffers[Field.SHUTDOWN], Field.SHUTDOWN) return + def sync_bounds(self): + self.receive_outerbounds() + self.receive_innerbounds() + self.send_boundsout() + class PHHub(Hub): @@ -317,22 +322,14 @@ def sync(self): """ Manages communication with Spokes """ - self.send_ws() - self.send_nonants() - self.send_boundsout() - self.receive_outerbounds() - self.receive_innerbounds() - if self.opt.extensions is not None: - self.opt.extobject.sync_with_spokes() + self.sync_Ws() + self.sync_nonants() + self.sync_bounds() + self.sync_extensions() def sync_with_spokes(self): self.sync() - def sync_bounds(self): - self.receive_outerbounds() - self.receive_innerbounds() - self.send_boundsout() - def sync_extensions(self): if self.opt.extensions is not None: self.opt.extobject.sync_with_spokes() @@ -431,8 +428,7 @@ def sync(self, send_nonants=True): """ if send_nonants: self.send_nonants() - self.receive_outerbounds() - self.receive_innerbounds() + self.sync_bounds() # in case LShaped ever gets extensions if getattr(self.opt, "extensions", None) is not None: self.opt.extobject.sync_with_spokes() From 7a6dbfc256599b83369e239baa14d1a5ee1d542d Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Wed, 2 Apr 2025 12:51:10 -0600 Subject: [PATCH 28/36] remove bound setter; replace with send_bound --- mpisppy/cylinders/fwph_spoke.py | 4 ++-- mpisppy/cylinders/lagranger_bounder.py | 4 ++-- mpisppy/cylinders/lagrangian_bounder.py | 6 +++--- mpisppy/cylinders/ph_ob.py | 6 +++--- mpisppy/cylinders/spoke.py | 15 +++++++-------- mpisppy/cylinders/subgradient_bounder.py | 4 ++-- 6 files changed, 19 insertions(+), 20 deletions(-) diff --git a/mpisppy/cylinders/fwph_spoke.py b/mpisppy/cylinders/fwph_spoke.py index 1530c514f..e1ac34585 100644 --- a/mpisppy/cylinders/fwph_spoke.py +++ b/mpisppy/cylinders/fwph_spoke.py @@ -25,7 +25,7 @@ def sync(self): if not hasattr(self.opt, '_local_bound'): return # Tell the hub about the most recent bound - self.bound = self.opt._local_bound + self.send_bound(self.opt._local_bound) def finalize(self): # The FWPH spoke can call "finalize" before it @@ -34,6 +34,6 @@ def finalize(self): # if we terminated early if not hasattr(self.opt, '_local_bound'): return - self.bound = self.opt._local_bound + self.send_bound(self.opt._local_bound) self.final_bound = self.opt._local_bound return self.final_bound diff --git a/mpisppy/cylinders/lagranger_bounder.py b/mpisppy/cylinders/lagranger_bounder.py index b4a893dbd..5e97bc83f 100644 --- a/mpisppy/cylinders/lagranger_bounder.py +++ b/mpisppy/cylinders/lagranger_bounder.py @@ -91,7 +91,7 @@ def main(self): if extensions: self.opt.extobject.post_iter0() - self.bound = self.trivial_bound + self.send_bound(self.trivial_bound) if extensions: self.opt.extobject.post_iter0_after_sync() @@ -103,7 +103,7 @@ def main(self): if extensions: self.opt.extobject.enditer() if bound is not None: - self.bound = bound + self.send_bound(bound) if extensions: self.opt.extobject.enditer_after_sync() self.A_iter += 1 diff --git a/mpisppy/cylinders/lagrangian_bounder.py b/mpisppy/cylinders/lagrangian_bounder.py index dc1548f0e..fe7468dca 100644 --- a/mpisppy/cylinders/lagrangian_bounder.py +++ b/mpisppy/cylinders/lagrangian_bounder.py @@ -73,7 +73,7 @@ def main(self, need_solution=False): self.opt.current_solver_options = self.opt.iterk_solver_options - self.bound = self.trivial_bound + self.send_bound(self.trivial_bound) if extensions: self.opt.extobject.post_iter0_after_sync() @@ -85,7 +85,7 @@ def main(self, need_solution=False): if extensions: self.opt.extobject.enditer() if bound is not None: - self.bound = bound + self.send_bound(bound) if extensions: self.opt.extobject.enditer_after_sync() self.dk_iter += 1 @@ -95,4 +95,4 @@ def main(self, need_solution=False): self.opt.Update_W(verbose) bound = self.lagrangian(need_solution=need_solution) if bound is not None: - self.bound = bound + self.send_bound(bound) diff --git a/mpisppy/cylinders/ph_ob.py b/mpisppy/cylinders/ph_ob.py index e22d0ee6f..7c1f077e1 100644 --- a/mpisppy/cylinders/ph_ob.py +++ b/mpisppy/cylinders/ph_ob.py @@ -150,7 +150,7 @@ def main(self): self._rescale_rho(self.opt.options["ph_ob_initial_rho_rescale_factor"] ) self.trivial_bound = self._phsolve(0) - self.bound = self.trivial_bound + self.send_bound(self.trivial_bound) self.opt.current_solver_options = self.opt.iterk_solver_options self.B_iter = 1 @@ -159,7 +159,7 @@ def main(self): while not self.got_kill_signal(): # because of aph, do not check for new data, just go for it - self.bound = self._update_weights_and_solve(self.B_iter) + self.send_bound(self._update_weights_and_solve(self.B_iter)) self.B_iter += 1 self.opt.B_iter = self.B_iter wtracker.grab_local_Ws() @@ -171,5 +171,5 @@ def finalize(self): and/or iteration limit is the cause of termination ''' self.final_bound = self._update_weights_and_solve(self.B_iter) - self.bound = self.final_bound + self.send_bound(self.final_bound) return self.final_bound diff --git a/mpisppy/cylinders/spoke.py b/mpisppy/cylinders/spoke.py index de172548c..7bbe7d073 100644 --- a/mpisppy/cylinders/spoke.py +++ b/mpisppy/cylinders/spoke.py @@ -84,8 +84,7 @@ def bound_type(self) -> Field: def bound(self): return self._bound[0] - @bound.setter - def bound(self, value): + def send_bound(self, value): self._append_trace(value) self._bound[0] = value self.put_send_buffer(self._bound, self.bound_type()) @@ -139,7 +138,7 @@ def _update_nonant_len_buffer(self) -> bool: class InnerBoundSpoke(_BoundSpoke): - """ For Spokes that provide an inner bound through self.bound to the + """ For Spokes that provide an inner bound through self.send_bound to the Hub, and do not need information from the main PH OPT hub. """ @@ -153,7 +152,7 @@ def bound_type(self) -> Field: class OuterBoundSpoke(_BoundSpoke): - """ For Spokes that provide an outer bound through self.bound to the + """ For Spokes that provide an outer bound through self.send_bound to the Hub, and do not need information from the main PH OPT hub. """ @@ -190,7 +189,7 @@ def update_Ws(self) -> bool: class OuterBoundWSpoke(_BoundWSpoke): """ For Spokes that provide an outer bound - through self.bound to the Hub, + through self.send_bound to the Hub, and receive the Ws (or weights) from the main PH OPT hub. """ @@ -227,7 +226,7 @@ def update_nonants(self) -> bool: class InnerBoundNonantSpoke(_BoundNonantSpoke): """ For Spokes that provide an inner (incumbent) - bound through self.bound to the Hub, + bound through self.send_bound to the Hub, and receive the nonants from the main SPOpt hub. @@ -257,7 +256,7 @@ def update_if_improving(self, candidate_inner_bound, update_best_solution_cache= if update: self.best_inner_bound = candidate_inner_bound # send to hub - self.bound = candidate_inner_bound + self.send_bound(candidate_inner_bound) return True return False @@ -274,7 +273,7 @@ def bound_type(self) -> Field: class OuterBoundNonantSpoke(_BoundNonantSpoke): """ For Spokes that provide an outer - bound through self.bound to the Hub, + bound through self.send_bound to the Hub, and receive the nonants from the main OPT hub. """ diff --git a/mpisppy/cylinders/subgradient_bounder.py b/mpisppy/cylinders/subgradient_bounder.py index fc125c23b..612afb8e7 100644 --- a/mpisppy/cylinders/subgradient_bounder.py +++ b/mpisppy/cylinders/subgradient_bounder.py @@ -26,7 +26,7 @@ def main(self): if extensions: self.opt.extobject.post_iter0() - self.bound = self.trivial_bound + self.send_bound(self.trivial_bound) if extensions: self.opt.extobject.post_iter0_after_sync() @@ -49,6 +49,6 @@ def main(self): if extensions: self.opt.extobject.enditer() if bound is not None: - self.bound = bound + self.send_bound(bound) if extensions: self.opt.extobject.enditer_after_sync() From 80c570cc0d4900613f2d9ca1bb43ac19e27e6a83 Mon Sep 17 00:00:00 2001 From: bknueven <30801372+bknueven@users.noreply.github.com> Date: Fri, 11 Apr 2025 09:26:26 -0600 Subject: [PATCH 29/36] Fix bool inversion bug --- mpisppy/cylinders/spcommunicator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mpisppy/cylinders/spcommunicator.py b/mpisppy/cylinders/spcommunicator.py index d8275322e..a4d404cc8 100644 --- a/mpisppy/cylinders/spcommunicator.py +++ b/mpisppy/cylinders/spcommunicator.py @@ -356,7 +356,7 @@ def get_receive_buffer(self, is_new (bool): Indicates whether the "gotten" values are new, based on the write_id. """ - if not synchronize: + if synchronize: self.cylinder_comm.Barrier() last_id = buf.id() From 89032c1868e63c57049dfafe6f52f1b4f0bb2ae3 Mon Sep 17 00:00:00 2001 From: bknueven <30801372+bknueven@users.noreply.github.com> Date: Fri, 11 Apr 2025 09:26:57 -0600 Subject: [PATCH 30/36] Use // for integer division Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- mpisppy/cylinders/spcommunicator.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mpisppy/cylinders/spcommunicator.py b/mpisppy/cylinders/spcommunicator.py index a4d404cc8..6f87654b9 100644 --- a/mpisppy/cylinders/spcommunicator.py +++ b/mpisppy/cylinders/spcommunicator.py @@ -371,8 +371,7 @@ def get_receive_buffer(self, self.cylinder_comm.Allreduce((local_val, MPI.INT), (sum_ids, MPI.INT), op=MPI.SUM) - if new_id != sum_ids[0] / self.cylinder_comm.size: - buf._is_new = False + if new_id != sum_ids[0] // self.cylinder_comm.size: return False if new_id > last_id: From 06292c2a07e39091b40105fdfecd63050740336d Mon Sep 17 00:00:00 2001 From: bknueven <30801372+bknueven@users.noreply.github.com> Date: Fri, 11 Apr 2025 12:22:13 -0600 Subject: [PATCH 31/36] Better final message --- mpisppy/spin_the_wheel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mpisppy/spin_the_wheel.py b/mpisppy/spin_the_wheel.py index 6d7562ee1..af5737c46 100644 --- a/mpisppy/spin_the_wheel.py +++ b/mpisppy/spin_the_wheel.py @@ -153,7 +153,7 @@ def run(self, comm_world=None): spcomm.free_windows() fullcomm.Barrier() - global_toc("Finalize Complete") + global_toc("Cylinder finalization complete") self.spcomm = spcomm From a32ae609a6c4cf8c6928a6667e6905b69b1cd050 Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Thu, 17 Apr 2025 13:58:39 -0600 Subject: [PATCH 32/36] Revert "Use // for integer division" This reverts commit 89032c1868e63c57049dfafe6f52f1b4f0bb2ae3. --- mpisppy/cylinders/spcommunicator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mpisppy/cylinders/spcommunicator.py b/mpisppy/cylinders/spcommunicator.py index 6f87654b9..a4d404cc8 100644 --- a/mpisppy/cylinders/spcommunicator.py +++ b/mpisppy/cylinders/spcommunicator.py @@ -371,7 +371,8 @@ def get_receive_buffer(self, self.cylinder_comm.Allreduce((local_val, MPI.INT), (sum_ids, MPI.INT), op=MPI.SUM) - if new_id != sum_ids[0] // self.cylinder_comm.size: + if new_id != sum_ids[0] / self.cylinder_comm.size: + buf._is_new = False return False if new_id > last_id: From a7ae1f1b5c6fc358cf3854924f4673358b4d1308 Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Thu, 17 Apr 2025 14:12:37 -0600 Subject: [PATCH 33/36] re-write sync check --- mpisppy/cylinders/spcommunicator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mpisppy/cylinders/spcommunicator.py b/mpisppy/cylinders/spcommunicator.py index a4d404cc8..00be40ead 100644 --- a/mpisppy/cylinders/spcommunicator.py +++ b/mpisppy/cylinders/spcommunicator.py @@ -371,7 +371,7 @@ def get_receive_buffer(self, self.cylinder_comm.Allreduce((local_val, MPI.INT), (sum_ids, MPI.INT), op=MPI.SUM) - if new_id != sum_ids[0] / self.cylinder_comm.size: + if new_id * self.cylinder_comm.size != sum_ids[0]: buf._is_new = False return False From c6f41cba95d72beb3a0248b224f02602416b62dd Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Thu, 17 Apr 2025 15:19:10 -0600 Subject: [PATCH 34/36] disable prox for one iteration after enforcing integers --- mpisppy/extensions/integer_relax_then_enforce.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mpisppy/extensions/integer_relax_then_enforce.py b/mpisppy/extensions/integer_relax_then_enforce.py index 2be92d834..125f4c1ca 100644 --- a/mpisppy/extensions/integer_relax_then_enforce.py +++ b/mpisppy/extensions/integer_relax_then_enforce.py @@ -31,6 +31,7 @@ def pre_iter0(self): for s in self.opt.local_scenarios.values(): self.integer_relaxer.apply_to(s) self._integers_relaxed = True + self._reenable_prox = False def _unrelax_integers(self): for sub in self.opt.local_subproblems.values(): @@ -45,8 +46,14 @@ def _unrelax_integers(self): for v in vlist: subproblem_solver.update_var(v) self._integers_relaxed = False + if not self.opt.prox_disabled: + self.opt._disable_prox() + self._reenable_prox = True def miditer(self): + if self._reenable_prox: + self.opt._reenable_prox() + self._reenable_prox = False if not self._integers_relaxed: return # time is running out From 7f0ba375798879b0f874da7d1f71f8888a1a693c Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Sat, 19 Apr 2025 15:33:42 -0600 Subject: [PATCH 35/36] adding rounding bias --- mpisppy/cylinders/slam_heuristic.py | 3 ++- mpisppy/extensions/xhatxbar.py | 3 ++- mpisppy/spopt.py | 6 ++++-- mpisppy/utils/cfg_vanilla.py | 1 + mpisppy/utils/config.py | 6 ++++++ mpisppy/utils/xhat_eval.py | 6 ++++-- 6 files changed, 19 insertions(+), 6 deletions(-) diff --git a/mpisppy/cylinders/slam_heuristic.py b/mpisppy/cylinders/slam_heuristic.py index b2977b829..2ca9e067e 100644 --- a/mpisppy/cylinders/slam_heuristic.py +++ b/mpisppy/cylinders/slam_heuristic.py @@ -89,6 +89,7 @@ def main(self): candidate = global_candidate.round() ''' + rounding_bias = self.opt.options.get("rounding_bias", 0.0) # Everyone has the candidate solution at this point for s in self.opt.local_scenarios.values(): is_pers = sputils.is_persistent(s._solver_plugin) @@ -99,7 +100,7 @@ def main(self): continue val = global_candidate[ix] if var.is_binary() or var.is_integer(): - val = round(val) + val = round(val + rounding_bias) var.fix(val) if (is_pers): solver.update_var(var) diff --git a/mpisppy/extensions/xhatxbar.py b/mpisppy/extensions/xhatxbar.py index a6ee4524d..33ad6506c 100644 --- a/mpisppy/extensions/xhatxbar.py +++ b/mpisppy/extensions/xhatxbar.py @@ -39,6 +39,7 @@ def _fix_nonants_xhat(self): You probably want to call _save_nonants right before calling this copy/pasted from phabse _fix_nonants """ + rounding_bias = self.opt.options.get("rounding_bias", 0.0) for k,s in self.opt.local_scenarios.items(): persistent_solver = None if (sputils.is_persistent(s._solver_plugin)): @@ -52,7 +53,7 @@ def _fix_nonants_xhat(self): if this_vardata in node.surrogate_vardatas: continue if this_vardata.is_integer() or this_vardata.is_binary(): - this_vardata._value = round(s._mpisppy_model.xbars[(ndn,i)]._value) + this_vardata._value = round(s._mpisppy_model.xbars[(ndn,i)]._value + rounding_bias) else: this_vardata._value = s._mpisppy_model.xbars[(ndn,i)]._value diff --git a/mpisppy/spopt.py b/mpisppy/spopt.py index 87d5e8159..f03cc3e8a 100644 --- a/mpisppy/spopt.py +++ b/mpisppy/spopt.py @@ -613,6 +613,7 @@ def _fix_nonants(self, cache): NOTE: You probably want to call _save_nonants right before calling this """ + rounding_bias = self.options.get("rounding_bias", 0.0) for k,s in self.local_scenarios.items(): persistent_solver = None @@ -635,7 +636,7 @@ def _fix_nonants(self, cache): if this_vardata in node.surrogate_vardatas: continue if this_vardata.is_binary() or this_vardata.is_integer(): - this_vardata._value = round(cache[ndn][i]) + this_vardata._value = round(cache[ndn][i] + rounding_bias) else: this_vardata._value = cache[ndn][i] this_vardata.fix() @@ -659,6 +660,7 @@ def _fix_root_nonants(self,root_cache): NOTE: You probably want to call _save_nonants right before calling this """ + rounding_bias = self.options.get("rounding_bias", 0.0) for k,s in self.local_scenarios.items(): persistent_solver = None @@ -687,7 +689,7 @@ def _fix_root_nonants(self,root_cache): if this_vardata in node.surrogate_vardatas: continue if this_vardata.is_binary() or this_vardata.is_integer(): - this_vardata._value = round(root_cache[i]) + this_vardata._value = round(root_cache[i] + rounding_bias) else: this_vardata._value = root_cache[i] this_vardata.fix() diff --git a/mpisppy/utils/cfg_vanilla.py b/mpisppy/utils/cfg_vanilla.py index 35d9d6f69..53a6fcd84 100644 --- a/mpisppy/utils/cfg_vanilla.py +++ b/mpisppy/utils/cfg_vanilla.py @@ -67,6 +67,7 @@ def shared_options(cfg): "tee-rank0-solves": cfg.tee_rank0_solves, "trace_prefix" : cfg.trace_prefix, "presolve" : cfg.presolve, + "rounding_bias" : cfg.rounding_bias, } if _hasit(cfg, "max_solver_threads"): shoptions["iter0_solver_options"]["threads"] = cfg.max_solver_threads diff --git a/mpisppy/utils/config.py b/mpisppy/utils/config.py index 953f3cfe9..09c272f84 100644 --- a/mpisppy/utils/config.py +++ b/mpisppy/utils/config.py @@ -247,6 +247,12 @@ def popular_args(self): domain=bool, default=False) + self.add_to_config("rounding_bias", + description="When rounding variables to integers, " + "add the given value first. (default = 0.0)", + domain=float, + default=0.0) + def ph_args(self): self.add_to_config("linearize_binary_proximal_terms", description="For PH, linearize the proximal terms for " diff --git a/mpisppy/utils/xhat_eval.py b/mpisppy/utils/xhat_eval.py index 58a665bd2..77137daed 100644 --- a/mpisppy/utils/xhat_eval.py +++ b/mpisppy/utils/xhat_eval.py @@ -300,6 +300,7 @@ def fix_nonants_upto_stage(self,t,cache): You probably want to call _save_nonants right before calling this """ self._lazy_create_solvers() + rounding_bias = self.options.get("rounding_bias", 0.0) for k,s in self.local_scenarios.items(): persistent_solver = None @@ -323,7 +324,7 @@ def fix_nonants_upto_stage(self,t,cache): if this_vardata in node.surrogate_vardatas: continue if this_vardata.is_binary() or this_vardata.is_integer(): - this_vardata._value = round(cache[ndn][i]) + this_vardata._value = round(cache[ndn][i] + rounding_bias) else: this_vardata._value = cache[ndn][i] this_vardata.fix() @@ -339,6 +340,7 @@ def _fix_nonants_at_value(self): Loop over the scenarios to restore, but loop over subproblems to alert persistent solvers. """ + rounding_bias = self.options.get("rounding_bias", 0.0) for k,s in self.local_scenarios.items(): persistent_solver = None @@ -350,7 +352,7 @@ def _fix_nonants_at_value(self): if var in s._mpisppy_data.all_surrogate_nonants: continue if var.is_binary() or var.is_integer(): - var._value = round(var._value) + var._value = round(var._value + rounding_bias) var.fix() if not self.bundling and persistent_solver is not None: persistent_solver.update_var(var) From c39a1cdea3db248397a71dff7a2eb1ce5a1019a4 Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Tue, 22 Apr 2025 13:43:26 -0600 Subject: [PATCH 36/36] Revert "disable prox for one iteration after enforcing integers" This reverts commit c6f41cba95d72beb3a0248b224f02602416b62dd. --- mpisppy/extensions/integer_relax_then_enforce.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/mpisppy/extensions/integer_relax_then_enforce.py b/mpisppy/extensions/integer_relax_then_enforce.py index 125f4c1ca..2be92d834 100644 --- a/mpisppy/extensions/integer_relax_then_enforce.py +++ b/mpisppy/extensions/integer_relax_then_enforce.py @@ -31,7 +31,6 @@ def pre_iter0(self): for s in self.opt.local_scenarios.values(): self.integer_relaxer.apply_to(s) self._integers_relaxed = True - self._reenable_prox = False def _unrelax_integers(self): for sub in self.opt.local_subproblems.values(): @@ -46,14 +45,8 @@ def _unrelax_integers(self): for v in vlist: subproblem_solver.update_var(v) self._integers_relaxed = False - if not self.opt.prox_disabled: - self.opt._disable_prox() - self._reenable_prox = True def miditer(self): - if self._reenable_prox: - self.opt._reenable_prox() - self._reenable_prox = False if not self._integers_relaxed: return # time is running out