Grad rho (#559)

* update farmer rho demo

* update UC example

* update docs

* convert config / generic_cylinders

* Update grad_rho.rst

* very minimal test for grad_rho

* ran ruff locally

---------

Co-authored-by: David L Woodruff <[email protected]>
Co-authored-by: Dave Woodruff <[email protected]>

Author: bknueven

.github/workflows/test_pr_and_main.yml

@@ -429,11 +429,6 @@ jobs:
         run: |
           pip install -e .
 
-      - name: Build Pyomo extensions
-        run: |
-          # some failures are expected, but this should succeed as long as pynumero is built correctly
-          pyomo build-extensions || python -c "from pyomo.contrib.pynumero.asl import AmplInterface; exit(0) if AmplInterface.available() else exit(1)"
-
       - name: run farmer tests
         timeout-minutes: 10
         run: |

doc/src/grad_rho.rst

@@ -1,77 +1,13 @@
 Gradient-based rho
 ==================
+You can find a detailed example using this code in ``examples.farmer.CI.farmer_rho_demo.py``.
 
-``mpisppy.utils.gradient.py`` computes rho 
-using the gradient and writes them in a file. 
+Grad-rho is also supported in ``generic_cylinders.py``.
 
-.. Note::
-   This only works for two-stage problems for now.
+There are options in ``cfg`` to control dynamic updates:
 
-To compute gradient-based rho, you can call ``grad_cost_and_rho`` 
-after creating your config object. 
-You will need to add ``gradient_args`` when you set up the config object.
-For example with farmer:
-
-.. code-block:: python
-
-   import mpisppy.utils.gradient as grad
-
-   def _parse_args():
-      cfg = config.Config()
-      ...
-      cfg.gradient_args()
-
-   def main():
-      ... #after the config object is created
-      grad.grad_cost_and_rho('farmer', cfg)
-
-.. Note::
-   This will write a rho file (resp. grad cost file) 
-   only if you include ``--grad-rho-file`` (resp. ``--grad-cost-file``) 
-   in your bash script.
-
-You can find a detailed example using this code in ``examples.farmer.farmer_rho_demo.py``.
-
-
-compute_grad
-------------
-
-This function computes the gradient of the objective function for each scenario. 
-It will write the resulting gradient costs in a csv file
-containing each scenario name, variable name and the corresponding value.
-
-To use it you should include the following in your bash script.
-
-.. code-block:: bash
-
-   --xhatpath #path of your xhat file (.npy)
-   --grad-cost-file #file where gradient costs will be written
-
-
-find_grad_rho
--------------
-
-This function computes rhos for each scenario and variable 
-using the previously computed gradient costs.
-The rho values depend on both the scenario and the variable: 
-``compute_rhos`` returns a dictionnary with the variable names
-and a list of corresponding rho values for each scenario.
-
-
-grad_cost_and_rho
------------------
-
-This function computes a rho for each variable using the dictionnary 
-returned by ``find_grad_rho``.
-To do so, it uses an order statistic which you should set with ``--order-stat``. 
-It needs to be a float between 0 and 1: 0 (resp. 1, 0.5) 
-corresponding to the min (resp. max, average).
-It will write the resulting rhos in a csv file
-containing each variable name and the corresponding value.
-
-To use it you should include the following in your bash script.
-
-.. code-block:: bash
-
-   --grad-rho-file #file where gradient rhos will be written
-   --order-stat #float between 0 and 1
+  * `--dynamic-rho-primal-crit` and `--dynamic-rho-dual-crit` are booleans that trigger dynamic rho
+  * `--dynamic-rho-primal-thresh` and `--dynamic-rho-dual-thresh` control how sensitive the trigger is.
+    They have default values, so do not need to be set. See
+   ``dyn_rho_base.py`` to see how the update is done if you don't like the default values.
+  * `--grad-rho-multiplier` is a cummulative multiplier when rho is set or updated. 

examples/farmer/CI/farmer_rho_demo.bash

@@ -2,11 +2,7 @@
 
 SOLVERNAME="cplex"
 
-# get an xhat
-# xhat output file name is hardwired to 'farmer_cyl_nonants.npy'
-mpiexec -np 3 python -m mpi4py farmer_cylinders.py  --num-scens 3 --lagrangian --xhatshuffle --bundles-per-rank=0 --max-iterations=50 --default-rho=1 --solver-name=${SOLVERNAME}
-
-mpiexec -np 3 python -m mpi4py farmer_rho_demo.py --num-scens 3 --bundles-per-rank=0 --max-iterations=100 --default-rho=1 --solver-name=${SOLVERNAME} --xhatpath=./farmer_cyl_nonants.npy --grad-order-stat 0.5 --xhatshuffle --lagrangian --max-stalled-iters 5000 --grad-rho-setter --rel-gap 0.001
+mpiexec -np 3 python -m mpi4py farmer_rho_demo.py --num-scens 3 --bundles-per-rank=0 --max-iterations=100 --default-rho=1 --solver-name=${SOLVERNAME} --grad-order-stat 0.5 --xhatshuffle --lagrangian --max-stalled-iters 5000 --grad-rho --rel-gap 0.001
 
 #--rho-relative-bound
 #--grad-rho-file=./grad_rhos_demo.csv --grad-cost-file=./grad_cost_demo.csv --whatpath=./grad_cost_demo.csv --order-stat=0.5

examples/farmer/CI/farmer_rho_demo.py

@@ -26,7 +26,7 @@
 
 from mpisppy.extensions.norm_rho_updater import NormRhoUpdater
 from mpisppy.convergers.norm_rho_converger import NormRhoConverger
-from mpisppy.extensions.gradient_extension import Gradient_extension
+from mpisppy.extensions.grad_rho import GradRho
 
 write_solution = False
 
@@ -44,7 +44,7 @@ def _parse_args():
     cfg.lagrangian_args()
     cfg.lagranger_args()
     cfg.xhatshuffle_args()
-    cfg.dynamic_gradient_args() # gets gradient args for free
+    cfg.dynamic_rho_args() # gets gradient args for free
     cfg.add_to_config("crops_mult",
                          description="There will be 3x this many crops (default 1)",
                          domain=int,
@@ -102,8 +102,6 @@ def main():
     beans = (cfg, scenario_creator, scenario_denouement, all_scenario_names)
 
     ext_classes = []
-    if cfg.grad_rho:
-        ext_classes.append(Gradient_extension)
 
     if cfg.run_async:
         raise RuntimeError("APH not supported in this example.")
@@ -119,8 +117,8 @@ def main():
 
     #gradient extension kwargs
     if cfg.grad_rho:
-        ext_classes.append(Gradient_extension)        
-        hub_dict['opt_kwargs']['options']['gradient_extension_options'] = {'cfg': cfg}
+        ext_classes.append(GradRho)
+        hub_dict['opt_kwargs']['options']['grad_rho_options'] = {'cfg': cfg}
 
     ## Gabe's (way pre-pandemic) adaptive rho
     if cfg.use_norm_rho_updater:

examples/uc/gradient_uc_cylinders.py

@@ -22,7 +22,7 @@
 from mpisppy.extensions.fixer import Fixer
 from mpisppy.extensions.mipgapper import Gapper
 from mpisppy.extensions.xhatclosest import XhatClosest
-from mpisppy.extensions.gradient_extension import Gradient_extension
+from mpisppy.extensions.grad_rho import GradRho
 from mpisppy.utils import config
 import mpisppy.utils.cfg_vanilla as vanilla
 from mpisppy.extensions.cross_scen_extension import CrossScenarioExtension
@@ -40,7 +40,9 @@ def _parse_args():
     cfg.xhatlooper_args()
     cfg.xhatshuffle_args()
     cfg.cross_scenario_cuts_args()
-    cfg.dynamic_gradient_args() # gets you gradient_args for free
+    cfg.dynamic_rho_args() # gets gradient args for free
+    cfg.ph_ob_args()
+    cfg.integer_relax_then_enforce_args()
     cfg.add_to_config("ph_mipgaps_json",
                          description="json file with mipgap schedule (default None)",
                          domain=str,
@@ -123,7 +125,8 @@ def main():
         ext_classes.append(XhatClosest)
 
     if cfg.grad_rho:
-        ext_classes.append(Gradient_extension)
+        ext_classes.append(GradRho)
+        hub_dict['opt_kwargs']['options']['grad_rho_options'] = {'cfg': cfg}
 
     hub_dict["opt_kwargs"]["extension_kwargs"] = {"ext_classes" : ext_classes}
     if cross_scenario_cuts:
@@ -142,9 +145,6 @@ def main():
             "keep_solution" : True
         }
 
-    if cfg.grad_rho:
-        hub_dict['opt_kwargs']['options']['gradient_extension_options'] = {'cfg': cfg}
-
     if cfg.ph_mipgaps_json is not None:
         with open(cfg.ph_mipgaps_json) as fin:
             din = json.load(fin)
@@ -159,6 +159,9 @@ def main():
     if cfg.default_rho is None:
         # since we are using a rho_setter anyway
         hub_dict.opt_kwcfg.options["defaultPHrho"] = 1  
+
+    if cfg.integer_relax_then_enforce:
+        vanilla.add_integer_relax_then_enforce(hub_dict, cfg)
     ### end ph spoke ###
 
     # FWPH spoke
@@ -177,6 +180,7 @@ def main():
 
     # ph outer bounder spoke removed August 2025
 
+
     # xhat shuffle bound spoke
     if xhatshuffle:
         xhatshuffle_spoke = vanilla.xhatshuffle_spoke(*beans, scenario_creator_kwargs=scenario_creator_kwargs)

examples/uc/gradient_uc_test.bash

@@ -1,20 +1,5 @@
 #!/bin/bash
 
-SOLVERNAME="cplex"
+SOLVERNAME="xpress_direct"
 
-echo "First solve the EF to get a npy file for xhat ; the uc_cyl_nonants.npy name is hard-wired (you don't really such a good xhat, but this one is just easy to get)"
-python simple_ef.py $SOLVERNAME
-
-echo "Now run the cylinders"
-mpiexec --oversubscribe -np 3 python -m mpi4py gradient_uc_cylinders.py --xhatshuffle --ph-ob --bundles-per-rank=0 --max-iterations=5 --default-rho=1 --num-scens=5 --max-solver-threads=2 --lagrangian-iter0-mipgap=1e-7 --ph-mipgaps-json=phmipgaps.json --solver-name=${SOLVERNAME} --xhatpath uc_cyl_nonants.npy --rel-gap 0.000001 --display-progress --grad-rho-setter --grad-order-stat 0.5
-
-####mpiexec --oversubscribe -np 1 python -m mpi4py gradient_uc_cylinders.py --bundles-per-rank=0 --max-iterations=20 --default-rho=1 --num-scens=5 --max-solver-threads=2 --lagrangian-iter0-mipgap=1e-7 --ph-mipgaps-json=phmipgaps.json --solver-name=${SOLVERNAME} --xhatpath uc_cyl_nonants.npy --rel-gap 0.000001 --display-progress --grad-rho-setter --grad-order-stat 0.5
-
-
-# ? do we need ph_ob_rho_rescale_factors_json", ph_ob_gradient_rho", ??
-
-# --fwph
-echo
-
-# --rho-setter --order-stat 0.5
-# --display-progress
+mpiexec -np 3 python -m mpi4py gradient_uc_cylinders.py --xhatshuffle --lagrangian --bundles-per-rank=0 --max-iterations=10 --default-rho=1 --num-scens=5 --max-solver-threads=2 --lagrangian-iter0-mipgap=1e-7 --ph-mipgaps-json=phmipgaps.json --solver-name=${SOLVERNAME} --rel-gap 0.000001 --display-progress --grad-rho --grad-order-stat 0.5 --integer-relax-then-enforce --integer-relax-then-enforce-ratio=0.9

mpisppy/extensions/gradient_extension.py

@@ -28,7 +28,7 @@
 from mpisppy.extensions.extension import MultiExtension, Extension
 from mpisppy.extensions.norm_rho_updater import NormRhoUpdater
 from mpisppy.extensions.primal_dual_rho import PrimalDualRho
-from mpisppy.extensions.gradient_extension import Gradient_extension
+from mpisppy.extensions.grad_rho import GradRho
 from mpisppy.extensions.scenario_lp_mps_files import Scenario_lp_mps_files
 
 from mpisppy.utils.wxbarwriter import WXBarWriter
@@ -237,8 +237,8 @@ def _do_decomp(module, cfg, scenario_creator, scenario_creator_kwargs, scenario_
         vanilla.add_integer_relax_then_enforce(hub_dict, cfg)
 
     if cfg.grad_rho:
-        ext_classes.append(Gradient_extension)
-        hub_dict['opt_kwargs']['options']['gradient_extension_options'] = {'cfg': cfg}        
+        ext_classes.append(GradRho)
+        hub_dict['opt_kwargs']['options']['grad_rho_options'] = {'cfg': cfg}
 
     if cfg.write_scenario_lp_mps_files_dir is not None:
         ext_classes.append(Scenario_lp_mps_files)