Merge branch 'main' into iden-vars-namedexpr

.github/workflows/test_branches.yml

@@ -40,7 +40,8 @@ jobs:
         python-version: '3.10'
     - name: Black Formatting Check
       run: |
-        pip install black
+        # Note v24.4.1 fails due to a bug in the parser
+        pip install 'black!=24.4.1'
         black . -S -C --check --diff --exclude examples/pyomobook/python-ch/BadIndent.py
     - name: Spell Check
       uses: crate-ci/typos@master
@@ -92,7 +93,7 @@ jobs:
           skip_doctest: 1
           TARGET: linux
           PYENV: conda
-          PACKAGES: mpi4py
+          PACKAGES: openmpi mpi4py
 
         - os: ubuntu-latest
           python: '3.10'
@@ -263,8 +264,7 @@ jobs:
         if test -z "${{matrix.slim}}"; then
             python -m pip install --cache-dir cache/pip cplex docplex \
                 || echo "WARNING: CPLEX Community Edition is not available"
-            python -m pip install --cache-dir cache/pip \
-                -i https://pypi.gurobi.com gurobipy==10.0.3 \
+            python -m pip install --cache-dir cache/pip gurobipy==10.0.3\
                 || echo "WARNING: Gurobi is not available"
             python -m pip install --cache-dir cache/pip xpress \
                 || echo "WARNING: Xpress Community Edition is not available"

.github/workflows/test_pr_and_main.yml

@@ -7,6 +7,11 @@ on:
   pull_request:
     branches:
       - main
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
   workflow_dispatch:
     inputs:
       git-ref:
@@ -34,6 +39,8 @@ jobs:
   lint:
     name: lint/style-and-typos
     runs-on: ubuntu-latest
+    if: |
+      contains(github.event.pull_request.title, '[WIP]') != true && !github.event.pull_request.draft
     steps:
     - name: Checkout Pyomo source
       uses: actions/checkout@v4
@@ -43,7 +50,8 @@ jobs:
         python-version: '3.10'
     - name: Black Formatting Check
       run: |
-        pip install black
+        # Note v24.4.1 fails due to a bug in the parser
+        pip install 'black!=24.4.1'
         black . -S -C --check --diff --exclude examples/pyomobook/python-ch/BadIndent.py
     - name: Spell Check
       uses: crate-ci/typos@master
@@ -93,7 +101,7 @@ jobs:
           skip_doctest: 1
           TARGET: linux
           PYENV: conda
-          PACKAGES: mpi4py
+          PACKAGES: openmpi mpi4py
 
         - os: ubuntu-latest
           python: '3.11'
@@ -293,8 +301,7 @@ jobs:
         if test -z "${{matrix.slim}}"; then
             python -m pip install --cache-dir cache/pip cplex docplex \
                 || echo "WARNING: CPLEX Community Edition is not available"
-            python -m pip install --cache-dir cache/pip \
-                -i https://pypi.gurobi.com gurobipy==10.0.3 \
+            python -m pip install --cache-dir cache/pip gurobipy==10.0.3 \
                 || echo "WARNING: Gurobi is not available"
             python -m pip install --cache-dir cache/pip xpress \
                 || echo "WARNING: Xpress Community Edition is not available"
@@ -605,7 +612,8 @@ jobs:
       if: ${{ ! matrix.slim }}
       shell: bash
       run: |
-        $PYTHON_EXE -m pip install --cache-dir cache/pip highspy \
+        echo "NOTE: temporarily pinning to highspy pre-release for testing"
+        $PYTHON_EXE -m pip install --cache-dir cache/pip "highspy>=1.7.1.dev1" \
             || echo "WARNING: highspy is not available"
 
     - name: Set up coverage tracking
@@ -733,7 +741,7 @@ jobs:
   cover:
     name: process-coverage-${{ matrix.TARGET }}
     needs: build
-    if: always() # run even if a build job fails
+    if: success() || failure() # run even if a build job fails, but not if cancelled
     runs-on: ${{ matrix.os }}
     timeout-minutes: 10
     strategy:

.github/workflows/typos.toml

@@ -40,4 +40,31 @@ WRONLY = "WRONLY"
 Hax = "Hax"
 # Big Sur
 Sur = "Sur"
+# contrib package named mis and the acronym whence the name comes
+mis = "mis"
+MIS = "MIS"
+# Ignore the shorthand ans for answer
+ans = "ans"
+# Ignore the keyword arange
+arange = "arange"
+# Ignore IIS
+IIS = "IIS"
+iis = "iis"
+# Ignore PN
+PN = "PN"
+# Ignore hd
+hd = "hd"
+# Ignore opf
+opf = "opf"
+# Ignore FRE
+FRE = "FRE"
+# Ignore MCH
+MCH = "MCH"
+# Ignore RO
+ro = "ro"
+RO = "RO"
+# Ignore EOF - end of file
+EOF = "EOF"
+# Ignore lst as shorthand for list
+lst = "lst"
 # AS NEEDED: Add More Words Below

README.md

@@ -71,6 +71,7 @@ version, we will remove testing for that Python version.
 
 ### Tutorials and Examples
 
+* [Pyomo — Optimization Modeling in Python](https://link.springer.com/book/10.1007/978-3-030-68928-5)
 * [Pyomo Workshop Slides](https://github.com/Pyomo/pyomo-tutorials/blob/main/Pyomo-Workshop-December-2023.pdf)
 * [Prof. Jeffrey Kantor's Pyomo Cookbook](https://jckantor.github.io/ND-Pyomo-Cookbook/)
 * The [companion notebooks](https://mobook.github.io/MO-book/intro.html)

doc/OnlineDocs/bibliography.rst

@@ -39,6 +39,8 @@ Bibliography
                   John D. Siirola, Jean-Paul Watson, and David L. Woodruff.
                   Pyomo - Optimization Modeling in Python, 3rd Edition.
                   Vol. 67. Springer, 2021.
+                  doi: `10.1007/978-3-030-68928-5
+                  <https://doi.org/10.1007/978-3-030-68928-5>`_
 
 .. [PyomoJournal] William E. Hart, Jean-Paul Watson, David L. Woodruff.
                   "Pyomo: modeling and solving mathematical programs in

doc/OnlineDocs/conf.py

@@ -84,6 +84,7 @@
     'sphinx.ext.todo',
     'sphinx_copybutton',
     'enum_tools.autoenum',
+    'sphinx.ext.autosectionlabel',
     #'sphinx.ext.githubpages',
 ]
 

doc/OnlineDocs/contributed_packages/iis.rst

@@ -1,6 +1,135 @@
+Infeasibility Diagnostics
+!!!!!!!!!!!!!!!!!!!!!!!!!
+
+There are two closely related tools for infeasibility diagnosis:
+
+  - :ref:`Infeasible Irreducible System (IIS) Tool`
+  - :ref:`Minimal Intractable System finder (MIS) Tool`
+
+The first simply provides a conduit for solvers that compute an
+infeasible irreducible system (e.g., Cplex, Gurobi, or Xpress).  The
+second provides similar functionality, but uses the ``mis`` package
+contributed to Pyomo.
+
+
 Infeasible Irreducible System (IIS) Tool
 ========================================
 
 .. automodule:: pyomo.contrib.iis.iis
 
 .. autofunction:: pyomo.contrib.iis.write_iis
+
+Minimal Intractable System finder (MIS) Tool
+============================================
+
+The file ``mis.py`` finds sets of actions that each, independently,
+would result in feasibility.  The zero-tolerance is whatever the
+solver uses, so users may want to post-process output if it is going
+to be used for analysis. It also computes a minimal intractable system
+(which is not guaranteed to be unique).  It was written by Ben Knueven
+as part of the watertap project (https://github.com/watertap-org/watertap)
+and is therefore governed by a license shown
+at the top of ``mis.py``.
+
+The algorithms come from John Chinneck's slides, see: https://www.sce.carleton.ca/faculty/chinneck/docs/CPAIOR07InfeasibilityTutorial.pdf
+
+Solver
+------
+
+At the time of this writing, you need to use IPopt even for LPs.
+
+Quick Start
+-----------
+
+The file ``trivial_mis.py`` is a tiny example listed at the bottom of
+this help file, which references a Pyomo model with the Python variable
+`m` and has these lines:
+
+.. code-block:: python
+
+   from pyomo.contrib.mis import compute_infeasibility_explanation
+   ipopt = pyo.SolverFactory("ipopt")
+   compute_infeasibility_explanation(m, solver=ipopt)
+
+.. Note::
+   This is done instead of solving the problem.
+
+.. Note::
+   IDAES users can pass ``get_solver()`` imported from ``ideas.core.solvers``
+   as the solver.
+
+Interpreting the Output
+-----------------------
+
+Assuming the dependencies are installed, running ``trivial_mis.py``
+(shown below) will
+produce a lot of warnings from IPopt and then meaningful output (using a logger).
+
+Repair Options
+^^^^^^^^^^^^^^
+
+This output for the trivial example shows three independent ways that the model could be rendered feasible:
+
+
+.. code-block:: text
+   
+   Model Trivial Quad may be infeasible. A feasible solution was found with only the following variable bounds relaxed:
+	ub of var x[1] by 4.464126126706818e-05
+	lb of var x[2] by 0.9999553410114216
+   Another feasible solution was found with only the following variable bounds relaxed:
+	lb of var x[1] by 0.7071067726864677
+	ub of var x[2] by 0.41421355687130673
+	ub of var y by 0.7071067651855212
+   Another feasible solution was found with only the following inequality constraints, equality constraints, and/or variable bounds relaxed:
+	constraint: c by 0.9999999861866736
+
+
+Minimal Intractable System (MIS)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This output shows a minimal intractable system:
+
+
+.. code-block:: text
+
+   Computed Minimal Intractable System (MIS)!
+   Constraints / bounds in MIS:
+	lb of var x[2]
+	lb of var x[1]
+	constraint: c
+   
+Constraints / bounds in guards for stability
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This part of the report is for nonlinear programs (NLPs).
+
+When we’re trying to reduce the constraint set, for an NLP there may be constraints that when missing cause the solver 
+to fail in some catastrophic fashion. In this implementation this is interpreted as failing to get a `results` 
+object back from the call to `solve`. In these cases we keep the constraint in the problem but it’s in the 
+set of “guard” constraints – we can’t really be sure they’re a source of infeasibility or not, 
+just that “bad things” happen when they’re not included.
+
+Perhaps ideally we would put a constraint in the “guard” set if IPopt failed to converge, and only put it in the 
+MIS if IPopt converged to a point of local infeasibility. However, right now the code generally makes the 
+assumption that if IPopt fails to converge the subproblem is infeasible, though obviously that is far from the truth. 
+Hence for difficult NLPs even the “Phase 1” may “fail” – in that when finished the subproblem containing just the 
+constraints in the elastic filter may be feasible -- because IPopt failed to converge and we assumed that meant the 
+subproblem was not feasible.
+
+Dealing with NLPs is far from clean, but that doesn’t mean the tool can’t return useful results even when its assumptions are not satisfied.
+
+trivial_mis.py
+--------------
+
+.. code-block:: python
+
+   import pyomo.environ as pyo
+   m = pyo.ConcreteModel("Trivial Quad")
+   m.x = pyo.Var([1,2], bounds=(0,1))
+   m.y = pyo.Var(bounds=(0, 1))
+   m.c = pyo.Constraint(expr=m.x[1] * m.x[2] == -1)
+   m.d = pyo.Constraint(expr=m.x[1] + m.y >= 1)
+
+   from pyomo.contrib.mis import compute_infeasibility_explanation
+   ipopt = pyo.SolverFactory("ipopt")
+   compute_infeasibility_explanation(m, solver=ipopt)

doc/OnlineDocs/contributed_packages/parmest/datarec.rst

@@ -3,56 +3,52 @@
 Data Reconciliation 
 ====================
 
-The method :class:`~pyomo.contrib.parmest.parmest.Estimator.theta_est`
-can optionally return model values.  This feature can be used to return
-reconciled data using a user specified objective. In this case, the list
-of variable names the user wants to estimate (theta_names) is set to an
-empty list and the objective function is defined to minimize
+The optional argument ``return_values`` in :class:`~pyomo.contrib.parmest.parmest.Estimator.theta_est`
+can be used for data reconciliation or to return model values based on the specified objective. 
+
+For data reconciliation, the ``m.unknown_parameters`` is empty
+and the objective function is defined to minimize
 measurement to model error. Note that the model used for data
 reconciliation may differ from the model used for parameter estimation.
 
-The following example illustrates the use of parmest for data
-reconciliation.  The functions
+The functions
 :class:`~pyomo.contrib.parmest.graphics.grouped_boxplot` or
 :class:`~pyomo.contrib.parmest.graphics.grouped_violinplot` can be used
 to visually compare the original and reconciled data.
 
-Here's a stylized code snippet showing how box plots might be created:
-
-.. doctest::
-    :skipif: True
-
-    >>> import pyomo.contrib.parmest.parmest as parmest
-    >>> pest = parmest.Estimator(model_function, data, [], objective_function)
-    >>> obj, theta, data_rec = pest.theta_est(return_values=['A', 'B']) 
-    >>> parmest.graphics.grouped_boxplot(data, data_rec)
-
-Returned Values
-^^^^^^^^^^^^^^^
+The following example from the reactor design subdirectory returns reconciled values for experiment outputs 
+(`ca`, `cb`, `cc`, and `cd`) and then uses those values in 
+parameter estimation (`k1`, `k2`, and `k3`).
 
-Here's a full program that can be run to see returned values (in this case it
-is the response function that is defined in the model file):
+.. literalinclude:: ../../../../pyomo/contrib/parmest/examples/reactor_design/datarec_example.py
+   :language: python
+
+The following example returns model values from a Pyomo Expression. 
 
 .. doctest::
    :skipif: not ipopt_available or not parmest_available
 
    >>> import pandas as pd   
    >>> import pyomo.contrib.parmest.parmest as parmest
-   >>> from pyomo.contrib.parmest.examples.rooney_biegler.rooney_biegler import rooney_biegler_model
-
-   >>> theta_names = ['asymptote', 'rate_constant']
+   >>> from pyomo.contrib.parmest.examples.rooney_biegler.rooney_biegler import RooneyBieglerExperiment
 
+   >>> # Generate data
    >>> data = pd.DataFrame(data=[[1,8.3],[2,10.3],[3,19.0],
    ...                           [4,16.0],[5,15.6],[7,19.8]],
    ...                     columns=['hour', 'y'])
 
-   >>> def SSE(model, data):  
-   ...     expr = sum((data.y[i]\
-   ...                 - model.response_function[data.hour[i]])**2 for i in data.index)
+   >>> # Create an experiment list
+   >>> exp_list = []
+   >>> for i in range(data.shape[0]):
+   ...     exp_list.append(RooneyBieglerExperiment(data.loc[i, :]))
+
+   >>> # Define objective
+   >>> def SSE(model):
+   ...     expr = (model.experiment_outputs[model.y]
+   ...             - model.response_function[model.experiment_outputs[model.hour]]
+   ...            ) ** 2
    ...     return expr
 
-   >>> pest = parmest.Estimator(rooney_biegler_model, data, theta_names, SSE,
-   ...                          solver_options=None)
+   >>> pest = parmest.Estimator(exp_list, obj_function=SSE, solver_options=None)
    >>> obj, theta, var_values = pest.theta_est(return_values=['response_function'])
    >>> #print(var_values)
-