Merge branch 'main' into cyipopt-extend-callback

.github/workflows/test_branches.yml

@@ -855,9 +855,6 @@ jobs:
         token: ${{ secrets.PYOMO_CODECOV_TOKEN }}
         name: ${{ matrix.TARGET }}
         flags: ${{ matrix.TARGET }}
-        # downgrading after v0.7.0 broke tokenless upload
-        # see codecov/codecov-action#1487
-        version: v0.6.0
         fail_ci_if_error: true
 
     - name: Upload other coverage reports
@@ -870,7 +867,4 @@ jobs:
         token: ${{ secrets.PYOMO_CODECOV_TOKEN }}
         name: ${{ matrix.TARGET }}/other
         flags: ${{ matrix.TARGET }},other
-        # downgrading after v0.7.0 broke tokenless upload
-        # see codecov/codecov-action#1487
-        version: v0.6.0
         fail_ci_if_error: true

.github/workflows/test_pr_and_main.yml

@@ -68,8 +68,9 @@ jobs:
         verbose: true
         # How many times to retry a failed request (defaults to 1)
         retry_count: 3
-        # Exclude Jenkins because it's behind a firewall; ignore RTD because
-        # a magically-generated string is triggering a failure
+        # Exclude:
+        #  - Jenkins because it's behind a firewall
+        #  - RTD because a magically-generated string triggers failures
         exclude_urls: https://pyomo-jenkins.sandia.gov/,https://pyomo.readthedocs.io/en/%s/errors.html
 
 
@@ -899,9 +900,6 @@ jobs:
         token: ${{ secrets.PYOMO_CODECOV_TOKEN }}
         name: ${{ matrix.TARGET }}
         flags: ${{ matrix.TARGET }}
-        # downgrading after v0.7.0 broke tokenless upload
-        # see codecov/codecov-action#1487
-        version: v0.6.0
         fail_ci_if_error: true
 
     - name: Upload other coverage reports
@@ -914,7 +912,4 @@ jobs:
         token: ${{ secrets.PYOMO_CODECOV_TOKEN }}
         name: ${{ matrix.TARGET }}/other
         flags: ${{ matrix.TARGET }},other
-        # downgrading after v0.7.0 broke tokenless upload
-        # see codecov/codecov-action#1487
-        version: v0.6.0
         fail_ci_if_error: true

.github/workflows/typos.toml

@@ -38,6 +38,8 @@ caf = "caf"
 WRONLY = "WRONLY"
 # Ignore the name Hax
 Hax = "Hax"
+# Ignore dout (short for dual output in SAS solvers)
+dout = "dout"
 # Big Sur
 Sur = "Sur"
 # contrib package named mis and the acronym whence the name comes
@@ -67,4 +69,6 @@ RO = "RO"
 EOF = "EOF"
 # Ignore lst as shorthand for list
 lst = "lst"
+# Abbreviation of gamma (used in stochpdegas1_automatic.py)
+gam = "gam"
 # AS NEEDED: Add More Words Below

.jenkins.sh

@@ -20,8 +20,11 @@
 #
 # CODECOV_TOKEN: the token to use when uploading results to codecov.io
 #
-# CODECOV_ARGS: additional arguments to pass to the codecov uploader
-#     (e.g., to support SSL certificates)
+# CODECOV_SOURCE_BRANCH: passed to the 'codecov-cli' command; branch of Pyomo
+#     (e.g., to enable correct codecov uploads)
+#
+# CODECOV_REPO_OWNER: passed to the 'codecov-cli' command; owner of repo
+#     (e.g., to enable correct codecov uploads)
 #
 # DISABLE_COVERAGE: if nonempty, then coverage analysis is disabled
 #
@@ -202,22 +205,43 @@ if test -z "$MODE" -o "$MODE" == test; then
         # Note, that the PWD should still be $WORKSPACE/pyomo
         #
         coverage combine || exit 1
-        coverage report -i
+        coverage report -i || exit 1
+        coverage xml -i || exit 1
         export OS=`uname`
-        if test -z "$CODECOV_TOKEN"; then
-            coverage xml
-        else
-            CODECOV_JOB_NAME=`echo ${JOB_NAME} | sed -r 's/^(.*autotest_)?Pyomo_([^\/]+).*/\2/'`.$BUILD_NUMBER.$python
+        if test -z "$PYOMO_SOURCE_SHA"; then
+            PYOMO_SOURCE_SHA=$GIT_COMMIT
+        fi
+        if test -n "$CODECOV_TOKEN" -a -n "$PYOMO_SOURCE_SHA"; then
+            CODECOV_JOB_NAME=$(echo ${JOB_NAME} \
+                | sed -r 's/^(.*autotest_)?Pyomo_([^\/]+).*/\2/').$BUILD_NUMBER.$python
+            if test -z "$CODECOV_REPO_OWNER"; then
+                if test -n "$PYOMO_SOURCE_REPO"; then
+                    CODECOV_REPO_OWNER=$(echo "$PYOMO_SOURCE_REPO" | cut -d '/' -f 4)
+                elif test -n "$GIT_URL"; then
+                    CODECOV_REPO_OWNER=$(echo "$GIT_URL" | cut -d '/' -f 4)
+                else
+                    CODECOV_REPO_OWNER=""
+                fi
+            fi
+            if test -z "$CODECOV_SOURCE_BRANCH"; then
+                CODECOV_SOURCE_BRANCH=$(git branch -av --contains "$PYOMO_SOURCE_SHA" \
+                    | grep "${PYOMO_SOURCE_SHA:0:7}" | grep "/origin/" \
+                    | cut -d '/' -f 3 | cut -d' ' -f 1)
+                if test -z "$CODECOV_SOURCE_BRANCH"; then
+                    CODECOV_SOURCE_BRANCH=main
+                fi
+            fi
             i=0
             while /bin/true; do
                 i=$[$i+1]
                 echo "Uploading coverage to codecov (attempt $i)"
-                codecov -X gcovcodecov -X gcov -X s3 --no-color \
-                    -t $CODECOV_TOKEN --root `pwd` -e OS,python \
-                    --name $CODECOV_JOB_NAME $CODECOV_ARGS \
-                    | tee .cover.upload
-                if test $? == 0 -a `grep -i error .cover.upload \
-                        | grep -v branch= | wc -l` -eq 0; then
+                codecovcli -v upload-process --sha $PYOMO_SOURCE_SHA \
+                    --fail-on-error --git-service github --token $CODECOV_TOKEN \
+                    --slug pyomo/pyomo --file coverage.xml --disable-search \
+                    --name $CODECOV_JOB_NAME \
+                    --branch $CODECOV_REPO_OWNER:$CODECOV_SOURCE_BRANCH \
+                    --env OS,python --network-root-folder `pwd` --plugin noop
+                if test $? == 0; then
                     break
                 elif test $i -ge 4; then
                     exit 1

.codecov.yml → codecov.yml

@@ -1,19 +1,21 @@
+codecov:
+  notify:
+    # GHA: 5, Jenkins: 11
+    # Accurate as of July 3, 2024
+    # Potential to change when Python versions change
+    after_n_builds: 16
+    wait_for_ci: true
 coverage:
-  range: "50...100"
+  range:
+  - 50.0
+  - 100.0
   status:
+    patch:
+      default:
+        # Force patches to be covered at the level of the codebase
+        threshold: 0.0
     project:
       default:
         # Allow overall coverage to drop to avoid failures due to code
         # cleanup or CI unavailability/lag
-        threshold: 5%
-    patch:
-      default:
-        # Force patches to be covered at the level of the codebase
-        threshold: 0%
-#  ci:
-#    - !ci.appveyor.com
-codecov:
-  notify:
-    # GHA: 4, Jenkins: 8
-    after_n_builds: 12  # all
-    wait_for_ci: yes
+        threshold: 5.0

pyomo/common/config.py

@@ -996,7 +996,7 @@ class will still create ``c`` instances that only have the single
 :py:meth:`generate_documentation()`.  The simplest is
 :py:meth:`display()`, which prints out the current values of the
 configuration object (and if it is a container type, all of it's
-children).  :py:meth:`generate_yaml_template` is simular to
+children).  :py:meth:`generate_yaml_template` is similar to
 :py:meth:`display`, but also includes the description fields as
 formatted comments.
 

pyomo/common/tests/test_dependencies.py

@@ -209,7 +209,7 @@ def test_and_or(self):
         _and_or = avail0 & avail1 | avail2
         self.assertTrue(_and_or)
 
-        # Verify operator prescedence
+        # Verify operator precedence
         _or_and = avail0 | avail2 & avail2
         self.assertTrue(_or_and)
         _or_and = (avail0 | avail2) & avail2

pyomo/contrib/appsi/solvers/highs.py

@@ -481,7 +481,7 @@ def _remove_constraints(self, cons: List[ConstraintData]):
             indices_to_remove.append(con_ndx)
             self._mutable_helpers.pop(con, None)
         self._solver_model.deleteRows(
-            len(indices_to_remove), np.array(indices_to_remove)
+            len(indices_to_remove), np.sort(np.array(indices_to_remove))
         )
         con_ndx = 0
         new_con_map = dict()

pyomo/contrib/appsi/solvers/tests/test_highs_persistent.py

@@ -80,6 +80,43 @@ def test_mutable_params_with_remove_vars(self):
         res = opt.solve(m)
         self.assertAlmostEqual(res.best_feasible_objective, -9)
 
+    def test_fix_and_unfix(self):
+        # Tests issue https://github.com/Pyomo/pyomo/issues/3127
+
+        m = pe.ConcreteModel()
+        m.x = pe.Var(domain=pe.Binary)
+        m.y = pe.Var(domain=pe.Binary)
+        m.fx = pe.Var(domain=pe.NonNegativeReals)
+        m.fy = pe.Var(domain=pe.NonNegativeReals)
+        m.c1 = pe.Constraint(expr=m.fx <= m.x)
+        m.c2 = pe.Constraint(expr=m.fy <= m.y)
+        m.c3 = pe.Constraint(expr=m.x + m.y <= 1)
+
+        m.obj = pe.Objective(expr=m.fx * 0.5 + m.fy * 0.4, sense=pe.maximize)
+
+        opt = Highs()
+
+        # solution 1 has m.x == 1 and m.y == 0
+        r = opt.solve(m)
+        self.assertAlmostEqual(m.fx.value, 1, places=5)
+        self.assertAlmostEqual(m.fy.value, 0, places=5)
+        self.assertAlmostEqual(r.best_feasible_objective, 0.5, places=5)
+
+        # solution 2 has m.x == 0 and m.y == 1
+        m.y.fix(1)
+        r = opt.solve(m)
+        self.assertAlmostEqual(m.fx.value, 0, places=5)
+        self.assertAlmostEqual(m.fy.value, 1, places=5)
+        self.assertAlmostEqual(r.best_feasible_objective, 0.4, places=5)
+
+        # solution 3 should be equal solution 1
+        m.y.unfix()
+        m.x.fix(1)
+        r = opt.solve(m)
+        self.assertAlmostEqual(m.fx.value, 1, places=5)
+        self.assertAlmostEqual(m.fy.value, 0, places=5)
+        self.assertAlmostEqual(r.best_feasible_objective, 0.5, places=5)
+
     def test_capture_highs_output(self):
         # tests issue #3003
         #

pyomo/contrib/pyros/tests/test_grcs.py

@@ -42,6 +42,7 @@
 from pyomo.contrib.pyros.util import get_vars_from_component
 from pyomo.contrib.pyros.util import identify_objective_functions
 from pyomo.common.collections import Bunch
+from pyomo.repn.plugins import nl_writer as pyomo_nl_writer
 import time
 import math
 from pyomo.contrib.pyros.util import time_code
@@ -68,7 +69,7 @@
 from pyomo.common.dependencies import numpy as np, numpy_available
 from pyomo.common.dependencies import scipy as sp, scipy_available
 from pyomo.environ import maximize as pyo_max
-from pyomo.common.errors import ApplicationError
+from pyomo.common.errors import ApplicationError, InfeasibleConstraintException
 from pyomo.opt import (
     SolverResults,
     SolverStatus,
@@ -4616,6 +4617,76 @@ def test_discrete_separation_subsolver_error(self):
             ),
         )
 
+    @unittest.skipUnless(ipopt_available, "IPOPT is not available.")
+    def test_pyros_nl_writer_tol(self):
+        """
+        Test PyROS subsolver call routine behavior
+        with respect to the NL writer tolerance is as
+        expected.
+        """
+        m = ConcreteModel()
+        m.q = Param(initialize=1, mutable=True)
+        m.x1 = Var(initialize=1, bounds=(0, 1))
+        m.x2 = Var(initialize=2, bounds=(0, m.q))
+        m.obj = Objective(expr=m.x1 + m.x2)
+
+        # fixed just inside the PyROS-specified NL writer tolerance.
+        m.x1.fix(m.x1.upper + 9.9e-5)
+
+        current_nl_writer_tol = pyomo_nl_writer.TOL
+        ipopt_solver = SolverFactory("ipopt")
+        pyros_solver = SolverFactory("pyros")
+
+        pyros_solver.solve(
+            model=m,
+            first_stage_variables=[m.x1],
+            second_stage_variables=[m.x2],
+            uncertain_params=[m.q],
+            uncertainty_set=BoxSet([[0, 1]]),
+            local_solver=ipopt_solver,
+            global_solver=ipopt_solver,
+            decision_rule_order=0,
+            solve_master_globally=False,
+            bypass_global_separation=True,
+        )
+
+        self.assertEqual(
+            pyomo_nl_writer.TOL,
+            current_nl_writer_tol,
+            msg="Pyomo NL writer tolerance not restored as expected.",
+        )
+
+        # fixed just outside the PyROS-specified NL writer tolerance.
+        # this should be exceptional.
+        m.x1.fix(m.x1.upper + 1.01e-4)
+
+        err_msg = (
+            "model contains a trivially infeasible variable.*x1"
+            ".*fixed.*outside bounds"
+        )
+        with self.assertRaisesRegex(InfeasibleConstraintException, err_msg):
+            pyros_solver.solve(
+                model=m,
+                first_stage_variables=[m.x1],
+                second_stage_variables=[m.x2],
+                uncertain_params=[m.q],
+                uncertainty_set=BoxSet([[0, 1]]),
+                local_solver=ipopt_solver,
+                global_solver=ipopt_solver,
+                decision_rule_order=0,
+                solve_master_globally=False,
+                bypass_global_separation=True,
+            )
+
+        self.assertEqual(
+            pyomo_nl_writer.TOL,
+            current_nl_writer_tol,
+            msg=(
+                "Pyomo NL writer tolerance not restored as expected "
+                "after exceptional test."
+            ),
+        )
+
     @unittest.skipUnless(
         baron_license_is_valid, "Global NLP solver is not available and licensed."
     )

pyomo/contrib/pyros/util.py

@@ -38,6 +38,7 @@
 from pyomo.core.expr import value
 from pyomo.core.expr.numeric_expr import NPV_MaxExpression, NPV_MinExpression
 from pyomo.repn.standard_repn import generate_standard_repn
+from pyomo.repn.plugins import nl_writer as pyomo_nl_writer
 from pyomo.core.expr.visitor import (
     identify_variables,
     identify_mutable_parameters,
@@ -377,7 +378,14 @@ def revert_solver_max_time_adjustment(
         elif isinstance(solver, SolverFactory.get_class("baron")):
             options_key = "MaxTime"
         elif isinstance(solver, SolverFactory.get_class("ipopt")):
-            options_key = "max_cpu_time"
+            options_key = (
+                # IPOPT 3.14.0+ added support for specifying
+                # wall time limit explicitly; this is preferred
+                # over CPU time limit
+                "max_wall_time"
+                if solver.version() >= (3, 14, 0, 0)
+                else "max_cpu_time"
+            )
         elif isinstance(solver, SolverFactory.get_class("scip")):
             options_key = "limits/time"
         else:
@@ -1809,6 +1817,16 @@ def call_solver(model, solver, config, timing_obj, timer_name, err_msg):
     timing_obj.start_timer(timer_name)
     tt_timer.tic(msg=None)
 
+    # tentative: reduce risk of InfeasibleConstraintException
+    # occurring due to discrepancies between Pyomo NL writer
+    # tolerance and (default) subordinate solver (e.g. IPOPT)
+    # feasibility tolerances.
+    # e.g., a Var fixed outside bounds beyond the Pyomo NL writer
+    # tolerance, but still within the default IPOPT feasibility
+    # tolerance
+    current_nl_writer_tol = pyomo_nl_writer.TOL
+    pyomo_nl_writer.TOL = 1e-4
+
     try:
         results = solver.solve(
             model,
@@ -1827,6 +1845,8 @@ def call_solver(model, solver, config, timing_obj, timer_name, err_msg):
             results.solver, TIC_TOC_SOLVE_TIME_ATTR, tt_timer.toc(msg=None, delta=True)
         )
     finally:
+        pyomo_nl_writer.TOL = current_nl_writer_tol
+
         timing_obj.stop_timer(timer_name)
         revert_solver_max_time_adjustment(
             solver, orig_setting, custom_setting_present, config