Fix nested shell handling

modules/core/src/main/java/org/locationtech/jts/algorithm/hull/ConcaveHullOfPolygons.java

@@ -58,7 +58,8 @@
  * via {@link #setHolesAllowed(boolean)}.
  * <p>
  * The hull can be specified as being "tight", via {@link #setTight(boolean)}.
- * This causes the result to follow the outer boundaries of the input polygons. 
+ * This causes the result to follow the outer boundaries of the input polygons
+ * which "face away" from other input polygons. 
  * <p>
  * Instead of the complete hull, the "fill area" between the input polygons 
  * can be computed using {@link #getFill()}.
@@ -68,6 +69,9 @@
  * If needed, a set of possibly-overlapping Polygons 
  * can be converted to a valid MultiPolygon
  * by using {@link Geometry#union()};
+ * <p>
+ * If the input contains holes (possibly containing nested polygon elements, 
+ * these will be preserved in the output.
  * 
  * @author Martin Davis
  *
@@ -297,7 +301,7 @@ private Geometry createEmptyHull() {
   }
 
   private void buildHullTris() {
-    polygonRings = extractShellRings(inputPolygons);
+    polygonRings = OuterShellsExtracter.extractShells(inputPolygons);
     Polygon frame = createFrame(inputPolygons.getEnvelopeInternal(), polygonRings, geomFactory);
     ConstrainedDelaunayTriangulator cdt = new ConstrainedDelaunayTriangulator(frame);
     List<Tri> tris = cdt.getTriangles();
@@ -589,12 +593,4 @@ private static Polygon createFrame(Envelope polygonsEnv, LinearRing[] polygonRin
     return frame;
   }
 
-  private static LinearRing[] extractShellRings(Geometry polygons) {
-    LinearRing[] rings = new LinearRing[polygons.getNumGeometries()];
-    for (int i = 0; i < polygons.getNumGeometries(); i++) {
-      Polygon consPoly = (Polygon) polygons.getGeometryN(i);
-      rings[i] = (LinearRing) consPoly.getExteriorRing().copy();
-    }
-    return rings;
-  }
 }

modules/core/src/main/java/org/locationtech/jts/algorithm/hull/OuterShellsExtracter.java

@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2024 Martin Davis.
+ *
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * and Eclipse Distribution License v. 1.0 which accompanies this distribution.
+ * The Eclipse Public License is available at http://www.eclipse.org/legal/epl-v20.html
+ * and the Eclipse Distribution License is available at
+ *
+ * http://www.eclipse.org/org/documents/edl-v10.php.
+ */
+package org.locationtech.jts.algorithm.hull;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+
+import org.locationtech.jts.algorithm.PointLocation;
+import org.locationtech.jts.geom.Coordinate;
+import org.locationtech.jts.geom.Geometry;
+import org.locationtech.jts.geom.GeometryFactory;
+import org.locationtech.jts.geom.LinearRing;
+import org.locationtech.jts.geom.Polygon;
+
+/**
+ * Extracts the rings of outer shells from a polygonal geometry.
+ * Outer shells are the shells of polygon elements which
+ * are not nested inside holes of other polygons.
+ * 
+ * @author mdavis
+ *
+ */
+class OuterShellsExtracter {
+
+  public static LinearRing[] extractShells(Geometry polygons) {
+    OuterShellsExtracter extracter = new OuterShellsExtracter(polygons);
+    return extracter.extractShells();
+  }
+
+  private Geometry polygons;
+
+  public OuterShellsExtracter(Geometry polygons) {
+    this.polygons = polygons;
+  }
+
+  private LinearRing[] extractShells() {
+    LinearRing[] shells = extractShellRings(polygons);
+    /**
+     * sort shells in order of increasing envelope area
+     * to ensure that shells are added before any of their inner shells
+     */
+    Arrays.sort(shells, new EnvelopeAreaComparator());
+    List<LinearRing> outerShells = new ArrayList<LinearRing>();
+    for (int i = shells.length - 1; i >= 0; i--) {
+      LinearRing shell = shells[i];
+      if (outerShells.size() == 0 
+          || isOuter(shell, outerShells)) {
+        outerShells.add(shell);
+      }
+    }
+    return GeometryFactory.toLinearRingArray(outerShells);
+  }
+
+  private boolean isOuter(LinearRing shell, List<LinearRing> outerShells) {
+    for (LinearRing outShell : outerShells) {
+      if (covers(outShell, shell)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  private boolean covers(LinearRing shellA, LinearRing shellB) {
+    //-- if shellB envelope is not covered then shell is not covered
+    if (! shellA.getEnvelopeInternal().covers(shellB.getEnvelopeInternal()))
+      return false;
+    //-- if a shellB point lies inside shellA, shell is covered (since shells do not overlap)
+    if (isPointInRing(shellB, shellA))
+      return true;
+    return false;
+  }
+
+  private boolean isPointInRing(LinearRing shell, LinearRing shellRing) {
+    //TODO: optimize this with cached index
+    Coordinate pt = shell.getCoordinate();
+    return PointLocation.isInRing(pt, shellRing.getCoordinates());
+  }
+
+  private static LinearRing[] extractShellRings(Geometry polygons) {
+    LinearRing[] rings = new LinearRing[polygons.getNumGeometries()];
+    for (int i = 0; i < polygons.getNumGeometries(); i++) {
+      Polygon consPoly = (Polygon) polygons.getGeometryN(i);
+      rings[i] = (LinearRing) consPoly.getExteriorRing().copy();
+    }
+    return rings;
+  }
+
+  private static class EnvelopeAreaComparator implements Comparator<Geometry> {
+
+    @Override
+    public int compare(Geometry o1, Geometry o2) {
+      return Double.compare(envArea(o1), envArea(o2));
+    }
+
+    private static double envArea(Geometry g) {
+      return g.getEnvelopeInternal().getArea();
+    }
+
+  }
+}

modules/core/src/test/java/org/locationtech/jts/algorithm/hull/ConcaveHullOfPolygonsTest.java

@@ -80,7 +80,7 @@ public void testPoly3Concave3() {
         "POLYGON ((9 10, 10 9, 10 3, 7 0, 4 0, 0 2, 0 7, 4 10, 9 10))" );
   }
 
-  public void testPoly3WithHole() {
+  public void testPoly3WithHullHole() {
     String wkt = "MULTIPOLYGON (((1 9, 5 9, 5 7, 3 7, 3 5, 1 5, 1 9)), ((1 4, 3 4, 3 2, 5 2, 5 0, 1 0, 1 4)), ((6 9, 8 9, 9 5, 8 0, 6 0, 6 2, 8 5, 6 7, 6 9)))";
     checkHullWithHoles( wkt, .9, wkt);
     checkHullWithHoles( wkt, 1, 
@@ -93,6 +93,20 @@ public void testPoly3WithHole() {
         "POLYGON ((6 9, 8 9, 9 5, 8 0, 6 0, 5 0, 1 0, 1 4, 1 5, 1 9, 5 9, 6 9))");
   }
 
+  public void testPolygonHole() {
+    checkHullByLenRatio(
+        "MULTIPOLYGON (((1 1, 10 3, 19 1, 16 8, 19 7, 19 19, 10 20, 8 17, 1 19, 1 1), (3 4, 5 10, 3 16, 9 14, 14 15, 15 9, 13 5, 3 4)))", 
+        0.9, 
+        "POLYGON ((10 20, 19 19, 19 7, 19 1, 10 3, 1 1, 1 19, 10 20), (13 5, 15 9, 14 15, 9 14, 3 16, 5 10, 3 4, 13 5))" );    
+  }
+
+  public void testPolygonNestedPoly() {
+    checkHullByLenRatio(
+        "MULTIPOLYGON (((1 1, 10 3, 19 1, 16 8, 19 7, 19 19, 10 20, 8 17, 1 19, 1 1), (3 4, 5 10, 3 16, 9 14, 14 15, 15 9, 13 5, 3 4)), ((6 10, 7 13, 10 12, 12 13, 13 11, 11 9, 13 8, 9 6, 6 6, 7 8, 6 10)))", 
+        0.9, 
+        "MULTIPOLYGON (((10 20, 19 19, 19 7, 19 1, 10 3, 1 1, 1 19, 10 20), (13 5, 15 9, 14 15, 9 14, 3 16, 5 10, 3 4, 13 5)), ((7 13, 10 12, 12 13, 13 11, 11 9, 13 8, 9 6, 6 6, 7 8, 6 10, 7 13)))" );    
+  }
+
   private void checkHull(String wkt, double maxLen, String wktExpected) {
     Geometry geom = read(wkt);
     Geometry actual = ConcaveHullOfPolygons.concaveHullByLength(geom, maxLen);