Merge remote-tracking branch 'origin/cf-point-writer' into cf-point-w…

…riter

cdm-test/src/test/java/ucar/nc2/ft/coverage/TestCoverageHorizSubset.java

@@ -133,7 +133,7 @@ public void testLongitudeSubset() throws Exception {
   @Category(NeedsExternalResource.class)
   public void testCdmRemoteSubset() throws Exception {
     String filename =
-        "cdmremote:https://thredds-dev.unidata.ucar.edu/thredds/cdmremote/grib/NCEP/NAM/CONUS_40km/conduit/best";
+        "cdmremote:https://thredds-test.unidata.ucar.edu/thredds/cdmremote/grib/NCEP/NAM/CONUS_40km/conduit/best";
     System.out.printf("open %s%n", filename);
 
     try (FeatureDatasetCoverage cc = CoverageDatasetFactory.open(filename)) {

cdm/core/src/main/java/ucar/ma2/Index.java

@@ -570,10 +570,10 @@ public Index set(int[] index) {
       throw new ArrayIndexOutOfBoundsException();
     if (rank == 0)
       return this;
-    int prefixrank = (hasvlen ? rank : rank - 1);
+    int prefixrank = (hasvlen ? rank - 1 : rank);
     System.arraycopy(index, 0, current, 0, prefixrank);
     if (hasvlen)
-      current[prefixrank] = -1;
+      current[rank] = -1; // ??
     return this;
   }
 

cdm/core/src/main/java/ucar/nc2/NetcdfFileSubclass.java

@@ -16,7 +16,9 @@
  *
  * @author caron
  * @since 10/29/2014
+ * @deprecated Use NetcdfFile.builder(). Remove in v6
  */
+@Deprecated
 public class NetcdfFileSubclass extends NetcdfFile {
 
   public NetcdfFileSubclass() {}

cdm/core/src/main/java/ucar/nc2/NetcdfFiles.java

@@ -805,7 +805,7 @@ private static IOServiceProvider getIosp(ucar.unidata.io.RandomAccessFile raf) t
     return null;
   }
 
-  private static NetcdfFile build(IOServiceProvider spi, ucar.unidata.io.RandomAccessFile raf, String location,
+  public static NetcdfFile build(IOServiceProvider spi, ucar.unidata.io.RandomAccessFile raf, String location,
       ucar.nc2.util.CancelTask cancelTask) throws IOException {
 
     NetcdfFile.Builder builder = NetcdfFile.builder().setIosp((AbstractIOServiceProvider) spi).setLocation(location);

cdm/core/src/main/java/ucar/nc2/dataset/CoordinateAxis1D.java

@@ -866,6 +866,7 @@ protected void readValues() {
     }
 
     coords = (double[]) data.get1DJavaArray(DataType.DOUBLE);
+    this.wasRead = true;
     // IndexIterator iter = data.getIndexIterator();
     // while (iter.hasNext())
     // coords[count++] = iter.getDoubleNext();

cdm/core/src/main/java/ucar/nc2/dataset/CoordinateAxis1DTime.java

@@ -337,12 +337,17 @@ private CoordinateAxis1DTime(NetcdfDataset ncd, VariableDS org, Formatter errMes
     int ncoords = (int) org.getSize();
     List<CalendarDate> result = new ArrayList<>(ncoords);
 
-    Array data = org.read();
+    if (org instanceof CoordinateAxis1D) {
+      coords = ((CoordinateAxis1D) org).getCoordValues();
+    } else {
+      Array data = org.read();
+      coords = (double[]) data.get1DJavaArray(DataType.DOUBLE);
+    }
+    this.wasRead = true;
 
     int count = 0;
-    IndexIterator ii = data.getIndexIterator();
     for (int i = 0; i < ncoords; i++) {
-      double val = ii.getDoubleNext();
+      double val = coords[i];
       if (Double.isNaN(val))
         continue; // WTF ??
       result.add(helper.makeCalendarDateFromOffset(val));
@@ -355,12 +360,11 @@ private CoordinateAxis1DTime(NetcdfDataset ncd, VariableDS org, Formatter errMes
       setDimension(0, localDim);
 
       // set the shortened values
-      Array shortData = Array.factory(data.getDataType(), new int[] {count});
+      Array shortData = Array.factory(org.getDataType(), new int[] {count});
       Index ima = shortData.getIndex();
       int count2 = 0;
-      ii = data.getIndexIterator();
       for (int i = 0; i < ncoords; i++) {
-        double val = ii.getDoubleNext();
+        double val = coords[i];
         if (Double.isNaN(val))
           continue;
         shortData.setDouble(ima.set0(count2), val);

cdm/core/src/main/java/ucar/nc2/dataset/NetcdfDataset.java

@@ -1344,12 +1344,16 @@ public void addCoordinateTransform(CoordinateTransform ct) {
   public CoordinateAxis addCoordinateAxis(VariableDS v) {
     if (v == null)
       return null;
+
+    final List<CoordinateAxis> coordCopy = new ArrayList<>(coordAxes);
+
     CoordinateAxis oldVar = findCoordinateAxis(v.getFullName());
     if (oldVar != null)
-      coordAxes.remove(oldVar);
+      coordCopy.remove(oldVar);
 
     CoordinateAxis ca = (v instanceof CoordinateAxis) ? (CoordinateAxis) v : CoordinateAxis.factory(this, v);
-    coordAxes.add(ca);
+    coordCopy.add(ca);
+    this.coordAxes = coordCopy;
 
     if (v.isMemberOfStructure()) {
       Structure parentOrg = v.getParentStructure(); // gotta be careful to get the wrapping parent

cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java

@@ -436,12 +436,12 @@ public void setCaching(boolean caching) {
   @Override
   protected Array _read() throws IOException {
     Array result;
-
     // check if already cached - caching in VariableDS only done explicitly by app
-    if (hasCachedData())
+    if (hasCachedData()) {
       result = super._read();
-    else
+    } else {
       result = proxyReader.reallyRead(this, null);
+    }
 
     return convert(result);
   }
@@ -450,15 +450,16 @@ protected Array _read() throws IOException {
   @Override
   protected Array _read(Section section) throws IOException, InvalidRangeException {
     // really a full read
-    if ((null == section) || section.computeSize() == getSize())
+    if ((null == section) || section.computeSize() == getSize()) {
       return _read();
+    }
 
     Array result;
-    if (hasCachedData())
+    if (hasCachedData()) {
       result = super._read(section);
-    else
+    } else {
       result = proxyReader.reallyRead(this, section, null);
-
+    }
     return convert(result);
   }
 
@@ -840,6 +841,7 @@ protected VariableDS(Builder<?> builder, Group parentGroup) {
 
     this.orgFileTypeId = builder.orgFileTypeId;
     this.enhanceProxy = new EnhancementsImpl(this, builder.units, builder.getDescription());
+    this.scaleOffset = builder.scaleOffset;
 
     createEnhancements();
 
@@ -858,7 +860,9 @@ private void createEnhancements() {
       this.dataType = unsignedConversion != null ? unsignedConversion.getOutType() : dataType;
     }
     if (this.enhanceMode.contains(Enhance.ApplyScaleOffset) && (dataType.isNumeric() || dataType == DataType.CHAR)) {
-      this.scaleOffset = ScaleOffset.createFromVariable(this);
+      if (this.scaleOffset == null) {
+        this.scaleOffset = ScaleOffset.createFromVariable(this);
+      }
       this.dataType = scaleOffset != null ? scaleOffset.getScaledOffsetType() : this.dataType;
     }
     Attribute standardizerAtt = findAttribute(CDM.STANDARDIZE);
@@ -902,6 +906,10 @@ protected Builder<?> addLocalFieldsToBuilder(Builder<? extends Builder<?>> build
     builder.setOriginalVariable(this.orgVar).setOriginalDataType(this.orgDataType).setOriginalName(this.orgName)
         .setOriginalFileTypeId(this.orgFileTypeId).setEnhanceMode(this.enhanceMode).setUnits(this.enhanceProxy.units)
         .setDesc(this.enhanceProxy.desc);
+    builder.scaleOffset = this.scaleOffset;
+    if (this.coordSysNames != null) {
+      this.coordSysNames.stream().forEach(s -> builder.addCoordinateSystemName(s));
+    }
 
     return (VariableDS.Builder<?>) super.addLocalFieldsToBuilder(builder);
   }
@@ -944,6 +952,8 @@ public static abstract class Builder<T extends Builder<T>> extends Variable.Buil
     private boolean fillValueIsMissing = NetcdfDataset.fillValueIsMissing;
     private boolean missingDataIsMissing = NetcdfDataset.missingDataIsMissing;
 
+    private ScaleOffset scaleOffset;
+
     private boolean built;
 
     protected abstract T self();
@@ -1040,6 +1050,7 @@ public T copyFrom(VariableDS.Builder<?> builder) {
       setFillValueIsMissing(builder.fillValueIsMissing);
       setInvalidDataIsMissing(builder.invalidDataIsMissing);
       setMissingDataIsMissing(builder.missingDataIsMissing);
+      this.scaleOffset = builder.scaleOffset;
       this.orgVar = builder.orgVar;
       this.orgDataType = builder.orgDataType;
       this.orgFileTypeId = builder.orgFileTypeId;

cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java

@@ -78,13 +78,16 @@ public static ScaleOffset createFromVariable(VariableDS var) {
     if (scaleAtt != null && !scaleAtt.isString()) {
       scaleType = FilterHelpers.getAttributeDataType(scaleAtt, signedness);
       scale = 1 / (var.convertUnsigned(scaleAtt.getNumericValue(), scaleType).doubleValue());
+      var.remove(scaleAtt);
     }
 
     Attribute offsetAtt = var.findAttribute(CDM.ADD_OFFSET);
     if (offsetAtt != null && !offsetAtt.isString()) {
       offsetType = FilterHelpers.getAttributeDataType(offsetAtt, signedness);
       offset = var.convertUnsigned(offsetAtt.getNumericValue(), offsetType).doubleValue();
+      var.remove(offsetAtt);
     }
+
     if (scale != DEFAULT_SCALE || offset != DEFAULT_OFFSET) {
       DataType scaledOffsetType =
           FilterHelpers.largestOf(var.getUnsignedConversionType(), scaleType, offsetType).withSignedness(signedness);

cdm/core/src/main/java/ucar/nc2/ft/FeatureDatasetFactoryManager.java

@@ -315,8 +315,8 @@ public static FeatureDataset wrap(FeatureType wantFeatureType, NetcdfDataset ncd
       }
     }
 
-    if (null == useFactory) {
-      errlog.format("**Failed to find FeatureDatasetFactory for= %s datatype=%s%n", ncd.getLocation(), wantFeatureType);
+    if (useFactory == null) {
+      errlog.format("**Failed to find FeatureDatasetFactory for datatype=%s%n", wantFeatureType);
       return null;
     }
 

cdm/core/src/main/java/ucar/nc2/ft/point/standard/TableAnalyzer.java

@@ -619,7 +619,6 @@ private void writeConfigXML(java.util.Formatter sf) {
   private Document makeDocument() {
     Element rootElem = new Element("featureDataset");
     Document doc = new Document(rootElem);
-    rootElem.setAttribute("location", ds.getLocation());
     rootElem.addContent(new Element("analyser").setAttribute("class", getName()));
     if (ft != null)
       rootElem.setAttribute("featureType", ft.toString());

cdm/core/src/main/java/ucar/nc2/ft2/coverage/adapter/DtCoverageDataset.java

@@ -13,11 +13,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import ucar.nc2.Attribute;
-import ucar.nc2.Dimension;
-import ucar.nc2.NetcdfFile;
-import ucar.nc2.Variable;
-import ucar.nc2.VariableSimpleIF;
+
+import ucar.nc2.*;
 import ucar.nc2.constants.CDM;
 import ucar.nc2.constants.FeatureType;
 import ucar.nc2.dataset.CoordinateAxis;
@@ -112,23 +109,26 @@ public DtCoverageDataset(NetcdfDataset ncd) throws IOException {
    * @param parseInfo put parse info here, may be null
    * @throws java.io.IOException on read error
    */
-  public DtCoverageDataset(NetcdfDataset ncd, Formatter parseInfo) throws IOException {
-    this.ncd = ncd;
-
-    Set<NetcdfDataset.Enhance> enhance = ncd.getEnhanceMode();
-    if (enhance == null || enhance.isEmpty())
+  public DtCoverageDataset(NetcdfDataset ds, Formatter parseInfo) throws IOException {
+    Set<NetcdfDataset.Enhance> enhance = ds.getEnhanceMode();
+    if (enhance == null || enhance.isEmpty()) {
       enhance = NetcdfDataset.getDefaultEnhanceMode();
-    ncd = NetcdfDatasets.enhance(ncd, enhance, null);
+    }
+    this.ncd = NetcdfDatasets.enhance(ds, enhance, null);
 
-    DtCoverageCSBuilder facc = DtCoverageCSBuilder.classify(ncd, parseInfo);
-    if (facc != null)
-      this.coverageType = facc.type;
+    // sort by largest size first
+    List<CoordinateSystem> csList = new ArrayList<>(ncd.getCoordinateSystems());
+    csList.sort((o1, o2) -> o2.getCoordinateAxes().size() - o1.getCoordinateAxes().size());
 
     Map<String, Gridset> csHash = new HashMap<>();
-    for (CoordinateSystem cs : ncd.getCoordinateSystems()) {
+    for (CoordinateSystem cs : csList) {
       DtCoverageCSBuilder fac = new DtCoverageCSBuilder(ncd, cs, parseInfo);
-      if (fac.type == null)
+      if (fac.type == null) {
         continue;
+      }
+      if (this.coverageType == null) {
+        this.coverageType = fac.type;
+      }
       DtCoverageCS ccs = fac.makeCoordSys();
       if (ccs == null)
         continue;

cdm/core/src/main/java/ucar/nc2/internal/dataset/CoordinatesHelper.java

@@ -56,7 +56,7 @@ public List<CoordinateTransform> getCoordTransforms() {
   private CoordinatesHelper(Builder builder, NetcdfDataset ncd) {
     List<CoordinateAxis> axes = new ArrayList<>();
     addAxes(ncd.getRootGroup(), axes);
-    this.coordAxes = ImmutableList.copyOf(axes);
+    coordAxes = ImmutableList.copyOf(axes);
 
     coordTransforms =
         builder.coordTransforms.stream().map(ct -> ct.build(ncd)).filter(Objects::nonNull).collect(Collectors.toList());

cdm/core/src/main/java/ucar/nc2/internal/ncml/AggDataset.java

@@ -145,6 +145,10 @@ protected Array read(Variable mainv, CancelTask cancelTask) throws IOException {
         return null;
 
       Variable v = findVariable(ncd, mainv);
+      if (v == null) {
+        Aggregation.logger.error("AggDataset can't find " + mainv.getFullName() + " in " + ncd.getLocation());
+        throw new IllegalArgumentException("Variable '" + mainv.getFullName() + "' does not exist in aggregation.");
+      }
       if (debugRead)
         System.out.printf("Agg.read %s from %s in %s%n", mainv.getNameAndDimensions(), v.getNameAndDimensions(),
             getLocation());

cdm/core/src/main/java/ucar/nc2/internal/ncml/AggDatasetOuter.java

@@ -285,10 +285,8 @@ protected Array read(Variable mainv, CancelTask cancelTask, List<Range> section)
 
       Variable v = findVariable(ncd, mainv);
       if (v == null) {
-        Aggregation.logger.error("AggOuterDimension cant find " + mainv.getFullName() + " in " + ncd.getLocation()
-            + "; return all zeroes!!!");
-        return Array.factory(mainv.getDataType(), new Section(section).getShape()); // all zeros LOOK need missing
-                                                                                    // value
+        Aggregation.logger.error("AggOuterDimension can't find " + mainv.getFullName() + " in " + ncd.getLocation());
+        throw new IllegalArgumentException("Variable '" + mainv.getFullName() + "' does not exist in aggregation.");
       }
 
       if (Aggregation.debugRead) {

cdm/core/src/main/java/ucar/nc2/internal/ncml/NcmlReader.java

@@ -989,7 +989,7 @@ private Optional<Builder> readVariableExisting(Group.Builder groupBuilder,
             .orElseThrow(() -> new IllegalStateException("Cant find variable " + nameInFile));
       }
     }
-    vb.setName(name).setDataType(dtype);
+    vb.setOriginalName(nameInFile).setName(name).setDataType(dtype);
     if (typedefS != null) {
       vb.setEnumTypeName(typedefS);
     }

cdm/core/src/main/java/ucar/nc2/util/cache/FileCache.java

@@ -4,6 +4,8 @@
  */
 package ucar.nc2.util.cache;
 
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import ucar.nc2.dataset.DatasetUrl;
 import ucar.nc2.time.CalendarDate;
 import ucar.nc2.time.CalendarDateFormatter;
@@ -697,13 +699,11 @@ public int compareTo(Tracker o) {
   synchronized void cleanup(int maxElements) {
 
     try {
-      /*
-       * int size = counter.get();
-       * int fsize = files.size();
-       * if (debug && (size != fsize)) {
-       * log.warn("FileCache " + name + " counter " + size + " doesnt match files().size=" + fsize);
-       * }
-       */
+      for (CacheElement.CacheFile cacheFile : files.values()) {
+        if (!Files.exists(Paths.get(cacheFile.ncfile.getLocation()))) {
+          remove(cacheFile);
+        }
+      }
 
       int size = files.size();
       if (size <= minElements)