diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/DsgCollectionImpl.java b/cdm/core/src/main/java/ucar/nc2/ft/point/DsgCollectionImpl.java index d5a301fd5e..ee61fdff45 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/DsgCollectionImpl.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/DsgCollectionImpl.java @@ -38,9 +38,9 @@ protected DsgCollectionImpl(String name, CalendarDateUnit timeUnit, String altUn protected DsgCollectionImpl(String name, String timeName, CalendarDateUnit timeUnit, String altName, String altUnits) { this.name = name; - // this.timeName = timeName; + this.timeName = timeName; this.timeUnit = timeUnit; - // this.altName = altName; + this.altName = altName; this.altUnits = altUnits; } diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/PointFeatureCCCImpl.java b/cdm/core/src/main/java/ucar/nc2/ft/point/PointFeatureCCCImpl.java index 1cdc24134f..e0df6ab101 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/PointFeatureCCCImpl.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/PointFeatureCCCImpl.java @@ -25,6 +25,12 @@ protected PointFeatureCCCImpl(String name, CalendarDateUnit timeUnit, String alt this.collectionFeatureType = collectionFeatureType; } + protected PointFeatureCCCImpl(String name, String timeName, CalendarDateUnit timeUnit, String altName, + String altUnits, FeatureType collectionFeatureType) { + super(name, timeName, timeUnit, altName, altUnits); + this.collectionFeatureType = collectionFeatureType; + } + // All features in this collection have this feature type @Nonnull @Override diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/ProfileFeatureImpl.java b/cdm/core/src/main/java/ucar/nc2/ft/point/ProfileFeatureImpl.java index c5680ab506..363c846877 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/ProfileFeatureImpl.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/ProfileFeatureImpl.java @@ -31,6 +31,17 @@ public ProfileFeatureImpl(String name, CalendarDateUnit timeUnit, String altUnit } } + public ProfileFeatureImpl(String name, String timeName, CalendarDateUnit timeUnit, String altName, String altUnits, + double lat, double lon, double time, int nfeatures) { + super(name, timeName, timeUnit, altName, altUnits); + this.latlonPoint = LatLonPoint.create(lat, lon); + this.time = time; + if (nfeatures >= 0) { + getInfo(); // create the object + info.nfeatures = nfeatures; + } + } + @Override @Nonnull public LatLonPoint getLatLon() { diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/SectionCollectionImpl.java b/cdm/core/src/main/java/ucar/nc2/ft/point/SectionCollectionImpl.java index 16c65590f6..edc4f2c2bd 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/SectionCollectionImpl.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/SectionCollectionImpl.java @@ -21,8 +21,9 @@ public abstract class SectionCollectionImpl extends PointFeatureCCCImpl implements TrajectoryProfileFeatureCollection { - protected SectionCollectionImpl(String name, CalendarDateUnit timeUnit, String altUnits) { - super(name, timeUnit, altUnits, FeatureType.TRAJECTORY_PROFILE); + protected SectionCollectionImpl(String name, String timeName, CalendarDateUnit timeUnit, String altName, + String altUnits) { + super(name, timeName, timeUnit, altName, altUnits, FeatureType.TRAJECTORY_PROFILE); } ///////////////////////////////////////////////////////////////////////////////////// diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/SectionFeatureImpl.java b/cdm/core/src/main/java/ucar/nc2/ft/point/SectionFeatureImpl.java index 9e3515a3a2..0522606176 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/SectionFeatureImpl.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/SectionFeatureImpl.java @@ -23,8 +23,9 @@ public abstract class SectionFeatureImpl extends PointFeatureCCImpl implements TrajectoryProfileFeature { - protected SectionFeatureImpl(String name, CalendarDateUnit timeUnit, String altUnits) { - super(name, timeUnit, altUnits, FeatureType.TRAJECTORY_PROFILE); + protected SectionFeatureImpl(String name, String timeName, CalendarDateUnit timeUnit, String altName, + String altUnits) { + super(name, timeName, timeUnit, altName, altUnits, FeatureType.TRAJECTORY_PROFILE); } ///////////////////////////////////////////////////////////////////////////////////// diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/StationProfileCollectionImpl.java b/cdm/core/src/main/java/ucar/nc2/ft/point/StationProfileCollectionImpl.java index 4cd9262254..1835602840 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/StationProfileCollectionImpl.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/StationProfileCollectionImpl.java @@ -34,6 +34,11 @@ public StationProfileCollectionImpl(String name, CalendarDateUnit timeUnit, Stri super(name, timeUnit, altUnits, FeatureType.STATION_PROFILE); } + public StationProfileCollectionImpl(String name, String timeName, CalendarDateUnit timeUnit, String altName, + String altUnits) { + super(name, timeName, timeUnit, altName, altUnits, FeatureType.STATION_PROFILE); + } + // Double-check idiom for lazy initialization of instance fields. See Effective Java 2nd Ed, p. 283. protected StationHelper getStationHelper() { if (stationHelper == null) { @@ -130,7 +135,7 @@ private static class StationProfileFeatureCollectionSubset extends StationProfil private final List stations; StationProfileFeatureCollectionSubset(StationProfileCollectionImpl from, List stations) { - super(from.getName(), from.getTimeUnit(), from.getAltUnits()); + super(from.getName(), from.getTimeName(), from.getTimeUnit(), from.getAltName(), from.getAltUnits()); this.from = from; this.stations = stations; } diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/StationProfileFeatureImpl.java b/cdm/core/src/main/java/ucar/nc2/ft/point/StationProfileFeatureImpl.java index b64e71d71a..9d356f94ba 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/StationProfileFeatureImpl.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/StationProfileFeatureImpl.java @@ -119,6 +119,24 @@ public int compareTo(@Nonnull Station so) { return station.getName().compareTo(so.getName()); } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof StationProfileFeatureImpl)) { + return false; + } + + StationProfileFeatureImpl that = (StationProfileFeatureImpl) o; + return name.equals(that.name); + } + + @Override + public int hashCode() { + return name.hashCode(); + } + // @Override public StationProfileFeature subset(LatLonRect boundingBox) { return this; // only one station - we could check if its in the bb diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/TrajectoryFeatureImpl.java b/cdm/core/src/main/java/ucar/nc2/ft/point/TrajectoryFeatureImpl.java index 588a8a8f10..4138148869 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/TrajectoryFeatureImpl.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/TrajectoryFeatureImpl.java @@ -17,6 +17,15 @@ */ public abstract class TrajectoryFeatureImpl extends PointCollectionImpl implements TrajectoryFeature { + public TrajectoryFeatureImpl(String name, String timeName, CalendarDateUnit timeUnit, String altName, String altUnits, + int nfeatures) { + super(name, timeName, timeUnit, altName, altUnits); + if (nfeatures >= 0) { + getInfo(); // create the object + info.nfeatures = nfeatures; + } + } + public TrajectoryFeatureImpl(String name, CalendarDateUnit timeUnit, String altUnits, int nfeatures) { super(name, timeUnit, altUnits); if (nfeatures >= 0) { diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/standard/StandardProfileCollectionImpl.java b/cdm/core/src/main/java/ucar/nc2/ft/point/standard/StandardProfileCollectionImpl.java index e99ae7582c..ec7b650c6c 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/standard/StandardProfileCollectionImpl.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/standard/StandardProfileCollectionImpl.java @@ -63,8 +63,9 @@ private class StandardProfileFeature extends ProfileFeatureImpl { StructureData profileData; StandardProfileFeature(Cursor cursor, double time, StructureData profileData) { - super(ft.getFeatureName(cursor), StandardProfileCollectionImpl.this.getTimeUnit(), - StandardProfileCollectionImpl.this.getAltUnits(), ft.getLatitude(cursor), ft.getLongitude(cursor), time, -1); + super(ft.getFeatureName(cursor), ft.getTimeName(), StandardProfileCollectionImpl.this.getTimeUnit(), + ft.getAltName(), StandardProfileCollectionImpl.this.getAltUnits(), ft.getLatitude(cursor), + ft.getLongitude(cursor), time, -1); this.cursor = cursor; this.profileData = profileData; diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/standard/StandardSectionCollectionImpl.java b/cdm/core/src/main/java/ucar/nc2/ft/point/standard/StandardSectionCollectionImpl.java index 8fa824b8a7..62fe72283b 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/standard/StandardSectionCollectionImpl.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/standard/StandardSectionCollectionImpl.java @@ -40,7 +40,7 @@ public class StandardSectionCollectionImpl extends SectionCollectionImpl { private NestedTable ft; StandardSectionCollectionImpl(NestedTable ft, CalendarDateUnit timeUnit, String altUnits) { - super(ft.getName(), timeUnit, altUnits); + super(ft.getName(), ft.getTimeName(), timeUnit, ft.getAltName(), altUnits); this.ft = ft; this.extras = ft.getExtras(); } @@ -122,8 +122,8 @@ private class StandardSectionFeature extends SectionFeatureImpl { StructureData sectionData; StandardSectionFeature(Cursor cursor, StructureData sectionData) { - super(ft.getFeatureName(cursor), StandardSectionCollectionImpl.this.getTimeUnit(), - StandardSectionCollectionImpl.this.getAltUnits()); + super(ft.getFeatureName(cursor), ft.getTimeName(), StandardSectionCollectionImpl.this.getTimeUnit(), + ft.getAltName(), StandardSectionCollectionImpl.this.getAltUnits()); this.cursor = cursor; this.sectionData = sectionData; } @@ -201,8 +201,8 @@ private class StandardSectionProfileFeature extends ProfileFeatureImpl { StructureData profileData; StandardSectionProfileFeature(Cursor cursor, double time, StructureData profileData) { - super(ft.getFeatureName(cursor), StandardSectionCollectionImpl.this.getTimeUnit(), - StandardSectionCollectionImpl.this.getAltUnits(), ft.getLatitude(cursor), ft.getLongitude(cursor), time, -1); + super(ft.getFeatureName(cursor), ft.getTimeName(), ft.getTimeUnit(), ft.getAltName(), ft.getAltUnits(), + ft.getLatitude(cursor), ft.getLongitude(cursor), time, -1); this.cursor = cursor; this.profileData = profileData; diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/standard/StandardStationCollectionImpl.java b/cdm/core/src/main/java/ucar/nc2/ft/point/standard/StandardStationCollectionImpl.java index 40e93df6d1..44bfe10505 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/standard/StandardStationCollectionImpl.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/standard/StandardStationCollectionImpl.java @@ -73,8 +73,7 @@ private class StandardStationFeatureImpl extends StationTimeSeriesFeatureImpl { StructureData stationData; StandardStationFeatureImpl(StationFeature s, CalendarDateUnit dateUnit, StructureData stationData, int recnum) { - super(s, StandardStationCollectionImpl.this.getTimeName(), dateUnit, - StandardStationCollectionImpl.this.getAltName(), StandardStationCollectionImpl.this.getAltUnits(), -1); + super(s, ft.getTimeName(), dateUnit, ft.getAltName(), ft.getAltUnits(), -1); this.recnum = recnum; this.stationData = stationData; } diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/standard/StandardStationProfileCollectionImpl.java b/cdm/core/src/main/java/ucar/nc2/ft/point/standard/StandardStationProfileCollectionImpl.java index 93f7ce5b42..ef33238784 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/standard/StandardStationProfileCollectionImpl.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/standard/StandardStationProfileCollectionImpl.java @@ -47,7 +47,7 @@ public class StandardStationProfileCollectionImpl extends StationProfileCollecti private NestedTable ft; StandardStationProfileCollectionImpl(NestedTable ft, CalendarDateUnit timeUnit, String altUnits) { - super(ft.getName(), timeUnit, altUnits); + super(ft.getName(), ft.getTimeName(), timeUnit, ft.getAltName(), altUnits); this.ft = ft; } @@ -228,8 +228,8 @@ public boolean hasNext() throws IOException { @Override public PointFeatureCollection next() { count++; - PointFeatureCollection result = new StandardProfileFeature(station, getTimeUnit(), getAltUnits(), - ft.getObsTime(cursor), cursor.copy(), profileData); + PointFeatureCollection result = new StandardProfileFeature(station, getTimeName(), getTimeUnit(), getAltName(), + getAltUnits(), ft.getObsTime(cursor), cursor.copy(), profileData); prev = (DsgCollectionImpl) result; return result; } @@ -247,10 +247,10 @@ private class StandardProfileFeature extends ProfileFeatureImpl { private Cursor cursor; StructureData profileData; - StandardProfileFeature(Station s, CalendarDateUnit timeUnit, String altUnits, double time, Cursor cursor, - StructureData profileData) { - super(timeUnit.makeCalendarDate(time).toString(), timeUnit, altUnits, s.getLatitude(), s.getLongitude(), time, - -1); + StandardProfileFeature(Station s, String timeName, CalendarDateUnit timeUnit, String altName, String altUnits, + double time, Cursor cursor, StructureData profileData) { + super(timeUnit.makeCalendarDate(time).toString(), timeName, timeUnit, altName, altUnits, s.getLatitude(), + s.getLongitude(), time, -1); this.cursor = cursor; this.profileData = profileData; diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/standard/StandardTrajectoryCollectionImpl.java b/cdm/core/src/main/java/ucar/nc2/ft/point/standard/StandardTrajectoryCollectionImpl.java index cc7c53e8c9..1431df38cb 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/standard/StandardTrajectoryCollectionImpl.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/standard/StandardTrajectoryCollectionImpl.java @@ -40,7 +40,7 @@ protected StandardTrajectoryCollectionImpl(String name, CalendarDateUnit timeUni } StandardTrajectoryCollectionImpl(NestedTable ft, CalendarDateUnit timeUnit, String altUnits) { - super(ft.getName(), timeUnit, altUnits, FeatureType.TRAJECTORY); + super(ft.getName(), ft.getTimeName(), timeUnit, ft.getAltName(), altUnits, FeatureType.TRAJECTORY); this.ft = ft; this.extras = ft.getExtras(); } @@ -57,8 +57,8 @@ private class StandardTrajectoryFeature extends TrajectoryFeatureImpl { StructureData trajData; StandardTrajectoryFeature(Cursor cursor, StructureData trajData) { - super(ft.getFeatureName(cursor), StandardTrajectoryCollectionImpl.this.getTimeUnit(), - StandardTrajectoryCollectionImpl.this.getAltUnits(), -1); + super(ft.getFeatureName(cursor), ft.getTimeName(), StandardTrajectoryCollectionImpl.this.getTimeUnit(), + ft.getAltName(), StandardTrajectoryCollectionImpl.this.getAltUnits(), -1); this.cursor = cursor; this.trajData = trajData; } diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/writer/CFPointWriter.java b/cdm/core/src/main/java/ucar/nc2/ft/point/writer/CFPointWriter.java index 66cdfc705d..2f3988e626 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/writer/CFPointWriter.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/writer/CFPointWriter.java @@ -18,8 +18,8 @@ import ucar.nc2.constants.*; import ucar.nc2.dataset.CoordinateAxis; import ucar.nc2.ft.*; +import ucar.nc2.ft.point.StationFeature; import ucar.nc2.ft.point.StationPointFeature; -import ucar.nc2.ft.point.StationTimeSeriesCollectionImpl; import ucar.nc2.time.CalendarDate; import ucar.nc2.time.CalendarDateFormatter; import ucar.nc2.time.CalendarDateUnit; @@ -124,11 +124,9 @@ private static int writePointFeatureCollection(FeatureDatasetPoint fdpoint, Poin pointWriter.setExtraVariables(pfc.getExtraVariables()); + pointWriter.writeHeader(pfc); int count = 0; for (PointFeature pf : pfc) { - if (count == 0) - pointWriter.writeHeader(pf); - pointWriter.writeRecord(pf, pf.getFeatureData()); count++; if (debug && count % 100 == 0) @@ -144,25 +142,33 @@ private static int writePointFeatureCollection(FeatureDatasetPoint fdpoint, Poin private static int writeStationFeatureCollection(FeatureDatasetPoint dataset, StationTimeSeriesFeatureCollection fc, String fileOut, CFPointWriterConfig config) throws IOException { + int count = 0; try (WriterCFStationCollection cfWriter = new WriterCFStationCollection(fileOut, dataset.getGlobalAttributes(), dataset.getDataVariables(), fc.getTimeUnit(), fc.getAltUnits(), config)) { - cfWriter.setExtraVariables(fc.getExtraVariables()); + List flattenFeatures = new ArrayList<>(); + List extraVariables = new ArrayList<>(); + for (DsgFeatureCollection station : dataset.getPointFeatureCollectionList()) { + extraVariables.addAll(station.getExtraVariables()); + flattenFeatures.addAll(((StationTimeSeriesFeatureCollection) station).getStationFeatures()); + } - cfWriter.writeHeader(fc); + cfWriter.setExtraVariables(extraVariables); + cfWriter.writeHeader(flattenFeatures, null); - int count = 0; - for (PointFeatureCollection pfc : fc) { - for (PointFeature pf : pfc) { - StationPointFeature spf = (StationPointFeature) pf; + for (DsgFeatureCollection featureCollection : dataset.getPointFeatureCollectionList()) { + cfWriter.obsRecno = 0; + for (StationTimeSeriesFeature station : (StationTimeSeriesFeatureCollection) featureCollection) { + for (PointFeature pf : station) { - cfWriter.writeRecord(spf.getStation(), pf, pf.getFeatureData()); - count++; - if (debug && count % 100 == 0) - logger.debug(String.format("%d ", count)); - if (debug && count % 1000 == 0) - logger.debug(String.format("%n ")); + cfWriter.writeRecord(((StationPointFeature) pf).getStation(), pf, pf.getFeatureData()); + count++; + if (debug && count % 100 == 0) + logger.debug(String.format("%d ", count)); + if (debug && count % 1000 == 0) + logger.debug(String.format("%n ")); + } } } cfWriter.finish(); @@ -175,21 +181,27 @@ private static int writeProfileFeatureCollection(FeatureDatasetPoint fdpoint, Pr try (WriterCFProfileCollection cfWriter = new WriterCFProfileCollection(fileOut, fdpoint.getGlobalAttributes(), fdpoint.getDataVariables(), fc.getTimeUnit(), fc.getAltUnits(), config)) { - - cfWriter.setExtraVariables(fc.getExtraVariables()); + List extraVariables = new ArrayList<>(); + List flattenFeatures = new ArrayList<>(); // LOOK not always needed int count = 0; + int nprofiles = 0; int name_strlen = 0; - int nprofiles = fc.size(); - if (nprofiles < 0) { - for (ProfileFeature pf : fc) { - name_strlen = Math.max(name_strlen, pf.getName().length()); - count++; + for (DsgFeatureCollection featureCollection : fdpoint.getPointFeatureCollectionList()) { + nprofiles += featureCollection.size(); + if (nprofiles < 0) { + for (ProfileFeature profile : (ProfileFeatureCollection) featureCollection) { + flattenFeatures.add(profile); + name_strlen = Math.max(name_strlen, profile.getName().length()); + count++; + } + nprofiles = count; } - nprofiles = count; } + cfWriter.setExtraVariables(extraVariables); cfWriter.setFeatureAuxInfo(nprofiles, name_strlen); + cfWriter.writeHeader(flattenFeatures); count = 0; for (ProfileFeature profile : fc) { @@ -210,29 +222,32 @@ private static int writeTrajectoryFeatureCollection(FeatureDatasetPoint fdpoint, try (WriterCFTrajectoryCollection cfWriter = new WriterCFTrajectoryCollection(fileOut, fdpoint.getGlobalAttributes(), fdpoint.getDataVariables(), fc.getTimeUnit(), fc.getAltUnits(), config)) { + List extraVariables = new ArrayList<>(); + List flattenFeatures = new ArrayList<>(); - cfWriter.setExtraVariables(fc.getExtraVariables()); - - // LOOK not always needed int count = 0; int name_strlen = 0; - int ntrajs = fc.size(); - if (ntrajs < 0) { - for (TrajectoryFeature traj : fc) { - name_strlen = Math.max(name_strlen, traj.getName().length()); - count++; + int ntrajs = 0; + for (DsgFeatureCollection featureCollection : fdpoint.getPointFeatureCollectionList()) { + for (TrajectoryFeature trajectory : (TrajectoryFeatureCollection) featureCollection) { + flattenFeatures.add(trajectory); + extraVariables.addAll(trajectory.getExtraVariables()); + name_strlen = Math.max(name_strlen, trajectory.getName().length()); + ntrajs++; } - ntrajs = count; } + cfWriter.setExtraVariables(extraVariables); cfWriter.setFeatureAuxInfo(ntrajs, name_strlen); - + cfWriter.writeHeader(flattenFeatures); count = 0; - for (TrajectoryFeature traj : fc) { - count += cfWriter.writeTrajectory(traj); - if (debug && count % 10 == 0) - logger.debug(String.format("%d ", count)); - if (debug && count % 100 == 0) - logger.debug(String.format("%n ")); + for (DsgFeatureCollection featureCollection : fdpoint.getPointFeatureCollectionList()) { + for (TrajectoryFeature trajectory : (TrajectoryFeatureCollection) featureCollection) { + count += cfWriter.writeTrajectory(trajectory); + if (debug && count % 10 == 0) + logger.debug(String.format("%d ", count)); + if (debug && count % 100 == 0) + logger.debug(String.format("%n ")); + } } cfWriter.finish(); @@ -246,34 +261,34 @@ private static int writeStationProfileFeatureCollection(FeatureDatasetPoint data try (WriterCFStationProfileCollection cfWriter = new WriterCFStationProfileCollection(fileOut, dataset.getGlobalAttributes(), dataset.getDataVariables(), fc.getTimeUnit(), fc.getAltUnits(), config)) { - cfWriter.setExtraVariables(fc.getExtraVariables()); - cfWriter.setStations(fc.getStationFeatures()); + List extraVariables = new ArrayList<>(); + List flattenFeatures = new ArrayList<>(); - int name_strlen = 0; - int countProfiles = 0; - for (StationProfileFeature spf : fc) { - name_strlen = Math.max(name_strlen, spf.getName().length()); - if (spf.size() >= 0) - countProfiles += spf.size(); - else { - for (ProfileFeature pf : spf) { - countProfiles++; - } - } + for (DsgFeatureCollection dsgFeatures : dataset.getPointFeatureCollectionList()) { + extraVariables.addAll(dsgFeatures.getExtraVariables()); + flattenFeatures.addAll(((StationProfileFeatureCollection) dsgFeatures).getStationFeatures()); } - cfWriter.setFeatureAuxInfo(countProfiles, name_strlen); - int count = 0; - for (StationProfileFeature spf : fc) { - for (ProfileFeature pf : spf) { - if (pf.getTime() == null) - continue; // assume this means its an "incomplete multidimensional" + cfWriter.setExtraVariables(extraVariables); + cfWriter.setStations(flattenFeatures); + cfWriter.setFeatureAuxInfo(0, 0); + cfWriter.writeHeader(flattenFeatures); - count += cfWriter.writeProfile(spf, pf); - if (debug && count % 100 == 0) - logger.debug(String.format("%d ", count)); - if (debug && count % 1000 == 0) - logger.debug(String.format("%n ")); + int count = 0; + for (DsgFeatureCollection featureCollection : dataset.getPointFeatureCollectionList()) { + cfWriter.resetObsIndex(); + for (StationFeature station : ((StationProfileFeatureCollection) featureCollection).getStationFeatures()) { + StationProfileFeature spf = (StationProfileFeature) station; + cfWriter.resetProfileIndex(); + for (ProfileFeature pf : spf) { + if (pf.getTime() == null) + continue; // assume this means its an "incomplete multidimensional" + count += cfWriter.writeProfile(spf, pf); + if (debug && count % 100 == 0) + logger.debug(String.format("%d ", count)); + if (debug && count % 1000 == 0) + logger.debug(String.format("%n ")); + } } } @@ -287,39 +302,47 @@ private static int writeTrajectoryProfileFeatureCollection(FeatureDatasetPoint d try (WriterCFTrajectoryProfileCollection cfWriter = new WriterCFTrajectoryProfileCollection(fileOut, dataset.getGlobalAttributes(), dataset.getDataVariables(), fc.getTimeUnit(), fc.getAltUnits(), config)) { - - cfWriter.setExtraVariables(fc.getExtraVariables()); + List extraVariables = new ArrayList<>(); + List flattenFeatures = new ArrayList<>(); int traj_strlen = 0; int prof_strlen = 0; int countTrajectories = 0; int countProfiles = 0; - for (TrajectoryProfileFeature spf : fc) { - countTrajectories++; - traj_strlen = Math.max(traj_strlen, spf.getName().length()); - if (spf.size() >= 0) - countProfiles += spf.size(); - else { - for (ProfileFeature profile : spf) { - prof_strlen = Math.max(prof_strlen, profile.getName().length()); - countProfiles++; + for (DsgFeatureCollection featureCollection : dataset.getPointFeatureCollectionList()) { + for (TrajectoryProfileFeature trajProfile : (TrajectoryProfileFeatureCollection) featureCollection) { + extraVariables.addAll(trajProfile.getExtraVariables()); + flattenFeatures.add(trajProfile); + countTrajectories++; + traj_strlen = Math.max(traj_strlen, trajProfile.getName().length()); + if (trajProfile.size() >= 0) + countProfiles += trajProfile.size(); + else { + for (ProfileFeature profile : trajProfile) { + prof_strlen = Math.max(prof_strlen, profile.getName().length()); + countProfiles++; + } } } } + cfWriter.setExtraVariables(extraVariables); cfWriter.setFeatureAuxInfo(countProfiles, prof_strlen); cfWriter.setFeatureAuxInfo2(countTrajectories, traj_strlen); + cfWriter.writeHeader(flattenFeatures); int count = 0; - for (TrajectoryProfileFeature spf : fc) { - for (ProfileFeature profile : spf) { - if (profile.getTime() == null) - continue; // assume this means its a "incomplete multidimensional" - - count += cfWriter.writeProfile(spf, profile); - if (debug && count % 100 == 0) - logger.debug(String.format("%d ", count)); - if (debug && count % 1000 == 0) - logger.debug(String.format("%n ")); + for (DsgFeatureCollection tpfc : dataset.getPointFeatureCollectionList()) { + for (TrajectoryProfileFeature spf : (TrajectoryProfileFeatureCollection) tpfc) { + for (ProfileFeature profile : spf) { + if (profile.getTime() == null) + continue; // assume this means its a "incomplete multidimensional" + + count += cfWriter.writeProfile(spf, profile); + if (debug && count % 100 == 0) + logger.debug(String.format("%d ", count)); + if (debug && count % 1000 == 0) + logger.debug(String.format("%n ")); + } } } @@ -359,7 +382,7 @@ private static int writeTrajectoryProfileFeatureCollection(FeatureDatasetPoint d protected List dataVars; private Map extraMap; // added as variables just as they are - protected List extra; + protected List extra = new ArrayList<>(); protected LatLonRect llbb; protected CalendarDate minDate; @@ -381,7 +404,7 @@ protected CFPointWriter(String fileOut, List atts, List extra) { } } - protected abstract void makeFeatureVariables(StructureData featureData, boolean isExtended); + protected abstract void makeFeatureVariables(List featureData, boolean isExtended); - protected void makeMiddleVariables(StructureData middleData, boolean isExtended) { + protected void makeMiddleVariables(List middleData, boolean isExtended) { // NOOP } - protected void writeHeader(List obsCoords, StationTimeSeriesCollectionImpl stationFeatures) - throws IOException { + protected void writeHeader(List obsCoords, List stationFeatures, + List featureDataStructs, List middleDataStructs) throws IOException { this.recordDim = writer.addUnlimitedDimension(recordDimName); addExtraVariables(); - addCoordinatesClassic(recordDim, obsCoords, dataMap); + if (writer.getVersion().isExtendedModel()) { + record = (Structure) writer.addVariable(null, recordName, DataType.STRUCTURE, recordDimName); + addCoordinatesExtended(record, obsCoords); + } - for (StationTimeSeriesFeature stnFeature : stationFeatures) { - StructureData featureData = stnFeature.getFeatureData(); - if (writer.getVersion().isExtendedModel()) { - makeFeatureVariables(featureData, true); - record = (Structure) writer.addVariable(null, recordName, DataType.STRUCTURE, recordDimName); - addCoordinatesExtended(record, obsCoords); + if (featureDataStructs != null) + makeFeatureVariables(featureDataStructs, writer.getVersion().isExtendedModel()); + + if (middleDataStructs != null) + makeMiddleVariables(middleDataStructs, writer.getVersion().isExtendedModel()); + + + for (PointFeatureCollection stnFeature : stationFeatures) { - } else { - if (writer.findDimension(stationDimName) == null) - makeFeatureVariables(featureData, false); - } PeekingIterator iter = Iterators.peekingIterator(stnFeature.iterator()); if (iter.hasNext()) { - PointFeature pointFeat = iter.peek(); - assert pointFeat instanceof StationPointFeature : "Expected pointFeat to be a StationPointFeature, not a " - + pointFeat.getClass().getSimpleName(); - StructureData obsData = pointFeat.getFeatureData(); + PointFeature pointFeat = iter.peek(); - Formatter coordNames = - new Formatter().format("%s %s %s", pointFeat.getFeatureCollection().getTimeName(), latName, lonName); + Formatter coordNames = new Formatter().format("%s %s %s", stnFeature.getTimeName(), latName, lonName); if (!Double.isNaN(pointFeat.getLocation().getAltitude())) { + altitudeCoordinateName = stnFeature.getAltName(); coordNames.format(" %s", altitudeCoordinateName); } if (writer.getVersion().isExtendedModel()) { - addDataVariablesExtended(obsData, coordNames.toString()); - + addDataVariablesExtended(pointFeat.getFeatureData(), coordNames.toString()); } - addDataVariablesClassic(recordDim, obsData, dataMap, coordNames.toString()); + addDataVariablesClassic(recordDim, pointFeat.getFeatureData(), dataMap, coordNames.toString()); } } + addCoordinatesClassic(recordDim, obsCoords, dataMap); + writer.create(); if (!(writer.getVersion().isExtendedModel())) record = writer.addRecordStructure(); // for netcdf3 @@ -504,7 +526,7 @@ record = writer.addRecordStructure(); // for netcdf3 } - protected void writeHeader(List obsCoords, StructureData featureData, StructureData obsData, + protected void writeHeader(List obsCoords, List featureData, StructureData obsData, String coordNames) throws IOException { this.recordDim = writer.addUnlimitedDimension(recordDimName); @@ -528,31 +550,6 @@ record = writer.addRecordStructure(); // for netcdf3 writeExtraVariables(); } - protected void writeHeader2(List obsCoords, StructureData featureData, StructureData middleData, - StructureData obsData, String coordNames) throws IOException { - this.recordDim = writer.addUnlimitedDimension(recordDimName); - - addExtraVariables(); - - if (writer.getVersion().isExtendedModel()) { - makeFeatureVariables(featureData, true); - makeMiddleVariables(middleData, true); - record = (Structure) writer.addVariable(null, recordName, DataType.STRUCTURE, recordDimName); - addCoordinatesExtended(record, obsCoords); - addDataVariablesExtended(obsData, coordNames); - writer.create(); - - } else { - makeFeatureVariables(featureData, false); - makeMiddleVariables(middleData, false); - addCoordinatesClassic(recordDim, obsCoords, dataMap); - addDataVariablesClassic(recordDim, obsData, dataMap, coordNames); - writer.create(); - record = writer.addRecordStructure(); // for netcdf3 - } - - writeExtraVariables(); - } protected void addExtraVariables() { if (extra == null) diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/writer/WriterCFPointCollection.java b/cdm/core/src/main/java/ucar/nc2/ft/point/writer/WriterCFPointCollection.java index 5f65a259a9..8447a34fef 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/writer/WriterCFPointCollection.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/writer/WriterCFPointCollection.java @@ -53,15 +53,36 @@ public void writeHeader(PointFeature pf) throws IOException { Formatter coordNames = new Formatter().format("%s %s %s", pf.getFeatureCollection().getTimeName(), latName, lonName); if (altUnits != null) { - coords.add(VariableSimpleBuilder.makeScalar(altName, "altitude of measurement", altUnits, DataType.DOUBLE) - .addAttribute(CF.POSITIVE, CF1Convention.getZisPositive(altName, altUnits)).build()); - coordNames.format(" %s", altName); + coords.add(VariableSimpleBuilder + .makeScalar(pf.getFeatureCollection().getAltName(), "altitude of measurement", altUnits, DataType.DOUBLE) + .addAttribute(CF.POSITIVE, CF1Convention.getZisPositive(pf.getFeatureCollection().getAltName(), altUnits)) + .build()); + coordNames.format(" %s", pf.getFeatureCollection().getAltName()); } super.writeHeader(coords, null, pf.getDataAll(), coordNames.toString()); } - protected void makeFeatureVariables(StructureData featureData, boolean isExtended) { + public void writeHeader(PointFeatureCollection pfc) throws IOException { + List coords = new ArrayList<>(); + coords.add(VariableSimpleBuilder + .makeScalar(pfc.getTimeName(), "time of measurement", timeUnit.getUdUnit(), DataType.DOUBLE) + .addAttribute(CF.CALENDAR, timeUnit.getCalendar().toString()).build()); + + coords.add( + VariableSimpleBuilder.makeScalar(latName, "latitude of measurement", CDM.LAT_UNITS, DataType.DOUBLE).build()); + coords.add( + VariableSimpleBuilder.makeScalar(lonName, "longitude of measurement", CDM.LON_UNITS, DataType.DOUBLE).build()); + if (altUnits != null) { + coords + .add(VariableSimpleBuilder.makeScalar(pfc.getAltName(), "altitude of measurement", altUnits, DataType.DOUBLE) + .addAttribute(CF.POSITIVE, CF1Convention.getZisPositive(altName, altUnits)).build()); + } + + super.writeHeader(coords, Arrays.asList(pfc), null, null); + } + + protected void makeFeatureVariables(List featureData, boolean isExtended) { // NOOP } @@ -69,13 +90,19 @@ protected void makeFeatureVariables(StructureData featureData, boolean isExtende // writing data public void writeRecord(PointFeature sobs, StructureData sdata) throws IOException { - writeRecord(sobs.getObservationTime(), sobs.getObservationTimeAsCalendarDate(), sobs.getLocation(), sdata); + writeRecord(sobs.getFeatureCollection().getTimeName(), sobs.getObservationTime(), + sobs.getObservationTimeAsCalendarDate(), sobs.getFeatureCollection().getAltName(), sobs.getLocation(), sdata); } private int obsRecno; public void writeRecord(double timeCoordValue, CalendarDate obsDate, EarthLocation loc, StructureData sdata) throws IOException { + writeRecord(timeName, timeCoordValue, obsDate, altName, loc, sdata); + } + + private void writeRecord(String timeName, double timeCoordValue, CalendarDate obsDate, String altName, + EarthLocation loc, StructureData sdata) throws IOException { trackBB(loc.getLatLon(), obsDate); StructureMembers.Builder smb = StructureMembers.builder().setName("Coords"); @@ -89,6 +116,6 @@ public void writeRecord(double timeCoordValue, CalendarDate obsDate, EarthLocati // coords first so it takes precedence StructureDataComposite sdall = StructureDataComposite.create(ImmutableList.of(coords, sdata)); obsRecno = super.writeStructureData(obsRecno, record, sdall, dataMap); - } + } } diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/writer/WriterCFProfileCollection.java b/cdm/core/src/main/java/ucar/nc2/ft/point/writer/WriterCFProfileCollection.java index e1c967ac4c..41294ebe9e 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/writer/WriterCFProfileCollection.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/writer/WriterCFProfileCollection.java @@ -39,7 +39,6 @@ public class WriterCFProfileCollection extends CFPointWriter { // private Formatter coordNames = new Formatter(); private Structure profileStruct; // used for netcdf4 extended private Map featureVarMap = new HashMap<>(); - private boolean headerDone; public WriterCFProfileCollection(String fileOut, List globalAtts, List dataVars, CalendarDateUnit timeUnit, String altUnits, CFPointWriterConfig config) throws IOException { @@ -50,14 +49,10 @@ public WriterCFProfileCollection(String fileOut, List globalAtts, Lis } public int writeProfile(ProfileFeature profile) throws IOException { + if (id_strlen == 0) + id_strlen = profile.getName().length() * 2; int count = 0; for (PointFeature pf : profile) { - if (!headerDone) { - if (id_strlen == 0) - id_strlen = profile.getName().length() * 2; - writeHeader(profile, pf); - headerDone = true; - } writeObsData(pf); count++; } @@ -66,21 +61,28 @@ public int writeProfile(ProfileFeature profile) throws IOException { return count; } - private void writeHeader(ProfileFeature profile, PointFeature obs) throws IOException { - - Formatter coordNames = new Formatter().format("%s %s %s", profileTimeName, latName, lonName); + protected void writeHeader(List profiles) throws IOException { List coords = new ArrayList<>(); - if (useAlt) { - coords.add(VariableSimpleBuilder.makeScalar(altitudeCoordinateName, "obs altitude", altUnits, DataType.DOUBLE) - .addAttribute(CF.STANDARD_NAME, "altitude") - .addAttribute(CF.POSITIVE, CF1Convention.getZisPositive(altitudeCoordinateName, altUnits)).build()); - coordNames.format(" %s", altitudeCoordinateName); + List profileData = new ArrayList<>(); + + for (ProfileFeature profile : profiles) { + profileData.add(profile.getFeatureData()); + coords.add(VariableSimpleBuilder + .makeScalar(profile.getTimeName(), "time of measurement", profile.getTimeUnit().getUdUnit(), DataType.DOUBLE) + .addAttribute(CF.CALENDAR, profile.getTimeUnit().getCalendar().toString()).build()); + + if (useAlt) { + altitudeCoordinateName = profile.getAltName(); + coords.add(VariableSimpleBuilder.makeScalar(altitudeCoordinateName, "obs altitude", altUnits, DataType.DOUBLE) + .addAttribute(CF.STANDARD_NAME, "altitude") + .addAttribute(CF.POSITIVE, CF1Convention.getZisPositive(altitudeCoordinateName, altUnits)).build()); + } } - super.writeHeader(coords, profile.getFeatureData(), obs.getFeatureData(), coordNames.toString()); + super.writeHeader(coords, profiles, profileData, null); } - protected void makeFeatureVariables(StructureData featureData, boolean isExtended) { + protected void makeFeatureVariables(List featureDataStructs, boolean isExtended) { // LOOK why not unlimited here ? Dimension profileDim = writer.addDimension(null, profileDimName, nfeatures); @@ -105,10 +107,12 @@ protected void makeFeatureVariables(StructureData featureData, boolean isExtende .addAttribute(CF.CALENDAR, timeUnit.getCalendar().toString()).build()); - for (StructureMembers.Member m : featureData.getMembers()) { - VariableSimpleIF dv = getDataVar(m.getName()); - if (dv != null) - profileVars.add(dv); + for (StructureData featureData : featureDataStructs) { + for (StructureMembers.Member m : featureData.getMembers()) { + VariableSimpleIF dv = getDataVar(m.getName()); + if (dv != null) + profileVars.add(dv); + } } if (isExtended) { @@ -146,8 +150,10 @@ private void writeProfileData(ProfileFeature profile, int nobs) throws IOExcepti private void writeObsData(PointFeature pf) throws IOException { StructureMembers.Builder smb = StructureMembers.builder().setName("Coords"); + smb.addMemberScalar(pf.getFeatureCollection().getTimeName(), null, null, DataType.DOUBLE, pf.getObservationTime()); if (useAlt) - smb.addMemberScalar(altitudeCoordinateName, null, null, DataType.DOUBLE, pf.getLocation().getAltitude()); + smb.addMemberScalar(pf.getFeatureCollection().getAltName(), null, null, DataType.DOUBLE, + pf.getLocation().getAltitude()); StructureData coords = new StructureDataFromMember(smb.build()); // coords first so it takes precedence diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/writer/WriterCFStationCollection.java b/cdm/core/src/main/java/ucar/nc2/ft/point/writer/WriterCFStationCollection.java index 12ba44db94..caad99e813 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/writer/WriterCFStationCollection.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/writer/WriterCFStationCollection.java @@ -10,16 +10,14 @@ import ucar.nc2.*; import ucar.nc2.constants.CDM; import ucar.nc2.constants.CF; -import ucar.nc2.dataset.conv.CF1Convention; -import ucar.nc2.ft.DsgFeatureCollection; -import ucar.nc2.ft.PointFeature; -import ucar.nc2.ft.StationTimeSeriesFeatureCollection; +import ucar.nc2.ft.*; import ucar.nc2.ft.point.StationFeature; import ucar.nc2.ft.point.StationPointFeature; -import ucar.nc2.ft.point.StationTimeSeriesCollectionImpl; import ucar.nc2.time.CalendarDate; import ucar.nc2.time.CalendarDateUnit; import ucar.unidata.geoloc.Station; + +import javax.annotation.Nullable; import java.io.IOException; import java.util.*; import java.util.stream.Collectors; @@ -64,64 +62,21 @@ public WriterCFStationCollection(String fileOut, List atts, List coords = new ArrayList<>(); - - // see if there's altitude, wmoId for any stations - for (Station stn : stations) { - if (!Double.isNaN(stn.getAltitude())) - useAlt = true; - if ((stn.getWmoId() != null) && (!stn.getWmoId().trim().isEmpty())) - useWmoId = true; - if ((stn.getDescription() != null) && (!stn.getDescription().trim().isEmpty())) - useDesc = true; - - // find string lengths - id_strlen = Math.max(id_strlen, stn.getName().length()); - if (stn.getDescription() != null) - desc_strlen = Math.max(desc_strlen, stn.getDescription().length()); - if (stn.getWmoId() != null) - wmo_strlen = Math.max(wmo_strlen, stn.getWmoId().length()); - - if (stn instanceof DsgFeatureCollection) { - DsgFeatureCollection dsgStation = (DsgFeatureCollection) stn; - if (coords.stream().noneMatch(x -> x.getShortName().equals(dsgStation.getTimeName()))) { - coords.add(VariableSimpleBuilder - .makeScalar(dsgStation.getTimeName(), "time of measurement", dsgStation.getTimeUnit().getUdUnit(), - DataType.DOUBLE) - .addAttribute(CF.CALENDAR, dsgStation.getTimeUnit().getCalendar().toString()).build()); - } else { - coords.add( - VariableSimpleBuilder.makeScalar(timeName, "time of measurement", timeUnit.getUdUnit(), DataType.DOUBLE) - .addAttribute(CF.CALENDAR, timeUnit.getCalendar().toString()).build()); - } - } - } - - llbb = CFPointWriterUtils.getBoundingBox(stnList); // gets written in super.finish(); - - coords.add(VariableSimpleBuilder - .makeScalar(stationIndexName, "station index for this observation record", null, DataType.INT) - .addAttribute(CF.INSTANCE_DIMENSION, stationDimName).build()); - - super.writeHeader(coords, (StationTimeSeriesCollectionImpl) stations); - - int count = 0; - stationIndexMap = new HashMap<>(stnList.size(), 1.0f); - for (StationFeature stn : stnList) { - writeStationData(stn); - stationIndexMap.put(stn.getName(), count); - count++; - } + writeHeader(stations.getStationFeatures(), null); } - public void writeHeader(List stns, StationPointFeature spf) throws IOException { - this.stnList = stns; + public void writeHeader(List stns, @Nullable StationPointFeature spf) throws IOException { + this.stnList = stns.stream().distinct().collect(Collectors.toList()); + + List coords = new ArrayList<>(); + List flattenStations = new ArrayList<>(); + List stationData = new ArrayList<>(); // see if there's altitude, wmoId for any stations - for (Station stn : stnList) { - if (!Double.isNaN(stn.getAltitude())) - useAlt = true; + for (StationFeature stn : stns) { + flattenStations.add((PointFeatureCollection) stn); + stationData.add(stn.getFeatureData()); + useAlt = !Double.isNaN(stn.getAltitude()); if ((stn.getWmoId() != null) && (!stn.getWmoId().trim().isEmpty())) useWmoId = true; if ((stn.getDescription() != null) && (!stn.getDescription().trim().isEmpty())) @@ -133,29 +88,23 @@ public void writeHeader(List stns, StationPointFeature spf) thro desc_strlen = Math.max(desc_strlen, stn.getDescription().length()); if (stn.getWmoId() != null) wmo_strlen = Math.max(wmo_strlen, stn.getWmoId().length()); + DsgFeatureCollection dsgStation = (DsgFeatureCollection) stn; + if (coords.stream().noneMatch(x -> x.getShortName().equals(dsgStation.getTimeName()))) { + coords + .add(VariableSimpleBuilder + .makeScalar(dsgStation.getTimeName(), "time of measurement", dsgStation.getTimeUnit().getUdUnit(), + DataType.DOUBLE) + .addAttribute(CF.CALENDAR, dsgStation.getTimeUnit().getCalendar().toString()).build()); + } } llbb = CFPointWriterUtils.getBoundingBox(stnList); // gets written in super.finish(); - StationFeature sf = spf.getStation(); - StructureData stnData = sf.getFeatureData(); - StructureData obsData = spf.getFeatureData(); - - List coords = new ArrayList<>(); - coords.add(VariableSimpleBuilder.makeScalar(spf.getFeatureCollection().getTimeName(), "time of measurement", - timeUnit.getUdUnit(), DataType.DOUBLE).addAttribute(CF.CALENDAR, timeUnit.getCalendar().toString()).build()); - coords.add(VariableSimpleBuilder .makeScalar(stationIndexName, "station index for this observation record", null, DataType.INT) .addAttribute(CF.INSTANCE_DIMENSION, stationDimName).build()); - - Formatter coordNames = - new Formatter().format("%s %s %s", spf.getFeatureCollection().getTimeName(), latName, lonName); - if (useAlt) - coordNames.format(" %s", stationAltName); - - super.writeHeader(coords, stnData, obsData, coordNames.toString()); + super.writeHeader(coords, flattenStations, stationData, null); int count = 0; stationIndexMap = new HashMap<>(stnList.size(), 1.0f); @@ -167,7 +116,7 @@ public void writeHeader(List stns, StationPointFeature spf) thro } - protected void makeFeatureVariables(StructureData featureData, boolean isExtended) { + protected void makeFeatureVariables(List featureDataStructs, boolean isExtended) { // add the dimensions : extendded model can use an unlimited dimension // Dimension stationDim = isExtended ? writer.addDimension(null, stationDimName, 0, true, true, false) : @@ -180,8 +129,7 @@ protected void makeFeatureVariables(StructureData featureData, boolean isExtende if (useAlt) { stnVars.add(VariableSimpleBuilder.makeScalar(stationAltName, "station altitude", altUnits, DataType.DOUBLE) - .addAttribute(CF.STANDARD_NAME, CF.SURFACE_ALTITUDE) - .addAttribute(CF.POSITIVE, CF1Convention.getZisPositive(altName, altUnits)).build()); + .addAttribute(CF.STANDARD_NAME, CF.STATION_ALTITUDE).build()); } stnVars.add(VariableSimpleBuilder.makeString(stationIdName, "station identifier", null, id_strlen) @@ -195,9 +143,12 @@ protected void makeFeatureVariables(StructureData featureData, boolean isExtende stnVars.add(VariableSimpleBuilder.makeString(wmoName, "station WMO id", null, wmo_strlen) .addAttribute(CF.STANDARD_NAME, CF.PLATFORM_ID).build()); - for (StructureMembers.Member m : featureData.getMembers()) { - if (getDataVar(m.getName()) != null) - stnVars.add(VariableSimpleBuilder.fromMember(m).build()); + for (StructureData featureData : featureDataStructs) { + for (StructureMembers.Member m : featureData.getMembers()) { + if (getDataVar(m.getName()) != null + && stnVars.stream().noneMatch(x -> x.getShortName().equals(m.getFullName()))) + stnVars.add(VariableSimpleBuilder.fromMember(m).build()); + } } if (isExtended) { @@ -232,19 +183,18 @@ public void writeRecord(Station s, PointFeature sobs, StructureData sdata) throw if (s instanceof DsgFeatureCollection) { DsgFeatureCollection dsgStation = (DsgFeatureCollection) s; writeRecord(dsgStation.getName(), dsgStation.getTimeName(), sobs.getObservationTime(), - sobs.getObservationTimeAsCalendarDate(), dsgStation.getAltName(), sobs.getLocation().getAltitude(), sdata); + sobs.getObservationTimeAsCalendarDate(), altitudeCoordinateName, sobs.getLocation().getAltitude(), sdata); } else { writeRecord(s.getName(), sobs.getFeatureCollection().getTimeName(), sobs.getObservationTime(), - sobs.getObservationTimeAsCalendarDate(), this.altitudeCoordinateName, sobs.getLocation().getAltitude(), - sdata); + sobs.getObservationTimeAsCalendarDate(), altitudeCoordinateName, sobs.getLocation().getAltitude(), sdata); } } - private int obsRecno; + protected int obsRecno; public void writeRecord(String stnName, double timeCoordValue, CalendarDate obsDate, StructureData sdata) throws IOException { - writeRecord(stnName, timeName, timeCoordValue, obsDate, altName, 0, sdata); + writeRecord(stnName, timeName, timeCoordValue, obsDate, altitudeCoordinateName, 0, sdata); } public void writeRecord(String stnName, String timeCoordName, double timeCoordValue, CalendarDate obsDate, diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/writer/WriterCFStationProfileCollection.java b/cdm/core/src/main/java/ucar/nc2/ft/point/writer/WriterCFStationProfileCollection.java index ca93065bca..e010e32f91 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/writer/WriterCFStationProfileCollection.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/writer/WriterCFStationProfileCollection.java @@ -6,6 +6,7 @@ package ucar.nc2.ft.point.writer; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import ucar.ma2.*; @@ -13,13 +14,12 @@ import ucar.nc2.constants.CDM; import ucar.nc2.constants.CF; import ucar.nc2.dataset.conv.CF1Convention; -import ucar.nc2.ft.PointFeature; -import ucar.nc2.ft.ProfileFeature; -import ucar.nc2.ft.StationProfileFeature; +import ucar.nc2.ft.*; import ucar.nc2.ft.point.StationFeature; import ucar.nc2.time.CalendarDateUnit; import java.io.IOException; import java.util.*; +import java.util.stream.Collectors; /** * Write a CF "Discrete Sample" station profile collection file. @@ -48,7 +48,6 @@ public class WriterCFStationProfileCollection extends CFPointWriter { // private Formatter coordNames = new Formatter(); private Structure profileStruct; // used for netcdf4 extended private Map profileVarMap = new HashMap<>(); - private boolean headerDone; public WriterCFStationProfileCollection(String fileOut, List globalAtts, List dataVars, CalendarDateUnit timeUnit, String altUnits, CFPointWriterConfig config) throws IOException { @@ -59,12 +58,11 @@ public WriterCFStationProfileCollection(String fileOut, List globalAt } public void setStations(List stns) { - this.stnList = stns; + this.stnList = stns.stream().distinct().collect(Collectors.toList()); // see if there's altitude, wmoId for any stations for (StationFeature stn : stnList) { - if (!Double.isNaN(stn.getAltitude())) - useAlt = true; + useAlt = !Double.isNaN(stn.getAltitude()); if ((stn.getWmoId() != null) && (!stn.getWmoId().trim().isEmpty())) useWmoId = true; if ((stn.getDescription() != null) && (!stn.getDescription().trim().isEmpty())) @@ -84,43 +82,41 @@ public void setStations(List stns) { public int writeProfile(StationProfileFeature spf, ProfileFeature profile) throws IOException { int count = 0; for (PointFeature pf : profile) { - if (!headerDone) { - if (id_strlen == 0) - id_strlen = profile.getName().length() * 2; - writeHeader(spf, profile, pf); - headerDone = true; - } writeObsData(pf); count++; } - - Integer stnIndex = stationIndexMap.get(spf.getName()); - if (stnIndex == null) { - log.warn("BAD station {}", spf.getName()); - } else { - writeProfileData(stnIndex, profile, count); - } - return count; } - private void writeHeader(StationProfileFeature stn, ProfileFeature profile, PointFeature obs) throws IOException { - StructureData stnData = stn.getFeatureData(); - StructureData profileData = profile.getFeatureData(); - StructureData obsData = obs.getFeatureData(); + public void writeHeader(List stations) throws IOException { List obsCoords = new ArrayList<>(); - // obsCoords.add(VariableSimpleBuilder.makeScalar(timeName, "time of measurement", timeUnit.getUnitsString(), - // DataType.DOUBLE)); // LOOK ?? - Formatter coordNames = new Formatter().format("%s %s %s", profileTimeName, latName, lonName); - // if (useAlt) { - obsCoords.add(VariableSimpleBuilder.makeScalar(altitudeCoordinateName, "obs altitude", altUnits, DataType.DOUBLE) - .addAttribute(CF.STANDARD_NAME, "altitude") - .addAttribute(CF.POSITIVE, CF1Convention.getZisPositive(altitudeCoordinateName, altUnits)).build()); - coordNames.format(" %s", altitudeCoordinateName); - // } - - super.writeHeader2(obsCoords, stnData, profileData, obsData, coordNames.toString()); + List flattenStations = new ArrayList<>(); + List featureData = new ArrayList<>(); + List profileData = new ArrayList<>(); + + for (StationFeature station : stations) { + StationProfileFeature stationProfile = (StationProfileFeature) station; + featureData.add(stationProfile.getFeatureData()); + for (ProfileFeature pfc : stationProfile) { + flattenStations.add(pfc); + profileData.add(pfc.getFeatureData()); + } + + + obsCoords.add(VariableSimpleBuilder + .makeScalar(stationProfile.getTimeName(), "time of measurement", timeUnit.toString(), DataType.DOUBLE) + .build()); + if (altUnits != null) { + altitudeCoordinateName = stationProfile.getAltName(); + obsCoords + .add(VariableSimpleBuilder.makeScalar(altitudeCoordinateName, "obs altitude", altUnits, DataType.DOUBLE) + .addAttribute(CF.STANDARD_NAME, "altitude") + .addAttribute(CF.POSITIVE, CF1Convention.getZisPositive(altitudeCoordinateName, altUnits)).build()); + } + } + + super.writeHeader(obsCoords, flattenStations, featureData, profileData); // write the stations int count = 0; @@ -128,12 +124,36 @@ private void writeHeader(StationProfileFeature stn, ProfileFeature profile, Poin for (StationFeature sf : stnList) { writeStationData(sf); stationIndexMap.put(sf.getName(), count); + for (ProfileFeature p : (StationProfileFeature) sf) { + int countPoints = 0; + if (p.size() >= 0) { + countPoints += p.size(); + } else { + countPoints += Iterables.size(p); + } + writeProfileData(count, p, countPoints); + } count++; } + } + @Override + public void setFeatureAuxInfo(int nfeatures, int id_strlen) { + int countProfiles = 0; + int name_strlen = 0; + for (StationFeature s : stnList) { + name_strlen = Math.max(name_strlen, s.getName().length()); + if (((StationProfileFeature) s).size() >= 0) + countProfiles += ((StationProfileFeature) s).size(); + else { + countProfiles += Iterables.size(((StationProfileFeature) s)); + } + } + this.nfeatures = countProfiles; + this.id_strlen = name_strlen; } - protected void makeFeatureVariables(StructureData stnData, boolean isExtended) { + protected void makeFeatureVariables(List stnDataStructs, boolean isExtended) { // add the dimensions : extended model can use an unlimited dimension // Dimension stationDim = isExtended ? writer.addDimension(null, stationDimName, 0, true, true, false) : // writer.addDimension(null, stationDimName, nstns); @@ -145,8 +165,8 @@ protected void makeFeatureVariables(StructureData stnData, boolean isExtended) { if (useAlt) { stnVars.add(VariableSimpleBuilder.makeScalar(stationAltName, "station altitude", altUnits, DataType.DOUBLE) - .addAttribute(CF.STANDARD_NAME, CF.SURFACE_ALTITUDE) - .addAttribute(CF.POSITIVE, CF1Convention.getZisPositive(altName, altUnits)).build()); + .addAttribute(CF.STANDARD_NAME, CF.STATION_ALTITUDE) + .addAttribute(CF.POSITIVE, CF1Convention.getZisPositive(stationAltName, altUnits)).build()); } stnVars.add(VariableSimpleBuilder.makeString(stationIdName, "station identifier", null, id_strlen) @@ -160,9 +180,11 @@ protected void makeFeatureVariables(StructureData stnData, boolean isExtended) { stnVars.add(VariableSimpleBuilder.makeString(wmoName, "station WMO id", null, wmo_strlen) .addAttribute(CF.STANDARD_NAME, CF.PLATFORM_ID).build()); - for (StructureMembers.Member m : stnData.getMembers()) { - if (getDataVar(m.getName()) != null) - stnVars.add(VariableSimpleBuilder.fromMember(m).build()); + for (StructureData stnData : stnDataStructs) { + for (StructureMembers.Member m : stnData.getMembers()) { + if (getDataVar(m.getName()) != null) + stnVars.add(VariableSimpleBuilder.fromMember(m).build()); + } } if (isExtended) { @@ -195,7 +217,7 @@ private void writeStationData(StationFeature stn) throws IOException { } @Override - protected void makeMiddleVariables(StructureData profileData, boolean isExtended) { + protected void makeMiddleVariables(List profileDataStructs, boolean isExtended) { Dimension profileDim = writer.addDimension(null, profileDimName, nfeatures); // add the profile Variables using the profile dimension @@ -216,10 +238,12 @@ protected void makeMiddleVariables(StructureData profileData, boolean isExtended .add(VariableSimpleBuilder.makeScalar(stationIndexName, "station index for this profile", null, DataType.INT) .addAttribute(CF.INSTANCE_DIMENSION, stationDimName).build()); - for (StructureMembers.Member m : profileData.getMembers()) { - VariableSimpleIF dv = getDataVar(m.getName()); - if (dv != null) - profileVars.add(dv); + for (StructureData profileData : profileDataStructs) { + for (StructureMembers.Member m : profileData.getMembers()) { + VariableSimpleIF dv = getDataVar(m.getName()); + if (dv != null) + profileVars.add(dv); + } } if (isExtended) { @@ -232,14 +256,16 @@ protected void makeMiddleVariables(StructureData profileData, boolean isExtended private int profileRecno; + protected void resetProfileIndex() { + profileRecno = 0; + } + public void writeProfileData(int stnIndex, ProfileFeature profile, int nobs) throws IOException { trackBB(profile.getLatLon(), profile.getTime()); StructureMembers.Builder smb = StructureMembers.builder().setName("Coords"); smb.addMemberScalar(latName, null, null, DataType.DOUBLE, profile.getLatLon().getLatitude()); smb.addMemberScalar(lonName, null, null, DataType.DOUBLE, profile.getLatLon().getLongitude()); - // Date date = (profile.getTime() != null) ? (double) profile.getTime().getTime() : 0.0; // LOOK (profile.getTime() - // != null) ??? double timeInMyUnits = timeUnit.makeOffsetFromRefDate(profile.getTime()); smb.addMemberScalar(profileTimeName, null, null, DataType.DOUBLE, timeInMyUnits); // LOOK time not always part // of profile @@ -257,9 +283,15 @@ public void writeProfileData(int stnIndex, ProfileFeature profile, int nobs) thr private int obsRecno; + protected void resetObsIndex() { + obsRecno = 0; + } + public void writeObsData(PointFeature pf) throws IOException { StructureMembers.Builder smb = StructureMembers.builder().setName("Coords"); - smb.addMemberScalar(altitudeCoordinateName, null, null, DataType.DOUBLE, pf.getLocation().getAltitude()); + smb.addMemberScalar(pf.getFeatureCollection().getTimeName(), null, null, DataType.DOUBLE, pf.getObservationTime()); + smb.addMemberScalar(pf.getFeatureCollection().getAltName(), null, null, DataType.DOUBLE, + pf.getLocation().getAltitude()); StructureData coords = new StructureDataFromMember(smb.build()); // coords first so it takes precedence diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/writer/WriterCFTrajectoryCollection.java b/cdm/core/src/main/java/ucar/nc2/ft/point/writer/WriterCFTrajectoryCollection.java index 31fb6cd468..cc5f2fe90c 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/writer/WriterCFTrajectoryCollection.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/writer/WriterCFTrajectoryCollection.java @@ -11,8 +11,7 @@ import ucar.nc2.constants.CDM; import ucar.nc2.constants.CF; import ucar.nc2.dataset.conv.CF1Convention; -import ucar.nc2.ft.PointFeature; -import ucar.nc2.ft.TrajectoryFeature; +import ucar.nc2.ft.*; import ucar.nc2.time.CalendarDateUnit; import ucar.unidata.geoloc.EarthLocation; import java.io.IOException; @@ -30,7 +29,6 @@ public class WriterCFTrajectoryCollection extends CFPointWriter { /////////////////////////////////////////////////// private Structure featureStruct; // used for netcdf4 extended private Map featureVarMap = new HashMap<>(); - private boolean headerDone; public WriterCFTrajectoryCollection(String fileOut, List globalAtts, List dataVars, CalendarDateUnit timeUnit, String altUnits, CFPointWriterConfig config) throws IOException { @@ -43,42 +41,42 @@ public WriterCFTrajectoryCollection(String fileOut, List globalAtts, public int writeTrajectory(TrajectoryFeature traj) throws IOException { int count = 0; for (PointFeature pf : traj) { - if (!headerDone) { - if (id_strlen == 0) - id_strlen = traj.getName().length() * 2; - writeHeader(traj, pf); - headerDone = true; - } + if (id_strlen == 0) + id_strlen = traj.getName().length() * 2; writeObsData(pf); count++; } - writeTrajectoryData(traj, count); return count; } - private void writeHeader(TrajectoryFeature feature, PointFeature obs) throws IOException { - // obs data - List coords = new ArrayList<>(); - coords.add(VariableSimpleBuilder.makeScalar(obs.getFeatureCollection().getTimeName(), "time of measurement", - timeUnit.getUdUnit(), DataType.DOUBLE).addAttribute(CF.CALENDAR, timeUnit.getCalendar().toString()).build()); - - coords.add( + protected void writeHeader(List trajectories) throws IOException { + List obsCoords = new ArrayList<>(); + List featureData = new ArrayList<>(); + + for (TrajectoryFeature trajectory : trajectories) { + featureData.add(trajectory.getFeatureData()); + // obs data + obsCoords.add(VariableSimpleBuilder + .makeScalar(trajectory.getTimeName(), "time of measurement", timeUnit.getUdUnit(), DataType.DOUBLE) + .addAttribute(CF.CALENDAR, timeUnit.getCalendar().toString()).build()); + + if (altUnits != null) { + altitudeCoordinateName = trajectory.getAltName(); + obsCoords.add(VariableSimpleBuilder + .makeScalar(altitudeCoordinateName, "altitude of measurement", altUnits, DataType.DOUBLE) + .addAttribute(CF.POSITIVE, CF1Convention.getZisPositive(altitudeCoordinateName, altUnits)).build()); + } + } + obsCoords.add( VariableSimpleBuilder.makeScalar(latName, "latitude of measurement", CDM.LAT_UNITS, DataType.DOUBLE).build()); - coords.add( + obsCoords.add( VariableSimpleBuilder.makeScalar(lonName, "longitude of measurement", CDM.LON_UNITS, DataType.DOUBLE).build()); - Formatter coordNames = - new Formatter().format("%s %s %s", obs.getFeatureCollection().getTimeName(), latName, lonName); - if (altUnits != null) { - coords.add(VariableSimpleBuilder.makeScalar(altName, "altitude of measurement", altUnits, DataType.DOUBLE) - .addAttribute(CF.POSITIVE, CF1Convention.getZisPositive(altName, altUnits)).build()); - coordNames.format(" %s", altName); - } - super.writeHeader(coords, feature.getFeatureData(), obs.getFeatureData(), coordNames.toString()); + super.writeHeader(obsCoords, trajectories, featureData, null); } - protected void makeFeatureVariables(StructureData featureData, boolean isExtended) { + protected void makeFeatureVariables(List featureDataStructs, boolean isExtended) { // LOOK why not unlimited here fro extended model ? Dimension profileDim = writer.addDimension(null, trajDimName, nfeatures); @@ -91,10 +89,12 @@ protected void makeFeatureVariables(StructureData featureData, boolean isExtende .add(VariableSimpleBuilder.makeScalar(numberOfObsName, "number of obs for this profile", null, DataType.INT) .addAttribute(CF.SAMPLE_DIMENSION, recordDimName).build()); - for (StructureMembers.Member m : featureData.getMembers()) { - VariableSimpleIF dv = getDataVar(m.getName()); - if (dv != null) - featureVars.add(dv); + for (StructureData featureData : featureDataStructs) { + for (StructureMembers.Member m : featureData.getMembers()) { + VariableSimpleIF dv = getDataVar(m.getName()); + if (dv != null) + featureVars.add(dv); + } } if (isExtended) { @@ -131,7 +131,7 @@ public void writeObsData(PointFeature pf) throws IOException { smb.addMemberScalar(latName, null, null, DataType.DOUBLE, loc.getLatitude()); smb.addMemberScalar(lonName, null, null, DataType.DOUBLE, loc.getLongitude()); if (altUnits != null) - smb.addMemberScalar(altName, null, null, DataType.DOUBLE, loc.getAltitude()); + smb.addMemberScalar(pf.getFeatureCollection().getAltName(), null, null, DataType.DOUBLE, loc.getAltitude()); StructureData coords = new StructureDataFromMember(smb.build()); // coords first so it takes precedence diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/writer/WriterCFTrajectoryProfileCollection.java b/cdm/core/src/main/java/ucar/nc2/ft/point/writer/WriterCFTrajectoryProfileCollection.java index af61ad8230..767b9fa125 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/writer/WriterCFTrajectoryProfileCollection.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/writer/WriterCFTrajectoryProfileCollection.java @@ -11,9 +11,7 @@ import ucar.nc2.constants.CDM; import ucar.nc2.constants.CF; import ucar.nc2.dataset.conv.CF1Convention; -import ucar.nc2.ft.PointFeature; -import ucar.nc2.ft.ProfileFeature; -import ucar.nc2.ft.TrajectoryProfileFeature; +import ucar.nc2.ft.*; import ucar.nc2.time.CalendarDateUnit; import java.io.IOException; import java.util.*; @@ -39,7 +37,6 @@ public class WriterCFTrajectoryProfileCollection extends CFPointWriter { /////////////////////////////////////////////////// private Structure profileStruct; // used for netcdf4 extended private Map profileVarMap = new HashMap<>(); - private boolean headerDone; public WriterCFTrajectoryProfileCollection(String fileOut, List globalAtts, List dataVars, CalendarDateUnit timeUnit, String altUnits, CFPointWriterConfig config) @@ -59,12 +56,8 @@ public void setFeatureAuxInfo2(int ntraj, int traj_strlen) { public int writeProfile(TrajectoryProfileFeature section, ProfileFeature profile) throws IOException { int count = 0; for (PointFeature pf : profile) { - if (!headerDone) { - if (id_strlen == 0) - id_strlen = profile.getName().length() * 2; - writeHeader(section, profile, pf); - headerDone = true; - } + if (id_strlen == 0) + id_strlen = profile.getName().length() * 2; writeObsData(pf); count++; } @@ -78,26 +71,34 @@ public int writeProfile(TrajectoryProfileFeature section, ProfileFeature profile return count; } - private void writeHeader(TrajectoryProfileFeature section, ProfileFeature profile, PointFeature obs) - throws IOException { - - StructureData sectionData = section.getFeatureData(); - StructureData profileData = profile.getFeatureData(); - StructureData obsData = obs.getFeatureData(); - - Formatter coordNames = new Formatter().format("%s %s %s", profileTimeName, latName, lonName); + protected void writeHeader(List trajectoryProfiles) throws IOException { List obsCoords = new ArrayList<>(); - if (useAlt) { - obsCoords.add(VariableSimpleBuilder.makeScalar(altitudeCoordinateName, "obs altitude", altUnits, DataType.DOUBLE) - .addAttribute(CF.STANDARD_NAME, "altitude") - .addAttribute(CF.POSITIVE, CF1Convention.getZisPositive(altitudeCoordinateName, altUnits)).build()); - coordNames.format(" %s", altitudeCoordinateName); + List profileFeatures = new ArrayList<>(); + List trajectoryData = new ArrayList<>(); + List profileData = new ArrayList<>(); + + for (TrajectoryProfileFeature trajectoryProfile : trajectoryProfiles) { + trajectoryData.add(trajectoryProfile.getFeatureData()); + for (ProfileFeature profile : trajectoryProfile) { + profileData.add(profile.getFeatureData()); + profileFeatures.add(profile); + } + obsCoords.add(VariableSimpleBuilder + .makeScalar(trajectoryProfile.getTimeName(), "time of measurement", timeUnit.toString(), DataType.DOUBLE) + .build()); + + if (altUnits != null) { + altitudeCoordinateName = trajectoryProfile.getAltName(); + obsCoords + .add(VariableSimpleBuilder.makeScalar(altitudeCoordinateName, "obs altitude", altUnits, DataType.DOUBLE) + .addAttribute(CF.STANDARD_NAME, "altitude") + .addAttribute(CF.POSITIVE, CF1Convention.getZisPositive(altitudeCoordinateName, altUnits)).build()); + } } - - super.writeHeader2(obsCoords, sectionData, profileData, obsData, coordNames.toString()); + super.writeHeader(obsCoords, profileFeatures, trajectoryData, profileData); } - protected void makeFeatureVariables(StructureData trajData, boolean isExtended) { + protected void makeFeatureVariables(List trajDataStructs, boolean isExtended) { // add the dimensions : extended model can use an unlimited dimension Dimension trajDim = writer.addDimension(null, trajDimName, ntraj); @@ -107,9 +108,11 @@ protected void makeFeatureVariables(StructureData trajData, boolean isExtended) trajVars.add(VariableSimpleBuilder.makeString(trajIdName, "trajectory identifier", null, traj_strlen) .addAttribute(CF.CF_ROLE, CF.TRAJECTORY_ID).build()); - for (StructureMembers.Member m : trajData.getMembers()) { - if (getDataVar(m.getName()) != null) - trajVars.add(VariableSimpleBuilder.fromMember(m).build()); + for (StructureData trajData : trajDataStructs) { + for (StructureMembers.Member m : trajData.getMembers()) { + if (getDataVar(m.getName()) != null) + trajVars.add(VariableSimpleBuilder.fromMember(m).build()); + } } if (isExtended) { @@ -136,7 +139,7 @@ private int writeSectionData(TrajectoryProfileFeature section) throws IOExceptio } @Override - protected void makeMiddleVariables(StructureData profileData, boolean isExtended) { + protected void makeMiddleVariables(List profileDataStructs, boolean isExtended) { Dimension profileDim = writer.addDimension(null, profileDimName, nfeatures); @@ -162,10 +165,12 @@ protected void makeMiddleVariables(StructureData profileData, boolean isExtended .add(VariableSimpleBuilder.makeScalar(numberOfObsName, "number of obs for this profile", null, DataType.INT) .addAttribute(CF.SAMPLE_DIMENSION, recordDimName).build()); - for (StructureMembers.Member m : profileData.getMembers()) { - VariableSimpleIF dv = getDataVar(m.getName()); - if (dv != null) - profileVars.add(dv); + for (StructureData profileData : profileDataStructs) { + for (StructureMembers.Member m : profileData.getMembers()) { + VariableSimpleIF dv = getDataVar(m.getName()); + if (dv != null) + profileVars.add(dv); + } } if (isExtended) { @@ -202,8 +207,10 @@ public void writeProfileData(int sectionIndex, ProfileFeature profile, int nobs) private void writeObsData(PointFeature pf) throws IOException { StructureMembers.Builder smb = StructureMembers.builder().setName("Coords"); - if (useAlt) - smb.addMemberScalar(altitudeCoordinateName, null, null, DataType.DOUBLE, pf.getLocation().getAltitude()); + smb.addMemberScalar(pf.getFeatureCollection().getTimeName(), null, null, DataType.DOUBLE, pf.getObservationTime()); + if (altUnits != null) + smb.addMemberScalar(pf.getFeatureCollection().getAltName(), null, null, DataType.DOUBLE, + pf.getLocation().getAltitude()); StructureData coords = new StructureDataFromMember(smb.build()); // coords first so it takes precedence diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/CFPointWriter.java b/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/CFPointWriter.java index eaa20d2bdc..f63e81f939 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/CFPointWriter.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/CFPointWriter.java @@ -16,10 +16,12 @@ import java.util.Comparator; import java.util.Formatter; import java.util.List; +import java.util.ArrayList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import ucar.nc2.NetcdfFileWriter; +import ucar.nc2.Variable; import ucar.nc2.constants.FeatureType; import ucar.nc2.ft.DsgFeatureCollection; import ucar.nc2.ft.FeatureDatasetFactoryManager; @@ -28,6 +30,7 @@ import ucar.nc2.ft.PointFeatureCollection; import ucar.nc2.ft.ProfileFeature; import ucar.nc2.ft.ProfileFeatureCollection; +import ucar.nc2.ft.point.StationFeature; import ucar.nc2.ft.StationProfileFeature; import ucar.nc2.ft.StationProfileFeatureCollection; import ucar.nc2.ft.StationTimeSeriesFeatureCollection; @@ -35,7 +38,6 @@ import ucar.nc2.ft.TrajectoryFeatureCollection; import ucar.nc2.ft.TrajectoryProfileFeature; import ucar.nc2.ft.TrajectoryProfileFeatureCollection; -import ucar.nc2.ft.point.StationPointFeature; import ucar.nc2.util.CancelTask; import ucar.nc2.write.Nc4Chunking; import ucar.nc2.write.Nc4ChunkingStrategy; @@ -94,18 +96,18 @@ private static int writePointFeatureCollection(FeatureDatasetPoint fdpoint, Poin fdpoint.getDataVariables(), pfc.getTimeUnit(), pfc.getAltUnits(), config)) { pointWriter.setExtraVariables(pfc.getExtraVariables()); + pointWriter.writeHeader(fdpoint.getPointFeatureCollectionList()); int count = 0; - for (PointFeature pf : pfc) { - if (count == 0) - pointWriter.writeHeader(pf); - - pointWriter.writeRecord(pf, pf.getFeatureData()); - count++; - if (debug && count % 100 == 0) - logger.debug(String.format("%d ", count)); - if (debug && count % 1000 == 0) - logger.debug(String.format("%n ")); + for (DsgFeatureCollection pointFeatures : fdpoint.getPointFeatureCollectionList()) { + for (PointFeature pf : (PointFeatureCollection) pointFeatures) { + pointWriter.writeRecord(pf); + count++; + if (debug && count % 100 == 0) + logger.debug(String.format("%d ", count)); + if (debug && count % 1000 == 0) + logger.debug(String.format("%n ")); + } } pointWriter.finish(); @@ -119,20 +121,28 @@ private static int writeStationFeatureCollection(FeatureDatasetPoint dataset, St try (WriterCFStationCollection cfWriter = new WriterCFStationCollection(fileOut, dataset.attributes(), dataset.getDataVariables(), fc.getTimeUnit(), fc.getAltUnits(), config)) { - cfWriter.setExtraVariables(fc.getExtraVariables()); - cfWriter.writeHeader(fc); + List flattenFeatures = new ArrayList<>(); + List extraVariables = new ArrayList<>(); + for (DsgFeatureCollection featureCollection : dataset.getPointFeatureCollectionList()) { + extraVariables.addAll(featureCollection.getExtraVariables()); + flattenFeatures.addAll(((StationTimeSeriesFeatureCollection) featureCollection).getStationFeatures()); + } + + cfWriter.setExtraVariables(extraVariables); + cfWriter.writeHeader(flattenFeatures); int count = 0; - for (PointFeatureCollection pfc : fc) { - for (PointFeature pf : pfc) { - StationPointFeature spf = (StationPointFeature) pf; - cfWriter.writeRecord(spf.getStation().getName(), pf.getObservationTime(), - pf.getObservationTimeAsCalendarDate(), pf.getLocation().getAltitude(), pf.getFeatureData()); - count++; - if (debug && count % 100 == 0) - logger.debug(String.format("%d ", count)); - if (debug && count % 1000 == 0) - logger.debug(String.format("%n ")); + for (DsgFeatureCollection featureCollection : dataset.getPointFeatureCollectionList()) { + cfWriter.resetObsIndex(); + for (PointFeatureCollection pointCollection : (StationTimeSeriesFeatureCollection) featureCollection) { + for (PointFeature point : pointCollection) { + cfWriter.writeObsData(point); + count++; + if (debug && count % 100 == 0) + logger.debug(String.format("%d ", count)); + if (debug && count % 1000 == 0) + logger.debug(String.format("%n ")); + } } } @@ -147,36 +157,47 @@ private static int writeProfileFeatureCollection(FeatureDatasetPoint fdpoint, Pr try (WriterCFProfileCollection cfWriter = new WriterCFProfileCollection(fileOut, fdpoint.attributes(), fdpoint.getDataVariables(), fc.getTimeUnit(), fc.getAltUnits(), config)) { - cfWriter.setExtraVariables(fc.getExtraVariables()); + List flattenFeatures = new ArrayList<>(); + List extraVariables = new ArrayList<>(); // LOOK this is lame // LOOK not always needed int count = 0; int name_strlen = 0; - int nprofiles = fc.size(); - if (nprofiles < 0) { - for (ProfileFeature pf : fc) { - name_strlen = Math.max(name_strlen, pf.getName().length()); - count++; - } - nprofiles = count; - } else { - for (ProfileFeature pf : fc) { - name_strlen = Math.max(name_strlen, pf.getName().length()); - count++; - if (count > 10) - break; + int nprofiles = 0; + for (DsgFeatureCollection featureCollection : fdpoint.getPointFeatureCollectionList()) { + nprofiles += featureCollection.size(); + extraVariables.addAll(featureCollection.getExtraVariables()); + if (nprofiles < 0) { + for (ProfileFeature profile : (ProfileFeatureCollection) featureCollection) { + flattenFeatures.add(profile); + name_strlen = Math.max(name_strlen, profile.getName().length()); + count++; + } + nprofiles = count; + } else { + for (ProfileFeature profile : (ProfileFeatureCollection) featureCollection) { + flattenFeatures.add(profile); + name_strlen = Math.max(name_strlen, profile.getName().length()); + count++; + if (count > 10) + break; + } } } + cfWriter.setExtraVariables(extraVariables); cfWriter.setFeatureAuxInfo(nprofiles, name_strlen); + cfWriter.writeHeader(flattenFeatures); count = 0; - for (ProfileFeature profile : fc) { - count += cfWriter.writeProfile(profile); - if (debug && count % 10 == 0) - logger.debug(String.format("%d ", count)); - if (debug && count % 100 == 0) - logger.debug(String.format("%n ")); + for (DsgFeatureCollection featureCollection : fdpoint.getPointFeatureCollectionList()) { + for (ProfileFeature profile : (ProfileFeatureCollection) featureCollection) { + count += cfWriter.writeProfile(profile); + if (debug && count % 10 == 0) + logger.debug(String.format("%d ", count)); + if (debug && count % 100 == 0) + logger.debug(String.format("%n ")); + } } cfWriter.finish(); @@ -190,32 +211,37 @@ private static int writeTrajectoryFeatureCollection(FeatureDatasetPoint fdpoint, try (WriterCFTrajectoryCollection cfWriter = new WriterCFTrajectoryCollection(fileOut, fdpoint.attributes(), fdpoint.getDataVariables(), fc.getTimeUnit(), fc.getAltUnits(), config)) { - cfWriter.setExtraVariables(fc.getExtraVariables()); - + List flattenFeatures = new ArrayList<>(); + List extraVariables = new ArrayList<>(); // LOOK not always needed - int count = 0; + int npoints = 0; int name_strlen = 0; - int ntrajs = fc.size(); - if (ntrajs < 0) { - for (TrajectoryFeature traj : fc) { - name_strlen = Math.max(name_strlen, traj.getName().length()); - count++; + int ntrajs = 0; + for (DsgFeatureCollection featureCollection : fdpoint.getPointFeatureCollectionList()) { + for (TrajectoryFeature trajectory : (TrajectoryFeatureCollection) featureCollection) { + flattenFeatures.add(trajectory); + extraVariables.addAll(trajectory.getExtraVariables()); + name_strlen = Math.max(name_strlen, trajectory.getName().length()); + npoints++; } - ntrajs = count; } + ntrajs = npoints; + cfWriter.setExtraVariables(extraVariables); cfWriter.setFeatureAuxInfo(ntrajs, name_strlen); - - count = 0; - for (TrajectoryFeature traj : fc) { - count += cfWriter.writeTrajectory(traj); - if (debug && count % 10 == 0) - logger.debug(String.format("%d ", count)); - if (debug && count % 100 == 0) - logger.debug(String.format("%n ")); + cfWriter.writeHeader(flattenFeatures); + + npoints = 0; + for (DsgFeatureCollection featureCollection : fdpoint.getPointFeatureCollectionList()) { + for (TrajectoryFeature trajectory : (TrajectoryFeatureCollection) featureCollection) { + npoints += cfWriter.writeTrajectory(trajectory); + if (debug && npoints % 10 == 0) + logger.debug(String.format("%d ", npoints)); + if (debug && npoints % 100 == 0) + logger.debug(String.format("%n ")); + } } - cfWriter.finish(); - return count; + return npoints; } } @@ -225,33 +251,44 @@ private static int writeStationProfileFeatureCollection(FeatureDatasetPoint data try (WriterCFStationProfileCollection cfWriter = new WriterCFStationProfileCollection(fileOut, dataset.attributes(), dataset.getDataVariables(), fc.getTimeUnit(), fc.getAltUnits(), config)) { - cfWriter.setExtraVariables(fc.getExtraVariables()); - cfWriter.setStations(fc.getStationFeatures()); + List extraVariables = new ArrayList<>(); + List flattenFeatures = new ArrayList<>(); int name_strlen = 0; int countProfiles = 0; - for (StationProfileFeature spf : fc) { - name_strlen = Math.max(name_strlen, spf.getName().length()); - if (spf.size() >= 0) - countProfiles += spf.size(); - else { - countProfiles += Iterables.size(spf); + for (DsgFeatureCollection featureCollection : dataset.getPointFeatureCollectionList()) { + extraVariables.addAll(featureCollection.getExtraVariables()); + for (StationFeature station : ((StationProfileFeatureCollection) featureCollection).getStationFeatures()) { + flattenFeatures.add(station); + name_strlen = Math.max(name_strlen, station.getName().length()); + if (((StationProfileFeature) station).size() >= 0) + countProfiles += ((StationProfileFeature) station).size(); + else { + countProfiles += Iterables.size(((StationProfileFeature) station)); + } } } + + cfWriter.setExtraVariables(extraVariables); + cfWriter.setStations(flattenFeatures); cfWriter.setFeatureAuxInfo(countProfiles, name_strlen); + cfWriter.writeHeader(flattenFeatures); + int count = 0; - for (StationProfileFeature spf : fc) { - for (ProfileFeature pf : spf) { - if (pf.getTime() == null) { - continue; // assume this means its an "incomplete multidimensional" + for (DsgFeatureCollection featureCollection : dataset.getPointFeatureCollectionList()) { + cfWriter.resetObsIndex(); + for (StationFeature station : ((StationProfileFeatureCollection) featureCollection).getStationFeatures()) { + for (ProfileFeature profile : (StationProfileFeature) station) { + if (profile.getTime() == null) { + continue; // assume this means its an "incomplete multidimensional" + } + count += cfWriter.writeProfile(profile); + if (debug && count % 100 == 0) + logger.debug(String.format("%d ", count)); + if (debug && count % 1000 == 0) + logger.debug(String.format("%n ")); } - - count += cfWriter.writeProfile(spf, pf); - if (debug && count % 100 == 0) - logger.debug(String.format("%d ", count)); - if (debug && count % 1000 == 0) - logger.debug(String.format("%n ")); } } @@ -266,38 +303,46 @@ private static int writeTrajectoryProfileFeatureCollection(FeatureDatasetPoint d try (WriterCFTrajectoryProfileCollection cfWriter = new WriterCFTrajectoryProfileCollection(fileOut, dataset.attributes(), dataset.getDataVariables(), fc.getTimeUnit(), fc.getAltUnits(), config)) { - cfWriter.setExtraVariables(fc.getExtraVariables()); + List extraVariables = new ArrayList<>(); + List flattenFeatures = new ArrayList<>(); int traj_strlen = 0; int prof_strlen = 0; int countTrajectories = 0; int countProfiles = 0; - for (TrajectoryProfileFeature spf : fc) { - countTrajectories++; - traj_strlen = Math.max(traj_strlen, spf.getName().length()); - if (spf.size() >= 0) - countProfiles += spf.size(); - else { - for (ProfileFeature profile : spf) { - prof_strlen = Math.max(prof_strlen, profile.getName().length()); - countProfiles++; + for (DsgFeatureCollection featureCollection : dataset.getPointFeatureCollectionList()) { + for (TrajectoryProfileFeature trajProf : ((TrajectoryProfileFeatureCollection) featureCollection)) { + flattenFeatures.add(trajProf); + extraVariables.addAll(trajProf.getExtraVariables()); + countTrajectories++; + traj_strlen = Math.max(traj_strlen, trajProf.getName().length()); + if (trajProf.size() >= 0) + countProfiles += trajProf.size(); + else { + for (ProfileFeature profile : trajProf) { + prof_strlen = Math.max(prof_strlen, profile.getName().length()); + countProfiles++; + } } } } + cfWriter.setExtraVariables(extraVariables); cfWriter.setFeatureAuxInfo(countProfiles, prof_strlen); cfWriter.setFeatureAuxInfo2(countTrajectories, traj_strlen); + cfWriter.writeHeader(flattenFeatures); int count = 0; - for (TrajectoryProfileFeature spf : fc) { - for (ProfileFeature profile : spf) { - if (profile.getTime() == null) - continue; // assume this means its a "incomplete multidimensional" - - count += cfWriter.writeProfile(spf, profile); - if (debug && count % 100 == 0) - logger.debug(String.format("%d ", count)); - if (debug && count % 1000 == 0) - logger.debug(String.format("%n ")); + for (DsgFeatureCollection featureCollection : dataset.getPointFeatureCollectionList()) { + for (TrajectoryProfileFeature trajProf : (TrajectoryProfileFeatureCollection) featureCollection) { + for (ProfileFeature profile : trajProf) { + if (profile.getTime() == null) + continue; // assume this means its a "incomplete multidimensional" + count += cfWriter.writeProfile(trajProf, profile); + if (debug && count % 100 == 0) + logger.debug(String.format("%d ", count)); + if (debug && count % 1000 == 0) + logger.debug(String.format("%n ")); + } } } diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFPointAbstract.java b/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFPointAbstract.java index 6bbdd99a74..a2dba6ea4a 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFPointAbstract.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFPointAbstract.java @@ -21,24 +21,14 @@ import ucar.ma2.InvalidRangeException; import ucar.ma2.StructureData; import ucar.ma2.StructureMembers; -import ucar.nc2.Attribute; -import ucar.nc2.AttributeContainer; -import ucar.nc2.Dimension; -import ucar.nc2.Dimensions; -import ucar.nc2.NetcdfFile; -import ucar.nc2.Structure; -import ucar.nc2.Variable; -import ucar.nc2.VariableSimpleIF; +import ucar.nc2.*; import ucar.nc2.constants.ACDD; import ucar.nc2.constants.AxisType; import ucar.nc2.constants.CDM; import ucar.nc2.constants.CF; import ucar.nc2.constants._Coordinate; import ucar.nc2.dataset.CoordinateAxis; -import ucar.nc2.ft.PointFeature; -import ucar.nc2.ft.StationTimeSeriesFeature; -import ucar.nc2.ft.point.StationPointFeature; -import ucar.nc2.ft.point.StationTimeSeriesCollectionImpl; +import ucar.nc2.ft.*; import ucar.nc2.time.CalendarDate; import ucar.nc2.time.CalendarDateFormatter; import ucar.nc2.time.CalendarDateUnit; @@ -104,7 +94,7 @@ abstract class WriterCFPointAbstract implements Closeable { Structure record; // used for netcdf3 and netcdf4 extended private Dimension recordDim; HashSet dataMap = new HashSet<>(); - private List extra; + private List extra = new ArrayList<>(); LatLonRect llbb; private CalendarDate minDate; @@ -121,7 +111,7 @@ abstract class WriterCFPointAbstract implements Closeable { * @param config configuration */ WriterCFPointAbstract(String fileOut, AttributeContainer atts, List dataVars, - CalendarDateUnit timeUnit, @Nullable String altUnits, CFPointWriterConfig config) throws IOException { + CalendarDateUnit timeUnit, @Nullable String altUnits, CFPointWriterConfig config) { this.dataVars = dataVars; this.timeUnit = timeUnit; @@ -191,33 +181,26 @@ VariableSimpleIF findDataVar(String name) { } // Always overridden - abstract void makeFeatureVariables(StructureData featureData, boolean isExtended); + abstract void makeFeatureVariables(List featureData, boolean isExtended); // Supplied when its a two level feature (station profile, trajectory profile) - void makeMiddleVariables(StructureData middleData, boolean isExtended) { + void makeMiddleVariables(List middleData, boolean isExtended) { // NOOP } - protected void writeHeader(List obsCoords, StationTimeSeriesCollectionImpl stationFeatures, - @Nullable StructureData middleData) throws IOException { + protected void writeHeader(List obsCoords, + Iterable stationFeatures, List featureDataStruct, + @Nullable List middleDataStruct) throws IOException { this.recordDim = Dimension.builder().setName(recordDimName).setIsUnlimited(true).build(); writerb.addDimension(recordDim); addExtraVariables(); + if (featureDataStruct != null) + makeFeatureVariables(featureDataStruct, isExtendedModel); + if (middleDataStruct != null) + makeMiddleVariables(middleDataStruct, isExtendedModel); - PeekingIterator stIter = Iterators.peekingIterator(stationFeatures.iterator()); - if (stIter.hasNext()) { - StationTimeSeriesFeature stationFeat = stIter.peek(); - - StructureData featureData = stationFeat.getFeatureData(); - if (featureData != null) { - makeFeatureVariables(featureData, isExtendedModel); - } - if (middleData != null) { - makeMiddleVariables(middleData, isExtendedModel); - } - } Structure.Builder recordb = null; if (isExtendedModel) { recordb = writerb.addStructure(recordName, recordDimName); @@ -226,24 +209,19 @@ protected void writeHeader(List obsCoords, StationTimeSeriesCo addCoordinatesClassic(recordDim, obsCoords, dataMap); } - for (StationTimeSeriesFeature stnFeature : stationFeatures) { + for (PointFeatureCollection stnFeature : stationFeatures) { PeekingIterator iter = Iterators.peekingIterator(stnFeature.iterator()); if (iter.hasNext()) { PointFeature pointFeat = iter.peek(); - assert pointFeat instanceof StationPointFeature : "Expected pointFeat to be a StationPointFeature, not a " - + pointFeat.getClass().getSimpleName(); StructureData obsData = pointFeat.getFeatureData(); - String timeCoordName = pointFeat.getFeatureCollection().getTimeName(); - - Formatter coordNames = new Formatter().format("%s %s %s", timeCoordName, latName, lonName); + Formatter coordNames = new Formatter().format("%s %s %s", stnFeature.getTimeName(), latName, lonName); if (!Double.isNaN(pointFeat.getLocation().getAltitude())) { - coordNames.format(" %s", altitudeCoordinateName); + coordNames.format(" %s", stnFeature.getAltName()); } if (isExtendedModel) { addDataVariablesExtended(recordb, obsData, coordNames.toString()); - } else { addDataVariablesClassic(recordDim, obsData, dataMap, coordNames.toString()); } @@ -255,49 +233,6 @@ protected void writeHeader(List obsCoords, StationTimeSeriesCo finishBuilding(); } - void writeHeader(List obsCoords, StructureData featureData, @Nullable StructureData middleData, - StructureData obsData, String coordNames) throws IOException { - this.recordDim = Dimension.builder().setName(recordDimName).setIsUnlimited(true).build(); - writerb.addDimension(recordDim); - - addExtraVariables(); - - if (isExtendedModel) { - if (featureData != null) { - makeFeatureVariables(featureData, true); - } - if (middleData != null) { - makeMiddleVariables(middleData, true); - } - Structure.Builder recordb = writerb.addStructure(recordName, recordDimName); - addCoordinatesExtended(recordb, obsCoords); - addDataVariablesExtended(recordb, obsData, coordNames); - - } else { - if (featureData != null) { - makeFeatureVariables(featureData, false); - } - if (middleData != null) { - makeMiddleVariables(middleData, false); - } - addCoordinatesClassic(recordDim, obsCoords, dataMap); - addDataVariablesClassic(recordDim, obsData, dataMap, coordNames); - // record = writer.addRecordStructure(); // for netcdf3 - } - - // Create the NetcdfFile and write variable metadata to it. - this.writer = writerb.build(); - /* - * NetcdfFormatWriter.Result result = writer.create(netcdfBuilder.build(), 0); - * if (!result.wasWritten()) { - * throw new IOException(result.getErrorMessage()); - * } - */ - - writeExtraVariables(); - finishBuilding(); - } - private void addExtraVariables() { if (extra == null) return; @@ -621,5 +556,4 @@ public void finish() throws IOException { public void close() throws IOException { writer.close(); } - } diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFPointCollection.java b/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFPointCollection.java index 808b1cd22b..7b83c2a3bf 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFPointCollection.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFPointCollection.java @@ -8,7 +8,6 @@ import com.google.common.collect.ImmutableList; import java.io.IOException; import java.util.ArrayList; -import java.util.Formatter; import java.util.List; import ucar.ma2.DataType; import ucar.ma2.StructureData; @@ -22,10 +21,10 @@ import ucar.nc2.constants.CDM; import ucar.nc2.constants.CF; import ucar.nc2.dataset.conv.CF1Convention; +import ucar.nc2.ft.DsgFeatureCollection; import ucar.nc2.ft.PointFeature; -import ucar.nc2.time.CalendarDate; +import ucar.nc2.ft.PointFeatureCollection; import ucar.nc2.time.CalendarDateUnit; -import ucar.unidata.geoloc.EarthLocation; /** * Write a CF 1.6 "Discrete Sample" point file. @@ -47,54 +46,53 @@ class WriterCFPointCollection extends WriterCFPointAbstract { writerb.addAttribute(new Attribute(CF.DSG_REPRESENTATION, "Point Data, H.1")); } - void writeHeader(PointFeature pf) throws IOException { + void writeHeader(List featureCollections) throws IOException { List coords = new ArrayList<>(); - coords.add(VariableSimpleBuilder.makeScalar(pf.getFeatureCollection().getTimeName(), "time of measurement", - timeUnit.getUdUnit(), DataType.DOUBLE).addAttribute(CF.CALENDAR, timeUnit.getCalendar().toString()).build()); + List pointCollections = new ArrayList<>(); + for (DsgFeatureCollection featureCollection : featureCollections) { + pointCollections.add((PointFeatureCollection) featureCollection); + coords.add(VariableSimpleBuilder + .makeScalar(featureCollection.getTimeName(), "time of measurement", timeUnit.getUdUnit(), DataType.DOUBLE) + .addAttribute(CF.CALENDAR, timeUnit.getCalendar().toString()).build()); + if (altUnits != null) { + altitudeCoordinateName = featureCollection.getAltName(); + coords.add(VariableSimpleBuilder + .makeScalar(altitudeCoordinateName, "altitude of measurement", altUnits, DataType.DOUBLE) + .addAttribute(CF.POSITIVE, CF1Convention.getZisPositive(altName, altUnits)).build()); + } + } coords.add( VariableSimpleBuilder.makeScalar(latName, "latitude of measurement", CDM.LAT_UNITS, DataType.DOUBLE).build()); coords.add( VariableSimpleBuilder.makeScalar(lonName, "longitude of measurement", CDM.LON_UNITS, DataType.DOUBLE).build()); - Formatter coordNames = - new Formatter().format("%s %s %s", pf.getFeatureCollection().getTimeName(), latName, lonName); - if (altUnits != null) { - coords.add(VariableSimpleBuilder.makeScalar(altName, "altitude of measurement", altUnits, DataType.DOUBLE) - .addAttribute(CF.POSITIVE, CF1Convention.getZisPositive(altName, altUnits)).build()); - coordNames.format(" %s", altName); - } - - super.writeHeader(coords, null, null, pf.getDataAll(), coordNames.toString()); + super.writeHeader(coords, pointCollections, null, null); } - @Override - void makeFeatureVariables(StructureData featureData, boolean isExtended) { - // NOOP - } ///////////////////////////////////////////////////////// // writing data private int obsRecno; - void writeRecord(PointFeature sobs, StructureData sdata) throws IOException { - writeRecord(sobs.getObservationTime(), sobs.getObservationTimeAsCalendarDate(), sobs.getLocation(), sdata); - } - - private void writeRecord(double timeCoordValue, CalendarDate obsDate, EarthLocation loc, StructureData sdata) - throws IOException { - trackBB(loc.getLatLon(), obsDate); + protected void writeRecord(PointFeature point) throws IOException { + trackBB(point.getLocation().getLatLon(), point.getObservationTimeAsCalendarDate()); StructureMembers.Builder smb = StructureMembers.builder().setName("Coords"); - smb.addMemberScalar(timeName, null, null, DataType.DOUBLE, timeCoordValue); - smb.addMemberScalar(latName, null, null, DataType.DOUBLE, loc.getLatitude()); - smb.addMemberScalar(lonName, null, null, DataType.DOUBLE, loc.getLongitude()); + smb.addMemberScalar(point.getFeatureCollection().getTimeName(), null, null, DataType.DOUBLE, + point.getObservationTime()); + smb.addMemberScalar(latName, null, null, DataType.DOUBLE, point.getLocation().getLatitude()); + smb.addMemberScalar(lonName, null, null, DataType.DOUBLE, point.getLocation().getLongitude()); if (altUnits != null) - smb.addMemberScalar(altitudeCoordinateName, null, null, DataType.DOUBLE, loc.getAltitude()); + smb.addMemberScalar(point.getFeatureCollection().getAltName(), null, null, DataType.DOUBLE, + point.getLocation().getAltitude()); StructureData coords = new StructureDataFromMember(smb.build()); // coords first so it takes precedence - StructureDataComposite sdall = StructureDataComposite.create(ImmutableList.of(coords, sdata)); + StructureDataComposite sdall = StructureDataComposite.create(ImmutableList.of(coords, point.getFeatureData())); obsRecno = super.writeStructureData(obsRecno, record, sdall, dataMap); } + @Override + void makeFeatureVariables(List featureData, boolean isExtended) {} + } diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFProfileCollection.java b/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFProfileCollection.java index 67a215b703..8afd2407be 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFProfileCollection.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFProfileCollection.java @@ -8,7 +8,6 @@ import com.google.common.collect.ImmutableList; import java.io.IOException; import java.util.ArrayList; -import java.util.Formatter; import java.util.HashSet; import java.util.List; import ucar.ma2.DataType; @@ -50,7 +49,6 @@ class WriterCFProfileCollection extends WriterCFPointAbstract { /////////////////////////////////////////////////// private Structure profileStruct; // used for netcdf4 extended private HashSet featureVarMap = new HashSet<>(); - private boolean headerDone; WriterCFProfileCollection(String fileOut, AttributeContainer globalAtts, List dataVars, CalendarDateUnit timeUnit, String altUnits, CFPointWriterConfig config) throws IOException { @@ -67,14 +65,10 @@ void finishBuilding() throws IOException { } int writeProfile(ProfileFeature profile) throws IOException { + if (id_strlen == 0) + id_strlen = profile.getName().length() * 2; int count = 0; for (PointFeature pf : profile) { - if (!headerDone) { - if (id_strlen == 0) - id_strlen = profile.getName().length() * 2; - writeHeader(profile, pf); - headerDone = true; - } writeObsData(pf); count++; } @@ -83,23 +77,27 @@ int writeProfile(ProfileFeature profile) throws IOException { return count; } - private void writeHeader(ProfileFeature profile, PointFeature obs) throws IOException { - - Formatter coordNames = new Formatter().format("%s %s %s", profileTimeName, latName, lonName); + protected void writeHeader(List profiles) throws IOException { List coords = new ArrayList<>(); - if (useAlt) { - coords.add(VariableSimpleBuilder.makeScalar(altitudeCoordinateName, "obs altitude", altUnits, DataType.DOUBLE) - .addAttribute(CF.STANDARD_NAME, "altitude") - .addAttribute(CF.POSITIVE, CF1Convention.getZisPositive(altitudeCoordinateName, altUnits)).build()); - coordNames.format(" %s", altitudeCoordinateName); + List profileData = new ArrayList<>(); + for (ProfileFeature profile : profiles) { + profileData.add(profile.getFeatureData()); + coords.add(VariableSimpleBuilder + .makeScalar(profile.getTimeName(), "time of measurement", profile.getTimeUnit().getUdUnit(), DataType.DOUBLE) + .addAttribute(CF.CALENDAR, profile.getTimeUnit().getCalendar().toString()).build()); + if (useAlt) { + altitudeCoordinateName = profile.getAltName(); + coords.add(VariableSimpleBuilder.makeScalar(altitudeCoordinateName, "obs altitude", altUnits, DataType.DOUBLE) + .addAttribute(CF.STANDARD_NAME, "altitude") + .addAttribute(CF.POSITIVE, CF1Convention.getZisPositive(altitudeCoordinateName, altUnits)).build()); + } } - super.writeHeader(coords, profile.getFeatureData(), null, obs.getFeatureData(), coordNames.toString()); + super.writeHeader(coords, profiles, profileData, null); } @Override - protected void makeFeatureVariables(StructureData featureData, boolean isExtended) { - + void makeFeatureVariables(List featureDataStruct, boolean isExtended) { // LOOK why not unlimited here ? Dimension profileDim = writerb.addDimension(profileDimName, nfeatures); @@ -120,11 +118,12 @@ protected void makeFeatureVariables(StructureData featureData, boolean isExtende .makeScalar(profileTimeName, "nominal time of profile", timeUnit.getUdUnit(), DataType.DOUBLE) .addAttribute(CF.CALENDAR, timeUnit.getCalendar().toString()).build()); - - for (StructureMembers.Member m : featureData.getMembers()) { - VariableSimpleIF dv = findDataVar(m.getName()); - if (dv != null) - profileVars.add(dv); + for (StructureData featureData : featureDataStruct) { + for (StructureMembers.Member m : featureData.getMembers()) { + VariableSimpleIF dv = findDataVar(m.getName()); + if (dv != null) + profileVars.add(dv); + } } if (isExtended) { @@ -162,8 +161,10 @@ private void writeProfileData(ProfileFeature profile, int nobs) throws IOExcepti private void writeObsData(PointFeature pf) throws IOException { StructureMembers.Builder smb = StructureMembers.builder().setName("Coords"); + smb.addMemberScalar(pf.getFeatureCollection().getTimeName(), null, null, DataType.DOUBLE, pf.getObservationTime()); if (useAlt) - smb.addMemberScalar(altitudeCoordinateName, null, null, DataType.DOUBLE, pf.getLocation().getAltitude()); + smb.addMemberScalar(pf.getFeatureCollection().getAltName(), null, null, DataType.DOUBLE, + pf.getLocation().getAltitude()); StructureData coords = new StructureDataFromMember(smb.build()); // coords first so it takes precedence diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFStationCollection.java b/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFStationCollection.java index 5120d8efec..e04b38a996 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFStationCollection.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFStationCollection.java @@ -26,14 +26,9 @@ import ucar.nc2.VariableSimpleIF; import ucar.nc2.constants.CDM; import ucar.nc2.constants.CF; -import ucar.nc2.dataset.conv.CF1Convention; -import ucar.nc2.ft.DsgFeatureCollection; -import ucar.nc2.ft.StationTimeSeriesFeatureCollection; +import ucar.nc2.ft.*; import ucar.nc2.ft.point.StationFeature; -import ucar.nc2.ft.point.StationTimeSeriesCollectionImpl; -import ucar.nc2.time.CalendarDate; import ucar.nc2.time.CalendarDateUnit; -import ucar.unidata.geoloc.Station; /** * Write a CF "Discrete Sample" station file. @@ -60,7 +55,6 @@ class WriterCFStationCollection extends WriterCFPointAbstract { private HashMap stationIndexMap; private boolean useDesc; - private boolean useAlt; private boolean useWmoId; private int desc_strlen = 1, wmo_strlen = 1; @@ -80,14 +74,16 @@ void finishBuilding() throws IOException { stationStruct = findStructure(stationStructName); } - protected void writeHeader(StationTimeSeriesFeatureCollection stations) throws IOException { - this.stnList = stations.getStationFeatures().stream().distinct().collect(Collectors.toList()); + protected void writeHeader(List stations) throws IOException { + this.stnList = stations.stream().distinct().collect(Collectors.toList()); List coords = new ArrayList<>(); + List flattenStations = new ArrayList<>(); + List stationData = new ArrayList<>(); // see if there's altitude, wmoId for any stations - for (Station stn : stations) { - if (!Double.isNaN(stn.getAltitude())) - useAlt = true; + for (StationFeature stn : stations) { + stationData.add(stn.getFeatureData()); + useAlt = altUnits != null; if ((stn.getWmoId() != null) && (!stn.getWmoId().trim().isEmpty())) useWmoId = true; if ((stn.getDescription() != null) && (!stn.getDescription().trim().isEmpty())) @@ -102,6 +98,7 @@ protected void writeHeader(StationTimeSeriesFeatureCollection stations) throws I if (stn instanceof DsgFeatureCollection) { DsgFeatureCollection dsgStation = (DsgFeatureCollection) stn; + flattenStations.add((PointFeatureCollection) dsgStation); if (coords.stream().noneMatch(x -> x.getShortName().equals(dsgStation.getTimeName()))) { coords.add(VariableSimpleBuilder .makeScalar(dsgStation.getTimeName(), "time of measurement", dsgStation.getTimeUnit().getUdUnit(), @@ -120,9 +117,10 @@ protected void writeHeader(StationTimeSeriesFeatureCollection stations) throws I .makeScalar(stationIndexName, "station index for this observation record", null, DataType.INT) .addAttribute(CF.INSTANCE_DIMENSION, stationDimName).build()); - super.writeHeader(coords, (StationTimeSeriesCollectionImpl) stations, null); + + super.writeHeader(coords, flattenStations, stationData, null); int count = 0; - stationIndexMap = new HashMap<>(2 * stnList.size()); + stationIndexMap = new HashMap<>(stnList.size(), 1.0f); for (StationFeature stn : stnList) { writeStationData(stn); stationIndexMap.put(stn.getName(), count); @@ -131,7 +129,7 @@ protected void writeHeader(StationTimeSeriesFeatureCollection stations) throws I } @Override - void makeFeatureVariables(StructureData featureData, boolean isExtended) { + void makeFeatureVariables(List featureDataStructs, boolean isExtended) { // add the dimensions : extendded model can use an unlimited dimension // Dimension stationDim = isExtended ? writer.addDimension(null, stationDimName, 0, true, true, false) : // writer.addDimension(null, stationDimName, nstns); @@ -143,8 +141,7 @@ void makeFeatureVariables(StructureData featureData, boolean isExtended) { if (useAlt) { stnVars.add(VariableSimpleBuilder.makeScalar(stationAltName, "station altitude", altUnits, DataType.DOUBLE) - .addAttribute(CF.STANDARD_NAME, CF.SURFACE_ALTITUDE) - .addAttribute(CF.POSITIVE, CF1Convention.getZisPositive(altName, altUnits)).build()); + .addAttribute(CF.STANDARD_NAME, CF.STATION_ALTITUDE).build()); } stnVars.add(VariableSimpleBuilder.makeString(stationIdName, "station identifier", null, id_strlen) @@ -158,9 +155,11 @@ void makeFeatureVariables(StructureData featureData, boolean isExtended) { stnVars.add(VariableSimpleBuilder.makeString(wmoName, "station WMO id", null, wmo_strlen) .addAttribute(CF.STANDARD_NAME, CF.PLATFORM_ID).build()); - for (StructureMembers.Member m : featureData.getMembers()) { - if (findDataVar(m.getName()) != null) { - stnVars.add(VariableSimpleBuilder.fromMember(m).build()); + for (StructureData featureData : featureDataStructs) { + for (StructureMembers.Member m : featureData.getMembers()) { + if (findDataVar(m.getName()) != null) { + stnVars.add(VariableSimpleBuilder.fromMember(m).build()); + } } } @@ -178,7 +177,8 @@ private void writeStationData(StationFeature stn) throws IOException { StructureMembers.Builder smb = StructureMembers.builder().setName("Coords"); smb.addMemberScalar(latName, null, null, DataType.DOUBLE, stn.getLatLon().getLatitude()); smb.addMemberScalar(lonName, null, null, DataType.DOUBLE, stn.getLatLon().getLongitude()); - smb.addMemberScalar(stationAltName, null, null, DataType.DOUBLE, stn.getAltitude()); + if (useAlt) + smb.addMemberScalar(stationAltName, null, null, DataType.DOUBLE, stn.getAltitude()); smb.addMemberString(stationIdName, null, null, stn.getName().trim(), id_strlen); if (useDesc) smb.addMemberString(descName, null, null, stn.getDescription().trim(), desc_strlen); @@ -192,24 +192,28 @@ private void writeStationData(StationFeature stn) throws IOException { private int obsRecno; - protected void writeRecord(String stnName, double timeCoordValue, CalendarDate obsDate, double altCoordValue, - StructureData sdata) throws IOException { - trackBB(null, obsDate); - - Integer parentIndex = stationIndexMap.get(stnName); + protected void writeObsData(PointFeature pf) throws IOException { + trackBB(null, pf.getObservationTimeAsCalendarDate()); + String stationName = pf.getFeatureCollection().getName(); + Integer parentIndex = stationIndexMap.get(stationName); if (parentIndex == null) - throw new RuntimeException("Cant find station " + stnName); + throw new RuntimeException("Cant find station " + stationName); StructureMembers.Builder smb = StructureMembers.builder().setName("Coords"); - smb.addMemberScalar(timeName, null, null, DataType.DOUBLE, timeCoordValue); - if (!Double.isNaN(altCoordValue)) - smb.addMemberScalar(altitudeCoordinateName, null, null, DataType.DOUBLE, altCoordValue); + smb.addMemberScalar(pf.getFeatureCollection().getTimeName(), null, null, DataType.DOUBLE, pf.getObservationTime()); + if (altUnits != null) + smb.addMemberScalar(pf.getFeatureCollection().getAltName(), null, null, DataType.DOUBLE, + pf.getLocation().getAltitude()); smb.addMemberScalar(stationIndexName, null, null, DataType.INT, parentIndex); StructureData coords = new StructureDataFromMember(smb.build()); // coords first so it takes precedence - StructureDataComposite sdall = StructureDataComposite.create(ImmutableList.of(coords, sdata)); + StructureDataComposite sdall = StructureDataComposite.create(ImmutableList.of(coords, pf.getFeatureData())); obsRecno = super.writeStructureData(obsRecno, record, sdall, dataMap); } + protected void resetObsIndex() { + obsRecno = 0; + } } + diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFStationProfileCollection.java b/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFStationProfileCollection.java index 034219f0bb..f43eba6616 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFStationProfileCollection.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFStationProfileCollection.java @@ -8,10 +8,12 @@ import com.google.common.collect.ImmutableList; import java.io.IOException; import java.util.ArrayList; -import java.util.Formatter; import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.stream.Collectors; + +import com.google.common.collect.Iterables; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import ucar.ma2.DataType; @@ -29,6 +31,7 @@ import ucar.nc2.constants.CF; import ucar.nc2.dataset.conv.CF1Convention; import ucar.nc2.ft.PointFeature; +import ucar.nc2.ft.PointFeatureCollection; import ucar.nc2.ft.ProfileFeature; import ucar.nc2.ft.StationProfileFeature; import ucar.nc2.ft.point.StationFeature; @@ -51,7 +54,6 @@ class WriterCFStationProfileCollection extends WriterCFPointAbstract { private HashMap stationIndexMap; private boolean useDesc; - private boolean useAlt; private boolean useWmoId; private int desc_strlen = 1, wmo_strlen = 1; @@ -61,7 +63,6 @@ class WriterCFStationProfileCollection extends WriterCFPointAbstract { // private Formatter coordNames = new Formatter(); private Structure profileStruct; // used for netcdf4 extended private HashSet profileVarMap = new HashSet<>(); - private boolean headerDone; WriterCFStationProfileCollection(String fileOut, AttributeContainer globalAtts, List dataVars, CalendarDateUnit timeUnit, String altUnits, CFPointWriterConfig config) throws IOException { @@ -72,12 +73,11 @@ class WriterCFStationProfileCollection extends WriterCFPointAbstract { } void setStations(List stns) { - this.stnList = stns; + this.stnList = stns.stream().distinct().collect(Collectors.toList()); // see if there's altitude, wmoId for any stations for (StationFeature stn : stnList) { - if (!Double.isNaN(stn.getAltitude())) - useAlt = true; + useAlt = !Double.isNaN(stn.getAltitude()); if ((stn.getWmoId() != null) && (!stn.getWmoId().trim().isEmpty())) useWmoId = true; if ((stn.getDescription() != null) && (!stn.getDescription().trim().isEmpty())) @@ -94,42 +94,41 @@ void setStations(List stns) { llbb = CFPointWriterUtils.getBoundingBox(stnList); // gets written in super.finish(); } - int writeProfile(StationProfileFeature spf, ProfileFeature profile) throws IOException { + int writeProfile(ProfileFeature profile) throws IOException { + if (id_strlen == 0) + id_strlen = profile.getName().length() * 2; int count = 0; for (PointFeature pf : profile) { - if (!headerDone) { - if (id_strlen == 0) - id_strlen = profile.getName().length() * 2; - writeHeader(spf, profile, pf); - headerDone = true; - } writeObsData(pf); count++; } - - Integer stnIndex = stationIndexMap.get(spf.getName()); - if (stnIndex == null) { - log.warn("BAD station {}", spf.getName()); - } else { - writeProfileData(stnIndex, profile, count); - } - return count; } - private void writeHeader(StationProfileFeature stn, ProfileFeature profile, PointFeature obs) throws IOException { - StructureData stnData = stn.getFeatureData(); - StructureData profileData = profile.getFeatureData(); - StructureData obsData = obs.getFeatureData(); - + protected void writeHeader(List stations) throws IOException { + List coverageCollections = new ArrayList<>(); + List stationData = new ArrayList<>(); + List profileData = new ArrayList<>(); List obsCoords = new ArrayList<>(); - Formatter coordNames = new Formatter().format("%s %s %s", profileTimeName, latName, lonName); - obsCoords.add(VariableSimpleBuilder.makeScalar(altitudeCoordinateName, "obs altitude", altUnits, DataType.DOUBLE) - .addAttribute(CF.STANDARD_NAME, "altitude") - .addAttribute(CF.POSITIVE, CF1Convention.getZisPositive(altitudeCoordinateName, altUnits)).build()); - coordNames.format(" %s", altitudeCoordinateName); - super.writeHeader(obsCoords, stnData, profileData, obsData, coordNames.toString()); + for (StationFeature station : stations) { + stationData.add(station.getFeatureData()); + for (ProfileFeature profile : (StationProfileFeature) station) { + profileData.add(profile.getFeatureData()); + coverageCollections.add(profile); + + obsCoords.add(VariableSimpleBuilder + .makeScalar(profile.getTimeName(), "time of measurement", timeUnit.toString(), DataType.DOUBLE).build()); + + altitudeCoordinateName = profile.getAltName(); + obsCoords + .add(VariableSimpleBuilder.makeScalar(altitudeCoordinateName, "obs altitude", altUnits, DataType.DOUBLE) + .addAttribute(CF.STANDARD_NAME, "altitude") + .addAttribute(CF.POSITIVE, CF1Convention.getZisPositive(altitudeCoordinateName, altUnits)).build()); + } + } + + super.writeHeader(obsCoords, coverageCollections, stationData, profileData); // write the stations int count = 0; @@ -137,12 +136,36 @@ private void writeHeader(StationProfileFeature stn, ProfileFeature profile, Poin for (StationFeature sf : stnList) { writeStationData(sf); stationIndexMap.put(sf.getName(), count); + for (ProfileFeature p : (StationProfileFeature) sf) { + int countPoints = 0; + if (p.size() >= 0) { + countPoints += p.size(); + } else { + countPoints += Iterables.size(p); + } + writeProfileData(count, p, countPoints); + } count++; } + } + @Override + void setFeatureAuxInfo(int nfeatures, int id_strlen) { + int countProfiles = 0; + int name_strlen = 0; + for (StationFeature s : stnList) { + name_strlen = Math.max(name_strlen, s.getName().length()); + if (((StationProfileFeature) s).size() >= 0) + countProfiles += ((StationProfileFeature) s).size(); + else { + countProfiles += Iterables.size(((StationProfileFeature) s)); + } + } + this.nfeatures = countProfiles; + this.id_strlen = name_strlen; } - void makeFeatureVariables(StructureData stnData, boolean isExtended) { + void makeFeatureVariables(List stnDataStructs, boolean isExtended) { // add the dimensions : extended model can use an unlimited dimension Dimension stationDim = writerb.addDimension(stationDimName, stnList.size()); @@ -152,8 +175,8 @@ void makeFeatureVariables(StructureData stnData, boolean isExtended) { if (useAlt) { stnVars.add(VariableSimpleBuilder.makeScalar(stationAltName, "station altitude", altUnits, DataType.DOUBLE) - .addAttribute(CF.STANDARD_NAME, CF.SURFACE_ALTITUDE) - .addAttribute(CF.POSITIVE, CF1Convention.getZisPositive(altName, altUnits)).build()); + .addAttribute(CF.STANDARD_NAME, CF.STATION_ALTITUDE) + .addAttribute(CF.POSITIVE, CF1Convention.getZisPositive(stationAltName, altUnits)).build()); } stnVars.add(VariableSimpleBuilder.makeString(stationIdName, "station identifier", null, id_strlen) @@ -167,9 +190,11 @@ void makeFeatureVariables(StructureData stnData, boolean isExtended) { stnVars.add(VariableSimpleBuilder.makeString(wmoName, "station WMO id", null, wmo_strlen) .addAttribute(CF.STANDARD_NAME, CF.PLATFORM_ID).build()); - for (StructureMembers.Member m : stnData.getMembers()) { - if (findDataVar(m.getName()) != null) - stnVars.add(VariableSimpleBuilder.fromMember(m).build()); + for (StructureData stnData : stnDataStructs) { + for (StructureMembers.Member m : stnData.getMembers()) { + if (findDataVar(m.getName()) != null) + stnVars.add(VariableSimpleBuilder.fromMember(m).build()); + } } if (isExtended) { @@ -201,8 +226,7 @@ private void writeStationData(StationFeature stn) throws IOException { stnRecno = super.writeStructureData(stnRecno, stationStruct, sdall, stationVarMap); } - @Override - void makeMiddleVariables(StructureData profileData, boolean isExtended) { + void makeMiddleVariables(List profileDataStructs, boolean isExtended) { Dimension profileDim = writerb.addDimension(profileDimName, nfeatures); // add the profile Variables using the profile dimension @@ -223,10 +247,12 @@ void makeMiddleVariables(StructureData profileData, boolean isExtended) { .add(VariableSimpleBuilder.makeScalar(stationIndexName, "station index for this profile", null, DataType.INT) .addAttribute(CF.INSTANCE_DIMENSION, stationDimName).build()); - for (StructureMembers.Member m : profileData.getMembers()) { - VariableSimpleIF dv = findDataVar(m.getName()); - if (dv != null) - profileVars.add(dv); + for (StructureData profileData : profileDataStructs) { + for (StructureMembers.Member m : profileData.getMembers()) { + VariableSimpleIF dv = findDataVar(m.getName()); + if (dv != null) + profileVars.add(dv); + } } if (isExtended) { @@ -256,7 +282,7 @@ private void writeProfileData(int stnIndex, ProfileFeature profile, int nobs) th // != null) ??? double timeInMyUnits = timeUnit.makeOffsetFromRefDate(profile.getTime()); smb.addMemberScalar(profileTimeName, null, null, DataType.DOUBLE, timeInMyUnits); // LOOK time not always part - // of profile + // of profile smb.addMemberString(profileIdName, null, null, profile.getName().trim(), id_strlen); smb.addMemberScalar(numberOfObsName, null, null, DataType.INT, nobs); smb.addMemberScalar(stationIndexName, null, null, DataType.INT, stnIndex); @@ -270,9 +296,15 @@ private void writeProfileData(int stnIndex, ProfileFeature profile, int nobs) th private int obsRecno; + protected void resetObsIndex() { + obsRecno = 0; + } + private void writeObsData(PointFeature pf) throws IOException { StructureMembers.Builder smb = StructureMembers.builder().setName("Coords"); - smb.addMemberScalar(altitudeCoordinateName, null, null, DataType.DOUBLE, pf.getLocation().getAltitude()); + smb.addMemberScalar(pf.getFeatureCollection().getTimeName(), null, null, DataType.DOUBLE, pf.getObservationTime()); + smb.addMemberScalar(pf.getFeatureCollection().getAltName(), null, null, DataType.DOUBLE, + pf.getLocation().getAltitude()); StructureData coords = new StructureDataFromMember(smb.build()); // coords first so it takes precedence @@ -280,4 +312,5 @@ private void writeObsData(PointFeature pf) throws IOException { obsRecno = super.writeStructureData(obsRecno, record, sdall, dataMap); } + } diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFTrajectoryCollection.java b/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFTrajectoryCollection.java index 72b431cd2e..5da5b03f82 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFTrajectoryCollection.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFTrajectoryCollection.java @@ -8,7 +8,6 @@ import com.google.common.collect.ImmutableList; import java.io.IOException; import java.util.ArrayList; -import java.util.Formatter; import java.util.HashSet; import java.util.List; import ucar.ma2.DataType; @@ -40,7 +39,6 @@ class WriterCFTrajectoryCollection extends WriterCFPointAbstract { private Structure featureStruct; // used for netcdf4 extended private HashSet featureVarMap = new HashSet<>(); - private boolean headerDone; WriterCFTrajectoryCollection(String fileOut, AttributeContainer globalAtts, List dataVars, CalendarDateUnit timeUnit, String altUnits, CFPointWriterConfig config) throws IOException { @@ -51,45 +49,46 @@ class WriterCFTrajectoryCollection extends WriterCFPointAbstract { } int writeTrajectory(TrajectoryFeature traj) throws IOException { + if (id_strlen == 0) + id_strlen = traj.getName().length() * 2; int count = 0; for (PointFeature pf : traj) { - if (!headerDone) { - if (id_strlen == 0) - id_strlen = traj.getName().length() * 2; - writeHeader(traj, pf); - headerDone = true; - } writeObsData(pf); count++; } - writeTrajectoryData(traj, count); return count; } - private void writeHeader(TrajectoryFeature feature, PointFeature obs) throws IOException { + protected void writeHeader(List trajectories) throws IOException { // obs data List coords = new ArrayList<>(); - coords.add(VariableSimpleBuilder.makeScalar(obs.getFeatureCollection().getTimeName(), "time of measurement", - timeUnit.getUdUnit(), DataType.DOUBLE).addAttribute(CF.CALENDAR, timeUnit.getCalendar().toString()).build()); + List trajectoryData = new ArrayList<>(); + + for (TrajectoryFeature trajectory : trajectories) { + trajectoryData.add(trajectory.getFeatureData()); + coords.add(VariableSimpleBuilder + .makeScalar(trajectory.getTimeName(), "time of measurement", timeUnit.getUdUnit(), DataType.DOUBLE) + .addAttribute(CF.CALENDAR, timeUnit.getCalendar().toString()).build()); + + if (altUnits != null) { + altitudeCoordinateName = trajectory.getAltName(); + coords.add(VariableSimpleBuilder + .makeScalar(altitudeCoordinateName, "altitude of measurement", altUnits, DataType.DOUBLE) + .addAttribute(CF.POSITIVE, CF1Convention.getZisPositive(altitudeCoordinateName, altUnits)).build()); + } + } coords.add( VariableSimpleBuilder.makeScalar(latName, "latitude of measurement", CDM.LAT_UNITS, DataType.DOUBLE).build()); coords.add( VariableSimpleBuilder.makeScalar(lonName, "longitude of measurement", CDM.LON_UNITS, DataType.DOUBLE).build()); - Formatter coordNames = - new Formatter().format("%s %s %s", obs.getFeatureCollection().getTimeName(), latName, lonName); - if (altUnits != null) { - coords.add(VariableSimpleBuilder.makeScalar(altName, "altitude of measurement", altUnits, DataType.DOUBLE) - .addAttribute(CF.POSITIVE, CF1Convention.getZisPositive(altName, altUnits)).build()); - coordNames.format(" %s", altName); - } - super.writeHeader(coords, feature.getFeatureData(), null, obs.getFeatureData(), coordNames.toString()); + super.writeHeader(coords, trajectories, trajectoryData, null); } @Override - void makeFeatureVariables(StructureData featureData, boolean isExtended) { + void makeFeatureVariables(List featureDataStructs, boolean isExtended) { // LOOK why not unlimited here fro extended model ? Dimension trajDim = writerb.addDimension(trajDimName, nfeatures); @@ -102,10 +101,12 @@ void makeFeatureVariables(StructureData featureData, boolean isExtended) { .add(VariableSimpleBuilder.makeScalar(numberOfObsName, "number of obs for this profile", null, DataType.INT) .addAttribute(CF.SAMPLE_DIMENSION, recordDimName).build()); - for (StructureMembers.Member m : featureData.getMembers()) { - VariableSimpleIF dv = findDataVar(m.getName()); - if (dv != null) - featureVars.add(dv); + for (StructureData featureData : featureDataStructs) { + for (StructureMembers.Member m : featureData.getMembers()) { + VariableSimpleIF dv = findDataVar(m.getName()); + if (dv != null) + featureVars.add(dv); + } } if (isExtended) { @@ -148,7 +149,7 @@ private void writeObsData(PointFeature pf) throws IOException { smb.addMemberScalar(latName, null, null, DataType.DOUBLE, loc.getLatitude()); smb.addMemberScalar(lonName, null, null, DataType.DOUBLE, loc.getLongitude()); if (altUnits != null) - smb.addMemberScalar(altName, null, null, DataType.DOUBLE, loc.getAltitude()); + smb.addMemberScalar(pf.getFeatureCollection().getAltName(), null, null, DataType.DOUBLE, loc.getAltitude()); StructureData coords = new StructureDataFromMember(smb.build()); // coords first so it takes precedence diff --git a/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFTrajectoryProfileCollection.java b/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFTrajectoryProfileCollection.java index 6041898436..6a80a03eca 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFTrajectoryProfileCollection.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/point/writer2/WriterCFTrajectoryProfileCollection.java @@ -7,11 +7,8 @@ import com.google.common.collect.ImmutableList; import java.io.IOException; -import java.util.ArrayList; -import java.util.Formatter; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; +import java.util.*; + import ucar.ma2.DataType; import ucar.ma2.StructureData; import ucar.ma2.StructureDataComposite; @@ -48,7 +45,6 @@ class WriterCFTrajectoryProfileCollection extends WriterCFPointAbstract { /////////////////////////////////////////////////// private Structure profileStruct; // used for netcdf4 extended private HashSet profileVarMap = new HashSet<>(); - private boolean headerDone; WriterCFTrajectoryProfileCollection(String fileOut, AttributeContainer globalAtts, List dataVars, CalendarDateUnit timeUnit, String altUnits, CFPointWriterConfig config) throws IOException { @@ -66,13 +62,10 @@ void setFeatureAuxInfo2(int ntraj, int traj_strlen) { int writeProfile(TrajectoryProfileFeature section, ProfileFeature profile) throws IOException { int count = 0; + if (id_strlen == 0) + id_strlen = profile.getName().length() * 2; + for (PointFeature pf : profile) { - if (!headerDone) { - if (id_strlen == 0) - id_strlen = profile.getName().length() * 2; - writeHeader(section, profile, pf); - headerDone = true; - } writeObsData(pf); count++; } @@ -86,27 +79,37 @@ int writeProfile(TrajectoryProfileFeature section, ProfileFeature profile) throw return count; } - private void writeHeader(TrajectoryProfileFeature section, ProfileFeature profile, PointFeature obs) - throws IOException { - - StructureData sectionData = section.getFeatureData(); - StructureData profileData = profile.getFeatureData(); - StructureData obsData = obs.getFeatureData(); - - Formatter coordNames = new Formatter().format("%s %s %s", profileTimeName, latName, lonName); + protected void writeHeader(List trajectoryProfiles) throws IOException { List obsCoords = new ArrayList<>(); - if (useAlt) { - obsCoords.add(VariableSimpleBuilder.makeScalar(altitudeCoordinateName, "obs altitude", altUnits, DataType.DOUBLE) - .addAttribute(CF.STANDARD_NAME, "altitude") - .addAttribute(CF.POSITIVE, CF1Convention.getZisPositive(altitudeCoordinateName, altUnits)).build()); - coordNames.format(" %s", altitudeCoordinateName); + List profileFeatures = new ArrayList<>(); + List trajectoryData = new ArrayList<>(); + List profileData = new ArrayList<>(); + + for (TrajectoryProfileFeature trajectoryProfile : trajectoryProfiles) { + trajectoryData.add(trajectoryProfile.getFeatureData()); + for (ProfileFeature profile : trajectoryProfile) { + profileData.add(profile.getFeatureData()); + profileFeatures.add(profile); + } + obsCoords.add(VariableSimpleBuilder + .makeScalar(trajectoryProfile.getTimeName(), "time of measurement", timeUnit.toString(), DataType.DOUBLE) + .build()); + + Formatter coordNames = new Formatter().format("%s %s %s", trajectoryProfile.getTimeName(), latName, lonName); + if (altUnits != null) { + altitudeCoordinateName = trajectoryProfile.getAltName(); + obsCoords.add( + VariableSimpleBuilder.makeScalar(trajectoryProfile.getAltName(), "obs altitude", altUnits, DataType.DOUBLE) + .addAttribute(CF.STANDARD_NAME, "altitude") + .addAttribute(CF.POSITIVE, CF1Convention.getZisPositive(altitudeCoordinateName, altUnits)).build()); + coordNames.format(" %s", trajectoryProfile.getAltName()); + } } - - super.writeHeader(obsCoords, sectionData, profileData, obsData, coordNames.toString()); + super.writeHeader(obsCoords, profileFeatures, trajectoryData, profileData); } @Override - void makeFeatureVariables(StructureData trajData, boolean isExtended) { + void makeFeatureVariables(List featureData, boolean isExtended) { // add the dimensions : extended model can use an unlimited dimension Dimension trajDim = writerb.addDimension(trajDimName, ntraj); @@ -115,9 +118,11 @@ void makeFeatureVariables(StructureData trajData, boolean isExtended) { trajVars.add(VariableSimpleBuilder.makeString(trajIdName, "trajectory identifier", null, traj_strlen) .addAttribute(CF.CF_ROLE, CF.TRAJECTORY_ID).build()); - for (StructureMembers.Member m : trajData.getMembers()) { - if (findDataVar(m.getName()) != null) - trajVars.add(VariableSimpleBuilder.fromMember(m).build()); + for (StructureData trajData : featureData) { + for (StructureMembers.Member m : trajData.getMembers()) { + if (findDataVar(m.getName()) != null) + trajVars.add(VariableSimpleBuilder.fromMember(m).build()); + } } if (isExtended) { @@ -130,28 +135,7 @@ void makeFeatureVariables(StructureData trajData, boolean isExtended) { } @Override - void finishBuilding() throws IOException { - super.finishBuilding(); - profileStruct = findStructure(profileStructName); - trajStructure = findStructure(trajStructName); - } - - private int trajRecno; - - private int writeSectionData(TrajectoryProfileFeature section) throws IOException { - - StructureMembers.Builder smb = StructureMembers.builder().setName("Coords"); - smb.addMemberString(trajIdName, null, null, section.getName().trim(), traj_strlen); - StructureData coords = new StructureDataFromMember(smb.build()); - - // coords first so it takes precedence - StructureDataComposite sdall = StructureDataComposite.create(ImmutableList.of(coords, section.getFeatureData())); - trajRecno = super.writeStructureData(trajRecno, trajStructure, sdall, trajVarMap); - return trajRecno - 1; - } - - @Override - protected void makeMiddleVariables(StructureData profileData, boolean isExtended) { + void makeMiddleVariables(List profileDataStructs, boolean isExtended) { Dimension profileDim = writerb.addDimension(profileDimName, nfeatures); // add the profile Variables using the profile dimension @@ -159,11 +143,14 @@ protected void makeMiddleVariables(StructureData profileData, boolean isExtended profileVars.add(VariableSimpleBuilder.makeString(profileIdName, "profile identifier", null, id_strlen) .addAttribute(CF.CF_ROLE, CF.PROFILE_ID) // profileId:cf_role = "profile_id"; .addAttribute(CDM.MISSING_VALUE, String.valueOf(idMissingValue)).build()); - profileVars .add(VariableSimpleBuilder.makeScalar(latName, "profile latitude", CDM.LAT_UNITS, DataType.DOUBLE).build()); profileVars .add(VariableSimpleBuilder.makeScalar(lonName, "profile longitude", CDM.LON_UNITS, DataType.DOUBLE).build()); + profileVars + .add(VariableSimpleBuilder.makeScalar(numberOfObsName, "number of obs for this profile", null, DataType.INT) + .addAttribute(CF.SAMPLE_DIMENSION, recordDimName).build()); // rowSize:sample_dimension = "obs" + profileVars.add(VariableSimpleBuilder .makeScalar(profileTimeName, "nominal time of profile", timeUnit.getUdUnit(), DataType.DOUBLE) .addAttribute(CF.CALENDAR, timeUnit.getCalendar().toString()).build()); @@ -172,14 +159,12 @@ protected void makeMiddleVariables(StructureData profileData, boolean isExtended VariableSimpleBuilder.makeScalar(trajectoryIndexName, "trajectory index for this profile", null, DataType.INT) .addAttribute(CF.INSTANCE_DIMENSION, trajDimName).build()); - profileVars - .add(VariableSimpleBuilder.makeScalar(numberOfObsName, "number of obs for this profile", null, DataType.INT) - .addAttribute(CF.SAMPLE_DIMENSION, recordDimName).build()); - - for (StructureMembers.Member m : profileData.getMembers()) { - VariableSimpleIF dv = findDataVar(m.getName()); - if (dv != null) - profileVars.add(dv); + for (StructureData profileData : profileDataStructs) { + for (StructureMembers.Member m : profileData.getMembers()) { + VariableSimpleIF dv = findDataVar(m.getName()); + if (dv != null) + profileVars.add(dv); + } } if (isExtended) { @@ -190,6 +175,27 @@ protected void makeMiddleVariables(StructureData profileData, boolean isExtended } } + @Override + void finishBuilding() throws IOException { + super.finishBuilding(); + profileStruct = findStructure(profileStructName); + trajStructure = findStructure(trajStructName); + } + + private int trajRecno; + + private int writeSectionData(TrajectoryProfileFeature section) throws IOException { + + StructureMembers.Builder smb = StructureMembers.builder().setName("Coords"); + smb.addMemberString(trajIdName, null, null, section.getName().trim(), traj_strlen); + StructureData coords = new StructureDataFromMember(smb.build()); + + // coords first so it takes precedence + StructureDataComposite sdall = StructureDataComposite.create(ImmutableList.of(coords, section.getFeatureData())); + trajRecno = super.writeStructureData(trajRecno, trajStructure, sdall, trajVarMap); + return trajRecno - 1; + } + private int profileRecno; private void writeProfileData(int sectionIndex, ProfileFeature profile, int nobs) throws IOException { @@ -216,8 +222,10 @@ private void writeProfileData(int sectionIndex, ProfileFeature profile, int nobs private void writeObsData(PointFeature pf) throws IOException { StructureMembers.Builder smb = StructureMembers.builder().setName("Coords"); - if (useAlt) - smb.addMemberScalar(altitudeCoordinateName, null, null, DataType.DOUBLE, pf.getLocation().getAltitude()); + smb.addMemberScalar(pf.getFeatureCollection().getTimeName(), null, null, DataType.DOUBLE, pf.getObservationTime()); + if (altUnits != null) + smb.addMemberScalar(pf.getFeatureCollection().getAltName(), null, null, DataType.DOUBLE, + pf.getLocation().getAltitude()); StructureData coords = new StructureDataFromMember(smb.build()); // coords first so it takes precedence diff --git a/cdm/core/src/main/java/ucar/nc2/ft2/coverage/writer/CoverageAsPoint.java b/cdm/core/src/main/java/ucar/nc2/ft2/coverage/writer/CoverageAsPoint.java index ce29905eaf..46439ee9d0 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft2/coverage/writer/CoverageAsPoint.java +++ b/cdm/core/src/main/java/ucar/nc2/ft2/coverage/writer/CoverageAsPoint.java @@ -201,6 +201,8 @@ private class CoverageAsStationFeatureCollection extends StationTimeSeriesCollec CoverageAsStationFeatureCollection(VarGroup varGroup) { super(varGroup.name + " AsStationFeatureCollection", varGroup.dateUnit, varGroup.zUnit); this.timeName = varGroup.timeAxis != null ? varGroup.timeAxis.getName() : "time"; + this.timeUnit = + varGroup.timeAxis != null ? varGroup.timeAxis.getCalendarDateUnit() : CalendarDateUnit.unixDateUnit; this.altName = varGroup.zAxis != null ? varGroup.zAxis.getName() : "altitude"; this.varGroup = varGroup; this.collectionFeatureType = varGroup.fType; @@ -307,9 +309,10 @@ public boolean hasNext() throws IOException { @Override public PointFeatureCollection next() throws IOException { double obsTime = this.timeAxis != null ? this.timeAxis.getCoordMidpoint(curr) : 0.0; + String timeName = this.timeAxis != null ? this.timeAxis.getName() : "time"; curr++; - return new CoverageAsProfileFeature(obsTime, varGroup.dateUnit, varGroup.zUnit, getLatitude(), getLongitude(), - this.varIters); + return new CoverageAsProfileFeature(obsTime, timeName, varGroup.dateUnit, varGroup.zAxis.getName(), + varGroup.zUnit, getLatitude(), getLongitude(), this.varIters); } } @@ -317,9 +320,9 @@ private class CoverageAsProfileFeature extends ProfileFeatureImpl { List varIters; - CoverageAsProfileFeature(double obsTime, CalendarDateUnit timeUnit, String altUnits, double lat, double lon, - List varIters) { - super("", timeUnit, altUnits, lat, lon, obsTime, -1); + CoverageAsProfileFeature(double obsTime, String timeName, CalendarDateUnit timeUnit, String altName, + String altUnits, double lat, double lon, List varIters) { + super("", timeName, timeUnit, altName, altUnits, lat, lon, obsTime, -1); this.varIters = varIters; } diff --git a/cdm/core/src/test/java/ucar/nc2/ft2/coverage/writer/TestCoverageAsPoint.java b/cdm/core/src/test/java/ucar/nc2/ft2/coverage/writer/TestCoverageAsPoint.java index f2e5195085..a98cc3c804 100644 --- a/cdm/core/src/test/java/ucar/nc2/ft2/coverage/writer/TestCoverageAsPoint.java +++ b/cdm/core/src/test/java/ucar/nc2/ft2/coverage/writer/TestCoverageAsPoint.java @@ -4,7 +4,6 @@ import static org.junit.Assert.assertThrows; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; import ucar.ma2.Array; import ucar.nc2.constants.FeatureType; @@ -86,7 +85,6 @@ public void testVarFeatureTypes() throws IOException { } @Test - @Ignore public void testCoverageAsPoint() throws IOException { double[] vals = Arrays.copyOfRange(expected, 0, 1); // test single point (no time) @@ -131,7 +129,6 @@ public void testExceptionThrownForPointOutsideCoverage() { } @Test - @Ignore public void testCoverageAsProfile() throws IOException { // test single profile (no time) List varNames = new ArrayList<>(); @@ -255,7 +252,7 @@ private void readCoverageAsPoint(List varNames, SubsetParams params, dou private void readCoverageAsProfile(List varNames, SubsetParams params, double[] alt, double[] time, double[] expected) throws IOException { - readCoverageAsProfile(varNames, params, alt, time, expected, 0, "time", "altitude"); + readCoverageAsProfile(varNames, params, alt, time, expected, 0, "time", "z"); } private void readCoverageAsProfile(List varNames, SubsetParams params, double[] alt, double[] time,