Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Feature/reader and writer #10

Merged
merged 6 commits into from
Aug 16, 2023
Merged

Feature/reader and writer #10

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
64cd21b
A number of small changes towards making the Reader and Writer symmet…
phlndrwd Aug 15, 2023
9225fb5
Removed some commented out functions from Reader as they're not curen…
phlndrwd Aug 15, 2023
bebc916
Renamed functions in AtlasWriter that use FileData objects. Updated c…
phlndrwd Aug 15, 2023
3391684
Minor alignment changes.
phlndrwd Aug 15, 2023
c4f2829
Changed access specifiers to expose only two public functions in Atla…
phlndrwd Aug 15, 2023
89b290b
Removed unused functions from AtlasReader.
phlndrwd Aug 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 0 additions & 24 deletions src/monio/AtlasReader.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,30 +84,6 @@ void monio::AtlasReader::populateFieldWithDataContainer(atlas::Field& field,
}
}

void monio::AtlasReader::populateFieldSetWithData(atlas::FieldSet& fieldSet,
const Data& data) {
oops::Log::debug() << "AtlasReader::populateFieldSetWithData()" << std::endl;
if (mpiCommunicator_.rank() == mpiRankOwner_) {
for (auto& field : fieldSet) {
populateFieldWithDataContainer(field, data.getContainer(field.name()));
}
}
}

void monio::AtlasReader::populateFieldSetWithData(atlas::FieldSet& fieldSet,
const Data& data,
const std::vector<std::string>& fieldNames) {
oops::Log::debug() << "AtlasReader::populateFieldSetWithData()" << std::endl;
if (mpiCommunicator_.rank() == mpiRankOwner_) {
auto fieldNameIt = fieldNames.begin();
for (auto& field : fieldSet) {
populateFieldWithDataContainer(field, data.getContainer(*fieldNameIt));
++fieldNameIt;
}
}
}


template<typename T>
void monio::AtlasReader::populateField(atlas::Field& field,
const std::vector<T>& dataVec,
Expand Down
8 changes: 1 addition & 7 deletions src/monio/AtlasReader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,18 +43,12 @@ class AtlasReader {
void populateFieldWithDataContainer(atlas::Field& field,
const std::shared_ptr<monio::DataContainerBase>& dataContainer,
const std::vector<size_t>& lfricToAtlasMap,
const bool copyFirstLevel = false); // Monio::readBackground
const bool copyFirstLevel = false);

void populateFieldWithDataContainer(atlas::Field& field,
const std::shared_ptr<monio::DataContainerBase>& dataContainer);

void populateFieldSetWithData(atlas::FieldSet& fieldSet, const Data& data);
void populateFieldSetWithData(atlas::FieldSet& fieldSet,
const Data& data,
const std::vector<std::string>& fieldNames);

private:
// AtlasWriter::populateFieldWithDataContainer
template<typename T> void populateField(atlas::Field& field,
const std::vector<T>& dataVec,
const std::vector<size_t>& lfricToAtlasMap,
Expand Down
125 changes: 63 additions & 62 deletions src/monio/AtlasWriter.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,64 +189,12 @@ void monio::AtlasWriter::populateDataContainerWithField(
}
}

template<typename T>
void monio::AtlasWriter::populateDataVec(std::vector<T>& dataVec,
const atlas::Field& field,
const std::vector<size_t>& lfricToAtlasMap) {
oops::Log::debug() << "AtlasWriter::populateDataVec() " << field.name() << std::endl;
atlas::idx_t numLevels = field.levels();
if ((lfricToAtlasMap.size() * numLevels) != dataVec.size()) {
utils::throwException("AtlasWriter::populateDataVec()> "
"Data container is not configured for the expected data...");
}
auto fieldView = atlas::array::make_view<T, 2>(field);
for (std::size_t i = 0; i < lfricToAtlasMap.size(); ++i) {
for (atlas::idx_t j = 0; j < numLevels; ++j) {
int index = lfricToAtlasMap[i] + (j * lfricToAtlasMap.size());
dataVec[index] = fieldView(i, j);
}
}
}

template void monio::AtlasWriter::populateDataVec<double>(std::vector<double>& dataVec,
const atlas::Field& field,
const std::vector<size_t>& lfricToAtlasMap);
template void monio::AtlasWriter::populateDataVec<float>(std::vector<float>& dataVec,
const atlas::Field& field,
const std::vector<size_t>& lfricToAtlasMap);
template void monio::AtlasWriter::populateDataVec<int>(std::vector<int>& dataVec,
const atlas::Field& field,
const std::vector<size_t>& lfricToAtlasMap);

template<typename T>
void monio::AtlasWriter::populateDataVec(std::vector<T>& dataVec,
const atlas::Field& field,
const std::vector<int>& dimensions) {
oops::Log::debug() << "AtlasWriter::populateDataVec()" << std::endl;
auto fieldView = atlas::array::make_view<T, 2>(field);
for (atlas::idx_t i = 0; i < dimensions[consts::eHorizontal]; ++i) { // Horizontal dimension
for (atlas::idx_t j = 0; j < dimensions[consts::eVertical]; ++j) { // Levels dimension
int index = j + (i * dimensions[consts::eVertical]);
dataVec[index] = fieldView(i, j);
}
}
}

template void monio::AtlasWriter::populateDataVec<double>(std::vector<double>& dataVec,
const atlas::Field& field,
const std::vector<int>& dimensions);
template void monio::AtlasWriter::populateDataVec<float>(std::vector<float>& dataVec,
const atlas::Field& field,
const std::vector<int>& dimensions);
template void monio::AtlasWriter::populateDataVec<int>(std::vector<int>& dataVec,
const atlas::Field& field,
const std::vector<int>& dimensions);

void monio::AtlasWriter::populateMetadataAndDataWithFieldSet(Metadata& metadata,
Data& data,
const atlas::FieldSet& fieldSet) {
void monio::AtlasWriter::populateFileDataWithFieldSet(FileData& fileData,
const atlas::FieldSet& fieldSet) {
oops::Log::debug() << "AtlasWriter::populateMetadataAndDataWithFieldSet()" << std::endl;
if (mpiCommunicator_.rank() == mpiRankOwner_) {
Metadata& metadata = fileData.getMetadata();
Data& data = fileData.getData();
int dimCount = 0;
bool createdLonLat = false;
for (const auto& field : fieldSet) {
Expand All @@ -258,7 +206,7 @@ void monio::AtlasWriter::populateMetadataAndDataWithFieldSet(Metadata& metadata,
std::string dimName = metadata.getDimensionName(dimSize);
if (dimName == consts::kNotFoundError) {
dimName = "dim" + std::to_string(dimCount);
metadata.addDimension(dimName, dimSize);
fileData.getMetadata().addDimension(dimName, dimSize);
dimCount++;
}
}
Expand Down Expand Up @@ -288,18 +236,18 @@ void monio::AtlasWriter::populateMetadataAndDataWithFieldSet(Metadata& metadata,

std::shared_ptr<monio::AttributeString> producedByAttr =
std::make_shared<AttributeString>(producedByName, producedByString);
metadata.addGlobalAttr(producedByName, producedByAttr);
fileData.getMetadata().addGlobalAttr(producedByName, producedByAttr);
}
}
}

void monio::AtlasWriter::populateMetadataAndDataWithLfricFieldSet(
Metadata& metadata,
Data& data,
void monio::AtlasWriter::populateFileDataWithLfricFieldSet(FileData& fileData,
const std::vector<consts::FieldMetadata>& fieldMetadataVec,
atlas::FieldSet& fieldSet,
const std::vector<size_t>& lfricToAtlasMap) {
oops::Log::debug() << "AtlasWriter::populateMetadataAndDataWithLfricFieldSet()" << std::endl;
Metadata& metadata = fileData.getMetadata();
Data& data = fileData.getData();
// Dimensions
metadata.addDimension(std::string(consts::kHorizontalName), lfricToAtlasMap.size());
metadata.addDimension(std::string(consts::kVerticalFullName), consts::kVerticalFullSize);
Expand All @@ -318,7 +266,7 @@ void monio::AtlasWriter::populateMetadataAndDataWithLfricFieldSet(

std::shared_ptr<monio::AttributeString> producedByAttr =
std::make_shared<AttributeString>(producedByName, producedByString);
metadata.addGlobalAttr(producedByName, producedByAttr);
fileData.getMetadata().addGlobalAttr(producedByName, producedByAttr);
}

void monio::AtlasWriter::populateDataWithField(Data& data,
Expand All @@ -339,3 +287,56 @@ void monio::AtlasWriter::populateDataWithField(Data& data,
populateDataContainerWithField(dataContainer, field, dimensions);
data.addContainer(dataContainer);
}

template<typename T>
void monio::AtlasWriter::populateDataVec(std::vector<T>& dataVec,
const atlas::Field& field,
const std::vector<size_t>& lfricToAtlasMap) {
oops::Log::debug() << "AtlasWriter::populateDataVec() " << field.name() << std::endl;
atlas::idx_t numLevels = field.levels();
if ((lfricToAtlasMap.size() * numLevels) != dataVec.size()) {
utils::throwException("AtlasWriter::populateDataVec()> "
"Data container is not configured for the expected data...");
}
auto fieldView = atlas::array::make_view<T, 2>(field);
for (std::size_t i = 0; i < lfricToAtlasMap.size(); ++i) {
for (atlas::idx_t j = 0; j < numLevels; ++j) {
int index = lfricToAtlasMap[i] + (j * lfricToAtlasMap.size());
dataVec[index] = fieldView(i, j);
}
}
}

template void monio::AtlasWriter::populateDataVec<double>(std::vector<double>& dataVec,
const atlas::Field& field,
const std::vector<size_t>& lfricToAtlasMap);
template void monio::AtlasWriter::populateDataVec<float>(std::vector<float>& dataVec,
const atlas::Field& field,
const std::vector<size_t>& lfricToAtlasMap);
template void monio::AtlasWriter::populateDataVec<int>(std::vector<int>& dataVec,
const atlas::Field& field,
const std::vector<size_t>& lfricToAtlasMap);

template<typename T>
void monio::AtlasWriter::populateDataVec(std::vector<T>& dataVec,
const atlas::Field& field,
const std::vector<int>& dimensions) {
oops::Log::debug() << "AtlasWriter::populateDataVec()" << std::endl;
auto fieldView = atlas::array::make_view<T, 2>(field);
for (atlas::idx_t i = 0; i < dimensions[consts::eHorizontal]; ++i) { // Horizontal dimension
for (atlas::idx_t j = 0; j < dimensions[consts::eVertical]; ++j) { // Levels dimension
int index = j + (i * dimensions[consts::eVertical]);
dataVec[index] = fieldView(i, j);
}
}
}

template void monio::AtlasWriter::populateDataVec<double>(std::vector<double>& dataVec,
const atlas::Field& field,
const std::vector<int>& dimensions);
template void monio::AtlasWriter::populateDataVec<float>(std::vector<float>& dataVec,
const atlas::Field& field,
const std::vector<int>& dimensions);
template void monio::AtlasWriter::populateDataVec<int>(std::vector<int>& dataVec,
const atlas::Field& field,
const std::vector<int>& dimensions);
35 changes: 16 additions & 19 deletions src/monio/AtlasWriter.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,7 @@
#include <vector>

#include "Constants.h"
#include "Data.h"
#include "Metadata.h"
#include "FileData.h"

#include "atlas/array/DataType.h"
#include "atlas/field.h"
Expand All @@ -36,16 +35,15 @@ class AtlasWriter {
AtlasWriter& operator=( AtlasWriter&&) = delete; //!< Deleted move assignment
AtlasWriter& operator=(const AtlasWriter&) = delete; //!< Deleted copy assignment

void populateMetadataAndDataWithFieldSet(Metadata& metadata,
Data& data,
const atlas::FieldSet& fieldSet);
void populateFileDataWithFieldSet(FileData& fileData,
const atlas::FieldSet& fieldSet);

void populateMetadataAndDataWithLfricFieldSet(Metadata& metadata,
Data& data,
const std::vector<consts::FieldMetadata>& fieldMetadataVec,
atlas::FieldSet& fieldSet,
const std::vector<size_t>& lfricToAtlasMap);
void populateFileDataWithLfricFieldSet(FileData& fileData,
const std::vector<consts::FieldMetadata>& fieldMetadataVec,
atlas::FieldSet& fieldSet,
const std::vector<size_t>& lfricToAtlasMap);

private:
void populateMetadataWithField(Metadata& metadata,
const atlas::Field& field,
const consts::FieldMetadata* fieldMetadata = nullptr,
Expand All @@ -60,15 +58,6 @@ class AtlasWriter {
const atlas::Field& field,
const std::vector<int>& dimensions);

template<typename T> void populateDataVec(std::vector<T>& dataVec,
const atlas::Field& field,
const std::vector<size_t>& lfricToAtlasMap);

template<typename T> void populateDataVec(std::vector<T>& dataVec,
const atlas::Field& field,
const std::vector<int>& dimensions);

private:
void populateDataWithField(Data& data,
const atlas::Field& field,
const std::vector<size_t>& lfricToAtlasMap,
Expand All @@ -78,6 +67,14 @@ class AtlasWriter {
const atlas::Field& field,
const std::vector<int> dimensions);

template<typename T> void populateDataVec(std::vector<T>& dataVec,
const atlas::Field& field,
const std::vector<size_t>& lfricToAtlasMap);

template<typename T> void populateDataVec(std::vector<T>& dataVec,
const atlas::Field& field,
const std::vector<int>& dimensions);

const eckit::mpi::Comm& mpiCommunicator_;
const std::size_t mpiRankOwner_;
};
Expand Down
10 changes: 0 additions & 10 deletions src/monio/File.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,56 +246,46 @@ void monio::File::readAttributes(Metadata& metadata) {

template<typename T>
void monio::File::readSingleDatum(const std::string& varName,
const int varSize,
std::vector<T>& dataVec) {
oops::Log::debug() << "File::readData()" << std::endl;
if (fileMode_ == netCDF::NcFile::read) {
auto var = getFile().getVar(varName);
dataVec.resize(varSize, 0);
var.getVar(dataVec.data());
} else {
utils::throwException("File::readSingleDatum()> Write file accessed for reading...");
}
}

template void monio::File::readSingleDatum<double>(const std::string& varName,
const int varSize,
std::vector<double>& dataVec);
template void monio::File::readSingleDatum<float>(const std::string& varName,
const int varSize,
std::vector<float>& dataVec);
template void monio::File::readSingleDatum<int>(const std::string& varName,
const int varSize,
std::vector<int>& dataVec);

template<typename T>
void monio::File::readFieldDatum(const std::string& fieldName,
const int varSize,
const std::vector<size_t>& startVec,
const std::vector<size_t>& countVec,
std::vector<T>& dataVec) {
oops::Log::debug() << "File::readFieldDatum()" << std::endl;
if (fileMode_ == netCDF::NcFile::read) {
auto var = getFile().getVar(fieldName);
dataVec.resize(varSize, 0);
var.getVar(startVec, countVec, dataVec.data());
} else {
utils::throwException("File::readFieldDatum()> Write file accessed for reading...");
}
}

template void monio::File::readFieldDatum<double>(const std::string& varName,
const int varSize,
const std::vector<size_t>& startVec,
const std::vector<size_t>& countVec,
std::vector<double>& dataVec);
template void monio::File::readFieldDatum<float>(const std::string& varName,
const int varSize,
const std::vector<size_t>& startVec,
const std::vector<size_t>& countVec,
std::vector<float>& dataVec);
template void monio::File::readFieldDatum<int>(const std::string& varName,
const int varSize,
const std::vector<size_t>& startVec,
const std::vector<size_t>& countVec,
std::vector<int>& dataVec);
Expand Down
2 changes: 0 additions & 2 deletions src/monio/File.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,9 @@ class File {
const std::vector<std::string>& varNames);

template<typename T> void readSingleDatum(const std::string& varName,
const int varSize,
std::vector<T>& dataVec);

template<typename T> void readFieldDatum(const std::string& fieldName,
const int varSize,
const std::vector<size_t>& startVec,
const std::vector<size_t>& countVec,
std::vector<T>& dataVec);
Expand Down
Loading