Skip to content

Commit

Permalink
Delete Stream Interface (novatel#67)
Browse files Browse the repository at this point in the history 
* fix todos

* use unique_ptr

* delete stream interface
---------

Co-authored-by: Jonathan McDermid <[email protected]>
  • Loading branch information
@jonathanmcdermid
jonathanmcdermid and Jonathan McDermid authored Oct 18, 2024
1 parent 4cbe730 commit 04d7433
Show file tree
Hide file tree
Showing 52 changed files with 136 additions and 3,155 deletions.
3 changes: 0 additions & 3 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ execute_process(COMMAND ${CMAKE_COMMAND}

# Build EDIE components
add_subdirectory(src/common)
add_subdirectory(src/stream_interface)
add_subdirectory(src/decoders/common)
add_subdirectory(src/decoders/oem)

Expand All @@ -51,15 +50,13 @@ add_library(novatel_edie INTERFACE)
add_library(novatel_edie::novatel_edie ALIAS novatel_edie)
target_link_libraries(novatel_edie INTERFACE
common
stream_interface
decoders_common
oem_decoder
)

if(BUILD_TESTS)
enable_testing()
add_subdirectory(src/common/test)
add_subdirectory(src/stream_interface/test)
add_subdirectory(src/decoders/common/test)
add_subdirectory(src/decoders/oem/test)
endif()
Expand Down
1 change: 0 additions & 1 deletion conanfile.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,6 @@ def package_info(self):
self.cpp_info.libs = [
# Note: the order of the listed libs matters when linking statically.
"edie_oem_decoder",
"edie_stream_interface",
"edie_decoders_common",
"edie_common",
]
Expand Down
5 changes: 2 additions & 3 deletions examples/novatel/command_encoding/command_encoding.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@

#include <novatel_edie/common/logger.hpp>
#include <novatel_edie/decoders/oem/commander.hpp>
#include <novatel_edie/stream_interface/outputfilestream.hpp>

namespace fs = std::filesystem;

Expand Down Expand Up @@ -94,7 +93,7 @@ int main(int argc, char* argv[])
}

std::string sOutFilename = std::string("COMMAND.").append(strEncodeFormat);
OutputFileStream clOutputFs(sOutFilename.c_str());
clOutputFs.WriteData(pcEncodedMessageBuffer, uiEncodeBufferLength);
std::ofstream ofs(sOutFilename.c_str(), std::ios::binary);
ofs.write(pcEncodedMessageBuffer, uiEncodeBufferLength);
return 0;
}
37 changes: 14 additions & 23 deletions examples/novatel/converter_components/converter_components.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,6 @@
#include <novatel_edie/decoders/oem/framer.hpp>
#include <novatel_edie/decoders/oem/header_decoder.hpp>
#include <novatel_edie/decoders/oem/message_decoder.hpp>
#include <novatel_edie/stream_interface/inputfilestream.hpp>
#include <novatel_edie/stream_interface/outputfilestream.hpp>
#include <novatel_edie/version.h>

namespace fs = std::filesystem;
Expand Down Expand Up @@ -127,6 +125,7 @@ int main(int argc, char* argv[])
Logger::AddRotatingFileLogger(clFilter.GetLogger());

// Set up buffers
std::array<char, MAX_ASCII_MESSAGE_LENGTH> cData;
unsigned char acFrameBuffer[MAX_ASCII_MESSAGE_LENGTH];
unsigned char acEncodeBuffer[MAX_ASCII_MESSAGE_LENGTH];
unsigned char* pucEncodedMessageBuffer = acEncodeBuffer;
Expand All @@ -142,24 +141,17 @@ int main(int argc, char* argv[])
MetaDataStruct stMetaData;
MessageDataStruct stMessageData;

// Initialize FS structures and buffers
StreamReadStatus stReadStatus;
ReadDataStructure stReadData;
unsigned char acIfsReadBuffer[MAX_ASCII_MESSAGE_LENGTH];
stReadData.cData = reinterpret_cast<char*>(acIfsReadBuffer);
stReadData.uiDataSize = sizeof(acIfsReadBuffer);

// Setup file streams
InputFileStream clIfs(pathInFilename.string().c_str());
OutputFileStream clConvertedLogsOfs(pathInFilename.string().append(".").append(sEncodeFormat).c_str());
OutputFileStream clUnknownBytesOfs(pathInFilename.string().append(".").append(sEncodeFormat).append(".UNKNOWN").c_str());
std::ifstream ifs(pathInFilename, std::ios::binary);
std::ofstream convertedOfs(pathInFilename.string() + "." + sEncodeFormat, std::ios::binary);
std::ofstream unknownOfs(pathInFilename.string() + "." + sEncodeFormat + ".UNKNOWN", std::ios::binary);

tStart = std::chrono::high_resolution_clock::now();

while (!stReadStatus.bEOS)
while (!ifs.eof())
{
stReadStatus = clIfs.ReadData(stReadData);
clFramer.Write(reinterpret_cast<unsigned char*>(stReadData.cData), stReadStatus.uiCurrentStreamRead);
ifs.read(cData.data(), cData.size());
clFramer.Write(reinterpret_cast<unsigned char*>(cData.data()), ifs.gcount());
// Clearing INCOMPLETE status when internal buffer needs more bytes.
eFramerStatus = STATUS::INCOMPLETE_MORE_DATA;

Expand All @@ -172,7 +164,7 @@ int main(int argc, char* argv[])
{
if (stMetaData.bResponse)
{
clUnknownBytesOfs.WriteData(reinterpret_cast<char*>(pucFrameBuffer), stMetaData.uiLength);
unknownOfs.write(reinterpret_cast<char*>(pucFrameBuffer), stMetaData.uiLength);
continue;
}

Expand All @@ -199,37 +191,36 @@ int main(int argc, char* argv[])

if (eEncoderStatus == STATUS::SUCCESS)
{
clConvertedLogsOfs.WriteData(reinterpret_cast<char*>(stMessageData.pucMessage), stMessageData.uiMessageLength);
convertedOfs.write(reinterpret_cast<char*>(stMessageData.pucMessage), stMessageData.uiMessageLength);
stMessageData.pucMessage[stMessageData.uiMessageLength] = '\0';
pclLogger->info("Encoded: ({}) {}", stMessageData.uiMessageLength, reinterpret_cast<char*>(pucEncodedMessageBuffer));
}
else
{
clUnknownBytesOfs.WriteData(reinterpret_cast<char*>(pucFrameBuffer), uiBodyLength);
unknownOfs.write(reinterpret_cast<char*>(pucFrameBuffer), uiBodyLength);
pclLogger->warn("Encoder returned with status code {}", static_cast<int>(eEncoderStatus));
}
}
else
{
clUnknownBytesOfs.WriteData(reinterpret_cast<char*>(pucFrameBuffer), uiBodyLength);
unknownOfs.write(reinterpret_cast<char*>(pucFrameBuffer), uiBodyLength);
pclLogger->warn("MessageDecoder returned with status code {}", static_cast<int>(eDecoderStatus));
}
}
else
{
clUnknownBytesOfs.WriteData(reinterpret_cast<char*>(pucFrameBuffer), stMetaData.uiLength);
unknownOfs.write(reinterpret_cast<char*>(pucFrameBuffer), stMetaData.uiLength);
pclLogger->warn("HeaderDecoder returned with status code {}", static_cast<int>(eDecoderStatus));
}
}
else if (eFramerStatus == STATUS::UNKNOWN) { clUnknownBytesOfs.WriteData(reinterpret_cast<char*>(pucFrameBuffer), stMetaData.uiLength); }
else if (eFramerStatus == STATUS::UNKNOWN) { unknownOfs.write(reinterpret_cast<char*>(pucFrameBuffer), stMetaData.uiLength); }
else { pclLogger->warn("Framer returned with status code {}", static_cast<int>(eFramerStatus)); }
}
}

// Clean up
uint32_t uiBytes = clFramer.Flush(acFrameBuffer, sizeof(acFrameBuffer));
clUnknownBytesOfs.WriteData(reinterpret_cast<char*>(acFrameBuffer), uiBytes);

unknownOfs.write(reinterpret_cast<char*>(acFrameBuffer), uiBytes);
Logger::Shutdown();
return 0;
}
20 changes: 6 additions & 14 deletions examples/novatel/converter_file_parser/converter_file_parser.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,6 @@

#include <novatel_edie/common/logger.hpp>
#include <novatel_edie/decoders/oem/file_parser.hpp>
#include <novatel_edie/stream_interface/inputfilestream.hpp>
#include <novatel_edie/stream_interface/outputfilestream.hpp>
#include <novatel_edie/version.h>

namespace fs = std::filesystem;
Expand Down Expand Up @@ -114,18 +112,12 @@ int main(int argc, char* argv[])
clFileParser.SetFilter(&clFilter);
clFileParser.SetEncodeFormat(eEncodeFormat);

// Initialize FS structures and buffers
ReadDataStructure stReadData;
unsigned char acIfsReadBuffer[MAX_ASCII_MESSAGE_LENGTH];
stReadData.cData = reinterpret_cast<char*>(acIfsReadBuffer);
stReadData.uiDataSize = sizeof(acIfsReadBuffer);

// Set up file streams
InputFileStream clIfs(pathInFilename.string().c_str());
OutputFileStream clConvertedLogsOfs(pathInFilename.string().append(".").append(sEncodeFormat).c_str());
OutputFileStream clUnknownBytesOfs(pathInFilename.string().append(".").append(sEncodeFormat).append(".UNKNOWN").c_str());
std::ifstream ifs(pathInFilename, std::ios::binary);
std::ofstream convertedOfs(pathInFilename.string() + "." + sEncodeFormat, std::ios::binary);
std::ofstream unknownOfs(pathInFilename.string() + "." + sEncodeFormat + ".UNKNOWN", std::ios::binary);

if (!clFileParser.SetStream(&clIfs))
if (!clFileParser.SetStream(&ifs))
{
pclLogger->error("Input stream could not be set. The stream is either unavailable or exhausted.");
exit(-1);
Expand All @@ -143,7 +135,7 @@ int main(int argc, char* argv[])
eStatus = clFileParser.Read(stMessageData, stMetaData);
if (eStatus == STATUS::SUCCESS)
{
clConvertedLogsOfs.WriteData(reinterpret_cast<char*>(stMessageData.pucMessage), stMessageData.uiMessageLength);
convertedOfs.write(reinterpret_cast<char*>(stMessageData.pucMessage), stMessageData.uiMessageLength);
stMessageData.pucMessage[stMessageData.uiMessageLength] = '\0';
pclLogger->info("Encoded: ({}) {}", stMessageData.uiMessageLength, reinterpret_cast<char*>(stMessageData.pucMessage));
uiCompleteMessages++;
Expand All @@ -158,7 +150,7 @@ int main(int argc, char* argv[])
if (std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - tLoop).count() > 1000)
{
uiCounter++;
pclLogger->info("{}% {} logs/s", clFileParser.GetPercentRead(), uiCompleteMessages / uiCounter);
pclLogger->info("{} logs/s", uiCompleteMessages / uiCounter);
tLoop = std::chrono::high_resolution_clock::now();
}
}
Expand Down
24 changes: 8 additions & 16 deletions examples/novatel/converter_parser/converter_parser.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,6 @@

#include <novatel_edie/common/logger.hpp>
#include <novatel_edie/decoders/oem/parser.hpp>
#include <novatel_edie/stream_interface/inputfilestream.hpp>
#include <novatel_edie/stream_interface/outputfilestream.hpp>
#include <novatel_edie/version.h>

namespace fs = std::filesystem;
Expand Down Expand Up @@ -110,37 +108,31 @@ int main(int argc, char* argv[])

clParser.SetFilter(&clFilter);

// Initialize FS structures and buffers
StreamReadStatus stReadStatus;
ReadDataStructure stReadData;
unsigned char acIfsReadBuffer[MAX_ASCII_MESSAGE_LENGTH];
stReadData.cData = reinterpret_cast<char*>(acIfsReadBuffer);
stReadData.uiDataSize = sizeof(acIfsReadBuffer);
std::array<char, MAX_ASCII_MESSAGE_LENGTH> cData;

// Set up file streams
InputFileStream clIfs(pathInFilename.string().c_str());
OutputFileStream clConvertedLogsOfs(pathInFilename.string().append(".").append(sEncodeFormat).c_str());
OutputFileStream clUnknownBytesOfs(pathInFilename.string().append(".").append(sEncodeFormat).append(".UNKNOWN").c_str());
std::ifstream ifs(pathInFilename, std::ios::binary);
std::ofstream convertedOfs(pathInFilename.string() + "." + sEncodeFormat, std::ios::binary);
std::ofstream unknownOfs(pathInFilename.string() + "." + sEncodeFormat + ".UNKNOWN", std::ios::binary);

uint32_t uiCompleteMessages = 0;
uint32_t uiCounter = 0;

tStart = std::chrono::high_resolution_clock::now();
tLoop = std::chrono::high_resolution_clock::now();

while (!stReadStatus.bEOS)
while (!ifs.eof())
{
stReadData.cData = reinterpret_cast<char*>(acIfsReadBuffer);
stReadStatus = clIfs.ReadData(stReadData);
clParser.Write(reinterpret_cast<unsigned char*>(stReadData.cData), stReadStatus.uiCurrentStreamRead);
ifs.read(cData.data(), cData.size());
clParser.Write(reinterpret_cast<unsigned char*>(cData.data()), ifs.gcount());

STATUS eStatus = clParser.Read(stMessageData, stMetaData);

while (eStatus != STATUS::BUFFER_EMPTY)
{
if (eStatus == STATUS::SUCCESS)
{
clConvertedLogsOfs.WriteData(reinterpret_cast<char*>(stMessageData.pucMessage), stMessageData.uiMessageLength);
convertedOfs.write(reinterpret_cast<char*>(stMessageData.pucMessage), stMessageData.uiMessageLength);
stMessageData.pucMessage[stMessageData.uiMessageLength] = '\0';
pclLogger->info("Encoded: ({}) {}", stMessageData.uiMessageLength, reinterpret_cast<char*>(stMessageData.pucMessage));
uiCompleteMessages++;
Expand Down
24 changes: 8 additions & 16 deletions examples/novatel/json_parser/json_parser.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,6 @@

#include <novatel_edie/common/logger.hpp>
#include <novatel_edie/decoders/oem/parser.hpp>
#include <novatel_edie/stream_interface/inputfilestream.hpp>
#include <novatel_edie/stream_interface/outputfilestream.hpp>
#include <novatel_edie/version.h>

namespace fs = std::filesystem;
Expand Down Expand Up @@ -119,37 +117,31 @@ int main(int argc, char* argv[])

clParser.SetFilter(&clFilter);

// Initialize FS structures and buffers
StreamReadStatus stReadStatus;
ReadDataStructure stReadData;
unsigned char acIfsReadBuffer[MAX_ASCII_MESSAGE_LENGTH];
stReadData.cData = reinterpret_cast<char*>(acIfsReadBuffer);
stReadData.uiDataSize = sizeof(acIfsReadBuffer);
std::array<char, MAX_ASCII_MESSAGE_LENGTH> cData;

// Set up file streams
InputFileStream clIfs(pathInFilename.string().c_str());
OutputFileStream clConvertedLogsOfs(pathInFilename.string().append(".").append(sEncodeFormat).c_str());
OutputFileStream clUnknownBytesOfs(pathInFilename.string().append(".").append(sEncodeFormat).append(".UNKNOWN").c_str());
std::ifstream ifs(pathInFilename, std::ios::binary);
std::ofstream convertedOfs(pathInFilename.string() + "." + sEncodeFormat, std::ios::binary);
std::ofstream unknownOfs(pathInFilename.string() + "." + sEncodeFormat + ".UNKNOWN", std::ios::binary);

uint32_t uiCompleteMessages = 0;
uint32_t uiCounter = 0;

tStart = std::chrono::high_resolution_clock::now();
tLoop = std::chrono::high_resolution_clock::now();

while (!stReadStatus.bEOS)
while (!ifs.eof())
{
stReadData.cData = reinterpret_cast<char*>(acIfsReadBuffer);
stReadStatus = clIfs.ReadData(stReadData);
clParser.Write(reinterpret_cast<unsigned char*>(stReadData.cData), stReadStatus.uiCurrentStreamRead);
ifs.read(cData.data(), cData.size());
clParser.Write(reinterpret_cast<unsigned char*>(cData.data()), ifs.gcount());

STATUS eStatus = clParser.Read(stMessageData, stMetaData);

while (eStatus != STATUS::BUFFER_EMPTY)
{
if (eStatus == STATUS::SUCCESS)
{
clConvertedLogsOfs.WriteData(reinterpret_cast<char*>(stMessageData.pucMessage), stMessageData.uiMessageLength);
convertedOfs.write(reinterpret_cast<char*>(stMessageData.pucMessage), stMessageData.uiMessageLength);
stMessageData.pucMessage[stMessageData.uiMessageLength] = '\0';
pclLogger->info("Encoded: ({}) {}", stMessageData.uiMessageLength, reinterpret_cast<char*>(stMessageData.pucMessage));
uiCompleteMessages++;
Expand Down
26 changes: 7 additions & 19 deletions examples/novatel/range_decompressor/range_decompressor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,6 @@
#include <novatel_edie/decoders/oem/framer.hpp>
#include <novatel_edie/decoders/oem/header_decoder.hpp>
#include <novatel_edie/decoders/oem/rangecmp/range_decompressor.hpp>
#include <novatel_edie/stream_interface/inputfilestream.hpp>
#include <novatel_edie/stream_interface/outputfilestream.hpp>
#include <novatel_edie/version.h>

namespace fs = std::filesystem;
Expand Down Expand Up @@ -118,18 +116,14 @@ int main(int argc, char* argv[])
clFramer.SetPayloadOnly(false);
clFramer.SetReportUnknownBytes(true);

unsigned char acFileStreamBuffer[MAX_ASCII_MESSAGE_LENGTH];
ReadDataStructure stReadData;
stReadData.cData = reinterpret_cast<char*>(acFileStreamBuffer);
stReadData.uiDataSize = sizeof(acFileStreamBuffer);
std::array<char, MAX_ASCII_MESSAGE_LENGTH> cData;

unsigned char acFrameBuffer[MAX_ASCII_MESSAGE_LENGTH];
unsigned char acEncodeBuffer[MAX_ASCII_MESSAGE_LENGTH];
unsigned char* pucEncodedMessageBuffer = acEncodeBuffer;

InputFileStream clIfs(pathInFilename.string().c_str());
OutputFileStream clOfs(pathInFilename.string().append(".DECOMPRESSED.").append(sEncodeFormat).c_str());
StreamReadStatus stReadStatus;
std::ifstream ifs(pathInFilename, std::ios::binary);
std::ofstream ofs(pathInFilename.string() + ".DECOMPRESSED." + sEncodeFormat, std::ios::binary);

auto eStatus = STATUS::UNKNOWN;

Expand Down Expand Up @@ -157,13 +151,7 @@ int main(int argc, char* argv[])
if (eStatus == STATUS::SUCCESS)
{
uiCompletedMessages++;
uint32_t uiBytesWritten = clOfs.WriteData(reinterpret_cast<char*>(pucReadBuffer), stMetaData.uiLength);
if (stMetaData.uiLength == uiBytesWritten)
{
pucReadBuffer[stMetaData.uiLength] = '\0';
pclLogger->info("Decompressed: ({}) {}", stMetaData.uiLength, reinterpret_cast<char*>(pucReadBuffer));
}
else { pclLogger->error("Could only write {}/{} bytes.", uiBytesWritten, stMessageData.uiMessageLength); }
ofs.write(reinterpret_cast<char*>(pucReadBuffer), stMetaData.uiLength);
}
else if (eStatus == STATUS::UNSUPPORTED)
{
Expand All @@ -189,14 +177,14 @@ int main(int argc, char* argv[])
else if (eStatus == STATUS::BUFFER_EMPTY || eStatus == STATUS::INCOMPLETE)
{
// Read from file, write to framer.
stReadStatus = clIfs.ReadData(stReadData);
if (stReadStatus.uiCurrentStreamRead == 0)
ifs.read(cData.data(), cData.size());
if (ifs.gcount() == 0)
{
pclLogger->info("Stream finished");
break;
}

clFramer.Write(reinterpret_cast<unsigned char*>(stReadData.cData), stReadStatus.uiCurrentStreamRead);
clFramer.Write(reinterpret_cast<unsigned char*>(cData.data()), ifs.gcount());
}
}

Expand Down
Loading

0 comments on commit 04d7433

Please sign in to comment.