From 59d814d611344114b8773685dfc3007df1b99b5d Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Thu, 31 Oct 2024 06:35:12 +0000 Subject: [PATCH] polish --- core/pipeline/Pipeline.cpp | 4 + core/plugin/flusher/sls/FlusherSLS.cpp | 4 + core/protobuf/sls/RawLog.cpp | 387 ------------------- core/protobuf/sls/RawLog.h | 126 ------ core/protobuf/sls/RawLogGroup.cpp | 222 ----------- core/protobuf/sls/RawLogGroup.h | 280 -------------- core/unittest/log_pb/CMakeLists.txt | 12 +- core/unittest/log_pb/PBUnittest.cpp | 405 -------------------- core/unittest/pipeline/PipelineUnittest.cpp | 2 + 9 files changed, 18 insertions(+), 1424 deletions(-) delete mode 100644 core/protobuf/sls/RawLog.cpp delete mode 100644 core/protobuf/sls/RawLog.h delete mode 100644 core/protobuf/sls/RawLogGroup.cpp delete mode 100644 core/protobuf/sls/RawLogGroup.h delete mode 100644 core/unittest/log_pb/PBUnittest.cpp diff --git a/core/pipeline/Pipeline.cpp b/core/pipeline/Pipeline.cpp index db517363f8..7be446e4cf 100644 --- a/core/pipeline/Pipeline.cpp +++ b/core/pipeline/Pipeline.cpp @@ -388,6 +388,10 @@ bool Pipeline::Send(vector&& groupList) { auto before = chrono::system_clock::now(); bool allSucceeded = true; for (auto& group : groupList) { + if (group.GetEvents().empty()) { + LOG_DEBUG(sLogger, ("empty event group", "discard")("config", mName)); + continue; + } auto res = mRouter.Route(group); for (auto& item : res) { if (item.first >= mFlushers.size()) { diff --git a/core/plugin/flusher/sls/FlusherSLS.cpp b/core/plugin/flusher/sls/FlusherSLS.cpp index 9f5b4cd08b..6ba452f564 100644 --- a/core/plugin/flusher/sls/FlusherSLS.cpp +++ b/core/plugin/flusher/sls/FlusherSLS.cpp @@ -486,7 +486,11 @@ bool FlusherSLS::Init(const Json::Value& config, Json::Value& optionalGoPipeline mCompressor = CompressorFactory::GetInstance()->Create(config, *mContext, sName, mPluginID, CompressType::LZ4); } +#ifdef __ENTERPRISE__ + mGroupSerializer = make_unique(this); +#else mGroupSerializer = make_unique(this); +#endif mGroupListSerializer = make_unique(this); // MaxSendRate diff --git a/core/protobuf/sls/RawLog.cpp b/core/protobuf/sls/RawLog.cpp deleted file mode 100644 index 359857ffc5..0000000000 --- a/core/protobuf/sls/RawLog.cpp +++ /dev/null @@ -1,387 +0,0 @@ -// Copyright 2022 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "RawLog.h" -#include "common/Flags.h" -#include "logger/Logger.h" - -DEFINE_FLAG_INT32(raw_log_init_buffer_size, "", 256); - -namespace logtail { - -// 1+5( 1 ---> header; 5 ---> uint32) -#define INIT_LOG_SIZE_BYTES 6L - -/** - * Return the number of bytes required to store a variable-length unsigned - * 32-bit integer in base-128 varint encoding. - * - * \param v - * Value to encode. - * \return - * Number of bytes required. - */ -static inline size_t uint32_size(uint32_t v) { - if (v < (1UL << 7)) { - return 1; - } else if (v < (1UL << 14)) { - return 2; - } else if (v < (1UL << 21)) { - return 3; - } else if (v < (1UL << 28)) { - return 4; - } else { - return 5; - } -} - -/** - * Pack an unsigned 32-bit integer in base-128 varint encoding and return the - * number of bytes written, which must be 5 or less. - * - * \param value - * Value to encode. - * \param[out] out - * Packed value. - * \return - * Number of bytes written to `out`. - */ -static inline size_t uint32_pack(uint32_t value, uint8_t* out) { - unsigned rv = 0; - - if (value >= 0x80) { - out[rv++] = value | 0x80; - value >>= 7; - if (value >= 0x80) { - out[rv++] = value | 0x80; - value >>= 7; - if (value >= 0x80) { - out[rv++] = value | 0x80; - value >>= 7; - if (value >= 0x80) { - out[rv++] = value | 0x80; - value >>= 7; - } - } - } - } - /* assert: value<128 */ - out[rv++] = value; - return rv; -} - -static inline uint32_t parse_uint32(unsigned len, const uint8_t* data) { - uint32_t rv = data[0] & 0x7f; - if (len > 1) { - rv |= ((uint32_t)(data[1] & 0x7f) << 7); - if (len > 2) { - rv |= ((uint32_t)(data[2] & 0x7f) << 14); - if (len > 3) { - rv |= ((uint32_t)(data[3] & 0x7f) << 21); - if (len > 4) - rv |= ((uint32_t)(data[4]) << 28); - } - } - } - return rv; -} - -static unsigned scan_varint(unsigned len, const uint8_t* data) { - unsigned i; - if (len > 10) - len = 10; - for (i = 0; i < len; i++) - if ((data[i] & 0x80) == 0) - break; - if (i == len) - return 0; - return i + 1; -} - -RawLog* -RawLog::AddLogFull(uint32_t logTime, std::vector keys, boost::match_results subMathValues) { - // limit logTime's min value, ensure varint size is 5 - if (logTime < 463563523) { - logTime = 463563523; - } - - int32_t i = 0; - uint32_t logSize = 6; - int32_t pair_count = (int32_t)keys.size(); - for (; i < pair_count; ++i) { - auto keyLen = uint32_t(keys[i].size()); - auto valLen = uint32_t(subMathValues[i + 1].length()); - uint32_t contSize = uint32_size(keyLen) + uint32_size(valLen) + keyLen + valLen + 2u; - logSize += 1 + uint32_size(contSize) + contSize; - } - uint32_t headerSize = 1 + uint32_size(logSize); - int32_t deltaSize = INIT_LOG_SIZE_BYTES - headerSize; - uint32_t totalBufferSize = logSize + headerSize; - - auto* rawLog = new RawLog(); - rawLog->mRawLen = totalBufferSize + deltaSize; - rawLog->mRawBuffer = (uint8_t*)malloc(totalBufferSize + deltaSize); - rawLog->mMaxLen = totalBufferSize + deltaSize; - - uint8_t* buf = (uint8_t*)rawLog->mRawBuffer + deltaSize; - - *buf++ = 0x0A; - buf += uint32_pack(logSize, buf); - - // time - *buf++ = 0x08; - buf += uint32_pack(logTime, buf); - - // Content - // header - i = 0; - for (; i < pair_count; ++i) { - auto keyLen = uint32_t(keys[i].size()); - auto valLen = uint32_t(subMathValues[i + 1].length()); - *buf++ = 0x12; - buf += uint32_pack(uint32_size(keyLen) + uint32_size(valLen) + 2 + keyLen + valLen, buf); - *buf++ = 0x0A; - buf += uint32_pack(keyLen, buf); - memcpy(buf, keys[i].c_str(), keyLen); - buf += keyLen; - *buf++ = 0x12; - buf += uint32_pack(valLen, buf); - memcpy(buf, subMathValues[i + 1].first, valLen); - buf += valLen; - } - rawLog->mMallocDelta = deltaSize; - assert(buf - rawLog->mRawBuffer == totalBufferSize + rawLog->mMallocDelta); - rawLog->mNowBuffer = buf; - return rawLog; -} - -void RawLog::AddTime(uint32_t logTime) { - // limit logTime's min value, ensure varint size is 5 - if (logTime < 463563523) { - logTime = 463563523; - } - // 1 header and 5 time - if (mRawLen + 6 > mMaxLen) { - // reset log_now_buffer - adjustBuffer(6); - } - - // time - *mNowBuffer++ = 0x08; - mNowBuffer += uint32_pack(logTime, mNowBuffer); - mRawLen += 6; -} - -uint32_t RawLog::AddKeyValue(const char* key, size_t key_len, const char* value, size_t value_len) { - // sum total size - uint32_t kv_size - = sizeof(char) * (key_len + value_len) + uint32_size((uint32_t)key_len) + uint32_size((uint32_t)value_len) + 2; - kv_size += 1 + uint32_size(kv_size); - // ensure buffer size - if (mRawLen + kv_size > mMaxLen) { - // reset log_now_buffer - adjustBuffer(kv_size); - } - mRawLen += kv_size; - uint8_t* buf = mNowBuffer; - // key_value header - *buf++ = 0x12; - buf += uint32_pack(uint32_size(key_len) + uint32_size(value_len) + 2 + key_len + value_len, buf); - // key len - *buf++ = 0x0A; - buf += uint32_pack(key_len, buf); - // key - memcpy(buf, key, key_len); - buf += key_len; - // value len - *buf++ = 0x12; - buf += uint32_pack(value_len, buf); - // value - memcpy(buf, value, value_len); - buf += value_len; - mNowBuffer = (uint8_t*)buf; - return kv_size; -} - - -void RawLog::AddLogStart(uint32_t logTime) { - if (mRawLen + INIT_LOG_SIZE_BYTES > mMaxLen) { - adjustBuffer(INIT_LOG_SIZE_BYTES); - } - mRawLen = INIT_LOG_SIZE_BYTES; - mNowBuffer += INIT_LOG_SIZE_BYTES; - AddTime(logTime); -} - -void RawLog::AddLogDone() { - uint32_t log_size = mRawLen - INIT_LOG_SIZE_BYTES; - // check total size and uint32_size(total size) - - int32_t header_size = uint32_size(log_size) + 1; - - if (header_size != INIT_LOG_SIZE_BYTES) { - mMallocDelta = (int8_t)(INIT_LOG_SIZE_BYTES - header_size); - } - // set log header - uint8_t* buf = (mRawBuffer + mMallocDelta); - *buf++ = 0x0A; - uint32_pack(log_size, buf); -} - -void RawLog::adjustBuffer(uint32_t newLen) { - if (mRawBuffer == NULL) { - mRawBuffer = (uint8_t*)malloc(INT32_FLAG(raw_log_init_buffer_size)); - mNowBuffer = mRawBuffer; - mMaxLen = INT32_FLAG(raw_log_init_buffer_size); - return; - } - uint32_t new_buffer_len = mMaxLen << 1; - - if (new_buffer_len < mMaxLen + newLen) { - new_buffer_len = mMaxLen + newLen; - } - - mRawBuffer = (uint8_t*)realloc(mRawBuffer, new_buffer_len); - mMaxLen = new_buffer_len; - mNowBuffer = mRawBuffer + mRawLen; -} - -void RawLog::AppendToString(std::string* output) const { - output->append((const char*)mRawBuffer + mMallocDelta, mRawLen - mMallocDelta); -} - -bool RawLog::NextKeyValue( - RawLog::iterator& iter, const char*& key, uint32_t& keyLen, const char*& value, uint32_t& valueLen) { - if (iter.mNowOffset >= (int32_t)mRawLen) { - return false; - } - assert(mRawBuffer[iter.mNowOffset] == 0x12); - iter.mLastOffset = iter.mNowOffset; - // skip log content - ++iter.mNowOffset; - uint32_t keyValueLenSize = scan_varint(5, mRawBuffer + iter.mNowOffset); - iter.mNowOffset += keyValueLenSize; - // parse key - ++iter.mNowOffset; - uint32_t keyLenSize = scan_varint(5, mRawBuffer + iter.mNowOffset); - keyLen = parse_uint32(keyLenSize, mRawBuffer + iter.mNowOffset); - iter.mNowOffset += keyLenSize; - key = (const char*)mRawBuffer + iter.mNowOffset; - iter.mNowOffset += keyLen; - // parse value - ++iter.mNowOffset; - uint32_t valueLenSize = scan_varint(5, mRawBuffer + iter.mNowOffset); - valueLen = parse_uint32(valueLenSize, mRawBuffer + iter.mNowOffset); - iter.mNowOffset += valueLenSize; - value = (const char*)mRawBuffer + iter.mNowOffset; - iter.mNowOffset += valueLen; - return true; -} - -bool RawLog::DeleteKeyValue(RawLog::iterator& iter) { - if (iter.mLastOffset == 0 || iter.mLastOffset == iter.mNowOffset) { - return false; - } - int32_t deltaSize = iter.mLastOffset - iter.mNowOffset; - memmove(mRawBuffer + iter.mLastOffset, mRawBuffer + iter.mNowOffset, mRawLen - iter.mNowOffset); - iter.mNowOffset = iter.mLastOffset; - adjustLogSize(deltaSize); - mRawLen += deltaSize; - return true; -} - -bool RawLog::UpdateKeyValue( - RawLog::iterator& iter, const char* key, size_t key_len, const char* value, size_t value_len) { - if (iter.mLastOffset == 0) { - return false; - } - // sum total size - int32_t kv_size - = sizeof(char) * (key_len + value_len) + uint32_size((uint32_t)key_len) + uint32_size((uint32_t)value_len) + 2; - int32_t lastSize = iter.mNowOffset - iter.mLastOffset; - kv_size += 1 + uint32_size(kv_size); - // ensure buffer size - int32_t deltaSize = kv_size - lastSize; - if (deltaSize > 0 && mRawLen + kv_size > mMaxLen) { - // reset log_now_buffer - adjustBuffer(kv_size); - } - - // move - if (deltaSize != 0) { - memmove(mRawBuffer + iter.mLastOffset + kv_size, mRawBuffer + iter.mNowOffset, mRawLen - iter.mNowOffset); - adjustLogSize(deltaSize); - } - - mRawLen += deltaSize; - uint8_t* buf = mRawBuffer + iter.mLastOffset; - // key_value header - *buf++ = 0x12; - buf += uint32_pack(uint32_size(key_len) + uint32_size(value_len) + 2 + key_len + value_len, buf); - // key len - *buf++ = 0x0A; - buf += uint32_pack(key_len, buf); - // key - memcpy(buf, key, key_len); - buf += key_len; - // value len - *buf++ = 0x12; - buf += uint32_pack(value_len, buf); - // value - memcpy(buf, value, value_len); - buf += value_len; - mNowBuffer = (uint8_t*)buf; - iter.mNowOffset = iter.mLastOffset + kv_size; - return true; -} - -bool RawLog::AppendKeyValue(const char* key, size_t kenLen, const char* value, size_t valueLen) { - uint32_t deltaSize = AddKeyValue(key, kenLen, value, valueLen); - adjustLogSize(deltaSize); - return false; -} - -RawLog::iterator RawLog::GetIterator() const { - auto iter = iterator{}; - // skip delta and log header - uint32_t startLen = mMallocDelta + 1; - // skip log header size - startLen += scan_varint(5, mRawBuffer + startLen); - // skip time - startLen += 6; - iter.mNowOffset = startLen; - assert(mRawBuffer[startLen] == 0x12); - return iter; -} - -void RawLog::adjustLogSize(int32_t deltaLen) { - if (deltaLen == 0) { - return; - } - - uint32_t logLenSize = scan_varint(5, mRawBuffer + mMallocDelta + 1); - int32_t logSize = parse_uint32(logLenSize, mRawBuffer + mMallocDelta + 1); - int32_t newSize = logSize + deltaLen; - uint32_t newHeaderSize = uint32_size(newSize) + 1; - - mMallocDelta = (int8_t)(INIT_LOG_SIZE_BYTES - newHeaderSize); - - // set log header - uint8_t* buf = (mRawBuffer + mMallocDelta); - *buf++ = 0x0A; - uint32_pack(newSize, buf); -} - - -} // namespace logtail diff --git a/core/protobuf/sls/RawLog.h b/core/protobuf/sls/RawLog.h deleted file mode 100644 index 0b303655d9..0000000000 --- a/core/protobuf/sls/RawLog.h +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright 2022 iLogtail Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef LOGTAIL_RAWLOG_H -#define LOGTAIL_RAWLOG_H - -#include -#include -#include - -namespace logtail { - -class RawLog { -public: - struct iterator { - int32_t mLastOffset = 0; - int32_t mNowOffset = 0; - }; - - RawLog() = default; - ~RawLog() { - if (mRawBuffer != NULL) { - free(mRawBuffer); - } - } - - static RawLog* - AddLogFull(uint32_t logTime, std::vector keys, boost::match_results subMathValues); - - void AddLogStart(uint32_t logTime); - - - void AddKeyValue(const std::string& key, const std::string& value) { - AddKeyValue(key.c_str(), key.size(), value.c_str(), value.size()); - } - - void AddKeyValue(const std::string& key, const char* value, size_t valueLen) { - AddKeyValue(key.c_str(), key.size(), value, valueLen); - } - - uint32_t AddKeyValue(const char* key, size_t kenLen, const char* value, size_t valueLen); - - void AddLogDone(); - - uint8_t* GetBuffer() { return mRawBuffer + mMallocDelta; } - - size_t GetBufferLength() { return mRawLen - mMallocDelta; } - - void AppendToString(std::string* output) const; - - /** - * init iterator - * @return - */ - iterator GetIterator() const; - - /** - * get next key value pair, return false if end of log - * @param iter - * @param key - * @param keyLen - * @param value - * @param valueLen - * @return - */ - bool NextKeyValue(iterator& iter, const char*& key, uint32_t& keyLen, const char*& value, uint32_t& valueLen); - - /** - * delete this key value pair, return false if iter is invalid - * @param iter - * @return - */ - bool DeleteKeyValue(iterator& iter); - - /** - * update this key value pair - * @param iter - * @param key - * @param kenLen - * @param value - * @param valueLen - * @return - */ - bool UpdateKeyValue(iterator& iter, const char* key, size_t kenLen, const char* value, size_t valueLen); - - /** - * append key value pair after end of log - * @param key - * @param kenLen - * @param value - * @param valueLen - * @return - */ - bool AppendKeyValue(const char* key, size_t kenLen, const char* value, size_t valueLen); - - -protected: - void adjustBuffer(uint32_t newLen); - void AddTime(uint32_t logTime); - void adjustLogSize(int32_t deltaLen); - - uint32_t mRawLen = 0; - uint32_t mMaxLen = 0; - uint8_t* mRawBuffer = NULL; - uint8_t* mNowBuffer = NULL; - int8_t mMallocDelta = 0; -}; - - -} // namespace logtail - - -#endif // LOGTAIL_RAWLOG_H diff --git a/core/protobuf/sls/RawLogGroup.cpp b/core/protobuf/sls/RawLogGroup.cpp deleted file mode 100644 index 9ce1215b7c..0000000000 --- a/core/protobuf/sls/RawLogGroup.cpp +++ /dev/null @@ -1,222 +0,0 @@ -// Copyright 2022 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "RawLogGroup.h" - -namespace logtail { - -/** - * Return the number of bytes required to store a variable-length unsigned - * 32-bit integer in base-128 varint encoding. - * - * \param v - * Value to encode. - * \return - * Number of bytes required. - */ -static inline size_t uint32_size(uint32_t v) { - if (v < (1UL << 7)) { - return 1; - } else if (v < (1UL << 14)) { - return 2; - } else if (v < (1UL << 21)) { - return 3; - } else if (v < (1UL << 28)) { - return 4; - } else { - return 5; - } -} - -/** - * Pack an unsigned 32-bit integer in base-128 varint encoding and return the - * number of bytes written, which must be 5 or less. - * - * \param value - * Value to encode. - * \param[out] out - * Packed value. - * \return - * Number of bytes written to `out`. - */ -static inline size_t uint32_pack(uint32_t value, std::string* output) { - unsigned rv = 1; - - if (value >= 0x80) { - output->push_back(value | 0x80); - ++rv; - value >>= 7; - if (value >= 0x80) { - output->push_back(value | 0x80); - ++rv; - value >>= 7; - if (value >= 0x80) { - output->push_back(value | 0x80); - ++rv; - value >>= 7; - if (value >= 0x80) { - output->push_back(value | 0x80); - ++rv; - value >>= 7; - } - } - } - } - /* assert: value<128 */ - output->push_back(value); - return rv; -} - - -int RawLogGroup::logtags_size() const { - return (int)logtags_.size(); -} - -void RawLogGroup::clear_logtags() { - logtags_.clear(); -} - -const LogTag& RawLogGroup::logtags(int index) const { - return logtags_[index]; -} - -LogTag* RawLogGroup::mutable_logtags(int index) { - return &logtags_[index]; -} - -void RawLogGroup::add_logtags(const std::string& key, const std::string& value) { - logtags_.push_back(::std::move(LogTag(key, value))); -} - -void RawLogGroup::add_logtags(const std::string& key, std::string&& value) { - logtags_.push_back(LogTag(key, ::std::move(value))); -} - -std::vector* RawLogGroup::mutable_logtags() { - return &logtags_; -} - -const std::vector& RawLogGroup::logtags() const { - return logtags_; -} - -RawLogGroup::~RawLogGroup() { - for (size_t size = 0; size < rawlogs_.size(); ++size) { - delete rawlogs_[size]; - } -} - -int RawLogGroup::logs_size() const { - return (int)rawlogs_.size(); -} - -void RawLogGroup::clear_logs() { - for (size_t size = 0; size < rawlogs_.size(); ++size) { - delete rawlogs_[size]; - } - rawlogs_.clear(); -} - -const RawLog& RawLogGroup::logs(int index) const { - return *rawlogs_[index]; -} - -RawLog* RawLogGroup::mutable_logs(int index) { - return rawlogs_[index]; -} - -void RawLogGroup::add_logs(RawLog* log) { - rawlogs_.push_back(log); -} - -std::vector* RawLogGroup::mutable_logs() { - return &rawlogs_; -} - -const std::vector& RawLogGroup::logs() { - return rawlogs_; -} - -bool RawLogGroup::SerializeToString(std::string* output) const { - output->clear(); - return AppendToString(output); -} - -bool RawLogGroup::AppendToString(std::string* output) const { - if (rawlogs_.empty()) { - return false; - } - pack_logs(output); - pack_others(output); - pack_logtags(output); - return true; -} - -void RawLogGroup::pack_logs(std::string* output) const { - for (size_t size = 0; size < rawlogs_.size(); ++size) { - RawLog* log = rawlogs_[size]; - log->AppendToString(output); - } -} - -void RawLogGroup::pack_logtags(std::string* output) const { - for (size_t size = 0; size < logtags_.size(); ++size) { - const LogTag& logTag = logtags_[size]; - - // use only 1 malloc - size_t k_len = logTag.first.size(); - size_t v_len = logTag.second.size(); - const char* k = logTag.first.c_str(); - const char* v = logTag.second.c_str(); - uint32_t tag_size - = sizeof(char) * (k_len + v_len) + uint32_size((uint32_t)k_len) + uint32_size((uint32_t)v_len) + 2; - output->push_back(0x32); - uint32_pack(tag_size, output); - output->push_back(0x0A); - uint32_pack((uint32_t)k_len, output); - output->append(k, k_len); - output->push_back(0x12); - uint32_pack((uint32_t)v_len, output); - output->append(v, v_len); - } -} - -void RawLogGroup::pack_others(std::string* output) const { - if (has_category()) { - output->push_back(0x12); - uint32_pack((uint32_t)category_.size(), output); - output->append(category_); - } - - if (has_topic()) { - output->push_back(0x1A); - uint32_pack((uint32_t)topic_.size(), output); - output->append(topic_); - } - - if (has_source()) { - output->push_back(0x22); - uint32_pack((uint32_t)source_.size(), output); - output->append(source_); - } - - if (has_machineuuid()) { - output->push_back(0x2A); - uint32_pack((uint32_t)machineuuid_.size(), output); - output->append(machineuuid_); - } -} - - -} // namespace logtail \ No newline at end of file diff --git a/core/protobuf/sls/RawLogGroup.h b/core/protobuf/sls/RawLogGroup.h deleted file mode 100644 index cf5a357f11..0000000000 --- a/core/protobuf/sls/RawLogGroup.h +++ /dev/null @@ -1,280 +0,0 @@ -/* - * Copyright 2022 iLogtail Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef LOGTAIL_RAWLOGGROUP_H -#define LOGTAIL_RAWLOGGROUP_H - -#include - -namespace logtail { - -typedef std::pair LogTag; - -class RawLogGroup { -public: - RawLogGroup() = default; - ~RawLogGroup(); - - int logs_size() const; - void clear_logs(); - const RawLog& logs(int index) const; - RawLog* mutable_logs(int index); - void add_logs(RawLog* log); - std::vector* mutable_logs(); - const std::vector& logs(); - - // repeated .sls_logs.LogTag LogTags = 6; - int logtags_size() const; - void clear_logtags(); - const LogTag& logtags(int index) const; - LogTag* mutable_logtags(int index); - void add_logtags(const std::string& key, const std::string& value); - void add_logtags(const std::string& key, std::string&& value); - std::vector* mutable_logtags(); - const std::vector& logtags() const; - - // optional string Category = 2; 0x12 - bool has_category() const; - void clear_category(); - const ::std::string& category() const; - void set_category(const ::std::string& value); - void set_category(::std::string&& value); - void set_category(const char* value, size_t size); - ::std::string* mutable_category(); - - // optional string Topic = 3; 0x1A - bool has_topic() const; - void clear_topic(); - const ::std::string& topic() const; - void set_topic(const ::std::string& value); - void set_topic(::std::string&& value); - void set_topic(const char* value, size_t size); - ::std::string* mutable_topic(); - - // optional string Source = 4; 0x22; - bool has_source() const; - void clear_source(); - const ::std::string& source() const; - void set_source(const ::std::string& value); - void set_source(::std::string&& value); - void set_source(const char* value, size_t size); - ::std::string* mutable_source(); - - // optional string MachineUUID = 5; 0x2A - bool has_machineuuid() const; - void clear_machineuuid(); - static const int kMachineUUIDFieldNumber = 5; - const ::std::string& machineuuid() const; - void set_machineuuid(const ::std::string& value); - void set_machineuuid(::std::string&& value); - void set_machineuuid(const char* value, size_t size); - ::std::string* mutable_machineuuid(); - - // Serialize the message and store it in the given string. All required - // fields must be set. - bool SerializeToString(std::string* output) const; - // Like SerializeToString(), but appends to the data to the string's existing - // contents. All required fields must be set. - bool AppendToString(std::string* output) const; - -private: - void set_has_category(); - void clear_has_category(); - void set_has_topic(); - void clear_has_topic(); - void set_has_source(); - void clear_has_source(); - void set_has_machineuuid(); - void clear_has_machineuuid(); - - void pack_logs(std::string* output) const; - void pack_logtags(std::string* output) const; - void pack_others(std::string* output) const; - - - uint64_t _has_bits_ = 0; - std::string category_; - std::string topic_; - std::string source_; - std::string machineuuid_; - std::vector logtags_; - std::vector rawlogs_; -}; - -// optional string Category = 2; -inline bool RawLogGroup::has_category() const { - return (_has_bits_ & 0x00000001u) != 0; -} -inline void RawLogGroup::set_has_category() { - _has_bits_ |= 0x00000001u; -} -inline void RawLogGroup::clear_has_category() { - _has_bits_ &= ~0x00000001u; -} -inline void RawLogGroup::clear_category() { - category_.clear(); - clear_has_category(); -} -inline const ::std::string& RawLogGroup::category() const { - // @@protoc_insertion_point(field_get:sls_logs.LogGroup.Category) - return category_; -} -inline void RawLogGroup::set_category(const ::std::string& value) { - set_has_category(); - category_ = value; - // @@protoc_insertion_point(field_set:sls_logs.LogGroup.Category) -} -inline void RawLogGroup::set_category(::std::string&& value) { - set_has_category(); - category_ = ::std::move(value); - // @@protoc_insertion_point(field_set_rvalue:sls_logs.LogGroup.Category) -} -inline void RawLogGroup::set_category(const char* value, size_t size) { - set_has_category(); - category_ = ::std::string(reinterpret_cast(value), size); - ; - // @@protoc_insertion_point(field_set_pointer:sls_logs.LogGroup.Category) -} -inline ::std::string* RawLogGroup::mutable_category() { - set_has_category(); - // @@protoc_insertion_point(field_mutable:sls_logs.LogGroup.Category) - return &category_; - ; -} - -// optional string Topic = 3; -inline bool RawLogGroup::has_topic() const { - return (_has_bits_ & 0x00000002u) != 0; -} -inline void RawLogGroup::set_has_topic() { - _has_bits_ |= 0x00000002u; -} -inline void RawLogGroup::clear_has_topic() { - _has_bits_ &= ~0x00000002u; -} -inline void RawLogGroup::clear_topic() { - topic_.clear(); - clear_has_topic(); -} -inline const ::std::string& RawLogGroup::topic() const { - // @@protoc_insertion_point(field_get:sls_logs.LogGroup.Topic) - return topic_; -} -inline void RawLogGroup::set_topic(const ::std::string& value) { - set_has_topic(); - topic_ = value; - // @@protoc_insertion_point(field_set:sls_logs.LogGroup.Topic) -} -inline void RawLogGroup::set_topic(::std::string&& value) { - set_has_topic(); - topic_ = ::std::move(value); - // @@protoc_insertion_point(field_set_rvalue:sls_logs.LogGroup.Topic) -} -inline void RawLogGroup::set_topic(const char* value, size_t size) { - set_has_topic(); - topic_ = ::std::string(reinterpret_cast(value), size); - // @@protoc_insertion_point(field_set_pointer:sls_logs.LogGroup.Topic) -} -inline ::std::string* RawLogGroup::mutable_topic() { - set_has_topic(); - // @@protoc_insertion_point(field_mutable:sls_logs.LogGroup.Topic) - return &topic_; - ; -} - -// optional string Source = 4; -inline bool RawLogGroup::has_source() const { - return (_has_bits_ & 0x00000004u) != 0; -} -inline void RawLogGroup::set_has_source() { - _has_bits_ |= 0x00000004u; -} -inline void RawLogGroup::clear_has_source() { - _has_bits_ &= ~0x00000004u; -} -inline void RawLogGroup::clear_source() { - source_.clear(); - clear_has_source(); -} -inline const ::std::string& RawLogGroup::source() const { - // @@protoc_insertion_point(field_get:sls_logs.LogGroup.Source) - return source_; -} -inline void RawLogGroup::set_source(const ::std::string& value) { - set_has_source(); - source_ = value; - // @@protoc_insertion_point(field_set:sls_logs.LogGroup.Source) -} -inline void RawLogGroup::set_source(::std::string&& value) { - set_has_source(); - source_ = ::std::move(value); - // @@protoc_insertion_point(field_set_rvalue:sls_logs.LogGroup.Source) -} -inline void RawLogGroup::set_source(const char* value, size_t size) { - set_has_source(); - source_ = ::std::string(reinterpret_cast(value), size); - // @@protoc_insertion_point(field_set_pointer:sls_logs.LogGroup.Source) -} -inline ::std::string* RawLogGroup::mutable_source() { - set_has_source(); - // @@protoc_insertion_point(field_mutable:sls_logs.LogGroup.Source) - return &source_; - ; -} - -// optional string MachineUUID = 5; -inline bool RawLogGroup::has_machineuuid() const { - return (_has_bits_ & 0x00000008u) != 0; -} -inline void RawLogGroup::set_has_machineuuid() { - _has_bits_ |= 0x00000008u; -} -inline void RawLogGroup::clear_has_machineuuid() { - _has_bits_ &= ~0x00000008u; -} -inline void RawLogGroup::clear_machineuuid() { - machineuuid_.clear(); - clear_has_machineuuid(); -} -inline const ::std::string& RawLogGroup::machineuuid() const { - // @@protoc_insertion_point(field_get:sls_logs.LogGroup.MachineUUID) - return machineuuid_; -} -inline void RawLogGroup::set_machineuuid(const ::std::string& value) { - set_has_machineuuid(); - machineuuid_ = value; - // @@protoc_insertion_point(field_set:sls_logs.LogGroup.MachineUUID) -} -inline void RawLogGroup::set_machineuuid(::std::string&& value) { - set_has_machineuuid(); - machineuuid_ = ::std::move(value); - // @@protoc_insertion_point(field_set_rvalue:sls_logs.LogGroup.MachineUUID) -} -inline void RawLogGroup::set_machineuuid(const char* value, size_t size) { - set_has_machineuuid(); - machineuuid_ = ::std::string(reinterpret_cast(value), size); - // @@protoc_insertion_point(field_set_pointer:sls_logs.LogGroup.MachineUUID) -} -inline ::std::string* RawLogGroup::mutable_machineuuid() { - set_has_machineuuid(); - // @@protoc_insertion_point(field_mutable:sls_logs.LogGroup.MachineUUID) - return &machineuuid_; - ; -} - -} // namespace logtail - -#endif // LOGTAIL_RAWLOGGROUP_H diff --git a/core/unittest/log_pb/CMakeLists.txt b/core/unittest/log_pb/CMakeLists.txt index 1941bc0a39..6a7ad3019c 100644 --- a/core/unittest/log_pb/CMakeLists.txt +++ b/core/unittest/log_pb/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2022 iLogtail Authors +# Copyright 2024 iLogtail Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,8 +15,12 @@ cmake_minimum_required(VERSION 3.22) project(log_pb_unittest) -add_executable(log_pb_unittest PBUnittest.cpp) -target_link_libraries(log_pb_unittest ${UT_BASE_TARGET}) +if (ENABLE_ENTERPRISE) + add_executable(log_group_serializer_unittest LogGroupSerializerUnittest.cpp) + target_link_libraries(log_group_serializer_unittest ${UT_BASE_TARGET}) +endif () include(GoogleTest) -gtest_discover_tests(log_pb_unittest) +if (ENABLE_ENTERPRISE) + gtest_discover_tests(log_group_serializer_unittest) +endif () diff --git a/core/unittest/log_pb/PBUnittest.cpp b/core/unittest/log_pb/PBUnittest.cpp deleted file mode 100644 index 0e60ffcb96..0000000000 --- a/core/unittest/log_pb/PBUnittest.cpp +++ /dev/null @@ -1,405 +0,0 @@ -// Copyright 2022 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include "unittest/Unittest.h" -#include "protobuf/sls/RawLogGroup.h" -#include "protobuf/sls/sls_logs.pb.h" - -using namespace std; -using namespace sls_logs; - -namespace logtail { - -class PBUnittest : public ::testing::Test { -public: - string longLogValue10 = string(10 * 1024, 'c'); - string longLogValue100 = string(100 * 1024, 'd'); - string longLogValue1000 = string(1000 * 1024, 'e'); - string longLogValue10000 = string(10000 * 1024, 'f'); - - static void SetUpTestCase() // void Setup(); - {} - static void TearDownTestCase() // void CleanUp(); - {} - - void TestFullWrite() { - boost::match_results what; - boost::regex reg("(\\w+) (\\w+) (\\w+) (\\w+)"); - boost::regex_match("1 234 567 890xyz", what, reg, boost::match_default); - vector keys; - keys.push_back("key1"); - keys.push_back("key2"); - keys.push_back("key3"); - keys.push_back(""); - - auto now = GetCurrentLogtailTime(); - RawLog* rawLog = RawLog::AddLogFull(now.tv_sec, keys, what); - - LogGroup loggroup; - Log* log = loggroup.add_logs(); - Log_Content* kv = log->add_contents(); - kv->set_key("key1"); - kv->set_value("1"); - kv = log->add_contents(); - kv->set_key("key2"); - kv->set_value("234"); - kv = log->add_contents(); - kv->set_key("key3"); - kv->set_value("567"); - kv = log->add_contents(); - kv->set_key(""); - kv->set_value("890xyz"); - SetLogTime(log, now.tv_sec); - - string rawLogStr; - rawLog->AppendToString(&rawLogStr); - string logStr; - EXPECT_EQ(loggroup.AppendToString(&logStr), true); - - EXPECT_EQ(rawLogStr, logStr); - - delete rawLog; - } - - void TestNormalWrite() { - RawLog rawLog; - auto now = GetCurrentLogtailTime(); - rawLog.AddLogStart(now.tv_sec); - rawLog.AddKeyValue("key1", strlen("key1"), "value", strlen("value")); - rawLog.AddKeyValue("key2", strlen("key2"), "", 0); - rawLog.AddKeyValue("key3", strlen("key3"), "value3", strlen("value3")); - rawLog.AddKeyValue("key4", strlen("key4"), "value", strlen("value")); - rawLog.AddLogDone(); - - LogGroup loggroup; - Log* log = loggroup.add_logs(); - Log_Content* kv = log->add_contents(); - kv->set_key("key1"); - kv->set_value("value"); - kv = log->add_contents(); - kv->set_key("key2"); - kv->set_value(""); - kv = log->add_contents(); - kv->set_key("key3"); - kv->set_value("value3"); - kv = log->add_contents(); - kv->set_key("key4"); - kv->set_value("value"); - SetLogTime(log, now.tv_sec); - - string rawLogStr; - rawLog.AppendToString(&rawLogStr); - string logStr; - EXPECT_EQ(loggroup.AppendToString(&logStr), true); - EXPECT_EQ(rawLogStr, logStr); - } - - void TestBigLog() { - RawLog rawLog; - auto now = GetCurrentLogtailTime(); - rawLog.AddLogStart(now.tv_sec); - rawLog.AddKeyValue("key1", strlen("key1"), "value", strlen("value")); - rawLog.AddKeyValue("key2", strlen("key2"), longLogValue100.c_str(), longLogValue100.size()); - rawLog.AddKeyValue("key3", strlen("key3"), longLogValue10.c_str(), longLogValue10.size()); - rawLog.AddKeyValue("key4", strlen("key4"), longLogValue1000.c_str(), longLogValue1000.size()); - rawLog.AddLogDone(); - - LogGroup loggroup; - Log* log = loggroup.add_logs(); - Log_Content* kv = log->add_contents(); - kv->set_key("key1"); - kv->set_value("value"); - kv = log->add_contents(); - kv->set_key("key2"); - kv->set_value(longLogValue100); - kv = log->add_contents(); - kv->set_key("key3"); - kv->set_value(longLogValue10); - kv = log->add_contents(); - kv->set_key("key4"); - kv->set_value(longLogValue1000); - SetLogTime(log, now.tv_sec); - - string rawLogStr; - rawLog.AppendToString(&rawLogStr); - string logStr; - EXPECT_EQ(loggroup.AppendToString(&logStr), true); - EXPECT_EQ(rawLogStr, logStr); - } - - void TestIterator() { - RawLog rawLog; - int32_t nowTime = time(NULL); - rawLog.AddLogStart(nowTime); - rawLog.AddKeyValue("key1", strlen("key1"), "value1", strlen("value1")); - rawLog.AddKeyValue("key2", strlen("key2"), "value2", strlen("value2")); - rawLog.AddKeyValue("key3", strlen("key3"), "value3", strlen("value3")); - rawLog.AddKeyValue("key4", strlen("key4"), "value4", strlen("value4")); - rawLog.AddLogDone(); - - RawLog::iterator iter = rawLog.GetIterator(); - const char* key = NULL; - uint32_t keyLen = 0; - const char* value = NULL; - uint32_t valueLen = 0; - int i = 0; - while (rawLog.NextKeyValue(iter, key, keyLen, value, valueLen)) { - ++i; - string keyStr(key, keyLen); - string valueStr(value, valueLen); - string expectKey = "key" + std::to_string(i); - string expectValue = "value" + std::to_string(i); - EXPECT_EQ(keyStr, expectKey); - EXPECT_EQ(valueStr, expectValue); - } - } - - void TestUpdateDeleteAppend() { - RawLog rawLog; - auto now = GetCurrentLogtailTime(); - rawLog.AddLogStart(now.tv_sec); - rawLog.AddKeyValue("key1", strlen("key1"), "value1", strlen("value1")); - rawLog.AddKeyValue("k", strlen("k"), "value", strlen("value")); - rawLog.AddKeyValue("k", strlen("k"), "value", strlen("value")); - rawLog.AddKeyValue("k", strlen("k"), "value", strlen("value")); - rawLog.AddLogDone(); - - RawLog::iterator iter = rawLog.GetIterator(); - const char* key = NULL; - uint32_t keyLen = 0; - const char* value = NULL; - uint32_t valueLen = 0; - { - EXPECT_TRUE(rawLog.NextKeyValue(iter, key, keyLen, value, valueLen)); - string keyStr(key, keyLen); - string valueStr(value, valueLen); - EXPECT_EQ(keyStr, string("key1")); - EXPECT_EQ(valueStr, string("value1")); - } - { - EXPECT_TRUE(rawLog.NextKeyValue(iter, key, keyLen, value, valueLen)); - rawLog.UpdateKeyValue(iter, "key2", strlen("key2"), "value2", strlen("value2")); - } - { - EXPECT_TRUE(rawLog.NextKeyValue(iter, key, keyLen, value, valueLen)); - rawLog.DeleteKeyValue(iter); - } - { - EXPECT_TRUE(rawLog.NextKeyValue(iter, key, keyLen, value, valueLen)); - rawLog.UpdateKeyValue(iter, "key3", strlen("key3"), "value3", strlen("value3")); - } - EXPECT_FALSE(rawLog.NextKeyValue(iter, key, keyLen, value, valueLen)); - rawLog.AppendKeyValue("key4", strlen("key4"), "value4", strlen("value4")); - - LogGroup loggroup; - Log* log = loggroup.add_logs(); - Log_Content* kv = log->add_contents(); - kv->set_key("key1"); - kv->set_value("value1"); - kv = log->add_contents(); - kv->set_key("key2"); - kv->set_value("value2"); - kv = log->add_contents(); - kv->set_key("key3"); - kv->set_value("value3"); - kv = log->add_contents(); - kv->set_key("key4"); - kv->set_value("value4"); - SetLogTime(log, now.tv_sec); - - string rawLogStr; - rawLog.AppendToString(&rawLogStr); - string logStr; - EXPECT_EQ(loggroup.AppendToString(&logStr), true); - EXPECT_EQ(rawLogStr, logStr); - } - - void TestLogGroup() { - RawLog* pRawLog = new RawLog(); - RawLog& rawLog = *pRawLog; - auto now = GetCurrentLogtailTime(); - rawLog.AddLogStart(now.tv_sec); - rawLog.AddKeyValue("key1", strlen("key1"), "value", strlen("value")); - rawLog.AddKeyValue("key2", strlen("key2"), "", 0); - rawLog.AddKeyValue("key3", strlen("key3"), "value3", strlen("value3")); - rawLog.AddKeyValue("key4", strlen("key4"), "value", strlen("value")); - rawLog.AddLogDone(); - - LogGroup loggroup; - Log* log = loggroup.add_logs(); - Log_Content* kv = log->add_contents(); - kv->set_key("key1"); - kv->set_value("value"); - kv = log->add_contents(); - kv->set_key("key2"); - kv->set_value(""); - kv = log->add_contents(); - kv->set_key("key3"); - kv->set_value("value3"); - kv = log->add_contents(); - kv->set_key("key4"); - kv->set_value("value"); - SetLogTime(log, now.tv_sec); - - - RawLogGroup* pRawLogGroup = new RawLogGroup; - pRawLogGroup->set_source("192.168.1.1"); - loggroup.set_source("192.168.1.1"); - - pRawLogGroup->set_category("logstore"); - loggroup.set_category("logstore"); - - pRawLogGroup->set_machineuuid("123-456"); - loggroup.set_machineuuid("123-456"); - - pRawLogGroup->set_topic("topic"); - loggroup.set_topic("topic"); - - pRawLogGroup->add_logtags("tagkey1", "tagvalue1"); - pRawLogGroup->add_logtags("tagkey2", "tagvalue2"); - - auto tag1 = loggroup.add_logtags(); - tag1->set_key("tagkey1"); - tag1->set_value("tagvalue1"); - - auto tag2 = loggroup.add_logtags(); - tag2->set_key("tagkey2"); - tag2->set_value("tagvalue2"); - - string rawLogStr; - pRawLogGroup->add_logs(pRawLog); - pRawLogGroup->AppendToString(&rawLogStr); - string logStr; - EXPECT_EQ(loggroup.AppendToString(&logStr), true); - EXPECT_EQ(rawLogStr, logStr); - delete pRawLogGroup; - } - - void TestNoOptionLogGroup() { - RawLog* pRawLog = new RawLog(); - RawLog& rawLog = *pRawLog; - auto now = GetCurrentLogtailTime(); - rawLog.AddLogStart(now.tv_sec); - rawLog.AddKeyValue("key1", strlen("key1"), "value", strlen("value")); - rawLog.AddKeyValue("key2", strlen("key2"), "", 0); - rawLog.AddKeyValue("key3", strlen("key3"), "value3", strlen("value3")); - rawLog.AddKeyValue("key4", strlen("key4"), "value", strlen("value")); - rawLog.AddLogDone(); - - LogGroup loggroup; - Log* log = loggroup.add_logs(); - Log_Content* kv = log->add_contents(); - kv->set_key("key1"); - kv->set_value("value"); - kv = log->add_contents(); - kv->set_key("key2"); - kv->set_value(""); - kv = log->add_contents(); - kv->set_key("key3"); - kv->set_value("value3"); - kv = log->add_contents(); - kv->set_key("key4"); - kv->set_value("value"); - SetLogTime(log, now.tv_sec); - - - RawLogGroup* pRawLogGroup = new RawLogGroup; - pRawLogGroup->set_source("192.168.1.1"); - loggroup.set_source("192.168.1.1"); - - pRawLogGroup->set_topic("topic"); - loggroup.set_topic("topic"); - - pRawLogGroup->add_logtags("tagkey1", "tagvalue1"); - pRawLogGroup->add_logtags("tagkey2", "tagvalue2"); - - auto tag1 = loggroup.add_logtags(); - tag1->set_key("tagkey1"); - tag1->set_value("tagvalue1"); - - auto tag2 = loggroup.add_logtags(); - tag2->set_key("tagkey2"); - tag2->set_value("tagvalue2"); - - string rawLogStr; - pRawLogGroup->add_logs(pRawLog); - pRawLogGroup->AppendToString(&rawLogStr); - string logStr; - EXPECT_EQ(loggroup.AppendToString(&logStr), true); - EXPECT_EQ(rawLogStr, logStr); - delete pRawLogGroup; - } - - void TestMultiLog() { - auto now = GetCurrentLogtailTime(); - string rawLogStr; - auto c1 = clock(); - for (int i = 0; i < 1000000; ++i) { - RawLog rawLog; - - rawLog.AddLogStart(now.tv_sec); - rawLog.AddKeyValue("key1", strlen("key1"), "value", strlen("value")); - rawLog.AddKeyValue("key2", strlen("key2"), "", 0); - rawLog.AddKeyValue("key3", strlen("key3"), "value3", strlen("value3")); - rawLog.AddKeyValue("key4", strlen("key4"), "value", strlen("value")); - rawLog.AddLogDone(); - rawLogStr.clear(); - rawLog.AppendToString(&rawLogStr); - } - auto c2 = clock(); - string logStr; - for (int i = 0; i < 1000000; ++i) { - LogGroup loggroup; - Log* log = loggroup.add_logs(); - Log_Content* kv = log->add_contents(); - kv->set_key("key1"); - kv->set_value("value"); - kv = log->add_contents(); - kv->set_key("key2"); - kv->set_value(""); - kv = log->add_contents(); - kv->set_key("key3"); - kv->set_value("value3"); - kv = log->add_contents(); - kv->set_key("key4"); - kv->set_value("value"); - SetLogTime(log, now.tv_sec); - logStr.clear(); - loggroup.AppendToString(&logStr); - } - auto c3 = clock(); - EXPECT_GT((c3 - c2) / 5, c2 - c1); - printf("%d %d \n", (int)(c3 - c2), (int)(c2 - c1)); - EXPECT_EQ(rawLogStr, logStr); - } -}; - -APSARA_UNIT_TEST_CASE(PBUnittest, TestFullWrite, 0); -APSARA_UNIT_TEST_CASE(PBUnittest, TestNormalWrite, 0); -APSARA_UNIT_TEST_CASE(PBUnittest, TestIterator, 0); -APSARA_UNIT_TEST_CASE(PBUnittest, TestUpdateDeleteAppend, 0); -APSARA_UNIT_TEST_CASE(PBUnittest, TestBigLog, 0); -APSARA_UNIT_TEST_CASE(PBUnittest, TestLogGroup, 0); -APSARA_UNIT_TEST_CASE(PBUnittest, TestNoOptionLogGroup, 0); -APSARA_UNIT_TEST_CASE(PBUnittest, TestMultiLog, 0); - -} // namespace logtail - - -int main(int argc, char** argv) { - logtail::Logger::Instance().InitGlobalLoggers(); - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/core/unittest/pipeline/PipelineUnittest.cpp b/core/unittest/pipeline/PipelineUnittest.cpp index c77a1e1a7a..95fe13faee 100644 --- a/core/unittest/pipeline/PipelineUnittest.cpp +++ b/core/unittest/pipeline/PipelineUnittest.cpp @@ -2762,6 +2762,7 @@ void PipelineUnittest::TestSend() const { { // all valid vector group; + group.back().AddLogEvent(); group.emplace_back(make_shared()); APSARA_TEST_TRUE(pipeline.Send(std::move(group))); } @@ -2771,6 +2772,7 @@ void PipelineUnittest::TestSend() const { = false; vector group; group.emplace_back(make_shared()); + group.back().AddLogEvent(); APSARA_TEST_FALSE(pipeline.Send(std::move(group))); const_cast(static_cast(pipeline.mFlushers[0]->GetPlugin()))->mIsValid = true;