Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/main' into kql-escape-char-fix
Browse files Browse the repository at this point in the history
  • Loading branch information
gibber9809 committed Nov 12, 2024
2 parents 56b89c9 + f3b1cf6 commit a62542b
Show file tree
Hide file tree
Showing 67 changed files with 5,268 additions and 3,480 deletions.
12 changes: 8 additions & 4 deletions .github/workflows/clp-core-build-macos.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ on:
- "components/core/CMakeLists.txt"
- "components/core/src/**"
- "components/core/tests/**"
- "components/core/tools/scripts/lib_install/macos-12/**"
- "components/core/tools/scripts/lib_install/macos/**"
- "components/core/tools/scripts/deps-download/**"
- "components/core/tools/scripts/utils/build-and-run-unit-tests.py"
- "deps-tasks.yml"
Expand All @@ -21,12 +21,15 @@ on:
- "components/core/CMakeLists.txt"
- "components/core/src/**"
- "components/core/tests/**"
- "components/core/tools/scripts/lib_install/macos-12/**"
- "components/core/tools/scripts/lib_install/macos/**"
- "components/core/tools/scripts/deps-download/**"
- "components/core/tools/scripts/utils/build-and-run-unit-tests.py"
- "deps-tasks.yml"
- "Taskfile.yml"
- "tools/scripts/deps-download/**"
schedule:
# Run daily at 00:15 UTC (the 15 is to avoid periods of high load)
- cron: "15 0 * * *"
workflow_dispatch:

concurrency:
Expand All @@ -38,8 +41,9 @@ jobs:
build-macos:
strategy:
matrix:
runner: ["macos-13", "macos-14"]
use_shared_libs: [true, false]
runs-on: "macos-12"
runs-on: "${{matrix.runner}}"
steps:
- uses: "actions/checkout@v4"
with:
Expand All @@ -55,7 +59,7 @@ jobs:
rm -f /usr/local/bin/python3*
- name: "Install dependencies"
run: "./components/core/tools/scripts/lib_install/macos-12/install-all.sh"
run: "./components/core/tools/scripts/lib_install/macos/install-all.sh"

- run: "./tools/scripts/deps-download/init.sh"
shell: "bash"
Expand Down
7 changes: 5 additions & 2 deletions .github/workflows/clp-core-build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ on:
- "deps-tasks.yml"
- "Taskfile.yml"
- "tools/scripts/deps-download/**"
- "!components/core/tools/scripts/lib_install/macos-12/**"
- "!components/core/tools/scripts/lib_install/macos/**"
push:
paths:
- ".github/actions/clp-core-build/action.yaml"
Expand All @@ -22,7 +22,10 @@ on:
- "deps-tasks.yml"
- "Taskfile.yml"
- "tools/scripts/deps-download/**"
- "!components/core/tools/scripts/lib_install/macos-12/**"
- "!components/core/tools/scripts/lib_install/macos/**"
schedule:
# Run daily at 00:15 UTC (the 15 is to avoid periods of high load)
- cron: "15 0 * * *"
workflow_dispatch:

env:
Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/clp-docs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@ name: "clp-docs"
on:
pull_request:
push:
schedule:
# Run daily at 00:15 UTC (the 15 is to avoid periods of high load)
- cron: "15 0 * * *"
workflow_dispatch:

concurrency:
Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/clp-execution-image-build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ on:
- ".github/actions/clp-execution-image-build/action.yaml"
- ".github/workflows/clp-execution-image-build.yaml"
- "tools/docker-images/**/*"
schedule:
# Run daily at 00:15 UTC (the 15 is to avoid periods of high load)
- cron: "15 0 * * *"
workflow_dispatch:

concurrency:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/clp-lint.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ on:
pull_request:
push:
schedule:
# Run at midnight UTC every day with 15 minutes delay added to avoid high load periods
# Run daily at 00:15 UTC (the 15 is to avoid periods of high load)
- cron: "15 0 * * *"
workflow_dispatch:

Expand Down
6 changes: 3 additions & 3 deletions Taskfile.yml
Original file line number Diff line number Diff line change
Expand Up @@ -228,10 +228,10 @@ tasks:
- "client/src/**/*.css"
- "client/src/**/*.jsx"
- "client/src/webpack.config.js"
- "yscope-log-viewer/.babelrc"
- "yscope-log-viewer/customized-packages/**/*"
- "yscope-log-viewer/package.json"
- "yscope-log-viewer/public/**/*"
- "yscope-log-viewer/src/**/*"
- "yscope-log-viewer/tsconfig.json"
- "yscope-log-viewer/webpack.common.js"
- "yscope-log-viewer/webpack.prod.js"
dir: "components/log-viewer-webui"
Expand Down Expand Up @@ -348,7 +348,7 @@ tasks:
- "mkdir -p '{{.OUTPUT_TMP_DIR}}'"
- >-
curl --fail --location --show-error
"{{.URL_PREFIX}}/{{.TAR_NAME}}"
"{{trimSuffix "/" .URL_PREFIX}}/{{.TAR_NAME}}"
--output "{{.TAR_PATH}}"
- "tar xf '{{.TAR_PATH}}' --directory '{{.OUTPUT_TMP_DIR}}'"
- "mv '{{.EXTRACTED_DIR}}' '{{.OUTPUT_DIR}}'"
Expand Down
2 changes: 1 addition & 1 deletion components/core/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,7 @@ else()
endif()

# Find and setup msgpack
find_package(msgpack-cxx 6.0.0 REQUIRED)
find_package(msgpack-cxx 7.0.0 REQUIRED)
if(msgpack-cxx_FOUND)
message(STATUS "Found msgpack-cxx ${msgpack-cxx_VERSION}")
else()
Expand Down
22 changes: 22 additions & 0 deletions components/core/cmake/Modules/FindMariaDBClient.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,28 @@ include(cmake/Modules/FindLibraryDependencies.cmake)
find_package(PkgConfig)
pkg_check_modules(mariadbclient_PKGCONF QUIET "lib${mariadbclient_LIBNAME}")

if(NOT mariadbclient_PKGCONF_FOUND AND APPLE)
execute_process(
COMMAND brew --prefix mariadb-connector-c
RESULT_VARIABLE mariadbclient_BREW_RESULT
OUTPUT_VARIABLE mariadbclient_MACOS_PREFIX
)
if(NOT mariadbclient_BREW_RESULT EQUAL 0)
message(
FATAL_ERROR
"pkg-config cannot find ${mariadbclient_LIBNAME} and mariadb-connector-c isn't"
" installed via Homebrew"
)
endif()
string(STRIP "${mariadbclient_MACOS_PREFIX}" mariadbclient_MACOS_PREFIX)
list(PREPEND CMAKE_PREFIX_PATH ${mariadbclient_MACOS_PREFIX})
pkg_check_modules(mariadbclient_PKGCONF QUIET "lib${mariadbclient_LIBNAME}")
endif()

if(NOT mariadbclient_PKGCONF_FOUND)
message(FATAL_ERROR "pkg-config cannot find ${mariadbclient_LIBNAME}")
endif()

# Set include directory
find_path(MariaDBClient_INCLUDE_DIR mysql.h
HINTS ${mariadbclient_PKGCONF_INCLUDEDIR}
Expand Down
62 changes: 57 additions & 5 deletions components/core/src/clp/CurlDownloadHandler.cpp
Original file line number Diff line number Diff line change
@@ -1,13 +1,22 @@
#include "CurlDownloadHandler.hpp"

#include <algorithm>
#include <array>
#include <cctype>
#include <chrono>
#include <cstddef>
#include <memory>
#include <optional>
#include <string>
#include <string_view>
#include <unordered_map>
#include <unordered_set>
#include <utility>

#include <curl/curl.h>
#include <fmt/core.h>

#include "ErrorCode.hpp"

namespace clp {
CurlDownloadHandler::CurlDownloadHandler(
Expand All @@ -19,7 +28,8 @@ CurlDownloadHandler::CurlDownloadHandler(
size_t offset,
bool disable_caching,
std::chrono::seconds connection_timeout,
std::chrono::seconds overall_timeout
std::chrono::seconds overall_timeout,
std::optional<std::unordered_map<std::string, std::string>> const& http_header_kv_pairs
)
: m_error_msg_buf{std::move(error_msg_buf)} {
if (nullptr != m_error_msg_buf) {
Expand Down Expand Up @@ -48,13 +58,55 @@ CurlDownloadHandler::CurlDownloadHandler(
m_easy_handle.set_option(CURLOPT_TIMEOUT, static_cast<long>(overall_timeout.count()));

// Set up http headers
constexpr std::string_view cRangeHeaderName{"range"};
constexpr std::string_view cCacheControlHeaderName{"cache-control"};
constexpr std::string_view cPragmaHeaderName{"pragma"};
std::unordered_set<std::string_view> const reserved_headers{
cRangeHeaderName,
cCacheControlHeaderName,
cPragmaHeaderName
};
if (0 != offset) {
std::string const range{"Range: bytes=" + std::to_string(offset) + "-"};
m_http_headers.append(range);
m_http_headers.append(fmt::format("{}: bytes={}-", cRangeHeaderName, offset));
}
if (disable_caching) {
m_http_headers.append("Cache-Control: no-cache");
m_http_headers.append("Pragma: no-cache");
m_http_headers.append(fmt::format("{}: no-cache", cCacheControlHeaderName));
m_http_headers.append(fmt::format("{}: no-cache", cPragmaHeaderName));
}
if (http_header_kv_pairs.has_value()) {
for (auto const& [key, value] : http_header_kv_pairs.value()) {
// HTTP header field-name (key) is case-insensitive:
// https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2
// Therefore, we convert keys to lowercase for comparison with the reserved keys.
// NOTE: We do not check for duplicate keys due to case insensitivity, leaving duplicate
// handling to the server.
auto lower_key{key};
std::transform(
lower_key.begin(),
lower_key.end(),
lower_key.begin(),
[](unsigned char c) -> char {
// Implicitly cast the input character into `unsigned char` to avoid UB:
// https://en.cppreference.com/w/cpp/string/byte/tolower
return static_cast<char>(std::tolower(c));
}
);
if (reserved_headers.contains(lower_key) || value.ends_with("\r\n")) {
throw CurlOperationFailed(
ErrorCode_Failure,
__FILE__,
__LINE__,
CURLE_BAD_FUNCTION_ARGUMENT,
fmt::format(
"`CurlDownloadHandler` failed to construct with the following "
"invalid header: {}:{}",
key,
value
)
);
}
m_http_headers.append(fmt::format("{}: {}", key, value));
}
}
if (false == m_http_headers.is_empty()) {
m_easy_handle.set_option(CURLOPT_HTTPHEADER, m_http_headers.get_raw_list());
Expand Down
10 changes: 9 additions & 1 deletion components/core/src/clp/CurlDownloadHandler.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,10 @@
#include <chrono>
#include <cstddef>
#include <memory>
#include <optional>
#include <string>
#include <string_view>
#include <unordered_map>

#include <curl/curl.h>

Expand Down Expand Up @@ -53,6 +56,9 @@ class CurlDownloadHandler {
* Doc: https://curl.se/libcurl/c/CURLOPT_CONNECTTIMEOUT.html
* @param overall_timeout Maximum time that the transfer may take. Note that this includes
* `connection_timeout`. Doc: https://curl.se/libcurl/c/CURLOPT_TIMEOUT.html
* @param http_header_kv_pairs Key-value pairs representing HTTP headers to pass to the server
* in the download request. Doc: https://curl.se/libcurl/c/CURLOPT_HTTPHEADER.html
* @throw CurlOperationFailed if an error occurs.
*/
explicit CurlDownloadHandler(
std::shared_ptr<ErrorMsgBuf> error_msg_buf,
Expand All @@ -63,7 +69,9 @@ class CurlDownloadHandler {
size_t offset = 0,
bool disable_caching = false,
std::chrono::seconds connection_timeout = cDefaultConnectionTimeout,
std::chrono::seconds overall_timeout = cDefaultOverallTimeout
std::chrono::seconds overall_timeout = cDefaultOverallTimeout,
std::optional<std::unordered_map<std::string, std::string>> const& http_header_kv_pairs
= std::nullopt
);

// Disable copy/move constructors/assignment operators
Expand Down
16 changes: 13 additions & 3 deletions components/core/src/clp/NetworkReader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@
#include <memory>
#include <mutex>
#include <optional>
#include <string>
#include <string_view>
#include <unordered_map>
#include <utility>

#include <curl/curl.h>

Expand Down Expand Up @@ -118,7 +121,8 @@ NetworkReader::NetworkReader(
std::chrono::seconds overall_timeout,
std::chrono::seconds connection_timeout,
size_t buffer_pool_size,
size_t buffer_size
size_t buffer_size,
std::optional<std::unordered_map<std::string, std::string>> http_header_kv_pairs
)
: m_src_url{src_url},
m_offset{offset},
Expand All @@ -130,7 +134,12 @@ NetworkReader::NetworkReader(
for (size_t i = 0; i < m_buffer_pool_size; ++i) {
m_buffer_pool.emplace_back(m_buffer_size);
}
m_downloader_thread = std::make_unique<DownloaderThread>(*this, offset, disable_caching);
m_downloader_thread = std::make_unique<DownloaderThread>(
*this,
offset,
disable_caching,
std::move(http_header_kv_pairs)
);
m_downloader_thread->start();
}

Expand Down Expand Up @@ -215,7 +224,8 @@ auto NetworkReader::DownloaderThread::thread_method() -> void {
m_offset,
m_disable_caching,
m_reader.m_connection_timeout,
m_reader.m_overall_timeout
m_reader.m_overall_timeout,
m_http_header_kv_pairs
};
auto const ret_code{curl_handler.perform()};
// Enqueue the last filled buffer, if any
Expand Down
21 changes: 18 additions & 3 deletions components/core/src/clp/NetworkReader.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
#include <span>
#include <string>
#include <string_view>
#include <unordered_map>
#include <utility>
#include <vector>

#include <curl/curl.h>
Expand Down Expand Up @@ -94,6 +96,8 @@ class NetworkReader : public ReaderInterface {
* Doc: https://curl.se/libcurl/c/CURLOPT_CONNECTTIMEOUT.html
* @param buffer_pool_size The required number of buffers in the buffer pool.
* @param buffer_size The size of each buffer in the buffer pool.
* @param http_header_kv_pairs Key-value pairs representing HTTP headers to pass to the server
* in the download request. Doc: https://curl.se/libcurl/c/CURLOPT_HTTPHEADER.html
*/
explicit NetworkReader(
std::string_view src_url,
Expand All @@ -103,7 +107,9 @@ class NetworkReader : public ReaderInterface {
std::chrono::seconds connection_timeout
= CurlDownloadHandler::cDefaultConnectionTimeout,
size_t buffer_pool_size = cDefaultBufferPoolSize,
size_t buffer_size = cDefaultBufferSize
size_t buffer_size = cDefaultBufferSize,
std::optional<std::unordered_map<std::string, std::string>> http_header_kv_pairs
= std::nullopt
);

// Destructor
Expand Down Expand Up @@ -242,11 +248,19 @@ class NetworkReader : public ReaderInterface {
* @param reader
* @param offset Index of the byte at which to start the download.
* @param disable_caching Whether to disable caching.
* @param http_header_kv_pairs Key-value pairs representing HTTP headers to pass to the
* server in the download request. Doc: https://curl.se/libcurl/c/CURLOPT_HTTPHEADER.html
*/
DownloaderThread(NetworkReader& reader, size_t offset, bool disable_caching)
DownloaderThread(
NetworkReader& reader,
size_t offset,
bool disable_caching,
std::optional<std::unordered_map<std::string, std::string>> http_header_kv_pairs
)
: m_reader{reader},
m_offset{offset},
m_disable_caching{disable_caching} {}
m_disable_caching{disable_caching},
m_http_header_kv_pairs{std::move(http_header_kv_pairs)} {}

private:
// Methods implementing `clp::Thread`
Expand All @@ -255,6 +269,7 @@ class NetworkReader : public ReaderInterface {
NetworkReader& m_reader;
size_t m_offset{0};
bool m_disable_caching{false};
std::optional<std::unordered_map<std::string, std::string>> m_http_header_kv_pairs;
};

/**
Expand Down
Loading

0 comments on commit a62542b

Please sign in to comment.