From 204ae95b2aae880746ac3ca6a04eb60fb100160d Mon Sep 17 00:00:00 2001 From: "zhuangbowei.zbw" Date: Wed, 13 Dec 2023 23:31:11 +0800 Subject: [PATCH 1/5] [bugfix] basic/bearer auth may coexist Signed-off-by: zhuangbowei.zbw --- src/overlaybd/base64.h | 8 +++++--- src/overlaybd/registryfs/registryfs.cpp | 7 ++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/overlaybd/base64.h b/src/overlaybd/base64.h index abc45996..b32a0258 100644 --- a/src/overlaybd/base64.h +++ b/src/overlaybd/base64.h @@ -1,3 +1,5 @@ +#pragma once + #ifndef _BASE64_H_ #define _BASE64_H_ @@ -13,7 +15,7 @@ static inline bool is_base64(BYTE c) { return (isalnum(c) || (c == '+') || (c == '/')); } -std::string base64_encode(BYTE const *buf, unsigned int bufLen) { +static inline std::string base64_encode(BYTE const *buf, unsigned int bufLen) { std::string ret; int i = 0; int j = 0; @@ -53,7 +55,7 @@ std::string base64_encode(BYTE const *buf, unsigned int bufLen) { return ret; } -std::string base64_decode(std::string const &encoded_string) { +static inline std::string base64_decode(std::string const &encoded_string) { int in_len = encoded_string.size(); int i = 0; int j = 0; @@ -96,4 +98,4 @@ std::string base64_decode(std::string const &encoded_string) { return ret; } -#endif \ No newline at end of file +#endif diff --git a/src/overlaybd/registryfs/registryfs.cpp b/src/overlaybd/registryfs/registryfs.cpp index 70719dfb..975a235a 100644 --- a/src/overlaybd/registryfs/registryfs.cpp +++ b/src/overlaybd/registryfs/registryfs.cpp @@ -14,6 +14,7 @@ limitations under the License. */ #include "registryfs.h" +#include "../base64.h" #include #include @@ -46,6 +47,7 @@ using namespace photon::fs; static const estring kDockerRegistryAuthChallengeKeyValuePrefix = "www-authenticate"; static const estring kAuthHeaderKey = "Authorization"; static const estring kBearerAuthPrefix = "Bearer "; +static const estring kBasicAuthPrefix = "Basic "; static const estring kDockerRegistryBlobReaderFailPrefix = "DockerRegistryBolbReader Failure: "; static const uint64_t kMinimalTokenLife = 30L * 1000 * 1000; // token lives atleast 30s static const uint64_t kMinimalAUrlLife = 300L * 1000 * 1000; // actual_url lives atleast 300s @@ -334,7 +336,10 @@ class RegistryFSImpl : public RegistryFS { DEFER({ release_cURL(req); }); photon::net::StringWriter writer; if (!username.empty()) { - req->set_user_passwd(username.c_str(), password.c_str()).set_redirect(3); + // req->set_user_passwd(username.c_str(), password.c_str()).set_redirect(3); + std::string basic_auth = username + ":" + password; + std::string encoded = base64_encode((const BYTE*) basic_auth.c_str(), basic_auth.length()); + req->append_header(kAuthHeaderKey, kBasicAuthPrefix + encoded); } auto ret = req->GET(auth_url, &writer, tmo.timeout_us()); From 9b873fde5066d13426555c8c39821997f7ef64f5 Mon Sep 17 00:00:00 2001 From: "zhuangbowei.zbw" Date: Wed, 13 Dec 2023 23:46:57 +0800 Subject: [PATCH 2/5] [NIT] minor fixes Signed-off-by: zhuangbowei.zbw --- CMakeLists.txt | 2 +- README.md | 2 +- src/main.cpp | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 223e274b..10fa7e20 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.15) +cmake_minimum_required(VERSION 3.14) project( overlaybd diff --git a/README.md b/README.md index a85b3130..8bc28d34 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ It is better to run `overlaybd-tcmu` as a service so that it can be restarted af To build overlaybd from source code, the following dependencies are required: -* CMake >= 3.15 +* CMake >= 3.14 * gcc/g++ >= 7 diff --git a/src/main.cpp b/src/main.cpp index 4f4f9f34..982fa049 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -32,6 +32,7 @@ #include #include #include +#include class TCMUDevLoop; @@ -401,6 +402,7 @@ void sigint_handler(int signal = SIGINT) { int main(int argc, char **argv) { mallopt(M_TRIM_THRESHOLD, 128 * 1024); + prctl(PR_SET_THP_DISABLE, 1); photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_DEFAULT); photon::block_all_signal(); From e6b9847234ae13d3b58e6bdac114349df9667f1a Mon Sep 17 00:00:00 2001 From: "zhuangbowei.zbw" Date: Wed, 13 Dec 2023 22:59:37 +0800 Subject: [PATCH 3/5] [feat] support client cert Signed-off-by: zhuangbowei.zbw --- CMake/Findphoton.cmake | 2 +- README.md | 2 + src/config.h | 8 ++ src/image_service.cpp | 5 +- src/overlaybd/registryfs/registryfs.cpp | 26 ++++-- src/overlaybd/registryfs/registryfs.h | 12 ++- src/overlaybd/registryfs/registryfs_v2.cpp | 98 ++++++++++++++++++---- 7 files changed, 124 insertions(+), 29 deletions(-) diff --git a/CMake/Findphoton.cmake b/CMake/Findphoton.cmake index 02927187..79b56972 100644 --- a/CMake/Findphoton.cmake +++ b/CMake/Findphoton.cmake @@ -5,7 +5,7 @@ set(PHOTON_ENABLE_EXTFS ON) FetchContent_Declare( photon GIT_REPOSITORY https://github.com/alibaba/PhotonLibOS.git - GIT_TAG v0.6.14 + GIT_TAG v0.6.15 ) if(BUILD_TESTING) diff --git a/README.md b/README.md index 8bc28d34..f257a486 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,8 @@ Default configure file `overlaybd.json` is installed to `/etc/overlaybd/`. | auditPath | The path for audit file, `/var/log/overlaybd-audit.log` is the default value. | | registryFsVersion | registry client version, 'v1' libcurl based, 'v2' is photon http based. 'v2' is the default value. | | prefetchConfig.concurrency | Prefetch concurrency for reloading trace, `16` is default | +| certConfig.certFile | The path for SSL/TLS client certificate file | +| certConfig.keyFile | The path for SSL/TLS client key file | > NOTE: `download` is the config for background downloading. After an overlaybd device is lauched, a background task will be running to fetch the whole blobs into local directories. After downloading, I/O requests are directed to local files. Unlike other options, download config is reloaded when a device launching. diff --git a/src/config.h b/src/config.h index 2d0f9db0..acbfc148 100644 --- a/src/config.h +++ b/src/config.h @@ -123,6 +123,13 @@ struct PrefetchConfig : public ConfigUtils::Config { APPCFG_PARA(concurrency, int, 16); }; +struct CertConfig : public ConfigUtils::Config { + APPCFG_CLASS + + APPCFG_PARA(certFile, std::string, ""); + APPCFG_PARA(keyFile, std::string, ""); +}; + struct GlobalConfig : public ConfigUtils::Config { APPCFG_CLASS @@ -145,6 +152,7 @@ struct GlobalConfig : public ConfigUtils::Config { APPCFG_PARA(gzipCacheConfig, GzipCacheConfig); APPCFG_PARA(logConfig, LogConfig); APPCFG_PARA(prefetchConfig, PrefetchConfig); + APPCFG_PARA(certConfig, CertConfig); }; struct AuthConfig : public ConfigUtils::Config { diff --git a/src/image_service.cpp b/src/image_service.cpp index 09544ca9..69657532 100644 --- a/src/image_service.cpp +++ b/src/image_service.cpp @@ -348,7 +348,8 @@ int ImageService::init() { registryfs_creator = new_registryfs_v2; global_fs.underlay_registryfs = registryfs_creator( - {this, &ImageService::reload_auth}, cafile, 30UL * 1000000); + {this, &ImageService::reload_auth}, cafile, 30UL * 1000000, + global_conf.certConfig().certFile().c_str(), global_conf.certConfig().keyFile().c_str()); if (global_fs.underlay_registryfs == nullptr) { LOG_ERROR_RETURN(0, -1, "create registryfs failed."); } @@ -512,4 +513,4 @@ ImageService *create_image_service(const char *config_path) { return nullptr; } return ret; -} \ No newline at end of file +} diff --git a/src/overlaybd/registryfs/registryfs.cpp b/src/overlaybd/registryfs/registryfs.cpp index 975a235a..3b2a8fa1 100644 --- a/src/overlaybd/registryfs/registryfs.cpp +++ b/src/overlaybd/registryfs/registryfs.cpp @@ -104,8 +104,10 @@ class RegistryFSImpl : public RegistryFS { return open(pathname, flags); // ignore mode } - RegistryFSImpl(PasswordCB callback, const char *caFile, uint64_t timeout) + RegistryFSImpl(PasswordCB callback, const char *caFile, uint64_t timeout, + const char *cert_file, const char *key_file) : m_callback(callback), m_caFile(caFile), m_timeout(timeout), + m_cert_file(cert_file), m_key_file(key_file), m_meta_size(kMinimalMetaLife), m_scope_token(kMinimalTokenLife), m_url_info(kMinimalAUrlLife) { } @@ -136,9 +138,6 @@ class RegistryFSImpl : public RegistryFS { { auto curl = get_cURL(); - if (photon::net::http::what_protocol(actual_url) == 2) { - curl->reset().clear_header().setopt(CURLOPT_SSL_VERIFYPEER, 0L); - } DEFER({ release_cURL(curl); }); curl->set_redirect(10); // set token if needed @@ -267,6 +266,8 @@ class RegistryFSImpl : public RegistryFS { estring m_accelerate; estring m_caFile; uint64_t m_timeout; + estring m_cert_file; + estring m_key_file; ObjectCache m_meta_size; ObjectCache m_scope_token; ObjectCache m_url_info; @@ -277,7 +278,14 @@ class RegistryFSImpl : public RegistryFS { auto curl = m_curl_pool.get(); mutex.unlock(); curl->reset_error(); - curl->reset().clear_header().set_cafile(m_caFile.c_str()); + curl->reset().clear_header().set_cafile(m_caFile.c_str()) + .setopt(CURLOPT_SSL_VERIFYPEER, 0L).setopt(CURLOPT_SSL_VERIFYHOST, 0L); + if (m_cert_file != "" && m_key_file != "" && + !::access(m_cert_file.c_str(), 0) && !::access(m_key_file.c_str(), 0)) { + LOG_DEBUG("curl with ` and `", m_cert_file.c_str(), m_key_file.c_str()); + curl->setopt(CURLOPT_SSLCERT, m_cert_file.c_str()); + curl->setopt(CURLOPT_SSLKEY, m_key_file.c_str()); + } return curl; }; @@ -336,7 +344,6 @@ class RegistryFSImpl : public RegistryFS { DEFER({ release_cURL(req); }); photon::net::StringWriter writer; if (!username.empty()) { - // req->set_user_passwd(username.c_str(), password.c_str()).set_redirect(3); std::string basic_auth = username + ":" + password; std::string encoded = base64_encode((const BYTE*) basic_auth.c_str(), basic_auth.length()); req->append_header(kAuthHeaderKey, kBasicAuthPrefix + encoded); @@ -520,9 +527,10 @@ inline IFile *RegistryFSImpl::open(const char *pathname, int) { return file; } -IFileSystem *new_registryfs_v1(PasswordCB callback, const char *caFile, - uint64_t timeout) { +IFileSystem *new_registryfs_v1(PasswordCB callback, const char *caFile, uint64_t timeout, + const char *cert_file, const char *key_file) { if (!callback) LOG_ERROR_RETURN(EINVAL, nullptr, "password callback not set"); - return new RegistryFSImpl(callback, caFile ? caFile : "", timeout); + return new RegistryFSImpl(callback, caFile ? caFile : "", timeout, + cert_file ? cert_file : "", key_file ? key_file : ""); } diff --git a/src/overlaybd/registryfs/registryfs.h b/src/overlaybd/registryfs/registryfs.h index 3414ff07..0bad0324 100644 --- a/src/overlaybd/registryfs/registryfs.h +++ b/src/overlaybd/registryfs/registryfs.h @@ -30,15 +30,21 @@ using PasswordCB = Delegate, const char *>; extern "C" { photon::fs::IFileSystem *new_registryfs_v1(PasswordCB callback, const char *caFile = nullptr, - uint64_t timeout = -1); + uint64_t timeout = -1, + const char *cert_file = nullptr, + const char *key_file = nullptr); photon::fs::IFileSystem *new_registryfs_v2(PasswordCB callback, const char *caFile = nullptr, - uint64_t timeout = -1); + uint64_t timeout = -1, + const char *cert_file = nullptr, + const char *key_file = nullptr); photon::fs::IFile* new_registry_uploader(photon::fs::IFile *lfile, std::string &upload_url, std::string &username, std::string &password, uint64_t timeout, - ssize_t upload_bs = -1); + ssize_t upload_bs = -1, + const char *cert_file = nullptr, + const char *key_file = nullptr); } diff --git a/src/overlaybd/registryfs/registryfs_v2.cpp b/src/overlaybd/registryfs/registryfs_v2.cpp index 6726fa05..693dd2c1 100644 --- a/src/overlaybd/registryfs/registryfs_v2.cpp +++ b/src/overlaybd/registryfs/registryfs_v2.cpp @@ -34,13 +34,16 @@ #include #include #include +#include #include #include +#include #include #include #include #include #include +#include using namespace photon::fs; using namespace photon::net::http; @@ -70,6 +73,8 @@ static std::unordered_map str_to_kvmap(estring &src) return ret; } +photon::net::TLSContext *new_tls_context_from_file(const char *cert_file, const char *key_file); + enum class UrlMode { Redirect, Self @@ -110,15 +115,19 @@ class RegistryFSImpl_v2 : public RegistryFS { return open(pathname, flags); // ignore mode } - RegistryFSImpl_v2(PasswordCB callback, const char *caFile, uint64_t timeout) - : m_callback(callback), m_caFile(caFile), m_timeout(timeout), + RegistryFSImpl_v2(PasswordCB callback, const char *caFile, uint64_t timeout, + photon::net::TLSContext *ctx) + : m_callback(callback), m_caFile(caFile), m_timeout(timeout), m_tls_ctx(ctx), m_meta_size(kMinimalMetaLife), m_scope_token(kMinimalTokenLife), m_url_info(kMinimalAUrlLife) { - m_client = new_http_client(); + + m_client = nullptr; + this->refresh_client(); } ~RegistryFSImpl_v2() { - delete m_client; + if (m_client) delete m_client; + if (m_tls_ctx) delete m_tls_ctx; } long get_data(const estring &url, off_t offset, size_t count, uint64_t timeout, HTTP_OP &op) { @@ -250,8 +259,10 @@ class RegistryFSImpl_v2 : public RegistryFS { } void refresh_client() { - delete m_client; - m_client = new_http_client(); + if (m_client) { + delete m_client; + } + m_client = new_http_client(nullptr, m_tls_ctx); } int refresh_token(const estring &url, estring &token) { @@ -273,7 +284,8 @@ class RegistryFSImpl_v2 : public RegistryFS { estring m_accelerate; estring m_caFile; uint64_t m_timeout; - photon::net::http::Client* m_client; + photon::net::TLSContext *m_tls_ctx; + photon::net::http::Client *m_client; ObjectCache m_meta_size; ObjectCache m_scope_token; ObjectCache m_url_info; @@ -476,10 +488,15 @@ inline IFile *RegistryFSImpl_v2::open(const char *pathname, int) { return file; } -IFileSystem *new_registryfs_v2(PasswordCB callback, const char *caFile, uint64_t timeout) { +IFileSystem *new_registryfs_v2(PasswordCB callback, const char *caFile, uint64_t timeout, + const char *cert_file, const char *key_file) { if (!callback) LOG_ERROR_RETURN(EINVAL, nullptr, "password callback not set"); - return new RegistryFSImpl_v2(callback, caFile ? caFile : "", timeout); + auto ctx = new_tls_context_from_file(cert_file, key_file); + if (!ctx) { + LOG_ERRNO_RETURN(0, nullptr, "failed to new tls context"); + } + return new RegistryFSImpl_v2(callback, caFile ? caFile : "", timeout, ctx); } class RegistryUploader : public VirtualFile { @@ -501,9 +518,10 @@ class RegistryUploader : public VirtualFile { estring m_token; RegistryUploader(IFile *lfile, std::string &upload_url, std::string &username, - std::string &password, uint64_t timeout = -1, ssize_t upload_bs = -1) + std::string &password, uint64_t timeout, ssize_t upload_bs, + photon::net::TLSContext *ctx) : m_local_file(lfile), m_origin_upload_url(upload_url), m_username(username), m_password(password), - m_timeout(timeout) { + m_timeout(timeout), m_tls_ctx(ctx) { if (upload_bs != -1) m_upload_chunk_size = upload_bs; SHA256_Init(&m_sha256_ctx); @@ -674,7 +692,8 @@ class RegistryUploader : public VirtualFile { int upload_thread() { photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_NONE); DEFER(photon::fini()); - m_upload_fs = new RegistryFSImpl_v2({this, &RegistryUploader::load_auth}, "", m_timeout); + m_upload_fs = new RegistryFSImpl_v2({this, &RegistryUploader::load_auth}, + "", m_timeout, m_tls_ctx); DEFER({ delete m_upload_fs; }); m_http_client_ts = photon::now; ::posix_memalign(&m_upload_buf, 4096, 1024 * 1024); @@ -768,14 +787,65 @@ class RegistryUploader : public VirtualFile { } LOG_ERROR_RETURN(0, -1, "failed to get upload url, code=`", op.status_code); } + +protected: + photon::net::TLSContext *m_tls_ctx; }; IFile *new_registry_uploader(IFile *lfile, std::string &upload_url, std::string &username, - std::string &password, uint64_t timeout, ssize_t upload_bs) { - auto ret = new RegistryUploader(lfile, upload_url, username, password, timeout, upload_bs); + std::string &password, uint64_t timeout, ssize_t upload_bs, + const char *cert_file, const char *key_file) { + auto ctx = new_tls_context_from_file(cert_file, key_file); + if (!ctx) { + LOG_ERRNO_RETURN(0, nullptr, "failed to new tls context"); + } + auto ret = new RegistryUploader(lfile, upload_url, username, password, + timeout, upload_bs, ctx); if (ret->init() < 0) { delete ret; return nullptr; } return ret; } + +photon::net::TLSContext *new_tls_context_from_file(const char *cert_file, const char *key_file) { + if (!cert_file || !key_file) + return photon::net::new_tls_context(nullptr, nullptr, nullptr); + + if (strcmp(cert_file, "") == 0 || strcmp(key_file, "") == 0) { + return photon::net::new_tls_context(nullptr, nullptr, nullptr); + } + + auto read_file = [](const char *fn, estring &target) -> int { + if (::access(fn, F_OK) != 0) { + LOG_ERRNO_RETURN(0, -1, "file ` not found", fn); + } + photon::fs::IFile *file = open_localfile_adaptor(fn, O_RDONLY, 0444); + if (file == nullptr) { + LOG_ERRNO_RETURN(0, -1, "failed to open file `", fn); + } + struct stat st; + int ret = file->fstat(&st); + if (ret) { + LOG_ERRNO_RETURN(0, ret, "failed to stat file `", fn); + } + char buf[4096]; + ret = file->pread(buf, st.st_size, 0); + if (ret != st.st_size) { + LOG_ERRNO_RETURN(0, -1, "failed to read file `", fn); + } + target = estring(buf); + return 0; + }; + + estring cert_str; + estring key_str; + + if (read_file(cert_file, cert_str)) { + LOG_ERRNO_RETURN(0, nullptr, "failed to read SSL/TLS client cert file `", cert_file); + } + if (read_file(key_file, key_str)) { + LOG_ERRNO_RETURN(0, nullptr, "failed to read SSL/TLS client key file `", key_file); + } + return photon::net::new_tls_context(cert_str.c_str(), key_str.c_str(), nullptr); +} From fb3e65fa0e02c282d35d731bcad5fc34f5a5faad Mon Sep 17 00:00:00 2001 From: "zhuangbowei.zbw" Date: Tue, 19 Dec 2023 18:07:14 +0800 Subject: [PATCH 4/5] [feat] fifo overlaybd-apply Signed-off-by: zhuangbowei.zbw --- src/tools/overlaybd-apply.cpp | 43 ++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/src/tools/overlaybd-apply.cpp b/src/tools/overlaybd-apply.cpp index 81010c37..0d2ff036 100644 --- a/src/tools/overlaybd-apply.cpp +++ b/src/tools/overlaybd-apply.cpp @@ -44,6 +44,40 @@ using namespace std; using namespace photon::fs; +class FIFOFile : public VirtualReadOnlyFile { +public: + IFile *m_fifo; + + FIFOFile(IFile *fifo): m_fifo(fifo) { } + ~FIFOFile() { delete m_fifo; } + + ssize_t read(void *buf, size_t count) override { + size_t left = count; + char *pos = (char *) buf; + LOG_DEBUG(VALUE(count)); + while (left > 0) { + ssize_t readn = m_fifo->read(pos, left); + if (readn < 0 || readn > (ssize_t) left) { + LOG_ERRNO_RETURN(0, -1, "failed to read fifo", VALUE(left), VALUE(readn)); + } + left -= (size_t) readn; + pos += readn; + LOG_DEBUG("fifo read", VALUE(readn)); + } + LOG_DEBUG(VALUE(left)); + return count - left; + } + + int fstat(struct stat *buf) override { + return m_fifo->fstat(buf); + } + + UNIMPLEMENTED_POINTER(IFileSystem *filesystem() override); + UNIMPLEMENTED(off_t lseek(off_t offset, int whence) override); + UNIMPLEMENTED(ssize_t readv(const struct iovec *iov, int iovcnt) override); + UNIMPLEMENTED(ssize_t preadv(const struct iovec *iov, int iovcnt, off_t offset) override); +}; + int main(int argc, char **argv) { std::string image_config_path, input_path, gz_index_path, config_path, sha256_checksum; string tarheader; @@ -93,7 +127,14 @@ int main(int argc, char **argv) { auto tarf = open_file(input_path.c_str(), O_RDONLY, 0666); DEFER(delete tarf); - if (is_gzfile(tarf)) { + struct stat st; + auto ret = tarf->fstat(&st); + if (ret) { + LOG_ERRNO_RETURN(0, -1, "failed to stat `", input_path, VALUE(ret)); + } + if (S_ISFIFO(st.st_mode)) { + src_file = new FIFOFile(tarf); + } else if (is_gzfile(tarf)) { if (gz_index_path != "") { auto res = create_gz_index(tarf, gz_index_path.c_str(), 1024*1024); LOG_INFO("create_gz_index ", VALUE(res)); From ce6c5574edf58768b74fab35379f3c7f5e4d231f Mon Sep 17 00:00:00 2001 From: "zhuangbowei.zbw" Date: Wed, 20 Dec 2023 12:11:20 +0800 Subject: [PATCH 5/5] [feat] streaming overlaybd-commit Signed-off-by: zhuangbowei.zbw --- src/tools/CMakeLists.txt | 2 +- src/tools/overlaybd-commit.cpp | 47 +++++++++++++++++++++++++++++++--- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index 68d56aaa..5149aac1 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -1,6 +1,6 @@ add_executable(overlaybd-commit overlaybd-commit.cpp) target_include_directories(overlaybd-commit PUBLIC ${PHOTON_INCLUDE_DIR}) -target_link_libraries(overlaybd-commit photon_static overlaybd_lib) +target_link_libraries(overlaybd-commit photon_static overlaybd_image_lib) add_executable(overlaybd-create overlaybd-create.cpp) target_include_directories(overlaybd-create PUBLIC ${PHOTON_INCLUDE_DIR}) diff --git a/src/tools/overlaybd-commit.cpp b/src/tools/overlaybd-commit.cpp index e3353113..18ad5c69 100644 --- a/src/tools/overlaybd-commit.cpp +++ b/src/tools/overlaybd-commit.cpp @@ -21,6 +21,8 @@ #include "../overlaybd/lsmt/file.h" #include "../overlaybd/zfile/zfile.h" #include "../overlaybd/tar/tar_file.h" +#include "../overlaybd/registryfs/registryfs.h" +#include "../image_service.h" #include #include #include @@ -59,6 +61,8 @@ int main(int argc, char **argv) { bool tar = false, rm_old = false, seal = false, commit_sealed = false; bool verbose = false; int compress_threads = 1; + std::string upload_url, cred_file_path; + ssize_t upload_bs = 262144; CLI::App app{"this is overlaybd-commit"}; app.add_option("-m", commit_msg, "add some custom message if needed"); @@ -80,6 +84,10 @@ int main(int argc, char **argv) { app.add_flag("--commit_sealed", commit_sealed, "commit sealed, index_file is output")->default_val(false); app.add_option("--compress_threads", compress_threads, "compress threads")->default_val(1); app.add_flag("--verbose", verbose, "output debug info")->default_val(false); + app.add_option("--upload", upload_url, "registry upload url"); + app.add_option("--upload_bs", upload_bs, "block size for upload, in KB"); + app.add_option("--cred_file_path", cred_file_path, "cred file path for registryfs")->type_name("FILEPATH")->check(CLI::ExistingFile); + CLI11_PARSE(app, argc, argv); build_turboOCI = build_turboOCI || build_fastoci; set_log_output_level(verbose ? 0 : 1); @@ -117,8 +125,11 @@ int main(int argc, char **argv) { } IFile *fout = nullptr; IFile *out = nullptr; + IFile *zfile_builder = nullptr; + IFile *upload_builder = nullptr; ZFile::CompressOptions opt; + ZFile::CompressArgs *zfile_args = nullptr; opt.verify = 1; if (compress_zfile) { if (algorithm == "") { @@ -146,9 +157,29 @@ int main(int argc, char **argv) { } fout = open_file(fs, commit_file_path.c_str(), O_RDWR | O_EXCL | O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); - ZFile::CompressArgs zfile_args(opt); - zfile_args.workers = compress_threads; - zfile_builder = ZFile::new_zfile_builder(fout, &zfile_args, false); + out = fout; + + zfile_args = new ZFile::CompressArgs(opt); + zfile_args->workers = compress_threads; + zfile_args->overwrite_header = true; + + if (!upload_url.empty()) { + zfile_args->overwrite_header = false; + LOG_INFO("upload to `", upload_url); + std::string username, password; + if (load_cred_from_file(cred_file_path, upload_url, username, password) < 0) { + fprintf(stderr, "failed to read upload cred file\n"); + exit(-1); + } + upload_builder = new_registry_uploader(out, upload_url, username, password, 2UL*60*1000*1000, upload_bs*1024); + if (upload_builder == nullptr) { + fprintf(stderr, "failed to init upload\n"); + exit(-1); + } + out = upload_builder; + } + + zfile_builder = ZFile::new_zfile_builder(out, zfile_args, false); out = zfile_builder; } else { if (algorithm != "" || block_size != 0) { @@ -179,6 +210,16 @@ int main(int argc, char **argv) { } out->close(); delete zfile_builder; + if (zfile_args) { + delete zfile_args; + } + + if (upload_builder != nullptr && upload_builder->fsync() < 0) { + fprintf(stderr, "failed to commit or upload"); + return -1; + } + + delete upload_builder; delete fout; delete fin; printf("overlaybd-commit has committed files SUCCESSFULLY\n");