From 13dc94d4cfafeae648d6164f1c7573a1f0a33508 Mon Sep 17 00:00:00 2001 From: Brian Martin Date: Wed, 25 Mar 2020 16:40:04 -0700 Subject: [PATCH] Squashed 'deps/ccommon/' changes from f5efe29..6b1d8d5 6b1d8d5 Improve the Rust build story (#240) 5955d6b move nodelay setting to the server socket, and accepted connections will inherit that (#238) aafd20f formatting (#237) 3b5c069 Update ccommon_rs to use bytes v5.0 (#235) df74087 Address clippy lints in rust code (#234) 7c5bbd1 Make metrics and options Send + Sync (#233) c2e617e Ensure that manually implemented C options use the correct name (#232) 6e76741 Add some docs b5b4c5a Change check_pipe to use nanosleep instead of usleep (#231) eb0a389 Use name of field instead of description for C metrics (#230) 16ddc76 Fix broken buf impls (#229) eefcdcb Avoid redundant rebuilds of rust packages in CI (#228) 33f62a8 Update bindgen to also generate bindings recursively (#227) 27ffc7c Implement bytes::Buf and Bytes::ButMut on Buf and OwnedBuf (#226) f873930 Various small bugfixes and usability improvements for rust code (#225) 37a1ecd Port option parsing module to Rust (#224) 38f7556 Fix failed test detection (#215) 0ab1604 Conditionally use std::any::type_name if it's supported (#223) 98176d3 Backport changes from twitter/pelikan#265 (#222) ba54096 Remove test for removed rust logging functionality (#221) 519118d Rewrite cmake cargo build wrapper (#220) 5d23b3a Fix some small typos found in twitter/pelikan#263 (#219) 475dda7 Clean up logging shim (#218) 1d28dd2 Expand rust bindings to add options, metrics, and ccbox (#217) 88b5400 Upstream changes from twitter/pelikan#261 (#216) 4e99e63 fix a bug and change how check is found (#214) 54067ef slightly simplify accept error-handling logic (#210) e9fe980 Fix synchronize ccommon with pelikan deps/ccommon (#212) 7eb6424 Cleanup libcheck related code (#211) 683bc1a cc_bstring simplify and fix (#207) 8737d99 continue on server socket on non-blocking errors (#209) 2a62281 add atoi64 to bstring (#206) f71c657 cc_option simplify _allowed_in_name (#205) 24e3131 Add ITT instrumentation option (#204) 236c98d Fix docs (#200) e58f6a8 cc_array and cc_ring_array NULL fixes (#201) 1c8df42 Add basic support of build type (#199) 7107988 Fix now_ns() (#198) da240e5 cc: extend cc_util module (#196) 4846b15 Fix TAILQ_REINIT (#195) 4f5dbb0 Update Cmake version to 2.8 (#197) 2e6f78a cc_mm use OS_DARWIN macro to detect OS (#194) 57acaf6 cc: extend queue module (#193) a64ada2 cc: extend duration module (#192) b117632 reverting CMake file changes (#191) dea5bee backport changes made to ccommon in pelikan (#190) a4c0334 add linebreak to stats_log() (#188) 05eb03e fix inconsistent naming and bump version (#187) 4acc53a Stats to file (#186) 2168fec minimize osx build config (#185) 42b24de Simplify rust options, specify fewer output targets (#183) c9fa905 update CMakeRust used to latest version, tweaks to make build work (#184) 2ef0163 Reorder dependency includes in cmake, don't parallel build (#182) a6a54d9 remove endian-specific logic from str*cmp (#177) 4c0668b epoll_create* ignores size hint in newer kernels, switch to new API (#179) c9c5ee5 improve cc_bstring string literal and cstring names (#176) 0184d73 Add unit tests for buffer, fix buf/dbuf bugs and refactor (#174) d7dab43 create a .cargo/config so intellij uses the same target dir as cmake (#173) e710712 use accept4 for tcp_accept when available (#171) 21ba10e Remove cargo lock for shared lib, closes #169 (#172) 24660f1 update style guide (#170) 17baf1e Per thread logging (#168) git-subtree-dir: deps/ccommon git-subtree-split: 6b1d8d56b214e2a9d448625fe53303d57b4ac6d1 --- .gitignore | 9 + .travis.yml | 110 +--- CMakeLists.txt | 85 ++- Cargo.toml | 17 + LICENSE | 2 +- NOTICE | 2 + ci/before-install.sh | 15 - ci/install-check.sh | 25 +- ci/local-run.sh | 55 ++ ci/run.sh | 5 +- cmake/CMakeCargo.cmake | 418 +++++++++++-- cmake/CMakeDetermineRustCompiler.cmake | 25 - cmake/CMakeRustCompiler.cmake.in | 15 - cmake/CMakeRustInformation.cmake | 106 ---- cmake/CMakeTestRustCompiler.cmake | 3 - cmake/CargoLink.cmake | 105 ++++ cmake/CargoTest.cmake | 99 +++ cmake/FindITTNOTIFY.cmake | 65 ++ cmake/FindRust.cmake | 30 +- config.h.in | 4 +- docs/c-styleguide.txt | 427 ------------- docs/coding_style.rst | 583 ++++++++++++++++++ docs/modules/cc_log.rst | 126 +++- docs/modules/cc_metric.rst | 133 +++- docs/modules/cc_option.rst | 95 ++- docs/modules/cc_ring_array.rst | 47 +- docs/overview.rst | 50 +- include/buffer/cc_buf.h | 47 +- include/buffer/cc_dbuf.h | 6 +- include/cc_bstring.h | 70 ++- include/cc_define.h | 15 +- include/cc_event.h | 4 +- include/cc_itt.h | 69 +++ include/cc_log.h | 4 + include/cc_mm.h | 4 + include/cc_queue.h | 18 + include/cc_stats_log.h | 38 ++ include/cc_util.h | 3 + .../{hash/cc_lookup3.h => rust/cc_log_rs.h} | 9 +- include/time/cc_timer.h | 23 +- rust/CMakeLists.txt | 5 - rust/Cargo.lock | 496 --------------- rust/Cargo.toml | 21 - rust/cc_binding/CMakeLists.txt | 4 - rust/cc_binding/Cargo.toml | 12 - rust/cc_binding/build.rs | 108 ---- rust/cc_binding/src/lib.rs | 8 - rust/cc_binding/wrapper.h | 2 - rust/ccommon-backend/Cargo.toml | 17 + rust/ccommon-backend/src/c_export.rs | 146 +++++ rust/ccommon-backend/src/compat.rs | 157 +++++ rust/ccommon-backend/src/lib.rs | 41 ++ rust/ccommon-backend/src/option/default.rs | 99 +++ rust/ccommon-backend/src/option/parse.rs | 519 ++++++++++++++++ rust/ccommon-backend/src/option/print.rs | 120 ++++ rust/ccommon-backend/tests/parse.rs | 190 ++++++ rust/ccommon-derive/Cargo.toml | 15 + rust/ccommon-derive/src/attrs.rs | 191 ++++++ rust/ccommon-derive/src/lib.rs | 488 +++++++++++++++ rust/ccommon-rs/Cargo.toml | 30 + rust/ccommon-rs/build.rs | 23 + .../{ccommon_rs => ccommon-rs}/src/bstring.rs | 79 ++- rust/ccommon-rs/src/buf.rs | 366 +++++++++++ rust/ccommon-rs/src/ccbox.rs | 222 +++++++ rust/ccommon-rs/src/error.rs | 108 ++++ rust/ccommon-rs/src/lib.rs | 32 + rust/ccommon-rs/src/log/debug.rs | 121 ++++ rust/ccommon-rs/src/log/mod.rs | 52 ++ rust/ccommon-rs/src/log/shim.rs | 227 +++++++ rust/ccommon-rs/src/metric/counter.rs | 113 ++++ rust/ccommon-rs/src/metric/fpn.rs | 73 +++ rust/ccommon-rs/src/metric/gauge.rs | 113 ++++ rust/ccommon-rs/src/metric/mod.rs | 364 +++++++++++ rust/ccommon-rs/src/option/boolean.rs | 81 +++ rust/ccommon-rs/src/option/fpn.rs | 81 +++ rust/ccommon-rs/src/option/mod.rs | 357 +++++++++++ rust/ccommon-rs/src/option/string.rs | 109 ++++ rust/ccommon-rs/src/option/uint.rs | 81 +++ rust/ccommon-rs/src/ptrs.rs | 40 ++ rust/ccommon-rs/tests/derive_metrics.rs | 46 ++ rust/ccommon-rs/tests/derive_options.rs | 54 ++ rust/ccommon-rs/tests/log.rs | 140 +++++ rust/ccommon-sys/Cargo.toml | 16 + rust/ccommon-sys/build.rs | 142 +++++ rust/ccommon-sys/src/lib.rs | 61 ++ rust/ccommon-sys/src/metric.rs | 99 +++ rust/ccommon-sys/wrapper.h | 51 ++ rust/ccommon_rs/CMakeLists.txt | 4 - rust/ccommon_rs/Cargo.toml | 12 - rust/ccommon_rs/src/lib.rs | 3 - src/CMakeLists.txt | 6 +- src/buffer/cc_buf.c | 4 +- src/buffer/cc_dbuf.c | 31 +- src/cc_array.c | 2 +- src/cc_bstring.c | 50 +- src/cc_debug.c | 1 - src/cc_log.c | 25 + src/cc_mm.c | 14 +- src/cc_option.c | 3 +- src/cc_ring_array.c | 4 +- src/channel/cc_tcp.c | 77 ++- src/event/cc_epoll.c | 16 +- src/hash/CMakeLists.txt | 1 - src/hash/cc_lookup3.c | 446 -------------- src/stats/CMakeLists.txt | 1 + src/stats/cc_stats_log.c | 89 +++ src/time/cc_timer_darwin.c | 6 + src/time/cc_timer_linux.c | 42 +- test-coverage.sh | 2 +- test/CMakeLists.txt | 5 +- test/array/CMakeLists.txt | 1 - test/bstring/CMakeLists.txt | 1 - test/bstring/check_bstring.c | 56 ++ test/buffer/CMakeLists.txt | 9 + test/buffer/check_buf.c | 429 +++++++++++++ test/channel/pipe/CMakeLists.txt | 1 - test/channel/pipe/check_pipe.c | 17 +- test/channel/tcp/CMakeLists.txt | 1 - test/event/CMakeLists.txt | 1 - test/log/CMakeLists.txt | 16 +- test/log/check_log.c | 2 +- test/metric/CMakeLists.txt | 1 - test/option/CMakeLists.txt | 1 - test/pool/CMakeLists.txt | 1 - test/rbuf/CMakeLists.txt | 1 - test/rbuf/check_rbuf.c | 1 - test/ring_array/CMakeLists.txt | 1 - test/ring_array/check_ring_array.c | 1 + test/time/timer/CMakeLists.txt | 1 - test/time/wheel/CMakeLists.txt | 1 - 130 files changed, 8070 insertions(+), 2203 deletions(-) create mode 100644 Cargo.toml create mode 100755 ci/local-run.sh delete mode 100644 cmake/CMakeDetermineRustCompiler.cmake delete mode 100644 cmake/CMakeRustCompiler.cmake.in delete mode 100644 cmake/CMakeRustInformation.cmake delete mode 100644 cmake/CMakeTestRustCompiler.cmake create mode 100644 cmake/CargoLink.cmake create mode 100644 cmake/CargoTest.cmake create mode 100644 cmake/FindITTNOTIFY.cmake delete mode 100644 docs/c-styleguide.txt create mode 100644 docs/coding_style.rst create mode 100644 include/cc_itt.h create mode 100644 include/cc_stats_log.h rename include/{hash/cc_lookup3.h => rust/cc_log_rs.h} (80%) delete mode 100644 rust/CMakeLists.txt delete mode 100644 rust/Cargo.lock delete mode 100644 rust/Cargo.toml delete mode 100644 rust/cc_binding/CMakeLists.txt delete mode 100644 rust/cc_binding/Cargo.toml delete mode 100644 rust/cc_binding/build.rs delete mode 100644 rust/cc_binding/src/lib.rs delete mode 100644 rust/cc_binding/wrapper.h create mode 100644 rust/ccommon-backend/Cargo.toml create mode 100644 rust/ccommon-backend/src/c_export.rs create mode 100644 rust/ccommon-backend/src/compat.rs create mode 100644 rust/ccommon-backend/src/lib.rs create mode 100644 rust/ccommon-backend/src/option/default.rs create mode 100644 rust/ccommon-backend/src/option/parse.rs create mode 100644 rust/ccommon-backend/src/option/print.rs create mode 100644 rust/ccommon-backend/tests/parse.rs create mode 100644 rust/ccommon-derive/Cargo.toml create mode 100644 rust/ccommon-derive/src/attrs.rs create mode 100644 rust/ccommon-derive/src/lib.rs create mode 100644 rust/ccommon-rs/Cargo.toml create mode 100644 rust/ccommon-rs/build.rs rename rust/{ccommon_rs => ccommon-rs}/src/bstring.rs (84%) create mode 100644 rust/ccommon-rs/src/buf.rs create mode 100644 rust/ccommon-rs/src/ccbox.rs create mode 100644 rust/ccommon-rs/src/error.rs create mode 100644 rust/ccommon-rs/src/lib.rs create mode 100644 rust/ccommon-rs/src/log/debug.rs create mode 100644 rust/ccommon-rs/src/log/mod.rs create mode 100644 rust/ccommon-rs/src/log/shim.rs create mode 100644 rust/ccommon-rs/src/metric/counter.rs create mode 100644 rust/ccommon-rs/src/metric/fpn.rs create mode 100644 rust/ccommon-rs/src/metric/gauge.rs create mode 100644 rust/ccommon-rs/src/metric/mod.rs create mode 100644 rust/ccommon-rs/src/option/boolean.rs create mode 100644 rust/ccommon-rs/src/option/fpn.rs create mode 100644 rust/ccommon-rs/src/option/mod.rs create mode 100644 rust/ccommon-rs/src/option/string.rs create mode 100644 rust/ccommon-rs/src/option/uint.rs create mode 100644 rust/ccommon-rs/src/ptrs.rs create mode 100644 rust/ccommon-rs/tests/derive_metrics.rs create mode 100644 rust/ccommon-rs/tests/derive_options.rs create mode 100644 rust/ccommon-rs/tests/log.rs create mode 100644 rust/ccommon-sys/Cargo.toml create mode 100644 rust/ccommon-sys/build.rs create mode 100644 rust/ccommon-sys/src/lib.rs create mode 100644 rust/ccommon-sys/src/metric.rs create mode 100644 rust/ccommon-sys/wrapper.h delete mode 100644 rust/ccommon_rs/CMakeLists.txt delete mode 100644 rust/ccommon_rs/Cargo.toml delete mode 100644 rust/ccommon_rs/src/lib.rs delete mode 100644 src/hash/cc_lookup3.c create mode 100644 src/stats/cc_stats_log.c create mode 100644 test/buffer/CMakeLists.txt create mode 100644 test/buffer/check_buf.c diff --git a/.gitignore b/.gitignore index 781ba3860..0682f193d 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,12 @@ cscope.* lcov CMAKE_BINARY_DIR + +# Generated Rust Bindings +bindings.rs + +# Cargo lock file +Cargo.lock + +# Cargo build path +target \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 41c2b344e..aa5e031a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,12 @@ sudo: false language: c +dist: xenial # using anchor to import sources into linux builds addons: apt: &apt sources: - ubuntu-toolchain-r-test - - llvm-toolchain-precise-3.6 - - llvm-toolchain-precise-3.7 - - llvm-toolchain-precise # important for allowed-to-fail matching # see https://docs.travis-ci.com/user/customizing-the-build#Rows-that-are-Allowed-to-Fail @@ -18,115 +16,37 @@ env: # travis currently does not support directly setting gcc/clang with versions # (e.g. gcc-4.8) as value for the compiler key. So we will have to manually # request these packages and use environment varibles to create the matrix. -# -# In the case of osx, use brew to install the paritcular versions, instead of -# specifying with packages. matrix: include: - # gcc 4.8 on linux - - env: - - C_COMPILER=gcc-4.8 + - name: "gcc-5 on Linux" + compiler: gcc addons: apt: <<: *apt packages: - - gcc-4.8 - libsubunit-dev + - os: osx + osx_image: xcode10.1 + compiler: clang - # gcc 4.9 on linux - - env: - - C_COMPILER=gcc-4.9 - addons: - apt: - <<: *apt - packages: - - gcc-4.9 - - libsubunit-dev - - # gcc 5 on linux - - env: - - C_COMPILER=gcc-5 - addons: - apt: - <<: *apt - packages: - - gcc-5 - - libsubunit-dev - - # gcc 5 on linux - - env: - - C_COMPILER=gcc-5 - - RUST_ENABLED=1 - addons: - apt: - <<: *apt - packages: - - gcc-5 - - libsubunit-dev - - # clang 3.6 on linux - - env: - - C_COMPILER=clang-3.6 - addons: - apt: - <<: *apt - packages: - - clang-3.6 - - libsubunit-dev - - # clang 3.7 on linux - - env: - - C_COMPILER=clang-3.7 + - name: "cargo build" + language: rust addons: apt: <<: *apt packages: - - clang-3.7 - libsubunit-dev - - ## gcc 4.8 on osx - #- os: osx - # env: FORMULA=gcc48 COMPILER=gcc C_COMPILER=gcc-4.8 - # - ## gcc 4.9 on osx - #- os: osx - # env: FORMULA=gcc49 COMPILER=gcc C_COMPILER=gcc-4.9 - # - ## gcc 5 on osx - #- os: osx - # env: FORMULA=gcc5 COMPILER=gcc C_COMPILER=gcc-5 - - # OSX 10.13 - # Apple LLVM version 9.1.0 (clang-902.0.39.2) - # Target: x86_64-apple-darwin17.6.0 - - os: osx - osx_image: xcode9.4 - env: - - C_COMPILER=clang - - ALLOWED_TO_FAIL=1 - - # OSX 10.12 - # Apple LLVM version 9.0.0 (clang-900.0.39.2) - # Target: x86_64-apple-darwin16.7.0 - - os: osx - osx_image: xcode9.2 - env: - - C_COMPILER=clang - - ALLOWED_TO_FAIL=1 + script: + - cargo build + - cargo test + - cargo build --release + - cargo test --release allow_failures: - os: osx - osx_image: xcode9.4 - env: - - C_COMPILER=clang - - ALLOWED_TO_FAIL=1 - - - os: osx - osx_image: xcode9.2 - env: - - C_COMPILER=clang - - ALLOWED_TO_FAIL=1 + osx_image: xcode10.1 + compiler: clang before_install: - ./ci/before-install.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 2d2a2c05a..e76d9e3f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,9 @@ -cmake_minimum_required(VERSION 2.6) +cmake_minimum_required(VERSION 2.8) project(ccommon C) +# Uncomment the following to output dependency graph debugging messages +# set_property(GLOBAL PROPERTY GLOBAL_DEPENDS_DEBUG_MODE 1) + enable_testing() ################### @@ -36,8 +39,8 @@ endif() # config.h.in has to include entries set/tested here for them to have effect # version info -set(${PROJECT_NAME}_VERSION_MAJOR 1) -set(${PROJECT_NAME}_VERSION_MINOR 2) +set(${PROJECT_NAME}_VERSION_MAJOR 2) +set(${PROJECT_NAME}_VERSION_MINOR 1) set(${PROJECT_NAME}_VERSION_PATCH 0) set(${PROJECT_NAME}_VERSION ${${PROJECT_NAME}_VERSION_MAJOR}.${${PROJECT_NAME}_VERSION_MINOR}.${${PROJECT_NAME}_VERSION_PATCH} @@ -54,6 +57,7 @@ option(HAVE_STATS "stats enabled by default" ON) option(HAVE_DEBUG_MM "debugging oriented memory management disabled by default" OFF) option(COVERAGE "code coverage" OFF) option(HAVE_RUST "rust bindings not built by default" OFF) +option(HAVE_ITT_INSTRUMENTATION "instrument code with ITT API" OFF) if(HAVE_RUST) option(RUST_VERBOSE_BUILD "pass -vv to cargo compilation" OFF) @@ -78,7 +82,6 @@ if(BUILD_AND_INSTALL_CHECK) set(CHECK_ROOT_DIR "${LIBCHECK_PREFIX}") set(CMAKE_REQUIRED_INCLUDES "${CHECK_ROOT_DIR}/include") # these make check link correctly in ccommon and pelikan - set(CMAKE_REQUIRED_LIBRARIES "${CHECK_ROOT_DIR}/lib") endif() include(CheckIncludeFiles) @@ -96,9 +99,7 @@ check_symbol_exists(sys_signame signal.h HAVE_SIGNAME) include(CheckFunctionExists) check_function_exists(backtrace HAVE_BACKTRACE) - -include(TestBigEndian) -test_big_endian(HAVE_BIG_ENDIAN) +check_function_exists(accept4 HAVE_ACCEPT4) # how to use config.h.in to generate config.h # this has to be set _after_ the above checks @@ -114,12 +115,23 @@ configure_file( # set compiler flags # string concat is easier in 3.0, but older versions don't have the concat subcommand # so we are using list as input until we move to new version -# TODO add build types add_definitions(-D_GNU_SOURCE -D_FILE_OFFSET_BITS=64) +# Set a default build type (Release) if none was specified + +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif() + +if(CMAKE_BUILD_TYPE MATCHES Debug) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O0") +else() + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2") +endif() + set(CMAKE_MACOSX_RPATH 1) set(CFLAGS_LIST "-std=c11 " - "-ggdb3 -O2 " + "-ggdb3 " "-Wall " "-Wmissing-prototypes -Wmissing-declarations -Wredundant-decls " "-Wunused-function -Wunused-value -Wunused-variable " @@ -132,13 +144,15 @@ if(CMAKE_COMPILER_IS_GNUCC) endif() if (COVERAGE) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -Wall -W -fprofile-arcs -ftest-coverage") + if(NOT ${CMAKE_BUILD_TYPE} MATCHES Debug) + message(WARNING "Code coverage results with an optimised (non-Debug) build may be misleading" ) + endif() + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage") endif(COVERAGE) # test dependencies include(FindPackageHandleStandardArgs) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}/cmake") - find_package(Check) if(NOT CHECK_FOUND) message(WARNING "Check is required to build and run tests") @@ -150,6 +164,17 @@ if(CHECK_FOUND) endif(NOT CHECK_WORKING) endif(CHECK_FOUND) +if (HAVE_ITT_INSTRUMENTATION) + if(PKG_CONFIG_FOUND) + pkg_check_modules(ITTNOTIFY REQUIRED ittnotify>=1.0) + else() + find_package(ITTNOTIFY REQUIRED 1.0) + endif() + include_directories(${ITTNOTIFY_INCLUDE_DIRS}) + link_directories(${ITTNOTIFY_LIBRARY_DIRS}) + link_libraries(${ITTNOTIFY_LIBRARIES}) +endif(HAVE_ITT_INSTRUMENTATION) + find_package(Threads) @@ -159,28 +184,41 @@ include_directories( "${PROJECT_BINARY_DIR}" "include") -if(HAVE_RUST) - enable_language(Rust) - include(CMakeCargo) - add_subdirectory(rust) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DHAVE_RUST=1") -endif() - - ################### # things to build # ################### add_subdirectory(src) -if(CHECK_WORKING) - include_directories(${include_directories} "${CHECK_INCLUDES}") + +if(CHECK_FOUND) + include_directories(${include_directories} ${CHECK_INCLUDES}) add_subdirectory(test) -endif(CHECK_WORKING) +endif(CHECK_FOUND) + +if(HAVE_RUST) + include(CMakeCargo) + add_subdirectory(rust) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DHAVE_RUST=1") + + if (${CMAKE_VERSION} VERSION_LESS "3.13.0") + # CMakeCargo requires the use of some newer features of cmake + # for changes to dependant libraries to cause a rebuild. This + # will never break the build, it is just annoying for development. + # Leave a warning here, but don't break the build for older cmake + # versions. + message( + WARNING + "Rust targets don't properly pick up changes to dependencies in cmake version <= 3.13" + ) + endif() +endif() ################### # print a summary # ################### +message(STATUS "CMAKE_BUILD_TYPE: " ${CMAKE_BUILD_TYPE}) + message(STATUS "PLATFORM: " ${OS_PLATFORM}) message(STATUS "CPPFLAGS: " ${CMAKE_CPP_FLAGS}) @@ -189,6 +227,5 @@ message(STATUS "CFLAGS: " ${CMAKE_C_FLAGS}) message(STATUS "HAVE_SIGNAME: " ${HAVE_SIGNAME}) message(STATUS "HAVE_BACKTRACE: " ${HAVE_BACKTRACE}) -message(STATUS "HAVE_BIG_ENDIAN: " ${HAVE_BIG_ENDIAN}) -message(STATUS "CHECK_WORKING: " ${CHECK_WORKING}) +message(STATUS "CHECK_FOUND: " ${CHECK_FOUND}) diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 000000000..fa126b5fc --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,17 @@ +[workspace] +members = [ + "rust/ccommon-backend", + "rust/ccommon-derive", + "rust/ccommon-rs", + "rust/ccommon-sys", +] + +[profile.bench] +debug = true +lto = true +codegen-units = 1 + +[profile.release] +debug = true +lto = true +codegen-units = 1 diff --git a/LICENSE b/LICENSE index ecc06d346..5c3aef4f8 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2013-2015 Twitter, Inc +Copyright 2013-2018 Twitter, Inc Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/NOTICE b/NOTICE index 2b7bb058e..e0672064d 100644 --- a/NOTICE +++ b/NOTICE @@ -22,3 +22,5 @@ * Boston, MA 02111-1307, USA. */ +We use the CMakeRust project (https://github.com/Devolutions/CMakeRust) under +the Apache 2.0 License. diff --git a/ci/before-install.sh b/ci/before-install.sh index cff9327c3..b5f49421a 100755 --- a/ci/before-install.sh +++ b/ci/before-install.sh @@ -11,21 +11,6 @@ trap cleanup EXIT TOPLEVEL="$(git -C "$(cd "$(dirname "$0")" >/dev/null || exit 1; pwd)" rev-parse --show-toplevel)" || die 'failed to find TOPLEVEL' -# for osx: 0. update brew; 1. install cmake if missing; 2. (gcc) unlink pre-installed gcc; 3. (gcc) install desired version of gcc - -if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - brew update &>/dev/null - brew install cmake || true # xcode 8.1 is missing cmake - - if [[ "$C_COMPILER" =~ ^gcc && -n "${FORMULA:-}" ]]; then - brew unlink gcc || true - brew unlink "$FORMULA" || true - brew install "$FORMULA" - fi -fi - -export CC="$C_COMPILER" - if [[ -n "${RUST_ENABLED:-}" ]]; then curl https://sh.rustup.rs -sSf | sh -s -- -y fi diff --git a/ci/install-check.sh b/ci/install-check.sh index 628d6d229..0658135c4 100755 --- a/ci/install-check.sh +++ b/ci/install-check.sh @@ -17,18 +17,37 @@ TEMP="$(mktemp -d -t TEMP.XXXXXXX)" || die "failed to make tmpdir" cleanup() { [[ -n "${TEMP:-}" ]] && rm -rf "${TEMP}"; } trap cleanup EXIT -TOPLEVEL="$(git -C "$(cd "$(dirname "$0")" >/dev/null || exit 1; pwd)" rev-parse --show-toplevel)" || die 'failed to find TOPLEVEL' +realpath() { python -c "from __future__ import print_function; import os,sys; print(os.path.realpath(sys.argv[1]))" "$1"; } + +TOPLEVEL="$(cd "$(dirname "$(realpath "$0" >/dev/null || exit 1)")" && git rev-parse --show-toplevel)" || die 'failed to find TOPLEVEL' + CHECK_VERSION=0.12.0 CHECK_TARBALL="check-${CHECK_VERSION}.tar.gz" CHECK_DIR="check-${CHECK_VERSION}" +echo "building and installing check" >&2 + ( cd "$TEMP" && wget "https://github.com/libcheck/check/releases/download/${CHECK_VERSION}/${CHECK_TARBALL}" && - tar xvfz "${CHECK_TARBALL}" && + tar xfz "${CHECK_TARBALL}" && cd "${CHECK_DIR}" && ./configure --prefix="$CHECK_PREFIX" && make && make install -) || die "check build failed" +) >$TEMP/cmake-build.log 2>&1 + +RESULT=$? +if [[ $RESULT -ne 0 ]]; then + cat >&2 <&2 +fi + +exit $RESULT diff --git a/ci/local-run.sh b/ci/local-run.sh new file mode 100755 index 000000000..ecc90c3ec --- /dev/null +++ b/ci/local-run.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +## This file is potentially useful for doing a clean environment build on MacOS ## +## This is my personal way of running the build, YMMV - jsimms ## + +set -euo pipefail +IFS=$'\n\t' + +die() { echo "fatal: $*" >&2; exit 1; } + +TOPLEVEL=$(git -C "$(cd "$(dirname "$0")" >/dev/null || exit 1; pwd)" rev-parse --show-toplevel) || die "TOPLEVEL fail" + +cd "$TOPLEVEL" + +TEMP="$(mktemp -d -t TEMP.XXXXXXX)" || die "failed to make tmpdir" +cleanup() { [[ -n "${TEMP:-}" ]] && rm -rf "${TEMP}"; } +trap cleanup EXIT + +BUILD_PATH=( + "$HOME/.cargo/bin" + "/usr/local/bin" + "/usr/local/sbin" + "/usr/bin" + "/usr/sbin" + "/bin" + "/sbin" + "/opt/X11/bin" + "/usr/X11R6/bin" +) + +PATH=$(echo "${BUILD_PATH[@]}"|tr ' ' ':') + +cat >&2 < +# The library package name as specified in Cargo.toml. +# This argument is required. +# TARGET_DIR +# Overrides the default directory for the cargo target +# directory which is ${CMAKE_BINARY_DIR}/target. +# BIN +# Indicates that this crate generated a binary and to +# expose that binary file as a target. +# STATIC +# Indicates this target generates a static library that +# can be consumed by other cmake targets. +# NO_TEST +# Disables generation of the test target. +# +# Notes: +# - If neither BIN or STATIC is defined then it is assumed +# that the target exports no artifacts (i.e. it is only used +# by other rust targets within the build) +# - Build and exported are currently mutually exclusive. If you +# want to have multiple targets like this then call cargo_build +# multiple times. +# +# Additional Things: +# - This also sets up the required cmake properties to delete the +# target directory when `make clean` or equivalent is run. +# +# How it Works: +# In general, cmake has a hard time integrating with external build +# systems within the same directory. However, cmake has extensive +# support for custom linkers and also allows you to define interface +# libraries when no artifact is created. Together, we can use these +# to trick cmake into thinking it is building the executable while +# actually using cargo to build them. This means that things like +# linker flags and dependencies should mostly "just work" (hopefully). +# +# What we have to do depends on what type of library we are compiling: +# - For pure rust libraries that don't export any C API, we define +# an interface library. This means that any linker flags are +# properly passed on to downstream dependencies. In addition, we +# define a custom target so the library is built. +# - For static libraries, we define a library that's built using a +# custom linker language. The custom linker command is really just +# a `cargo build` in disguise, but one that passes the correct +# flags in. +# - For binaries, we define an executable target also using a custom +# linker language similar to a static library. +# +# To pass the proper linker flags to the rust process, we have a special +# target .linkflags.txt that echoes the linker flags into a known +# file. These are then picked up by the import-link-flags crate which +# uses a build script to pass them to rustc. function(cargo_build) - cmake_parse_arguments(CARGO "" "NAME" "" ${ARGN}) + cmake_parse_arguments( + CARGO + "BIN;STATIC;NO_TEST" + "NAME;TARGET_DIR;COPY_TO" + "" + ${ARGN} + ) + string(REPLACE "-" "_" LIB_NAME ${CARGO_NAME}) + + if(NOT (DEFINED CARGO_TARGET_DIR)) + if ($ENV{CI}) + set(CARGO_TARGET_DIR ${CMAKE_BINARY_DIR}/target) + else() + set(CARGO_TARGET_DIR ${CMAKE_CURRENT_BINARY_DIR}/target) + endif() + endif() - get_filename_component(CARGO_TARGET_DIR "${CMAKE_CURRENT_BINARY_DIR}" ABSOLUTE) + if(CARGO_BIN AND CARGO_STATIC) + message( + FATAL_ERROR + "Cannot create a cargo target that has " + "both a binary and a static library. Use multiple " + "targets instead." + ) + endif() - cargo_set_lib_target(LIB_NAME) + cargo_build_private_get_target(CRATE_TARGET) + cargo_build_private_get_build_type(CRATE_BUILD_TYPE) - if(NOT CMAKE_BUILD_TYPE) - set(LIB_BUILD_TYPE "debug") - elseif(${CMAKE_BUILD_TYPE} STREQUAL "Release") - set(LIB_BUILD_TYPE "release") + # The CONFIGURE_DEPENDS flag will rerun the glob at build time if the + # the build system supports it. + file( + GLOB_RECURSE CRATE_SOURCES + CONFIGURE_DEPENDS "*.rs" + ) + + list(APPEND CRATE_SOURCES Cargo.toml) + + # Clean the target directory when make clean is run + set_directory_properties(PROPERTIES + ADDITIONAL_CLEAN_FILES + ${CARGO_TARGET_DIR} + ) + + if(CARGO_BIN OR CARGO_STATIC) + set(LINK_FLAGS_FILE $.linkflags.txt) else() - set(LIB_BUILD_TYPE "debug") + set(LINK_FLAGS_FILE $) endif() - set(SHARED_LIB_FNAME "${CMAKE_SHARED_LIBRARY_PREFIX}${LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}") - set(STATIC_LIB_FNAME "${CMAKE_STATIC_LIBRARY_PREFIX}${LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}") - - set(LIB_BASE_DIR "${CARGO_TARGET_DIR}/${LIB_TARGET}/${LIB_BUILD_TYPE}") + set(FORWARDED_VARS + # So that internal invocations of cmake are consistent + "CMAKE=${CMAKE_COMMAND}" + # So that build scripts can configure themselves based + # on whether cmake is driving the build or not + "CCOMMON_CMAKE_IS_DRIVING_BUILD=1" + # Needed to configure the correct target directory + "CARGO_TARGET_DIR=${CARGO_TARGET_DIR}" + # Needed for build scripts + "CMAKE_BINARY_DIR=${CMAKE_BINARY_DIR}" + ) - get_filename_component(SHARED_LIB_FILE "${LIB_BASE_DIR}/${SHARED_LIB_FNAME}" ABSOLUTE) - get_filename_component(STATIC_LIB_FILE "${LIB_BASE_DIR}/${STATIC_LIB_FNAME}" ABSOLUTE) + if(CARGO_BIN) + set(OUTPUT_FILE_NAME ${LIB_NAME}${CMAKE_EXECUTABLE_SUFFIX}) + set(OUTPUT_FILE ${CARGO_TARGET_DIR}/${CRATE_TARGET}/${CRATE_BUILD_TYPE}/${OUTPUT_FILE_NAME}) + elseif(CARGO_STATIC) + set(OUTPUT_FILE_NAME ${CMAKE_STATIC_LIBRARY_PREFIX}${LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}) + set(OUTPUT_FILE ${CARGO_TARGET_DIR}/${CRATE_TARGET}/${CRATE_BUILD_TYPE}/${OUTPUT_FILE_NAME}) + endif() if(IOS) - set(CARGO_ARGS "lipo") - else() - set(CARGO_ARGS "build") - list(APPEND CARGO_ARGS "--target" ${LIB_TARGET}) + # Since we're going through cargo rustc to pass linker flags + # this won't work. The previous build script used cargo lipo + # here. However, the likelyhood of someone wanting to use + # a library for cache servers on IOS is low at this time so + # this is OK. + message(FATAL_ERROR "Compiling for IOS is not supported") endif() - if(RUST_VERBOSE_BUILD) - list(APPEND CARGO_ARGS "-vv") - endif() + # Arguments to cargo + set(CRATE_ARGS "") + list(APPEND CRATE_ARGS "--target" ${CRATE_TARGET}) - if(${LIB_BUILD_TYPE} STREQUAL "release") - list(APPEND CARGO_ARGS "--release") + if(${CRATE_BUILD_TYPE} STREQUAL "release") + list(APPEND CRATE_ARGS "--release") endif() - file(GLOB_RECURSE LIB_SOURCES "*.rs") + # Convert CRATE_ARGS from a list to a string + string(REPLACE ";" " " CRATE_ARGS_STR "${CRATE_ARGS}") - set(CARGO_ENV_COMMAND ${CMAKE_COMMAND} -E env "CARGO_TARGET_DIR=${CARGO_TARGET_DIR}") + set(LINK_FLAGS_FILE "${CMAKE_CURRENT_BINARY_DIR}/${CARGO_NAME}.linkflags.txt") - add_custom_command( - OUTPUT ${STATIC_LIB_FILE} - COMMAND ${CARGO_ENV_COMMAND} ${CARGO_EXECUTABLE} ARGS ${CARGO_ARGS} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - DEPENDS ${LIB_SOURCES} - COMMENT "running cargo") - add_custom_target(${CARGO_NAME}_static_target ALL DEPENDS ${STATIC_LIB_FILE}) - add_library(${CARGO_NAME}_static STATIC IMPORTED GLOBAL) - add_dependencies(${CARGO_NAME}_static ${CARGO_NAME}_static_target) - set_target_properties(${CARGO_NAME}_static PROPERTIES IMPORTED_LOCATION ${STATIC_LIB_FILE}) + # The following is a hack. It takes advantage of the fact that + # cmake allows us to define arbitrary linker languages in order + # to invoke cargo as our linker command. To do this, we define + # a custom language specific to only our target that also takes + # in the required environment variables we want to set and the + # output file generated by cargo. + # + # This works by running a cmake script during build-time which + # parses the link flags and sends them off to cargo as -Clink-flag=... + # For the full details in what's done during build see CargoLink.cmake. + # + # The upside of this hack is that it allows libraries and binaries + # generated by cargo to be used as normal cmake targets. This + # means that stuff like target_link_libraries works properly. + # + # Extra Note: The variables that look like in the string + # below are called expansion variables by the community wiki. They + # don't seem to be documented anywhere in the official docs but + # are hopefully stable. + # + # Another Note: Since these are required to be global variables we + # push them in the cache as hidden variables. + # + # Note 3: There is a expansion rule in the docs which I would + # expect to expand to what was passed in to target_compile_flags. + # Unfortunately, it doesn't, so I've instead just shoved the + # compile flags into linker command template directory. + # + # The inspiration for this hack came from this SO answer + # https://stackoverflow.com/questions/34165365/retrieve-all-link-flags-in-cmake + # + # "Docs" for the expansion rules can be found here + # https://gitlab.kitware.com/cmake/community/wikis/doc/cmake/Build-Rules + set(LINK_COMMAND " -P ${FILE_LIST_DIR}/CargoLink.cmake 'TARGET=' 'LINK_FLAGS=' 'LINK_LIBRARIES=' 'FLAGS=${CRATE_ARGS_STR}' 'OUTPUT=${OUTPUT_FILE}' 'CMAKE_CURRENT_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}' 'LINK_FLAGS_FILE=${LINK_FLAGS_FILE}' -- ${FORWARDED_VARS}") + + foreach(VAR ${FORWARDED_VARS}) + string(APPEND LINK_COMMAND " ${VAR}") + endforeach() + + # TODO(sean): disambiguate this based on bin/lib so that multiple targets in + # the same directory don't clash. + set(CMAKE_${CARGO_NAME}_LINK_EXECUTABLE ${LINK_COMMAND} CACHE INTERNAL "") + set(CMAKE_${CARGO_NAME}_CREATE_STATIC_LIBRARY ${LINK_COMMAND} CACHE INTERNAL "") + set(CMAKE_${CARGO_NAME}_CREATE_SHARED_LIBRARY ${LINK_COMMAND} CACHE INTERNAL "") + set(CMAKE_${CARGO_NAME}_LINK_FLAGS ${CMAKE_C_LINK_FLAGS} CACHE INTERNAL "") + set(CMAKE_${CARGO_NAME}_LINK_DIRECTORIES ${CMAKE_C_LINK_DIRECTORIES} CACHE INTERNAL "") + + # Needed to build when we have test targets + set( + CMAKE_ECHO_LINK_EXECUTABLE + "bash -c \" -E echo_append > \"" + CACHE INTERNAL "" + ) + set(CMAKE_ECHO_LINK_FLAGS ${CMAKE_C_LINK_FLAGS} CACHE INTERNAL "") + set(CMAKE_ECHO_LINK_DIRECTORIES ${CMAKE_C_LINK_DIRECTORIES} CACHE INTERNAL "") + set_source_files_properties( + ${CRATE_SOURCES} + PROPERTY + HEADER_FILE_ONLY ON + ) + + # CMake doesn't seem to add dependencies on the crate + # sources. This means that making changes to a rust + # target is rather painful. + # + # We work around this by creating a stamp file that depends + # on all the source files in the crate. When a source file + # is modified, the stamp file is updated, and this is picked + # up since we set the stamp file as a link dependency of + # library/binary. add_custom_command( - OUTPUT ${SHARED_LIB_FILE} - COMMAND ${CARGO_ENV_COMMAND} ${CARGO_EXECUTABLE} ARGS ${CARGO_ARGS} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - DEPENDS ${LIB_SOURCES} - COMMENT "running cargo") - add_custom_target(${CARGO_NAME}_shared_target ALL DEPENDS ${SHARED_LIB_FILE}) - add_library(${CARGO_NAME}_shared SHARED IMPORTED GLOBAL) - add_dependencies(${CARGO_NAME}_shared ${CARGO_NAME}_shared_target) - set_target_properties(${CARGO_NAME}_shared PROPERTIES IMPORTED_LOCATION ${SHARED_LIB_FILE}) + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/up-to-date.stamp + DEPENDS ${CRATE_SOURCES} + COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/up-to-date.stamp + COMMENT "checking whether crate sources are up-to-date" + ) + + # Targets + if(CARGO_BIN) + # We are building a binary executable + add_executable( + ${CARGO_NAME} + ${CRATE_SOURCES} + # Needed so that this file is usable in LINK_DEPENDENCY + ${CMAKE_CURRENT_BINARY_DIR}/up-to-date.stamp + ) + + # Ensure that we use our custom "linker" + set_target_properties( + ${CARGO_NAME} PROPERTIES + LINKER_LANGUAGE ${CARGO_NAME} + LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/up-to-date.stamp + ) + + target_compile_options(${CARGO_NAME} PRIVATE ${CRATE_ARGS}) + elseif(CARGO_STATIC) + # We are building a static library that will be used from C code + add_library( + ${CARGO_NAME} + STATIC + ${CRATE_SOURCES} + ${CMAKE_CURRENT_BINARY_DIR}/up-to-date.stamp + ) + + # Ensure that we use our custom "linker" + set_target_properties( + ${CARGO_NAME} PROPERTIES + LINKER_LANGUAGE ${CARGO_NAME} + # Needed so that this file is usable in LINK_DEPENDENCY + LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/up-to-date.stamp + ) + + target_compile_options(${CARGO_NAME} PRIVATE ${CRATE_ARGS}) + else() + set(CARGO_ENV_COMMAND ${CMAKE_COMMAND} -E env ${FORWARDED_VARS}) + + # We are building a rust-only library. Define it as a interface library + add_library( + ${CARGO_NAME} + INTERFACE + ) + + if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.13.0") + # This is required for dependency detection to work correctly. + # However, nothing breaks on a clean build if you don't have it + # so we don't make it required. + set_target_properties( + ${CARGO_NAME} PROPERTIES + INTERFACE_LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/build.stamp + ) + endif() + + # However, we still want to build the library, so define a custom command for that + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/build.stamp + COMMAND ${CARGO_ENV_COMMAND} cargo build ${CRATE_ARGS} + COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/build.stamp + DEPENDS ${CRATE_SOURCES} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + COMMENT "running cargo for target ${CARGO_NAME}" + ) + + add_custom_target( + ${CARGO_NAME}-build + ALL DEPENDS + ${CMAKE_CURRENT_BINARY_DIR}/build.stamp + ) + + add_dependencies(${CARGO_NAME} ${CARGO_NAME}-build) + add_dependencies(${CARGO_NAME} ${CARGO_NAME}-link-export) + + add_executable(${CARGO_NAME}-link-export Cargo.toml) + set_target_properties( + ${CARGO_NAME}-link-export PROPERTIES + LINKER_LANGUAGE ECHO + SUFFIX ".txt" + TARGET_MESSAGES OFF + ) + target_link_libraries(${CARGO_NAME}-link-export $) + + set(LINK_FLAGS_FILE $) + endif() + + if(NOT CARGO_NO_TEST) + add_test( + NAME test-${CARGO_NAME} + COMMAND ${CMAKE_COMMAND} -P "${FILE_LIST_DIR}/CargoTest.cmake" + "LINK_FLAGS_FILE=${LINK_FLAGS_FILE}" + "FLAGS=${CRATE_ARGS_STR}" + "CMAKE_CURRENT_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}" + "CMAKE_CURRENT_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR}" + -- + ${FORWARDED_VARS} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + ) + endif() endfunction() diff --git a/cmake/CMakeDetermineRustCompiler.cmake b/cmake/CMakeDetermineRustCompiler.cmake deleted file mode 100644 index 33a459c0a..000000000 --- a/cmake/CMakeDetermineRustCompiler.cmake +++ /dev/null @@ -1,25 +0,0 @@ - -if(NOT CMAKE_Rust_COMPILER) - find_package(Rust) - if(RUST_FOUND) - set(CMAKE_Rust_COMPILER "${RUSTC_EXECUTABLE}") - set(CMAKE_Rust_COMPILER_ID "Rust") - set(CMAKE_Rust_COMPILER_VERSION "${RUST_VERSION}") - set(CMAKE_Rust_PLATFORM_ID "Rust") - endif() -endif() - -message(STATUS "Cargo Prefix: ${CARGO_PREFIX}") -message(STATUS "Rust Compiler Version: ${RUSTC_VERSION}") - -mark_as_advanced(CMAKE_Rust_COMPILER) - -if(CMAKE_Rust_COMPILER) - set(CMAKE_Rust_COMPILER_LOADED 1) -endif(CMAKE_Rust_COMPILER) - -configure_file(${CMAKE_SOURCE_DIR}/cmake/CMakeRustCompiler.cmake.in - ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/${CMAKE_VERSION}/CMakeRustCompiler.cmake IMMEDIATE @ONLY) - -set(CMAKE_Rust_COMPILER_ENV_VAR "RUSTC") - diff --git a/cmake/CMakeRustCompiler.cmake.in b/cmake/CMakeRustCompiler.cmake.in deleted file mode 100644 index 5916c1c3d..000000000 --- a/cmake/CMakeRustCompiler.cmake.in +++ /dev/null @@ -1,15 +0,0 @@ - -set(CMAKE_Rust_COMPILER "@CMAKE_Rust_COMPILER@") -set(CMAKE_Rust_COMPILER_ID "@CMAKE_Rust_COMPILER_ID@") -set(CMAKE_Rust_COMPILER_VERSION "@CMAKE_Rust_COMPILER_VERSION@") -set(CMAKE_Rust_COMPILER_LOADED @CMAKE_Rust_COMPILER_LOADED@) -set(CMAKE_Rust_PLATFORM_ID "@CMAKE_Rust_PLATFORM_ID@") - -SET(CMAKE_Rust_SOURCE_FILE_EXTENSIONS rs) -SET(CMAKE_Rust_LINKER_PREFERENCE 40) -#SET(CMAKE_Rust_OUTPUT_EXTENSION_REPLACE 1) -SET(CMAKE_STATIC_LIBRARY_PREFIX_Rust "") -SET(CMAKE_STATIC_LIBRARY_SUFFIX_Rust .a) - -set(CMAKE_Rust_COMPILER_ENV_VAR "RUSTC") - diff --git a/cmake/CMakeRustInformation.cmake b/cmake/CMakeRustInformation.cmake deleted file mode 100644 index 05d96c94b..000000000 --- a/cmake/CMakeRustInformation.cmake +++ /dev/null @@ -1,106 +0,0 @@ - -# -# Usage: rustc [OPTIONS] INPUT -# -# Options: -# -h --help Display this message -# --cfg SPEC Configure the compilation environment -# -L [KIND=]PATH Add a directory to the library search path. The -# optional KIND can be one of dependency, crate, native, -# framework or all (the default). -# -l [KIND=]NAME Link the generated crate(s) to the specified native -# library NAME. The optional KIND can be one of static, -# dylib, or framework. If omitted, dylib is assumed. -# --crate-type [bin|lib|rlib|dylib|cdylib|staticlib|metadata] -# Comma separated list of types of crates for the -# compiler to emit -# --crate-name NAME Specify the name of the crate being built -# --emit [asm|llvm-bc|llvm-ir|obj|link|dep-info] -# Comma separated list of types of output for the -# compiler to emit -# --print [crate-name|file-names|sysroot|cfg|target-list|target-cpus|target-features|relocation-models|code-models] -# Comma separated list of compiler information to print -# on stdout -# -g Equivalent to -C debuginfo=2 -# -O Equivalent to -C opt-level=2 -# -o FILENAME Write output to -# --out-dir DIR Write output to compiler-chosen filename in -# --explain OPT Provide a detailed explanation of an error message -# --test Build a test harness -# --target TARGET Target triple for which the code is compiled -# -W --warn OPT Set lint warnings -# -A --allow OPT Set lint allowed -# -D --deny OPT Set lint denied -# -F --forbid OPT Set lint forbidden -# --cap-lints LEVEL Set the most restrictive lint level. More restrictive -# lints are capped at this level -# -C --codegen OPT[=VALUE] -# Set a codegen option -# -V --version Print version info and exit -# -v --verbose Use verbose output -# -# Additional help: -# -C help Print codegen options -# -W help Print 'lint' options and default settings -# -Z help Print internal options for debugging rustc -# --help -v Print the full set of options rustc accepts -# - -# - -include(CMakeLanguageInformation) - -if(UNIX) - set(CMAKE_Rust_OUTPUT_EXTENSION .o) -else() - set(CMAKE_Rust_OUTPUT_EXTENSION .obj) -endif() - -set(CMAKE_Rust_ECHO_ALL "echo \"TARGET: TARGET_BASE: ") -set(CMAKE_Rust_ECHO_ALL "${CMAKE_Rust_ECHO_ALL} OBJECT: OBJECTS: OBJECT_DIR: SOURCE: SOURCES: ") -set(CMAKE_Rust_ECHO_ALL "${CMAKE_Rust_ECHO_ALL} LINK_LIBRARIES: FLAGS: LINK_FLAGS: \"") - -if(NOT CMAKE_Rust_CREATE_SHARED_LIBRARY) - set(CMAKE_Rust_CREATE_SHARED_LIBRARY - "echo \"CMAKE_Rust_CREATE_SHARED_LIBRARY\"" - "${CMAKE_Rust_ECHO_ALL}" - ) -endif() - -if(NOT CMAKE_Rust_CREATE_SHARED_MODULE) - set(CMAKE_Rust_CREATE_SHARED_MODULE - "echo \"CMAKE_Rust_CREATE_SHARED_MODULE\"" - "${CMAKE_Rust_ECHO_ALL}" - ) -endif() - -if(NOT CMAKE_Rust_CREATE_STATIC_LIBRARY) - set(CMAKE_Rust_CREATE_STATIC_LIBRARY - "echo \"CMAKE_Rust_CREATE_STATIC_LIBRARY\"" - "${CMAKE_Rust_ECHO_ALL}" - ) -endif() - -if(NOT CMAKE_Rust_COMPILE_OBJECT) - set(CMAKE_Rust_COMPILE_OBJECT - "echo \"CMAKE_Rust_COMPILE_OBJECT\"" - "${CMAKE_Rust_ECHO_ALL}" - "${CMAKE_Rust_COMPILER} --emit obj -o ") -endif() - -if(NOT CMAKE_Rust_LINK_EXECUTABLE) - set(CMAKE_Rust_LINK_EXECUTABLE - "echo \"CMAKE_Rust_LINK_EXECUTABLE\"" - "${CMAKE_Rust_ECHO_ALL}" - ) -endif() - -mark_as_advanced( - CMAKE_Rust_FLAGS - CMAKE_Rust_FLAGS_DEBUG - CMAKE_Rust_FLAGS_MINSIZEREL - CMAKE_Rust_FLAGS_RELEASE - CMAKE_Rust_FLAGS_RELWITHDEBINFO) - -set(CMAKE_Rust_INFORMATION_LOADED 1) - diff --git a/cmake/CMakeTestRustCompiler.cmake b/cmake/CMakeTestRustCompiler.cmake deleted file mode 100644 index ff75bb3e0..000000000 --- a/cmake/CMakeTestRustCompiler.cmake +++ /dev/null @@ -1,3 +0,0 @@ - -set(CMAKE_Rust_COMPILER_WORKS 1 CACHE INTERNAL "") - diff --git a/cmake/CargoLink.cmake b/cmake/CargoLink.cmake new file mode 100644 index 000000000..940eb907f --- /dev/null +++ b/cmake/CargoLink.cmake @@ -0,0 +1,105 @@ +# ccommon - a cache common library. +# Copyright (C) 2019 Twitter, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Ensure that empty elements in lists aren't deleted +cmake_policy(SET CMP0007 NEW) + +# Ignore the first 3 arguments since they will always be cmake -P /LinkRust.cmake +set(ARGI 3) +# This flips once we see -- in the arguments +set(PARSING_ENV_VARS OFF) +# Arguments to be passed directly to the build command +set(PASSTHROUGH_VARS ) + +# Split up the command-line arguments to this script +# into two groups +# +# Arguments before '--' are cmake variables that we set +# in this script. These are parameters which control the +# behaviour here. +# +# Arguments after '--' are environment variables to pass +# through to the cargo invocation, they are used by build +# scripts and to control cargo behaviour. +while(ARGI LESS ${CMAKE_ARGC}) + set(CURRENT_ARG ${CMAKE_ARGV${ARGI}}) + + if(NOT PARSING_ENV_VARS) + if(CURRENT_ARG STREQUAL "--") + set(PARSING_ENV_VARS ON) + else() + string(REPLACE "=" ";" ARGLIST "${CURRENT_ARG}") + + list(GET ARGLIST 0 VAR) + list(REMOVE_AT ARGLIST 0) + string(REPLACE ";" "=" VALUE "${ARGLIST}") + + set(${VAR} "${VALUE}") + endif() + else() + list(APPEND PASSTHROUGH_VARS "${CURRENT_ARG}") + endif() + + math(EXPR ARGI "${ARGI} + 1") +endwhile() + +file( + WRITE + "${LINK_FLAGS_FILE}" + ${LINK_FLAGS} ${LINK_LIBRARIES} +) + +# This converts a space-delimited string to a cmake list +string(REPLACE " " ";" LINK_FLAGS_LIST "${LINK_LIBRARIES}" "${LINK_FLAGS}") +set(LINK_FLAGS ) + +# To pass linker args through cargo we need to use +# the -Clink-arg= syntax. +foreach(FLAG ${LINK_FLAGS_LIST}) + if(EXISTS "${FLAG}") + get_filename_component(FLAG "${FLAG}" ABSOLUTE) + endif() + + list(APPEND LINK_FLAGS "-Clink-arg=${FLAG}") +endforeach() + +string(REPLACE ";" " " LINK_FLAGS "${LINK_FLAGS}") +string(REPLACE " " ";" FLAGS "${FLAGS}") + +# TODO(sean): We don't always want to colour the output. Is +# there a way to autodetect this properly? +set(CARGO_COMMAND cargo build --color always ${FLAGS}) + +execute_process( + COMMAND ${CMAKE_COMMAND} -E env ${PASSTHROUGH_VARS} "RUSTFLAGS=${LINK_FLAGS}" ${CARGO_COMMAND} + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + RESULT_VARIABLE STATUS +) + +# Ensure that our script exits with the correct error code. +# The only way to get a cmake script to exit with an error +# code is to print a message so that's what we do here. +if(NOT STATUS EQUAL 0) + message(FATAL_ERROR "Cargo build failed") +endif() + +# Get the directory above TARGET since file(COPY ...) +# uses a directory as the destination +get_filename_component( + TARGET "${TARGET}/.." + ABSOLUTE +) + +file(COPY "${OUTPUT}" DESTINATION "${TARGET}") diff --git a/cmake/CargoTest.cmake b/cmake/CargoTest.cmake new file mode 100644 index 000000000..88bfeebb6 --- /dev/null +++ b/cmake/CargoTest.cmake @@ -0,0 +1,99 @@ +# ccommon - a cache common library. +# Copyright (C) 2019 Twitter, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Ensure that empty elements in lists aren't deleted +cmake_policy(SET CMP0007 NEW) + +# Ignore the first 3 arguments since they will always be cmake -P /LinkRust.cmake +set(ARGI 3) +# This flips once we see -- in the arguments +set(PARSING_ENV_VARS OFF) +# Arguments to be passed directly to the build command +set(PASSTHROUGH_VARS ) + +# Split up the command-line arguments to this script +# into two groups +# +# Arguments before '--' are cmake variables that we set +# in this script. These are parameters which control the +# behaviour here. +# +# Arguments after '--' are environment variables to pass +# through to the cargo invocation, they are used by build +# scripts and to control cargo behaviour. +while(ARGI LESS ${CMAKE_ARGC}) + set(CURRENT_ARG ${CMAKE_ARGV${ARGI}}) + + if(NOT PARSING_ENV_VARS) + if(CURRENT_ARG STREQUAL "--") + set(PARSING_ENV_VARS ON) + else() + string(REPLACE "=" ";" ARGLIST "${CURRENT_ARG}") + + list(GET ARGLIST 0 VAR) + list(REMOVE_AT ARGLIST 0) + string(REPLACE ";" "=" VALUE "${ARGLIST}") + + set(${VAR} "${VALUE}") + endif() + else() + list(APPEND PASSTHROUGH_VARS "${CURRENT_ARG}") + endif() + + math(EXPR ARGI "${ARGI} + 1") +endwhile() + +file( + READ + "${LINK_FLAGS_FILE}" + LINK_FLAGS +) + +# This converts a space-delimited string to a cmake list +string(REPLACE " " ";" LINK_FLAGS_LIST "${LINK_FLAGS}") +set(LINK_FLAGS ) + +# To pass linker args through cargo we need to use +# the -Clink-arg= syntax. +foreach(FLAG ${LINK_FLAGS_LIST}) + if(EXISTS "${CMAKE_CURRENT_BINARY_DIR}/${FLAG}") + get_filename_component(FLAG "${CMAKE_CURRENT_BINARY_DIR}/${FLAG}" ABSOLUTE) + endif() + + list(APPEND LINK_FLAGS "-Clink-arg=${FLAG}") +endforeach() + +string(REPLACE ";" " " LINK_FLAGS "${LINK_FLAGS}") +string(REPLACE " " ";" FLAGS "${FLAGS}") + +# TODO(sean): We don't always want to colour the output. Is +# there a way to autodetect this properly? +set(CARGO_COMMAND cargo test --color always ${FLAGS}) + +execute_process( + COMMAND ${CMAKE_COMMAND} -E env ${PASSTHROUGH_VARS} "RUSTFLAGS=${LINK_FLAGS}" ${CARGO_COMMAND} + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + RESULT_VARIABLE STATUS +) + +# Ensure that our script exits with the correct error code. +# The only way to get a cmake script to exit with an error +# code is to print a message so that's what we do here. +if(NOT STATUS EQUAL 0) + # Dump some script state to help with debugging + message(STATUS "PASSTHROUGH_VARS=${PASSTHROUGH_VARS}") + + message(FATAL_ERROR "cargo test failed") +endif() diff --git a/cmake/FindITTNOTIFY.cmake b/cmake/FindITTNOTIFY.cmake new file mode 100644 index 000000000..0c1ae6bc5 --- /dev/null +++ b/cmake/FindITTNOTIFY.cmake @@ -0,0 +1,65 @@ +# ITTNOTIFY is the instrumentation and tracing technology (ITT) APIs provided by +# the Intel® VTune™Amplifier enable your application to generate and control +# the collection of trace data during its execution. +# +# The following variables are set when ITTNOTIFY is found: +# ITTNOTIFY_FOUND = Set to true, if all components of ITTNOTIFY have been found. +# ITTNOTIFY_INCLUDE_DIRS = Include path for the header files of ITTNOTIFY. +# ITTNOTIFY_LIBRARY_DIRS = Library search path for the ITTNOTIFY libraries. +# ITTNOTIFY_LIBRARIES = Link these to use ITTNOTIFY. +# ITTNOTIFY_LFLAGS = Linker flags (optional). + +if (NOT ITTNOTIFY_FOUND) + + find_program(VTUNE_EXECUTABLE amplxe-cl) + + if(NOT VTUNE_EXECUTABLE) + set(ITTNOTIFY_FOUND false) + return() + endif() + + get_filename_component(VTUNE_DIR ${VTUNE_EXECUTABLE} PATH) + set(ITTNOTIFY_ROOT_DIR "${VTUNE_DIR}/..") + + ##_____________________________________________________________________________ + ## Check for the header files + + find_path (ITTNOTIFY_INCLUDE_DIRS + NAMES ittnotify.h + HINTS ${ITTNOTIFY_ROOT_DIR} + PATHS /usr /usr/local + PATH_SUFFIXES include + ) + + ##_____________________________________________________________________________ + ## Check for the library + + find_library (ITTNOTIFY_LIBRARIES ittnotify + HINTS ${ITTNOTIFY_ROOT_DIR} + PATHS /usr /usr/local /opt/local + PATH_SUFFIXES lib64 lib + ) + + ##_____________________________________________________________________________ + ## Actions taken when all components have been found + + find_package_handle_standard_args (ITTNOTIFY DEFAULT_MSG ITTNOTIFY_LIBRARIES ITTNOTIFY_INCLUDE_DIRS) + + if (ITTNOTIFY_FOUND) + if (NOT ITTNOTIFY_FIND_QUIETLY) + message (STATUS "Found components for ITTNOTIFY") + message (STATUS "ITTNOTIFY_ROOT_DIR = ${ITTNOTIFY_ROOT_DIR}") + message (STATUS "ITTNOTIFY_INCLUDE_DIRS = ${ITTNOTIFY_INCLUDE_DIRS}") + message (STATUS "ITTNOTIFY_LIBRARIES = ${ITTNOTIFY_LIBRARIES}") + endif (NOT ITTNOTIFY_FIND_QUIETLY) + else (ITTNOTIFY_FOUND) + if (ITTNOTIFY_FIND_REQUIRED) + message (FATAL_ERROR "Could not find ITTNOTIFY!") + endif (ITTNOTIFY_FIND_REQUIRED) + endif (ITTNOTIFY_FOUND) + + if(UNIX) + list(APPEND ITTNOTIFY_LIBRARIES dl) + endif() + +endif (NOT ITTNOTIFY_FOUND) diff --git a/cmake/FindRust.cmake b/cmake/FindRust.cmake index 2dbf14ddf..53127a103 100644 --- a/cmake/FindRust.cmake +++ b/cmake/FindRust.cmake @@ -4,55 +4,53 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM BOTH) set(_CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ${CMAKE_FIND_ROOT_PATH_MODE_INCLUDE}) set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE BOTH) -if(WIN32) +if(CMAKE_HOST_WIN32) set(USER_HOME "$ENV{USERPROFILE}") else() set(USER_HOME "$ENV{HOME}") endif() -# Find cargo prefix -find_path(CARGO_PREFIX ".cargo" - PATHS "${USER_HOME}") - -if(CARGO_PREFIX MATCHES "NOTFOUND") - message(FATAL_ERROR "Could not find Rust!") -else() - set(CARGO_PREFIX "${CARGO_PREFIX}/.cargo") +if(NOT DEFINED CARGO_HOME) + if("$ENV{CARGO_HOME}" STREQUAL "") + set(CARGO_HOME "${USER_HOME}/.cargo") + else() + set(CARGO_HOME "$ENV{CARGO_HOME}") + endif() endif() # Find cargo executable find_program(CARGO_EXECUTABLE cargo - HINTS "${CARGO_PREFIX}" + HINTS "${CARGO_HOME}" PATH_SUFFIXES "bin") mark_as_advanced(CARGO_EXECUTABLE) # Find rustc executable find_program(RUSTC_EXECUTABLE rustc - HINTS "${CARGO_PREFIX}" + HINTS "${CARGO_HOME}" PATH_SUFFIXES "bin") mark_as_advanced(RUSTC_EXECUTABLE) # Find rustdoc executable find_program(RUSTDOC_EXECUTABLE rustdoc - HINTS "${CARGO_PREFIX}" + HINTS "${CARGO_HOME}" PATH_SUFFIXES "bin") mark_as_advanced(RUSTDOC_EXECUTABLE) # Find rust-gdb executable find_program(RUST_GDB_EXECUTABLE rust-gdb - HINTS "${CARGO_PREFIX}" + HINTS "${CARGO_HOME}" PATH_SUFFIXES "bin") mark_as_advanced(RUST_GDB_EXECUTABLE) # Find rust-lldb executable find_program(RUST_LLDB_EXECUTABLE rust-lldb - HINTS "${CARGO_PREFIX}" + HINTS "${CARGO_HOME}" PATH_SUFFIXES "bin") mark_as_advanced(RUST_LLDB_EXECUTABLE) # Find rustup executable find_program(RUSTUP_EXECUTABLE rustup - HINTS "${CARGO_PREFIX}" + HINTS "${CARGO_HOME}" PATH_SUFFIXES "bin") mark_as_advanced(RUSTUP_EXECUTABLE) @@ -61,7 +59,7 @@ set(RUST_FOUND FALSE CACHE INTERNAL "") if(CARGO_EXECUTABLE AND RUSTC_EXECUTABLE AND RUSTDOC_EXECUTABLE) set(RUST_FOUND TRUE CACHE INTERNAL "") - set(CARGO_PREFIX "${CARGO_PREFIX}" CACHE PATH "Rust Cargo prefix") + set(CARGO_HOME "${CARGO_HOME}" CACHE PATH "Rust Cargo Home") execute_process(COMMAND ${RUSTC_EXECUTABLE} --version OUTPUT_VARIABLE RUSTC_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE) string(REGEX REPLACE "rustc ([^ ]+) .*" "\\1" RUSTC_VERSION "${RUSTC_VERSION}") diff --git a/config.h.in b/config.h.in index f1dc18bd9..f2fe91ba5 100644 --- a/config.h.in +++ b/config.h.in @@ -12,10 +12,12 @@ #cmakedefine HAVE_BACKTRACE -#cmakedefine HAVE_BIG_ENDIAN +#cmakedefine HAVE_ACCEPT4 #cmakedefine HAVE_LOGGING #cmakedefine HAVE_STATS #cmakedefine HAVE_DEBUG_MM + +#cmakedefine HAVE_ITT_INSTRUMENTATION diff --git a/docs/c-styleguide.txt b/docs/c-styleguide.txt deleted file mode 100644 index 69661c0d8..000000000 --- a/docs/c-styleguide.txt +++ /dev/null @@ -1,427 +0,0 @@ -- No literal tabs. Expand tabs to 4 spaces. -- Indentation is 4 spaces. -- No more than 3 levels of indentation, otherwise you should think about - refactoring your code. -- Use one statement per line. -- Make sure that your editor does not leave space at the end of the line. -- snake_case for variable, function and file names. -- Use your own judgment when naming variables and functions. Be as Spartan - as possible. Eg: Using name like this_variable_is_a_temporary_counter - will usually be frowned upon. -- Don’t use local variables or parameters that shadow global identifiers. - GCC’s ‘-Wshadow’ option can help you to detect this problem. - -- Avoid using int, char, short, long. Instead use int8_t uint8_t, int16_t, - uint16_t, int32_t, uint32_t, int64_t, uint64_t, which are available in - . However, when interfacing with system calls and libraries - you cannot get away from using int and char. -- Use bool for boolean variables. You have to include -- If memory usage or alignment is a concern, avoid using a bool as type for - struct member names. Instead use unsigned 1-bit bit field. e.g. - struct foo { - unsigned is_bar:1; - }; - However, if neither memory usage or alignment will be significantly impacted - by the struct, opt for using bool for the sake of readability. -- Always use size_t type when dealing with sizes of objects or memory ranges. -- Your code should be 64-bit and 32-bit friendly. Bear in mind problems - of printing, comparisons, and structure alignment. You have to include - to get generic format specifier macros for printing. - -- 80 column line limit. -- If you have to wrap a long statement (> 80 column), put the operator at the - end of the line and indent the next line at the same column as the arguments - in the previous column. Eg: - while (cnt < 20 && this_variable_name_is_too_long && - ep != NULL) { - z = a + really + long + statement + that + needs + three + lines + - gets + indented + on + the + same + column + as + the + - previous + column - } - - and: - - int a = function(param_a, param_b, param_c, param_d, param_e, param_f, - param_g, param_h, param_i, param_j, param_k, param_l); - -- Always use braces for all conditional blocks (if, switch, for, while, do). - This holds good even for single statement conditional blocks. Eg: - if (cond) { - stmt; - } -- Placement of braces for non-function statement blocks - put opening brace - last on the line and closing brace first. Eg: - if (x is true) { - we do y - } -- Placement of brace for functions - put the opening brace at the beginning - of the next line and closing brace first. This is useful because several - tools look for opening brace in column one to find beginning of C - functions. Eg: -int -function(int x) -{ - body of the function -} - -- Closing brace is empty on a line of its own, except in cases where it is - followed by a continuation of the same statement, i.e. a "while" in a - do-statement or an "else" in an if-statement, like this: - do { - body of do-loop - } while (condition); - - and, - - if (x == y) { - .. - } else if (x > y) { - ... - } else { - .... - } - -- Column align switch keyword and the corresponding case/default keyword. Eg: - switch (alphabet) { - case 'a': - case 'b': - printf("I am a or b\n"); - break; - - default: - break; - } - -- Forever loops are done with for, and not while. Eg: - for (;;) { - stmt; - } - -- Don't use a space after a function name. -- Do not needlessly surround the return expression with parentheses. -- Use space after keywords. Exceptions are sizeof, typeof, alignof and - __attribute__, which look like functions. -- Do not add spaces around (inside) parenthesized expressions. - s = sizeof( sizeof(*p)) ); /* bad example */ - s = sizeof(sizeof(*p)); /* good example */ -- Casts should not be followed by space. Eg: - int q = *(int *)&p -- There is no need to type cast when assigning a void pointer to a non-void - pointer, or vice versa. -- Avoid using goto statements. However there are some exceptions to this rule - when a single goto label within a function and one or more goto statements - come in handy when a function exits from multiple locations and some common - work such as cleanup has to be done. Eg: -int -fun(void) -{ - int result = 0; - char *buffer; - - buffer = malloc(1024); - if (buffer == NULL) { - return -1; - } - - if (condition1) { - while (loop1) { - ... - } - result = 1; - goto out; - } - - ... -out: - free(buffer); - return result; -} -- When declaring pointer data, use '*' adjacent to the data name and not - adjacent to the type name. Eg: - int - function(int *p) - { - char *p; - - } -- Use one space around (on each side of) most binary and ternary operators, - such as any of these: - = + - < > * / % | & ^ <= >= == != ? : - but no space after unary operators: - & * + - ~ ! sizeof typeof alignof __attribute__ defined - no space before the postfix increment & decrement unary operators: - ++ -- - and no space around the '.' and "->" structure member operators. - -- 0 and NULL; use 0 for integers, 0.0 for doubles, NULL for pointers, and - '\0' for chars. -- Test pointers against NULL. E.g, use: - if (p == NULL) - - not: - - !(p) - -- Do not use ! for tests unless it is a boolean. E.g. use: - if (*p == '\0') - - not: - - if (!*p) - -- Don't use assignments inside if or while-conditions. E.g, use: - - struct foo *foo; - foo = malloc(sizeof(*foo)); - if (foo == NULL) { - return -1 - } - - not: - - struct foo *foo; - if ((foo = malloc(sizeof(*foo))) == NULL) { - return -1; - } - -- Don't ever use typedef for structure types. Typedefs are problematic - because they do not properly hide their underlying type; for example you - need to know if the typedef is the structure itself or a pointer to the - structure. In addition they must be declared exactly once, whereas an - incomplete structure type can be mentioned as many times as necessary. - Typedefs are difficult to use in stand-alone header files: the header - that defines the typedef must be included before the header that uses it, - or by the header that uses it (which causes namespace pollution), or - there must be a back-door mechanism for obtaining the typedef. -- The only exception for using a typedef is when you are defining a type - for a function pointer or a type for an enum. Eg: - - typedef void (*foo_handler_t)(int, void *); - - or: - - typedef enum types { - TYPE_1, - TYPE_2 - } types_t; - -- Use just one variable declaration per line when variables are part of a - struct. This leaves you room for a small comment on each item, explaining - its use. Declarations should also be aligned. Eg, use: - - struct foo { - int *foo_a; /* comment for foo_a */ - int foo_b; /* comment for foo_b */ - unsigned foo_c:1; /* comment for foo_c */ - }; - - and not: - - struct foo { - int *foo_a, foo_b; - unsigned foo_c:1; - }; - -- For variable declaration outside a struct, either collect all the - declarations of the same type on a single line, or use one variable - per line if the variables purpose needs to be commented. Eg: - char *a, *b, c; - - or: - - char *a, *b; - char c; /* comments for c */ - -- Avoid magic numbers because no-one has a clue (including the author) of - what it means after a month. - -- Function definitions should start the name of the function in column - one. This is useful because it makes searching for function definitions - fairly trivial. Eg: -static char * -concat(char *s1, char *s2) -{ - body of the function -} - -- Function and variables local to a file should be static. -- Separate two successive functions with one blank line. -- Include parameter names with their datypes in function declaration. Eg: -void function(int param); - -- Functions should be short and sweet, and do just one thing. They should - fit on one or two screenfuls of text (80 x 24 screen size), and do one - thing and do that well. - The maximum length of a function is inversely proportional to the - complexity and indentation level of that function. So, if you have a - conceptually simple function that is just one long (but simple) - case-statement, where you have to do lots of small things for a lot of - different cases, it's OK to have a longer function. - Another measure of the function is the number of local variables. They - shouldn't exceed 5-10, or you're doing something wrong. Re-think the - function, and split it into smaller pieces. A human brain can - generally easily keep track of about 7 different things, anything more - and it gets confused. You know you're brilliant, but maybe you'd like - to understand what you did 2 weeks from now. -- Use const for function parameters passed by reference, if the passed - pointer has no side effect. - -- C style comments only. Don't use // for single line comments. Instead - use /* ... */ style. -- For multi-line comments use the following style - /* - * This is the preferred style for multi-line - * comments in the Linux kernel source code. - * Please use it consistently. - * - * Description: A column of asterisks on the left side, - * with beginning and ending almost-blank lines. - */ - -- To comment out block of code spanning several lines use preprocessor - directive "#ifdef 0 ... #endif" - -- Please write a brief comment at the start of each source file, with the - file name and a line or two about the overall purpose of the file. - -- All major functions should have comments describing what they do at the - head of the function. Avoid putting comments in the function body unless - absolutely needed. If possible, add a comment on what sorts of arguments - the function gets, and what the possible values of arguments mean and - what they are used for and the significance of return value if there is - one. It is not necessary to duplicate in words the meaning of the C - argument declarations, if a C type is being used in its customary fashion. - If there is anything nonstandard about its use (such as an argument of - type char * which is really the address of the second character of a - string, not the first), or any possible values that would not work the - way one would expect (such as, that strings containing newlines are not - guaranteed to work), be sure to say so. Eg: - -/* - * Try to acquire a physical address lock while a pmap is locked. If we - * fail to trylock we unlock and lock the pmap directly and cache the - * locked pa in *locked. The caller should then restart their loop in case - * the virtual to physical mapping has changed. - * - * Returns 0 on success and -1 on failure. - */ -int -vm_page_pa_tryrelock(pmap_t pmap, vm_paddr_t pa, vm_paddr_t *locked) -{ - ... - -- The comment on a function is much clearer if you use the argument names - to speak about the argument values. The variable name itself should be - lower case, but write it in upper case when you are speaking about the - value rather than the variable itself. Thus, “the inode number NODE_NUM” - rather than “an inode”. - -- Every struct definition should have an accompanying comment that - describes what it is for and how it should be used. - -- Finally, while comments are absolutely important to keep the code readable, - remember that the best code is self-documenting. Giving sensible names to - types and variables is much better than using obscure names that you must - then explain through comments. - -- Recommend using UPPERCASE for macro names. However, sometimes using - lowercase for macro names makes sense when macros masquerade as well-known - function calls. Eg, it makes sense to write the wrapper for the - standard free() function in lowercase to keep the readability - consistent: - -#define my_free(_p) do { \ - free(_p); \ - (_p) = NULL; \ -} while (0) - -- Use enums when defining more than one related constants. All enumeration - values are in UPPERCASE. -- Avoid macros as much as possible and use inline functions, enums and const - variables wherever you can. -- For macros encapsulating compound statements, right justify the backslashes - and enclose it in do { ... } while (0) -- For parameterized macros, all the parameters used in the macro body must - be surrounded by parentheses. Eg: - #define ADD_1(_x) ((_x) + 1) - -- Use sizeof(varname) instead of sizeof(type) whenever possible. Eg: - char *p; - p = malloc(sizeof(*p)); /* good example */ - p = malloc(sizeof(char)); /* bad example */ - -- All variables should be declared at the beginning of a scope block {..}. - It is even preferred to declare all variables at the beginning of the - function so that all the local variable declarations is in one place and - we can see the comprehensive list in one glance. -- Global structs should be declared at the top of the file in which they - are used, or in separate header files if they are used in multiple - source files. -- Declarations of external functions and functions to appear later in the - source file should all go in one place near the beginning of the file, - somewhere before the first function definition in the file or else - should go in a header file. -- Use of extern should be considered as evil, if it is used in header files - to reference global variables. -- Don’t put extern declarations inside functions. - -- Usually every *.c file should have an associated *.h file. There are some - exceptions to this rule, such as unit tests and small *.c files containing - just the main() function. -- Every header file in the source code must have preprocessor conditional - to prevent the header file from being scanned multiple times and avoiding - mutual dependency cycles. Alternatively you can use #pragma once directive, - as it avoids name clashes and increases the compile speed. Eg, for a - header file named foo.h, the entire contents of the header file must be - between the guard macros as follows: - -#ifndef _FOO_H_ -#define _FOO_H_ -... -#endif /* _FOO_H_ */ - -Or, - -#pragma once -#ifndef _FOO_H_ -#define _FOO_H_ -... -#endif /* _FOO_H_ */ - -- Don't use #include when a forward declaration would suffice. -- Functions defined in header files should be static inline. - -- Don’t make the program ugly just to placate GCC when extra warnings options - such as ‘-Wconversion’ or ‘-Wundef’ are used. These options can help in - finding bugs, but they can also generate so many false alarms that that - it hurts readability to silence them with unnecessary casts, wrappers, and - other complications. - -- Conditional compilation: when supporting configuration options already - known when building your program we prefer using if (... ) over conditional - compilation, as in the former case the compiler is able to perform more - extensive checking of all possible code paths. Eg, use: - - if (HAS_FOO) - ... - else - ... - -instead of: - - #ifdef HAS_FOO - ... - #else - ... - #endif - - A modern compiler such as GCC will generate exactly the same code in both - cases and of course, the former method assumes that HAS_FOO is defined as - either 0 or 1. - -- Finally, rules are rules. Sometimes they are sensible and sometimes not - and regardless of your preference, we would like you to follow them. - A project is easier to follow if all project contributors follow the style - rules so that they can all read and understand everyone's code easily. But - remember, like all good rules, they are exceptions where it makes sense not - to be too rigid on the grounds of common sense and consistency! diff --git a/docs/coding_style.rst b/docs/coding_style.rst new file mode 100644 index 000000000..0a6a7bdfe --- /dev/null +++ b/docs/coding_style.rst @@ -0,0 +1,583 @@ +Introduction +============ + +The C Style Guide is a set of guidelines and conventions that encourage +good code. While some suggestions are more strict than others, you should +always practice good judgement. + +If following the guide causes unnecessary hoop-jumping or otherwise +less-readable code, *readability trumps the guide*. However, if the more +readable variant comes with perils or pitfalls, readability may be +sacrificed. + +Consistency is crucial. Without consistent application, there simply is no style +to speak of [#fn1]_. Stay in sync with the rest of the codebase; when you want +to change a rule or style, change it everywhere. + +Contents +-------- + +.. toctree:: + :maxdepth: 1 + + coding_style + + +C Standard +========== + +- Use ``-std=c11`` when compiling +- Avoid ``_Atomic``, ``_Generic`` and ``_Thread_local``, for now. We will + embrace ``C11`` fully when Twitter's official ``GCC`` is bumped to 4.9. + +Indentation +=========== + +- Do not use literal tabs. Expand tabs to **four** spaces instead. +- Use **four** spaces for every indentation level. +- Do not use more than **four** levels of indentation unless there's a good + reason. +- Make sure that your editor does not leave space at the end of each line. +- When a block of code resembles a table, such as in a macro-block or struct + definition, start each field on a ``4k+1`` column, so all fields are **four** + space aligned. + + .. code-block:: c + + /* name type description */ + #define FOO_METRIC(ACTION) \ + ACTION( foo_free, METRIC_GAUGE, "# free foo" )\ + ACTION( foo_borrow, METRIC_COUNTER, "# foos borrowed" )\ + ACTION( foo_return, METRIC_COUNTER, "# foos returned" )\ + /* name starts on column 13 + * type starts on column 29 + * description on column 45 + */ + + struct foo { + struct foo *next; + struct mumble amumble; + int bar; + }; + /* type starts on column 5, name on column 21 */ + + +Switch alignment +---------------- + +Align the ``switch`` keyword and the corresponding ``case`` and ``default`` +keywords to the same column. For example: + +.. code-block:: c + + switch (alphabet) { + case 'a': + case 'b': + printf("I am a or b\n"); + break; + default: + break; + } + + +Naming +====== + +- Use ``snake_case`` for the names of variables, functions, and files. +- Use your own judgement when you name variables and be as spartan as possible, + abbreviation is common in C. + For example, do not use a name like ``this_variable_is_a_temporary_counter``. + + +Types +===== + +- For variables of the following types: + + - ``int`` + - ``char`` + - ``short`` + - ``long`` + + Prefer the following types declared in the ```` header: + + - ``int8_t`` + - ``uint8_t`` + - ``int16_t`` + - ``uint16_t`` + - ``int32_t`` + - ``uint32_t`` + - ``int64_t`` + - ``uint64_t`` + + The latter types give us more predictable value range and memory layout on + different platforms. + +- Use the `bool` type for boolean data. You have to include the ```` + header. +- Always use the ``size_t`` type when you work with: + + - Sizes of objects + - Memory ranges + + +Line length +=========== + +- Limit each line to 80 columns or less. +- If you have to wrap a longer statement, put the operator at the end of the + line and use **eight** spaces to indent the next line. Indentation on the + next level is not affected. For example: + + .. code-block:: c + + while (cnt < 20 && this_variable_name_is_too_long && + ep != NULL) { + z = a + really + long + statement + that + needs + + two + lines + gets + indented + four + spaces + + on + the + second + and + subsequent + lines; + } + + and: + + .. code-block:: c + + int a = function(param_a, param_b, param_c, param_d, param_e, param_f, + param_g, param_h, param_i, param_j, param_k, param_l); + + +Braces +====== + +- Always use braces for all conditional blocks (``if``, ``switch``, ``for``, + ``while``, and ``do``), even for single statement conditional blocks (remember + the ``goto fail`` bug by Apple?). For example: + + .. code-block:: c + + if (cond) { + stmt; + } + +- For non-function statement blocks, put the opening brace at the end of the + first line and the closing brace in a new line. For example: + + .. code-block:: c + + if (x) { + foo(); + } + +- For functions, put the opening brace at the beginning of the second line + and the closing brace in a new line. For example: + + .. code-block:: c + + int + function(int x) + { + body of the function + } + +- Place the closing brace in its own line, except when it is part of the same + statement, such as a ``while`` in a ``do``-statement or an ``else`` in + an ``if``-statement. For example: + + .. code-block:: c + + do { + body of do-loop + } while (condition); + + and, + + if (x == y) { + .. + } else if (x > y) { + ... + } else { + .... + } + + +Infinite loops +============= + +Create infinite loops with ``for`` statements, not ``while`` statements. +For example: + +.. code-block:: c + + for (;;) { + stmt; + } + + +Spaces +====== + +- Do not use a space after a function name. +- Use space after keywords, except after the ``sizeof``, ``typeof``, ``alignof`` + and ``__attribute__`` keywords, since they are used like functions. +- Do not add spaces inside parenthesized expressions. For example: + + .. code-block:: c + + s = sizeof( sizeof(*p)) ); /* Bad example */ + s = sizeof(sizeof(*p)); /* Good example */ + +- When declaring pointers, place the asterisk ('*'') adjacent to the variable + name, not the type name. For example: + + .. code-block:: c + + int + function(int *p) + { + char *p; + body of the function + } + +- Use one space around most binary and ternary operators, such as any of these: + + ``=`` ``+`` ``-`` ``<`` ``>`` ``*`` ``/`` ``%`` ``|`` ``&`` ``^`` + ``<=`` ``>=`` ``==`` ``!=`` ``?`` ``:`` + + Do not add spaces after unary operators: + + ``&`` ``*`` ``+`` ``-`` ``~`` ``!`` ``sizeof`` ``typeof`` ``alignof`` + ``__attribute__`` ``defined`` + + Do not add spaces before the postfix increment and decrement unary operators: + + ``++`` ``--`` + + Do not add spaces around the ``.`` and ``->`` structure member operators. + +- Do not add spaces after casts. For example: + + .. code-block:: c + + int q = *(int *)&p + + +Type definitions +================ + +In general, do not use ``typedef`` for the purpose of hiding structures. +Typedefs used this way are problematic because they do not properly hide their +underlying type; for example, you need to know if the typedef is the structure +itself or a pointer to the structure. In addition, they must be declared exactly +once, whereas an incomplete structure type can be mentioned as many times as +necessary. Typedefs are difficult to use in stand-alone header files: the header +that defines the typedef must be included before the header that uses it, or by +the header that uses it (which causes namespace pollution), or there must be a +back-door mechanism for obtaining the typedef. + +That said, ``typedef`` can be helpful sometimes. For example, it is routinely +used to clarify the nature of an argument or return value, which can be of a +rather generic type such as ``void *``. It is also common to rename ``enum``. + +To make ``typedef`` names more informative and regular, we use the following +suffixes: + + - ``_e`` for ``enum`` + - ``_f`` for floating point numbers, regardless of size + - ``_i`` for signed integers, regardless of size + - ``_u`` for unsigned integers, regardless of size + - ``_fn`` for function pointers + - ``_p`` for other pointer type + - ``_st`` for ``struct`` + + +Forward Declaration +=================== + +Prefer using forward declaration over including another header for type- +declaration only. Forward declaration such as ``struct request;`` is feasible +if none of its members are directly accessed, such as when it's used in function +declaration. + + +Functions +========= + +- Declare functions that are local to a file as static. +- Place function types in their own line preceding the function. For example: + + .. code-block:: c + + static char * + function(int a1, int a2, float fl, int a4) + { + ... + +- Separate two successive functions with one blank line. +- Include parameter names with their dataypes in the function declaration. For + example: + + .. code-block:: c + + void function(int param); + +- When you use a wrapper function, name the wrapped function with the same name + as the wrapper function preceded by an underscore ('_'). Wrapped functions + are usually static. For example: + + .. code-block:: c + + static int + _fib(int n) + { + ... + } + + int + fib(int n) + { + ... + _fib(n); + ... + } + +- Create functions that are short and sweet. Functions should do just one + thing and fit on one or two screenfuls of text (80x24 screen size). + + The maximum length of a function is inversely proportional to the + complexity and indentation level of that function. So, if you have a + conceptually simple function that is just one long (but simple) + case-statement, where you have to do lots of small things for many + different cases, it is acceptable to have a longer function. + + Another measure of function complexity is the number of local variables. They + should not exceed 5-10. If your function has more than that, re-think the + function and split it into smaller pieces. A human brain can + generally easily keep track of about seven different things; anything more + and it gets confused. You may need to come back to your function and + understand what you did two weeks from now. + + +Goto statements +=============== + +- Use ``goto`` statements judiciously. Never use them to jump out of the + current function. Almost the only case where ``goto`` statements are helpful + is when a flow can exit from multiple locations within a function, and the + same clean-up logic applies to all of them. + + + .. code-block:: c + + int + fun(void) + { + int result = 0; + char *buffer; + buffer = malloc(1024); + if (buffer == NULL) { + return -1; + } + if (condition1) { + while (loop1) { + ... + } + result = 1; + goto out; + } + ... + out: + free(buffer); + return result; + } + + + +Comments +======== + +- Do not use ``//`` for single line comments. Instead, use the ``/* ... */`` + style. + +- For multi-line comments, use the following style: + + .. code-block:: c + + /* + * This is the preferred style for multi-line + * comments in the Linux kernel source code. + * Please use it consistently. + * + * Description: A column of asterisks on the left side, + * with beginning and ending almost-blank lines. + */ + +- To comment out blocks of code spanning several lines, use + ``#ifdef 0 ... #endif``. + +- Add comments before all major functions to describe what they do. Do not put + comments in the function body unless absolutely needed. For example: + + .. code-block:: c + + /* + * Try to acquire a physical address lock while a pmap is locked. If we + * fail to trylock we unlock and lock the pmap directly and cache the + * locked pa in *locked. The caller should then restart their loop in + * case the virtual to physical mapping has changed. + */ + int + vm_page_pa_tryrelock(pmap_t pmap, vm_paddr_t pa, vm_paddr_t *locked) + { + ... + + + +Other naming conventions +======================== + +- Use UPPERCASE for macro names. + +- Use ``enum`` to define several related constants. Use UPPERCASE for all + enumeration values. + +- Avoid macros as much as possible and use inline functions wherever you can. + +- For macros encapsulating compound statements, right-justify the backslashes + and enclose the statements in a ``do { ... } while (0)`` block. + +- For parameterized macros, add parentheses to all the parameters. For example: + + .. code-block:: c + + #define ADD_1(x) ((x) + 1) + + + +Inclusion +========= + +- Rule of thumb- local to global: first include header of the same name as + source, followed by headers in the same project, and external/system headers + last. +- Organize header inclusion in blocks, separated by blank line(s). For example, + headers that are shipped with the project and system headers should be in + separate clusters. +- Sort inclusions within the same block in alphabetic order. + + .. code-block:: c + + /* File: foo.c */ + #include "foo.h" /* first block: own header */ + + #include "bar.h" /* second block: headers from current project */ + #include "util/baz.h" + + #include /* third block: system/library headers */ + #include + #include + + +Structures +========== + +- To determine the size of a data structure, use some data of that type instead + of the type itself. For example: + + .. code-block:: c + + char *p; + p = malloc(sizeof(*p)); /* Good example */ + p = malloc(sizeof(char)); /* Bad example */ + +- Declare each variable in a structure in a separate line. Try to make the + structure readable by aligning the member names and comments using spaces. + Use a modest number of spaces if they suffice to align most of the member + names. Separate names that follow extremely long types with a single space. + + .. code-block:: c + + struct foo { + struct foo *next; /* List of active foo. */ + struct mumble amumble; /* Comment for mumble. */ + int bar; /* Try to align the comments. */ + struct verylongtypename *baz; /* Won't fit in 2 tabs. */ + }; + struct foo *foohead; /* Head of global foo list. */ + +- Declare major structures at the top of the file in which they are used, or + in separate header files if they are used in multiple source files. + Use of the structures should be by separate declarations and should be + ``extern`` if they are declared in a header file. + + +Pointers +======== + +- Use ``NULL`` as the null pointer constant (instead of ``0``). + +- Compare pointers to ``NULL``. For example: + + .. code-block:: c + + (p = f()) == NULL + + Do not compare to zero the integer. For example: + + .. code-block:: c + + !(p = f()) + +- Do not use ``!`` for comparisons (unless the variable is of boolean type). For + example: + + .. code-block:: c + + if (*p == '\0') + + The following snippet is a bad example: + + .. code-block:: c + + if (!*p) /* assume p is of type char * */ + +- Use ``const`` for function parameters if the pointer has no side effect. + +- Functions in charge of freeing an object should take a pointer to the intended + pointer to be freed, and set the pointer to the object to ``NULL`` before + returning. This prevents dangling pointers that are often discovered long + after ``free`` is called. + + .. code-block:: c + + void + destroy_buffer(struct buffer **pb) + { + free(*pb); + *pb = NULL; + } + +- Dynamically allocated structures should always initialize their members of + pointer type as soon as possible, to avoid the dangling pointer problem. + + +Macros +====== + +- Prefer ``static inline`` functions over macros. Macros often have unintended + side effects, for example: + + .. code-block:: c + + #define MAX(a,b) ((a) > (b) ? (a) : (b)) + + When used as in ``MAX(x++, y++)``, will increment either ``x`` or ``y`` twice, + which is probably not intended by the caller. + + +.. rubric:: Footnotes + +.. [#fn1] Frederick Brooks gave a definition of "style" in his book, The Design + of Design, which begins with "Style is a set of different repeated + microdecisions...". The book talked about the importance of Consistency in + the pages leading to this definition, starting from page 142, where the + author claimed that "consistency underlies all principles of quality". diff --git a/docs/modules/cc_log.rst b/docs/modules/cc_log.rst index 122dfe9c1..6a206210f 100644 --- a/docs/modules/cc_log.rst +++ b/docs/modules/cc_log.rst @@ -1,23 +1,60 @@ Log === -The log module provides a pause-less logging utility that incurs very little latency, and works well on latency-sensitive paths. It prioritizes performance over reliability, and is suitable for debug logging, event logging but not transaction logging. +The log module provides a pause-less logging utility that incurs very little +latency, and works well on latency-sensitive paths. It prioritizes performance +over reliability, and is suitable for debug logging, event logging but not +transaction logging. Background ---------- -Logging is the essential in recording important events, abnormal behavior and data samples. Unfortunately, for in-memory cache or other latency-sensitive applications, logging often presents a dilemma for developers. The naive log implementation usually writes directly to a file, an operation that is most likely buffered, but not guaranteed to return immediately. In production, it is not unheard of for a background task that nobody knows about to create contention around disk I/O, and to cause "mysterious" slowdown among mission critical services. As a result, developers often invoke logging somewhat hesitantly, and shy away from high-frequency logging practice such as request recording, fearing the negative impact on core performance. +Logging is the essential in recording important events, abnormal behavior and +data samples. Unfortunately, for in-memory cache or other latency-sensitive +applications, logging often presents a dilemma for developers. The naive log +implementation usually writes directly to a file, an operation that is most +likely buffered, but not guaranteed to return immediately. In production, it +is not unheard of for a background task that nobody knows about to create +contention around disk I/O, and to cause "mysterious" slowdown among mission +critical services. As a result, developers often invoke logging somewhat +hesitantly, and shy away from high-frequency logging practice such as request +recording, fearing the negative impact on core performance. Design ------ -The nature of I/O operations against persistent media means there is always a trade-off between having data stored reliably, and introducing overhead to performance critical processing. For the purpose we are considering here- debug logging, data sampling, etc- we think it is acceptable to give up a little bit of reliability for predictable performance. - -The trade-off is made with the consistent use of memory as an intermediate logging destination, a ring buffer that supports a single producer and a single consumer to be exact. A thread on the performance-critical path writes to the ring buffer which always returns immediately. Users are supposed to set up another, asynchronous (background) thread to flush data in the ring buffer from memory to their final location, which is a file. When the buffer becomes full, logged data are dropped until the next flush succeeds. - -The rationale for this design comes from a few observations in production: first, when everything is working as expected, the producer/consumer logging setup is actually quite similar to buffered write. Configurable buffer size and flush interval give user some control, and I/O operations are generally more efficient with a larger chunk of contiguous data than many smaller ones, reducing the overall contention at storage devices. Second, when the system is under high load, it is usually a sensible decision to sacrifice log fidelity for maximum application throughput. Finally, if an error condition is triggering abundant exception logging that overwhelms the buffer, it is very likely for the dropped logs to be similar to existing ones. Therefore, we can afford significant data loss/drop without losing much insight into the problem. The last condition is also helped if high-performance metrics are used to provide robust statistics. - -We have retained the ability to log directly to a file with the right configuration options, which are explained below. We recognize that it is nice to have a straightforward way of logging that does not require much setup, e.g. a background thread, for development or more relaxed scenarios. +The nature of I/O operations against persistent media means there is always a +trade-off between having data stored reliably, and introducing overhead to +performance critical processing. For the purpose we are considering here- +debug logging, data sampling, etc- we think it is acceptable to give up a +little bit of reliability for predictable performance. + +The trade-off is made with the consistent use of memory as an intermediate +logging destination, a ring buffer that supports a single producer and a +single consumer to be exact. A thread on the performance-critical path writes +to the ring buffer which always returns immediately. Users are supposed to set +up another, asynchronous (background) thread to flush data in the ring buffer +from memory to their final location, which is a file. When the buffer becomes +full, logged data are dropped until the next flush succeeds. + +The rationale for this design comes from a few observations in production: +first, when everything is working as expected, the producer/consumer logging +setup is actually quite similar to buffered write. Configurable buffer size +and flush interval give user some control, and I/O operations are generally +more efficient with a larger chunk of contiguous data than many smaller ones, +reducing the overall contention at storage devices. Second, when the system is +under high load, it is usually a sensible decision to sacrifice log fidelity +for maximum application throughput. Finally, if an error condition is +triggering abundant exception logging that overwhelms the buffer, it is very +likely for the dropped logs to be similar to existing ones. Therefore, we can +afford significant data loss/drop without losing much insight into the problem. +The last condition is also helped if high-performance metrics are used to +provide robust statistics. + +We have retained the ability to log directly to a file with the right +configuration options, which are explained below. We recognize that it is nice +to have a straightforward way of logging that does not require much setup, e.g. +a background thread, for development or more relaxed scenarios. Data Structure @@ -34,7 +71,10 @@ Data Structure ``log_metrics_st`` declares metrics native to the ``log`` module. -A ``logger`` has three fields: ``name`` points to the name of the log file, which may be ``NULL`` e.g. if using standard outputs; ``fd`` is the file descriptor for the log file; ``buf`` points to the ring buffer used for temporary log storage, buffering is disabled if ``buf`` is set to ``NULL``. +A ``logger`` has three fields: ``name`` points to the name of the log file, +which may be ``NULL`` e.g. if using standard outputs; ``fd`` is the file +descriptor for the log file; ``buf`` points to the ring buffer used for +temporary log storage, buffering is disabled if ``buf`` is set to ``NULL``. Synopsis -------- @@ -53,7 +93,7 @@ Synopsis void log_flush(struct logger *logger); - rstatus_i log_reopen(struct logger *logger); + rstatus_i log_reopen(struct logger *logger, char *target); Description ----------- @@ -74,9 +114,17 @@ Create and destroy struct logger *log_create(char *filename, uint32_t buf_cap); void log_destroy(struct logger **logger); -``log_create`` returns a logger with the information given or ``NULL`` if an error has occurred. If ``filename`` is not ``NULL``, it will attempt to open the file with flags ``O_WRONLY | O_APPEND | O_CREAT`` and masks ``0644``. Otherwise, it defaults the output to ``stderr``. ``log_create`` uses ``buf_cap`` in creating the ring buffer. A buffer of capacity ``buf_cap`` is allocated upon successful return. However, if ``cap_buf`` equals ``0``, buffering is turned off and ``write`` syscall will be used directly. +``log_create`` returns a logger with the information given or ``NULL`` if an +error has occurred. If ``filename`` is not ``NULL``, it will attempt to open +the file with flags ``O_WRONLY | O_APPEND | O_CREAT`` and masks ``0644``. +Otherwise, it defaults the output to ``stderr``. ``log_create`` uses +``buf_cap`` in creating the ring buffer. A buffer of capacity ``buf_cap`` is +allocated upon successful return. However, if ``cap_buf`` equals ``0``, +buffering is turned off and ``write`` syscall will be used directly. -``log_destroy`` will flush to the log file and release all memory resources whenever applicable. Note that the argument is of type ``struct logger **`` to avoid dangling pointers. +``log_destroy`` will flush to the log file and release all memory resources +whenever applicable. Note that the argument is of type ``struct logger **`` to +avoid dangling pointers. Write to logger ^^^^^^^^^^^^^^^ @@ -87,34 +135,66 @@ Write to logger #define log_stdout(...) _log_fd(STDOUT_FILENO, __VA_ARGS__) bool log_write(struct logger *logger, char *buf, uint32_t len); -``log_stderr`` and ``log_stdout`` are two convenience wrappers that make it easy to log to standard outputs. The arguments follow the same convention as in ``printf``. +``log_stderr`` and ``log_stdout`` are two convenience wrappers that make it +easy to log to standard outputs. The arguments follow the same convention as +in ``printf``. -``log_write`` takes a formatted string of length ``len`` stored in ``buf``, and logs it according to the way ``logger`` is created. If buffering is enabled, data is copied to the ring buffer. If the ring buffer does not have enough free capacity for the log, the entire log is skipped. Without buffering, ``log_write`` writes directly to the ``fd`` it is setup with in a best-effort fashion. +``log_write`` takes a formatted string of length ``len`` stored in ``buf``, +and logs it according to the way ``logger`` is created. If buffering is enabled, +data is copied to the ring buffer. If the ring buffer does not have enough +free capacity for the log, the entire log is skipped. Without buffering, +``log_write`` writes directly to the ``fd`` it is setup with in a best-effort +fashion. Flush to file ^^^^^^^^^^^^^ .. code-block:: C - void log_flush(struct logger *logger); + size_t log_flush(struct logger *logger); -``log_flush`` writes as much data to the log file as possible, and updates the (read) marker in the ring buffer. Data that cannot be written to the file will be kept until next call. If the ring buffer or the file was never setup, no action is taken. +``log_flush`` writes as much data to the log file as possible, and updates the +(read) marker in the ring buffer. Data that cannot be written to the file will +be kept until next call. If the ring buffer or the file was never setup, no +action is taken. Return the number of bytes flushed. Log reopen ^^^^^^^^^^ .. code-block:: C - rstatus_i log_reopen(struct logger *logger); + rstatus_i log_reopen(struct logger *logger, char *target); -``log_reopen`` reopens the log file according to ``name``, and does nothing if standard outputs are used. It returns ``CC_OK`` for success or ``CC_ERROR`` if reopen failed (at which point ``logger`` will no longer have a valid ``fd``). +``log_reopen`` reopens the log file according to ``name``, and does nothing if +standard outputs are used. It returns ``CC_OK`` for success or ``CC_ERROR`` if +reopen failed (at which point ``logger`` will no longer have a valid ``fd``). +If ``target`` is specified function will rename original log file to the +provided target filename and reopen the log file. -This function can be used to reopen the log file when an exception has happened, or another party such as ``logrotate`` instructs the application to do so. Log rotation in a ``nocopytruncate`` manner- i.e. the content in the file is not copied, but the file is simply renamed- is more efficient in high-load systems. But doing so requires signaling the application to reopen the log file after renaming. This function makes it possible to achieve that when used with proper signal handling. +This function can be used to reopen the log file when an exception has happened, +or another party such as ``logrotate`` instructs the application to do so. Log +rotation in a ``nocopytruncate`` manner- i.e. the content in the file is not +copied, but the file is simply renamed- is more efficient in high-load systems. +But doing so requires signaling the application to reopen the log file after +renaming. This function makes it possible to achieve that when used with +proper signal handling. Thread-safety ------------- -The logger is not thread-safe in the general sense. However, it is safe to use one thread as the producer, which writes to the logger, while using another thread as the consumer, which flushes the logger. A typical setup would have a worker thread being the producer and a background maintenance thread as the consumer. - -If ``log_reopen`` is used with a signal, it might invalidate the previous file descriptor in the middle of ``log_flush`` execution, regardless of thread model. The impact of this is to see an exception in ``write`` and failure in clearing up the ring buffer. But as long as ``log_flush`` is scheduled periodically, it is not fatal. To avoid such conflict, ``log_reopen`` should be scheduled on the same thread that performs ``log_flush``, and executed sequentially. One way of setting up signals to achieve this behavior requires masking the signal used for log rotation and having the thread check for pending signals using ``sigpending``. +The logger is not thread-safe in the general sense. However, it is safe to use +one thread as the producer, which writes to the logger, while using another +thread as the consumer, which flushes the logger. A typical setup would have a +worker thread being the producer and a background maintenance thread as the +consumer. + +If ``log_reopen`` is used with a signal, it might invalidate the previous file +descriptor in the middle of ``log_flush`` execution, regardless of thread model. +The impact of this is to see an exception in ``write`` and failure in clearing +up the ring buffer. But as long as ``log_flush`` is scheduled periodically, it +is not fatal. To avoid such conflict, ``log_reopen`` should be scheduled on +the same thread that performs ``log_flush``, and executed sequentially. One +way of setting up signals to achieve this behavior requires masking the signal +used for log rotation and having the thread check for pending signals using +``sigpending``. Examples -------- diff --git a/docs/modules/cc_metric.rst b/docs/modules/cc_metric.rst index bfc6adad0..8ca95c3ac 100644 --- a/docs/modules/cc_metric.rst +++ b/docs/modules/cc_metric.rst @@ -1,43 +1,89 @@ Metrics ======= -The metrics module provides a light-weight library to declare, update and report metrics. +The metrics module provides a light-weight library to declare, update and +report metrics. -Metrics can be turned on or off at compile time. They can also be enabled, disabled and composed at the module level. For example, if an application uses both the ``event`` module and ``tcp`` module, it is perfectly fine to disable metrics on the ``event`` module while enabling it for the ``tcp`` module. You only pay for what you use. +Metrics can be turned on or off at compile time. They can also be enabled, +disabled and composed at the module level. For example, if an application uses +both the ``event`` module and ``tcp`` module, it is perfectly fine to disable +metrics on the ``event`` module while enabling it for the ``tcp`` module. You +only pay for what you use. -The desire to compose metric groups drives a lot of the design decisions of metrics, as we shall explain below. +The desire to compose metric groups drives a lot of the design decisions of +metrics, as we shall explain below. Background ---------- -If good abstraction works when it hides as much unnecessary details from users as possible, then good observability plays the complimentary role. Instead of enclosing functionalities in a black box, it offers a glimpse into their internal state and structure. If everything works perfectly, abstraction is all that is needed. However, when things go wrong, as they inevitable do eventually, observability is the first thing developers turn to for monitoring and debugging. +If good abstraction works when it hides as much unnecessary details from users +as possible, then good observability plays the complimentary role. Instead of +enclosing functionalities in a black box, it offers a glimpse into their +internal state and structure. If everything works perfectly, abstraction is +all that is needed. However, when things go wrong, as they inevitable do +eventually, observability is the first thing developers turn to for monitoring +and debugging. -Good observability is one of the defining properties of production systems, as compared to prototypes and toy systems which often lack it. But because it is orthogonal to the functionalities of the system being built, and often not included in the initial design and implementation, a lot of systems end up having observability that feels bolted on and alien. +Good observability is one of the defining properties of production systems, as +compared to prototypes and toy systems which often lack it. But because it is +orthogonal to the functionalities of the system being built, and often not +included in the initial design and implementation, a lot of systems end up +having observability that feels bolted on and alien. Goals ----- Through practice, we found there are a few properties that we really appreciate: -#. metrics should be cheap to update and retrieve, so developers don't have to worry excessively about overhead and performance degradation caused by observability. -#. there needs to be an easy way to provide and look up the meaning of every metric, so readings can readily translate into understanding and insight. -#. for reusable modules or libraries, having metrics reported the same way everywhere is highly desirable, so developers know what to look for and can rely on their experiences from elsewhere. - -In reality, achieving all these goals at once is often a luxury. For example, many libraries offer no visibility by themselves, forcing developers to create and maintain metrics above the library APIs. This means what's visible is limited by API exposure, which is often less than what the internal states capture. It also means duplicate work among different users. When libraries do provide metrics, they are offered as standalone and cannot be easily combined. - -Given the scope of this project, we not only have control over how functionalities are implemented, but also can influence the desirable way of organizing and using these functionalities. In other words, we can afford to dictate how we think observability should be provided. With this assumption, it turns out we can find a way to achieve all these goals at once. +#. metrics should be cheap to update and retrieve, so developers don't have to + worry excessively about overhead and performance degradation caused by + observability. +#. there needs to be an easy way to provide and look up the meaning of every + metric, so readings can readily translate into understanding and insight. +#. for reusable modules or libraries, having metrics reported the same way + everywhere is highly desirable, so developers know what to look for and can + rely on their experiences from elsewhere. + +In reality, achieving all these goals at once is often a luxury. For example, +many libraries offer no visibility by themselves, forcing developers to create +and maintain metrics above the library APIs. This means what's visible is +limited by API exposure, which is often less than what the internal states +capture. It also means duplicate work among different users. When libraries do +provide metrics, they are offered as standalone and cannot be easily combined. + +Given the scope of this project, we not only have control over how +functionalities are implemented, but also can influence the desirable way of +organizing and using these functionalities. In other words, we can afford to +dictate how we think observability should be provided. With this assumption, +it turns out we can find a way to achieve all these goals at once. Design ------ -Metrics are part of each module. Almost every module comes with a predefined group of metrics that capture its state and usage. This saves developers the trouble of managing metrics at the application level and minimizes duplicate work among similar use cases. Users can follow the same convention to group and define metrics outside of this library, and mix everything together. - -A metric groups takes on the dual format of a structure whose members are metrics, and an array whose elements are metrics. This is made possible with the use of C preprocessors, and makes both updating (lookup by name, via the structure interpretation) and reporting (traverse by iteration, via the array interpretation) efficient. Users can further create a global structure containing multiple metrics groups, whiles still using the same flattened array view to process all metrics. - -Metrics should be allocated globally, and the address for each module's metric group(s) should be provided during module setup (or ``NULL`` to turn off metrics). This means users have total control of the memory layout of metrics, at the cost of having to setup module up with such information. In practice, this often means global metric reporting is extremely straightforward. - -Metrics are updated and reported using atomic instructions, not locks. When combined with zero- or low-contention threading model, metric access is extremely efficient. +Metrics are part of each module. Almost every module comes with a predefined +group of metrics that capture its state and usage. This saves developers the +trouble of managing metrics at the application level and minimizes duplicate +work among similar use cases. Users can follow the same convention to group +and define metrics outside of this library, and mix everything together. + +A metric groups takes on the dual format of a structure whose members are +metrics, and an array whose elements are metrics. This is made possible with +the use of C preprocessors, and makes both updating (lookup by name, via the +structure interpretation) and reporting (traverse by iteration, via the array +interpretation) efficient. Users can further create a global structure +containing multiple metrics groups, whiles still using the same flattened +array view to process all metrics. + +Metrics should be allocated globally, and the address for each module's metric +group(s) should be provided during module setup (or ``NULL`` to turn off +metrics). This means users have total control of the memory layout of metrics, +at the cost of having to setup module up with such information. In practice, +this often means global metric reporting is extremely straightforward. + +Metrics are updated and reported using atomic instructions, not locks. When +combined with zero- or low-contention threading model, metric access is +extremely efficient. Data Structure -------------- @@ -61,9 +107,12 @@ Data Structure }; }; -``metric`` is the fundamental data structure, each metric has (for now) three types, a printable name, a short description, and value. +``metric`` is the fundamental data structure, each metric has (for now) three +types, a printable name, a short description, and value. -If a metric is of type ``METRIC_COUNTER``, its value always increases monotonically. A metric of type ``METRIC_GAUGE`` has a signed integer value. Type ``METRIC_FPN`` means the value is a floating-point number. +If a metric is of type ``METRIC_COUNTER``, its value always increases +monotonically. A metric of type ``METRIC_GAUGE`` has a signed integer value. +Type ``METRIC_FPN`` means the value is a floating-point number. Usage @@ -77,9 +126,12 @@ Declare and initialize METRIC_INIT(_name, _type, _description) METRIC_NAME(_name, _type, _description) -To use these macros, ``_name`` *must* be a legal identifier [C11]_. See ``cc_metric.h`` for related implementation details. +To use these macros, ``_name`` *must* be a legal identifier [C11]_. See +``cc_metric.h`` for related implementation details. -A C preprocessor convention allows the above macros to be applied against a "list" of metrics. For example, one can define request related metrics, ``REQUEST_METRIC``, as such: +A C preprocessor convention allows the above macros to be applied against a +"list" of metrics. For example, one can define request related metrics, +``REQUEST_METRIC``, as such: .. code-block:: C @@ -90,7 +142,8 @@ A C preprocessor convention allows the above macros to be applied against a "lis ACTION( request_create, METRIC_COUNTER, "# reqs created" )\ ACTION( request_destroy, METRIC_COUNTER, "# reqs destroyed" ) -A metric group for the request module can be defined by using the ``METRIC_DECLARE`` macro against the list above: +A metric group for the request module can be defined by using the +``METRIC_DECLARE`` macro against the list above: .. code-block:: C @@ -111,10 +164,11 @@ Helper functions .. code-block:: C void metric_reset(struct metric sarr[], unsigned int nmetric); - size_t metric_print(char *buf, size_t nbuf, struct metric *m); + size_t metric_print(char *buf, size_t nbuf, char *fmt, struct metric *m); ``metric_reset`` resets the values of an array of metrics. -``metric_print`` prints the name and value of a metric, in human readable format, to buffer ``buf``, with a single space separating the two fields. This simple style is compatible with how Memcached currently reports metrics ([Memcached]_). Helper functions for other formats (e.g. Redis [Redis]_, StatsD [StatsD]_) may be introduced in the future. +``metric_print`` prints the name and value of a metric, in human readable +format specified by ``fmt``, to buffer ``buf``. Update @@ -127,20 +181,28 @@ Update DECR_N(_base, _metric, _delta) UPDATE_VAL(_base, _metric, _val) -The ``_base`` field reflects the starting address of the metric group. Therefore, if ``request_metrics`` is of type ``request_metrics_st *``, we can use it and the metric name, e.g. ``request_free`` as listed in ``REQUEST_METRIC`` to update the metric value: +The ``_base`` field reflects the starting address of the metric group. +Therefore, if ``request_metrics`` is of type ``request_metrics_st *``, we can +use it and the metric name, e.g. ``request_free`` as listed in +``REQUEST_METRIC`` to update the metric value: .. code-block:: C DECR(request_metrics, request_free); -``UPDATE_VAL`` applies to all three metric types. ``INCR_N`` and ``INCR``, which is short for ``INCR_N(_, _, 1)``, apply to both counters and gauges. ``DECR_N`` and ``DECR`` apply to gauges only. +``UPDATE_VAL`` applies to all three metric types. ``INCR_N`` and ``INCR``, +which is short for ``INCR_N(_, _, 1)``, apply to both counters and gauges. +``DECR_N`` and ``DECR`` apply to gauges only. Report ^^^^^^ -Often, reporting metrics means iterating through and read/print them all. This is when the array view of metrics becomes handy. +Often, reporting metrics means iterating through and read/print them all. This +is when the array view of metrics becomes handy. -The object that the aforementioned ``request_metrics`` points to has the same memory layout as an array of ``struct metric``. We only need to know the size of this array to traverse it, which we can get via the following macro: +The object that the aforementioned ``request_metrics`` points to has the same +memory layout as an array of ``struct metric``. We only need to know the size +of this array to traverse it, which we can get via the following macro: .. code-block:: C @@ -163,7 +225,8 @@ Which helps us to loop through all request related metrics: Hierarchical composition ^^^^^^^^^^^^^^^^^^^^^^^^ -A full-fledged application uses many modules. Similarly, metric groups can be further assembled to provide observability of the entire service: +A full-fledged application uses many modules. Similarly, metric groups can be +further assembled to provide observability of the entire service: .. code-block:: c @@ -173,12 +236,16 @@ A full-fledged application uses many modules. Similarly, metric groups can be fu storage_metrics_st storage_metrics; } app_stats; -To work with this setup, individual modules should be initialized with the correct base address of their metric group, e.g. ``&app_stats.storage_metrics`` for the storage module. Reporting multiple metric groups works almost exactly the same as a single metric group. +To work with this setup, individual modules should be initialized with the +correct base address of their metric group, e.g. +``&app_stats.storage_metrics`` for the storage module. Reporting multiple +metric groups works almost exactly the same as a single metric group. Compile-time switch ^^^^^^^^^^^^^^^^^^^ -All macros can be turned to no-op by turning off ``HAVE_STATS`` at compile time, which in turn undefines ``CC_STATS``. +All macros can be turned to no-op by turning off ``HAVE_STATS`` at compile time +, which in turn undefines ``CC_STATS``. .. code-block:: bash diff --git a/docs/modules/cc_option.rst b/docs/modules/cc_option.rst index ecde3dbe6..38983fe56 100644 --- a/docs/modules/cc_option.rst +++ b/docs/modules/cc_option.rst @@ -1,18 +1,37 @@ Option ====== -The option module facilitates the declaration, loading and combination of per-module and global configuration options. +The option module facilitates the declaration, loading and combination of per- +module and global configuration options. -The ability to compose options minimizes the work developers put into related boilerplate, and keeps naming and default values consistent. +The ability to compose options minimizes the work developers put into related +boilerplate, and keeps naming and default values consistent. Background ---------- -Every service needs some amount of configuration. While hugely useful and necessary, code that does configuration is mostly boilerplate and isn't exactly fun to write. Furthermore, reusing libraries or modules often does not spare the developers from creating and passing along aggregated options for these components, repeated for each service or application. Such boilerplate thus often takes a large code footprint among application-specific logic despite being mundane. - -When it comes to the format of configuration, there are flavors and varieties. Most notable is the choice of using primarily command line options or config files. In general, command line options are easier and quicker to start with, making them great for testing and development. However, they become less manageable as the number of options increases. And as part of the command that launched the process, they are potentially confusing if applications ever want config-reload support. Config files work better if a large of tuning knobs are present and commonly used, or when it is desirable to keep history or share configurations. On the downside, an extra file is used. - -Within each route there are also varieties. Several conventions exist for command line, such as short vs long name, single or double dash prefix. For config files, many languages and/or formats have been used, such as YAML, XML, and numerous simpler formats created ad hoc by the developers. +Every service needs some amount of configuration. While hugely useful and +necessary, code that does configuration is mostly boilerplate and isn't exactly +fun to write. Furthermore, reusing libraries or modules often does not spare the +developers from creating and passing along aggregated options for these +components, repeated for each service or application. Such boilerplate thus often +takes a large code footprint among application-specific logic despite being +mundane. + +When it comes to the format of configuration, there are flavors and varieties. +Most notable is the choice of using primarily command line options or config +files. In general, command line options are easier and quicker to start with, +making them great for testing and development. However, they become less +manageable as the number of options increases. And as part of the command that +launched the process, they are potentially confusing if applications ever want +config-reload support. Config files work better if a large of tuning knobs are +present and commonly used, or when it is desirable to keep history or share +configurations. On the downside, an extra file is used. + +Within each route there are also varieties. Several conventions exist for +command line, such as short vs long name, single or double dash prefix. For +config files, many languages and/or formats have been used, such as YAML, XML, +and numerous simpler formats created ad hoc by the developers. Goals ----- @@ -25,13 +44,33 @@ We want to have configuration options that: Design ------ -Options are part of each module, declared with macro with a table-like layout. Information of each option include name, type, default value, and description. They can be combined and manipulated with other macros and functions. - -Options are grouped and viewed in two ways: as members of a higher-level struct, or elements in an array. The former makes it easy to access individual option by name, such as passing the right options to initialize a specific module. The latter facilitates option traversal, which is useful to print all the options available and their default/current values. - -We opt for a file-oriented approach to pass most options into an application. Our estimate of the number of options in the services we want to create makes config files the more readable choice, and it paves way for dynamic reload. - -The format within the config file needs to balance a few things: expressiveness, human readability, and parsing complexity. We researched a few existing solutions: JSON, YAML, Java's simple Option class, whitespace delimiter-ed options used by Redis, etc. Looking at the needs of our target application group, as represented by Memcached and Redis, it seems the simple key value is either sufficient or "almost sufficient". Thus we choose to use a key/value format, ``: `` (colon and space ``: ``), which happens to be the YAML mappings format. We also adopted the YAML comment that begins with ``#``. It is quite trivial to write a parser for these basic structures from scratch. And being YAML-compatible makes two other things possible: 1) loading the config file by any YAML parser, and 2) extending the config format by borrowing more from the YAML protocol or even using a full-fledged YAML library, if ever needed. +Options are part of each module, declared with macro with a table-like layout. +Information of each option include name, type, default value, and description. +They can be combined and manipulated with other macros and functions. + +Options are grouped and viewed in two ways: as members of a higher-level +struct, or elements in an array. The former makes it easy to access individual +option by name, such as passing the right options to initialize a specific +module. The latter facilitates option traversal, which is useful to print all +the options available and their default/current values. + +We opt for a file-oriented approach to pass most options into an application. +Our estimate of the number of options in the services we want to create makes +config files the more readable choice, and it paves way for dynamic reload. + +The format within the config file needs to balance a few things: expressiveness, +human readability, and parsing complexity. We researched a few existing +solutions: JSON, YAML, Java's simple Option class, whitespace delimiter-ed +options used by Redis, etc. Looking at the needs of our target application +group, as represented by Memcached and Redis, it seems the simple key value is +either sufficient or "almost sufficient". Thus we choose to use a key/value +format, ``: `` (colon and space ``: ``), which happens to be the +YAML mappings format. We also adopted the YAML comment that begins with ``#``. +It is quite trivial to write a parser for these basic structures from scratch. +And being YAML-compatible makes two other things possible: 1) loading the +config file by any YAML parser, and 2) extending the config format by +borrowing more from the YAML protocol or even using a full-fledged YAML library, +if ever needed. Data Structure -------------- @@ -40,6 +79,7 @@ Data Structure typedef enum option_type { OPTION_TYPE_BOOL, OPTION_TYPE_UINT, + OPTION_TYPE_FPN, OPTION_TYPE_STR, OPTION_TYPE_SENTINEL } option_type_e; @@ -47,6 +87,7 @@ Data Structure typedef union option_val { bool vbool; uintmax_t vuint; + double vfpn; char *vstr; } option_val_u; @@ -59,7 +100,13 @@ Data Structure char *description; }; -The core data structure ``struct option`` has six members. ``name`` and ``description`` help identify and explain the purpose of the option. ``type`` decides how input should be interpreted, which currently can be boolean, unsigned integer or C string. Both the default value and current value are kept around, with values matching the type. Keeping the default separately will make it easy to reset the option to original. Finally, boolean ``set`` tells if an option has been set, and thus usable. +The core data structure ``struct option`` has six members. ``name`` and `` +description`` help identify and explain the purpose of the option. ``type`` +decides how input should be interpreted, which currently can be boolean, +unsigned integer, double or C string. Both the default value and current value +are kept around, with values matching the type. Keeping the default separately +will make it easy to reset the option to original. Finally, boolean ``set`` +tells if an option has been set, and thus usable. Synopsis -------- @@ -71,8 +118,8 @@ Synopsis rstatus_i option_load_file(FILE *fp, struct option options[], unsigned int nopt); void option_print(struct option *opt); - void option_printall(struct option options[], unsigned int nopt); - void option_printall_default(struct option options[], unsigned int nopt); + void option_print_all(struct option options[], unsigned int nopt); + void option_describe_all(struct option options[], unsigned int nopt); void option_free(struct option options[], unsigned int nopt); @@ -86,9 +133,12 @@ Declare and initialize #define OPTION_DECLARE(_name, _type, _default, _description) #define OPTION_INIT(_name, _type, _default, _description) -To use these macros, ``_name`` *must* be a legal identifier [C11]_. See ``cc_option.h`` for related implementation details. +To use these macros, ``_name`` *must* be a legal identifier [C11]_. See +``cc_option.h`` for related implementation details. -A C preprocessor convention allows the above macros to be applied against a "list" of options. For example, one can define options, ``BUF_OPTION``, for a buffer module as such: +A C preprocessor convention allows the above macros to be applied against a +"list" of options. For example, one can define options, ``BUF_OPTION``, for a +buffer module as such: .. code-block:: C @@ -96,7 +146,8 @@ A C preprocessor convention allows the above macros to be applied against a "lis ACTION( buf_init_size, OPTION_TYPE_UINT, BUF_DEFAULT_SIZE, "default size when buf is created" )\ ACTION( buf_poolsize, OPTION_TYPE_UINT, BUF_POOLSIZE, "buf pool size" ) -An option struct for the buffer module can be defined by using the ``OPTION_DECLARE`` macro against the list above: +An option struct for the buffer module can be defined by using the +``OPTION_DECLARE`` macro against the list above: .. code-block:: C @@ -118,8 +169,8 @@ Print option info .. code-block:: C void option_print(struct option *opt); - void option_printall(struct option options[], unsigned int nopt); - void option_printall_default(struct option options[], unsigned int nopt); + void option_print_all(struct option options[], unsigned int nopt); + void option_describe_all(struct option options[], unsigned int nopt); Examples diff --git a/docs/modules/cc_ring_array.rst b/docs/modules/cc_ring_array.rst index 1b8b36654..e4c618baa 100644 --- a/docs/modules/cc_ring_array.rst +++ b/docs/modules/cc_ring_array.rst @@ -1,9 +1,15 @@ ring array ========== -Ring array is a circular array data structure that allows elements to be pushed/popped in FIFO order. This data structure is designed to facilitate the sharing of resources between two threads with a producer/consumer relationship; that is, one thread only pushes to the ring array and the other thread only pops from it. +Ring array is a circular array data structure that allows elements to be +pushed/popped in FIFO order. This data structure is designed to facilitate the +sharing of resources between two threads with a producer/consumer relationship; +that is, one thread only pushes to the ring array and the other thread only pops +from it. -The main difference between ring array and ring buffer is that the former organizes and processes data as elements, while the latter treats data as flexible-length binary string. +The main difference between ring array and ring buffer is that the former +organizes and processes data as elements, while the latter treats data as +flexible-length binary string. Synopsis -------- @@ -26,7 +32,8 @@ Synopsis Description ----------- -This section contains descriptions of what the functions in the ccommon ring array module do. +This section contains descriptions of what the functions in the ccommon ring +array module do. Creation/Destruction ^^^^^^^^^^^^^^^^^^^^ @@ -35,9 +42,15 @@ Creation/Destruction struct ring_array *ring_array_create(size_t elem_size, uint32_t cap); void ring_array_destroy(struct ring_array **arr); -In order to create a ccommon ``ring_array`` data structure, call ``ring_array_create()`` with ``elem_size`` as the ``sizeof`` the elements the ``ring_array`` contains and with ``cap`` as the maximum number of elements the ``ring_array`` should be able to hold. This function returns a pointer to the ``ring_array`` that it creates. +In order to create a ccommon ``ring_array`` data structure, call +``ring_array_create()`` with ``elem_size`` as the ``sizeof`` the elements the +``ring_array`` contains and with ``cap`` as the maximum number of elements the +``ring_array`` should be able to hold. This function returns a pointer to the +``ring_array`` that it creates. -After the ``ring_array`` is no longer needed, ``ring_array_destroy`` should be called with the ``ring_array`` as its argument to free the memory allocated for it. +After the ``ring_array`` is no longer needed, ``ring_array_destroy`` should be +called with the ``ring_array`` as its argument to free the memory allocated for +it. Element Access ^^^^^^^^^^^^^^ @@ -46,9 +59,17 @@ Element Access rstatus_i ring_array_push(const void *elem, struct ring_array *arr); rstatus_i ring_array_pop(void *elem, struct ring_array *arr); -These functions are used to push/pop elements in the ``ring_array``. To push an element into the ``ring_array``, call ``ring_array_push()`` with ``elem`` being a pointer to the element being stored and ``arr`` being the ``ring_array`` being pushed to. ``ring_array_push()`` returns ``CC_OK`` if the element is stored, and ``CC_ERROR`` if the element could not be stored (i.e. the ``ring_array`` is full). +These functions are used to push/pop elements in the ``ring_array``. To push an +element into the ``ring_array``, call ``ring_array_push()`` with ``elem`` being +a pointer to the element being stored and ``arr`` being the ``ring_array`` being +pushed to. ``ring_array_push()`` returns ``CC_OK`` if the element is stored, and +``CC_ERROR`` if the element could not be stored i.e. the ``ring_array`` is full. -To pop an element from the ``ring_array``, call ``ring_array_pop()`` with ``elem`` being a pointer to the memory location for where the element should be popped to, and ``arr`` being the ``ring_array`` being popped from. ``ring_array_pop()`` returns ``CC_OK`` if the element was successfully popped, and ``CC_ERROR`` if not successful (i.e. the ``ring_array`` is empty). +To pop an element from the ``ring_array``, call ``ring_array_pop()`` with +``elem`` being a pointer to the memory location for where the element should be +popped to, and ``arr`` being the ``ring_array`` being popped from. +``ring_array_pop()`` returns ``CC_OK`` if the element was successfully popped, +and ``CC_ERROR`` if not successful (i.e. the ``ring_array`` is empty). State ^^^^^ @@ -56,20 +77,26 @@ State bool ring_array_full(const struct ring_array *arr); bool ring_array_empty(const struct ring_array *arr); -These functions tell the caller about the state of the ``ring_array``, specifically whether it is full or empty. In a producer/consumer model, ``ring_array_full()`` is a producer facing API, and ``ring_array_empty()`` is a consumer facing API. This is so that the producer can check whether or not the ``ring_array`` is full before pushing more elements into the array; likewise, the consumer can check whether or not the ``ring_array`` is empty before attempting to pop. +These functions tell the caller about the state of the ``ring_array``, +specifically whether it is full or empty. In a producer/consumer model, +``ring_array_full()`` is a producer facing API, and ``ring_array_empty()`` is a +consumer facing API. This is so that the producer can check whether or not the +``ring_array`` is full before pushing more elements into the array; likewise, the +consumer can check whether or not the ``ring_array`` is empty before attempting +to pop. Flush ^^^^^ ..code-block:: C void ring_array_flush(struct ring_array *arr); -This function is a consumer facing API that discards everything in the ``ring_array``. +This function is an external API that discards everything in the ``ring_array``. Examples -------- -Multi threaded Hello World! with ccommon ``ring_array``: +Multi threaded ``Hello World!`` with ccommon ``ring_array``: .. code-block:: c diff --git a/docs/overview.rst b/docs/overview.rst index a029e3bd0..47ebc0092 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -5,27 +5,47 @@ Overview Background ========== -Building a Cache Common library has a lot to do with the historical context that gave rise to the idea. The current open-source caching field is both sparse and crowded- most of what people use today trace back to either Memcached or Redis, maybe forked over them. On the other hand, many of the customization and improvement individuals come up with don't make their way back into the trunk version very easily, indicating an architectural problem with the original codebases. - -Fundamentally, there hasn't been enough modularity in either project, or the many that derived from them, to encourage sharing and reuse. During our own, multiple attempts to create Twemperf, Twemproxy, Fatcache and Slimcache, regardless of whether we were writing a server, a proxy or a client, we had to resort to copy-and-paste, and littered our changes across the landscape. - -It certainly *feels* that there was a lot in common among these projects, so we formalized it by abstracting the commonality as modules and putting them in a single library, which is ccommon, short for Cache Common. - -We went through the existing code bases that have implemented some or all of the core functionalities and have been tested in production for years, synthesized them, made changes whenever necessary, to make the APIs generic enough for all the use cases we know of. Inheriting code makes it faster and more reliable to build the library; holding the APIs against known implementations allow the core library to be generic and flexible enough for our needs. - -Given that multi-threaded programs are much harder to get right than their single-threaded counterpart, and sometimes incur non-trivial synchronization overhead, the priority is to get single-threading right first, with the possibility of investing in multi-threading in the future on a module-by-module basis. +Building a Cache Common library has a lot to do with the historical context that +gave rise to the idea. The current open-source caching field is both sparse and +crowded- most of what people use today trace back to either Memcached or Redis, +maybe forked over them. On the other hand, many of the customization and +improvement individuals come up with don't make their way back into the trunk +version very easily, indicating an architectural problem with the original +codebases. + +Fundamentally, there hasn't been enough modularity in either project, or the +many that derived from them, to encourage sharing and reuse. During our own, +multiple attempts to create Twemperf, Twemproxy, Fatcache and Slimcache, +regardless of whether we were writing a server, a proxy or a client, we had to +resort to copy-and-paste, and littered our changes across the landscape. + +It certainly *feels* that there was a lot in common among these projects, so we +formalized it by abstracting the commonality as modules and putting them in a +single library, which is ccommon, short for Cache Common. + +We went through the existing code bases that have implemented some or all of the +core functionalities and have been tested in production for years, synthesized +them, made changes whenever necessary, to make the APIs generic enough for all +the use cases we know of. Inheriting code makes it faster and more reliable to +build the library; holding the APIs against known implementations allow the core +library to be generic and flexible enough for our needs. + +Given that multi-threaded programs are much harder to get right than their +single-threaded counterpart, and sometimes incur non-trivial synchronization +overhead, the priority is to get single-threading right first, with the +possibility of investing in multi-threading in the future on a module-by-module +basis. Goals ===== #. Modularized functionality with consistent, intuitive interface. - -#. Use abstraction to allow compile-time choice from multiple implementations of the same interface, but avoid excessive, multi-layered abstraction unless absolutely necessary. - -#. Production quality code that has built-in configuration, monitoring and logging support. - - +#. Use abstraction to allow compile-time choice from multiple implementations of + the same interface, but avoid excessive, multi-layered abstraction unless + absolutely necessary. +#. Production quality code that has built-in configuration, monitoring and + logging support. #. Work well on platforms emerging in the next 3-5 years. diff --git a/include/buffer/cc_buf.h b/include/buffer/cc_buf.h index 4fe111dd3..279be3d89 100644 --- a/include/buffer/cc_buf.h +++ b/include/buffer/cc_buf.h @@ -38,25 +38,25 @@ extern "C" { /* name type default description */ -#define BUF_OPTION(ACTION) \ - ACTION( buf_init_size, OPTION_TYPE_UINT, BUF_DEFAULT_SIZE, "default size when buf is created" )\ - ACTION( buf_poolsize, OPTION_TYPE_UINT, BUF_POOLSIZE, "buf pool size" ) +#define BUF_OPTION(ACTION) \ + ACTION( buf_init_size, OPTION_TYPE_UINT, BUF_DEFAULT_SIZE, "init buf size incl header" )\ + ACTION( buf_poolsize, OPTION_TYPE_UINT, BUF_POOLSIZE, "buf pool size" ) typedef struct { BUF_OPTION(OPTION_DECLARE) } buf_options_st; /* name type description */ -#define BUF_METRIC(ACTION) \ - ACTION( buf_curr, METRIC_GAUGE, "# buf allocated" )\ - ACTION( buf_active, METRIC_GAUGE, "# buf in use/borrowed" )\ - ACTION( buf_create, METRIC_COUNTER, "# buf creates" )\ - ACTION( buf_create_ex, METRIC_COUNTER, "# buf create exceptions")\ - ACTION( buf_destroy, METRIC_COUNTER, "# buf destroys" )\ - ACTION( buf_borrow, METRIC_COUNTER, "# buf borrows" )\ - ACTION( buf_borrow_ex, METRIC_COUNTER, "# buf borrow exceptions")\ - ACTION( buf_return, METRIC_COUNTER, "# buf returns" )\ - ACTION( buf_memory, METRIC_GAUGE, "memory allocated to buf") +#define BUF_METRIC(ACTION) \ + ACTION( buf_curr, METRIC_GAUGE, "# buf allocated" )\ + ACTION( buf_active, METRIC_GAUGE, "# buf in use/borrowed" )\ + ACTION( buf_create, METRIC_COUNTER, "# buf creates" )\ + ACTION( buf_create_ex, METRIC_COUNTER, "# buf create exceptions" )\ + ACTION( buf_destroy, METRIC_COUNTER, "# buf destroys" )\ + ACTION( buf_borrow, METRIC_COUNTER, "# buf borrows" )\ + ACTION( buf_borrow_ex, METRIC_COUNTER, "# buf borrow exceptions" )\ + ACTION( buf_return, METRIC_COUNTER, "# buf returns" )\ + ACTION( buf_memory, METRIC_GAUGE, "memory alloc'd to buf including header" ) typedef struct { BUF_METRIC(METRIC_DECLARE) @@ -68,7 +68,7 @@ struct buf { char *wpos; /* write marker */ char *end; /* end of buffer */ bool free; /* is this buf free? */ - char begin[1]; /* beginning of buffer */ + char begin[]; /* beginning of buffer */ }; #define BUF_HDR_SIZE offsetof(struct buf, begin) @@ -103,7 +103,7 @@ void buf_destroy(struct buf **buf); /* Size of data that has yet to be read */ static inline uint32_t -buf_rsize(struct buf *buf) +buf_rsize(const struct buf *buf) { ASSERT(buf->rpos <= buf->wpos); @@ -112,37 +112,38 @@ buf_rsize(struct buf *buf) /* Amount of room left in buffer for writing new data */ static inline uint32_t -buf_wsize(struct buf *buf) +buf_wsize(const struct buf *buf) { ASSERT(buf->wpos <= buf->end); return (uint32_t)(buf->end - buf->wpos); } -/* Total capacity of given buf */ +/* Total size of given buf, including header */ static inline uint32_t -buf_size(struct buf *buf) +buf_size(const struct buf *buf) { ASSERT(buf->begin < buf->end); - return (uint32_t)(buf->end - (char*)buf); + return (uint32_t)(buf->end - (char *)buf); } +/* Size of given buf, not including header */ static inline uint32_t -buf_capacity(struct buf *buf) +buf_capacity(const struct buf *buf) { ASSERT(buf->begin < buf->end); return (uint32_t)(buf->end - buf->begin); } -/* new capacity needed to append count bytes to the buffer */ +/* new capacity needed to write count bytes to the buffer */ static inline uint32_t -buf_new_cap(struct buf *buf, uint32_t count) +buf_new_cap(const struct buf *buf, uint32_t count) { ASSERT(buf->begin <= buf->wpos); - return buf->wpos - buf->begin + count; + return count <= buf_wsize(buf) ? 0 : count - buf_wsize(buf); } static inline void diff --git a/include/buffer/cc_dbuf.h b/include/buffer/cc_dbuf.h index 185de4734..07ad34f96 100644 --- a/include/buffer/cc_dbuf.h +++ b/include/buffer/cc_dbuf.h @@ -29,8 +29,8 @@ extern "C" { /* name type default description */ -#define DBUF_OPTION(ACTION) \ - ACTION( dbuf_max_power, OPTION_TYPE_UINT, DBUF_DEFAULT_MAX, "max number of doubling" ) +#define DBUF_OPTION(ACTION) \ + ACTION( dbuf_max_power, OPTION_TYPE_UINT, DBUF_DEFAULT_MAX, "max number of doubles") typedef struct { DBUF_OPTION(OPTION_DECLARE) @@ -43,7 +43,7 @@ typedef struct { ACTION( dbuf_double, METRIC_COUNTER, "# double completed" )\ ACTION( dbuf_double_ex, METRIC_COUNTER, "# double failed" )\ ACTION( dbuf_shrink, METRIC_COUNTER, "# shrink completed" )\ - ACTION( dbuf_shrink_ex, METRIC_COUNTER, "# srhink failed" )\ + ACTION( dbuf_shrink_ex, METRIC_COUNTER, "# shrink failed" )\ ACTION( dbuf_fit, METRIC_COUNTER, "# fit completed" )\ ACTION( dbuf_fit_ex, METRIC_COUNTER, "# fit failed" ) diff --git a/include/cc_bstring.h b/include/cc_bstring.h index b82c8b3f5..86c1aadb7 100644 --- a/include/cc_bstring.h +++ b/include/cc_bstring.h @@ -38,15 +38,14 @@ struct bstring { #define str2bstr(_str) (struct bstring){ sizeof(_str) - 1, (_str) } #define null_bstring (struct bstring){ 0, NULL } -#define bstring_set_text(_str, _text) do { \ - (_str)->len = (uint32_t)(sizeof(_text) - 1); \ - (_str)->data = (_text); \ +#define bstring_set_literal(_str, _literal) do { \ + (_str)->len = (uint32_t)(sizeof(_literal) - 1); \ + (_str)->data = (_literal); \ } while (0); -/* TODO(yao): rename this */ -#define bstring_set_raw(_str, _raw) do { \ - (_str)->len = (uint32_t)(cc_strlen(_raw)); \ - (_str)->data = (char *)(_raw); \ +#define bstring_set_cstr(_str, _cstr) do { \ + (_str)->len = (uint32_t)(cc_strlen(_cstr)); \ + (_str)->data = (char *)(_cstr); \ } while (0); void bstring_init(struct bstring *str); @@ -59,7 +58,6 @@ int bstring_compare(const struct bstring *s1, const struct bstring *s2); struct bstring *bstring_alloc(uint32_t size); void bstring_free(struct bstring **bstring); -/* TODO(yao): is this endian thing really useful? */ /* efficient implementation of string comparion of short strings */ #define str2cmp(m, c0, c1) \ (m[0] == c0 && m[1] == c1) @@ -67,70 +65,79 @@ void bstring_free(struct bstring **bstring); #define str3cmp(m, c0, c1, c2) \ (m[0] == c0 && m[1] == c1 && m[2] == c2) -#ifdef CC_LITTLE_ENDIAN - #define str4cmp(m, c0, c1, c2, c3) \ - (*(uint32_t *) m == ((c3 << 24) | (c2 << 16) | (c1 << 8) | c0)) + ((m[0] << 24 | m[1] << 16 | m[2] << 8 | m[3]) == \ + ((c0 << 24) | (c1 << 16) | (c2 << 8) | c3)) #define str5cmp(m, c0, c1, c2, c3, c4) \ (str4cmp(m, c0, c1, c2, c3) && (m[4] == c4)) #define str6cmp(m, c0, c1, c2, c3, c4, c5) \ - (str4cmp(m, c0, c1, c2, c3) && \ - (((uint32_t *) m)[1] & 0xffff) == ((c5 << 8) | c4)) + (str5cmp(m, c0, c1, c2, c3, c4) && m[5] == c5) #define str7cmp(m, c0, c1, c2, c3, c4, c5, c6) \ - (str6cmp(m, c0, c1, c2, c3, c4, c5) && (m[6] == c6)) + (str6cmp(m, c0, c1, c2, c3, c4, c5) && m[6] == c6) #define str8cmp(m, c0, c1, c2, c3, c4, c5, c6, c7) \ (str4cmp(m, c0, c1, c2, c3) && \ - (((uint32_t *) m)[1] == ((c7 << 24) | (c6 << 16) | (c5 << 8) | c4))) + (m[4] << 24 | m[5] << 16 | m[6] << 8 | m[7]) == \ + ((c4 << 24) | (c5 << 16) | (c6 << 8) | c7)) #define str9cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8) \ (str8cmp(m, c0, c1, c2, c3, c4, c5, c6, c7) && m[8] == c8) #define str10cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9) \ - (str8cmp(m, c0, c1, c2, c3, c4, c5, c6, c7) && \ - (((uint32_t *) m)[2] & 0xffff) == ((c9 << 8) | c8)) + (str9cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8) && m[9] == c9) #define str11cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10) \ - (str10cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9) && (m[10] == c10)) + (str10cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9) && m[10] == c10) #define str12cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11) \ (str8cmp(m, c0, c1, c2, c3, c4, c5, c6, c7) && \ - (((uint32_t *) m)[2] == ((c11 << 24) | (c10 << 16) | (c9 << 8) | c8))) - -#else // BIG ENDIAN + (m[8] << 24 | m[9] << 16 | m[10] << 8 | m[11]) == \ + ((c8 << 24) | (c9 << 16) | (c10 << 8) | c11)) + +/* below is a more efficient implementation for little-endian only, it takes + * about 50% the cycles compared to the generic implementation above in the + * extreme cases (e.g. string length being multiples of 4), however, our + * profiling showed that string comparison does not contribute meaningfully to + * overall processing cost, both events and hashes are far more notable, and + * therefore we can choose the generic implementation until profiling results + * indicate otherwise. + */ +/* #define str4cmp(m, c0, c1, c2, c3) \ - (str3cmp(m, c0, c1, c2) && (m3 == c3)) + (*(uint32_t *) m == ((c3 << 24) | (c2 << 16) | (c1 << 8) | c0)) #define str5cmp(m, c0, c1, c2, c3, c4) \ (str4cmp(m, c0, c1, c2, c3) && (m[4] == c4)) #define str6cmp(m, c0, c1, c2, c3, c4, c5) \ - (str5cmp(m, c0, c1, c2, c3, c4) && m[5] == c5) + (str4cmp(m, c0, c1, c2, c3) && \ + (((uint32_t *) m)[1] & 0xffff) == ((c5 << 8) | c4)) #define str7cmp(m, c0, c1, c2, c3, c4, c5, c6) \ - (str6cmp(m, c0, c1, c2, c3, c4, c5) && m[6] == c6) + (str6cmp(m, c0, c1, c2, c3, c4, c5) && (m[6] == c6)) #define str8cmp(m, c0, c1, c2, c3, c4, c5, c6, c7) \ - (str7cmp(m, c0, c1, c2, c3, c4, c5, c6) && m[7] == c7) + (str4cmp(m, c0, c1, c2, c3) && \ + (((uint32_t *) m)[1] == ((c7 << 24) | (c6 << 16) | (c5 << 8) | c4))) #define str9cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8) \ (str8cmp(m, c0, c1, c2, c3, c4, c5, c6, c7) && m[8] == c8) #define str10cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9) \ - (str9cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8) && m[9] == c9) + (str8cmp(m, c0, c1, c2, c3, c4, c5, c6, c7) && \ + (((uint32_t *) m)[2] & 0xffff) == ((c9 << 8) | c8)) #define str11cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10) \ - (str10cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9) && m[10] == c10) + (str10cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9) && (m[10] == c10)) #define str12cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11) \ - (str11cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10) && m[11] == c11) - -#endif // CC_LITTLE_ENDIAN - + (str8cmp(m, c0, c1, c2, c3, c4, c5, c6, c7) && \ + (((uint32_t *) m)[2] == ((c11 << 24) | (c10 << 16) | (c9 << 8) | c8))) +*/ /* * Wrapper around common routines for manipulating C character strings @@ -163,6 +170,7 @@ void bstring_free(struct bstring **bstring); /* bstring to uint conversion */ rstatus_i bstring_atou64(uint64_t *u64, struct bstring *str); +rstatus_i bstring_atoi64(int64_t *i64, struct bstring *str); #ifdef __cplusplus } diff --git a/include/cc_define.h b/include/cc_define.h index b521b133e..6a207036f 100644 --- a/include/cc_define.h +++ b/include/cc_define.h @@ -27,13 +27,6 @@ extern "C" { # define CC_HAVE_SIGNAME 1 #endif - -#ifdef HAVE_BIG_ENDIAN -# define CC_BIG_ENDIAN 1 -#else -# define CC_LITTLE_ENDIAN 1 -#endif - #ifdef HAVE_STATS # define CC_STATS 1 #endif @@ -54,10 +47,18 @@ extern "C" { # define CC_BACKTRACE 1 #endif +#ifdef HAVE_ACCEPT4 +# define CC_ACCEPT4 1 +#endif + #ifdef HAVE_DEBUG_MM #define CC_DEBUG_MM 1 #endif +#ifdef HAVE_ITT_INSTRUMENTATION +#define CC_ITT 1 +#endif + #define CC_OK 0 #define CC_ERROR -1 diff --git a/include/cc_event.h b/include/cc_event.h index 2f38caa6b..f5a31d281 100644 --- a/include/cc_event.h +++ b/include/cc_event.h @@ -26,8 +26,6 @@ extern "C" { #include -#define EVENT_SIZE 1024 - #define EVENT_READ 0x0000ff #define EVENT_WRITE 0x00ff00 #define EVENT_ERR 0xff0000 @@ -51,7 +49,7 @@ void event_setup(event_metrics_st *metrics); void event_teardown(void); /* event base */ -struct event_base *event_base_create(int size, event_cb_fn cb); +struct event_base *event_base_create(int nevent, event_cb_fn cb); void event_base_destroy(struct event_base **evb); /* event control */ diff --git a/include/cc_itt.h b/include/cc_itt.h new file mode 100644 index 000000000..583c72d93 --- /dev/null +++ b/include/cc_itt.h @@ -0,0 +1,69 @@ +#pragma once + +#include + +#ifdef CC_ITT +#include "ittnotify.h" +#endif + +#ifdef __cplusplus +extern "C" { +#endif +#ifdef CC_ITT + +#define ITT_DOMAIN_NAME "cc_itt" + +#define cc_declare_itt_function(_keyword, _name) \ + _keyword __itt_heap_function _name + +#define cc_create_itt_malloc(_name) \ + _name = __itt_heap_function_create(#_name, ITT_DOMAIN_NAME) + +#define cc_create_itt_free(_name) \ + _name = __itt_heap_function_create(#_name, ITT_DOMAIN_NAME) + +#define cc_create_itt_realloc(_name) \ + _name = __itt_heap_function_create(#_name, ITT_DOMAIN_NAME) + +#define cc_itt_alloc(_itt_heap_f, _p, _s) do { \ + __itt_heap_allocate_begin(_itt_heap_f, (size_t)(_s), 0); \ + __itt_heap_allocate_end(_itt_heap_f, (void *)&(_p), (size_t)(_s), 0); \ +} while (0) + +#define cc_itt_zalloc(_itt_heap_f, _p, _s) do { \ + __itt_heap_allocate_begin(_itt_heap_f, (size_t)(_s), 1); \ + __itt_heap_allocate_end(_itt_heap_f, (void *)&(_p), (size_t)(_s), 1); \ +} while (0) + +#define cc_itt_free(_itt_heap_f, _p) do { \ + __itt_heap_free_begin(_itt_heap_f, _p); \ + __itt_heap_free_end(_itt_heap_f, _p); \ +} while (0) + +#define cc_itt_realloc(_itt_heap_f, _p, _np, _s) do { \ + __itt_heap_reallocate_begin(_itt_heap_f, _p, (size_t)(_s), 0); \ + __itt_heap_reallocate_end(_itt_heap_f, _p, (void *)&(_np), (size_t)(_s), 0); \ +} while (0) + +#define cc_itt_heap_internal_access() \ + __itt_heap_internal_access_begin() + +#define cc_itt_heap_internal_access_end() \ + __itt_heap_internal_access_end() + +#else +#define cc_declare_itt_function(_keyword, _name) +#define cc_create_itt_malloc(_name) +#define cc_create_itt_free(_name) +#define cc_create_itt_realloc(_name) +#define cc_itt_alloc(_itt_heap_f, _p, _s) +#define cc_itt_zalloc(_itt_heap_f, _p, _s) +#define cc_itt_free(_itt_heap_f, _p) +#define cc_itt_realloc(_itt_heap_f, _p, _np, _s) +#define cc_itt_heap_internal_access_begin() +#define cc_itt_heap_internal_access_end() +#endif /* CC_ITT */ + +#ifdef __cplusplus +} +#endif diff --git a/include/cc_log.h b/include/cc_log.h index d61fcff91..f308842d8 100644 --- a/include/cc_log.h +++ b/include/cc_log.h @@ -61,6 +61,10 @@ typedef struct { void log_setup(log_metrics_st *metrics); void log_teardown(void); +/* these two are for testing purposes only */ +log_metrics_st *log_metrics_create(void); +void log_metrics_destroy(log_metrics_st **p); + /** * Create a logger. If filename is NULL, created logger writes to stderr. * buf_cap is the size of the buffer used for pauseless logging. specify diff --git a/include/cc_mm.h b/include/cc_mm.h index 2d982196e..a9effa23b 100644 --- a/include/cc_mm.h +++ b/include/cc_mm.h @@ -66,6 +66,9 @@ extern "C" { #define cc_munmap(_p, _s) \ _cc_munmap(_p, (size_t)(_s), __FILE__, __LINE__) +#define cc_alloc_usable_size(_p) \ + _cc_alloc_usable_size(_p, __FILE__, __LINE__) + void * _cc_alloc(size_t size, const char *name, int line); void * _cc_zalloc(size_t size, const char *name, int line); void * _cc_calloc(size_t nmemb, size_t size, const char *name, int line); @@ -74,6 +77,7 @@ void * _cc_realloc_move(void *ptr, size_t size, const char *name, int line); void _cc_free(void *ptr, const char *name, int line); void * _cc_mmap(size_t size, const char *name, int line); int _cc_munmap(void *p, size_t size, const char *name, int line); +size_t _cc_alloc_usable_size(void *ptr, const char *name, int line); #ifdef __cplusplus } diff --git a/include/cc_queue.h b/include/cc_queue.h index c8f8a5ce2..06bf9c31a 100644 --- a/include/cc_queue.h +++ b/include/cc_queue.h @@ -127,6 +127,7 @@ extern "C" { * _INSERT_TAIL - - + + + * _REMOVE_HEAD + - + - - * _REMOVE + + + + + + * _REINIT - - - + - * */ @@ -691,6 +692,23 @@ struct { \ (head2)->tqh_last = &(head2)->tqh_first; \ } while (0) +#define TAILQ_REINIT(head, var, field, offset) do { \ + TAILQ_FIRST((head)) = var; \ + TAILQ_FOREACH(var, head, field) { \ + if ((TAILQ_NEXT((var), field)) != NULL) { \ + TAILQ_NEXT((var), field) = \ + (void *)((char *)(TAILQ_NEXT((var), field)) + (offset));\ + } \ + if ((var) == TAILQ_FIRST(head)) { \ + (var)->field.tqe_prev = &TAILQ_FIRST(head); \ + } else { \ + (var)->field.tqe_prev = \ + (void *)((char *)((var)->field.tqe_prev) + (offset)); \ + } \ + (head)->tqh_last = &TAILQ_NEXT((var), field); \ + } \ +} while (0) + /* * Circular queue declarations. */ diff --git a/include/cc_stats_log.h b/include/cc_stats_log.h new file mode 100644 index 000000000..b33972770 --- /dev/null +++ b/include/cc_stats_log.h @@ -0,0 +1,38 @@ +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + + +#define STATS_LOG_FILE NULL /* default log file */ +#define STATS_LOG_NBUF 0 /* default log buf size */ + +/* name type default description */ +#define STATS_LOG_OPTION(ACTION) \ + ACTION( stats_log_file, OPTION_TYPE_STR, NULL, "file storing stats" )\ + ACTION( stats_log_nbuf, OPTION_TYPE_UINT, STATS_LOG_NBUF, "stats log buf size" ) + +typedef struct { + STATS_LOG_OPTION(OPTION_DECLARE) +} stats_log_options_st; + + +/* dump stats as CSV records into a log file, this allows metrics to be captured + * locally without setting up an observability infrastructure + */ +void stats_log_setup(stats_log_options_st *options); +void stats_log_teardown(void); + +void stats_log(struct metric metrics[], unsigned int nmetric); + +void stats_log_flush(void); + + +#ifdef __cplusplus +} +#endif + diff --git a/include/cc_util.h b/include/cc_util.h index d9c13955b..f8a770059 100644 --- a/include/cc_util.h +++ b/include/cc_util.h @@ -86,6 +86,9 @@ extern "C" { #define cc_strlen(_s) \ strlen((char *)(_s)) +#define cc_strnlen(_s, _n) \ + strnlen((char *)(_s), (size_t)(_n)) + #define cc_strcmp(_s1, _s2) \ strcmp((char *)(_s1), (char *)(_s2)) diff --git a/include/hash/cc_lookup3.h b/include/rust/cc_log_rs.h similarity index 80% rename from include/hash/cc_lookup3.h rename to include/rust/cc_log_rs.h index af6f8318c..27c22dfb7 100644 --- a/include/hash/cc_lookup3.h +++ b/include/rust/cc_log_rs.h @@ -1,5 +1,4 @@ -/* - * ccommon - a cache common library. +/* ccommon - a cache common library. * Copyright (C) 2013 Twitter, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,10 +22,10 @@ extern "C" { #include -#include -#include +/* NOTE: for documentation see ccommon/rust/ccommon_rs/src/log/mod.rs */ -uint32_t hash_lookup3(const void *key, size_t length, const uint32_t initval); +rstatus_i rust_log_setup(void); +void rust_log_teardown(void); #ifdef __cplusplus } diff --git a/include/time/cc_timer.h b/include/time/cc_timer.h index 6a8c382ee..86b871a7a 100644 --- a/include/time/cc_timer.h +++ b/include/time/cc_timer.h @@ -49,6 +49,14 @@ struct duration; /* data structure to measure duration */ * relationship between this unit and nanosecond can be obtained via another * syscall */ + +enum duration_type { + DURATION_PRECISE, /* default */ + DURATION_FAST, + + MAX_DURATION_TYPE +}; + #ifdef OS_DARWIN struct duration { bool started; @@ -58,6 +66,7 @@ struct duration { }; #elif defined OS_LINUX struct duration { + enum duration_type type; bool started; bool stopped; struct timespec start; @@ -86,12 +95,12 @@ struct timeout { bool is_intvl; }; - /* update duration */ void duration_reset(struct duration *d); /* get a reading of duration and copy it without stopping the original timer */ void duration_snapshot(struct duration *s, const struct duration *d); void duration_start(struct duration *d); +void duration_start_type(struct duration *d, enum duration_type type); void duration_stop(struct duration *d); /* read duration */ double duration_ns(struct duration *d); @@ -99,6 +108,18 @@ double duration_us(struct duration *d); double duration_ms(struct duration *d); double duration_sec(struct duration *d); +static inline int +duration_compare(const void *lhs, const void *rhs) +{ + double lns = duration_ns((struct duration *)lhs); + double rns = duration_ns((struct duration *)rhs); + if (lns < rns) + return -1; + if (lns > rns) + return 1; + + return 0; +} /* * Not all possible granularity can be meaningfully used for sleep or event. diff --git a/rust/CMakeLists.txt b/rust/CMakeLists.txt deleted file mode 100644 index 808fb4391..000000000 --- a/rust/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -file(WRITE CMAKE_BINARY_DIR "${CMAKE_BINARY_DIR}\n") - -if(HAVE_RUST) - add_subdirectory(ccommon_rs) -endif() diff --git a/rust/Cargo.lock b/rust/Cargo.lock deleted file mode 100644 index 5ae870301..000000000 --- a/rust/Cargo.lock +++ /dev/null @@ -1,496 +0,0 @@ -[[package]] -name = "aho-corasick" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "ansi_term" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "atty" -version = "0.2.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "backtrace" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "backtrace-sys 0.1.23 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-demangle 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "backtrace-sys" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cc 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "bindgen" -version = "0.37.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cexpr 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "clang-sys 0.23.0 (registry+https://github.com/rust-lang/crates.io-index)", - "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)", - "env_logger 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", - "peeking_take_while 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro2 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "which 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "bitflags" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "cc" -version = "1.0.17" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "cc_binding" -version = "0.1.0" -dependencies = [ - "bindgen 0.37.4 (registry+https://github.com/rust-lang/crates.io-index)", - "failure 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "ccommon_rs" -version = "0.1.0" -dependencies = [ - "cc_binding 0.1.0", - "failure 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "cexpr" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "nom 3.2.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "cfg-if" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "clang-sys" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "libloading 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "clap" -version = "2.32.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", - "atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", - "bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "textwrap 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "env_logger" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", - "humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "failure" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "failure_derive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "failure_derive" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", - "synstructure 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "glob" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "humantime" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "lazy_static" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "libc" -version = "0.2.42" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "libloading" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cc 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "log" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "memchr" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "memchr" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "nom" -version = "3.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "proc-macro2" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "quick-error" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "quote" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "quote" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "proc-macro2 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "redox_syscall" -version = "0.1.40" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "redox_termios" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "regex" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "aho-corasick 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", - "thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "regex-syntax" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "rustc-demangle" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "strsim" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "syn" -version = "0.11.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", - "synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "synom" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "synstructure" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "termcolor" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "termion" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "textwrap" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "thread_local" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "ucd-util" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "unicode-width" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "unicode-xid" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "unicode-xid" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "unreachable" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "utf8-ranges" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "vec_map" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "void" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "which" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "winapi" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "wincolor" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[metadata] -"checksum aho-corasick 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f0ba20154ea1f47ce2793322f049c5646cc6d0fa9759d5f333f286e507bf8080" -"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" -"checksum atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "2fc4a1aa4c24c0718a250f0681885c1af91419d242f29eb8f2ab28502d80dbd1" -"checksum backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "89a47830402e9981c5c41223151efcced65a0510c13097c769cede7efb34782a" -"checksum backtrace-sys 0.1.23 (registry+https://github.com/rust-lang/crates.io-index)" = "bff67d0c06556c0b8e6b5f090f0eac52d950d9dfd1d35ba04e4ca3543eaf6a7e" -"checksum bindgen 0.37.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1b25ab82877ea8fe6ce1ce1f8ac54361f0218bad900af9eb11803994bf67c221" -"checksum bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d0c54bb8f454c567f21197eefcdbf5679d0bd99f2ddbe52e84c77061952e6789" -"checksum cc 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)" = "49ec142f5768efb5b7622aebc3fdbdbb8950a4b9ba996393cb76ef7466e8747d" -"checksum cexpr 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "42aac45e9567d97474a834efdee3081b3c942b2205be932092f53354ce503d6c" -"checksum cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "efe5c877e17a9c717a0bf3613b2709f723202c4e4675cc8f12926ded29bcb17e" -"checksum clang-sys 0.23.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d7f7c04e52c35222fffcc3a115b5daf5f7e2bfb71c13c4e2321afe1fc71859c2" -"checksum clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b957d88f4b6a63b9d70d5f454ac8011819c6efa7727858f458ab71c756ce2d3e" -"checksum env_logger 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)" = "0e6e40ebb0e66918a37b38c7acab4e10d299e0463fe2af5d29b9cc86710cfd2a" -"checksum failure 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "934799b6c1de475a012a02dab0ace1ace43789ee4b99bcfbf1a2e3e8ced5de82" -"checksum failure_derive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c7cdda555bb90c9bb67a3b670a0f42de8e73f5981524123ad8578aafec8ddb8b" -"checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" -"checksum humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0484fda3e7007f2a4a0d9c3a703ca38c71c54c55602ce4660c419fd32e188c9e" -"checksum lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e6412c5e2ad9584b0b8e979393122026cdd6d2a80b933f890dcd694ddbe73739" -"checksum libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" = "b685088df2b950fccadf07a7187c8ef846a959c142338a48f9dc0b94517eb5f1" -"checksum libloading 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9c3ad660d7cb8c5822cd83d10897b0f1f1526792737a179e73896152f85b88c2" -"checksum log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "61bd98ae7f7b754bc53dca7d44b604f733c6bba044ea6f41bc8d89272d8161d2" -"checksum memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "148fab2e51b4f1cfc66da2a7c32981d1d3c083a803978268bb11fe4b86925e7a" -"checksum memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "796fba70e76612589ed2ce7f45282f5af869e0fdd7cc6199fa1aa1f1d591ba9d" -"checksum nom 3.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05aec50c70fd288702bcd93284a8444607f3292dbdf2a30de5ea5dcdbe72287b" -"checksum peeking_take_while 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" -"checksum proc-macro2 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "77997c53ae6edd6d187fec07ec41b207063b5ee6f33680e9fa86d405cdd313d4" -"checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0" -"checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" -"checksum quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9949cfe66888ffe1d53e6ec9d9f3b70714083854be20fd5e271b232a017401e8" -"checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1" -"checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" -"checksum regex 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "13c93d55961981ba9226a213b385216f83ab43bd6ac53ab16b2eeb47e337cf4e" -"checksum regex-syntax 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05b06a75f5217880fc5e905952a42750bf44787e56a6c6d6852ed0992f5e1d54" -"checksum rustc-demangle 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "76d7ba1feafada44f2d38eed812bd2489a03c0f5abb975799251518b68848649" -"checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550" -"checksum syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" -"checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" -"checksum synstructure 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3a761d12e6d8dcb4dcf952a7a89b475e3a9d69e4a69307e01a470977642914bd" -"checksum termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "adc4587ead41bf016f11af03e55a624c06568b5a19db4e90fde573d805074f83" -"checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" -"checksum textwrap 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "307686869c93e71f94da64286f9a9524c0f308a9e1c87a583de8e9c9039ad3f6" -"checksum thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "279ef31c19ededf577bfd12dfae728040a21f635b06a24cd670ff510edd38963" -"checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d" -"checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" -"checksum unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" -"checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" -"checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" -"checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122" -"checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" -"checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" -"checksum which 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e84a603e7e0b1ce1aa1ee2b109c7be00155ce52df5081590d1ffb93f4f515cb2" -"checksum winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "773ef9dcc5f24b7d850d0ff101e542ff24c3b090a9768e03ff889fdef41f00fd" -"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" -"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -"checksum wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "eeb06499a3a4d44302791052df005d5232b927ed1a9658146d842165c4de7767" diff --git a/rust/Cargo.toml b/rust/Cargo.toml deleted file mode 100644 index ae5821dad..000000000 --- a/rust/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -[workspace] - -members = [ - "cc_binding", - "ccommon_rs", -] - -[profile.release] -opt-level = 3 -debug = true -rpath = false -lto = true -debug-assertions = false -codegen-units = 1 - -[profile.dev] -debug = true -opt-level = 0 - -[term] -verbose = true diff --git a/rust/cc_binding/CMakeLists.txt b/rust/cc_binding/CMakeLists.txt deleted file mode 100644 index 6c8be94ae..000000000 --- a/rust/cc_binding/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -cargo_build(NAME cc_binding) - -add_dependencies(cc_binding_static ccommon-static) -add_dependencies(cc_binding_shared ccommon-shared) diff --git a/rust/cc_binding/Cargo.toml b/rust/cc_binding/Cargo.toml deleted file mode 100644 index 0ad06b1df..000000000 --- a/rust/cc_binding/Cargo.toml +++ /dev/null @@ -1,12 +0,0 @@ -[package] -name = "cc_binding" -version = "0.1.0" -authors = ["Jonathan Simms "] - -[build-dependencies] -bindgen = "0.37.0" -failure = "~0.1.1" - -[lib] -name = "cc_binding" -crate-type = ["rlib", "dylib", "staticlib", "lib"] diff --git a/rust/cc_binding/build.rs b/rust/cc_binding/build.rs deleted file mode 100644 index 4e6dc243d..000000000 --- a/rust/cc_binding/build.rs +++ /dev/null @@ -1,108 +0,0 @@ -extern crate bindgen; -#[macro_use] -extern crate failure; - -use std::env; -use std::fs; -use std::io; -use std::io::BufReader; -use std::io::prelude::*; -use std::ffi::OsString; -use std::os::unix::fs as unix_fs; -use std::path::Path; -use std::path::PathBuf; -use std::result; - -type Result = result::Result; - -fn get_cmake_binary_dir() -> io::Result { - // this file is written by cmake on each run, updated with the location of - // the build directory. - let mut fp = fs::File::open("../CMAKE_BINARY_DIR")?; - let mut buf = String::new(); - let n = fp.read_to_string(&mut buf)?; - assert!(n > 0, "file was empty"); - Ok(String::from(buf.trim_right())) -} - -const CMAKE_CACHE: &str = "CMakeCache.txt"; -const CCOMMON_BINARY_DIR_KEY: &str = "ccommon_BINARY_DIR:STATIC"; - -fn get_cmake_cache_value(binary_dir: &Path, key: &str) -> Result> { - let cache_path = binary_dir.join(CMAKE_CACHE); - let fp = BufReader::new(fs::File::open(cache_path)?); - - for x in fp.lines() { - let line = x?; - let needle = format!("{}=", key); - if line.starts_with(&needle[..]) { - if let Some(v) = line.rsplit("=").take(1).last() { - return Ok(Some(v.to_owned())) - } else { - bail!("bad line: {:#?}", line); - } - } - } - - Ok(None) -} - -fn main() { - println!("cargo:rustc-link-lib=static=ccommon-1.2.0"); - - let include_path = fs::canonicalize("./../../include").unwrap(); - - let cmake_binary_dir = match get_cmake_binary_dir() { - Ok(p) => p, - Err(err) => panic!("Failed locating the CMAKE_BINARY_DIR file: {:#?}", err), - }; - - let cbd = PathBuf::from(cmake_binary_dir); - - let mut config_h_dir = cbd.clone(); - config_h_dir.push("ccommon"); - - let lib_dir: String = { - let cbd = get_cmake_cache_value(&cbd, CCOMMON_BINARY_DIR_KEY) - .map(|o| o.map(OsString::from)) - .unwrap() - .expect( - format!("could not find {} in {}", CCOMMON_BINARY_DIR_KEY, CMAKE_CACHE).as_ref() - ); - - let cbd = Path::new(&cbd); - cbd.join("lib").to_str().unwrap().to_string() - }; - - println!("cargo:rustc-link-search=native={}", lib_dir); - - let bindings = bindgen::Builder::default() - .clang_args(vec![ - "-I", include_path.to_str().unwrap(), - "-I", config_h_dir.to_str().unwrap(), - "-I", cbd.to_str().unwrap(), - "-L", &lib_dir, - ]) - .header("wrapper.h") - .blacklist_type("max_align_t") // https://github.com/rust-lang-nursery/rust-bindgen/issues/550 - .generate() - .expect("Unable to generate bindings"); - - let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); - bindings - .write_to_file(out_path.join("bindings.rs")) - .expect("Couldn't write bindings!"); - - // ./target/debug/build/cc_binding-27eac70f0fa2e180/out <<- starts here - - // cc_binding-27eac70f0fa2e180 - let symlink_content = - out_path.parent().unwrap().file_name().unwrap(); - - let build_dir = out_path.parent().and_then(|p| p.parent()).unwrap(); - - let link_location = build_dir.join("cc_binding"); - let _ = fs::remove_file(link_location.as_path()); - unix_fs::symlink(symlink_content, link_location).unwrap(); -} - diff --git a/rust/cc_binding/src/lib.rs b/rust/cc_binding/src/lib.rs deleted file mode 100644 index 2cf071191..000000000 --- a/rust/cc_binding/src/lib.rs +++ /dev/null @@ -1,8 +0,0 @@ -#![allow(unknown_lints)] -#![allow(clippy)] -#![allow(clippy_pedantic)] -#![allow(non_upper_case_globals)] -#![allow(non_camel_case_types)] -#![allow(non_snake_case)] -#![allow(dead_code)] -include!(concat!(env!("OUT_DIR"), "/bindings.rs")); diff --git a/rust/cc_binding/wrapper.h b/rust/cc_binding/wrapper.h deleted file mode 100644 index c5b21b148..000000000 --- a/rust/cc_binding/wrapper.h +++ /dev/null @@ -1,2 +0,0 @@ -#include -#include diff --git a/rust/ccommon-backend/Cargo.toml b/rust/ccommon-backend/Cargo.toml new file mode 100644 index 000000000..b4483ac43 --- /dev/null +++ b/rust/ccommon-backend/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "ccommon-backend" +version = "0.1.0" +authors = ["Sean Lynch "] +edition = "2018" + +[features] +c-export = [] + +[dependencies] +ccommon-sys = { path = "../ccommon-sys" } +libc = "0.2" + +# Note: We can depend on ccommon_rs for testing purposes +# but not otherwise to avoid circular dependencies. +[dev-dependencies] +ccommon-rs = { path = "../ccommon-rs" } diff --git a/rust/ccommon-backend/src/c_export.rs b/rust/ccommon-backend/src/c_export.rs new file mode 100644 index 000000000..ea5b35d18 --- /dev/null +++ b/rust/ccommon-backend/src/c_export.rs @@ -0,0 +1,146 @@ +// ccommon - a cache common library. +// Copyright (C) 2019 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use ccommon_sys::{option, rstatus_i, CC_ENOMEM, CC_ERROR}; +use libc::{c_char, c_uint, FILE}; + +use std::ffi::CStr; +use std::io::BufReader; + +use crate::compat::CFileRef; + +const CC_OK: rstatus_i = ccommon_sys::CC_OK as rstatus_i; + +/// Set an option using the value parsed from the string. +/// +/// Returns 0 for success, otherwise prints an error message +/// and returns an error code. +#[no_mangle] +unsafe extern "C" fn option_set(opt: *mut option, val_str: *const c_char) -> rstatus_i { + use crate::option::ParseErrorKind; + + assert!(!val_str.is_null()); + assert!(!opt.is_null()); + + let s = CStr::from_ptr(val_str); + let bytes = s.to_bytes(); + + match crate::option::option_set(&mut *opt, bytes) { + Ok(()) => CC_OK, + Err(e) => { + eprintln!("Failed to parse option value: {}", e); + + match e.kind() { + ParseErrorKind::OutOfMemory => CC_ENOMEM, + _ => CC_ERROR, + } + } + } +} + +/// Initialize an option to it's default value. +/// +/// Returns `0` for success, otherise prints an error message +/// and returns `CC_ENOMEM`. +/// +/// Note: The only case where an error can occur here is when +/// we fail to allocate memory for a string. +#[no_mangle] +unsafe extern "C" fn option_default(opt: *mut option) -> rstatus_i { + assert!(!opt.is_null()); + + match crate::option::option_default(&mut *opt) { + Ok(()) => CC_OK, + Err(_) => { + // This seems unlikely to work if we ran out of memory + // while trying to set an option. Myabe it was a really + // big string? + eprintln!("Ran out of memory while attempting to set option to default"); + + CC_ENOMEM + } + } +} + +/// Print a single option stdout. +/// +/// Will panic if printing to stdout fails. +#[no_mangle] +unsafe extern "C" fn option_print(opt: *const option) { + assert!(!opt.is_null()); + + crate::option::option_print(&mut std::io::stdout(), &*opt).expect("Failed to write to stdout"); +} + +/// Print all options in an array. +/// +/// Will panic if printing to stdout fails. +#[no_mangle] +unsafe extern "C" fn option_print_all(opts: *const option, nopt: c_uint) { + let slice = std::slice::from_raw_parts(opts, nopt as usize); + + crate::option::option_print_all(&mut std::io::stdout(), slice) + .expect("Failed to write to stdout"); +} + +/// Print a description of all options in an array. +/// +/// Will panic if printing to stdout fails. +#[no_mangle] +unsafe extern "C" fn option_describe_all(opts: *const option, nopt: c_uint) { + let slice = std::slice::from_raw_parts(opts, nopt as usize); + + crate::option::option_describe_all(&mut std::io::stdout(), slice) + .expect("Failed to write to stdout"); +} + +/// Initialize an array of options to the default values. +/// +/// This returns `CC_OK` on success and `CC_ENOMEM` if it +/// failed to allocate memory for a string copy. +#[no_mangle] +unsafe extern "C" fn option_load_default(opts: *mut option, nopt: c_uint) -> rstatus_i { + let slice = std::slice::from_raw_parts_mut(opts, nopt as usize); + + match crate::option::option_load_default(slice) { + Ok(()) => CC_OK, + Err(_) => CC_ENOMEM, + } +} + +/// Parse options in `.ini` format from a file. +/// +/// Returns 0 on success. On error prints out an error message +/// and returns a non-zero error code. +#[no_mangle] +unsafe extern "C" fn option_load_file(fp: *mut FILE, opts: *mut option, nopt: c_uint) -> rstatus_i { + use crate::option::ParseErrorKind; + + let slice = std::slice::from_raw_parts_mut(opts, nopt as usize); + let file = CFileRef::from_ptr_mut(fp); + let mut file = BufReader::new(file); + + match crate::option::option_load(slice, &mut file) { + Ok(()) => CC_OK, + Err(e) => { + eprintln!("Failed to load options from file: {}", e); + + match e.kind() { + ParseErrorKind::OutOfMemory => CC_ENOMEM, + _ => CC_ERROR, + } + } + } +} diff --git a/rust/ccommon-backend/src/compat.rs b/rust/ccommon-backend/src/compat.rs new file mode 100644 index 000000000..c9fc4cbf8 --- /dev/null +++ b/rust/ccommon-backend/src/compat.rs @@ -0,0 +1,157 @@ +// ccommon - a cache common library. +// Copyright (C) 2019 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use libc::{ + c_long, c_void, feof, fflush, fread, fseek, ftell, fwrite, FILE, SEEK_CUR, SEEK_END, SEEK_SET, +}; +use std::io::{Error, ErrorKind, Read, Result, Seek, SeekFrom, Write}; + +/// A reference to a C `FILE` instance. +/// +/// This is a convenience wrapper that supports `Read`, `Write` +/// and `Seek` from [`std::io`][io]. It does not take any ownership +/// over the `FILE` pointer. +/// +/// [io]: std::io +pub struct CFileRef { + _ptr: FILE, +} + +impl CFileRef { + /// Create a new reference from a const file pointer. + /// + /// # Safety + /// It is undefined behaviour + /// - To call this function with an invalid pointer. + /// - If the reference created through this function outlives + /// the original pointer + /// - If the pointer is modified while the reference is still live. + /// + /// # Panics + /// This function will panic if `ptr` is null. + pub unsafe fn from_ptr<'a>(ptr: *const FILE) -> &'a Self { + assert!(!ptr.is_null()); + + &*(ptr as *const Self) + } + + /// Create a new reference from a mutable file pointer. + /// + /// # Safety + /// It is undefined behaviour + /// - To call this function with an invalid pointer. + /// - If the reference created through this function outlives + /// the original pointer + /// - If the pointer is modified while the reference is still live. + /// + /// # Panics + /// This function will panic if `ptr` is null. + pub unsafe fn from_ptr_mut<'a>(ptr: *mut FILE) -> &'a mut Self { + assert!(!ptr.is_null()); + + &mut *(ptr as *mut Self) + } + + /// Convert the reference back to a `FILE` pointer. + pub fn as_ptr(&self) -> *const FILE { + self as *const Self as *const FILE + } + + /// Convert the reference back to a `FILE` pointer. + pub fn as_mut_ptr(&mut self) -> *mut FILE { + self as *mut Self as *mut FILE + } +} + +impl Write for CFileRef { + fn write(&mut self, bytes: &[u8]) -> Result { + let written = unsafe { + fwrite( + bytes.as_ptr() as *const c_void, + 1, + bytes.len(), + self.as_mut_ptr(), + ) + }; + + if written != bytes.len() { + return Err(Error::last_os_error()); + } + + Ok(written) + } + + fn flush(&mut self) -> Result<()> { + let res = unsafe { fflush(self.as_mut_ptr()) }; + + if res != 0 { + Err(Error::last_os_error()) + } else { + Ok(()) + } + } +} + +impl Read for CFileRef { + fn read(&mut self, buf: &mut [u8]) -> Result { + let read = unsafe { + fread( + buf.as_mut_ptr() as *mut c_void, + 1, + buf.len(), + self.as_mut_ptr(), + ) + }; + + if read == buf.len() { + return Ok(read); + } + + if unsafe { feof(self.as_mut_ptr()) != 0 } { + return Ok(read); + } + + Err(Error::last_os_error()) + } +} + +impl Seek for CFileRef { + fn seek(&mut self, pos: SeekFrom) -> Result { + use std::convert::TryInto; + + let (offset, origin) = match pos { + SeekFrom::Start(offset) => (offset as i64, SEEK_SET), + SeekFrom::Current(offset) => (offset, SEEK_CUR), + SeekFrom::End(offset) => (offset, SEEK_END), + }; + + let offset: c_long = match offset.try_into() { + Ok(off) => off, + Err(e) => return Err(Error::new(ErrorKind::Other, Box::new(e))), + }; + + let ret = unsafe { fseek(self.as_mut_ptr(), offset, origin) }; + if ret != 0 { + return Err(Error::last_os_error()); + } + + let off = unsafe { ftell(self.as_mut_ptr()) }; + if off < 0 { + return Err(Error::last_os_error()); + } + + Ok(off as u64) + } +} diff --git a/rust/ccommon-backend/src/lib.rs b/rust/ccommon-backend/src/lib.rs new file mode 100644 index 000000000..441143e8e --- /dev/null +++ b/rust/ccommon-backend/src/lib.rs @@ -0,0 +1,41 @@ +// ccommon - a cache common library. +// Copyright (C) 2019 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Rust reimplementations of parts of ccommon. +//! +//! This is a separate library as it is intended to be used as a dependency +//! on C code within ccommon. + +pub mod option { + mod default; + mod parse; + mod print; + + pub use self::default::{option_default, option_load_default, OutOfMemoryError}; + pub use self::parse::{option_load, option_set, ParseError, ParseErrorKind}; + pub use self::print::{option_describe, option_describe_all, option_print, option_print_all}; +} + +pub mod compat; + +#[cfg(feature = "c-export")] +mod c_export; + +/// Rexports of the functions defined in this crate that correspond +/// to the C functions used in ccommon. +#[cfg(feature = "c-export")] +pub mod c { + pub use crate::c_export::*; +} diff --git a/rust/ccommon-backend/src/option/default.rs b/rust/ccommon-backend/src/option/default.rs new file mode 100644 index 000000000..f0955257d --- /dev/null +++ b/rust/ccommon-backend/src/option/default.rs @@ -0,0 +1,99 @@ +// ccommon - a cache common library. +// Copyright (C) 2019 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use ccommon_sys::{ + _cc_alloc, option, OPTION_TYPE_BOOL, OPTION_TYPE_FPN, OPTION_TYPE_STR, OPTION_TYPE_UINT, +}; + +use std::convert::TryInto; +use std::ffi::CStr; +use std::fmt; + +macro_rules! c_str { + ($s:expr) => { + concat!($s, "\0").as_ptr() as *const i8 + }; +} + +#[derive(Debug)] +pub struct OutOfMemoryError(()); + +impl fmt::Display for OutOfMemoryError { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + write!(fmt, "Out of memory") + } +} + +impl std::error::Error for OutOfMemoryError {} + +/// Initialize an option to it's default value. +/// +/// # Safety +/// This function assumes that the description and name +/// pointers within the options always point to a valid +/// C string. It also assumes that the value for string +/// type options is either a valid C string or null. +pub unsafe fn option_default(opt: &mut option) -> Result<(), OutOfMemoryError> { + use std::mem::MaybeUninit; + + opt.set = true; + + match opt.type_ { + OPTION_TYPE_BOOL => opt.val.vbool = opt.default_val.vbool, + OPTION_TYPE_UINT => opt.val.vuint = opt.default_val.vuint, + OPTION_TYPE_FPN => opt.val.vfpn = opt.default_val.vfpn, + OPTION_TYPE_STR => { + let default = opt.default_val.vstr; + + if default.is_null() { + opt.val.vstr = default; + } else { + let s = CStr::from_ptr(default); + let bytes = s.to_bytes_with_nul(); + let mem = _cc_alloc( + bytes.len().try_into().unwrap(), + c_str!(module_path!()), + line!() as std::os::raw::c_int, + ) as *mut MaybeUninit; + + if mem.is_null() { + return Err(OutOfMemoryError(())); + } + + std::ptr::copy_nonoverlapping(bytes.as_ptr(), mem as *mut u8, bytes.len()); + + opt.val.vstr = mem as *mut libc::c_char + } + } + _ => opt.val = opt.default_val, + }; + + Ok(()) +} + +/// Initialize all the options to their default values. +/// +/// # Safety +/// This function assumes that the description and name +/// pointers within the options always point to a valid +/// C string. It also assumes that the value for string +/// type options is either a valid C string or null. +pub unsafe fn option_load_default(options: &mut [option]) -> Result<(), OutOfMemoryError> { + for opt in options.iter_mut() { + option_default(opt)?; + } + + Ok(()) +} diff --git a/rust/ccommon-backend/src/option/parse.rs b/rust/ccommon-backend/src/option/parse.rs new file mode 100644 index 000000000..970daf23c --- /dev/null +++ b/rust/ccommon-backend/src/option/parse.rs @@ -0,0 +1,519 @@ +// ccommon - a cache common library. +// Copyright (C) 2019 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Disabling this lint since I find that the usage of the lifetimes +// in this file makes things clearer. +#![allow(clippy::needless_lifetimes)] + +use ccommon_sys::{ + option, option_type, OPTION_TYPE_BOOL, OPTION_TYPE_FPN, OPTION_TYPE_STR, OPTION_TYPE_UINT, +}; + +use std::borrow::Cow; +use std::convert::TryInto; +use std::error::Error; +use std::ffi::CStr; +use std::fmt; +use std::io::{BufRead, Error as IOError}; + +macro_rules! c_str { + ($s:expr) => { + concat!($s, "\0").as_ptr() as *const i8 + }; +} + +/// Error for when a config file fails to parse. +/// +/// This covers everything from an IO error to running out of memory +/// to a missing colon within a line. +pub struct ParseError<'a> { + span: Span<'a>, + data: ParseErrorType, +} + +impl<'a> ParseError<'a> { + pub fn kind(&self) -> ParseErrorKind { + match &self.data { + ParseErrorType::InvalidBool => ParseErrorKind::InvalidBool, + ParseErrorType::InvalidUInt => ParseErrorKind::InvalidUInt, + ParseErrorType::StringWithNull => ParseErrorKind::InvalidString, + ParseErrorType::InvalidFloat => ParseErrorKind::InvalidFloat, + ParseErrorType::NoColonInLine => ParseErrorKind::IncorrectKVSyntax, + ParseErrorType::UnknownOptionType => ParseErrorKind::UnknownOptionType, + ParseErrorType::OutOfMemory => ParseErrorKind::OutOfMemory, + ParseErrorType::IOError(_) => ParseErrorKind::IOError, + ParseErrorType::InvalidKey => ParseErrorKind::UnknownOption, + } + } +} + +impl<'a> ParseError<'a> { + fn invalid_bool(span: Span<'a>) -> Self { + Self { + span, + data: ParseErrorType::InvalidBool, + } + } + + fn invalid_uint(span: Span<'a>) -> Self { + Self { + span, + data: ParseErrorType::InvalidUInt, + } + } + + fn invalid_fpn(span: Span<'a>) -> Self { + Self { + span, + data: ParseErrorType::InvalidFloat, + } + } + + fn string_contained_null(span: Span<'a>) -> Self { + Self { + span, + data: ParseErrorType::StringWithNull, + } + } + + fn invalid_key(span: Span<'a>) -> Self { + Self { + span, + data: ParseErrorType::InvalidKey, + } + } + + fn missing_colon(span: Span<'a>) -> Self { + Self { + span, + data: ParseErrorType::NoColonInLine, + } + } + + fn unknown_option_type(span: Span<'a>) -> Self { + Self { + span, + data: ParseErrorType::UnknownOptionType, + } + } + + fn ioerror(err: IOError) -> Self { + Self { + span: Span::new(&[], 0), + data: ParseErrorType::IOError(err), + } + } + + fn oom() -> Self { + Self { + span: Span { + line: 0, + text: Cow::Borrowed(&[]), + }, + data: ParseErrorType::OutOfMemory, + } + } + + fn into_owned(self) -> ParseError<'static> { + ParseError { + span: Span { + line: self.span.line, + text: Cow::Owned(self.span.text.into_owned()), + }, + data: self.data, + } + } +} + +#[derive(Debug)] +enum ParseErrorType { + InvalidBool, + InvalidUInt, + StringWithNull, + InvalidFloat, + + InvalidKey, + UnknownOptionType, + NoColonInLine, + IOError(IOError), + OutOfMemory, +} + +/// The type of the parse error, +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] +pub enum ParseErrorKind { + /// Tried to parse a bool, but it wasn't valid. + InvalidBool, + /// Tried to parse an unsigned integer, but it wasn't valid. + InvalidUInt, + /// Tried to parse a string, but it contained a nul character + InvalidString, + /// Tried to parse a float, but it wasn't valid. + InvalidFloat, + /// Tried to parse a `key: value` pair but a colon was missing. + IncorrectKVSyntax, + + /// Parsed a valid option, but it doesn't correspond to any + /// of the options that were provided. + UnknownOption, + /// An option had an unrecognized option type. + UnknownOptionType, + /// Was unable to allocate memory when it was needed while parsing. + /// + /// Note that this doesn't apply to all OOM cases. + OutOfMemory, + /// An IO error occurred when reading from the stream. + IOError, +} + +/// Load options from a type which implements `BufRead`. +/// +/// In most cases you'll want to use [`OptionExt::load`][0] +/// instead. +/// +/// [0]: crate::option::OptionExt::load +pub fn option_load( + options: &mut [option], + source: &mut R, +) -> Result<(), ParseError<'static>> { + let mut linebuf = Vec::new(); + + let mut lineno = 0; + + while source + .read_until(b'\n', &mut linebuf) + .map_err(ParseError::ioerror)? + != 0 + { + // Strip off any comments before doing parsing + let line = linebuf.split(|&x| x == b'#').next().unwrap(); + + if line.iter().copied().all(|x| x.is_ascii_whitespace()) { + linebuf.clear(); + continue; + } + + let (k, v) = parse_kv(&line, lineno).map_err(|x| x.into_owned())?; + + let opt = match unsafe { find_option(options, k) } { + Some(opt) => opt, + None => return Err(ParseError::invalid_key(Span::new(k, lineno)).into_owned()), + }; + + let value = parse_value(v, lineno, opt.type_).map_err(|x| x.into_owned())?; + unsafe { set_option_value(opt, value)? }; + + lineno += 1; + linebuf.clear(); + } + + Ok(()) +} + +/// Load a single option from a byte string. +pub fn option_set<'a>(option: &mut option, value: &'a [u8]) -> Result<(), ParseError<'a>> { + let value = parse_value(value, 0, option.type_)?; + + unsafe { set_option_value(option, value) } +} + +#[derive(Clone, Debug, Eq, PartialEq, Hash)] +struct Span<'a> { + text: Cow<'a, [u8]>, + line: u32, +} + +impl<'a> Span<'a> { + pub fn new(text: &'a [u8], line: u32) -> Self { + Self { + text: Cow::Borrowed(text), + line, + } + } +} + +enum Value<'a> { + Str(&'a [u8]), + Float(f64), + UInt(u64), + Bool(bool), +} + +fn is_space(c: Option) -> bool { + c.map(|x| x.is_ascii_whitespace()).unwrap_or(false) +} + +/// Trim starting and ending spaces off of a byte slice +fn trim_bytes(mut slice: &[u8]) -> &[u8] { + while is_space(slice.first().copied()) { + slice = slice.split_first().unwrap().1; + } + + while is_space(slice.last().copied()) { + slice = slice.split_last().unwrap().1 + } + + slice +} + +/// Parse a key-value pair separated by `:` then strip the whitespace +/// off of both the key and the value. +fn parse_kv<'a>(input: &'a [u8], line: u32) -> Result<(&'a [u8], &'a [u8]), ParseError<'a>> { + let mut first = true; + + let mut split = input.split(|&x| { + if x == b':' && first { + first = false; + return true; + } + + false + }); + + let key = match split.next() { + Some(x) => x, + // There will always be at least one subslice + None => unreachable!(), + }; + let value = match split.next() { + Some(x) => x, + None => return Err(ParseError::missing_colon(Span::new(input, line))), + }; + + let key = trim_bytes(key); + + if !is_valid_name(key) { + return Err(ParseError::invalid_key(Span::new(key, line))); + } + + Ok((key, trim_bytes(value))) +} + +/// Validate a key name. +/// +/// A valid key can only contain characters in `[a-zA-Z0-9_]`. +fn is_valid_name(key: &[u8]) -> bool { + key.iter() + .copied() + .all(|x: u8| x.is_ascii_alphanumeric() || x == b'_') +} + +/// Parse an unsigned integer. Does not implement expression handling +/// that was present in the original ccommon implmentation since it +/// was deemed that it probably wasn't needed. +fn parse_uint<'a>(input: &'a [u8], line: u32) -> Result> { + use std::str; + + let s = str::from_utf8(input).map_err(|_| ParseError::invalid_uint(Span::new(input, line)))?; + + s.parse() + .map_err(|_| ParseError::invalid_uint(Span::new(input, line))) +} + +/// Parse a string value. The only thing this method does is verify that +/// the string doesn't contain any nul characters. +fn parse_str<'a>(input: &'a [u8], line: u32) -> Result<&'a [u8], ParseError<'a>> { + if input.iter().any(|&c| c == 0) { + return Err(ParseError::string_contained_null(Span::new(input, line))); + } + + Ok(input) +} + +/// Parse a boolean. A boolean is either the literal `yes` or the literal `no. +fn parse_bool<'a>(input: &'a [u8], line: u32) -> Result> { + match input { + b"yes" => Ok(true), + b"no" => Ok(false), + _ => Err(ParseError::invalid_bool(Span::new(input, line))), + } +} + +/// Parse a floating point number. This just calls `f64::parse`. +fn parse_fpn<'a>(input: &'a [u8], line: u32) -> Result> { + use std::str; + + let s = str::from_utf8(input).map_err(|_| ParseError::invalid_fpn(Span::new(input, line)))?; + + s.parse() + .map_err(|_| ParseError::invalid_fpn(Span::new(input, line))) +} + +/// Given the type of the option, parse a value +fn parse_value<'a>( + input: &'a [u8], + line: u32, + ty: option_type, +) -> Result, ParseError<'a>> { + match ty { + OPTION_TYPE_UINT => Ok(Value::UInt(parse_uint(input, line)?)), + OPTION_TYPE_BOOL => Ok(Value::Bool(parse_bool(input, line)?)), + OPTION_TYPE_STR => Ok(Value::Str(parse_str(input, line)?)), + OPTION_TYPE_FPN => Ok(Value::Float(parse_fpn(input, line)?)), + + _ => Err(ParseError::unknown_option_type(Span::new(input, line))), + } +} + +/// Search for a named option within the options array. +unsafe fn find_option<'a>(options: &'a mut [option], name: &[u8]) -> Option<&'a mut option> { + for opt in options.iter_mut() { + assert!(!opt.name.is_null()); + + let opt_name = CStr::from_ptr(opt.name); + + if opt_name.to_bytes() == name { + return Some(opt); + } + } + + None +} + +unsafe fn set_option_value(option: &mut option, value: Value) -> Result<(), ParseError<'static>> { + use ccommon_sys::{_cc_alloc, _cc_free}; + use std::mem::MaybeUninit; + + match value { + Value::Bool(v) => option.val.vbool = v, + Value::Float(v) => option.val.vfpn = v, + Value::UInt(v) => option.val.vuint = v, + Value::Str(v) => { + if option.set && !option.val.vstr.is_null() { + // Avoid leaking memory in the case where the option has + // already been initialized. + _cc_free( + option.val.vstr as *mut libc::c_void, + c_str!(module_path!()), + line!() as std::os::raw::c_int, + ); + } + + let mem = _cc_alloc( + (v.len() + 1).try_into().unwrap(), + c_str!(module_path!()), + line!() as std::os::raw::c_int, + ) as *mut MaybeUninit; + + if mem.is_null() { + return Err(ParseError::oom()); + } + + // Copy the string over + std::ptr::copy_nonoverlapping(v.as_ptr() as *const MaybeUninit, mem, v.len()); + + // Add nul terminator + std::ptr::write(mem.wrapping_add(v.len()), MaybeUninit::new(0)); + + option.val.vstr = mem as *mut i8 + } + } + + option.set = true; + + Ok(()) +} + +fn escape_string(bytes: &[u8]) -> String { + bytes + .iter() + .flat_map(|x| std::ascii::escape_default(*x)) + .map(|c| c as char) + .collect() +} + +impl fmt::Debug for Value<'_> { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + match self { + Value::Bool(v) => fmt.debug_tuple("Bool").field(v).finish(), + Value::Float(v) => fmt.debug_tuple("Float").field(v).finish(), + Value::UInt(v) => fmt.debug_tuple("UInt").field(v).finish(), + Value::Str(s) => { + let escaped = &escape_string(s); + fmt.debug_tuple("Str").field(escaped).finish() + } + } + } +} + +impl fmt::Display for ParseError<'_> { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + use ParseErrorType::*; + + let spanmsg: String = (*self.span.text) + .iter() + .copied() + .flat_map(std::ascii::escape_default) + .map(|c| c as char) + .collect(); + + match &self.data { + InvalidBool => write!( + fmt, + "line {}: '{}' is not a valid boolean expression, expected 'yes' or 'no'", + self.span.line, spanmsg + ), + InvalidUInt => write!( + fmt, + "line {}: '{}' is not a valid integer", + self.span.line, spanmsg + ), + StringWithNull => write!( + fmt, + "line {}: string literal contained nul character", + self.span.line + ), + InvalidFloat => write!( + fmt, + "line {}: '{}' is not a valid floating point number", + self.span.line, spanmsg + ), + InvalidKey => write!( + fmt, + "line {}: option '{}' not recognized", + self.span.line, spanmsg + ), + UnknownOptionType => write!(fmt, "unknown option type"), + NoColonInLine => write!( + fmt, + "line {}: invalid formatting, expected ': '", + self.span.line + ), + IOError(e) => write!(fmt, "IO error: {}", e), + OutOfMemory => write!(fmt, "ran out of memory while parsing file"), + } + } +} + +impl fmt::Debug for ParseError<'_> { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + // Create a debug representation that actually shows the error + // message since the internals are fairly unhelpful when they + // show up in a `unwrap` or `expect` call. + fmt.debug_struct("ParseError") + .field("message", &format_args!("{}", self)) + .field("kind", &self.kind()) + .finish() + } +} + +impl<'a> Error for ParseError<'a> { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match &self.data { + ParseErrorType::IOError(e) => Some(e), + _ => None, + } + } +} diff --git a/rust/ccommon-backend/src/option/print.rs b/rust/ccommon-backend/src/option/print.rs new file mode 100644 index 000000000..9f5c4731e --- /dev/null +++ b/rust/ccommon-backend/src/option/print.rs @@ -0,0 +1,120 @@ +// ccommon - a cache common library. +// Copyright (C) 2019 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use ccommon_sys::{ + option, option_type, option_val, OPTION_TYPE_BOOL, OPTION_TYPE_FPN, OPTION_TYPE_STR, + OPTION_TYPE_UINT, +}; + +use std::ffi::CStr; +use std::io::{Result, Write}; + +unsafe fn fmt_cstr(ptr: *const libc::c_char) -> &'static str { + CStr::from_ptr(ptr).to_str().unwrap() +} + +fn fmt_type(ty: option_type) -> &'static str { + match ty { + OPTION_TYPE_BOOL => "bool", + OPTION_TYPE_UINT => "unsigned int", + OPTION_TYPE_FPN => "double", + OPTION_TYPE_STR => "string", + _ => "", + } +} + +fn fmt_value(writer: &mut W, val: option_val, ty: option_type) -> Result<()> { + unsafe { + match ty { + OPTION_TYPE_BOOL => write!(writer, "{: <20}", if val.vbool { "yes" } else { "no" }), + OPTION_TYPE_UINT => write!(writer, "{: <20}", val.vuint), + OPTION_TYPE_FPN => write!(writer, "{: <20}", val.vfpn), + OPTION_TYPE_STR => { + if val.vstr.is_null() { + write!(writer, "{: <20}", "NULL") + } else { + write!(writer, "{: <20}", fmt_cstr(val.vstr)) + } + } + _ => write!(writer, "{: <20}", ""), + } + } +} + +/// Print a single option. +/// +/// # Safety +/// This function assumes that the description and name +/// pointers within the options always point to a valid +/// C string. It also assumes that the value for string +/// type options is either a valid C string or null. +pub unsafe fn option_print(writer: &mut W, opt: &option) -> Result<()> { + write!( + writer, + "name: {: <31} type: {: <15} current: ", + fmt_cstr(opt.name), + fmt_type(opt.type_) + )?; + fmt_value(writer, opt.val, opt.type_)?; + write!(writer, " (default: ")?; + fmt_value(writer, opt.default_val, opt.type_)?; + writeln!(writer, ")") +} + +/// Print all options. +/// +/// # Safety +/// This function assumes that the description and name +/// pointers within the options always point to a valid +/// C string. It also assumes that the value for string +/// type options is either a valid C string or null. +pub unsafe fn option_print_all(writer: &mut W, options: &[option]) -> Result<()> { + for opt in options.iter() { + option_print(writer, opt)?; + } + + Ok(()) +} + +/// Describe a single option. +/// +/// # Safety +/// This function assumes that the description and name +/// pointers within the options always point to a valid +/// C string. It also assumes that the value for string +/// type options is either a valid C string or null. +pub unsafe fn option_describe(writer: &mut W, opt: &option) -> Result<()> { + let name = fmt_cstr(opt.name); + let desc = fmt_cstr(opt.description); + + write!(writer, "{: <31} {: <15} ", name, fmt_type(opt.type_))?; + fmt_value(writer, opt.default_val, opt.type_)?; + writeln!(writer, " {}\n", desc) +} + +/// Describe all options. +/// +/// # Safety +/// This function assumes that the description and name +/// pointers within the options always point to a valid +/// C string. It also assumes that the value for string +/// type options is either a valid C string or null. +pub unsafe fn option_describe_all(writer: &mut W, options: &[option]) -> Result<()> { + for opt in options.iter() { + option_describe(writer, opt)?; + } + + Ok(()) +} diff --git a/rust/ccommon-backend/tests/parse.rs b/rust/ccommon-backend/tests/parse.rs new file mode 100644 index 000000000..f6b253ac5 --- /dev/null +++ b/rust/ccommon-backend/tests/parse.rs @@ -0,0 +1,190 @@ +// ccommon - a cache common library. +// Copyright (C) 2019 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use ccommon_backend::option::*; +use ccommon_rs::{option::*, Options}; + +use std::ffi::CStr; +use std::io::Cursor; + +#[derive(Options)] +#[repr(C)] +pub struct TestOptions { + #[option(desc = "float option")] + fpn_option: Float, + #[option(desc = "uint option")] + uint_option: UInt, + #[option(desc = "bool option")] + bool_option: Bool, + #[option(desc = "string option")] + str_option: Str, +} + +macro_rules! c_str_ffi { + ($s:expr) => { + unsafe { CStr::from_bytes_with_nul_unchecked(concat!($s, "\0").as_bytes()) } + }; +} + +fn cstr_to_str(ptr: *const libc::c_char) -> Option<&'static str> { + if ptr.is_null() { + return None; + } + + unsafe { + let cstr = CStr::from_ptr(ptr); + Some(cstr.to_str().unwrap()) + } +} + +#[test] +fn parse_sanity_test() { + let mut options = TestOptions::new(); + let config: &[u8] = br#" + fpn_option: 53.1 + uint_option: 1000 + bool_option: yes + str_option: Hello, World! + "#; + + let mut cursor = Cursor::new(config); + options.load(&mut cursor).expect("Failed to parse config"); + + assert_eq!(options.fpn_option.value(), 53.1); + assert_eq!(options.uint_option.value(), 1000); + assert_eq!(options.bool_option.value(), true); + assert_eq!( + options.str_option.as_cstr(), + Some(c_str_ffi!("Hello, World!")) + ); +} + +#[test] +fn unknown_option() { + let mut options = TestOptions::new(); + let config: &[u8] = b"unknown: FOO"; + + let mut cursor = Cursor::new(config); + let err = options.load(&mut cursor).unwrap_err(); + + assert_eq!(err.kind(), ParseErrorKind::UnknownOption); +} + +#[test] +fn parse_bool() { + let mut options = TestOptions::new(); + let config1: &[u8] = b"bool_option: yes"; + let config2: &[u8] = b"bool_option: no"; + + let mut cursor1 = Cursor::new(config1); + options.load(&mut cursor1).unwrap(); + + assert_eq!(options.bool_option.value(), true); + + let mut cursor2 = Cursor::new(config2); + options.load(&mut cursor2).unwrap(); + + assert_eq!(options.bool_option.value(), false); +} + +#[test] +fn overlarge_uint() { + let mut options = TestOptions::new(); + let config: &[u8] = b"uint_option: 100000000000000000000000000000000000000000"; + + let mut cursor = Cursor::new(config); + let err = options.load(&mut cursor).unwrap_err(); + + assert_eq!(err.kind(), ParseErrorKind::InvalidUInt); +} + +#[test] +fn non_colon_separated_line() { + let mut options = TestOptions::new(); + let config: &[u8] = b"foobar"; + + let mut cursor = Cursor::new(config); + let err = options.load(&mut cursor).unwrap_err(); + + assert_eq!(err.kind(), ParseErrorKind::IncorrectKVSyntax); +} + +#[test] +fn test_parse_uint() { + let mut options = TestOptions::new(); + + let mut cursor = Cursor::new(b"uint_option: invalid"); + assert!(options.load(&mut cursor).is_err()); + + let mut cursor = Cursor::new(b"uint_option: -1"); + assert!(options.load(&mut cursor).is_err()); + + let mut cursor = Cursor::new(b"uint_option: 1"); + options.load(&mut cursor).unwrap(); + assert_eq!(options.uint_option.value(), 1); + + let mut cursor = Cursor::new(b"uint_option: 1A"); + assert!(options.load(&mut cursor).is_err()); +} + +#[test] +fn test_parse_float() { + let mut options = TestOptions::new(); + + let mut cursor = Cursor::new(b"fpn_option: invalid"); + assert!(options.load(&mut cursor).is_err()); + + let mut cursor = Cursor::new(b"fpn_option: 1.0e-5X"); + assert!(options.load(&mut cursor).is_err()); + + let mut cursor = Cursor::new(&b"fpn_option: 1.0e10000000000000000000000"[..]); + options.load(&mut cursor).unwrap(); + assert!(options.fpn_option.value().is_infinite()); + + let mut cursor = Cursor::new("fpn_option: 1.0"); + options.load(&mut cursor).unwrap(); + assert_eq!(options.fpn_option.value(), 1.0); + + let mut cursor = Cursor::new("fpn_option: -1e20"); + options.load(&mut cursor).unwrap(); + assert_eq!(options.fpn_option.value(), -1e20); +} + +#[test] +fn test_parse_string() { + let mut options = TestOptions::new(); + + let mut cursor = Cursor::new(b"str_option: foo"); + options.load(&mut cursor).unwrap(); + assert_eq!(cstr_to_str(options.str_option.value()), Some("foo")); + + let mut cursor = Cursor::new(b"str_option: foo\0bar"); + assert!(options.load(&mut cursor).is_err()); +} + +#[test] +fn test_parse_comments() { + let mut options = TestOptions::new(); + + let mut cursor = Cursor::new( + &br#" + # Test Comment + uint_option: 22 + # uint_option: 34 + "#[..], + ); + options.load(&mut cursor).unwrap(); + assert_eq!(options.uint_option.value(), 22); +} diff --git a/rust/ccommon-derive/Cargo.toml b/rust/ccommon-derive/Cargo.toml new file mode 100644 index 000000000..c9616d41f --- /dev/null +++ b/rust/ccommon-derive/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "ccommon-derive" +version = "0.1.0" +authors = ["Sean Lynch "] +edition = "2018" +publish = false + +[lib] +proc-macro = true + +[dependencies] +syn = { version = "1.0.0", features = [ "parsing" ] } +quote = "1.0.0" +proc-macro2 = "1.0.0" +proc-macro-crate = "0.1.4" diff --git a/rust/ccommon-derive/src/attrs.rs b/rust/ccommon-derive/src/attrs.rs new file mode 100644 index 000000000..34e6503f6 --- /dev/null +++ b/rust/ccommon-derive/src/attrs.rs @@ -0,0 +1,191 @@ +// ccommon - a cache common library. +// Copyright (C) 2019 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use proc_macro2::TokenStream; +use quote::ToTokens; +use syn::parse::*; +use syn::punctuated::*; +use syn::spanned::Spanned; +use syn::*; + +use std::mem; + +pub struct AttrOption { + pub name: Ident, + pub eq: Token![=], + pub val: T, +} + +pub type EqOption = AttrOption; +pub type ExprOption = AttrOption; + +pub struct MetricAttr { + pub desc: EqOption, + pub name: Option, +} + +pub struct OptionAttr { + pub desc: EqOption, + pub name: Option, + pub default: Option, +} + +enum StrOrExpr { + Str(LitStr), + Expr(Expr), +} + +impl Parse for StrOrExpr { + fn parse(buf: &ParseBuffer) -> Result { + if buf.fork().parse::().is_ok() { + return Ok(StrOrExpr::Str(buf.parse()?)); + } + + buf.parse().map(StrOrExpr::Expr) + } +} + +impl AttrOption { + pub fn into_lit(self) -> Result { + match self.val { + StrOrExpr::Str(s) => Ok(EqOption { + name: self.name, + eq: self.eq, + val: Lit::Str(s), + }), + StrOrExpr::Expr(e) => Err(Error::new( + e.span(), + "Found expression, expected a literal string", + )), + } + } + + pub fn into_expr(self) -> ExprOption { + let expr = match self.val { + StrOrExpr::Str(s) => parse_quote!(#s), + StrOrExpr::Expr(e) => e, + }; + + ExprOption { + name: self.name, + eq: self.eq, + val: expr, + } + } +} + +impl Parse for AttrOption { + fn parse(buf: &ParseBuffer) -> Result { + Ok(Self { + name: buf.parse()?, + eq: buf.parse()?, + val: buf.parse()?, + }) + } +} + +impl Parse for MetricAttr { + fn parse(buf: &ParseBuffer) -> Result { + let seq: Punctuated = Punctuated::parse_terminated(buf)?; + + let mut desc = None; + let mut name = None; + + for opt in seq { + let span = opt.name.span(); + let param = opt.name.to_string(); + let seen = match &*param { + "desc" => mem::replace(&mut desc, Some(opt)).is_some(), + "name" => mem::replace(&mut name, Some(opt)).is_some(), + _ => { + return Err(Error::new( + opt.name.span(), + format!("Unknown option `{}`", opt.name), + )) + } + }; + + if seen { + return Err(Error::new( + span, + format!("`{}` may only be specified once", param), + )); + } + } + + let desc = match desc { + Some(x) => x, + None => return Err(buf.error("Expected a `desc` parameter here")), + }; + + Ok(Self { desc, name }) + } +} + +impl Parse for OptionAttr { + fn parse(buf: &ParseBuffer) -> Result { + let seq: Punctuated, Token![,]> = Punctuated::parse_terminated(buf)?; + + let mut desc = None; + let mut name = None; + let mut default = None; + + for val in seq { + let span = val.span(); + let param = val.name.to_string(); + let seen = match &*param { + "desc" => mem::replace(&mut desc, Some(val.into_lit()?)).is_some(), + "name" => mem::replace(&mut name, Some(val.into_lit()?)).is_some(), + "default" => mem::replace(&mut default, Some(val.into_expr())).is_some(), + _ => return Err(Error::new(span, format!("Unknown option `{}`", param))), + }; + + if seen { + return Err(Error::new( + span, + format!("`{}` may only be specified once", param), + )); + } + } + + let desc = match desc { + Some(x) => x, + None => return Err(buf.error("Expected a `desc` parameter here")), + }; + + Ok(Self { + desc, + name, + default, + }) + } +} + +impl ToTokens for AttrOption { + fn to_tokens(&self, stream: &mut TokenStream) { + self.name.to_tokens(stream); + self.eq.to_tokens(stream); + self.val.to_tokens(stream); + } +} + +impl ToTokens for StrOrExpr { + fn to_tokens(&self, stream: &mut TokenStream) { + match self { + Self::Str(s) => s.to_tokens(stream), + Self::Expr(e) => e.to_tokens(stream), + } + } +} diff --git a/rust/ccommon-derive/src/lib.rs b/rust/ccommon-derive/src/lib.rs new file mode 100644 index 000000000..7aae4650e --- /dev/null +++ b/rust/ccommon-derive/src/lib.rs @@ -0,0 +1,488 @@ +// ccommon - a cache common library. +// Copyright (C) 2019 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Derive macros for `ccommon_rs` + +extern crate proc_macro; + +mod attrs; + +use proc_macro2::{Span, TokenStream}; +use quote::{quote, quote_spanned}; +use syn::spanned::Spanned; +use syn::{ + parse_macro_input, Attribute, Data, DeriveInput, Error, Field, Fields, Generics, Ident, Lit, + LitStr, +}; + +use self::attrs::*; + +/// Derive macro for `Metrics`. +/// +/// Fields in the struct attempting to use this derive macro +/// must either implement `SingleMetric` or `Metrics`. This is +/// decided upon based on the field being decorated with the +/// `metric` attribute. +/// +/// # Example +/// ```rust,ignore +/// #[derive(Metrics)] +/// struct MyMetrics { +/// // This metric will have a name of `metric1` and a description +/// // of `"The first metric"`. +/// // This field requires that `Gauge` implements `SingleMetric`. +/// #[metric(desc = "The first metric")] +/// metric1: Gauge, +/// +/// // This metric will have a name of `mymetric.metric2` and a +/// // a description of `"The second metric"`. +/// // This field requires that `Counter` implements `SingleMetric`. +/// #[metric(desc = "The second metric", "mymetric.metric2")] +/// metric2: Counter +/// } +/// +/// #[derive(Metrics)] +/// struct OtherMetrics { +/// // This field must implement `Metrics` +/// my_metrics: MyMetrics +/// } +/// ``` +#[proc_macro_derive(Metrics, attributes(metric))] +pub fn derive_metrics(stream: proc_macro::TokenStream) -> proc_macro::TokenStream { + let input = parse_macro_input!(stream as DeriveInput); + + match derive_metrics_impl(input) { + Ok(x) => x, + Err(e) => e.to_compile_error(), + } + .into() +} + +/// Derive macro for `Options`. +/// +/// Fields in the struct attempting to use this derive macro +/// must either implement `SingleOption` or `Options`. Which +/// one of these will be required is decided by whether the +/// field is decorated by the `option` attribute. +/// +/// The following parameters can be passed to the `option` attribute +/// - `desc`: Sets the description of the field. +/// - `name`: Overrides the name of the field, the default +/// name is the name of the field within the struct. +/// - `default`: Override the default value for the field. +/// By default, this is `Default::default()` or `std::ptr::null_mut()` +/// in the case of strings. This can be any valid rust expression +/// that returns the correct type. +/// +/// # Example +/// ```rust,ignore +/// #[derive(Options)] +/// #[repr(C)] +/// struct MyOptions { +/// // This option will have a name of `options.option1`, a description +/// // of `Option 1`, and a default value of `false`. +/// #[option(desc = "Option 1", name = "options.option1")] +/// opt1: Bool, +/// +/// #[option( +/// desc = "Function result option", +/// default = example_factory_function() +/// )] +/// opt2: UInt +/// } +/// +/// #[derive(Option)] +/// #[repr(transparent)] +/// struct OtherOptions { +/// inner: MyOptions +/// } +/// ``` +#[proc_macro_derive(Options, attributes(option))] +pub fn derive_options(stream: proc_macro::TokenStream) -> proc_macro::TokenStream { + let input = parse_macro_input!(stream as DeriveInput); + + match derive_options_impl(input) { + Ok(x) => x, + Err(e) => e.to_compile_error(), + } + .into() +} + +fn derive_metrics_impl(input: DeriveInput) -> Result { + let krate = crate_name("ccommon-rs")?; + + let data = match input.data { + Data::Struct(data) => data, + Data::Enum(data) => { + return Err(Error::new( + data.enum_token.span(), + "Can only derive Metrics for a struct", + )) + } + Data::Union(data) => { + return Err(Error::new( + data.union_token.span(), + "Can only derive Metrics for a struct", + )) + } + }; + + if !is_repr_c_or_transparent(&input.attrs) { + return Err(Error::new( + input.ident.span(), + format!( + "`{}` must be either #[repr(C)] or #[repr(transparent)] to implement Metrics", + input.ident + ), + )); + } + + if has_generics(&input.generics) { + return Err(Error::new( + input.generics.span(), + "Cannot derive Metrics for a struct with generics", + )); + } + + let ident = input.ident; + let process_field = |is_tuple| { + let krate = krate.clone(); + move |(i, field): (usize, &Field)| { + let ty = &field.ty; + let name = &field.ident; + let label = if is_tuple { + quote! {} + } else { + quote! { #name: } + }; + + Ok(match get_metric_attr(&field.attrs)? { + Some(attr) => { + let desc = to_c_str(&attr.desc.val); + let namestr = match attr.name { + Some(name) => to_c_str(&name.val), + None => match field.ident.as_ref() { + Some(name) => to_c_str(&name_as_lit(name)), + None => { + to_c_str(&Lit::Str(LitStr::new(&format!("{}", i), field.span()))) + } + }, + }; + + quote! { + #label <#ty as #krate::metric::SingleMetric>::new(#namestr, #desc) + } + } + None => quote! { + #label <#ty as #krate::metric::Metrics>::new() + }, + }) + } + }; + + let initializer = match data.fields { + Fields::Named(fields) => { + let initializers: Vec<_> = fields + .named + .iter() + .enumerate() + .map(process_field(false)) + .collect::>()?; + + quote! { + Self { + #( #initializers, )* + } + } + } + Fields::Unnamed(fields) => { + let initializers: Vec<_> = fields + .unnamed + .iter() + .enumerate() + .map(process_field(true)) + .collect::>()?; + + quote! { + Self ( + #( #initializers, )* + ) + } + } + Fields::Unit => quote!(Self), + }; + + Ok(quote! { + unsafe impl #krate::metric::Metrics for #ident { + fn new() -> Self { + #initializer + } + } + }) +} + +fn derive_options_impl(input: DeriveInput) -> Result { + let krate = crate_name("ccommon-rs")?; + + let data = match input.data { + Data::Struct(data) => data, + Data::Enum(data) => { + return Err(Error::new( + data.enum_token.span(), + "Can only derive Metrics for a struct", + )) + } + Data::Union(data) => { + return Err(Error::new( + data.union_token.span(), + "Can only derive Metrics for a struct", + )) + } + }; + + if !is_repr_c_or_transparent(&input.attrs) { + return Err(Error::new( + input.ident.span(), + format!( + "`{}` must be either #[repr(C)] or #[repr(transparent)] to implement Metrics", + input.ident + ), + )); + } + + if has_generics(&input.generics) { + return Err(Error::new( + input.generics.span(), + "Cannot derive Metrics for a struct with generics", + )); + } + + let ident = input.ident; + let process_field = |is_tuple| { + let krate = krate.clone(); + move |(i, field): (usize, &Field)| { + let ty = &field.ty; + let name = &field.ident; + let label = if is_tuple { + quote! {} + } else { + quote! { #name: } + }; + + Ok(match get_option_attr(&field.attrs)? { + Some(attr) => { + let desc = to_c_str(&attr.desc.val); + let namestr = match attr.name { + Some(name) => to_c_str(&name.val), + None => match field.ident.as_ref() { + Some(name) => to_c_str(&name_as_lit(name)), + None => { + to_c_str(&Lit::Str(LitStr::new(&format!("{}", i), field.span()))) + } + }, + }; + + match attr.default.map(|x| x.val) { + Some(default) => quote! { + #label <#ty as #krate::option::SingleOption>::new( + #default, + #namestr, + #desc + ) + }, + None => quote! { + #label <#ty as #krate::option::SingleOption>::defaulted(#namestr, #desc) + }, + } + } + None => quote! { + #label <#ty as #krate::option::Options>::new() + }, + }) + } + }; + + let initializer = match data.fields { + Fields::Named(fields) => { + let initializers: Vec<_> = fields + .named + .iter() + .enumerate() + .map(process_field(false)) + .collect::>()?; + + quote! { + Self { #( #initializers, )* } + } + } + Fields::Unnamed(fields) => { + let initializers: Vec<_> = fields + .unnamed + .iter() + .enumerate() + .map(process_field(true)) + .collect::>()?; + + quote! { + Self ( #( #initializers, )* ) + } + } + Fields::Unit => quote!(Self), + }; + + Ok(quote! { + unsafe impl #krate::option::Options for #ident { + fn new() -> Self { + #initializer + } + } + }) +} + +fn crate_name(name: &'static str) -> Result { + if std::env::var("CARGO_PKG_NAME").unwrap() == "ccommon-rs" { + return Ok(quote! { ::ccommon_rs }); + } + + let name = match proc_macro_crate::crate_name(name) { + Ok(name) => name, + Err(e) => return Err(Error::new(Span::call_site(), e)), + }; + + let ident = Ident::new(&name, Span::call_site()); + + Ok(quote! { ::#ident }) +} + +fn name_as_lit(name: &Ident) -> Lit { + Lit::Str(LitStr::new(&format!("{}", name), name.span())) +} + +fn to_c_str(desc: &Lit) -> TokenStream { + use syn::LitByteStr; + + let span = desc.span(); + + let mut value = match desc { + Lit::Str(lit) => lit.value().into_bytes(), + Lit::ByteStr(lit) => lit.value(), + _ => unreachable!(), + }; + + for &byte in &value { + if byte == b'\0' { + return quote_spanned! { span => + compile_error!( + "Description contained a nul character" + ) + }; + } + } + + value.push(b'\0'); + + let lit = LitByteStr::new(&value, span); + + quote! { + unsafe { + ::std::ffi::CStr::from_bytes_with_nul_unchecked(#lit) + } + } +} + +fn get_metric_attr(attrs: &[Attribute]) -> Result, Error> { + for attr in attrs { + let ident = match attr.path.get_ident() { + Some(x) => x, + None => continue, + }; + + if ident != "metric" { + continue; + } + + return Ok(Some(attr.parse_args()?)); + } + + Ok(None) +} + +fn get_option_attr(attrs: &[Attribute]) -> Result, Error> { + for attr in attrs { + let ident = match attr.path.get_ident() { + Some(x) => x, + None => continue, + }; + + if ident != "option" { + continue; + } + + return Ok(Some(attr.parse_args()?)); + } + + Ok(None) +} + +fn is_repr_c_or_transparent(attrs: &[Attribute]) -> bool { + use syn::{Meta, NestedMeta}; + + fn is_correct_meta(nested: &NestedMeta) -> bool { + let meta = match nested { + NestedMeta::Meta(meta) => meta, + _ => return false, + }; + + let path = match meta { + Meta::Path(path) => path, + _ => return false, + }; + + let ident = match path.get_ident() { + Some(ident) => ident, + _ => return false, + }; + + ident == "C" || ident == "transparent" + } + + for attr in attrs { + let ident = match attr.path.get_ident() { + Some(ident) => ident, + None => continue, + }; + + if ident != "repr" { + continue; + } + + let list = match attr.parse_meta() { + Ok(Meta::List(list)) => list, + _ => continue, + }; + + for nested in &list.nested { + if is_correct_meta(nested) { + return true; + } + } + } + + false +} + +fn has_generics(generics: &Generics) -> bool { + generics.lt_token.is_some() || generics.where_clause.is_some() +} diff --git a/rust/ccommon-rs/Cargo.toml b/rust/ccommon-rs/Cargo.toml new file mode 100644 index 000000000..836ae1526 --- /dev/null +++ b/rust/ccommon-rs/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "ccommon-rs" +version = "0.1.0" +authors = [ + "Jonathan Simms ", + "Sean Lynch " +] +edition = "2018" + +[features] +default = [ "derive" ] +derive = [ "ccommon-derive" ] + +[dependencies] +ccommon-sys = { path = "../ccommon-sys" } +ccommon-backend = { path = "../ccommon-backend" } +libc = "0.2.0" +log = "0.4.0" +bytes = "0.5.0" + +[dependencies.ccommon-derive] +path = "../ccommon-derive" +optional = true + +[build-dependencies] +autocfg = "0.1.7" + +[dev-dependencies] +rusty-fork = "0.2.0" +gag = "0.1.10" diff --git a/rust/ccommon-rs/build.rs b/rust/ccommon-rs/build.rs new file mode 100644 index 000000000..0433a3c10 --- /dev/null +++ b/rust/ccommon-rs/build.rs @@ -0,0 +1,23 @@ +// ccommon - a cache common library. +// Copyright (C) 2019 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use autocfg::AutoCfg; + +fn main() { + let cfg = AutoCfg::new().unwrap(); + + // For rust versions >= 1.38 we can use std::any::type_name + cfg.emit_rustc_version(1, 38); +} diff --git a/rust/ccommon_rs/src/bstring.rs b/rust/ccommon-rs/src/bstring.rs similarity index 84% rename from rust/ccommon_rs/src/bstring.rs rename to rust/ccommon-rs/src/bstring.rs index f8588d608..8c70b669e 100644 --- a/rust/ccommon_rs/src/bstring.rs +++ b/rust/ccommon-rs/src/bstring.rs @@ -1,3 +1,18 @@ +// ccommon - a cache common library. +// Copyright (C) 2019 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //! BString is a wrapper around a foreign allocated and freed pointer to a cc_bstring. //! It takes care of creating and freeing the foreign pointer within the normal //! Rust lifetime rules. It has a companion reference object BStr, and the relation @@ -22,8 +37,8 @@ //! //! [nasal demons]: http://www.catb.org/jargon/html/N/nasal-demons.html -use cc_binding as bind; use std::borrow::Borrow; +use std::borrow::BorrowMut; use std::boxed::Box; use std::cell::UnsafeCell; use std::fmt; @@ -32,32 +47,22 @@ use std::fmt::Formatter; use std::mem; use std::ops::{Deref, DerefMut}; use std::slice; -use std::str; -use std::borrow::BorrowMut; - - -pub type CCbstring = bind::bstring; +use std::str::{self, Utf8Error}; +pub type CCbstring = ccommon_sys::bstring; #[doc(hidden)] #[inline] unsafe fn raw_ptr_to_bytes<'a>(ptr: *const CCbstring) -> &'a [u8] { - slice::from_raw_parts( - (*ptr).data as *const _ as *const u8, - (*ptr).len as usize - ) + slice::from_raw_parts((*ptr).data as *const _ as *const u8, (*ptr).len as usize) } #[doc(hidden)] #[inline] unsafe fn raw_ptr_to_bytes_mut<'a>(ptr: *mut CCbstring) -> &'a mut [u8] { - slice::from_raw_parts_mut( - (*ptr).data as *mut _ as *mut u8, - (*ptr).len as usize - ) + slice::from_raw_parts_mut((*ptr).data as *mut _ as *mut u8, (*ptr).len as usize) } - // this pattern lifted from https://docs.rs/foreign-types-shared/0.1.1/src/foreign_types_shared/lib.rs.html struct Opaque(UnsafeCell<()>); @@ -91,6 +96,18 @@ impl BStr { pub fn as_ptr(&self) -> *mut CCbstring { self as *const _ as *mut _ } + + pub fn from_ref(ccb: &CCbstring) -> &Self { + unsafe { Self::from_ptr(ccb as *const CCbstring as *mut _) } + } + + pub fn to_utf8_str(&self) -> Result<&str, Utf8Error> { + str::from_utf8(&self[..]) + } + + pub fn to_utf8_string(&self) -> Result { + self.to_utf8_str().map(|x| x.to_owned()) + } } impl Deref for BStr { @@ -117,7 +134,7 @@ impl AsRef for BStr { impl AsMut for BStr { fn as_mut(&mut self) -> &mut CCbstring { - unsafe { &mut *(self.as_ptr() as *mut _)} + unsafe { &mut *(self.as_ptr() as *mut _) } } } @@ -129,7 +146,7 @@ impl Borrow for BStr { impl BorrowMut for BStr { fn borrow_mut(&mut self) -> &mut CCbstring { - unsafe { &mut *(self.as_ptr() as *mut _)} + unsafe { &mut *(self.as_ptr() as *mut _) } } } @@ -227,7 +244,7 @@ pub struct BString(*mut CCbstring); impl BString { pub fn new(size: u32) -> Self { - let bsp: *mut CCbstring = unsafe { bind::bstring_alloc(size) }; + let bsp: *mut CCbstring = unsafe { ccommon_sys::bstring_alloc(size) }; assert!(!bsp.is_null()); BString(bsp) @@ -249,7 +266,7 @@ impl BString { /// Takes byte slice `&[u8]` and copies it into an owned BString. #[inline] pub fn from_bytes(s: &[u8]) -> Self { - let bsp: *mut CCbstring = unsafe { bind::bstring_alloc(s.len() as u32) }; + let bsp: *mut CCbstring = unsafe { ccommon_sys::bstring_alloc(s.len() as u32) }; assert!(!bsp.is_null()); @@ -264,6 +281,7 @@ impl BString { /// /// This method will panic if `src.len() != self.len()` #[inline] + // Note: Used for tests #[allow(dead_code)] fn copy_from_slice(&mut self, src: &[u8]) { assert_eq!(src.len(), self.len()); @@ -271,12 +289,12 @@ impl BString { } #[inline] - fn as_bytes(&self) -> &[u8] { + pub fn as_bytes(&self) -> &[u8] { unsafe { raw_ptr_to_bytes(self.0) } } #[inline] - fn as_bytes_mut(&mut self) -> &mut [u8] { + pub fn as_bytes_mut(&mut self) -> &mut [u8] { unsafe { raw_ptr_to_bytes_mut(self.0) } } @@ -284,6 +302,14 @@ impl BString { fn len(&self) -> usize { unsafe { (*self.0).len as usize } } + + pub fn to_utf8_str(&self) -> Result<&str, Utf8Error> { + str::from_utf8(self.as_bytes()) + } + + pub fn to_utf8_string(&self) -> Result { + self.to_utf8_str().map(|x| x.to_owned()) + } } impl Debug for BString { @@ -305,7 +331,7 @@ impl PartialEq for BString { impl Drop for BString { #[inline] fn drop(&mut self) { - unsafe { bind::bstring_free(&mut self.0) }; + unsafe { ccommon_sys::bstring_free(&mut self.0) }; } } @@ -386,7 +412,6 @@ impl<'a> From<&'a str> for BString { unsafe impl Send for BString {} unsafe impl Sync for BString {} - #[cfg(test)] mod test { use super::*; @@ -395,14 +420,16 @@ mod test { fn test_raw_ptr_to_bytes() { let bs = CCbstring { len: 5, - data: String::from("abcde").as_ptr() as *mut i8 + data: "abcde".as_ptr() as *mut i8, }; let ptr: *const CCbstring = &bs as *const CCbstring; let slice = unsafe { raw_ptr_to_bytes(ptr) }; + + assert_eq!(slice.as_ptr(), bs.data as *mut u8); assert_eq!(slice.len(), 5); - assert_eq!(&slice[..], "abcde".as_bytes()); + assert_eq!(slice, "abcde".as_bytes()); } #[test] @@ -450,7 +477,7 @@ mod test { } fn foreign_code(s: &str) -> *mut CCbstring { - BString::into_raw(BString::from(s)) + BString::into_raw(BString::from(s)) } #[test] diff --git a/rust/ccommon-rs/src/buf.rs b/rust/ccommon-rs/src/buf.rs new file mode 100644 index 000000000..613589447 --- /dev/null +++ b/rust/ccommon-rs/src/buf.rs @@ -0,0 +1,366 @@ +// ccommon - a cache common library. +// Copyright (C) 2019 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use ccommon_sys::buf; + +use std::io::{self, Read, Write}; +use std::mem::MaybeUninit; +use std::ops::{Deref, DerefMut}; + +use bytes::{Buf as BytesBuf, BufMut as BytesBufMut}; + +/// A non-owned buffer, can be read from and written to. +/// +/// # Safety +/// This is a self-referential struct that assumes it is +/// followed by it's data. Attempting to move it or create +/// a new instance (via transmute) will cause UB. +#[repr(transparent)] +#[derive(Debug)] +pub struct Buf { + buf: buf, +} + +impl Buf { + pub unsafe fn from_ptr<'a>(buf: *const buf) -> &'a Buf { + &*(buf as *const Buf) + } + + pub unsafe fn from_mut_ptr<'a>(buf: *mut buf) -> &'a mut Buf { + &mut *(buf as *mut Buf) + } + + pub fn as_ptr(&self) -> *const buf { + &self.buf as *const buf + } + + pub fn as_mut_ptr(&mut self) -> *mut buf { + &mut self.buf as *mut buf + } + + #[deprecated(note = "use from_ptr instead")] + pub unsafe fn from_raw<'a>(buf: *const buf) -> &'a Buf { + &*(buf as *const Buf) + } + + #[deprecated(note = "use from_mut_ptr instead")] + pub unsafe fn from_raw_mut<'a>(buf: *mut buf) -> &'a mut Buf { + &mut *(buf as *mut Buf) + } + + #[deprecated(note = "use as_ptr instead")] + #[allow(clippy::wrong_self_convention)] + pub fn into_raw(&self) -> *const buf { + &self.buf as *const _ + } + + #[deprecated(note = "use as_mut_ptr instead")] + #[allow(clippy::wrong_self_convention)] + pub fn into_raw_mut(&mut self) -> *mut buf { + &mut self.buf as *mut _ + } + + /// The number of bytes that can still be written to the + /// buffer before it is full. + pub fn write_size(&self) -> usize { + assert!(self.buf.wpos as usize <= self.buf.end as usize); + + self.buf.end as usize - self.buf.wpos as usize + } + + /// The number of bytes left to read from the buffer + pub fn read_size(&self) -> usize { + assert!(self.buf.rpos as usize <= self.buf.wpos as usize); + + self.buf.wpos as usize - self.buf.rpos as usize + } + + pub fn capacity(&self) -> usize { + assert!(self.buf.begin.as_ptr() as usize <= self.buf.end as usize); + + self.buf.end as usize - self.buf.begin.as_ptr() as usize + } + + /// Additional capacity required to write count bytes to the buffer + pub fn new_cap(&self, bytes: usize) -> usize { + assert!(self.buf.begin.as_ptr() as usize <= self.buf.wpos as usize); + + bytes.saturating_sub(self.write_size()) + } + + /// Clear the buffer and remove it from any allocation pool + pub fn reset(&mut self) { + self.buf.next.stqe_next = std::ptr::null_mut(); + self.buf.free = false; + self.buf.rpos = self.buf.begin.as_mut_ptr(); + self.buf.wpos = self.buf.rpos; + } + + /// Shift data in the buffer to the end of the buffer. + pub fn rshift(&mut self) { + let size = self.read_size(); + + if size > 0 { + unsafe { + std::ptr::copy(self.buf.rpos, self.buf.rpos.offset(-(size as isize)), size); + } + } + + self.buf.rpos = unsafe { self.buf.end.offset(-(size as isize)) }; + self.buf.wpos = self.buf.end; + } + + /// Shift data in the buffer to the start of the buffer. + pub fn lshift(&mut self) { + let size = self.read_size(); + + if size > 0 { + unsafe { + std::ptr::copy(self.buf.rpos, self.buf.begin.as_mut_ptr(), size); + } + } + + self.buf.rpos = self.buf.begin.as_mut_ptr(); + self.buf.wpos = self.buf.rpos.wrapping_add(size); + } + + /// Get the currently valid part of the buffer as a slice + pub fn as_slice(&self) -> &[u8] { + use std::slice; + + unsafe { slice::from_raw_parts(self.buf.rpos as *const u8, self.read_size()) } + } + + /// Get the currently valid part of the buffer as a slice + pub fn as_slice_mut(&mut self) -> &mut [u8] { + use std::slice; + + unsafe { slice::from_raw_parts_mut(self.buf.rpos as *mut u8, self.read_size()) } + } +} + +/// An owned buffer. +/// +/// In addition to all the operations supported by `Buf` +/// an `OwnedBuf` also supports some resizing operations. +#[repr(transparent)] +#[derive(Debug)] +pub struct OwnedBuf { + buf: *mut buf, +} + +impl OwnedBuf { + pub unsafe fn from_raw(buf: *mut buf) -> Self { + Self { buf } + } + + pub fn into_raw(self) -> *mut buf { + let buf = self.buf; + // Don't want to run drop on the destructor + std::mem::forget(self); + buf + } + + pub fn as_ptr(&self) -> *const buf { + self.buf + } + + pub fn as_mut_ptr(&mut self) -> *mut buf { + self.buf + } + + /// Double the size of the buffer. + pub fn double(&mut self) -> Result<(), crate::Error> { + use ccommon_sys::dbuf_double; + + unsafe { + let status = dbuf_double(&mut self.buf as *mut _); + + if status != 0 { + Err(status.into()) + } else { + Ok(()) + } + } + } + + /// Shrink the buffer to the max of the initial size or the content size. + pub fn shrink(&mut self) -> Result<(), crate::Error> { + use ccommon_sys::dbuf_shrink; + + unsafe { + let status = dbuf_shrink(&mut self.buf as *mut _); + + if status != 0 { + Err(status.into()) + } else { + Ok(()) + } + } + } + + /// Resize the buffer to fit the required capacity. + /// + /// # Panics + /// Panics if `cap` is greater than `u32::MAX`. + pub fn fit(&mut self, cap: usize) -> Result<(), crate::Error> { + use ccommon_sys::dbuf_fit; + + assert!((cap as u64) <= std::u32::MAX as u64); + + unsafe { + let status = dbuf_fit(&mut self.buf as *mut _, cap as u32); + + if status != 0 { + Err(status.into()) + } else { + Ok(()) + } + } + } +} + +unsafe impl Send for OwnedBuf {} +unsafe impl Sync for OwnedBuf {} + +impl Drop for OwnedBuf { + fn drop(&mut self) { + use ccommon_sys::buf_destroy; + + unsafe { + buf_destroy(&mut self.buf as *mut _); + } + } +} + +impl Deref for OwnedBuf { + type Target = Buf; + + fn deref(&self) -> &Buf { + unsafe { Buf::from_ptr(self.buf) } + } +} + +impl DerefMut for OwnedBuf { + fn deref_mut(&mut self) -> &mut Buf { + unsafe { Buf::from_mut_ptr(self.buf) } + } +} + +impl Read for Buf { + fn read(&mut self, out: &mut [u8]) -> io::Result { + let len = self.read_size().min(out.len()); + + unsafe { std::ptr::copy_nonoverlapping(self.buf.rpos as *const u8, out.as_mut_ptr(), len) } + self.buf.rpos = self.buf.rpos.wrapping_add(len); + + Ok(len) + } +} + +impl Write for Buf { + fn write(&mut self, src: &[u8]) -> io::Result { + if src.is_empty() { + return Ok(0); + } + + let len = self.write_size().min(src.len()); + + unsafe { std::ptr::copy_nonoverlapping(src.as_ptr(), self.buf.wpos as *mut _, src.len()) } + self.buf.wpos = self.buf.wpos.wrapping_add(len); + + Ok(len) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl Read for OwnedBuf { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + (**self).read(buf) + } +} + +impl Write for OwnedBuf { + fn write(&mut self, buf: &[u8]) -> io::Result { + (**self).write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + (**self).flush() + } +} + +impl BytesBuf for Buf { + fn remaining(&self) -> usize { + self.read_size() + } + + fn bytes(&self) -> &[u8] { + self.as_slice() + } + + fn advance(&mut self, cnt: usize) { + assert!(cnt <= self.read_size()); + self.buf.rpos = self.buf.rpos.wrapping_add(cnt); + } +} + +impl BytesBufMut for Buf { + fn remaining_mut(&self) -> usize { + self.write_size() + } + + unsafe fn advance_mut(&mut self, cnt: usize) { + assert!(cnt <= self.write_size()); + self.buf.wpos = self.buf.wpos.wrapping_add(cnt); + } + + fn bytes_mut(&mut self) -> &mut [MaybeUninit] { + unsafe { + std::slice::from_raw_parts_mut(self.buf.wpos as *mut MaybeUninit, self.write_size()) + } + } +} + +impl BytesBuf for OwnedBuf { + fn remaining(&self) -> usize { + (**self).remaining() + } + + fn bytes(&self) -> &[u8] { + BytesBuf::bytes(&**self) + } + + fn advance(&mut self, cnt: usize) { + (**self).advance(cnt) + } +} + +impl BytesBufMut for OwnedBuf { + fn remaining_mut(&self) -> usize { + (**self).remaining_mut() + } + + unsafe fn advance_mut(&mut self, cnt: usize) { + (**self).advance_mut(cnt) + } + + fn bytes_mut(&mut self) -> &mut [MaybeUninit] { + BytesBufMut::bytes_mut(&mut **self) + } +} diff --git a/rust/ccommon-rs/src/ccbox.rs b/rust/ccommon-rs/src/ccbox.rs new file mode 100644 index 000000000..10fe4e415 --- /dev/null +++ b/rust/ccommon-rs/src/ccbox.rs @@ -0,0 +1,222 @@ +// ccommon - a cache common library. +// Copyright (C) 2019 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::convert::TryInto; +use std::fmt; +use std::mem::{self, MaybeUninit}; +use std::ops::*; +use std::ptr::NonNull; + +use ccommon_sys::{_cc_alloc, _cc_free}; + +use crate::error::AllocationError; + +macro_rules! c_str { + ($s:expr) => { + concat!($s, "\0").as_ptr() as *const i8 + }; +} + +/// A box that allocates and frees using `cc_alloc` +/// and `cc_free`. +#[repr(transparent)] +pub struct CCBox(NonNull); + +unsafe impl Send for CCBox {} +unsafe impl Sync for CCBox {} + +impl CCBox { + /// Create a new `CCBox` with `val` inside. + /// + /// # Panics + /// Panics if the underlying allocator fails to allocate + /// or if `T` requires an alignment of greater than 16. + pub fn new(val: T) -> CCBox { + // Most malloc implementations give 16-byte alignment + assert!(mem::align_of::() <= 16); + + match Self::try_new(val) { + Ok(x) => x, + Err(e) => panic!("{}", e), + } + } + + /// Attempt to create a new `CCBox` with `val` inside. + /// + /// Since the underlying allocator does not support + /// passing alignments, any allocation with alignment + /// greater than 16 will fail. + /// + /// In addition, returns an error whenever the underlying + /// allocator returns `NULL`. + pub fn try_new(val: T) -> Result, AllocationError> { + if mem::size_of_val(&val) == 0 { + return Ok(CCBox(NonNull::dangling())); + } + + if mem::align_of_val(&val) > 16 { + return Err(AllocationError::new()); + } + + unsafe { + let ptr = _cc_alloc( + mem::size_of_val(&val).try_into().unwrap(), + c_str!(module_path!()), + line!() as std::os::raw::c_int, + ) as *mut MaybeUninit; + + *ptr = MaybeUninit::new(val); + + Ok(CCBox(match NonNull::new(ptr as *mut T) { + Some(x) => x, + None => return Err(AllocationError::new()), + })) + } + } +} + +impl CCBox { + /// Construct a `CCBox` from a raw pointer. + /// + /// After this function is called the raw pointer is owned by + /// the resulting `CCBox`. When the box is dropped, it will run + /// the destructor for the stored value and free the pointer + /// using [`_cc_free`](ccommon_sys::_cc_free). + /// + /// # Safety + /// For this to be safe the pointer must have been allocated + /// using [`_cc_alloc`](ccommon_sys::_cc_alloc) (this includes + /// values allocated through `CCBox::new`). + pub unsafe fn from_raw(raw: *mut T) -> Self { + assert!(!raw.is_null()); + + CCBox(NonNull::new_unchecked(raw)) + } + + /// Consumes the box and returns the pointer stored inside. + /// + /// The pointer should be freed using [`_cc_free`](ccommon_sys::_cc_free) + /// or by recreating a `CCBox` from the pointer. + pub fn into_raw(b: Self) -> *mut T { + let ptr = b.0.as_ptr(); + // Don't want to free the pointer + mem::forget(b); + ptr + } +} + +impl Drop for CCBox { + fn drop(&mut self) { + use std::ptr; + + if mem::size_of_val(&**self) != 0 { + unsafe { + ptr::drop_in_place(self.0.as_ptr()); + + _cc_free( + self.0.as_ptr() as *mut std::ffi::c_void, + c_str!(module_path!()), + line!() as std::os::raw::c_int, + ); + } + } + } +} + +impl Deref for CCBox { + type Target = T; + + fn deref(&self) -> &T { + unsafe { self.0.as_ref() } + } +} + +impl DerefMut for CCBox { + fn deref_mut(&mut self) -> &mut T { + unsafe { self.0.as_mut() } + } +} + +impl Default for CCBox { + fn default() -> Self { + Self::new(T::default()) + } +} + +impl AsRef for CCBox { + fn as_ref(&self) -> &T { + &*self + } +} + +impl AsMut for CCBox { + fn as_mut(&mut self) -> &mut T { + &mut *self + } +} + +impl Clone for CCBox { + fn clone(&self) -> Self { + CCBox::new(self.as_ref().clone()) + } +} + +impl fmt::Debug for CCBox { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + (**self).fmt(fmt) + } +} + +impl fmt::Display for CCBox { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + (**self).fmt(fmt) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ccbox_size_tests() { + use std::mem::size_of; + + assert_eq!(size_of::>(), size_of::<*mut ()>()); + + assert_eq!(size_of::>>(), size_of::<*mut ()>()); + } + + #[test] + fn ptr_round_trip() { + unsafe { + let ptr1 = _cc_alloc(16, c_str!("test"), 0); + let boxed = CCBox::from_raw(ptr1); + let ptr2 = CCBox::into_raw(boxed); + + // Ensure the pointer is properly freed + let _boxed = CCBox::from_raw(ptr2); + + assert_eq!(ptr1, ptr2); + } + } + + #[test] + fn overaligned() { + #[repr(align(128))] + struct OverAligned(u8); + + assert!(CCBox::try_new(OverAligned(0)).is_err()); + } +} diff --git a/rust/ccommon-rs/src/error.rs b/rust/ccommon-rs/src/error.rs new file mode 100644 index 000000000..25dd0da4a --- /dev/null +++ b/rust/ccommon-rs/src/error.rs @@ -0,0 +1,108 @@ +// ccommon - a cache common library. +// Copyright (C) 2019 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::error::Error as StdError; +use std::fmt::{self, Debug, Display, Formatter}; + +/// Error codes that could be returned by ccommon functions. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +pub enum Error { + Generic = -1, + EAgain = -2, + ERetry = -3, + ENoMem = -4, + EEmpty = -5, + ERdHup = -6, + EInval = -7, + EOther = -8, +} + +impl From for Error { + fn from(val: std::os::raw::c_int) -> Self { + match val { + -1 => Error::Generic, + -2 => Error::EAgain, + -3 => Error::ERetry, + -4 => Error::ENoMem, + -5 => Error::EEmpty, + -6 => Error::ERdHup, + -7 => Error::EInval, + _ => Error::EOther, + } + } +} + +impl Display for Error { + fn fmt(&self, fmt: &mut Formatter) -> fmt::Result { + match self { + Error::Generic => write!(fmt, "Generic Error"), + Error::EAgain => write!(fmt, "EAGAIN"), + Error::ERetry => write!(fmt, "ERETRY"), + Error::ENoMem => write!(fmt, "ENOMEM"), + Error::EEmpty => write!(fmt, "EEMPTY"), + Error::ERdHup => write!(fmt, "ERDHUP"), + Error::EInval => write!(fmt, "EINVAL"), + Error::EOther => write!(fmt, "EOTHER"), + } + } +} + +impl StdError for Error {} + +/// An allocation handler failed to allocate a value. +/// +/// Converts to `Error::ENoMem`. +pub struct AllocationError(pub(crate) std::marker::PhantomData); + +impl AllocationError { + pub(crate) fn new() -> Self { + AllocationError(std::marker::PhantomData) + } +} + +impl Display for AllocationError { + fn fmt(&self, fmt: &mut Formatter) -> fmt::Result { + #[cfg(rustc_1_38)] + let res = write!( + fmt, + "Unable to allocate a value of type {} with size {}", + std::any::type_name::(), + std::mem::size_of::() + ); + + #[cfg(not(rustc_1_38))] + let res = write!( + fmt, + "Unable to allocate a value of size {}", + std::mem::size_of::() + ); + + res + } +} +impl Debug for AllocationError { + fn fmt(&self, fmt: &mut Formatter) -> fmt::Result { + fmt.debug_struct("AllocationError").finish() + } +} + +impl StdError for AllocationError {} + +impl From> for Error { + fn from(_: AllocationError) -> Self { + Self::ENoMem + } +} diff --git a/rust/ccommon-rs/src/lib.rs b/rust/ccommon-rs/src/lib.rs new file mode 100644 index 000000000..b32730b09 --- /dev/null +++ b/rust/ccommon-rs/src/lib.rs @@ -0,0 +1,32 @@ +// ccommon - a cache common library. +// Copyright (C) 2018 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Needed to allow derive macros within this crate +extern crate self as ccommon_rs; + +#[cfg(feature = "derive")] +pub use ccommon_derive::{Metrics, Options}; + +pub mod bstring; +pub mod buf; +pub mod log; +pub mod metric; +pub mod option; + +mod ccbox; +mod error; + +pub use self::ccbox::CCBox; +pub use self::error::{AllocationError, Error}; diff --git a/rust/ccommon-rs/src/log/debug.rs b/rust/ccommon-rs/src/log/debug.rs new file mode 100644 index 000000000..1f9d31311 --- /dev/null +++ b/rust/ccommon-rs/src/log/debug.rs @@ -0,0 +1,121 @@ +// ccommon - a cache common library. +// Copyright (C) 2019 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::atomic::{AtomicBool, Ordering}; + +use ccommon_sys::{debug_options_st, debug_setup, debug_teardown}; + +use crate::Error; + +static DEBUG_INIT: AtomicBool = AtomicBool::new(false); + +/// RAII Wrapper around the ccommon debug logger. +/// +/// # Safety +/// It is unsafe to call `debug_setup` or `debug_teardown` +/// while an instance of DebugLogger exists. +/// +/// If a debug logger has already been created using `debug_setup` +/// then use [`from_existing`](#fn.from_existing) to create a +/// `DebugLogger` instance wrapping it. +pub struct DebugLogger(()); + +impl DebugLogger { + /// Create a new debug logger. + /// + /// If another debug logger is currently active then + /// this will return `Error::EInval`, otherwise the + /// error status reflects the return code of `debug_setup`. + pub fn new<'a, I>(opts: I) -> Result + where + I: Into>, + { + if DEBUG_INIT + .compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed) + .is_err() + { + return Err(Error::EInval); + } + + let opts = opts + .into() + .map(|x| x as *const _ as *mut _) + .unwrap_or(std::ptr::null_mut()); + + let status = unsafe { debug_setup(opts) }; + + if status < 0 { + Err(status.into()) + } else { + Ok(Self(())) + } + } + + /// Overwrite the current debug logger with a new one. + /// + /// # Safety + /// It is unsafe to call this method while logging is + /// happening concurrently. + pub unsafe fn overwrite<'a, I>(&self, opts: I) -> Result + where + I: Into>, + { + let opts = opts + .into() + .map(|x| x as *const _ as *mut _) + .unwrap_or(std::ptr::null_mut()); + + let status = debug_setup(opts); + + if status < 0 { + Err(status.into()) + } else { + Ok(Self(())) + } + } + + /// Create a debug logger from one that has already been + /// set up through `debug_setup` or via + /// [`release`](#fn.release). + /// + /// # Safety + /// It is unsafe to call this method while an existing + /// instance of `DebugLogger` is live. + pub unsafe fn from_existing() -> Self { + DEBUG_INIT.store(true, Ordering::Relaxed); + + Self(()) + } + + /// Release the current global debug logger from the lifetime + /// of this particular `DebugLogger`. + /// + /// # Safety + /// It is unsafe to create a new debug logger before + /// `debug_teardown` is called. + pub unsafe fn release(self) { + std::mem::forget(self); + DEBUG_INIT.store(false, Ordering::Relaxed); + } +} + +impl Drop for DebugLogger { + fn drop(&mut self) { + unsafe { + debug_teardown(); + DEBUG_INIT.store(false, Ordering::Relaxed); + } + } +} diff --git a/rust/ccommon-rs/src/log/mod.rs b/rust/ccommon-rs/src/log/mod.rs new file mode 100644 index 000000000..56f4bb2ad --- /dev/null +++ b/rust/ccommon-rs/src/log/mod.rs @@ -0,0 +1,52 @@ +// ccommon - a cache common library. +// Copyright (C) 2019 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Bindings related to logging. +//! +//! There are currently two parts to this module. +//! 1. `DebugLogger`: A RAII wrapper around ccommon's debug module. It is +//! needed to set up the debug logging infrastructure that is used +//! as a backend by +//! 2. The `log` shim. This is a logger that forwards the logging macros +//! provided by the [`log`](https://docs.rs/log) crate into the debug +//! logger initialized by part 1. +//! 3. A panic handler which will utilize ccommon's logging infrastructure +//! in addition to printing out normal log messages. +//! +//! To use this module initialize the logging shim by calling [`init()`][0] +//! (alternatively, use [`rust_log_setup`][1] and [`rust_log_teardown`][2] from +//! C code). In addition, create an instance of [`DebugLogger`][3] (or +//! initialize it from C code). +//! +//! Now, just use the log macros as normal and all logs will be run through +//! ccommon's logging infrastructure. +//! +//! An optional extra thing that can be down is to use +//! [`set_panic_handler`][4] to set a panic hook which +//! will utilize common's logging infrastructure in addition to printing +//! to stderr. +//! +//! [0]: ccommon_rs::log::init +//! [1]: ccommon_rs::log::rust_log_setup +//! [2]: ccommon_rs::log::rust_log_teardown +//! [3]: ccommon_rs::log::DebugLogger +//! [4]: ccommon_rs::log::set_panic_handler + +// Backend for the standard logging shim +mod debug; +mod shim; + +pub use self::debug::DebugLogger; +pub use self::shim::{init, rust_log_setup, rust_log_teardown, set_panic_handler}; diff --git a/rust/ccommon-rs/src/log/shim.rs b/rust/ccommon-rs/src/log/shim.rs new file mode 100644 index 000000000..ad284b221 --- /dev/null +++ b/rust/ccommon-rs/src/log/shim.rs @@ -0,0 +1,227 @@ +// ccommon - a cache common library. +// Copyright (C) 2019 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io::Write; +use std::os::raw::{c_char, c_int}; +use std::panic::{self, PanicInfo}; +use std::sync::atomic::{AtomicPtr, Ordering}; +use std::thread; + +use ccommon_sys::{_log, debug_log_flush, debug_logger, LOG_MAX_LEN}; +use log::{Level, Log, Metadata, Record, SetLoggerError}; + +fn ccommon_dlog() -> &'static AtomicPtr { + unsafe { &*(&ccommon_sys::dlog as *const _ as *const AtomicPtr) } +} + +const FILENAME_MAX_LEN: usize = 1024; + +/// A log implementation which wraps the debug logger from ccommon +struct CCLog; + +unsafe impl Send for CCLog {} + +unsafe impl Sync for CCLog {} + +fn level_to_int(level: Level) -> c_int { + use ccommon_sys::*; + + let level = match level { + Level::Trace => LOG_VERB, + Level::Debug => LOG_DEBUG, + Level::Info => LOG_INFO, + Level::Warn => LOG_WARN, + Level::Error => LOG_ERROR, + }; + + level as c_int +} + +impl CCLog { + fn enabled_ptr(metadata: &Metadata, ptr: *const debug_logger) -> bool { + if ptr.is_null() { + return false; + } + + unsafe { + // Note: Assumes the level doesn't change concurrently + (*ptr).level >= level_to_int(metadata.level()) + } + } +} + +impl Log for CCLog { + #[inline] + fn enabled(&self, metadata: &Metadata) -> bool { + Self::enabled_ptr(metadata, ccommon_dlog().load(Ordering::Relaxed)) + } + + fn log(&self, record: &Record) { + let ptr = ccommon_dlog().load(Ordering::Relaxed); + + if !Self::enabled_ptr(record.metadata(), ptr) { + return; + } + + // +1 bytes are trailing nul bytes + let mut buffer = [0; LOG_MAX_LEN as usize + 1]; + let mut filename = [0; FILENAME_MAX_LEN + 1]; + + let _ = write!(&mut buffer[0..LOG_MAX_LEN as usize], "{}", record.args()); + + // Make a best-effort attempt to provide a meainingful + // location that is consistent with how ccommon does + // log messages. + let filestr = record + .file() + .or_else(|| record.module_path()) + .unwrap_or_else(|| record.target()); + let _ = write!(&mut filename[0..FILENAME_MAX_LEN], "{}", filestr); + + unsafe { + _log( + ptr, + (&filename).as_ptr() as *const c_char, + record.line().map(|x| x as c_int).unwrap_or(-1), + level_to_int(record.level()), + "%s\0".as_ptr() as *const c_char, + (&buffer).as_ptr() as *const c_char, + ) + } + } + + fn flush(&self) { + unsafe { + // The argument here is unused and ccommon uses it for + // compatibility with other modules such as timer_wheel + debug_log_flush(std::ptr::null_mut()); + } + } +} + +// The logger has no state so a global zero-sized instance +// works just fine. +static LOGGER: CCLog = CCLog; + +/// Initialize the logger. It will automatically use the +/// ccommon debug logger once that is set up. +pub fn init() -> Result<(), SetLoggerError> { + use log::LevelFilter; + + // TODO: dynamically set this based on dlog? + log::set_max_level(LevelFilter::Trace); + + log::set_logger(&LOGGER) +} + +/// Set the global panic handler to one which logs the +/// panic at `LOG_CRIT` level before calling the original +/// panic hook. +pub fn set_panic_handler() { + use ccommon_sys::LOG_CRIT; + + use std::env; + use std::io::Cursor; + + // After logging the message we want to call whatever existing + // panic hook was in place. In most cases this should be the + // default hook. + let old_hook = panic::take_hook(); + + panic::set_hook(Box::new(move |info: &PanicInfo| { + let mut buffer = [0u8; LOG_MAX_LEN as usize + 1]; + let ptr = ccommon_dlog().load(Ordering::Relaxed); + + let msg = if let Some(s) = info.payload().downcast_ref::<&str>() { + *s + } else if let Some(s) = info.payload().downcast_ref::() { + &s[..] + } else { + "Box" + }; + + // Cursor so that we can get the number of bytes written afterwards + let mut cursor = Cursor::new(&mut buffer[0..LOG_MAX_LEN as usize]); + + // If the thread has a name then use it. (This mirrors std's standard + // debug handler) + let current_thread = thread::current(); + let name = current_thread.name().unwrap_or(""); + + if let Some(ref location) = info.location() { + let _ = write!( + &mut cursor, + "thread '{}' panicked at '{}', {}", + name, msg, *location + ); + } else { + let _ = write!(&mut cursor, "thread '{}' panicked at '{}'", name, msg); + }; + + if !ptr.is_null() { + // Ensure that the panic message is logged + // TODO(sean): Do we want to log the backtrace as well? + if let Some(location) = info.location() { + unsafe { + _log( + ptr, + location.file().as_ptr() as *const c_char, + location.line() as c_int, + LOG_CRIT as c_int, + (&buffer).as_ptr() as *const c_char, + ); + } + } else { + unsafe { + _log( + ptr, + file!().as_ptr() as *const c_char, + line!() as c_int, + LOG_CRIT as c_int, + (&buffer).as_ptr() as *const c_char, + ) + } + } + } + + let old_val = env::var_os("RUST_BACKTRACE"); + // If possible, have the stdlib display a backtrace + env::set_var("RUST_BACKTRACE", "full"); + old_hook(info); + + if let Some(var) = old_val { + env::set_var("RUST_BACKTRACE", var); + } + })) +} + +//================================================ +// C Interface Methods +//================================================ + +#[no_mangle] +pub extern "C" fn rust_log_setup() -> ccommon_sys::rstatus_i { + use ccommon_sys::*; + + match init() { + Ok(()) => CC_OK as rstatus_i, + Err(_) => CC_ERROR, + } +} + +#[no_mangle] +pub extern "C" fn rust_log_teardown() { + // Provided for symmetry with the rest of ccommon, currently a no-op +} diff --git a/rust/ccommon-rs/src/metric/counter.rs b/rust/ccommon-rs/src/metric/counter.rs new file mode 100644 index 000000000..8807a1cab --- /dev/null +++ b/rust/ccommon-rs/src/metric/counter.rs @@ -0,0 +1,113 @@ +// ccommon - a cache common library. +// Copyright (C) 2019 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::ffi::CStr; +use std::fmt; +use std::ops::{AddAssign, SubAssign}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use ccommon_sys::{metric, metric_anon_union, METRIC_COUNTER}; + +use super::private::Sealed; +use super::SingleMetric; + +/// An atomic counter metric. +/// +/// Exposes `incr` and `decr` operations that can also be +/// used via operators `+=` and `-=`. +#[derive(Copy, Clone)] +#[repr(transparent)] +pub struct Counter(metric); + +unsafe impl Send for Counter {} +unsafe impl Sync for Counter {} + +impl Counter { + fn as_ref(&self) -> &AtomicU64 { + unsafe { &*self.0.data.as_ptr::() } + } + + /// Increment the counter by `n` atomically. + pub fn incr_n(&self, n: u64) { + self.as_ref().fetch_add(n, Ordering::Relaxed); + } + + /// Increment the counter by `1` atomically. + pub fn incr(&self) { + self.incr_n(1) + } + + /// Decrement the counter by `n` atomically. + pub fn decr_n(&self, n: u64) { + self.as_ref().fetch_sub(n, Ordering::Relaxed); + } + + /// Decrement the counter by `1` atomically. + pub fn decr(&self) { + self.decr_n(1) + } + + /// Atomically store a value in the counter. + pub fn update(&self, val: u64) { + self.as_ref().store(val, Ordering::Relaxed) + } + + /// Atomically get the value out of the counter. + pub fn value(&self) -> u64 { + self.as_ref().load(Ordering::Relaxed) + } +} + +impl Sealed for Counter {} + +unsafe impl SingleMetric for Counter { + fn new(name: &CStr, desc: &CStr) -> Self { + Self(metric { + name: name.as_ptr() as *mut i8, + desc: desc.as_ptr() as *mut i8, + type_: METRIC_COUNTER, + data: metric_anon_union::counter(0), + }) + } + + fn name(&self) -> &'static CStr { + unsafe { CStr::from_ptr(self.0.name) } + } + fn desc(&self) -> &'static CStr { + unsafe { CStr::from_ptr(self.0.desc) } + } +} + +impl fmt::Debug for Counter { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + fmt.debug_struct("Counter") + .field("name", unsafe { &CStr::from_ptr(self.0.name as *const i8) }) + .field("desc", unsafe { &CStr::from_ptr(self.0.desc as *const i8) }) + .field("counter", &self.value()) + .finish() + } +} + +impl AddAssign for Counter { + fn add_assign(&mut self, val: u64) { + self.incr_n(val); + } +} + +impl SubAssign for Counter { + fn sub_assign(&mut self, val: u64) { + self.decr_n(val); + } +} diff --git a/rust/ccommon-rs/src/metric/fpn.rs b/rust/ccommon-rs/src/metric/fpn.rs new file mode 100644 index 000000000..3e1b02d4d --- /dev/null +++ b/rust/ccommon-rs/src/metric/fpn.rs @@ -0,0 +1,73 @@ +// ccommon - a cache common library. +// Copyright (C) 2019 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::ffi::CStr; +use std::fmt; +use std::sync::atomic::{AtomicU64, Ordering}; + +use ccommon_sys::{metric, metric_anon_union, METRIC_FPN}; + +use super::private::Sealed; +use super::SingleMetric; + +/// A `f64` metric that can be updated atomically. +#[derive(Copy, Clone)] +#[repr(transparent)] +pub struct Fpn(metric); + +impl Fpn { + /// Get the value atomically. + pub fn value(&self) -> f64 { + f64::from_bits(unsafe { (*self.0.data.as_ptr::()).load(Ordering::Relaxed) }) + } + + /// Update the value atomically. + pub fn update(&self, val: f64) { + unsafe { (*self.0.data.as_ptr::()).store(val.to_bits(), Ordering::Relaxed) } + } +} + +impl Sealed for Fpn {} + +unsafe impl Send for Fpn {} +unsafe impl Sync for Fpn {} + +unsafe impl SingleMetric for Fpn { + fn new(name: &CStr, desc: &CStr) -> Self { + Self(metric { + name: name.as_ptr() as *mut i8, + desc: desc.as_ptr() as *mut i8, + type_: METRIC_FPN, + data: metric_anon_union::gauge(0), + }) + } + + fn name(&self) -> &'static CStr { + unsafe { CStr::from_ptr(self.0.name) } + } + fn desc(&self) -> &'static CStr { + unsafe { CStr::from_ptr(self.0.desc) } + } +} + +impl fmt::Debug for Fpn { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + fmt.debug_struct("Fpn") + .field("name", unsafe { &CStr::from_ptr(self.0.name as *const i8) }) + .field("desc", unsafe { &CStr::from_ptr(self.0.desc as *const i8) }) + .field("fpn", &self.value()) + .finish() + } +} diff --git a/rust/ccommon-rs/src/metric/gauge.rs b/rust/ccommon-rs/src/metric/gauge.rs new file mode 100644 index 000000000..334dabe8f --- /dev/null +++ b/rust/ccommon-rs/src/metric/gauge.rs @@ -0,0 +1,113 @@ +// ccommon - a cache common library. +// Copyright (C) 2019 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::ffi::CStr; +use std::fmt; +use std::ops::{AddAssign, SubAssign}; +use std::sync::atomic::{AtomicI64, Ordering}; + +use ccommon_sys::{metric, metric_anon_union, METRIC_GAUGE}; + +use super::private::Sealed; +use super::SingleMetric; + +/// A gauge metric that can be updated atomically. +/// +/// Exposes `incr`, `decr`, and `update`. `incr` and `decr` +/// can be used through operators `+=` and `-=`. +#[derive(Copy, Clone)] +#[repr(transparent)] +pub struct Gauge(metric); + +impl Gauge { + fn as_ref(&self) -> &AtomicI64 { + unsafe { &*self.0.data.as_ptr::() } + } + + /// Increment the gauge atomically by `n`. + pub fn incr_n(&self, n: i64) { + self.as_ref().fetch_add(n, Ordering::Relaxed); + } + + /// Increment the gauge atomically by `1`. + pub fn incr(&self) { + self.incr_n(1) + } + + /// Decrement the gauge atomically by `n`. + pub fn decr_n(&self, n: i64) { + self.as_ref().fetch_sub(n, Ordering::Relaxed); + } + + /// Decrement the gauge atomically by `1`. + pub fn decr(&self) { + self.decr_n(1) + } + + /// Set the value of the gauge atomically. + pub fn update(&self, val: i64) { + self.as_ref().store(val, Ordering::Relaxed) + } + + /// Get the value of the gauge atomically. + pub fn value(&self) -> i64 { + self.as_ref().load(Ordering::Relaxed) + } +} + +impl Sealed for Gauge {} + +unsafe impl Send for Gauge {} +unsafe impl Sync for Gauge {} + +unsafe impl SingleMetric for Gauge { + fn new(name: &CStr, desc: &CStr) -> Self { + Self(metric { + name: name.as_ptr() as *mut i8, + desc: desc.as_ptr() as *mut i8, + type_: METRIC_GAUGE, + data: metric_anon_union::gauge(0), + }) + } + + fn name(&self) -> &'static CStr { + unsafe { CStr::from_ptr(self.0.name) } + } + fn desc(&self) -> &'static CStr { + unsafe { CStr::from_ptr(self.0.desc) } + } +} + +impl fmt::Debug for Gauge { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + fmt.debug_struct("Counter") + .field("name", unsafe { &CStr::from_ptr(self.0.name as *const i8) }) + .field("desc", unsafe { &CStr::from_ptr(self.0.desc as *const i8) }) + .field("counter", &self.value()) + .finish() + } +} + +impl AddAssign for Gauge { + fn add_assign(&mut self, val: i64) { + self.incr_n(val); + } +} + +impl SubAssign for Gauge { + fn sub_assign(&mut self, val: i64) { + self.decr_n(val); + } +} diff --git a/rust/ccommon-rs/src/metric/mod.rs b/rust/ccommon-rs/src/metric/mod.rs new file mode 100644 index 000000000..b07070fc8 --- /dev/null +++ b/rust/ccommon-rs/src/metric/mod.rs @@ -0,0 +1,364 @@ +// ccommon - a cache common library. +// Copyright (C) 2019 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Types and methods for dealing with ccommon metrics. +//! +//! To use this any collection of metrics (i.e. a struct containing metrics) +//! should implement the `Metrics` trait through the derive macro. This +//! will (usually) assert that the struct you are using is equivalent in +//! memory to an array of `metric` structs. + +use std::ffi::CStr; + +use ccommon_sys::{metric, metric_describe_all, metric_reset}; + +// Sealed trait to prevent SingleMetric from ever being implemented +// from outside of this crate. +mod private { + pub trait Sealed {} +} + +mod counter; +mod fpn; +mod gauge; + +pub use self::counter::Counter; +pub use self::fpn::Fpn; +pub use self::gauge::Gauge; + +/// A single metric value. +/// +/// This trait is sealed and cannot be implemented outside +/// of ccommon_rs. +pub unsafe trait SingleMetric: self::private::Sealed { + /// Create a metric with the given description and name. + /// + /// Normally this should only be called by the + /// derive macro for `Metrics`. + fn new(name: &'static CStr, desc: &'static CStr) -> Self; + + /// The metric's name + fn name(&self) -> &'static CStr; + /// The metric's description + fn desc(&self) -> &'static CStr; +} + +/// A type that can be safely viewed as a contiguous array +/// of [`metric`s][0]. +/// +/// It should usually only be implemented through `#[derive(Metrics)]`. +/// However, it must be implemented manually for C types that have +/// been bound through bindgen. +/// +/// [0]: ../../cc_binding/struct.metric.html +pub unsafe trait Metrics: Sized { + fn new() -> Self; +} + +pub trait MetricExt: Metrics { + /// The number of metrics in this object when + /// it is interpreted as an array. + /// + /// # Panics + /// Panics if the size of this type is not a multiple + /// of the size of `metric`. + fn num_metrics() -> usize { + use std::mem::size_of; + + // If this assert fails then there was no way that + // options upholds it's safety requirements so it's + // better to fail here. + assert!(size_of::() % size_of::() == 0); + + // If this assert fails then we'll pass an invalid + // size to several ccommon methods. + assert!(size_of::() / size_of::() < std::u32::MAX as usize); + + size_of::() / size_of::() + } + + /// Get `self` as a const pointer to an array of `metric`s. + /// + /// # Panics + /// Panics if the size of this type is not a multiple + /// of the size of `metric`. + fn as_ptr(&self) -> *const metric { + assert!(std::mem::size_of::() % std::mem::size_of::() == 0); + + self as *const _ as *const metric + } + + /// Get `self` as a mutable pointer to an array of `metric`s. + /// + /// # Panics + /// Panics if the size of this type is not a multiple + /// of the size of `metric`. + fn as_mut_ptr(&mut self) -> *mut metric { + assert!(std::mem::size_of::() % std::mem::size_of::() == 0); + + self as *mut _ as *mut metric + } + + /// Print a description of all metricss in the current object + /// given using the name and description. + /// + /// Internally this calls out to `metric_describe_all`. + fn describe_all(&self) { + unsafe { + metric_describe_all( + // Note: ccommon uses a mutable pointer but it + // should really be a const pointer. + self.as_ptr() as *mut _, + Self::num_metrics() as u32, + ) + } + } + + /// Reset all metrics to their default values. + /// + /// This means that all 3 types of metrics are reset to 0. + /// + /// Internally this calls out to `metric_reset`. + fn reset_all(&mut self) { + unsafe { metric_reset(self.as_mut_ptr(), Self::num_metrics() as u32) } + } +} + +impl MetricExt for T {} + +/// Impls of Metrics for cc_bindings types +mod impls { + use super::Metrics; + use ccommon_sys::*; + + macro_rules! c_str { + ($s:expr) => { + concat!($s, "\0").as_bytes().as_ptr() as *const i8 as *mut _ + }; + } + + macro_rules! initialize_metric_value { + (METRIC_GAUGE) => { + metric_anon_union::gauge(0) + }; + (METRIC_COUNTER) => { + metric_anon_union::counter(0) + }; + (METRIC_FPN) => { + metric_anon_union::fpn(0.0) + }; + } + + macro_rules! impl_metrics { + { + $( + impl Metrics for $metrics_ty:ty { + $( + ACTION( $field:ident, $type:ident, $desc:expr ) + )* + } + )* + } => { + $( + unsafe impl Metrics for $metrics_ty { + fn new() -> Self { + Self { + $( + $field: metric { + name: c_str!(stringify!($field)), + type_: $type, + desc: c_str!($desc), + data: initialize_metric_value!($type) + }, + )* + } + } + } + )* + } + } + + impl_metrics! { + impl Metrics for buf_metrics_st { + ACTION( buf_curr, METRIC_GAUGE, "# buf allocated" ) + ACTION( buf_active, METRIC_GAUGE, "# buf in use/borrowed" ) + ACTION( buf_create, METRIC_COUNTER, "# buf creates" ) + ACTION( buf_create_ex, METRIC_COUNTER, "# buf create exceptions" ) + ACTION( buf_destroy, METRIC_COUNTER, "# buf destroys" ) + ACTION( buf_borrow, METRIC_COUNTER, "# buf borrows" ) + ACTION( buf_borrow_ex, METRIC_COUNTER, "# buf borrow exceptions" ) + ACTION( buf_return, METRIC_COUNTER, "# buf returns" ) + ACTION( buf_memory, METRIC_GAUGE, "memory alloc'd to buf including header" ) + } + + impl Metrics for dbuf_metrics_st { + ACTION( dbuf_double, METRIC_COUNTER, "# double completed" ) + ACTION( dbuf_double_ex, METRIC_COUNTER, "# double failed" ) + ACTION( dbuf_shrink, METRIC_COUNTER, "# shrink completed" ) + ACTION( dbuf_shrink_ex, METRIC_COUNTER, "# shrink failed" ) + ACTION( dbuf_fit, METRIC_COUNTER, "# fit completed" ) + ACTION( dbuf_fit_ex, METRIC_COUNTER, "# fit failed" ) + } + + impl Metrics for pipe_metrics_st { + ACTION( pipe_conn_create, METRIC_COUNTER, "# pipe connections created" ) + ACTION( pipe_conn_create_ex, METRIC_COUNTER, "# pipe conn create exceptions" ) + ACTION( pipe_conn_destroy, METRIC_COUNTER, "# pipe connections destroyed" ) + ACTION( pipe_conn_curr , METRIC_GAUGE, "# pipe conn allocated" ) + ACTION( pipe_conn_borrow, METRIC_COUNTER, "# pipe connections borrowed" ) + ACTION( pipe_conn_borrow_ex, METRIC_COUNTER, "# pipe conn borrow exceptions" ) + ACTION( pipe_conn_return, METRIC_COUNTER, "# pipe connections returned" ) + ACTION( pipe_conn_active, METRIC_GAUGE, "# pipe conn being borrowed" ) + ACTION( pipe_open, METRIC_COUNTER, "# pipe connects made" ) + ACTION( pipe_open_ex, METRIC_COUNTER, "# pipe connect exceptions" ) + ACTION( pipe_close, METRIC_COUNTER, "# pipe connection closed" ) + ACTION( pipe_recv, METRIC_COUNTER, "# recv attempted" ) + ACTION( pipe_recv_ex, METRIC_COUNTER, "# recv exceptions" ) + ACTION( pipe_recv_byte, METRIC_COUNTER, "# bytes received" ) + ACTION( pipe_send, METRIC_COUNTER, "# send attempted" ) + ACTION( pipe_send_ex, METRIC_COUNTER, "# send exceptions" ) + ACTION( pipe_send_byte, METRIC_COUNTER, "# bytes sent" ) + ACTION( pipe_flag_ex, METRIC_COUNTER, "# pipe flag exceptions" ) + } + + impl Metrics for tcp_metrics_st { + ACTION( tcp_conn_create, METRIC_COUNTER, "# tcp connections created" ) + ACTION( tcp_conn_create_ex, METRIC_COUNTER, "# tcp conn create exceptions" ) + ACTION( tcp_conn_destroy, METRIC_COUNTER, "# tcp connections destroyed" ) + ACTION( tcp_conn_curr, METRIC_GAUGE, "# tcp conn allocated" ) + ACTION( tcp_conn_borrow, METRIC_COUNTER, "# tcp connections borrowed" ) + ACTION( tcp_conn_borrow_ex, METRIC_COUNTER, "# tcp conn borrow exceptions" ) + ACTION( tcp_conn_return, METRIC_COUNTER, "# tcp connections returned" ) + ACTION( tcp_conn_active, METRIC_GAUGE, "# tcp conn being borrowed" ) + ACTION( tcp_accept, METRIC_COUNTER, "# tcp connection accepts" ) + ACTION( tcp_accept_ex, METRIC_COUNTER, "# tcp accept exceptions" ) + ACTION( tcp_reject, METRIC_COUNTER, "# tcp connection rejects" ) + ACTION( tcp_reject_ex, METRIC_COUNTER, "# tcp reject exceptions" ) + ACTION( tcp_connect, METRIC_COUNTER, "# tcp connects made" ) + ACTION( tcp_connect_ex, METRIC_COUNTER, "# tcp connect exceptions " ) + ACTION( tcp_close, METRIC_COUNTER, "# tcp connection closed" ) + ACTION( tcp_recv, METRIC_COUNTER, "# recv attempted" ) + ACTION( tcp_recv_ex, METRIC_COUNTER, "# recv exceptions" ) + ACTION( tcp_recv_byte, METRIC_COUNTER, "# bytes received" ) + ACTION( tcp_send, METRIC_COUNTER, "# send attempted" ) + ACTION( tcp_send_ex, METRIC_COUNTER, "# send exceptions" ) + ACTION( tcp_send_byte, METRIC_COUNTER, "# bytes sent" ) + } + + impl Metrics for sockio_metrics_st { + ACTION( buf_sock_create, METRIC_COUNTER, "# buf sock created" ) + ACTION( buf_sock_create_ex, METRIC_COUNTER, "# buf sock create exceptions" ) + ACTION( buf_sock_destroy, METRIC_COUNTER, "# buf sock destroyed" ) + ACTION( buf_sock_curr, METRIC_GAUGE, "# buf sock allocated" ) + ACTION( buf_sock_borrow, METRIC_COUNTER, "# buf sock borrowed" ) + ACTION( buf_sock_borrow_ex, METRIC_COUNTER, "# buf sock borrow exceptions" ) + ACTION( buf_sock_return, METRIC_COUNTER, "# buf sock returned" ) + ACTION( buf_sock_active, METRIC_GAUGE, "# buf sock being borrowed" ) + } + + impl Metrics for timing_wheel_metrics_st { + ACTION( timeout_event_curr, METRIC_GAUGE, "# timeout events allocated" ) + ACTION( timeout_event_active, METRIC_GAUGE, "# timeout events in use" ) + ACTION( timeout_event_borrow, METRIC_COUNTER, "# timeout events borrowed" ) + ACTION( timeout_event_borrow_ex,METRIC_COUNTER, "# tevents borrow errors" ) + ACTION( timeout_event_return, METRIC_COUNTER, "# timeout events returned" ) + ACTION( timing_wheel_insert, METRIC_COUNTER, "# tevent insertions" ) + ACTION( timing_wheel_remove, METRIC_COUNTER, "# tevent removal" ) + ACTION( timing_wheel_event, METRIC_GAUGE, "# tevents in timing wheels" ) + ACTION( timing_wheel_process, METRIC_COUNTER, "# tevents processed" ) + ACTION( timing_wheel_tick, METRIC_COUNTER, "# ticks processed" ) + ACTION( timing_wheel_exec, METRIC_COUNTER, "# timing wheel executions " ) + } + + impl Metrics for event_metrics_st { + ACTION( event_total, METRIC_COUNTER, "# events returned" ) + ACTION( event_loop, METRIC_COUNTER, "# event loop returns" ) + ACTION( event_read, METRIC_COUNTER, "# reads registered" ) + ACTION( event_write, METRIC_COUNTER, "# writes registered" ) + } + + impl Metrics for log_metrics_st { + ACTION( log_create, METRIC_COUNTER, "# loggers created" ) + ACTION( log_create_ex, METRIC_COUNTER, "# log create errors" ) + ACTION( log_destroy, METRIC_COUNTER, "# loggers destroyed" ) + ACTION( log_curr, METRIC_GAUGE, "current # loggers" ) + ACTION( log_open, METRIC_COUNTER, "# files opened by loggers" ) + ACTION( log_open_ex, METRIC_COUNTER, "# logger open file errors" ) + ACTION( log_write, METRIC_COUNTER, "# log messages written" ) + ACTION( log_write_byte, METRIC_COUNTER, "# bytes written by log" ) + ACTION( log_write_ex, METRIC_COUNTER, "# log write errors" ) + ACTION( log_skip, METRIC_COUNTER, "# messages not completely logged" ) + ACTION( log_skip_byte, METRIC_COUNTER, "# bytes unable to be logged" ) + ACTION( log_flush, METRIC_COUNTER, "# log flushes to disk" ) + ACTION( log_flush_ex, METRIC_COUNTER, "# errors flushing to disk" ) + } + + impl Metrics for rbuf_metrics_st { + ACTION( rbuf_create, METRIC_COUNTER, "# rbuf created" ) + ACTION( rbuf_create_ex, METRIC_COUNTER, "# rbuf create errors" ) + ACTION( rbuf_destroy, METRIC_COUNTER, "# rbuf destroyed" ) + ACTION( rbuf_curr, METRIC_GAUGE, "# rbuf allocated" ) + ACTION( rbuf_byte, METRIC_GAUGE, "# rbuf bytes allocated" ) + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + macro_rules! c_str { + ($s:expr) => { + unsafe { CStr::from_bytes_with_nul_unchecked(concat!($s, "\0").as_bytes()) } + }; + } + + #[test] + fn gauge_basic_use() { + let gauge = Gauge::new(c_str!("test"), c_str!("test desc")); + + assert_eq!(gauge.value(), 0); + gauge.incr(); + assert_eq!(gauge.value(), 1); + gauge.decr_n(10); + assert_eq!(gauge.value(), -9); + } + + #[test] + fn counter_basic_use() { + let ctr = Counter::new(c_str!("test"), c_str!("test desc")); + + assert_eq!(ctr.value(), 0); + ctr.incr(); + assert_eq!(ctr.value(), 1); + ctr.incr_n(400); + assert_eq!(ctr.value(), 401); + } + + #[test] + fn fpn_basic_use() { + let fpn = Fpn::new(c_str!("test"), c_str!("test desc")); + + assert_eq!(fpn.value(), 0.0); + fpn.update(500.0); + assert_eq!(fpn.value(), 500.0); + } + + #[test] + fn size_sanity_test() { + // Protect against a bad bindgen run + assert!(std::mem::size_of::() != 0); + } +} diff --git a/rust/ccommon-rs/src/option/boolean.rs b/rust/ccommon-rs/src/option/boolean.rs new file mode 100644 index 000000000..713acf5ce --- /dev/null +++ b/rust/ccommon-rs/src/option/boolean.rs @@ -0,0 +1,81 @@ +// ccommon - a cache common library. +// Copyright (C) 2019 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::ffi::CStr; +use std::fmt; + +use ccommon_sys::{option, option_val_u, OPTION_TYPE_BOOL}; + +use super::{Sealed, SingleOption}; + +/// A boolean option. +#[derive(Copy, Clone)] +#[repr(transparent)] +pub struct Bool(option); + +impl Sealed for Bool {} + +unsafe impl Send for Bool {} + +unsafe impl SingleOption for Bool { + type Value = bool; + + fn new(default: bool, name: &'static CStr, desc: &'static CStr) -> Self { + Self(option { + name: name.as_ptr() as *mut _, + set: false, + type_: OPTION_TYPE_BOOL, + default_val: option_val_u { vbool: default }, + val: option_val_u { vbool: default }, + description: desc.as_ptr() as *mut _, + }) + } + fn defaulted(name: &'static CStr, desc: &'static CStr) -> Self { + Self::new(Default::default(), name, desc) + } + + fn name(&self) -> &'static CStr { + unsafe { CStr::from_ptr(self.0.name) } + } + fn desc(&self) -> &'static CStr { + unsafe { CStr::from_ptr(self.0.description) } + } + fn value(&self) -> Self::Value { + unsafe { self.0.val.vbool } + } + fn default(&self) -> Self::Value { + unsafe { self.0.default_val.vbool } + } + fn is_set(&self) -> bool { + self.0.set + } + + fn set_value(&mut self, val: Self::Value) { + self.0.set = true; + self.0.val = option_val_u { vbool: val } + } +} + +impl fmt::Debug for Bool { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + fmt.debug_struct("Bool") + .field("name", &self.name()) + .field("desc", &self.desc()) + .field("value", &self.value()) + .field("default", &self.default()) + .field("is_set", &self.is_set()) + .finish() + } +} diff --git a/rust/ccommon-rs/src/option/fpn.rs b/rust/ccommon-rs/src/option/fpn.rs new file mode 100644 index 000000000..4f392a023 --- /dev/null +++ b/rust/ccommon-rs/src/option/fpn.rs @@ -0,0 +1,81 @@ +// ccommon - a cache common library. +// Copyright (C) 2019 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::ffi::CStr; +use std::fmt; + +use ccommon_sys::{option, option_val_u, OPTION_TYPE_FPN}; + +use super::{Sealed, SingleOption}; + +/// A floating-point option. +#[derive(Copy, Clone)] +#[repr(transparent)] +pub struct Float(option); + +impl Sealed for Float {} + +unsafe impl Send for Float {} + +unsafe impl SingleOption for Float { + type Value = f64; + + fn new(default: Self::Value, name: &'static CStr, desc: &'static CStr) -> Self { + Self(option { + name: name.as_ptr() as *mut _, + set: false, + type_: OPTION_TYPE_FPN, + default_val: option_val_u { vfpn: default }, + val: option_val_u { vfpn: default }, + description: desc.as_ptr() as *mut _, + }) + } + fn defaulted(name: &'static CStr, desc: &'static CStr) -> Self { + Self::new(Default::default(), name, desc) + } + + fn name(&self) -> &'static CStr { + unsafe { CStr::from_ptr(self.0.name) } + } + fn desc(&self) -> &'static CStr { + unsafe { CStr::from_ptr(self.0.description) } + } + fn value(&self) -> Self::Value { + unsafe { self.0.val.vfpn } + } + fn default(&self) -> Self::Value { + unsafe { self.0.default_val.vfpn } + } + fn is_set(&self) -> bool { + self.0.set + } + + fn set_value(&mut self, val: Self::Value) { + self.0.set = true; + self.0.val = option_val_u { vfpn: val } + } +} + +impl fmt::Debug for Float { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + fmt.debug_struct("Float") + .field("name", &self.name()) + .field("desc", &self.desc()) + .field("value", &self.value()) + .field("default", &self.default()) + .field("is_set", &self.is_set()) + .finish() + } +} diff --git a/rust/ccommon-rs/src/option/mod.rs b/rust/ccommon-rs/src/option/mod.rs new file mode 100644 index 000000000..b2e45d584 --- /dev/null +++ b/rust/ccommon-rs/src/option/mod.rs @@ -0,0 +1,357 @@ +// ccommon - a cache common library. +// Copyright (C) 2019 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Types and methods for dealing with ccommon options. +//! +//! See the docs on the `Options` macro to see how to create +//! new options structs. + +use std::ffi::CStr; + +use ccommon_backend::option::{ + option_describe_all, option_load, option_load_default, option_print_all, ParseError, +}; +use ccommon_sys::option; + +// Sealed trait to prevent SingleOption from ever being implemented +// from outside of this crate. +mod private { + pub trait Sealed {} +} + +use self::private::Sealed; + +mod boolean; +mod fpn; +mod string; +mod uint; + +pub use self::boolean::Bool; +pub use self::fpn::Float; +pub use self::string::Str; +pub use self::uint::UInt; + +/// A single option. +/// +/// This trait is sealed and cannot be implemented outside +/// of ccommon_rs. +pub unsafe trait SingleOption: self::private::Sealed { + /// The type of the value contained within this option. + type Value; + + /// Create an option with the given description, name, + /// and default value. + /// + /// Normally, this should only be called by the derive + /// macro for `Options`. + fn new(default: Self::Value, name: &'static CStr, desc: &'static CStr) -> Self; + + /// Create an option with the given description, name, + /// and `Default::default()` as the default value. + /// + /// The only exception is that [`Str`](crate::option::Str) + /// uses `std::ptr::null_mut()` as it's default value since + /// pointers do not implement `Default`. + /// + /// Normally, this should only be called by the derive macro + /// for `Options`. + fn defaulted(name: &'static CStr, desc: &'static CStr) -> Self; + + /// The name of this option + fn name(&self) -> &'static CStr; + /// A C string describing this option + fn desc(&self) -> &'static CStr; + /// The current value of this option + fn value(&self) -> Self::Value; + /// The default value for this option + fn default(&self) -> Self::Value; + /// Whether the option has been set externally + fn is_set(&self) -> bool; + + /// Overwrite the current value for the option. + /// + /// This will always set `is_set` to true. + fn set_value(&mut self, val: Self::Value); +} + +/// A type that can be safely viewed as a contiguous +/// array of [`option`s][0]. +/// +/// See [`OptionsExt`] for more useful functions on +/// `Options`. +/// +/// This trait should normally only be implemented through +/// `#[derive(Options)]`. However, it must be implemented +/// manually for C types which have been bound using bindgen. +/// +/// [0]: ../../cc_binding/struct.option.html +pub unsafe trait Options: Sized { + fn new() -> Self; +} + +pub trait OptionExt: Options { + /// The number of options in this object when it + /// is interpreted as an array. + /// + /// # Panics + /// Panics if the size of this type is not a multiple + /// of thie size of `option`. + fn num_options() -> usize { + use std::mem::size_of; + + // If this assert fails then there was no way that + // options upholds it's safety requirements so it's + // better to fail here. + assert!(size_of::() % size_of::