Skip to content

Commit

Permalink
Update C API
Browse files Browse the repository at this point in the history
  • Loading branch information
@yinggeh
yinggeh committed Aug 13, 2024
1 parent fd5c44b commit edb0533
Show file tree
Hide file tree
Showing 8 changed files with 199 additions and 99 deletions.
77 changes: 63 additions & 14 deletions include/triton/core/tritonserver.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ struct TRITONSERVER_Server;
struct TRITONSERVER_ServerOptions;
struct TRITONSERVER_Metric;
struct TRITONSERVER_MetricFamily;
struct TRITONSERVER_MetricArgs;

///
/// TRITONSERVER API Version
Expand Down Expand Up @@ -2645,6 +2646,42 @@ TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricFamilyNew(
TRITONSERVER_DECLSPEC struct TRITONSERVER_Error*
TRITONSERVER_MetricFamilyDelete(struct TRITONSERVER_MetricFamily* family);

/// Get the TRITONSERVER_MetricKind of the metric family.
///
/// \param metric The metric family object to query.
/// \param kind Returns the TRITONSERVER_MetricKind of metric.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC struct TRITONSERVER_Error*
TRITONSERVER_GetMetricFamilyKind(
struct TRITONSERVER_MetricFamily* family, TRITONSERVER_MetricKind* kind);

/// Create a new metric args object. The caller takes ownership of the
/// TRITONSERVER_MetricArgs object and must call TRITONSERVER_MetricArgsDelete
/// to release the object.
///
/// \param args Returns the new metric args object.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricArgsNew(
struct TRITONSERVER_MetricArgs** args);

/// Set metric args with prometheus histogram metric parameter.
///
/// \param args The metric args object to set.
/// \param buckets The array of bucket boundaries.
/// \param buckets_count The number of bucket boundaries.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC struct TRITONSERVER_Error*
TRITONSERVER_MetricArgsSetHistogram(
struct TRITONSERVER_MetricArgs* args, const double* buckets,
const uint64_t buckets_count);

/// Delete a metric args object.
///
/// \param args The metric args object.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricArgsDelete(
struct TRITONSERVER_MetricArgs* args);

/// Create a new metric object. The caller takes ownership of the
/// TRITONSERVER_Metric object and must call
/// TRITONSERVER_MetricDelete to release the object. The caller is also
Expand All @@ -2660,10 +2697,31 @@ TRITONSERVER_MetricFamilyDelete(struct TRITONSERVER_MetricFamily* family);
/// bucket boundaries. For histogram only.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricNew(
struct TRITONSERVER_Metric** metric,
struct TRITONSERVER_MetricFamily* family,
const struct TRITONSERVER_Parameter** labels, const uint64_t label_count);

/// Create a new metric object. The caller takes ownership of the
/// TRITONSERVER_Metric object and must call
/// TRITONSERVER_MetricDelete to release the object. The caller is also
/// responsible for ownership of the labels passed in.
/// Each label can be deleted immediately after creating the metric with
/// TRITONSERVER_ParameterDelete if not re-using the labels.
/// Metric args can be deleted immediately after creating the metric with
/// TRITONSERVER_MetricArgsDelete if not re-using the metric args.
///
/// \param metric Returns the new metric object.
/// \param family The metric family to add this new metric to.
/// \param labels The array of labels to associate with this new metric.
/// \param label_count The number of labels.
/// \param args Metric args that store additional arguments to construct
/// particular metric types, e.g. histogram.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricNewWithArgs(
struct TRITONSERVER_Metric** metric,
struct TRITONSERVER_MetricFamily* family,
const struct TRITONSERVER_Parameter** labels, const uint64_t label_count,
const void* buckets = nullptr);
const struct TRITONSERVER_MetricArgs* args);

/// Delete a metric object.
/// All TRITONSERVER_Metric* objects should be deleted BEFORE their
Expand Down Expand Up @@ -2700,8 +2758,9 @@ TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricValue(
TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricIncrement(
struct TRITONSERVER_Metric* metric, double value);

/// Set the current value of metric to value.
/// Supports metrics of kind TRITONSERVER_METRIC_KIND_GAUGE and returns
/// Set the current value of metric to value or observe the value to metric.
/// Supports metrics of kind TRITONSERVER_METRIC_KIND_GAUGE and
/// TRITONSERVER_METRIC_KIND_HISTOGRAM. Returns
/// TRITONSERVER_ERROR_UNSUPPORTED for unsupported TRITONSERVER_MetricKind.
///
/// \param metric The metric object to update.
Expand All @@ -2710,16 +2769,6 @@ TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricIncrement(
TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricSet(
struct TRITONSERVER_Metric* metric, double value);

/// Observe the current value of metric to value.
/// Supports metrics of kind TRITONSERVER_METRIC_KIND_HISTOGRAM and returns
/// TRITONSERVER_ERROR_UNSUPPORTED for unsupported TRITONSERVER_MetricKind.
///
/// \param metric The metric object to update.
/// \param value The amount to observe metric's value to.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricObserve(
struct TRITONSERVER_Metric* metric, double value);

/// Collect metrics.
/// Supports metrics of kind TRITONSERVER_METRIC_KIND_COUNTER,
/// TRITONSERVER_METRIC_KIND_GAUGE, TRITONSERVER_METRIC_KIND_HISTOGRAM and
Expand All @@ -2732,7 +2781,7 @@ TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricObserve(
TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricCollect(
struct TRITONSERVER_Metric* metric, void* value);

/// Get the TRITONSERVER_MetricKind of metric and its corresponding family.
/// Get the TRITONSERVER_MetricKind of metric of its corresponding family.
///
/// \param metric The metric object to query.
/// \param kind Returns the TRITONSERVER_MetricKind of metric.
Expand Down
2 changes: 0 additions & 2 deletions python/tritonserver/_c/triton_bindings.pyi
View file
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,6 @@ class TRITONSERVER_Metric:
) -> None: ...
def increment(self, arg0: float) -> None: ...
def set_value(self, arg0: float) -> None: ...
def observe(self, arg0: float) -> None: ...
@property
def kind(self) -> TRITONSERVER_MetricKind: ...
@property
Expand Down Expand Up @@ -385,7 +384,6 @@ class TRITONSERVER_MetricKind:
__members__: ClassVar[dict] = ... # read-only
COUNTER: ClassVar[TRITONSERVER_MetricKind] = ...
GAUGE: ClassVar[TRITONSERVER_MetricKind] = ...
HISTOGRAM: ClassVar[TRITONSERVER_MetricKind] = ...
__entries: ClassVar[dict] = ...
def __init__(self, value: int) -> None: ...
def __eq__(self, other: object) -> bool: ...
Expand Down
19 changes: 5 additions & 14 deletions python/tritonserver/_c/tritonserver_pybind.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1672,16 +1672,14 @@ class PyMetric : public PyWrapper<struct TRITONSERVER_Metric> {
DESTRUCTOR_WITH_LOG(PyMetric, TRITONSERVER_MetricDelete);
PyMetric(
PyMetricFamily& family,
const std::vector<std::shared_ptr<PyParameter>>& labels,
const std::vector<double>* buckets)
const std::vector<std::shared_ptr<PyParameter>>& labels)
{
std::vector<const struct TRITONSERVER_Parameter*> params;
for (const auto& label : labels) {
params.emplace_back(label->Ptr());
}
ThrowIfError(TRITONSERVER_MetricNew(
&triton_object_, family.Ptr(), params.data(), params.size(),
reinterpret_cast<const void*>(buckets)));
&triton_object_, family.Ptr(), params.data(), params.size()));
owned_ = true;
}

Expand All @@ -1702,11 +1700,6 @@ class PyMetric : public PyWrapper<struct TRITONSERVER_Metric> {
ThrowIfError(TRITONSERVER_MetricSet(triton_object_, val));
}

void Observe(double val) const
{
ThrowIfError(TRITONSERVER_MetricObserve(triton_object_, val));
}

TRITONSERVER_MetricKind Kind() const
{
TRITONSERVER_MetricKind val = TRITONSERVER_METRIC_KIND_COUNTER;
Expand Down Expand Up @@ -2147,8 +2140,7 @@ PYBIND11_MODULE(triton_bindings, m)
// TRITONSERVER_MetricKind
py::enum_<TRITONSERVER_MetricKind>(m, "TRITONSERVER_MetricKind")
.value("COUNTER", TRITONSERVER_METRIC_KIND_COUNTER)
.value("GAUGE", TRITONSERVER_METRIC_KIND_GAUGE)
.value("HISTOGRAM", TRITONSERVER_METRIC_KIND_HISTOGRAM);
.value("GAUGE", TRITONSERVER_METRIC_KIND_GAUGE);
// TRITONSERVER_MetricFamily
py::class_<PyMetricFamily>(m, "TRITONSERVER_MetricFamily")
.def(py::init<
Expand All @@ -2157,13 +2149,12 @@ PYBIND11_MODULE(triton_bindings, m)
py::class_<PyMetric>(m, "TRITONSERVER_Metric")
.def(
py::init<
PyMetricFamily&, const std::vector<std::shared_ptr<PyParameter>>&,
const std::vector<double>*>(),
PyMetricFamily&,
const std::vector<std::shared_ptr<PyParameter>>&>(),
py::keep_alive<1, 2>())
.def_property_readonly("value", &PyMetric::Value)
.def("increment", &PyMetric::Increment)
.def("set_value", &PyMetric::SetValue)
.def("observe", &PyMetric::Observe)
.def_property_readonly("kind", &PyMetric::Kind);
}

Expand Down
55 changes: 11 additions & 44 deletions src/metric_family.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,14 +72,14 @@ MetricFamily::MetricFamily(
void*
MetricFamily::Add(
std::map<std::string, std::string> label_map, Metric* metric,
const std::vector<double>* buckets)
const TritonServerMetricArgs* args)
{
void* prom_metric = nullptr;
switch (kind_) {
case TRITONSERVER_METRIC_KIND_COUNTER: {
if (buckets != nullptr) {
if (args != nullptr) {
throw std::invalid_argument(
"Unexpected buckets found in counter Metric constructor.");
"Unexpected args found in counter Metric constructor.");
}
auto counter_family_ptr =
reinterpret_cast<prometheus::Family<prometheus::Counter>*>(family_);
Expand All @@ -88,9 +88,9 @@ MetricFamily::Add(
break;
}
case TRITONSERVER_METRIC_KIND_GAUGE: {
if (buckets != nullptr) {
if (args != nullptr) {
throw std::invalid_argument(
"Unexpected buckets found in gauge Metric constructor.");
"Unexpected args found in gauge Metric constructor.");
}
auto gauge_family_ptr =
reinterpret_cast<prometheus::Family<prometheus::Gauge>*>(family_);
Expand All @@ -99,13 +99,14 @@ MetricFamily::Add(
break;
}
case TRITONSERVER_METRIC_KIND_HISTOGRAM: {
if (buckets == nullptr) {
if (args == nullptr) {
throw std::invalid_argument(
"Missing required buckets in histogram Metric constructor.");
"Bucket boundaries not found in Metric constructor args.");
}
auto histogram_family_ptr =
reinterpret_cast<prometheus::Family<prometheus::Histogram>*>(family_);
auto histogram_ptr = &histogram_family_ptr->Add(label_map, *buckets);
auto histogram_ptr =
&histogram_family_ptr->Add(label_map, args->buckets());
prom_metric = reinterpret_cast<void*>(histogram_ptr);
break;
}
Expand Down Expand Up @@ -206,7 +207,7 @@ MetricFamily::~MetricFamily()
Metric::Metric(
TRITONSERVER_MetricFamily* family,
std::vector<const InferenceParameter*> labels,
const std::vector<double>* buckets)
const TritonServerMetricArgs* args)
{
family_ = reinterpret_cast<MetricFamily*>(family);
kind_ = family_->Kind();
Expand All @@ -225,7 +226,7 @@ Metric::Metric(
std::string(reinterpret_cast<const char*>(param->ValuePointer()));
}

metric_ = family_->Add(label_map, this, buckets);
metric_ = family_->Add(label_map, this, args);
}

Metric::~Metric()
Expand Down Expand Up @@ -355,40 +356,6 @@ Metric::Set(double value)
gauge_ptr->Set(value);
break;
}
case TRITONSERVER_METRIC_KIND_HISTOGRAM: {
return TRITONSERVER_ErrorNew(
TRITONSERVER_ERROR_UNSUPPORTED,
"TRITONSERVER_METRIC_KIND_HISTOGRAM does not support Set");
}
default:
return TRITONSERVER_ErrorNew(
TRITONSERVER_ERROR_UNSUPPORTED,
"Unsupported TRITONSERVER_MetricKind");
}

return nullptr; // Success
}

TRITONSERVER_Error*
Metric::Observe(double value)
{
if (metric_ == nullptr) {
return TRITONSERVER_ErrorNew(
TRITONSERVER_ERROR_INTERNAL,
"Could not set metric value. Metric has been invalidated.");
}

switch (kind_) {
case TRITONSERVER_METRIC_KIND_COUNTER: {
return TRITONSERVER_ErrorNew(
TRITONSERVER_ERROR_UNSUPPORTED,
"TRITONSERVER_METRIC_KIND_COUNTER does not support Observe");
}
case TRITONSERVER_METRIC_KIND_GAUGE: {
return TRITONSERVER_ErrorNew(
TRITONSERVER_ERROR_UNSUPPORTED,
"TRITONSERVER_METRIC_KIND_GAUGE does not support Observe");
}
case TRITONSERVER_METRIC_KIND_HISTOGRAM: {
auto histogram_ptr = reinterpret_cast<prometheus::Histogram*>(metric_);
histogram_ptr->Observe(value);
Expand Down
29 changes: 27 additions & 2 deletions src/metric_family.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@

#ifdef TRITON_ENABLE_METRICS

#include <cstring>
#include <mutex>
#include <set>
#include <unordered_map>
Expand All @@ -38,6 +39,30 @@

namespace triton { namespace core {

//
// TritonServerMetricArgs
//
// Implementation for TRITONSERVER_MetricArgs.
//
class TritonServerMetricArgs {
public:
TritonServerMetricArgs() = default;

void* SetHistogramArgs(const double* buckets, uint64_t bucket_count)
{
kind_ = TRITONSERVER_MetricKind::TRITONSERVER_METRIC_KIND_HISTOGRAM;
buckets_.resize(bucket_count);
std::memcpy(buckets_.data(), buckets, sizeof(double) * bucket_count);
return nullptr;
}

const std::vector<double>& buckets() const { return buckets_; }

private:
TRITONSERVER_MetricKind kind_;
std::vector<double> buckets_;
};

//
// Implementation for TRITONSERVER_MetricFamily.
//
Expand All @@ -53,7 +78,7 @@ class MetricFamily {

void* Add(
std::map<std::string, std::string> label_map, Metric* metric,
const std::vector<double>* buckets = nullptr);
const TritonServerMetricArgs* buckets);
void Remove(void* prom_metric, Metric* metric);

int NumMetrics()
Expand Down Expand Up @@ -90,7 +115,7 @@ class Metric {
Metric(
TRITONSERVER_MetricFamily* family,
std::vector<const InferenceParameter*> labels,
const std::vector<double>* buckets = nullptr);
const TritonServerMetricArgs* args);
~Metric();

MetricFamily* Family() const { return family_; }
Expand Down
Loading

0 comments on commit edb0533

Please sign in to comment.