From 72a10c99e8fc67b3fa9e5a9d47b99f882b8b0e3c Mon Sep 17 00:00:00 2001 From: Elias Bermudez Date: Fri, 21 Jul 2023 12:40:56 -0700 Subject: [PATCH 1/8] Add json schema for output file and examples --- .../examples/coupled_output_file.json | 39 ++++++++ .../examples/decoupled_output_file.json | 41 ++++++++ src/c++/perf_analyzer/schema.json | 96 +++++++++++++++++++ 3 files changed, 176 insertions(+) create mode 100644 src/c++/perf_analyzer/examples/coupled_output_file.json create mode 100644 src/c++/perf_analyzer/examples/decoupled_output_file.json create mode 100644 src/c++/perf_analyzer/schema.json diff --git a/src/c++/perf_analyzer/examples/coupled_output_file.json b/src/c++/perf_analyzer/examples/coupled_output_file.json new file mode 100644 index 000000000..6d0dd7cb6 --- /dev/null +++ b/src/c++/perf_analyzer/examples/coupled_output_file.json @@ -0,0 +1,39 @@ +{ + "experiments": [ + { + "experiment": { + "mode": "concurrency", + "value": 4 + }, + "requests": [ + { + "timestamp": 1, + "sequence_id": 1, + "responses_timestamps": [ + 2 + ] + }, + { + "timestamp": 5, + "sequence_id": 2, + "responses_timestamps": [ + 6 + ] + }, + { + "timestamp": 7, + "sequence_id": 3, + "responses_timestamps": [ + 8 + ] + } + ], + "window_boundaries": [ + 1, + 5, + 7 + ] + } + ], + "version": "1.2.3" +} \ No newline at end of file diff --git a/src/c++/perf_analyzer/examples/decoupled_output_file.json b/src/c++/perf_analyzer/examples/decoupled_output_file.json new file mode 100644 index 000000000..a7ceafef0 --- /dev/null +++ b/src/c++/perf_analyzer/examples/decoupled_output_file.json @@ -0,0 +1,41 @@ +{ + "experiments": [ + { + "experiment": { + "mode": "concurrency", + "value": 4 + }, + "requests": [ + { + "timestamp": 1, + "sequence_id": 1, + "responses_timestamps": [ + 2, + 3, + 4 + ] + }, + { + "timestamp": 5, + "sequence_id": 2, + "responses_timestamps": [] + }, + { + "timestamp": 6, + "sequence_id": 2, + "responses_timestamps": [ + 7, + 8, + 9 + ] + } + ], + "window_boundaries": [ + 1, + 5, + 6 + ] + } + ], + "version": "1.2.3" +} \ No newline at end of file diff --git a/src/c++/perf_analyzer/schema.json b/src/c++/perf_analyzer/schema.json new file mode 100644 index 000000000..7a9be1e31 --- /dev/null +++ b/src/c++/perf_analyzer/schema.json @@ -0,0 +1,96 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/triton-inference-server/client/tree/main/src/c%2B%2B/perf_analyzer/output", + "title": "Perf Analyzer output data", + "description": "A json file describing the output from a perf analyzer run", + "type": "object", + "properties": { + "experiments": { + "description": "The array of all experiments run by perf analyzer", + "type": "array", + "items": { + "type": "object" + }, + "minItems": 1, + "uniqueItems": true, + "properties": { + "experiment": { + "description": "A single experiment run by perf analyzer", + "type": "object", + "minItems": 1, + "maxItems": 1, + "properties": { + "mode": { + "description": "Operating mode of perf analyzer: either concurrency or request rate", + "type": "string" + }, + "value": { + "description": "Concurrency or request rate for the current experiment", + "type": "integer" + } + }, + "required": [ + "mode", + "value" + ] + }, + "requests": { + "description": "The array of requests sent by perf_analyzer for this experiment", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/properties/experiments/properties/$defs/request" + } + }, + "$defs": { + "request": { + "description": "Info for a single request", + "type": "object", + "properties": { + "timestamp": { + "description": "Time stamp of the request", + "type": "integer" + }, + "sequence_id": { + "description": "sequence_id of the request", + "type": "integer" + }, + "responses_timestamps": { + "description": "All associated responses to this request", + "type": "array", + "items": { + "type": "integer" + } + } + }, + "required": [ + "timestamp", + "responses_timestamps" + ] + } + }, + "window_boundaries": { + "description": "An array of time stamps describing window boundaries", + "type": "array", + "items": { + "type": "integer" + }, + "uniqueItems": true + } + }, + "required": [ + "experiment", + "requests", + "window_boundaries" + ] + }, + "version": { + "description": "The version of perf analyzer that generated the report", + "type": "string" + } + }, + "required": [ + "experiments", + "version" + ] +} \ No newline at end of file From 3889e0656b8e9eee1440e7abe0db6a0c0111ee87 Mon Sep 17 00:00:00 2001 From: Elias Bermudez Date: Fri, 21 Jul 2023 12:47:25 -0700 Subject: [PATCH 2/8] Add newlines at end of files --- src/c++/perf_analyzer/examples/coupled_output_file.json | 2 +- src/c++/perf_analyzer/examples/decoupled_output_file.json | 2 +- src/c++/perf_analyzer/schema.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/c++/perf_analyzer/examples/coupled_output_file.json b/src/c++/perf_analyzer/examples/coupled_output_file.json index 6d0dd7cb6..b04e247b9 100644 --- a/src/c++/perf_analyzer/examples/coupled_output_file.json +++ b/src/c++/perf_analyzer/examples/coupled_output_file.json @@ -36,4 +36,4 @@ } ], "version": "1.2.3" -} \ No newline at end of file +} diff --git a/src/c++/perf_analyzer/examples/decoupled_output_file.json b/src/c++/perf_analyzer/examples/decoupled_output_file.json index a7ceafef0..76b201611 100644 --- a/src/c++/perf_analyzer/examples/decoupled_output_file.json +++ b/src/c++/perf_analyzer/examples/decoupled_output_file.json @@ -38,4 +38,4 @@ } ], "version": "1.2.3" -} \ No newline at end of file +} diff --git a/src/c++/perf_analyzer/schema.json b/src/c++/perf_analyzer/schema.json index 7a9be1e31..fbf2febb7 100644 --- a/src/c++/perf_analyzer/schema.json +++ b/src/c++/perf_analyzer/schema.json @@ -93,4 +93,4 @@ "experiments", "version" ] -} \ No newline at end of file +} From ade52404b8390eeebfdd7d8b03de908d6622285d Mon Sep 17 00:00:00 2001 From: Elias Bermudez Date: Fri, 21 Jul 2023 16:38:30 -0700 Subject: [PATCH 3/8] Update json type to correct keyword --- src/c++/perf_analyzer/schema.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/c++/perf_analyzer/schema.json b/src/c++/perf_analyzer/schema.json index fbf2febb7..4bc748b2b 100644 --- a/src/c++/perf_analyzer/schema.json +++ b/src/c++/perf_analyzer/schema.json @@ -26,7 +26,7 @@ }, "value": { "description": "Concurrency or request rate for the current experiment", - "type": "integer" + "type": "number" } }, "required": [ @@ -49,17 +49,17 @@ "properties": { "timestamp": { "description": "Time stamp of the request", - "type": "integer" + "type": "number" }, "sequence_id": { "description": "sequence_id of the request", - "type": "integer" + "type": "number" }, "responses_timestamps": { "description": "All associated responses to this request", "type": "array", "items": { - "type": "integer" + "type": "number" } } }, @@ -73,7 +73,7 @@ "description": "An array of time stamps describing window boundaries", "type": "array", "items": { - "type": "integer" + "type": "number" }, "uniqueItems": true } From d1fdc9cac8c5655e86a4811bc177264ef3c9b4ed Mon Sep 17 00:00:00 2001 From: Elias Bermudez Date: Mon, 24 Jul 2023 16:59:55 -0700 Subject: [PATCH 4/8] Update schema to fix hierarchy and address feedback --- src/c++/perf_analyzer/schema.json | 146 +++++++++++++++--------------- 1 file changed, 73 insertions(+), 73 deletions(-) diff --git a/src/c++/perf_analyzer/schema.json b/src/c++/perf_analyzer/schema.json index 4bc748b2b..a00730b5a 100644 --- a/src/c++/perf_analyzer/schema.json +++ b/src/c++/perf_analyzer/schema.json @@ -1,96 +1,96 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/triton-inference-server/client/tree/main/src/c%2B%2B/perf_analyzer/output", + "$id": "https://github.com/triton-inference-server/client/blob/main/src/c%2B%2B/perf_analyzer/examples/schema.json", "title": "Perf Analyzer output data", - "description": "A json file describing the output from a perf analyzer run", + "description": "A json file describing the output from a perf analyzer run.", "type": "object", + "required": [ + "experiments", + "version" + ], "properties": { "experiments": { - "description": "The array of all experiments run by perf analyzer", + "description": "The array of all experiments run by perf analyzer.", "type": "array", - "items": { - "type": "object" - }, + "required": [ + "experiment", + "requests", + "window_boundaries" + ], "minItems": 1, "uniqueItems": true, - "properties": { - "experiment": { - "description": "A single experiment run by perf analyzer", - "type": "object", - "minItems": 1, - "maxItems": 1, - "properties": { - "mode": { - "description": "Operating mode of perf analyzer: either concurrency or request rate", - "type": "string" - }, - "value": { - "description": "Concurrency or request rate for the current experiment", - "type": "number" - } - }, - "required": [ - "mode", - "value" - ] - }, - "requests": { - "description": "The array of requests sent by perf_analyzer for this experiment", - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/properties/experiments/properties/$defs/request" - } - }, - "$defs": { - "request": { - "description": "Info for a single request", + "items": { + "type": "object", + "properties": { + "experiment": { + "description": "A single experiment run by perf analyzer.", "type": "object", + "required": [ + "mode", + "value" + ], + "minItems": 1, + "maxItems": 1, "properties": { - "timestamp": { - "description": "Time stamp of the request", - "type": "number" + "mode": { + "description": "Operating mode of perf analyzer: For example, concurrency or request rate.", + "type": "string" }, - "sequence_id": { - "description": "sequence_id of the request", + "value": { + "description": "Concurrency or request rate for the current experiment.", "type": "number" - }, - "responses_timestamps": { - "description": "All associated responses to this request", - "type": "array", - "items": { + } + } + }, + "requests": { + "description": "The array of requests sent by perf_analyzer for this experiment.", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/properties/experiments/items/properties/$defs/request" + } + }, + "$defs": { + "request": { + "description": "Info for a single request.", + "type": "object", + "required": [ + "timestamp", + "responses_timestamps" + ], + "properties": { + "timestamp": { + "description": "Time stamp of the request.", + "type": "number" + }, + "sequence_id": { + "description": "The sequence_id of the request.", "type": "number" + }, + "responses_timestamps": { + "description": "All associated responses to this request.", + "type": "array", + "items": { + "type": "number" + } } } + } + }, + "window_boundaries": { + "description": "An array of time stamps describing window boundaries.", + "type": "array", + "items": { + "type": "number" }, - "required": [ - "timestamp", - "responses_timestamps" - ] + "uniqueItems": true } - }, - "window_boundaries": { - "description": "An array of time stamps describing window boundaries", - "type": "array", - "items": { - "type": "number" - }, - "uniqueItems": true } - }, - "required": [ - "experiment", - "requests", - "window_boundaries" - ] + } }, "version": { - "description": "The version of perf analyzer that generated the report", + "description": "The version of perf analyzer that generated the report.", "type": "string" } - }, - "required": [ - "experiments", - "version" - ] -} + } +} \ No newline at end of file From f93e475ae9c36876316e515b20af17376c97865a Mon Sep 17 00:00:00 2001 From: Elias Bermudez Date: Mon, 24 Jul 2023 17:00:13 -0700 Subject: [PATCH 5/8] Add json file with known error for testing --- .../examples/known_error_file.json | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 src/c++/perf_analyzer/examples/known_error_file.json diff --git a/src/c++/perf_analyzer/examples/known_error_file.json b/src/c++/perf_analyzer/examples/known_error_file.json new file mode 100644 index 000000000..60f2b3dce --- /dev/null +++ b/src/c++/perf_analyzer/examples/known_error_file.json @@ -0,0 +1,39 @@ +{ + "experiments": [ + { + "experiment": { + "mode": "concurrency", + "value": 4 + }, + "requests": [ + { + "timestamp": "This should be a number", + "sequence_id": 1, + "responses_timestamps": [ + 2 + ] + }, + { + "timestamp": 5, + "sequence_id": 2, + "responses_timestamps": [ + 6 + ] + }, + { + "timestamp": 7, + "sequence_id": 3, + "responses_timestamps": [ + 8 + ] + } + ], + "window_boundaries": [ + 1, + 5, + 7 + ] + } + ], + "version": "1.2.3" +} From 21dae61cc240baf26f39a52e7d06aba2f5fa8a70 Mon Sep 17 00:00:00 2001 From: Elias Bermudez Date: Tue, 25 Jul 2023 15:18:42 -0700 Subject: [PATCH 6/8] Move example files to the docs directory --- .../perf_analyzer/{ => docs}/examples/coupled_output_file.json | 0 .../{ => docs}/examples/decoupled_output_file.json | 0 src/c++/perf_analyzer/{ => docs}/examples/known_error_file.json | 0 src/c++/perf_analyzer/schema.json | 2 +- 4 files changed, 1 insertion(+), 1 deletion(-) rename src/c++/perf_analyzer/{ => docs}/examples/coupled_output_file.json (100%) rename src/c++/perf_analyzer/{ => docs}/examples/decoupled_output_file.json (100%) rename src/c++/perf_analyzer/{ => docs}/examples/known_error_file.json (100%) diff --git a/src/c++/perf_analyzer/examples/coupled_output_file.json b/src/c++/perf_analyzer/docs/examples/coupled_output_file.json similarity index 100% rename from src/c++/perf_analyzer/examples/coupled_output_file.json rename to src/c++/perf_analyzer/docs/examples/coupled_output_file.json diff --git a/src/c++/perf_analyzer/examples/decoupled_output_file.json b/src/c++/perf_analyzer/docs/examples/decoupled_output_file.json similarity index 100% rename from src/c++/perf_analyzer/examples/decoupled_output_file.json rename to src/c++/perf_analyzer/docs/examples/decoupled_output_file.json diff --git a/src/c++/perf_analyzer/examples/known_error_file.json b/src/c++/perf_analyzer/docs/examples/known_error_file.json similarity index 100% rename from src/c++/perf_analyzer/examples/known_error_file.json rename to src/c++/perf_analyzer/docs/examples/known_error_file.json diff --git a/src/c++/perf_analyzer/schema.json b/src/c++/perf_analyzer/schema.json index a00730b5a..9c3360ab8 100644 --- a/src/c++/perf_analyzer/schema.json +++ b/src/c++/perf_analyzer/schema.json @@ -93,4 +93,4 @@ "type": "string" } } -} \ No newline at end of file +} From 1ac9814037da4d313b5f6e069c9690c959a9e1bb Mon Sep 17 00:00:00 2001 From: Elias Bermudez Date: Wed, 26 Jul 2023 16:35:30 -0700 Subject: [PATCH 7/8] Remove some example json files --- .../docs/examples/coupled_output_file.json | 39 ------------------- .../docs/examples/known_error_file.json | 39 ------------------- 2 files changed, 78 deletions(-) delete mode 100644 src/c++/perf_analyzer/docs/examples/coupled_output_file.json delete mode 100644 src/c++/perf_analyzer/docs/examples/known_error_file.json diff --git a/src/c++/perf_analyzer/docs/examples/coupled_output_file.json b/src/c++/perf_analyzer/docs/examples/coupled_output_file.json deleted file mode 100644 index b04e247b9..000000000 --- a/src/c++/perf_analyzer/docs/examples/coupled_output_file.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "experiments": [ - { - "experiment": { - "mode": "concurrency", - "value": 4 - }, - "requests": [ - { - "timestamp": 1, - "sequence_id": 1, - "responses_timestamps": [ - 2 - ] - }, - { - "timestamp": 5, - "sequence_id": 2, - "responses_timestamps": [ - 6 - ] - }, - { - "timestamp": 7, - "sequence_id": 3, - "responses_timestamps": [ - 8 - ] - } - ], - "window_boundaries": [ - 1, - 5, - 7 - ] - } - ], - "version": "1.2.3" -} diff --git a/src/c++/perf_analyzer/docs/examples/known_error_file.json b/src/c++/perf_analyzer/docs/examples/known_error_file.json deleted file mode 100644 index 60f2b3dce..000000000 --- a/src/c++/perf_analyzer/docs/examples/known_error_file.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "experiments": [ - { - "experiment": { - "mode": "concurrency", - "value": 4 - }, - "requests": [ - { - "timestamp": "This should be a number", - "sequence_id": 1, - "responses_timestamps": [ - 2 - ] - }, - { - "timestamp": 5, - "sequence_id": 2, - "responses_timestamps": [ - 6 - ] - }, - { - "timestamp": 7, - "sequence_id": 3, - "responses_timestamps": [ - 8 - ] - } - ], - "window_boundaries": [ - 1, - 5, - 7 - ] - } - ], - "version": "1.2.3" -} From 5e7df191c04ab10f97cbd2695171cbe19d7687b4 Mon Sep 17 00:00:00 2001 From: Elias Bermudez Date: Wed, 26 Jul 2023 16:46:37 -0700 Subject: [PATCH 8/8] Update schema to use integer and remove uniqueness from timestamps --- src/c++/perf_analyzer/schema.json | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/c++/perf_analyzer/schema.json b/src/c++/perf_analyzer/schema.json index 9c3360ab8..f6080058c 100644 --- a/src/c++/perf_analyzer/schema.json +++ b/src/c++/perf_analyzer/schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/triton-inference-server/client/blob/main/src/c%2B%2B/perf_analyzer/examples/schema.json", "title": "Perf Analyzer output data", - "description": "A json file describing the output from a perf analyzer run.", + "description": "A json file describing the output from a Perf Analyzer run.", "type": "object", "required": [ "experiments", @@ -10,7 +10,7 @@ ], "properties": { "experiments": { - "description": "The array of all experiments run by perf analyzer.", + "description": "The array of all experiments run by Perf Analyzer.", "type": "array", "required": [ "experiment", @@ -23,7 +23,7 @@ "type": "object", "properties": { "experiment": { - "description": "A single experiment run by perf analyzer.", + "description": "A single experiment run by Perf Analyzer.", "type": "object", "required": [ "mode", @@ -33,19 +33,18 @@ "maxItems": 1, "properties": { "mode": { - "description": "Operating mode of perf analyzer: For example, concurrency or request rate.", + "description": "Operating mode of Perf Analyzer: For example, 'concurrency' or 'request rate'.", "type": "string" }, "value": { "description": "Concurrency or request rate for the current experiment.", - "type": "number" + "type": "integer" } } }, "requests": { - "description": "The array of requests sent by perf_analyzer for this experiment.", + "description": "The array of requests sent by Perf Analyzer for this experiment.", "type": "array", - "uniqueItems": true, "items": { "$ref": "#/properties/experiments/items/properties/$defs/request" } @@ -61,17 +60,17 @@ "properties": { "timestamp": { "description": "Time stamp of the request.", - "type": "number" + "type": "integer" }, "sequence_id": { "description": "The sequence_id of the request.", - "type": "number" + "type": "integer" }, "responses_timestamps": { "description": "All associated responses to this request.", "type": "array", "items": { - "type": "number" + "type": "integer" } } } @@ -81,7 +80,7 @@ "description": "An array of time stamps describing window boundaries.", "type": "array", "items": { - "type": "number" + "type": "integer" }, "uniqueItems": true } @@ -89,7 +88,7 @@ } }, "version": { - "description": "The version of perf analyzer that generated the report.", + "description": "The version of Perf Analyzer that generated the report.", "type": "string" } }