Skip to content

Commit

Permalink
Add nightly runs for zstd codec (#3904)
Browse files Browse the repository at this point in the history 
Signed-off-by: Rishabh Singh <[email protected]>
  • Loading branch information
@rishabh6788
rishabh6788 authored Aug 17, 2023
1 parent 1efb889 commit bbdd8ce
Show file tree
Hide file tree
Showing 3 changed files with 30 additions and 0 deletions.
10 changes: 10 additions & 0 deletions jenkins/opensearch/benchmark-test.jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,16 @@ pipeline {
H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-4-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"}
H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-6-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:6;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"}

H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-zstd-codec;WORKLOAD_PARAMS={"index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}}
H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-zstd-codec;WORKLOAD_PARAMS={"index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}}
H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-zstd-codec;WORKLOAD_PARAMS={"index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}}
H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-zstd-codec;WORKLOAD_PARAMS={"index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}}

H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-zstd-codec;WORKLOAD_PARAMS={"index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}}
H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-zstd-codec;WORKLOAD_PARAMS={"index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}}
H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-zstd-codec;WORKLOAD_PARAMS={"index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}}
H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-zstd-codec;WORKLOAD_PARAMS={"index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}}

H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"}
H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"}
H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"}
Expand Down
10 changes: 10 additions & 0 deletions tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test.jenkinsfile.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,16 @@
H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-4-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"}
H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-6-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:6;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"}

H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-zstd-codec;WORKLOAD_PARAMS={"index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}}
H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-zstd-codec;WORKLOAD_PARAMS={"index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}}
H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-zstd-codec;WORKLOAD_PARAMS={"index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}}
H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-zstd-codec;WORKLOAD_PARAMS={"index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}}

H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-zstd-codec;WORKLOAD_PARAMS={"index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}}
H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-zstd-codec;WORKLOAD_PARAMS={"index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}}
H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-zstd-codec;WORKLOAD_PARAMS={"index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}}
H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-zstd-codec;WORKLOAD_PARAMS={"index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}}

H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"}
H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"}
H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"}
Expand Down
Loading

0 comments on commit bbdd8ce

Please sign in to comment.