From 870334f569f5216d3f8e0ee1a90f0bb6fb72f555 Mon Sep 17 00:00:00 2001 From: grig777 Date: Wed, 22 Jan 2025 19:36:39 -0500 Subject: [PATCH 01/22] Add action to upload helm charts --- .github/workflows/release_charts.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/release_charts.yml diff --git a/.github/workflows/release_charts.yml b/.github/workflows/release_charts.yml new file mode 100644 index 000000000..548a54ff1 --- /dev/null +++ b/.github/workflows/release_charts.yml @@ -0,0 +1,26 @@ +name: Release Charts + +on: + push: + branches: + - main + +jobs: + release: + # depending on default permission settings for your org (contents being read-only or read-write for workloads), you will have to add permissions + # see: https://docs.github.com/en/actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token + permissions: + contents: write + runs-on: forge-amd64-medium + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Run chart-releaser + uses: helm/chart-releaser-action@v1.7.0 + env: + CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + charts_dir: "./stable/" + skip_existing: true \ No newline at end of file From 065e30d747a264e68226ddd3115c51ccb9226def Mon Sep 17 00:00:00 2001 From: grig777 Date: Wed, 22 Jan 2025 19:37:16 -0500 Subject: [PATCH 02/22] Fix the action running branch --- .github/workflows/release_charts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_charts.yml b/.github/workflows/release_charts.yml index 548a54ff1..30a661b69 100644 --- a/.github/workflows/release_charts.yml +++ b/.github/workflows/release_charts.yml @@ -3,7 +3,7 @@ name: Release Charts on: push: branches: - - main + - master jobs: release: From f68d6481e1b304eb944abda3b964dd191c5504b1 Mon Sep 17 00:00:00 2001 From: grig777 Date: Wed, 22 Jan 2025 19:40:02 -0500 Subject: [PATCH 03/22] Try another runner --- .github/workflows/release_charts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_charts.yml b/.github/workflows/release_charts.yml index 30a661b69..481a99795 100644 --- a/.github/workflows/release_charts.yml +++ b/.github/workflows/release_charts.yml @@ -11,7 +11,7 @@ jobs: # see: https://docs.github.com/en/actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token permissions: contents: write - runs-on: forge-amd64-medium + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 From 5bebc3e50583057a13a7c98cdb174c92c8f136ad Mon Sep 17 00:00:00 2001 From: grig777 Date: Wed, 22 Jan 2025 19:42:25 -0500 Subject: [PATCH 04/22] Change chart to test upload --- stable/jfrog-platform/Chart.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/stable/jfrog-platform/Chart.yaml b/stable/jfrog-platform/Chart.yaml index d8d240537..bb7faa80f 100644 --- a/stable/jfrog-platform/Chart.yaml +++ b/stable/jfrog-platform/Chart.yaml @@ -54,5 +54,6 @@ maintainers: name: jfrog-platform sources: - https://github.com/jfrog/charts +- https://github.com/fastly/jfrog-helm-charts type: application version: 10.20.4 From 2138f4a25764b77bcf7b47dbba25692670a5b25c Mon Sep 17 00:00:00 2001 From: grig777 Date: Wed, 22 Jan 2025 19:44:33 -0500 Subject: [PATCH 05/22] Change chart to test upload --- .github/workflows/release_charts.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release_charts.yml b/.github/workflows/release_charts.yml index 481a99795..791cb5f99 100644 --- a/.github/workflows/release_charts.yml +++ b/.github/workflows/release_charts.yml @@ -23,4 +23,3 @@ jobs: env: CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" charts_dir: "./stable/" - skip_existing: true \ No newline at end of file From 1540f474acad07545e8422f7ba712c83653ca603 Mon Sep 17 00:00:00 2001 From: grig777 Date: Wed, 22 Jan 2025 19:51:30 -0500 Subject: [PATCH 06/22] Test upload --- .github/workflows/release_charts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_charts.yml b/.github/workflows/release_charts.yml index 791cb5f99..9fa294542 100644 --- a/.github/workflows/release_charts.yml +++ b/.github/workflows/release_charts.yml @@ -22,4 +22,4 @@ jobs: uses: helm/chart-releaser-action@v1.7.0 env: CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - charts_dir: "./stable/" + charts_dir: "./stable/jfrog-platform/" From 9434461acb1c8e8bdb5e470ea17c45beb3ed7cfa Mon Sep 17 00:00:00 2001 From: grig777 Date: Wed, 22 Jan 2025 19:52:43 -0500 Subject: [PATCH 07/22] Test upload --- .github/workflows/release_charts.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_charts.yml b/.github/workflows/release_charts.yml index 9fa294542..d847e752f 100644 --- a/.github/workflows/release_charts.yml +++ b/.github/workflows/release_charts.yml @@ -17,9 +17,9 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Run chart-releaser uses: helm/chart-releaser-action@v1.7.0 + with: + charts_dir: "./stable/" env: CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - charts_dir: "./stable/jfrog-platform/" From f5add1f6c6dfdd5f18432c60a53d37151e91ff27 Mon Sep 17 00:00:00 2001 From: grig777 Date: Wed, 22 Jan 2025 19:54:19 -0500 Subject: [PATCH 08/22] Test upload --- .github/workflows/release_charts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_charts.yml b/.github/workflows/release_charts.yml index d847e752f..12c7381ed 100644 --- a/.github/workflows/release_charts.yml +++ b/.github/workflows/release_charts.yml @@ -20,6 +20,6 @@ jobs: - name: Run chart-releaser uses: helm/chart-releaser-action@v1.7.0 with: - charts_dir: "./stable/" + charts_dir: "./stable/jfrog-platform/" env: CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" From 35ffc61bcb99594bab103c91929ef1e77cb92eb6 Mon Sep 17 00:00:00 2001 From: grig777 Date: Wed, 22 Jan 2025 20:04:55 -0500 Subject: [PATCH 09/22] Test upload --- .github/workflows/release_charts.yml | 49 +++++++++++++++++++--------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/.github/workflows/release_charts.yml b/.github/workflows/release_charts.yml index 12c7381ed..634710798 100644 --- a/.github/workflows/release_charts.yml +++ b/.github/workflows/release_charts.yml @@ -1,25 +1,42 @@ -name: Release Charts +name: Publish Helm Charts to GHCR on: push: branches: - master + paths: + - 'charts/**' # Adjust this path to match the location of your Helm charts jobs: - release: - # depending on default permission settings for your org (contents being read-only or read-write for workloads), you will have to add permissions - # see: https://docs.github.com/en/actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token - permissions: - contents: write + package-and-push: runs-on: ubuntu-latest + steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Run chart-releaser - uses: helm/chart-releaser-action@v1.7.0 - with: - charts_dir: "./stable/jfrog-platform/" - env: - CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Helm + uses: azure/setup-helm@v3 + with: + version: v4.2.0 # Replace with your preferred Helm version + + - name: Authenticate with GitHub Container Registry + env: + CR_PAT: ${{ secrets.GITHUB_TOKEN }} + run: | + echo ${{ secrets.GITHUB_TOKEN }} | helm registry login ghcr.io --username ${{ github.actor }} --password-stdin + + - name: Package and Push Helm Charts + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + for chart_dir in stable/*; do + if [ -d "$chart_dir" ]; then + # Package the Helm chart + helm package "$chart_dir" --destination packaged_charts + # Push the packaged chart to GitHub Container Registry + CHART_NAME=$(basename "$chart_dir") + CHART_VERSION=$(helm show chart "$chart_dir" | grep 'version:' | awk '{print $2}') + helm push "packaged_charts/${CHART_NAME}-${CHART_VERSION}.tgz" "oci://ghcr.io/${{ github.repository_owner }}/charts" + fi + done \ No newline at end of file From a505572ac2d357793755c9ed3907af5298a22e00 Mon Sep 17 00:00:00 2001 From: grig777 Date: Wed, 22 Jan 2025 20:05:41 -0500 Subject: [PATCH 10/22] Test upload --- .github/workflows/release_charts.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_charts.yml b/.github/workflows/release_charts.yml index 634710798..ca67eadd7 100644 --- a/.github/workflows/release_charts.yml +++ b/.github/workflows/release_charts.yml @@ -5,7 +5,7 @@ on: branches: - master paths: - - 'charts/**' # Adjust this path to match the location of your Helm charts + - 'stable/**' jobs: package-and-push: @@ -18,7 +18,7 @@ jobs: - name: Set up Helm uses: azure/setup-helm@v3 with: - version: v4.2.0 # Replace with your preferred Helm version + version: v4.2.0 - name: Authenticate with GitHub Container Registry env: From 9deb925a73b2ac7c324b0f5715e064122fef3f3d Mon Sep 17 00:00:00 2001 From: grig777 Date: Wed, 22 Jan 2025 20:05:56 -0500 Subject: [PATCH 11/22] Test upload --- .github/workflows/release_charts.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_charts.yml b/.github/workflows/release_charts.yml index ca67eadd7..cba981ca5 100644 --- a/.github/workflows/release_charts.yml +++ b/.github/workflows/release_charts.yml @@ -4,8 +4,8 @@ on: push: branches: - master - paths: - - 'stable/**' + # paths: + # - 'stable/**' jobs: package-and-push: From cdedb10072fd4da40bbedc75246332c3d8a2e906 Mon Sep 17 00:00:00 2001 From: grig777 Date: Wed, 22 Jan 2025 20:07:19 -0500 Subject: [PATCH 12/22] Test upload --- .github/workflows/release_charts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_charts.yml b/.github/workflows/release_charts.yml index cba981ca5..8111543ea 100644 --- a/.github/workflows/release_charts.yml +++ b/.github/workflows/release_charts.yml @@ -18,7 +18,7 @@ jobs: - name: Set up Helm uses: azure/setup-helm@v3 with: - version: v4.2.0 + version: v3.16.2 - name: Authenticate with GitHub Container Registry env: From e389862ea9b404468c72a841746d6c898b945744 Mon Sep 17 00:00:00 2001 From: grig777 Date: Wed, 22 Jan 2025 20:09:20 -0500 Subject: [PATCH 13/22] Test upload --- .github/workflows/release_charts.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release_charts.yml b/.github/workflows/release_charts.yml index 8111543ea..7bbbbec79 100644 --- a/.github/workflows/release_charts.yml +++ b/.github/workflows/release_charts.yml @@ -32,6 +32,8 @@ jobs: run: | for chart_dir in stable/*; do if [ -d "$chart_dir" ]; then + # Update dependencies + helm dependency update "$chart_dir" # Package the Helm chart helm package "$chart_dir" --destination packaged_charts # Push the packaged chart to GitHub Container Registry From 2def256d1926e537c369b0132761b2294305a843 Mon Sep 17 00:00:00 2001 From: grig777 Date: Wed, 22 Jan 2025 20:13:14 -0500 Subject: [PATCH 14/22] Test upload --- .github/workflows/release_charts.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release_charts.yml b/.github/workflows/release_charts.yml index 7bbbbec79..1805e8a91 100644 --- a/.github/workflows/release_charts.yml +++ b/.github/workflows/release_charts.yml @@ -37,8 +37,7 @@ jobs: # Package the Helm chart helm package "$chart_dir" --destination packaged_charts # Push the packaged chart to GitHub Container Registry - CHART_NAME=$(basename "$chart_dir") - CHART_VERSION=$(helm show chart "$chart_dir" | grep 'version:' | awk '{print $2}') - helm push "packaged_charts/${CHART_NAME}-${CHART_VERSION}.tgz" "oci://ghcr.io/${{ github.repository_owner }}/charts" + CHART_FILE=$(ls packaged_charts/*.tgz | tail -n 1) + helm push "$CHART_FILE" "oci://ghcr.io/${{ github.repository_owner }}/charts" fi done \ No newline at end of file From 7aaee761d6135a7b1f00ee840861b68349d7aa52 Mon Sep 17 00:00:00 2001 From: grig777 Date: Wed, 22 Jan 2025 20:18:46 -0500 Subject: [PATCH 15/22] Test upload --- stable/artifactory-cpp-ce/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stable/artifactory-cpp-ce/Chart.yaml b/stable/artifactory-cpp-ce/Chart.yaml index 8e3e1d8ff..92e31aad8 100644 --- a/stable/artifactory-cpp-ce/Chart.yaml +++ b/stable/artifactory-cpp-ce/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 appVersion: 7.98.13 dependencies: - name: artifactory - repository: file://charts/artifactory + repository: file://artifactory version: 107.98.13 description: JFrog Artifactory CE for C++ home: https://www.jfrog.com/artifactory/ From 000f7194632dc895de49c69d8338cc274f4d6673 Mon Sep 17 00:00:00 2001 From: grig777 Date: Wed, 22 Jan 2025 20:20:11 -0500 Subject: [PATCH 16/22] Test upload --- stable/artifactory-cpp-ce/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stable/artifactory-cpp-ce/Chart.yaml b/stable/artifactory-cpp-ce/Chart.yaml index 92e31aad8..50542e1f2 100644 --- a/stable/artifactory-cpp-ce/Chart.yaml +++ b/stable/artifactory-cpp-ce/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 appVersion: 7.98.13 dependencies: - name: artifactory - repository: file://artifactory + repository: file://../artifactory version: 107.98.13 description: JFrog Artifactory CE for C++ home: https://www.jfrog.com/artifactory/ From 4a237ea9eac4e459274063e5ac2deebd337fca89 Mon Sep 17 00:00:00 2001 From: grig777 Date: Wed, 22 Jan 2025 22:22:01 -0500 Subject: [PATCH 17/22] Upload only jfrog platform chart --- .github/workflows/release_charts.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_charts.yml b/.github/workflows/release_charts.yml index 1805e8a91..062ba16cd 100644 --- a/.github/workflows/release_charts.yml +++ b/.github/workflows/release_charts.yml @@ -30,7 +30,8 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - for chart_dir in stable/*; do + CHARTS = ("stable/jfrog-platform") + for chart_dir in ${CHARTS[@]}"; do if [ -d "$chart_dir" ]; then # Update dependencies helm dependency update "$chart_dir" From 912d3013e7082b5f5d63ef3fd216a4b25440fbcd Mon Sep 17 00:00:00 2001 From: grig777 Date: Wed, 22 Jan 2025 22:22:58 -0500 Subject: [PATCH 18/22] fix: typo on Upload only jfrog platform chart --- .github/workflows/release_charts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_charts.yml b/.github/workflows/release_charts.yml index 062ba16cd..fe43551b3 100644 --- a/.github/workflows/release_charts.yml +++ b/.github/workflows/release_charts.yml @@ -30,7 +30,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - CHARTS = ("stable/jfrog-platform") + CHARTS=("stable/jfrog-platform") for chart_dir in ${CHARTS[@]}"; do if [ -d "$chart_dir" ]; then # Update dependencies From ef85064f193228d2bc3c51ffe152de379d58c006 Mon Sep 17 00:00:00 2001 From: grig777 Date: Wed, 22 Jan 2025 22:24:07 -0500 Subject: [PATCH 19/22] fix: typo on Upload only jfrog platform chart --- .github/workflows/release_charts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_charts.yml b/.github/workflows/release_charts.yml index fe43551b3..e8025ef1e 100644 --- a/.github/workflows/release_charts.yml +++ b/.github/workflows/release_charts.yml @@ -31,7 +31,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | CHARTS=("stable/jfrog-platform") - for chart_dir in ${CHARTS[@]}"; do + for chart_dir in ${CHARTS[@]}; do if [ -d "$chart_dir" ]; then # Update dependencies helm dependency update "$chart_dir" From 6daab1d668bd0f355c8b45982ba4395dd3976c13 Mon Sep 17 00:00:00 2001 From: grig777 Date: Mon, 27 Jan 2025 11:34:47 -0500 Subject: [PATCH 20/22] Disable action and add custom resources to template --- .github/workflows/release_charts.yml | 12 +- stable/jfrog-platform/templates/custom.yaml | 853 ++++++++++++++++++++ 2 files changed, 859 insertions(+), 6 deletions(-) create mode 100644 stable/jfrog-platform/templates/custom.yaml diff --git a/.github/workflows/release_charts.yml b/.github/workflows/release_charts.yml index e8025ef1e..a796829be 100644 --- a/.github/workflows/release_charts.yml +++ b/.github/workflows/release_charts.yml @@ -1,11 +1,11 @@ name: Publish Helm Charts to GHCR -on: - push: - branches: - - master - # paths: - # - 'stable/**' +on: workflow_dispatch +# push: +# branches: +# - master +# paths: +# - 'stable/**' jobs: package-and-push: diff --git a/stable/jfrog-platform/templates/custom.yaml b/stable/jfrog-platform/templates/custom.yaml new file mode 100644 index 000000000..027a46a79 --- /dev/null +++ b/stable/jfrog-platform/templates/custom.yaml @@ -0,0 +1,853 @@ +# Added by FBS team +{{- range .Values.secrets }} +apiVersion: secrets-manager.tuenti.io/v1alpha1 +kind: SecretDefinition +metadata: + name: {{ .name }} +spec: + keysMap: + {{- range .keys }} + {{ .k8s_key }}: + key: {{ .vault_key }} + path: {{ .vault_path }} + {{- end }} + name: {{ .name }} + type: Opaque +--- +{{- end }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + gen-wi-role: "true" + name: artifactory +automountServiceAccountToken: true +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + gen-wi-role: "true" + name: xray +automountServiceAccountToken: true +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: artifactory-fluentd-config +data: + fluentd.conf: | + # METRICS SEGMENT - BEGIN + + @type prometheus + + + + @type tail + @id artifactory_metrics_tail + path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-metrics.log" + pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-metrics.log.pos" + tag jfrog.metrics.rt + read_from_head true + refresh_interval 10 + follow_inodes true + skip_refresh_on_startup true + read_bytes_limit_per_second 104857600 + + @type none + + + + @type tail + @id observability_metrics_tail + path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/observability-metrics.log" + pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/observability-metrics.log.pos" + tag jfrog.metrics.rt + read_from_head true + refresh_interval 10 + follow_inodes true + skip_refresh_on_startup true + read_bytes_limit_per_second 104857600 + + @type none + + + + + @type parser + key_name message + + @type regexp + expression ^(?[\w]+)\ (?[[+-]?\d(\.\d+)?[Ee][+-]?\d+]+)\ (?[\d]+)$ + time_type string + time_key timestamp + time_format %s + + emit_invalid_record_to_error false + + + + @type record_transformer + enable_ruby true + renew_record false + + ${record["key"]} ${if record["value"]["."] then record["value"].to_f else record["value"].to_i end} + + + + + @type prometheus + + + name jfrt_runtime_heap_freememory_bytes + desc Heap Free Memory in Bytes + type gauge + key jfrt_runtime_heap_freememory_bytes + + agent fluentd + host ${hostname} + service artifactory + + + + + name jfrt_runtime_heap_maxmemory_bytes + desc Heap Max Memory in Bytes + type gauge + key jfrt_runtime_heap_maxmemory_bytes + + agent fluentd + host ${hostname} + service artifactory + + + + + name jfrt_runtime_heap_totalmemory_bytes + desc Heap Total Memory in Bytes + type gauge + key jfrt_runtime_heap_totalmemory_bytes + + agent fluentd + host ${hostname} + service artifactory + + + + + name jfrt_runtime_heap_processors_total + desc Total Heap Processors + type gauge + key jfrt_runtime_heap_processors_total + + agent fluentd + host ${hostname} + service artifactory + + + + + name jfrt_db_connections_active_total + desc Active DB Connections + type gauge + key jfrt_db_connections_active_total + + agent fluentd + host ${hostname} + service artifactory + + + + + name jfrt_db_connections_idle_total + desc Idle DB Connections + type gauge + key jfrt_db_connections_idle_total + + agent fluentd + host ${hostname} + service artifactory + + + + + name jfrt_db_connections_max_active_total + desc Max Active DB Connections + type gauge + key jfrt_db_connections_max_active_total + + agent fluentd + host ${hostname} + service artifactory + + + + + name jfrt_db_connections_min_idle_total + desc Min Idle DB Connections + type gauge + key jfrt_db_connections_min_idle_total + + agent fluentd + host ${hostname} + service artifactory + + + + + name jfrt_storage_current_total_size_bytes + desc Artifactory Storage in Bytes + type gauge + key jfrt_storage_current_total_size_bytes + + agent fluentd + host ${hostname} + service artifactory + + + + + name app_disk_used_bytes + desc Application Disk Used in Bytes + type gauge + key app_disk_used_bytes + + agent fluentd + host ${hostname} + service artifactory + + + + + name app_disk_free_bytes + desc Application Disk Free in Bytes + type gauge + key app_disk_free_bytes + + agent fluentd + host ${hostname} + service artifactory + + + + + name sys_memory_used_bytes + desc System Memory Used in Bytes + type gauge + key sys_memory_used_bytes + + agent fluentd + host ${hostname} + service artifactory + + + + + name sys_memory_free_bytes + desc System Memory Free in Bytes + type gauge + key sys_memory_free_bytes + + agent fluentd + host ${hostname} + service artifactory + + + + + name sys_cpu_ratio + desc System CPU Ratio + type gauge + key sys_cpu_ratio + + agent fluentd + host ${hostname} + service artifactory + + + + + + # METRICS SEGMENT - END + + # LOG SEGMENT - BEGIN + + + @type tail + @id access_service_tail + path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-service.log" + pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-service.log.pos" + tag jfrog.rt.access.service + + @type none + + + + @type tail + @id artifactory_service_tail + path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-service.log" + pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-service.log.pos" + tag jfrog.rt.artifactory.service + + @type none + + + + @type tail + @id frontend_service_tail + path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-service.log" + pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-service.log.pos" + tag jfrog.rt.frontend.service + + @type none + + + + @type tail + @id metadata_service_tail + path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/metadata-service.log" + pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/metadata-service.log.pos" + tag jfrog.rt.metadata.service + + @type none + + + + @type tail + @id router_service_tail + path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-service.log" + pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-service.log.pos" + tag jfrog.rt.router.service + + @type none + + + # Strip out color codes then field extract the service fields + + @type record_transformer + enable_ruby true + + message ${record["message"].gsub(/\e\[([;\d]+)?m/, '')} + + + + @type parser + key_name message + + @type multiline + format_firstline /\d{4}-\d{1,2}-\d{1,2}/ + format1 /^(?[^ ]*) \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?.*)\] \[(?.*)\] -(?.*)$/ + time_key timestamp + time_format %Y-%m-%dT%H:%M:%S.%LZ + + emit_invalid_record_to_error false + + # End Service Fields Extraction + + @type tail + @id router_traefik_tail + path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-traefik.log" + pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-traefik.log.pos" + tag jfrog.rt.router.traefik + + @type multiline + format_firstline /\d{4}-\d{1,2}-\d{1,2}/ + format1 /^(?[^ ]*) \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?.*)\] \[(?.*)\] - (?.+)$/ + time_key timestamp + time_format %Y-%m-%dT%H:%M:%S.%LZ + + + + + @type tail + @id access_request_tail + path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-request.log" + pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-request.log.pos" + tag jfrog.rt.access.request + + @type regexp + expression /^(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?.+)$/m + time_key timestamp + time_format %Y-%m-%dT%H:%M:%S.%LZ + + + + @type tail + @id artifactory_request_tail + path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-request.log" + pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-request.log.pos" + tag jfrog.rt.artifactory.request + + @type regexp + expression /^(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?.+)$/m + time_key timestamp + time_type mixed + time_format %Y-%m-%dT%H:%M:%S.%LZ + time_format_fallbacks %Y-%m-%dT%H:%M:%SZ + types response_content_length:integer, request_content_length:integer, return_status_code:integer + + emit_invalid_record_to_error false + + + @type record_transformer + enable_ruby true + + user ${!record["username"].strip().start_with?("token") ? (record["username"].strip()) : ("")} + repo ${record["request_url"].strip().include?("/api/docker") && !record["request_url"].include?("/api/docker/null") && !record["request_url"].include?("/api/docker/v2") && !record["request_url"].include?("v2/token") ? (record["request_url"].strip())[/api\/docker\/(.+)\/(?:v2)/, 1] : ("")} + artifact ${record["request_url"].strip().include?("/api/docker") && !record["request_url"].include?("/api/docker/null") && !record["request_url"].include?("/api/docker/v2") && !record["request_url"].include?("v2/token") ? (record["request_url"].strip())[/v2\/(.+)\/(?:manifests|blobs)/, 1] : ("")} + docker_repo ${record["request_url"].strip().include?("/api/docker") && !record["request_url"].include?("/api/docker/null") && !record["request_url"].include?("/api/docker/v2") && !record["request_url"].include?("v2/token") ? (record["request_url"].strip())[/api\/docker\/(.+)\/(?:v2)/, 1] : ("")} + docker_image ${record["request_url"].strip().include?("/api/docker") && !record["request_url"].include?("/api/docker/null") && !record["request_url"].include?("/api/docker/v2") && !record["request_url"].include?("v2/token") ? (record["request_url"].strip())[/v2\/(.+)\/(?:manifests|blobs)/, 1] : ("")} + data_download ${record["response_content_length"] == -1 ? 0 : record["response_content_length"]} + data_upload ${record["request_content_length"] == -1 ? 0 : record["request_content_length"]} + + + + @type tail + @id frontend_request_tail + path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-request.log" + pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-request.log.pos" + tag jfrog.rt.frontend.request + + @type regexp + expression /^(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?.+)$/m + time_key timestamp + time_format %Y-%m-%dT%H:%M:%S.%LZ + + + + @type record_transformer + enable_ruby true + + user ${!record["username"].strip().start_with?("token") ? (record["username"].strip()) : ("")} + + + + @type tail + @id metadata_request_tail + path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/metadata-request.log" + pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/metadata-request.log.pos" + tag jfrog.rt.metadata.request + + @type regexp + expression /^(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?.+)$/m + time_key timestamp + time_format %Y-%m-%dT%H:%M:%S.%LZ + + + + @type record_transformer + enable_ruby true + + user ${!record["username"].strip().start_with?("token") ? (record["username"].strip()) : ("")} + + + + @type tail + @id router_request_tail + path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-request.log" + pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-request.log.pos" + tag jfrog.rt.router.request + + @type json + time_key time + time_format %Y-%m-%dT%H:%M:%SZ + + + # example access logs: + # 2024-04-09T22:47:56.186Z [4147ce79c667c62c ] [ACCEPTED LOGIN] for client : jfmc@01fdq6vwvc110f0bsyy5cc1xxr / 127.0.0.1 [token] + # 2024-04-09T22:48:09.057Z [23805d42fe9d707e] [ACCEPTED DOWNLOAD] for client : jfmc@01fdq6vwvc110f0bsyy5cc1xxr / 127.0.0.1 [apiKey] + # 2024-04-09T22:48:35.240Z [581d24eaf7913e4b] [DENIED LOGIN] for client : NA / 198.145.32.100 + + @type tail + @id artifactory_access_tail + path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-access.log" + pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-access.log.pos" + tag jfrog.rt.artifactory.access + + @type regexp + expression /^(?[^\s]*) \[(?[^\s\]]*)\s*\] \[(?[^\]\r\n]*)\] (?[^\s]*)? *for client : *(?[^\s]*) *\/ *(?[^\s]*)? ?(\[(?[^\s]*)\])?$/m + time_key timestamp + time_format %Y-%m-%dT%H:%M:%S.%LZ + + + + @type tail + @id access_security_audit_tail + path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-security-audit.log" + pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-security-audit.log.pos" + tag jfrog.rt.access.audit + + @type regexp + expression /^(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?.*)$/m + time_key timestamp + time_format %Y-%m-%dT%H:%M:%S.%LZ + + + + # LOG SEGMENT - END + + # WHAT LOG IT WAS INTO THE JSON + + @type record_transformer + + hostname "#{Socket.gethostname}" + log_source ${tag} + + + + + @type stdout + + + + @type stdout + flush_at_shutdown true + output_type json + + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: xray-fluentd-config +data: + fluentd.conf: | + # METRICS SEGMENT - BEGIN + + @type prometheus + + + + @type tail + @id xray_metrics_tail + path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-server-metrics.log" + pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-server-metrics.log.pos" + tag jfrog.metrics.xr + read_from_head true + refresh_interval 10 + follow_inodes true + skip_refresh_on_startup true + read_bytes_limit_per_second 104857600 + + @type none + + + + @type tail + @id observability_metrics_tail + path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/observability-metrics.log" + pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/observability-metrics.log.pos" + tag jfrog.metrics.xr + read_from_head true + refresh_interval 10 + follow_inodes true + skip_refresh_on_startup true + read_bytes_limit_per_second 104857600 + + @type none + + + + + @type parser + key_name message + + @type regexp + expression ^(?[\w]+)\ (?[[+-]?\d(\.\d+)?[Ee][+-]?\d+]+)\ (?[\d]+)$ + time_type string + time_key timestamp + time_format %s + + emit_invalid_record_to_error false + + + + @type record_transformer + enable_ruby true + renew_record false + + ${record["key"]} ${if record["value"]["."] then record["value"].to_f else record["value"].to_i end} + + + + + @type prometheus + + + name sys_cpu_ratio + desc System CPU Ratio + type gauge + key _sys_cpu_ratio + + agent fluentd + host ${hostname} + service xray + + + + + name sys_memory_used_bytes + desc System Memory Used in Bytes + type gauge + key _sys_memory_used_bytes + + agent fluentd + host ${hostname} + service xray + + + + + name sys_memory_free_bytes + desc System Memory Free in Bytes + type gauge + key _sys_memory_free_bytes + + agent fluentd + host ${hostname} + service xray + + + + + name app_disk_free_bytes + desc Application Disk Free in Bytes + type gauge + key _app_disk_free_bytes + + agent fluentd + host ${hostname} + service xray + + + + + name app_disk_used_bytes + desc Application Disk Used in Bytes + type gauge + key _app_disk_used_bytes + + agent fluentd + host ${hostname} + service xray + + + + + name go_memstats_heap_objects_total + desc GO Memory Statistics Total Heap Objects + type gauge + key _go_memstats_heap_objects_total + + agent fluentd + host ${hostname} + service xray + + + + + name go_memstats_heap_allocated_bytes + desc GO Memory Statistics Heap Allocated in Bytes + type gauge + key _go_memstats_heap_allocated_bytes + + agent fluentd + host ${hostname} + service xray + + + + + name go_memstats_heap_in_use_bytes + desc GO Memory Statistics Heap Used in Bytes + type gauge + key _go_memstats_heap_in_use_bytes + + agent fluentd + host ${hostname} + service xray + + + + + name go_memstats_heap_reserved_bytes + desc GO Memory Statistics Heap Reserved in Bytes + type gauge + key _go_memstats_heap_reserved_bytes + + agent fluentd + host ${hostname} + service xray + + + + + name db_connection_pool_in_use_total + desc Total DB Connections in Use + type gauge + key db_connection_pool_in_use_total + + agent fluentd + host ${hostname} + service xray + + + + + name db_connection_pool_idle_total + desc Total DB Connections Idle + type gauge + key db_connection_pool_idle_total + + agent fluentd + host ${hostname} + service xray + + + + + name db_connection_pool_max_open_total + desc Max DB Connections Active + type gauge + key db_connection_pool_max_open_total + + agent fluentd + host ${hostname} + service xray + + + + + + # METRICS SEGMENT - END + + # LOG SEGMENT - BEGIN + + + @type tail + @id xray_server_tail + path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-server-service.log" + pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-server-service.log.pos" + tag jfrog.xray.server.service + + @type none + + + + @type tail + @id xray_persist_tail + path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-persist-service.log" + pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-persist-service.log.pos" + tag jfrog.xray.persist.service + + @type none + + + + @type tail + @id xray_indexer_tail + path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-indexer-service.log" + pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-indexer-service.log.pos" + tag jfrog.xray.indexer.service + + @type none + + + + @type tail + @id xray_analysis_tail + path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-analysis-service.log" + pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-analysis-service.log.pos" + tag jfrog.xray.analysis.service + + @type none + + + + @type tail + @id xray_router_tail + path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-service.log" + pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-service.log.pos" + tag jfrog.xray.router.service + + @type none + + + + @type tail + @id xray_router_traefik_tail + path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-traefik.log" + pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-traefik.log.pos" + tag jfrog.xray.router.traefik + + @type regexp + expression /^(?[^ ]*) \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?.*)\] \[(?.*)\] - (?.+)$/ + time_key timestamp + time_format %Y-%m-%dT%H:%M:%S.%LZ + + + + @type tail + @id xray_router_request_tail + path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-request.log" + pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-request.log.pos" + tag jfrog.xray.router.request + + @type json + time_key time + time_format %Y-%m-%dT%H:%M:%SZ + + + + @type tail + @id xray_request_tail + path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-request.log" + pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-request.log.pos" + tag jfrog.xray.xray.request + + @type regexp + expression /^(?[^\|]*)\|(?[^\|]*)\|(?[^\|]++)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?.*)$/ + time_key timestamp + time_format %Y-%m-%dT%H:%M:%S.%LZ + + + # STRIP COLOR CODES FROM SERVICE LOGS + + @type record_transformer + enable_ruby true + + message ${record["message"].gsub(/\e\[([;\d]+)?m/, '')} + + + # FIELD EXTRACT SERVICE LOG + + @type parser + key_name message + + @type multiline + format_firstline /\d{4}-\d{1,2}-\d{1,2}/ + format1 /^(?[^ ]*) \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?.*)\] \[(?.*)\] (?.*)$/ + time_key timestamp + time_format %Y-%m-%dT%H:%M:%S.%LZ + + emit_invalid_record_to_error false + + + @type record_transformer + + log_source ${tag} + + + + # LOG SEGMENT - END + + + @type stdout + + + + @type stdout + flush_at_shutdown true + output_type json + From d190233afb7827ad812eace1c32a6dbd02b600b1 Mon Sep 17 00:00:00 2001 From: grig777 Date: Fri, 21 Feb 2025 10:31:07 -0500 Subject: [PATCH 21/22] Delete .github/workflows/release_charts.yml --- .github/workflows/release_charts.yml | 44 ---------------------------- 1 file changed, 44 deletions(-) delete mode 100644 .github/workflows/release_charts.yml diff --git a/.github/workflows/release_charts.yml b/.github/workflows/release_charts.yml deleted file mode 100644 index a796829be..000000000 --- a/.github/workflows/release_charts.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Publish Helm Charts to GHCR - -on: workflow_dispatch -# push: -# branches: -# - master -# paths: -# - 'stable/**' - -jobs: - package-and-push: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Set up Helm - uses: azure/setup-helm@v3 - with: - version: v3.16.2 - - - name: Authenticate with GitHub Container Registry - env: - CR_PAT: ${{ secrets.GITHUB_TOKEN }} - run: | - echo ${{ secrets.GITHUB_TOKEN }} | helm registry login ghcr.io --username ${{ github.actor }} --password-stdin - - - name: Package and Push Helm Charts - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - CHARTS=("stable/jfrog-platform") - for chart_dir in ${CHARTS[@]}; do - if [ -d "$chart_dir" ]; then - # Update dependencies - helm dependency update "$chart_dir" - # Package the Helm chart - helm package "$chart_dir" --destination packaged_charts - # Push the packaged chart to GitHub Container Registry - CHART_FILE=$(ls packaged_charts/*.tgz | tail -n 1) - helm push "$CHART_FILE" "oci://ghcr.io/${{ github.repository_owner }}/charts" - fi - done \ No newline at end of file From 0e13eecb5c83b854731f24c2a405902f260e6258 Mon Sep 17 00:00:00 2001 From: grig777 Date: Fri, 21 Feb 2025 10:31:48 -0500 Subject: [PATCH 22/22] Update Chart.yaml --- stable/jfrog-platform/Chart.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/stable/jfrog-platform/Chart.yaml b/stable/jfrog-platform/Chart.yaml index 5ba48799d..355798573 100644 --- a/stable/jfrog-platform/Chart.yaml +++ b/stable/jfrog-platform/Chart.yaml @@ -48,6 +48,5 @@ maintainers: name: jfrog-platform sources: - https://github.com/jfrog/charts -- https://github.com/fastly/jfrog-helm-charts type: application version: 11.0.1