diff --git a/Dockerfile b/Dockerfile index d394a9d..bea201a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,6 +57,9 @@ ENV AWS_REGION=${AWS_REGION:-us-west-2} ENV AWS_ACCESS_KEY=${AWS_ACCESS_KEY:-YOUR_AWS_ACCESS_KEY} ENV AWS_SECRET_KEY_ID=${AWS_SECRET_KEY_ID:-YOUR_AWS_SECRET_KEY_ID} ENV AWS_BUCKET_NAME=${AWS_BUCKET_NAME:-devopscorner-bedrock} +ENV AMAZON_BEDROCK_AGENT_ID=${AMAZON_BEDROCK_AGENT_ID:-YOUR_AMAZON_BEDROCK_AGENT_ID} +ENV AMAZON_BEDROCK_MODEL_ID=${AMAZON_BEDROCK_MODEL_ID:-anthropic.claude-3-haiku-20240307-v1:0} +ENV AMAZON_BEDROCK_VERSION=${AMAZON_BEDROCK_VERSION:-bedrock-2023-05-31} # OpenSearch ENV OPENSEARCH_ENDPOINT=${OPENSEARCH_ENDPOINT:-https://opensearch.us-west-2.es.amazonaws.com} @@ -92,7 +95,7 @@ ENV OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES} ENV OTEL_TIME_INTERVAL=${OTEL_TIME_INTERVAL:-1} ENV OTEL_RANDOM_TIME_ALIVE_INCREMENTER=${OTEL_RANDOM_TIME_ALIVE_INCREMENTER:-1} ENV OTEL_RANDOM_TOTAL_HEAP_SIZE_UPPER_BOUND=${OTEL_RANDOM_TOTAL_HEAP_SIZE_UPPER_BOUND:-100} -ENV OTEL_RANDOM_THREAD_ACTIVE_UPPOR_BOUND=${OTEL_RANDOM_THREAD_ACTIVE_UPPOR_BOUND:-10} +ENV OTEL_RANDOM_THREAD_ACTIVE_UPPER_BOUND=${OTEL_RANDOM_THREAD_ACTIVE_UPPER_BOUND:-10} ENV OTEL_RANDOM_CPU_USAGE_UPPER_BOUND=${OTEL_RANDOM_CPU_USAGE_UPPER_BOUND:-100} # X-Ray @@ -156,8 +159,8 @@ COPY --from=devopscorner/k8s-context:latest /usr/local/bin/k8s-context /usr/loca COPY --from=devopscorner/k8s-context:latest /usr/local/bin/kc /usr/local/bin/kc COPY src /go/src -COPY _infra /go/src COPY .aws /go/src +COPY .codecatalyst /go/src COPY src/.env.example /go/.env COPY entrypoint.sh /go/entrypoint diff --git a/README.md b/README.md index f8eaba9..588fc30 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ ObservabilityAI: Memanfaatkan Amazon Bedrock Untuk Memantau Kinerja RESTful API ├── go.mod ├── go.sum ├── golang-bedrock.db +├── .env.example ├── main.go ├── middleware │ └── auth_middleware.go @@ -67,6 +68,7 @@ ObservabilityAI: Memanfaatkan Amazon Bedrock Untuk Memantau Kinerja RESTful API │ └── main_routes.go ├── utility │ ├── bedrock.go +│ ├── genid.go │ ├── loki.go │ ├── otel.go │ ├── prometheus.go @@ -76,7 +78,7 @@ ObservabilityAI: Memanfaatkan Amazon Bedrock Untuk Memantau Kinerja RESTful API ├── file_view.go └── login_view.go -11 directories, 28 files +11 directories, 30 files ``` ## Coverages: diff --git a/dashboard/xignals-logs-dashboard.json b/dashboard/xignals-logs-dashboard.json new file mode 100644 index 0000000..4553eba --- /dev/null +++ b/dashboard/xignals-logs-dashboard.json @@ -0,0 +1,1797 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Xignals Logs (xignals-logs-*) Filter", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 5, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": ["opensearch", "xignals", "prometheus", "cluster"], + "targetBlank": true, + "title": "Xignals Cluster", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 8, + "panels": [], + "title": "Summary", + "type": "row" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "description": "This panel displays the total number of ingestions over time.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "displayName": "Total", + "links": [ + { + "title": "Status-ALL", + "url": "https://console.awscb.id/d/xignals-log/nonprod-xignals-log?orgId=1&refresh=1m&var-Query=&var-datasource=adqd3v1qncnb4c" + } + ], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#5e5eaa", + "value": null + }, + { + "color": "#EAB839", + "value": 100000 + }, + { + "color": "red", + "value": 1000000 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 13, + "links": [ + { + "title": "Status-ALL", + "url": "https://console.awscb.id/d/xignals-log/nonprod-xignals-log?orgId=1&refresh=1m&var-Query=&var-datasource=adqd3v1qncnb4c" + } + ], + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["sum"], + "fields": "", + "values": false + }, + "showPercentChange": true, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "alias": "FILTERS", + "bucketAggs": [ + { + "field": "${Filters}", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "format": "table", + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "${Query}", + "queryType": "lucene", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Total Ingestions @ Time", + "type": "stat" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "description": "This panel displays the total number of ingestions over time for the \"INFO\" status.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "decimals": 0, + "fieldMinMax": false, + "links": [ + { + "title": "Status-INFO", + "url": "https://console.awscb.id/d/xignals-log/nonprod-xignals-log?orgId=1&refresh=1m&var-Query=info&var-datasource=adqd3v1qncnb4c" + } + ], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 10, + "links": [ + { + "title": "Status-INFO", + "url": "https://console.awscb.id/d/xignals-log/nonprod-xignals-log?orgId=1&refresh=1m&var-Query=info&var-datasource=adqd3v1qncnb4c" + } + ], + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["sum"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "alias": "INFO", + "bucketAggs": [ + { + "field": "${Filters}", + "id": "1", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "format": "table", + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "info", + "queryType": "lucene", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Status INFO @ Time", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "number", + "targetField": "INFO" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "description": "This panel displays the total number of ingestions over time for the \"ERROR\" status.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "fixed" + }, + "decimals": 0, + "fieldMinMax": false, + "links": [ + { + "title": "Status-ERROR", + "url": "https://console.awscb.id/d/xignals-log/nonprod-xignals-log?orgId=1&refresh=1m&var-Query=error&var-datasource=adqd3v1qncnb4c" + } + ], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 11, + "links": [ + { + "title": "Status-ERROR", + "url": "https://console.awscb.id/d/xignals-log/nonprod-xignals-log?orgId=1&refresh=1m&var-Query=error&var-datasource=adqd3v1qncnb4c" + } + ], + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["sum"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "alias": "ERROR", + "bucketAggs": [ + { + "field": "${Filters}", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "format": "table", + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "error", + "queryType": "lucene", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Status ERROR @ Time", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "number", + "targetField": "ERROR" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "description": "This panel displays the total number of ingestions over time for the \"DEBUG\" status.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "dark-blue", + "mode": "fixed" + }, + "decimals": 0, + "fieldMinMax": false, + "links": [ + { + "title": "Status-DEBUG", + "url": "https://console.awscb.id/d/xignals-log/nonprod-xignals-log?orgId=1&refresh=1m&var-Query=&var-datasource=adqd3v1qncnb4c&var-Filters=log%7C%21~%7Cinfo&var-Filters=log%7C%21~%7Cerror" + } + ], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-blue", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 37, + "links": [ + { + "title": "Status-DEBUG", + "url": "https://console.awscb.id/d/xignals-log/nonprod-xignals-log?orgId=1&refresh=1m&var-Query=&var-datasource=adqd3v1qncnb4c&var-Filters=log%7C%21~%7Cinfo&var-Filters=log%7C%21~%7Cerror" + } + ], + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["sum"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "alias": "DEBUG", + "bucketAggs": [ + { + "field": "${Filters}", + "id": "1", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "!(info) OR !(error) OR !(warning) OR !(critical) OR !(trace)", + "queryType": "lucene", + "refId": "C", + "timeField": "@timestamp" + } + ], + "title": "Status DEBUG @ Time", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "number", + "targetField": "DEBUG" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "description": "This panel displays the total number of ingestions over time for the \"WARNING\" status.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "fixed" + }, + "decimals": 0, + "fieldMinMax": false, + "links": [ + { + "title": "Status-INFO", + "url": "https://console.awscb.id/d/xignals-log/nonprod-xignals-log?orgId=1&refresh=1m&var-Query=warning&var-datasource=adqd3v1qncnb4c" + } + ], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 5 + }, + "id": 31, + "links": [ + { + "title": "Status-INFO", + "url": "https://console.awscb.id/d/xignals-log/nonprod-xignals-log?orgId=1&refresh=1m&var-Query=warning&var-datasource=adqd3v1qncnb4c" + } + ], + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["sum"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "alias": "WARNING", + "bucketAggs": [ + { + "field": "${Filters}", + "id": "1", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "format": "table", + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "warning", + "queryType": "lucene", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Status WARNING @ Time", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "number", + "targetField": "INFO" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "description": "This panel displays the total number of ingestions over time for the \"CRITICAL\" status.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "decimals": 0, + "fieldMinMax": false, + "links": [ + { + "title": "Status-INFO", + "url": "https://console.awscb.id/d/xignals-log/nonprod-xignals-log?orgId=1&refresh=1m&var-Query=critical&var-datasource=adqd3v1qncnb4c" + } + ], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 5 + }, + "id": 30, + "links": [ + { + "title": "Status-INFO", + "url": "https://console.awscb.id/d/xignals-log/nonprod-xignals-log?orgId=1&refresh=1m&var-Query=critical&var-datasource=adqd3v1qncnb4c" + } + ], + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["sum"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "alias": "CRITICAL", + "bucketAggs": [ + { + "field": "${Filters}", + "id": "1", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "format": "table", + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "critical", + "queryType": "lucene", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Status CRITICAL @ Time", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "number", + "targetField": "INFO" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "description": "This panel displays the total number of ingestions over time for the \"TRACE\" status.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "super-light-blue", + "mode": "fixed" + }, + "decimals": 0, + "fieldMinMax": false, + "links": [ + { + "title": "Status-INFO", + "url": "https://console.awscb.id/d/xignals-log/nonprod-xignals-log?orgId=1&refresh=1m&var-Query=trace&var-datasource=adqd3v1qncnb4c" + } + ], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "super-light-blue", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 5 + }, + "id": 32, + "links": [ + { + "title": "Status-INFO", + "url": "https://console.awscb.id/d/xignals-log/nonprod-xignals-log?orgId=1&refresh=1m&var-Query=trace&var-datasource=adqd3v1qncnb4c" + } + ], + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["sum"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "alias": "TRACE", + "bucketAggs": [ + { + "field": "${Filters}", + "id": "1", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "format": "table", + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "trace", + "queryType": "lucene", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Status TRACE @ Time", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "number", + "targetField": "INFO" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 7, + "panels": [], + "title": "Stream Logs", + "type": "row" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "description": "This panel titled \"Panel Title\" displays a table format of counts over time. It uses the \"Xignals Logs\" datasource and is part of the \"Xignals EMI\" dashboard.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 100000 + }, + { + "color": "red", + "value": 1000000 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 10 + }, + "id": 33, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "timezone": ["browser"], + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "alias": "FILTERS", + "bucketAggs": [ + { + "field": "${Filters}", + "id": "1", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "format": "table", + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "${Query}", + "queryType": "lucene", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Ingestion Logs - Graph", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "description": "This panel displays the count of INFO, ERROR, and DEBUG log messages over time for non-production environments.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "gauge" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "applyToRow": true, + "mode": "gradient", + "type": "color-background" + } + }, + { + "id": "color", + "value": { + "fixedColor": "transparent", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "INFO" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ERROR" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "DEBUG" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "WARNING" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "CRITICAL" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "TRACE" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "super-light-blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 34, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": false, + "fields": "", + "reducer": ["sum"], + "show": true + }, + "frameIndex": 0, + "showHeader": true + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "alias": "INFO", + "bucketAggs": [ + { + "field": "${Filters}", + "id": "1", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "format": "table", + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "info", + "queryType": "lucene", + "refId": "A", + "timeField": "@timestamp" + }, + { + "alias": "ERROR", + "bucketAggs": [ + { + "field": "${Filters}", + "id": "1", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "error", + "queryType": "lucene", + "refId": "B", + "timeField": "@timestamp" + }, + { + "alias": "DEBUG", + "bucketAggs": [ + { + "field": "${Filters}", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "!(info) OR !(error) OR !(warning) OR !(critical) OR !(trace)", + "queryType": "lucene", + "refId": "D", + "timeField": "@timestamp" + }, + { + "alias": "WARNING", + "bucketAggs": [ + { + "field": "${Filters}", + "id": "1", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "warning", + "queryType": "lucene", + "refId": "C", + "timeField": "@timestamp" + }, + { + "alias": "CRITICAL", + "bucketAggs": [ + { + "field": "${Filters}", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "critical", + "queryType": "lucene", + "refId": "E", + "timeField": "@timestamp" + }, + { + "alias": "TRACE", + "bucketAggs": [ + { + "field": "${Filters}", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "trace", + "queryType": "lucene", + "refId": "F", + "timeField": "@timestamp" + } + ], + "title": "Ingestion Logs - Hits", + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 14, + "panels": [], + "title": "Stream Logs", + "type": "row" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "description": "Displays a detailed log index segmented by information, error, and debug levels with time-based histograms.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "json-view" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "@timestamp" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "json-view" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 35, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "@timestamp" + } + ] + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "alias": "INFO", + "bucketAggs": [ + { + "field": "@timestamp", + "id": "3", + "settings": { + "min_doc_count": "0", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "${Filters}", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "format": "table", + "metrics": [ + { + "id": "1", + "type": "logs" + } + ], + "query": "${Query}", + "queryType": "lucene", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Logs Index", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "byVariable": false, + "include": { + "names": ["@timestamp", "traceId"] + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "description": "Displays detailed logs with timestamps to monitor and assess system activities within the non-production environment.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "transparent", + "mode": "shades" + }, + "custom": { + "align": "auto", + "cellOptions": { + "applyToRow": false, + "type": "color-background" + }, + "filterable": true, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "@timestamp" + }, + "properties": [ + { + "id": "custom.width", + "value": 226 + }, + { + "id": "unit", + "value": "" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "_source" + }, + "properties": [ + { + "id": "custom.width" + }, + { + "id": "custom.align", + "value": "left" + }, + { + "id": "custom.inspect", + "value": true + }, + { + "id": "custom.cellOptions", + "value": { + "type": "json-view" + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "pattern": "\\s+level=(info)", + "result": { + "color": "green", + "index": 0, + "text": "info" + } + }, + "type": "regex" + }, + { + "options": { + "pattern": "\\s+level=(warn)", + "result": { + "color": "orange", + "index": 1, + "text": "warn" + } + }, + "type": "regex" + }, + { + "options": { + "pattern": "\\s+level=(error)", + "result": { + "color": "red", + "index": 2, + "text": "error" + } + }, + "type": "regex" + }, + { + "options": { + "pattern": "\\s+level=(debug)", + "result": { + "color": "blue", + "index": 3, + "text": "debug" + } + }, + "type": "regex" + }, + { + "options": { + "pattern": "\\s+level=(critical)", + "result": { + "color": "purple", + "index": 4, + "text": "critical" + } + }, + "type": "regex" + }, + { + "options": { + "pattern": "\\s+level=(trace)", + "result": { + "color": "yellow", + "index": 5, + "text": "trace" + } + }, + "type": "regex" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 15, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 36, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": true, + "enablePagination": true, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "@timestamp", + "id": "2", + "settings": { + "min_doc_count": "0", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "${Filters}", + "id": "1", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "format": "table", + "metrics": [ + { + "id": "1", + "type": "logs" + } + ], + "query": "${Query}", + "queryType": "lucene", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Logs Detail", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": { + "@timestamp": true, + "_id": false, + "_index": false, + "_source": true, + "kubernetes.container_hash": false, + "log": true + }, + "indexByName": { + "@timestamp": 0, + "_id": 2, + "_index": 4, + "_p": 5, + "_source": 1, + "_type": 6, + "kubernetes.annotations.checksum/config": 7, + "kubernetes.annotations.checksum/sc-dashboard-provider-config": 8, + "kubernetes.annotations.checksum/secret": 9, + "kubernetes.annotations.cluster-autoscaler_kubernetes_io/safe-to-evict": 10, + "kubernetes.annotations.kubectl_kubernetes_io/default-container": 11, + "kubernetes.annotations.kubectl_kubernetes_io/restartedAt": 12, + "kubernetes.annotations.prometheus_io/port": 13, + "kubernetes.annotations.prometheus_io/scrape": 14, + "kubernetes.container_hash": 15, + "kubernetes.container_image": 16, + "kubernetes.container_name": 17, + "kubernetes.docker_id": 18, + "kubernetes.host": 19, + "kubernetes.labels.app": 20, + "kubernetes.labels.app_kubernetes_io/component": 21, + "kubernetes.labels.app_kubernetes_io/instance": 22, + "kubernetes.labels.app_kubernetes_io/managed-by": 23, + "kubernetes.labels.app_kubernetes_io/name": 24, + "kubernetes.labels.app_kubernetes_io/part-of": 25, + "kubernetes.labels.app_kubernetes_io/version": 26, + "kubernetes.labels.apps_kubernetes_io/pod-index": 27, + "kubernetes.labels.controller-revision-hash": 28, + "kubernetes.labels.helm_sh/chart": 29, + "kubernetes.labels.pod-template-hash": 30, + "kubernetes.labels.role": 31, + "kubernetes.labels.statefulset_kubernetes_io/pod-name": 32, + "kubernetes.namespace_name": 33, + "kubernetes.pod_id": 34, + "kubernetes.pod_name": 35, + "log": 3, + "stream": 36, + "time": 37 + }, + "renameByName": { + "@timestamp": "Time", + "_source": "_source", + "log": "Log" + } + } + } + ], + "type": "table" + } + ], + "refresh": "1m", + "schemaVersion": 39, + "tags": ["opensearch", "log", "xignals", "kubernetes", "prometheus", "alva"], + "templating": { + "list": [ + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${datasource}" + }, + "description": "", + "filters": [], + "hide": 0, + "name": "Filters", + "skipUrlSync": false, + "type": "adhoc" + }, + { + "current": { + "selected": true, + "text": "", + "value": "" + }, + "hide": 0, + "name": "Query", + "options": [ + { + "selected": true, + "text": "", + "value": "" + } + ], + "query": "", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": false, + "text": "OpenSearch", + "value": "P9744FCCEAAFBD98F" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "grafana-opensearch-datasource", + "queryValue": "", + "refresh": 1, + "regex": "^OpenSearch", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "[NONPROD] XIGNALS / LOG / XIGNALS-OTEL", + "uid": "xignals-log-otel", + "version": 6, + "weekStart": "" +} diff --git a/docker-compose.yaml b/docker-compose.yaml index a308693..6cc7caa 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -112,6 +112,9 @@ services: - AWS_ACCESS_KEY=${AWS_ACCESS_KEY:-YOUR_AWS_ACCESS_KEY} - AWS_SECRET_KEY_ID=${AWS_SECRET_KEY_ID:-YOUR_AWS_SECRET_KEY_ID} - AWS_BUCKET_NAME=${AWS_BUCKET_NAME:-devopscorner-bedrock} + - AMAZON_BEDROCK_AGENT_ID=${AMAZON_BEDROCK_AGENT_ID:-YOUR_AMAZON_BEDROCK_AGENT_ID} + - AMAZON_BEDROCK_MODEL_ID=${AMAZON_BEDROCK_MODEL_ID:-anthropic.claude-3-haiku-20240307-v1:0} + - AMAZON_BEDROCK_VERSION=${AMAZON_BEDROCK_VERSION:-bedrock-2023-05-31} # OpenSearch - OPENSEARCH_ENDPOINT=${OPENSEARCH_ENDPOINT:-https://opensearch.us-west-2.es.amazonaws.com} - OPENSEARCH_USERNAME=${OPENSEARCH_USERNAME:-devopscorner} @@ -140,7 +143,7 @@ services: - OTEL_TIME_INTERVAL=${OTEL_TIME_INTERVAL:-1} - OTEL_RANDOM_TIME_ALIVE_INCREMENTER=${OTEL_RANDOM_TIME_ALIVE_INCREMENTER:-1} - OTEL_RANDOM_TOTAL_HEAP_SIZE_UPPER_BOUND=${OTEL_RANDOM_TOTAL_HEAP_SIZE_UPPER_BOUND:-100} - - OTEL_RANDOM_THREAD_ACTIVE_UPPOR_BOUND=${OTEL_RANDOM_THREAD_ACTIVE_UPPOR_BOUND:-10} + - OTEL_RANDOM_THREAD_ACTIVE_UPPER_BOUND=${OTEL_RANDOM_THREAD_ACTIVE_UPPER_BOUND:-10} - OTEL_RANDOM_CPU_USAGE_UPPER_BOUND=${OTEL_RANDOM_CPU_USAGE_UPPER_BOUND:-100} # Jaeger - JAEGER_AGENT_PORT=${JAEGER_AGENT_PORT:-6831} diff --git a/manifest/helm-value-fluentbit.yaml b/manifest/helm-value-fluentbit.yaml index 4d2d04b..5848f73 100644 --- a/manifest/helm-value-fluentbit.yaml +++ b/manifest/helm-value-fluentbit.yaml @@ -173,7 +173,7 @@ config: Host opsearch-master.monitoring.svc Port 9200 HTTP_User admin - HTTP_Passwd aw5cb2024 + HTTP_Passwd s3cr3tpassw0rd Logstash_Format On Logstash_Prefix devopscorner-nonprod Retry_Limit Off diff --git a/manifest/helm-value-golang-bedrock.yaml b/manifest/helm-value-golang-bedrock.yaml new file mode 100644 index 0000000..589b11f --- /dev/null +++ b/manifest/helm-value-golang-bedrock.yaml @@ -0,0 +1,332 @@ +--- +replicaCount: 1 + +secret: + enabled: false + name: "secret-golang-bedrock" + mountPath: {} + subPath: {} + readOnly: true + data: {} + +configMap: + enabled: false + name: "config-golang-bedrock" + mountPath: /app/core/config + readOnly: true + data: + .app.config.json: |- + { + "AppName": "Golang Bedrock Service", + "GRPCTimeout": 10, + "CacheExpiry": 300, + "CacheCleanup": 600, + "DefaultPageLimit": 3, + "ClientTimeout": 10 + } + +image: + repository: devopscorner/golang-bedrock + pullPolicy: Always # IfNotPresent + tag: "alpine" + +imagePullSecrets: + - name: "devopspoc-docker-secret" + +nameOverride: "devopspoc-golang-bedrock" +fullnameOverride: "devopspoc-golang-bedrock" + +serviceAccount: + create: true + annotations: {} + name: devopspoc-golang-bedrock + namespace: devops-tools + +## Persist data to a persistent volume +persistence: + enabled: true + storageClass: "nfs-client-metrics" + accessMode: ReadWriteOnce + capacity: 500Mi + size: 50Mi + annotations: {} + selector: {} + nfs: + path: "/devopscorner/golang-bedrock" + server: "fs-0987612345.efs.us-west-2.amazonaws.com" + +service: + enabled: true + type: ClusterIP + ports: + - name: http + port: 80 + targetPort: 8080 + protocol: TCP + +containers: + ports: + - name: http + containerPort: 8080 + protocol: TCP + +ingress: + enabled: true + ingressName: devopspoc-golang-bedrocks-ingress + ingressClassName: nginx + annotations: + # cert-manager.io/acme-challenge-type: "http01" + # cert-manager.io/cluster-issuer: "devopspoc-nonprod" + ingress.kubernetes.io/ssl-passthrough: "true" + ingress.kubernetes.io/whitelist-source-range: "10.103.0.0/16" + kubernetes.io/ingress.allow-http: "false" + kubernetes.io/ingress.class: "nginx" + kubernetes.io/tls-acme: "true" + nginx.ingress.kubernetes.io/affinity: "cookie" + # nginx.ingress.kubernetes.io/auth-realm: "Authentication Required - Golang Bedrock" + # nginx.ingress.kubernetes.io/auth-secret: "golang-bedrock-auth" + # nginx.ingress.kubernetes.io/auth-type: "basic" + nginx.ingress.kubernetes.io/backend-protocol: "HTTP" + nginx.ingress.kubernetes.io/cors-allow-headers: "*" + nginx.ingress.kubernetes.io/cors-allow-methods: "*" + nginx.ingress.kubernetes.io/cors-allow-origin: "*" + nginx.ingress.kubernetes.io/enable-cors: "true" + nginx.ingress.kubernetes.io/from-to-www-redirect: "true" + nginx.ingress.kubernetes.io/proxy-connect-timeout: "600" + nginx.ingress.kubernetes.io/proxy-read-timeout: "600" + nginx.ingress.kubernetes.io/proxy-send-timeout: "600" + hosts: + - host: golang-bedrock.awscb.id + http: + paths: + - path: / + pathType: Prefix # Prefix -or - ImplementationSpecific + backend: + service: + name: devopspoc-golang-bedrock + port: + number: 80 + tls: + - secretName: devopspoc-nonprod-golang-bedrocks-tls + hosts: + - golang-bedrock.awscb.id + extraPaths: [] + +application: + enabled: true + env: + - name: HELM_TEMPLATE_NAME + value: "devopscorner-bedrock" + ## ALPINE ## + - name: ALPINE_VERSION + value: 3.18 + ## GIN RESTful ## + - name: GIN_MODE + value: "release" + - name: APP_URL + value: "http://0.0.0.0" + - name: APP_PORT + value: 8080 + ## DATABASE ## + - name: DB_CONNECTION + value: sqlite + - name: DB_HOST + value: "0.0.0.0" + - name: DB_PORT + value: 5000 + - name: DB_DATABASE + value: "golang-bedrock.db" + - name: DB_USERNAME + valueFrom: + secretKeyRef: + name: golang-bedrock + key: DB_USERNAME + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: golang-bedrock + key: DB_PASSWORD + ## JWT Token ## + - name: JWT_AUTH_USERNAME + valueFrom: + secretKeyRef: + name: golang-bedrock + key: JWT_AUTH_USERNAME + - name: JWT_AUTH_PASSWORD + valueFrom: + secretKeyRef: + name: golang-bedrock + key: JWT_AUTH_PASSWORD + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: golang-bedrock + key: JWT_SECRET + ## LOGGER ## + - name: LOG_LEVEL + value: INFO + ## AWS Credentials ## + - name: AWS_REGION + value: us-west-2 + - name: AWS_ACCESS_KEY + valueFrom: + secretKeyRef: + name: golang-bedrock + key: AWS_ACCESS_KEY + - name: AWS_SECRET_KEY_ID + valueFrom: + secretKeyRef: + name: golang-bedrock + key: AWS_SECRET_KEY_ID + - name: AWS_BUCKET_NAME + value: devopscorner-bedrock + - name: AMAZON_BEDROCK_AGENT_ID + valueFrom: + secretKeyRef: + name: golang-bedrock + key: AMAZON_BEDROCK_AGENT_ID + - name: AMAZON_BEDROCK_MODEL_ID + value: "anthropic.claude-3-sonnet-20240229-v1:0" + - name: AMAZON_BEDROCK_VERSION + value: "bedrock-2023-05-31" + ## OPENSEARCH ## + - name: OPENSEARCH_ENDPOINT + value: "https://opensearch.us-west-2.es.amazonaws.com" + - name: OPENSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: golang-bedrock + key: OPENSEARCH_USERNAME + - name: OPENSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: golang-bedrock + key: OPENSEARCH_PASSWORD + ## PROMETHEUS ## + - name: PROMETHEUS_ENDPOINT + value: "http://xignals-prometheus.monitoring.svc:9090" + - name: PROMETHEUS_PORT + value: 9090 + ## LOKI ## + - name: LOKI_ENDPOINT + value: "http://xignals-loki.monitoring.svc:3100" + - name: LOKI_PORT + value: 3100 + ## GRAFANA ## + - name: GRAFANA_ENDPOINT + value: "http://xignals-grafana.monitoring.svc:3000" + - name: GRAFANA_PORT + value: 3000 + - name: GRAFANA_API_KEY + valueFrom: + secretKeyRef: + name: golang-bedrock + key: GRAFANA_API_KEY + ## OPENTELEMETRY ## + - name: OTEL_INSTRUMENTATION_METRIC_ENABLED + value: true + - name: OTEL_INSTRUMENTATION_TRACE_ENABLED + value: true + - name: OTEL_INSTRUMENTATION_LOG_ENABLED + value: true + - name: OTEL_INSTRUMENTATION_TRACE_NAME + value: "jaeger" + - name: OTEL_ENVIRONMENT + value: "nonprod" + - name: OTEL_SERVICE_NAME + value: "golang-bedrock" + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://xignals-poc-otelcol.monitoring.svc:4317" + - name: OTEL_EXPORTER_OTLP_INSECURE + value: true + - name: OTEL_EXPORTER_OTLP_HEADERS + value: "" + - name: OTEL_RESOURCE_ATTRIBUTES + value: "" + - name: OTEL_TIME_INTERVAL + value: 1 + - name: OTEL_RANDOM_TIME_ALIVE_INCREMENTER + value: 1 + - name: OTEL_RANDOM_TOTAL_HEAP_SIZE_UPPER_BOUND + value: 100 + - name: OTEL_RANDOM_THREAD_ACTIVE_UPPER_BOUND + value: 10 + - name: OTEL_RANDOM_CPU_USAGE_UPPER_BOUND + value: 100 + ## JAEGER ## + - name: JAEGER_AGENT_PORT + value: 6831 + - name: JAEGER_SAMPLER_TYPE + value: "const" + - name: JAEGER_SAMPLER_PARAM + value: 1 + - name: JAEGER_SAMPLER_MANAGER_HOST_PORT + value: "http://xignals-poc-jaeger-agent.monitoring.svc:5778" + - name: JAEGER_REPORTER_LOG_SPANS + value: true + - name: JAEGER_REPORTER_BUFFER_FLUSH_INTERVAL + value: "5*time.Second" + - name: JAEGER_REPORTER_MAX_QUEUE_SIZE + value: 100 + - name: JAEGER_REPORTER_LOCAL_AGENT_HOST_PORT + value: "http://xignals-poc-jaeger-agent.monitoring.svc:6831" + - name: JAEGER_REPORTER_COLLECTOR_ENDPOINT + value: "http://xignals-poc-jaeger-collector.monitoring.svc:14268/api/traces" + - name: JAEGER_REPORTER_COLLECTOR_USER=${JAEGER_REPORTER_COLLECTOR_USER:-devopscorner} + valueFrom: + secretKeyRef: + name: golang-bedrock + key: JAEGER_REPORTER_COLLECTOR_USER + - name: JAEGER_REPORTER_COLLECTOR_PASSWORD + valueFrom: + secretKeyRef: + name: golang-bedrock + key: JAEGER_REPORTER_COLLECTOR_PASSWORD + - name: JAEGER_TAGS + value: "golang,otel,restful,api,bedrock" + ## XRAY ## + - name: XRAY_VERSION + value: latest + - name: XRAY_DAEMON_ENDPOINT + value: "https://xray.us-west-2.amazonaws.com" + - name: XRAY_DAEMON_PORT + value: 2000 + +envFrom: + enabled: false + +resources: + limits: + cpu: 300m + memory: 300Mi + requests: + cpu: 150m + memory: 150Mi + +autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 5 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +nodeSelector: + enabled: false + select: + node: "devopspoc-monitoring" # DEV/UAT Cluster + +tolerations: [] + +podAnnotations: {} + +podSecurityContext: + {} + # fsGroup: 2000 + +livenessProbe: {} + +readinessProbe: {} + +affinity: {} + +securityContext: {} diff --git a/manifest/manifest-metrics-server.yaml b/manifest/manifest-metrics-server.yaml new file mode 100644 index 0000000..4e01766 --- /dev/null +++ b/manifest/manifest-metrics-server.yaml @@ -0,0 +1,197 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-app: metrics-server + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + name: system:aggregated-metrics-reader +rules: +- apiGroups: + - metrics.k8s.io + resources: + - pods + - nodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server +rules: +- apiGroups: + - "" + resources: + - pods + - nodes + - nodes/stats + - namespaces + - configmaps + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + k8s-app: metrics-server + name: metrics-server-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-app: metrics-server + name: metrics-server:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: https + selector: + k8s-app: metrics-server +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: metrics-server + strategy: + rollingUpdate: + maxUnavailable: 0 + template: + metadata: + labels: + k8s-app: metrics-server + spec: + containers: + - args: + - --cert-dir=/tmp + - --secure-port=4443 + ## Use unsecure for private / internal metrics + - --kubelet-insecure-tls + ## Use wide preferred address + #- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + ## Use only for InternalIP / External IP + - --kubelet-preferred-address-types=InternalIP,ExternalIP + - --kubelet-use-node-status-port + - --metric-resolution=15s + image: k8s.gcr.io/metrics-server/metrics-server:v0.5.2 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /livez + port: https + scheme: HTTPS + periodSeconds: 10 + name: metrics-server + ports: + - containerPort: 4443 + name: https + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: https + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 10 + resources: + requests: + cpu: 100m + memory: 200Mi + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + volumeMounts: + - mountPath: /tmp + name: tmp-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: metrics-server + volumes: + - emptyDir: {} + name: tmp-dir +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + labels: + k8s-app: metrics-server + name: v1beta1.metrics.k8s.io +spec: + group: metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: metrics-server + namespace: kube-system + version: v1beta1 + versionPriority: 100 diff --git a/manifest/manifest-storage-class.yaml b/manifest/manifest-storage-class.yaml new file mode 100644 index 0000000..c5a7ba5 --- /dev/null +++ b/manifest/manifest-storage-class.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + annotations: + storageclass.beta.kubernetes.io/is-default-class: "true" + name: gp2 +provisioner: kubernetes.io/aws-ebs +reclaimPolicy: Delete +allowVolumeExpansion: true +volumeBindingMode: WaitForFirstConsumer +#volumeBindingMode: Immediate +parameters: + type: gp2 + fsType: ext4 +mountOptions: + - debug + +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + annotations: + storageclass.beta.kubernetes.io/is-default-class: "false" + name: gp3 +provisioner: kubernetes.io/aws-ebs +reclaimPolicy: Retain +allowVolumeExpansion: true +volumeBindingMode: WaitForFirstConsumer +#volumeBindingMode: Immediate +parameters: + type: gp3 + fsType: ext4 +mountOptions: + - debug diff --git a/manifest/secret/manifest-secret-devopspoc-auth.yaml b/manifest/secret/manifest-secret-devopspoc-auth.yaml new file mode 100644 index 0000000..a0632b6 --- /dev/null +++ b/manifest/secret/manifest-secret-devopspoc-auth.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Secret +metadata: + name: devopspoc-auth + namespace: monitoring +type: Opaque +data: + # https://bcrypt-generator.com/ + auth: "" + username: "" + password: "" diff --git a/manifest/secret/manifest-secret-golang-bedrock.yaml b/manifest/secret/manifest-secret-golang-bedrock.yaml new file mode 100644 index 0000000..7860023 --- /dev/null +++ b/manifest/secret/manifest-secret-golang-bedrock.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Secret +metadata: + name: golang-bedrock + namespace: monitoring +type: Opaque +data: + DB_USERNAME: "" + DB_PASSWORD: "" + JWT_AUTH_USERNAME: "" + JWT_AUTH_PASSWORD: "" + AWS_ACCESS_KEY: "" + AWS_SECRET_KEY_ID: "" + AMAZON_BEDROCK_AGENT_ID: "" + OPENSEARCH_USERNAME: "" + OPENSEARCH_PASSWORD: "" + GRAFANA_API_KEY: "" + JAEGER_REPORTER_COLLECTOR_USER: "" + JAEGER_REPORTER_COLLECTOR_PASSWORD: "" diff --git a/manifest/secret/manifest-secret-postgresql.yaml b/manifest/secret/manifest-secret-postgresql.yaml new file mode 100644 index 0000000..02f41cc --- /dev/null +++ b/manifest/secret/manifest-secret-postgresql.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-secret + namespace: monitoring + labels: + app.kubernetes.io/instance: postgresql + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: postgresql + app.kubernetes.io/version: 16.3.0 + helm.sh/chart: postgresql-15.5.16 + annotations: + meta.helm.sh/release-name: postgresql + meta.helm.sh/release-namespace: monitoring +type: Opaque +data: + postgres-user: "" + POSTGRES_USERNAME: "" + postgres-password: "" + postgres-root-password: "" + POSTGRES_PASSWORD: "" + POSTGRES_ROOT_PASSWORD: "" \ No newline at end of file diff --git a/manifest/secret/manifest-secret-xignals-auth.yaml b/manifest/secret/manifest-secret-xignals-auth.yaml new file mode 100644 index 0000000..e990936 --- /dev/null +++ b/manifest/secret/manifest-secret-xignals-auth.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Secret +metadata: + name: xignals-auth + namespace: monitoring +type: Opaque +data: + # https://bcrypt-generator.com/ + auth: "" + username: "" + password: "" \ No newline at end of file diff --git a/postman/RESTful API - Golang Bedrock SQLite ORM.postman_collection.json b/postman/RESTful API - Golang Bedrock SQLite ORM.postman_collection.json index 19407fa..2d1337e 100644 --- a/postman/RESTful API - Golang Bedrock SQLite ORM.postman_collection.json +++ b/postman/RESTful API - Golang Bedrock SQLite ORM.postman_collection.json @@ -265,7 +265,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"username\": \"devopscorner\",\n \"password\": \"DevOpsCorner2024\"\n}" + "raw": "{\n \"username\": \"devopscorner-user\",\n \"password\": \"DevOpsCorner2024\"\n}" }, "url": { "raw": "{{api_url}}:8080/login", @@ -313,7 +313,7 @@ "response": [] }, { - "name": "Find Uploaded File Id", + "name": "Find Uploaded File Id 1", "request": { "method": "GET", "header": [ @@ -329,7 +329,7 @@ } ], "url": { - "raw": "{{api_url}}:8080/v1/files/1724454386171356000", + "raw": "{{api_url}}:8080/v1/files/1725417360394229000", "host": [ "{{api_url}}" ], @@ -337,7 +337,71 @@ "path": [ "v1", "files", - "1724454386171356000" + "1725417360394229000" + ] + }, + "description": "localhost:8080/books" + }, + "response": [] + }, + { + "name": "Find Uploaded File Id 2", + "request": { + "method": "GET", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + }, + { + "key": "Authorization", + "value": "Bearer {{auth_token}}", + "type": "text" + } + ], + "url": { + "raw": "{{api_url}}:8080/v1/files/1725383792358030000", + "host": [ + "{{api_url}}" + ], + "port": "8080", + "path": [ + "v1", + "files", + "1725383792358030000" + ] + }, + "description": "localhost:8080/books" + }, + "response": [] + }, + { + "name": "Find Uploaded File Id 3", + "request": { + "method": "GET", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + }, + { + "key": "Authorization", + "value": "Bearer {{auth_token}}", + "type": "text" + } + ], + "url": { + "raw": "{{api_url}}:8080/v1/files/1725417214616798000", + "host": [ + "{{api_url}}" + ], + "port": "8080", + "path": [ + "v1", + "files", + "1725417214616798000" ] }, "description": "localhost:8080/books" @@ -361,8 +425,24 @@ } ], "body": { - "mode": "raw", - "raw": "{\n \"FileName\": \"document1.pdf\",\n \"FileSize\": \"1024 * 1024\",\n \"FileType\": \"application/pdf\",\n \"UploadedBy\": \"user1@example.com\"\n}" + "mode": "formdata", + "formdata": [ + { + "key": "FileType", + "value": "application/pdf", + "type": "text" + }, + { + "key": "UploadedBy", + "value": "user1@example.com", + "type": "text" + }, + { + "key": "file", + "type": "file", + "src": "/Users/dfdenni/Desktop/Learning-DevOps.pdf" + } + ] }, "url": { "raw": "{{api_url}}:8080/v1/files", @@ -377,7 +457,76 @@ }, "description": "localhost:8080/books" }, - "response": [] + "response": [ + { + "name": "Example Result File 1", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + }, + { + "key": "Authorization", + "value": "Bearer {{auth_token}}", + "type": "text" + } + ], + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "FileType", + "value": "application/pdf", + "type": "text" + }, + { + "key": "UploadedBy", + "value": "user1@example.com", + "type": "text" + }, + { + "key": "file", + "type": "file", + "src": "/Users/dfdenni/Desktop/Learning-DevOps.pdf" + } + ] + }, + "url": { + "raw": "{{api_url}}:8080/v1/files", + "host": [ + "{{api_url}}" + ], + "port": "8080", + "path": [ + "v1", + "files" + ] + } + }, + "status": "Created", + "code": 201, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Date", + "value": "Wed, 04 Sep 2024 02:26:53 GMT" + }, + { + "key": "Transfer-Encoding", + "value": "chunked" + } + ], + "cookie": [], + "body": "{\n \"data\": {\n \"analysis\": \"Based on the information provided, here's an analysis of the file upload:\\n\\n1. **Filename**: `1725416786086072000_Learning-DevOps.pdf`\\n - The filename suggests that it is a PDF file related to learning DevOps.\\n - The filename contains a long numerical sequence (`1725416786086072000`) before the descriptive part (`Learning-DevOps.pdf`). This numerical sequence could be a unique identifier, timestamp, or some other form of file naming convention.\\n\\n2. **File Size**: `19141577 bytes`\\n - The file size is approximately 19.14 MB (megabytes).\\n - For a PDF file, this size is relatively large, indicating that the file may contain a substantial amount of content, such as text, images, or other embedded media.\\n\\n3. **File Type**: `application/pdf`\\n - The file type is a Portable Document Format (PDF), which is a widely used format for sharing documents across different platforms and devices.\\n - PDF files can contain text, images, multimedia, and interactive elements, making them suitable for various purposes, including educational materials, manuals, reports, and presentations.\\n\\nBased on the filename and file type, it appears that this file is likely a PDF document related to learning DevOps practices. The relatively large file size suggests that it may contain extensive content, potentially including text, diagrams, or other multimedia elements.\\n\\nPDF files are generally considered safe for sharing and viewing, as long as they are obtained from trusted sources and opened with updated PDF viewers or readers. However, it's always a good practice to scan files for potential security risks before opening them, especially if they come from untrusted sources.\",\n \"createdAt\": \"2024-09-04T09:26:43.750343+07:00\",\n \"fileName\": \"1725416786086072000_Learning-DevOps.pdf\",\n \"fileSize\": 19141577,\n \"fileType\": \"application/pdf\",\n \"fileURL\": \"https://example.s3.amazonaws.com/1725416786086072000_Learning-DevOps.pdf\",\n \"id\": \"1725416786086072000\",\n \"metrics\": {\n \"analysisLatency\": \"9.313159125s\",\n \"inputTokens\": 30,\n \"outputTokens\": 419,\n \"totalLatency\": \"9.314087583s\",\n \"uploadLatency\": \"610.458µs\"\n },\n \"updatedAt\": \"2024-09-04T09:26:53.071124+07:00\",\n \"uploadedBy\": \"user1@example.com\"\n }\n}" + } + ] }, { "name": "Upload File 2", @@ -396,8 +545,24 @@ } ], "body": { - "mode": "raw", - "raw": "{\n \"FileName\": \"image1.jpg\",\n \"FileSize\": \"512 * 1024\",\n \"FileType\": \"image/jpeg\",\n \"UploadedBy\": \"user2@example.com\"\n}" + "mode": "formdata", + "formdata": [ + { + "key": "FileType", + "value": "application/pdf", + "type": "text" + }, + { + "key": "UploadedBy", + "value": "user1@example.com", + "type": "text" + }, + { + "key": "file", + "type": "file", + "src": "/Users/dfdenni/Desktop/Learning-DevOps.pdf" + } + ] }, "url": { "raw": "{{api_url}}:8080/v1/files", @@ -431,8 +596,24 @@ } ], "body": { - "mode": "raw", - "raw": "{\n \"FileName\": \"spreadsheet1.xlsx\",\n \"FileSize\": \"2048 * 1024\",\n \"FileType\": \"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet\",\n \"UploadedBy\": \"user3@example.com\"\n}" + "mode": "formdata", + "formdata": [ + { + "key": "FileType", + "value": "application/pdf", + "type": "text" + }, + { + "key": "UploadedBy", + "value": "user1@example.com", + "type": "text" + }, + { + "key": "file", + "type": "file", + "src": "/Users/dfdenni/Desktop/Learning-DevOps.pdf" + } + ] }, "url": { "raw": "{{api_url}}:8080/v1/files", @@ -467,10 +648,10 @@ ], "body": { "mode": "raw", - "raw": "{\n \"FileName\": \"document2.pdf\",\n \"FileSize\": \"2048 * 2048\",\n \"FileType\": \"application/pdf\",\n \"UploadedBy\": \"user1@example.com\"\n}" + "raw": "{\n \"fileName\": \"1725386092471912000_Learning-DevOps.pdf\",\n \"fileSize\": 19141577,\n \"fileType\": \"application/pdf\",\n \"fileURL\": \"https://example.s3.amazonaws.com/1725386092471912000_Learning-DevOps.pdf\",\n \"uploadedBy\": \"user2@example.com\",\n \"analysis\": \"Analysis failed: Bedrock model not found. Please check configuration.\"\n}" }, "url": { - "raw": "{{api_url}}:8080/v1/files/1724454386171356000", + "raw": "{{api_url}}:8080/v1/files/1725383792358030000", "host": [ "{{api_url}}" ], @@ -478,7 +659,7 @@ "path": [ "v1", "files", - "1724454386171356000" + "1725383792358030000" ] }, "description": "localhost:8080/books/3" @@ -502,7 +683,7 @@ } ], "url": { - "raw": "{{api_url}}:8080/v1/files/1724454386171356000", + "raw": "{{api_url}}:8080/v1/files/1725383792358030000", "host": [ "{{api_url}}" ], @@ -510,7 +691,7 @@ "path": [ "v1", "files", - "1724454386171356000" + "1725383792358030000" ] }, "description": "localhost:8080/books/3" diff --git a/postman/eg_results.json b/postman/eg_results.json new file mode 100644 index 0000000..73fbc67 --- /dev/null +++ b/postman/eg_results.json @@ -0,0 +1,20 @@ +{ + "data": { + "analysis": "Based on the information provided, here's an analysis of the file upload:\n\n1. **Filename**: `1725417360394229000_Learning-DevOps.pdf`\n - The filename appears to be a combination of a long numeric value (`1725417360394229000`) followed by an underscore and a descriptive name (`Learning-DevOps`).\n - The numeric value could potentially be a timestamp or a unique identifier.\n - The descriptive name suggests that the file is a PDF document related to learning DevOps practices.\n - The file extension `.pdf` indicates that it is a Portable Document Format (PDF) file.\n\n2. **File Size**: `19141577 bytes`\n - The file size is approximately 18.27 MB (19,141,577 bytes).\n - This is a relatively large file size for a PDF document, which could indicate that the document contains a significant amount of content, possibly including images or other media.\n\n3. **File Type**: `application/pdf`\n - The MIME type `application/pdf` is the standard type for PDF files.\n - This confirms that the file is indeed a PDF document, which is consistent with the file extension.\n\nOverall, the provided information suggests that the file upload is a PDF document related to learning DevOps practices, with a unique identifier or timestamp in the filename. The file size is relatively large, indicating that it may contain substantial content or embedded media. However, without further context or access to the file itself, it's difficult to provide a more detailed analysis of the file's contents or purpose.", + "createdAt": "2024-09-04T09:36:25.159682+07:00", + "fileName": "1725417360394229000_Learning-DevOps.pdf", + "fileSize": 19141577, + "fileType": "application/pdf", + "fileURL": "https://example.s3.amazonaws.com/1725417360394229000_Learning-DevOps.pdf", + "id": "1725417360394229000", + "metrics": { + "analysisLatency": "11.85446525s", + "inputTokens": 30, + "outputTokens": 375, + "totalLatency": "11.854582167s", + "uploadLatency": "10.667µs" + }, + "updatedAt": "2024-09-04T09:36:37.017314+07:00", + "uploadedBy": "user1@example.com" + } +} \ No newline at end of file diff --git a/python/test_bedrock.py b/python/test_bedrock.py new file mode 100755 index 0000000..e51d75f --- /dev/null +++ b/python/test_bedrock.py @@ -0,0 +1,72 @@ +import boto3 +import json +import os +from botocore.exceptions import ClientError + +# Set up AWS credentials and region +aws_access_key = os.getenv('AWS_ACCESS_KEY_ID') +aws_secret_key = os.getenv('AWS_SECRET_ACCESS_KEY') +aws_region = os.getenv('AWS_REGION', 'us-west-2') # Replace with your region if different +model_id = os.getenv('AMAZON_BEDROCK_MODEL_ID', 'anthropic.claude-3-sonnet-20240229-v1:0') + +# Initialize Bedrock clients +bedrock_runtime = boto3.client( + service_name='bedrock-runtime', + region_name=aws_region, + aws_access_key_id=aws_access_key, + aws_secret_access_key=aws_secret_key +) + +bedrock = boto3.client( + service_name='bedrock', + region_name=aws_region, + aws_access_key_id=aws_access_key, + aws_secret_access_key=aws_secret_key +) + +def list_models(): + try: + response = bedrock.list_foundation_models() + print("\nAvailable Bedrock models:") + for model in response['modelSummaries']: + print(f"- {model['modelId']}") + except ClientError as e: + print(f"Error listing models: {e}") + +def invoke_model(model_id): + try: + body = json.dumps({ + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": "Hello, Claude. Please respond with 'Hello, Human!'" + } + ] + }) + + response = bedrock_runtime.invoke_model_with_response_stream( + modelId=model_id, + contentType='application/json', + accept='application/json', + body=body + ) + + for event in response['body']: + chunk = json.loads(event['chunk']['bytes'].decode()) + if chunk['type'] == 'content_block_delta': + print(chunk['delta']['text'], end='') + elif chunk['type'] == 'message_stop': + print() # New line at the end of the response + + except ClientError as e: + print(f"Error invoking model: {e}") + +if __name__ == "__main__": + print(f"Testing Bedrock in region: {aws_region}") + print(f"Using model ID: {model_id}") + + list_models() + print("\nAttempting to invoke the model:") + invoke_model(model_id) \ No newline at end of file diff --git a/src/.env.example b/src/.env.example index 5b9f445..50688e2 100644 --- a/src/.env.example +++ b/src/.env.example @@ -20,6 +20,9 @@ AWS_REGION=us-west-2 AWS_ACCESS_KEY=YOUR_AWS_KEY AWS_SECRET_KEY_ID=YOUR_SECRET_KEY AWS_BUCKET_NAME=devopscorner-bedrock +AMAZON_BEDROCK_AGENT_ID=YOUR_AMAZON_BEDROCK_AGENT_ID +AMAZON_BEDROCK_MODEL_ID=anthropic.claude-3-haiku-20240307-v1:0 +AMAZON_BEDROCK_VERSION=bedrock-2023-05-31 OPENSEARCH_ENDPOINT=https://opensearch.us-west-2.es.amazonaws.com OPENSEARCH_USERNAME=devopscorner @@ -52,13 +55,9 @@ OTEL_RESOURCE_ATTRIBUTES= OTEL_TIME_INTERVAL=1 OTEL_RANDOM_TIME_ALIVE_INCREMENTER=1 OTEL_RANDOM_TOTAL_HEAP_SIZE_UPPER_BOUND=100 -OTEL_RANDOM_THREAD_ACTIVE_UPPOR_BOUND=10 +OTEL_RANDOM_THREAD_ACTIVE_UPPER_BOUND=10 OTEL_RANDOM_CPU_USAGE_UPPER_BOUND=100 -XRAY_VERSION=latest -XRAY_DAEMON_ENDPOINT=https://xray.us-west-2.amazonaws.com -XRAY_DAEMON_PORT=2000 - JAEGER_AGENT_PORT=6831 # Sampler Type: const / probabilistic / rateLimiting / remote JAEGER_SAMPLER_TYPE=const @@ -72,4 +71,8 @@ JAEGER_REPORTER_LOCAL_AGENT_HOST_PORT=http://0.0.0.0:6831 JAEGER_REPORTER_COLLECTOR_ENDPOINT=http://0.0.0.0:14268/api/traces JAEGER_REPORTER_COLLECTOR_USER=devopscorner JAEGER_REPORTER_COLLECTOR_PASSWORD=DevOpsCorner2024 -JAEGER_TAGS=golang,otel,restful,api,generativeai,genai,bedrock \ No newline at end of file +JAEGER_TAGS=golang,otel,restful,api,generativeai,genai,bedrock + +XRAY_VERSION=latest +XRAY_DAEMON_ENDPOINT=https://xray.us-west-2.amazonaws.com +XRAY_DAEMON_PORT=2000 diff --git a/src/cmd/migrate_file_upload.go b/src/cmd/migrate_file_upload.go index 3fda453..7d838d7 100644 --- a/src/cmd/migrate_file_upload.go +++ b/src/cmd/migrate_file_upload.go @@ -4,12 +4,12 @@ package main import ( "fmt" "log" - "strconv" "time" "github.com/devopscorner/golang-bedrock/src/config" "github.com/devopscorner/golang-bedrock/src/driver" "github.com/devopscorner/golang-bedrock/src/model" + "github.com/devopscorner/golang-bedrock/src/utility" "gorm.io/gorm" ) @@ -40,18 +40,17 @@ func MigrateUploader(db *gorm.DB) error { } for _, file := range files { - t := time.Now().UnixNano() - id_time := strconv.FormatInt(t, 10) - file.ID = id_time + file.ID = utility.GenerateID() file.CreatedAt = time.Now() file.UpdatedAt = time.Now() file.FileURL = fmt.Sprintf("https://example-bucket.s3.amazonaws.com/%s", file.FileName) // Example S3 URL + file.Analysis = "Sample analysis for " + file.FileName // Add a sample analysis if err := db.Create(&file).Error; err != nil { fmt.Printf("Failed to insert data for file: %+v\n", file) fmt.Printf("Error: %v\n", err) return err } else { - fmt.Printf("Sample file: %+v, created successfully! \n", file.FileName) + fmt.Printf("Sample file: %s, created successfully!\n", file.FileName) } } diff --git a/src/config/config.go b/src/config/config.go index 2bb64a9..26ee2a3 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -6,65 +6,68 @@ import ( ) type Config struct { - GinMode string - AppUrl string - AppPort int - DbConnection string - DbHost string - DbPort int - DbDatabase string - DbUsername string - DbPassword string - JwtAuthUsername string - JwtAuthPassword string - JwtIssuer string - JwtSecret string - LogLevel string - AWSRegion string - AWSAccessKey string - AWSSecretKey string - AWSBucketName string - OpenSearchEndpoint string - OpenSearchUsername string - OpenSearchPassword string - PrometheusEndpoint string - PrometheusPort int - LokiEndpoint string - LokiPort int - GrafanaEndpoint string - GrafanaPort int - GrafanaApiKey string - OtelEnvironment string - OtelMetricEnable string - OtelTraceEnable string - OtelTraceName string - OtelLogEnable string - OtelServiceName string - OtelOtlpEndpoint string - OtelOtlpPort int - OtelOtlpInsecure string - OtelOtlpHeader string - OtelAttributes string - OtelTimeInterval int64 `mapstructure:"TimeInterval"` - OtelTimeAliveIncrementer int64 `mapstructure:"RandomTimeAliveIncrementer"` - OtelTotalHeapSizeUpperBound int64 `mapstructure:"RandomTotalHeapSizeUpperBound"` - OtelThreadsActiveUpperBound int64 `mapstructure:"RandomThreadsActiveUpperBound"` - OtelCpuUsageUpperBound int64 `mapstructure:"RandomCpuUsageUpperBound"` - JaegerAgentPort int - JaegerSamplerType string - JaegerSamplerParam int - JaegerSamplerManagerHostPort string - JaegerReporterLogSpan string - JaegerReporterBufferFlushInterval int - JaegerReporterMaxQueueSize int - JaegerReporterLocalAgentHostPort string - JaegerReporterCollectorEndpoint string - JaegerReporterCollectorUser string - JaegerReporterCollectorPassword string - JaegerTags string - XRayDaemonEndpoint string - XRayDaemonPort int - XRayVersion string + GinMode string `mapstructure:"GIN_MODE"` + AppUrl string `mapstructure:"APP_URL"` + AppPort int `mapstructure:"APP_PORT"` + DbConnection string `mapstructure:"DB_CONNECTION"` + DbHost string `mapstructure:"DB_HOST"` + DbPort int `mapstructure:"DB_PORT"` + DbDatabase string `mapstructure:"DB_DATABASE"` + DbUsername string `mapstructure:"DB_USERNAME"` + DbPassword string `mapstructure:"DB_PASSWORD"` + JwtAuthUsername string `mapstructure:"JWT_AUTH_USERNAME"` + JwtAuthPassword string `mapstructure:"JWT_AUTH_PASSWORD"` + JwtIssuer string `mapstructure:"JWT_AUTH_USERNAME"` + JwtSecret string `mapstructure:"JWT_SECRET"` + LogLevel string `mapstructure:"LOG_LEVEL"` + AWSRegion string `mapstructure:"AWS_REGION"` + AWSAccessKey string `mapstructure:"AWS_ACCESS_KEY"` + AWSSecretKey string `mapstructure:"AWS_SECRET_KEY_ID"` + AWSBucketName string `mapstructure:"AWS_BUCKET_NAME"` + AmazonBedrockAgentId string `mapstructure:"AMAZON_BEDROCK_AGENT_ID"` + AmazonBedrockModelId string `mapstructure:"AMAZON_BEDROCK_MODEL_ID"` + AmazonBedrockVersion string `mapstructure:"AMAZON_BEDROCK_VERSION"` + OpenSearchEndpoint string `mapstructure:"OPENSEARCH_ENDPOINT"` + OpenSearchUsername string `mapstructure:"OPENSEARCH_USERNAME"` + OpenSearchPassword string `mapstructure:"OPENSEARCH_PASSWORD"` + PrometheusEndpoint string `mapstructure:"PROMETHEUS_ENDPOINT"` + PrometheusPort int `mapstructure:"PROMETHEUS_PORT"` + LokiEndpoint string `mapstructure:"LOKI_ENDPOINT"` + LokiPort int `mapstructure:"LOKI_PORT"` + GrafanaEndpoint string `mapstructure:"GRAFANA_ENDPOINT"` + GrafanaPort int `mapstructure:"GRAFANA_PORT"` + GrafanaApiKey string `mapstructure:"GRAFANA_API_KEY"` + OtelEnvironment string `mapstructure:"OTEL_ENVIRONMENT"` + OtelServiceName string `mapstructure:"OTEL_SERVICE_NAME"` + OtelMetricEnable string `mapstructure:"OTEL_INSTRUMENTATION_METRIC_ENABLED"` + OtelTraceEnable string `mapstructure:"OTEL_INSTRUMENTATION_TRACE_ENABLED"` + OtelTraceName string `mapstructure:"OTEL_INSTRUMENTATION_TRACE_NAME"` + OtelLogEnable string `mapstructure:"OTEL_INSTRUMENTATION_LOG_ENABLED"` + OtelOtlpEndpoint string `mapstructure:"OTEL_EXPORTER_OTLP_ENDPOINT"` + OtelOtlpPort int `mapstructure:"OTEL_EXPORTER_OTLP_PORT"` + OtelOtlpInsecure string `mapstructure:"OTEL_EXPORTER_OTLP_INSECURE"` + OtelOtlpHeader string `mapstructure:"OTEL_EXPORTER_OTLP_HEADERS"` + OtelAttributes string `mapstructure:"OTEL_RESOURCE_ATTRIBUTES"` + OtelTimeInterval int64 `mapstructure:"OTEL_TIME_INTERVAL"` + OtelTimeAliveIncrementer int64 `mapstructure:"OTEL_RANDOM_TIME_ALIVE_INCREMENTER"` + OtelTotalHeapSizeUpperBound int64 `mapstructure:"OTEL_RANDOM_TOTAL_HEAP_SIZE_UPPER_BOUND"` + OtelThreadsActiveUpperBound int64 `mapstructure:"OTEL_RANDOM_THREAD_ACTIVE_UPPER_BOUND"` + OtelCpuUsageUpperBound int64 `mapstructure:"OTEL_RANDOM_CPU_USAGE_UPPER_BOUND"` + JaegerAgentPort int `mapstructure:"JAEGER_AGENT_PORT"` + JaegerSamplerType string `mapstructure:"JAEGER_SAMPLER_TYPE"` + JaegerSamplerParam int `mapstructure:"JAEGER_SAMPLER_PARAM"` + JaegerSamplerManagerHostPort string `mapstructure:"JAEGER_SAMPLER_MANAGER_HOST_PORT"` + JaegerReporterLogSpan string `mapstructure:"JAEGER_REPORTER_LOG_SPANS"` + JaegerReporterBufferFlushInterval int `mapstructure:"JAEGER_REPORTER_BUFFER_FLUSH_INTERVAL"` + JaegerReporterMaxQueueSize int `mapstructure:"JAEGER_REPORTER_MAX_QUEUE_SIZE"` + JaegerReporterLocalAgentHostPort string `mapstructure:"JAEGER_REPORTER_LOCAL_AGENT_HOST_PORT"` + JaegerReporterCollectorEndpoint string `mapstructure:"JAEGER_REPORTER_COLLECTOR_ENDPOINT"` + JaegerReporterCollectorUser string `mapstructure:"JAEGER_REPORTER_COLLECTOR_USER"` + JaegerReporterCollectorPassword string `mapstructure:"JAEGER_REPORTER_COLLECTOR_PASSWORD"` + JaegerTags string `mapstructure:"JAEGER_TAGS"` + XRayDaemonEndpoint string `mapstructure:"XRAY_DAEMON_ENDPOINT"` + XRayDaemonPort int `mapstructure:"XRAY_DAEMON_PORT"` + XRayVersion string `mapstructure:"XRAY_VERSION"` } func LoadConfig() (*Config, error) { @@ -98,13 +101,20 @@ func LoadConfig() (*Config, error) { viper.SetDefault("AWS_ACCESS_KEY", "YOUR_AWS_ACCESS_KEY") viper.SetDefault("AWS_SECRET_KEY_ID", "YOUR_AWS_SECRET_KEY_ID") viper.SetDefault("AWS_BUCKET_NAME", "devopscorner-bedrock") + viper.SetDefault("AMAZON_BEDROCK_AGENT_ID", "YOUR_AMAZON_BEDROCK_AGENT_ID") + viper.SetDefault("AMAZON_BEDROCK_MODEL_ID", "anthropic.claude-3-haiku-20240307-v1:0") + viper.SetDefault("AMAZON_BEDROCK_VERSION", "bedrock-2023-05-31") + viper.SetDefault("OPENSEARCH_ENDPOINT", "https://opensearch.us-west-2.es.amazonaws.com") viper.SetDefault("OPENSEARCH_USERNAME", "OPENSEARCH_USERNAME") viper.SetDefault("OPENSEARCH_PASSWORD", "OPENSEARCH_PASSWORD") + viper.SetDefault("PROMETHEUS_ENDPOINT", "http://0.0.0.0:9090") viper.SetDefault("PROMETHEUS_PORT", 9090) + viper.SetDefault("LOKI_ENDPOINT", "http://0.0.0.0:3100") viper.SetDefault("LOKI_PORT", 3100) + viper.SetDefault("GRAFANA_ENDPOINT", "http://0.0.0.0:3000") viper.SetDefault("GRAFANA_PORT", 3000) viper.SetDefault("GRAFANA_API_KEY", "GRAFANA_API_KEY") @@ -125,7 +135,7 @@ func LoadConfig() (*Config, error) { viper.SetDefault("OTEL_TIME_INTERVAL", 1) viper.SetDefault("OTEL_RANDOM_TIME_ALIVE_INCREMENTER", 1) viper.SetDefault("OTEL_RANDOM_TOTAL_HEAP_SIZE_UPPER_BOUND", 100) - viper.SetDefault("OTEL_RANDOM_THREAD_ACTIVE_UPPOR_BOUND", 10) + viper.SetDefault("OTEL_RANDOM_THREAD_ACTIVE_UPPER_BOUND", 10) viper.SetDefault("OTEL_RANDOM_CPU_USAGE_UPPER_BOUND", 100) // TRACING with XRAY @@ -164,6 +174,8 @@ func LoadConfig() (*Config, error) { AWSAccessKey: viper.GetString("AWS_ACCESS_KEY"), AWSSecretKey: viper.GetString("AWS_SECRET_KEY_ID"), AWSBucketName: viper.GetString("AWS_BUCKET_NAME"), + AmazonBedrockAgentId: viper.GetString("AMAZON_BEDROCK_AGENT_ID"), + AmazonBedrockModelId: viper.GetString("AMAZON_BEDROCK_MODEL_ID"), OpenSearchEndpoint: viper.GetString("OPENSEARCH_ENDPOINT"), OpenSearchUsername: viper.GetString("OPENSEARCH_USERNAME"), OpenSearchPassword: viper.GetString("OPENSEARCH_PASSWORD"), @@ -171,6 +183,8 @@ func LoadConfig() (*Config, error) { PrometheusPort: viper.GetInt("PROMETHEUS_PORT"), GrafanaEndpoint: viper.GetString("GRAFANA_ENDPOINT"), GrafanaApiKey: viper.GetString("GRAFANA_API_KEY"), + LokiEndpoint: viper.GetString("LOKI_ENDPOINT"), + LokiPort: viper.GetInt("LOKI_PORT"), OtelEnvironment: viper.GetString("OTEL_ENVIRONMENT"), OtelMetricEnable: viper.GetString("OTEL_INSTRUMENTATION_METRIC_ENABLED"), OtelTraceEnable: viper.GetString("OTEL_INSTRUMENTATION_TRACE_ENABLED"), @@ -185,7 +199,7 @@ func LoadConfig() (*Config, error) { OtelTimeInterval: viper.GetInt64("OTEL_TIME_INTERVAL"), OtelTimeAliveIncrementer: viper.GetInt64("OTEL_RANDOM_TIME_ALIVE_INCREMENTER"), OtelTotalHeapSizeUpperBound: viper.GetInt64("OTEL_RANDOM_TOTAL_HEAP_SIZE_UPPER_BOUND"), - OtelThreadsActiveUpperBound: viper.GetInt64("OTEL_RANDOM_THREAD_ACTIVE_UPPOR_BOUND"), + OtelThreadsActiveUpperBound: viper.GetInt64("OTEL_RANDOM_THREAD_ACTIVE_UPPER_BOUND"), OtelCpuUsageUpperBound: viper.GetInt64("OTEL_RANDOM_CPU_USAGE_UPPER_BOUND"), JaegerAgentPort: viper.GetInt("JAEGER_AGENT_PORT"), JaegerSamplerType: viper.GetString("JAEGER_SAMPLER_TYPE"), diff --git a/src/config/const.go b/src/config/const.go index b575df3..31d29e7 100644 --- a/src/config/const.go +++ b/src/config/const.go @@ -14,7 +14,7 @@ const ( ERR_GENERATE_TOKEN = "Failed to generate token" ERR_GENERATE_REFRESH_TOKEN = "Failed to generate refresh token" ERR_MISSING_AUTH_HEADER = "Missing authorization header" - ERR_FILE_NOT_FOUND = "Book not found" + ERR_FILE_NOT_FOUND = "File id not found" ERR_UPDATE_FILE = "Failed to update file" ERR_DELETE_FILE = "Failed to delete file" ) diff --git a/src/config/value.go b/src/config/value.go index e72aaf4..c7a549d 100644 --- a/src/config/value.go +++ b/src/config/value.go @@ -149,6 +149,27 @@ func AWSBucketName() string { return config.AWSBucketName } +func AmazonBedrockAgentId() string { + config := &Config{ + AmazonBedrockAgentId: viper.GetString("AMAZON_BEDROCK_AGENT_ID"), + } + return config.AmazonBedrockAgentId +} + +func AmazonBedrockModelId() string { + config := &Config{ + AmazonBedrockModelId: viper.GetString("AMAZON_BEDROCK_MODEL_ID"), + } + return config.AmazonBedrockModelId +} + +func AmazonBedrockVersion() string { + config := &Config{ + AmazonBedrockVersion: viper.GetString("AMAZON_BEDROCK_VERSION"), + } + return config.AmazonBedrockVersion +} + // ----------------------------------- // OPENSEARCH // ----------------------------------- @@ -327,7 +348,7 @@ func OtelTotalHeapSizeUpperBound() int64 { func OtelThreadsActiveUpperBound() int64 { config := &Config{ - OtelThreadsActiveUpperBound: viper.GetInt64("OTEL_RANDOM_THREAD_ACTIVE_UPPOR_BOUND"), + OtelThreadsActiveUpperBound: viper.GetInt64("OTEL_RANDOM_THREAD_ACTIVE_UPPER_BOUND"), } return config.OtelThreadsActiveUpperBound } diff --git a/src/controller/file_controller.go b/src/controller/file_controller.go index 9140e6f..0ad25bc 100644 --- a/src/controller/file_controller.go +++ b/src/controller/file_controller.go @@ -2,8 +2,10 @@ package controller import ( + "context" "fmt" "mime/multipart" + "net/http" "time" "github.com/aws/aws-sdk-go-v2/service/s3" @@ -14,11 +16,15 @@ import ( "github.com/devopscorner/golang-bedrock/src/view" "github.com/gin-gonic/gin" validator "github.com/go-playground/validator/v10" + "github.com/sirupsen/logrus" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" ) -var fileTracer = otel.Tracer("file-controller") +var ( + fileTracer = otel.Tracer("file-controller") + log = logrus.New() +) type FileController struct { repo repository.FileRepository @@ -30,54 +36,74 @@ func NewFileController(repo repository.FileRepository, s3Client *s3.Client) *Fil } func (fc *FileController) CreateFile(c *gin.Context) { - ctxTrace, span := fileTracer.Start(c.Request.Context(), "CreateFile") + ctx, span := fileTracer.Start(c.Request.Context(), "CreateFile") defer span.End() file, err := c.FormFile("file") if err != nil { - utility.RecordError(ctxTrace, err) - utility.LogError(c, "Invalid file upload", err) - view.ErrorBadRequest(c, err) + fc.handleError(ctx, c, "Invalid file upload", err, http.StatusBadRequest) return } - filename := generateUniqueFilename(file.Filename) + fileID := utility.GenerateID() + filename := generateUniqueFilename(file.Filename, fileID) - s3URL, err := fc.uploadFileToS3(c, file, filename) + s3URL, err := fc.uploadFileToS3(ctx, file, filename) if err != nil { - utility.RecordError(ctxTrace, err) - utility.LogError(c, "Failed to upload file to S3", err) - view.ErrorInternalServer(c, err) + fc.handleError(ctx, c, "Failed to upload file to S3", err, http.StatusInternalServerError) return } fileUpload := model.FileUpload{ + ID: fileID, FileName: filename, FileSize: file.Size, FileType: file.Header.Get("Content-Type"), FileURL: s3URL, - UploadedBy: c.GetString("user"), + UploadedBy: "user1@example.com", + Analysis: "", } validate := validator.New() if err := validate.Struct(fileUpload); err != nil { - utility.RecordError(ctxTrace, err) - utility.LogError(c, "File validation failed", err) - view.ErrorBadRequest(c, err) + fc.handleError(ctx, c, "File validation failed", err, http.StatusBadRequest) return } - err = fc.repo.CreateFile(ctxTrace, &fileUpload) + err = fc.repo.CreateFile(ctx, &fileUpload) if err != nil { - utility.RecordError(ctxTrace, err) - utility.LogError(c, "Failed to create file record", err) - view.ErrorInternalServer(c, err) + fc.handleError(ctx, c, "Failed to create file record", err, http.StatusInternalServerError) return } utility.RecordFileUpload(fileUpload.FileType, float64(fileUpload.FileSize)) - go fc.analyzeUploadWithBedrock(c, &fileUpload) + // Use a channel to handle the analysis result and metrics + resultChan := make(chan struct { + analysis string + metrics utility.Metrics + }) + go fc.analyzeUploadWithBedrock(context.Background(), &fileUpload, resultChan) + + var analysisMetrics utility.Metrics + // Wait for the analysis result or timeout + select { + case result := <-resultChan: + fileUpload.Analysis = result.analysis + analysisMetrics = result.metrics + fc.logAnalysisMetrics(fileUpload.FileName, result.metrics) + case <-time.After(30 * time.Second): + fileUpload.Analysis = "Analysis timed out" + log.WithField("filename", fileUpload.FileName).Warn("Bedrock analysis timed out") + } + + // Update the file with the analysis result + if err := fc.repo.UpdateFile(ctx, &fileUpload); err != nil { + log.WithFields(logrus.Fields{ + "error": err, + "filename": fileUpload.FileName, + }).Error("Failed to update file with analysis result") + } span.SetAttributes( attribute.String("filename", fileUpload.FileName), @@ -85,54 +111,59 @@ func (fc *FileController) CreateFile(c *gin.Context) { attribute.String("file_type", fileUpload.FileType), ) - utility.LogInfo(c, fmt.Sprintf("Created file: %s, Size: %d, Type: %s", fileUpload.FileName, fileUpload.FileSize, fileUpload.FileType)) - view.ViewCreateFile(c, fileUpload) + log.WithFields(logrus.Fields{ + "filename": fileUpload.FileName, + "size": fileUpload.FileSize, + "type": fileUpload.FileType, + }).Info("Created file") + + // Pass the metrics to the view function + view.ViewCreateFile(c, fileUpload, analysisMetrics) } func (fc *FileController) FindAll(c *gin.Context) { - ctxTrace, span := fileTracer.Start(c.Request.Context(), "FindAll") + ctx, span := fileTracer.Start(c.Request.Context(), "FindAll") defer span.End() - files, err := fc.repo.FindAll(ctxTrace) + files, err := fc.repo.FindAll(ctx) if err != nil { - utility.RecordError(ctxTrace, err) - utility.LogError(c, "Failed to find files", err) - view.ErrorInternalServer(c, err) + fc.handleError(ctx, c, "Failed to find files", err, http.StatusInternalServerError) return } span.SetAttributes(attribute.Int("file_count", len(files))) - utility.LogInfo(c, fmt.Sprintf("Retrieved %d files", len(files))) + log.WithFields(logrus.Fields{ + "count": len(files), + }).Info("Retrieved files") view.ViewFindAllFiles(c, files) } func (fc *FileController) FindByID(c *gin.Context) { - ctxTrace, span := fileTracer.Start(c.Request.Context(), "FindByID") + ctx, span := fileTracer.Start(c.Request.Context(), "FindByID") defer span.End() id := c.Param("id") span.SetAttributes(attribute.String("file_id", id)) - file, err := fc.repo.FindByID(ctxTrace, id) + file, err := fc.repo.FindByID(ctx, id) if err != nil { - utility.RecordError(ctxTrace, err) - utility.LogError(c, fmt.Sprintf("Failed to find file: %s", id), err) - view.ErrorInternalServer(c, err) + fc.handleError(ctx, c, "Failed to find file", err, http.StatusInternalServerError) return } if file == nil { - utility.LogWarn(c, fmt.Sprintf("File not found: %s", id)) - view.ErrorNotFound(c) + fc.handleError(ctx, c, "File not found", nil, http.StatusNotFound) return } - utility.LogInfo(c, fmt.Sprintf("Retrieved file: %s", id)) + log.WithFields(logrus.Fields{ + "id": id, + }).Info("Retrieved file") view.ViewFindFileByID(c, file) } func (fc *FileController) UpdateFile(c *gin.Context) { - ctxTrace, span := fileTracer.Start(c.Request.Context(), "UpdateFile") + ctx, span := fileTracer.Start(c.Request.Context(), "UpdateFile") defer span.End() id := c.Param("id") @@ -140,17 +171,13 @@ func (fc *FileController) UpdateFile(c *gin.Context) { var input model.FileUpload if err := c.ShouldBindJSON(&input); err != nil { - utility.RecordError(ctxTrace, err) - utility.LogError(c, "Invalid input for file update", err) - view.ErrorBadRequest(c, err) + fc.handleError(ctx, c, "Invalid input for file update", err, http.StatusBadRequest) return } - file, err := fc.repo.FindByID(c, id) + file, err := fc.repo.FindByID(ctx, id) if err != nil { - utility.RecordError(ctxTrace, err) - utility.LogError(c, fmt.Sprintf("Failed to find file for update: %s", id), err) - view.ErrorNotFound(c) + fc.handleError(ctx, c, "Failed to find file for update", err, http.StatusNotFound) return } @@ -159,65 +186,115 @@ func (fc *FileController) UpdateFile(c *gin.Context) { file.FileType = input.FileType file.FileURL = input.FileURL file.UploadedBy = input.UploadedBy + file.Analysis = input.Analysis - if err := fc.repo.UpdateFile(c, file); err != nil { - utility.RecordError(ctxTrace, err) - utility.LogError(c, fmt.Sprintf("Failed to update file: %s", id), err) - view.ErrorInternalServer(c, err) + if err := fc.repo.UpdateFile(ctx, file); err != nil { + fc.handleError(ctx, c, "Failed to update file", err, http.StatusInternalServerError) return } - utility.LogInfo(c, fmt.Sprintf("Updated file: %s", id)) + log.WithFields(logrus.Fields{ + "id": id, + }).Info("Updated file") view.ViewUpdateFile(c, file) } func (fc *FileController) DeleteFile(c *gin.Context) { - ctxTrace, span := fileTracer.Start(c.Request.Context(), "DeleteFile") + ctx, span := fileTracer.Start(c.Request.Context(), "DeleteFile") defer span.End() id := c.Param("id") span.SetAttributes(attribute.String("file_id", id)) - if err := fc.repo.DeleteFile(c, id); err != nil { - utility.RecordError(ctxTrace, err) - utility.LogError(c, fmt.Sprintf("Failed to delete file: %s", id), err) - view.ErrorInternalServer(c, err) + if err := fc.repo.DeleteFile(ctx, id); err != nil { + fc.handleError(ctx, c, "Failed to delete file", err, http.StatusInternalServerError) return } - utility.LogInfo(c, fmt.Sprintf("Deleted file: %s", id)) + log.WithFields(logrus.Fields{ + "id": id, + }).Info("Deleted file") view.ViewDeleteFile(c) } -func (fc *FileController) uploadFileToS3(c *gin.Context, file *multipart.FileHeader, filename string) (string, error) { - ctxTrace, span := fileTracer.Start(c, "uploadFileToS3") +func (fc *FileController) uploadFileToS3(ctx context.Context, file *multipart.FileHeader, filename string) (string, error) { + ctx, span := fileTracer.Start(ctx, "uploadFileToS3") defer span.End() f, err := file.Open() if err != nil { - utility.RecordError(ctxTrace, err) - return "", err + return "", fmt.Errorf("failed to open file: %w", err) } defer f.Close() - return utility.UploadFileToS3(c, fc.s3Client, config.AWSBucketName(), filename, f) + return utility.UploadFileToS3(ctx, fc.s3Client, config.AWSBucketName(), filename, f) } -func (fc *FileController) analyzeUploadWithBedrock(c *gin.Context, fileInfo *model.FileUpload) { - ctxTrace, span := fileTracer.Start(c, "analyzeUploadWithBedrock") +func (fc *FileController) analyzeUploadWithBedrock(ctx context.Context, fileInfo *model.FileUpload, resultChan chan<- struct { + analysis string + metrics utility.Metrics +}) { + ctx, cancel := context.WithTimeout(ctx, 25*time.Second) + defer cancel() + + _, span := fileTracer.Start(ctx, "analyzeUploadWithBedrock") defer span.End() - analysis, err := utility.AnalyzeWithBedrock(c, fmt.Sprintf("Analyze this file upload: Filename: %s, Size: %d bytes, Type: %s", fileInfo.FileName, fileInfo.FileSize, fileInfo.FileType)) + analysis, metrics, err := utility.AnalyzeWithBedrock(ctx, fmt.Sprintf("Analyze this file upload: Filename: %s, Size: %d bytes, Type: %s", fileInfo.FileName, fileInfo.FileSize, fileInfo.FileType)) if err != nil { - utility.RecordError(ctxTrace, err) - utility.LogError(c, "Failed to analyze upload with Bedrock", err) - return + log.WithFields(logrus.Fields{ + "error": err, + "filename": fileInfo.FileName, + }).Error("Failed to analyze upload with Bedrock") + analysis = fc.getErrorAnalysis(err) } span.SetAttributes(attribute.String("analysis_result", analysis)) - utility.LogInfo(c, fmt.Sprintf("Bedrock analysis completed for file: %s. Result: %s", fileInfo.FileName, analysis)) + log.WithFields(logrus.Fields{ + "filename": fileInfo.FileName, + "analysis": analysis, + }).Info("Bedrock analysis completed for file") + + resultChan <- struct { + analysis string + metrics utility.Metrics + }{analysis, metrics} +} + +func (fc *FileController) getErrorAnalysis(err error) string { + switch { + case err.Error() == "Bedrock model not found. Please check your model ID and permissions": + return "Analysis failed: Bedrock model not found. Please check configuration." + case err.Error() == "Access denied to Bedrock model. Please check your IAM permissions": + return "Analysis failed: Access denied to Bedrock model. Please check permissions." + case err.Error() == "Invalid input for Bedrock model": + return "Analysis failed: Invalid input for Bedrock model." + case err.Error() == "Bedrock analysis timed out": + return "Analysis failed: Bedrock request timed out." + default: + return "Analysis failed due to an error with Bedrock." + } +} + +func (fc *FileController) logAnalysisMetrics(filename string, metrics utility.Metrics) { + log.WithFields(logrus.Fields{ + "filename": filename, + "totalLatency": metrics.TotalLatency, + "uploadLatency": metrics.UploadLatency, + "analysisLatency": metrics.AnalysisLatency, + "inputTokens": metrics.InputTokens, + "outputTokens": metrics.OutputTokens, + }).Info("Bedrock analysis metrics") +} + +func (fc *FileController) handleError(ctx context.Context, c *gin.Context, message string, err error, statusCode int) { + utility.RecordError(ctx, err) + log.WithFields(logrus.Fields{ + "error": err, + }).Error(message) + view.ErrorResponse(c, statusCode, message) } -func generateUniqueFilename(originalFilename string) string { - return fmt.Sprintf("%d_%s", time.Now().UnixNano(), originalFilename) +func generateUniqueFilename(originalFilename string, fileId string) string { + return fmt.Sprintf("%s_%s", fileId, originalFilename) } diff --git a/src/go.mod b/src/go.mod index 5a23017..d97afad 100644 --- a/src/go.mod +++ b/src/go.mod @@ -3,14 +3,17 @@ module github.com/devopscorner/golang-bedrock/src go 1.21.4 require ( - github.com/aws/aws-sdk-go-v2 v1.30.4 - github.com/aws/aws-sdk-go-v2/config v1.27.29 - github.com/aws/aws-sdk-go-v2/credentials v1.17.29 + github.com/aws/aws-sdk-go-v2 v1.30.5 + github.com/aws/aws-sdk-go-v2/config v1.27.32 + github.com/aws/aws-sdk-go-v2/credentials v1.17.31 github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.15.1 github.com/aws/aws-sdk-go-v2/service/s3 v1.60.1 + github.com/aws/smithy-go v1.20.4 + github.com/gin-contrib/cors v1.7.2 github.com/gin-gonic/gin v1.10.0 github.com/go-playground/validator/v10 v10.22.0 github.com/golang-jwt/jwt v3.2.2+incompatible + github.com/google/uuid v1.6.0 github.com/grafana/loki-client-go v0.0.0-20230116142646-e7494d0ef70c github.com/prometheus/client_golang v1.20.2 github.com/prometheus/common v0.55.0 @@ -32,19 +35,18 @@ require ( require ( github.com/aws/aws-sdk-go v1.55.5 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.22.5 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.30.5 // indirect - github.com/aws/smithy-go v1.20.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.22.6 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.30.6 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bytedance/sonic v1.11.9 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect @@ -68,7 +70,6 @@ require ( github.com/goccy/go-json v0.10.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/uuid v1.6.0 // indirect github.com/grafana/regexp v0.0.0-20220304095617-2e8d9baf4ac2 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect diff --git a/src/go.sum b/src/go.sum index 218aa92..9b109a2 100644 --- a/src/go.sum +++ b/src/go.sum @@ -144,20 +144,20 @@ github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4 github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= -github.com/aws/aws-sdk-go-v2 v1.30.4 h1:frhcagrVNrzmT95RJImMHgabt99vkXGslubDaDagTk8= -github.com/aws/aws-sdk-go-v2 v1.30.4/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0= +github.com/aws/aws-sdk-go-v2 v1.30.5 h1:mWSRTwQAb0aLE17dSzztCVJWI9+cRMgqebndjwDyK0g= +github.com/aws/aws-sdk-go-v2 v1.30.5/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 h1:70PVAiL15/aBMh5LThwgXdSQorVr91L127ttckI9QQU= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4/go.mod h1:/MQxMqci8tlqDH+pjmoLu1i0tbWCUP1hhyMRuFxpQCw= -github.com/aws/aws-sdk-go-v2/config v1.27.29 h1:+ZPKb3u9Up4KZWLGTtpTmC5T3XmRD1ZQ8XQjRCHUvJw= -github.com/aws/aws-sdk-go-v2/config v1.27.29/go.mod h1:yxqvuubha9Vw8stEgNiStO+yZpP68Wm9hLmcm+R/Qk4= -github.com/aws/aws-sdk-go-v2/credentials v1.17.29 h1:CwGsupsXIlAFYuDVHv1nnK0wnxO0wZ/g1L8DSK/xiIw= -github.com/aws/aws-sdk-go-v2/credentials v1.17.29/go.mod h1:BPJ/yXV92ZVq6G8uYvbU0gSl8q94UB63nMT5ctNO38g= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 h1:yjwoSyDZF8Jth+mUk5lSPJCkMC0lMy6FaCD51jm6ayE= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12/go.mod h1:fuR57fAgMk7ot3WcNQfb6rSEn+SUffl7ri+aa8uKysI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 h1:TNyt/+X43KJ9IJJMjKfa3bNTiZbUP7DeCxfbTROESwY= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16/go.mod h1:2DwJF39FlNAUiX5pAc0UNeiz16lK2t7IaFcm0LFHEgc= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 h1:jYfy8UPmd+6kJW5YhY0L1/KftReOGxI/4NtVSTh9O/I= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16/go.mod h1:7ZfEPZxkW42Afq4uQB8H2E2e6ebh6mXTueEpYzjCzcs= +github.com/aws/aws-sdk-go-v2/config v1.27.32 h1:jnAMVTJTpAQlePCUUlnXnllHEMGVWmvUJOiGjgtS9S0= +github.com/aws/aws-sdk-go-v2/config v1.27.32/go.mod h1:JibtzKJoXT0M/MhoYL6qfCk7nm/MppwukDFZtdgVRoY= +github.com/aws/aws-sdk-go-v2/credentials v1.17.31 h1:jtyfcOfgoqWA2hW/E8sFbwdfgwD3APnF9CLCKE8dTyw= +github.com/aws/aws-sdk-go-v2/credentials v1.17.31/go.mod h1:RSgY5lfCfw+FoyKWtOpLolPlfQVdDBQWTUniAaE+NKY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 h1:pfQ2sqNpMVK6xz2RbqLEL0GH87JOwSxPV2rzm8Zsb74= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13/go.mod h1:NG7RXPUlqfsCLLFfi0+IpKN4sCB9D9fw/qTaSB+xRoU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 h1:pI7Bzt0BJtYA0N/JEC6B8fJ4RBrEMi1LBrkMdFYNSnQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17/go.mod h1:Dh5zzJYMtxfIjYW+/evjQ8uj2OyR/ve2KROHGHlSFqE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 h1:Mqr/V5gvrhA2gvgnF42Zh5iMiQNcOYthFYwCyrnuWlc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17/go.mod h1:aLJpZlCmjE+V+KtN1q1uyZkfnUWpQGpbsn89XPKyzfU= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16 h1:mimdLQkIX1zr8GIPY1ZtALdBQGxcASiBd2MOp8m/dMc= @@ -168,18 +168,18 @@ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 h1:KypMCbL github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4/go.mod h1:Vz1JQXliGcQktFTN/LN6uGppAIRoLBR2bMvIMP0gOjc= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18 h1:GckUnpm4EJOAio1c8o25a+b3lVfwVzC9gnSBqiiNmZM= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18/go.mod h1:Br6+bxfG33Dk3ynmkhsW2Z/t9D4+lRqdLDNCKi85w0U= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18 h1:tJ5RnkHCiSH0jyd6gROjlJtNwov0eGYNz8s8nFcR0jQ= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18/go.mod h1:++NHzT+nAF7ZPrHPsA+ENvsXkOO8wEu+C6RXltAG4/c= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 h1:rfprUlsdzgl7ZL2KlXiUAoJnI/VxfHCvDFr2QDFj6u4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19/go.mod h1:SCWkEdRq8/7EK60NcvvQ6NXKuTcchAD4ROAsC37VEZE= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16 h1:jg16PhLPUiHIj8zYIW6bqzeQSuHVEiWnGA0Brz5Xv2I= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16/go.mod h1:Uyk1zE1VVdsHSU7096h/rwnXDzOzYQVl+FNPhPw7ShY= github.com/aws/aws-sdk-go-v2/service/s3 v1.60.1 h1:mx2ucgtv+MWzJesJY9Ig/8AFHgoE5FwLXwUVgW/FGdI= github.com/aws/aws-sdk-go-v2/service/s3 v1.60.1/go.mod h1:BSPI0EfnYUuNHPS0uqIo5VrRwzie+Fp+YhQOUs16sKI= -github.com/aws/aws-sdk-go-v2/service/sso v1.22.5 h1:zCsFCKvbj25i7p1u94imVoO447I/sFv8qq+lGJhRN0c= -github.com/aws/aws-sdk-go-v2/service/sso v1.22.5/go.mod h1:ZeDX1SnKsVlejeuz41GiajjZpRSWR7/42q/EyA/QEiM= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 h1:SKvPgvdvmiTWoi0GAJ7AsJfOz3ngVkD/ERbs5pUnHNI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5/go.mod h1:20sz31hv/WsPa3HhU3hfrIet2kxM4Pe0r20eBZ20Tac= -github.com/aws/aws-sdk-go-v2/service/sts v1.30.5 h1:OMsEmCyz2i89XwRwPouAJvhj81wINh+4UK+k/0Yo/q8= -github.com/aws/aws-sdk-go-v2/service/sts v1.30.5/go.mod h1:vmSqFK+BVIwVpDAGZB3CoCXHzurt4qBE8lf+I/kRTh0= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.6 h1:o++HUDXlbrTl4PSal3YHtdErQxB8mDGAtkKNXBWPfIU= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.6/go.mod h1:eEygMHnTKH/3kNp9Jr1n3PdejuSNcgwLe1dWgQtO0VQ= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.6 h1:yCHcQCOwTfIsc8DoEhM3qXPxD+j8CbI6t1K3dNzsWV0= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.6/go.mod h1:bCbAxKDqNvkHxRaIMnyVPXPo+OaPRwvmgzMxbz1VKSA= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.6 h1:TrQadF7GcqvQ63kgwEcjlrVc2Fa0wpgLT0xtc73uAd8= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.6/go.mod h1:NXi1dIAGteSaRLqYgarlhP/Ij0cFT+qmCwiJqWh/U5o= github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4= github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= @@ -459,6 +459,8 @@ github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSy github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/cors v1.7.2 h1:oLDHxdg8W/XDoN/8zamqk/Drgt4oVZDvaV0YmvVICQw= +github.com/gin-contrib/cors v1.7.2/go.mod h1:SUJVARKgQ40dmrzgXEVxj2m7Ig1v1qIboQkPDTQ9t2E= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= diff --git a/src/main.go b/src/main.go index ccec034..e4ff9e3 100644 --- a/src/main.go +++ b/src/main.go @@ -4,11 +4,13 @@ package main import ( "fmt" "log" + "time" "github.com/devopscorner/golang-bedrock/src/config" "github.com/devopscorner/golang-bedrock/src/driver" "github.com/devopscorner/golang-bedrock/src/routes" "github.com/devopscorner/golang-bedrock/src/utility" + "github.com/gin-contrib/cors" "github.com/gin-gonic/gin" ) @@ -16,7 +18,7 @@ func main() { // Load configuration cfg, err := config.LoadConfig() if err != nil { - log.Fatalf("Failed to load configuration: %v", err) + log.Fatalf("✗ Failed to load configuration: %v", err) } // Initialize logger @@ -32,13 +34,13 @@ func main() { // Initialize S3 client s3Client, err := utility.InitS3Client(cfg) if err != nil { - log.Fatalf("Failed to initialize S3 client: %v", err) + log.Fatalf("✗ Failed to initialize S3 client: %v", err) } // Initialize Bedrock client err = utility.InitBedrock(cfg) if err != nil { - log.Fatalf("Failed to initialize Bedrock client: %v", err) + log.Fatalf("✗ Failed to initialize Bedrock client: %v", err) } // Initialize Prometheus metrics @@ -47,7 +49,10 @@ func main() { // Initialize Loki logger err = utility.InitLokiLogger(cfg) if err != nil { - log.Fatalf("Failed to initialize Loki logger: %v", err) + log.Printf("❗ Warning: Failed to initialize Loki logger: %v", err) + log.Println("✔ Continuing without Loki logging...") + } else { + log.Println("✔ Loki logger initialized successfully") } // Set Gin mode @@ -58,13 +63,23 @@ func main() { // Initialize router router := gin.Default() + // Setup CORS + router.Use(cors.New(cors.Config{ + AllowOrigins: []string{"*"}, + AllowMethods: []string{"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"}, + AllowHeaders: []string{"Origin", "Content-Length", "Content-Type", "Authorization"}, + ExposeHeaders: []string{"Content-Length"}, + AllowCredentials: true, + MaxAge: 12 * time.Hour, + })) + // Setup routes routes.SetupRoutes(router, cfg, s3Client, driver.DB) // Start the server port := fmt.Sprintf(":%d", cfg.AppPort) - log.Printf("Server is running on %s", port) + log.Printf("✔ Server is running on %s", port) if err := router.Run(port); err != nil { - log.Fatalf("Failed to start server: %v", err) + log.Fatalf("✗ Failed to start server: %v", err) } } diff --git a/src/model/file.go b/src/model/file.go index 3a5d086..bcf97f7 100644 --- a/src/model/file.go +++ b/src/model/file.go @@ -3,12 +3,13 @@ package model import "time" type FileUpload struct { - ID string `gorm:"primaryKey"` - FileName string `json:"fileName"` - FileSize int64 `json:"fileSize"` - FileType string `json:"fileType"` - FileURL string `json:"fileURL"` - UploadedBy string `json:"uploadedBy"` - CreatedAt time.Time - UpdatedAt time.Time + ID string `gorm:"primaryKey" json:"id"` + FileName string `json:"fileName"` + FileSize int64 `json:"fileSize"` + FileType string `json:"fileType"` + FileURL string `json:"fileURL"` + UploadedBy string `json:"uploadedBy"` + Analysis string `json:"analysis"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` } diff --git a/src/repository/file_repository.go b/src/repository/file_repository.go index 16dc22f..2ae0f60 100644 --- a/src/repository/file_repository.go +++ b/src/repository/file_repository.go @@ -51,7 +51,7 @@ func (r *fileRepository) FindByID(ctx context.Context, id string) (*model.FileUp span.SetAttributes(attribute.String("file_id", id)) var file model.FileUpload - result := r.db.WithContext(ctx).First(&file, "id = ?", id) + result := r.db.WithContext(ctx).First(&file, "ID = ?", id) if result.Error != nil { utility.RecordError(ctx, result.Error) return nil, result.Error @@ -85,6 +85,7 @@ func (r *fileRepository) UpdateFile(ctx context.Context, file *model.FileUpload) attribute.String("filename", file.FileName), attribute.Int64("file_size", file.FileSize), attribute.String("file_type", file.FileType), + attribute.String("analysis", file.Analysis), ) result := r.db.WithContext(ctx).Save(file) @@ -100,7 +101,7 @@ func (r *fileRepository) DeleteFile(ctx context.Context, id string) error { span.SetAttributes(attribute.String("file_id", id)) - result := r.db.WithContext(ctx).Delete(&model.FileUpload{}, "id = ?", id) + result := r.db.WithContext(ctx).Delete(&model.FileUpload{}, "ID = ?", id) if result.Error != nil { utility.RecordError(ctx, result.Error) } diff --git a/src/utility/bedrock.go b/src/utility/bedrock.go index 3d2843d..25c9a2b 100644 --- a/src/utility/bedrock.go +++ b/src/utility/bedrock.go @@ -4,15 +4,37 @@ package utility import ( "context" "encoding/json" + "errors" "fmt" + "time" + "github.com/aws/aws-sdk-go-v2/aws" awsCfg "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/bedrockruntime" + "github.com/aws/smithy-go" "github.com/devopscorner/golang-bedrock/src/config" + "github.com/sirupsen/logrus" ) -var bedrockClient *bedrockruntime.Client +var ( + bedrockClient *bedrockruntime.Client + logger *logrus.Logger +) + +// Metrics struct to hold various metrics +type Metrics struct { + TotalLatency time.Duration + UploadLatency time.Duration + AnalysisLatency time.Duration + InputTokens int + OutputTokens int +} + +// InitLogger should be called from your main application to set up the logger +func InitLogger(l *logrus.Logger) { + logger = l +} func InitBedrock(cfg *config.Config) error { bedrockCfg, err := awsCfg.LoadDefaultConfig(context.TODO(), @@ -31,35 +53,183 @@ func InitBedrock(cfg *config.Config) error { return nil } -func AnalyzeWithBedrock(ctx context.Context, content string) (string, error) { +func AnalyzeWithBedrock(ctx context.Context, content string) (string, Metrics, error) { + startTime := time.Now() + metrics := Metrics{} + + modelId := config.AmazonBedrockModelId() + if modelId == "" { + return "", metrics, fmt.Errorf("Bedrock model ID is not configured") + } + + if logger != nil { + logger.WithFields(logrus.Fields{ + "modelId": modelId, + }).Info("Analyzing with Bedrock") + } + + var inputBytes []byte + var err error + + uploadStart := time.Now() + if isClaudeV3Model(modelId) { + inputBytes, err = constructClaudeV3Input(content) + } else { + inputBytes, err = constructStandardInput(content) + } + metrics.UploadLatency = time.Since(uploadStart) + + if err != nil { + return "", metrics, fmt.Errorf("✗ Failed to construct input: %w", err) + } + + metrics.InputTokens = estimateTokenCount(content) + input := &bedrockruntime.InvokeModelInput{ - Body: []byte(constructPrompt(content)), - ModelId: ptrString("anthropic.claude-v2"), - ContentType: ptrString("application/json"), + Body: inputBytes, + ModelId: aws.String(modelId), + ContentType: aws.String("application/json"), } + ctx, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + + analysisStart := time.Now() output, err := bedrockClient.InvokeModel(ctx, input) + metrics.AnalysisLatency = time.Since(analysisStart) + + if err != nil { + var ae smithy.APIError + if errors.As(err, &ae) { + if logger != nil { + logger.WithFields(logrus.Fields{ + "errorCode": ae.ErrorCode(), + "errorMessage": ae.ErrorMessage(), + }).Error("✗ Bedrock API error") + } + + switch ae.ErrorCode() { + case "ResourceNotFoundException": + return "", metrics, fmt.Errorf("✗ Bedrock model not found. Please check your model ID and permissions") + case "AccessDeniedException": + return "", metrics, fmt.Errorf("✗ Access denied to Bedrock model. Please check your IAM permissions") + case "ValidationException": + return "", metrics, fmt.Errorf("✗ Invalid input for Bedrock model: %s", ae.ErrorMessage()) + default: + return "", metrics, fmt.Errorf("❗ Bedrock error: %s", ae.ErrorMessage()) + } + } + + if ctx.Err() == context.DeadlineExceeded { + return "", metrics, fmt.Errorf("❗ Bedrock analysis timed out") + } + + return "", metrics, fmt.Errorf("✗ Failed to invoke Bedrock model: %w", err) + } + + response, err := parseBedrockResponse(output.Body, modelId) if err != nil { - return "", err + return "", metrics, err + } + + metrics.OutputTokens = estimateTokenCount(response) + metrics.TotalLatency = time.Since(startTime) + + return response, metrics, nil +} + +// estimateTokenCount is a simple function to estimate the number of tokens in a string +// This is a very rough estimate and should be replaced with a proper tokenizer for production use +func estimateTokenCount(s string) int { + return len(s) / 4 // Assuming an average of 4 characters per token +} + +func isClaudeV3Model(modelId string) bool { + return modelId == "anthropic.claude-3-sonnet-20240229-v1:0" || + modelId == "anthropic.claude-3-haiku-20240307-v1:0" || + modelId == "anthropic.claude-v2" +} + +func constructClaudeV3Input(content string) ([]byte, error) { + input := struct { + BedrockVersion string `json:"anthropic_version"` + MaxTokens int `json:"max_tokens"` + Temperature float64 `json:"temperature"` + TopP float64 `json:"top_p"` + Messages []Message `json:"messages"` + }{ + BedrockVersion: config.AmazonBedrockVersion(), + MaxTokens: 500, + Temperature: 0.7, + TopP: 1, + Messages: []Message{ + { + Role: "user", + Content: content, + }, + }, } - var response map[string]interface{} - if err := json.Unmarshal(output.Body, &response); err != nil { - return "", err + return json.Marshal(input) +} + +func constructStandardInput(content string) ([]byte, error) { + input := struct { + Prompt string `json:"prompt"` + BedrockVersion string `json:"anthropic_version"` + MaxTokensToSample int `json:"max_tokens_to_sample"` + Temperature float64 `json:"temperature"` + TopP float64 `json:"top_p"` + }{ + Prompt: fmt.Sprintf("Human: %s\n\nAssistant: Here's my analysis:", content), + BedrockVersion: config.AmazonBedrockVersion(), + MaxTokensToSample: 500, + Temperature: 0.7, + TopP: 1, } - return response["completion"].(string), nil + return json.Marshal(input) } -func constructPrompt(content string) string { - return fmt.Sprintf(`{ - "prompt": "Human: %s\n\nAssistant: Here's my analysis:", - "max_tokens_to_sample": 500, - "temperature": 0.7, - "top_p": 1 - }`, content) +func parseBedrockResponse(responseBody []byte, modelId string) (string, error) { + if isClaudeV3Model(modelId) { + var response struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + } + if err := json.Unmarshal(responseBody, &response); err != nil { + return "", fmt.Errorf("✗ Failed to unmarshal Claude V3 response: %w", err) + } + if len(response.Content) == 0 || response.Content[0].Text == "" { + return "", fmt.Errorf("Empty response from Claude V3") + } + return response.Content[0].Text, nil + } else { + var response struct { + Completion string `json:"completion"` + } + if err := json.Unmarshal(responseBody, &response); err != nil { + return "", fmt.Errorf("✗ Failed to unmarshal standard response: %w", err) + } + return response.Completion, nil + } } -func ptrString(s string) *string { - return &s +type Message struct { + Role string `json:"role"` + Content string `json:"content"` +} + +// LogMetrics logs the collected metrics +func LogMetrics(metrics Metrics) { + if logger != nil { + logger.WithFields(logrus.Fields{ + "totalLatency": metrics.TotalLatency, + "uploadLatency": metrics.UploadLatency, + "analysisLatency": metrics.AnalysisLatency, + "inputTokens": metrics.InputTokens, + "outputTokens": metrics.OutputTokens, + }).Info("Bedrock analysis metrics") + } } diff --git a/src/utility/genid.go b/src/utility/genid.go new file mode 100644 index 0000000..e4f52d3 --- /dev/null +++ b/src/utility/genid.go @@ -0,0 +1,19 @@ +// utility/genid.go +package utility + +import ( + "strconv" + "time" + + "github.com/google/uuid" +) + +func GenerateID() string { + t := time.Now().UnixNano() + id_time := strconv.FormatInt(t, 10) + return id_time +} + +func GenerateFileID() string { + return uuid.New().String() +} diff --git a/src/utility/loki.go b/src/utility/loki.go index 7f9e40e..da989a9 100644 --- a/src/utility/loki.go +++ b/src/utility/loki.go @@ -18,14 +18,24 @@ var lokiClient *loki.Client var lokiURL *url.URL func InitLokiLogger(cfg *config.Config) error { + if cfg.LokiEndpoint == "" { + return fmt.Errorf("❗ Loki endpoint is not configured") + } + var err error lokiURL, err = url.Parse(cfg.LokiEndpoint) if err != nil { - return fmt.Errorf("invalid Loki URL: %v", err) + return fmt.Errorf("❗ Invalid Loki URL: %v", err) + } + + if lokiURL.Scheme == "" { + return fmt.Errorf("❗ Loki URL scheme is missing (e.g., http:// or https://)") } lokiConfig := loki.Config{ - URL: urlutil.URLValue{URL: lokiURL}, + URL: urlutil.URLValue{ + URL: lokiURL, + }, BatchWait: 1 * time.Second, BatchSize: 1024 * 1024, Timeout: 10 * time.Second, @@ -33,7 +43,7 @@ func InitLokiLogger(cfg *config.Config) error { client, err := loki.New(lokiConfig) if err != nil { - return fmt.Errorf("failed to create Loki client: %v", err) + return fmt.Errorf("✗ Failed to create Loki client: %v", err) } lokiClient = client @@ -42,7 +52,7 @@ func InitLokiLogger(cfg *config.Config) error { func LogWithLoki(ctx *gin.Context, level, message string, err error) { if lokiClient == nil { - fmt.Println("Loki client not initialized") + fmt.Println("❗ Loki client not initialized") return } diff --git a/src/utility/otel.go b/src/utility/otel.go index 9f88ca1..10a7d5c 100644 --- a/src/utility/otel.go +++ b/src/utility/otel.go @@ -34,33 +34,34 @@ func InitTracer(cfg *config.Config) func() { switch strings.ToLower(cfg.OtelTraceName) { case "jaeger": - log.Println("Jaeger tracing is not implemented in this example") - return func() {} + log.Println("❗ Jaeger tracing is not implemented. Falling back to OTLP HTTP exporter.") + fallthrough case "xray": - log.Println("X-Ray tracing is not implemented in this example") + log.Println("❗ X-Ray tracing is not implemented in this example") return func() {} + case "otlphttp": + exporter, err = otlptracehttp.New(ctx, + otlptracehttp.WithEndpoint(fmt.Sprintf("%s:%d", cfg.OtelOtlpEndpoint, cfg.OtelOtlpPort)), + otlptracehttp.WithInsecure(), + ) + case "otlpgrpc": + exporter, err = otlptrace.New( + ctx, + otlptracegrpc.NewClient( + otlptracegrpc.WithEndpoint(fmt.Sprintf("%s:%d", cfg.OtelOtlpEndpoint, cfg.OtelOtlpPort)), + otlptracegrpc.WithInsecure(), + ), + ) default: - // OTLP exporter setup - if strings.HasPrefix(cfg.OtelOtlpEndpoint, "http") { - // HTTP exporter - exporter, err = otlptracehttp.New(ctx, - otlptracehttp.WithEndpoint(fmt.Sprintf("%s:%d", cfg.OtelOtlpEndpoint, cfg.OtelOtlpPort)), - otlptracehttp.WithInsecure(), - ) - } else { - // gRPC exporter - exporter, err = otlptrace.New( - ctx, - otlptracegrpc.NewClient( - otlptracegrpc.WithEndpoint(fmt.Sprintf("%s:%d", cfg.OtelOtlpEndpoint, cfg.OtelOtlpPort)), - otlptracegrpc.WithInsecure(), - ), - ) - } + log.Printf("❗ Unknown tracer: %s. Falling back to OTLP HTTP exporter.", cfg.OtelTraceName) + exporter, err = otlptracehttp.New(ctx, + otlptracehttp.WithEndpoint(fmt.Sprintf("%s:%d", cfg.OtelOtlpEndpoint, cfg.OtelOtlpPort)), + otlptracehttp.WithInsecure(), + ) + } - if err != nil { - log.Fatalf("Failed to create exporter: %v", err) - } + if err != nil { + log.Fatalf("✗ Failed to create exporter: %v", err) } res, err := resource.New( @@ -73,7 +74,7 @@ func InitTracer(cfg *config.Config) func() { resource.WithProcess(), ) if err != nil { - log.Fatalf("Failed to create resource: %v", err) + log.Fatalf("✗ Failed to create resource: %v", err) } tp := sdktrace.NewTracerProvider( @@ -87,7 +88,7 @@ func InitTracer(cfg *config.Config) func() { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() if err := tp.Shutdown(ctx); err != nil { - log.Printf("Error shutting down tracer provider: %v", err) + log.Printf("❗ Error shutting down tracer provider: %v", err) } } } diff --git a/src/view/error_view.go b/src/view/error_view.go index 741b443..a2684e0 100644 --- a/src/view/error_view.go +++ b/src/view/error_view.go @@ -69,3 +69,7 @@ func ErrorUpdate(ctx *gin.Context) { func ErrorDelete(ctx *gin.Context) { ctx.JSON(http.StatusInternalServerError, gin.H{"error": config.ERR_DELETE_FILE}) } + +func ErrorResponse(ctx *gin.Context, statusCode int, message string) { + ctx.JSON(statusCode, gin.H{"error": message}) +} diff --git a/src/view/file_view.go b/src/view/file_view.go index 450be00..b419490 100644 --- a/src/view/file_view.go +++ b/src/view/file_view.go @@ -5,6 +5,7 @@ import ( "net/http" "github.com/devopscorner/golang-bedrock/src/model" + "github.com/devopscorner/golang-bedrock/src/utility" "github.com/gin-gonic/gin" ) @@ -23,9 +24,28 @@ func ViewFindFileByID(ctx *gin.Context, viewFile *model.FileUpload) { } // POST /files -// Create new file -func ViewCreateFile(ctx *gin.Context, viewFile model.FileUpload) { - ctx.JSON(http.StatusCreated, gin.H{"data": viewFile}) +// ViewCreateFile includes metrics in the response +func ViewCreateFile(c *gin.Context, file model.FileUpload, metrics utility.Metrics) { + c.JSON(http.StatusCreated, gin.H{ + "data": gin.H{ + "id": file.ID, + "fileName": file.FileName, + "fileSize": file.FileSize, + "fileType": file.FileType, + "fileURL": file.FileURL, + "uploadedBy": file.UploadedBy, + "analysis": file.Analysis, + "createdAt": file.CreatedAt, + "updatedAt": file.UpdatedAt, + "metrics": gin.H{ + "totalLatency": metrics.TotalLatency.String(), + "uploadLatency": metrics.UploadLatency.String(), + "analysisLatency": metrics.AnalysisLatency.String(), + "inputTokens": metrics.InputTokens, + "outputTokens": metrics.OutputTokens, + }, + }, + }) } // PUT /files/:id diff --git a/start-build.sh b/start-build.sh index 5d48136..0cbc034 100755 --- a/start-build.sh +++ b/start-build.sh @@ -20,7 +20,7 @@ export ECR_IMAGE="$CI_REGISTRY/$CI_PROJECT_PATH/$CI_PROJECT_NAME" export TARGETPLATFORM="linux/amd64" export ARCH="amd64" -export VERSION="1.1.4" +export VERSION=`git describe --tags` build_golang_bedrock() { TAGS="3.18 \ @@ -33,8 +33,8 @@ build_golang_bedrock() { ### DockerHub Build Images ### for TAG in $TAGS; do echo " Build Image => $IMAGE:$TAG" - docker build \ - -f Dockerfile-$ARCH \ + docker build --no-cache \ + -f Dockerfile \ -t $IMAGE:$TAG . echo '' done @@ -42,8 +42,8 @@ build_golang_bedrock() { ### ECR Build Images ### for TAG in $TAGS; do echo " Build Image => $ECR_IMAGE:$TAG" - docker build \ - -f Dockerfile-$ARCH \ + docker build --no-cache \ + -f Dockerfile \ -t $ECR_IMAGE:$TAG . echo '' done