add logs module with API for parsing and retrieving logs

.gitignore

@@ -1,2 +1,4 @@
 dist
 gateway/warehouse.db
+logs/data/source/
+logs/data/db/

README.md

@@ -109,6 +109,12 @@ The module sets up a simple HTTP file server that serves files from the `data` d
 
 Mount data volume to `/usr/src/app/data/`.
 
+## logs
+
+The module processes logs and returns them by request.
+
+Mount data volumes to `/usr/src/app/data/auth/`, `/usr/src/app/data/db/` and `/usr/src/app/data/source/`.
+
 ## Release
 
 Run `make release TAG=1.0.0`.

go.work

@@ -4,5 +4,6 @@ use (
 	./blog
 	./framework
 	./gateway
+	./logs
 	./static
 )

logs/Dockerfile

@@ -0,0 +1,13 @@
+FROM golang:1.23
+
+WORKDIR /usr/src/app
+
+COPY go.mod go.sum ./
+RUN go mod download && go mod verify
+
+COPY . .
+RUN go build -v -o /usr/local/bin ./...
+
+EXPOSE 8080
+
+CMD ["logs"]

logs/cloudbuild.yaml

@@ -0,0 +1,25 @@
+steps:
+  - name: gcr.io/cloud-builders/docker
+    args:
+      - build
+      - "--no-cache"
+      - "-t"
+      - "$_GCR_HOSTNAME/$PROJECT_ID/$REPO_NAME/$_SERVICE_NAME:$_TAG"
+      - "-f"
+      - ./logs/Dockerfile
+      - ./logs
+    id: Build
+  - name: gcr.io/cloud-builders/docker
+    args:
+      - push
+      - "$_GCR_HOSTNAME/$PROJECT_ID/$REPO_NAME/$_SERVICE_NAME:$_TAG"
+    id: Push
+images:
+  - "$_GCR_HOSTNAME/$PROJECT_ID/$REPO_NAME/$_SERVICE_NAME:$_TAG"
+options:
+  substitutionOption: ALLOW_LOOSE
+substitutions:
+  _GCR_HOSTNAME: eu.gcr.io
+  _TAG: latest
+tags:
+  - $_SERVICE_NAME

logs/controller/controller.go

@@ -0,0 +1,73 @@
+package controller
+
+import (
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/abaldeweg/warehouse-server/logs/db"
+	"github.com/abaldeweg/warehouse-server/logs/parser"
+	"github.com/gin-gonic/gin"
+)
+
+// GetLogs handles the GET request to retrieve logs.
+func GetLogs(c *gin.Context) {
+	fromParam := c.Param("from")
+	toParam := c.Param("to")
+
+	if len(fromParam) != 8 || len(toParam) != 8 {
+		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "'from' and 'to' parameters must be 8 characters long"})
+		return
+	}
+
+	from, err := strconv.Atoi(fromParam)
+	if err != nil {
+		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid 'from' parameter"})
+		return
+	}
+
+	to, err := strconv.Atoi(toParam)
+	if err != nil {
+		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid 'to' parameter"})
+		return
+	}
+
+	h, _ := db.NewDBHandler()
+	d, _ := h.Get(from, to)
+	defer h.Close()
+
+	c.JSON(http.StatusOK, d)
+}
+
+// CreateLog handles the POST request to parse and store logs.
+func CreateLog(c *gin.Context) {
+	entries, err := parser.ReadLogEntries()
+	if err != nil {
+		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": "Internal Server Error"})
+		return
+	}
+
+	h, err := db.NewDBHandler()
+	if err != nil {
+		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": "Internal Server Error"})
+		return
+	}
+	defer h.Close()
+
+	for _, entry := range entries {
+		date, _ := strconv.Atoi(time.Time(entry.Time).Format("20060102"))
+		exists, err := h.Exists(date, entry)
+		if err != nil {
+			c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": "Internal Server Error"})
+			return
+		}
+		if !exists {
+			if err := h.Write(date, entry); err != nil {
+				c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": "Internal Server Error"})
+				return
+			}
+		}
+	}
+
+	c.JSON(http.StatusOK, gin.H{"message": "success"})
+}

logs/data/auth/api_keys.json

@@ -0,0 +1,8 @@
+{
+  "keys": [
+    {
+      "key": "test",
+      "permissions": []
+    }
+  ]
+}

logs/db/db.go

@@ -0,0 +1,115 @@
+package db
+
+import (
+	"database/sql"
+	"encoding/json"
+	"os"
+
+	"github.com/abaldeweg/warehouse-server/logs/entity"
+	_ "github.com/mattn/go-sqlite3"
+)
+
+// DBHandler handles database operations for logs.
+type DBHandler struct {
+	db *sql.DB
+}
+
+// NewDBHandler creates a new DBHandler.
+func NewDBHandler() (*DBHandler, error) {
+	setup()
+
+	db, err := sql.Open("sqlite3", "data/db/events.db")
+	if err != nil {
+		return nil, err
+	}
+
+	migrate(db)
+
+	return &DBHandler{db: db}, nil
+}
+
+// Close closes the database connection.
+func (handler *DBHandler) Close() error {
+	return handler.db.Close()
+}
+
+// Get fetches all log entries from the database.
+func (handler *DBHandler) Get(from int, to int) ([]entity.LogEntry, error) {
+	query := `SELECT data FROM logs WHERE date >= ? AND date <= ?`
+	rows, err := handler.db.Query(query, from, to)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var logs []entity.LogEntry
+	for rows.Next() {
+		var jsonData []byte
+		if err := rows.Scan(&jsonData); err != nil {
+			return nil, err
+		}
+		var data entity.LogEntry
+		if err = json.Unmarshal(jsonData, &data); err != nil {
+			return nil, err
+		}
+		logs = append(logs, data)
+	}
+	return logs, nil
+}
+
+// Write inserts a log entry into the database.
+func (handler *DBHandler) Write(date int, data entity.LogEntry) error {
+	jsonData, err := json.Marshal(data)
+	if err != nil {
+		return err
+	}
+	query := `INSERT INTO logs (date, data) VALUES (?, ?)`
+	_, err = handler.db.Exec(query, date, jsonData)
+	return err
+}
+
+// Exists checks if a log entry already exists in the database.
+func (handler *DBHandler) Exists(date int, data entity.LogEntry) (bool, error) {
+	jsonData, err := json.Marshal(data)
+	if err != nil {
+		return false, err
+	}
+	query := `SELECT COUNT(*) FROM logs WHERE date = ? AND data = ?`
+	var count int
+	err = handler.db.QueryRow(query, date, jsonData).Scan(&count)
+	if err != nil {
+		return false, err
+	}
+	return count > 0, nil
+}
+
+// setup creates the necessary directories and files for the database.
+func setup() error {
+	if err := os.MkdirAll("data/db", os.ModePerm); err != nil {
+		return err
+	}
+	_, err := os.Stat("data/db/events.db")
+	if os.IsNotExist(err) {
+		file, err := os.Create("data/db/events.db")
+		if err != nil {
+			return err
+		}
+		file.Close()
+	}
+	return nil
+}
+
+// migrate creates the necessary tables for the database.
+func migrate(db *sql.DB) error {
+	createTableQuery := `
+	CREATE TABLE IF NOT EXISTS logs (
+		id INTEGER PRIMARY KEY AUTOINCREMENT,
+		date INTEGER,
+		data TEXT
+	)`
+	_, err := db.Exec(createTableQuery)
+	if err != nil {
+		return err
+	}
+	return nil
+}

logs/entity/entity.go

@@ -0,0 +1,40 @@
+package entity
+
+import "time"
+
+type LogEntry struct {
+	ClientAddr            string    `json:"ClientAddr"`
+	ClientHost            string    `json:"ClientHost"`
+	ClientPort            string    `json:"ClientPort"`
+	ClientUsername        string    `json:"ClientUsername"`
+	DownstreamContentSize int64     `json:"DownstreamContentSize"`
+	DownstreamStatus      int       `json:"DownstreamStatus"`
+	Duration              int64     `json:"Duration"`
+	GzipRatio             float64   `json:"GzipRatio"`
+	OriginContentSize     int64     `json:"OriginContentSize"`
+	OriginDuration        int64     `json:"OriginDuration"`
+	OriginStatus          int       `json:"OriginStatus"`
+	Overhead              int64     `json:"Overhead"`
+	RequestAddr           string    `json:"RequestAddr"`
+	RequestContentSize    int64     `json:"RequestContentSize"`
+	RequestCount          int       `json:"RequestCount"`
+	RequestHost           string    `json:"RequestHost"`
+	RequestMethod         string    `json:"RequestMethod"`
+	RequestPath           string    `json:"RequestPath"`
+	RequestPort           string    `json:"RequestPort"`
+	RequestProtocol       string    `json:"RequestProtocol"`
+	RequestScheme         string    `json:"RequestScheme"`
+	RetryAttempts         int       `json:"RetryAttempts"`
+	RouterName            string    `json:"RouterName"`
+	ServiceAddr           string    `json:"ServiceAddr"`
+	ServiceName           string    `json:"ServiceName"`
+	ServiceURL            string    `json:"ServiceURL"`
+	StartLocal            time.Time `json:"StartLocal"`
+	StartUTC              time.Time `json:"StartUTC"`
+	TLSCipher             string    `json:"TLSCipher"`
+	TLSVersion            string    `json:"TLSVersion"`
+	EntryPointName        string    `json:"entryPointName"`
+	Level                 string    `json:"level"`
+	Msg                   string    `json:"msg"`
+	Time                  time.Time `json:"time"`
+}

logs/go.mod

@@ -0,0 +1,81 @@
+module github.com/abaldeweg/warehouse-server/logs
+
+go 1.23
+
+require (
+	github.com/abaldeweg/warehouse-server/framework v0.12.1
+	github.com/gin-gonic/gin v1.10.0
+	github.com/mattn/go-sqlite3 v1.14.24
+)
+
+require (
+	cel.dev/expr v0.16.2 // indirect
+	cloud.google.com/go v0.116.0 // indirect
+	cloud.google.com/go/auth v0.11.0 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.6 // indirect
+	cloud.google.com/go/compute/metadata v0.5.2 // indirect
+	cloud.google.com/go/iam v1.2.2 // indirect
+	cloud.google.com/go/monitoring v1.21.2 // indirect
+	cloud.google.com/go/storage v1.48.0 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.3 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.3 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.3 // indirect
+	github.com/bytedance/sonic v1.12.6 // indirect
+	github.com/bytedance/sonic/loader v0.2.1 // indirect
+	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/cloudwego/base64x v0.1.4 // indirect
+	github.com/cloudwego/iasm v0.2.0 // indirect
+	github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect
+	github.com/envoyproxy/go-control-plane v0.13.1 // indirect
+	github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.7 // indirect
+	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.23.0 // indirect
+	github.com/goccy/go-json v0.10.4 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/google/s2a-go v0.1.8 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
+	github.com/googleapis/gax-go/v2 v2.14.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.9 // indirect
+	github.com/leodido/go-urn v1.4.0 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
+	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+	github.com/ugorji/go/codec v1.2.12 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/contrib/detectors/gcp v1.31.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.56.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 // indirect
+	go.opentelemetry.io/otel v1.31.0 // indirect
+	go.opentelemetry.io/otel/metric v1.31.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.31.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.31.0 // indirect
+	go.opentelemetry.io/otel/trace v1.31.0 // indirect
+	golang.org/x/arch v0.12.0 // indirect
+	golang.org/x/crypto v0.31.0 // indirect
+	golang.org/x/net v0.33.0 // indirect
+	golang.org/x/oauth2 v0.24.0 // indirect
+	golang.org/x/sync v0.10.0 // indirect
+	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
+	golang.org/x/time v0.8.0 // indirect
+	google.golang.org/api v0.210.0 // indirect
+	google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20241113202542-65e8d215514f // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20241118233622-e639e219e697 // indirect
+	google.golang.org/grpc v1.67.2 // indirect
+	google.golang.org/grpc/stats/opentelemetry v0.0.0-20241018153737-98959d9a4904 // indirect
+	google.golang.org/protobuf v1.36.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)