From ab2f161ffbb592a96d07c9b73b54c1f20c595ce0 Mon Sep 17 00:00:00 2001 From: LbP22 Date: Mon, 26 Feb 2024 15:32:44 +0200 Subject: [PATCH] dbs usage upd, fix init --- application/Dockerfile | 4 --- .../backend/app/containerdb/containerdb.go | 23 +++++++----- application/backend/app/daemon/daemon.go | 16 ++++++--- application/backend/app/db/db.go | 33 ++++++++--------- application/backend/app/routes/routes.go | 13 +++---- .../backend/app/statistics/statistics.go | 4 +-- application/backend/app/streamer/streamer.go | 3 +- application/backend/app/util/util.go | 22 ++++++++++-- application/backend/app/vars/vars.go | 36 +++++++++++-------- application/build.sh | 3 +- 10 files changed, 94 insertions(+), 63 deletions(-) diff --git a/application/Dockerfile b/application/Dockerfile index d1cb6d5..1f47cf2 100644 --- a/application/Dockerfile +++ b/application/Dockerfile @@ -12,8 +12,6 @@ RUN npm run build COPY . /code/ FROM alpine -RUN apk add bash curl -# tmp COPY --from=frontbuilder /code/dist/ /backend/dist/ @@ -26,8 +24,6 @@ RUN go mod download \ && go build -o main . FROM alpine -RUN apk add bash curl -# tmp COPY --from=frontbuilder /code/dist/ /dist/ COPY --from=backendbuilder /backend/main /backend/main diff --git a/application/backend/app/containerdb/containerdb.go b/application/backend/app/containerdb/containerdb.go index dc0e964..0518432 100644 --- a/application/backend/app/containerdb/containerdb.go +++ b/application/backend/app/containerdb/containerdb.go @@ -28,6 +28,9 @@ func PutLogMessage(db *leveldb.DB, host string, container string, message_item [ panic("Host is not mentioned!") } location := host + "/" + container + if vars.Statuses_DBs[location] == nil { + vars.Statuses_DBs[location] = util.GetDB(host, container, "statuses") + } if strings.Contains(message_item[1], "ERROR") || strings.Contains(message_item[1], "ERR") || // const statuses_errors = ["ERROR", "ERR", "Error", "Err"]; strings.Contains(message_item[1], "Error") || strings.Contains(message_item[1], "Err") { @@ -49,13 +52,20 @@ func PutLogMessage(db *leveldb.DB, host string, container string, message_item [ } else if strings.Contains(message_item[1], "ONLOGS") { vars.Counters_For_Containers_Last_30_Min[location]["meta"]++ vars.Statuses_DBs[location].Put([]byte(message_item[0]), []byte("meta"), nil) - } else { vars.Counters_For_Containers_Last_30_Min[location]["other"]++ vars.Statuses_DBs[location].Put([]byte(message_item[0]), []byte("other"), nil) } - return db.Put([]byte(message_item[0]), []byte(message_item[1]), nil) + err := db.Put([]byte(message_item[0]), []byte(message_item[1]), nil) + if err != nil { + db = util.GetDB(host, container, "logs") + err = db.Put([]byte(message_item[0]), []byte(message_item[1]), nil) + if err != nil { + panic(err) + } + } + return err } func GetLogsByStatus(host string, container string, message string, status string, limit int, startWith string, getPrev bool, include bool, caseSensetivity bool) [][]string { @@ -215,18 +225,15 @@ func DeleteContainer(host string, container string, fullDelete bool) { if vars.ActiveDBs[container] != nil { vars.ActiveDBs[container].Close() - newActiveDB, _ := leveldb.OpenFile("leveldb/hosts/"+host+"/containers/"+container+"/logs", nil) - vars.ActiveDBs[container] = newActiveDB + vars.ActiveDBs[container] = util.GetDB(host, container, "active") } if vars.Statuses_DBs[host+"/"+container] != nil { vars.Statuses_DBs[host+"/"+container].Close() - newStatusesDB, _ := leveldb.OpenFile("leveldb/hosts/"+host+"/containers/"+container+"/statuses", nil) - vars.Statuses_DBs[host+"/"+container] = newStatusesDB + vars.Statuses_DBs[host+"/"+container] = util.GetDB(host, container, "statuses") } if vars.Stat_Containers_DBs[host+"/"+container] != nil { vars.Stat_Containers_DBs[host+"/"+container].Close() - newStatDB, _ := leveldb.OpenFile("leveldb/hosts/"+host+"/containers/"+container+"/statistics", nil) - vars.Statuses_DBs[host+"/"+container] = newStatDB + vars.Statuses_DBs[host+"/"+container] = util.GetDB(host, container, "statistics") } vars.Counters_For_Containers_Last_30_Min[host+"/"+container] = map[string]uint64{"error": 0, "debug": 0, "info": 0, "warn": 0, "meta": 0, "other": 0} } diff --git a/application/backend/app/daemon/daemon.go b/application/backend/app/daemon/daemon.go index cb1a281..65ca00f 100644 --- a/application/backend/app/daemon/daemon.go +++ b/application/backend/app/daemon/daemon.go @@ -171,17 +171,23 @@ func GetContainersList() []string { json.Unmarshal([]byte(body), &result) var names []string - containersDB, err := leveldb.OpenFile("leveldb/hosts/"+util.GetHost()+"/containersMeta", nil) - if err != nil { - panic(err) + + containersMetaDB := vars.ContainersMeta_DBs[util.GetHost()] + if containersMetaDB == nil { + containersMetaDB, err := leveldb.OpenFile("leveldb/hosts/"+util.GetHost()+"/containersMeta", nil) + if err != nil { + panic(err) + } + vars.ContainersMeta_DBs[util.GetHost()] = containersMetaDB } - defer containersDB.Close() + containersMetaDB = vars.ContainersMeta_DBs[util.GetHost()] + for i := 0; i < len(result); i++ { name := fmt.Sprintf("%v", result[i]["Names"].([]interface{})[0].(string))[1:] id := result[i]["Id"].(string) names = append(names, name) - containersDB.Put([]byte(name), []byte(id), nil) + containersMetaDB.Put([]byte(name), []byte(id), nil) } return names diff --git a/application/backend/app/db/db.go b/application/backend/app/db/db.go index 93686db..4566c5f 100644 --- a/application/backend/app/db/db.go +++ b/application/backend/app/db/db.go @@ -1,50 +1,51 @@ package db import ( + "fmt" "time" "github.com/devforth/OnLogs/app/util" + "github.com/devforth/OnLogs/app/vars" "github.com/syndtr/goleveldb/leveldb" ) func CreateOnLogsToken() string { - tokensDB, _ := leveldb.OpenFile("leveldb/tokens", nil) - defer tokensDB.Close() - token := util.GenerateJWTSecret() to_put := time.Now().UTC().Add(24 * time.Hour).String() - tokensDB.Put([]byte(token), []byte(to_put), nil) + err := vars.TokensDB.Put([]byte(token), []byte(to_put), nil) + if err != nil { + vars.TokensDB.Close() + vars.TokensDB, vars.TokensDBErr = leveldb.OpenFile("leveldb/tokens", nil) + + err = vars.TokensDB.Put([]byte(token), []byte(to_put), nil) + if err != nil { + panic(err) + } + } + fmt.Println("created token " + token) return token } func IsTokenExists(token string) bool { - tokensDB, _ := leveldb.OpenFile("leveldb/tokens", nil) - defer tokensDB.Close() - - iter := tokensDB.NewIterator(nil, nil) + iter := vars.TokensDB.NewIterator(nil, nil) defer iter.Release() iter.First() if string(iter.Key()) == token { - tokensDB.Put([]byte(token), []byte("was used"), nil) + vars.TokensDB.Put([]byte(token), []byte("was used"), nil) return true } for iter.Next() { if string(iter.Key()) == token { - tokensDB.Put([]byte(token), []byte("was used"), nil) + vars.TokensDB.Put([]byte(token), []byte("was used"), nil) return true } } - return false } func DeleteUnusedTokens() { for { - db, r := leveldb.OpenFile("leveldb/tokens", nil) - if r != nil { - panic(r) - } - + db := vars.TokensDB iter := db.NewIterator(nil, nil) for iter.Next() { wasUsed := string(iter.Value()) diff --git a/application/backend/app/routes/routes.go b/application/backend/app/routes/routes.go index 086d160..7cadf90 100644 --- a/application/backend/app/routes/routes.go +++ b/application/backend/app/routes/routes.go @@ -22,7 +22,6 @@ import ( "github.com/devforth/OnLogs/app/util" "github.com/devforth/OnLogs/app/vars" "github.com/gorilla/websocket" - "github.com/syndtr/goleveldb/leveldb" ) func enableCors(w *http.ResponseWriter) { @@ -128,16 +127,14 @@ func AddLogLine(w http.ResponseWriter, req *http.Request) { if vars.Counters_For_Hosts_Last_30_Min[logItem.Host] == nil { go statistics.RunStatisticForContainer(logItem.Host, logItem.Container) } - location := logItem.Host + "/" + logItem.Container - if vars.Statuses_DBs[location] == nil { - vars.Statuses_DBs[location] = util.GetDB(logItem.Host, logItem.Container, "statuses") + err := containerdb.PutLogMessage(util.GetDB(logItem.Host, logItem.Container, "logs"), logItem.Host, logItem.Container, logItem.LogLine) + if err != nil { + defer w.WriteHeader(http.StatusInternalServerError) + panic(err) } - current_db, _ := leveldb.OpenFile("leveldb/hosts/"+logItem.Host+"/containers/"+logItem.Container+"/logs", nil) - containerdb.PutLogMessage(current_db, logItem.Host, logItem.Container, logItem.LogLine) - defer current_db.Close() to_send, _ := json.Marshal([]string{logItem.LogLine[0], logItem.LogLine[1]}) - for _, c := range vars.Connections[location] { + for _, c := range vars.Connections[logItem.Host+"/"+logItem.Container] { c.WriteMessage(1, to_send) } } diff --git a/application/backend/app/statistics/statistics.go b/application/backend/app/statistics/statistics.go index 1d237e9..b3c553e 100644 --- a/application/backend/app/statistics/statistics.go +++ b/application/backend/app/statistics/statistics.go @@ -42,8 +42,8 @@ func RunStatisticForContainer(host string, container string) { location := host + "/" + container vars.Counters_For_Containers_Last_30_Min[location] = map[string]uint64{"error": 0, "debug": 0, "info": 0, "warn": 0, "meta": 0, "other": 0} if vars.Stat_Containers_DBs[location] == nil { - current_db, _ := leveldb.OpenFile("leveldb/hosts/"+host+"/containers/"+container+"/statistics", nil) - defer current_db.Close() + current_db := util.GetDB(host, container, "statistics") + // defer current_db.Close() vars.Stat_Containers_DBs[location] = current_db } defer delete(vars.Stat_Containers_DBs, location) diff --git a/application/backend/app/streamer/streamer.go b/application/backend/app/streamer/streamer.go index b66e0ee..af84c84 100644 --- a/application/backend/app/streamer/streamer.go +++ b/application/backend/app/streamer/streamer.go @@ -30,8 +30,7 @@ func createStreams(containers []string) { } } if vars.Statuses_DBs[util.GetHost()+"/"+container] == nil { - statusesDB, _ := leveldb.OpenFile("leveldb/hosts/"+util.GetHost()+"/containers/"+container+"/statuses", nil) - vars.Statuses_DBs[util.GetHost()+"/"+container] = statusesDB + vars.Statuses_DBs[util.GetHost()+"/"+container] = util.GetDB(util.GetHost(), container, "/statuses") } vars.ActiveDBs[container] = newDB vars.Active_Daemon_Streams = append(vars.Active_Daemon_Streams, container) diff --git a/application/backend/app/util/util.go b/application/backend/app/util/util.go index a9e66e4..d34e54d 100644 --- a/application/backend/app/util/util.go +++ b/application/backend/app/util/util.go @@ -102,6 +102,14 @@ func GetDB(host string, container string, dbType string) *leveldb.DB { fmt.Println("ERROR: unable to open db for "+host+"/"+container+"/"+dbType, err) } + if dbType == "logs" { + vars.ActiveDBs[container] = res_db + } else if dbType == "statuses" { + vars.Statuses_DBs[host+"/"+container] = res_db + } else if dbType == "statistics" { + vars.Stat_Containers_DBs[host+"/"+container] = res_db + } + return res_db } @@ -185,9 +193,17 @@ func GetDockerContainerID(host string, container string) string { return "" } - idDB, _ := leveldb.OpenFile("leveldb/hosts/"+host+"/containersMeta", nil) - defer idDB.Close() - iter := idDB.NewIterator(nil, nil) + containersMetaDB := vars.ContainersMeta_DBs[host] + if containersMetaDB == nil { + containersMetaDB, err := leveldb.OpenFile("leveldb/hosts/"+host+"/containersMeta", nil) + if err != nil { + panic(err) + } + vars.ContainersMeta_DBs[host] = containersMetaDB + } + containersMetaDB = vars.ContainersMeta_DBs[host] + + iter := containersMetaDB.NewIterator(nil, nil) defer iter.Release() iter.Last() diff --git a/application/backend/app/vars/vars.go b/application/backend/app/vars/vars.go index 16c12ca..47b20dd 100644 --- a/application/backend/app/vars/vars.go +++ b/application/backend/app/vars/vars.go @@ -9,22 +9,30 @@ import ( ) var ( - ActiveDBs = map[string]*leveldb.DB{} - Stat_Containers_DBs = map[string]*leveldb.DB{} - Stat_Hosts_DBs = map[string]*leveldb.DB{} - Statuses_DBs = map[string]*leveldb.DB{} - BrokenLogs_DBs = map[string]*leveldb.DB{} - Active_Daemon_Streams = []string{} - DockerContainers = []string{} - AgentsActiveContainers = map[string][]string{} - ToDelete = map[string][]string{} - Connections = map[string][]websocket.Conn{} + ActiveDBs = map[string]*leveldb.DB{} + Stat_Containers_DBs = map[string]*leveldb.DB{} + Stat_Hosts_DBs = map[string]*leveldb.DB{} + Statuses_DBs = map[string]*leveldb.DB{} + BrokenLogs_DBs = map[string]*leveldb.DB{} + ContainersMeta_DBs = map[string]*leveldb.DB{} + + Active_Daemon_Streams = []string{} + + DockerContainers = []string{} + AgentsActiveContainers = map[string][]string{} + + ToDelete = map[string][]string{} + Connections = map[string][]websocket.Conn{} + Counters_For_Hosts_Last_30_Min = map[string]map[string]uint64{} Counters_For_Containers_Last_30_Min = map[string]map[string]uint64{} - FavsDB, FavsDBErr = leveldb.OpenFile("leveldb/favourites", nil) - StateDB, StateDBErr = leveldb.OpenFile("leveldb/state", nil) - UsersDB, UsersDBErr = leveldb.OpenFile("leveldb/users", nil) // should i ever close it? - Year = strconv.Itoa(time.Now().UTC().Year()) + + FavsDB, FavsDBErr = leveldb.OpenFile("leveldb/favourites", nil) + StateDB, StateDBErr = leveldb.OpenFile("leveldb/state", nil) + UsersDB, UsersDBErr = leveldb.OpenFile("leveldb/users", nil) + TokensDB, TokensDBErr = leveldb.OpenFile("leveldb/tokens", nil) + + Year = strconv.Itoa(time.Now().UTC().Year()) ) type UserData struct { diff --git a/application/build.sh b/application/build.sh index 16d4006..8062213 100644 --- a/application/build.sh +++ b/application/build.sh @@ -1,2 +1,3 @@ docker buildx create --use -docker buildx build --platform=linux/amd64,linux/arm64 --tag "devforth/onlogs:latest" --tag "devforth/onlogs:1.0.4" --push . +# docker buildx build --platform=linux/amd64,linux/arm64 --tag "devforth/onlogs:latest" --tag "devforth/onlogs:1.0.5" --push . +docker buildx build --platform=linux/amd64,linux/arm64 --tag "devforth/onlogs:1.0.4-beta-7" --push .