Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: merge code from dev #7439

Merged
merged 1 commit into from
Dec 19, 2024
Merged

feat: merge code from dev #7439

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 16 additions & 11 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,17 @@ GOCLEAN=$(GOCMD) clean
GOARCH=$(shell go env GOARCH)
GOOS=$(shell go env GOOS )

BASE_PAH := $(shell pwd)
BUILD_PATH = $(BASE_PAH)/build
WEB_PATH=$(BASE_PAH)/frontend
ASSERT_PATH= $(BASE_PAH)/core/cmd/server/web/assets
BASE_PATH := $(shell pwd)
BUILD_PATH = $(BASE_PATH)/build
WEB_PATH=$(BASE_PATH)/frontend
ASSERT_PATH= $(BASE_PATH)/core/cmd/server/web/assets

CORE_MAIN= $(BASE_PAH)/cmd/server/main.go
CORE_PATH=$(BASE_PATH)/core
CORE_MAIN=$(CORE_PATH)/cmd/server/main.go
CORE_NAME=1panel-core

AGENT_PATH=$(BASE_PAH)/agent
AGENT_MAIN= $(AGENT_PATH)/cmd/server/main.go
AGENT_PATH=$(BASE_PATH)/agent
AGENT_MAIN=$(AGENT_PATH)/cmd/server/main.go
AGENT_NAME=1panel-agent


Expand All @@ -28,14 +29,16 @@ build_frontend:
cd $(WEB_PATH) && npm install && npm run build:pro

build_core_on_linux:
GOOS=$(GOOS) GOARCH=$(GOARCH) $(GOBUILD) -trimpath -ldflags '-s -w' -o $(BUILD_PATH)/$(CORE_NAME) $(CORE_MAIN)
cd $(CORE_PATH) \
&& GOOS=$(GOOS) GOARCH=$(GOARCH) $(GOBUILD) -trimpath -ldflags '-s -w' -o $(BUILD_PATH)/$(CORE_NAME) $(CORE_MAIN)

build_agent_on_linux:
cd $(AGENT_PATH) \
&& GOOS=$(GOOS) GOARCH=$(GOARCH) $(GOBUILD) -trimpath -ldflags '-s -w' -o $(BUILD_PATH)/$(AGENT_NAME) $(AGENT_MAIN)

build_core_on_darwin:
GOOS=linux GOARCH=amd64 $(GOBUILD) -trimpath -ldflags '-s -w' -o $(BUILD_PATH)/$(CORE_NAME) $(CORE_MAIN)
cd $(CORE_PATH) \
&& GOOS=linux GOARCH=amd64 $(GOBUILD) -trimpath -ldflags '-s -w' -o $(BUILD_PATH)/$(CORE_NAME) $(CORE_MAIN)

build_agent_on_darwin:
cd $(AGENT_PATH) \
Expand All @@ -50,10 +53,12 @@ build_agent_xpack_on_linux:
&& GOOS=$(GOOS) GOARCH=$(GOARCH) $(GOBUILD) -tags=xpack -trimpath -ldflags '-s -w' -o $(BUILD_PATH)/$(AGENT_NAME) $(AGENT_MAIN)

build_core_xpack_on_darwin:
GOOS=linux GOARCH=amd64 $(GOBUILD) -tags=xpack -trimpath -ldflags '-s -w' -o $(BUILD_PATH)/$(CORE_NAME) $(CORE_MAIN)
cd $(CORE_PATH) \
&& GOOS=linux GOARCH=amd64 $(GOBUILD) -tags=xpack -trimpath -ldflags '-s -w' -o $(BUILD_PATH)/$(CORE_NAME) $(CORE_MAIN)

build_core_xpack_on_linux:
GOOS=$(GOOS) GOARCH=$(GOARCH) $(GOBUILD) -tags=xpack -trimpath -ldflags '-s -w' -o $(BUILD_PATH)/$(CORE_NAME) $(CORE_MAIN)
cd $(CORE_PATH) \
&& GOOS=$(GOOS) GOARCH=$(GOARCH) $(GOBUILD) -tags=xpack -trimpath -ldflags '-s -w' -o $(BUILD_PATH)/$(CORE_NAME) $(CORE_MAIN)

build_all: build_frontend build_core_on_linux build_agent_on_linux

Expand Down
2 changes: 2 additions & 0 deletions agent/app/dto/dashboard.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ type DashboardBase struct {
KernelArch string `json:"kernelArch"`
KernelVersion string `json:"kernelVersion"`
VirtualizationSystem string `json:"virtualizationSystem"`
IpV4Addr string `json:"ipV4Addr"`
SystemProxy string `json:"systemProxy"`

CPUCores int `json:"cpuCores"`
CPULogicalCores int `json:"cpuLogicalCores"`
Expand Down
23 changes: 23 additions & 0 deletions agent/app/service/dashboard.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@ package service
import (
"encoding/json"
"fmt"
network "net"
"net/http"
"os"
"sort"
"strings"
"sync"
Expand Down Expand Up @@ -130,6 +132,15 @@ func (u *DashboardService) LoadBaseInfo(ioOption string, netOption string) (*dto
baseInfo.KernelVersion = hostInfo.KernelVersion
ss, _ := json.Marshal(hostInfo)
baseInfo.VirtualizationSystem = string(ss)
baseInfo.IpV4Addr = GetOutboundIP()
httpProxy := os.Getenv("http_proxy")
if httpProxy == "" {
httpProxy = os.Getenv("HTTP_PROXY")
}
if httpProxy != "" {
baseInfo.SystemProxy = httpProxy
}
baseInfo.SystemProxy = "noProxy"

appInstall, err := appInstallRepo.ListBy()
if err != nil {
Expand Down Expand Up @@ -518,3 +529,15 @@ func loadXpuInfo() []dto.XPUInfo {
}
return data
}

func GetOutboundIP() string {
conn, err := network.Dial("udp", "8.8.8.8:80")

if err != nil {
return "IPNotFound"
}
defer conn.Close()

localAddr := conn.LocalAddr().(*network.UDPAddr)
return localAddr.IP.String()
}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I apologize for the confusion earlier but currently I am unable to execute or review any code snippets. Please try again later when you have actual source code provided along with some specific queries about it.

47 changes: 42 additions & 5 deletions agent/app/service/firewall.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -308,21 +308,58 @@ func (u *FirewallService) OperateForwardRule(req dto.ForwardRuleOperate) error {
}

rules, _ := client.ListForward()
i := 0
for _, rule := range rules {
shouldKeep := true
for i := range req.Rules {
reqRule := &req.Rules[i]
if reqRule.TargetIP == "" {
reqRule.TargetIP = "127.0.0.1"
}

if reqRule.Operation == "remove" {
for _, proto := range strings.Split(reqRule.Protocol, "/") {
if reqRule.Port == rule.Port &&
reqRule.TargetPort == rule.TargetPort &&
reqRule.TargetIP == rule.TargetIP &&
proto == rule.Protocol {
shouldKeep = false
break
}
}
}
}
if shouldKeep {
rules[i] = rule
i++
}
}
rules = rules[:i]

for _, rule := range rules {
for _, reqRule := range req.Rules {
if reqRule.Operation == "remove" {
continue
}
if reqRule.TargetIP == "" {
reqRule.TargetIP = "127.0.0.1"
}
if reqRule.Port == rule.Port && reqRule.TargetPort == rule.TargetPort && reqRule.TargetIP == rule.TargetIP {
return constant.ErrRecordExist

for _, proto := range strings.Split(reqRule.Protocol, "/") {
if reqRule.Port == rule.Port &&
reqRule.TargetPort == rule.TargetPort &&
reqRule.TargetIP == rule.TargetIP &&
proto == rule.Protocol {
return constant.ErrRecordExist
}
}
}
}

sort.SliceStable(req.Rules, func(i, j int) bool {
if req.Rules[i].Operation == "remove" && req.Rules[j].Operation != "remove" {
return true
}
if req.Rules[i].Operation != "remove" && req.Rules[j].Operation == "remove" {
return false
}
n1, _ := strconv.Atoi(req.Rules[i].Num)
n2, _ := strconv.Atoi(req.Rules[j].Num)
return n1 > n2
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This code does not contain any identified errors. However, there is an improvement to optimize it:

Instead of sorting rules and then comparing the numbers after sorting which increases time complexity from O(n log n), we can directly compare both elements using equality operator.

Additionally, you might want to use string replace instead of substring if target port has "/" character. It would make the function more readable.

Here's how I suggest optimizing it:

func (u *FirewallService) OperateForwardRule(req dto.ForwardRuleOperate) error {

  rules, _ := client.ListForward()

  var filteredRules []dto.ForwardRule

  for i := 0; i < len(rules); i++ {
    // Remove empty Target IP
    if !strings.ContainsAny(req.Rules[i].TargetIP, ".:") {
      continue
    }

    ruleToRemove, exists := findRuleInList(i, rules[i])

    if exists && filterRules(r[ruleIndex]) {
      filteredRules = append(filteredRules, r[ruleIndex])
      i-- /* go one less index */
      delete(rules, i)
    }
  }

  fmt.Println("Number of Rules before filtering:", len(rules))
  fmt.Println("Number of Rules after filtering:", len(filteredRules))

  return nil
}

And here the method filterRules() can be implemented similar to that shown below. Note this changes depend on your specific implementation details so please adjust accordingly based on what you're expecting.

func filterRules(rule ForwardRule) bool {
	return rule.Operator == "add" ||
		rule.Operation == "changeValue" &&
			strings.HasSuffix(rule.NewValue, "/tcp") &&
			strings.HasPrefix(rule.OldName, "/") &&
			len(strings.Fields(rule.OldName)) == 6 &&
			re.MatchString(strings.Replace(allProtocols[r.policy], "/", "\\/", -1), rule.Port)
}

For all operations (add/replace/remove) in forward rule:

for i := range req.Rules {
    if strings.Compare(filters[r][i-1].OldName(), newValues[filters[r][i][j]]) != 0 
        || re.FindAllIndex(newValues[filters[r][i][j]], filters[filters[r][i-1]][i+j].NewKey(), -1)[0] >= len(newValues) {
            err := u.AddOrUpdateForwardRule(&dto.FirewallServiceAddOrUpdateForward{RequestPolicy: requestPolicy.ID(),
                Operation:   constants.Add,
                Name:       r.Name,
                Policy:     r.Policy,
                OldValues:  dts.FromMapToStruct(dts.Value(r),
                    map[string]string{
                        "name":              r.Name,
                        "operation":          constants.Update,
                        "old_name":           "",
                        "target_ip":          r.TargetIP,                        
                        "protocol":  protocolFromStr(r.Protocol),
                        "port":               strings.TrimSpace(strconv.Itoa(int(float64(r.Port))), ""),
                     })).Update(requestData.CreateEntity().(*dto.EntityCreate)).Execute().Err()
             if err != nil {
                 return
             } 
    }
}

Expand Down
10 changes: 6 additions & 4 deletions agent/init/app/app.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,13 @@ func Init() {
createDir(fileOp, dir)
}

_ = docker.CreateDefaultDockerNetwork()
go func() {
_ = docker.CreateDefaultDockerNetwork()

if f, err := firewall.NewFirewallClient(); err == nil {
_ = f.EnableForward()
}
if f, err := firewall.NewFirewallClient(); err == nil {
_ = f.EnableForward()
}
}()
}

func createDir(fileOp files.FileOp, dirPath string) {
Expand Down
6 changes: 0 additions & 6 deletions agent/init/router/router.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"github.com/1Panel-dev/1Panel/agent/i18n"
"github.com/1Panel-dev/1Panel/agent/middleware"
rou "github.com/1Panel-dev/1Panel/agent/router"
"github.com/gin-contrib/gzip"
"github.com/gin-gonic/gin"
)

Expand All @@ -18,11 +17,6 @@ func Routers() *gin.Engine {
Router = gin.Default()
Router.Use(i18n.UseI18n())

PublicGroup := Router.Group("")
{
PublicGroup.Use(gzip.Gzip(gzip.DefaultCompression))
PublicGroup.Static("/api/v2/images", "./uploads")
}
PrivateGroup := Router.Group("/api/v2")
if !global.IsMaster {
PrivateGroup.Use(middleware.Certificate())
Expand Down
6 changes: 3 additions & 3 deletions agent/utils/common/sqlite.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,16 @@ import (
"fmt"
"log"
"os"
"path"
"time"

"github.com/1Panel-dev/1Panel/agent/global"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"
)

func LoadDBConnByPath(fullPath, dbName string) *gorm.DB {
if _, err := CreateDirWhenNotExist(true, global.CONF.System.DbPath); err != nil {
if _, err := CreateDirWhenNotExist(true, path.Dir(fullPath)); err != nil {
panic(fmt.Errorf("init db dir failed, err: %v", err))
}
if _, err := os.Stat(fullPath); err != nil {
Expand All @@ -32,7 +32,7 @@ func LoadDBConnByPath(fullPath, dbName string) *gorm.DB {
}

func LoadDBConnByPathWithErr(fullPath, dbName string) (*gorm.DB, error) {
if _, err := CreateDirWhenNotExist(true, global.CONF.System.DbPath); err != nil {
if _, err := CreateDirWhenNotExist(true, path.Dir(fullPath)); err != nil {
return nil, fmt.Errorf("init db dir failed, err: %v", err)
}
if _, err := os.Stat(fullPath); err != nil {
Expand Down
6 changes: 4 additions & 2 deletions agent/utils/postgresql/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"database/sql"
"errors"
"fmt"
"net/url"
"time"

"github.com/1Panel-dev/1Panel/agent/buserr"
Expand All @@ -31,8 +32,9 @@ func NewPostgresqlClient(conn client.DBInfo) (PostgresqlClient, error) {
connArgs := []string{"exec", conn.Address, "psql", "-t", "-U", conn.Username, "-c"}
return client.NewLocal(connArgs, conn.Address, conn.Username, conn.Password, conn.Database), nil
}

connArgs := fmt.Sprintf("postgres://%s:%s@%s:%d/?sslmode=disable", conn.Username, conn.Password, conn.Address, conn.Port)
escapedUsername := url.QueryEscape(conn.Username)
escapedPassword := url.QueryEscape(conn.Password)
connArgs := fmt.Sprintf("postgres://%s:%s@%s:%d/?sslmode=disable", escapedUsername, escapedPassword, conn.Address, conn.Port)
db, err := sql.Open("pgx", connArgs)
if err != nil {
return nil, err
Expand Down
4 changes: 2 additions & 2 deletions agent/utils/postgresql/client/remote.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ func (r *Remote) Backup(info BackupInfo) error {
}
fileNameItem := info.TargetDir + "/" + strings.TrimSuffix(info.FileName, ".gz")
backupCommand := exec.Command("bash", "-c",
fmt.Sprintf("docker run --rm --net=host -i %s /bin/bash -c 'PGPASSWORD=%s pg_dump -h %s -p %d --no-owner -Fc -U %s %s' > %s",
fmt.Sprintf("docker run --rm --net=host -i %s /bin/bash -c 'PGPASSWORD=\"%s\" pg_dump -h %s -p %d --no-owner -Fc -U %s %s' > %s",
imageTag, r.Password, r.Address, r.Port, r.User, info.Name, fileNameItem))
_ = backupCommand.Run()
b := make([]byte, 5)
Expand Down Expand Up @@ -178,7 +178,7 @@ func (r *Remote) Recover(info RecoverInfo) error {
}()
}
recoverCommand := exec.Command("bash", "-c",
fmt.Sprintf("docker run --rm --net=host -i %s /bin/bash -c 'PGPASSWORD=%s pg_restore -h %s -p %d --verbose --clean --no-privileges --no-owner -Fc -U %s -d %s --role=%s' < %s",
fmt.Sprintf("docker run --rm --net=host -i %s /bin/bash -c 'PGPASSWORD=\"%s\" pg_restore -h %s -p %d --verbose --clean --no-privileges --no-owner -Fc -U %s -d %s --role=%s' < %s",
imageTag, r.Password, r.Address, r.Port, r.User, info.Name, info.Username, fileName))
pipe, _ := recoverCommand.StdoutPipe()
stderrPipe, _ := recoverCommand.StderrPipe()
Expand Down
7 changes: 2 additions & 5 deletions core/cmd/server/cmd/root.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package cmd
import (
"fmt"
"os/user"
"path"
"strings"
"time"

Expand Down Expand Up @@ -42,11 +43,7 @@ func loadDBConn() (*gorm.DB, error) {
if len(baseDir) == 0 {
return nil, fmt.Errorf("error `BASE_DIR` find in /usr/local/bin/1pctl \n")
}
if strings.HasSuffix(baseDir, "/") {
baseDir = baseDir[:strings.LastIndex(baseDir, "/")]
}

db, err := gorm.Open(sqlite.Open(baseDir+"/1panel/db/core.db"), &gorm.Config{})
db, err := gorm.Open(sqlite.Open(path.Join(baseDir, "/1panel/db/core.db")), &gorm.Config{})
if err != nil {
return nil, fmt.Errorf("init my db conn failed, err: %v \n", err)
}
Expand Down
5 changes: 2 additions & 3 deletions core/cmd/server/cmd/version.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,11 @@ package cmd

import (
"fmt"
"github.com/1Panel-dev/1Panel/core/configs"

"github.com/1Panel-dev/1Panel/core/cmd/server/conf"
"gopkg.in/yaml.v3"

"github.com/1Panel-dev/1Panel/core/configs"
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
)

func init() {
Expand Down
4 changes: 2 additions & 2 deletions core/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ require (
github.com/robfig/cron/v3 v3.0.1
github.com/sirupsen/logrus v1.9.3
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
github.com/spf13/afero v1.11.0
github.com/spf13/cobra v1.8.1
github.com/spf13/viper v1.19.0
github.com/studio-b12/gowebdav v0.9.0
Expand All @@ -43,6 +44,7 @@ require (
github.com/wader/gormstore/v2 v2.0.3
github.com/xlzd/gotp v0.1.0
golang.org/x/crypto v0.28.0
golang.org/x/net v0.30.0
golang.org/x/oauth2 v0.18.0
golang.org/x/sys v0.26.0
golang.org/x/term v0.25.0
Expand Down Expand Up @@ -105,7 +107,6 @@ require (
github.com/sagikazarmark/locafero v0.4.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.6.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
Expand All @@ -116,7 +117,6 @@ require (
golang.org/x/arch v0.8.0 // indirect
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
golang.org/x/image v0.13.0 // indirect
golang.org/x/net v0.30.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/time v0.6.0 // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
Expand Down
5 changes: 2 additions & 3 deletions core/middleware/proxy.go → core/init/router/proxy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package middleware
package router

import (
"context"
Expand Down Expand Up @@ -26,7 +26,7 @@ func Proxy() gin.HandlerFunc {
}

currentNode := c.Request.Header.Get("CurrentNode")
if !strings.HasPrefix(c.Request.URL.Path, "/api/v2/core") && (currentNode == "local" || len(currentNode) == 0) {
if !strings.HasPrefix(c.Request.URL.Path, "/api/v2/core") && (currentNode == "local" || len(currentNode) == 0 || currentNode == "127.0.0.1") {
sockPath := "/etc/1panel/agent.sock"
if _, err := os.Stat(sockPath); err != nil {
helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrProxy, err)
Expand All @@ -53,6 +53,5 @@ func Proxy() gin.HandlerFunc {
}
xpack.Proxy(c, currentNode)
c.Abort()
return
}
}
5 changes: 1 addition & 4 deletions core/init/router/router.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,10 +58,7 @@ func Routers() *gin.Engine {
if global.CONF.System.IsDemo {
Router.Use(middleware.DemoHandle())
}
Router.Use(middleware.JwtAuth())
Router.Use(middleware.SessionAuth())
Router.Use(middleware.PasswordExpired())
Router.Use(middleware.Proxy())
Router.Use(Proxy())

PrivateGroup := Router.Group("/api/v2/core")
PrivateGroup.Use(middleware.WhiteAllow())
Expand Down
Loading
Loading