Auto-update with remote registry URL (#13)

* go mod tidy + .vscode in gitignore

* registry pkg

* remove registryutil.go

* lint, update startAction and cleaning

* remove log

* channel great type + gather "close" in torProxy.Close method

* remove TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256

* Revert "remove TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256"

This reverts commit 81b9308.

* remove "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA"

* auto-update-period flag

.gitignore

@@ -3,3 +3,5 @@
 /dist
 localhost.key
 localhost.crt
+
+.vscode

Makefile

@@ -14,7 +14,7 @@ clean:
 ## fmt: Go Format
 fmt:
 	@echo "Gofmt..."
-	if [ -n "$(gofmt -l ./...)" ]; then echo "Go code is not formatted"; exit 1; fi
+	@if [ -n "$(gofmt -l ./...)" ]; then echo "Go code is not formatted"; exit 1; fi
 
 ## help: prints this help message
 help:

README.md

@@ -39,6 +39,7 @@ $ torproxy start --domain mywebsite.com --registry '[{"endpoint": "http://somewh
 $ torproxy start --domain mywebsite.com --registry https://raw.githubusercontent.com/tdex-network/tdex-registry/master/registry.json
 ```
 
+With a URL, the proxy will refetch the registry every 12 hours in order to auto-update the set of endpoints to redirects.
 
 * Load registry from local path to file
 

cmd/start.go

@@ -4,11 +4,12 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"net/url"
 	"os"
 	"os/signal"
 	"syscall"
+	"time"
 
+	registrypkg "github.com/tdex-network/tor-proxy/pkg/registry"
 	"github.com/tdex-network/tor-proxy/pkg/torproxy"
 	"github.com/urfave/cli/v2"
 	"github.com/weppos/publicsuffix-go/publicsuffix"
@@ -64,25 +65,19 @@ var start = cli.Command{
 			Usage: "the socks5 port exposed by the tor client",
 			Value: 9050,
 		},
+		&cli.IntFlag{
+			Name:  "auto-update-period",
+			Usage: "period in hours to check for new endpoints",
+			Value: 12,
+		},
 	},
 	Action: startAction,
 }
 
 func startAction(ctx *cli.Context) error {
-
-	// load registry json
-	registryBytes, err := getRegistryJSON(ctx.String("registry"))
-	if err != nil {
-		return fmt.Errorf("laoding json: %w", err)
-	}
-
-	// parse registry json
-	redirects, err := registryJSONToRedirects(registryBytes)
-	if err != nil {
-		return fmt.Errorf("validating json: %w", err)
-	}
-
 	var proxy *torproxy.TorProxy
+	var err error
+
 	if ctx.Bool("use-tor") {
 		// use the embedded tor client and expose it on :9050
 		proxy, err = torproxy.NewTorProxy()
@@ -97,8 +92,27 @@ func startAction(ctx *cli.Context) error {
 		return fmt.Errorf("creating tor instance: %w", err)
 	}
 
-	// Add redirects to the proxy
-	proxy.WithRedirects(redirects)
+	// create registry
+	registry, err := registrypkg.NewRegistry(ctx.String("registry"))
+	if err != nil {
+		return fmt.Errorf("loading json: %w", err)
+	}
+
+	// Add registry to the proxy
+	// this will init the set of redirects
+	// in case of remote registry (an URL): start auto-updater
+	proxy.WithRegistry(registry)
+
+	if proxy.Registry.RegistryType() == registrypkg.RemoteRegistryType {
+		errorHandler := func (err error) {
+			log.Println("registry auto update error: %w", err) 
+		}
+
+		period := ctx.Int("auto-update-period")
+		autoUpdatePeriod := time.Duration(period) * time.Hour
+		log.Printf("starting registry auto update every %s", autoUpdatePeriod)
+		proxy.WithAutoUpdater(autoUpdatePeriod, errorHandler)
+	}
 
 	// check if insecure flag, otherwise either domain or key & cert paths MUST be present to serve with TLS
 	var address string
@@ -136,7 +150,8 @@ func startAction(ctx *cli.Context) error {
 	if err := proxy.Serve(address, tlsOptions); err != nil {
 		return fmt.Errorf("serving proxy: %w", err)
 	}
-	defer proxy.Listener.Close()
+	// close the proxy when the process is interrupted
+	defer proxy.Close() // close the auto-updater in case of remote registry
 
 	// Catch SIGTERM and SIGINT signals
 	sigChan := make(chan os.Signal, 1)
@@ -148,41 +163,7 @@ func startAction(ctx *cli.Context) error {
 	return nil
 }
 
-func isValidURL(s string) bool {
-	_, err := url.ParseRequestURI(s)
-	if err != nil {
-		return false
-	}
-
-	return true
-}
-
 func isValidDomain(d string) bool {
 	_, err := publicsuffix.Parse(d)
-	if err != nil {
-		return false
-	}
-
-	return true
-}
-
-// getRegistryJSON will check if the given string is a) a JSON by itself b) if is a path to a file c) remote url
-func getRegistryJSON(source string) ([]byte, error) {
-
-	// check if it is a json the given source already
-	if isArrayOfObjectsJSON(source) {
-		return []byte(source), nil
-	}
-
-	// check if is a valid URL
-	if isValidURL(source) {
-		return fetchFromRemoteURL(source)
-	}
-
-	// in the end check if is a path to a file. If it exists try to read
-	if _, err := os.Stat(source); !os.IsNotExist(err) {
-		return fetchFromFilePath(source)
-	}
-
-	return nil, errors.New("source must be either a valid JSON string, a remote URL or a valid path to a JSON file")
+	return err == nil
 }

go.mod

@@ -3,12 +3,17 @@ module github.com/tdex-network/tor-proxy
 go 1.15
 
 require (
-	github.com/caddyserver/certmagic v0.12.0
-	github.com/cretz/bine v0.1.0
-	github.com/ipsn/go-libtor v1.0.366
+	github.com/caddyserver/certmagic v0.15.2
+	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
+	github.com/cretz/bine v0.2.0
+	github.com/ipsn/go-libtor v1.0.380
 	github.com/urfave/cli/v2 v2.3.0
-	github.com/weppos/publicsuffix-go v0.13.0
-	golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect
-	golang.org/x/net v0.0.0-20210119194325-5f4716e94777
-	golang.org/x/sys v0.0.0-20210217105451-b926d437f341 // indirect
+	github.com/weppos/publicsuffix-go v0.15.0
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/multierr v1.7.0 // indirect
+	go.uber.org/zap v1.19.1 // indirect
+	golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 // indirect
+	golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9
+	golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 // indirect
+	golang.org/x/text v0.3.7 // indirect
 )