From 44507efe421d84d82d86f55641e71d19688df58c Mon Sep 17 00:00:00 2001 From: Sameh Abouel-saad Date: Thu, 31 Oct 2024 14:46:43 +0300 Subject: [PATCH] implement alwaysAllowedIDs --- .app.env.example | 4 +- README.md | 9 ++- internal/clients/idenfy/idenfy.go | 36 ++++------ internal/configs/config.go | 50 ++++++++++++- internal/server/server.go | 77 ++++++++++++--------- internal/services/kyc_service.go | 18 ++++- scripts/dev/auth/generate-test-auth-data.go | 2 +- 7 files changed, 131 insertions(+), 65 deletions(-) diff --git a/.app.env.example b/.app.env.example index 37f4ba5..ef425f6 100644 --- a/.app.env.example +++ b/.app.env.example @@ -18,4 +18,6 @@ IP_LIMITER_TOKEN_EXPIRATION=1440 ID_LIMITER_MAX_TOKEN_REQUESTS=5 ID_LIMITER_TOKEN_EXPIRATION=1440 DEBUG=false -IDENFY_CALLBACK_URL=https://kyc.dev.grid.tf/webhooks/idenfy/verification-update \ No newline at end of file +IDENFY_CALLBACK_URL=https://kyc.dev.grid.tf/webhooks/idenfy/verification-update +IDENFY_NAMESPACE= +VERIFICATION_ALWAYS_VERIFIED_IDS= \ No newline at end of file diff --git a/README.md b/README.md index fe0042b..a3f53c6 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ The application uses environment variables for configuration. Here's a list of a - `IDENFY_API_KEY`: API key for iDenfy service (required) (note: make sure to use correct iDenfy API key for the environment dev, test, and production) (iDenfy dev -> TFChain Devnet, iDenfy test -> TFChain QAnet, iDenfy prod -> TFChain Testnet and Mainnet) - `IDENFY_API_SECRET`: API secret for iDenfy service (required) - `IDENFY_BASE_URL`: Base URL for iDenfy API (default: "") -- `IDENFY_CALLBACK_SIGN_KEY`: Callback signing key for iDenfy webhooks (required) (note: should match the signing key in iDenfy dashboard for the related environment) +- `IDENFY_CALLBACK_SIGN_KEY`: Callback signing key for iDenfy webhooks (required) (note: should match the signing key in iDenfy dashboard for the related environment and should be at least 32 characters long) - `IDENFY_WHITELISTED_IPS`: Comma-separated list of whitelisted IPs for iDenfy callbacks - `IDENFY_DEV_MODE`: Enable development mode for iDenfy integration (default: false) (note: works only in iDenfy dev environment, enabling it in test or production environment will cause iDenfy to reject the requests) - `IDENFY_CALLBACK_URL`: URL for iDenfy verification update callbacks. (example: `https://{KYC-SERVICE-DOMAIN}/webhooks/idenfy/verification-update`) @@ -94,6 +94,13 @@ The application uses environment variables for configuration. Here's a list of a To configure these options, you can either set them as environment variables or include them in your `.env` file. +Regarding the iDenfy signing key, it's best to use key composed of alphanumeric characters to avoid such issues. +You can generate a random key using the following command: + +```bash +cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1 +``` + Refer to `internal/configs/config.go` for the implementation details of these configuration options. ## Running the Application diff --git a/internal/clients/idenfy/idenfy.go b/internal/clients/idenfy/idenfy.go index 2872746..7a31c50 100644 --- a/internal/clients/idenfy/idenfy.go +++ b/internal/clients/idenfy/idenfy.go @@ -18,14 +18,9 @@ import ( ) type Idenfy struct { - client *fasthttp.Client - accessKey string - secretKey string - baseURL string - callbackSignKey []byte - callbackUrl string - devMode bool - logger *logger.Logger + client *fasthttp.Client + config *configs.Idenfy + logger *logger.Logger } const ( @@ -34,19 +29,14 @@ const ( func New(config configs.Idenfy, logger *logger.Logger) *Idenfy { return &Idenfy{ - baseURL: config.BaseURL, - client: &fasthttp.Client{}, - accessKey: config.APIKey, - secretKey: config.APISecret, - callbackSignKey: []byte(config.CallbackSignKey), - callbackUrl: config.CallbackUrl, - devMode: config.DevMode, - logger: logger, + client: &fasthttp.Client{}, + config: &config, + logger: logger, } } func (c *Idenfy) CreateVerificationSession(ctx context.Context, clientID string) (models.Token, error) { // TODO: Refactor - url := c.baseURL + VerificationSessionEndpoint + url := c.config.BaseURL + VerificationSessionEndpoint req := fasthttp.AcquireRequest() defer fasthttp.ReleaseRequest(req) @@ -56,11 +46,11 @@ func (c *Idenfy) CreateVerificationSession(ctx context.Context, clientID string) req.Header.Set("Content-Type", "application/json") // Set basic auth - authStr := c.accessKey + ":" + c.secretKey + authStr := c.config.APIKey + ":" + c.config.APISecret auth := base64.StdEncoding.EncodeToString([]byte(authStr)) req.Header.Set("Authorization", "Basic "+auth) - RequestBody := c.createVerificationSessionRequestBody(clientID, c.devMode) + RequestBody := c.createVerificationSessionRequestBody(clientID, c.config.DevMode) jsonBody, err := json.Marshal(RequestBody) if err != nil { @@ -91,19 +81,19 @@ func (c *Idenfy) CreateVerificationSession(ctx context.Context, clientID string) // verify signature of the callback func (c *Idenfy) VerifyCallbackSignature(ctx context.Context, body []byte, sigHeader string) error { - if len(c.callbackSignKey) < 1 { + if len(c.config.CallbackSignKey) < 1 { return errors.New("callback was received but no signature key was provided") } sig, err := hex.DecodeString(sigHeader) if err != nil { return err } - mac := hmac.New(sha256.New, c.callbackSignKey) + mac := hmac.New(sha256.New, []byte(c.config.CallbackSignKey)) mac.Write(body) if !hmac.Equal(sig, mac.Sum(nil)) { - c.logger.Error("Signature verification failed", zap.String("sigHeader", sigHeader), zap.String("key", string(c.callbackSignKey)), zap.String("mac", hex.EncodeToString(mac.Sum(nil)))) + c.logger.Error("Signature verification failed", zap.String("sigHeader", sigHeader), zap.String("key", string(c.config.CallbackSignKey)), zap.String("mac", hex.EncodeToString(mac.Sum(nil)))) return errors.New("signature verification failed") } return nil @@ -114,7 +104,7 @@ func (c *Idenfy) createVerificationSessionRequestBody(clientID string, devMode b RequestBody := map[string]interface{}{ "clientId": clientID, "generateDigitString": true, - "callbackUrl": c.callbackUrl, + "callbackUrl": c.config.CallbackUrl, } if devMode { RequestBody["expiryTime"] = 30 diff --git a/internal/configs/config.go b/internal/configs/config.go index 428b89b..dae147f 100644 --- a/internal/configs/config.go +++ b/internal/configs/config.go @@ -1,6 +1,9 @@ package configs import ( + "net/url" + "slices" + "example.com/tfgrid-kyc-service/internal/errors" "github.com/ilyakaznacheev/cleanenv" ) @@ -32,14 +35,16 @@ type Idenfy struct { WhitelistedIPs []string `env:"IDENFY_WHITELISTED_IPS" env-separator:","` DevMode bool `env:"IDENFY_DEV_MODE" env-default:"false"` CallbackUrl string `env:"IDENFY_CALLBACK_URL" env-required:"false"` + Namespace string `env:"IDENFY_NAMESPACE" env-default:""` } type TFChain struct { WsProviderURL string `env:"TFCHAIN_WS_PROVIDER_URL" env-default:"wss://tfchain.grid.tf"` } type Verification struct { - SuspiciousVerificationOutcome string `env:"VERIFICATION_SUSPICIOUS_VERIFICATION_OUTCOME" env-default:"APPROVED"` - ExpiredDocumentOutcome string `env:"VERIFICATION_EXPIRED_DOCUMENT_OUTCOME" env-default:"REJECTED"` - MinBalanceToVerifyAccount uint64 `env:"VERIFICATION_MIN_BALANCE_TO_VERIFY_ACCOUNT" env-default:"10000000"` + SuspiciousVerificationOutcome string `env:"VERIFICATION_SUSPICIOUS_VERIFICATION_OUTCOME" env-default:"APPROVED"` + ExpiredDocumentOutcome string `env:"VERIFICATION_EXPIRED_DOCUMENT_OUTCOME" env-default:"REJECTED"` + MinBalanceToVerifyAccount uint64 `env:"VERIFICATION_MIN_BALANCE_TO_VERIFY_ACCOUNT" env-default:"10000000"` + AlwaysVerifiedIDs []string `env:"VERIFICATION_ALWAYS_VERIFIED_IDS" env-separator:","` } type IPLimiter struct { MaxTokenRequests int `env:"IP_LIMITER_MAX_TOKEN_REQUESTS" env-default:"4"` @@ -63,5 +68,44 @@ func LoadConfig() (*Config, error) { if err != nil { return nil, errors.NewInternalError("error loading config", err) } + cfg.Validate() return cfg, nil } + +// validate config +func (c *Config) Validate() error { + // iDenfy base URL should be https://ivs.idenfy.com + if c.Idenfy.BaseURL != "https://ivs.idenfy.com" { + panic("invalid iDenfy base URL") + } + // CallbackUrl should be valid URL + parsedCallbackUrl, err := url.ParseRequestURI(c.Idenfy.CallbackUrl) + if err != nil { + panic("invalid CallbackUrl") + } + // CallbackSignKey should not be empty + if len(c.Idenfy.CallbackSignKey) < 16 { + panic("CallbackSignKey should be at least 16 characters long") + } + // WsProviderURL should be valid URL and start with wss:// + if u, err := url.ParseRequestURI(c.TFChain.WsProviderURL); err != nil || u.Scheme != "wss" { + panic("invalid WsProviderURL") + } + // domain should not be empty and same as domain in CallbackUrl + if parsedCallbackUrl.Host != c.Challenge.Domain { + panic("invalid Challenge Domain. It should be same as domain in CallbackUrl") + } + // Window should be greater than 2 + if c.Challenge.Window < 2 { + panic("invalid Challenge Window. It should be greater than 2 otherwise it will be too short and verification can fail in slow networks") + } + // SuspiciousVerificationOutcome should be either APPROVED or REJECTED + if !slices.Contains([]string{"APPROVED", "REJECTED"}, c.Verification.SuspiciousVerificationOutcome) { + panic("invalid SuspiciousVerificationOutcome") + } + // ExpiredDocumentOutcome should be either APPROVED or REJECTED + if !slices.Contains([]string{"APPROVED", "REJECTED"}, c.Verification.ExpiredDocumentOutcome) { + panic("invalid ExpiredDocumentOutcome") + } + return nil +} diff --git a/internal/server/server.go b/internal/server/server.go index eb0140c..19ff6cb 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -53,44 +53,22 @@ func New(config *configs.Config, logger *logger.Logger) *Server { ipLimiterConfig := limiter.Config{ Max: config.IPLimiter.MaxTokenRequests, Expiration: time.Duration(config.IPLimiter.TokenExpiration) * time.Minute, - SkipFailedRequests: false, + SkipFailedRequests: true, SkipSuccessfulRequests: false, Storage: ipLimiterstore, // skip the limiter for localhost Next: func(c *fiber.Ctx) bool { - return c.IP() == "127.0.0.1" + // skip the limiter if the keyGenerator returns "127.0.0.1" + return extractIPFromRequest(c) == "127.0.0.1" }, KeyGenerator: func(c *fiber.Ctx) string { - // Check for X-Forwarded-For header - if ip := c.Get("X-Forwarded-For"); ip != "" { - ips := strings.Split(ip, ",") - if len(ips) > 0 { - // return the first non-private ip in the list - for _, ip := range ips { - if net.ParseIP(strings.TrimSpace(ip)) != nil && !net.ParseIP(strings.TrimSpace(ip)).IsPrivate() { - return strings.TrimSpace(ip) - } - } - } - } - - // Check for X-Real-IP header if not a private IP - if ip := c.Get("X-Real-IP"); ip != "" { - if net.ParseIP(strings.TrimSpace(ip)) != nil && !net.ParseIP(strings.TrimSpace(ip)).IsPrivate() { - return strings.TrimSpace(ip) - } - } - - // Fall back to RemoteIP() if no proxy headers are present - ip := c.IP() - if parsedIP := net.ParseIP(ip); parsedIP != nil { - if !parsedIP.IsPrivate() { - return ip - } - } - - // If we still have a private IP, return a default value that will be skipped by the limiter - return "127.0.0.1" + logger.Debug("client IPs detected by the limiter", + zap.String("remoteIp", c.IP()), + zap.String("X-Forwarded-For", c.Get("X-Forwarded-For")), + zap.String("X-Real-IP", c.Get("X-Real-IP")), + zap.Strings("ips", c.IPs()), + ) + return extractIPFromRequest(c) }, } idLimiterStore := mongodb.New(mongodb.Config{ @@ -103,7 +81,7 @@ func New(config *configs.Config, logger *logger.Logger) *Server { idLimiterConfig := limiter.Config{ Max: config.IDLimiter.MaxTokenRequests, Expiration: time.Duration(config.IDLimiter.TokenExpiration) * time.Minute, - SkipFailedRequests: false, + SkipFailedRequests: true, SkipSuccessfulRequests: false, Storage: idLimiterStore, // Use client id as key to limit the number of requests per client @@ -136,7 +114,7 @@ func New(config *configs.Config, logger *logger.Logger) *Server { if err != nil { logger.Fatal("Failed to initialize substrate client", zap.Error(err)) } - kycService := services.NewKYCService(verificationRepo, tokenRepo, idenfyClient, substrateClient, &config.Verification, logger) + kycService := services.NewKYCService(verificationRepo, tokenRepo, idenfyClient, substrateClient, config, logger) // Initialize handler handler := handlers.NewHandler(kycService, logger) @@ -158,6 +136,37 @@ func New(config *configs.Config, logger *logger.Logger) *Server { return &Server{app: app, config: config, logger: logger} } +func extractIPFromRequest(c *fiber.Ctx) string { + // Check for X-Forwarded-For header + if ip := c.Get("X-Forwarded-For"); ip != "" { + ips := strings.Split(ip, ",") + if len(ips) > 0 { + + for _, ip := range ips { + // return the first non-private ip in the list + if net.ParseIP(strings.TrimSpace(ip)) != nil && !net.ParseIP(strings.TrimSpace(ip)).IsPrivate() { + return strings.TrimSpace(ip) + } + } + } + } + // Check for X-Real-IP header if not a private IP + if ip := c.Get("X-Real-IP"); ip != "" { + if net.ParseIP(strings.TrimSpace(ip)) != nil && !net.ParseIP(strings.TrimSpace(ip)).IsPrivate() { + return strings.TrimSpace(ip) + } + } + // Fall back to RemoteIP() if no proxy headers are present + ip := c.IP() + if parsedIP := net.ParseIP(ip); parsedIP != nil { + if !parsedIP.IsPrivate() { + return ip + } + } + // If we still have a private IP, return a default value that will be skipped by the limiter + return "127.0.0.1" +} + func (s *Server) Start() { go func() { sigChan := make(chan os.Signal, 1) diff --git a/internal/services/kyc_service.go b/internal/services/kyc_service.go index de5011e..a4793b7 100644 --- a/internal/services/kyc_service.go +++ b/internal/services/kyc_service.go @@ -3,6 +3,7 @@ package services import ( "context" "math/big" + "slices" "strings" "time" @@ -26,14 +27,17 @@ type kycService struct { IdenfySuffix string } -func NewKYCService(verificationRepo repository.VerificationRepository, tokenRepo repository.TokenRepository, idenfy *idenfy.Idenfy, substrateClient *substrate.Substrate, config *configs.Verification, logger *logger.Logger) KYCService { +func NewKYCService(verificationRepo repository.VerificationRepository, tokenRepo repository.TokenRepository, idenfy *idenfy.Idenfy, substrateClient *substrate.Substrate, config *configs.Config, logger *logger.Logger) KYCService { chainName, err := substrateClient.GetChainName() if err != nil { panic(errors.NewInternalError("error getting chain name", err)) } chainNameParts := strings.Split(chainName, " ") chainNetworkName := strings.ToLower(chainNameParts[len(chainNameParts)-1]) - return &kycService{verificationRepo: verificationRepo, tokenRepo: tokenRepo, idenfy: idenfy, substrate: substrateClient, config: config, logger: logger, IdenfySuffix: chainNetworkName} + if config.Idenfy.Namespace != "" { + chainNetworkName = config.Idenfy.Namespace + ":" + chainNetworkName + } + return &kycService{verificationRepo: verificationRepo, tokenRepo: tokenRepo, idenfy: idenfy, substrate: substrateClient, config: &config.Verification, logger: logger, IdenfySuffix: chainNetworkName} } // --------------------------------------------------------------------------------------------------------------------- @@ -127,6 +131,16 @@ func (s *kycService) GetVerificationData(ctx context.Context, clientID string) ( } func (s *kycService) GetVerificationStatus(ctx context.Context, clientID string) (*models.VerificationOutcome, error) { + // check first if the clientID is in alwaysVerifiedAddresses + if s.config.AlwaysVerifiedIDs != nil && slices.Contains(s.config.AlwaysVerifiedIDs, clientID) { + final := true + return &models.VerificationOutcome{ + Final: &final, + ClientID: clientID, + IdenfyRef: "", + Outcome: models.OutcomeApproved, + }, nil + } verification, err := s.verificationRepo.GetVerification(ctx, clientID) if err != nil { s.logger.Error("Error getting verification from database", zap.String("clientID", clientID), zap.Error(err)) diff --git a/scripts/dev/auth/generate-test-auth-data.go b/scripts/dev/auth/generate-test-auth-data.go index 932dd9e..5305f99 100644 --- a/scripts/dev/auth/generate-test-auth-data.go +++ b/scripts/dev/auth/generate-test-auth-data.go @@ -12,7 +12,7 @@ import ( ) const ( - domain = "kyc1.gent01.dev.grid.tf" + domain = "kyc.qa.grid.tf" ) // Generate test auth data for development use