Merge pull request #945 from areed/saas-metrics

Record each time an install script is downloaded

cmd/server/main.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"net"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
@@ -61,7 +62,20 @@ func bundle(w http.ResponseWriter, r *http.Request) {
 	base := path.Base(r.URL.Path)
 	installerID := strings.TrimSuffix(base, ".tar.gz")
 	installerURL := fmt.Sprintf("%s/bundle/%s", upstream, installerID)
-	resp, err := http.Get(installerURL)
+	request, err := http.NewRequest("GET", installerURL, nil)
+	if err != nil {
+		log.Printf("Error building request for %s: %v", installerURL, err)
+		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+		return
+	}
+	// forward request headers for metrics
+	request.Header = r.Header
+	if request.Header.Get("X-Forwarded-For") == "" {
+		if host, _, _ := net.SplitHostPort(r.RemoteAddr); host != "" {
+			request.Header.Set("X-Forwarded-For", host)
+		}
+	}
+	resp, err := http.DefaultClient.Do(request)
 	if err != nil {
 		log.Printf("Error fetching %s: %v", installerURL, err)
 		http.Error(w, "Internal Server Error", http.StatusInternalServerError)

web/package.json

@@ -21,11 +21,13 @@
     "promise-mysql": "^4.1.0",
     "replicated-lint": "^0.13.3",
     "request": "^2.88.0",
+    "request-ip": "^2.1.3",
     "request-promise": "^4.2.2",
     "sigsci-module-nodejs": "https://dl.signalsciences.net/sigsci-module-nodejs/1.4.8/sigsci-module-nodejs-1.4.8.tgz",
     "statsd-client": "^0.4.2",
     "superagent": "^5.1.0",
-    "ts-express-decorators": "^5.24.1"
+    "ts-express-decorators": "^5.24.1",
+    "uuid": "^8.3.1"
   },
   "devDependencies": {
     "@pact-foundation/pact": "^8.2.0",

web/src/controllers/Bundles.ts

@@ -6,10 +6,13 @@ import {
   Controller,
   Get,
   PathParams,
+  Req,
   Res } from "ts-express-decorators";
 import { Templates } from "../util/services/templates";
 import { InstallerStore } from "../installers";
 import { logger } from "../logger";
+import { MetricsStore } from "../util/services/metrics";
+import * as requestIP from "request-ip";
 
 interface ErrorResponse {
   error: any;
@@ -27,7 +30,7 @@ interface FilepathContentsMap {
 
 // Manifest for building an airgap bundle.
 interface BundleManifest {
-  layers: Array<string>;
+  layers: string[];
   files: FilepathContentsMap;
 };
 
@@ -39,6 +42,7 @@ export class Bundle {
   constructor(
     private readonly templates: Templates,
     private readonly installers: InstallerStore,
+    private readonly metricsStore: MetricsStore,
   ) {
     this.replicatedAppURL = process.env["REPLICATED_APP_URL"] || "https://replicated.app";
     this.distURL = `https://${process.env["KURL_BUCKET"]}.s3.amazonaws.com`;
@@ -58,6 +62,7 @@ export class Bundle {
   @Get("/:installerID")
   public async redirect(
     @Res() response: Express.Response,
+    @Req() req: Express.Request,
     @PathParams("installerID") installerID: string,
   ): Promise<BundleManifest|ErrorResponse> {
 
@@ -69,6 +74,18 @@ export class Bundle {
     }
     installer = installer.resolve();
 
+    try {
+      await this.metricsStore.saveSaasScriptEvent({
+        installerID,
+        timestamp: new Date(),
+        isAirgap: true,
+        clientIP: requestIP.getClientIp(req),
+        userAgent: req.get("User-Agent"),
+      });
+    } catch (err) {
+      logger.error(`Failed to save saas script event: ${err.message}`);
+    }
+
     response.type("application/json");
 
     const ret: BundleManifest = {layers: [], files: {}};

web/src/controllers/InstallerAPI.ts

@@ -237,20 +237,20 @@ export class Installers {
       response.status(401);
       return unauthenticatedResponse;
     }
-  
+
     let teamID: string;
     try {
       teamID = await decode(auth);
     } catch(error) {
       response.status(401);
       return unauthenticatedResponse;
     }
-  
+
     if (!teamID) {
       response.status(401);
       return unauthenticatedResponse;
     }
-  
+
     if (Installer.isSHA(id)) {
       response.status(400);
       return teamWithGeneratedIDResponse;
@@ -263,7 +263,7 @@ export class Installers {
       response.status(400);
       return slugReservedResponse;
     }
-  
+
     let i: Installer;
     try {
       i = Installer.parse(request.body, teamID);
@@ -272,21 +272,21 @@ export class Installers {
       return { error };
     }
     i.id = id;
-  
+
     if (i.spec.kotsadm && !i.spec.kotsadm.applicationSlug) {
-      if (slug != "") {
+      if (slug !== "") {
         i.spec.kotsadm.applicationSlug = slug
       }
     }
-  
+
     const err = await i.validate();
     if (err) {
       response.status(400);
       return err;
     }
-  
+
     await this.installerStore.saveTeamInstaller(i);
-  
+
     response.contentType("text/plain");
     response.status(201);
     return `${this.kurlURL}/${i.id}`;

web/src/controllers/Scripts.ts

@@ -3,10 +3,14 @@ import {
   Controller,
   Get,
   PathParams,
+  Req,
   Res } from "ts-express-decorators";
 import { instrumented } from "monkit";
 import { Installer, InstallerStore } from "../installers";
 import { Templates } from "../util/services/templates";
+import { MetricsStore } from "../util/services/metrics";
+import { logger } from "../logger";
+import * as requestIP from "request-ip";
 
 interface ErrorResponse {
   error: any;
@@ -24,6 +28,7 @@ export class Installers {
   constructor (
     private readonly installerStore: InstallerStore,
     private readonly templates: Templates,
+    private readonly metricsStore: MetricsStore,
   ) {}
 
   /**
@@ -38,6 +43,7 @@ export class Installers {
   @instrumented
   public async getInstaller(
     @Res() response: Express.Response,
+    @Req() request: Express.Request,
     @PathParams("installerID") installerID: string,
   ): Promise<string | ErrorResponse> {
 
@@ -48,6 +54,18 @@ export class Installers {
     }
     installer = installer.resolve();
 
+    try {
+      await this.metricsStore.saveSaasScriptEvent({
+        installerID,
+        timestamp: new Date(),
+        isAirgap: false,
+        clientIP: requestIP.getClientIp(request),
+        userAgent: request.get("User-Agent"),
+      });
+    } catch (err) {
+      logger.error(`Failed to save saas script event: ${err.message}`);
+    }
+
     response.status(200);
     return this.templates.renderInstallScript(installer);
   }

web/src/util/services/metrics.ts

@@ -0,0 +1,51 @@
+import { Service } from "ts-express-decorators";
+import * as mysql from "promise-mysql";
+import { MysqlWrapper } from "./mysql";
+import { Installer } from "../../installers";
+import * as uuid from "uuid";
+
+export interface GetInstallScriptEvent {
+  id?: string;
+  installerID: string;
+  timestamp: Date;
+  isAirgap: boolean;
+  clientIP?: string;
+  userAgent?: string;
+}
+
+@Service()
+export class MetricsStore {
+  private readonly pool: mysql.Pool;
+
+  constructor({ pool }: MysqlWrapper) {
+    this.pool = pool;
+  }
+
+  public async saveSaasScriptEvent(e: GetInstallScriptEvent): Promise<void> {
+    const q = `INSERT INTO kurl_saas_script_metrics (
+      id,
+      installer_id,
+      timestamp,
+      is_airgap,
+      client_ip,
+      user_agent
+    ) VALUES (
+      ?,
+      ?,
+      ?,
+      ?,
+      ?,
+      ?
+    )`;
+    const v = [
+      uuid.v4(),
+      e.installerID,
+      e.timestamp,
+      e.isAirgap,
+      e.clientIP,
+      e.userAgent,
+    ];
+
+    await this.pool.query(q, v);
+  }
+}

web/yarn.lock

@@ -3454,6 +3454,11 @@ is-wsl@^1.1.0:
   resolved "https://registry.yarnpkg.com/is-wsl/-/is-wsl-1.1.0.tgz#1f16e4aa22b04d1336b66188a66af3c600c3a66d"
   integrity sha1-HxbkqiKwTRM2tmGIpmrzxgDDpm0=
 
+is_js@^0.9.0:
+  version "0.9.0"
+  resolved "https://registry.yarnpkg.com/is_js/-/is_js-0.9.0.tgz#0ab94540502ba7afa24c856aa985561669e9c52d"
+  integrity sha1-CrlFQFArp6+iTIVqqYVWFmnpxS0=
+
 [email protected]:
   version "0.0.1"
   resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf"
@@ -5514,6 +5519,13 @@ replicated-lint@^0.13.3:
     yaml-ast-parser "^0.0.33"
     yargs "^9.0.1"
 
+request-ip@^2.1.3:
+  version "2.1.3"
+  resolved "https://registry.yarnpkg.com/request-ip/-/request-ip-2.1.3.tgz#99ab2bafdeaf2002626e28083cb10597511d9e14"
+  integrity sha512-J3qdE/IhVM3BXkwMIVO4yFrvhJlU3H7JH16+6yHucadT4fePnR8dyh+vEs6FIx0S2x5TCt2ptiPfHcn0sqhbYQ==
+  dependencies:
+    is_js "^0.9.0"
+
 [email protected]:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/request-promise-core/-/request-promise-core-1.1.2.tgz#339f6aababcafdb31c799ff158700336301d3346"
@@ -7040,6 +7052,11 @@ [email protected], uuid@^3.0.1, uuid@^3.1.0, uuid@^3.3.2:
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131"
   integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==
 
+uuid@^8.3.1:
+  version "8.3.1"
+  resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.1.tgz#2ba2e6ca000da60fce5a196954ab241131e05a31"
+  integrity sha512-FOmRr+FmWEIG8uhZv6C2bTgEVXsHk08kE7mPlrBbEe+c3r9pjceVPgupIfNIhc4yx55H69OXANrUaSuu9eInKg==
+
 v8flags@^3.0.0:
   version "3.1.3"
   resolved "https://registry.yarnpkg.com/v8flags/-/v8flags-3.1.3.tgz#fc9dc23521ca20c5433f81cc4eb9b3033bb105d8"