Added websocket support (#45)

* adding clickhouse start

* wip

* prep

* adding duckdb

* wip

* added background service

* replaced data types

* added todo

* added stub for proxy

* added proxy for serve

* small fix for buld

* should work

* improving for hive syntax

* wip

* wip

* wip

* wip

* finishing up

* adding tables

* standardizing uuids

* wip

* added websocket

* wip

* moved to new url format

* adding ws

* updating socket

* wip

* wip

.gitignore

@@ -5,6 +5,7 @@
 *.so
 *.dylib
 
+config.*.json
 # Test binary, build with `go test -c`
 *.test
 

.setup/chrome/background.js

@@ -6,7 +6,7 @@ chrome.contextMenus.create({
   onclick: function (word,tab) {
     var link = word.linkUrl || tab.url || word.selectionText;
     if (link) {      
-      chrome.tabs.create({ url: "https://tr.sfpl.io/pub/v1/track_url.html?tu=1&url=" + encodeURIComponent(link) });
+      chrome.tabs.create({ url: "https://tr.sfpl.io/v1/pub/track_url.html?tu=1&url=" + encodeURIComponent(link) });
     } else {
       console.log("Couldn't determine link");
       alert("Couldn't determine link");

.setup/old/track.js

@@ -125,7 +125,7 @@ export default function track(params, force) {
         json.uid = path(["sfpl", "pub", "uid"], user)
         json.uname = path(["sfpl", "pub", "uname"], user)
     }
-    request(`${URL_TRACK}/tr/v1/`, {
+    request(`${URL_TRACK}/v1/tr/`, {
         method: "POST",
         body: JSON.stringify(json),
         ...ta

.setup/parcel/config.yaml

@@ -31,6 +31,13 @@ Routes:
         PageType: settings
 Tracker:
     Track: true
+    ApiVersion: v1
+    WebSocket: true
+    WS:
+        Development: ws://localhost:8880/v1/ws
+        Staging: ws://tr.sfpl.io/v1/ws
+        Production: wss://tr.sfpl.io/v1/ws
+
     Url: 
         Development: http://localhost:8880
         Staging: https://tr.sfpl.io

.setup/parcel/src/lib/tracker/track.js

@@ -1,4 +1,3 @@
-
 /* From https://github.com/sfproductlabs/tracker (this is v1 - keep an eye on current v2 updates!)
  * https://github.com/keithws/browser-report
  * Report browser settings like whatsmybrowseorg
@@ -12,8 +11,41 @@ import { v1 as uuidv1 } from 'uuid';
 import report from './report';
 import request from './request';
 import device from './device';
-import {getHostRoot} from './network';
+import { getHostRoot } from './network';
 import config from '../../../config.yaml';
+import { compress } from '../../../../../public/lz4'
+
+// Initial WebSocket setup
+let socket;
+const wsurl = path(["Tracker", "WS", process.env.TARGET || path(["Application", "Target"], config) || "Development"], config);
+// Define WebSocket event handlers
+const wsHandlers = {
+    onopen: () => {
+        console.debug('WS TR connected');
+    },
+    onmessage: (event) => {
+        const data = JSON.parse(event.data);
+        console.debug('WS TR Received:', data);
+    },
+    onerror: (event) => {
+        console.error('WS TR error:', event);
+    },
+    onclose: () => {
+        console.debug('WS TR Disconnected from server');
+        // Attempt to reconnect after a delay
+        setTimeout(setupWebSocket, 1000);
+    }
+};
+
+// Function to setup WebSocket with event handlers
+function setupWebSocket() {
+    socket = new WebSocket(wsurl);
+    socket.addEventListener('open', wsHandlers.onopen);
+    socket.addEventListener('message', wsHandlers.onmessage);
+    socket.addEventListener('error', wsHandlers.onerror);
+    socket.addEventListener('close', wsHandlers.onclose);
+}
+setupWebSocket();
 
 /**
  * @param {object} params - event parameters
@@ -23,7 +55,7 @@ import config from '../../../config.yaml';
  * @returns {null|undefined}
  */
 export default function track(params) {
-    if (!config || !config.Tracker || !config.Tracker.Track) {        
+    if (!config || !config.Tracker || !config.Tracker.Track) {
         return;
     }
     params = defaultTo({})(params);
@@ -68,7 +100,7 @@ export default function track(params) {
     if (email) {
         json.email = email;
     }
-    
+
     if (!json.eid) {
         json.eid = uuidv1()
     }
@@ -128,10 +160,10 @@ export default function track(params) {
         json.params.fbc = fbc;
     }
     json.now = now / 1000
-    json.ehash = json.ehash || cookies.get(config.Cookies.Names.COOKIE_EMAIL_HASH) || undefined;    
+    json.ehash = json.ehash || cookies.get(config.Cookies.Names.COOKIE_EMAIL_HASH) || undefined;
     json.cflags = json.cflags || cookies.get(config.Cookies.Names.COOKIE_CONSENT) || undefined;
     json.uri = window.location.origin + window.location.pathname;
-    
+
     //Existing Query Params
     //Ex. http://localhost:3003/?gu=1&ptyp=blog&utm_source=news_service&utm_medium=email&utm_campaign=campaign&aff=promo&ref=60c59df0ed0811e8a766de1a241fb011&uname=admin
     try {
@@ -165,11 +197,19 @@ export default function track(params) {
     json.params = { ...json, ...temp }
 
     let tr = function (obj) {
-        request(`${getTrackerUrl()}/tr/v1/`, {
-            method: "POST",
-            body: JSON.stringify(obj),
-            ...ta
-        });
+        if (config.Tracker.WebSocket && socket.readyState === WebSocket.OPEN) {
+            const str = JSON.stringify(obj);
+            const bytes = new TextEncoder().encode(str);
+            compress(bytes, true).then(compressed => {
+                socket.send(compressed);
+            });
+        } else {
+            request(`${getTrackerUrl()}/v1/tr/`, {
+                method: "POST",
+                body: JSON.stringify(obj),
+                ...ta
+            });
+        }
     };
 
     if (json.first) {
@@ -205,7 +245,7 @@ export const getPageType = () => {
         }
     }
     return null;
-} 
+}
 
 export function getTrackerUrl() {
     return path(["Tracker", "Url", process.env.TARGET || path(["Application", "Target"], config) || "Development"], config);
@@ -216,21 +256,161 @@ export function resetUserCookies() {
     let vid = cookies.get(config.Cookies.Names.COOKIE_VISITOR);
     let found = true;
     if (!vid) {
-      vid = uuidv1();
-      found = false;
+        vid = uuidv1();
+        found = false;
     }
     const sid = cookies.get(config.Cookies.Names.COOKIE_SESSION) || vid;
-    cookies.setLax(config.Cookies.Names.COOKIE_VISITOR, vid, { 
-      sameSite: 'lax', 
-      domain
+    cookies.setLax(config.Cookies.Names.COOKIE_VISITOR, vid, {
+        sameSite: 'lax',
+        domain
     });
-    cookies.setLax(config.Cookies.Names.COOKIE_SESSION, sid, { 
-      sameSite: 'lax', 
-      domain
+    cookies.setLax(config.Cookies.Names.COOKIE_SESSION, sid, {
+        sameSite: 'lax',
+        domain
     });
     if (!found) {
-      try { track({ename:'visited', eid: vid, sid: vid, vid:vid, etyp: "cookie"}); } catch {};
+        try { track({ ename: 'visited', eid: vid, sid: vid, vid: vid, etyp: "cookie" }); } catch { };
     }
-  }
+}
+
+resetUserCookies();
+
+// Track mouse position, scroll position and click events
+// Events are batched and sent every 100ms to reduce server load and improve performance
+// Click events include target element information (id, class, tag) for better analytics
+let mouseEvents = [];
+let scrollEvents = [];
+let clickEvents = [];
+let keyboardEvents = [];
+let trackingTimeout;
+let keyboardTrackingTimeout;
+
+const sensitiveClasses = ['password', 'credit-card', 'ssn'];
+const sensitiveAttributes = ['data-sensitive', 'data-private'];
+
+const shouldMaskElement = (element) => {
+    return sensitiveClasses.some(cls => element.className.includes(cls)) ||
+        sensitiveAttributes.some(attr => element.hasAttribute(attr));
+};
+const getElementDetails = (element) => {
+    return {
+        id: element.id || null,
+        className: element.className || null,
+        tagName: element.tagName?.toLowerCase() || null,
+        text: shouldMaskElement(element) ? '***' : element.textContent?.trim() || null,
+        type: element.type || null,
+        href: element.href || null,
+        // Get closest parent with data-track attribute if exists
+        trackingData: element.closest('[data-track]')?.dataset?.track || null
+    };
+};
+
+const trackClickEvent = (e) => {
+    const target = e.target;
+    clickEvents.push({
+        timestamp: Date.now(),
+        x: e.clientX,
+        y: e.clientY,
+        element: getElementDetails(target)
+    });
+    scheduleTracking();
+};
+
+const trackMouseMovement = (e) => {
+    mouseEvents.push({
+        x: e.clientX,
+        y: e.clientY,
+        timestamp: Date.now()
+    });
+    scheduleTracking();
+};
+
+const trackScrollMovement = () => {
+    scrollEvents.push({
+        x: window.scrollX,
+        y: window.scrollY,
+        timestamp: Date.now()
+    });
+    scheduleTracking();
+};
+
+const trackKeyboardEvent = (e) => {
+    // Don't track actual key values from sensitive fields
+    const isSensitiveField = shouldMaskElement(e.target);
+    keyboardEvents.push({
+        timestamp: Date.now(),
+        key: isSensitiveField ? '***' : e.key,
+        type: e.type, // 'keydown', 'keyup', etc.
+        element: getElementDetails(e.target)
+    });
+    scheduleKeyboardTracking();
+};
+
+const scheduleKeyboardTracking = () => {
+    if (keyboardTrackingTimeout) {
+        clearTimeout(keyboardTrackingTimeout);
+    }
+    keyboardTrackingTimeout = setTimeout(sendKeyboardEvents, 2000);
+};
+
+const sendKeyboardEvents = () => {
+    if (keyboardEvents.length > 0) {
+        track({
+            ename: "keyboard_input",
+            values: keyboardEvents
+        });
+        keyboardEvents = [];
+    }
+};
+
+const scheduleTracking = () => {
+    if (trackingTimeout) {
+        clearTimeout(trackingTimeout);
+    }
+    trackingTimeout = setTimeout(sendTrackingEvents, 100);
+};
+
+const sendTrackingEvents = () => {
+    if (mouseEvents.length > 0) {
+        track({
+            ename: "mouse_move",
+            values: mouseEvents
+        });
+        mouseEvents = [];
+    }
+    if (scrollEvents.length > 0) {
+        track({
+            ename: "scroll_move",
+            values: scrollEvents
+        });
+        scrollEvents = [];
+    }
+    if (clickEvents.length > 0) {
+        track({
+            ename: "element_click",
+            values: clickEvents
+        });
+        clickEvents = [];
+    }
+};
+
+const trackPerformance = () => {
+    if (window.performance && window.performance.timing) {
+        const timing = performance.timing;
+        track({
+            ename: "page_performance",
+            values: [{
+                loadTime: timing.loadEventEnd - timing.navigationStart,
+                domReady: timing.domContentLoadedEventEnd - timing.navigationStart,
+                firstPaint: performance.getEntriesByType('paint')[0]?.startTime,
+                timestamp: Date.now()
+            }]
+        });
+    }
+};
 
-  resetUserCookies();
+window.addEventListener('mousemove', trackMouseMovement);
+window.addEventListener('scroll', trackScrollMovement);
+window.addEventListener('click', trackClickEvent);
+window.addEventListener('load', trackPerformance);
+window.addEventListener('keydown', trackKeyboardEvent);