Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(telemetry): local event count store #1306

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

fix(telemetry): local event count store #1306

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion add-on/src/lib/telemetry.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@ export function trackView (view: string, segments: Record<string, string>): void
* TrackView is a wrapper around ignite-metrics trackView
*
* @param event
* @param segments
*/
export function trackEvent (event: CountlyEvent): void {
log('trackEvent called for event: ', event)
Expand Down
83 changes: 72 additions & 11 deletions add-on/src/lib/trackers/requestTracker.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,21 @@
import debug from 'debug'
import type browser from 'webextension-polyfill'
import browser from 'webextension-polyfill'
import { trackEvent } from '../telemetry.js'

export const DEFAULT_REQUEST_TRACKER_FLUSH_INTERVAL = 1000 * 60 * 60
export const REQUEST_TRACKER_SYNC_INTERVAL = 1000 * 60 * 60 * 24
export const REQUEST_TRACKER_LOCAL_STORAGE_KEY = 'request-tracker'

interface RequestTrackerPersistedState {
lastSync: number
requestTypeStore: { [key in browser.WebRequest.ResourceType]?: number }
}

export class RequestTracker {
private readonly eventKey: 'url-observed' | 'url-resolved'
private readonly flushInterval: number
private readonly log: debug.Debugger & { error?: debug.Debugger }
private lastSync: number = Date.now()
private requestTypeStore: { [key in browser.WebRequest.ResourceType]?: number } = {}
private requestTypeStore: RequestTrackerPersistedState['requestTypeStore'] = {}

constructor (eventKey: 'url-observed' | 'url-resolved', flushInterval = DEFAULT_REQUEST_TRACKER_FLUSH_INTERVAL) {
this.eventKey = eventKey
Expand All @@ -24,27 +30,82 @@ export class RequestTracker {
this.requestTypeStore[type] = (this.requestTypeStore[type] ?? 0) + 1
}

private flushStore (): void {
this.log('flushing')
const count = Object.values(this.requestTypeStore).reduce((a, b): number => a + b, 0)
private async flushStoreToMemory (): Promise<void> {
this.log('flushing to memory')

const persistedState = await browser.storage.local.get(REQUEST_TRACKER_LOCAL_STORAGE_KEY) as RequestTrackerPersistedState ?? {
lastSync: Date.now(),
requestTypeStore: {}
}

// merge
const { lastSync, requestTypeStore } = persistedState

const mergedRequestTypeKeys: Set<browser.WebRequest.ResourceType> = new Set([
...Object.keys(requestTypeStore) as browser.WebRequest.ResourceType[],
...Object.keys(this.requestTypeStore) as browser.WebRequest.ResourceType[]
])

const mergedRequestTypeStore = Object.fromEntries([...mergedRequestTypeKeys].map((key): [
browser.WebRequest.ResourceType,
number
] => ([
key,
(requestTypeStore?.[key] ?? 0) + (this.requestTypeStore?.[key] ?? 0)
])))

await browser.storage.local.set({
[REQUEST_TRACKER_LOCAL_STORAGE_KEY]: {
lastSync,
requestTypeStore: mergedRequestTypeStore
}
})

// reset
this.requestTypeStore = {}
await this.syncEventsToTelemetry()
}

private async syncEventsToTelemetry (): Promise<void> {
this.log('syncing')
const currentTimestamp = Date.now()
const persistedState = await browser.storage.local.get(REQUEST_TRACKER_LOCAL_STORAGE_KEY) as RequestTrackerPersistedState
const { lastSync, requestTypeStore } = persistedState

// skip if we already synced recently
if (lastSync + REQUEST_TRACKER_SYNC_INTERVAL > currentTimestamp) {
this.log('sync skipped')
return
}

// skip if there is nothing to sync
const count = Object.values(requestTypeStore).reduce((a, b): number => a + b, 0)

if (count === 0) {
this.log('nothing to flush')
return
}

// sync
trackEvent({
key: this.eventKey,
count,
dur: Date.now() - this.lastSync,
dur: currentTimestamp - lastSync,
segmentation: Object.assign({}, this.requestTypeStore) as unknown as Record<string, string>
})

// reset
this.lastSync = Date.now()
this.requestTypeStore = {}
await browser.storage.local.set({
[REQUEST_TRACKER_LOCAL_STORAGE_KEY]: {
lastSync: currentTimestamp,
requestTypeStore: {}
}
})
}

private setupFlushScheduler (): void {
setTimeout(() => {
this.flushStore()
setTimeout(async () => {
await this.flushStoreToMemory()
this.setupFlushScheduler()
}, this.flushInterval)
}
Expand Down
31 changes: 18 additions & 13 deletions test/functional/lib/trackers/requestTrackers.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,14 @@ import { expect } from 'chai';
import sinon from 'sinon';
import browser from 'sinon-chrome';
import PatchedCountly from 'countly-sdk-web'
import { DEFAULT_REQUEST_TRACKER_FLUSH_INTERVAL, RequestTracker } from './../../../../add-on/src/lib/trackers/requestTracker.js'
import { DEFAULT_REQUEST_TRACKER_FLUSH_INTERVAL, REQUEST_TRACKER_SYNC_INTERVAL, RequestTracker } from './../../../../add-on/src/lib/trackers/requestTracker.js'

const sinonSandBox = sinon.createSandbox()
describe('lib/trackers/requestTracker', () => {

let requestTracker: RequestTracker
let countlySDKStub: sinon.SinonStub
let clock: sinon.SinonFakeTimers
let requestTracker

before(() => {
clock = sinonSandBox.useFakeTimers()
Expand All @@ -18,6 +18,8 @@ describe('lib/trackers/requestTracker', () => {

afterEach(() => {
sinonSandBox.resetHistory()
browser.storage.local.get.reset()
browser.storage.local.set.reset()
})

describe('url-observed', () => {
Expand All @@ -26,13 +28,15 @@ describe('lib/trackers/requestTracker', () => {
})

it('should init a Tracker', () => {
expect(requestTracker).to.be.instanceOf(RequestTracker)
expect(requestTracker).to.have.property('track')
})

it('should track a request', async () => {
await requestTracker.track({ type: 'main_frame' } as browser.WebRequest.OnBeforeRequestDetailsType)
it.only('should track a request', async () => {
requestTracker.track({ type: 'main_frame' } as browser.WebRequest.OnBeforeRequestDetailsType)
browser.storage.local.get.returns(null)
clock.tick(DEFAULT_REQUEST_TRACKER_FLUSH_INTERVAL)
browser.storage.local.get.returns({ lastSync: Date.now(), requestTypeStore: { main_frame: 1 } })
clock.tick(REQUEST_TRACKER_SYNC_INTERVAL)
sinon.assert.calledWith(countlySDKStub.add_event, {
key: 'url-observed',
count: 1,
Expand All @@ -44,9 +48,9 @@ describe('lib/trackers/requestTracker', () => {
})

it('should track multiple requests', async () => {
await requestTracker.track({ type: 'main_frame' } as browser.WebRequest.OnBeforeRequestDetailsType)
await requestTracker.track({ type: 'sub_frame' } as browser.WebRequest.OnBeforeRequestDetailsType)
await requestTracker.track({ type: 'xmlHTTPRequest' } as browser.WebRequest.OnBeforeRequestDetailsType)
requestTracker.track({ type: 'main_frame' } as browser.WebRequest.OnBeforeRequestDetailsType)
requestTracker.track({ type: 'sub_frame' } as browser.WebRequest.OnBeforeRequestDetailsType)
requestTracker.track({ type: 'xmlHTTPRequest' } as browser.WebRequest.OnBeforeRequestDetailsType)
clock.tick(DEFAULT_REQUEST_TRACKER_FLUSH_INTERVAL)
sinon.assert.calledWith(countlySDKStub.add_event, {
key: 'url-observed',
Expand Down Expand Up @@ -78,8 +82,9 @@ describe('lib/trackers/requestTracker', () => {
})

it('should track a request', async () => {
await requestTracker.track({ type: 'main_frame' } as browser.WebRequest.OnBeforeRequestDetailsType)
clock.tick(DEFAULT_REQUEST_TRACKER_FLUSH_INTERVAL)
requestTracker.track({ type: 'main_frame' } as browser.WebRequest.OnBeforeRequestDetailsType)
clock.tick(DEFAULT_REQUEST_TRACKER_FLUSH_INTERVAL + 1000)
expect(browser.storage.local.set.lastCall.args).to.deep.equal([])
sinon.assert.calledWith(countlySDKStub.add_event, {
key: 'url-resolved',
count: 1,
Expand All @@ -91,9 +96,9 @@ describe('lib/trackers/requestTracker', () => {
})

it('should track multiple requests', async () => {
await requestTracker.track({ type: 'main_frame' } as browser.WebRequest.OnBeforeRequestDetailsType)
await requestTracker.track({ type: 'sub_frame' } as browser.WebRequest.OnBeforeRequestDetailsType)
await requestTracker.track({ type: 'xmlHTTPRequest' } as browser.WebRequest.OnBeforeRequestDetailsType)
requestTracker.track({ type: 'main_frame' } as browser.WebRequest.OnBeforeRequestDetailsType)
requestTracker.track({ type: 'sub_frame' } as browser.WebRequest.OnBeforeRequestDetailsType)
requestTracker.track({ type: 'xmlHTTPRequest' } as browser.WebRequest.OnBeforeRequestDetailsType)
clock.tick(DEFAULT_REQUEST_TRACKER_FLUSH_INTERVAL)
sinon.assert.calledWith(countlySDKStub.add_event, {
key: 'url-resolved',
Expand Down
Loading