diff --git a/CHANGELOG.md b/CHANGELOG.md index b5a5961bafc..7dd6fd4ad84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,27 @@ -## Version 24.10.xx +## Version 24.10.6 + +Fixes: +- [push] Using apns-id header as message result in debug mode +- [server-stats] Fix data point calculation in job +- [TopEventsJob] preserver previous state if overwriting fails +- [ui] scroll top on step changes in drawers + Enterprise fixes: +- [drill] Encoding url component before changing history state - [drill] Fixed drill meta regeneration - +- [drill] [license] Update license loader to enable supplying db client +- [users] Format data points displayed in user sidebar +- [cohorts] Unescape drill texts in cohort component + Dependencies: - Bump fs-extra from 11.2.0 to 11.3.0 - Bump nodemailer from 6.9.16 to 6.10.0 -## Version 24.10.5 +Enterprise Dependencies: +- Bump nanoid in /plugins/cognito from 2.1.11 to 3.3.8 +- Bump shortid in /plugins/cognito from 2.2.16 to 2.2.17 +## Version 24.10.5 Fixes: - [core] Fixed a bug causing events to not being loaded when there's an escaped character in the event name - [core] Fixed a bug that was causing drill to crash when there's a percentage symbol in the event name diff --git a/api/jobs/topEvents.js b/api/jobs/topEvents.js index 44913509632..20d3af998ac 100644 --- a/api/jobs/topEvents.js +++ b/api/jobs/topEvents.js @@ -19,8 +19,8 @@ class TopEventsJob extends job.Job { /** * TopEvents initialize function */ - init() { - this.getAllApps(); + async init() { + return this.getAllApps(); } /** @@ -144,6 +144,7 @@ class TopEventsJob extends job.Job { } catch (error) { log.e("TopEvents Job has a error: ", error); + throw error; } } @@ -157,7 +158,18 @@ class TopEventsJob extends job.Job { const encodedData = this.encodeEvents(data); const timeSecond = this.timeSecond(); const currentPeriood = this.mutatePeriod(period); - await new Promise((res, rej) => common.db.collection(TopEventsJob.COLLECTION_NAME).insert({ app_id: _id, ts: timeSecond, period: currentPeriood, data: encodedData, totalCount: totalCount, prevTotalCount: prevTotalCount, totalSum: totalSum, prevTotalSum: prevTotalSum, totalDuration: totalDuration, prevTotalDuration: prevTotalDuration, prevSessionCount: sessionData.prevSessionCount, totalSessionCount: sessionData.totalSessionCount, prevUsersCount: usersData.prevUsersCount, totalUsersCount: usersData.totalUsersCount }, (error, records) => !error && records ? res(records) : rej(error))); + await new Promise((res, rej) => common.db.collection(TopEventsJob.COLLECTION_NAME).findOneAndReplace( + { + app_id: _id, period: currentPeriood, + }, + { + app_id: _id, ts: timeSecond, period: currentPeriood, data: encodedData, totalCount: totalCount, prevTotalCount: prevTotalCount, totalSum: totalSum, prevTotalSum: prevTotalSum, totalDuration: totalDuration, prevTotalDuration: prevTotalDuration, prevSessionCount: sessionData.prevSessionCount, totalSessionCount: sessionData.totalSessionCount, prevUsersCount: usersData.prevUsersCount, totalUsersCount: usersData.totalUsersCount + }, + { + upsert: true + }, + (error, records) => !error && records ? res(records) : rej(error)) + ); } /** @@ -169,7 +181,6 @@ class TopEventsJob extends job.Job { const getEvents = await new Promise((res, rej) => common.db.collection("events").findOne({ _id: app._id }, (errorEvents, result) => errorEvents ? rej(errorEvents) : res(result))); if (getEvents && 'list' in getEvents) { const eventMap = this.eventsFilter(getEvents.list); - await new Promise((res, rej) => common.db.collection(TopEventsJob.COLLECTION_NAME).remove({ app_id: app._id }, (error, result) => error ? rej(error) : res(result))); if (eventMap && eventMap instanceof Array) { for (const period of TopEventsJob.PERIODS) { const data = {}; @@ -211,9 +222,14 @@ class TopEventsJob extends job.Job { * @param {Db} db connection * @param {done} done callback */ - run(db, done) { - this.init(); - done(); + async run(db, done) { + try { + await this.init(); + done(); + } + catch (error) { + done(error); + } } } diff --git a/plugins/push/api/send/platforms/i.js b/plugins/push/api/send/platforms/i.js index 93a22f2b6cf..78caf34eb09 100644 --- a/plugins/push/api/send/platforms/i.js +++ b/plugins/push/api/send/platforms/i.js @@ -708,7 +708,7 @@ class APN extends Base { status = headers[':status']; // self.log.d('%d: status %d: %j', i, status, self.session.state); if (status === 200) { - const apnsUniqueId = headers["apns-unique-id"]; + const apnsUniqueId = headers["apns-id"] ?? headers["apns-unique-id"]; oks.push({ p: p._id, r: apnsUniqueId }); stream.destroy(); streamDone(); diff --git a/plugins/server-stats/api/jobs/stats.js b/plugins/server-stats/api/jobs/stats.js index d112fc1913c..894ca43e416 100644 --- a/plugins/server-stats/api/jobs/stats.js +++ b/plugins/server-stats/api/jobs/stats.js @@ -3,10 +3,20 @@ const job = require('../../../../api/parts/jobs/job.js'), tracker = require('../../../../api/parts/mgmt/tracker.js'), log = require('../../../../api/utils/log.js')('job:stats'), - config = require("../../../../frontend/express/config.js"), + config = require('../../../../frontend/express/config.js'), pluginManager = require('../../../pluginManager.js'), + serverStats = require('../parts/stats.js'), moment = require('moment-timezone'), - request = require('countly-request')(pluginManager.getConfig("security")); + request = require('countly-request')(pluginManager.getConfig('security')); + +let drill; +try { + drill = require('../../../drill/api/parts/data/drill.js'); +} +catch (ex) { + log.e(ex); + drill = null; +} const promisedLoadConfigs = function(db) { return new Promise((resolve) => { @@ -27,6 +37,90 @@ class StatsJob extends job.Job { super(name, data); } + /** + * @param {Object} allData - All server stats data from the beginning of time + * @returns {Object} Sum of all data, average data per month, and last three month data + */ + static generateDataSummary(allData) { + const data = {}; + data.all = 0; + data.month3 = []; + const utcMoment = moment.utc(); + const months = {}; + for (let i = 0; i < 3; i++) { + months[utcMoment.format('YYYY:M')] = true; + utcMoment.subtract(1, 'months'); + } + for (const [key, value] of Object.entries(allData)) { + data.all += value; + if (months[key]) { + data.month3.push(key + ' - ' + (value)); + } + } + data.avg = Math.round((data.all / Object.keys(allData).length) * 100) / 100; + + return data; + } + + /** + * @param {Object} allData - All server stats data from the beginning of time + * @returns {Object} Monthly data + */ + static generateDataMonthly(allData) { + const data = {}; + const utcMoment = moment.utc(); + const ids = {}; + const ids6 = {}; + const ids0 = {}; + const order = []; + + for (let i = 0; i < 12; i++) { + order.push(utcMoment.format('MMM YYYY')); + ids[utcMoment.format('YYYY:M')] = utcMoment.format('MMM YYYY'); + if (i < 7) { + ids6[utcMoment.format('YYYY:M')] = utcMoment.format('MMM YYYY'); + } + if (i === 0) { + ids0[utcMoment.format('YYYY:M')] = utcMoment.format('MMM YYYY'); + } + utcMoment.subtract(1, 'months'); + } + + const DP = {}; + data.DP = []; + let avg12monthDP = 0; + let avg6monthDP = 0; + + let avg12 = 0; + let avg6 = 0; + for (const [key, value] of Object.entries(allData)) { + if (ids[key]) { + DP[ids[key]] = value; + + if (!ids0[key]) { + avg12monthDP += DP[ids[key]]; + avg12++; + } + if (ids6[key] && !ids0[key]) { + avg6monthDP += DP[ids[key]]; + avg6++; + } + } + } + + for (let i = 0; i < order.length; i++) { + data.DP.push((i < 9 ? '0' + (i + 1) : i + 1) + '. ' + order[i] + ': ' + ((DP[order[i]] || 0).toLocaleString())); + } + if (avg12) { + data['Last 12 months'] = Math.round(avg12monthDP / avg12).toLocaleString(); + } + if (avg6) { + data['Last 6 months'] = Math.round(avg6monthDP / avg6).toLocaleString(); + } + + return data; + } + /** * Returns a human readable name given application id. * @param {Object} db - Database object, used for querying @@ -34,139 +128,80 @@ class StatsJob extends job.Job { * @returns {undefined} Returns nothing, only callback **/ run(db, done) { - if (config.web.track !== "none") { - db.collection("members").find({global_admin: true}).toArray(function(err, members) { + if (config.web.track !== 'none') { + db.collection('members').find({global_admin: true}).toArray(async(err, members) => { if (!err && members.length > 0) { - db.collection("server_stats_data_points").aggregate([ - { - $group: { - _id: "$m", - e: { $sum: "$e"}, - s: { $sum: "$s"} - } + let license = {}; + if (drill) { + try { + license = await drill.loadLicense(undefined, db); } - ], { allowDiskUse: true }, async function(error, allData) { - if (!error) { - var data = {}; - data.all = 0; - data.month3 = []; - var utcMoment = moment.utc(); - var months = {}; - for (let i = 0; i < 3; i++) { - months[utcMoment.format("YYYY:M")] = true; - utcMoment.subtract(1, 'months'); - } - for (let i = 0; i < allData.length; i++) { - data.all += allData[i].e + allData[i].s; - if (months[allData[i]._id]) { - data.month3.push(allData[i]._id + " - " + (allData[i].e + allData[i].s)); - } - } - data.avg = Math.round((data.all / allData.length) * 100) / 100; - var date = new Date(); - var usersData = []; - - await promisedLoadConfigs(db); - - let domain = ''; - - try { - // try to extract hostname from full domain url - const urlObj = new URL(pluginManager.getConfig('api').domain); - domain = urlObj.hostname; - } - catch (_) { - // do nothing, domain from config will be used as is - } - - usersData.push({ - device_id: domain, - timestamp: Math.floor(date.getTime() / 1000), - hour: date.getHours(), - dow: date.getDay(), - user_details: JSON.stringify({ - custom: { - dataPointsAll: data.all, - dataPointsMonthlyAvg: data.avg, - dataPointsLast3Months: data.month3 - } - }) - }); + catch (error) { + log.e(error); + // do nothing, most likely there is no license + } + } - var formData = { - app_key: "e70ec21cbe19e799472dfaee0adb9223516d238f", - requests: JSON.stringify(usersData) - }; - - request.post({ - url: 'https://stats.count.ly/i/bulk', - body: formData - }, function(a) { - log.d('Done running stats job: %j', a); - done(); - }); + const options = { + monthlyBreakdown: true, + license_hosting: license.license_hosting, + }; + + serverStats.fetchDatapoints(db, {}, options, async(allData) => { + const dataSummary = StatsJob.generateDataSummary(allData); - if (tracker.isEnabled()) { - utcMoment = moment.utc(); - data = {}; - var ids = {}; - var ids6 = {}; - var ids0 = {}; - var order = []; - var Countly = tracker.getSDK(); - for (let i = 0; i < 12; i++) { - order.push(utcMoment.format("MMM YYYY")); - ids[utcMoment.format("YYYY:M")] = utcMoment.format("MMM YYYY"); - if (i < 7) { - ids6[utcMoment.format("YYYY:M")] = utcMoment.format("MMM YYYY"); - } - if (i === 0) { - ids0[utcMoment.format("YYYY:M")] = utcMoment.format("MMM YYYY"); - } - utcMoment.subtract(1, 'months'); - } - - var DP = {}; - data.DP = []; - var avg12monthDP = 0; - var avg6monthDP = 0; - - var avg12 = 0; - var avg6 = 0; - for (let i = 0; i < allData.length; i++) { - if (ids[allData[i]._id]) { - var val = allData[i].e + allData[i].s; - DP[ids[allData[i]._id]] = val; - if (!ids0[allData[i]._id]) { - avg12monthDP += DP[ids[allData[i]._id]]; - avg12++; - } - if (ids6[allData[i]._id] && !ids0[allData[i]._id]) { - avg6monthDP += DP[ids[allData[i]._id]]; - avg6++; - } - } - } - - for (let i = 0; i < order.length; i++) { - data.DP.push((i < 9 ? "0" + (i + 1) : i + 1) + ". " + order[i] + ": " + ((DP[order[i]] || 0).toLocaleString())); - } - - if (avg12) { - data["Last 12 months"] = Math.round(avg12monthDP / avg12).toLocaleString(); - } - if (avg6) { - data["Last 6 months"] = Math.round(avg6monthDP / avg6).toLocaleString(); - } - Countly.user_details({ - "custom": data - }); - - Countly.userData.save(); - } + let date = new Date(); + const usersData = []; + + await promisedLoadConfigs(db); + + let domain = ''; + + try { + // try to extract hostname from full domain url + const urlObj = new URL(pluginManager.getConfig('api').domain); + domain = urlObj.hostname; } - else { + catch (_) { + // do nothing, domain from config will be used as is + } + + usersData.push({ + device_id: domain, + timestamp: Math.floor(date.getTime() / 1000), + hour: date.getHours(), + dow: date.getDay(), + user_details: JSON.stringify({ + custom: { + dataPointsAll: dataSummary.all, + dataPointsMonthlyAvg: dataSummary.avg, + dataPointsLast3Months: dataSummary.month3, + }, + }), + }); + + var formData = { + app_key: 'e70ec21cbe19e799472dfaee0adb9223516d238f', + requests: JSON.stringify(usersData) + }; + + request.post({ + url: 'https://stats.count.ly/i/bulk', + json: formData + }, function(a) { + log.d('Done running stats job: %j', a); done(); + }); + + if (tracker.isEnabled()) { + const dataMonthly = StatsJob.generateDataMonthly(allData); + + const Countly = tracker.getSDK(); + Countly.user_details({ + 'custom': dataMonthly, + }); + + Countly.userData.save(); } }); } @@ -184,4 +219,4 @@ class StatsJob extends job.Job { } } -module.exports = StatsJob; \ No newline at end of file +module.exports = StatsJob; diff --git a/plugins/server-stats/tests.js b/plugins/server-stats/tests/api.js similarity index 99% rename from plugins/server-stats/tests.js rename to plugins/server-stats/tests/api.js index 6c3b6f18e8f..0be383a560b 100644 --- a/plugins/server-stats/tests.js +++ b/plugins/server-stats/tests/api.js @@ -1,8 +1,8 @@ var request = require('supertest'); var should = require('should'); -var testUtils = require("../../test/testUtils"); -const pluginManager = require('../pluginManager.js'); -var statInternalEvents = require('../server-stats/api/parts/stats.js').internalEventsEnum; +var testUtils = require('../../../test/testUtils'); +const pluginManager = require('../../pluginManager.js'); +var statInternalEvents = require('../../server-stats/api/parts/stats.js').internalEventsEnum; request = request.agent(testUtils.url); const dataPointTimeout = 1100; @@ -617,4 +617,4 @@ describe('Testing data points plugin', function() { }); }); }); -}); \ No newline at end of file +}); diff --git a/plugins/server-stats/tests/index.js b/plugins/server-stats/tests/index.js new file mode 100644 index 00000000000..0e5818bf8dd --- /dev/null +++ b/plugins/server-stats/tests/index.js @@ -0,0 +1,2 @@ +require('./api.js'); +require('./job.js'); diff --git a/plugins/server-stats/tests/job.js b/plugins/server-stats/tests/job.js new file mode 100644 index 00000000000..0657d113fda --- /dev/null +++ b/plugins/server-stats/tests/job.js @@ -0,0 +1,51 @@ +const moment = require('moment'); +const should = require('should'); + +const StatsJob = require('../api/jobs/stats.js'); + +const allData = {}; + +for (let count = 0; count < 12; count += 1) { + const utcM = moment.utc(); + utcM.subtract(count, 'month'); + + allData[utcM.format('YYYY:M')] = 1000; +} + +describe('Stats job', () => { + it('Generates data summary', () => { + const { all, avg, month3 } = StatsJob.generateDataSummary(allData); + + should(all).equal(12000); + should(avg).equal(1000); + + const expectedMonth3 = []; + for (let count = 0; count < 3; count += 1) { + const utcM = moment.utc(); + utcM.subtract(count, 'month'); + + expectedMonth3.push(`${utcM.format('YYYY:M')} - 1000`); + } + + should(month3).eql(expectedMonth3); + }); + + it('Generates data monthly', () => { + const monthlyData = StatsJob.generateDataMonthly(allData); + + should(monthlyData['Last 6 months']).equal((1000).toLocaleString()); + should(monthlyData['Last 12 months']).equal((1000).toLocaleString()); + + const expectedDP = []; + for (let count = 0; count < 12; count += 1) { + const utcM = moment.utc(); + utcM.subtract(count, 'month'); + + const idx = count < 9 ? `0${count + 1}` : `${count + 1}`; + + expectedDP.push(`${idx}. ${utcM.format('MMM YYYY')}: ${(1000).toLocaleString()}`); + } + + should(monthlyData.DP).eql(expectedDP); + }); +});