From df75dd013d332b9fed5e7c6530911407d0eb6f96 Mon Sep 17 00:00:00 2001 From: Jacques Labuschagne Date: Tue, 24 Jul 2018 00:23:22 +0100 Subject: [PATCH] Bugfixes and multi-contract per org investigation. --- api/lib/data_store.js | 10 +++-- api/lib/data_sync.js | 8 ++-- api/lib/data_sync_utils.js | 7 +--- api/lib/espo.js | 24 ++++++++---- api/lib/get_customer.js | 3 +- api/lib/get_customer_list.js | 8 +++- api/lib/get_sla_hours.js | 3 +- api/lib/get_sla_unquoted.js | 2 + api/lib/get_users.js | 4 +- api/lib/org_data.js | 71 +++++++++++------------------------- frontend/proxy/urls.py | 2 + frontend/proxy/views.py | 3 +- frontend/static/index.js | 15 ++++++-- 13 files changed, 81 insertions(+), 79 deletions(-) diff --git a/api/lib/data_store.js b/api/lib/data_store.js index f8365c1..fc55a25 100644 --- a/api/lib/data_store.js +++ b/api/lib/data_store.js @@ -1,7 +1,7 @@ var util = require('./util'), sqlite3 = require('sqlite3').verbose(); -const DEBUG = true; +const DEBUG = false; 'use strict'; @@ -83,10 +83,14 @@ let sql = { dump: { contracts: util.trim ` SELECT c.id as contract_id,c.org_id,c.start_date,c.end_date, - cs.system_id, - b.id as budget_id,b.base_hours,b.base_hours_spent,b.sla_quote_hours,b.additional_hours + cs.system_id FROM contracts c LEFT JOIN contract_system_link cs ON cs.contract_id=c.id + `, + budgets: util.trim ` + SELECT c.id as contract_id, + b.id as budget_id,b.base_hours,b.base_hours_spent,b.sla_quote_hours,b.additional_hours + FROM contracts c LEFT JOIN contract_budget_link cb ON cb.contract_id=c.id LEFT JOIN budgets b ON b.id=cb.budget_id `, diff --git a/api/lib/data_sync.js b/api/lib/data_sync.js index 97c4abf..e602ae1 100644 --- a/api/lib/data_sync.js +++ b/api/lib/data_sync.js @@ -277,7 +277,7 @@ function process_wrms_data(resolve, reject, contract, wr_rows){ timesheet_buckets = {}, timesheet_budgets = {}; - // We only reach this point if there are no live quotes. + // We only reach this point if there are no live quotes; add up the timesheets. for (; iw < wr_rows.length && wr_rows[iw].request_id === wr.request_id; ++iw){ let iw_hours = wr_rows[iw].timesheet_hours, @@ -335,8 +335,8 @@ function process_wrms_data(resolve, reject, contract, wr_rows){ try{ await sqlite_promise( store.dbs.syncing, - 'UPDATE budgets SET base_hours_spent=? WHERE id=?', - b.base_hours_spent + n, + 'UPDATE budgets SET base_hours_spent=base_hours_spent + ? WHERE id=?', + n, b.id ); }catch(err){ @@ -451,7 +451,7 @@ function select_best_budget(contract, period){ store.dbs.syncing.all( // TODO FIXME: figure out the syntax of prepared statement with placeholder // containing % and using ESCAPE clause. Right now this is an easy sqli vector. - `SELECT id,base_hours,base_hours_spent,sla_quote_hours,additional_hours FROM budgets WHERE id LIKE '${contract.name} %'`, + `SELECT id,base_hours,base_hours_spent,sla_quote_hours,additional_hours FROM budgets WHERE id LIKE '${contract.name.replace(/'/, '')} %'`, (err, budgets) => { if (err){ reject(err); diff --git a/api/lib/data_sync_utils.js b/api/lib/data_sync_utils.js index 0407432..26a053b 100644 --- a/api/lib/data_sync_utils.js +++ b/api/lib/data_sync_utils.js @@ -29,13 +29,10 @@ function match_non_monthly_budget_name(id, ctx){ let from = new Date(m[1]), to = new Date(m[2]); - // force "to" to be the end of the month - to.setMonth(to.getMonth()+1); - to.setDate(-1); - let pdate = new Date(ctx.period); - return pdate > from && pdate < to; + // Budget names are [start,end) + return pdate >= from && pdate < to; } return false; } diff --git a/api/lib/espo.js b/api/lib/espo.js index 4837048..547dac4 100644 --- a/api/lib/espo.js +++ b/api/lib/espo.js @@ -161,11 +161,15 @@ function query_espo_accounts(context){ // type: "Service Level Agreement" function merge_espo_data(context){ let active = {}; + util.log_debug(__filename, '==================== Raw contracts: ', DEBUG); context.contracts.list.forEach(c => { util.log_debug(__filename, JSON.stringify(c), DEBUG); if (c.status === 'Active' && c.type === 'Service Level Agreement'){ - active[c.accountId] = { + if (!active[c.accountId]){ + active[c.accountId] = []; + } + active[c.accountId].push({ name: c.name, org_name: c.accountName, type: (c.sLAFrequency || 'unknown').toLowerCase().trim(), @@ -173,16 +177,22 @@ function merge_espo_data(context){ start_date: util.date_fmt(new Date(c.startDate)), end_date: util.date_fmt(new Date(c.endrenewalDate)), systems: (c.systemID ? c.systemID.split(/,\s*/).map(n => parseInt(n)).filter(n => n !== null && !isNaN(n)) : []) - }; + }); } }); + + let result = []; + util.log_debug(__filename, '==================== Raw accounts: ', DEBUG); - context.accounts.list.forEach(a => { - util.log_debug(__filename, JSON.stringify(a), DEBUG); - if (active[a.id]){ - active[a.id].org_id = a.orgID; + context.accounts.list.forEach(arr => { + util.log_debug(__filename, JSON.stringify(arr), DEBUG); + if (active[arr.id]){ + active[arr.id].forEach(a => { + a.org_id = arr.orgID; + result.push(a); + }); } }); - return Object.values(active); + return Object.values(result); } diff --git a/api/lib/get_customer.js b/api/lib/get_customer.js index 44d8c74..c6c542c 100644 --- a/api/lib/get_customer.js +++ b/api/lib/get_customer.js @@ -14,7 +14,8 @@ module.exports = function(req, res, next, ctx){ FROM contracts c JOIN contract_system_link cs ON c.id=cs.contract_id WHERE c.org_id=? - AND cs.system_id IN (${ctx.sys.join(',')})`, + AND cs.system_id IN (${ctx.sys.join(',')}) + GROUP BY c.id`, ctx.org, handler(data => { if (!Array.isArray(data) || data.length < 1){ diff --git a/api/lib/get_customer_list.js b/api/lib/get_customer_list.js index 7a5fdc8..5a32ffc 100644 --- a/api/lib/get_customer_list.js +++ b/api/lib/get_customer_list.js @@ -7,8 +7,10 @@ module.exports = function(req, res, next, ctx){ store.query( util.trim `SELECT c.id, c.org_name, - c.org_id + c.org_id, + s.system_id FROM contracts c + JOIN contract_system_link s ON c.id=s.contract_id ORDER BY c.org_name,c.id`, handler(data => { if (!Array.isArray(data)){ @@ -18,7 +20,9 @@ module.exports = function(req, res, next, ctx){ let r = {}; data.forEach(row => { - r[row.id] = row.org_id; + let o = r[row.id] || {org_id: row.org_id, systems: []}; + o.systems.push(row.system_id); + r[row.id] = o; }) return r; diff --git a/api/lib/get_sla_hours.js b/api/lib/get_sla_hours.js index 663d727..879e5c8 100644 --- a/api/lib/get_sla_hours.js +++ b/api/lib/get_sla_hours.js @@ -19,7 +19,8 @@ module.exports = function(req, res, next, ctx){ JOIN contracts c ON c.id=cb.contract_id JOIN contract_system_link cs ON cs.contract_id=c.id WHERE c.org_id=? - AND cs.system_id IN (${ctx.sys.join(',')})`, + AND cs.system_id IN (${ctx.sys.join(',')}) + GROUP BY b.id`, ctx.org, handler(data => { if (!Array.isArray(data) || data.length < 1){ diff --git a/api/lib/get_sla_unquoted.js b/api/lib/get_sla_unquoted.js index 984ee21..5a27f18 100644 --- a/api/lib/get_sla_unquoted.js +++ b/api/lib/get_sla_unquoted.js @@ -26,6 +26,8 @@ module.exports = function(req, res, next, ctx){ handler(data => { let r = {result: [{wr: "None", result: 0}]}; + util.log_debug(__filename, 'raw data: ' + JSON.stringify(data, null, 2)); + if (Array.isArray(data) && data.length > 0){ // Compress the list to one element per WR let wrs = {}; diff --git a/api/lib/get_users.js b/api/lib/get_users.js index 9c936be..810210f 100644 --- a/api/lib/get_users.js +++ b/api/lib/get_users.js @@ -14,7 +14,9 @@ module.exports = query.prepare( next(r); }, (key, ctx, next, error) => { - if (!util.get_org(ctx) || + // TODO: decide where to fetch tokens from now that the config no longer exists. + if (true || + !util.get_org(ctx) || !util.get_org(ctx).users || !util.get_org(ctx).users.hostname || !util.get_org(ctx).users.token) diff --git a/api/lib/org_data.js b/api/lib/org_data.js index cfbfae0..2370525 100644 --- a/api/lib/org_data.js +++ b/api/lib/org_data.js @@ -4,33 +4,44 @@ var util = require('./util'); var orgs = {}; +const DEBUG = false; + exports.__raw = function(){ return orgs } exports.add_org = function(contract){ - util.log_debug(__filename, `add_org(${JSON.stringify(contract)})`); + util.log_debug(__filename, `add_org(${JSON.stringify(contract)})`, DEBUG); orgs[contract.name] = JSON.parse(JSON.stringify(contract)); } exports.add_system = function(contract, system){ - util.log_debug(__filename, `add_system(${JSON.stringify(contract)}, ${system})`); + util.log_debug(__filename, `add_system(${JSON.stringify(contract)}, ${system})`, DEBUG); let o = orgs[contract.name]; if (o){ let s = o.systems || []; - s.push(system); - orgs[contract.name].systems = s; + if (!s.includes(system)){ + s.push(system); + } + o.systems = s; }else{ throw new Error(`add_system(${system}) called before add_org(${contract.name})`); } } -function get_org_by_key(field, val){ +// Succeed if either argument is null, or if the arrays match exactly. +function systems_match(a, b){ + return !Array.isArray(a) || + !Array.isArray(b) || + a.sort().join(',') === b.sort().join(','); +} + +function get_org_by_key(field, val, systems){ let o = null; Object.values(orgs).forEach(org => { - if (org[field] === val){ + if (org[field] === val && systems_match(systems, org.systems)){ o = org; } }); @@ -45,8 +56,6 @@ function get_org_by_key(field, val){ // // Returns contracts.* or null exports.get_org = function(id){ - util.log_debug(__filename, 'get_org(' + JSON.stringify({id:id}) + ')'); - if (id === undefined){ throw new Error('get_org() with no ID specified'); } @@ -55,53 +64,17 @@ exports.get_org = function(id){ n = parseInt(id); // in case it's a bare number if (id.org || !isNaN(n)){ - o = get_org_by_key('org_id', id.org || n); + // Numeric lookups may need to be disambiguated by system + o = get_org_by_key('org_id', id.org || n, id.systems); }else{ + // Name lookups are already unique... Unless someone has messed up in the CRM, + // in which case we'll just return the first match o = get_org_by_key('name', id); } - return o; -} - + util.log_debug(__filename, 'get_org(' + JSON.stringify({id:id}) + ') => ' + JSON.stringify(o), DEBUG); -/* -var config = require('config'), - fs = require('fs'); - -var __orgs = (function(cfg){ - let o = JSON.parse(JSON.stringify(cfg)); - Object.keys(o).forEach(name => { - o[name].name = name; - o[ o[name].id ] = o[name]; - }); return o; -})(config.get('orgs')); - -exports.get_org = function(id){ - if (id === undefined){ - return null; - } - return __orgs[id.org ? id.org : id]; } -exports.get_all_orgs = function(){ - // not using __orgs because it doubles up ID and name keys - let o = JSON.parse(JSON.stringify(config.get('orgs'))); - return Object.keys(o).map(name => { - return o[name].id; - }); -} -exports.get_all_systems = function(){ - let arr = []; - // not using __orgs because it doubles up ID and name keys - let o = JSON.parse(JSON.stringify(config.get('orgs'))); - Object.keys(o).forEach(name => { - let s = o[name].default_system; - if (s && s.match(/^[0-9,]+$/)){ - arr = arr.concat(s.split(/,/)); - } - }); - return arr; -} -*/ diff --git a/frontend/proxy/urls.py b/frontend/proxy/urls.py index 395d693..9ce489f 100644 --- a/frontend/proxy/urls.py +++ b/frontend/proxy/urls.py @@ -2,6 +2,8 @@ from . import views +# TODO: to support multiple active contracts per client, the "dashboard" and "api" URLs need +# to match (?P[0-9,]+) between and . urlpatterns = [ url(r'^$', views.IndexView.as_view(), name='index'), url(r'^dashboard/(?P[_a-z0-9A-Z ]+)/$', views.DashboardView.as_view(), name='dashboard'), diff --git a/frontend/proxy/views.py b/frontend/proxy/views.py index 6f444e5..918a1c1 100644 --- a/frontend/proxy/views.py +++ b/frontend/proxy/views.py @@ -124,8 +124,7 @@ def get(self, request, item, client, month): if not request.user.is_superuser: # If not admin, cannot view any months earlier than July 2017. - # TODO: fix this properly with a database for the client SLA - # dates instead of hard coding + # TODO: respect the SLA dates in the CRM instead. if month_dt < min_dt: month = min_dt.strftime("%Y-%m") diff --git a/frontend/static/index.js b/frontend/static/index.js index c4219c2..3d97411 100644 --- a/frontend/static/index.js +++ b/frontend/static/index.js @@ -2,6 +2,13 @@ URI_EXT = '__vendor/default/2017-7'; var tile_exists = {}; +function mkuri(org){ + // TODO: if we need to support multiple active contracts per client, then + // after making the django proxy changes, insert org.systems.join(',') + // instead of "default" here: + return org.org_id + "/default/" + PERIOD; +} + query('/customer_list', function(err, data){ if (err){ console.log('customer_list: ' + err); @@ -15,7 +22,7 @@ query('/customer_list', function(err, data){ } } - const org = {name: name, org_id: data[name]}; + const org = {name: name, org_id: data[name].org_id, systems: data[name].systems}; if (!tile_exists[name]){ draw_tile(org, i, 'blue'); @@ -26,7 +33,7 @@ query('/customer_list', function(err, data){ handle_hours(org, i, after_fetches), undefined, 0, - org.org_id + "/default/" + PERIOD + mkuri(org) ); }); }); @@ -64,7 +71,7 @@ function handle_hours(org, i, next){ handle_count(org, i, color, next), undefined, 0, - org.org_id + "/default/" + PERIOD + mkuri(org) ); }; } @@ -93,7 +100,7 @@ function draw_tile(org, i, color, count){ '' + '
' + '' + count + '' + - '' + org.name + '' + + '' + org.name.replace(/ SLA /, ' ').replace(/\d\d\d\d\s?-\s?/, '') + '' + '
' + '' );