domestic data patch WIP

package.json

@@ -11,7 +11,7 @@
     "type": "npm"
   },
   "scripts": {
-    "test": "jest src/modules/dw/UnbundlingAid/unbundlingAid.test.ts",
+    "test": "jest",
     "lint": "tslint -c tslint.json 'src/**/*.ts'",
     "test:watch": "npm test -- --watch",
     "codecov": "npm run build && jest -u && codecov",

src/modules/utils/__snapshots__/utils.test.ts.snap

@@ -1,21 +1,23 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`Utility functions test should create an aggregate sql query for a single year i.e unbundling aid 1`] = `
-"Object {
-  \\"queryA\\": \\"SELECT to_di_id, year, sum(value) AS value from fact.oda_2015 WHERE value > 0 AND year = 2015 AND sector = 'banking-and-business' GROUP BY to_di_id, year\\",
-  \\"queryB\\": \\"SELECT bundle, year, sum(value) AS value from fact.oda_2015 WHERE value > 0 AND from_di_id = 'afdb' AND to_di_id = 'UG' AND year = 2013 GROUP BY bundle, year\\",
-}"
-`;
-
-exports[`Utility functions test should create an aggregate sql query for multiple years 1`] = `"\\"SELECT bundle, year, sum(value) AS value from fact.oda_2015 WHERE value > 0 AND from_di_id = 'afdb' AND to_di_id = 'UG' AND year >= 2013 AND year <= 2015 GROUP BY bundle, year\\""`;
-
-exports[`Utility functions test should create human friendly numbers i.e 1.5k for 1500 1`] = `
-"Object {
-  \\"formattedB\\": Array [
-    \\"150\\",
-    \\"1.5k\\",
-    \\"15k\\",
-    \\"200m\\",
+exports[`Utility functions test should make tree data 1`] = `
+"Array [
+  Array [],
+  Array [
+    Object {
+      \\"budget_type\\": \\"Actual\\",
+      \\"color\\": \\"#5da3d9\\",
+      \\"id\\": \\"NG\\",
+      \\"levels\\": Array [
+        \\"Total Expenditure\\",
+        \\"Spending From Excess Crude Account /Sovereign Wealth Fund\\",
+        \\"Other Excess Crude Account\\",
+      ],
+      \\"uid\\": \\"Total ExpenditureSpending From Excess Crude Account /Sovereign Wealth FundOther Excess Crude Account\\",
+      \\"value\\": 539768571.48,
+      \\"value_ncu\\": 198000000000,
+      \\"year\\": 2006,
+    },
   ],
-}"
+]"
 `;

src/modules/utils/index.ts

@@ -9,7 +9,7 @@ import {isError} from '@devinit/prelude';
 import {getBudgetLevels, IBudgetLevelRef} from '../refs/countryProfile';
 import {IGetIndicatorArgs, IGetIndicatorValueArgs, IhasDiId, IProcessedSimple,
         IRAW, IRAWDomestic, IToolTipArgs, ISpotlightGetIndicatorArgs,
-        IGetIndicatorArgsSimple} from './types';
+        IGetIndicatorArgsSimple, IMissingDomesticParent} from './types';
 import {approximate, toNumericFields, toId, getTableNameFromSql} from '@devinit/prelude';
 
 export const RECIPIENT = 'recipient';
@@ -206,3 +206,52 @@ export const makeSqlAggregateQuery = (queryArgs: any, groupByField: string, tabl
                 : `${query} ${field} = '${queryArgs[field]}' ${AND}`; // we need to enclose field values in quotes
         }, `SELECT ${groupByField}, year, sum(value) AS value from ${table} WHERE value > 0 AND`);
 };
+
+// A patch of domestic data with missing parent entries
+// Ideally every entry in domestic table should have a parent entry i,e data of the form [l1: a, l2: b, l3:c]
+// l1: a, l2: b, l3:d]  should have a corresponding entry with [l1: a, l2: b] as parent so to speak.
+// solution find data with missing parent levels, get that datas children and sum it up to create new parent entry
+
+export const missingParentsData = (data: DH.IDomestic[]): DH.IDomestic[][] => {
+    return [3, 4].map((level) => {
+        const levelParentData = data.filter(obj => obj.levels.length === level - 1);
+        const levelChildrenData = data.filter(obj => obj.levels.length === level);
+        // parents that are missing in the raw data
+        const missingParents = levelChildrenData.map(child => {
+            // for each child confirm it has a standalone parent
+            const childParent = levelParentData.filter(parent => {
+                // the last parent item in level 2 should be the 2nd last item in level 3
+                return R.last(parent.levels) === R.last(R.init(child.levels));
+            });
+            if (!childParent.length) {
+                return {
+                    levels: R.init(child.levels),
+                    uid: R.init(child.levels).join(''),
+                    level: `l${level - 1}`,
+                    child
+                };
+            }
+            return undefined;
+        })
+        .filter(item => item && item.level) as IMissingDomesticParent[];
+        // create the missing parents
+        return missingParents.reduce((acc, obj) => {
+            const baseParent = {...obj.child, levels: obj.levels, uid: obj.uid};
+            if (acc.length) {
+                const similarParent = acc.find(parent => parent.uid === obj.uid);
+                if (similarParent) {
+                    const joinedParent = {
+                        ...similarParent,
+                        value: obj.child.value + similarParent.value,
+                        value_ncu: obj.child.value_ncu + similarParent.value_ncu
+                    };
+                    // remove previous parent from list
+                    const newParentsList = acc.filter(objP => objP.uid !== joinedParent.uid);
+                    return [...newParentsList, joinedParent];
+                }
+                return [...acc, baseParent];
+            }
+            return [baseParent];
+        }, [] as DH.IDomestic[]);
+    });
+};

src/modules/utils/testData/tree.ts

@@ -0,0 +1,185 @@
+export default [{
+        year: 2006,
+        budget_type: 'Actual',
+        value: 539768571.4,
+        value_ncu: 198000000000,
+        uid: 'BkMRW3iFH4Tf',
+        levels: ['Total Expenditure',
+            'Federal Government Exoenditure',
+            'Federal Government Extrabudgetary Funds'
+        ],
+        color: '#5da3d9'
+    },
+    {
+        year: 2006,
+        budget_type: 'Actual',
+
+        value: 1349421429,
+        value_ncu: 495000000000,
+
+        uid: 'BJmCbniYrN6f',
+        levels: ['Total Expenditure',
+            'Federal Government Exoenditure',
+            'Federal Government Capital Expenditure'
+        ],
+        color: '#5da3d9'
+    },
+    {
+        year: 2006,
+        budget_type: 'Actual',
+        value: 528864155.9,
+        value_ncu: 194000000000,
+
+        uid: 'HyV0-2oYrNTz',
+        levels: ['Total Expenditure',
+            'Spending From Excess Crude Account /Sovereign Wealth Fund',
+            'Other Excess Crude Account',
+            'Shared Infrastructure And Social Spending'
+        ],
+        color: '#5da3d9'
+    },
+    {
+        year: 2006,
+        budget_type: 'Actual',
+
+        value: 6439057403,
+        value_ncu: 2360000000000,
+
+        uid: 'SJHAW2stBV6G',
+        levels: ['Total Expenditure', 'State And Local Government'],
+        color: '#5da3d9'
+    },
+    {
+        year: 2006,
+        budget_type: 'Actual',
+
+        value: 2153622078,
+        value_ncu: 790000000000,
+
+        uid: 'SkLAZ2sYSV6G',
+        levels: ['Total Expenditure', 'Other Expenditure'],
+        color: '#e8443a'
+    },
+    {
+        year: 2006,
+        budget_type: 'Actual',
+
+        value: 577934026,
+        value_ncu: 212000000000,
+
+        uid: 'HkPRb3jtHEpz',
+        levels: ['Total Expenditure', 'Extrabudgetary Funds'],
+        color: '#5da3d9'
+    },
+    {
+        year: 2006,
+        budget_type: 'Actual',
+
+        value: 10904415.58,
+        value_ncu: 4000000000,
+
+        uid: 'HyuRZnjtHE6z',
+        levels: ['Total Expenditure',
+            'Spending From Excess Crude Account /Sovereign Wealth Fund',
+            'Other Excess Crude Account',
+            'Explicit Fuel Subsidy'
+        ],
+        color: '#5da3d9'
+    },
+    {
+        year: 2006,
+        budget_type: 'Actual',
+
+        value: 14260249481,
+        value_ncu: 5230000000000,
+
+        uid: 'BJF0ZnjKSEpf',
+        levels: ['Total Expenditure'],
+        color: '#7b3b89'
+    },
+    {
+        year: 2006,
+        budget_type: 'Actual',
+
+        value: 607921168.8,
+        value_ncu: 223000000000,
+
+        uid: 'H19R-3oFBEaM',
+        levels: ['Total Expenditure',
+            'Federal Government Exoenditure',
+            'Federal Government Recurrent Expenditure',
+            'Overhead Cost'
+        ],
+        color: '#5da3d9'
+    },
+    {
+        year: 2006,
+        budget_type: 'Actual',
+
+        value: 3200445974,
+        value_ncu: 1170000000000,
+
+        uid: 'S1i0ZhjYBVpM',
+        levels: ['Total Expenditure',
+            'Federal Government Exoenditure',
+            'Federal Government Recurrent Expenditure'
+        ],
+        color: '#5da3d9'
+    },
+    {
+        year: 2006,
+        budget_type: 'Actual',
+
+        value: 509781428.6,
+        value_ncu: 187000000000,
+
+        uid: 'H13RW3oKBV6G',
+        levels: ['Total Expenditure',
+            'Federal Government Exoenditure',
+            'Federal Government Recurrent Expenditure',
+            'Interest Payments'
+        ],
+        color: '#5da3d9'
+    },
+    {
+        year: 2006,
+        budget_type: 'Actual',
+
+        value: 269884285.7,
+        value_ncu: 99000000000,
+
+        uid: 'ByaAb3jFH4af',
+        levels: ['Total Expenditure',
+            'Federal Government Exoenditure',
+            'Federal Government Recurrent Expenditure',
+            'Transfers'
+        ],
+        color: '#5da3d9'
+    },
+    {
+        year: 2006,
+        budget_type: 'Actual',
+
+        value: 5089635974,
+        value_ncu: 1870000000000,
+
+        uid: 'SkA0-hjKHVaM',
+        levels: ['Total Expenditure', 'Federal Government Exoenditure'],
+        color: '#5da3d9'
+    },
+    {
+        year: 2006,
+        budget_type: 'Actual',
+
+        value: 1812859091,
+        value_ncu: 665000000000,
+
+        uid: 'S11kMnsFS4Tz',
+        levels: ['Total Expenditure',
+            'Federal Government Exoenditure',
+            'Federal Government Recurrent Expenditure',
+            'Personnel'
+        ],
+        color: '#5da3d9'
+    }
+];

src/modules/utils/types.ts

@@ -1,5 +1,12 @@
 import {IDB} from '@devinit/graphql-next/lib/db';
 
+export interface IMissingDomesticParent {
+    level: string;
+    levels: string[]; // for debugging
+    child: DH.IDomestic;
+    uid: string;
+}
+
 export interface IGetIndicatorArgs {
     id?: string;
     query: string;
@@ -84,10 +91,12 @@ export interface IRAWDomestic {
     di_id: string;
     year: string;
     budget_type: string;
+    value: number;
+    value_ncu: number;
     l1: string;
-    l2: string;
-    l3: string;
-    l4: string;
+    l2?: string;
+    l3?: string;
+    l4?: string;
 }
 export interface IToolTipArgs {
     query?: string;
@@ -112,3 +121,12 @@ export interface IGetIndicatorValueArgs {
     db: IDB;
     format: boolean;
 }
+
+export interface IDomesticTree {
+    id: string;
+    year: string;
+    budget_type: string;
+    value: number;
+    value_ncu: number;
+    children: IDomesticTree[];
+}

src/modules/utils/utils.test.ts

@@ -1,5 +1,6 @@
-import {makeSqlAggregateQuery, isDonor} from '.';
+import {makeSqlAggregateQuery, isDonor, missingParentsData} from '.';
 import * as prettyFormat from 'pretty-format';
+import data from './testData/tree';
 import {approximate, toId, getTotal, normalizeKeyName} from '@devinit/prelude';
 
 const dataA = [
@@ -61,4 +62,8 @@ describe('Utility functions test', () => {
         expect(isDonorCountryA).toBe(false);
         expect(isDonorCountryB).toBe(true);
     }, 10000);
+    it('should make tree data', () => {
+        const result = missingParentsData(data);
+        expect(prettyFormat(result)).toMatchSnapshot();
+    });
 });