Merge pull request #133 from musoke/test-csv

Test and refactor csvtojson

.github/workflows/nodejs.yml

@@ -29,5 +29,8 @@ jobs:
       - name: Run unit tests
         run: yarn test
 
+      - name: Check if JSON conversions changed by running tests
+        run: git diff --exit-code "src/data/*.json"
+
       - name: Generate production files
         run: yarn build

src/data/__tests__/csvtojson.ts

@@ -0,0 +1,34 @@
+import { expect, test } from '@jest/globals'
+
+// NOTE: these imports implicitly regenerate the JSON files that store the conversions.
+import {
+  BOULDER_GRADE_TABLE,
+  ROUTE_GRADE_TABLE,
+  ICE_GRADE_TABLE,
+  AID_GRADE_TABLE
+} from '../csvtojson'
+
+describe.each([
+  { table: BOULDER_GRADE_TABLE },
+  { table: ROUTE_GRADE_TABLE },
+  { table: ICE_GRADE_TABLE },
+  { table: AID_GRADE_TABLE }
+])('Grade tables are parsable', ({ table }) => {
+  test('each table is an array', async () => {
+    expect.hasAssertions() // double check that async tests run
+    const TABLE = await table
+
+    expect(TABLE).toBeInstanceOf(Array)
+
+    expect(TABLE.length).toBeGreaterThan(1)
+  })
+
+  test('each table has a numeric score in first row', async () => {
+    expect.hasAssertions() // double check that async tests run
+    const TABLE = await table
+
+    expect(TABLE[1]).toMatchObject(
+      { score: expect.any(Number) }
+    )
+  })
+})

src/data/aid.csv

@@ -1,4 +1,4 @@
-Score,Aid
+score,aid
 0,A0
 1,A0
 2,A0

src/data/boulder.csv

@@ -1,4 +1,4 @@
-Score,V Scale,Font Scale
+score,v,font
 0,VB-,1a
 1,VB-,1a
 2,VB-,1a+
@@ -106,4 +106,4 @@ Score,V Scale,Font Scale
 104,V21,9c
 105,V21,9c
 106,V22,9c+
-107,V22,9c+
+107,V22,9c+

src/data/boulder.json

@@ -1 +1 @@
-[{"score":0,"v":"VB-","font":"1a","band":"unknown"},{"score":1,"v":"VB-","font":"1a","band":"unknown"},{"score":2,"v":"VB-","font":"1a+","band":"unknown"},{"score":3,"v":"VB-","font":"1a+","band":"unknown"},{"score":4,"v":"VB-","font":"1b","band":"unknown"},{"score":5,"v":"VB-","font":"1b","band":"unknown"},{"score":6,"v":"VB-","font":"1b+","band":"unknown"},{"score":7,"v":"VB-","font":"1b+","band":"unknown"},{"score":8,"v":"VB-","font":"1c","band":"unknown"},{"score":9,"v":"VB-","font":"1c","band":"unknown"},{"score":10,"v":"VB-","font":"1c+","band":"unknown"},{"score":11,"v":"VB-","font":"1c+","band":"unknown"},{"score":12,"v":"VB","font":"2a","band":"unknown"},{"score":13,"v":"VB","font":"2a","band":"unknown"},{"score":14,"v":"VB","font":"2a+","band":"unknown"},{"score":15,"v":"VB","font":"2a+","band":"unknown"},{"score":16,"v":"VB","font":"2b","band":"unknown"},{"score":17,"v":"VB","font":"2b","band":"unknown"},{"score":18,"v":"VB","font":"2b+","band":"unknown"},{"score":19,"v":"VB","font":"2b+","band":"unknown"},{"score":20,"v":"VB","font":"2c","band":"unknown"},{"score":21,"v":"VB","font":"2c","band":"unknown"},{"score":22,"v":"VB","font":"2c+","band":"unknown"},{"score":23,"v":"VB","font":"2c+","band":"unknown"},{"score":24,"v":"VB+","font":"3a","band":"unknown"},{"score":25,"v":"VB+","font":"3a","band":"unknown"},{"score":26,"v":"VB+","font":"3a+","band":"unknown"},{"score":27,"v":"VB+","font":"3a+","band":"unknown"},{"score":28,"v":"VB+","font":"3b","band":"unknown"},{"score":29,"v":"VB+","font":"3b","band":"unknown"},{"score":30,"v":"VB+","font":"3b+","band":"unknown"},{"score":31,"v":"VB+","font":"3b+","band":"unknown"},{"score":32,"v":"VB+","font":"3c","band":"unknown"},{"score":33,"v":"VB+","font":"3c","band":"unknown"},{"score":34,"v":"VB+","font":"3c+","band":"unknown"},{"score":35,"v":"VB+","font":"3c+","band":"unknown"},{"score":36,"v":"V0-","font":"4a","band":"unknown"},{"score":37,"v":"V0-","font":"4a","band":"unknown"},{"score":38,"v":"V0-","font":"4a+","band":"unknown"},{"score":39,"v":"V0-","font":"4a+","band":"unknown"},{"score":40,"v":"V0","font":"4b","band":"unknown"},{"score":41,"v":"V0","font":"4b","band":"unknown"},{"score":42,"v":"V0","font":"4b+","band":"unknown"},{"score":43,"v":"V0","font":"4b+","band":"unknown"},{"score":44,"v":"V0+","font":"4c","band":"unknown"},{"score":45,"v":"V0+","font":"4c","band":"unknown"},{"score":46,"v":"V0+","font":"4c+","band":"unknown"},{"score":47,"v":"V0+","font":"4c+","band":"unknown"},{"score":48,"v":"V1","font":"5a","band":"unknown"},{"score":49,"v":"V1","font":"5a","band":"unknown"},{"score":50,"v":"V1","font":"5a+","band":"unknown"},{"score":51,"v":"V1","font":"5a+","band":"unknown"},{"score":52,"v":"V1","font":"5b","band":"unknown"},{"score":53,"v":"V2","font":"5b","band":"unknown"},{"score":54,"v":"V2","font":"5b+","band":"unknown"},{"score":55,"v":"V2","font":"5b+","band":"unknown"},{"score":56,"v":"V2","font":"5c","band":"unknown"},{"score":57,"v":"V2","font":"5c","band":"unknown"},{"score":58,"v":"V3","font":"5c+","band":"unknown"},{"score":59,"v":"V3","font":"5c+","band":"unknown"},{"score":60,"v":"V3","font":"6a","band":"unknown"},{"score":61,"v":"V3","font":"6a","band":"unknown"},{"score":62,"v":"V3","font":"6a+","band":"unknown"},{"score":63,"v":"V4","font":"6a+","band":"unknown"},{"score":64,"v":"V4","font":"6b","band":"unknown"},{"score":65,"v":"V4","font":"6b","band":"unknown"},{"score":66,"v":"V4","font":"6b+","band":"unknown"},{"score":67,"v":"V4","font":"6b+","band":"unknown"},{"score":68,"v":"V5","font":"6c","band":"unknown"},{"score":69,"v":"V5","font":"6c","band":"unknown"},{"score":70,"v":"V5","font":"6c+","band":"unknown"},{"score":71,"v":"V6","font":"6c+","band":"unknown"},{"score":72,"v":"V6","font":"7a","band":"unknown"},{"score":73,"v":"V6","font":"7a","band":"unknown"},{"score":74,"v":"V7","font":"7a+","band":"unknown"},{"score":75,"v":"V7","font":"7a+","band":"unknown"},{"score":76,"v":"V7","font":"7b","band":"unknown"},{"score":77,"v":"V8","font":"7b","band":"unknown"},{"score":78,"v":"V8","font":"7b+","band":"unknown"},{"score":79,"v":"V8","font":"7b+","band":"unknown"},{"score":80,"v":"V9","font":"7c","band":"unknown"},{"score":81,"v":"V9","font":"7c","band":"unknown"},{"score":82,"v":"V10","font":"7c+","band":"unknown"},{"score":83,"v":"V10","font":"7c+","band":"unknown"},{"score":84,"v":"V11","font":"8a","band":"unknown"},{"score":85,"v":"V11","font":"8a","band":"unknown"},{"score":86,"v":"V12","font":"8a+","band":"unknown"},{"score":87,"v":"V12","font":"8a+","band":"unknown"},{"score":88,"v":"V13","font":"8b","band":"unknown"},{"score":89,"v":"V13","font":"8b","band":"unknown"},{"score":90,"v":"V14","font":"8b+","band":"unknown"},{"score":91,"v":"V14","font":"8b+","band":"unknown"},{"score":92,"v":"V15","font":"8c","band":"unknown"},{"score":93,"v":"V15","font":"8c","band":"unknown"},{"score":94,"v":"V16","font":"8c+","band":"unknown"},{"score":95,"v":"V16","font":"8c+","band":"unknown"},{"score":96,"v":"V17","font":"9a","band":"unknown"},{"score":97,"v":"V17","font":"9a","band":"unknown"},{"score":98,"v":"V18","font":"9a+","band":"unknown"},{"score":99,"v":"V18","font":"9a+","band":"unknown"},{"score":100,"v":"V19","font":"9b","band":"unknown"},{"score":101,"v":"V19","font":"9b","band":"unknown"},{"score":102,"v":"V20","font":"9b+","band":"unknown"},{"score":103,"v":"V20","font":"9b+","band":"unknown"},{"score":104,"v":"V21","font":"9c","band":"unknown"},{"score":105,"v":"V21","font":"9c","band":"unknown"},{"score":106,"v":"V22","font":"9c+","band":"unknown"},{"score":107,"v":"V22","font":"9c+","band":"unknown"}]
+[{"score":0,"v":"VB-","font":"1a"},{"score":1,"v":"VB-","font":"1a"},{"score":2,"v":"VB-","font":"1a+"},{"score":3,"v":"VB-","font":"1a+"},{"score":4,"v":"VB-","font":"1b"},{"score":5,"v":"VB-","font":"1b"},{"score":6,"v":"VB-","font":"1b+"},{"score":7,"v":"VB-","font":"1b+"},{"score":8,"v":"VB-","font":"1c"},{"score":9,"v":"VB-","font":"1c"},{"score":10,"v":"VB-","font":"1c+"},{"score":11,"v":"VB-","font":"1c+"},{"score":12,"v":"VB","font":"2a"},{"score":13,"v":"VB","font":"2a"},{"score":14,"v":"VB","font":"2a+"},{"score":15,"v":"VB","font":"2a+"},{"score":16,"v":"VB","font":"2b"},{"score":17,"v":"VB","font":"2b"},{"score":18,"v":"VB","font":"2b+"},{"score":19,"v":"VB","font":"2b+"},{"score":20,"v":"VB","font":"2c"},{"score":21,"v":"VB","font":"2c"},{"score":22,"v":"VB","font":"2c+"},{"score":23,"v":"VB","font":"2c+"},{"score":24,"v":"VB+","font":"3a"},{"score":25,"v":"VB+","font":"3a"},{"score":26,"v":"VB+","font":"3a+"},{"score":27,"v":"VB+","font":"3a+"},{"score":28,"v":"VB+","font":"3b"},{"score":29,"v":"VB+","font":"3b"},{"score":30,"v":"VB+","font":"3b+"},{"score":31,"v":"VB+","font":"3b+"},{"score":32,"v":"VB+","font":"3c"},{"score":33,"v":"VB+","font":"3c"},{"score":34,"v":"VB+","font":"3c+"},{"score":35,"v":"VB+","font":"3c+"},{"score":36,"v":"V0-","font":"4a"},{"score":37,"v":"V0-","font":"4a"},{"score":38,"v":"V0-","font":"4a+"},{"score":39,"v":"V0-","font":"4a+"},{"score":40,"v":"V0","font":"4b"},{"score":41,"v":"V0","font":"4b"},{"score":42,"v":"V0","font":"4b+"},{"score":43,"v":"V0","font":"4b+"},{"score":44,"v":"V0+","font":"4c"},{"score":45,"v":"V0+","font":"4c"},{"score":46,"v":"V0+","font":"4c+"},{"score":47,"v":"V0+","font":"4c+"},{"score":48,"v":"V1","font":"5a"},{"score":49,"v":"V1","font":"5a"},{"score":50,"v":"V1","font":"5a+"},{"score":51,"v":"V1","font":"5a+"},{"score":52,"v":"V1","font":"5b"},{"score":53,"v":"V2","font":"5b"},{"score":54,"v":"V2","font":"5b+"},{"score":55,"v":"V2","font":"5b+"},{"score":56,"v":"V2","font":"5c"},{"score":57,"v":"V2","font":"5c"},{"score":58,"v":"V3","font":"5c+"},{"score":59,"v":"V3","font":"5c+"},{"score":60,"v":"V3","font":"6a"},{"score":61,"v":"V3","font":"6a"},{"score":62,"v":"V3","font":"6a+"},{"score":63,"v":"V4","font":"6a+"},{"score":64,"v":"V4","font":"6b"},{"score":65,"v":"V4","font":"6b"},{"score":66,"v":"V4","font":"6b+"},{"score":67,"v":"V4","font":"6b+"},{"score":68,"v":"V5","font":"6c"},{"score":69,"v":"V5","font":"6c"},{"score":70,"v":"V5","font":"6c+"},{"score":71,"v":"V6","font":"6c+"},{"score":72,"v":"V6","font":"7a"},{"score":73,"v":"V6","font":"7a"},{"score":74,"v":"V7","font":"7a+"},{"score":75,"v":"V7","font":"7a+"},{"score":76,"v":"V7","font":"7b"},{"score":77,"v":"V8","font":"7b"},{"score":78,"v":"V8","font":"7b+"},{"score":79,"v":"V8","font":"7b+"},{"score":80,"v":"V9","font":"7c"},{"score":81,"v":"V9","font":"7c"},{"score":82,"v":"V10","font":"7c+"},{"score":83,"v":"V10","font":"7c+"},{"score":84,"v":"V11","font":"8a"},{"score":85,"v":"V11","font":"8a"},{"score":86,"v":"V12","font":"8a+"},{"score":87,"v":"V12","font":"8a+"},{"score":88,"v":"V13","font":"8b"},{"score":89,"v":"V13","font":"8b"},{"score":90,"v":"V14","font":"8b+"},{"score":91,"v":"V14","font":"8b+"},{"score":92,"v":"V15","font":"8c"},{"score":93,"v":"V15","font":"8c"},{"score":94,"v":"V16","font":"8c+"},{"score":95,"v":"V16","font":"8c+"},{"score":96,"v":"V17","font":"9a"},{"score":97,"v":"V17","font":"9a"},{"score":98,"v":"V18","font":"9a+"},{"score":99,"v":"V18","font":"9a+"},{"score":100,"v":"V19","font":"9b"},{"score":101,"v":"V19","font":"9b"},{"score":102,"v":"V20","font":"9b+"},{"score":103,"v":"V20","font":"9b+"},{"score":104,"v":"V21","font":"9c"},{"score":105,"v":"V21","font":"9c"},{"score":106,"v":"V22","font":"9c+"},{"score":107,"v":"V22","font":"9c+"}]

src/data/csvtojson.ts

@@ -6,84 +6,43 @@ import * as fs from 'fs'
 import { AidGrade, Boulder, Route, IceGrade } from '../scales'
 import path from 'path'
 
-const boulderGrades: Boulder[] = []
-const routeGrades: Route[] = []
+const dataDir = path.join(process.cwd(), 'src', 'data')
 
-const writeDir = path.join(process.cwd(), 'src/data')
-/* Use 'unknown' for default band property as grade band is assigned in each individual grade scale.
- */
+async function getData<T> (pathCsv: any, pathJson): Promise<T[]> {
+  const data: T[] = []
+  return await new Promise((resolve, reject) => {
+    fs.createReadStream(pathCsv)
+      .on('error', error => {
+        console.warn('error in parsing:', error)
+        reject(error)
+      })
+      .pipe(csv({
+        mapValues: ({ header, index, value }) => { return header === 'score' ? parseInt(value, 10) : value }
+      }))
+      .on('data', (row) => {
+        data.push(row)
+      })
+      .on('end', () => {
+        const parsedData = JSON.stringify(data)
+        fs.writeFileSync(pathJson, parsedData)
 
-fs.createReadStream(path.join(process.cwd(), 'src/data/boulder.csv'))
-  .pipe(csv())
-  .on('data', (data) => {
-    if (data['V Scale'] === '' && data['Font Scale'] === '') {
-      return
-    }
-    boulderGrades.push({
-      score: parseInt(data.Score, 10),
-      v: data['V Scale'],
-      font: data['Font Scale'],
-      band: 'unknown'
-    })
-  })
-  .on('end', () => {
-    const data = JSON.stringify(boulderGrades)
-    fs.writeFileSync(`${writeDir}/boulder.json`, data)
+        resolve(data)
+      })
   })
+}
 
-fs.createReadStream(path.join(process.cwd(), 'src/data/routes.csv'))
-  .pipe(csv())
-  .on('data', (data) => {
-    if (data.Yosemite === '' && data.Yosemite === '') {
-      return
-    }
-    routeGrades.push({
-      score: parseInt(data.Score, 10),
-      yds: data.Yosemite,
-      french: data.French,
-      uiaa: data.UIAA,
-      ewbank: data.Ewbank,
-      saxon: data.Saxon,
-      norwegian: data.Norwegian,
-      band: 'unknown'
-    })
-  })
-  .on('end', () => {
-    const data = JSON.stringify(routeGrades)
-    fs.writeFileSync(`${writeDir}/routes.json`, data)
-  })
+const CSV_PATH_BOULDER = path.join(dataDir, 'boulder.csv')
+const JSON_PATH_BOULDER = path.join(dataDir, 'boulder.json')
+export const BOULDER_GRADE_TABLE: Promise<Boulder[]> = getData(CSV_PATH_BOULDER, JSON_PATH_BOULDER)
 
-const iceGrades: IceGrade[] = []
-fs.createReadStream(path.join(process.cwd(), 'src/data/ice.csv'))
-  .pipe(csv())
-  .on('data', (data) => {
-    if (data.AI === '' && data.WI === '') {
-      return
-    }
-    iceGrades.push({
-      score: parseInt(data.Score, 10),
-      wi: data.WI,
-      ai: data.AI
-    })
-  })
-  .on('end', () => {
-    const data = JSON.stringify(iceGrades)
-    fs.writeFileSync(`${writeDir}/ice.json`, data)
-  })
+const CSV_PATH_ROUTES = path.join(dataDir, 'routes.csv')
+const JSON_PATH_ROUTES = path.join(dataDir, 'routes.json')
+export const ROUTE_GRADE_TABLE: Promise<Route[]> = getData(CSV_PATH_ROUTES, JSON_PATH_ROUTES)
 
-const aidGrades: AidGrade[] = []
-fs.createReadStream(path.join(process.cwd(), 'src/data/aid.csv'))
-  .pipe(csv())
-  .on('data', (data) => {
-    if (data.Aid === '' && data.Aid === '') {
-      return
-    }
-    aidGrades.push({
-      score: parseInt(data.Score, 10),
-      aid: data.Aid
-    })
-  })
-  .on('end', () => {
-    const data = JSON.stringify(aidGrades)
-    fs.writeFileSync(`${writeDir}/aid.json`, data)
-  })
+const CSV_PATH_ICE = path.join(dataDir, 'ice.csv')
+const JSON_PATH_ICE = path.join(dataDir, 'ice.json')
+export const ICE_GRADE_TABLE: Promise<IceGrade[]> = getData(CSV_PATH_ICE, JSON_PATH_ICE)
+
+const CSV_PATH_AID = path.join(dataDir, 'aid.csv')
+const JSON_PATH_AID = path.join(dataDir, 'aid.json')
+export const AID_GRADE_TABLE: Promise<AidGrade[]> = getData(CSV_PATH_AID, JSON_PATH_AID)

src/data/ice.csv

@@ -1,4 +1,4 @@
-Score,WI,AI
+score,wi,ai
 0,WI1,AI1
 1,WI1,AI1
 2,WI1,AI1

src/data/routes.csv

@@ -1,4 +1,4 @@
-Score,Yosemite,French,UIAA,Ewbank,Saxon,Norwegian
+score,yds,french,uiaa,ewbank,saxon,norwegian
 0,5.0,1a,1,1,1,1-
 1,5.0,1a,1,1,1,1-
 2,5.0,1a+,1,1,1,1-
@@ -106,4 +106,4 @@ Score,Yosemite,French,UIAA,Ewbank,Saxon,Norwegian
 104,5.15d,9c,12-,39,13c,11
 105,5.15d,9c,12-,39,13c,11
 106,5.16a,9c+,12,40,13c,11+
-107,5.16a,9c+,12,40,13c,11+
+107,5.16a,9c+,12,40,13c,11+