Zoobot/feature/data sources (#126)

* move broken url field to end for FE checkbox in data

* fix merge conflicts

* sources

* zoobot/feature/data-sources

* change id_name to id_source_name

* Revert "fix merge conflicts"

This reverts commit 6cfd267.

* sources changes

* sources massaging

* sources fixes

* sources

* testing changes

* fixed post after nested crosswalks

* remove console and jest

* Add all cities not explicitely specified for sources download (#128)

* Add all cities not explicitely specified for sources download

* lengthen time for geojson for temporarily

* Picking up trees from a year ago tried to load all of SF as geojson because we modified ids in 2022 june (#134)

* Add GET tests for sources=all and sources array

* fixed security issues

* code review cleanup

* removed find cities and countries as we are not currently using

* change queries to prepared statements

server/routes/csv/csv-router.js

@@ -18,7 +18,7 @@ csvRouter.get('/', async (req, res) => {
 
   const jsonData = JSON.parse(JSON.stringify(data));
   const cityName = city
-    ? city.toLowerCase().replaceAll(' ', '_')
+    ? city.toLowerCase().replaceAll(' ', '-')
     : 'all-cities';
   const csvPath = `server/csv-downloads/${cityName}.csv`;
   const ws = fs.createWriteStream(csvPath);

server/routes/shared-routes-utils.js

@@ -1,8 +1,34 @@
+// function camelToSnakeCase(camelIn) {
+//   return camelIn.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
+// }
+
+function snakeToCamelCase(snakeIn) {
+  return snakeIn.replace(/([-_][a-z])/g, (group) =>
+    group.toUpperCase().replace('-', '').replace('_', ''),
+  );
+}
+
+export const convertObjectKeysToCamelCase = (obj) => {
+  const newObj = {};
+  for (const key in obj) {
+    newObj[snakeToCamelCase(key)] = obj[key];
+  }
+  return newObj;
+};
+
 function camelToSnakeCase(camelIn) {
-  return camelIn.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
+  return camelIn.replace(/[A-Z0-9]/g, (letter) => {
+    if (/[A-Z]/.test(letter)) {
+      return `_${letter.toLowerCase()}`;
+    } else if (/[0-9]/.test(letter)) {
+      return `_${letter}`;
+    } else {
+      return letter;
+    }
+  });
 }
 
-export default function convertObjectKeysToSnakeCase(obj) {
+export const convertObjectKeysToSnakeCase = (obj) => {
   const newObj = {};
 
   // eslint-disable-next-line no-restricted-syntax
@@ -14,4 +40,4 @@ export default function convertObjectKeysToSnakeCase(obj) {
     }
   }
   return newObj;
-}
+};

server/routes/sources/sources-queries.js

@@ -1,64 +1,168 @@
 import { db, pgPromise } from '../../db/index.js';
-import convertObjectKeysToSnakeCase from '../shared-routes-utils.js';
+
+import {
+  convertObjectKeysToSnakeCase,
+  convertObjectKeysToCamelCase,
+} from '../shared-routes-utils.js';
+
+const SOURCE_FIELDS = `id_source_name as "idSourceName", 
+    iso_alpha_3 as "isoAlpha3", 
+    country,
+    state, 
+    city, 
+    email, 
+    contact, 
+    phone, 
+    info, 
+    download, 
+    notes, 
+    filename,
+    format,
+    longitude,
+    latitude,
+    license,
+    broken`;
+
+const CROSSWALK_FIELDS = `id_source_name as "idSourceName", 
+  common,
+  species, 
+  genus, 
+  scientific, 
+  family, 
+  variety, 
+  class, 
+  dbh, 
+  height, 
+  structure, 
+  trunks, 
+  age, 
+  health, 
+  crown, 
+  spread, 
+  planted, 
+  updated, 
+  location, 
+  note, 
+  address, 
+  id_reference as "idReference", 
+  owner, 
+  ule, 
+  ule_min as "uleMin", 
+  ule_max as "uleMax", 
+  cost,
+  audited, 
+  longitude, 
+  latitude, 
+  city, 
+  state, 
+  zip, 
+  country, 
+  neighborhood, 
+  url, 
+  urlimage, 
+  status, 
+  email, 
+  volunteer, 
+  notes, 
+  legal_status as legalStatus, 
+  irrigation, 
+  count, 
+  dbh_min as "dbhMin", 
+  dbh_max as "dbhMax", 
+  height_min as "heightMin", 
+  height_max as "heightMax",
+  crown_min as "crownMin", 
+  crown_max as "crownMax"`;
 
 export async function createSource(data) {
   // eslint-disable-next-line no-unused-vars
   const { crosswalk, destinations, ...source } = data;
 
+  const dataInSnakeCase = convertObjectKeysToSnakeCase(source);
+
   const query = `
-    INSERT INTO sources(\${this:name})
-    VALUES(\${this:csv})
-    RETURNING country, city, id, created
-  `;
+      INSERT INTO sources(\${this:name})
+      VALUES(\${this:csv})
+      RETURNING id_source_name as "idSourceName"
+    `;
 
-  const response = await db.one(query, source);
+  const response = await db.one(query, dataInSnakeCase);
   return response;
 }
 
 export async function createCrosswalk(data) {
+  const dataInSnakeCase = convertObjectKeysToSnakeCase(data);
   const query = `
-    INSERT INTO crosswalk(\${this:name})
-    VALUES(\${this:csv})
-    RETURNING id
-  `;
+      INSERT INTO crosswalk(\${this:name})
+      VALUES(\${this:csv})
+      RETURNING id_source_name as "idSourceName"
+    `;
 
-  const response = await db.one(query, data);
+  const response = await db.one(query, dataInSnakeCase);
   return response;
 }
 
-export async function findSourceCountry(country) {
-  const query = `SELECT 
-    id, iso_alpha_3  as country, state, city, 
-    email, contact, who, phone, 
-    info, download, broken, broken_reason as note
-    FROM sources
-    WHERE country = $1;`;
-  const values = [country];
-  const source = await db.any(query, values);
+export async function getAllSources() {
+  const query = {
+    name: 'find-sources',
+    text: `SELECT ${SOURCE_FIELDS} FROM sources;`,
+  };
+  const source = await db.any(query);
   return source;
 }
 
-export async function getAllSources() {
-  const query = `SELECT id, iso_alpha_3  as country, state, city, 
-    email, contact, who, phone, 
-    info, download, broken, broken_reason as note 
-    FROM sources;`;
-  const source = await db.any(query);
+export async function getSourceByIdSourceName(idSourceName) {
+  const query = {
+    name: 'find-source',
+    text: `SELECT ${SOURCE_FIELDS} 
+      FROM sources 
+      WHERE id_source_name =  $1`,
+    values: idSourceName,
+  };
+
+  const source = await db.one(query);
+  return source;
+}
+
+export async function getCrosswalkByIdSourceName(idSourceName) {
+  const query = {
+    name: 'find-crosswalk',
+    text: `SELECT ${CROSSWALK_FIELDS}
+    FROM crosswalk where id_source_name = $1`,
+    values: idSourceName,
+  };
+  const source = await db.one(query);
   return source;
 }
 
-export async function updateSourceById(updatedSourceData, id) {
-  const updatedSourceDataInSnakeCase =
-    convertObjectKeysToSnakeCase(updatedSourceData);
+export async function updateSourceByIdSourceName(data) {
+  const dataInSnakeCase = convertObjectKeysToSnakeCase(data);
+  const keys = Object.keys(dataInSnakeCase);
+  const keysString = keys.join(', ');
+
+  const condition = pgPromise.as.format(
+    ` WHERE id_source_name = '${data.idSourceName}' 
+      RETURNING ${keysString};`,
+  );
+  const query =
+    pgPromise.helpers.update(dataInSnakeCase, keys, 'sources') + condition;
+  const updatedResponse = await db.one(query, dataInSnakeCase);
+  const camelCaseResponse = convertObjectKeysToCamelCase(await updatedResponse);
+  return camelCaseResponse;
+}
+
+export async function updateCrosswalkByIdSourceName(data) {
+  const { id_source_name, ...dataInSnakeCase } =
+    convertObjectKeysToSnakeCase(data);
+  const keys = Object.keys(dataInSnakeCase);
+  const keysString = keys.join(', ');
 
-  const condition = pgPromise.as.format(`WHERE id = ${id} RETURNING *`);
+  const condition = pgPromise.as.format(
+    ` WHERE id_source_name = '${id_source_name}' RETURNING ${keysString};`,
+  );
   const query =
-    pgPromise.helpers.update(
-      updatedSourceDataInSnakeCase,
-      Object.keys(updatedSourceDataInSnakeCase),
-      'sources',
-    ) + condition;
-  const updatedSource = await db.one(query, updatedSourceDataInSnakeCase);
-
-  return updatedSource;
+    pgPromise.helpers.update(dataInSnakeCase, keys, 'crosswalk') + condition;
+  const updatedResponse = await db.one(query, dataInSnakeCase);
+  const camelCaseSource = convertObjectKeysToCamelCase(await updatedResponse);
+  return { idSourceName: data.idSourceName, ...camelCaseSource };
 }

server/routes/sources/sources-router.js

@@ -1,54 +1,76 @@
 import express from 'express';
 import AppError from '../../errors/AppError.js';
 import {
-  findSourceCountry,
-  updateSourceById,
+  getSourceByIdSourceName,
+  getCrosswalkByIdSourceName,
+  updateSourceByIdSourceName,
   createSource,
   createCrosswalk,
   getAllSources,
+  updateCrosswalkByIdSourceName,
 } from './sources-queries.js';
+import validateSource from './sources-validations.js';
 
 const sourcesRouter = express.Router();
 
 sourcesRouter.get('/', async (req, res) => {
-  const { id, country } = req.query;
+  const { idSourceName, sources } = req.query;
 
-  const idSource = !id ? '*' : id;
-  const source = !country
-    ? await getAllSources()
-    : await findSourceCountry(country, idSource);
-  res.status(200).json(source ?? {});
+  if (!idSourceName && sources === 'All') {
+    let foundSources = await getAllSources();
+    if (!foundSources || foundSources.length === 0)
+      throw new AppError(400, 'Error getting source');
+    res.status(200).json(foundSources);
+  }
+
+  if (idSourceName) {
+    const responseSource = await getSourceByIdSourceName(idSourceName);
+    const responseCrosswalk = await getCrosswalkByIdSourceName(idSourceName);
+    if (!responseSource) throw new AppError(400, 'Error getting source');
+    res
+      .status(200)
+      .json({ source: responseSource, crosswalk: responseCrosswalk });
+  }
 });
 
 sourcesRouter.post('/', async (req, res) => {
   // eslint-disable-next-line no-unused-vars
-  const { crosswalk, ...source } = req.body;
-  const responseSource = await createSource(source);
-  if (!responseSource) throw new AppError(400, 'Error creating source');
+  const validated = await validateSource(req);
+  if (!validated) throw new AppError(400, 'Error validating source');
+
+  const { crosswalk = null, source = null } = req.body;
+  let responseSource, responseCrosswalk;
+  if (source) {
+    responseSource = await createSource(source);
+    if (!responseSource) throw new AppError(400, 'Error creating source');
+  }
+
   if (crosswalk) {
-    const responseCrosswalk = await createCrosswalk({
-      id: source.id,
-      ...crosswalk,
-    });
+    responseCrosswalk = await createCrosswalk(crosswalk);
     if (!responseCrosswalk) throw new AppError(400, 'Error creating Crosswalk');
   }
-
-  res.status(201).json(responseSource);
+  const response = { source: responseSource, crosswalk: responseCrosswalk };
+  res.status(200).json(response);
 });
 
 sourcesRouter.put('/', async (req, res) => {
-  const { id, ...body } = req.body;
+  // eslint-disable-next-line no-unused-vars
+  const validated = await validateSource(req);
+  if (!validated) throw new AppError(400, 'Error validating source');
 
-  if (!id) {
-    throw new AppError(
-      400,
-      'sourcesRouter.put Missing required parameter: id.',
-    );
+  const { crosswalk = null, source = null } = req.body;
+  let responseSource, responseCrosswalk;
+  if (source) {
+    responseSource = await updateSourceByIdSourceName(source);
+    if (!responseSource) throw new AppError(400, 'Error creating source');
   }
 
-  const updatedSource = await updateSourceById(body, id);
-
-  res.status(200).json(updatedSource);
+  if (crosswalk) {
+    responseCrosswalk = await updateCrosswalkByIdSourceName(crosswalk);
+    if (!responseCrosswalk) throw new AppError(400, 'Error creating Crosswalk');
+  }
+  const response = { source: responseSource, crosswalk: responseCrosswalk };
+  res.status(200).json(response);
 });
 
 export default sourcesRouter;

server/routes/sources/sources-validations.js

@@ -0,0 +1,12 @@
+export default function validateSource(req) {
+  if (!req?.body) return false;
+  const { crosswalk = null, source = null } = req.body;
+  if (source) {
+    if (source?.idSourceName === undefined) return false;
+  }
+  if (crosswalk) {
+    if (crosswalk?.idSourceName === undefined) return false;
+  }
+
+  return true;
+}