diff --git a/.eslintrc b/.eslintrc new file mode 100644 index 0000000..aa206ec --- /dev/null +++ b/.eslintrc @@ -0,0 +1,12 @@ +{ + "extends": ["standard"], + "env": { + "es6": true, + "browser": true + }, + "rules": { + "semi": [2, "always"], + "no-extra-semi": 2, + "semi-spacing": [2, { "before": false, "after": true }] + } +} diff --git a/libs/queries.js b/libs/queries.js index e2f6442..5f9c56c 100644 --- a/libs/queries.js +++ b/libs/queries.js @@ -4,6 +4,8 @@ var _ = require('lodash'); var ejs = require('elastic.js'); var gjv = require('geojson-validation'); +var geojsonError = new Error('Invalid Geojson'); + /** * @apiDefine search * @apiParam {string} [search] Supports Lucene search syntax for all available fields @@ -16,7 +18,7 @@ var legacyParams = function (params, q) { var geojsonQueryBuilder = function (feature, query) { var shape = ejs.Shape(feature.geometry.type, feature.geometry.coordinates); - query = query.should(ejs.GeoShapeQuery() + query = query.must(ejs.GeoShapeQuery() .field('data_geometry') .shape(shape)); return query; @@ -31,17 +33,17 @@ var geojsonQueryBuilder = function (feature, query) { * with no spaces in between. Example: `contains=23,21` **/ var contains = function (params, query) { - var correct_query = new RegExp('^[0-9\.\,\-]+$'); - if (correct_query.test(params)) { + var correctQuery = new RegExp('^[0-9\.\,\-]+$'); + if (correctQuery.test(params)) { var coordinates = params.split(','); coordinates = coordinates.map(parseFloat); if (coordinates[0] < -180 || coordinates[0] > 180) { - return err.incorrectCoordinatesError(params); + throw 'Invalid coordinates'; } if (coordinates[1] < -90 || coordinates[1] > 90) { - return err.incorrectCoordinatesError(params); + throw 'Invalid coordinates'; } var shape = ejs.Shape('circle', coordinates).radius('1km'); @@ -51,7 +53,7 @@ var contains = function (params, query) { .shape(shape)); return query; } else { - err.incorrectCoordinatesError(params); + throw 'Invalid coordinates'; } }; @@ -64,11 +66,19 @@ var contains = function (params, query) { **/ var intersects = function (geojson, query) { // if we receive an object, assume it's GeoJSON, if not, try and parse + if (typeof geojson === 'string') { + try { + geojson = JSON.parse(geojson); + } catch (e) { + throw geojsonError; + } + } + if (gjv.valid(geojson)) { // If it is smaller than Nigeria use geohash // if (tools.areaNotLarge(geojson)) { if (geojson.type === 'FeatureCollection') { - for (var i=0; i < geojson.features.length; i++) { + for (var i = 0; i < geojson.features.length; i++) { var feature = geojson.features[i]; query = geojsonQueryBuilder(feature, query); } @@ -77,7 +87,7 @@ var intersects = function (geojson, query) { } return query; } else { - err.invalidGeoJsonError(); + throw geojsonError; } }; @@ -130,7 +140,7 @@ module.exports = function (params, q) { // Do legacy search if (params.search) { return legacyParams(params, q); - }; + } // contain search if (params.contains) { @@ -145,7 +155,7 @@ module.exports = function (params, q) { } // select parameters that have _from or _to - _.forEach(params, function(value, key) { + _.forEach(params, function (value, key) { var field = _.replace(key, '_from', ''); field = _.replace(field, '_to', ''); @@ -155,15 +165,14 @@ module.exports = function (params, q) { to: 'cloud_to', field: 'cloud_coverage' }; - } - else if (_.endsWith(key, '_from')) { + } else if (_.endsWith(key, '_from')) { if (_.isUndefined(rangeFields[field])) { rangeFields[field] = {}; } rangeFields[field]['from'] = key; rangeFields[field]['field'] = field; - } else if ( _.endsWith(key, '_to')) { + } else if (_.endsWith(key, '_to')) { if (_.isUndefined(rangeFields[field])) { rangeFields[field] = {}; } @@ -176,7 +185,7 @@ module.exports = function (params, q) { }); // Range search - _.forEach(rangeFields, function(value, key) { + _.forEach(rangeFields, function (value, key) { query = rangeQuery( _.get(params, _.get(value, 'from')), _.get(params, _.get(value, 'to')), @@ -195,7 +204,7 @@ module.exports = function (params, q) { } // For all items that were not matched pass the key to the term query - _.forEach(params, function(value, key) { + _.forEach(params, function (value, key) { query = matchQuery(key, value, query); }); diff --git a/libs/search.js b/libs/search.js index 59e087c..ae2416f 100644 --- a/libs/search.js +++ b/libs/search.js @@ -13,20 +13,19 @@ var client = new elasticsearch.Client({ requestTimeout: 50000 // milliseconds }); - var Search = function (event) { var params; - if (_.has(event, 'query')) { + if (_.has(event, 'query') && !_.isEmpty(event.query)) { params = event.query; - } else if (_.has(event, 'body')) { + } else if (_.has(event, 'body') && !_.isEmpty(event.body)) { params = event.body; } else { - throw('Event must either have query or body'); + params = {}; } // get page number - var page = parseInt((params.page) ? params.page: 1); + var page = parseInt((params.page) ? params.page : 1); // Build Elastic Search Query this.q = ejs.Request(); @@ -41,7 +40,6 @@ var Search = function (event) { }; Search.prototype.buildSearch = function () { - var fields; // if fields are included remove it from params @@ -95,7 +93,7 @@ Search.prototype.buildAggregation = function () { if (_.has(this.params, 'fields')) { var fields = this.params.fields.split(','); - _.forEach(fields, function(field) { + _.forEach(fields, function (field) { if (_.has(aggr, field)) { self.q.agg(aggr[field](field).field(field)); } @@ -130,10 +128,14 @@ Search.prototype.legacy = function (callback) { self.params.search = sat; } - var search_params = this.buildSearch(); + try { + var searchParams = this.buildSearch(); + } catch (e) { + return callback(e, null); + } // limit search to only landsat - client.search(search_params).then(function (body) { + client.search(searchParams).then(function (body) { var response = []; var count = 0; @@ -149,7 +151,7 @@ Search.prototype.legacy = function (callback) { skip: self.frm, limit: self.size, total: count - }, + } }, results: response }; @@ -162,9 +164,15 @@ Search.prototype.legacy = function (callback) { Search.prototype.simple = function (callback) { var self = this; - var search_params = this.buildSearch(); + var searchParams; + + try { + searchParams = this.buildSearch(); + } catch (e) { + return callback(e, null); + } - client.search(search_params).then(function (body) { + client.search(searchParams).then(function (body) { var response = []; var count = 0; @@ -193,10 +201,15 @@ Search.prototype.simple = function (callback) { Search.prototype.geojson = function (callback) { var self = this; - var search_params = this.buildSearch(); + var searchParams; - client.search(search_params).then(function (body) { + try { + searchParams = this.buildSearch(); + } catch (e) { + return callback(e, null); + } + client.search(searchParams).then(function (body) { var count = body.hits.total; var response = { @@ -230,10 +243,15 @@ Search.prototype.geojson = function (callback) { }; Search.prototype.count = function (callback) { - var search_params = this.buildAggregation(); + var searchParams; - client.search(search_params).then(function (body) { + try { + searchParams = this.buildAggregation(); + } catch (e) { + return callback(e, null); + } + client.search(searchParams).then(function (body) { var count = 0; count = body.hits.total; @@ -243,7 +261,7 @@ Search.prototype.count = function (callback) { found: count, name: process.env.NAME || 'sat-api', license: 'CC0-1.0', - website: process.env.WEBSITE || 'https://api.developmentseed.org/satellites/', + website: process.env.WEBSITE || 'https://api.developmentseed.org/satellites/' }, counts: body.aggregations }; @@ -252,8 +270,6 @@ Search.prototype.count = function (callback) { }, function (err) { return callback(err); }); - }; - module.exports = Search; diff --git a/package.json b/package.json index 27b8833..b079a0f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sat-api-lib", - "version": "0.1.0", + "version": "0.2.0", "description": "A library for creating a search API of public Satellites metadata using Elasticsearch", "main": "index.js", "scripts": { diff --git a/test/events/simple.json b/test/events/simple.json index b5e2a31..b141d26 100644 --- a/test/events/simple.json +++ b/test/events/simple.json @@ -96,5 +96,21 @@ ] } } + }, + "getIntersectsInvalid": { + "query": { + "intersects": "{" + } + }, + "getIntersectsString": { + "query": { + "intersects": "{\"type\":\"FeatureCollection\",\"features\":[{\"type\":\"Feature\",\"properties\":{},\"geometry\":{\"type\":\"Polygon\",\"coordinates\":[[[-41.41845703125,35.209721645221386],[-41.41845703125,35.746512259918504],[-40.71533203125,35.746512259918504],[-40.71533203125,35.209721645221386],[-41.41845703125,35.209721645221386]]]}}]}" + } + }, + "getIntersectsWithSatellineName": { + "query": { + "intersects": "{\"type\":\"Feature\",\"properties\":{},\"geometry\":{\"type\":\"Polygon\",\"coordinates\":[[[-76.5032958984375,36.7520891569463],[-76.5032958984375,37.081475648860525],[-75.926513671875,37.081475648860525],[-75.926513671875,36.7520891569463],[-76.5032958984375,36.7520891569463]]]}}", + "satellite_name": "sentinel" + } } } diff --git a/test/fixtures/count-getIntersects.json b/test/fixtures/count-getIntersects.json index 9bcd954..0be3905 100644 --- a/test/fixtures/count-getIntersects.json +++ b/test/fixtures/count-getIntersects.json @@ -6,7 +6,7 @@ "body": { "query": { "bool": { - "should": [ + "must": [ { "geo_shape": { "data_geometry": { diff --git a/test/fixtures/count-postIntersects.json b/test/fixtures/count-postIntersects.json index edd2659..83a5711 100644 --- a/test/fixtures/count-postIntersects.json +++ b/test/fixtures/count-postIntersects.json @@ -6,7 +6,7 @@ "body": { "query": { "bool": { - "should": [ + "must": [ { "geo_shape": { "data_geometry": { diff --git a/test/fixtures/geojson-getIntersects.json b/test/fixtures/geojson-getIntersects.json index e052744..0a74820 100644 --- a/test/fixtures/geojson-getIntersects.json +++ b/test/fixtures/geojson-getIntersects.json @@ -6,7 +6,7 @@ "body": { "query": { "bool": { - "should": [ + "must": [ { "geo_shape": { "data_geometry": { diff --git a/test/fixtures/geojson-postIntersects.json b/test/fixtures/geojson-postIntersects.json index bd9a02e..9bec752 100644 --- a/test/fixtures/geojson-postIntersects.json +++ b/test/fixtures/geojson-postIntersects.json @@ -6,7 +6,7 @@ "body": { "query": { "bool": { - "should": [ + "must": [ { "geo_shape": { "data_geometry": { diff --git a/test/fixtures/simple-getIntersects.json b/test/fixtures/simple-getIntersects.json index c542c81..6c27523 100644 --- a/test/fixtures/simple-getIntersects.json +++ b/test/fixtures/simple-getIntersects.json @@ -6,7 +6,7 @@ "body": { "query": { "bool": { - "should": [ + "must": [ { "geo_shape": { "data_geometry": { diff --git a/test/fixtures/simple-getIntersectsInvalid.json b/test/fixtures/simple-getIntersectsInvalid.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/test/fixtures/simple-getIntersectsInvalid.json @@ -0,0 +1 @@ +[] diff --git a/test/fixtures/simple-getIntersectsString.json b/test/fixtures/simple-getIntersectsString.json new file mode 100644 index 0000000..03bdf84 --- /dev/null +++ b/test/fixtures/simple-getIntersectsString.json @@ -0,0 +1,77 @@ +[ + { + "scope": "http://localhost:9200", + "method": "POST", + "path": "/sat-api/_search?size=1&from=0", + "body": { + "query": { + "bool": { + "must": [ + { + "geo_shape": { + "data_geometry": { + "shape": { + "type": "polygon", + "coordinates": [ + [ + [ + -41.41845703125, + 35.209721645221386 + ], + [ + -41.41845703125, + 35.746512259918504 + ], + [ + -40.71533203125, + 35.746512259918504 + ], + [ + -40.71533203125, + 35.209721645221386 + ], + [ + -41.41845703125, + 35.209721645221386 + ] + ] + ] + } + } + } + } + ] + } + }, + "sort": [ + { + "date": { + "order": "desc" + } + } + ] + }, + "status": 200, + "response": { + "took": 55, + "timed_out": false, + "_shards": { + "total": 5, + "successful": 5, + "failed": 0 + }, + "hits": { + "total": 0, + "max_score": null, + "hits": [] + } + }, + "headers": { + "access-control-allow-origin": "*", + "content-type": "application/json; charset=UTF-8", + "server": "Jetty(8.1.12.v20130726)", + "content-length": "123", + "connection": "keep-alive" + } + } +] diff --git a/test/fixtures/simple-getIntersectsWithSatellineName.json b/test/fixtures/simple-getIntersectsWithSatellineName.json new file mode 100644 index 0000000..208c145 --- /dev/null +++ b/test/fixtures/simple-getIntersectsWithSatellineName.json @@ -0,0 +1,224 @@ +[ + { + "scope": "http://localhost:9200", + "method": "POST", + "path": "/sat-api/_search?size=1&from=0", + "body": { + "query": { + "bool": { + "must": [ + { + "geo_shape": { + "data_geometry": { + "shape": { + "type": "polygon", + "coordinates": [ + [ + [ + -76.5032958984375, + 36.7520891569463 + ], + [ + -76.5032958984375, + 37.081475648860525 + ], + [ + -75.926513671875, + 37.081475648860525 + ], + [ + -75.926513671875, + 36.7520891569463 + ], + [ + -76.5032958984375, + 36.7520891569463 + ] + ] + ] + } + } + } + }, + { + "match": { + "satellite_name": { + "query": "sentinel", + "lenient": false, + "zero_terms_query": "none" + } + } + } + ] + } + }, + "sort": [ + { + "date": { + "order": "desc" + } + } + ] + }, + "status": 200, + "response": { + "took": 9, + "timed_out": false, + "_shards": { + "total": 5, + "successful": 5, + "failed": 0 + }, + "hits": { + "total": 53, + "max_score": null, + "hits": [ + { + "_index": "sat-api", + "_type": "sentinel2", + "_id": "S2A_tile_20160521_18SVG_0", + "_score": null, + "_source": { + "scene_id": "S2A_tile_20160521_18SVG_0", + "original_scene_id": "S2A_OPER_MSI_L1C_TL_MTI__20160521T205720_A004770_T18SVG_N02.02", + "satellite_name": "Sentinel-2A", + "cloud_coverage": 100, + "date": "2016-05-21", + "thumbnail": "http://sentinel-s2-l1c.s3.amazonaws.com/tiles/18/S/VG/2016/5/21/0/preview.jp2", + "tile_geometry": { + "crs": { + "type": "name", + "properties": { + "name": "urn:ogc:def:crs:EPSG:8.9:4326" + } + }, + "type": "Polygon", + "coordinates": [ + [ + [ + -76.13852907879546, + 37.94208091609699 + ], + [ + -74.88891399416816, + 37.94753713237227 + ], + [ + -74.89036811332919, + 36.957830487563065 + ], + [ + -76.12362950841117, + 36.95256487221862 + ], + [ + -76.13852907879546, + 37.94208091609699 + ] + ] + ] + }, + "data_coverage_percentage": 96.32, + "cloudy_pixel_percentage": 100, + "latitude_band": "S", + "product_name": "S2A_OPER_PRD_MSIL1C_PDMC_20160521T231436_R054_V20160521T155250_20160521T155250", + "tile_origin": { + "crs": { + "type": "name", + "properties": { + "name": "urn:ogc:def:crs:EPSG:8.9:4326" + } + }, + "type": "Point", + "coordinates": [ + -76.13852907879546, + 37.94208091609699 + ] + }, + "utm_zone": 18, + "product_path": "products/2016/5/21/S2A_OPER_PRD_MSIL1C_PDMC_20160521T231436_R054_V20160521T155250_20160521T155250", + "timestamp": "2016-05-21T15:52:50.531Z", + "path": "tiles/18/S/VG/2016/5/21/0", + "grid_square": "VG", + "spacecraft_name": "Sentinel-2A", + "product_stop_time": "2016-05-21T15:52:50.531Z", + "processing_level": "Level-1C", + "product_type": "S2MSI1C", + "processing_baseline": "02.02", + "sensing_orbit_number": 54, + "sensing_orbit_direction": "DESCENDING", + "product_format": "SAFE", + "product_cloud_coverage_assessment": 94.65677857142857, + "product_meta_link": "http://sentinel-s2-l1c.s3.amazonaws.com/products/2016/5/21/S2A_OPER_PRD_MSIL1C_PDMC_20160521T231436_R054_V20160521T155250_20160521T155250/metadata.xml", + "download_links": { + "aws_s3": [ + "http://sentinel-s2-l1c.s3.amazonaws.com/tiles/18/S/VG/2016/5/21/0/B01.jp2", + "http://sentinel-s2-l1c.s3.amazonaws.com/tiles/18/S/VG/2016/5/21/0/B02.jp2", + "http://sentinel-s2-l1c.s3.amazonaws.com/tiles/18/S/VG/2016/5/21/0/B03.jp2", + "http://sentinel-s2-l1c.s3.amazonaws.com/tiles/18/S/VG/2016/5/21/0/B04.jp2", + "http://sentinel-s2-l1c.s3.amazonaws.com/tiles/18/S/VG/2016/5/21/0/B05.jp2", + "http://sentinel-s2-l1c.s3.amazonaws.com/tiles/18/S/VG/2016/5/21/0/B06.jp2", + "http://sentinel-s2-l1c.s3.amazonaws.com/tiles/18/S/VG/2016/5/21/0/B07.jp2", + "http://sentinel-s2-l1c.s3.amazonaws.com/tiles/18/S/VG/2016/5/21/0/B08.jp2", + "http://sentinel-s2-l1c.s3.amazonaws.com/tiles/18/S/VG/2016/5/21/0/B09.jp2", + "http://sentinel-s2-l1c.s3.amazonaws.com/tiles/18/S/VG/2016/5/21/0/B10.jp2", + "http://sentinel-s2-l1c.s3.amazonaws.com/tiles/18/S/VG/2016/5/21/0/B11.jp2", + "http://sentinel-s2-l1c.s3.amazonaws.com/tiles/18/S/VG/2016/5/21/0/B12.jp2", + "http://sentinel-s2-l1c.s3.amazonaws.com/tiles/18/S/VG/2016/5/21/0/B8A.jp2" + ] + }, + "original_tile_meta": "http://sentinel-s2-l1c.s3.amazonaws.com/tiles/18/S/VG/2016/5/21/0/tileInfo.json", + "data_geometry": { + "crs": { + "type": "name", + "properties": { + "name": "urn:ogc:def:crs:EPSG:8.9:4326" + } + }, + "type": "Polygon", + "coordinates": [ + [ + [ + -76.13851756108318, + 37.942072015005024 + ], + [ + -74.88892538947994, + 37.94752813015372 + ], + [ + -74.88962473195204, + 37.477548468640364 + ], + [ + -75.06286437531902, + 36.957873469625454 + ], + [ + -76.12361841098011, + 36.95257399124932 + ], + [ + -76.13851756108318, + 37.942072015005024 + ] + ] + ] + } + }, + "sort": [ + 1463788800000 + ] + } + ] + } + }, + "headers": { + "access-control-allow-origin": "*", + "content-type": "application/json; charset=UTF-8", + "server": "Jetty(8.1.12.v20130726)", + "content-length": "3512", + "connection": "keep-alive" + } + } +] diff --git a/test/fixtures/simple-postIntersects.json b/test/fixtures/simple-postIntersects.json index a784222..481b977 100644 --- a/test/fixtures/simple-postIntersects.json +++ b/test/fixtures/simple-postIntersects.json @@ -6,7 +6,7 @@ "body": { "query": { "bool": { - "should": [ + "must": [ { "geo_shape": { "data_geometry": { diff --git a/test/test_simple.js b/test/test_simple.js index 9078d3d..26cdc23 100644 --- a/test/test_simple.js +++ b/test/test_simple.js @@ -2,17 +2,17 @@ process.env.ES_HOST = 'localhost:9200'; var _ = require('lodash'); +var path = require('path'); var nock = require('nock'); var test = require('tap').test; var Search = require('../index.js'); var payload = require('./events/simple.json'); - -nock.back.fixtures = __dirname + '/fixtures'; +nock.back.fixtures = path.join(__dirname + '/fixtures'); nock.back.setMode('record'); -var nockBack = function(key, func) { - nock.back('simple-' + key + '.json', function(nockDone) { +var nockBack = function (key, func) { + nock.back('simple-' + key + '.json', function (nockDone) { var search = new Search(payload[key]); search.simple(function (err, response) { nockDone(); @@ -21,11 +21,11 @@ var nockBack = function(key, func) { }); }; - test('root endpoint with simple GET/POST should return 1 result', function (t) { var keys = ['simpleGet', 'simplePost']; - keys.forEach(function(key, index) { - nockBack(key, function(err, response) { + keys.forEach(function (key, index) { + nockBack(key, function (err, response) { + t.error(err); t.equals(response.meta.limit, 1); t.equals(response.results.length, 1); t.ok(_.has(response.results[0], 'scene_id')); @@ -38,8 +38,9 @@ test('root endpoint with simple GET/POST should return 1 result', function (t) { test('root endpoint with simple GET/POST should return 1 result', function (t) { var keys = ['simplePostLimit2WithFields']; - keys.forEach(function(key) { - nockBack(key, function(err, response) { + keys.forEach(function (key) { + nockBack(key, function (err, response) { + t.error(err); t.equals(response.meta.limit, 2); t.equals(response.results.length, 2); t.notOk(_.has(response.results[0], 'scene_id')); @@ -52,7 +53,8 @@ test('root endpoint with simple GET/POST should return 1 result', function (t) { test('root endpoint with simple POST with limit 2 should return 2 result', function (t) { var key = 'simplePostLimit2'; - nockBack(key, function(err, response) { + nockBack(key, function (err, response) { + t.error(err); t.equals(response.meta.limit, 2); t.equals(response.results.length, 2); t.end(); @@ -61,7 +63,8 @@ test('root endpoint with simple POST with limit 2 should return 2 result', funct test('root endpoint with POST date range', function (t) { var key = 'postDatRange'; - nockBack(key, function(err, response) { + nockBack(key, function (err, response) { + t.error(err); t.equals(response.meta.found, 454226); t.equals(response.meta.limit, 1); t.equals(response.results.length, 1); @@ -71,7 +74,8 @@ test('root endpoint with POST date range', function (t) { test('root endpoint POST intersects', function (t) { var key = 'postIntersects'; - nockBack(key, function(err, response) { + nockBack(key, function (err, response) { + t.error(err); t.equals(response.meta.found, 237); t.equals(response.meta.limit, 1); t.equals(response.results.length, 1); @@ -81,10 +85,42 @@ test('root endpoint POST intersects', function (t) { test('root endpoint GET intersects with no match', function (t) { var key = 'getIntersects'; - nockBack(key, function(err, response) { + nockBack(key, function (err, response) { + t.error(err); t.equals(response.meta.found, 0); - t.equals(response.meta.limit, 1); - t.equals(response.results.length, 0); + t.equals(response.meta.limit, 1); + t.equals(response.results.length, 0); + t.end(); + }); +}); + +test('root endpoint GET invalid intersect', function (t) { + var key = 'getIntersectsInvalid'; + nockBack(key, function (err, response) { + t.match(err, Error); + t.match(err.message, 'Invalid Geojson'); + t.end(); + }); +}); + +test('root endpoint GET string intersects', function (t) { + var key = 'getIntersectsString'; + nockBack(key, function (err, response) { + t.error(err); + t.equals(response.meta.found, 0); + t.equals(response.meta.limit, 1); + t.equals(response.results.length, 0); + t.end(); + }); +}); + +test('root endpoint GET string intersects with satelline_name', function (t) { + var key = 'getIntersectsWithSatellineName'; + nockBack(key, function (err, response) { + t.error(err); + t.equals(response.meta.found, 53); + t.equals(response.meta.limit, 1); + t.equals(response.results.length, 1); t.end(); }); });