From 66ee3f5bafdd53cc177f20d809811c29df007bf3 Mon Sep 17 00:00:00 2001 From: Kurtis Houser Date: Fri, 23 Feb 2018 16:53:24 -0800 Subject: [PATCH 1/3] Add scaffolding to modify external data returned from api --- app/config/dataSource.js | 80 ++++++++++++++++++++++++++++++++++----- app/repositories/index.js | 6 ++- 2 files changed, 74 insertions(+), 12 deletions(-) diff --git a/app/config/dataSource.js b/app/config/dataSource.js index 9f29a32..f94be9e 100644 --- a/app/config/dataSource.js +++ b/app/config/dataSource.js @@ -11,17 +11,77 @@ exports.internal = { exports.external = { /* --- SF Open Data --- */ - communityResiliencyIndicators: 'https://data.sfgov.org/resource/wsj2-27m9.json', + communityResiliencyIndicators: { + url:'https://data.sfgov.org/resource/wsj2-27m9.json', + modifyDataset: (dataset) => { + // do something here as needed + return dataset; + }, + }, /* --- SF Open Data Markers --- */ - privatelyOwnedPublicOpenSpaces: 'https://data.sfgov.org/resource/3ub7-d4yy.geojson?$where=the_geom IS NOT NULL', - parksAndOpenSpaces: 'https://data.sfgov.org/resource/94uf-amnx.geojson?$where=location_1 IS NOT NULL', - schools: 'https://data.sfgov.org/resource/mmsr-vumy.geojson?$where=location_1 IS NOT NULL', + privatelyOwnedPublicOpenSpaces: { + url: 'https://data.sfgov.org/resource/3ub7-d4yy.geojson?$where=the_geom IS NOT NULL', + modifyDataset: (dataset) => { + // do something here as needed + return dataset; + }, + }, + parksAndOpenSpaces: { + url: 'https://data.sfgov.org/resource/94uf-amnx.geojson?$where=location_1 IS NOT NULL', + modifyDataset: (dataset) => { + // do something here as needed + return dataset; + }, + }, + schools: { + url: 'https://data.sfgov.org/resource/mmsr-vumy.geojson?$where=location_1 IS NOT NULL', + modifyDataset: (dataset) => { + // do something here as needed + return dataset; + }, +}, // very large, nation-wide data set, need to limit what's returned - businesses: 'https://data.sfgov.org/resource/vbiu-2p9h.geojson?$where=location IS NOT NULL', - cityFacilities: 'https://data.sfgov.org/resource/i5wr-crji.geojson?$where=geom IS NOT NULL', - healthCareFacilities: 'https://data.sfgov.org/resource/sci7-7q9i.geojson?$where=location IS NOT NULL', - pitStops: 'https://data.sfgov.org/resource/snkr-6jdf.geojson?$where=geom IS NOT NULL', + businesses: { + url: 'https://data.sfgov.org/resource/vbiu-2p9h.geojson?$where=location IS NOT NULL', + modifyDataset: (dataset) => { + // do something here as needed + return dataset; + }, + }, + cityFacilities: { + url: 'https://data.sfgov.org/resource/i5wr-crji.geojson?$where=geom IS NOT NULL', + modifyDataset: (dataset) => { + // do something here as needed + return dataset; + }, + }, + healthCareFacilities: { + url: 'https://data.sfgov.org/resource/sci7-7q9i.geojson?$where=location IS NOT NULL', + modifyDataset: (dataset) => { + // do something here as needed + return dataset; + }, + }, + pitStops: { + url:'https://data.sfgov.org/resource/snkr-6jdf.geojson?$where=geom IS NOT NULL', + modifyDataset: (dataset) => { + // do something here as needed + return dataset; + }, + }, /* --- SF Open Data Regions --- */ - seizmicHazardZones: 'https://data.sfgov.org/resource/t2cc-dy6b.geojson?$where=the_geom IS NOT NULL', - neighborhoods: 'https://data.sfgov.org/resource/6ia5-2f8k.geojson?$where=the_geom IS NOT NULL', + seizmicHazardZones: { + url: 'https://data.sfgov.org/resource/t2cc-dy6b.geojson?$where=the_geom IS NOT NULL', + modifyDataset: (dataset) => { + // do something here as needed + return dataset; + }, + }, + neighborhoods:{ + url: 'https://data.sfgov.org/resource/6ia5-2f8k.geojson?$where=the_geom IS NOT NULL', + modifyDataset: (dataset) => { + // do something here as needed + return dataset; + }, + }, }; diff --git a/app/repositories/index.js b/app/repositories/index.js index b27331b..c73219e 100644 --- a/app/repositories/index.js +++ b/app/repositories/index.js @@ -25,10 +25,12 @@ async function getById(mapDataset) { async function fetchById(mapDataset) { try { - const response = await fetch(external[mapDataset]); + const response = await fetch(external[mapDataset].url); const dataset = await response.json(); - return dataset; + const modifiedDataset = external[mapDataset].modifyDataset(dataset); + + return modifiedDataset; } catch (error) { throw new Error(error); } From e3c0eaf8a02a17c5c6d78c3909db0e8a19fdfbc2 Mon Sep 17 00:00:00 2001 From: Jeff Appareti Date: Wed, 7 Mar 2018 17:18:16 -0800 Subject: [PATCH 2/3] Updated SOQL query strings for DataSF sources --- app/config/dataSource.js | 90 +++++++++++++++++--------------------- app/repositories/index.js | 5 ++- app/utilities/addAuthor.js | 8 ++++ 3 files changed, 51 insertions(+), 52 deletions(-) create mode 100644 app/utilities/addAuthor.js diff --git a/app/config/dataSource.js b/app/config/dataSource.js index f94be9e..35229bd 100644 --- a/app/config/dataSource.js +++ b/app/config/dataSource.js @@ -12,76 +12,66 @@ exports.internal = { exports.external = { /* --- SF Open Data --- */ communityResiliencyIndicators: { - url:'https://data.sfgov.org/resource/wsj2-27m9.json', - modifyDataset: (dataset) => { + url: "https://data.sfgov.org/resource/wsj2-27m9.json", + modifyDataset: dataset => { // do something here as needed return dataset; - }, + } }, /* --- SF Open Data Markers --- */ privatelyOwnedPublicOpenSpaces: { - url: 'https://data.sfgov.org/resource/3ub7-d4yy.geojson?$where=the_geom IS NOT NULL', - modifyDataset: (dataset) => { - // do something here as needed - return dataset; - }, + url: "https://data.sfgov.org/resource/3ub7-d4yy.geojson", + query: + "$query=SELECT name AS title, descriptio AS description, the_geom WHERE the_geom IS NOT NULL", + author: "DataSF" }, parksAndOpenSpaces: { - url: 'https://data.sfgov.org/resource/94uf-amnx.geojson?$where=location_1 IS NOT NULL', - modifyDataset: (dataset) => { - // do something here as needed - return dataset; - }, + url: "https://data.sfgov.org/resource/94uf-amnx.geojson", + query: + "$query=SELECT parkid AS _id, parktype AS description, parkname AS name, location_1 WHERE location_1 IS NOT NULL", + author: "DataSF" }, schools: { - url: 'https://data.sfgov.org/resource/mmsr-vumy.geojson?$where=location_1 IS NOT NULL', - modifyDataset: (dataset) => { - // do something here as needed - return dataset; + url: "https://data.sfgov.org/resource/mmsr-vumy.geojson", + query: + "$query=SELECT location_1, campus_name AS name, campus_address AS description, county_fips AS _id WHERE location_1 IS NOT NULL", + author: "DataSF" }, -}, // very large, nation-wide data set, need to limit what's returned businesses: { - url: 'https://data.sfgov.org/resource/vbiu-2p9h.geojson?$where=location IS NOT NULL', - modifyDataset: (dataset) => { - // do something here as needed - return dataset; - }, + url: "https://data.sfgov.org/resource/vbiu-2p9h.geojson", + query: + "$query=SELECT * WHERE location IS NOT NULL AND city = 'San Francisco'", + author: "DataSF" }, cityFacilities: { - url: 'https://data.sfgov.org/resource/i5wr-crji.geojson?$where=geom IS NOT NULL', - modifyDataset: (dataset) => { - // do something here as needed - return dataset; - }, + url: "https://data.sfgov.org/resource/i5wr-crji.geojson", + query: + "$query=SELECT facility_id AS _id, common_name AS name, address AS description, geom WHERE geom IS NOT NULL", + author: "DataSF" }, healthCareFacilities: { - url: 'https://data.sfgov.org/resource/sci7-7q9i.geojson?$where=location IS NOT NULL', - modifyDataset: (dataset) => { - // do something here as needed - return dataset; - }, + url: "https://data.sfgov.org/resource/sci7-7q9i.geojson", + query: + "$query=SELECT uid AS _id, facility_name AS name, facility_type AS description WHERE location IS NOT NULL", + author: "DataSF" }, pitStops: { - url:'https://data.sfgov.org/resource/snkr-6jdf.geojson?$where=geom IS NOT NULL', - modifyDataset: (dataset) => { - // do something here as needed - return dataset; - }, + url: "https://data.sfgov.org/resource/snkr-6jdf.geojson", + query: + "$query=SELECT id AS _id, location AS name, facilitytype AS description, geom WHERE geom IS NOT NULL", + author: "DataSF" }, /* --- SF Open Data Regions --- */ seizmicHazardZones: { - url: 'https://data.sfgov.org/resource/t2cc-dy6b.geojson?$where=the_geom IS NOT NULL', - modifyDataset: (dataset) => { - // do something here as needed - return dataset; - }, - }, - neighborhoods:{ - url: 'https://data.sfgov.org/resource/6ia5-2f8k.geojson?$where=the_geom IS NOT NULL', - modifyDataset: (dataset) => { - // do something here as needed - return dataset; - }, + url: "https://data.sfgov.org/resource/t2cc-dy6b.geojson", + query: "$query=SELECT id AS _id, the_geom WHERE the_geom IS NOT NULL", + author: "DataSF" }, + neighborhoods: { + url: "https://data.sfgov.org/resource/6ia5-2f8k.geojson", + query: + "$query=SELECT the_geom, name, link AS description WHERE the_geom IS NOT NULL", + author: "DataSF" + } }; diff --git a/app/repositories/index.js b/app/repositories/index.js index c73219e..00c0f2a 100644 --- a/app/repositories/index.js +++ b/app/repositories/index.js @@ -3,6 +3,7 @@ const geojson = require('geojson'); const Marker = require('../models/marker.js'); const mapDatasets = require('../config/mapDatasets'); const { internal, external } = require('../config/dataSource'); +const addAuthor = require("../utilities/addAuthor"); async function getAll() { const datasets = await mapDatasets; // simulated db call @@ -25,10 +26,10 @@ async function getById(mapDataset) { async function fetchById(mapDataset) { try { - const response = await fetch(external[mapDataset].url); + const response = await fetch(`${external[mapDataset].url}?${external[mapDataset].query}`); const dataset = await response.json(); - const modifiedDataset = external[mapDataset].modifyDataset(dataset); + const modifiedDataset = addAuthor(dataset, external[mapDataset].author); return modifiedDataset; } catch (error) { diff --git a/app/utilities/addAuthor.js b/app/utilities/addAuthor.js new file mode 100644 index 0000000..1dcc24a --- /dev/null +++ b/app/utilities/addAuthor.js @@ -0,0 +1,8 @@ +function addAuthor(dataset, author) { + dataset.features.map(feature => { + feature.properties.author = author; + }); + return dataset; +} + +module.exports = addAuthor; From bbd22b3d99360d1324168bc9613486d3ca8d272f Mon Sep 17 00:00:00 2001 From: Jeff Appareti Date: Wed, 7 Mar 2018 19:45:59 -0800 Subject: [PATCH 3/3] Fixed health care facilities query --- app/config/dataSource.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config/dataSource.js b/app/config/dataSource.js index 57193ab..2cedceb 100644 --- a/app/config/dataSource.js +++ b/app/config/dataSource.js @@ -53,7 +53,7 @@ exports.external = { healthCareFacilities: { url: "https://data.sfgov.org/resource/sci7-7q9i.geojson", query: - "$query=SELECT uid AS _id, facility_name AS name, facility_type AS description WHERE location IS NOT NULL", + "$query=SELECT uid AS _id, facility_name AS name, facility_type, location AS description WHERE location IS NOT NULL", author: "DataSF" }, pitStops: {