From c98ccdf1d45dc75de1de018814f11f408ae1ea8e Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Fri, 22 Sep 2017 14:06:44 -0400 Subject: [PATCH 01/21] fix(project-settings): fix type for default location lat/lng --- lib/manager/components/GeneralSettings.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/manager/components/GeneralSettings.js b/lib/manager/components/GeneralSettings.js index 02a1ab5ab..4968b496d 100644 --- a/lib/manager/components/GeneralSettings.js +++ b/lib/manager/components/GeneralSettings.js @@ -117,9 +117,8 @@ export default class GeneralSettings extends Component { bounds: bounds, onConfirm: (marker) => { if (marker) { - const defaultLocationLat = marker.lat.toFixed(6) - const defaultLocationLon = marker.lng.toFixed(6) - // ReactDOM.findDOMNode(this.refs.defaultLocation).value = `${defaultLocationLat},${defaultLocationLon}` + const defaultLocationLat = +marker.lat.toFixed(6) + const defaultLocationLon = +marker.lng.toFixed(6) this.setState(update(this.state, { general: { $merge: {defaultLocationLat, defaultLocationLon} } })) } } @@ -137,6 +136,7 @@ export default class GeneralSettings extends Component { } render () { + console.log(this.state) const messages = getComponentMessages('ProjectSettings') const {project, editDisabled} = this.props const noEdits = Object.keys(this.state.general).length === 0 && this.state.general.constructor === Object From 90accf4b8d8781b5997737b3c9ce81fceb51848f Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Mon, 16 Oct 2017 12:29:09 -0400 Subject: [PATCH 02/21] test(seedData): update seed data script to perform concurrent upload and query requests --- scripts/package.json | 3 +- scripts/seedData.js | 199 ++++++++-- scripts/yarn.lock | 913 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 1091 insertions(+), 24 deletions(-) diff --git a/scripts/package.json b/scripts/package.json index 5b8e8a7ac..ca132824a 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -1,6 +1,7 @@ { "dependencies": { "auth0": "^2.7.0", - "make-runnable": "^1.3.6" + "make-runnable": "^1.3.6", + "request-promise-native": "^1.0.5" } } diff --git a/scripts/seedData.js b/scripts/seedData.js index e63679ec2..7399431a8 100755 --- a/scripts/seedData.js +++ b/scripts/seedData.js @@ -1,36 +1,67 @@ -// This script seeds datatools-server with test project and +// This script seeds datatools-server with test project and multiple feed versions. +// The datatools-server application should be run with NO_AUTH set to true for the test. // NOTE: make sure you have a recent version of node (e.g., v8) for async/await -// USAGE: app_url='http://localhost:4000' num_feedsources=2 feed_path='/tmp/gtfs.zip' node seedData.js run +// USAGE: app_url='http://localhost:4000' concurrency=10 num_feedsources=2 feed_path='/tmp/gtfs.zip' node seedData.js run var fetch = require('isomorphic-fetch') var fs = require('fs') -var request = require('request') +var request = require('request-promise-native') const API_ENDPOINT = `${process.env.app_url}/api/manager/secure` +const GRAPHQL_ENDPOINT = `${process.env.app_url}/api/manager/graphql` +const CONCURRENCY = process.env.concurrency || 4 const FEED_PATH = process.env.feed_path const method = 'post' const headers = {'Content-Type': 'application/json'} const NUM_FEEDSOURCES = +process.env.num_feedsources || 5 +const versionIdForJobId = {} +const taskMap = {} +const GRAPHQL_STOPS_QUERY = ` + query stops($namespace: String) { + feed(namespace: $namespace) { + namespace + feed_id + feed_version + filename + row_counts { + stops + } + stops { + stop_id + stop_name + stop_lat + stop_lon + } + } + } + ` async function createProject (data) { - return await fetch(`${API_ENDPOINT}/project`, { + return fetch(`${API_ENDPOINT}/project`, { method, headers, body: JSON.stringify(data) }) .then(res => res.json()) - .then(p => p) + .catch(err => console.log(err)) +} + +async function getFeedVersion (feedVersionId) { + return fetch(`${API_ENDPOINT}/feedversion/${feedVersionId}`, { + method: 'get', + headers + }) + .then(res => res.json()) .catch(err => console.log(err)) } async function createFeedSource (data) { - return await fetch(`${API_ENDPOINT}/feedsource`, { + return fetch(`${API_ENDPOINT}/feedsource`, { method, headers, body: JSON.stringify(data) }) .then(res => res.json()) - .then(fs => fs) .catch(err => console.log(err)) } @@ -38,33 +69,155 @@ async function uploadFeedVersion (feedSource, filePath) { const file = fs.createReadStream(filePath) const formData = {file} const url = `${API_ENDPOINT}/feedversion?feedSourceId=${feedSource.id}` - await request.post({url, formData}, (err, resp, body) => { - if (err) { - console.log('Upload error!', err) - // Exit script if upload fails - process.exit(1) - } else { - console.log(`Upload success: ` + body) - } + return request.post({url, formData}) + .then(jobId => { // make sure to close read stream file.destroy() + return jobId }) + .catch(err => { + console.log(err) + + // // Exit script if upload fails + // process.exit(1) + }) +} + +async function jobIsActive (jobId) { + const job = await fetch(`${API_ENDPOINT}/status/jobs/${jobId}`) + .then(res => res.json()) + .catch(err => console.log(err)) + if (job !== null) { + // console.log(`waiting to process feedversion: ${job.feedVersion.id}`) + if (!versionIdForJobId.hasOwnProperty(jobId)) { + versionIdForJobId[jobId] = job.feedVersionId + } + } + return job !== null && !job.status.completed +} + +async function waitForJobToFinish (jobId) { + while (await jobIsActive(jobId)) { + sleep(1000) + } + console.log(`Job ${jobId} just finished`) +} + +async function doFeedVersionThing (taskName, projectId) { + const name = `test-${taskName}` + // Create a new feed source for each version because otherwise the server will detect duplicate uploads + const feedSource = await createFeedSource({name, projectId}) + console.log(`Created feedSource: ${feedSource.name} (${feedSource.id})`) + console.log(`Uploading file from ${FEED_PATH}`) + const jobId = await uploadFeedVersion(feedSource, FEED_PATH) + const testStatus = { + startTime: new Date(), + finishedProcessingTime: null, + finishedGraphQLRequest: null, + passed: false + } + taskMap[jobId] = testStatus + // listOfVersions.push(feedVersion.id) + console.log(`Processing feed version job: ${jobId}`) + // wait for job to finish processing before making graphql requests + await waitForJobToFinish(jobId) + testStatus.finishedProcessingTime = new Date() + // make graphql requests against newly created feed version + const feedVersionId = versionIdForJobId[jobId] + const graphqlResponse = await makeGraphQLRequests(feedVersionId) + testStatus.finishedGraphQLRequest = new Date() + validateGraphQlResponse(testStatus, graphqlResponse) + return testStatus.passed +} + +function validateGraphQlResponse (testStatus, graphqlResponse) { + testStatus.passed = graphqlResponse.feed.row_counts.stops === graphqlResponse.feed.stops.length +} + +async function makeGraphQLRequests (feedVersionId) { + // get postgres unique id + const feedVersion = await getFeedVersion(feedVersionId) + const {namespace} = feedVersion + console.log(`fetching graphql for ${namespace}`) + + // make graphql request + return fetch(GRAPHQL_ENDPOINT, + { + method, + body: JSON.stringify({ + query: GRAPHQL_STOPS_QUERY, + variables: JSON.stringify({namespace}) + }) + }) + .then(res => res.json()) + // .then(json => { + // // console.log(`graphql response: ${JSON.stringify(json)}`) + // + // }) + .catch(err => console.log(err)) +} + +function sleep (msec) { + return new Promise(resolve => setTimeout(resolve, msec)) } +async function task (threadId, taskId, projectId) { + const taskName = threadId + '-' + taskId + console.log('starting ' + taskName) + await doFeedVersionThing(taskName, projectId) + // await sleep(1000) + console.log('finished ' + taskName) +} + +async function seriesWork (threadId, projectId) { + for (var i = 0; i < NUM_FEEDSOURCES; i++) { + await task(threadId, 'task_' + i, projectId) + } +} + +async function parallelWork (projectId) { + var seriesTasks = [] + for (var i = 0; i < CONCURRENCY; i++) { + seriesTasks.push(seriesWork('thread_' + i, projectId)) + } + await Promise.all(seriesTasks) + const executionTimes = [] + const testStatusList = Object.values(taskMap) + for (const testStatus of testStatusList) { + try { + if (testStatus.passed) { + const executionTime = testStatus.finishedProcessingTime - testStatus.startTime + executionTimes.push(executionTime) + } + } catch (e) { + // do not record execution time because the test failed to complete + } + } + let nTotalTests = CONCURRENCY * NUM_FEEDSOURCES + let nFailedTests = nTotalTests - executionTimes.length + console.log(`Number of failed tests: ${nFailedTests} out of ${nTotalTests} total`) + executionTimes.sort() + const min = executionTimes[0] + const max = executionTimes[executionTimes.length - 1] + const median = executionTimes[Math.floor(executionTimes.length / 2)] + console.log(`Feed processing time (milliseconds): MIN - ${min} MAX - ${max} MED - ${median}`) +} + +// Total volume of data to load, plus how much of it you want to run parallel +// Rename functions, parameters +// Make typesafe (use types.js) exports.run = async function () { console.log(`Using application URL: ${process.env.app_url}`) + + // create initial project const proj = {name: 'tester 1'} const p = await createProject(proj) const {id: projectId} = p console.log(`Created project: ${projectId}`) - console.log(`Creating ${NUM_FEEDSOURCES} feed sources`) - for (var i = 0; i < NUM_FEEDSOURCES; i++) { - const name = `test-${i}` - const feedSource = await createFeedSource({name, projectId}) - console.log(`Created feedSource: ${feedSource.name} (${feedSource.id})`) - console.log(`Uploading file from ${FEED_PATH}`) - await uploadFeedVersion(feedSource, FEED_PATH) - } + console.log(`Creating ${NUM_FEEDSOURCES * CONCURRENCY} feed sources`) + + // Make concurrent parallel requests to create feed versions + parallelWork(projectId) } require('make-runnable') diff --git a/scripts/yarn.lock b/scripts/yarn.lock index 521ac6ca1..f4dcf7077 100644 --- a/scripts/yarn.lock +++ b/scripts/yarn.lock @@ -2,22 +2,136 @@ # yarn lockfile v1 +agent-base@2, agent-base@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-2.1.1.tgz#d6de10d5af6132d5bd692427d46fc538539094c7" + dependencies: + extend "~3.0.0" + semver "~5.0.1" + +agent-base@^4.1.0: + version "4.1.1" + resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-4.1.1.tgz#92d8a4fc2524a3b09b3666a33b6c97960f23d6a4" + dependencies: + es6-promisify "^5.0.0" + +ajv@^5.1.0: + version "5.2.3" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-5.2.3.tgz#c06f598778c44c6b161abafe3466b81ad1814ed2" + dependencies: + co "^4.6.0" + fast-deep-equal "^1.0.0" + json-schema-traverse "^0.3.0" + json-stable-stringify "^1.0.1" + ansi-regex@^2.0.0: version "2.1.1" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df" +asn1@~0.2.3: + version "0.2.3" + resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.3.tgz#dac8787713c9966849fc8180777ebe9c1ddf3b86" + +assert-plus@1.0.0, assert-plus@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525" + +ast-types@0.x.x: + version "0.9.14" + resolved "https://registry.yarnpkg.com/ast-types/-/ast-types-0.9.14.tgz#d34ba5dffb9d15a44351fd2a9d82e4ab2838b5ba" + +asynckit@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" + +auth0@^2.7.0: + version "2.8.0" + resolved "https://registry.yarnpkg.com/auth0/-/auth0-2.8.0.tgz#cd3c4aa78a433c1c9ca2f4c29cfc8f9baf09af13" + dependencies: + bluebird "^2.10.2" + lru-memoizer "^1.11.1" + object.assign "^4.0.4" + request "^2.83.0" + rest-facade "^1.5.0" + +aws-sign2@~0.7.0: + version "0.7.0" + resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" + +aws4@^1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e" + +bcrypt-pbkdf@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz#63bc5dcb61331b92bc05fd528953c33462a06f8d" + dependencies: + tweetnacl "^0.14.3" + +bluebird@^2.10.2: + version "2.11.0" + resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-2.11.0.tgz#534b9033c022c9579c56ba3b3e5a5caafbb650e1" + bluebird@^3.5.0: version "3.5.0" resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.0.tgz#791420d7f551eea2897453a8a77653f96606d67c" +boom@4.x.x: + version "4.3.1" + resolved "https://registry.yarnpkg.com/boom/-/boom-4.3.1.tgz#4f8a3005cb4a7e3889f749030fd25b96e01d2e31" + dependencies: + hoek "4.x.x" + +boom@5.x.x: + version "5.2.0" + resolved "https://registry.yarnpkg.com/boom/-/boom-5.2.0.tgz#5dd9da6ee3a5f302077436290cb717d3f4a54e02" + dependencies: + hoek "4.x.x" + builtin-modules@^1.0.0: version "1.1.1" resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-1.1.1.tgz#270f076c5a72c02f5b65a47df94c5fe3a278892f" +bytes@3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.0.0.tgz#d32815404d689699f85a4ea4fa8755dd13a96048" + +camel-case@^1.1.1: + version "1.2.2" + resolved "https://registry.yarnpkg.com/camel-case/-/camel-case-1.2.2.tgz#1aca7c4d195359a2ce9955793433c6e5542511f2" + dependencies: + sentence-case "^1.1.1" + upper-case "^1.1.1" + camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" +caseless@~0.12.0: + version "0.12.0" + resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" + +change-case@^2.3.0: + version "2.3.1" + resolved "https://registry.yarnpkg.com/change-case/-/change-case-2.3.1.tgz#2c4fde3f063bb41d00cd68e0d5a09db61cbe894f" + dependencies: + camel-case "^1.1.1" + constant-case "^1.1.0" + dot-case "^1.1.0" + is-lower-case "^1.1.0" + is-upper-case "^1.1.0" + lower-case "^1.1.1" + lower-case-first "^1.0.0" + param-case "^1.1.0" + pascal-case "^1.1.0" + path-case "^1.1.0" + sentence-case "^1.1.1" + snake-case "^1.1.0" + swap-case "^1.1.0" + title-case "^1.1.0" + upper-case "^1.1.1" + upper-case-first "^1.1.0" + cliui@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/cliui/-/cliui-3.2.0.tgz#120601537a916d29940f934da3b48d585a39213d" @@ -26,20 +140,173 @@ cliui@^3.2.0: strip-ansi "^3.0.1" wrap-ansi "^2.0.0" +co@^4.6.0: + version "4.6.0" + resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184" + code-point-at@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77" +combined-stream@^1.0.5, combined-stream@~1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.5.tgz#938370a57b4a51dea2c77c15d5c5fdf895164009" + dependencies: + delayed-stream "~1.0.0" + +component-emitter@^1.2.0: + version "1.2.1" + resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.2.1.tgz#137918d6d78283f7df7a6b7c5a63e140e69425e6" + +constant-case@^1.1.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/constant-case/-/constant-case-1.1.2.tgz#8ec2ca5ba343e00aa38dbf4e200fd5ac907efd63" + dependencies: + snake-case "^1.1.0" + upper-case "^1.1.1" + +cookiejar@^2.1.0: + version "2.1.1" + resolved "https://registry.yarnpkg.com/cookiejar/-/cookiejar-2.1.1.tgz#41ad57b1b555951ec171412a81942b1e8200d34a" + +core-util-is@1.0.2, core-util-is@~1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" + +cryptiles@3.x.x: + version "3.1.2" + resolved "https://registry.yarnpkg.com/cryptiles/-/cryptiles-3.1.2.tgz#a89fbb220f5ce25ec56e8c4aa8a4fd7b5b0d29fe" + dependencies: + boom "5.x.x" + +dashdash@^1.12.0: + version "1.14.1" + resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" + dependencies: + assert-plus "^1.0.0" + +data-uri-to-buffer@1: + version "1.2.0" + resolved "https://registry.yarnpkg.com/data-uri-to-buffer/-/data-uri-to-buffer-1.2.0.tgz#77163ea9c20d8641b4707e8f18abdf9a78f34835" + +debug@2, debug@^2.6.8: + version "2.6.9" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" + dependencies: + ms "2.0.0" + +debug@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" + dependencies: + ms "2.0.0" + decamelize@^1.1.1: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" +deep-is@~0.1.3: + version "0.1.3" + resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34" + +deepmerge@^1.5.1: + version "1.5.2" + resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-1.5.2.tgz#10499d868844cdad4fee0842df8c7f6f0c95a753" + +define-properties@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.2.tgz#83a73f2fea569898fb737193c8f873caf6d45c94" + dependencies: + foreach "^2.0.5" + object-keys "^1.0.8" + +degenerator@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/degenerator/-/degenerator-1.0.4.tgz#fcf490a37ece266464d9cc431ab98c5819ced095" + dependencies: + ast-types "0.x.x" + escodegen "1.x.x" + esprima "3.x.x" + +delayed-stream@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" + +depd@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.1.tgz#5783b4e1c459f06fa5ca27f991f3d06e7a310359" + +dot-case@^1.1.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/dot-case/-/dot-case-1.1.2.tgz#1e73826900de28d6de5480bc1de31d0842b06bec" + dependencies: + sentence-case "^1.1.2" + +ecc-jsbn@~0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz#0fc73a9ed5f0d53c38193398523ef7e543777505" + dependencies: + jsbn "~0.1.0" + error-ex@^1.2.0: version "1.3.1" resolved "https://registry.yarnpkg.com/error-ex/-/error-ex-1.3.1.tgz#f855a86ce61adc4e8621c3cda21e7a7612c3a8dc" dependencies: is-arrayish "^0.2.1" +es6-promise@^4.0.3: + version "4.1.1" + resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-4.1.1.tgz#8811e90915d9a0dba36274f0b242dbda78f9c92a" + +es6-promisify@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/es6-promisify/-/es6-promisify-5.0.0.tgz#5109d62f3e56ea967c4b63505aef08291c8a5203" + dependencies: + es6-promise "^4.0.3" + +escodegen@1.x.x: + version "1.9.0" + resolved "https://registry.yarnpkg.com/escodegen/-/escodegen-1.9.0.tgz#9811a2f265dc1cd3894420ee3717064b632b8852" + dependencies: + esprima "^3.1.3" + estraverse "^4.2.0" + esutils "^2.0.2" + optionator "^0.8.1" + optionalDependencies: + source-map "~0.5.6" + +esprima@3.x.x, esprima@^3.1.3: + version "3.1.3" + resolved "https://registry.yarnpkg.com/esprima/-/esprima-3.1.3.tgz#fdca51cee6133895e3c88d535ce49dbff62a4633" + +estraverse@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-4.2.0.tgz#0dee3fed31fcd469618ce7342099fc1afa0bdb13" + +esutils@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.2.tgz#0abf4f1caa5bcb1f7a9d8acc6dea4faaa04bac9b" + +extend@3, extend@^3.0.0, extend@~3.0.0, extend@~3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.1.tgz#a755ea7bc1adfcc5a31ce7e762dbaadc5e636444" + +extsprintf@1.3.0, extsprintf@^1.2.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" + +fast-deep-equal@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-1.0.0.tgz#96256a3bc975595eb36d82e9929d060d893439ff" + +fast-levenshtein@~2.0.4: + version "2.0.6" + resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" + +file-uri-to-path@1: + version "1.0.0" + resolved "https://registry.yarnpkg.com/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd" + find-up@^1.0.0: version "1.1.2" resolved "https://registry.yarnpkg.com/find-up/-/find-up-1.1.2.tgz#6b2e9822b1a2ce0a60ab64d610eccad53cb24d0f" @@ -47,22 +314,139 @@ find-up@^1.0.0: path-exists "^2.0.0" pinkie-promise "^2.0.0" +foreach@^2.0.5: + version "2.0.5" + resolved "https://registry.yarnpkg.com/foreach/-/foreach-2.0.5.tgz#0bee005018aeb260d0a3af3ae658dd0136ec1b99" + +forever-agent@~0.6.1: + version "0.6.1" + resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" + +form-data@^2.3.1, form-data@~2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.1.tgz#6fb94fbd71885306d73d15cc497fe4cc4ecd44bf" + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.5" + mime-types "^2.1.12" + +formidable@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/formidable/-/formidable-1.1.1.tgz#96b8886f7c3c3508b932d6bd70c4d3a88f35f1a9" + +ftp@~0.3.10: + version "0.3.10" + resolved "https://registry.yarnpkg.com/ftp/-/ftp-0.3.10.tgz#9197d861ad8142f3e63d5a83bfe4c59f7330885d" + dependencies: + readable-stream "1.1.x" + xregexp "2.0.0" + +function-bind@^1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" + get-caller-file@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.2.tgz#f702e63127e7e231c160a80c1554acb70d5047e5" +get-uri@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/get-uri/-/get-uri-2.0.1.tgz#dbdcacacd8c608a38316869368117697a1631c59" + dependencies: + data-uri-to-buffer "1" + debug "2" + extend "3" + file-uri-to-path "1" + ftp "~0.3.10" + readable-stream "2" + +getpass@^0.1.1: + version "0.1.7" + resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" + dependencies: + assert-plus "^1.0.0" + graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" +har-schema@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" + +har-validator@~5.0.3: + version "5.0.3" + resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.0.3.tgz#ba402c266194f15956ef15e0fcf242993f6a7dfd" + dependencies: + ajv "^5.1.0" + har-schema "^2.0.0" + +hawk@~6.0.2: + version "6.0.2" + resolved "https://registry.yarnpkg.com/hawk/-/hawk-6.0.2.tgz#af4d914eb065f9b5ce4d9d11c1cb2126eecc3038" + dependencies: + boom "4.x.x" + cryptiles "3.x.x" + hoek "4.x.x" + sntp "2.x.x" + +hoek@4.x.x: + version "4.2.0" + resolved "https://registry.yarnpkg.com/hoek/-/hoek-4.2.0.tgz#72d9d0754f7fe25ca2d01ad8f8f9a9449a89526d" + hosted-git-info@^2.1.4: version "2.4.2" resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.4.2.tgz#0076b9f46a270506ddbaaea56496897460612a67" +http-errors@1.6.2: + version "1.6.2" + resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.6.2.tgz#0a002cc85707192a7e7946ceedc11155f60ec736" + dependencies: + depd "1.1.1" + inherits "2.0.3" + setprototypeof "1.0.3" + statuses ">= 1.3.1 < 2" + +http-proxy-agent@1, http-proxy-agent@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-1.0.0.tgz#cc1ce38e453bf984a0f7702d2dd59c73d081284a" + dependencies: + agent-base "2" + debug "2" + extend "3" + +http-signature@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" + dependencies: + assert-plus "^1.0.0" + jsprim "^1.2.2" + sshpk "^1.7.0" + +https-proxy-agent@1, https-proxy-agent@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-1.0.0.tgz#35f7da6c48ce4ddbfa264891ac593ee5ff8671e6" + dependencies: + agent-base "2" + debug "2" + extend "3" + +iconv-lite@0.4.19: + version "0.4.19" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.19.tgz#f7468f60135f5e5dad3399c0a81be9a1603a082b" + +inherits@2.0.3, inherits@~2.0.1, inherits@~2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" + invert-kv@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-1.0.0.tgz#104a8e4aaca6d3d8cd157a8ef8bfab2d7a3ffdb6" +ip@^1.1.4, ip@^1.1.5: + version "1.1.5" + resolved "https://registry.yarnpkg.com/ip/-/ip-1.1.5.tgz#bdded70114290828c0a039e72ef25f5aaec4354a" + is-arrayish@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" @@ -79,16 +463,86 @@ is-fullwidth-code-point@^1.0.0: dependencies: number-is-nan "^1.0.0" +is-lower-case@^1.1.0: + version "1.1.3" + resolved "https://registry.yarnpkg.com/is-lower-case/-/is-lower-case-1.1.3.tgz#7e147be4768dc466db3bfb21cc60b31e6ad69393" + dependencies: + lower-case "^1.1.0" + +is-typedarray@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" + +is-upper-case@^1.1.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/is-upper-case/-/is-upper-case-1.1.2.tgz#8d0b1fa7e7933a1e58483600ec7d9661cbaf756f" + dependencies: + upper-case "^1.1.0" + is-utf8@^0.2.0: version "0.2.1" resolved "https://registry.yarnpkg.com/is-utf8/-/is-utf8-0.2.1.tgz#4b0da1442104d1b336340e80797e865cf39f7d72" +isarray@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf" + +isarray@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" + +isstream@~0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" + +jsbn@~0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" + +json-schema-traverse@^0.3.0: + version "0.3.1" + resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz#349a6d44c53a51de89b40805c5d5e59b417d3340" + +json-schema@0.2.3: + version "0.2.3" + resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13" + +json-stable-stringify@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz#9a759d39c5f2ff503fd5300646ed445f88c4f9af" + dependencies: + jsonify "~0.0.0" + +json-stringify-safe@~5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" + +jsonify@~0.0.0: + version "0.0.0" + resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.0.tgz#2c74b6ee41d93ca51b7b5aaee8f503631d252a73" + +jsprim@^1.2.2: + version "1.4.1" + resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2" + dependencies: + assert-plus "1.0.0" + extsprintf "1.3.0" + json-schema "0.2.3" + verror "1.10.0" + lcid@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/lcid/-/lcid-1.0.0.tgz#308accafa0bc483a3867b4b6f2b9506251d1b835" dependencies: invert-kv "^1.0.0" +levn@~0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/levn/-/levn-0.3.0.tgz#3b09924edf9f083c0490fdd4c0bc4421e04764ee" + dependencies: + prelude-ls "~1.1.2" + type-check "~0.3.2" + load-json-file@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-1.1.0.tgz#956905708d58b4bab4c2261b04f59f31c99374c0" @@ -99,10 +553,52 @@ load-json-file@^1.0.0: pinkie-promise "^2.0.0" strip-bom "^2.0.0" +lock@~0.1.2: + version "0.1.4" + resolved "https://registry.yarnpkg.com/lock/-/lock-0.1.4.tgz#fec7deaef17e7c3a0a55e1da042803e25d91745d" + lodash.assign@^4.0.3, lodash.assign@^4.0.6: version "4.2.0" resolved "https://registry.yarnpkg.com/lodash.assign/-/lodash.assign-4.2.0.tgz#0d99f3ccd7a6d261d19bdaeb9245005d285808e7" +lodash@^4.13.1: + version "4.17.4" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae" + +lodash@~4.5.1: + version "4.5.1" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.5.1.tgz#80e8a074ca5f3893a6b1c10b2a636492d710c316" + +lower-case-first@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/lower-case-first/-/lower-case-first-1.0.2.tgz#e5da7c26f29a7073be02d52bac9980e5922adfa1" + dependencies: + lower-case "^1.1.2" + +lower-case@^1.1.0, lower-case@^1.1.1, lower-case@^1.1.2: + version "1.1.4" + resolved "https://registry.yarnpkg.com/lower-case/-/lower-case-1.1.4.tgz#9a2cabd1b9e8e0ae993a4bf7d5875c39c42e8eac" + +lru-cache@~2.6.5: + version "2.6.5" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-2.6.5.tgz#e56d6354148ede8d7707b58d143220fd08df0fd5" + +lru-cache@~4.0.0: + version "4.0.2" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-4.0.2.tgz#1d17679c069cda5d040991a09dbc2c0db377e55e" + dependencies: + pseudomap "^1.0.1" + yallist "^2.0.0" + +lru-memoizer@^1.11.1: + version "1.11.1" + resolved "https://registry.yarnpkg.com/lru-memoizer/-/lru-memoizer-1.11.1.tgz#0693f6100593914c02e192bf9b8d93884cbf50d3" + dependencies: + lock "~0.1.2" + lodash "~4.5.1" + lru-cache "~4.0.0" + very-fast-args "^1.1.0" + make-runnable@^1.3.6: version "1.3.6" resolved "https://registry.yarnpkg.com/make-runnable/-/make-runnable-1.3.6.tgz#ca9b1d31b06f051e37570fb7ad98bc5369f982be" @@ -110,6 +606,32 @@ make-runnable@^1.3.6: bluebird "^3.5.0" yargs "^4.7.1" +methods@^1.1.1: + version "1.1.2" + resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" + +mime-db@~1.30.0: + version "1.30.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.30.0.tgz#74c643da2dd9d6a45399963465b26d5ca7d71f01" + +mime-types@^2.1.12, mime-types@~2.1.17: + version "2.1.17" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.17.tgz#09d7a393f03e995a79f8af857b70a9e0ab16557a" + dependencies: + mime-db "~1.30.0" + +mime@^1.4.1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/mime/-/mime-1.4.1.tgz#121f9ebc49e3766f311a76e1fa1c8003c4b03aa6" + +ms@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" + +netmask@^1.0.6: + version "1.0.6" + resolved "https://registry.yarnpkg.com/netmask/-/netmask-1.0.6.tgz#20297e89d86f6f6400f250d9f4f6b4c1945fcd35" + normalize-package-data@^2.3.2: version "2.3.8" resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.3.8.tgz#d819eda2a9dedbd1ffa563ea4071d936782295bb" @@ -123,18 +645,87 @@ number-is-nan@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d" +oauth-sign@~0.8.2: + version "0.8.2" + resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" + +object-keys@^1.0.10, object-keys@^1.0.8: + version "1.0.11" + resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.0.11.tgz#c54601778ad560f1142ce0e01bcca8b56d13426d" + +object.assign@^4.0.4: + version "4.0.4" + resolved "https://registry.yarnpkg.com/object.assign/-/object.assign-4.0.4.tgz#b1c9cc044ef1b9fe63606fc141abbb32e14730cc" + dependencies: + define-properties "^1.1.2" + function-bind "^1.1.0" + object-keys "^1.0.10" + +optionator@^0.8.1: + version "0.8.2" + resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.8.2.tgz#364c5e409d3f4d6301d6c0b4c05bba50180aeb64" + dependencies: + deep-is "~0.1.3" + fast-levenshtein "~2.0.4" + levn "~0.3.0" + prelude-ls "~1.1.2" + type-check "~0.3.2" + wordwrap "~1.0.0" + os-locale@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/os-locale/-/os-locale-1.4.0.tgz#20f9f17ae29ed345e8bde583b13d2009803c14d9" dependencies: lcid "^1.0.0" +pac-proxy-agent@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/pac-proxy-agent/-/pac-proxy-agent-2.0.0.tgz#beb17cd2b06a20b379d57e1b2e2c29be0dfe5f9a" + dependencies: + agent-base "^2.1.1" + debug "^2.6.8" + get-uri "^2.0.0" + http-proxy-agent "^1.0.0" + https-proxy-agent "^1.0.0" + pac-resolver "^3.0.0" + raw-body "^2.2.0" + socks-proxy-agent "^3.0.0" + +pac-resolver@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/pac-resolver/-/pac-resolver-3.0.0.tgz#6aea30787db0a891704deb7800a722a7615a6f26" + dependencies: + co "^4.6.0" + degenerator "^1.0.4" + ip "^1.1.5" + netmask "^1.0.6" + thunkify "^2.1.2" + +param-case@^1.1.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/param-case/-/param-case-1.1.2.tgz#dcb091a43c259b9228f1c341e7b6a44ea0bf9743" + dependencies: + sentence-case "^1.1.2" + parse-json@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-2.2.0.tgz#f480f40434ef80741f8469099f8dea18f55a4dc9" dependencies: error-ex "^1.2.0" +pascal-case@^1.1.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/pascal-case/-/pascal-case-1.1.2.tgz#3e5d64a20043830a7c49344c2d74b41be0c9c99b" + dependencies: + camel-case "^1.1.1" + upper-case-first "^1.1.0" + +path-case@^1.1.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/path-case/-/path-case-1.1.2.tgz#50ce6ba0d3bed3dd0b5c2a9c4553697434409514" + dependencies: + sentence-case "^1.1.2" + path-exists@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-2.1.0.tgz#0feb6c64f0fc518d9a754dd5efb62c7022761f4b" @@ -149,6 +740,10 @@ path-type@^1.0.0: pify "^2.0.0" pinkie-promise "^2.0.0" +performance-now@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" + pify@^2.0.0: version "2.3.0" resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c" @@ -163,6 +758,48 @@ pinkie@^2.0.0: version "2.0.4" resolved "https://registry.yarnpkg.com/pinkie/-/pinkie-2.0.4.tgz#72556b80cfa0d48a974e80e77248e80ed4f7f870" +prelude-ls@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54" + +process-nextick-args@~1.0.6: + version "1.0.7" + resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-1.0.7.tgz#150e20b756590ad3f91093f25a4f2ad8bff30ba3" + +proxy-agent@2: + version "2.1.0" + resolved "https://registry.yarnpkg.com/proxy-agent/-/proxy-agent-2.1.0.tgz#a3a2b3866debfeb79bb791f345dc9bc876e7ff86" + dependencies: + agent-base "2" + debug "2" + extend "3" + http-proxy-agent "1" + https-proxy-agent "1" + lru-cache "~2.6.5" + pac-proxy-agent "^2.0.0" + socks-proxy-agent "2" + +pseudomap@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" + +punycode@^1.4.1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" + +qs@^6.5.1, qs@~6.5.1: + version "6.5.1" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.1.tgz#349cdf6eef89ec45c12d7d5eb3fc0c870343a6d8" + +raw-body@^2.2.0: + version "2.3.2" + resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.3.2.tgz#bcd60c77d3eb93cde0050295c3f379389bc88f89" + dependencies: + bytes "3.0.0" + http-errors "1.6.2" + iconv-lite "0.4.19" + unpipe "1.0.0" + read-pkg-up@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-1.0.1.tgz#9d63c13276c065918d57f002a57f40a1b643fb02" @@ -178,6 +815,68 @@ read-pkg@^1.0.0: normalize-package-data "^2.3.2" path-type "^1.0.0" +readable-stream@1.1.x: + version "1.1.14" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9" + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.1" + isarray "0.0.1" + string_decoder "~0.10.x" + +readable-stream@2, readable-stream@^2.0.5: + version "2.3.3" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.3.tgz#368f2512d79f9d46fdfc71349ae7878bbc1eb95c" + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.3" + isarray "~1.0.0" + process-nextick-args "~1.0.6" + safe-buffer "~5.1.1" + string_decoder "~1.0.3" + util-deprecate "~1.0.1" + +request-promise-core@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/request-promise-core/-/request-promise-core-1.1.1.tgz#3eee00b2c5aa83239cfb04c5700da36f81cd08b6" + dependencies: + lodash "^4.13.1" + +request-promise-native@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/request-promise-native/-/request-promise-native-1.0.5.tgz#5281770f68e0c9719e5163fd3fab482215f4fda5" + dependencies: + request-promise-core "1.1.1" + stealthy-require "^1.1.0" + tough-cookie ">=2.3.3" + +request@^2.83.0: + version "2.83.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.83.0.tgz#ca0b65da02ed62935887808e6f510381034e3356" + dependencies: + aws-sign2 "~0.7.0" + aws4 "^1.6.0" + caseless "~0.12.0" + combined-stream "~1.0.5" + extend "~3.0.1" + forever-agent "~0.6.1" + form-data "~2.3.1" + har-validator "~5.0.3" + hawk "~6.0.2" + http-signature "~1.2.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.17" + oauth-sign "~0.8.2" + performance-now "^2.1.0" + qs "~6.5.1" + safe-buffer "^5.1.1" + stringstream "~0.0.5" + tough-cookie "~2.3.3" + tunnel-agent "^0.6.0" + uuid "^3.1.0" + require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" @@ -186,14 +885,84 @@ require-main-filename@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-1.0.1.tgz#97f717b69d48784f5f526a6c5aa8ffdda055a4d1" +rest-facade@^1.5.0: + version "1.10.0" + resolved "https://registry.yarnpkg.com/rest-facade/-/rest-facade-1.10.0.tgz#527847f2900257813ccc99e5dec3c225c6fb9f34" + dependencies: + bluebird "^2.10.2" + change-case "^2.3.0" + deepmerge "^1.5.1" + superagent "^3.5.2" + superagent-proxy "^1.0.2" + +safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1: + version "5.1.1" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" + "semver@2 || 3 || 4 || 5": version "5.3.0" resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" +semver@~5.0.1: + version "5.0.3" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a" + +sentence-case@^1.1.1, sentence-case@^1.1.2: + version "1.1.3" + resolved "https://registry.yarnpkg.com/sentence-case/-/sentence-case-1.1.3.tgz#8034aafc2145772d3abe1509aa42c9e1042dc139" + dependencies: + lower-case "^1.1.1" + set-blocking@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" +setprototypeof@1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.0.3.tgz#66567e37043eeb4f04d91bd658c0cbefb55b8e04" + +smart-buffer@^1.0.13: + version "1.1.15" + resolved "https://registry.yarnpkg.com/smart-buffer/-/smart-buffer-1.1.15.tgz#7f114b5b65fab3e2a35aa775bb12f0d1c649bf16" + +snake-case@^1.1.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/snake-case/-/snake-case-1.1.2.tgz#0c2f25e305158d9a18d3d977066187fef8a5a66a" + dependencies: + sentence-case "^1.1.2" + +sntp@2.x.x: + version "2.0.2" + resolved "https://registry.yarnpkg.com/sntp/-/sntp-2.0.2.tgz#5064110f0af85f7cfdb7d6b67a40028ce52b4b2b" + dependencies: + hoek "4.x.x" + +socks-proxy-agent@2: + version "2.1.1" + resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-2.1.1.tgz#86ebb07193258637870e13b7bd99f26c663df3d3" + dependencies: + agent-base "2" + extend "3" + socks "~1.1.5" + +socks-proxy-agent@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-3.0.1.tgz#2eae7cf8e2a82d34565761539a7f9718c5617659" + dependencies: + agent-base "^4.1.0" + socks "^1.1.10" + +socks@^1.1.10, socks@~1.1.5: + version "1.1.10" + resolved "https://registry.yarnpkg.com/socks/-/socks-1.1.10.tgz#5b8b7fc7c8f341c53ed056e929b7bf4de8ba7b5a" + dependencies: + ip "^1.1.4" + smart-buffer "^1.0.13" + +source-map@~0.5.6: + version "0.5.7" + resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc" + spdx-correct@~1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/spdx-correct/-/spdx-correct-1.0.2.tgz#4b3073d933ff51f3912f03ac5519498a4150db40" @@ -208,6 +977,28 @@ spdx-license-ids@^1.0.2: version "1.2.2" resolved "https://registry.yarnpkg.com/spdx-license-ids/-/spdx-license-ids-1.2.2.tgz#c9df7a3424594ade6bd11900d596696dc06bac57" +sshpk@^1.7.0: + version "1.13.1" + resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.13.1.tgz#512df6da6287144316dc4c18fe1cf1d940739be3" + dependencies: + asn1 "~0.2.3" + assert-plus "^1.0.0" + dashdash "^1.12.0" + getpass "^0.1.1" + optionalDependencies: + bcrypt-pbkdf "^1.0.0" + ecc-jsbn "~0.1.1" + jsbn "~0.1.0" + tweetnacl "~0.14.0" + +"statuses@>= 1.3.1 < 2": + version "1.3.1" + resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.3.1.tgz#faf51b9eb74aaef3b3acf4ad5f61abf24cb7b93e" + +stealthy-require@^1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/stealthy-require/-/stealthy-require-1.1.1.tgz#35b09875b4ff49f26a777e509b3090a3226bf24b" + string-width@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/string-width/-/string-width-1.0.2.tgz#118bdf5b8cdc51a2a7e70d211e07e2b0b9b107d3" @@ -216,6 +1007,20 @@ string-width@^1.0.1: is-fullwidth-code-point "^1.0.0" strip-ansi "^3.0.0" +string_decoder@~0.10.x: + version "0.10.31" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" + +string_decoder@~1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.3.tgz#0fc67d7c141825de94282dd536bec6b9bce860ab" + dependencies: + safe-buffer "~5.1.0" + +stringstream@~0.0.5: + version "0.0.5" + resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.5.tgz#4e484cd4de5a0bbbee18e46307710a8a81621878" + strip-ansi@^3.0.0, strip-ansi@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf" @@ -228,6 +1033,90 @@ strip-bom@^2.0.0: dependencies: is-utf8 "^0.2.0" +superagent-proxy@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/superagent-proxy/-/superagent-proxy-1.0.2.tgz#92d3660578f618ed43a82cf8cac799fe2938ba2d" + dependencies: + debug "2" + proxy-agent "2" + +superagent@^3.5.2: + version "3.6.3" + resolved "https://registry.yarnpkg.com/superagent/-/superagent-3.6.3.tgz#eb95fcb576a9d23a730a9d0789731b5379a36cdc" + dependencies: + component-emitter "^1.2.0" + cookiejar "^2.1.0" + debug "^3.1.0" + extend "^3.0.0" + form-data "^2.3.1" + formidable "^1.1.1" + methods "^1.1.1" + mime "^1.4.1" + qs "^6.5.1" + readable-stream "^2.0.5" + +swap-case@^1.1.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/swap-case/-/swap-case-1.1.2.tgz#c39203a4587385fad3c850a0bd1bcafa081974e3" + dependencies: + lower-case "^1.1.1" + upper-case "^1.1.1" + +thunkify@^2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/thunkify/-/thunkify-2.1.2.tgz#faa0e9d230c51acc95ca13a361ac05ca7e04553d" + +title-case@^1.1.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/title-case/-/title-case-1.1.2.tgz#fae4a6ae546bfa22d083a0eea910a40d12ed4f5a" + dependencies: + sentence-case "^1.1.1" + upper-case "^1.0.3" + +tough-cookie@>=2.3.3, tough-cookie@~2.3.3: + version "2.3.3" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.3.tgz#0b618a5565b6dea90bf3425d04d55edc475a7561" + dependencies: + punycode "^1.4.1" + +tunnel-agent@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" + dependencies: + safe-buffer "^5.0.1" + +tweetnacl@^0.14.3, tweetnacl@~0.14.0: + version "0.14.5" + resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" + +type-check@~0.3.2: + version "0.3.2" + resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.3.2.tgz#5884cab512cf1d355e3fb784f30804b2b520db72" + dependencies: + prelude-ls "~1.1.2" + +unpipe@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" + +upper-case-first@^1.1.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/upper-case-first/-/upper-case-first-1.1.2.tgz#5d79bedcff14419518fd2edb0a0507c9b6859115" + dependencies: + upper-case "^1.1.1" + +upper-case@^1.0.3, upper-case@^1.1.0, upper-case@^1.1.1: + version "1.1.3" + resolved "https://registry.yarnpkg.com/upper-case/-/upper-case-1.1.3.tgz#f6b4501c2ec4cdd26ba78be7222961de77621598" + +util-deprecate@~1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" + +uuid@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04" + validate-npm-package-license@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/validate-npm-package-license/-/validate-npm-package-license-3.0.1.tgz#2804babe712ad3379459acfbe24746ab2c303fbc" @@ -235,6 +1124,18 @@ validate-npm-package-license@^3.0.1: spdx-correct "~1.0.0" spdx-expression-parse "~1.0.0" +verror@1.10.0: + version "1.10.0" + resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" + dependencies: + assert-plus "^1.0.0" + core-util-is "1.0.2" + extsprintf "^1.2.0" + +very-fast-args@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/very-fast-args/-/very-fast-args-1.1.0.tgz#e16d1d1faf8a6e596a246421fd90a77963d0b396" + which-module@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/which-module/-/which-module-1.0.0.tgz#bba63ca861948994ff307736089e3b96026c2a4f" @@ -243,6 +1144,10 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" +wordwrap@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb" + wrap-ansi@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-2.1.0.tgz#d8fc3d284dd05794fe84973caecdd1cf824fdd85" @@ -250,10 +1155,18 @@ wrap-ansi@^2.0.0: string-width "^1.0.1" strip-ansi "^3.0.1" +xregexp@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/xregexp/-/xregexp-2.0.0.tgz#52a63e56ca0b84a7f3a5f3d61872f126ad7a5943" + y18n@^3.2.1: version "3.2.1" resolved "https://registry.yarnpkg.com/y18n/-/y18n-3.2.1.tgz#6d15fba884c08679c0d77e88e7759e811e07fa41" +yallist@^2.0.0: + version "2.1.2" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52" + yargs-parser@^2.4.1: version "2.4.1" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-2.4.1.tgz#85568de3cf150ff49fa51825f03a8c880ddcc5c4" From b9c2605008a05c0ed404d63b7cf5e1e2ca5a5305 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Thu, 19 Oct 2017 10:39:12 -0400 Subject: [PATCH 03/21] display status message in job monitor for retired jobs --- lib/common/components/JobMonitor.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/common/components/JobMonitor.js b/lib/common/components/JobMonitor.js index 422e78522..231b76a7c 100644 --- a/lib/common/components/JobMonitor.js +++ b/lib/common/components/JobMonitor.js @@ -92,9 +92,7 @@ class RetiredJob extends Pure { style: PropTypes.object } - removeJob = () => { - this.props.removeRetiredJob(this.props.job) - } + removeJob = () => this.props.removeRetiredJob(this.props.job) render () { const {job, style, statusStyle} = this.props @@ -127,7 +125,7 @@ class RetiredJob extends Pure {
- {job.status && job.status.error ? 'Error!' : 'Completed!'} + {job.status.message}
From b28c7cac77759e139cf541cb0d977c299606315d Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Thu, 19 Oct 2017 10:39:44 -0400 Subject: [PATCH 04/21] remove unused code --- lib/common/util/util.js | 32 +------------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/lib/common/util/util.js b/lib/common/util/util.js index e9313a5be..0f0afa787 100644 --- a/lib/common/util/util.js +++ b/lib/common/util/util.js @@ -47,37 +47,7 @@ export function generateRandomColor (): string { } return color } -// export function invertHex (hexnum) { -// if (hexnum.length != 6) { -// alert('Hex color must be six hex numbers in length.') -// return false -// } -// -// hexnum = hexnum.toUpperCase() -// var splitnum = hexnum.split('') -// var resultnum = '' -// var simplenum = 'FEDCBA9876'.split('') -// var complexnum = new Array() -// complexnum.A = '5' -// complexnum.B = '4' -// complexnum.C = '3' -// complexnum.D = '2' -// complexnum.E = '1' -// complexnum.F = '0' -// -// for(var i=0; i<6; i++){ -// if (!isNaN(splitnum[i])) { -// resultnum += simplenum[splitnum[i]] -// } else if (complexnum[splitnum[i]]){ -// resultnum += complexnum[splitnum[i]] -// } else { -// alert('Hex colors must only include hex numbers 0-9, and A-F') -// return false -// } -// } -// -// return resultnum -// } + export function idealTextColor (bgColor: string): string { var nThreshold = 105 var components = getRGBComponents(bgColor) From 1c50b4f18bd932cff8d0479b96d776b8c79c53a4 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Wed, 1 Nov 2017 14:31:37 -0400 Subject: [PATCH 05/21] fix(feed-summary): fix tripsPerDate chart to use new validationResult --- .../components/validation/TripsChart.js | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/manager/components/validation/TripsChart.js b/lib/manager/components/validation/TripsChart.js index 62de34769..8f1f9edbb 100644 --- a/lib/manager/components/validation/TripsChart.js +++ b/lib/manager/components/validation/TripsChart.js @@ -18,8 +18,11 @@ export default class TripsChart extends Component { if (!this.props.validationResult) { return } - const tripsPerDate = this.props.validationResult.tripsPerDate - const data = Object.keys(tripsPerDate).map(key => [key, tripsPerDate[key]]) + const {dailyTripCounts, firstCalendarDate, lastCalendarDate} = this.props.validationResult + const firstDate = moment(firstCalendarDate) + const lastDate = moment(lastCalendarDate) + const data = dailyTripCounts.map((count, index) => + [firstDate.clone().add(index, 'days'), count]) const graphHeight = 300 const spacing = 8 const leftMargin = 50 @@ -52,22 +55,26 @@ export default class TripsChart extends Component { x1={0} y1={y} x2={svgWidth} y2={y} stroke='gray' - strokesvgWidth={1} /> + strokeWidth={1} + /> {l} })} {data.map((d, index) => { - const dow = moment(d[0]).day() + const dow = d[0].day() + const dateString = d[0].format('YYYY-MM-DD') const x = leftMargin + (spacing / 2) + (index * spacing) // generate the bar for this date return ( + {dateString}: {d[1]} trips - {index % 14 === 0 /* label the date every 14 days */ + {index % 14 === 0 /* label x-axis with dates every 14 days */ ? - {d[0]} + {dateString} : null From 0fd9d75e6b901cd9adf039340d63783813d8bca8 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Thu, 2 Nov 2017 16:33:49 -0400 Subject: [PATCH 06/21] fix(feed-summary): remove flow from trips chart --- lib/manager/components/validation/TripsChart.js | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/manager/components/validation/TripsChart.js b/lib/manager/components/validation/TripsChart.js index 8f1f9edbb..6c87ebda2 100644 --- a/lib/manager/components/validation/TripsChart.js +++ b/lib/manager/components/validation/TripsChart.js @@ -18,9 +18,8 @@ export default class TripsChart extends Component { if (!this.props.validationResult) { return } - const {dailyTripCounts, firstCalendarDate, lastCalendarDate} = this.props.validationResult + const {dailyTripCounts, firstCalendarDate} = this.props.validationResult const firstDate = moment(firstCalendarDate) - const lastDate = moment(lastCalendarDate) const data = dailyTripCounts.map((count, index) => [firstDate.clone().add(index, 'days'), count]) const graphHeight = 300 @@ -31,8 +30,6 @@ export default class TripsChart extends Component { const svgHeight = graphHeight + bottomMargin const maxTrips = Math.max.apply(Math, data.map(d => d[1])) const yAxisMax = Math.ceil(maxTrips / 100) * 100 - // console.log(maxTrips, yAxisMax) - const yAxisPeriod = maxTrips > 1000 ? 1000 : 100 const yAxisLabels = [] for (var i = yAxisPeriod; i <= yAxisMax; i += yAxisPeriod) { @@ -82,7 +79,8 @@ export default class TripsChart extends Component { : '#8da0cb' } strokeWidth={7} /> - {index % 14 === 0 /* label x-axis with dates every 14 days */ + {/* label x-axis with dates every 14 days */} + {index % 14 === 0 ? @@ -93,8 +91,8 @@ export default class TripsChart extends Component { } ) - } - )} + })} + {/* Add baseline to chart */} Date: Tue, 7 Nov 2017 18:12:32 -0500 Subject: [PATCH 07/21] docs(deployment): update docs for mongo-pojo setup --- docs/dev/deployment.md | 114 +++++++++++++++++++++++++++++++---------- 1 file changed, 87 insertions(+), 27 deletions(-) diff --git a/docs/dev/deployment.md b/docs/dev/deployment.md index fd143df5b..f4002d038 100644 --- a/docs/dev/deployment.md +++ b/docs/dev/deployment.md @@ -2,9 +2,16 @@ ## Prerequisites -The application consists of two repositories: a [Spark-powered Java backend](https://github.com/conveyal/datatools-server) and a [Javascript frontend written with React and Redux](https://github.com/conveyal/datatools-ui). To install and deploy the application, you will need Java 8, Maven, Node/npm, yarn, and [mastarm](https://github.com/conveyal/mastarm). +The application consists of two repositories: a [Spark-powered Java backend](https://github.com/conveyal/datatools-server) +and a [Javascript frontend written with React and Redux](https://github.com/conveyal/datatools-ui). +To install and deploy the application, you will need Java 8, Maven, Node/npm, +yarn, and [mastarm](https://github.com/conveyal/mastarm). -User authentication is done via [Auth0](http://auth0.com). You will need an Auth0 account and application to use the Data Manager. +User authentication is done via [Auth0](http://auth0.com). You will need an +Auth0 account and application to use the Data Manager. + +Two databases are required for the application: [MongoDB](https://www.mongodb.com/) +and a SQL database ([PostgreSQL](https://www.postgresql.org/) is recommended). ## Installation and Basic Configuration @@ -22,9 +29,13 @@ $ cp datatools-ui/configurations/default/env.yml.tmp datatools-ui/configurations $ cp datatools-server/configurations/default/env.yml.tmp datatools-server/configurations/default/env.yml ``` -You'll then need to supply Auth0 account information (see below) and API keys for various services used in the application. +You'll then need to supply Auth0 account information (see below) and API keys +for various services used in the application. -The default `server.yml` (for `datatools-server`) and `settings.yml` (for `datatools-ui`) should work out of the box, but you may want to specify alternative settings files outside of these repositories. These can be specified as a directory during `datatools-ui` build with mastarm: +The default `server.yml` (for `datatools-server`) and `settings.yml` (for +`datatools-ui`) should work out of the box, but you may want to specify +alternative settings files outside of these repositories. These can be specified +as a directory during `datatools-ui` build with mastarm: ```bash $ mastarm build --config /path/to/configurations/dir @@ -35,24 +46,56 @@ AND as individual file args for `datatools-server`: ```bash $ java -jar target/dt-v1.0.0.jar /path/to/env.yml /path/to/server.yml ``` -In `datatools-server:server.yml`, be sure to update the paths for where the databases will be stored: +In `datatools-server:server.yml`, be sure to update the paths for where the +databases will be stored: ```yaml application: data: - mapdb: /path/to/mapdb - gtfs: /path/to/gtfs - editor_mapdb: /path/to/editor - regions: /path/to/regions/geojson + gtfs: /path/to/gtfs/storage ``` + +### Database setup + +#### GTFS data storage +GTFS data storage is handled by whichever standard RDBMS you prefer. However, +the application has been significantly tuned and optimized for PostgreSQL 9, so +we highly recommend using PostgreSQL. + +Once PostgreSQL is installed and the service has been started, create the +database: +```bash +$ createdb gtfs_storage_db +``` +Pass the URL of the database in the server's `env.yml` (and optionally add +additional connection variables): +```yaml +GTFS_DATABASE_URL: jdbc:postgresql://localhost/gtfs_storage_db +# GTFS_DATABASE_USER: +# GTFS_DATABASE_PASSWORD: +``` + +#### Application data storage +Application data storage (i.e., where projects, feed sources, and feed versions +are stored) is handled by MongoDB. There is no need to manually initialize a +database in MongoDB (MongoDB will handle this automatically if you prefer). +Connection details for MongoDB are also set in the server's `env.yml`: +```yaml +MONGO_URI: # defaults to localhost:27017 (MongoDB default) if empty +MONGO_DB_NAME: application_db +``` + ### Setting up Auth0 #### Creating account and application (client) 1. Create an [Auth0](https://auth0.com) account (free). -2. Once you've created an Auth0 account, create an application (client) in Auth0 to use with the Data Manager with the following settings: - - enable only `Username-Password-Authentication` connections (i.e., turn off Google) +2. Once you've created an Auth0 account, create an application (client) in Auth0 + to use with the Data Manager with the following settings: + - enable only `Username-Password-Authentication` connections (i.e., turn off + Google) - set `Use Auth0 instead of the IdP to do Single Sign On` to true - - update the following application- and account-level settings to include `http://localhost:9000` (or the domain where the application will be hosted): + - update the following application- and account-level settings to include + `http://localhost:9000` (or the domain where the application will be hosted): - Account level (Account Settings > Advanced) - Allowed logout URLs - Application level @@ -61,7 +104,10 @@ application: - keep all other default settings #### Creating your first user -Create your first Auth0 user through Auth0 web console (Users > Create User). In addition to an email and password, you'll need to supply the user with the following default application admin `app_metadata` (`user_metadata` should remain empty): +Create your first Auth0 user through Auth0 web console (Users > Create User). In +addition to an email and password, you'll need to supply the user with the +following default application admin `app_metadata` (`user_metadata` should +remain empty): ```json { @@ -78,7 +124,7 @@ Create your first Auth0 user through Auth0 web console (Users > Create User). In } ``` -#### Update `env.yml` for both +#### Update `env.yml` for server and UI repos Update the following properties in **both** `env.yml` files to reflect the public Auth0 application settings: ```yaml @@ -116,7 +162,6 @@ Build and deploy the frontend to s3 using npm script (which calls [mastarm](http ```bash $ npm run deploy -- s3://$S3_BUCKET_NAME/dist ->>>>>>> Stashed changes ``` Package the application using Maven: @@ -132,12 +177,18 @@ $ java -jar target/dt-v1.0.0.jar /path/to/env.yml /path/to/server.yml ``` -The application back-end should now be running at `http://localhost:9000` (or whatever port you specified in `server.yml`). -The front-end assets are pointed to by the back end at whatever s3 bucket name is specified in `server.yml` at `application.assets_bucket`. +The application back-end should now be running at `http://localhost:9000` (or +whatever port you specified in `server.yml`). The front-end assets are pointed +to by the back end at whatever s3 bucket name is specified in `server.yml` at +`application.assets_bucket`. ## Configuring Modules -The application contains a number of optional modules that each require their own configuration settings and resources. At minimum, each module must be set to `enabled: true` and may require additional configuration. +The application contains a number of optional modules that each require their +own configuration settings and resources. At minimum, each module must be set to +`enabled: true` and may require additional configuration. + +**Note:** for datatools-server v2.0.0, the `editor` and `r5_network` ### Editor @@ -150,13 +201,16 @@ Enables the GTFS Editor module. - `MAPBOX_ACCESS_TOKEN` - `R5_URL` (optional parameter for r5 routing in editor pattern drawing) -### Validator +### R5 network validation -While the application handles basic validation even without the validator module enabled, the validator allows for enhanced accessibility- and map-based validation. +While the application handles basic validation even without the `r5_network` +module enabled, this module allows for enhanced accessibility- and map-based +validation. #### List of configuration settings -- `OSM_VEX` - `datatools-server:env.yml` the validator requires the URL of a running instance of the [OSM vex server](https://github.com/conveyal/vanilla-extract). +- `OSM_VEX` - `datatools-server:env.yml` the validator requires the URL of a +running instance of the [OSM vex server](https://github.com/conveyal/vanilla-extract). ### Sign Configurations @@ -189,26 +243,31 @@ Enables the OTP automated deployments module. #### List of configuration settings -- `OSM_VEX` - `datatools-server:env.yml` the validator requires the URL of a running instance of the [OSM vex server](https://github.com/conveyal/vanilla-extract). +- `OSM_VEX` - `datatools-server:env.yml` the validator requires the URL of a +running instance of the [OSM vex server](https://github.com/conveyal/vanilla-extract). ### GTFS API -Supports other modules with API endpoints for getting GTFS entities (routes, stops, etc.) +Supports other modules with API endpoints for getting GTFS entities (routes, +stops, etc.) #### List of configuration settings - `load_on_fetch` - whether to load GTFS feeds when new feed is detected - `load_on_startup` - whether to load GTFS feeds on application startup - `use_extension` - which extension to connect to -- `update_frequency` - update frequency for GTFS API (in seconds). Comment out to disable updates. +- `update_frequency` - update frequency for GTFS API (in seconds). Comment + out to disable updates. ## Configuring Extensions -The application supports integration with several third-party resources for retrieving feeds. +The application supports integration with several third-party resources for +retrieving feeds. ### Integration with [transit.land](https://transit.land/) -Ensure that the `extensions:transitland:enabled` flag is set to `true` in `config.yml`: +Ensure that the `extensions:transitland:enabled` flag is set to `true` in +`config.yml`: ```yaml extensions: @@ -219,7 +278,8 @@ extensions: ### Integration with [TransitFeeds](http://transitfeeds.com/) -Ensure that the `extensions:transitfeeds:enabled` flag is set to `true` in `config.yml`, and provide your API key: +Ensure that the `extensions:transitfeeds:enabled` flag is set to `true` in +`config.yml`, and provide your API key: ```yaml extensions: From 16380125e50eb06cf7c9985a58c175dcf4fcfecb Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Tue, 7 Nov 2017 18:15:36 -0500 Subject: [PATCH 08/21] docs(deployment): fix message about editor/r5 modules --- docs/dev/deployment.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/dev/deployment.md b/docs/dev/deployment.md index f4002d038..15de0aa38 100644 --- a/docs/dev/deployment.md +++ b/docs/dev/deployment.md @@ -188,7 +188,10 @@ The application contains a number of optional modules that each require their own configuration settings and resources. At minimum, each module must be set to `enabled: true` and may require additional configuration. -**Note:** for datatools-server v2.0.0, the `editor` and `r5_network` +**Note:** for `datatools-server` `v3.0.0`, the `editor` and `r5_network` should be +disabled because they have not been refactored to handle updates to the loading +of GTFS data into an RDBMS. Please use `v2.0.0` or wait for releases following +`v3.0.0`. ### Editor From 6036e205afa553d55d13a7df852c2ed96459beb9 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Wed, 8 Nov 2017 10:09:48 -0500 Subject: [PATCH 09/21] refactor(status): update job monitor for server refactor --- lib/common/components/JobMonitor.js | 117 +++++++++++++++++----------- lib/manager/actions/status.js | 2 +- lib/manager/reducers/status.js | 4 +- lib/style.css | 28 +++++++ 4 files changed, 103 insertions(+), 48 deletions(-) diff --git a/lib/common/components/JobMonitor.js b/lib/common/components/JobMonitor.js index 231b76a7c..cf0f4bd2d 100644 --- a/lib/common/components/JobMonitor.js +++ b/lib/common/components/JobMonitor.js @@ -1,7 +1,7 @@ import Icon from '@conveyal/woonerf/components/icon' import Pure from '@conveyal/woonerf/components/pure' import React, {PropTypes} from 'react' -import {ProgressBar, Button} from 'react-bootstrap' +import {ProgressBar, Button, OverlayTrigger, Popover} from 'react-bootstrap' import SidebarPopover from './SidebarPopover' @@ -28,17 +28,9 @@ export default class JobMonitor extends Pure { this.props.jobMonitor.retired.forEach(job => this.props.removeRetiredJob(job)) } + sortByDate = (a, b) => new Date(b.status.initialized) - new Date(a.status.initialized) + render () { - const jobContainerStyle = { - marginBottom: 20 - } - const progressBarStyle = { - marginBottom: 2 - } - const statusMessageStyle = { - fontSize: '12px', - color: 'darkGray' - } const {jobMonitor, removeRetiredJob} = this.props const {jobs, retired} = jobMonitor return ( @@ -46,39 +38,40 @@ export default class JobMonitor extends Pure { ref={(SidebarPopover) => { this.popover = SidebarPopover }} title='Server Jobs' {...this.props}> - {retired.map(job => ( - - ))} - {jobs.length - ? jobs.map(job => ( -
-
+
    + {retired.sort(this.sortByDate).map(job => ( + + ))} + {jobs.sort(this.sortByDate).map(job => ( +
  • +
    -
    +
    {job.name}
    - -
    {job.status ? job.status.message : 'waiting'}
    + +
    + {job.status ? job.status.message : 'waiting'} +
    -
    - )) - :

    No active jobs.

    - } - {retired.length - ? - : null - } +
  • + ))} +
+

{jobs.length ? jobs.length : 'No'} active jobs.

+ ) } @@ -95,17 +88,16 @@ class RetiredJob extends Pure { removeJob = () => this.props.removeRetiredJob(this.props.job) render () { - const {job, style, statusStyle} = this.props + const {job} = this.props return ( -
+
  • {job.status && job.status.error ? : }
    -
    +
    -
    +
    {job.status.message} + {job.status.exceptionDetails + ? + Oh no! Looks like an error has occurred. + + }> +

    + To submit an error report email a screenshot of your browser + window, the following text (current URL and error details), + and a detailed description of the steps you followed + to support@conveyal.com. +

    +

    {window.location.href}

    + + {job.status.exceptionDetails} + + + }> + +
    + : null + }
    -
    +
  • ) } } diff --git a/lib/manager/actions/status.js b/lib/manager/actions/status.js index daf74e4b8..736a30c21 100644 --- a/lib/manager/actions/status.js +++ b/lib/manager/actions/status.js @@ -140,7 +140,7 @@ export function handleFinishedJob (job) { dispatch(handlingFinishedJob(job)) switch (job.type) { case 'VALIDATE_FEED': - dispatch(fetchFeedSource(job.feedVersion.feedSource.id, true, true)) + dispatch(fetchFeedSource(job.feedSourceId, true, true)) break case 'PROCESS_SNAPSHOT': dispatch(fetchSnapshots(job.feedVersion.feedSource)) diff --git a/lib/manager/reducers/status.js b/lib/manager/reducers/status.js index b1d394b40..1fae12410 100644 --- a/lib/manager/reducers/status.js +++ b/lib/manager/reducers/status.js @@ -104,8 +104,8 @@ const config = (state = { modal: {$set: {title: 'Warning!', body: action.message, action: action.action}}, message: {$set: null} }) - case 'RECEIVED_FETCH_FEED': - return update(state, {modal: {$set: {title: 'Feed fetched successfully!', body: `New version for ${action.feedSource.name} fetched at ${moment().format('MMMM Do YYYY, h:mm:ss a')}`}}}) + // case 'RECEIVED_FETCH_FEED': + // return update(state, {modal: {$set: {title: 'Feed fetched successfully!', body: `New version for ${action.feedSource.name} fetched at ${moment().format('MMMM Do YYYY, h:mm:ss a')}`}}}) // case 'UPLOADED_FEED': // return update(state, {modal: {$set: {title: 'Feed uploaded successfully!', body: `New version for ${action.feedSource.name} uploaded at ${moment().format('MMMM Do YYYY, h:mm:ss a')}`}}}) case 'FEED_NOT_MODIFIED': diff --git a/lib/style.css b/lib/style.css index 418093b6d..82fdbbad9 100644 --- a/lib/style.css +++ b/lib/style.css @@ -212,3 +212,31 @@ h4.line:after { text-overflow: ellipsis; white-space: nowrap; } + +/* Job Monitor */ + +.job-list { + max-height: 200px; + overflow-y: scroll; +} + +.job-container { + margin-bottom: 20px; +} + +.job-container-inner { + margin-left: 25px; +} + +.job-status-progress-bar { + margin-bottom: 2px; +} + +.job-status-message { + font-size: 12px; + color: darkGray; +} + +.job-spinner-div { + float: left; +} From 71b46e6f356d797f4fe01076de58382d171784dd Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Wed, 8 Nov 2017 10:13:19 -0500 Subject: [PATCH 10/21] refactor(manager): use constants for API URL fetches --- lib/alerts/actions/alerts.js | 5 ++++- lib/common/constants/index.js | 9 +++++++++ lib/gtfs/actions/general.js | 3 ++- lib/gtfs/components/gtfsmapsearch.js | 3 ++- lib/gtfs/util/graphql.js | 4 +++- lib/manager/actions/versions.js | 13 +++++++------ 6 files changed, 27 insertions(+), 10 deletions(-) create mode 100644 lib/common/constants/index.js diff --git a/lib/alerts/actions/alerts.js b/lib/alerts/actions/alerts.js index 7204bd4b2..afb9b7bb6 100644 --- a/lib/alerts/actions/alerts.js +++ b/lib/alerts/actions/alerts.js @@ -3,6 +3,7 @@ import fetch from 'isomorphic-fetch' import { fetchStopsAndRoutes } from '../../gtfs/actions/general' import { secureFetch } from '../../common/actions' +import {GTFS_API_PREFIX} from '../../common/constants' import { getAlertsUrl, getFeedId } from '../../common/util/modules' import { setErrorMessage } from '../../manager/actions/status' import {getActiveProject} from '../../manager/selectors' @@ -129,7 +130,9 @@ export function editAlert (alert) { export function fetchEntity (entity, activeProject) { const feed = activeProject.feedSources.find(f => getFeedId(f) === entity.entity.AgencyId) const feedId = getFeedId(feed) - const url = entity.type === 'stop' ? `/api/manager/stops/${entity.entity.StopId}?feed=${feedId}` : `/api/manager/routes/${entity.entity.RouteId}?feed=${feedId}` + const url = entity.type === 'stop' + ? `${GTFS_API_PREFIX}stops/${entity.entity.StopId}?feed=${feedId}` + : `${GTFS_API_PREFIX}routes/${entity.entity.RouteId}?feed=${feedId}` return fetch(url) .then((response) => { return response.json() diff --git a/lib/common/constants/index.js b/lib/common/constants/index.js new file mode 100644 index 000000000..86a69d0ea --- /dev/null +++ b/lib/common/constants/index.js @@ -0,0 +1,9 @@ +const SECURE: string = 'secure/' +export const API_PREFIX: string = `/api/manager/` +export const SECURE_API_PREFIX: string = `${API_PREFIX}${SECURE}` +// TODO: move GTFS_API_PREFIX to gtfs sub path +// export const GTFS_API_PREFIX: string = `${API_PREFIX}gtfs/` +export const GTFS_API_PREFIX: string = `${API_PREFIX}` +export const GTFS_GRAPHQL_PREFIX: string = `${GTFS_API_PREFIX}graphql` +export const EDITOR_PREFIX: string = `/api/editor/` +export const SECURE_EDITOR_PREFIX: string = `${EDITOR_PREFIX}${SECURE}` diff --git a/lib/gtfs/actions/general.js b/lib/gtfs/actions/general.js index 83c78fdf9..c7aabbab3 100644 --- a/lib/gtfs/actions/general.js +++ b/lib/gtfs/actions/general.js @@ -4,6 +4,7 @@ import { clearStops } from './stops' import { clearPatterns } from './patterns' import { clearRoutes } from './routes' import { stopsAndRoutes, compose, patternsAndStopsForBoundingBox } from '../util/graphql' +import {GTFS_GRAPHQL_PREFIX} from '../../common/constants' import { getFeedId } from '../../common/util/modules' import {getActiveProject} from '../../manager/selectors' @@ -78,7 +79,7 @@ export function fetchStopsAndRoutes (entities, module) { variables: JSON.stringify({feedId, routeId, stopId}) }) dispatch(requestStopsAndRoutes(feedId, routeId, stopId, module)) - return fetch(`/api/manager/graphql`, {method, body}) + return fetch(GTFS_GRAPHQL_PREFIX, {method, body}) .then(response => response.json()) .then(results => dispatch(receivedStopsAndRoutes(results, module))) } diff --git a/lib/gtfs/components/gtfsmapsearch.js b/lib/gtfs/components/gtfsmapsearch.js index 9ed20cf4d..db3347a66 100644 --- a/lib/gtfs/components/gtfsmapsearch.js +++ b/lib/gtfs/components/gtfsmapsearch.js @@ -3,6 +3,7 @@ import {connect} from 'react-redux' import fetch from 'isomorphic-fetch' import { Button } from 'react-bootstrap' +import {GTFS_API_PREFIX} from '../../common/constants' import ActiveGtfsMap from '../containers/ActiveGtfsMap' import GtfsSearch from './gtfssearch' @@ -24,7 +25,7 @@ class GtfsMapSearch extends Component { } getPatterns (input) { - return fetch(`/api/manager/patterns?route=${input.route.route_id}&feed=${input.route.feed_id}`) + return fetch(`${GTFS_API_PREFIX}patterns?route=${input.route.route_id}&feed=${input.route.feed_id}`) .then((response) => { return response.json() }) diff --git a/lib/gtfs/util/graphql.js b/lib/gtfs/util/graphql.js index 99a9d85a6..d9ec46972 100644 --- a/lib/gtfs/util/graphql.js +++ b/lib/gtfs/util/graphql.js @@ -1,8 +1,10 @@ // @flow +import {GTFS_GRAPHQL_PREFIX} from '../../common/constants' + // variable names/keys must match those specified in GraphQL schema export function compose (query: string, variables: Object) { - return `/api/manager/graphql?query=${encodeURIComponent(query)}&variables=${encodeURIComponent(JSON.stringify(variables))}` + return `${GTFS_GRAPHQL_PREFIX}?query=${encodeURIComponent(query)}&variables=${encodeURIComponent(JSON.stringify(variables))}` } export const feed = ` diff --git a/lib/manager/actions/versions.js b/lib/manager/actions/versions.js index 54701846a..a5ef41d82 100644 --- a/lib/manager/actions/versions.js +++ b/lib/manager/actions/versions.js @@ -4,6 +4,7 @@ import S3 from 'aws-sdk/clients/s3' import { secureFetch } from '../../common/actions' import { getConfigProperty } from '../../common/util/config' +import {GTFS_GRAPHQL_PREFIX, SECURE_API_PREFIX} from '../../common/constants' import fileDownload from '../../common/util/file-download' import { setErrorMessage, startJobMonitor } from './status' import { fetchFeedSource } from './feeds' @@ -59,7 +60,7 @@ export function receiveFeedVersion (feedVersion) { export function fetchFeedVersion (feedVersionId) { return function (dispatch, getState) { dispatch(requestingFeedVersion()) - const url = `/api/manager/secure/feedversion/${feedVersionId}` + const url = `${SECURE_API_PREFIX}feedversion/${feedVersionId}` return dispatch(secureFetch(url)) .then(response => response.json()) .then(version => { @@ -85,7 +86,7 @@ export function publishedFeedVersion (feedVersion) { export function publishFeedVersion (feedVersion) { return function (dispatch, getState) { dispatch(publishingFeedVersion(feedVersion)) - const url = `/api/manager/secure/feedversion/${feedVersion.id}/publish` + const url = `${SECURE_API_PREFIX}feedversion/${feedVersion.id}/publish` return dispatch(secureFetch(url, 'post')) .then(response => response.json()) .then(version => { @@ -170,7 +171,7 @@ export function deletingFeedVersion (feedVersion) { export function deleteFeedVersion (feedVersion, changes) { return function (dispatch, getState) { dispatch(deletingFeedVersion(feedVersion)) - const url = '/api/manager/secure/feedversion/' + feedVersion.id + const url = `${SECURE_API_PREFIX}feedversion/${feedVersion.id}` return dispatch(secureFetch(url, 'delete')) .then((res) => { // fetch feed source with versions + snapshots @@ -243,7 +244,7 @@ export function fetchFeedVersionIsochrones (feedVersion, fromLat, fromLon, toLat } dispatch(requestingFeedVersionIsochrones(feedVersion, fromLat, fromLon, toLat, toLon, date, fromTime, toTime)) const params = {fromLat, fromLon, toLat, toLon, date, fromTime, toTime} - const url = `/api/manager/secure/feedversion/${feedVersion.id}/isochrones?${qs.stringify(params)}` + const url = `${SECURE_API_PREFIX}feedversion/${feedVersion.id}/isochrones?${qs.stringify(params)}` return dispatch(secureFetch(url)) .then(res => { console.log(res.status) @@ -310,7 +311,7 @@ export function creatingFeedVersionFromSnapshot () { export function createFeedVersionFromSnapshot (feedSource, snapshotId) { return function (dispatch, getState) { dispatch(creatingFeedVersionFromSnapshot()) - const url = `/api/manager/secure/feedversion/fromsnapshot?feedSourceId=${feedSource.id}&snapshotId=${snapshotId}` + const url = `${SECURE_API_PREFIX}feedversion/fromsnapshot?feedSourceId=${feedSource.id}&snapshotId=${snapshotId}` return dispatch(secureFetch(url, 'post')) .then((res) => { if (res) dispatch(startJobMonitor()) @@ -327,7 +328,7 @@ export function renamingFeedVersion () { export function renameFeedVersion (feedVersion, name) { return function (dispatch, getState) { dispatch(renamingFeedVersion()) - const url = `/api/manager/secure/feedversion/${feedVersion.id}/rename?name=${name}` + const url = `${SECURE_API_PREFIX}feedversion/${feedVersion.id}/rename?name=${name}` return dispatch(secureFetch(url, 'put')) .then((res) => { dispatch(fetchFeedVersion(feedVersion.id)) From 92194c392693cbeb12ec0aab21da22f663697aef Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Wed, 8 Nov 2017 10:29:31 -0500 Subject: [PATCH 11/21] fix(file-upload): Place file uploads directly in request body This change removes the use of multipart form data uploads in favor of placing the file directly in the request body. This change is in response to catalogueglobal/datatools-server#57. --- lib/common/util/upload-file.js | 12 +++++++ lib/gtfsplus/actions/gtfsplus.js | 24 ++++++-------- lib/manager/actions/versions.js | 56 +++++++++++++++----------------- 3 files changed, 48 insertions(+), 44 deletions(-) create mode 100644 lib/common/util/upload-file.js diff --git a/lib/common/util/upload-file.js b/lib/common/util/upload-file.js new file mode 100644 index 000000000..0ce856f40 --- /dev/null +++ b/lib/common/util/upload-file.js @@ -0,0 +1,12 @@ +import fetch from 'isomorphic-fetch' + +export function uploadFile ({file, url, token}) { + return fetch(url, { + method: 'post', + headers: { + 'Authorization': 'Bearer ' + token, + // 'Content-Type': 'application/zip' + }, + body: file + }) +} diff --git a/lib/gtfsplus/actions/gtfsplus.js b/lib/gtfsplus/actions/gtfsplus.js index 83abf9625..de28d01ff 100644 --- a/lib/gtfsplus/actions/gtfsplus.js +++ b/lib/gtfsplus/actions/gtfsplus.js @@ -2,10 +2,11 @@ import JSZip from 'jszip' import fetch from 'isomorphic-fetch' import {createAction} from 'redux-actions' -import { secureFetch } from '../../common/actions' -import { getGtfsPlusSpec } from '../../common/util/config' +import {secureFetch} from '../../common/actions' +import {getGtfsPlusSpec} from '../../common/util/config' +import {uploadFile} from '../../common/util/upload-file' // import {stopsAndRoutes, compose} from '../../gtfs/util/graphql' -import { fetchFeedVersions } from '../../manager/actions/versions' +import {fetchFeedVersions} from '../../manager/actions/versions' // EDIT ACTIVE GTFS+ ACTIONS @@ -142,17 +143,12 @@ export function uploadGtfsPlusFeed (feedVersionId, file) { return function (dispatch, getState) { dispatch(uploadingGtfsPlusFeed(feedVersionId, file)) const url = `/api/manager/secure/gtfsplus/${feedVersionId}` - var data = new window.FormData() - data.append('file', file) - - return fetch(url, { - method: 'post', - headers: { 'Authorization': 'Bearer ' + getState().user.token }, - body: data - }).then(result => { - console.log(result) - return dispatch(uploadedGtfsPlusFeed(feedVersionId, file)) - }) + + return uploadFile({file, url, token: getState().user.token}) + .then(result => { + console.log(result) + return dispatch(uploadedGtfsPlusFeed(feedVersionId, file)) + }) } } diff --git a/lib/manager/actions/versions.js b/lib/manager/actions/versions.js index a5ef41d82..3c551e9dc 100644 --- a/lib/manager/actions/versions.js +++ b/lib/manager/actions/versions.js @@ -1,19 +1,17 @@ import fetch from 'isomorphic-fetch' import qs from 'qs' import S3 from 'aws-sdk/clients/s3' +import {createAction} from 'redux-actions' -import { secureFetch } from '../../common/actions' -import { getConfigProperty } from '../../common/util/config' +import {secureFetch} from '../../common/actions' import {GTFS_GRAPHQL_PREFIX, SECURE_API_PREFIX} from '../../common/constants' +import {getConfigProperty} from '../../common/util/config' +import {uploadFile} from '../../common/util/upload-file' import fileDownload from '../../common/util/file-download' -import { setErrorMessage, startJobMonitor } from './status' -import { fetchFeedSource } from './feeds' +import {setErrorMessage, startJobMonitor} from './status' +import {fetchFeedSource} from './feeds' -export function requestingFeedVersions () { - return { - type: 'REQUESTING_FEEDVERSIONS' - } -} +export const requestingFeedVersions = createAction('REQUESTING_FEEDVERSIONS') export function receiveFeedVersions (feedSource, feedVersions) { return { @@ -135,29 +133,27 @@ export function feedNotModified (feedSource, message) { export function uploadFeed (feedSource, file) { return function (dispatch, getState) { dispatch(uploadingFeed(feedSource, file)) - const url = `/api/manager/secure/feedversion?feedSourceId=${feedSource.id}&lastModified=${file.lastModified}` + const url = `${SECURE_API_PREFIX}feedversion?feedSourceId=${feedSource.id}&lastModified=${file.lastModified}` - var data = new window.FormData() - data.append('file', file) - - return fetch(url, { - method: 'post', - headers: { 'Authorization': 'Bearer ' + getState().user.token }, - body: data - }).then(res => { - if (res.status === 304) { - dispatch(feedNotModified(feedSource, 'Feed upload cancelled because it matches latest feed version.')) - } else if (res.status >= 400) { - dispatch(setErrorMessage('Error uploading feed source')) - } else { - dispatch(uploadedFeed(feedSource)) - dispatch(startJobMonitor()) - } - console.log('uploadFeed result', res) + return uploadFile({file, url, token: getState().user.token}) + .then(res => { + // 304 halt thrown by server if uploaded feed matches the hash of the + // latest version + if (res.status === 304) { + // Do not start job monitor if feed matches latest version. Display the + // status modal with a message about the cancelled upload. + dispatch(feedNotModified(feedSource, 'Feed upload cancelled because it matches latest feed version.')) + } else if (res.status >= 400) { + dispatch(setErrorMessage('Error uploading feed source')) + } else { + dispatch(uploadedFeed(feedSource)) + dispatch(startJobMonitor()) + } + console.log('uploadFeed result', res) - // fetch feed source with versions - return dispatch(fetchFeedSource(feedSource.id, true)) - }) + // fetch feed source with versions + return dispatch(fetchFeedSource(feedSource.id, true)) + }) } } From b53d7e85736a2757e5bfcaa3f1cbe11b6356c604 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Wed, 8 Nov 2017 12:06:13 -0500 Subject: [PATCH 12/21] fix(graphql): update graphql to match new gtfs-api --- lib/gtfs/actions/feed.js | 10 +-- lib/gtfs/actions/patterns.js | 59 +++++--------- lib/gtfs/actions/routes.js | 40 +++------ lib/gtfs/reducers/patterns.js | 15 ++-- lib/gtfs/reducers/routes.js | 4 +- lib/gtfs/util/graphql.js | 149 ++++++++++++++++++---------------- 6 files changed, 124 insertions(+), 153 deletions(-) diff --git a/lib/gtfs/actions/feed.js b/lib/gtfs/actions/feed.js index 5bd35efe2..41f26088c 100644 --- a/lib/gtfs/actions/feed.js +++ b/lib/gtfs/actions/feed.js @@ -31,21 +31,21 @@ function receiveFeed (feedId, data) { } } -export function fetchFeed (feedId, date, from, to) { +export function fetchFeed (namespace, date, from, to) { return function (dispatch, getState) { - dispatch(fetchingFeed(feedId, date, from, to)) + dispatch(fetchingFeed(namespace, date, from, to)) return fetch(compose( feed // (feedId, date, from, to) - , {feedId, date, from, to}) + , {namespace, date, from, to}) ) .then((response) => { if (response.status >= 300) { - return dispatch(errorFetchingFeed(feedId, date, from, to)) + return dispatch(errorFetchingFeed(namespace, date, from, to)) } return response.json() }) .then(json => { - dispatch(receiveFeed(feedId, json)) + dispatch(receiveFeed(namespace, json)) }) } } diff --git a/lib/gtfs/actions/patterns.js b/lib/gtfs/actions/patterns.js index 2af1281da..a4c48811b 100644 --- a/lib/gtfs/actions/patterns.js +++ b/lib/gtfs/actions/patterns.js @@ -1,58 +1,43 @@ import fetch from 'isomorphic-fetch' +import {createAction} from 'redux-actions' import { compose, patterns } from '../../gtfs/util/graphql' -export function fetchingPatterns (feedId, routeId) { - return { - type: 'FETCH_GRAPHQL_PATTERNS', - feedId, - routeId - } -} - -export function clearPatterns () { - return { - type: 'CLEAR_GRAPHQL_PATTERNS' - } -} - -export function errorFetchingPatterns (feedId, data) { - return { - type: 'FETCH_GRAPHQL_PATTERNS_REJECTED', - data - } -} - -export function receivePatterns (feedId, data) { - return { - type: 'FETCH_GRAPHQL_PATTERNS_FULFILLED', - data - } -} +export const fetchingPatterns = createAction('FETCH_GRAPHQL_PATTERNS') +export const clearPatterns = createAction('CLEAR_GRAPHQL_PATTERNS') +export const errorFetchingPatterns = createAction('FETCH_GRAPHQL_PATTERNS_REJECTED') +export const receivePatterns = createAction('FETCH_GRAPHQL_PATTERNS_FULFILLED') -export function patternDateTimeFilterChange (feedId, props) { +export function patternDateTimeFilterChange (namespace, props) { return function (dispatch, getState) { const routeId = getState().gtfs.patterns.routeFilter const { date, from, to } = getState().gtfs.filter.dateTimeFilter - dispatch(fetchPatterns(feedId, routeId, date, from, to)) + dispatch(fetchPatterns(namespace, routeId, date, from, to)) } } -export function fetchPatterns (feedId, routeId, date, from, to) { +export function fetchPatterns (namespace, routeId, date, from, to) { return function (dispatch, getState) { - dispatch(fetchingPatterns(feedId, routeId, date, from, to)) + dispatch(fetchingPatterns({namespace, routeId, date, from, to})) + // FIXME: if routeId is null, clear current routes so we can fetch all routes if (!routeId) { - return dispatch(receivePatterns(feedId, {routes: []})) + const routes = [] + return dispatch(receivePatterns({namespace, routes})) } - return fetch(compose(patterns, {feedId, routeId, date, from, to})) + return fetch(compose(patterns, {namespace, routeId, date, from, to})) .then((response) => { if (response.status >= 300) { - return dispatch(errorFetchingPatterns(feedId, routeId)) + return dispatch(errorFetchingPatterns(namespace, routeId)) } return response.json() }) .then(json => { - dispatch(receivePatterns(feedId, json)) + if (json && json.feed) { + const {routes} = json.feed + dispatch(receivePatterns({namespace, routes})) + } else { + console.log('Error fetching patterns') + } }) } } @@ -64,11 +49,11 @@ export function updateRouteFilter (routeId) { } } -export function patternRouteFilterChange (feedId, routeData) { +export function patternRouteFilterChange (namespace, routeData) { return function (dispatch, getState) { const newRouteId = (routeData && routeData.route_id) ? routeData.route_id : null const {date, from, to} = getState().gtfs.filter.dateTimeFilter dispatch(updateRouteFilter(newRouteId)) - dispatch(fetchPatterns(feedId, newRouteId, date, from, to)) + dispatch(fetchPatterns(namespace, newRouteId, date, from, to)) } } diff --git a/lib/gtfs/actions/routes.js b/lib/gtfs/actions/routes.js index 9f1d404c4..bbcd1d975 100644 --- a/lib/gtfs/actions/routes.js +++ b/lib/gtfs/actions/routes.js @@ -1,42 +1,22 @@ import fetch from 'isomorphic-fetch' +import {createAction} from 'redux-actions' import { compose, routes } from '../../gtfs/util/graphql' -export function fetchingRoutes (feedId) { - return { - type: 'FETCH_GRAPHQL_ROUTES', - feedId - } -} - -export function clearRoutes () { - return { - type: 'CLEAR_GRAPHQL_ROUTES' - } -} - -export function errorFetchingRoutes (feedId, data) { - return { - type: 'FETCH_GRAPHQL_ROUTES_REJECTED', - data - } -} - -export function receiveRoutes (feedId, data) { - return { - type: 'FETCH_GRAPHQL_ROUTES_FULFILLED', - data - } -} +export const fetchingRoutes = createAction('FETCH_GRAPHQL_ROUTES') +export const clearRoutes = createAction('CLEAR_GRAPHQL_ROUTES') +export const errorFetchingRoutes = createAction('FETCH_GRAPHQL_ROUTES_REJECTED') +const receiveRoutes = createAction('FETCH_GRAPHQL_ROUTES_FULFILLED') -export function fetchRoutes (feedId) { +export function fetchRoutes (namespace) { return function (dispatch, getState) { - dispatch(fetchingRoutes(feedId)) - return fetch(compose(routes, { feedId: feedId })) + dispatch(fetchingRoutes(namespace)) + return fetch(compose(routes, { namespace })) .then((response) => { return response.json() }) .then(json => { - dispatch(receiveRoutes(feedId, json)) + const {routes} = json.feed + dispatch(receiveRoutes({namespace, routes})) }) } } diff --git a/lib/gtfs/reducers/patterns.js b/lib/gtfs/reducers/patterns.js index 90bf2527e..d9bb8d06c 100644 --- a/lib/gtfs/reducers/patterns.js +++ b/lib/gtfs/reducers/patterns.js @@ -62,15 +62,16 @@ export default function reducer (state = defaultState, action) { data: {$set: action.patterns} }) case 'FETCH_GRAPHQL_PATTERNS_FULFILLED': - const allRoutes = action.data ? action.data.routes : [] + const {routes} = action.payload const allPatterns = [] - for (let i = 0; i < allRoutes.length; i++) { - const curRouteId = allRoutes[i].route_id - const curRouteName = getRouteName(allRoutes[i]) - - for (let j = 0; j < allRoutes[i].patterns.length; j++) { - const curPattern = allRoutes[i].patterns[j] + // iterate over routes from graphql to set route name + for (let i = 0; i < routes.length; i++) { + const curRouteId = routes[i].route_id + const curRouteName = getRouteName(routes[i]) + // iterate over patterns from graphql to add route ref fields + for (let j = 0; j < routes[i].patterns.length; j++) { + const curPattern = routes[i].patterns[j] curPattern.route_id = curRouteId curPattern.route_name = curRouteName allPatterns.push(curPattern) diff --git a/lib/gtfs/reducers/routes.js b/lib/gtfs/reducers/routes.js index 3186a14eb..bcc222cb3 100644 --- a/lib/gtfs/reducers/routes.js +++ b/lib/gtfs/reducers/routes.js @@ -36,8 +36,8 @@ export default function reducer (state = defaultState, action) { }) case 'FETCH_GRAPHQL_ROUTES_FULFILLED': const newRoutes = [] - for (let i = 0; i < action.data.routes.length; i++) { - const curRoute = action.data.routes[i] + for (let i = 0; i < action.payload.routes.length; i++) { + const curRoute = action.payload.routes[i] curRoute.route_name = getRouteName(curRoute) newRoutes.push(curRoute) } diff --git a/lib/gtfs/util/graphql.js b/lib/gtfs/util/graphql.js index d9ec46972..5c53b81ee 100644 --- a/lib/gtfs/util/graphql.js +++ b/lib/gtfs/util/graphql.js @@ -8,9 +8,9 @@ export function compose (query: string, variables: Object) { } export const feed = ` -query feedQuery($feedId: [String]) { - feeds (feed_id: $feedId) { - feed_id, +query feedQuery($namespace: String) { + feeds (namespace: $namespace) { + namespace, # feed_publisher_name, # feed_publisher_url, # feed_lang, @@ -22,45 +22,49 @@ query feedQuery($feedId: [String]) { } ` +// FIXME: add back in counts to route query export const routes = ` -query routeQuery($feedId: [String]) { - routes(feed_id: $feedId) { - route_id - route_short_name - route_long_name, - route_desc, - route_url, - trip_count, - pattern_count +query routeQuery($namespace: String) { + feed(namespace: $namespace) { + routes { + route_id + route_short_name + route_long_name, + route_desc, + route_url, + # trip_count, + # pattern_count + } } } ` export const patterns = ` -query patternsQuery($feedId: [String], $routeId: [String], $date: String, $from: Long, $to: Long) { - routes (feed_id: $feedId, route_id: $routeId) { - route_id, - route_short_name, - route_long_name, - patterns { - pattern_id, - name, - geometry, - stop_count, - trip_count, - stats(date: $date, from: $from, to: $to){ - headway, - avgSpeed - }, +query patternsQuery($namespace: String, $routeId: [String], $date: String, $from: Long, $to: Long) { + feed (namespace: $namespace, route_id: $routeId) { + routes { + route_id, + route_short_name, + route_long_name, + patterns { + pattern_id, + name, + geometry, + stop_count, + trip_count, + stats(date: $date, from: $from, to: $to){ + headway, + avgSpeed + }, + } } } } ` -export const stops = () => { - return ` - query allStopsQuery($feedId: [String]) { - stops(feed_id: $feedId) { +export const stops = ` + query allStopsQuery($namespace: String) { + stops(namespace: $namespace) { stops { stop_id, stop_name, @@ -76,11 +80,10 @@ export const stops = () => { } } ` -} export const allStops = ` -query allStopsQuery($feedId: [String]) { - stops(feed_id: $feedId) { +query allStopsQuery($namespace: String) { + stops(namespace: $namespace) { stops { stop_id, stop_name, @@ -96,18 +99,18 @@ query allStopsQuery($feedId: [String]) { } } ` - +// FIXME: allows for multiple feedIds export const patternsAndStopsForBoundingBox = ( - feedId: string, + namespace: string, entities: Array, maxLat: number, maxLon: number, minLat: number, minLon: number ) => ` - query patternsAndStopsGeo($feedId: [String], $max_lat: Float, $max_lon: Float, $min_lat: Float, $min_lon: Float){ + query patternsAndStopsGeo($namespace: String, $max_lat: Float, $max_lon: Float, $min_lat: Float, $min_lon: Float){ ${entities.indexOf('routes') !== -1 - ? `patterns(feed_id: $feedId, max_lat: $max_lat, max_lon: $max_lon, min_lat: $min_lat, min_lon: $min_lon){ + ? `patterns(namespace: $namespace, max_lat: $max_lat, max_lon: $max_lon, min_lat: $min_lat, min_lon: $min_lon){ pattern_id, geometry, name, @@ -117,14 +120,14 @@ export const patternsAndStopsForBoundingBox = ( route_long_name, route_color, feed{ - feed_id + namespace }, } },` : '' } ${entities.indexOf('stops') !== -1 - ? `stops(feed_id: $feedId, max_lat: $max_lat, max_lon: $max_lon, min_lat: $min_lat, min_lon: $min_lon){ + ? `stops(namespace: $namespace, max_lat: $max_lat, max_lon: $max_lon, min_lat: $min_lat, min_lon: $min_lon){ stop_id, stop_code, stop_name, @@ -132,7 +135,7 @@ export const patternsAndStopsForBoundingBox = ( stop_lat, stop_lon, feed{ - feed_id + namespace } }` : '' @@ -141,10 +144,10 @@ export const patternsAndStopsForBoundingBox = ( ` // for use in entity fetching for signs / alerts -export const stopsAndRoutes = (feedId: string, routeId: ?string, stopId: ?string) => ` - query routeStopQuery($feedId: [String], ${routeId ? '$routeId: [String],' : ''}, ${stopId ? '$stopId: [String],' : ''}){ +export const stopsAndRoutes = (namespace: string, routeId: ?string, stopId: ?string) => ` + query routeStopQuery($namespace: String, ${routeId ? '$routeId: [String],' : ''}, ${stopId ? '$stopId: [String],' : ''}){ feeds(feed_id: $feedId){ - feed_id, + namespace, ${stopId ? `stops (stop_id: $stopId){ stop_id, @@ -167,7 +170,7 @@ export const stopsAndRoutes = (feedId: string, routeId: ?string, stopId: ?string // TODO: add back in patternId filter export function stopsFiltered ( - feedId: ?string, + namespace: ?string, routeId: ?string, patternId: ?string, date: string, @@ -178,39 +181,41 @@ export function stopsFiltered ( const hasTo = typeof to !== 'undefined' && to !== null const query = ` query filteredStopsQuery( - ${feedId ? '$feedId: [String],' : ''} + ${namespace ? '$namespace: String,' : ''} ${routeId ? '$routeId: [String],' : ''} ${patternId ? '$patternId: [String],' : ''} ${date ? '$date: String,' : ''} ${hasFrom ? '$from: Long,' : ''} ${hasTo ? '$to: Long' : ''} ) { - stops(${feedId ? 'feed_id: $feedId,' : ''} ${routeId ? 'route_id: $routeId,' : ''} ${patternId ? 'pattern_id: $patternId,' : ''}) { - stop_id, - stop_name, - stop_name, - stop_code, - stop_desc, - stop_lon, - stop_lat, - ${date && hasFrom && hasTo - ? ` - stats(date: $date, from: $from, to: $to){ - headway, - tripCount - }, - transferPerformance(date: $date, from: $from, to: $to){ - fromRoute, - toRoute, - bestCase, - worstCase, - typicalCase - }, - ` - : ''} - # zone_id, - # stop_url, - # stop_timezone + feed(${namespace ? 'namespace: $namespace,' : ''} ${routeId ? 'route_id: $routeId,' : ''} ${patternId ? 'pattern_id: $patternId,' : ''}) { + stops { + stop_id, + stop_name, + stop_name, + stop_code, + stop_desc, + stop_lon, + stop_lat, + ${date && hasFrom && hasTo + ? ` + # stats(date: $date, from: $from, to: $to){ + # headway, + # tripCount + # }, + # transferPerformance(date: $date, from: $from, to: $to){ + # fromRoute, + # toRoute, + # bestCase, + # worstCase, + # typicalCase + # }, + ` + : ''} + # zone_id, + # stop_url, + # stop_timezone + } } } ` From aa7dffa459e3ab5dcc20d7bc9301fa7ea475e4aa Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Wed, 8 Nov 2017 12:07:20 -0500 Subject: [PATCH 13/21] fix(manager): do not request feedsource after fetch --- lib/manager/actions/feeds.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/manager/actions/feeds.js b/lib/manager/actions/feeds.js index 97dbff652..a8ebc9285 100644 --- a/lib/manager/actions/feeds.js +++ b/lib/manager/actions/feeds.js @@ -236,7 +236,7 @@ export function runFetchFeed (feedSource) { .then(result => { console.log('fetchFeed result', result) // fetch feed source with versions - return dispatch(fetchFeedSource(feedSource.id, true)) + // return dispatch(fetchFeedSource(feedSource.id, true)) }) } } From 4cdb527e551d196c3908829b64711750d5e73dab Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Wed, 8 Nov 2017 12:08:02 -0500 Subject: [PATCH 14/21] fix(manager): do not request project after fetch --- lib/manager/actions/projects.js | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/lib/manager/actions/projects.js b/lib/manager/actions/projects.js index d80d4015a..91d1e6fe6 100644 --- a/lib/manager/actions/projects.js +++ b/lib/manager/actions/projects.js @@ -216,13 +216,6 @@ export function runningFetchFeedsForProject () { } } -export function receiveFetchFeedsForProject (project) { - return { - type: 'RECEIVE_FETCH_FEED_FOR_PROJECT', - project - } -} - export function fetchFeedsForProject (project) { return function (dispatch, getState) { dispatch(runningFetchFeedsForProject()) @@ -235,15 +228,14 @@ export function fetchFeedsForProject (project) { } else if (res.status >= 400) { dispatch(setErrorMessage('Error fetching project feeds')) } else { - dispatch(receiveFetchFeedsForProject(project)) dispatch(startJobMonitor()) return res.json() } }) .then(result => { - console.log('fetchFeed result', result) - dispatch(receiveFetchFeedsForProject()) - dispatch((fetchProjectWithFeeds(project.id))) + console.log('fetchProject result', result) + // Do nothing here. Fetch feed jobs will kick off if there are any + // updated feeds, with their own handlers for finished jobs. }) } } From e7a7aa6a7a08a002c273731526c8c9dcc1139945 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Wed, 8 Nov 2017 12:09:00 -0500 Subject: [PATCH 15/21] fix(graphql): update graphql queries to use namespace --- lib/manager/components/reporter/containers/Feed.js | 4 ++-- .../components/reporter/containers/Patterns.js | 12 ++++++------ lib/manager/components/reporter/containers/Routes.js | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/manager/components/reporter/containers/Feed.js b/lib/manager/components/reporter/containers/Feed.js index 86c9c9d42..c7b8c2e4b 100644 --- a/lib/manager/components/reporter/containers/Feed.js +++ b/lib/manager/components/reporter/containers/Feed.js @@ -10,11 +10,11 @@ const mapStateToProps = (state, ownProps) => { } const mapDispatchToProps = (dispatch, ownProps) => { - const feedId = ownProps.version.id.replace('.zip', '') + const {namespace} = ownProps.version return { onComponentMount: (initialProps) => { if (!initialProps.feed.fetchStatus.fetched) { - dispatch(fetchFeed(feedId)) + dispatch(fetchFeed(namespace)) } } } diff --git a/lib/manager/components/reporter/containers/Patterns.js b/lib/manager/components/reporter/containers/Patterns.js index 079c04a3b..37aabe833 100644 --- a/lib/manager/components/reporter/containers/Patterns.js +++ b/lib/manager/components/reporter/containers/Patterns.js @@ -13,25 +13,25 @@ const mapStateToProps = (state, ownProps) => { } const mapDispatchToProps = (dispatch, ownProps) => { - const feedId = ownProps.version.id.replace('.zip', '') + const {namespace} = ownProps.version return { onComponentMount: (initialProps) => { if (!initialProps.routes.fetchStatus.fetched) { - dispatch(fetchRoutes(feedId)) + dispatch(fetchRoutes(namespace)) } // if (!initialProps.patterns.fetchStatus.fetched) { - // dispatch(fetchPatterns(feedId, null)) + // dispatch(fetchPatterns(namespace, null)) // } }, patternRouteFilterChange: (newValue) => { - dispatch(patternRouteFilterChange(feedId, newValue)) + dispatch(patternRouteFilterChange(namespace, newValue)) }, viewStops: (row) => { - dispatch(stopPatternFilterChange(feedId, row)) + dispatch(stopPatternFilterChange(namespace, row)) ownProps.selectTab('stops') }, patternDateTimeFilterChange: (props) => { - dispatch(patternDateTimeFilterChange(feedId, props)) + dispatch(patternDateTimeFilterChange(namespace, props)) } } } diff --git a/lib/manager/components/reporter/containers/Routes.js b/lib/manager/components/reporter/containers/Routes.js index a0964f82b..a7018106e 100644 --- a/lib/manager/components/reporter/containers/Routes.js +++ b/lib/manager/components/reporter/containers/Routes.js @@ -11,15 +11,15 @@ const mapStateToProps = (state, ownProps) => { } const mapDispatchToProps = (dispatch, ownProps) => { - const feedId = ownProps.version.id.replace('.zip', '') + const {namespace} = ownProps.version return { onComponentMount: (initialProps) => { if (!initialProps.routes.fetchStatus.fetched) { - dispatch(fetchRoutes(feedId)) + dispatch(fetchRoutes(namespace)) } }, viewPatterns: (row) => { - dispatch(patternRouteFilterChange(feedId, row)) + dispatch(patternRouteFilterChange(namespace, row)) ownProps.selectTab('patterns') } } From f95fd57f6ba381c4dd6eb9e5d5e702a8033292e1 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Wed, 8 Nov 2017 12:10:54 -0500 Subject: [PATCH 16/21] fix(manager): update validation errors to use graphql --- lib/manager/actions/versions.js | 76 +++++++++++++- .../validation/GtfsValidationSummary.js | 4 +- .../validation/GtfsValidationViewer.js | 99 +++++++++++++++---- .../version/FeedVersionNavigator.js | 10 +- .../components/version/FeedVersionViewer.js | 2 +- .../containers/ActiveFeedVersionNavigator.js | 4 +- 6 files changed, 164 insertions(+), 31 deletions(-) diff --git a/lib/manager/actions/versions.js b/lib/manager/actions/versions.js index 3c551e9dc..c940d61ed 100644 --- a/lib/manager/actions/versions.js +++ b/lib/manager/actions/versions.js @@ -188,16 +188,84 @@ export function receiveValidationResult (feedVersion, validationResult) { validationResult } } + +const fetchingValidationErrors = createAction('FETCHING_VALIDATION_ERRORS') + +const receiveValidationErrors = createAction('RECEIVE_VALIDATION_ERRORS') + +export function fetchValidationErrors ({feedVersion, errorType, limit, offset}) { + return function (dispatch, getState) { + dispatch(fetchingValidationErrors) + // FIXME: why does namespace need to appear twice? + const query = ` + query errorsQuery($namespace: String, $errorType: [String], $limit: Int, $offset: Int) { + feed(namespace: $namespace) { + feed_id + feed_version + filename + errors (error_type: $errorType, limit: $limit, offset: $offset) { + error_type + entity_type + entity_id + line_number + bad_value + } + } + } + ` + const {namespace} = feedVersion + const method = 'post' + const body = JSON.stringify({ + query, + variables: JSON.stringify({namespace, errorType: [errorType], limit, offset}) + }) + return fetch(GTFS_GRAPHQL_PREFIX, {method, body}) + .then(response => response.json()) + .then(result => { + if (result.feed) { + const {errors} = result.feed + dispatch(receiveValidationErrors({feedVersion, errorType, limit, offset, errors})) + } + }) + .catch(err => console.log(err)) + } +} + export function fetchValidationResult (feedVersion, isPublic) { return function (dispatch, getState) { dispatch(requestingValidationResult(feedVersion)) - const route = isPublic ? 'public' : 'secure' - const url = `/api/manager/${route}/feedversion/${feedVersion.id}/validation` - return dispatch(secureFetch(url)) + const {namespace} = feedVersion + const query = ` + query countsQuery($namespace: String) { + feed(namespace: $namespace) { + feed_id + feed_version + filename + row_counts { + stops + trips + calendar_dates + errors + } + error_counts { + type, count + } + } + } + ` + const method = 'post' + const body = JSON.stringify({ + query, + variables: JSON.stringify({namespace}) + }) + return fetch(GTFS_GRAPHQL_PREFIX, {method, body}) .then(response => response.json()) .then(result => { - dispatch(receiveValidationResult(feedVersion, result)) + if (result.feed) { + dispatch(receiveValidationResult(feedVersion, result.feed)) + } }) + .catch(err => console.log(err)) } } diff --git a/lib/manager/components/validation/GtfsValidationSummary.js b/lib/manager/components/validation/GtfsValidationSummary.js index 38bf4ff22..528a0f43f 100644 --- a/lib/manager/components/validation/GtfsValidationSummary.js +++ b/lib/manager/components/validation/GtfsValidationSummary.js @@ -55,7 +55,7 @@ export class ValidationSummaryTable extends Component { const { version } = this.props - if (version && version.validationResult && version.validationResult.errors.length === 0) { + if (version && version.validationResult && version.validationResult.error_counts.length === 0) { return
    No validation issues found.
    } else if (!version || !version.validationResult) { return
    Feed has not yet been validated.
    @@ -65,7 +65,7 @@ export class ValidationSummaryTable extends Component { overflowWrap: 'break-word' } const problemMap = {} - version.validationResult.errors && version.validationResult.errors.map(val => { + version.validationResult.error_counts && version.validationResult.error_counts.map(val => { if (!problemMap[val.errorType]) { problemMap[val.errorType] = { count: 0, diff --git a/lib/manager/components/validation/GtfsValidationViewer.js b/lib/manager/components/validation/GtfsValidationViewer.js index 448a9a6ab..80a8d463d 100644 --- a/lib/manager/components/validation/GtfsValidationViewer.js +++ b/lib/manager/components/validation/GtfsValidationViewer.js @@ -1,13 +1,13 @@ +import Icon from '@conveyal/woonerf/components/icon' import moment from 'moment' import React, {Component, PropTypes} from 'react' -import { - // Badge, - // Button -} from 'react-bootstrap' +import {ListGroup, ListGroupItem, Panel} from 'react-bootstrap' import BootstrapTable from 'react-bootstrap-table/lib/BootstrapTable' import TableHeaderColumn from 'react-bootstrap-table/lib/TableHeaderColumn' -import {ValidationSummaryTable} from './GtfsValidationSummary' +import OptionButton from '../../../common/components/OptionButton' + +const DEFAULT_LIMIT = 10 export default class GtfsValidationViewer extends Component { static propTypes = { @@ -15,6 +15,10 @@ export default class GtfsValidationViewer extends Component { validationResult: PropTypes.object, version: PropTypes.object } + state = { + offset: 0, + limit: DEFAULT_LIMIT + } componentWillMount () { this.props.fetchValidationResult() } @@ -40,11 +44,35 @@ export default class GtfsValidationViewer extends Component { formatter (cell, row) { return {cell} } + + _onClickErrorType = errorType => { + const {version: feedVersion, fetchValidationErrors} = this.props + const {active} = this.state + if (active === errorType) { + this.setState({active: null}) + } else { + const offset = 0 + const limit = DEFAULT_LIMIT + // reset active error type, limit, and offset + this.setState({active: errorType, limit, offset}) + fetchValidationErrors({feedVersion, errorType, offset, limit}) + } + } + + _onClickLoadMoreErrors = errorType => { + const {version: feedVersion, fetchValidationErrors} = this.props + const {limit} = this.state + const offset = this.state.offset * limit + 1 + this.setState({offset}) + fetchValidationErrors({feedVersion, errorType, offset, limit}) + } + render () { const { validationResult: result, version } = this.props + const {active} = this.state const dateFormat = 'MMM. DD, YYYY' const timeFormat = 'h:MMa' // const messages = getComponentMessages('GtfsValidationViewer') @@ -61,21 +89,58 @@ export default class GtfsValidationViewer extends Component { sizePerPageList: [10, 20, 50, 100] } } - // let report = null - const files = ['routes', 'stops', 'trips', 'shapes', 'stop_times'] - const errors = {} - result && result.errors.map((error, i) => { - error.index = i - const key = files.indexOf(error.file) !== -1 ? error.file : 'other' - if (!errors[error.file]) { - errors[key] = [] - } - errors[key].push(error) - }) + const hasValidation = result && result.error_counts + const hasErrors = hasValidation && result.error_counts.length + const listGroupItemStyle = { fontSize: '18px', textAlign: 'center' } return (

    {version.name} {moment(version.updated).format(dateFormat + ', ' + timeFormat)}

    - + Validation errors}> + {hasErrors + ? result.error_counts.map((category, index) => { + const activeWithErrors = category.errors && active === category.type + return ( + + + {category.type}: {category.count} + {' '} + + + + + {activeWithErrors + ? category.errors.map((error, index) => ( + + line: {error.line_number}{' '} + entity_type: {error.entity_type}{' '} + entity_id: {error.entity_id}{' '} + bad_value: {error.bad_value} + + )) + : null + } + {activeWithErrors && category.errors.length < category.count + ? + + Load more + + + : activeWithErrors + ? + No more errors of this type + + : null + } + + ) + }) + : null + } + + user={user} />
    diff --git a/lib/manager/components/version/FeedVersionViewer.js b/lib/manager/components/version/FeedVersionViewer.js index f6b895205..36ec0b67e 100644 --- a/lib/manager/components/version/FeedVersionViewer.js +++ b/lib/manager/components/version/FeedVersionViewer.js @@ -71,8 +71,8 @@ export default class FeedVersionViewer extends Component { ? : versionSection === 'issues' ? : versionSection === 'gtfsplus' && isModuleEnabled('gtfsplus') ? { } const { jobs } = state.status.jobMonitor const validationJob = feedVersionIndex >= 1 - ? jobs.find(j => j.type === 'VALIDATE_FEED' && j.feedVersion.id === feedVersions[feedVersionIndex - 1].id) + ? jobs.find(j => j.type === 'VALIDATE_FEED' && j.feedVersionId === feedVersions[feedVersionIndex - 1].id) : null const hasVersions = feedVersions && feedVersions.length > 0 @@ -89,6 +90,7 @@ const mapDispatchToProps = (dispatch, ownProps) => { newNotePostedForVersion: (feedVersion, note) => dispatch(postNoteForFeedVersion(feedVersion, note)), notesRequestedForVersion: (feedVersion) => dispatch(fetchNotesForFeedVersion(feedVersion)), fetchValidationResult: (feedVersion, isPublic) => dispatch(fetchValidationResult(feedVersion, isPublic)), + fetchValidationErrors: payload => dispatch(fetchValidationErrors(payload)), publishFeedVersion: (feedVersion) => dispatch(publishFeedVersion(feedVersion)) } } From 3162f3e1410ad5c41e83f234fd6164ac8f6acf21 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Wed, 8 Nov 2017 12:21:07 -0500 Subject: [PATCH 17/21] fix(reducer): clean up reducer, changes for feed validation issues --- lib/manager/reducers/projects.js | 103 ++++++++++++++++++------------- 1 file changed, 59 insertions(+), 44 deletions(-) diff --git a/lib/manager/reducers/projects.js b/lib/manager/reducers/projects.js index 96be39760..f43eb5f01 100644 --- a/lib/manager/reducers/projects.js +++ b/lib/manager/reducers/projects.js @@ -1,7 +1,6 @@ import update from 'react-addons-update' import { getConfigProperty } from '../../common/util/config' import { defaultSorter } from '../../common/util/util' - const projects = (state = { isFetching: false, all: null, @@ -132,14 +131,12 @@ const projects = (state = { } }) } else if (feeds.length) { // if projectId does not match active project - return update(state, - { - isFetching: {$set: false}, - all: { - [projectIndex]: {$merge: {feedSources: feeds}} - } + return update(state, { + isFetching: {$set: false}, + all: { + [projectIndex]: {$merge: {feedSources: feeds}} } - ) + }) } else { return update(state, { @@ -154,26 +151,14 @@ const projects = (state = { } projectIndex = state.all.findIndex(p => p.id === action.feedSource.projectId) const existingSources = state.all[projectIndex].feedSources || [] - let updatedSources sourceIndex = existingSources.findIndex(s => s.id === action.feedSource.id) - if (sourceIndex === -1) { // source does not currently; add it - updatedSources = [ - ...existingSources, - action.feedSource - ] - } else { // existing feedsource array includes this one, replace it - updatedSources = [ - ...existingSources.slice(0, sourceIndex), - action.feedSource, - ...existingSources.slice(sourceIndex + 1) - ] - } + const updatedSources = sourceIndex !== -1 + ? {[sourceIndex]: {$set: action.feedSource}} + : {$set: [action.feedSource]} return update(state, { all: { [projectIndex]: { - $merge: { - feedSources: updatedSources - } + feedSources: updatedSources } }, isFetching: { $set: false } @@ -208,28 +193,15 @@ const projects = (state = { versionIndex = state.all[projectIndex].feedSources[sourceIndex].feedVersions ? state.all[projectIndex].feedSources[sourceIndex].feedVersions.findIndex(v => v.id === action.feedVersion.id) : -1 - const existingVersions = state.all[projectIndex].feedSources[sourceIndex].feedVersions || [] - let updatedVersions - if (versionIndex === -1) { // version does not currently; add it - updatedVersions = [ - ...existingVersions, - action.feedVersion - ] - } else { // existing feedversion array includes this one, replace it - updatedVersions = [ - ...existingVersions.slice(0, versionIndex), - action.feedVersion, - ...existingVersions.slice(versionIndex + 1) - ] - } + const updatedVersions = versionIndex !== -1 + ? {[versionIndex]: {$set: action.feedVersion}} + : {$set: [action.feedVersion]} return update(state, { all: { [projectIndex]: { feedSources: { [sourceIndex]: { - $merge: { - feedVersions: updatedVersions - } + feedVersions: updatedVersions } } } @@ -280,6 +252,33 @@ const projects = (state = { } } }) + case 'RECEIVE_VALIDATION_ERRORS': + projectIndex = state.all.findIndex(p => p.id === action.payload.feedVersion.feedSource.projectId) + sourceIndex = state.all[projectIndex].feedSources.findIndex(s => s.id === action.payload.feedVersion.feedSource.id) + versionIndex = state.all[projectIndex].feedSources[sourceIndex].feedVersions.findIndex(v => v.id === action.payload.feedVersion.id) + const errorIndex = state.all[projectIndex].feedSources[sourceIndex].feedVersions[versionIndex].validationResult.error_counts.findIndex(e => e.type === action.payload.errorType) + const newErrors = state.all[projectIndex].feedSources[sourceIndex].feedVersions[versionIndex].validationResult.error_counts[errorIndex].errors + ? {$push: action.payload.errors} + : {$set: action.payload.errors} + return update(state, { + all: { + [projectIndex]: { + feedSources: { + [sourceIndex]: { + feedVersions: { + [versionIndex]: { + validationResult: { + error_counts: { + [errorIndex]: {errors: newErrors} + } + } + } + } + } + } + } + } + }) case 'RECEIVE_FEEDVERSION_ISOCHRONES': projectIndex = state.all.findIndex(p => p.id === action.feedSource.projectId) sourceIndex = state.all[projectIndex].feedSources.findIndex(s => s.id === action.feedSource.id) @@ -366,14 +365,13 @@ const projects = (state = { case 'RECEIVE_NOTES_FOR_FEEDSOURCE': projectIndex = state.all.findIndex(p => p.id === action.feedSource.projectId) sourceIndex = state.all[projectIndex].feedSources.findIndex(s => s.id === action.feedSource.id) + const {notes} = action return update(state, { all: { [projectIndex]: { feedSources: { [sourceIndex]: { - $merge: { - notes: action.notes - } + $merge: {notes} } } } @@ -424,4 +422,21 @@ const projects = (state = { } } +// TODO: Use this function to get indexes (perhaps set to idx variable?) +// function getIndexesFromFeed ({state, feedVersion, feedSource, projectId}) { +// if (!feedSource && feedVersion) ({feedSource} = feedVersion) +// if (!projectId) ({projectId} = feedSource) +// const projectIndex = state.all.findIndex(p => p.id === projectId) +// const sources = state.all[projectIndex].feedSources || [] +// const sourceIndex = feedSource && sources.findIndex(s => s.id === feedSource.id) +// const versionIndex = feedVersion && sources[sourceIndex].feedVersions +// ? sources[sourceIndex].feedVersions.findIndex(v => v.id === feedVersion.id) +// : -1 +// return { +// projectIndex, +// sourceIndex, +// versionIndex +// } +// } + export default projects From 89a70ef9febd933f54081650fa3f114292ec6f3c Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Wed, 8 Nov 2017 12:33:57 -0500 Subject: [PATCH 18/21] fix(flow): add flow declaration --- lib/common/constants/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/common/constants/index.js b/lib/common/constants/index.js index 86a69d0ea..dc2ea6b46 100644 --- a/lib/common/constants/index.js +++ b/lib/common/constants/index.js @@ -1,3 +1,4 @@ +// @flow const SECURE: string = 'secure/' export const API_PREFIX: string = `/api/manager/` export const SECURE_API_PREFIX: string = `${API_PREFIX}${SECURE}` From e6a3971c5b80cb49c3a7783d81528d7c243eb89b Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Wed, 8 Nov 2017 12:34:38 -0500 Subject: [PATCH 19/21] fix(upload-file): add contentType --- lib/common/util/upload-file.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/common/util/upload-file.js b/lib/common/util/upload-file.js index 0ce856f40..25e13c921 100644 --- a/lib/common/util/upload-file.js +++ b/lib/common/util/upload-file.js @@ -5,7 +5,7 @@ export function uploadFile ({file, url, token}) { method: 'post', headers: { 'Authorization': 'Bearer ' + token, - // 'Content-Type': 'application/zip' + 'Content-Type': 'application/zip' }, body: file }) From bd9302af27d270f6a94880a0d192329a3040f776 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Wed, 8 Nov 2017 12:48:51 -0500 Subject: [PATCH 20/21] fix(lint): remove unused moment --- lib/manager/reducers/status.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/manager/reducers/status.js b/lib/manager/reducers/status.js index 1fae12410..01afc82dc 100644 --- a/lib/manager/reducers/status.js +++ b/lib/manager/reducers/status.js @@ -1,5 +1,5 @@ import update from 'react-addons-update' -import moment from 'moment' + const config = (state = { message: null, modal: null, From 61784803993be9b4bb098355e8a63e4e20535a3d Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Wed, 8 Nov 2017 17:59:05 -0500 Subject: [PATCH 21/21] docs(graphql): comments about queries that need fixing in v3.0.x BREAKING CHANGE: file uploads and GraphQL queries (namespace variable) --- lib/gtfs/util/graphql.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/gtfs/util/graphql.js b/lib/gtfs/util/graphql.js index 5c53b81ee..c0ed49204 100644 --- a/lib/gtfs/util/graphql.js +++ b/lib/gtfs/util/graphql.js @@ -39,6 +39,7 @@ query routeQuery($namespace: String) { } ` +// FIXME: Pattern stats are not supported yet in gtfs-api, so this is broken. export const patterns = ` query patternsQuery($namespace: String, $routeId: [String], $date: String, $from: Long, $to: Long) { feed (namespace: $namespace, route_id: $routeId) { @@ -168,6 +169,7 @@ export const stopsAndRoutes = (namespace: string, routeId: ?string, stopId: ?str } ` +// FIXME: Stop stats are not supported yet in gtfs-api, so this is broken. // TODO: add back in patternId filter export function stopsFiltered ( namespace: ?string,