Skip to content

Commit

Permalink
GPII-4014: Improve verify.js for GPII-4014 deployment to verify docs …
Browse files Browse the repository at this point in the history 
…in batches.
  • Loading branch information
@cindyli
cindyli committed Aug 8, 2019
1 parent 6011017 commit 908e9be
Show file tree
Hide file tree
Showing 3 changed files with 85 additions and 57 deletions.
4 changes: 2 additions & 2 deletions scripts/deleteExpiredAccessTokens.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ gpii.accessTokens.initOptions = function (processArgv) {
// the view index is created. See:
// https://stackoverflow.com/questions/29854776/couchdb-filter-timestamps-in-a-reduce-function-some-sort-of-date-now
var currentTime = Date.now();
options.accessTokensUrl = options.couchDbUrl + "/_design/views/_view/findExpiredAccessTokens?ascending=true&startkey=0&endkey=" + currentTime + "&limit=" + options.maxDocsInBatchPerRequest;
options.accessTokensUrl = options.couchDbUrl + "/_design/views/_view/findAccessTokenByExpires?descending=true&startkey=" + currentTime + "&endkey=0&limit=" + options.maxDocsInBatchPerRequest;
options.accessTokens = [];
options.totalDeleted = 0;
options.parsedCouchDbUrl = url.parse(options.couchDbUrl);
Expand Down Expand Up @@ -151,7 +151,7 @@ gpii.accessTokens.findExpiredAccessTokens = function (responseString, options) {
*/
gpii.accessTokens.logDeletion = function (responseString, options) {
options.totalDeleted = Number(options.totalDeleted) + Number(options.accessTokens.length);
fluid.log("Deleted ", options.accessTokens.length, " of ", options.totalExpiredInThisBatch, " expired access tokens.");
fluid.log("Deleted ", options.accessTokens.length, " of ", options.totalExpiredInThisBatch, " access tokens.");
return options.accessTokens.length;
};

Expand Down
136 changes: 82 additions & 54 deletions scripts/migration/schema-0.2-GPII-4014/verify.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,14 @@ https://github.com/GPII/universal/blob/master/LICENSE.txt
// 2. All "schemaVersion" in documents have been set to 0.2. Note that "_design/views" doc doesn't have this field.
// 3. All "timestampUpdated" have been set.

// Usage: node scripts/migration/schema-0.2-GPII-4014/verify.js CouchDB-url clientCredentialId-for-NOVA ...
// Usage: node scripts/migration/schema-0.2-GPII-4014/verify.js CouchDB-url maxDocsInBatchPerRequest clientCredentialId-for-NOVA ...
// @param {String} CouchDB-url - The url to the CouchDB where docoments should be verified.
// @param {Number} maxDocsInBatchPerRequest - Limit the number of documents to be verified in a batch.
// @param {Strings} clientCredentialIds-for-NOVA - The "_id" value of the NOVA client credential. There could be any
// number of client credential parameters from here onwards.

// A sample command that runs this script in the universal root directory:
// node scripts/migration/schema-0.2-GPII-4014/verify.js http://localhost:25984 "clientCredential-nova1" "clientCredential-nova2"
// node scripts/migration/schema-0.2-GPII-4014/verify.js http://localhost:25984 100 "clientCredential-nova1" "clientCredential-nova2"

"use strict";

Expand All @@ -42,10 +43,12 @@ require("../../../gpii/node_modules/gpii-db-operation/src/DbUtils.js");
gpii.migration.GPII4014.initOptions = function (processArgv) {
var options = {};
options.couchDbUrl = processArgv[2] + "/gpii";
options.novaClientCredentials = process.argv.splice(3);
options.maxDocsInBatchPerRequest = processArgv[3];
options.novaClientCredentials = process.argv.splice(4);
options.numOfVerified = 0;
options.numOfErrorDocs = 0;

// Set up database specific options
options.allDocsUrl = options.couchDbUrl + "/_all_docs?include_docs=true";
options.allDocs = [];
options.parsedCouchDbUrl = url.parse(options.couchDbUrl);
options.postOptions = {
Expand Down Expand Up @@ -75,7 +78,7 @@ gpii.migration.GPII4014.initOptions = function (processArgv) {
* @param {Array} options.allDocsUrl - The url for retrieving all documents in the database.
* @return {Promise} - A promise whose resolved value is the verification result.
*/
gpii.migration.GPII4014.verifyAllDocs = function (options) {
gpii.migration.GPII4014.verifyDocsInBatch = function (options) {
var details = {
requestUrl: options.allDocsUrl,
requestErrMsg: "Error retrieving documents from the database: ",
Expand All @@ -90,76 +93,101 @@ gpii.migration.GPII4014.verifyAllDocs = function (options) {
* @param {String} responseString - the response from the request to get all documents.
* @param {Object} options - Where to store the to-be-updated documents:
* @param {Array} options.novaClientCredentials - An array of NOVA client credential IDs.
* @param {Array} options.numOfVerified - The number of verified documents.
* @param {Array} options.numOfErrorDocs - The number of verified documents that have verification errors.
* @return {Promise} - A promise whose resolved value is the verification result.
*/
gpii.migration.GPII4014.verifyDocsData = function (responseString, options) {
var allDocs = JSON.parse(responseString);
var totalNumOfDocs = allDocs.total_rows;
var numOfVerified = 0;
var numOfErrorDocs = 0;
options.totalNumOfDocs = allDocs.total_rows;
var togo = fluid.promise();

fluid.each(allDocs.rows, function (aRow) {
var hasError = false;
var aDoc = aRow.doc;
// To filter out the "_design/views" doc that doesn't have the "schemaVersion" field
if (aDoc.schemaVersion) {
numOfVerified++;
if (aDoc.schemaVersion !== gpii.migration.GPII4014.newSchemaVersion) {
console.log("Error with the document _id \"" + aDoc._id + "\": schema version is ", aDoc.schemaVersion, ", not ", gpii.migration.GPII4014.newSchemaVersion);
hasError = true;
}
if (aDoc.timestampUpdated === null) {
console.log("Error with the document _id \"" + aDoc._id + "\": the value of timestampUpdated is empty");
hasError = true;
}
if (aDoc.type === "clientCredential") {
var docFields = {
allowedIPBlocks: aDoc.allowedIPBlocks,
allowedPrefsToWrite: aDoc.allowedPrefsToWrite,
isCreateGpiiKeyAllowed: aDoc.isCreateGpiiKeyAllowed,
isCreatePrefsSafeAllowed: aDoc.isCreatePrefsSafeAllowed
};
var diffResult = fluid.model.diff(
docFields,
options.novaClientCredentials.includes(aDoc._id) ? gpii.migration.GPII4014.newValuesForNovaClientCredential : gpii.migration.GPII4014.newValuesForPublicClientCredential
);
if (!diffResult) {
console.log("Error with the document _id \"" + aDoc._id + "\": new field values for client credential are incorrect - ", docFields);
if (allDocs.rows.length === 0) {
togo.reject({
errorCode: "GPII-NO-MORE-DOCS",
message: "No more documents to verify."
});
} else {
fluid.each(allDocs.rows, function (aRow) {
var hasError = false;
var aDoc = aRow.doc;
// To filter out the "_design/views" doc that doesn't have the "schemaVersion" field
if (aDoc.schemaVersion) {
if (aDoc.schemaVersion !== gpii.migration.GPII4014.newSchemaVersion) {
console.log("Error with the document _id \"" + aDoc._id + "\": schema version is ", aDoc.schemaVersion, ", not ", gpii.migration.GPII4014.newSchemaVersion);
hasError = true;
}
if (aDoc.timestampUpdated === null) {
console.log("Error with the document _id \"" + aDoc._id + "\": the value of timestampUpdated is empty");
hasError = true;
}
if (aDoc.type === "clientCredential") {
var docFields = {
allowedIPBlocks: aDoc.allowedIPBlocks,
allowedPrefsToWrite: aDoc.allowedPrefsToWrite,
isCreateGpiiKeyAllowed: aDoc.isCreateGpiiKeyAllowed,
isCreatePrefsSafeAllowed: aDoc.isCreatePrefsSafeAllowed
};
var diffResult = fluid.model.diff(
docFields,
options.novaClientCredentials.includes(aDoc._id) ? gpii.migration.GPII4014.newValuesForNovaClientCredential : gpii.migration.GPII4014.newValuesForPublicClientCredential
);
if (!diffResult) {
console.log("Error with the document _id \"" + aDoc._id + "\": new field values for client credential are incorrect - ", docFields);
hasError = true;
}
}
}
}
if (hasError) {
numOfErrorDocs++;
}
});
console.log("The database has " + totalNumOfDocs + " documents. " + numOfVerified + " documents are qualified for verifications.");
if (numOfErrorDocs > 0) {
togo.reject("Fail: " + numOfErrorDocs + " documents have errors.");
} else {
togo.resolve("All passed.");
if (hasError) {
options.numOfErrorDocs++;
}
});
options.numOfVerified = Number(options.numOfVerified) + Number(allDocs.rows.length);
console.log("Verified " + options.numOfVerified + " of " + options.totalNumOfDocs + " documents.");
togo.resolve();
}
return togo;
};

/**
* Create and execute the steps to verify documents.
* Verify recursively in small batches.
* @param {Object} options - Where to store information for verification as well as its progress:
* @param {Array} options.allDocsUrl - The url to query the database in small batches.
* @param {Array} options.novaClientCredentials - An array of NOVA client credential IDs.
* @param {Array} options.numOfVerified - The number of verified documents.
* @param {Array} options.numOfErrorDocs - The number of verified documents that have verification errors.
*/
gpii.migration.GPII4014.verify = function () {
var options = gpii.migration.GPII4014.initOptions(process.argv);
var verifyPromise = gpii.migration.GPII4014.verifyAllDocs(options);
gpii.migration.GPII4014.verifyRecursive = function (options) {
options.allDocsUrl = options.couchDbUrl + "/_all_docs?include_docs=true&descending=true&skip=" + options.numOfVerified + "&limit=" + options.maxDocsInBatchPerRequest;
var verifyPromise = gpii.migration.GPII4014.verifyDocsInBatch(options);

verifyPromise.then(
function (result) {
console.log("Done - " + result);
process.exit(0);
function () {
gpii.migration.GPII4014.verifyRecursive(options);
},
function (error) {
console.log(error);
process.exit(1);
if (error.errorCode === "GPII-NO-MORE-DOCS") {
if (options.numOfErrorDocs > 0) {
console.log("Fail: " + options.numOfErrorDocs + " documents have errors.");
} else {
console.log("All passed.");
}
console.log("Done: " + error.message + " Verified " + options.totalNumOfDocs + " documents in total.");
process.exit(0);
} else {
console.log(error);
process.exit(1);
}
}
);
};

/**
* Create and execute the steps to verify documents.
*/
gpii.migration.GPII4014.verify = function () {
var options = gpii.migration.GPII4014.initOptions(process.argv);
gpii.migration.GPII4014.verifyRecursive(options);
};

gpii.migration.GPII4014.verify();
2 changes: 1 addition & 1 deletion testData/dbData/views.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
"findAllGpiiKeys": {
"map": "function(doc) {if (doc.type === 'gpiiKey') { emit(doc._id, doc); }}"
},
"findExpiredAccessTokens": {
"findAccessTokenByExpires": {
"map": "function(doc) {if (doc.type === 'gpiiAppInstallationAuthorization') emit(Date.parse(doc.timestampExpires), doc); }"
},
"findDocsBySchemaVersion": {
Expand Down

0 comments on commit 908e9be

Please sign in to comment.