diff --git a/functions/index.js b/functions/index.js index 8b495ba..169e046 100644 --- a/functions/index.js +++ b/functions/index.js @@ -238,20 +238,45 @@ exports.generateExcelSheet = functions.https.onRequest(async (req, res) => { // read data from Firebase let data; + // TODO: clean up error catches, test error catches + const limitDocumentRef = db.collection('TODO').doc('TODO'); try { - let snapshot; - if (recordType === 'recorder') { - snapshot = await db.collection('route_records').where('startTime', '>', rangeStart).where('startTime', '<', rangeEnd).get(); - } else { // recordType === 'checkin' - snapshot = await db.collection('checkin_records').where('checkoutTime', '>', rangeStart).where('checkoutTime', '<', rangeEnd).get(); - } - data = snapshot.docs.map(doc => doc.data()); + await db.runTransaction(async (t) => { + const doc = await t.get(limitDocumentRef).data; + const { TODODAILYLIMIT, TODODAILYTIME } = doc.data(); + let newDailyTime; + let limit; + if (areSameDayPacific(etc, etc)) { + limit = 49000; + newDailyTime = etc; //beginning of today + } else { + limit = TODODAILYLIMIT; + newDailyTime = TODODAILYTIME; + } + const queryLimit = max(0, min(limit, 2000)); + try { + let snapshot; + if (recordType === 'recorder') { + snapshot = await db.collection('route_records').where('startTime', '>', rangeStart).where('startTime', '<', rangeEnd).limit(queryLimit).get(); + } else { // recordType === 'checkin' + snapshot = await db.collection('checkin_records').where('checkoutTime', '>', rangeStart).where('checkoutTime', '<', rangeEnd).limit(queryLimit).get(); + } + data = snapshot.docs.map(doc => doc.data()); + t.update(limitDocumentRef, { + TODODAILYLIMIT: limit - snapshot.size, + TODODAILYTIME: newDailyTime + }); + } catch (e) { + cors(req, res, () => { + res.sendStatus(500); + }); + return; + } + }) } catch (e) { - cors(req, res, () => { - res.sendStatus(500); - }); - return; + console.log('Transaction failure:', e); } + // handle no records if (data.length === 0) { diff --git a/src/app/modules/backend/services/implementations/firebase/checkin/firebase-checkin.service.ts b/src/app/modules/backend/services/implementations/firebase/checkin/firebase-checkin.service.ts index 30591ac..9a8b6cb 100644 --- a/src/app/modules/backend/services/implementations/firebase/checkin/firebase-checkin.service.ts +++ b/src/app/modules/backend/services/implementations/firebase/checkin/firebase-checkin.service.ts @@ -22,7 +22,11 @@ export class FirebaseCheckinService { } public getRecords(startDate: Date, endDate: Date): Observable { - const query = this.firestore.collection('checkin_records', ref => ref.where('checkoutTime', '>', startDate).where('checkoutTime', '<', endDate)); + const query = this.firestore.collection('checkin_records', ref => ref + .where('checkoutTime', '>', startDate) + .where('checkoutTime', '<', endDate) + .limit(2000) + ); return query.valueChanges({ idField: 'id' }).pipe( map((rawRecords: IFirestoreCheckinRecord[]): ICheckinRecord[] => { return rawRecords.map((rawRecord: IFirestoreCheckinRecord): ICheckinRecord => { diff --git a/src/app/modules/backend/services/implementations/firebase/routes/firebase-routes.service.ts b/src/app/modules/backend/services/implementations/firebase/routes/firebase-routes.service.ts index 63540db..3f15e06 100644 --- a/src/app/modules/backend/services/implementations/firebase/routes/firebase-routes.service.ts +++ b/src/app/modules/backend/services/implementations/firebase/routes/firebase-routes.service.ts @@ -22,7 +22,11 @@ export class FirebaseRoutesService { } public getRecords(startDate: Date, endDate: Date): Observable { - const query = this.firestore.collection('route_records', ref => ref.where('startTime', '>', startDate).where('startTime', '<', endDate)); + const query = this.firestore.collection('route_records', ref => ref + .where('startTime', '>', startDate) + .where('startTime', '<', endDate) + .limit(2000) + ); return query.valueChanges({ idField: 'id' }).pipe( map((rawRecords: IFirestoreRouteRecord[]): IRouteRecord[] => { return rawRecords.map((rawRecord: IFirestoreRouteRecord): IRouteRecord => { diff --git a/src/app/modules/checkin/pages/checkin-records/checkin-records.component.html b/src/app/modules/checkin/pages/checkin-records/checkin-records.component.html index 8fa07ad..7794db3 100644 --- a/src/app/modules/checkin/pages/checkin-records/checkin-records.component.html +++ b/src/app/modules/checkin/pages/checkin-records/checkin-records.component.html @@ -21,6 +21,14 @@
+
+

+ To save on costs, we allow for a maximum of 2,000 records to be viewed on the website at a time or downloaded at a time, + and we recommend a daily total of less than 50,000 records to be viewed on the website or downloaded. Both views and + downloads will contribute to this 50,000-record limit. If you're getting less records than you expect, try reducing your + date range. Reach out to DSC if you have concerns about large-scale data retrieval. +

+
diff --git a/src/app/modules/checkin/pages/checkin-records/checkin-records.component.scss b/src/app/modules/checkin/pages/checkin-records/checkin-records.component.scss index f712a03..7722b3d 100644 --- a/src/app/modules/checkin/pages/checkin-records/checkin-records.component.scss +++ b/src/app/modules/checkin/pages/checkin-records/checkin-records.component.scss @@ -3,6 +3,12 @@ font-size: 24px; } +#small-text { + padding-top: 20px; + font-size: 0.9rem; + color: rgb(100,100,100); +} + #checkin-records-wrapper { margin: 10px; padding-top: 20px; diff --git a/src/app/modules/routes/pages/route-records/route-records.component.html b/src/app/modules/routes/pages/route-records/route-records.component.html index b82ad4a..5929cc3 100644 --- a/src/app/modules/routes/pages/route-records/route-records.component.html +++ b/src/app/modules/routes/pages/route-records/route-records.component.html @@ -22,6 +22,14 @@ +
+

+ To save on costs, we allow for a maximum of 2,000 records to be viewed on the website at a time or downloaded at a time, + and we recommend a daily total of less than 50,000 records to be viewed on the website or downloaded. Both views and + downloads will contribute to this 50,000-record limit. If you're getting less records than you expect, try reducing your + date range. Reach out to DSC if you have concerns about large-scale data retrieval. +

+
diff --git a/src/app/modules/routes/pages/route-records/route-records.component.scss b/src/app/modules/routes/pages/route-records/route-records.component.scss index cb694f2..e5b6fb9 100644 --- a/src/app/modules/routes/pages/route-records/route-records.component.scss +++ b/src/app/modules/routes/pages/route-records/route-records.component.scss @@ -8,6 +8,12 @@ mat-card { font-size: 24px; } +#small-text { + padding-top: 20px; + font-size: 0.9rem; + color: rgb(100,100,100); +} + #route-records-wrapper { margin: 10px; padding-top: 20px;