Record retrieval limits

functions/index.js

@@ -238,20 +238,45 @@ exports.generateExcelSheet = functions.https.onRequest(async (req, res) => {
 
   // read data from Firebase
   let data;
+  // TODO: clean up error catches, test error catches
+  const limitDocumentRef = db.collection('TODO').doc('TODO');
   try {
-    let snapshot;
-    if (recordType === 'recorder') {
-      snapshot = await db.collection('route_records').where('startTime', '>', rangeStart).where('startTime', '<', rangeEnd).get();
-    } else { // recordType === 'checkin'
-      snapshot = await db.collection('checkin_records').where('checkoutTime', '>', rangeStart).where('checkoutTime', '<', rangeEnd).get();
-    }
-    data = snapshot.docs.map(doc => doc.data());
+    await db.runTransaction(async (t) => {
+      const doc = await t.get(limitDocumentRef).data;
+      const { TODODAILYLIMIT, TODODAILYTIME } = doc.data();
+      let newDailyTime;
+      let limit;
+      if (areSameDayPacific(etc, etc)) {
+        limit = 49000;
+        newDailyTime = etc; //beginning of today
+      } else {
+        limit = TODODAILYLIMIT;
+        newDailyTime = TODODAILYTIME;
+      }
+      const queryLimit = max(0, min(limit, 2000));
+      try {
+        let snapshot;
+        if (recordType === 'recorder') {
+          snapshot = await db.collection('route_records').where('startTime', '>', rangeStart).where('startTime', '<', rangeEnd).limit(queryLimit).get();
+        } else { // recordType === 'checkin'
+          snapshot = await db.collection('checkin_records').where('checkoutTime', '>', rangeStart).where('checkoutTime', '<', rangeEnd).limit(queryLimit).get();
+        }
+        data = snapshot.docs.map(doc => doc.data());
+        t.update(limitDocumentRef, {
+          TODODAILYLIMIT: limit - snapshot.size,
+          TODODAILYTIME: newDailyTime
+        });
+      } catch (e) {
+        cors(req, res, () => {
+          res.sendStatus(500);
+        });
+        return;
+      }
+    })
   } catch (e) {
-    cors(req, res, () => {
-      res.sendStatus(500);
-    });
-    return;
+    console.log('Transaction failure:', e);
   }
+
 
   // handle no records
   if (data.length === 0) {

src/app/modules/backend/services/implementations/firebase/checkin/firebase-checkin.service.ts

@@ -22,7 +22,11 @@ export class FirebaseCheckinService {
   }
 
   public getRecords(startDate: Date, endDate: Date): Observable<ICheckinRecord[]> {
-    const query = this.firestore.collection<IFirestoreCheckinRecord>('checkin_records', ref => ref.where('checkoutTime', '>', startDate).where('checkoutTime', '<', endDate));
+    const query = this.firestore.collection<IFirestoreCheckinRecord>('checkin_records', ref => ref
+      .where('checkoutTime', '>', startDate)
+      .where('checkoutTime', '<', endDate)
+      .limit(2000)
+    );
     return query.valueChanges({ idField: 'id' }).pipe(
       map((rawRecords: IFirestoreCheckinRecord[]): ICheckinRecord[] => {
         return rawRecords.map((rawRecord: IFirestoreCheckinRecord): ICheckinRecord => {

src/app/modules/backend/services/implementations/firebase/routes/firebase-routes.service.ts

@@ -22,7 +22,11 @@ export class FirebaseRoutesService {
   }
 
   public getRecords(startDate: Date, endDate: Date): Observable<IRouteRecord[]> {
-    const query = this.firestore.collection<IFirestoreRouteRecord>('route_records', ref => ref.where('startTime', '>', startDate).where('startTime', '<', endDate));
+    const query = this.firestore.collection<IFirestoreRouteRecord>('route_records', ref => ref
+      .where('startTime', '>', startDate)
+      .where('startTime', '<', endDate)
+      .limit(2000)
+    );
     return query.valueChanges({ idField: 'id' }).pipe(
       map((rawRecords: IFirestoreRouteRecord[]): IRouteRecord[] => {
         return rawRecords.map((rawRecord: IFirestoreRouteRecord): IRouteRecord => {

src/app/modules/checkin/pages/checkin-records/checkin-records.component.html

@@ -21,6 +21,14 @@
     <div>
       <button mat-raised-button class="select-screen-button" color="primary" (click)="handleDownload()" [disabled]="disableButton">Download</button>
     </div>
+    <div>
+      <p id='small-text'>
+        To save on costs, we allow for a maximum of 2,000 records to be viewed on the website at a time or downloaded at a time,
+        and we recommend a daily total of less than 50,000 records to be viewed on the website or downloaded. Both views and
+        downloads will contribute to this 50,000-record limit. If you're getting less records than you expect, try reducing your
+        date range. Reach out to DSC if you have concerns about large-scale data retrieval.
+      </p>
+    </div>
   </div>
 </div>
 <!-- END date range selection screen -->

src/app/modules/checkin/pages/checkin-records/checkin-records.component.scss

@@ -3,6 +3,12 @@
   font-size: 24px;
 }
 
+#small-text {
+  padding-top: 20px;
+  font-size: 0.9rem;
+  color: rgb(100,100,100);
+}
+
 #checkin-records-wrapper {
   margin: 10px;
   padding-top: 20px;

src/app/modules/routes/pages/route-records/route-records.component.html

@@ -22,6 +22,14 @@
       <button mat-raised-button class="select-screen-button" color="primary" (click)="handleDownload()"
         [disabled]="disableButton">Download</button>
     </div>
+    <div>
+      <p id='small-text'>
+        To save on costs, we allow for a maximum of 2,000 records to be viewed on the website at a time or downloaded at a time,
+        and we recommend a daily total of less than 50,000 records to be viewed on the website or downloaded. Both views and
+        downloads will contribute to this 50,000-record limit. If you're getting less records than you expect, try reducing your
+        date range. Reach out to DSC if you have concerns about large-scale data retrieval.
+      </p>
+    </div>
   </div>
 </div>
 <!-- END date range selection screen -->

src/app/modules/routes/pages/route-records/route-records.component.scss

@@ -8,6 +8,12 @@ mat-card {
   font-size: 24px;
 }
 
+#small-text {
+  padding-top: 20px;
+  font-size: 0.9rem;
+  color: rgb(100,100,100);
+}
+
 #route-records-wrapper {
   margin: 10px;
   padding-top: 20px;