Add limited report mode for large datasets

catalyst · Jan 3, 2025 · 848dced · 848dced
1 parent c823bbe
commit 848dced
Show file tree

Hide file tree

Showing 3 changed files with 68 additions and 0 deletions.
diff --git a/classes/helper.php b/classes/helper.php
@@ -171,4 +171,32 @@ public static function merge_duplicate_records(array $records): \stdClass {
 
         return $baserecord;
     }
+
+    /**
+     * Estimate row count for local_csp table
+     *
+     * @return int
+     */
+    public static function get_row_estimate(): int {
+        global $CFG, $DB;
+
+        if ($DB->get_dbfamily() === 'mysql') {
+            $sql = "SELECT table_rows AS row_count
+                      FROM information_schema.tables
+                     WHERE table_schema = DATABASE()
+                       AND table_type = 'BASE TABLE'
+                       AND table_name = :tablename";
+        } else if ($DB->get_dbfamily() === 'postgres') {
+            $sql = "SELECT relname AS table_name, reltuples::BIGINT AS row_count
+                      FROM pg_class
+                     WHERE relname = :tablename";
+        } else {
+            // Estimates not currently supported for other databases, use a full count.
+            return $DB->count_records('local_csp');
+        }
+
+        $params = ['tablename' => $CFG->prefix . 'local_csp'];
+        $record = $DB->get_record_sql($sql, $params);
+        return $record->row_count ?? 0;
+    }
 }
diff --git a/csp_report.php b/csp_report.php
@@ -113,6 +113,10 @@
     $action,
 ));
 
+// The local_csp table can grow very large if a policy is not set up correctly.
+// This report can be slow even with indexes, so this can be used to load a limited version.
+$limitedreport = \local_csp\helper::get_row_estimate() > 5000000;
+
 // If user has clicked on a violation to view all violation entries.
 if ($viewviolation) {
     $fields = 'id, sha1hash, blockeduri, blockeddomain, violateddirective, failcounter, timeupdated, documenturi, violationhash';
@@ -138,6 +142,37 @@
         $action,
     ));
 
+    // Disable sorting on large tables - sorting usually requires the full table.
+    if ($limitedreport) {
+        $table->sortable(false);
+    }
+
+} else if ($limitedreport) {
+    // If a table is too large only load the fields that can be loaded using hashes and no aggregation.
+    $table->define_columns([
+        'failcounter',
+        'blockeddomain',
+        'violateddirective',
+        'timecreated',
+        'action',
+    ]);
+    $table->define_headers([
+        $failcounter,
+        $blockeddomain,
+        $violateddirective,
+        $timeupdated,
+        $action,
+    ]);
+
+    // Row count is large so it can be used a rough indicator instead of failcounter.
+    $fields = 'id, violationcount as failcounter, blockeddomain, violateddirective, violationhash, timecreated';
+    $from = "(SELECT MAX(id) as maxid, count(*) as violationcount
+                FROM {local_csp}
+            GROUP BY violationhash) AS A
+                JOIN {local_csp} ON id = maxid";
+    $where = '1 = 1';
+    $params = [];
+
 } else {
     $fields = 'id, blockeddomain, violateddirective, violationhash, failcounter, timecreated';
     // Select the first blockedURI of a type, and collapse the rest while summing failcounter.
@@ -157,6 +192,10 @@
     echo $OUTPUT->header();
     echo $OUTPUT->heading($title);
 
+    if ($limitedreport) {
+        echo $OUTPUT->notification(get_string('limitedreport', 'local_csp'), 'warning');
+    }
+
     $action = new \confirm_action(get_string('areyousuretodeleteallrecords', 'local_csp'));
     $urlresetallcspstatistics = new moodle_url($PAGE->url, array(
         'resetallcspstatistics' => 1,

diff --git a/lang/en/local_csp.php b/lang/en/local_csp.php
@@ -56,6 +56,7 @@
 $string['failcounter'] = '#';
 $string['highestviolaters'] = 'Top Violation Sources';
 $string['invalidblockeduri'] = 'Invalid Blocked URI: {$a}';
+$string['limitedreport'] = 'The report columns and sorting options have been limited due to a large amount of data. This is often an indication that the policy is not set up correctly, consider updating the policy and resetting the statistics.';
 $string['loaddata'] = 'Load data';
 $string['loadexternaljavascript'] = 'Load external javascript from {$a}';
 $string['loadingmixedcontentdescription'] = 'When accessing moodle website via HTTPS browser prohibits displaying of the below resources because they origin from HTTP.<br />You should be able to see it in your browser\'s Javascript console.';