Merge pull request #21 from aaronweeden/fix-filters-10000

Remove limit on number of results returned from `get_filter_values()`.

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Main development branch
 - Update tests and testing instructions ([\#14](https://github.com/ubccr/xdmod-data/pull/14)).
+- Remove limit on number of results returned from `get_filter_values()` ([\#21](https://github.com/ubccr/xdmod-data/pull/21)).
 
 ## v1.0.0 (2023-07-21)
 - Initial release.

tests/regression/test_datawarehouse_regression.py

@@ -97,3 +97,15 @@ def test_get_filter_values(valid_dw):
         'jobs-fieldofscience-filter-values.csv',
         valid_dw.get_filter_values('Jobs', 'Field of Science'),
     )
+
+
+def test_get_data_filter_user(valid_dw):
+    # Make sure the filter validation works for a user whose list position is
+    # greater than 10000 — this will raise an exception if it doesn't work.
+    valid_dw.get_data(
+        duration=('2023-01-01', '2023-01-01'),
+        realm='Jobs',
+        metric='CPU Hours: Total',
+        dataset_type='aggregate',
+        filters={'User': '10332'},
+    )

xdmod_data/_http_requester.py

@@ -62,6 +62,27 @@ def _request_raw_data(self, params):
             print(progress_msg + 'DONE')
         return (data, response['fields'])
 
+    def _request_filter_values(self, realm_id, dimension_id):
+        limit = 10000
+        data = []
+        num_rows = limit
+        offset = 0
+        while num_rows == limit:
+            response = self._request_json(
+                path='/controllers/metric_explorer.php',
+                post_fields={
+                    'operation': 'get_dimension',
+                    'realm': realm_id,
+                    'dimension_id': dimension_id,
+                    'start': offset,
+                    'limit': limit,
+                },
+            )
+            data += response['data']
+            num_rows = len(response['data'])
+            offset += limit
+        return data
+
     def _request_json(self, path, post_fields=None):
         response = self.__request(path, post_fields)
         return json.loads(response)

xdmod_data/warehouse.py

@@ -315,15 +315,11 @@ def get_filter_values(self, realm, dimension):
             realm_id,
             dimension,
         )
-        path = '/controllers/metric_explorer.php'
-        post_fields = {
-            'operation': 'get_dimension',
-            'dimension_id': dimension_id,
-            'realm': realm_id,
-            'limit': 10000,
-        }
-        response = self.__http_requester._request_json(path, post_fields)
-        data = [(datum['id'], datum['name']) for datum in response['data']]
+        response_data = self.__http_requester._request_filter_values(
+            realm_id,
+            dimension_id,
+        )
+        data = [(datum['id'], datum['name']) for datum in response_data]
         result = self.__get_data_frame(data, ('id', 'label'), 'id')
         return result