diff --git a/CHANGELOG.md b/CHANGELOG.md index 85942c07..5924bb9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Main development branch - Update tests and testing instructions ([\#14](https://github.com/ubccr/xdmod-data/pull/14)). +- Remove limit on number of results returned from `get_filter_values()` ([\#21](https://github.com/ubccr/xdmod-data/pull/21)). ## v1.0.0 (2023-07-21) - Initial release. diff --git a/tests/regression/test_datawarehouse_regression.py b/tests/regression/test_datawarehouse_regression.py index 3c93c08c..437f5821 100644 --- a/tests/regression/test_datawarehouse_regression.py +++ b/tests/regression/test_datawarehouse_regression.py @@ -97,3 +97,15 @@ def test_get_filter_values(valid_dw): 'jobs-fieldofscience-filter-values.csv', valid_dw.get_filter_values('Jobs', 'Field of Science'), ) + + +def test_get_data_filter_user(valid_dw): + # Make sure the filter validation works for a user whose list position is + # greater than 10000 — this will raise an exception if it doesn't work. + valid_dw.get_data( + duration=('2023-01-01', '2023-01-01'), + realm='Jobs', + metric='CPU Hours: Total', + dataset_type='aggregate', + filters={'User': '10332'}, + ) diff --git a/xdmod_data/_http_requester.py b/xdmod_data/_http_requester.py index 5b9009ad..d90e3af7 100644 --- a/xdmod_data/_http_requester.py +++ b/xdmod_data/_http_requester.py @@ -62,6 +62,27 @@ def _request_raw_data(self, params): print(progress_msg + 'DONE') return (data, response['fields']) + def _request_filter_values(self, realm_id, dimension_id): + limit = 10000 + data = [] + num_rows = limit + offset = 0 + while num_rows == limit: + response = self._request_json( + path='/controllers/metric_explorer.php', + post_fields={ + 'operation': 'get_dimension', + 'realm': realm_id, + 'dimension_id': dimension_id, + 'start': offset, + 'limit': limit, + }, + ) + data += response['data'] + num_rows = len(response['data']) + offset += limit + return data + def _request_json(self, path, post_fields=None): response = self.__request(path, post_fields) return json.loads(response) diff --git a/xdmod_data/warehouse.py b/xdmod_data/warehouse.py index 17cddef6..c1c56945 100644 --- a/xdmod_data/warehouse.py +++ b/xdmod_data/warehouse.py @@ -315,15 +315,11 @@ def get_filter_values(self, realm, dimension): realm_id, dimension, ) - path = '/controllers/metric_explorer.php' - post_fields = { - 'operation': 'get_dimension', - 'dimension_id': dimension_id, - 'realm': realm_id, - 'limit': 10000, - } - response = self.__http_requester._request_json(path, post_fields) - data = [(datum['id'], datum['name']) for datum in response['data']] + response_data = self.__http_requester._request_filter_values( + realm_id, + dimension_id, + ) + data = [(datum['id'], datum['name']) for datum in response_data] result = self.__get_data_frame(data, ('id', 'label'), 'id') return result