Update tests and testing instructions.

.github/PULL_REQUEST_TEMPLATE.md

@@ -27,4 +27,4 @@
 - [ ] The pull request description is suitable for a Changelog entry
 - [ ] The milestone is set correctly on the pull request
 - [ ] The appropriate labels have been added to the pull request
-- [ ] The following command produces no errors (replace your own token from https://xdmod.access-ci.org): `XDMOD_API_TOKEN=YOUR_TOKEN_HERE pytest -vvs -o log_cli=true tests/`
+- [ ] Running the automated tests (see `docs/developing.md`) produces no errors.

docs/developing.md

@@ -1,3 +1,27 @@
+# Testing the code
+1. Start up a virtual environment, e.g.:
+    ```
+    $ env_dir=~/xdmod-data-test-env                                            
+    $ python3 -m venv ${env_dir}                                                
+    $ source ${env_dir}/bin/activate
+    ```
+1. Install your local development copy of `xdmod-data` in editable mode:
+    ```
+    (env) $ python3 -m pip install --force-reinstall -e /path/to/your/xdmod-data
+    ```
+1. Install `pytest`:
+    ```
+    (env) $ python3 -m pip install --upgrade pytest
+    ```
+1. Create an empty file in your home directory at `~/.xdmod-data-token` and change the permissions to 600.
+1. With an [https://xdmod.access-ci.org](ACCESS XDMoD) account with "User" as the Top Role, create an API token if you do not already have one (sign in and click My Profile -> API Token).
+1. Add your token to the file `~/.xdmod-data-token`.
+1. Change directories to your local development copy of `xdmod-data`.
+1. Run the following command:
+    ```
+    (env) $ pytest -vvs -o log_cli=true tests/ 
+    ```
+
 # Developing a new version
 1. Make sure the version number is updated in `xdmod_data/__version__.py`.
 1. Create a Pull Request for the new version.

tests/integration/test_datawarehouse_integration.py

@@ -1,8 +1,12 @@
+import os
+from os.path import expanduser
+import pandas
+from pathlib import Path
 import pytest
 from xdmod_data.warehouse import DataWarehouse
-import pandas
 
-VALID_XDMOD_URL = 'https://xdmod-dev.ccr.xdmod.org:9001'
+VALID_XDMOD_URL = 'https://xdmod.access-ci.org'
+TOKEN_PATH = '~/.xdmod-data-token'
 INVALID_STR = 'asdlkfjsdlkfisdjkfjd'
 METHOD_PARAMS = {
     'get_data': (
@@ -99,6 +103,11 @@
             key_error_test_params += [(method, {'filters': value}, match)]
 
 
+with open(Path(expanduser(TOKEN_PATH)), 'r') as token_file:
+    token = token_file.read().replace('\n', '').strip()
+os.environ['XDMOD_API_TOKEN'] = token
+
+
 @pytest.fixture(scope='module')
 def dw_methods():
     with DataWarehouse(VALID_XDMOD_URL) as dw:

tests/regression/data/xdmod-dev-jobs-dimensions.csv → tests/regression/data/jobs-dimensions.csv

@@ -11,6 +11,8 @@ nsfdirectorate,NSF Directorate,The NSF directorate of the field of science indic
 nodecount,Node Count,A categorization of jobs into discrete groups based on node count.
 pi,PI,The principal investigator of a project.
 pi_institution,PI Institution,Organizations that have PIs with allocations.
+pi_institution_country,PI Institution Country,The country of the institution of the PI of the project associated with compute jobs.
+pi_institution_state,PI Institution State,The location of the institution of the PI of the project associated with the compute jobs.
 parentscience,Parent Science,The parent of the field of science indiciated on the allocation request pertaining to the running jobs.
 queue,Queue,Queue pertains to the low level job queues on each resource.
 resource,Resource,A resource is a remote computer that can run jobs.
@@ -19,4 +21,6 @@ provider,Service Provider,A service provider is an institution that hosts resour
 username,System Username,The specific system username of the users who ran jobs.
 person,User,"A person who is on a PIs allocation, hence able run jobs on resources."
 institution,User Institution,Organizations that have users with allocations.
+institution_country,User Institution Country,The name of the country of the institution of the person who ran the compute job.
+institution_state,User Institution State,The location of the institution of the person who ran the compute job.
 nsfstatus,User NSF Status,Categorization of the users who ran jobs.

tests/regression/data/xdmod-dev-jobs-fieldofscience-filter-values.csv → tests/regression/data/jobs-fieldofscience-filter-values.csv

@@ -76,4 +76,5 @@ id,label
 216,Statistics and Probability
 251,Systematics and Population Biology
 208,Training
+191,Veterinary Science
 222,Visualization and Human-Computer Systems

tests/regression/data/machine-learning-notebook-example-every-1000.csv

@@ -1,36 +1,44 @@
 ,Nodes,Requested Wall Time,Wait Time,Wall Time,CPU User,"Mount point ""home"" data written","Mount point ""scratch"" data written",Total memory used
-0,1,18000,2223,61,45.33680231265288,,,
-1000,1,18000,3725,287,32.42974787000051,,,49033524736
-2000,1,172800,49,2571,79.97259330857212,,,30924976176.761906
-3000,4,3600,42922,2532,96.81470878098752,45.018830174107066,176860091.8520887,252683009.7777778
-4000,1,43200,1489,4273,98.65262019923018,,,53486231639.14893
-5000,16,172800,380054,19,83.2019551884715,47.13914246437847,193884.53690558145,213282869
-6000,10,144000,112975,3986,94.96697493527975,619.5369574236191,497097.00057766255,761111556.7857141
-7000,1,21600,142222,6067,6.010297266432706,,,107759541905.19403
-8000,1,3600,1,240,0.2383380917742573,,,7726677333.333334
-9000,1,3600,1,195,0.3623863357343296,,,7948964864
-10000,1,780,2664,345,0.9211134064869447,0,20498764.587838568,3695674410
-11000,1,3600,6,966,38.232643491545396,,,40409398340.26666
-12000,1,3600,13,875,38.44329943700583,,,41062250723.55555
-13000,1,21600,154321,5853,6.006929680701932,,,117623635458.65286
-14000,1,3600,135,1445,69.2786738348236,,,146436012299.13046
-15000,1,28800,132001,250,59.259627771523135,,,39645850477.71429
-16000,1,7200,12,711,46.9863178239425,,,184524522837.3333
-17000,1,7200,10,837,47.48688897176907,,,175949883864.6154
-18000,1,172800,54302,3717,94.54949009872816,,,24592065888.524582
-19000,1,7200,150,1122,48.242235742621304,,,207471162595.55554
-20000,1,172800,53263,21002,61.33315672532931,,,45945555454.53298
-21000,1,172800,50221,27861,70.3833748019495,,,65631446785.93313
-22000,1,3600,1090,11,2.1611174966916664,0,1280262.9868873185,210875512
-23000,1,172800,49372,38029,81.86028948409559,,,28993637866.161156
-24000,1,172800,47292,46989,75.1580595471091,,,63021351985.75967
-25000,1,10800,168258,519,67.63795467248386,,,93487184281.60002
-26000,1,43200,91318,2377,1.9121205307623317,,,24738851002.181816
-27000,4,1800,114,434,78.82404727566929,1044.8869758885673,1181195931.4220536,311622745.5
-28000,1,480,530,40,2.640800690755364,0,676348399.5515662,299178751
-29000,1,28800,183935,42190,11.050664559030599,,,45653222140.898186
-30000,1,43200,141566,1090,1.8172013314352236,,,17360479729.371433
-31000,1,21600,257981,5779,5.9772409694676165,,,170223749152.16763
-32000,1,86400,5160,229,94.98545157786833,,,43320737792
-33000,4,14400,3,1788,67.71721566083241,2281.993725064994,1398054207.6041634,887018972
-34000,1,28800,250083,2206,99.43961023903431,,,234280343040.00003
+0,1,172800,11,506,,,,
+1000,1,86400,1,66,,,,
+2000,1,86400,18,752,,,,
+3000,1,86400,8,5434,,,,
+4000,1,86400,6,1572,,,,
+5000,1,172800,7,2592,,,,
+6000,1,14400,7,2800,,,,
+7000,1,3600,2894,1357,,,,
+8000,1,21600,116,7277,,,,
+9000,1,21600,2173,6764,,,,
+10000,1,21600,3574,7095,,,,
+11000,1,9000,4,3564,88.01518903173182,992.4354304606816,267087841.6178405,811231171.4285715
+12000,1,21600,158,5565,,,,
+13000,1,21600,59,6965,,,,
+14000,1,21600,9,7760,,,,
+15000,1,3600,22122,1335,,,,
+16000,1,28800,130,9421,12.262731018331898,19749.432156075072,0,787292327.46875
+17000,1,28800,6,1990,,,,
+18000,1,172800,13,73,,,,
+19000,1,172800,7,129,,,,
+20000,1,25200,4,25211,82.16279473845965,0,5113844.667942916,240912572.72941175
+21000,1,21600,18,6099,,,,
+22000,1,21600,27,7131,,,,
+23000,1,1800,61,1079,35.02319701051263,5818814744.200479,0,91742777.25
+24000,1,3600,5,2306,0.11814596015380158,,,33983854837.760006
+25000,1,960,1,59,2.025062333453586,0,0,118141168
+26000,1,172800,1,20494,87.54061396105656,548.1123048956289,0,224020798.15942028
+27000,4,7200,2,7214,99.2396948622311,441.0514348202534,34392345950.27519,1104895888.4
+28000,1,21600,13,55,1.2148444482641405,,,
+29000,1,21600,171,40,,,,
+30000,1,960,0,42,1.5504320217730077,0,0,112133180
+31000,1,1800,11,183,25.94758412119134,,,129784697856
+32000,1,21600,372,114,1.5571541609296509,,,92681043968
+33000,2,1800,134,139,55.875186246345784,533.6963000565588,754284385.8435647,137041136
+34000,1,7200,74,9,0.94096807333301,4681.666820975073,0,145688575
+35000,1,172800,22,83953,98.95217460976379,,,91787361316.73543
+36000,1,6000,8,152,0.4601673251104144,,,85277047466.66667
+37000,1,900,124,137,96.18834348033303,,,35703571797.333336
+38000,1,21600,12,56,24.892228849477622,,,
+39000,1,21600,12,134,26.487756894710913,,,113801609216
+40000,1,21600,12,229,45.74138522053433,,,42761861802.66667
+41000,1,21600,20,307,0.9428384414161763,,,27184930360.88889
+42000,1,21600,130,386,1.68608777466353,,,49510031732.36363

tests/regression/data/xdmod-dev-realms.csv → tests/regression/data/realms.csv

@@ -5,5 +5,4 @@ Cloud,Cloud
 Gateways,Gateways
 Jobs,Jobs
 Requests,Requests
-ResourceAllocations,Resource Allocations
 SUPREMM,SUPREMM

tests/regression/test_datawarehouse_regression.py

@@ -1,13 +1,21 @@
-import pytest
-from xdmod_data.warehouse import DataWarehouse
-import pandas
 import numpy
 import os
+from os.path import expanduser
+import pandas
+from pathlib import Path
+import pytest
+from xdmod_data.warehouse import DataWarehouse
 
-XDMOD_URL = 'https://xdmod-dev.ccr.xdmod.org:9001'
+XDMOD_URL = 'https://xdmod.access-ci.org'
+TOKEN_PATH = '~/.xdmod-data-token'
 DATA_DIR = os.path.dirname(__file__) + '/data'
 
 
+with open(Path(expanduser(TOKEN_PATH)), 'r') as token_file:
+    token = token_file.read().replace('\n', '').strip()
+os.environ['XDMOD_API_TOKEN'] = token
+
+
 @pytest.fixture(scope='module')
 def valid_dw():
     with DataWarehouse(XDMOD_URL) as dw:
@@ -33,7 +41,7 @@ def __assert_dfs_equal(
 
 def test_get_raw_data(valid_dw):
     data = valid_dw.get_raw_data(
-        duration=('2021-05-01', '2021-05-02'),
+        duration=('2023-05-01', '2023-05-02'),
         realm='SUPREMM',
         fields=(
             'CPU User',
@@ -67,27 +75,27 @@ def __assert_descriptor_dfs_equal(data_file, actual):
 
 def test_describe_realms(valid_dw):
     __assert_descriptor_dfs_equal(
-        'xdmod-dev-realms.csv',
+        'realms.csv',
         valid_dw.describe_realms(),
     )
 
 
 def test_describe_metrics(valid_dw):
     __assert_descriptor_dfs_equal(
-        'xdmod-dev-jobs-metrics.csv',
+        'jobs-metrics.csv',
         valid_dw.describe_metrics('Jobs'),
     )
 
 
 def test_describe_dimensions(valid_dw):
     __assert_descriptor_dfs_equal(
-        'xdmod-dev-jobs-dimensions.csv',
+        'jobs-dimensions.csv',
         valid_dw.describe_dimensions('Jobs'),
     )
 
 
 def test_get_filter_values(valid_dw):
     __assert_descriptor_dfs_equal(
-        'xdmod-dev-jobs-fieldofscience-filter-values.csv',
+        'jobs-fieldofscience-filter-values.csv',
         valid_dw.get_filter_values('Jobs', 'Field of Science'),
     )

tests/unit/test_datawarehouse_unit.py

@@ -43,11 +43,7 @@ def test___enter___RuntimeError_xdmod_host_malformed():
 
 def test___enter___RuntimeError_xdmod_host_unresolved():
     invalid_host = 'https://' + INVALID_STR + '.xdmod.org'
-    with pytest.raises(
-        requests.exceptions.ConnectionError,
-        match='Failed to resolve \'' + INVALID_STR + '.xdmod.org\''
-        + ' \\(\\[Errno -2\\] Name or service not known\\)'
-    ):
+    with pytest.raises(Exception):
         with DataWarehouse(invalid_host):
             pass