Merge pull request #9 from OpenSourceBrain/development

To v1.0.3; improved testing on windows

.github/workflows/ci.yml

@@ -32,30 +32,55 @@ jobs:
       run: |
         pip install .
 
-    - name: Set up config
+    - name: Set up config non windows
       env:
         NSGR_USERNAME: ${{ secrets.NSGR_USERNAME }}
         NSGR_PASSWORD: ${{ secrets.NSGR_PASSWORD }}
         NSGR_APPID: ${{ secrets.NSGR_APPID }}
+      if: ${{ matrix.runs-on != 'windows-latest'  }}
       run: |
-        echo "APPNAME=PY_EXPANSE" > ~/nsgrest.conf
-        echo "URL=https://nsgr.sdsc.edu:8443/cipresrest/v1" >> ~/nsgrest.conf
+        echo "APPNAME=PY_EXPANSE" > nsgrest.conf
+        echo "URL=https://nsgr.sdsc.edu:8443/cipresrest/v1" >> nsgrest.conf
 
-        more ~/nsgrest.conf # print contents so far...
+        echo "USERNAME=${NSGR_USERNAME}" >> nsgrest.conf
+        echo "PASSWORD=${NSGR_PASSWORD}" >> nsgrest.conf
+        echo "APPID=${NSGR_APPID}" >> nsgrest.conf
 
-        echo "USERNAME=${NSGR_USERNAME}" >> ~/nsgrest.conf
-        echo "PASSWORD=${NSGR_PASSWORD}" >> ~/nsgrest.conf
-        echo "APPID=${NSGR_APPID}" >> ~/nsgrest.conf
+        more nsgrest.conf # print contents so far...
+
+    - name: Set up config windows
+      env:
+        NSGR_USERNAME: ${{ secrets.NSGR_USERNAME }}
+        NSGR_PASSWORD: ${{ secrets.NSGR_PASSWORD }}
+        NSGR_APPID: ${{ secrets.NSGR_APPID }}
+      if: ${{ matrix.runs-on == 'windows-latest'  }}
+      run: |
+        echo "APPNAME=PY_EXPANSE" > nsgrest.conf
+        echo "URL=https://nsgr.sdsc.edu:8443/cipresrest/v1" >> nsgrest.conf
+
+        $outtext_username = "USERNAME=$($env:NSGR_USERNAME)"
+        $outtext_username | Out-File -FilePath "nsgrest.conf" -Append
+        $outtext_password = "PASSWORD=$($env:NSGR_PASSWORD)"
+        $outtext_password | Out-File -FilePath "nsgrest.conf" -Append
+        $outtext_appid = "APPID=$($env:NSGR_APPID)"
+        $outtext_appid | Out-File -FilePath "nsgrest.conf" -Append
+
+        more nsgrest.conf # print contents so far...
 
     - name: Version info
       run: |
         pip list
 
     - name: Test listing
       run: |
-        nsgr_job -l
+        nsgr_job -c nsgrest.conf -l
+
+    - name: Test submission non windows
+      if: ${{ matrix.runs-on != 'windows-latest'  }}
+      run: |
+        cd example && nsgr_submit -c ../nsgrest.conf . validate && nsgr_submit -c ../nsgrest.conf . run && cd NGBW* && tar -xvf output.tar.gz && echo "" && echo "" && echo "Output file contents:" && echo "" && cd example_dir && cat output.txt
 
-    - name: Test submission        
-      if: ${{ matrix.runs-on != 'windows-latest' }}
+    - name: Test submission windows
+      if: ${{ matrix.runs-on == 'windows-latest'  }}
       run: |
-        cd example/ && nsgr_submit . validate && nsgr_submit . run && cd NGBW* && tar -xvf output.tar.gz && echo ; echo ; echo "Output file contents:" ; echo && cat example_dir/output.txt
+        cd example && nsgr_submit -c ..\nsgrest.conf . validate && nsgr_submit -c ..\nsgrest.conf . run && cd NGBW* && tar -xvf output.tar.gz && echo "" && echo "" &&  echo "Output file contents:" && echo "" && cd example_dir && cat output.txt

.gitignore

@@ -5,3 +5,5 @@
 /dist
 /release.*
 example/NGBW*
+/example/output
+/example/makezip.sh

example/example_dir/example.py

@@ -10,9 +10,13 @@
 import sys
 import pkg_resources
 
+print("Running example script to list Python packages...")
+
 with open("output.txt", "w") as f:
-    print(f"Python version: {sys.version}", file=f)
-    print("Installed packages:", file=f)
+    print(f"Python version is: {sys.version}", file=f)
+    print("Installed packages on NSG:", file=f)
     dists = [str(d).replace(" ", "==") for d in pkg_resources.working_set]
     for i in dists:
         print(i, file=f)
+
+print("Done!")

example/testParam.properties

@@ -1,4 +1,4 @@
-toolId = PY_EXPANSE
+toolId=NEURON_EXPANSE
 number_cores_=1
 number_nodes_=1
 tasks_per_node_=1

pynsgr/client.py

@@ -453,7 +453,7 @@ def __init__(self):
             raise
     """
 
-    def __init__(self):
+    def __init__(self, conf_filepath=None):
         found = False
         OSB2_USER_DIR = (
             "/opt/user"  # OSBv2 user's directory shared across all their workspaces
@@ -465,22 +465,41 @@ def __init__(self):
                 self.props.load(infile)
 
             found = True
-        except IOError as e:
+        except IOError:
+            pass
+
+        if conf_filepath is not None:
+            try:
+                with open(conf_filepath) as infile:
+                    self.props.load(infile)
+                found = True
+            except IOError:
+                print(f"Could not find specified configuration file at {conf_filepath}, trying default locations.")
+                pass
+
+        # look for ~/.CONF_FILENAME
+        confFile = os.path.join(os.path.expanduser("~"), f".{CONF_FILENAME}")
+        try:
+            with open(confFile) as infile:
+                self.props.load(infile)
+            found = True
+        except IOError:
             pass
 
+        # look for ~/CONF_FILENAME (no dot)
         confFile = os.path.join(os.path.expanduser("~"), CONF_FILENAME)
         try:
             with open(confFile) as infile:
                 self.props.load(infile)
             found = True
-        except IOError as e:
+        except IOError:
             pass
 
         requiredProperties = set(["APPNAME", "APPID", "USERNAME", "PASSWORD", "URL"])
         if not found:
             raise Exception(
-                "Didn't find the file: %s (which should contain properties %s) in user's the home directory (or %s on Open Source Brain v2)."
-                % (CONF_FILENAME, requiredProperties, OSB2_USER_DIR)
+                "Didn't find the file: ~/.%s or ~/%s (which should contain properties %s) or %s/%s on Open Source Brain v2."
+                % (CONF_FILENAME, CONF_FILENAME, requiredProperties, OSB2_USER_DIR, CONF_FILENAME)
             )
 
         if not requiredProperties.issubset(self.props.propertyNames()):

pynsgr/commands/nsgr_job.py

@@ -22,15 +22,23 @@ def nsgr_job(argv):
 
     -h
         help
+    -c
+        path to config file
+
+        By default, it looks in the users's home directory (~) for .nsgrest.conf, then for nsgrest.conf (without a dot)
+
     -l
         list all the user's jobs
+
     -j JOBHANDLE
         choose a job to act upon.  If no other action (like download) is selected, shows the job's status.
+
     -d results_directory
         download job results to specified directory.  Directory (but not intermediate directories) will
         be created if it doesn't exist.
 
         Use with -j to specify the job.
+
     -v
         verbose (use with -l to get a verbose job listing)
 
@@ -48,18 +56,21 @@ def nsgr_job(argv):
         nsgr_job -j JOBHANDLE -r
             cancel and remove the specified job.
     """
+    conf_filepath = None
     jobHandle = None
     verbose = False
     action = "status"
     resultsdir = None
     try:
-        options, remainder = getopt.getopt(argv[1:], "j:hld:vr")
+        options, remainder = getopt.getopt(argv[1:], "j:hld:vrc:")
     except getopt.GetoptError as ge:
         print(ge)
         return 1
     for opt, arg in options:
         if opt in ("-j"):
             jobHandle = arg
+        elif opt in ("-c"):
+            conf_filepath = arg
         elif opt in ("-h"):
             print((nsgr_job.__doc__))
             return 0
@@ -73,7 +84,7 @@ def nsgr_job(argv):
         elif opt in ("-v"):
             verbose = True
 
-    properties = CipresClient.Application().getProperties()
+    properties = CipresClient.Application(conf_filepath).getProperties()
     client = CipresClient.Client(
         properties.APPNAME,
         properties.APPID,
@@ -143,12 +154,13 @@ def nsgr_job(argv):
     except ET.ParseError as pe:
         print("Unexpected response cannot be parsed.  Parsing error message: %s" % (pe))
         return 2
+    return 0
 
 
 # required because console scripts cannot take argument lists
 def main():
     """Main runner"""
-    nsgr_job(sys.argv)
+    sys.exit(nsgr_job(sys.argv))
 
 
 if __name__ == "__main__":

pynsgr/commands/nsgr_submit.py

@@ -20,6 +20,11 @@ def nsgr_submit(argv):
     Where TEMPLATE_DIRECTORY is the name of a directory that contains the job's input data files and
     two property files named testInput.properties and testParam.properties.
 
+    -c
+        path to config file
+
+        By default, it looks in the users's home directory (~) for .nsgrest.conf, then for nsgrest.conf (without a dot)
+
     validate
         Ask's the REST API whether the job is valid.  If valid, prints to stdout, the command line that
         would be run on the execution host if the job were submitted to run.  If invalid, prints an
@@ -39,6 +44,16 @@ def nsgr_submit(argv):
     if not argv or len(argv) < 3:
         print(nsgr_submit.__doc__)
         return 1
+
+    # get config file path and remove from list
+    try:
+        c_path = argv.index("-c")
+        argv.pop(c_path)
+        conf_filepath = argv[c_path]
+        argv.pop(c_path)
+    except ValueError:
+        conf_filepath = None
+
     template = argv[1]
     action = argv[2]
     if not os.path.isdir(template):
@@ -53,7 +68,7 @@ def nsgr_submit(argv):
     if len(argv) > 3:
         resultsdir = argv[3]
 
-    properties = CipresClient.Application().getProperties()
+    properties = CipresClient.Application(conf_filepath).getProperties()
     client = CipresClient.Client(
         properties.APPNAME,
         properties.APPID,
@@ -106,7 +121,7 @@ def nsgr_submit(argv):
 # required because console scripts cannot take argument lists
 def main():
     """Main runner"""
-    nsgr_submit(sys.argv)
+    sys.exit(nsgr_submit(sys.argv))
 
 
 if __name__ == "__main__":

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = pynsgr
-version = 1.0.2
+version = 1.0.3
 author = "Terry Schwartz, Padraig Gleeson"
 author_email = [email protected]
 url = https://github.com/OpenSourceBrain/pynsgr