multi-path resources

.github/workflows/python-publish.yml

@@ -77,11 +77,11 @@ jobs:
         asset_path: ./dist/${{ steps.get_release_info.outputs.file_name }}
         asset_name: ${{ steps.get_release_info.outputs.file_name }}
         asset_content_type: application/gzip
-    - name: Publish pypi source package
-      uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
+    - name: Store the source package
+      uses: actions/upload-artifact@v2
       with:
-        user: __token__
-        password: ${{ secrets.PYPI_BLUESKY }}
+        name: python-package-distributions
+        path: dist
 
   buildbin:
     name: Build binary packages
@@ -134,7 +134,14 @@ jobs:
         asset_path: ./dist/${{ steps.get_release_info.outputs.file_name }}
         asset_name: ${{ steps.get_release_info.outputs.file_name }}
         asset_content_type: application/zip
-    - name: Publish pypi binary package for ${{matrix.TARGET}}
+    - name: Store the binary wheel
+      uses: actions/upload-artifact@v2
+      with:
+        name: python-package-distributions
+        path: dist
+
+
+    - name: Publish pypi packages
       uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
       with:
         user: __token__

.gitignore

@@ -1,11 +1,10 @@
 output
 scenario/ic.scn
-scenario/Nino/*
-data/cache
-data/grib
-data/netcdf
-data/performance/BADA/*
-!/data/performance/BADA/README.md
+cache
+bluesky/resources/grib
+bluesky/resources/netcdf
+bluesky/resources/performance/BADA/*
+!bluesky/resources/performance/BADA/README.md
 
 # Sphinx documentation
 docs/_build/
@@ -85,14 +84,9 @@ docs/_build/
 # PyBuilder
 target/
 
-data/cache
-data/performance/BADA/*
-!/data/performance/BADA/README
 ssd.obj
 
 # IPython
 *-checkpoint.py
 *-checkpoint.ipynb
 .ropeproject/config.py
-/scenario/andu.scn
-plugins/invdKDtree.py

BlueSky.py

@@ -1,8 +1,5 @@
 #!/usr/bin/env python
 """ Main BlueSky start script """
-
-
-
 import sys
 from bluesky.__main__ import main
 

MANIFEST.in

@@ -10,7 +10,3 @@ include BlueSky*.py
 include check.py
 include requirements.txt
 include Makefile
-
-# reduce package size for PyPI
-exclude bluesky/resources/data/graphics/world.8192x4096.dds
-exclude bluesky/resources/data/graphics/world.16384x8192.dds

bluesky/__init__.py

@@ -1,6 +1,7 @@
 ''' BlueSky: The open-source ATM simulator.'''
 from bluesky import settings
 from bluesky.core import Signal
+from bluesky.resourcepath import resource
 from bluesky import stack
 from bluesky import tools
 
@@ -28,7 +29,7 @@
 
 
 def init(mode='sim', configfile=None, scenfile=None, discoverable=False,
-         gui=None, detached=False, updatedirs=False, **kwargs):
+         gui=None, detached=False, workdir=None, **kwargs):
     ''' Initialize bluesky modules.
 
         Arguments:
@@ -39,6 +40,7 @@ def init(mode='sim', configfile=None, scenfile=None, discoverable=False,
           when this process is running a server) [True/False]
         - gui: Gui type (only when mode is client or server) [qtgl/pygame/console]
         - detached: Run with or without networking (only when mode is sim) [True/False]
+        - workdir: Pass a custom working directory (instead of cwd or ~/bluesky)
     '''
 
     # Argument checking
@@ -59,8 +61,12 @@ def init(mode='sim', configfile=None, scenfile=None, discoverable=False,
     globals()['mode'] = mode
     globals()['gui'] = gui
 
-    # Initialize global settings first, possibly loading a custom config file
-    settings.init(configfile, updatedirs)
+    # Initialise resource localisation, and set custom working directory if present
+    from bluesky import resourcepath
+    resourcepath.init(workdir)
+
+    # Initialize global settings, possibly loading a custom config file
+    settings.init(configfile)
 
     # Initialise tools
     tools.init()

bluesky/cmdargs.py

@@ -46,8 +46,8 @@ def __call__(self, parser, namespace, values=None, option_string=''):
     parser.add_argument("--discoverable", dest="discoverable", action="store_const", const=True,
                         default=False, help="Make simulation server discoverable. (Default in headless mode).")
 
-    parser.add_argument("--updatedirs", dest="updatedirs", action="store_const", const=True,
-                        default=False, help="Update files and folders in the local bluesky folder.")
+    parser.add_argument("--workdir", dest="workdir",
+                        help="Set BlueSky working directory (if other than cwd or ~/bluesky).")
 
     cmdargs = parser.parse_args()
 

bluesky/core/plugin.py

@@ -6,6 +6,7 @@
 import importlib
 import bluesky as bs
 from bluesky import settings
+from bluesky.resourcepath import ResourcePath
 from bluesky.core import timed_function, varexplorer as ve
 from bluesky import stack
 
@@ -100,7 +101,7 @@ def load(cls, name):
     @classmethod
     def find_plugins(cls, reqtype):
         ''' Create plugin wrapper objects based on source code of potential plug-in files. '''
-        for fname in settings.resolve_path(settings.plugin_path).glob('**/*.py'):
+        for fname in bs.resource(settings.plugin_path).glob('**/*.py'):
             with open(fname, 'rb') as f:
                 source = f.read()
                 try:
@@ -160,7 +161,12 @@ def find_plugins(cls, reqtype):
 def init(mode):
     ''' Initialization function of the plugin system.'''
     # Add plugin path to module search path
-    sys.path.append(settings.resolve_path(settings.plugin_path).absolute().as_posix())
+    plugin_path = bs.resource(settings.plugin_path)
+    if isinstance(plugin_path, ResourcePath):
+        for path in plugin_path.bases():
+            sys.path.append(path.absolute().as_posix())
+    else:
+        sys.path.append(plugin_path.absolute().as_posix())
     # Set plugin type for this instance of BlueSky
     req_type = 'sim' if mode[:3] == 'sim' else 'gui'
     oth_type = 'gui' if mode[:3] == 'sim' else 'sim'

bluesky/navdatabase/loadnavdata.py

@@ -11,9 +11,7 @@
 navdb_version = 'v20170101'
 
 ## Default settings
-settings.set_variable_defaults(navdata_path='data/navdata')
-
-sourcedir = settings.resolve_path(settings.navdata_path)
+settings.set_variable_defaults(navdata_path='navdata')
 
 
 def load_navdata():

bluesky/navdatabase/loadnavdata_txt.py

@@ -1,8 +1,8 @@
 ''' Load navigation data from text files.'''
-from pathlib import Path
 import numpy as np
 from zipfile import ZipFile
 from bluesky import settings
+import bluesky as bs
 from bluesky.tools.aero import ft
 
 
@@ -23,7 +23,7 @@ def loadnavdata_txt():
     wptdata['wpdesc']  = []              # description
 
 
-    with open(settings.resolve_path(settings.navdata_path) / 'nav.dat', 'rb') as f:
+    with open(bs.resource(settings.navdata_path) / 'nav.dat', 'rb') as f:
         print("Reading nav.dat")
 
         for line in f:
@@ -101,7 +101,7 @@ def loadnavdata_txt():
                 wptdata['wpdesc'].append("   ")  # Description
 
     #----------  Read  fix.dat file ----------
-    with open(settings.resolve_path(settings.navdata_path) / 'fix.dat', 'rb') as f:
+    with open(bs.resource(settings.navdata_path) / 'fix.dat', 'rb') as f:
         print("Reading fix.dat")
         for line in f:
             line = line.decode(encoding="ascii", errors="ignore").strip()
@@ -152,7 +152,7 @@ def loadnavdata_txt():
     awydata['awupfl']      = []              # highest flight level (int)
 
 
-    with open(settings.resolve_path(settings.navdata_path) / 'awy.dat', 'rb') as f:
+    with open(bs.resource(settings.navdata_path) / 'awy.dat', 'rb') as f:
         print("Reading awy.dat")
 
         for line in f:
@@ -220,7 +220,7 @@ def loadnavdata_txt():
     aptdata['aptype']    = []              # type (int, 1=large, 2=medium, 3=small)
     aptdata['apco']      = []              # two char country code (string)
     aptdata['apelev']    = []              # field elevation ft-> m
-    with open(settings.resolve_path(settings.navdata_path) / 'airports.dat', 'rb') as f:
+    with open(bs.resource(settings.navdata_path) / 'airports.dat', 'rb') as f:
         types = {'L': 1, 'M': 2, 'S': 3}
         for line in f:
             line = line.decode(encoding="ascii", errors="ignore").strip()
@@ -277,7 +277,7 @@ def loadnavdata_txt():
     firdata['firlon1'] = []
 
     # Get fir names
-    for filname in (settings.resolve_path(settings.navdata_path) / 'fir').iterdir():
+    for filname in (bs.resource(settings.navdata_path) / 'fir').iterdir():
         if filname.suffix == '.txt':
             firname = filname.stem
             firdata['fir'].append([firname, [], []])
@@ -323,7 +323,7 @@ def loadnavdata_txt():
     codata['cocode2']  = []              # 2 char code
     codata['cocode3']  = []              # 3 char code
     codata['conr']     = []              # country nr
-    with open(settings.resolve_path(settings.navdata_path) / 'icao-countries.dat', 'rb') as f:
+    with open(bs.resource(settings.navdata_path) / 'icao-countries.dat', 'rb') as f:
         for line in f:
             line = line.decode(encoding="ascii", errors="ignore").strip()
             # Skip empty lines or comments
@@ -356,7 +356,7 @@ def loadthresholds_txt():
     ''' Runway threshold loader for navdatabase. '''
     rwythresholds = dict()
     curthresholds = None
-    zfile = ZipFile(settings.resolve_path(settings.navdata_path) / 'apt.zip')
+    zfile = ZipFile(bs.resource(settings.navdata_path) / 'apt.zip')
     print("Reading apt.dat from apt.zip")
     with zfile.open('apt.dat', 'r') as f:
         for line in f:

bluesky/resourcepath.py

@@ -0,0 +1,117 @@
+''' BlueSky resource access '''
+import shutil
+import itertools
+from pathlib import Path
+from importlib.resources import files
+from importlib.readers import MultiplexedPath
+
+
+class ResourcePath(MultiplexedPath):
+    def __init__(self, *paths):
+        base = files('bluesky.resources')
+        paths = list(paths) + base._paths if isinstance(base, MultiplexedPath) else base
+        super().__init__(*paths)
+
+    def appendpath(self, path):
+        self._paths.append(path)
+
+    def insertpath(self, path, pos=0):
+        self._paths.insert(pos, path)
+
+    def bases(self):
+        for p in self._paths:
+            yield p
+
+    def base(self, idx):
+        return self._paths[idx]
+
+    @property
+    def nbases(self):
+        return len(self._paths)
+
+    def as_posix(self, idx=0):
+        return self._paths[idx].as_posix()
+
+    def glob(self, pattern: str):
+        files = set()
+        for res in itertools.chain.from_iterable(p.glob(pattern) for p in self._paths):
+            if res.is_file() and res.name in files:
+                continue
+            files.add(res.name)
+            yield res
+
+    def joinpath(self, *descendants):
+        # first try to find child in current paths
+        paths = []
+        if not descendants:
+            return self
+
+        for path in (p.joinpath(*descendants) for p in self._paths):
+            if path.exists():
+                if path.is_dir():
+                    # If it's a dir, try to combine it with others
+                    paths.append(path)
+                else:
+                    # If it's a file, immediately return it
+                    return path
+
+        # if it does not exist, construct it with the first path
+        return ResourcePath(*paths) if len(paths) > 1 else \
+            paths[0] if len(paths) == 1 else self._paths[0].joinpath(*descendants)
+
+    __truediv__ = joinpath
+
+
+def resource(*descendants):
+    ''' Get a path pointing to a BlueSky resource.
+    
+        Arguments:
+        - descendants: Zero or more path-like objects (Path or str)
+
+        Returns:
+        - Path pointing to resource (file or directory)
+          If arguments form an absolute path it is returned directly,
+          otherwise a path relative to BlueSky's resource paths is returned.
+    '''
+    if (ret:=Path(*descendants)).is_absolute():
+        return ret
+
+    return resource.path.joinpath(*descendants)
+resource.path = ResourcePath()
+
+
+def init(workdir=None):
+    ''' Initialise BlueSky resource paths. '''
+    if workdir is None:
+        if files('bluesky').parent == Path.cwd():
+            # Assume BlueSky is running from source, e.g., cloned from GitHub
+            # In this case additional resources (scenarios, plugins)
+            # are found in the current working directory
+            workdir = Path.cwd()
+        else:
+            # Assume BlueSky is running as a pip package
+            # In this case additional resources are found in $home/bluesky
+            workdir = Path.home().joinpath('bluesky')
+
+            # Initialise this folder if it doesn't exist
+            if not workdir.exists():
+                # Create directory and populate with basics
+                print(f'Creating BlueSky base directory "{workdir.absolute()}"')
+                workdir.mkdir()
+
+    elif not Path(workdir).exists():
+        print(f"Specified working directory {workdir} doesn't exist!")
+
+    # Ensure existence of scenario, plugins, output, and cache directories
+    for subdir in map(workdir.joinpath, ('scenario', 'plugins', 'output', 'cache')):
+        if not subdir.exists():
+            print(f'Creating directory "{subdir}"')
+            subdir.mkdir()
+    # Ensure existence of config file
+    cfgfile = workdir.joinpath("settings.cfg")
+    if not cfgfile.exists():
+        print(f'Copying default configfile to {cfgfile}')
+        shutil.copy(resource('default.cfg'), cfgfile)
+
+    # Set correct search paths for resource function
+    resource.path = ResourcePath(workdir)