Merge pull request #10 from mcg1969/otherlang

Handle non-python/R kernels, improve logging

project_inspect/__main__.py

@@ -92,4 +92,4 @@ def main(**kwargs):
     return 0
 
 
-main(**(parser.parse_args().__dict__))
+main(**(parser.parse_args().__dict__))

project_inspect/environments.py

@@ -5,12 +5,12 @@
 import re
 import os
 
-from os.path import basename, dirname, join, exists, isfile, isdir, abspath
-from glob import glob, iglob
+from os.path import basename, dirname, join, exists, isfile, isdir
+from glob import glob
 
 from . import config
-from .imports import find_python_imports, find_r_imports, find_file_imports
-from .utils import load_file
+from .imports import find_file_imports
+from .utils import load_file, warn_file
 
 import logging
 import pkg_resources
@@ -22,12 +22,12 @@
 def get_python_builtins(pybin):
     '''
     Determines the python modules that have been compiled into the Python executable.
-    
+
     Attempts to run the executable and dump sys.builtin_module_names. If this is not
     possible, simply returns a set containing sys.builtin_module_names for the current
     executable. This is a sufficiently close approximation of what we need, so just
     a warning is raised, and execution proceeds.
-    
+
     Args:
         pybin (str): path to a valid Python executable.
     Returns:
@@ -105,7 +105,7 @@ def parse_conda_meta(mpath):
 def get_eggs(sp_dir):
     '''
     Returns a list of all egg files/directories in the given site-packages directory.
-    
+
     Args:
         sp_dir (str): the site packages directory to scan
     Returns:
@@ -120,7 +120,7 @@ def get_eggs(sp_dir):
         try:
             dists = list(factory(fullpath))
         except Exception as e:
-            logger.warning('Error reading eggs in {}:\n{}'.format(fullpath, e))
+            warn_file(fullpath, 'ERROR READING EGGS', e)
             dists = []
         pdata = {'name': None,
                  'version': None,
@@ -139,18 +139,17 @@ def get_eggs(sp_dir):
                     sources = list(map(str.strip, dist.get_metadata(sources).splitlines()))
                     top_level = list(map(str.strip, dist.get_metadata('top_level.txt').splitlines()))
                     for top in top_level:
-                        top_s = top + '/'
                         for src in sources:
-                                src = src.split(',', 1)[0]
-                                if src.endswith('__init__.py'):
-                                    src = dirname(src)
-                                elif src.endswith(('.py', '.so')):
-                                    src = src[:-3]
-                                else:
-                                    continue
-                                pdata['modules']['python'].add(src.replace('/', '.'))
+                            src = src.split(',', 1)[0]
+                            if src.endswith('__init__.py'):
+                                src = dirname(src)
+                            elif src.endswith(('.py', '.so')):
+                                src = src[:-3]
+                            else:
+                                continue
+                            pdata['modules']['python'].add(src.replace('/', '.'))
             except Exception as e:
-                logger.warning('Unexpected error processing {}:\n{}'.format(fn, str(e)))
+                warn_file(fn, 'UNEXPECTED ERROR', e)
         if not pdata['name']:
             name, version = fn.rsplit('.', 1)[0], '<dev>'
             if fn.endswith('.dist-info'):
@@ -181,8 +180,8 @@ def get_python_importables(path, level=0):
         level -= 1
     root_len = len(root_path) + 1
     for root, dirs, files in gen:
-        dirs[:] = [d for d in dirs if not d.startswith('.')
-                   and exists(join(root, d, '__init__.py'))]
+        dirs[:] = [d for d in dirs if not d.startswith('.') and
+                   exists(join(root, d, '__init__.py'))]
         base_module = root[root_len:].replace('/', '.')
         for file in files:
             if file.startswith('.'):
@@ -201,6 +200,7 @@ def get_python_importables(path, level=0):
 @functools.lru_cache()
 def get_local_packages(path):
     packages = {}
+
     def _create(bname):
         if bname not in packages:
             packages[bname] = {'name': bname,
@@ -226,12 +226,11 @@ def _create(bname):
         pdata['modules']['python'].add(module)
         pdata['imports']['python'].update(imports)
     for fpath in glob(join(path, '*.R')) + glob(join(path, '*.ipynb')):
-        if not basename(fpath).startswith('.'):
-            bname = './' + basename(fpath)
-            pdata = _create(bname)
-            imports, language = find_file_imports(fpath, submodules=True)
-            if language in pdata['imports']:
-                pdata['imports'][language] = imports
+        bname = './' + basename(fpath)
+        pdata = _create(bname)
+        imports, language = find_file_imports(fpath, submodules=True)
+        if language in pdata['imports']:
+            pdata['imports'][language] = imports
     return packages
 
 
@@ -261,8 +260,7 @@ def environment_by_prefix(envdir, local=None):
                     if dep is not None:
                         depends.add(dep)
         return envdata
-
-    envname = basename(envdir)
+
     envdata = {'prefix': envdir}
     imports = envdata['imports'] = {'python': {}, 'r': {}}
     packages = envdata['packages'] = {}
@@ -316,8 +314,8 @@ def kernel_name_to_prefix(project_home, kernel_name):
         kernel_base = 'conda-env-{}-'.format(project_name)
         if kernel_loc.startswith(kernel_base):
             return join(project_home, 'envs', kernel_loc[len(kernel_base):])
-    
-    
+
+
 def modules_to_packages(environment, modules, language):
     requested = set()
     missing = set()

project_inspect/imports.py

@@ -1,20 +1,13 @@
 import re
-import os
-import json
-import functools
 
-from glob import glob, iglob
-from os.path import join, basename, dirname, exists, isfile
+from os.path import isfile
 
 from lib2to3 import pygram
 from lib2to3 import pytree
-from lib2to3.pgen2 import driver, parse, token
-from lib2to3.pgen2.parse import ParseError
-from lib2to3.pgen2.tokenize import TokenError
+from lib2to3.pgen2 import driver
 from lib2to3.pygram import python_symbols as syms
-from lib2to3.pytree import Leaf, Node
 
-from .utils import load_file, set_log_root, shortpath
+from .utils import load_file
 
 
 def stringify(content):
@@ -39,7 +32,7 @@ def yield_imports(node):
         base = stringify(node.children[1:right])
         if not base.endswith('.'):
             base += '.'
-        node = node.children[right+1]
+        node = node.children[right + 1]
     elif node.type == syms.import_name:
         # import a, b as c
         base = ''
@@ -55,7 +48,7 @@ def yield_imports(node):
     else:
         yield base + stringify(node)
 
-                
+
 p3_grammar = pygram.python_grammar_no_print_statement
 p3_driver = driver.Driver(p3_grammar, convert=pytree.convert)
 p2_grammar = pygram.python_grammar
@@ -68,11 +61,11 @@ def find_python_imports(code, recurse=True):
     try:
         tree = p3_driver.parse_string(code, debug=False)
         imports.update(yield_imports(tree))
-    except:
+    except Exception:
         try:
             tree = p2_driver.parse_string(code, debug=False)
             imports.update(yield_imports(tree))
-        except:
+        except Exception:
             if recurse:
                 for line in map(str.strip, code.splitlines()):
                     if line and not line.startswith('#'):
@@ -111,21 +104,20 @@ def strip_python_magic(cell):
 def find_notebook_imports(ndata):
     try:
         language = ndata['metadata']['kernelspec']['language'].lower()
-    except KeyError:
-        return set(), 'python'
-    if language not in ('python', 'r'):
-        raise RuntimeError('Unsupported language: {}'.format(language))
+    except (KeyError, TypeError):
+        language = 'unknown'
     modules = set()
-    modules.add('ipykernel' if language == 'python' else 'IRkernel')
-    for cell in ndata['cells']:
-        if cell['cell_type'] == 'code':
-            if language == 'python':
-                source = '\n'.join(strip_python_magic(cell['source']))
-                processor = find_python_imports
-            elif language == 'r':
-                source = '\n'.join(cell['source'])
-                processor = find_r_imports
-            modules.update(processor(source))
+    if language in ('python', 'r'):
+        modules.add('ipykernel' if language == 'python' else 'IRkernel')
+        for cell in ndata['cells']:
+            if cell['cell_type'] == 'code':
+                if language == 'python':
+                    source = '\n'.join(strip_python_magic(cell['source']))
+                    processor = find_python_imports
+                elif language == 'r':
+                    source = '\n'.join(cell['source'])
+                    processor = find_r_imports
+                modules.update(processor(source))
     return modules, language
 
 
@@ -134,17 +126,16 @@ def find_file_imports(fpath, submodules=False, locals=False):
         return set(), None
     data = load_file(fpath)
     if data is None:
-    	return set(), None
+        return set(), None
     elif fpath.endswith('.ipynb'):
         imports, language = find_notebook_imports(data)
     elif fpath.endswith('.py'):
         imports, language = find_python_imports(data), 'python'
-    else: # .R
+    else:  # .R
         imports, language = find_r_imports(data), 'r'
     if language == 'python':
         if not submodules:
             imports = set('.' if imp.startswith('.') else imp.split('.', 1)[0] for imp in imports)
         if not locals:
             imports = set(imp for imp in imports if not imp.startswith('.'))
     return imports, language
-