Merge branch '0.8.6'

.travis.yml

@@ -19,7 +19,7 @@ install:
   - pip install .
 before_script:
   # stop the build if there are Python syntax errors or undefined names
-  - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude=ipynb_to_percent
+  - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics
   # exit-zero treats all errors as warnings.  The GitHub editor is 127 chars wide
   - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
 script:

HISTORY.rst

@@ -3,6 +3,17 @@
 Release History
 ---------------
 
+0.8.6 (2018-11-??)
+++++++++++++++++++++++
+
+**Improvements**
+
+- The ``language_info`` section is not part of the default header any more. Language information is now taken from metadata ``kernelspec.language``. (#105).
+- When opening a paired notebook, the active file is now the file that was originally opened (#118). When saving a notebook, timestamps of all the alternative representations are tested to ensure that Jupyter's autosave does not override manual modifications.
+- Jupyter magic commands are now commented per default in the ``percent`` format (#126, #132). Version for the ``percent`` format increases from '1.1' to '1.2'. Set an option ``comment_magics`` to ``false`` either per notebook, or globally on Jupytext's contents manager, or on `jupytext`'s command line, if you prefer not to comment Jupyter magics.
+- Jupytext command line has a pre-commit mode (#121).
+
+
 0.8.5 (2018-11-13)
 ++++++++++++++++++++++
 

jupytext/cell_reader.py

@@ -408,7 +408,7 @@ def find_cell_end(self, lines):
 
 class DoublePercentScriptCellReader(ScriptCellReader):
     """Read notebook cells from Hydrogen/Spyder/VScode scripts (#59)"""
-    default_comment_magics = False
+    default_comment_magics = True
 
     def __init__(self, ext, comment_magics=None):
         ScriptCellReader.__init__(self, ext, comment_magics)
@@ -443,9 +443,10 @@ def find_cell_content(self, lines):
         # Cell content
         source = lines[cell_start:cell_end_marker]
 
-        if self.cell_type != 'code' or (self.metadata and not is_active('py', self.metadata)):
+        if self.cell_type != 'code' or (self.metadata and not is_active('py', self.metadata)) \
+                or (self.language is not None and self.language != self.default_language):
             source = uncomment(source, self.comment)
-        if self.comment_magics:
+        elif self.metadata is not None and self.comment_magics:
             source = self.uncomment_code_and_magics(source)
 
         self.content = source

jupytext/cell_to_text.py

@@ -60,8 +60,7 @@ def __init__(self, cell, default_language, ext, comment_magics=None, cell_metada
 
         # how many blank lines before next cell
         self.lines_to_next_cell = cell.metadata.get('lines_to_next_cell', 1)
-        self.lines_to_end_of_cell_marker = \
-            cell.metadata.get('lines_to_end_of_cell_marker', 0)
+        self.lines_to_end_of_cell_marker = cell.metadata.get('lines_to_end_of_cell_marker', 0)
 
         # for compatibility with v0.5.4 and lower (to be removed)
         if 'skipline' in cell.metadata:
@@ -286,8 +285,8 @@ def code_to_text(self):
 class DoublePercentCellExporter(BaseCellExporter):
     """A class that can represent a notebook cell as an
     Hydrogen/Spyder/VScode script (#59)"""
-    default_comment_magics = False
-    parse_cell_language = False
+    default_comment_magics = True
+    parse_cell_language = True
 
     def code_to_text(self):
         """Not used"""
@@ -299,6 +298,8 @@ def cell_to_text(self):
             self.metadata['cell_type'] = self.cell_type
 
         active = is_active('py', self.metadata)
+        if self.language != self.default_language and 'active' not in self.metadata:
+            active = False
         if self.cell_type == 'raw' and 'active' in self.metadata and self.metadata['active'] == '':
             del self.metadata['active']
 
@@ -308,11 +309,10 @@ def cell_to_text(self):
         else:
             lines = [self.comment + ' %% ' + options]
 
-        if self.cell_type == 'code':
+        if self.cell_type == 'code' and active:
             source = copy(self.source)
             comment_magic(source, self.language, self.comment_magics)
-            if active:
-                return lines + source
+            return lines + source
 
         return lines + comment_lines(self.source, self.comment)
 

jupytext/cli.py

@@ -2,7 +2,9 @@
 """
 
 import os
+import re
 import sys
+import subprocess
 import argparse
 from .jupytext import readf, reads, writef, writes
 from .formats import NOTEBOOK_EXTENSIONS, JUPYTEXT_FORMATS, check_file_version, one_format_as_string, parse_one_format
@@ -12,20 +14,23 @@
 from .version import __version__
 
 
-def convert_notebook_files(nb_files, fmt, input_format=None, output=None,
+def convert_notebook_files(nb_files, fmt, input_format=None, output=None, pre_commit=False,
                            test_round_trip=False, test_round_trip_strict=False, stop_on_first_error=True,
-                           update=True, freeze_metadata=False):
+                           update=True, freeze_metadata=False, comment_magics=None):
     """
     Export R markdown notebooks, python or R scripts, or Jupyter notebooks,
     to the opposite format
     :param nb_files: one or more notebooks files
     :param input_format: input format, e.g. "py:percent"
     :param fmt: destination format, e.g. "py:percent"
     :param output: None, destination file, or '-' for stdout
+    :param pre_commit: convert notebooks in the git index?
     :param test_round_trip: should round trip conversion be tested?
     :param test_round_trip_strict: should round trip conversion be tested, with strict notebook comparison?
     :param stop_on_first_error: when testing, should we stop on first error, or compare the full notebook?
     :param update: preserve the current outputs of .ipynb file
+    :param freeze_metadata: set metadata filters equal to the current script metadata
+    :param comment_magics: comment, or not, Jupyter magics
     when possible
     :return:
     """
@@ -34,6 +39,23 @@ def convert_notebook_files(nb_files, fmt, input_format=None, output=None,
     if ext not in NOTEBOOK_EXTENSIONS:
         raise TypeError('Destination extension {} is not a notebook'.format(ext))
 
+    if pre_commit:
+        input_format = input_format or 'ipynb'
+        input_ext, _ = parse_one_format(input_format)
+        modified, deleted = modified_and_deleted_files(input_ext)
+
+        for file in modified:
+            dest_file = file[:-len(input_ext)] + ext
+            nb = readf(file)
+            writef(nb, dest_file, format_name=format_name)
+            system('git', 'add', dest_file)
+
+        for file in deleted:
+            dest_file = file[:-len(input_ext)] + ext
+            system('git', 'rm', dest_file)
+
+        return
+
     if not nb_files:
         if not input_format:
             raise ValueError('Reading notebook from the standard input requires the --from field.')
@@ -81,6 +103,9 @@ def convert_notebook_files(nb_files, fmt, input_format=None, output=None,
                 print('{}: {}'.format(nb_file, str(error)))
             continue
 
+        if comment_magics is not None:
+            notebook.metadata.setdefault('jupytext', {})['comment_magics'] = comment_magics
+
         if output == '-':
             sys.stdout.write(writes(notebook, ext=ext, format_name=format_name))
             continue
@@ -111,6 +136,22 @@ def convert_notebook_files(nb_files, fmt, input_format=None, output=None,
         exit(notebooks_in_error)
 
 
+def system(*args, **kwargs):
+    """Execute the given bash command"""
+    kwargs.setdefault('stdout', subprocess.PIPE)
+    proc = subprocess.Popen(args, **kwargs)
+    out, err = proc.communicate()
+    return out
+
+
+def modified_and_deleted_files(ext):
+    """Return the list of modified and deleted ipynb files in the git index"""
+    re_modified = re.compile(r'^[AM]+\s+(?P<name>.*{})'.format(ext.replace('.', r'\.')), re.MULTILINE)
+    re_deleted = re.compile(r'^[D]+\s+(?P<name>.*{})'.format(ext.replace('.', r'\.')), re.MULTILINE)
+    files = system('git', 'status', '--porcelain').decode('utf-8')
+    return re_modified.findall(files), re_deleted.findall(files)
+
+
 def save_notebook_as(notebook, nb_file, nb_dest, format_name, combine):
     """Save notebook to file, in desired format"""
     if combine and os.path.isfile(nb_dest) and os.path.splitext(nb_dest)[1] == '.ipynb':
@@ -145,11 +186,23 @@ def canonize_format(format_or_ext, file_path=None):
     return {'notebook': 'ipynb', 'markdown': 'md', 'rmarkdown': 'Rmd'}[format_or_ext]
 
 
+def str2bool(input):
+    """Parse Yes/No/Default string
+    https://stackoverflow.com/questions/15008758/parsing-boolean-values-with-argparse"""
+    if input.lower() in ('yes', 'true', 't', 'y', '1'):
+        return True
+    if input.lower() in ('no', 'false', 'f', 'n', '0'):
+        return False
+    if input.lower() in ('d', 'default', ''):
+        return None
+    raise argparse.ArgumentTypeError('Expected: (Y)es/(T)rue/(N)o/(F)alse/(D)efault')
+
+
 def cli_jupytext(args=None):
     """Command line parser for jupytext"""
     parser = argparse.ArgumentParser(
-        description='Jupyter notebooks as markdown documents, '
-                    'Julia, Python or R scripts')
+        description='Jupyter notebooks as markdown documents, Julia, Python or R scripts',
+        formatter_class=argparse.RawTextHelpFormatter)
 
     notebook_formats = (['notebook', 'rmarkdown', 'markdown'] +
                         [_SCRIPT_EXTENSIONS[ext]['language'] for ext in _SCRIPT_EXTENSIONS] +
@@ -165,18 +218,28 @@ def cli_jupytext(args=None):
                         choices=notebook_formats,
                         help="Input format")
     parser.add_argument('notebooks',
-                        help='One or more notebook(s) to be converted. Input '
-                             'is read from stdin when no notebook is '
-                             'provided , but then the --from field is '
-                             'mandatory',
+                        help='One or more notebook(s). Input is read from stdin when no notebook '
+                             'is provided , but then the --from field is mandatory',
                         nargs='*')
+    parser.add_argument('--pre-commit', action='store_true',
+                        help="""Run Jupytext on the ipynb files in the git index. 
+Create a pre-commit hook with:                        
+
+echo '#!/bin/sh
+jupytext --to py:light --pre-commit' > .git/hooks/pre-commit
+chmod +x .git/hooks/pre-commit""")
     parser.add_argument('-o', '--output',
                         help='Destination file. Defaults to original file, '
                              'with extension changed to destination format. '
                              "Use '-' for printing the notebook on stdout.")
     parser.add_argument('--update', action='store_true',
                         help='Preserve outputs of .ipynb destination '
                              '(when file exists and inputs match)')
+    parser.add_argument('--comment-magics',
+                        type=str2bool,
+                        nargs='?',
+                        default=None,
+                        help='Should Jupyter magic commands be commented? (Y)es/(T)rue/(N)o/(F)alse/(D)efault')
     parser.add_argument('--freeze-metadata', action='store_true',
                         help='Set a metadata filter (unless one exists already) '
                              'equal to the current metadata of the notebook. Use this '
@@ -205,12 +268,18 @@ def cli_jupytext(args=None):
             args.output = '-'
 
     if not args.input_format:
-        if not args.notebooks:
-            raise ValueError('Please specificy either --from or notebooks')
+        if not args.notebooks and not args.pre_commit:
+            raise ValueError('Please specificy either --from, --pre-commit or notebooks')
 
     if args.update and not (args.test or args.test_strict) and args.to != 'ipynb':
         raise ValueError('--update works exclusively with --to notebook ')
 
+    if args.pre_commit:
+        if args.notebooks:
+            raise ValueError('--pre-commit takes notebooks from the git index. Do not pass any notebook here.')
+        if args.test or args.test_strict:
+            raise ValueError('--pre-commit cannot be used with --test or --test-strict')
+
     return args
 
 
@@ -227,11 +296,13 @@ def jupytext(args=None):
                                fmt=args.to,
                                input_format=args.input_format,
                                output=args.output,
+                               pre_commit=args.pre_commit,
                                test_round_trip=args.test,
                                test_round_trip_strict=args.test_strict,
                                stop_on_first_error=args.stop_on_first_error,
                                update=args.update,
-                               freeze_metadata=args.freeze_metadata)
+                               freeze_metadata=args.freeze_metadata,
+                               comment_magics=args.comment_magics)
     except ValueError as err:  # (ValueError, TypeError, IOError) as err:
         print('jupytext: error: ' + str(err))
         exit(1)