diff --git a/.ci/environment-ci.yml b/.ci/environment-ci.yml index 1582dcf5e..cc9f3a5ca 100644 --- a/.ci/environment-ci.yml +++ b/.ci/environment-ci.yml @@ -12,4 +12,4 @@ channels: dependencies: - jupyterlab - nbformat>=5.1.2 - - pandoc==2.16.2 + - pandoc>=3.0 diff --git a/environment.yml b/environment.yml index 3e4df5174..52f63845a 100644 --- a/environment.yml +++ b/environment.yml @@ -8,4 +8,4 @@ dependencies: - nbformat>=5.1.2 - pre-commit - nodejs>=20 - - pandoc==2.16.2 + - pandoc>=3.0 diff --git a/src/jupytext/jupytext.py b/src/jupytext/jupytext.py index 780f412ae..80a9ed84e 100644 --- a/src/jupytext/jupytext.py +++ b/src/jupytext/jupytext.py @@ -177,7 +177,7 @@ def reads(self, s, **_): return new_notebook(cells=cells, metadata=metadata) - def filter_notebook(self, nb, metadata): + def filter_notebook(self, nb, metadata, preserve_cell_ids=False): self.update_fmt_with_notebook_options(nb.metadata) unsupported_keys = set() metadata = insert_jupytext_info_and_filter_metadata( @@ -192,14 +192,23 @@ def filter_notebook(self, nb, metadata): _IGNORE_CELL_METADATA, unsupported_keys=unsupported_keys, ) + + if preserve_cell_ids and hasattr(cell, "id"): + id = {"id": cell.id} + else: + id = {} + if cell.cell_type == "code": - cells.append(new_code_cell(source=cell.source, metadata=cell_metadata)) + cells.append( + new_code_cell(source=cell.source, metadata=cell_metadata, **id) + ) else: cells.append( NotebookNode( source=cell.source, metadata=cell_metadata, cell_type=cell.cell_type, + **id, ) ) @@ -215,7 +224,9 @@ def filter_notebook(self, nb, metadata): def writes(self, nb, metadata=None, **kwargs): """Return the text representation of the notebook""" if self.fmt.get("format_name") == "pandoc": - return notebook_to_md(self.filter_notebook(nb, metadata)) + return notebook_to_md( + self.filter_notebook(nb, metadata, preserve_cell_ids=True) + ) if self.fmt.get("format_name") == "quarto" or self.ext == ".qmd": return notebook_to_qmd(self.filter_notebook(nb, metadata)) if self.fmt.get( diff --git a/tests/conftest.py b/tests/conftest.py index 8fd89c621..3d76dfbfb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -368,8 +368,8 @@ def pytest_runtest_setup(item): # https://github.com/mwouts/jupytext/commit/c07d919702999056ce47f92b74f63a15c8361c5d # The mirror files changed again when Pandoc 2.16 was introduced # https://github.com/mwouts/jupytext/pull/919/commits/1fa1451ecdaa6ad8d803bcb6fb0c0cf09e5371bf - if not is_pandoc_available(min_version="2.16.2", max_version="2.16.2"): - pytest.skip("pandoc==2.16.2 is not available") + if not is_pandoc_available(min_version="3.0"): + pytest.skip("pandoc>=3.0 is not available") if mark.name == "requires_quarto": if not is_quarto_available(min_version="0.2.0"): pytest.skip("quarto>=0.2 is not available") @@ -410,15 +410,15 @@ def pytest_collection_modifyitems(config, items): item.add_marker(pytest.mark.pre_commit) -"""To make sure that cell ids are distinct we use a global counter. +@pytest.fixture(autouse=True) +def cell_id(): + """To make sure that cell ids are distinct we use a global counter. This solves https://github.com/mwouts/jupytext/issues/747""" -global_cell_count = 0 + local_cell_count = 0 + def enumerate_cell_ids(): + nonlocal local_cell_count + local_cell_count += 1 + return f"cell-{local_cell_count}" -def generate_corpus_id(): - global global_cell_count - global_cell_count += 1 - return f"cell-{global_cell_count}" - - -nbbase.random_cell_id = generate_corpus_id + nbbase.random_cell_id = enumerate_cell_ids diff --git a/tests/data/notebooks/outputs/ipynb_to_pandoc/Notebook_with_R_magic.md b/tests/data/notebooks/outputs/ipynb_to_pandoc/Notebook_with_R_magic.md index 0d90a5863..2a734eb41 100644 --- a/tests/data/notebooks/outputs/ipynb_to_pandoc/Notebook_with_R_magic.md +++ b/tests/data/notebooks/outputs/ipynb_to_pandoc/Notebook_with_R_magic.md @@ -14,20 +14,20 @@ jupyter: This notebook shows the use of R cells to generate plots ::: -::: {.cell .code} +::: {#cell-1 .cell .code} ``` python %load_ext rpy2.ipython ``` ::: -::: {.cell .code} +::: {#cell-2 .cell .code} ``` python %%R suppressMessages(require(tidyverse)) ``` ::: -::: {.cell .code} +::: {#cell-3 .cell .code} ``` python %%R ggplot(iris, aes(x = Sepal.Length, y = Petal.Length, color=Species)) + geom_point() @@ -38,7 +38,7 @@ ggplot(iris, aes(x = Sepal.Length, y = Petal.Length, color=Species)) + geom_poin The default plot dimensions are not good for us, so we use the -w and -h parameters in %%R magic to set the plot size ::: -::: {.cell .code} +::: {#cell-4 .cell .code} ``` python %%R -w 400 -h 240 ggplot(iris, aes(x = Sepal.Length, y = Petal.Length, color=Species)) + geom_point() diff --git a/tests/data/notebooks/outputs/ipynb_to_pandoc/Notebook_with_more_R_magic_111.md b/tests/data/notebooks/outputs/ipynb_to_pandoc/Notebook_with_more_R_magic_111.md index d6ae7f7a4..4efd02dbf 100644 --- a/tests/data/notebooks/outputs/ipynb_to_pandoc/Notebook_with_more_R_magic_111.md +++ b/tests/data/notebooks/outputs/ipynb_to_pandoc/Notebook_with_more_R_magic_111.md @@ -8,7 +8,7 @@ jupyter: nbformat_minor: 2 --- -::: {.cell .code} +::: {#cell-1 .cell .code} ``` python %load_ext rpy2.ipython import pandas as pd @@ -24,7 +24,7 @@ df = pd.DataFrame( ``` ::: -::: {.cell .code} +::: {#cell-2 .cell .code} ``` python %%R -i df library("ggplot2") diff --git a/tests/data/notebooks/outputs/ipynb_to_pandoc/cat_variable.md b/tests/data/notebooks/outputs/ipynb_to_pandoc/cat_variable.md index 659bec56a..3b075fb9e 100644 --- a/tests/data/notebooks/outputs/ipynb_to_pandoc/cat_variable.md +++ b/tests/data/notebooks/outputs/ipynb_to_pandoc/cat_variable.md @@ -8,7 +8,7 @@ jupyter: nbformat_minor: 2 --- -::: {.cell .code} +::: {#cell-1 .cell .code} ``` python cat = 42 ``` diff --git a/tests/data/notebooks/outputs/ipynb_to_pandoc/convert_to_py_then_test_with_update83.md b/tests/data/notebooks/outputs/ipynb_to_pandoc/convert_to_py_then_test_with_update83.md index 57612c4a2..0ceb4488f 100644 --- a/tests/data/notebooks/outputs/ipynb_to_pandoc/convert_to_py_then_test_with_update83.md +++ b/tests/data/notebooks/outputs/ipynb_to_pandoc/convert_to_py_then_test_with_update83.md @@ -8,7 +8,7 @@ jupyter: nbformat_minor: 2 --- -::: {.cell .code} +::: {#cell-1 .cell .code} ``` python %%time @@ -20,7 +20,7 @@ print('asdf') Thanks for jupytext! ::: -::: {.cell .code} +::: {#cell-2 .cell .code} ``` python ``` ::: diff --git a/tests/data/notebooks/outputs/ipynb_to_pandoc/frozen_cell.md b/tests/data/notebooks/outputs/ipynb_to_pandoc/frozen_cell.md index c5fd9796f..f760b31e9 100644 --- a/tests/data/notebooks/outputs/ipynb_to_pandoc/frozen_cell.md +++ b/tests/data/notebooks/outputs/ipynb_to_pandoc/frozen_cell.md @@ -8,14 +8,14 @@ jupyter: nbformat_minor: 2 --- -::: {.cell .code} +::: {#cell-1 .cell .code} ``` python # This is an unfrozen cell. Works as usual. print("I'm a regular cell so I run and print!") ``` ::: -::: {.cell .code deletable="false" editable="false" run_control="{\"frozen\":true}"} +::: {#cell-2 .cell .code deletable="false" editable="false" run_control="{\"frozen\":true}"} ``` python # This is an frozen cell print("I'm frozen so Im not executed :(") diff --git a/tests/data/notebooks/outputs/ipynb_to_pandoc/ir_notebook.md b/tests/data/notebooks/outputs/ipynb_to_pandoc/ir_notebook.md index c4b6b8bc5..6673b3a2c 100644 --- a/tests/data/notebooks/outputs/ipynb_to_pandoc/ir_notebook.md +++ b/tests/data/notebooks/outputs/ipynb_to_pandoc/ir_notebook.md @@ -12,19 +12,19 @@ jupyter: This is a jupyter notebook that uses the IR kernel. ::: -::: {.cell .code} +::: {#cell-1 .cell .code} ``` R sum(1:10) ``` ::: -::: {.cell .code} +::: {#cell-2 .cell .code} ``` R plot(cars) ``` ::: -::: {.cell .code} +::: {#cell-3 .cell .code} ``` R ``` ::: diff --git a/tests/data/notebooks/outputs/ipynb_to_pandoc/julia_benchmark_plotly_barchart.md b/tests/data/notebooks/outputs/ipynb_to_pandoc/julia_benchmark_plotly_barchart.md index e62c607ca..08959bda9 100644 --- a/tests/data/notebooks/outputs/ipynb_to_pandoc/julia_benchmark_plotly_barchart.md +++ b/tests/data/notebooks/outputs/ipynb_to_pandoc/julia_benchmark_plotly_barchart.md @@ -8,7 +8,7 @@ jupyter: nbformat_minor: 1 --- -::: {.cell .code} +::: {#cell-1 .cell .code} ``` julia # IJulia rocks! So does Plotly. Check it out @@ -20,7 +20,7 @@ Plotly.signin(username, api_key) ``` ::: -::: {.cell .code} +::: {#cell-2 .cell .code} ``` julia # Following data taken from http://julialang.org/ frontpage benchmarks = ["fib", "parse_int", "quicksort3", "mandel", "pi_sum", "rand_mat_stat", "rand_mat_mul"] @@ -69,7 +69,7 @@ display("text/html", s) ``` ::: -::: {.cell .code} +::: {#cell-3 .cell .code} ``` julia # checkout https://plot.ly/api/ for more Julia examples! # But to show off some other Plotly features: diff --git a/tests/data/notebooks/outputs/ipynb_to_pandoc/jupyter.md b/tests/data/notebooks/outputs/ipynb_to_pandoc/jupyter.md index 5564be9f2..f40716625 100644 --- a/tests/data/notebooks/outputs/ipynb_to_pandoc/jupyter.md +++ b/tests/data/notebooks/outputs/ipynb_to_pandoc/jupyter.md @@ -14,7 +14,7 @@ jupyter: This notebook is a simple jupyter notebook. It only has markdown and code cells. And it does not contain consecutive markdown cells. We start with an addition: ::: -::: {.cell .code} +::: {#cell-1 .cell .code} ``` python a = 1 b = 2 @@ -26,13 +26,13 @@ a + b Now we return a few tuples ::: -::: {.cell .code} +::: {#cell-2 .cell .code} ``` python a, b ``` ::: -::: {.cell .code} +::: {#cell-3 .cell .code} ``` python a, b, a+b ``` diff --git a/tests/data/notebooks/outputs/ipynb_to_pandoc/jupyter_again.md b/tests/data/notebooks/outputs/ipynb_to_pandoc/jupyter_again.md index 2569427fa..0089724a8 100644 --- a/tests/data/notebooks/outputs/ipynb_to_pandoc/jupyter_again.md +++ b/tests/data/notebooks/outputs/ipynb_to_pandoc/jupyter_again.md @@ -8,7 +8,7 @@ jupyter: nbformat_minor: 2 --- -::: {.cell .code} +::: {#cell-1 .cell .code} ``` python c = ''' title: "Quick test" @@ -22,14 +22,14 @@ editor_options: ``` ::: -::: {.cell .code} +::: {#cell-2 .cell .code} ``` python import yaml print(yaml.dump(yaml.load(c))) ``` ::: -::: {.cell .code} +::: {#cell-3 .cell .code} ``` python ?next ``` diff --git a/tests/data/notebooks/outputs/ipynb_to_pandoc/jupyter_with_raw_cell_in_body.md b/tests/data/notebooks/outputs/ipynb_to_pandoc/jupyter_with_raw_cell_in_body.md index 838e15b9d..a0d7b071b 100644 --- a/tests/data/notebooks/outputs/ipynb_to_pandoc/jupyter_with_raw_cell_in_body.md +++ b/tests/data/notebooks/outputs/ipynb_to_pandoc/jupyter_with_raw_cell_in_body.md @@ -8,7 +8,7 @@ jupyter: nbformat_minor: 2 --- -::: {.cell .code} +::: {#cell-1 .cell .code} ``` python 1+2+3 ``` diff --git a/tests/data/notebooks/outputs/ipynb_to_pandoc/jupyter_with_raw_cell_on_top.md b/tests/data/notebooks/outputs/ipynb_to_pandoc/jupyter_with_raw_cell_on_top.md index e33e0c0da..d242c1afa 100644 --- a/tests/data/notebooks/outputs/ipynb_to_pandoc/jupyter_with_raw_cell_on_top.md +++ b/tests/data/notebooks/outputs/ipynb_to_pandoc/jupyter_with_raw_cell_on_top.md @@ -22,13 +22,13 @@ editor_options: ``` ::: -::: {.cell .code} +::: {#cell-1 .cell .code} ``` python 1+2+3 ``` ::: -::: {.cell .code} +::: {#cell-2 .cell .code} ``` python ``` ::: diff --git a/tests/data/notebooks/outputs/ipynb_to_pandoc/notebook_with_complex_metadata.md b/tests/data/notebooks/outputs/ipynb_to_pandoc/notebook_with_complex_metadata.md index ea8f36039..1bf6d88a6 100644 --- a/tests/data/notebooks/outputs/ipynb_to_pandoc/notebook_with_complex_metadata.md +++ b/tests/data/notebooks/outputs/ipynb_to_pandoc/notebook_with_complex_metadata.md @@ -8,7 +8,7 @@ jupyter: nbformat_minor: 2 --- -::: {.cell .code} +::: {#cell-1 .cell .code} ``` python ``` ::: diff --git a/tests/data/notebooks/outputs/ipynb_to_pandoc/nteract_with_parameter.md b/tests/data/notebooks/outputs/ipynb_to_pandoc/nteract_with_parameter.md index fed8a8f44..f01126798 100644 --- a/tests/data/notebooks/outputs/ipynb_to_pandoc/nteract_with_parameter.md +++ b/tests/data/notebooks/outputs/ipynb_to_pandoc/nteract_with_parameter.md @@ -10,19 +10,19 @@ jupyter: nbformat_minor: 2 --- -::: {.cell .code inputHidden="false" outputHidden="false" tags="[\"parameters\"]"} +::: {#cell-1 .cell .code inputHidden="false" outputHidden="false" tags="[\"parameters\"]"} ``` python param = 4 ``` ::: -::: {.cell .code inputHidden="false" outputHidden="false"} +::: {#cell-2 .cell .code inputHidden="false" outputHidden="false"} ``` python import pandas as pd ``` ::: -::: {.cell .code inputHidden="false" outputHidden="false"} +::: {#cell-3 .cell .code inputHidden="false" outputHidden="false"} ``` python df = pd.DataFrame({'A': [1, 2], 'B': [3 + param, 4]}, index=pd.Index(['x0', 'x1'], name='x')) @@ -30,7 +30,7 @@ df ``` ::: -::: {.cell .code inputHidden="false" outputHidden="false"} +::: {#cell-4 .cell .code inputHidden="false" outputHidden="false"} ``` python %matplotlib inline df.plot(kind='bar') diff --git a/tests/data/notebooks/outputs/ipynb_to_pandoc/plotly_graphs.md b/tests/data/notebooks/outputs/ipynb_to_pandoc/plotly_graphs.md index 736b0c362..3efc20765 100644 --- a/tests/data/notebooks/outputs/ipynb_to_pandoc/plotly_graphs.md +++ b/tests/data/notebooks/outputs/ipynb_to_pandoc/plotly_graphs.md @@ -20,14 +20,14 @@ This notebook contains complex outputs, including plotly javascript graphs. We use Plotly\'s connected mode to make the notebook lighter - when connected, the notebook downloads the `plotly.js` library from the web. ::: -::: {.cell .code} +::: {#cell-1 .cell .code} ``` python import plotly.offline as offline offline.init_notebook_mode(connected=True) ``` ::: -::: {.cell .code} +::: {#cell-2 .cell .code} ``` python import plotly.graph_objects as go fig = go.Figure( diff --git a/tests/data/notebooks/outputs/ipynb_to_pandoc/sample_rise_notebook_66.md b/tests/data/notebooks/outputs/ipynb_to_pandoc/sample_rise_notebook_66.md index 6fe251eea..729655f19 100644 --- a/tests/data/notebooks/outputs/ipynb_to_pandoc/sample_rise_notebook_66.md +++ b/tests/data/notebooks/outputs/ipynb_to_pandoc/sample_rise_notebook_66.md @@ -12,7 +12,7 @@ jupyter: A markdown cell ::: -::: {.cell .code slideshow="{\"slide_type\":\"\"}"} +::: {#cell-1 .cell .code slideshow="{\"slide_type\":\"\"}"} ``` python 1+1 ``` diff --git a/tests/data/notebooks/outputs/ipynb_to_pandoc/text_outputs_and_images.md b/tests/data/notebooks/outputs/ipynb_to_pandoc/text_outputs_and_images.md index 58c707fa7..e5457869c 100644 --- a/tests/data/notebooks/outputs/ipynb_to_pandoc/text_outputs_and_images.md +++ b/tests/data/notebooks/outputs/ipynb_to_pandoc/text_outputs_and_images.md @@ -18,7 +18,7 @@ This notebook contains outputs of many different types: text, HTML, plots and er Using `print`, `sys.stdout` and `sys.stderr` ::: -::: {.cell .code} +::: {#cell-1 .cell .code} ``` python import sys print('using print') @@ -27,7 +27,7 @@ sys.stderr.write('using sys.stderr.write') ``` ::: -::: {.cell .code} +::: {#cell-2 .cell .code} ``` python import logging logging.debug('Debug') @@ -43,14 +43,14 @@ logging.error('Error') Using `pandas`. Here we find two representations: both text and HTML. ::: -::: {.cell .code} +::: {#cell-3 .cell .code} ``` python import pandas as pd pd.DataFrame([4]) ``` ::: -::: {.cell .code} +::: {#cell-4 .cell .code} ``` python from IPython.display import display display(pd.DataFrame([5])) @@ -62,13 +62,13 @@ display(pd.DataFrame([6])) # Images ::: -::: {.cell .code} +::: {#cell-5 .cell .code} ``` python %matplotlib inline ``` ::: -::: {.cell .code} +::: {#cell-6 .cell .code} ``` python # First plot from matplotlib import pyplot as plt @@ -94,7 +94,7 @@ plt.show() # Errors ::: -::: {.cell .code} +::: {#cell-7 .cell .code} ``` python undefined_variable ``` diff --git a/tests/external/pre_commit/test_pre_commit_scripts.py b/tests/external/pre_commit/test_pre_commit_scripts.py index 885986db9..bd19a48b7 100644 --- a/tests/external/pre_commit/test_pre_commit_scripts.py +++ b/tests/external/pre_commit/test_pre_commit_scripts.py @@ -334,7 +334,7 @@ def test_wrap_markdown_cell(tmpdir): fp.write( "#!/bin/sh\n" "jupytext --pre-commit --sync --pipe-fmt ipynb --pipe \\\n" - " 'pandoc --from ipynb --to ipynb --atx-headers'\n" + " 'pandoc --from ipynb --to ipynb --markdown-headings=atx'\n" ) st = os.stat(hook) diff --git a/tests/external/simple_external_notebooks/test_read_simple_pandoc.py b/tests/external/simple_external_notebooks/test_read_simple_pandoc.py index 406a254b7..37d845cd1 100644 --- a/tests/external/simple_external_notebooks/test_read_simple_pandoc.py +++ b/tests/external/simple_external_notebooks/test_read_simple_pandoc.py @@ -9,6 +9,7 @@ @pytest.mark.requires_pandoc def test_pandoc_implicit( + cell_id, markdown="""# Lorem ipsum **Lorem ipsum** dolor sit amet, consectetur adipiscing elit. Nunc luctus @@ -31,7 +32,7 @@ def test_pandoc_implicit( @pytest.mark.requires_pandoc def test_pandoc_explicit( - markdown="""::: {.cell .markdown} + markdown="""::: {#cell_id .cell .markdown} # Lorem **Lorem ipsum** dolor sit amet, consectetur adipiscing elit. Nunc luctus @@ -46,7 +47,7 @@ def test_pandoc_explicit( @pytest.mark.requires_pandoc def test_pandoc_utf8_in_md( - markdown="""::: {.cell .markdown} + markdown="""::: {#cell_id .cell .markdown} # Utf-8 support This is the greek letter $\\pi$: π diff --git a/tests/functional/cli/test_cli.py b/tests/functional/cli/test_cli.py index a3d229083..b921ee460 100644 --- a/tests/functional/cli/test_cli.py +++ b/tests/functional/cli/test_cli.py @@ -1057,7 +1057,7 @@ def test_sync_pipe_config(tmpdir): "--pipe-fmt", "ipynb", "--pipe", - "pandoc --from ipynb --to ipynb --atx-headers", + "pandoc --from ipynb --to ipynb --markdown-headings=atx", str(nb_file), ] ) diff --git a/tests/unit/test_cell_id.py b/tests/unit/test_cell_id.py index 4d65e3a83..b2d0e0664 100644 --- a/tests/unit/test_cell_id.py +++ b/tests/unit/test_cell_id.py @@ -1,10 +1,8 @@ -from nbformat.v4.nbbase import new_code_cell +from nbformat.v4.nbbase import new_code_cell, new_markdown_cell, new_raw_cell def test_cell_id_is_not_random(): - id1 = new_code_cell().id - id2 = new_code_cell().id - - n1 = int(id1.split("-")[1]) - n2 = int(id2.split("-")[1]) - assert n2 == n1 + 1 + assert new_code_cell().id == "cell-1" + assert new_code_cell().id == "cell-2" + assert new_markdown_cell().id == "cell-3" + assert new_raw_cell().id == "cell-4"