🪓 Split Output into Outputs → Output[]

.changeset/flat-pumas-prove.md

@@ -0,0 +1,10 @@
+---
+"myst-spec-ext": minor
+"mystmd": minor
+"myst-directives": patch
+"myst-execute": patch
+"myst-compat": patch
+"myst-cli": patch
+---
+
+Add support for new Outputs node

packages/myst-cli/package.json

@@ -70,6 +70,7 @@
     "mime-types": "^2.1.35",
     "myst-cli-utils": "^2.0.11",
     "myst-common": "^1.7.6",
+    "myst-compat": "^0.0.0",
     "myst-config": "^1.7.6",
     "myst-execute": "^0.1.2",
     "myst-ext-card": "^1.0.9",

packages/myst-cli/src/process/file.ts

@@ -9,20 +9,27 @@ import type { GenericParent } from 'myst-common';
 import { RuleId, toText, fileError, fileWarn } from 'myst-common';
 import type { PageFrontmatter } from 'myst-frontmatter';
 import { validatePageFrontmatter, fillProjectFrontmatter } from 'myst-frontmatter';
-import { SourceFileKind } from 'myst-spec-ext';
+import { SourceFileKind, VERSION } from 'myst-spec-ext';
 import { frontmatterValidationOpts, getPageFrontmatter } from '../frontmatter.js';
 import type { ISession, ISessionWithCache } from '../session/types.js';
 import { castSession } from '../session/cache.js';
 import { config, projects, warnings, watch } from '../store/reducers.js';
 import type { PreRendererData, RendererData } from '../transforms/types.js';
+import { makeCompatible } from 'myst-compat';
 import { logMessagesFromVFile } from '../utils/logging.js';
 import { isValidFile, parseFilePath } from '../utils/resolveExtension.js';
 import { addWarningForFile } from '../utils/addWarningForFile.js';
 import { loadBibTeXCitationRenderers } from './citations.js';
 import { parseMyst } from './myst.js';
 import { processNotebookFull } from './notebook.js';
 import { selectors } from '../store/index.js';
-import { defined, incrementOptions, validateObjectKeys, validateEnum } from 'simple-validators';
+import {
+  defined,
+  incrementOptions,
+  validateObjectKeys,
+  validateEnum,
+  validateString,
+} from 'simple-validators';
 import type { ValidationOptions } from 'simple-validators';
 
 type LoadFileOptions = {
@@ -144,10 +151,12 @@ export function mystJSONValidationOpts(
 function validateMySTJSON(
   input: any,
   opts: ValidationOptions,
-): { mdast: GenericParent; kind: SourceFileKind; frontmatter: PageFrontmatter } | undefined {
+):
+  | { mdast: GenericParent; kind: SourceFileKind; frontmatter: PageFrontmatter; version: string }
+  | undefined {
   const value = validateObjectKeys(
     input,
-    { required: ['mdast'], optional: ['kind', 'frontmatter'] },
+    { required: ['mdast'], optional: ['kind', 'frontmatter', 'version'] },
     opts,
   );
   if (value === undefined) {
@@ -156,22 +165,39 @@ function validateMySTJSON(
 
   const { mdast } = value;
 
+  // Check kind
   let kind: undefined | SourceFileKind;
   if (defined(value.kind)) {
     kind = validateEnum<SourceFileKind>(value.kind, {
       ...incrementOptions('kind', opts),
       enum: SourceFileKind,
     });
   }
+
+  // Check frontmatter
   let frontmatter: undefined | PageFrontmatter;
   if (defined(value.frontmatter)) {
     frontmatter = validatePageFrontmatter(value.frontmatter, incrementOptions('frontmatter', opts));
   }
 
+  let version: undefined | string;
+  if (defined(value.version)) {
+    version = validateString(value.version, incrementOptions('version', opts));
+    // Need a valid version if given
+    if (version === undefined) {
+      return undefined;
+    }
+  }
+  version = version ?? '1';
+
+  // Ingest "legacy" mdast and upgrade it
+  makeCompatible(version, '2', mdast);
+
   return {
     mdast,
     kind: kind ?? SourceFileKind.Article,
     frontmatter: frontmatter ?? {},
+    version: version,
   };
 }
 

packages/myst-cli/src/process/mdast.ts

@@ -3,8 +3,9 @@ import { tic } from 'myst-cli-utils';
 import type { GenericParent, IExpressionResult, PluginUtils, References } from 'myst-common';
 import { fileError, fileWarn, RuleId, slugToUrl } from 'myst-common';
 import type { PageFrontmatter } from 'myst-frontmatter';
-import { SourceFileKind } from 'myst-spec-ext';
+import { SourceFileKind, VERSION } from 'myst-spec-ext';
 import type { LinkTransformer } from 'myst-transforms';
+import { downgrade as downgradeAST } from 'myst-compat';
 import {
   basicTransformationsPlugin,
   htmlPlugin,
@@ -273,6 +274,7 @@ export async function transformMdast(
     mdast,
     references,
     widgets,
+    version: VERSION,
   } as any;
   const cachedMdast = cache.$getMdast(file);
   if (cachedMdast) cachedMdast.post = data;
@@ -429,5 +431,9 @@ export async function finalizeMdast(
     postData.widgets = cache.$getMdast(file)?.pre.widgets;
     updateFileInfoFromFrontmatter(session, file, frontmatter);
   }
+  // TODO[ast-downgrade]: remove this once AST downgrade support is widespread
+  // i.e. once VERSION from myst-spec-ext is 2
+  downgradeAST('2', '1', mdast as any);
+
   logMessagesFromVFile(session, vfile);
 }

packages/myst-cli/src/process/notebook.ts

@@ -165,17 +165,21 @@ export async function processNotebookFull(
           value: ensureString(cell.source),
         };
 
-        // Embed outputs in an output block
-        const output: { type: 'output'; id: string; data: IOutput[] } = {
-          type: 'output',
+        const outputsChildren = (cell.outputs as IOutput[]).map((output) => {
+          // Embed outputs in an output block
+          const result = {
+            type: 'output',
+            jupyter_data: output,
+            children: [],
+          };
+          return result;
+        });
+        const outputs = {
+          type: 'outputs',
           id: nanoid(),
-          data: [],
+          children: outputsChildren,
         };
-
-        if (cell.outputs && (cell.outputs as IOutput[]).length > 0) {
-          output.data = cell.outputs as IOutput[];
-        }
-        return acc.concat(blockParent(cell, [code, output]));
+        return acc.concat(blockParent(cell, [code, outputs]));
       }
       return acc;
     },

packages/myst-cli/src/transforms/code.spec.ts

@@ -162,7 +162,10 @@ function build_mdast(tags: string[], has_output: boolean) {
     ],
   };
   if (has_output) {
-    mdast.children[0].children.push({ type: 'output' });
+    mdast.children[0].children.push({
+      type: 'outputs',
+      children: [{ type: 'output', children: [] }],
+    });
   }
   return mdast;
 }
@@ -261,7 +264,7 @@ describe('propagateBlockDataToCode', () => {
           const mdast = build_mdast([tag], has_output);
           propagateBlockDataToCode(new Session(), new VFile(), mdast);
           let result = '';
-          const outputNode = mdast.children[0].children[1];
+          const outputsNode = mdast.children[0].children[1];
           switch (target) {
             case 'cell':
               result = mdast.children[0].visibility;
@@ -270,12 +273,14 @@ describe('propagateBlockDataToCode', () => {
               result = mdast.children[0].children[0].visibility;
               break;
             case 'output':
-              if (!has_output && target == 'output') {
-                expect(outputNode).toEqual(undefined);
+              if (!has_output) {
+                expect(outputsNode).toEqual(undefined);
                 continue;
               }
-              result = outputNode.visibility;
+              result = outputsNode.visibility;
               break;
+            default:
+              throw new Error();
           }
           expect(result).toEqual(action);
         }
@@ -290,13 +295,13 @@ describe('propagateBlockDataToCode', () => {
         propagateBlockDataToCode(new Session(), new VFile(), mdast);
         const blockNode = mdast.children[0];
         const codeNode = mdast.children[0].children[0];
-        const outputNode = mdast.children[0].children[1];
+        const outputsNode = mdast.children[0].children[1];
         expect(blockNode.visibility).toEqual(action);
         expect(codeNode.visibility).toEqual(action);
         if (has_output) {
-          expect(outputNode.visibility).toEqual(action);
+          expect(outputsNode.visibility).toEqual(action);
         } else {
-          expect(outputNode).toEqual(undefined);
+          expect(outputsNode).toEqual(undefined);
         }
       }
     }
@@ -313,7 +318,8 @@ describe('propagateBlockDataToCode', () => {
               executable: true,
             },
             {
-              type: 'output',
+              type: 'outputs',
+              children: [],
             },
           ],
           data: {
@@ -323,10 +329,10 @@ describe('propagateBlockDataToCode', () => {
       ],
     };
     propagateBlockDataToCode(new Session(), new VFile(), mdast);
-    const outputNode = mdast.children[0].children[1];
-    expect(outputNode.children?.length).toEqual(1);
-    expect(outputNode.children[0].type).toEqual('image');
-    expect(outputNode.children[0].placeholder).toBeTruthy();
+    const outputsNode = mdast.children[0].children[1];
+    expect(outputsNode.children?.length).toEqual(1);
+    expect(outputsNode.children[0].type).toEqual('image');
+    expect(outputsNode.children[0].placeholder).toBeTruthy();
   });
   it('placeholder passes with no output', async () => {
     const mdast: any = {

packages/myst-cli/src/transforms/code.ts

@@ -1,6 +1,6 @@
 import type { GenericNode, GenericParent } from 'myst-common';
 import { NotebookCellTags, RuleId, fileError, fileWarn } from 'myst-common';
-import type { Image, Output } from 'myst-spec-ext';
+import type { Image, Outputs } from 'myst-spec-ext';
 import { select, selectAll } from 'unist-util-select';
 import yaml from 'js-yaml';
 import type { VFile } from 'vfile';
@@ -156,10 +156,9 @@ export function propagateBlockDataToCode(session: ISession, vfile: VFile, mdast:
   const blocks = selectAll('block', mdast) as GenericNode[];
   blocks.forEach((block) => {
     if (!block.data) return;
-    const outputNode = select('output', block) as Output | null;
-    if (block.data.placeholder && outputNode) {
-      if (!outputNode.children) outputNode.children = [];
-      outputNode.children.push({
+    const outputsNode = select('outputs', block) as Outputs | null;
+    if (block.data.placeholder && outputsNode) {
+      outputsNode.children.push({
         type: 'image',
         placeholder: true,
         url: block.data.placeholder as string,
@@ -195,18 +194,18 @@ export function propagateBlockDataToCode(session: ISession, vfile: VFile, mdast:
           if (codeNode) codeNode.visibility = 'remove';
           break;
         case NotebookCellTags.hideOutput:
-          if (outputNode) outputNode.visibility = 'hide';
+          if (outputsNode) outputsNode.visibility = 'hide';
           break;
         case NotebookCellTags.removeOutput:
-          if (outputNode) outputNode.visibility = 'remove';
+          if (outputsNode) outputsNode.visibility = 'remove';
           break;
         default:
           session.log.debug(`tag '${tag}' is not valid in code-cell tags'`);
       }
     });
     if (!block.visibility) block.visibility = 'show';
     if (codeNode && !codeNode.visibility) codeNode.visibility = 'show';
-    if (outputNode && !outputNode.visibility) outputNode.visibility = 'show';
+    if (outputsNode && !outputsNode.visibility) outputsNode.visibility = 'show';
   });
 }
 
@@ -233,7 +232,7 @@ export function transformLiftCodeBlocksInJupytext(mdast: GenericParent) {
         child.type === 'block' &&
         child.children?.length === 2 &&
         child.children?.[0].type === 'code' &&
-        child.children?.[1].type === 'output'
+        child.children?.[1].type === 'outputs'
       ) {
         newBlocks.push(child as GenericParent);
         newBlocks.push({ type: 'block', children: [] });