☣️ Refactor transformMdast and postProcessMdast to a single pipeline

packages/myst-cli/src/build/utils/getFileContent.ts

@@ -1,16 +1,20 @@
-import { resolve } from 'node:path';
+import path from 'node:path';
 import { plural } from 'myst-common';
 import { tic } from 'myst-cli-utils';
 import type { LinkTransformer } from 'myst-transforms';
 import { combineProjectCitationRenderers } from '../../process/citations.js';
 import { loadFile, selectFile } from '../../process/file.js';
 import { loadReferences } from '../../process/loadReferences.js';
 import type { TransformFn } from '../../process/mdast.js';
-import { postProcessMdast, transformMdast } from '../../process/mdast.js';
-import { loadProject, selectPageReferenceStates } from '../../process/site.js';
+import { transformMdast } from '../../process/mdast.js';
+import { loadProject, selectPageReferenceStates, makeBarrier } from '../../process/site.js';
+import { buildIndexTransform, MultiPageReferenceResolver } from 'myst-transforms';
 import type { ISession } from '../../session/types.js';
 import { selectors } from '../../store/index.js';
 import type { ImageExtensions } from '../../utils/resolveExtension.js';
+import { castSession } from '../../session/cache.js';
+import { VFile } from 'vfile';
+import { logMessagesFromVFile } from '../../utils/logging.js';
 
 export async function getFileContent(
   session: ISession,
@@ -34,13 +38,13 @@ export async function getFileContent(
   },
 ) {
   const toc = tic();
-  files = files.map((file) => resolve(file));
-  projectPath = projectPath ?? resolve('.');
+  files = files.map((file) => path.resolve(file));
+  projectPath = projectPath ?? path.resolve('.');
   const { project, pages } = await loadProject(session, projectPath);
   const projectFiles = pages.map((page) => page.file).filter((file) => !files.includes(file));
   await Promise.all([
     // Load all citations (.bib)
-    ...project.bibliography.map((path) => loadFile(session, path, projectPath, '.bib')),
+    ...project.bibliography.map((bib) => loadFile(session, bib, projectPath, '.bib')),
     // Load all content (.md, .tex, .myst.json, or .ipynb)
     ...[...files, ...projectFiles].map((file, ind) => {
       const preFrontmatter = Array.isArray(preFrontmatters)
@@ -60,11 +64,45 @@ export async function getFileContent(
   // Keep 'files' indices consistent in 'allFiles' as index is used for other fields.
   const allFiles = [...files, ...projectFiles, ...projectParts];
 
+  const { wait: waitReferencing, promise: referencingPromise } = makeBarrier(allFiles.length);
+  const { wait: waitIndexing, promise: indexingPromise } = makeBarrier(allFiles.length);
+
+  // TODO: maybe move transformMdast into a multi-file function
+  const referenceStateContext: {
+    referenceStates: ReturnType<typeof selectPageReferenceStates>;
+  } = { referenceStates: [] };
+  const referencingPages = allFiles.map((file) => {
+    return { file };
+  });
+  referencingPromise.then(() => {
+    const pageReferenceStates = selectPageReferenceStates(session, referencingPages);
+    referenceStateContext.referenceStates.push(...pageReferenceStates);
+  });
+  indexingPromise.then(() => {
+    const cache = castSession(session);
+    referencingPages.forEach((page) => {
+      const fileState = cache.$internalReferences[page.file];
+      if (!fileState) return;
+      const { mdast } = cache.$getMdast(page.file)?.post ?? {};
+      if (!mdast) return;
+      const vfile = new VFile();
+      vfile.path = page.file;
+      buildIndexTransform(
+        mdast,
+        vfile,
+        fileState,
+        new MultiPageReferenceResolver(referenceStateContext.referenceStates, fileState.filePath),
+      );
+      logMessagesFromVFile(session, vfile);
+    });
+  });
   await Promise.all(
     allFiles.map(async (file, ind) => {
       const pageSlug = pages.find((page) => page.file === file)?.slug;
       const titleDepth = typeof titleDepths === 'number' ? titleDepths : titleDepths?.[ind];
       await transformMdast(session, {
+        referenceResolutionBlocker: waitReferencing,
+        indexGenerationBlocker: waitIndexing,
         file,
         imageExtensions,
         projectPath,
@@ -74,24 +112,13 @@ export async function getFileContent(
         titleDepth,
         extraTransforms,
         execute,
-      });
-    }),
-  );
-  const pageReferenceStates = selectPageReferenceStates(
-    session,
-    allFiles.map((file) => {
-      return { file };
-    }),
-  );
-  await Promise.all(
-    [...files, ...projectParts].map(async (file) => {
-      await postProcessMdast(session, {
-        file,
         extraLinkTransformers,
-        pageReferenceStates,
+        runPostProcess: [...files, ...projectParts].includes(file),
+        referenceStateContext,
       });
     }),
   );
+
   const selectedFiles = await Promise.all(
     files.map(async (file) => {
       const selectedFile = selectFile(session, file);