wip: add markdown rendering to content layer

benchmark/bench/memory.js

@@ -18,7 +18,7 @@ export async function run(projectDir, outputFile) {
 	const outputFilePath = fileURLToPath(outputFile);
 
 	console.log('Building and benchmarking...');
-	await execaCommand(`node --expose-gc --max_old_space_size=256 ${astroBin} build`, {
+	await execaCommand(`node --expose-gc --max_old_space_size=10000 ${astroBin} build`, {
 		cwd: root,
 		stdio: 'inherit',
 		env: {

benchmark/make-project/markdown-cc1.js

@@ -0,0 +1,58 @@
+import fs from 'node:fs/promises';
+import { loremIpsumMd } from './_util.js';
+
+/**
+ * @param {URL} projectDir
+ */
+export async function run(projectDir) {
+	await fs.rm(projectDir, { recursive: true, force: true });
+	await fs.mkdir(new URL('./src/pages/blog', projectDir), { recursive: true });
+	await fs.mkdir(new URL('./src/content/blog', projectDir), { recursive: true });
+
+	const promises = [];
+
+
+	for (let i = 0; i < 10000; i++) {
+		const content = `\
+# Article ${i}
+
+${loremIpsumMd}
+`;
+		promises.push(
+			fs.writeFile(new URL(`./src/content/blog/article-${i}.md`, projectDir), content, 'utf-8')
+		);
+	}
+
+
+	await fs.writeFile(
+		new URL(`./src/pages/blog/[...slug].astro`, projectDir),
+		`\
+---
+import { getCollection } from 'astro:content';
+export async function getStaticPaths() {
+  const blogEntries = await getCollection('blog');
+  return blogEntries.map(entry => ({
+    params: { slug: entry.slug }, props: { entry },
+  }));
+}
+const { entry } = Astro.props;
+const { Content } = await entry.render();
+---
+<h1>{entry.data.title}</h1>
+<Content />
+`,
+		'utf-8'
+	);
+
+	await Promise.all(promises);
+
+	await fs.writeFile(
+		new URL('./astro.config.js', projectDir),
+		`\
+import { defineConfig } from 'astro/config';
+
+export default defineConfig({
+});`,
+		'utf-8'
+	);
+}

benchmark/make-project/markdown-cc2.js

@@ -0,0 +1,74 @@
+import fs from 'node:fs/promises';
+import { loremIpsumMd } from './_util.js';
+
+/**
+ * @param {URL} projectDir
+ */
+export async function run(projectDir) {
+	await fs.rm(projectDir, { recursive: true, force: true });
+	await fs.mkdir(new URL('./src/pages/blog', projectDir), { recursive: true });
+	await fs.mkdir(new URL('./data/blog', projectDir), { recursive: true });
+	await fs.mkdir(new URL('./src/content', projectDir), { recursive: true });
+
+	const promises = [];
+
+	for (let i = 0; i < 10000; i++) {
+		const content = `\
+# Article ${i}
+
+${loremIpsumMd}
+`;
+		promises.push(
+			fs.writeFile(new URL(`./data/blog/article-${i}.md`, projectDir), content, 'utf-8')
+		);
+	}
+
+	await fs.writeFile(
+		new URL(`./src/content/config.ts`, projectDir),
+		/*ts */ `
+		import { defineCollection, z } from 'astro:content';
+		import { glob } from 'astro/loaders';
+
+		const blog = defineCollection({
+			type: 'experimental_data',
+			loader: glob({ pattern: '*', base: './data/blog' }),
+		});
+		
+		export const collections = { blog }
+
+		`
+	);
+
+	await fs.writeFile(
+		new URL(`./src/pages/blog/[...slug].astro`, projectDir),
+		`\
+---
+import { getCollection } from 'astro:content';
+export async function getStaticPaths() {
+  const blogEntries = await getCollection('blog');
+  return blogEntries.map(entry => ({
+    params: { slug: entry.id }, props: { entry },
+  }));
+}
+const { entry } = Astro.props;
+const { Content } = await entry.render();
+
+---
+<h1>{entry.data.title}</h1>
+<Content />
+`,
+		'utf-8'
+	);
+
+	await Promise.all(promises);
+
+	await fs.writeFile(
+		new URL('./astro.config.js', projectDir),
+		`\
+import { defineConfig } from 'astro/config';
+
+export default defineConfig({
+});`,
+		'utf-8'
+	);
+}

packages/astro/package.json

@@ -185,6 +185,7 @@
     "vite": "^5.3.2",
     "vitefu": "^0.2.5",
     "which-pm": "^2.2.0",
+    "xxhash-wasm": "^1.0.2",
     "yargs-parser": "^21.1.1",
     "zod": "^3.23.8",
     "zod-to-json-schema": "^3.23.1",

packages/astro/src/@types/astro.ts

@@ -48,6 +48,7 @@ import type {
 } from '../transitions/events.js';
 import type { DeepPartial, OmitIndexSignature, Simplify } from '../type-utils.js';
 import type { SUPPORTED_MARKDOWN_FILE_EXTENSIONS } from './../core/constants.js';
+import type { DataEntry } from '../content/data-store.js';
 
 export type { AstroIntegrationLogger, ToolbarServerHelpers };
 
@@ -2276,6 +2277,12 @@ export type DataEntryModule = {
 	};
 };
 
+export interface RenderResult {
+	code: string;
+	metadata?: Record<string, any>;
+}
+export type RenderFunction = (entry: DataEntry) => Promise<RenderResult>;
+
 export interface ContentEntryType {
 	extensions: string[];
 	getEntryInfo(params: {
@@ -2291,6 +2298,8 @@ export interface ContentEntryType {
 		}
 	): rollup.LoadResult | Promise<rollup.LoadResult>;
 	contentModuleTypes?: string;
+	getRenderFunction?(settings: AstroSettings): Promise<RenderFunction>;
+
 	/**
 	 * Handle asset propagation for rendered content to avoid bleed.
 	 * Ex. MDX content can import styles and scripts, so `handlePropagation` should be true.

packages/astro/src/content/data-store.ts

@@ -7,6 +7,11 @@ export interface DataEntry {
 	data: Record<string, unknown>;
 	filePath?: string;
 	body?: string;
+	digest?: number | string;
+	rendered?: {
+		html: string;
+		metadata?: Record<string, unknown>;
+	};
 }
 
 export class DataStore {
@@ -90,25 +95,38 @@ export class DataStore {
 			entries: () => this.entries(collectionName),
 			values: () => this.values(collectionName),
 			keys: () => this.keys(collectionName),
-			set: (key, data, body, filePath) => {
+			set: ({ id: key, data, body, filePath, digest, rendered }) => {
 				if (!key) {
-					throw new Error(`Key must be a non-empty string`);
+					throw new Error(`ID must be a non-empty string`);
 				}
 				const id = String(key);
+				if (digest) {
+					const existing = this.get<DataEntry>(collectionName, id);
+					if (existing && existing.digest === digest) {
+						return false;
+					}
+				}
 				const entry: DataEntry = {
 					id,
 					data,
 				};
 				// We do it like this so we don't waste space stringifying
-				// the body and filePath if they are not set
+				// the fields if they are not set
 				if (body) {
 					entry.body = body;
 				}
 				if (filePath) {
 					entry.filePath = filePath;
 				}
+				if (digest) {
+					entry.digest = digest;
+				}
+				if (rendered) {
+					entry.rendered = rendered;
+				}
 
 				this.set(collectionName, id, entry);
+				return true;
 			},
 			delete: (key: string) => this.delete(collectionName, key),
 			clear: () => this.clear(collectionName),
@@ -183,7 +201,17 @@ export class DataStore {
 export interface ScopedDataStore {
 	get: (key: string) => DataEntry | undefined;
 	entries: () => Array<[id: string, DataEntry]>;
-	set: (key: string, data: Record<string, unknown>, body?: string, filePath?: string) => void;
+	set: (opts: {
+		id: string;
+		data: Record<string, unknown>;
+		body?: string;
+		filePath?: string;
+		digest?: number | string;
+		rendered?: {
+			html: string;
+			metadata?: Record<string, unknown>;
+		};
+	}) => boolean;
 	values: () => Array<DataEntry>;
 	keys: () => Array<string>;
 	delete: (key: string) => void;

packages/astro/src/content/loaders/file.ts

@@ -37,16 +37,16 @@ export function file(fileName: string): Loader {
 					logger.error(`Item in ${fileName} is missing an id or slug field.`);
 					continue;
 				}
-				const item = await parseData({ id, data: rawItem, filePath });
-				store.set(id, item, undefined, filePath);
+				const data = await parseData({ id, data: rawItem, filePath });
+				store.set({ id, data, filePath });
 			}
 		} else if (typeof json === 'object') {
 			const entries = Object.entries<Record<string, unknown>>(json);
 			logger.debug(`Found object with ${entries.length} entries in ${fileName}`);
 			store.clear();
 			for (const [id, rawItem] of entries) {
-				const item = await parseData({ id, data: rawItem, filePath });
-				store.set(id, item);
+				const data = await parseData({ id, data: rawItem, filePath });
+				store.set({ id, data });
 			}
 		} else {
 			logger.error(`Invalid data in ${fileName}. Must be an array or object.`);