Merge pull request #4112 from aloisklink/add-jsonschema-schema-for-config

Use JSON Schema to define and document `MermaidConfig`

packages/mermaid/.lintstagedrc.mjs

@@ -4,4 +4,5 @@ export default {
   'src/docs/**': ['pnpm --filter mermaid run docs:build --git'],
   'src/docs.mts': ['pnpm --filter mermaid run docs:build --git'],
   'src/(defaultConfig|config|mermaidAPI).ts': ['pnpm --filter mermaid run docs:build --git'],
+  'src/schemas/config.schema.yaml': ['pnpm --filter mermaid run types:build-config --git'],
 };

packages/mermaid/package.json

@@ -33,6 +33,8 @@
     "docs:dev:docker": "pnpm docs:pre:vitepress && concurrently \"pnpm --filter ./src/vitepress dev:docker\" \"ts-node-esm src/docs.mts --watch --vitepress\"",
     "docs:serve": "pnpm docs:build:vitepress && vitepress serve src/vitepress",
     "docs:spellcheck": "cspell --config ../../cSpell.json \"src/docs/**/*.md\"",
+    "types:build-config": "ts-node-esm --transpileOnly scripts/create-types-from-json-schema.mts",
+    "types:verify-config": "ts-node-esm scripts/create-types-from-json-schema.mts --verify",
     "release": "pnpm build",
     "prepublishOnly": "cpy '../../README.*' ./ --cwd=. && pnpm -w run build"
   },
@@ -75,6 +77,7 @@
     "web-worker": "^1.2.0"
   },
   "devDependencies": {
+    "@adobe/jsonschema2md": "^7.1.4",
     "@types/cytoscape": "^3.19.9",
     "@types/d3": "^7.4.0",
     "@types/d3-sankey": "^0.12.1",
@@ -88,6 +91,7 @@
     "@types/uuid": "^9.0.1",
     "@typescript-eslint/eslint-plugin": "^5.59.0",
     "@typescript-eslint/parser": "^5.59.0",
+    "ajv": "^8.11.2",
     "chokidar": "^3.5.3",
     "concurrently": "^8.0.1",
     "coveralls": "^3.1.1",
@@ -98,6 +102,7 @@
     "jison": "^0.4.18",
     "js-base64": "^3.7.5",
     "jsdom": "^22.0.0",
+    "json-schema-to-typescript": "^11.0.3",
     "micromatch": "^4.0.5",
     "path-browserify": "^1.0.1",
     "prettier": "^2.8.8",
@@ -110,6 +115,7 @@
     "typedoc-plugin-markdown": "^3.15.2",
     "typescript": "^5.0.4",
     "unist-util-flatmap": "^1.0.0",
+    "unist-util-visit": "^4.1.2",
     "vitepress": "^1.0.0-alpha.72",
     "vitepress-plugin-search": "^1.0.4-alpha.20"
   },

packages/mermaid/scripts/create-types-from-json-schema.mts

@@ -0,0 +1,252 @@
+/**
+ * Script to load Mermaid Config JSON Schema from YAML and to:
+ *
+ * - Validate JSON Schema
+ *
+ * Then to generate:
+ *
+ * - config.types.ts TypeScript file
+ */
+
+/* eslint-disable no-console */
+
+import { readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import assert from 'node:assert';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+
+import { load, JSON_SCHEMA } from 'js-yaml';
+import { compile, type JSONSchema } from 'json-schema-to-typescript';
+
+import _Ajv2019, { type JSONSchemaType } from 'ajv/dist/2019.js';
+
+// Workaround for wrong AJV types, see
+// https://github.com/ajv-validator/ajv/issues/2132#issuecomment-1290409907
+const Ajv2019 = _Ajv2019 as unknown as typeof _Ajv2019.default;
+
+// !!! -- The config.type.js file is created by this script -- !!!
+import type { MermaidConfig } from '../src/config.type.js';
+
+// options for running the main command
+const verifyOnly = process.argv.includes('--verify');
+/** If `true`, automatically `git add` any changes (i.e. during `pnpm run pre-commit`)*/
+const git = process.argv.includes('--git');
+
+/**
+ * All of the keys in the mermaid config that have a mermaid diagram config.
+ */
+const MERMAID_CONFIG_DIAGRAM_KEYS = [
+  'flowchart',
+  'sequence',
+  'gantt',
+  'journey',
+  'class',
+  'state',
+  'er',
+  'pie',
+  'quadrantChart',
+  'requirement',
+  'mindmap',
+  'timeline',
+  'gitGraph',
+  'c4',
+  'sankey',
+];
+
+/**
+ * Loads the MermaidConfig JSON schema YAML file.
+ *
+ * @returns The loaded JSON Schema, use {@link validateSchema} to confirm it is a valid JSON Schema.
+ */
+async function loadJsonSchemaFromYaml() {
+  const configSchemaFile = join('src', 'schemas', 'config.schema.yaml');
+  const contentsYaml = await readFile(configSchemaFile, { encoding: 'utf8' });
+  const jsonSchema = load(contentsYaml, {
+    filename: configSchemaFile,
+    // only allow JSON types in our YAML doc (will probably be default in YAML 1.3)
+    // e.g. `true` will be parsed a boolean `true`, `True` will be parsed as string `"True"`.
+    schema: JSON_SCHEMA,
+  });
+  return jsonSchema;
+}
+
+/**
+ * Asserts that the given value is a valid JSON Schema object.
+ *
+ * @param jsonSchema - The value to validate as JSON Schema 2019-09
+ * @throws {Error} if the given object is invalid.
+ */
+function validateSchema(jsonSchema: unknown): asserts jsonSchema is JSONSchemaType<MermaidConfig> {
+  if (typeof jsonSchema !== 'object') {
+    throw new Error(`jsonSchema param is not an object: actual type is ${typeof jsonSchema}`);
+  }
+  if (jsonSchema === null) {
+    throw new Error('jsonSchema param must not be null');
+  }
+  const ajv = new Ajv2019({
+    allErrors: true,
+    allowUnionTypes: true,
+    strict: true,
+  });
+
+  ajv.addKeyword({
+    keyword: 'meta:enum', // used by jsonschema2md (in docs.mts script)
+    errors: false,
+  });
+  ajv.addKeyword({
+    keyword: 'tsType', // used by json-schema-to-typescript
+    errors: false,
+  });
+
+  ajv.compile(jsonSchema);
+}
+
+/**
+ * Generate a typescript definition from a JSON Schema using json-schema-to-typescript.
+ *
+ * @param mermaidConfigSchema - The input JSON Schema.
+ */
+async function generateTypescript(mermaidConfigSchema: JSONSchemaType<MermaidConfig>) {
+  /**
+   * Replace all usages of `allOf` with `extends`.
+   *
+   * `extends` is only valid JSON Schema in very old versions of JSON Schema.
+   * However, json-schema-to-typescript creates much nicer types when using
+   * `extends`, so we should use them instead when possible.
+   *
+   * @param schema - The input schema.
+   * @returns The schema with `allOf` replaced with `extends`.
+   */
+  function replaceAllOfWithExtends(schema: JSONSchemaType<Record<string, any>>) {
+    if (schema['allOf']) {
+      const { allOf, ...schemaWithoutAllOf } = schema;
+      return {
+        ...schemaWithoutAllOf,
+        extends: allOf,
+      };
+    }
+    return schema;
+  }
+
+  /**
+   * For backwards compatibility with older Mermaid Typescript defs,
+   * we need to make all value optional instead of required.
+   *
+   * This is because the `MermaidConfig` type is used as an input, and everything is optional,
+   * since all the required values have default values.s
+   *
+   * In the future, we should make make the input to Mermaid `Partial<MermaidConfig>`.
+   *
+   * @todo TODO: Remove this function when Mermaid releases a new breaking change.
+   * @param schema - The input schema.
+   * @returns The schema with all required values removed.
+   */
+  function removeRequired(schema: JSONSchemaType<Record<string, any>>) {
+    return { ...schema, required: [] };
+  }
+
+  /**
+   * This is a temporary hack to control the order the types are generated in.
+   *
+   * By default, json-schema-to-typescript outputs the $defs in the order they
+   * are used, then any unused schemas at the end.
+   *
+   * **The only purpose of this function is to make the `git diff` simpler**
+   * **We should remove this later to simplify the code**
+   *
+   * @todo TODO: Remove this function in a future PR.
+   * @param schema - The input schema.
+   * @returns The schema with all `$ref`s removed.
+   */
+  function unrefSubschemas(schema: JSONSchemaType<Record<string, any>>) {
+    return {
+      ...schema,
+      properties: Object.fromEntries(
+        Object.entries(schema.properties).map(([key, propertySchema]) => {
+          if (MERMAID_CONFIG_DIAGRAM_KEYS.includes(key)) {
+            const { $ref, ...propertySchemaWithoutRef } = propertySchema as JSONSchemaType<unknown>;
+            if ($ref === undefined) {
+              throw Error(
+                `subSchema ${key} is in MERMAID_CONFIG_DIAGRAM_KEYS but does not have a $ref field`
+              );
+            }
+            const [
+              _root, // eslint-disable-line @typescript-eslint/no-unused-vars
+              _defs, // eslint-disable-line @typescript-eslint/no-unused-vars
+              defName,
+            ] = $ref.split('/');
+            return [
+              key,
+              {
+                ...propertySchemaWithoutRef,
+                tsType: defName,
+              },
+            ];
+          }
+          return [key, propertySchema];
+        })
+      ),
+    };
+  }
+
+  assert.ok(mermaidConfigSchema.$defs);
+  const modifiedSchema = {
+    ...unrefSubschemas(removeRequired(mermaidConfigSchema)),
+
+    $defs: Object.fromEntries(
+      Object.entries(mermaidConfigSchema.$defs).map(([key, subSchema]) => {
+        return [key, removeRequired(replaceAllOfWithExtends(subSchema))];
+      })
+    ),
+  };
+
+  const typescriptFile = await compile(
+    modifiedSchema as JSONSchema, // json-schema-to-typescript only allows JSON Schema 4 as input type
+    'MermaidConfig',
+    {
+      additionalProperties: false, // in JSON Schema 2019-09, these are called `unevaluatedProperties`
+      unreachableDefinitions: true, // definition for FontConfig is unreachable
+    }
+  );
+
+  // TODO, should we somehow use the functions from `docs.mts` instead?
+  if (verifyOnly) {
+    const originalFile = await readFile('./src/config.type.ts', { encoding: 'utf-8' });
+    if (typescriptFile !== originalFile) {
+      console.error('❌ Error: ./src/config.type.ts will be changed.');
+      console.error("Please run 'pnpm run --filter mermaid types:build-config' to update this");
+      process.exitCode = 1;
+    } else {
+      console.log('✅ ./src/config.type.ts will be unchanged');
+    }
+  } else {
+    console.log('Writing typescript file to ./src/config.type.ts');
+    await writeFile('./src/config.type.ts', typescriptFile, { encoding: 'utf8' });
+    if (git) {
+      console.log('📧 Git: Adding ./src/config.type.ts changed');
+      await promisify(execFile)('git', ['add', './src/config.type.ts']);
+    }
+  }
+}
+
+/** Main function */
+async function main() {
+  if (verifyOnly) {
+    console.log(
+      'Verifying that ./src/config.type.ts is in sync with src/schemas/config.schema.yaml'
+    );
+  }
+
+  const configJsonSchema = await loadJsonSchemaFromYaml();
+
+  validateSchema(configJsonSchema);
+
+  // Generate types from JSON Schema
+  await generateTypescript(configJsonSchema);
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});

packages/mermaid/src/docs.mts

@@ -34,7 +34,7 @@ import { readFileSync, writeFileSync, mkdirSync, existsSync, rmSync, rmdirSync }
 import { exec } from 'child_process';
 import { globby } from 'globby';
 import { JSDOM } from 'jsdom';
-import type { Code, Root } from 'mdast';
+import type { Code, ListItem, Root, Text } from 'mdast';
 import { posix, dirname, relative, join } from 'path';
 import prettier from 'prettier';
 import { remark } from 'remark';
@@ -44,6 +44,7 @@ import chokidar from 'chokidar';
 import mm from 'micromatch';
 // @ts-ignore No typescript declaration file
 import flatmap from 'unist-util-flatmap';
+import { visit } from 'unist-util-visit';
 
 const MERMAID_MAJOR_VERSION = (
   JSON.parse(readFileSync('../mermaid/package.json', 'utf8')).version as string
@@ -150,6 +151,7 @@ const copyTransformedContents = (filename: string, doCopy = false, transformedCo
   }
 
   filesTransformed.add(fileInFinalDocDir);
+
   if (doCopy) {
     writeFileSync(fileInFinalDocDir, newBuffer);
   }
@@ -321,6 +323,123 @@ const transformMarkdown = (file: string) => {
   copyTransformedContents(file, !verifyOnly, formatted);
 };
 
+import { load, JSON_SCHEMA } from 'js-yaml';
+// @ts-ignore: we're importing internal jsonschema2md functions
+import { default as schemaLoader } from '@adobe/jsonschema2md/lib/schemaProxy.js';
+// @ts-ignore: we're importing internal jsonschema2md functions
+import { default as traverseSchemas } from '@adobe/jsonschema2md/lib/traverseSchema.js';
+// @ts-ignore: we're importing internal jsonschema2md functions
+import { default as buildMarkdownFromSchema } from '@adobe/jsonschema2md/lib/markdownBuilder.js';
+// @ts-ignore: we're importing internal jsonschema2md functions
+import { default as jsonSchemaReadmeBuilder } from '@adobe/jsonschema2md/lib/readmeBuilder.js';
+
+/**
+ * Transforms the given JSON Schema into Markdown documentation
+ */
+async function transormJsonSchema(file: string) {
+  const yamlContents = readSyncedUTF8file(file);
+  const jsonSchema = load(yamlContents, {
+    filename: file,
+    // only allow JSON types in our YAML doc (will probably be default in YAML 1.3)
+    // e.g. `true` will be parsed a boolean `true`, `True` will be parsed as string `"True"`.
+    schema: JSON_SCHEMA,
+  });
+
+  /** Location of the `schema.yaml` files */
+  const SCHEMA_INPUT_DIR = 'src/schemas/';
+  /**
+   * Location to store the generated `schema.json` file for the website
+   *
+   * Because Vitepress doesn't handle bundling `.json` files properly, we need
+   * to instead place it into a `public/` subdirectory.
+   */
+  const SCHEMA_OUTPUT_DIR = 'src/docs/public/schemas/';
+  const VITEPRESS_PUBLIC_DIR = 'src/docs/public';
+  /**
+   * Location to store the generated Schema Markdown docs.
+   * Links to JSON Schemas should automatically be rewritten to point to
+   * `SCHEMA_OUTPUT_DIR`.
+   */
+  const SCHEMA_MARKDOWN_OUTPUT_DIR = join('src', 'docs', 'config', 'schema-docs');
+
+  // write .schema.json files
+  const jsonFileName = file
+    .replace('.schema.yaml', '.schema.json')
+    .replace(SCHEMA_INPUT_DIR, SCHEMA_OUTPUT_DIR);
+  copyTransformedContents(jsonFileName, !verifyOnly, JSON.stringify(jsonSchema, undefined, 2));
+
+  const schemas = traverseSchemas([schemaLoader()(jsonFileName, jsonSchema)]);
+
+  // ignore output of this function
+  // for some reason, without calling this function, we get some broken links
+  // this is probably a bug in @adobe/jsonschema2md
+  jsonSchemaReadmeBuilder({ readme: true })(schemas);
+
+  // write Markdown files
+  const markdownFiles = buildMarkdownFromSchema({
+    header: true,
+    // links,
+    includeProperties: ['tsType'], // Custom TypeScript type
+    exampleFormat: 'json',
+    // skipProperties,
+    /**
+     * Automatically rewrite schema paths passed to `schemaLoader`
+     * (e.g. src/docs/schemas/config.schema.json)
+     * to relative links (e.g. /schemas/config.schema.json)
+     *
+     * See https://vitepress.vuejs.org/guide/asset-handling
+     *
+     * @param origin - Original schema path (relative to this script).
+     * @returns New absolute Vitepress path to schema
+     */
+    rewritelinks: (origin: string) => {
+      return `/${relative(VITEPRESS_PUBLIC_DIR, origin)}`;
+    },
+  })(schemas);
+
+  for (const [name, markdownAst] of Object.entries(markdownFiles)) {
+    /*
+     * Converts list entries of shape '- tsType: () => Partial<FontConfig>'
+     * into '- tsType: `() => Partial<FontConfig>`' (e.g. escaping with back-ticks),
+     * as otherwise VitePress doesn't like the <FontConfig> bit.
+     */
+    visit(markdownAst as Root, 'listItem', (listEntry: ListItem) => {
+      let listText: Text;
+      const blockItem = listEntry.children[0];
+      if (blockItem.type === 'paragraph' && blockItem.children[0].type === 'text') {
+        listText = blockItem.children[0];
+      } // @ts-expect-error: MD AST output from @adobe/jsonschema2md is technically wrong
+      else if (blockItem.type === 'text') {
+        listText = blockItem;
+      } else {
+        return; // skip
+      }
+
+      if (listText.value.startsWith('tsType: ')) {
+        listText.value = listText.value.replace(/tsType: (.*)/g, 'tsType: `$1`');
+      }
+    });
+
+    const transformed = remark()
+      .use(remarkGfm)
+      .use(remarkFrontmatter, ['yaml']) // support YAML front-matter in Markdown
+      .use(transformMarkdownAst, {
+        // mermaid project specific plugin
+        originalFilename: file,
+        addAutogeneratedWarning: !noHeader,
+        removeYAML: !noHeader,
+      })
+      .stringify(markdownAst as Root);
+
+    const formatted = prettier.format(transformed, {
+      parser: 'markdown',
+      ...prettierConfig,
+    });
+    const newFileName = join(SCHEMA_MARKDOWN_OUTPUT_DIR, `${name}.md`);
+    copyTransformedContents(newFileName, !verifyOnly, formatted);
+  }
+}
+
 /**
  * Transform an HTML file and write the transformed file to the directory for published
  * documentation
@@ -388,6 +507,14 @@ const main = async () => {
   const sourceDirGlob = posix.join('.', SOURCE_DOCS_DIR, '**');
   const action = verifyOnly ? 'Verifying' : 'Transforming';
 
+  if (vitepress) {
+    console.log(`${action} 1 .schema.yaml file`);
+    await transormJsonSchema('src/schemas/config.schema.yaml');
+  } else {
+    // skip because this creates so many Markdown files that it lags git
+    console.log('Skipping 1 .schema.yaml file');
+  }
+
   const mdFileGlobs = getGlobs([posix.join(sourceDirGlob, '*.md')]);
   const mdFiles = await getFilesFromGlobs(mdFileGlobs);
   console.log(`${action} ${mdFiles.length} markdown files...`);

packages/mermaid/src/docs/.vitepress/config.ts

@@ -155,6 +155,7 @@ function sidebarConfig() {
         { text: 'Tutorials', link: '/config/Tutorials' },
         { text: 'API-Usage', link: '/config/usage' },
         { text: 'Mermaid API Configuration', link: '/config/setup/README' },
+        { text: 'Mermaid Configuration Options', link: '/config/schema-docs/config' },
         { text: 'Directives', link: '/config/directives' },
         { text: 'Theming', link: '/config/theming' },
         { text: 'Accessibility', link: '/config/accessibility' },

packages/mermaid/src/docs/.vitepress/mermaid-markdown-all.ts

@@ -54,6 +54,15 @@ const MermaidExample = async (md: MarkdownRenderer) => {
       return `<div class="tip custom-block"><p class="custom-block-title">NOTE</p><p>${token.content}}</p></div>`;
     }
 
+    if (token.info.trim() === 'regexp') {
+      // shiki doesn't yet support regexp code blocks, but the javascript
+      // one still makes RegExes look good
+      token.info = 'javascript';
+      // use trimEnd to move trailing `\n` outside if the JavaScript regex `/` block
+      token.content = `/${token.content.trimEnd()}/\n`;
+      return defaultRenderer(tokens, index, options, env, slf);
+    }
+
     if (token.info.trim() === 'jison') {
       return `<div class="language-">
       <button class="copy"></button>