feat: Record time difference in tests

.github/workflows/e2e.yml

@@ -69,6 +69,15 @@ jobs:
           start: pnpm run dev
           wait-on: 'http://localhost:9000'
           browser: chrome
+          spec: |
+            cypress/integration/rendering/classDiagram.spec.js
+            cypress/integration/rendering/flowchart-v2.spec.js
+
+      - name: Move runtime data
+        if: ${{ steps.cache-snapshot.outputs.cache-hit != 'true' }}
+        run: |
+          mkdir -p cypress/snapshots/runtimes
+          mv cypress/runtimes cypress/snapshots/runtimes/base
 
   e2e:
     runs-on: ubuntu-latest
@@ -140,6 +149,9 @@ jobs:
           # e.g. if this action was run from a fork
           record: ${{ secrets.CYPRESS_RECORD_KEY != '' }}
           parallel: ${{ secrets.CYPRESS_RECORD_KEY != '' }}
+          spec: |
+            cypress/integration/rendering/classDiagram.spec.js
+            cypress/integration/rendering/flowchart-v2.spec.js
         env:
           CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
           VITEST_COVERAGE: true
@@ -171,6 +183,16 @@ jobs:
     runs-on: ubuntu-latest
     if: ${{ always() }}
     steps:
+      - uses: actions/checkout@v4
+
+      - uses: pnpm/action-setup@v2
+        # uses version from "packageManager" field in package.json
+
+      - name: Setup Node.js 18.x
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18.x
+
       # Download all snapshot artifacts and merge them into a single folder
       - name: Download All Artifacts
         uses: actions/download-artifact@v4
@@ -179,6 +201,32 @@ jobs:
           pattern: snapshots-*
           merge-multiple: true
 
+      - name: Build
+        id: runtime
+        if: ${{ needs.e2e.result != 'failure' && github.event_name == 'pull_request' }}
+        run: |
+          ls -l cypress/snapshots/runtimes
+          mv cypress/snapshots/runtimes cypress/snapshots/runtimes/head
+          ls -l cypress/snapshots/runtimes
+          ls -l cypress/snapshots/runtimes/base
+          ls -l cypress/snapshots/runtimes/head
+          tree cypress/snapshots/runtimes
+          npm config set ignore-scripts true
+          pnpm install --frozen-lockfile
+          {
+            echo 'runtime_diff<<EOF'
+            npx tsx scripts/runTime.ts
+            echo EOF
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Comment PR runtime difference
+        if: ${{ github.event_name == 'pull_request' }}
+        uses: thollander/actions-comment-pull-request@v2
+        with:
+          message: |
+            ${{ steps.runtime.outputs.runtime_diff }}
+          comment_tag: size-diff
+
       # For successful push events, we save the snapshots cache
       - name: Save snapshots cache
         id: cache-upload

.gitignore

@@ -29,6 +29,7 @@ Gemfile.lock
 
 cypress/screenshots/
 cypress/snapshots/
+cypress/runtimes/
 
 # eslint --cache file
 .eslintcache

cSpell.json

@@ -122,6 +122,7 @@
     "rehype",
     "roledescription",
     "rozhkov",
+    "runtimes",
     "sandboxed",
     "sankey",
     "setupgraphviewbox",

cypress.config.ts

@@ -1,4 +1,6 @@
 import { defineConfig } from 'cypress';
+import fs from 'fs';
+import path from 'path';
 import { addMatchImageSnapshotPlugin } from 'cypress-image-snapshot/plugin';
 import coverage from '@cypress/code-coverage/task';
 import eyesPlugin from '@applitools/eyes-cypress';
@@ -17,6 +19,19 @@ export default eyesPlugin(
           }
           return launchOptions;
         });
+        on('task', {
+          recordRenderTime({ fileName, testName, timeTaken }) {
+            const resultsPath = path.join('cypress', 'runtimes');
+            if (!fs.existsSync(resultsPath)) {
+              fs.mkdirSync(resultsPath, { recursive: true });
+            }
+            fs.appendFileSync(
+              path.join(resultsPath, `${fileName}.csv`),
+              `${testName},${timeTaken}\n`
+            );
+            return true;
+          },
+        });
         addMatchImageSnapshotPlugin(on, config);
         // copy any needed variables from process.env to config.env
         config.env.useAppli = process.env.USE_APPLI ? true : false;

cypress/helpers/util.ts

@@ -110,6 +110,14 @@ export const openURLAndVerifyRendering = (
 
   cy.visit(url);
   cy.window().should('have.property', 'rendered', true);
+  cy.window().then((win) => {
+    cy.task('recordRenderTime', {
+      fileName: Cypress.spec.name,
+      testName: name,
+      // @ts-ignore Dynamically added property.
+      timeTaken: win.renderTime,
+    });
+  });
   cy.get('svg').should('be.visible');
 
   if (validation) {

cypress/platform/viewer.js

@@ -8,6 +8,7 @@ function b64ToUtf8(str) {
 
 // Adds a rendered flag to window when rendering is done, so cypress can wait for it.
 function markRendered() {
+  window.renderTime = Date.now() - window.loadTime;
   if (window.Cypress) {
     window.rendered = true;
   }
@@ -131,6 +132,7 @@ if (typeof document !== 'undefined') {
   window.addEventListener(
     'load',
     function () {
+      this.window.loadTime = Date.now();
       if (this.location.href.match('xss.html')) {
         this.console.log('Using api');
         void contentLoadedApi().finally(markRendered);

scripts/runTime.ts

@@ -0,0 +1,95 @@
+/* eslint-disable no-console */
+import { readFile } from 'fs/promises';
+import { globby } from 'globby';
+import { markdownTable } from 'markdown-table';
+
+interface RunTimes {
+  [key: string]: number;
+}
+interface TestResult {
+  [key: string]: RunTimes;
+}
+
+const getRuntimes = (csv: string): RunTimes => {
+  const lines = csv.split('\n');
+  const runtimes: RunTimes = {};
+  for (const line of lines) {
+    const [testName, timeTaken] = line.split(',');
+    if (testName && timeTaken) {
+      runtimes[testName] = Number(timeTaken);
+    }
+  }
+  return runtimes;
+};
+
+const readStats = async (path: string): Promise<TestResult> => {
+  const files = await globby(path);
+  const contents = await Promise.all(
+    files.map(async (file) => [file, await readFile(file, 'utf-8')])
+  );
+  const sizes = contents.map(([file, content]) => [file.split('/').pop(), getRuntimes(content)]);
+  return Object.fromEntries(sizes);
+};
+
+const percentChangeThreshold = 5;
+const percentageDifference = (
+  oldValue: number,
+  newValue: number
+): { change: string; crossedThreshold: boolean } => {
+  const difference = Math.abs(newValue - oldValue);
+  const avg = (newValue + oldValue) / 2;
+  const percentage = (difference / avg) * 100;
+  const roundedPercentage = percentage.toFixed(2); // Round to two decimal places
+  if (roundedPercentage === '0.00') {
+    return { change: '0.00%', crossedThreshold: false };
+  }
+  const sign = newValue > oldValue ? '+' : '-';
+  return {
+    change: `${sign}${roundedPercentage}%`,
+    crossedThreshold: percentage > percentChangeThreshold,
+  };
+};
+
+const main = async () => {
+  const oldStats = await readStats('./cypress/snapshots/runtimes/base/**/*.csv');
+  const newStats = await readStats('./cypress/snapshots/runtimes/head/**/*.csv');
+  const fullData: string[][] = [];
+  const changed: string[][] = [];
+  for (const [fileName, runtimes] of Object.entries(newStats)) {
+    const oldStat = oldStats[fileName];
+    if (!oldStat) {
+      continue;
+    }
+    for (const [testName, timeTaken] of Object.entries(runtimes)) {
+      const oldTimeTaken = oldStat[testName];
+      if (!oldTimeTaken) {
+        continue;
+      }
+      const delta = timeTaken - oldTimeTaken;
+      const { change, crossedThreshold } = percentageDifference(oldTimeTaken, timeTaken);
+      const out = [
+        fileName,
+        testName,
+        oldTimeTaken.toString(),
+        timeTaken.toString(),
+        change,
+        `${delta.toString()}ms`,
+      ];
+      if (crossedThreshold) {
+        changed.push(out);
+        console.warn(`${testName} (${fileName}): ${timeTaken}ms (${delta}ms, ${change})`);
+      }
+      fullData.push(out);
+    }
+  }
+  const headers = ['File', 'Test', 'Old Time', 'New Time', '% Change', 'Difference'];
+  console.log(markdownTable([headers, ...changed]));
+  console.log(`
+  <details>
+  <summary>Full Data</summary>
+  ${markdownTable([headers, ...fullData])}
+</details>
+`);
+};
+
+void main().catch((e) => console.error(e));