Merge pull request #5240 from mermaid-js/sidv/runTimeDiffTest

Test

.github/workflows/e2e.yml

@@ -69,15 +69,11 @@ jobs:
           start: pnpm run dev
           wait-on: 'http://localhost:9000'
           browser: chrome
-          spec: |
-            cypress/integration/rendering/classDiagram.spec.js
-            cypress/integration/rendering/flowchart-v2.spec.js
 
       - name: Move runtime data
         if: ${{ steps.cache-snapshot.outputs.cache-hit != 'true' }}
         run: |
-          mkdir -p cypress/snapshots/runtimes
+          mv cypress/snapshots/runtimes/current cypress/snapshots/runtimes/base
-          mv cypress/runtimes cypress/snapshots/runtimes/base
 
   e2e:
     runs-on: ubuntu-latest
@@ -149,9 +145,6 @@ jobs:
           # e.g. if this action was run from a fork
           record: ${{ secrets.CYPRESS_RECORD_KEY != '' }}
           parallel: ${{ secrets.CYPRESS_RECORD_KEY != '' }}
-          spec: |
-            cypress/integration/rendering/classDiagram.spec.js
-            cypress/integration/rendering/flowchart-v2.spec.js
         env:
           CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
           VITEST_COVERAGE: true
@@ -205,17 +198,12 @@ jobs:
         id: runtime
         if: ${{ needs.e2e.result != 'failure' && github.event_name == 'pull_request' }}
         run: |
-          ls -l cypress/snapshots/runtimes
+          mv ./snapshots/runtimes/current ./snapshots/runtimes/head
-          mv cypress/snapshots/runtimes cypress/snapshots/runtimes/head
-          ls -l cypress/snapshots/runtimes
-          ls -l cypress/snapshots/runtimes/base
-          ls -l cypress/snapshots/runtimes/head
-          tree cypress/snapshots/runtimes
           npm config set ignore-scripts true
           pnpm install --frozen-lockfile
           {
             echo 'runtime_diff<<EOF'
-            npx tsx scripts/runTime.ts
+            npx tsx scripts/runTime.ts ./snapshots
             echo EOF
           } >> "$GITHUB_OUTPUT"
 

cypress.config.ts

@@ -21,7 +21,7 @@ export default eyesPlugin(
         });
         on('task', {
           recordRenderTime({ fileName, testName, timeTaken }) {
-            const resultsPath = path.join('cypress', 'runtimes');
+            const resultsPath = path.join('cypress', 'snapshots', 'runtimes', 'current');
             if (!fs.existsSync(resultsPath)) {
               fs.mkdirSync(resultsPath, { recursive: true });
             }

scripts/runTime.ts

@@ -1,7 +1,6 @@
 /* eslint-disable no-console */
 import { readFile } from 'fs/promises';
 import { globby } from 'globby';
-import { markdownTable } from 'markdown-table';
 
 interface RunTimes {
   [key: string]: number;
@@ -51,10 +50,14 @@ const percentageDifference = (
 };
 
 const main = async () => {
-  const oldStats = await readStats('./cypress/snapshots/runtimes/base/**/*.csv');
+  const base = process.argv[2] || './cypress/snapshots';
-  const newStats = await readStats('./cypress/snapshots/runtimes/head/**/*.csv');
+  const oldStats = await readStats(`${base}/runtimes/base/**/*.csv`);
+  const newStats = await readStats(`${base}/runtimes/head/**/*.csv`);
   const fullData: string[][] = [];
   const changed: string[][] = [];
+  let oldRuntimeSum = 0;
+  let newRuntimeSum = 0;
+  let testCount = 0;
   for (const [fileName, runtimes] of Object.entries(newStats)) {
     const oldStat = oldStats[fileName];
     if (!oldStat) {
@@ -65,31 +68,68 @@ const main = async () => {
       if (!oldTimeTaken) {
         continue;
       }
+      oldRuntimeSum += oldTimeTaken;
+      newRuntimeSum += timeTaken;
+      testCount++;
       const delta = timeTaken - oldTimeTaken;
+
       const { change, crossedThreshold } = percentageDifference(oldTimeTaken, timeTaken);
       const out = [
         fileName,
-        testName,
+        testName.replace('#', ''),
-        oldTimeTaken.toString(),
+        `${oldTimeTaken}/${timeTaken}`,
-        timeTaken.toString(),
+        `${delta.toString()}ms ${change}`,
-        change,
-        `${delta.toString()}ms`,
       ];
-      if (crossedThreshold) {
+      if (crossedThreshold && Math.abs(delta) > 25) {
         changed.push(out);
-        console.warn(`${testName} (${fileName}): ${timeTaken}ms (${delta}ms, ${change})`);
       }
       fullData.push(out);
     }
   }
-  const headers = ['File', 'Test', 'Old Time', 'New Time', '% Change', 'Difference'];
+  const oldAverage = oldRuntimeSum / testCount;
-  console.log(markdownTable([headers, ...changed]));
+  const newAverage = newRuntimeSum / testCount;
+  const { change, crossedThreshold } = percentageDifference(oldAverage, newAverage);
+
+  const headers = ['File', 'Test', 'Time Old/New', 'Change (%)'];
+  console.log(`## Runtime Changes
+Old runtime average: ${oldAverage.toFixed(2)}ms
+New runtime average: ${newAverage.toFixed(2)}ms
+Change: ${change} ${crossedThreshold ? '⚠️' : ''}
+  `);
+  console.log(`
+  <details>
+  <summary>Changed tests</summary>
+  ${htmlTable([headers, ...changed])}
+</details>
+`);
   console.log(`
   <details>
   <summary>Full Data</summary>
-  ${markdownTable([headers, ...fullData])}
+  ${htmlTable([headers, ...fullData])}
 </details>
 `);
 };
 
+const htmlTable = (data: string[][]): string => {
+  let table = `<table border='1' style="border-collapse: collapse">`;
+
+  // Generate table header
+  table += `<tr>
+    ${data
+      .shift()!
+      .map((header) => `<th>${header}</th>`)
+      .join('')}
+    </tr>`;
+
+  // Generate table rows
+  for (const row of data) {
+    table += `<tr>
+    ${row.map((cell) => `<td>${cell}</td>`).join('')}
+    </tr>`;
+  }
+
+  table += '</table>';
+  return table;
+};
+
 void main().catch((e) => console.error(e));