From 441deaffc9828cacbcfb29df9ed78e243c76e8c5 Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Fri, 26 Jan 2024 01:05:59 +0530 Subject: [PATCH 1/7] Test --- scripts/runTime.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/runTime.ts b/scripts/runTime.ts index f7c9822b2..a1dd18277 100644 --- a/scripts/runTime.ts +++ b/scripts/runTime.ts @@ -17,6 +17,11 @@ const getRuntimes = (csv: string): RunTimes => { const [testName, timeTaken] = line.split(','); if (testName && timeTaken) { runtimes[testName] = Number(timeTaken); + + // TODO: Add some variation to test logging. Should remove. + if (Math.random() < 0.3) { + runtimes[testName] *= Math.random() * 2; + } } } return runtimes; From 6702d4184061149249c1d63d08ed9991dcfe0369 Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Fri, 26 Jan 2024 01:15:43 +0530 Subject: [PATCH 2/7] Change path --- .github/workflows/e2e.yml | 12 ++++++------ scripts/runTime.ts | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 03a1f1c8e..254059dcb 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -205,12 +205,12 @@ jobs: id: runtime if: ${{ needs.e2e.result != 'failure' && github.event_name == 'pull_request' }} run: | - ls -l cypress/snapshots/runtimes - mv cypress/snapshots/runtimes cypress/snapshots/runtimes/head - ls -l cypress/snapshots/runtimes - ls -l cypress/snapshots/runtimes/base - ls -l cypress/snapshots/runtimes/head - tree cypress/snapshots/runtimes + ls -l ./snapshots/runtimes + mv ./snapshots/runtimes ./snapshots/runtimes/head + ls -l ./snapshots/runtimes + ls -l ./snapshots/runtimes/base + ls -l ./snapshots/runtimes/head + tree ./snapshots/runtimes npm config set ignore-scripts true pnpm install --frozen-lockfile { diff --git a/scripts/runTime.ts b/scripts/runTime.ts index a1dd18277..d7a4edb69 100644 --- a/scripts/runTime.ts +++ b/scripts/runTime.ts @@ -56,8 +56,8 @@ const percentageDifference = ( }; const main = async () => { - const oldStats = await readStats('./cypress/snapshots/runtimes/base/**/*.csv'); - const newStats = await readStats('./cypress/snapshots/runtimes/head/**/*.csv'); + const oldStats = await readStats('./snapshots/runtimes/base/**/*.csv'); + const newStats = await readStats('./snapshots/runtimes/head/**/*.csv'); const fullData: string[][] = []; const changed: string[][] = []; for (const [fileName, runtimes] of Object.entries(newStats)) { From 32db724752309d2f9f8c7bd3fd1a4cd35827c555 Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Fri, 26 Jan 2024 01:23:30 +0530 Subject: [PATCH 3/7] Change path --- .github/workflows/e2e.yml | 5 ++--- cypress.config.ts | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 254059dcb..e04de875c 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -76,8 +76,7 @@ jobs: - name: Move runtime data if: ${{ steps.cache-snapshot.outputs.cache-hit != 'true' }} run: | - mkdir -p cypress/snapshots/runtimes - mv cypress/runtimes cypress/snapshots/runtimes/base + mv cypress/snapshots/runtimes/current cypress/snapshots/runtimes/base e2e: runs-on: ubuntu-latest @@ -206,7 +205,7 @@ jobs: if: ${{ needs.e2e.result != 'failure' && github.event_name == 'pull_request' }} run: | ls -l ./snapshots/runtimes - mv ./snapshots/runtimes ./snapshots/runtimes/head + mv ./snapshots/runtimes/current ./snapshots/runtimes/head ls -l ./snapshots/runtimes ls -l ./snapshots/runtimes/base ls -l ./snapshots/runtimes/head diff --git a/cypress.config.ts b/cypress.config.ts index 957863060..2ee4ca788 100644 --- a/cypress.config.ts +++ b/cypress.config.ts @@ -21,7 +21,7 @@ export default eyesPlugin( }); on('task', { recordRenderTime({ fileName, testName, timeTaken }) { - const resultsPath = path.join('cypress', 'runtimes'); + const resultsPath = path.join('cypress', 'snapshots', 'runtimes', 'current'); if (!fs.existsSync(resultsPath)) { fs.mkdirSync(resultsPath, { recursive: true }); } From c6cf5953a1a20610cee47ea3116119e6b9426628 Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Fri, 26 Jan 2024 01:40:30 +0530 Subject: [PATCH 4/7] HTML table --- .github/workflows/e2e.yml | 7 +----- scripts/runTime.ts | 45 ++++++++++++++++++++++++++++----------- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index e04de875c..2ab827d0c 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -204,17 +204,12 @@ jobs: id: runtime if: ${{ needs.e2e.result != 'failure' && github.event_name == 'pull_request' }} run: | - ls -l ./snapshots/runtimes mv ./snapshots/runtimes/current ./snapshots/runtimes/head - ls -l ./snapshots/runtimes - ls -l ./snapshots/runtimes/base - ls -l ./snapshots/runtimes/head - tree ./snapshots/runtimes npm config set ignore-scripts true pnpm install --frozen-lockfile { echo 'runtime_diff<> "$GITHUB_OUTPUT" diff --git a/scripts/runTime.ts b/scripts/runTime.ts index d7a4edb69..e968edcdb 100644 --- a/scripts/runTime.ts +++ b/scripts/runTime.ts @@ -19,9 +19,9 @@ const getRuntimes = (csv: string): RunTimes => { runtimes[testName] = Number(timeTaken); // TODO: Add some variation to test logging. Should remove. - if (Math.random() < 0.3) { - runtimes[testName] *= Math.random() * 2; - } + // if (Math.random() < 0.3) { + // runtimes[testName] *= Math.random() * 2; + // } } } return runtimes; @@ -56,8 +56,9 @@ const percentageDifference = ( }; const main = async () => { - const oldStats = await readStats('./snapshots/runtimes/base/**/*.csv'); - const newStats = await readStats('./snapshots/runtimes/head/**/*.csv'); + const base = process.argv[2] || './cypress/snapshots'; + const oldStats = await readStats(`${base}/runtimes/base/**/*.csv`); + const newStats = await readStats(`${base}/runtimes/head/**/*.csv`); const fullData: string[][] = []; const changed: string[][] = []; for (const [fileName, runtimes] of Object.entries(newStats)) { @@ -75,26 +76,46 @@ const main = async () => { const out = [ fileName, testName, - oldTimeTaken.toString(), - timeTaken.toString(), - change, - `${delta.toString()}ms`, + `${oldTimeTaken}/${timeTaken}`, + `${delta.toString()}ms ${change}`, ]; if (crossedThreshold) { changed.push(out); - console.warn(`${testName} (${fileName}): ${timeTaken}ms (${delta}ms, ${change})`); } fullData.push(out); } } - const headers = ['File', 'Test', 'Old Time', 'New Time', '% Change', 'Difference']; + const headers = ['File', 'Test', 'Time Old/New', 'Change (%)']; console.log(markdownTable([headers, ...changed])); console.log(`
Full Data - ${markdownTable([headers, ...fullData])} + ${htmlTable([headers, ...fullData])}
`); }; +const htmlTable = (data: string[][]): string => { + let table = ``; + + // Generate table header + table += ''; + for (const header of data[0]) { + table += ``; + } + table += ''; + + // Generate table rows + for (let i = 1; i < data.length; i++) { + table += ''; + for (let j = 0; j < data[i].length; j++) { + table += ``; + } + table += ''; + } + + table += '
${header}
${data[i][j]}
'; + return table; +}; + void main().catch((e) => console.error(e)); From eb6c92b0d95eafb74018482e22398d9a3ca03afb Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Fri, 26 Jan 2024 01:47:36 +0530 Subject: [PATCH 5/7] Remove small differences --- scripts/runTime.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/scripts/runTime.ts b/scripts/runTime.ts index e968edcdb..1cce8e339 100644 --- a/scripts/runTime.ts +++ b/scripts/runTime.ts @@ -17,11 +17,6 @@ const getRuntimes = (csv: string): RunTimes => { const [testName, timeTaken] = line.split(','); if (testName && timeTaken) { runtimes[testName] = Number(timeTaken); - - // TODO: Add some variation to test logging. Should remove. - // if (Math.random() < 0.3) { - // runtimes[testName] *= Math.random() * 2; - // } } } return runtimes; @@ -72,14 +67,15 @@ const main = async () => { continue; } const delta = timeTaken - oldTimeTaken; + const { change, crossedThreshold } = percentageDifference(oldTimeTaken, timeTaken); const out = [ fileName, - testName, + testName.replace('#', ''), `${oldTimeTaken}/${timeTaken}`, `${delta.toString()}ms ${change}`, ]; - if (crossedThreshold) { + if (crossedThreshold && Math.abs(delta) > 25) { changed.push(out); } fullData.push(out); From dc476233ba96cc170e0def9d0844369508d718f6 Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Fri, 26 Jan 2024 01:54:09 +0530 Subject: [PATCH 6/7] Add summary --- scripts/runTime.ts | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/scripts/runTime.ts b/scripts/runTime.ts index 1cce8e339..22c44bba3 100644 --- a/scripts/runTime.ts +++ b/scripts/runTime.ts @@ -1,7 +1,6 @@ /* eslint-disable no-console */ import { readFile } from 'fs/promises'; import { globby } from 'globby'; -import { markdownTable } from 'markdown-table'; interface RunTimes { [key: string]: number; @@ -56,6 +55,9 @@ const main = async () => { const newStats = await readStats(`${base}/runtimes/head/**/*.csv`); const fullData: string[][] = []; const changed: string[][] = []; + let oldRuntimeSum = 0; + let newRuntimeSum = 0; + let testCount = 0; for (const [fileName, runtimes] of Object.entries(newStats)) { const oldStat = oldStats[fileName]; if (!oldStat) { @@ -66,6 +68,9 @@ const main = async () => { if (!oldTimeTaken) { continue; } + oldRuntimeSum += oldTimeTaken; + newRuntimeSum += timeTaken; + testCount++; const delta = timeTaken - oldTimeTaken; const { change, crossedThreshold } = percentageDifference(oldTimeTaken, timeTaken); @@ -81,8 +86,22 @@ const main = async () => { fullData.push(out); } } + const oldAverage = oldRuntimeSum / testCount; + const newAverage = newRuntimeSum / testCount; + const { change, crossedThreshold } = percentageDifference(oldAverage, newAverage); + const headers = ['File', 'Test', 'Time Old/New', 'Change (%)']; - console.log(markdownTable([headers, ...changed])); + console.log(`## Runtime Changes +Old runtime average: ${oldAverage.toFixed(2)}ms +New runtime average: ${newAverage.toFixed(2)}ms +Change: ${change} ${crossedThreshold ? '⚠️' : ''} + `); + console.log(` +
+ Changed tests + ${htmlTable([headers, ...changed])} +
+`); console.log(`
Full Data From 781945325d72b1e1f43bd55b27bab5b5127c58e7 Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Fri, 26 Jan 2024 02:07:55 +0530 Subject: [PATCH 7/7] Cleanup runTime.ts --- .github/workflows/e2e.yml | 6 ------ scripts/runTime.ts | 21 ++++++++++----------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 2ab827d0c..2613f33b2 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -69,9 +69,6 @@ jobs: start: pnpm run dev wait-on: 'http://localhost:9000' browser: chrome - spec: | - cypress/integration/rendering/classDiagram.spec.js - cypress/integration/rendering/flowchart-v2.spec.js - name: Move runtime data if: ${{ steps.cache-snapshot.outputs.cache-hit != 'true' }} @@ -148,9 +145,6 @@ jobs: # e.g. if this action was run from a fork record: ${{ secrets.CYPRESS_RECORD_KEY != '' }} parallel: ${{ secrets.CYPRESS_RECORD_KEY != '' }} - spec: | - cypress/integration/rendering/classDiagram.spec.js - cypress/integration/rendering/flowchart-v2.spec.js env: CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} VITEST_COVERAGE: true diff --git a/scripts/runTime.ts b/scripts/runTime.ts index 22c44bba3..d8da3203d 100644 --- a/scripts/runTime.ts +++ b/scripts/runTime.ts @@ -114,19 +114,18 @@ const htmlTable = (data: string[][]): string => { let table = ``; // Generate table header - table += ''; - for (const header of data[0]) { - table += ``; - } - table += ''; + table += ` + ${data + .shift()! + .map((header) => ``) + .join('')} + `; // Generate table rows - for (let i = 1; i < data.length; i++) { - table += ''; - for (let j = 0; j < data[i].length; j++) { - table += ``; - } - table += ''; + for (const row of data) { + table += ` + ${row.map((cell) => ``).join('')} + `; } table += '
${header}
${header}
${data[i][j]}
${cell}
';