From 7a5f999f42b28ce0d9ec77b9ad2752720e9eec4b Mon Sep 17 00:00:00 2001
From: Sidharth Vinod <sidharthv96@gmail.com>
Date: Wed, 9 Apr 2025 12:08:57 +0530
Subject: [PATCH] chore: Add compare-timings script

Avoid creating unnecessary PRs when there is no significant timings change.
---
 .github/workflows/e2e-timings.yml |  8 +++
 scripts/compare-timings.ts        | 88 +++++++++++++++++++++++++++++++
 2 files changed, 96 insertions(+)
 create mode 100644 scripts/compare-timings.ts

diff --git a/.github/workflows/e2e-timings.yml b/.github/workflows/e2e-timings.yml
index 4c36bc238..85778f421 100644
--- a/.github/workflows/e2e-timings.yml
+++ b/.github/workflows/e2e-timings.yml
@@ -30,6 +30,10 @@ jobs:
         uses: cypress-io/github-action@18a6541367f4580a515371905f499a27a44e8dbe # v6.7.12
         with:
           runTests: false
+
+      - name: Copy previous timings
+        run: cp cypress/timings.json cypress/timings-old.json
+
       - name: Cypress run
         uses: cypress-io/github-action@18a6541367f4580a515371905f499a27a44e8dbe # v6.7.12
         id: cypress
@@ -45,6 +49,10 @@ jobs:
           SPLIT: 1
           SPLIT_INDEX: 0
           SPLIT_FILE: 'cypress/timings.json'
+
+      - name: Compare timings
+        run: pnpm tsx scripts/compare-timings.ts
+
       - name: Commit and create pull request
         uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e
         with:
diff --git a/scripts/compare-timings.ts b/scripts/compare-timings.ts
new file mode 100644
index 000000000..6c63190ad
--- /dev/null
+++ b/scripts/compare-timings.ts
@@ -0,0 +1,88 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+interface Timing {
+  spec: string;
+  duration: number;
+}
+
+interface TimingsFile {
+  durations: Timing[];
+}
+
+const TIMINGS_PATH = path.join(process.cwd(), 'cypress', 'timings.json');
+const TIMINGS_OLD_PATH = path.join(process.cwd(), 'cypress', 'timings-old.json');
+
+function log(message: string): void {
+  // eslint-disable-next-line no-console
+  console.log(message);
+}
+
+function readTimings(filePath: string): TimingsFile {
+  return JSON.parse(fs.readFileSync(filePath, 'utf8'));
+}
+
+interface CleanupOptions {
+  keepNew: boolean;
+  reason: string;
+}
+
+function cleanupFiles({ keepNew, reason }: CleanupOptions): void {
+  if (keepNew) {
+    log(`Keeping new timings: ${reason}`);
+    fs.unlinkSync(TIMINGS_OLD_PATH);
+  } else {
+    log(`Reverting to old timings: ${reason}`);
+    fs.unlinkSync(TIMINGS_PATH);
+    fs.renameSync(TIMINGS_OLD_PATH, TIMINGS_PATH);
+  }
+}
+
+function compareTimings(): void {
+  const oldTimings = readTimings(TIMINGS_OLD_PATH);
+  const newTimings = readTimings(TIMINGS_PATH);
+
+  const oldSpecs = new Set(oldTimings.durations.map((d) => d.spec));
+  const newSpecs = new Set(newTimings.durations.map((d) => d.spec));
+
+  // Check if specs were added or removed
+  const addedSpecs = [...newSpecs].filter((spec) => !oldSpecs.has(spec));
+  const removedSpecs = [...oldSpecs].filter((spec) => !newSpecs.has(spec));
+
+  if (addedSpecs.length > 0 || removedSpecs.length > 0) {
+    log('Specs changed:');
+    if (addedSpecs.length > 0) {
+      log(`Added: ${addedSpecs.join(', ')}`);
+    }
+    if (removedSpecs.length > 0) {
+      log(`Removed: ${removedSpecs.join(', ')}`);
+    }
+    return cleanupFiles({ keepNew: true, reason: 'Specs were added or removed' });
+  }
+
+  // Check timing variations
+  const timingChanges = newTimings.durations.map((newTiming) => {
+    const oldTiming = oldTimings.durations.find((d) => d.spec === newTiming.spec);
+    if (!oldTiming) {
+      throw new Error(`Could not find old timing for spec: ${newTiming.spec}`);
+    }
+    const change = Math.abs(newTiming.duration - oldTiming.duration) / oldTiming.duration;
+    return { spec: newTiming.spec, change };
+  });
+
+  const significantChanges = timingChanges.filter((t) => t.change >= 0.2);
+
+  if (significantChanges.length === 0) {
+    log('No significant timing changes detected (threshold: 20%)');
+    return cleanupFiles({ keepNew: false, reason: 'No significant timing changes' });
+  }
+
+  log('Significant timing changes:');
+  significantChanges.forEach((t) => {
+    log(`${t.spec}: ${(t.change * 100).toFixed(1)}%`);
+  });
+
+  cleanupFiles({ keepNew: true, reason: 'Significant timing changes detected' });
+}
+
+compareTimings();