mermaid/scripts/compare-timings.ts

/**
 * Compares new E2E test timings with previous timings and determines whether to keep the new timings.
 *
 * The script will:
 * 1. Read old timings from git HEAD
 * 2. Read new timings from the current file
 * 3. Compare the timings and specs
 * 4. Keep new timings if:
 *    - Specs were added/removed
 *    - Any timing changed by 20% or more
 * 5. Revert to old timings if:
 *    - No significant timing changes
 *
 * This helps prevent unnecessary timing updates when test performance hasn't changed significantly.
 */

import fs from 'node:fs';
import path from 'node:path';
import { execSync } from 'node:child_process';

interface Timing {
  spec: string;
  duration: number;
}

interface TimingsFile {
  durations: Timing[];
}

interface CleanupOptions {
  keepNew: boolean;
  reason: string;
}

const TIMINGS_FILE = 'cypress/timings.json';
const TIMINGS_PATH = path.join(process.cwd(), TIMINGS_FILE);

function log(message: string): void {
  // eslint-disable-next-line no-console
  console.log(message);
}

function readOldTimings(): TimingsFile {
  try {
    const oldContent = execSync(`git show HEAD:${TIMINGS_FILE}`, { encoding: 'utf8' });
    return JSON.parse(oldContent);
  } catch {
    log('Error getting old timings, using empty file');
    return { durations: [] };
  }
}

function readNewTimings(): TimingsFile {
  return JSON.parse(fs.readFileSync(TIMINGS_PATH, 'utf8'));
}

function cleanupFiles({ keepNew, reason }: CleanupOptions): void {
  if (keepNew) {
    log(`Keeping new timings: ${reason}`);
  } else {
    log(`Reverting to old timings: ${reason}`);
    execSync(`git checkout HEAD -- ${TIMINGS_FILE}`);
  }
}

function compareTimings(): void {
  const oldTimings = readOldTimings();
  const newTimings = readNewTimings();

  const oldSpecs = new Set(oldTimings.durations.map((d) => d.spec));
  const newSpecs = new Set(newTimings.durations.map((d) => d.spec));

  // Check if specs were added or removed
  const addedSpecs = [...newSpecs].filter((spec) => !oldSpecs.has(spec));
  const removedSpecs = [...oldSpecs].filter((spec) => !newSpecs.has(spec));

  if (addedSpecs.length > 0 || removedSpecs.length > 0) {
    log('Specs changed:');
    if (addedSpecs.length > 0) {
      log(`Added: ${addedSpecs.join(', ')}`);
    }
    if (removedSpecs.length > 0) {
      log(`Removed: ${removedSpecs.join(', ')}`);
    }
    return cleanupFiles({ keepNew: true, reason: 'Specs were added or removed' });
  }

  // Check timing variations
  const timingChanges = newTimings.durations.map((newTiming) => {
    const oldTiming = oldTimings.durations.find((d) => d.spec === newTiming.spec);
    if (!oldTiming) {
      throw new Error(`Could not find old timing for spec: ${newTiming.spec}`);
    }
    const change = Math.abs(newTiming.duration - oldTiming.duration);
    const changePercent = change / oldTiming.duration;
    return { spec: newTiming.spec, change, changePercent };
  });

  // Filter changes that's more than 5 seconds and 20% different
  const significantChanges = timingChanges.filter((t) => t.change > 5000 && t.changePercent >= 0.2);

  if (significantChanges.length === 0) {
    log('No significant timing changes detected (threshold: 5s and 20%)');
    return cleanupFiles({ keepNew: false, reason: 'No significant timing changes' });
  }

  log('Significant timing changes:');
  significantChanges.forEach((t) => {
    log(`${t.spec}: ${t.change.toFixed(1)}ms (${(t.changePercent * 100).toFixed(1)}%)`);
  });

  cleanupFiles({ keepNew: true, reason: 'Significant timing changes detected' });
}

compareTimings();