repos / gbc

GBC - Go B Compiler
git clone https://github.com/xplshn/gbc.git

gbc / cmd / gtest
xplshn  ·  2025-08-13

main.go

Go
  1package main
  2
  3import (
  4	"bytes"
  5	"context"
  6	"encoding/json"
  7	"flag"
  8	"fmt"
  9	"io"
 10	"log"
 11	"os"
 12	"os/exec"
 13	"os/signal"
 14	"path/filepath"
 15	"sort"
 16	"strings"
 17	"sync"
 18	"time"
 19
 20	"github.com/cespare/xxhash/v2"
 21	"github.com/google/go-cmp/cmp"
 22)
 23
 24type Execution struct {
 25	Stdout         string        `json:"stdout"`
 26	Stderr         string        `json:"stderr"`
 27	ExitCode       int           `json:"exitCode"`
 28	Duration       time.Duration `json:"duration"`
 29	TimedOut       bool          `json:"timed_out"`
 30	UnstableOutput bool          `json:"unstable_output,omitempty"`
 31}
 32
 33type TestRun struct {
 34	Name   string   `json:"name"`
 35	Args   []string `json:"args,omitempty"`
 36	Input  string   `json:"input,omitempty"`
 37	Result Execution `json:"result"`
 38}
 39
 40type TargetResult struct {
 41	BinaryPath string    `json:"binary_path,omitempty"`
 42	Compile    Execution `json:"compile"`
 43	Runs       []TestRun `json:"runs"`
 44}
 45
 46type FileTestResult struct {
 47	File      string        `json:"file"`
 48	Status    string        `json:"status"` // PASS, FAIL, SKIP, ERROR
 49	Message   string        `json:"message,omitempty"`
 50	Diff      string        `json:"diff,omitempty"`
 51	Reference *TargetResult `json:"reference,omitempty"`
 52	Target    *TargetResult `json:"target,omitempty"`
 53}
 54
 55type TestSuiteResults map[string]*FileTestResult
 56
 57var (
 58	refCompiler    = flag.String("ref-compiler", "b", "Path to the reference compiler.")
 59	refArgs        = flag.String("ref-args", "", "Arguments for the reference compiler (space-separated).")
 60	targetCompiler = flag.String("target-compiler", "./gbc", "Path to the target compiler to test.")
 61	targetArgs     = flag.String("target-args", "", "Arguments for the target compiler (space-separated).")
 62	generateGolden = flag.String("generate-golden", "", "Generate a golden .json file for a given source file.")
 63	testFiles      = flag.String("test-files", "tests/*.b", "Glob pattern(s) for files to test (space-separated).")
 64	skipFiles      = flag.String("skip-files", "", "Files to skip (space-separated).")
 65	outputJSON     = flag.String("output", ".test_results.json", "Output file for the JSON test report.")
 66	timeout        = flag.Duration("timeout", 5*time.Second, "Timeout for each command execution.")
 67	jobs           = flag.Int("j", 4, "Number of parallel test jobs.")
 68	runs           = flag.Int("runs", 5, "Number of times to run each test case to find the minimum duration.")
 69	verbose        = flag.Bool("v", false, "Enable verbose logging.")
 70	useCache       = flag.Bool("cached", false, "Use cached golden files if available.")
 71	jsonDir        = flag.String("dir", "", "Directory to store/read golden JSON files (defaults to source file dir).")
 72	ignoreLines    = flag.String("ignore-lines", "", "Comma-separated substrings to ignore during output comparison.")
 73)
 74
 75const (
 76	cRed     = "\x1b[91m"
 77	cYellow  = "\x1b[93m"
 78	cGreen   = "\x1b[92m"
 79	cCyan    = "\x1b[96m"
 80	cMagenta = "\x1b[95m"
 81	cBold    = "\x1b[1m"
 82	cNone    = "\x1b[0m"
 83)
 84
 85func main() {
 86	flag.Parse()
 87	log.SetFlags(0)
 88
 89	if *runs < 1 {
 90		*runs = 1
 91	}
 92
 93	// Single tempDir for all test artifacts
 94	tempDir, err := os.MkdirTemp("", "gtest-*")
 95	if err != nil {
 96		log.Fatalf("%s[ERROR]%s Failed to create temp directory: %v\n", cRed, cNone, err)
 97	}
 98	defer os.RemoveAll(tempDir)
 99	setupInterruptHandler(tempDir)
100
101	if *generateGolden != "" {
102		handleGenerateGolden(*generateGolden, tempDir)
103		return
104	}
105
106	handleRunTestSuite(tempDir)
107}
108
109// setupInterruptHandler is used to clean up on CTRL+C
110func setupInterruptHandler(tempDir string) {
111	c := make(chan os.Signal, 1)
112	signal.Notify(c, os.Interrupt)
113	go func() {
114		<-c
115		os.RemoveAll(tempDir)
116		fmt.Printf("\n%s[INTERRUPT]%s Test run cancelled. Cleaning up...\n", cYellow, cNone)
117		os.Exit(1)
118	}()
119}
120
121func getJSONPath(sourceFile string) string {
122	jsonFileName := "." + filepath.Base(sourceFile) + ".json"
123	if *jsonDir != "" {
124		return filepath.Join(*jsonDir, jsonFileName)
125	}
126	return filepath.Join(filepath.Dir(sourceFile), jsonFileName)
127}
128
129// hashFile computes the xxhash of a file's content
130func hashFile(path string) (string, error) {
131	f, err := os.Open(path)
132	if err != nil {
133		return "", err
134	}
135	defer f.Close()
136	h := xxhash.New()
137	if _, err := io.Copy(h, f); err != nil {
138		return "", err
139	}
140	return fmt.Sprintf("%x", h.Sum64()), nil
141}
142
143func handleGenerateGolden(sourceFile, tempDir string) {
144	log.Printf("Generating golden file for %s...\n", sourceFile)
145
146	fileHash, err := hashFile(sourceFile)
147	if err != nil {
148		log.Fatalf("%s[ERROR]%s Could not hash source file %s: %v\n", cRed, cNone, sourceFile, err)
149	}
150
151	targetResult, err := compileAndRun(*targetCompiler, strings.Fields(*targetArgs), sourceFile, tempDir, fileHash)
152	if err != nil {
153		log.Fatalf("%s[ERROR]%s Could not generate golden file for %s: %v\n", cRed, cNone, sourceFile, err)
154	}
155
156	jsonData, err := json.MarshalIndent(targetResult, "", "  ")
157	if err != nil {
158		log.Fatalf("%s[ERROR]%s Failed to marshal golden data to JSON: %v\n", cRed, cNone, err)
159	}
160
161	goldenFileName := getJSONPath(sourceFile)
162	if *jsonDir != "" {
163		if err := os.MkdirAll(*jsonDir, 0755); err != nil {
164			log.Fatalf("%s[ERROR]%s Failed to create directory %s: %v\n", cRed, cNone, *jsonDir, err)
165		}
166	}
167
168	if err := os.WriteFile(goldenFileName, jsonData, 0644); err != nil {
169		log.Fatalf("%s[ERROR]%s Failed to write golden file %s: %v\n", cRed, cNone, goldenFileName, err)
170	}
171
172	log.Printf("%s[SUCCESS]%s Golden file created at %s\n", cGreen, cNone, goldenFileName)
173}
174
175func handleRunTestSuite(tempDir string) {
176	_, err := exec.LookPath(*refCompiler)
177	refCompilerFound := err == nil
178	if !refCompilerFound && !*useCache {
179		log.Printf("%s[WARN]%s Reference compiler '%s' not found. Will rely on golden files. Use --cached to suppress this warning.\n", cYellow, cNone, *refCompiler)
180	}
181
182	files, err := expandGlobPatterns(*testFiles)
183	if err != nil {
184		log.Fatalf("%s[ERROR]%s Invalid glob pattern(s): %v\n", cRed, cNone, err)
185	}
186	if len(files) == 0 {
187		log.Println("No test files found matching the pattern(s).")
188		return
189	}
190
191	// Load previous results for caching reference compiler output
192	previousResults := make(TestSuiteResults)
193	outputFile := *outputJSON
194	if *jsonDir != "" {
195		outputFile = filepath.Join(*jsonDir, *outputJSON)
196	}
197	if prevData, err := os.ReadFile(outputFile); err == nil {
198		if json.Unmarshal(prevData, &previousResults) != nil {
199			log.Printf("%s[WARN]%s Could not parse previous results file %s. Cache will not be used.\n", cYellow, cNone, outputFile)
200			previousResults = make(TestSuiteResults) // Reset on parse error
201		}
202	}
203
204	skipList := make(map[string]bool)
205	for _, f := range strings.Fields(*skipFiles) {
206		skipList[f] = true
207	}
208
209	tasks := make(chan string, len(files))
210	resultsChan := make(chan *FileTestResult, len(files))
211	var wg sync.WaitGroup
212
213	for i := 0; i < *jobs; i++ {
214		wg.Add(1)
215		go func() {
216			defer wg.Done()
217			for file := range tasks {
218				fileHash, err := hashFile(file)
219				if err != nil {
220					resultsChan <- &FileTestResult{File: file, Status: "ERROR", Message: "Failed to hash source file"}
221					continue
222				}
223				resultsChan <- testFile(file, tempDir, fileHash, refCompilerFound, previousResults)
224			}
225		}()
226	}
227
228	// Feed the tasks channel, skipping files with identical content
229	seenHashes := make(map[string]string)
230	for _, file := range files {
231		if skipList[file] {
232			resultsChan <- &FileTestResult{File: file, Status: "SKIP", Message: "Explicitly skipped"}
233			continue
234		}
235		fileHash, err := hashFile(file)
236		if err != nil {
237			resultsChan <- &FileTestResult{File: file, Status: "ERROR", Message: fmt.Sprintf("Failed to read file for hashing: %v", err)}
238			continue
239		}
240		if originalFile, seen := seenHashes[fileHash]; seen {
241			resultsChan <- &FileTestResult{File: file, Status: "SKIP", Message: fmt.Sprintf("Content is identical to %s", originalFile)}
242			continue
243		}
244		seenHashes[fileHash] = file
245		tasks <- file
246	}
247	close(tasks)
248
249	wg.Wait()
250	close(resultsChan)
251
252	var allResults []*FileTestResult
253	for result := range resultsChan {
254		allResults = append(allResults, result)
255	}
256
257	sort.Slice(allResults, func(i, j int) bool {
258		return allResults[i].File < allResults[j].File
259	})
260
261	printSummary(allResults)
262	resultsMap := writeJSONReport(allResults)
263
264	if hasFailures(resultsMap) {
265		os.Exit(1)
266	}
267}
268
269func testFile(file, tempDir, fileHash string, refCompilerFound bool, previousResults TestSuiteResults) *FileTestResult {
270	goldenFile := getJSONPath(file)
271	_, err := os.Stat(goldenFile)
272	hasGoldenFile := err == nil
273
274	// 1st try: Use golden file if --cached is set or for non-standard extensions
275	if (*useCache && hasGoldenFile) || !strings.HasSuffix(file, ".b") {
276		if hasGoldenFile {
277			return testWithGoldenFile(file, goldenFile, tempDir, fileHash)
278		}
279		return &FileTestResult{File: file, Status: "SKIP", Message: "Cannot test without a corresponding .json golden file"}
280	}
281
282	// 2nd try: It's a .b file, try reference compiler if it exists
283	if refCompilerFound {
284		return testWithReferenceCompiler(file, tempDir, fileHash)
285	}
286
287	// 3rd try: No reference compiler, but a golden file exists. Use it as a fallback
288	if hasGoldenFile {
289		log.Printf("[%s] No reference compiler, falling back to golden file: %s", file, goldenFile)
290		return testWithGoldenFile(file, goldenFile, tempDir, fileHash)
291	}
292
293	// 4th try: Use the existing .test_results.json if it exists
294	if prevResult, ok := previousResults[file]; ok && prevResult.Reference != nil {
295		log.Printf("[%s] Using cached reference result from previous test run.", file)
296		targetResult, err := compileAndRun(*targetCompiler, strings.Fields(*targetArgs), file, tempDir, fileHash)
297		if err != nil {
298			return &FileTestResult{
299				File:      file,
300				Status:    "FAIL",
301				Message:   "Target compiler failed to compile, but cached reference expected success.",
302				Diff:      fmt.Sprintf("Target Compiler STDERR:\n%s", targetResult.Compile.Stderr),
303				Reference: prevResult.Reference,
304				Target:    targetResult,
305			}
306		}
307		comparisonResult := compareRuntimeResults(file, prevResult.Reference, targetResult)
308		comparisonResult.Message += " (against cached reference)"
309		return comparisonResult
310	}
311
312	// No way to test this file
313	return &FileTestResult{File: file, Status: "SKIP", Message: fmt.Sprintf("Reference compiler '%s' not found and no golden file or cached result exists", *refCompiler)}
314}
315
316func testWithGoldenFile(file, goldenFile, tempDir, fileHash string) *FileTestResult {
317	goldenData, err := os.ReadFile(goldenFile)
318	if err != nil {
319		return &FileTestResult{File: file, Status: "ERROR", Message: fmt.Sprintf("Could not read golden file %s: %v", goldenFile, err)}
320	}
321	var goldenResult TargetResult
322	if err := json.Unmarshal(goldenData, &goldenResult); err != nil {
323		return &FileTestResult{File: file, Status: "ERROR", Message: fmt.Sprintf("Could not parse golden file %s: %v", goldenFile, err)}
324	}
325
326	targetResult, err := compileAndRun(*targetCompiler, strings.Fields(*targetArgs), file, tempDir, fileHash)
327	if err != nil {
328		return &FileTestResult{
329			File:      file,
330			Status:    "FAIL",
331			Message:   "Target compiler failed to compile, but golden file expected success.",
332			Diff:      fmt.Sprintf("Target Compiler STDERR:\n%s", targetResult.Compile.Stderr),
333			Reference: &goldenResult,
334			Target:    targetResult,
335		}
336	}
337
338	return compareRuntimeResults(file, &goldenResult, targetResult)
339}
340
341func testWithReferenceCompiler(file, tempDir, fileHash string) *FileTestResult {
342	var refResult, targetResult *TargetResult
343	var refErr, targetErr error
344	var wg sync.WaitGroup
345
346	wg.Add(2)
347	go func() {
348		defer wg.Done()
349		refResult, refErr = compileAndRun(*refCompiler, strings.Fields(*refArgs), file, tempDir, "ref-"+fileHash)
350	}()
351	go func() {
352		defer wg.Done()
353		targetResult, targetErr = compileAndRun(*targetCompiler, strings.Fields(*targetArgs), file, tempDir, "target-"+fileHash)
354	}()
355	wg.Wait()
356
357	refCompiled := refErr == nil
358	targetCompiled := targetErr == nil
359
360	if !refCompiled && !targetCompiled {
361		return &FileTestResult{File: file, Status: "PASS", Message: "Both compilers failed to compile as expected", Reference: refResult, Target: targetResult}
362	}
363	if !targetCompiled && refCompiled {
364		return &FileTestResult{
365			File:      file,
366			Status:    "FAIL",
367			Message:   "Target compiler failed, but reference compiler succeeded",
368			Diff:      fmt.Sprintf("Target Compiler STDERR:\n%s", targetResult.Compile.Stderr),
369			Reference: refResult,
370			Target:    targetResult,
371		}
372	}
373	if targetCompiled && !refCompiled {
374		return &FileTestResult{
375			File:      file,
376			Status:    "FAIL",
377			Message:   "Target compiler succeeded, but reference compiler failed",
378			Diff:      fmt.Sprintf("Reference Compiler STDERR:\n%s", refResult.Compile.Stderr),
379			Reference: refResult,
380			Target:    targetResult,
381		}
382	}
383
384	return compareRuntimeResults(file, refResult, targetResult)
385}
386
387func compareRuntimeResults(file string, refResult, targetResult *TargetResult) *FileTestResult {
388	var diffs strings.Builder
389	var failed bool
390
391	targetRuns := make(map[string]TestRun)
392	for _, run := range targetResult.Runs {
393		targetRuns[run.Name] = run
394	}
395
396	sort.Slice(refResult.Runs, func(i, j int) bool {
397		return refResult.Runs[i].Name < refResult.Runs[j].Name
398	})
399
400	ignoredSubstrings := []string{}
401	if *ignoreLines != "" {
402		ignoredSubstrings = strings.Split(*ignoreLines, ",")
403	}
404
405	for _, refRun := range refResult.Runs {
406		targetRun, ok := targetRuns[refRun.Name]
407		if !ok {
408			failed = true
409			diffs.WriteString(fmt.Sprintf("Test run '%s' missing in target results.\n", refRun.Name))
410			continue
411		}
412
413		if refRun.Result.UnstableOutput != targetRun.Result.UnstableOutput {
414			failed = true
415			diffs.WriteString(fmt.Sprintf("Run '%s' Output Stability Mismatch:\n  - Ref:    %v\n  - Target: %v\n", refRun.Name, refRun.Result.UnstableOutput, targetRun.Result.UnstableOutput))
416		}
417
418		if refRun.Result.ExitCode != targetRun.Result.ExitCode {
419			failed = true
420			diffs.WriteString(fmt.Sprintf("Run '%s' Exit Code mismatch:\n  - Ref:    %d\n  - Target: %d\n", refRun.Name, refRun.Result.ExitCode, targetRun.Result.ExitCode))
421		}
422
423		// Filter output based on --ignore-lines first
424		refStdout := filterOutput(refRun.Result.Stdout, ignoredSubstrings)
425		targetStdout := filterOutput(targetRun.Result.Stdout, ignoredSubstrings)
426		refStderr := filterOutput(refRun.Result.Stderr, ignoredSubstrings)
427		targetStderr := filterOutput(targetRun.Result.Stderr, ignoredSubstrings)
428
429		// Normalize by replacing binary paths (argv[0]) if they exist.
430		// This handles cases where a program prints its own name.
431		// We replace both the full path and the basename with a generic placeholder.
432		const binaryPlaceholder = "__BINARY__"
433		if refResult.BinaryPath != "" {
434			refStdout = strings.ReplaceAll(refStdout, refResult.BinaryPath, binaryPlaceholder)
435			refStderr = strings.ReplaceAll(refStderr, refResult.BinaryPath, binaryPlaceholder)
436			refBase := filepath.Base(refResult.BinaryPath)
437			refStdout = strings.ReplaceAll(refStdout, refBase, binaryPlaceholder)
438			refStderr = strings.ReplaceAll(refStderr, refBase, binaryPlaceholder)
439		}
440		if targetResult.BinaryPath != "" {
441			targetStdout = strings.ReplaceAll(targetStdout, targetResult.BinaryPath, binaryPlaceholder)
442			targetStderr = strings.ReplaceAll(targetStderr, targetResult.BinaryPath, binaryPlaceholder)
443			targetBase := filepath.Base(targetResult.BinaryPath)
444			targetStdout = strings.ReplaceAll(targetStdout, targetBase, binaryPlaceholder)
445			targetStderr = strings.ReplaceAll(targetStderr, targetBase, binaryPlaceholder)
446		}
447
448		if refStdout != targetStdout {
449			failed = true
450			// Show the diff of the original, unmodified output for clarity.
451			diffs.WriteString(fmt.Sprintf("Run '%s' STDOUT mismatch:\n%s", refRun.Name, cmp.Diff(refRun.Result.Stdout, targetRun.Result.Stdout)))
452		}
453
454		if refStderr != targetStderr {
455			failed = true
456			// Show the diff of the original, unmodified output for clarity.
457			diffs.WriteString(fmt.Sprintf("Run '%s' STDERR mismatch:\n%s", refRun.Name, cmp.Diff(refRun.Result.Stderr, targetRun.Result.Stderr)))
458		}
459	}
460
461	if failed {
462		return &FileTestResult{
463			File:      file,
464			Status:    "FAIL",
465			Message:   "Runtime output or exit code mismatch",
466			Diff:      diffs.String(),
467			Reference: refResult,
468			Target:    targetResult,
469		}
470	}
471
472	return &FileTestResult{
473		File:      file,
474		Status:    "PASS",
475		Message:   "All test cases passed",
476		Reference: refResult,
477		Target:    targetResult,
478	}
479}
480
481// executeCommand runs a command with a timeout and captures its output, optionally piping data to stdin
482func executeCommand(ctx context.Context, command string, stdinData string, args ...string) Execution {
483	startTime := time.Now()
484	cmd := exec.CommandContext(ctx, command, args...)
485	var stdout, stderr bytes.Buffer
486	cmd.Stdout = &stdout
487	cmd.Stderr = &stderr
488
489	if stdinData != "" {
490		cmd.Stdin = strings.NewReader(stdinData)
491	}
492
493	err := cmd.Run()
494	duration := time.Since(startTime)
495
496	execResult := Execution{
497		Stdout:   stdout.String(),
498		Stderr:   stderr.String(),
499		Duration: duration,
500	}
501
502	if ctx.Err() == context.DeadlineExceeded {
503		execResult.TimedOut = true
504		execResult.ExitCode = -1
505	} else if err != nil {
506		if exitErr, ok := err.(*exec.ExitError); ok {
507			execResult.ExitCode = exitErr.ExitCode()
508		} else {
509			execResult.ExitCode = -2 // Should not happen often
510			execResult.Stderr += "\nExecution error: " + err.Error()
511		}
512	} else {
513		execResult.ExitCode = 0
514	}
515
516	return execResult
517}
518
519func compileAndRun(compiler string, compilerArgs []string, sourceFile, tempDir, binaryHash string) (*TargetResult, error) {
520	ctx, cancel := context.WithTimeout(context.Background(), *timeout)
521	defer cancel()
522
523	// Use the hash for a unique, deterministic binary name
524	binaryPath := filepath.Join(tempDir, binaryHash)
525
526	allArgs := []string{"-o", binaryPath}
527	allArgs = append(allArgs, compilerArgs...)
528	allArgs = append(allArgs, sourceFile)
529
530	compileResult := executeCommand(ctx, compiler, "", allArgs...)
531	if compileResult.ExitCode != 0 || compileResult.TimedOut {
532		return &TargetResult{Compile: compileResult}, fmt.Errorf("compilation failed with exit code %d", compileResult.ExitCode)
533	}
534
535	if _, err := os.Stat(binaryPath); os.IsNotExist(err) {
536		return &TargetResult{Compile: compileResult}, fmt.Errorf("compilation succeeded but binary was not created at %s", binaryPath)
537	}
538
539	// Probe to see if the binary waits for stdin by running it with a very short timeout.
540	// If it times out, it's likely waiting for input.
541	probeCtx, probeCancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
542	defer probeCancel()
543	probeResult := executeCommand(probeCtx, binaryPath, "")
544	readsStdin := probeResult.TimedOut
545
546	testCases := map[string][]string{
547		"no_args":         {},
548		"quit":            {"q"},
549		"hashTable":       {"s foo 10\ns bar 50\ng\ng foo\ng bar\np\nq\n"},
550		"fold":            {"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzAB\n"},
551		"fold2":           {"ABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWX\n"},
552		"string_arg":      {"test"},
553		"numeric_arg_pos": {"5"},
554		"numeric_arg_neg": {"-5"},
555		"numeric_arg_0":   {"0"},
556	}
557	var testCaseNames []string
558	for name := range testCases {
559		testCaseNames = append(testCaseNames, name)
560	}
561	sort.Strings(testCaseNames)
562
563	ignoredSubstrings := []string{}
564	if *ignoreLines != "" {
565		ignoredSubstrings = strings.Split(*ignoreLines, ",")
566	}
567
568	runResults := make([]TestRun, 0, len(testCaseNames))
569	for _, name := range testCaseNames {
570		args := testCases[name]
571		var durations []time.Duration
572		var firstRunResult Execution
573		var inputData string
574		var unstableOutput bool
575
576		// If the program is detected to read from stdin, we join the args to form the input string.
577		// Otherwise, we pass them as command-line arguments.
578		if readsStdin {
579			inputData = strings.Join(args, "\n")
580			if len(args) > 0 {
581				inputData += "\n"
582			}
583			args = []string{} // Clear args as they are now used for stdin
584		}
585
586		for i := 0; i < *runs; i++ {
587			runCtx, runCancel := context.WithTimeout(context.Background(), *timeout)
588			runResult := executeCommand(runCtx, binaryPath, inputData, args...)
589			runCancel()
590
591			if i == 0 {
592				firstRunResult = runResult
593			} else {
594				// Compare with the first run's output, after filtering
595				filteredFirstStdout := filterOutput(firstRunResult.Stdout, ignoredSubstrings)
596				filteredCurrentStdout := filterOutput(runResult.Stdout, ignoredSubstrings)
597				filteredFirstStderr := filterOutput(firstRunResult.Stderr, ignoredSubstrings)
598				filteredCurrentStderr := filterOutput(runResult.Stderr, ignoredSubstrings)
599
600				if firstRunResult.ExitCode != runResult.ExitCode || filteredFirstStdout != filteredCurrentStdout || filteredFirstStderr != filteredCurrentStderr {
601					unstableOutput = true
602					// Inconsistent output. We'll use the first run's result but mark it.
603					// We stop iterating because finding the "fastest" run is meaningless
604					// if the output is different each time.
605					break
606				}
607			}
608
609			if runResult.ExitCode != 0 || runResult.TimedOut {
610				firstRunResult = runResult
611				break
612			}
613			durations = append(durations, runResult.Duration)
614		}
615
616		if len(durations) > 0 {
617			sort.Slice(durations, func(i, j int) bool { return durations[i] < durations[j] })
618			firstRunResult.Duration = durations[0]
619		}
620		firstRunResult.UnstableOutput = unstableOutput
621
622		runResults = append(runResults, TestRun{Name: name, Args: args, Input: inputData, Result: firstRunResult})
623		if firstRunResult.ExitCode != 0 || firstRunResult.TimedOut {
624			break
625		}
626	}
627
628	return &TargetResult{Compile: compileResult, Runs: runResults, BinaryPath: binaryPath}, nil
629}
630
631// filterOutput removes lines containing any of the given substrings.
632func filterOutput(output string, ignoredSubstrings []string) string {
633	if len(ignoredSubstrings) == 0 || output == "" {
634		return output
635	}
636	// To preserve original line endings, we can split by \n and rejoin
637	lines := strings.Split(output, "\n")
638	filteredLines := make([]string, 0, len(lines))
639
640	for _, line := range lines {
641		ignore := false
642		for _, sub := range ignoredSubstrings {
643			if sub != "" && strings.Contains(line, sub) {
644				ignore = true
645				break
646			}
647		}
648		if !ignore {
649			filteredLines = append(filteredLines, line)
650		}
651	}
652	return strings.Join(filteredLines, "\n")
653}
654
655func formatDuration(d time.Duration) string {
656	if d < time.Millisecond {
657		return fmt.Sprintf("%6dµs", d.Microseconds())
658	}
659	return fmt.Sprintf("%6dms", d.Milliseconds())
660}
661
662func printSummary(results []*FileTestResult) {
663	var passed, failed, skipped, errored int
664	var totalTargetCompile, totalRefCompile, totalTargetRuntime, totalRefRuntime time.Duration
665	var comparedFileCount, runtimeFileCount int
666
667	// Pre-calculate max lengths for alignment.
668	var maxTestNameLen int
669	targetName := filepath.Base(*targetCompiler)
670	refName := filepath.Base(*refCompiler)
671
672	// Calculate max label length for alignment of performance stats
673	labels := []string{
674		targetName,
675		refName,
676		targetName + "_comp",
677		refName + "_comp",
678		targetName + "_runt",
679		refName + "_runt",
680	}
681	var maxLabelLen int
682	for _, label := range labels {
683		if len(label) > maxLabelLen {
684			maxLabelLen = len(label)
685		}
686	}
687
688	// Only calculate maxTestNameLen if in verbose mode, as it's only used there.
689	if *verbose {
690		for _, result := range results {
691			isBothFailed := result.Message == "Both compilers failed to compile as expected"
692			if result.Status == "PASS" && result.Target != nil && result.Reference != nil && !isBothFailed {
693				for _, run := range result.Target.Runs {
694					if len(run.Name) > maxTestNameLen {
695						maxTestNameLen = len(run.Name)
696					}
697				}
698			}
699		}
700	}
701
702	for _, result := range results {
703		fmt.Println("----------------------------------------------------------------------")
704		fmt.Printf("Testing %s%s%s...\n", cCyan, result.File, cNone)
705
706		switch result.Status {
707		case "PASS":
708			passed++
709			fmt.Printf("  [%sPASS%s] %s\n", cGreen, cNone, result.Message)
710		case "FAIL":
711			failed++
712			fmt.Printf("  [%sFAIL%s] %s\n", cRed, cNone, result.Message)
713			fmt.Println(formatDiff(result.Diff))
714		case "SKIP":
715			skipped++
716			fmt.Printf("  [%sSKIP%s] %s\n", cYellow, cNone, result.Message)
717		case "ERROR":
718			errored++
719			fmt.Printf("  [%sERROR%s] %s\n", cRed, cNone, result.Message)
720		}
721
722		isBothFailed := result.Message == "Both compilers failed to compile as expected"
723
724		if (result.Status == "PASS" || result.Status == "FAIL") && result.Target != nil && result.Reference != nil {
725			comparedFileCount++
726			totalTargetCompile += result.Target.Compile.Duration
727			totalRefCompile += result.Reference.Compile.Duration
728
729			if !isBothFailed {
730				runtimeFileCount++
731				for _, run := range result.Target.Runs {
732					totalTargetRuntime += run.Result.Duration
733				}
734				for _, run := range result.Reference.Runs {
735					totalRefRuntime += run.Result.Duration
736				}
737			}
738		}
739
740		if (result.Status == "PASS") && result.Target != nil && result.Reference != nil {
741			if *verbose && !isBothFailed {
742				refRunsMap := make(map[string]TestRun, len(result.Reference.Runs))
743				for _, run := range result.Reference.Runs {
744					refRunsMap[run.Name] = run
745				}
746				sortedTargetRuns := result.Target.Runs
747				sort.Slice(sortedTargetRuns, func(i, j int) bool {
748					return sortedTargetRuns[i].Name < sortedTargetRuns[j].Name
749				})
750
751				for _, targetRun := range sortedTargetRuns {
752					if refRun, ok := refRunsMap[targetRun.Name]; ok {
753						targetColor, refColor := cNone, cNone
754						if targetRun.Result.Duration < refRun.Result.Duration {
755							targetColor = cMagenta
756						} else if refRun.Result.Duration < targetRun.Result.Duration {
757							refColor = cMagenta
758						}
759						leftPart := fmt.Sprintf("%-*s: %s%s%s", maxLabelLen, targetName, targetColor, formatDuration(targetRun.Result.Duration), cNone)
760						rightPart := fmt.Sprintf("%-*s: %s%s%s", maxLabelLen, refName, refColor, formatDuration(refRun.Result.Duration), cNone)
761
762						fmt.Printf("  [%sPASS%s] %-*s [%s | %s]\n",
763							cGreen, cNone, maxTestNameLen, targetRun.Name,
764							leftPart, rightPart)
765					}
766				}
767			}
768
769			var summaryPadding string
770			if *verbose && maxTestNameLen > 0 {
771				// Aligns the summary block with the performance block in verbose mode.
772				// Prefix is "  [PASS] " (8) + name (maxTestNameLen) + " " (1)
773				summaryPadding = strings.Repeat(" ", 8+maxTestNameLen+1)
774			} else {
775				// In non-verbose mode, just indent slightly.
776				summaryPadding = "  "
777			}
778
779			refCompileColor, targetCompileColor := cNone, cNone
780			if result.Reference.Compile.Duration < result.Target.Compile.Duration {
781				refCompileColor = cMagenta
782			} else if result.Target.Compile.Duration < result.Reference.Compile.Duration {
783				targetCompileColor = cMagenta
784			}
785			leftCompilePart := fmt.Sprintf("%-*s: %s%s%s", maxLabelLen, targetName+"_comp", targetCompileColor, formatDuration(result.Target.Compile.Duration), cNone)
786			rightCompilePart := fmt.Sprintf("%-*s: %s%s%s", maxLabelLen, refName+"_comp", refCompileColor, formatDuration(result.Reference.Compile.Duration), cNone)
787			fmt.Printf("%s[%s | %s]\n", summaryPadding, leftCompilePart, rightCompilePart)
788
789			if !isBothFailed {
790				var totalTargetRun, totalRefRun time.Duration
791				for _, run := range result.Target.Runs {
792					totalTargetRun += run.Result.Duration
793				}
794				for _, run := range result.Reference.Runs {
795					totalRefRun += run.Result.Duration
796				}
797				refRunColor, targetRunColor := cNone, cNone
798				if totalRefRun < totalTargetRun {
799					refRunColor = cMagenta
800				} else if totalTargetRun < totalRefRun {
801					targetRunColor = cMagenta
802				}
803				leftRuntPart := fmt.Sprintf("%-*s: %s%s%s", maxLabelLen, targetName+"_runt", targetRunColor, formatDuration(totalTargetRun), cNone)
804				rightRuntPart := fmt.Sprintf("%-*s: %s%s%s", maxLabelLen, refName+"_runt", refRunColor, formatDuration(totalRefRun), cNone)
805				fmt.Printf("%s[%s | %s]\n", summaryPadding, leftRuntPart, rightRuntPart)
806			}
807
808		} else if result.Status != "PASS" && result.Target != nil && result.Reference != nil {
809			fmt.Printf("    %s compile: %s, %s compile: %s\n",
810				refName, result.Reference.Compile.Duration,
811				targetName, result.Target.Compile.Duration)
812		}
813	}
814
815	fmt.Println("----------------------------------------------------------------------")
816	fmt.Printf("%sTest Summary:%s %s%d Passed%s, %s%d Failed%s, %s%d Skipped%s, %s%d Errored%s, %d Total\n",
817		cBold, cNone, cGreen, passed, cNone, cRed, failed, cNone, cYellow, skipped, cNone, cRed, errored, cNone, len(results))
818
819	if comparedFileCount > 0 {
820		targetName, refName := filepath.Base(*targetCompiler), filepath.Base(*refCompiler)
821		avgTargetCompile := totalTargetCompile / time.Duration(comparedFileCount)
822		avgRefCompile := totalRefCompile / time.Duration(comparedFileCount)
823
824		fmt.Println("---")
825		if avgTargetCompile > avgRefCompile {
826			if avgRefCompile > 0 {
827				factor := float64(avgTargetCompile) / float64(avgRefCompile)
828				fmt.Printf("On average, %s%s%s was %s%.2fx%s slower to compile than %s.\n", cBold, targetName, cNone, cRed, factor, cNone, refName)
829			}
830		} else if avgRefCompile > avgTargetCompile {
831			if avgTargetCompile > 0 {
832				factor := float64(avgRefCompile) / float64(avgTargetCompile)
833				fmt.Printf("On average, %s%s%s was %s%.2fx%s faster to compile than %s.\n", cBold, targetName, cNone, cGreen, factor, cNone, refName)
834			}
835		}
836
837		if runtimeFileCount > 0 {
838			avgTargetRuntime := totalTargetRuntime / time.Duration(runtimeFileCount)
839			avgRefRuntime := totalRefRuntime / time.Duration(runtimeFileCount)
840			if avgTargetRuntime > avgRefRuntime {
841				if avgRefRuntime > 0 {
842					factor := float64(avgTargetRuntime) / float64(avgRefRuntime)
843					fmt.Printf("Binaries from %s%s%s ran %s%.2fx%s slower than those from %s.\n", cBold, targetName, cNone, cRed, factor, cNone, refName)
844				}
845			} else if avgRefRuntime > avgTargetRuntime {
846				if avgTargetRuntime > 0 {
847					factor := float64(avgRefRuntime) / float64(avgTargetRuntime)
848					fmt.Printf("Binaries from %s%s%s ran %s%.2fx%s faster than those from %s.\n", cBold, targetName, cNone, cGreen, factor, cNone, refName)
849				}
850			}
851		}
852	}
853}
854
855func formatDiff(diff string) string {
856	if diff == "" {
857		return ""
858	}
859	var builder strings.Builder
860	builder.WriteString("    --- Diff ---\n")
861	for _, line := range strings.Split(diff, "\n") {
862		lineWithIndent := "    " + line
863		trimmedLine := strings.TrimSpace(line)
864		if strings.HasPrefix(trimmedLine, "-") {
865			builder.WriteString(cRed)
866		} else if strings.HasPrefix(trimmedLine, "+") {
867			builder.WriteString(cGreen)
868		}
869		builder.WriteString(lineWithIndent)
870		builder.WriteString(cNone)
871		builder.WriteString("\n")
872	}
873	return builder.String()
874}
875
876func writeJSONReport(results []*FileTestResult) TestSuiteResults {
877	resultsMap := make(TestSuiteResults, len(results))
878	for _, r := range results {
879		resultsMap[r.File] = r
880	}
881
882	jsonData, err := json.MarshalIndent(resultsMap, "", "  ")
883	if err != nil {
884		log.Printf("%s[ERROR]%s Failed to marshal results to JSON: %v\n", cRed, cNone, err)
885		return resultsMap
886	}
887
888	outputFile := *outputJSON
889	if *jsonDir != "" {
890		if err := os.MkdirAll(*jsonDir, 0755); err != nil {
891			log.Printf("%s[ERROR]%s Failed to create dir %s: %v\n", cRed, cNone, *jsonDir, err)
892		}
893		outputFile = filepath.Join(*jsonDir, *outputJSON)
894	}
895
896	if err := os.WriteFile(outputFile, jsonData, 0644); err != nil {
897		log.Printf("%s[ERROR]%s Failed to write JSON report to %s: %v\n", cRed, cNone, outputFile, err)
898	} else {
899		fmt.Printf("Full test report saved to %s\n", outputFile)
900	}
901	return resultsMap
902}
903
904func hasFailures(results TestSuiteResults) bool {
905	for _, result := range results {
906		if result.Status == "FAIL" || result.Status == "ERROR" {
907			return true
908		}
909	}
910	return false
911}
912
913func expandGlobPatterns(patterns string) ([]string, error) {
914	var allFiles []string
915	seen := make(map[string]bool)
916	for _, pattern := range strings.Fields(patterns) {
917		files, err := filepath.Glob(pattern)
918		if err != nil {
919			return nil, fmt.Errorf("bad pattern %s: %w", pattern, err)
920		}
921		for _, file := range files {
922			absFile, err := filepath.Abs(file)
923			if err != nil {
924				continue // Skip files we can't resolve
925			}
926			if !seen[absFile] {
927				if info, err := os.Stat(absFile); err == nil && info.Mode().IsRegular() {
928					allFiles = append(allFiles, absFile)
929					seen[absFile] = true
930				}
931			}
932		}
933	}
934	return allFiles, nil
935}