xplshn
·
2025-09-10
main.go
Go
1// indentation logic: vibecoded
2// bad logic: written by me
3// sucks: absolutely, but instead of bitching about it, open a PR. Thanks.
4package main
5
6import (
7 "bytes"
8 "context"
9 "encoding/json"
10 "flag"
11 "fmt"
12 "io"
13 "log"
14 "os"
15 "os/exec"
16 "os/signal"
17 "path/filepath"
18 "sort"
19 "strings"
20 "sync"
21 "time"
22
23 "github.com/cespare/xxhash/v2"
24 "github.com/google/go-cmp/cmp"
25)
26
27type Execution struct {
28 Stdout string `json:"stdout"`
29 Stderr string `json:"stderr"`
30 ExitCode int `json:"exitCode"`
31 Duration time.Duration `json:"duration"`
32 TimedOut bool `json:"timed_out"`
33 UnstableOutput bool `json:"unstable_output,omitempty"`
34}
35
36type TestRun struct {
37 Name string `json:"name"`
38 Args []string `json:"args,omitempty"`
39 Input string `json:"input,omitempty"`
40 Result Execution `json:"result"`
41}
42
43type TargetResult struct {
44 BinaryPath string `json:"binary_path,omitempty"`
45 Compile Execution `json:"compile"`
46 Runs []TestRun `json:"runs"`
47}
48
49type FileTestResult struct {
50 File string `json:"file"`
51 Status string `json:"status"` // PASS, FAIL, SKIP, ERROR
52 Message string `json:"message,omitempty"`
53 Diff string `json:"diff,omitempty"`
54 Reference *TargetResult `json:"reference,omitempty"`
55 Target *TargetResult `json:"target,omitempty"`
56}
57
58type TestSuiteResults map[string]*FileTestResult
59
60var (
61 refCompiler = flag.String("ref-compiler", "b", "Path to the reference compiler.")
62 refArgs = flag.String("ref-args", "", "Arguments for the reference compiler (space-separated).")
63 targetCompiler = flag.String("target-compiler", "./gbc", "Path to the target compiler to test.")
64 targetArgs = flag.String("target-args", "", "Arguments for the target compiler (space-separated).")
65 generateGolden = flag.String("generate-golden", "", "Generate a golden .json file for a given source file.")
66 testFiles = flag.String("test-files", "tests/*.b", "Glob pattern(s) for files to test (space-separated).")
67 skipFiles = flag.String("skip-files", "", "Files to skip (space-separated).")
68 outputJSON = flag.String("output", ".test_results.json", "Output file for the JSON test report.")
69 timeout = flag.Duration("timeout", 5*time.Second, "Timeout for each command execution.")
70 jobs = flag.Int("j", 4, "Number of parallel test jobs.")
71 runs = flag.Int("runs", 5, "Number of times to run each test case to find the minimum duration.")
72 verbose = flag.Bool("v", false, "Enable verbose logging.")
73 useCache = flag.Bool("cached", false, "Use cached golden files if available.")
74 jsonDir = flag.String("dir", "", "Directory to store/read golden JSON files (defaults to source file dir).")
75 ignoreLines = flag.String("ignore-lines", "", "Comma-separated substrings to ignore during output comparison.")
76)
77
78const (
79 cRed = "\x1b[91m"
80 cYellow = "\x1b[93m"
81 cGreen = "\x1b[92m"
82 cCyan = "\x1b[96m"
83 cMagenta = "\x1b[95m"
84 cBold = "\x1b[1m"
85 cNone = "\x1b[0m"
86)
87
88func main() {
89 flag.Parse()
90 log.SetFlags(0)
91
92 if *runs < 1 {
93 *runs = 1
94 }
95
96 // Single tempDir for all test artifacts
97 tempDir, err := os.MkdirTemp("", "gtest-*")
98 if err != nil {
99 log.Fatalf("%s[ERROR]%s Failed to create temp directory: %v\n", cRed, cNone, err)
100 }
101 defer os.RemoveAll(tempDir)
102 setupInterruptHandler(tempDir)
103
104 if *generateGolden != "" {
105 handleGenerateGolden(*generateGolden, tempDir)
106 return
107 }
108
109 handleRunTestSuite(tempDir)
110}
111
112// Cleanup on Ctrl+C
113func setupInterruptHandler(tempDir string) {
114 c := make(chan os.Signal, 1)
115 signal.Notify(c, os.Interrupt)
116 go func() {
117 <-c
118 os.RemoveAll(tempDir)
119 fmt.Printf("\n%s[INTERRUPT]%s Test run cancelled. Cleaning up...\n", cYellow, cNone)
120 os.Exit(1)
121 }()
122}
123
124func getJSONPath(sourceFile string) string {
125 jsonFileName := "." + filepath.Base(sourceFile) + ".json"
126 if *jsonDir != "" {
127 return filepath.Join(*jsonDir, jsonFileName)
128 }
129 return filepath.Join(filepath.Dir(sourceFile), jsonFileName)
130}
131
132// Fast hash of file contents
133func hashFile(path string) (string, error) {
134 f, err := os.Open(path)
135 if err != nil {
136 return "", err
137 }
138 defer f.Close()
139 h := xxhash.New()
140 if _, err := io.Copy(h, f); err != nil {
141 return "", err
142 }
143 return fmt.Sprintf("%x", h.Sum64()), nil
144}
145
146func handleGenerateGolden(sourceFile, tempDir string) {
147 log.Printf("Generating golden file for %s...\n", sourceFile)
148
149 fileHash, err := hashFile(sourceFile)
150 if err != nil {
151 log.Fatalf("%s[ERROR]%s Could not hash source file %s: %v\n", cRed, cNone, sourceFile, err)
152 }
153
154 targetResult, err := compileAndRun(*targetCompiler, strings.Fields(*targetArgs), sourceFile, tempDir, fileHash)
155 if err != nil {
156 log.Fatalf("%s[ERROR]%s Could not generate golden file for %s: %v\n", cRed, cNone, sourceFile, err)
157 }
158
159 jsonData, err := json.MarshalIndent(targetResult, "", " ")
160 if err != nil {
161 log.Fatalf("%s[ERROR]%s Failed to marshal golden data to JSON: %v\n", cRed, cNone, err)
162 }
163
164 goldenFileName := getJSONPath(sourceFile)
165 if *jsonDir != "" {
166 if err := os.MkdirAll(*jsonDir, 0755); err != nil {
167 log.Fatalf("%s[ERROR]%s Failed to create directory %s: %v\n", cRed, cNone, *jsonDir, err)
168 }
169 }
170
171 if err := os.WriteFile(goldenFileName, jsonData, 0644); err != nil {
172 log.Fatalf("%s[ERROR]%s Failed to write golden file %s: %v\n", cRed, cNone, goldenFileName, err)
173 }
174
175 log.Printf("%s[SUCCESS]%s Golden file created at %s\n", cGreen, cNone, goldenFileName)
176}
177
178func handleRunTestSuite(tempDir string) {
179 _, err := exec.LookPath(*refCompiler)
180 refCompilerFound := err == nil
181 if !refCompilerFound && !*useCache {
182 log.Printf("%s[WARN]%s Reference compiler '%s' not found. Will rely on golden files. Use --cached to suppress this warning.\n", cYellow, cNone, *refCompiler)
183 }
184
185 files, err := expandGlobPatterns(*testFiles)
186 if err != nil {
187 log.Fatalf("%s[ERROR]%s Invalid glob pattern(s): %v\n", cRed, cNone, err)
188 }
189 if len(files) == 0 {
190 log.Println("No test files found matching the pattern(s).")
191 return
192 }
193
194 // Check if there's cached results to speed things up
195 previousResults := make(TestSuiteResults)
196 outputFile := *outputJSON
197 if *jsonDir != "" {
198 outputFile = filepath.Join(*jsonDir, *outputJSON)
199 }
200 if prevData, err := os.ReadFile(outputFile); err == nil {
201 if json.Unmarshal(prevData, &previousResults) != nil {
202 log.Printf("%s[WARN]%s Could not parse previous results file %s. Cache will not be used.\n", cYellow, cNone, outputFile)
203 previousResults = make(TestSuiteResults) // Reset on parse error
204 }
205 }
206
207 skipList := make(map[string]bool)
208 for _, f := range strings.Fields(*skipFiles) {
209 skipList[f] = true
210 }
211
212 tasks := make(chan string, len(files))
213 resultsChan := make(chan *FileTestResult, len(files))
214 var wg sync.WaitGroup
215
216 for i := 0; i < *jobs; i++ {
217 wg.Add(1)
218 go func() {
219 defer wg.Done()
220 for file := range tasks {
221 fileHash, err := hashFile(file)
222 if err != nil {
223 resultsChan <- &FileTestResult{File: file, Status: "ERROR", Message: "Failed to hash source file"}
224 continue
225 }
226 resultsChan <- testFile(file, tempDir, fileHash, refCompilerFound, previousResults)
227 }
228 }()
229 }
230
231 // Feed the tasks channel, skipping files with identical content
232 seenHashes := make(map[string]string)
233 for _, file := range files {
234 if skipList[file] {
235 resultsChan <- &FileTestResult{File: file, Status: "SKIP", Message: "Explicitly skipped"}
236 continue
237 }
238 fileHash, err := hashFile(file)
239 if err != nil {
240 resultsChan <- &FileTestResult{File: file, Status: "ERROR", Message: fmt.Sprintf("Failed to read file for hashing: %v", err)}
241 continue
242 }
243 if originalFile, seen := seenHashes[fileHash]; seen {
244 resultsChan <- &FileTestResult{File: file, Status: "SKIP", Message: fmt.Sprintf("Content is identical to %s", originalFile)}
245 continue
246 }
247 seenHashes[fileHash] = file
248 tasks <- file
249 }
250 close(tasks)
251
252 wg.Wait()
253 close(resultsChan)
254
255 var allResults []*FileTestResult
256 for result := range resultsChan {
257 allResults = append(allResults, result)
258 }
259
260 sort.Slice(allResults, func(i, j int) bool {
261 return allResults[i].File < allResults[j].File
262 })
263
264 printSummary(allResults)
265 resultsMap := writeJSONReport(allResults)
266
267 if hasFailures(resultsMap) {
268 os.Exit(1)
269 }
270}
271
272func testFile(file, tempDir, fileHash string, refCompilerFound bool, previousResults TestSuiteResults) *FileTestResult {
273 goldenFile := getJSONPath(file)
274 _, err := os.Stat(goldenFile)
275 hasGoldenFile := err == nil
276
277 // 1st try: Use golden file if --cached is set or for non-standard extensions
278 if (*useCache && hasGoldenFile) || !strings.HasSuffix(file, ".b") {
279 if hasGoldenFile {
280 return testWithGoldenFile(file, goldenFile, tempDir, fileHash)
281 }
282 return &FileTestResult{File: file, Status: "SKIP", Message: "Cannot test without a corresponding .json golden file"}
283 }
284
285 // 2nd try: It's a .b file, try reference compiler if it exists
286 if refCompilerFound {
287 return testWithReferenceCompiler(file, tempDir, fileHash)
288 }
289
290 // 3rd try: No reference compiler, but a golden file exists. Use it as a fallback
291 if hasGoldenFile {
292 log.Printf("[%s] No reference compiler, falling back to golden file: %s", file, goldenFile)
293 return testWithGoldenFile(file, goldenFile, tempDir, fileHash)
294 }
295
296 // 4th try: Use the existing .test_results.json if it exists
297 if prevResult, ok := previousResults[file]; ok && prevResult.Reference != nil {
298 log.Printf("[%s] Using cached reference result from previous test run.", file)
299 targetResult, err := compileAndRun(*targetCompiler, strings.Fields(*targetArgs), file, tempDir, fileHash)
300 if err != nil {
301 return &FileTestResult{
302 File: file,
303 Status: "FAIL",
304 Message: "Target compiler failed to compile, but cached reference expected success.",
305 Diff: fmt.Sprintf("Target Compiler STDERR:\n%s", targetResult.Compile.Stderr),
306 Reference: prevResult.Reference,
307 Target: targetResult,
308 }
309 }
310 comparisonResult := compareRuntimeResults(file, prevResult.Reference, targetResult)
311 comparisonResult.Message += " (against cached reference)"
312 return comparisonResult
313 }
314
315 // No way to test this file
316 return &FileTestResult{File: file, Status: "SKIP", Message: fmt.Sprintf("Reference compiler '%s' not found and no golden file or cached result exists", *refCompiler)}
317}
318
319func testWithGoldenFile(file, goldenFile, tempDir, fileHash string) *FileTestResult {
320 goldenData, err := os.ReadFile(goldenFile)
321 if err != nil {
322 return &FileTestResult{File: file, Status: "ERROR", Message: fmt.Sprintf("Could not read golden file %s: %v", goldenFile, err)}
323 }
324 var goldenResult TargetResult
325 if err := json.Unmarshal(goldenData, &goldenResult); err != nil {
326 return &FileTestResult{File: file, Status: "ERROR", Message: fmt.Sprintf("Could not parse golden file %s: %v", goldenFile, err)}
327 }
328
329 targetResult, err := compileAndRun(*targetCompiler, strings.Fields(*targetArgs), file, tempDir, fileHash)
330 if err != nil {
331 return &FileTestResult{
332 File: file,
333 Status: "FAIL",
334 Message: "Target compiler failed to compile, but golden file expected success.",
335 Diff: fmt.Sprintf("Target Compiler STDERR:\n%s", targetResult.Compile.Stderr),
336 Reference: &goldenResult,
337 Target: targetResult,
338 }
339 }
340
341 return compareRuntimeResults(file, &goldenResult, targetResult)
342}
343
344func testWithReferenceCompiler(file, tempDir, fileHash string) *FileTestResult {
345 var refResult, targetResult *TargetResult
346 var refErr, targetErr error
347 var wg sync.WaitGroup
348
349 wg.Add(2)
350 go func() {
351 defer wg.Done()
352 refResult, refErr = compileAndRun(*refCompiler, strings.Fields(*refArgs), file, tempDir, "ref-"+fileHash)
353 }()
354 go func() {
355 defer wg.Done()
356 targetResult, targetErr = compileAndRun(*targetCompiler, strings.Fields(*targetArgs), file, tempDir, "target-"+fileHash)
357 }()
358 wg.Wait()
359
360 refCompiled := refErr == nil
361 targetCompiled := targetErr == nil
362
363 if !refCompiled && !targetCompiled {
364 return &FileTestResult{File: file, Status: "PASS", Message: "Both compilers failed to compile as expected", Reference: refResult, Target: targetResult}
365 }
366 if !targetCompiled && refCompiled {
367 return &FileTestResult{
368 File: file,
369 Status: "FAIL",
370 Message: "Target compiler failed, but reference compiler succeeded",
371 Diff: fmt.Sprintf("Target Compiler STDERR:\n%s", targetResult.Compile.Stderr),
372 Reference: refResult,
373 Target: targetResult,
374 }
375 }
376 if targetCompiled && !refCompiled {
377 return &FileTestResult{
378 File: file,
379 Status: "FAIL",
380 Message: "Target compiler succeeded, but reference compiler failed",
381 Diff: fmt.Sprintf("Reference Compiler STDERR:\n%s", refResult.Compile.Stderr),
382 Reference: refResult,
383 Target: targetResult,
384 }
385 }
386
387 return compareRuntimeResults(file, refResult, targetResult)
388}
389
390func compareRuntimeResults(file string, refResult, targetResult *TargetResult) *FileTestResult {
391 var diffs strings.Builder
392 var failed bool
393
394 targetRuns := make(map[string]TestRun)
395 for _, run := range targetResult.Runs {
396 targetRuns[run.Name] = run
397 }
398
399 sort.Slice(refResult.Runs, func(i, j int) bool {
400 return refResult.Runs[i].Name < refResult.Runs[j].Name
401 })
402
403 ignoredSubstrings := []string{}
404 if *ignoreLines != "" {
405 ignoredSubstrings = strings.Split(*ignoreLines, ",")
406 }
407
408 for _, refRun := range refResult.Runs {
409 targetRun, ok := targetRuns[refRun.Name]
410 if !ok {
411 failed = true
412 diffs.WriteString(fmt.Sprintf("Test run '%s' missing in target results.\n", refRun.Name))
413 continue
414 }
415
416 if refRun.Result.UnstableOutput != targetRun.Result.UnstableOutput {
417 failed = true
418 diffs.WriteString(fmt.Sprintf("Run '%s' Output Stability Mismatch:\n - Ref: %v\n - Target: %v\n", refRun.Name, refRun.Result.UnstableOutput, targetRun.Result.UnstableOutput))
419 }
420
421 if refRun.Result.ExitCode != targetRun.Result.ExitCode {
422 failed = true
423 diffs.WriteString(fmt.Sprintf("Run '%s' Exit Code mismatch:\n - Ref: %d\n - Target: %d\n", refRun.Name, refRun.Result.ExitCode, targetRun.Result.ExitCode))
424 }
425
426 // Filter output based on --ignore-lines first
427 refStdout := filterOutput(refRun.Result.Stdout, ignoredSubstrings)
428 targetStdout := filterOutput(targetRun.Result.Stdout, ignoredSubstrings)
429 refStderr := filterOutput(refRun.Result.Stderr, ignoredSubstrings)
430 targetStderr := filterOutput(targetRun.Result.Stderr, ignoredSubstrings)
431
432 // Normalize by replacing binary paths (argv[0]) if they exist
433 // This handles cases where a program prints its own name
434 // We replace both the full path and the basename with a generic placeholder
435 const binaryPlaceholder = "__BINARY__"
436 if refResult.BinaryPath != "" {
437 refStdout = strings.ReplaceAll(refStdout, refResult.BinaryPath, binaryPlaceholder)
438 refStderr = strings.ReplaceAll(refStderr, refResult.BinaryPath, binaryPlaceholder)
439 refBase := filepath.Base(refResult.BinaryPath)
440 refStdout = strings.ReplaceAll(refStdout, refBase, binaryPlaceholder)
441 refStderr = strings.ReplaceAll(refStderr, refBase, binaryPlaceholder)
442 }
443 if targetResult.BinaryPath != "" {
444 targetStdout = strings.ReplaceAll(targetStdout, targetResult.BinaryPath, binaryPlaceholder)
445 targetStderr = strings.ReplaceAll(targetStderr, targetResult.BinaryPath, binaryPlaceholder)
446 targetBase := filepath.Base(targetResult.BinaryPath)
447 targetStdout = strings.ReplaceAll(targetStdout, targetBase, binaryPlaceholder)
448 targetStderr = strings.ReplaceAll(targetStderr, targetBase, binaryPlaceholder)
449 }
450
451 if refStdout != targetStdout {
452 failed = true
453 // Show the diff of the original, unmodified output for clarity
454 diffs.WriteString(fmt.Sprintf("Run '%s' STDOUT mismatch:\n%s", refRun.Name, cmp.Diff(refRun.Result.Stdout, targetRun.Result.Stdout)))
455 }
456
457 if refStderr != targetStderr {
458 failed = true
459 // Show the diff of the original, unmodified output for clarity
460 diffs.WriteString(fmt.Sprintf("Run '%s' STDERR mismatch:\n%s", refRun.Name, cmp.Diff(refRun.Result.Stderr, targetRun.Result.Stderr)))
461 }
462 }
463
464 if failed {
465 return &FileTestResult{
466 File: file,
467 Status: "FAIL",
468 Message: "Runtime output or exit code mismatch",
469 Diff: diffs.String(),
470 Reference: refResult,
471 Target: targetResult,
472 }
473 }
474
475 return &FileTestResult{
476 File: file,
477 Status: "PASS",
478 Message: "All test cases passed",
479 Reference: refResult,
480 Target: targetResult,
481 }
482}
483
484// Run a command with timeout, capturing output and optionally feeding stdin
485func executeCommand(ctx context.Context, command string, stdinData string, args ...string) Execution {
486 startTime := time.Now()
487 cmd := exec.CommandContext(ctx, command, args...)
488 var stdout, stderr bytes.Buffer
489 cmd.Stdout = &stdout
490 cmd.Stderr = &stderr
491
492 if stdinData != "" {
493 cmd.Stdin = strings.NewReader(stdinData)
494 }
495
496 err := cmd.Run()
497 duration := time.Since(startTime)
498
499 execResult := Execution{
500 Stdout: stdout.String(),
501 Stderr: stderr.String(),
502 Duration: duration,
503 }
504
505 if ctx.Err() == context.DeadlineExceeded {
506 execResult.TimedOut = true
507 execResult.ExitCode = -1
508 } else if err != nil {
509 if exitErr, ok := err.(*exec.ExitError); ok {
510 execResult.ExitCode = exitErr.ExitCode()
511 } else {
512 execResult.ExitCode = -2 // Should not happen often
513 execResult.Stderr += "\nExecution error: " + err.Error()
514 }
515 } else {
516 execResult.ExitCode = 0
517 }
518
519 return execResult
520}
521
522func compileAndRun(compiler string, compilerArgs []string, sourceFile, tempDir, binaryHash string) (*TargetResult, error) {
523 ctx, cancel := context.WithTimeout(context.Background(), *timeout)
524 defer cancel()
525
526 // Use the hash for a unique, deterministic binary name
527 binaryPath := filepath.Join(tempDir, binaryHash)
528
529 allArgs := []string{"-o", binaryPath}
530 allArgs = append(allArgs, compilerArgs...)
531 allArgs = append(allArgs, sourceFile)
532
533 compileResult := executeCommand(ctx, compiler, "", allArgs...)
534 if compileResult.ExitCode != 0 || compileResult.TimedOut {
535 return &TargetResult{Compile: compileResult}, fmt.Errorf("compilation failed with exit code %d", compileResult.ExitCode)
536 }
537
538 if _, err := os.Stat(binaryPath); os.IsNotExist(err) {
539 return &TargetResult{Compile: compileResult}, fmt.Errorf("compilation succeeded but binary was not created at %s", binaryPath)
540 }
541
542 // Quick test: does this program expect stdin?
543 probeCtx, probeCancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
544 defer probeCancel()
545 probeResult := executeCommand(probeCtx, binaryPath, "")
546 readsStdin := probeResult.TimedOut
547
548 testCases := map[string][]string{
549 "no_args": {},
550 "quit": {"q"},
551 "hashTable": {"s foo 10\ns bar 50\ng\ng foo\ng bar\np\nq\n"},
552 "fold": {"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzAB\n"},
553 "fold2": {"ABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWX\n"},
554 "string_arg": {"test"},
555 "numeric_arg_pos": {"5"},
556 "numeric_arg_neg": {"-5"},
557 "numeric_arg_0": {"0"},
558 }
559 var testCaseNames []string
560 for name := range testCases {
561 testCaseNames = append(testCaseNames, name)
562 }
563 sort.Strings(testCaseNames)
564
565 ignoredSubstrings := []string{}
566 if *ignoreLines != "" {
567 ignoredSubstrings = strings.Split(*ignoreLines, ",")
568 }
569
570 runResults := make([]TestRun, 0, len(testCaseNames))
571 for _, name := range testCaseNames {
572 args := testCases[name]
573 var durations []time.Duration
574 var firstRunResult Execution
575 var inputData string
576 var unstableOutput bool
577
578 // If the program is detected to read from stdin, we join the args to form the input string
579 // Otherwise, we pass them as command-line arguments
580 if readsStdin {
581 inputData = strings.Join(args, "\n")
582 if len(args) > 0 {
583 inputData += "\n"
584 }
585 args = []string{} // Clear args as they are now used for stdin
586 }
587
588 for i := 0; i < *runs; i++ {
589 runCtx, runCancel := context.WithTimeout(context.Background(), *timeout)
590 runResult := executeCommand(runCtx, binaryPath, inputData, args...)
591 runCancel()
592
593 if i == 0 {
594 firstRunResult = runResult
595 } else {
596 // Compare with the first run's output, after filtering
597 filteredFirstStdout := filterOutput(firstRunResult.Stdout, ignoredSubstrings)
598 filteredCurrentStdout := filterOutput(runResult.Stdout, ignoredSubstrings)
599 filteredFirstStderr := filterOutput(firstRunResult.Stderr, ignoredSubstrings)
600 filteredCurrentStderr := filterOutput(runResult.Stderr, ignoredSubstrings)
601
602 if firstRunResult.ExitCode != runResult.ExitCode || filteredFirstStdout != filteredCurrentStdout || filteredFirstStderr != filteredCurrentStderr {
603 unstableOutput = true
604 // Inconsistent output. We'll use the first run's result but mark it
605 // We stop iterating because finding the "fastest" run is meaningless
606 // if the output is different each time
607 break
608 }
609 }
610
611 if runResult.ExitCode != 0 || runResult.TimedOut {
612 firstRunResult = runResult
613 break
614 }
615 durations = append(durations, runResult.Duration)
616 }
617
618 if len(durations) > 0 {
619 sort.Slice(durations, func(i, j int) bool { return durations[i] < durations[j] })
620 firstRunResult.Duration = durations[0]
621 }
622 firstRunResult.UnstableOutput = unstableOutput
623
624 runResults = append(runResults, TestRun{Name: name, Args: args, Input: inputData, Result: firstRunResult})
625 if firstRunResult.ExitCode != 0 || firstRunResult.TimedOut {
626 break
627 }
628 }
629
630 return &TargetResult{Compile: compileResult, Runs: runResults, BinaryPath: binaryPath}, nil
631}
632
633// Remove lines containing any of these substrings
634func filterOutput(output string, ignoredSubstrings []string) string {
635 if len(ignoredSubstrings) == 0 || output == "" {
636 return output
637 }
638 // To preserve original line endings, we can split by \n and rejoin
639 lines := strings.Split(output, "\n")
640 filteredLines := make([]string, 0, len(lines))
641
642 for _, line := range lines {
643 ignore := false
644 for _, sub := range ignoredSubstrings {
645 if sub != "" && strings.Contains(line, sub) {
646 ignore = true
647 break
648 }
649 }
650 if !ignore {
651 filteredLines = append(filteredLines, line)
652 }
653 }
654 return strings.Join(filteredLines, "\n")
655}
656
657func formatDuration(d time.Duration) string {
658 if d < time.Millisecond {
659 return fmt.Sprintf("%6dµs", d.Microseconds())
660 }
661 return fmt.Sprintf("%6dms", d.Milliseconds())
662}
663
664func printSummary(results []*FileTestResult) {
665 var passed, failed, skipped, errored int
666 var totalTargetCompile, totalRefCompile, totalTargetRuntime, totalRefRuntime time.Duration
667 var comparedFileCount, runtimeFileCount int
668
669 // Pre-calculate max lengths for alignment
670 var maxTestNameLen int
671 targetName := filepath.Base(*targetCompiler)
672 refName := filepath.Base(*refCompiler)
673
674 // Calculate max label length for alignment of performance stats
675 labels := []string{
676 targetName,
677 refName,
678 targetName + "_comp",
679 refName + "_comp",
680 targetName + "_runt",
681 refName + "_runt",
682 }
683 var maxLabelLen int
684 for _, label := range labels {
685 if len(label) > maxLabelLen {
686 maxLabelLen = len(label)
687 }
688 }
689
690 // Only calculate maxTestNameLen if in verbose mode, as it's only used there
691 if *verbose {
692 for _, result := range results {
693 isBothFailed := result.Message == "Both compilers failed to compile as expected"
694 if result.Status == "PASS" && result.Target != nil && result.Reference != nil && !isBothFailed {
695 for _, run := range result.Target.Runs {
696 if len(run.Name) > maxTestNameLen {
697 maxTestNameLen = len(run.Name)
698 }
699 }
700 }
701 }
702 }
703
704 for _, result := range results {
705 fmt.Println("----------------------------------------------------------------------")
706 fmt.Printf("Testing %s%s%s...\n", cCyan, result.File, cNone)
707
708 switch result.Status {
709 case "PASS":
710 passed++
711 fmt.Printf(" [%sPASS%s] %s\n", cGreen, cNone, result.Message)
712 case "FAIL":
713 failed++
714 fmt.Printf(" [%sFAIL%s] %s\n", cRed, cNone, result.Message)
715 fmt.Println(formatDiff(result.Diff))
716 case "SKIP":
717 skipped++
718 fmt.Printf(" [%sSKIP%s] %s\n", cYellow, cNone, result.Message)
719 case "ERROR":
720 errored++
721 fmt.Printf(" [%sERROR%s] %s\n", cRed, cNone, result.Message)
722 }
723
724 isBothFailed := result.Message == "Both compilers failed to compile as expected"
725
726 if (result.Status == "PASS" || result.Status == "FAIL") && result.Target != nil && result.Reference != nil {
727 comparedFileCount++
728 totalTargetCompile += result.Target.Compile.Duration
729 totalRefCompile += result.Reference.Compile.Duration
730
731 if !isBothFailed {
732 runtimeFileCount++
733 for _, run := range result.Target.Runs {
734 totalTargetRuntime += run.Result.Duration
735 }
736 for _, run := range result.Reference.Runs {
737 totalRefRuntime += run.Result.Duration
738 }
739 }
740 }
741
742 if (result.Status == "PASS") && result.Target != nil && result.Reference != nil {
743 if *verbose && !isBothFailed {
744 refRunsMap := make(map[string]TestRun, len(result.Reference.Runs))
745 for _, run := range result.Reference.Runs {
746 refRunsMap[run.Name] = run
747 }
748 sortedTargetRuns := result.Target.Runs
749 sort.Slice(sortedTargetRuns, func(i, j int) bool {
750 return sortedTargetRuns[i].Name < sortedTargetRuns[j].Name
751 })
752
753 for _, targetRun := range sortedTargetRuns {
754 if refRun, ok := refRunsMap[targetRun.Name]; ok {
755 targetColor, refColor := cNone, cNone
756 if targetRun.Result.Duration < refRun.Result.Duration {
757 targetColor = cMagenta
758 } else if refRun.Result.Duration < targetRun.Result.Duration {
759 refColor = cMagenta
760 }
761 leftPart := fmt.Sprintf("%-*s: %s%s%s", maxLabelLen, targetName, targetColor, formatDuration(targetRun.Result.Duration), cNone)
762 rightPart := fmt.Sprintf("%-*s: %s%s%s", maxLabelLen, refName, refColor, formatDuration(refRun.Result.Duration), cNone)
763
764 fmt.Printf(" [%sPASS%s] %-*s [%s | %s]\n",
765 cGreen, cNone, maxTestNameLen, targetRun.Name,
766 leftPart, rightPart)
767 }
768 }
769 }
770
771 var summaryPadding string
772 if *verbose && maxTestNameLen > 0 {
773 // Aligns the summary block with the performance block in verbose mode
774 // Prefix is " [PASS] " (8) + name (maxTestNameLen) + " " (1)
775 summaryPadding = strings.Repeat(" ", 8+maxTestNameLen+1)
776 } else {
777 // In non-verbose mode, just indent slightly
778 summaryPadding = " "
779 }
780
781 refCompileColor, targetCompileColor := cNone, cNone
782 if result.Reference.Compile.Duration < result.Target.Compile.Duration {
783 refCompileColor = cMagenta
784 } else if result.Target.Compile.Duration < result.Reference.Compile.Duration {
785 targetCompileColor = cMagenta
786 }
787 leftCompilePart := fmt.Sprintf("%-*s: %s%s%s", maxLabelLen, targetName+"_comp", targetCompileColor, formatDuration(result.Target.Compile.Duration), cNone)
788 rightCompilePart := fmt.Sprintf("%-*s: %s%s%s", maxLabelLen, refName+"_comp", refCompileColor, formatDuration(result.Reference.Compile.Duration), cNone)
789 fmt.Printf("%s[%s | %s]\n", summaryPadding, leftCompilePart, rightCompilePart)
790
791 if !isBothFailed {
792 var totalTargetRun, totalRefRun time.Duration
793 for _, run := range result.Target.Runs {
794 totalTargetRun += run.Result.Duration
795 }
796 for _, run := range result.Reference.Runs {
797 totalRefRun += run.Result.Duration
798 }
799 refRunColor, targetRunColor := cNone, cNone
800 if totalRefRun < totalTargetRun {
801 refRunColor = cMagenta
802 } else if totalTargetRun < totalRefRun {
803 targetRunColor = cMagenta
804 }
805 leftRuntPart := fmt.Sprintf("%-*s: %s%s%s", maxLabelLen, targetName+"_runt", targetRunColor, formatDuration(totalTargetRun), cNone)
806 rightRuntPart := fmt.Sprintf("%-*s: %s%s%s", maxLabelLen, refName+"_runt", refRunColor, formatDuration(totalRefRun), cNone)
807 fmt.Printf("%s[%s | %s]\n", summaryPadding, leftRuntPart, rightRuntPart)
808 }
809
810 } else if result.Status != "PASS" && result.Target != nil && result.Reference != nil {
811 fmt.Printf(" %s compile: %s, %s compile: %s\n",
812 refName, result.Reference.Compile.Duration,
813 targetName, result.Target.Compile.Duration)
814 }
815 }
816
817 fmt.Println("----------------------------------------------------------------------")
818 fmt.Printf("%sTest Summary:%s %s%d Passed%s, %s%d Failed%s, %s%d Skipped%s, %s%d Errored%s, %d Total\n",
819 cBold, cNone, cGreen, passed, cNone, cRed, failed, cNone, cYellow, skipped, cNone, cRed, errored, cNone, len(results))
820
821 if comparedFileCount > 0 {
822 targetName, refName := filepath.Base(*targetCompiler), filepath.Base(*refCompiler)
823 avgTargetCompile := totalTargetCompile / time.Duration(comparedFileCount)
824 avgRefCompile := totalRefCompile / time.Duration(comparedFileCount)
825
826 fmt.Println("---")
827 if avgTargetCompile > avgRefCompile {
828 if avgRefCompile > 0 {
829 factor := float64(avgTargetCompile) / float64(avgRefCompile)
830 fmt.Printf("On average, %s%s%s was %s%.2fx%s slower to compile than %s.\n", cBold, targetName, cNone, cRed, factor, cNone, refName)
831 }
832 } else if avgRefCompile > avgTargetCompile {
833 if avgTargetCompile > 0 {
834 factor := float64(avgRefCompile) / float64(avgTargetCompile)
835 fmt.Printf("On average, %s%s%s was %s%.2fx%s faster to compile than %s.\n", cBold, targetName, cNone, cGreen, factor, cNone, refName)
836 }
837 }
838
839 if runtimeFileCount > 0 {
840 avgTargetRuntime := totalTargetRuntime / time.Duration(runtimeFileCount)
841 avgRefRuntime := totalRefRuntime / time.Duration(runtimeFileCount)
842 if avgTargetRuntime > avgRefRuntime {
843 if avgRefRuntime > 0 {
844 factor := float64(avgTargetRuntime) / float64(avgRefRuntime)
845 fmt.Printf("Binaries from %s%s%s ran %s%.2fx%s slower than those from %s.\n", cBold, targetName, cNone, cRed, factor, cNone, refName)
846 }
847 } else if avgRefRuntime > avgTargetRuntime {
848 if avgTargetRuntime > 0 {
849 factor := float64(avgRefRuntime) / float64(avgTargetRuntime)
850 fmt.Printf("Binaries from %s%s%s ran %s%.2fx%s faster than those from %s.\n", cBold, targetName, cNone, cGreen, factor, cNone, refName)
851 }
852 }
853 }
854 }
855}
856
857func formatDiff(diff string) string {
858 if diff == "" {
859 return ""
860 }
861 var builder strings.Builder
862 builder.WriteString(" --- Diff ---\n")
863 for _, line := range strings.Split(diff, "\n") {
864 lineWithIndent := " " + line
865 trimmedLine := strings.TrimSpace(line)
866 if strings.HasPrefix(trimmedLine, "-") {
867 builder.WriteString(cRed)
868 } else if strings.HasPrefix(trimmedLine, "+") {
869 builder.WriteString(cGreen)
870 }
871 builder.WriteString(lineWithIndent)
872 builder.WriteString(cNone)
873 builder.WriteString("\n")
874 }
875 return builder.String()
876}
877
878func writeJSONReport(results []*FileTestResult) TestSuiteResults {
879 resultsMap := make(TestSuiteResults, len(results))
880 for _, r := range results {
881 resultsMap[r.File] = r
882 }
883
884 jsonData, err := json.MarshalIndent(resultsMap, "", " ")
885 if err != nil {
886 log.Printf("%s[ERROR]%s Failed to marshal results to JSON: %v\n", cRed, cNone, err)
887 return resultsMap
888 }
889
890 outputFile := *outputJSON
891 if *jsonDir != "" {
892 if err := os.MkdirAll(*jsonDir, 0755); err != nil {
893 log.Printf("%s[ERROR]%s Failed to create dir %s: %v\n", cRed, cNone, *jsonDir, err)
894 }
895 outputFile = filepath.Join(*jsonDir, *outputJSON)
896 }
897
898 if err := os.WriteFile(outputFile, jsonData, 0644); err != nil {
899 log.Printf("%s[ERROR]%s Failed to write JSON report to %s: %v\n", cRed, cNone, outputFile, err)
900 } else {
901 fmt.Printf("Full test report saved to %s\n", outputFile)
902 }
903 return resultsMap
904}
905
906func hasFailures(results TestSuiteResults) bool {
907 for _, result := range results {
908 if result.Status == "FAIL" || result.Status == "ERROR" {
909 return true
910 }
911 }
912 return false
913}
914
915func expandGlobPatterns(patterns string) ([]string, error) {
916 var allFiles []string
917 seen := make(map[string]bool)
918 for _, pattern := range strings.Fields(patterns) {
919 files, err := filepath.Glob(pattern)
920 if err != nil {
921 return nil, fmt.Errorf("bad pattern %s: %w", pattern, err)
922 }
923 for _, file := range files {
924 absFile, err := filepath.Abs(file)
925 if err != nil {
926 continue // Skip files we can't resolve
927 }
928 if !seen[absFile] {
929 if info, err := os.Stat(absFile); err == nil && info.Mode().IsRegular() {
930 allFiles = append(allFiles, absFile)
931 seen[absFile] = true
932 }
933 }
934 }
935 }
936 return allFiles, nil
937}