xplshn
·
2025-08-13
main.go
Go
1package main
2
3import (
4 "bytes"
5 "context"
6 "encoding/json"
7 "flag"
8 "fmt"
9 "io"
10 "log"
11 "os"
12 "os/exec"
13 "os/signal"
14 "path/filepath"
15 "sort"
16 "strings"
17 "sync"
18 "time"
19
20 "github.com/cespare/xxhash/v2"
21 "github.com/google/go-cmp/cmp"
22)
23
24type Execution struct {
25 Stdout string `json:"stdout"`
26 Stderr string `json:"stderr"`
27 ExitCode int `json:"exitCode"`
28 Duration time.Duration `json:"duration"`
29 TimedOut bool `json:"timed_out"`
30 UnstableOutput bool `json:"unstable_output,omitempty"`
31}
32
33type TestRun struct {
34 Name string `json:"name"`
35 Args []string `json:"args,omitempty"`
36 Input string `json:"input,omitempty"`
37 Result Execution `json:"result"`
38}
39
40type TargetResult struct {
41 BinaryPath string `json:"binary_path,omitempty"`
42 Compile Execution `json:"compile"`
43 Runs []TestRun `json:"runs"`
44}
45
46type FileTestResult struct {
47 File string `json:"file"`
48 Status string `json:"status"` // PASS, FAIL, SKIP, ERROR
49 Message string `json:"message,omitempty"`
50 Diff string `json:"diff,omitempty"`
51 Reference *TargetResult `json:"reference,omitempty"`
52 Target *TargetResult `json:"target,omitempty"`
53}
54
55type TestSuiteResults map[string]*FileTestResult
56
57var (
58 refCompiler = flag.String("ref-compiler", "b", "Path to the reference compiler.")
59 refArgs = flag.String("ref-args", "", "Arguments for the reference compiler (space-separated).")
60 targetCompiler = flag.String("target-compiler", "./gbc", "Path to the target compiler to test.")
61 targetArgs = flag.String("target-args", "", "Arguments for the target compiler (space-separated).")
62 generateGolden = flag.String("generate-golden", "", "Generate a golden .json file for a given source file.")
63 testFiles = flag.String("test-files", "tests/*.b", "Glob pattern(s) for files to test (space-separated).")
64 skipFiles = flag.String("skip-files", "", "Files to skip (space-separated).")
65 outputJSON = flag.String("output", ".test_results.json", "Output file for the JSON test report.")
66 timeout = flag.Duration("timeout", 5*time.Second, "Timeout for each command execution.")
67 jobs = flag.Int("j", 4, "Number of parallel test jobs.")
68 runs = flag.Int("runs", 5, "Number of times to run each test case to find the minimum duration.")
69 verbose = flag.Bool("v", false, "Enable verbose logging.")
70 useCache = flag.Bool("cached", false, "Use cached golden files if available.")
71 jsonDir = flag.String("dir", "", "Directory to store/read golden JSON files (defaults to source file dir).")
72 ignoreLines = flag.String("ignore-lines", "", "Comma-separated substrings to ignore during output comparison.")
73)
74
75const (
76 cRed = "\x1b[91m"
77 cYellow = "\x1b[93m"
78 cGreen = "\x1b[92m"
79 cCyan = "\x1b[96m"
80 cMagenta = "\x1b[95m"
81 cBold = "\x1b[1m"
82 cNone = "\x1b[0m"
83)
84
85func main() {
86 flag.Parse()
87 log.SetFlags(0)
88
89 if *runs < 1 {
90 *runs = 1
91 }
92
93 // Single tempDir for all test artifacts
94 tempDir, err := os.MkdirTemp("", "gtest-*")
95 if err != nil {
96 log.Fatalf("%s[ERROR]%s Failed to create temp directory: %v\n", cRed, cNone, err)
97 }
98 defer os.RemoveAll(tempDir)
99 setupInterruptHandler(tempDir)
100
101 if *generateGolden != "" {
102 handleGenerateGolden(*generateGolden, tempDir)
103 return
104 }
105
106 handleRunTestSuite(tempDir)
107}
108
109// setupInterruptHandler is used to clean up on CTRL+C
110func setupInterruptHandler(tempDir string) {
111 c := make(chan os.Signal, 1)
112 signal.Notify(c, os.Interrupt)
113 go func() {
114 <-c
115 os.RemoveAll(tempDir)
116 fmt.Printf("\n%s[INTERRUPT]%s Test run cancelled. Cleaning up...\n", cYellow, cNone)
117 os.Exit(1)
118 }()
119}
120
121func getJSONPath(sourceFile string) string {
122 jsonFileName := "." + filepath.Base(sourceFile) + ".json"
123 if *jsonDir != "" {
124 return filepath.Join(*jsonDir, jsonFileName)
125 }
126 return filepath.Join(filepath.Dir(sourceFile), jsonFileName)
127}
128
129// hashFile computes the xxhash of a file's content
130func hashFile(path string) (string, error) {
131 f, err := os.Open(path)
132 if err != nil {
133 return "", err
134 }
135 defer f.Close()
136 h := xxhash.New()
137 if _, err := io.Copy(h, f); err != nil {
138 return "", err
139 }
140 return fmt.Sprintf("%x", h.Sum64()), nil
141}
142
143func handleGenerateGolden(sourceFile, tempDir string) {
144 log.Printf("Generating golden file for %s...\n", sourceFile)
145
146 fileHash, err := hashFile(sourceFile)
147 if err != nil {
148 log.Fatalf("%s[ERROR]%s Could not hash source file %s: %v\n", cRed, cNone, sourceFile, err)
149 }
150
151 targetResult, err := compileAndRun(*targetCompiler, strings.Fields(*targetArgs), sourceFile, tempDir, fileHash)
152 if err != nil {
153 log.Fatalf("%s[ERROR]%s Could not generate golden file for %s: %v\n", cRed, cNone, sourceFile, err)
154 }
155
156 jsonData, err := json.MarshalIndent(targetResult, "", " ")
157 if err != nil {
158 log.Fatalf("%s[ERROR]%s Failed to marshal golden data to JSON: %v\n", cRed, cNone, err)
159 }
160
161 goldenFileName := getJSONPath(sourceFile)
162 if *jsonDir != "" {
163 if err := os.MkdirAll(*jsonDir, 0755); err != nil {
164 log.Fatalf("%s[ERROR]%s Failed to create directory %s: %v\n", cRed, cNone, *jsonDir, err)
165 }
166 }
167
168 if err := os.WriteFile(goldenFileName, jsonData, 0644); err != nil {
169 log.Fatalf("%s[ERROR]%s Failed to write golden file %s: %v\n", cRed, cNone, goldenFileName, err)
170 }
171
172 log.Printf("%s[SUCCESS]%s Golden file created at %s\n", cGreen, cNone, goldenFileName)
173}
174
175func handleRunTestSuite(tempDir string) {
176 _, err := exec.LookPath(*refCompiler)
177 refCompilerFound := err == nil
178 if !refCompilerFound && !*useCache {
179 log.Printf("%s[WARN]%s Reference compiler '%s' not found. Will rely on golden files. Use --cached to suppress this warning.\n", cYellow, cNone, *refCompiler)
180 }
181
182 files, err := expandGlobPatterns(*testFiles)
183 if err != nil {
184 log.Fatalf("%s[ERROR]%s Invalid glob pattern(s): %v\n", cRed, cNone, err)
185 }
186 if len(files) == 0 {
187 log.Println("No test files found matching the pattern(s).")
188 return
189 }
190
191 // Load previous results for caching reference compiler output
192 previousResults := make(TestSuiteResults)
193 outputFile := *outputJSON
194 if *jsonDir != "" {
195 outputFile = filepath.Join(*jsonDir, *outputJSON)
196 }
197 if prevData, err := os.ReadFile(outputFile); err == nil {
198 if json.Unmarshal(prevData, &previousResults) != nil {
199 log.Printf("%s[WARN]%s Could not parse previous results file %s. Cache will not be used.\n", cYellow, cNone, outputFile)
200 previousResults = make(TestSuiteResults) // Reset on parse error
201 }
202 }
203
204 skipList := make(map[string]bool)
205 for _, f := range strings.Fields(*skipFiles) {
206 skipList[f] = true
207 }
208
209 tasks := make(chan string, len(files))
210 resultsChan := make(chan *FileTestResult, len(files))
211 var wg sync.WaitGroup
212
213 for i := 0; i < *jobs; i++ {
214 wg.Add(1)
215 go func() {
216 defer wg.Done()
217 for file := range tasks {
218 fileHash, err := hashFile(file)
219 if err != nil {
220 resultsChan <- &FileTestResult{File: file, Status: "ERROR", Message: "Failed to hash source file"}
221 continue
222 }
223 resultsChan <- testFile(file, tempDir, fileHash, refCompilerFound, previousResults)
224 }
225 }()
226 }
227
228 // Feed the tasks channel, skipping files with identical content
229 seenHashes := make(map[string]string)
230 for _, file := range files {
231 if skipList[file] {
232 resultsChan <- &FileTestResult{File: file, Status: "SKIP", Message: "Explicitly skipped"}
233 continue
234 }
235 fileHash, err := hashFile(file)
236 if err != nil {
237 resultsChan <- &FileTestResult{File: file, Status: "ERROR", Message: fmt.Sprintf("Failed to read file for hashing: %v", err)}
238 continue
239 }
240 if originalFile, seen := seenHashes[fileHash]; seen {
241 resultsChan <- &FileTestResult{File: file, Status: "SKIP", Message: fmt.Sprintf("Content is identical to %s", originalFile)}
242 continue
243 }
244 seenHashes[fileHash] = file
245 tasks <- file
246 }
247 close(tasks)
248
249 wg.Wait()
250 close(resultsChan)
251
252 var allResults []*FileTestResult
253 for result := range resultsChan {
254 allResults = append(allResults, result)
255 }
256
257 sort.Slice(allResults, func(i, j int) bool {
258 return allResults[i].File < allResults[j].File
259 })
260
261 printSummary(allResults)
262 resultsMap := writeJSONReport(allResults)
263
264 if hasFailures(resultsMap) {
265 os.Exit(1)
266 }
267}
268
269func testFile(file, tempDir, fileHash string, refCompilerFound bool, previousResults TestSuiteResults) *FileTestResult {
270 goldenFile := getJSONPath(file)
271 _, err := os.Stat(goldenFile)
272 hasGoldenFile := err == nil
273
274 // 1st try: Use golden file if --cached is set or for non-standard extensions
275 if (*useCache && hasGoldenFile) || !strings.HasSuffix(file, ".b") {
276 if hasGoldenFile {
277 return testWithGoldenFile(file, goldenFile, tempDir, fileHash)
278 }
279 return &FileTestResult{File: file, Status: "SKIP", Message: "Cannot test without a corresponding .json golden file"}
280 }
281
282 // 2nd try: It's a .b file, try reference compiler if it exists
283 if refCompilerFound {
284 return testWithReferenceCompiler(file, tempDir, fileHash)
285 }
286
287 // 3rd try: No reference compiler, but a golden file exists. Use it as a fallback
288 if hasGoldenFile {
289 log.Printf("[%s] No reference compiler, falling back to golden file: %s", file, goldenFile)
290 return testWithGoldenFile(file, goldenFile, tempDir, fileHash)
291 }
292
293 // 4th try: Use the existing .test_results.json if it exists
294 if prevResult, ok := previousResults[file]; ok && prevResult.Reference != nil {
295 log.Printf("[%s] Using cached reference result from previous test run.", file)
296 targetResult, err := compileAndRun(*targetCompiler, strings.Fields(*targetArgs), file, tempDir, fileHash)
297 if err != nil {
298 return &FileTestResult{
299 File: file,
300 Status: "FAIL",
301 Message: "Target compiler failed to compile, but cached reference expected success.",
302 Diff: fmt.Sprintf("Target Compiler STDERR:\n%s", targetResult.Compile.Stderr),
303 Reference: prevResult.Reference,
304 Target: targetResult,
305 }
306 }
307 comparisonResult := compareRuntimeResults(file, prevResult.Reference, targetResult)
308 comparisonResult.Message += " (against cached reference)"
309 return comparisonResult
310 }
311
312 // No way to test this file
313 return &FileTestResult{File: file, Status: "SKIP", Message: fmt.Sprintf("Reference compiler '%s' not found and no golden file or cached result exists", *refCompiler)}
314}
315
316func testWithGoldenFile(file, goldenFile, tempDir, fileHash string) *FileTestResult {
317 goldenData, err := os.ReadFile(goldenFile)
318 if err != nil {
319 return &FileTestResult{File: file, Status: "ERROR", Message: fmt.Sprintf("Could not read golden file %s: %v", goldenFile, err)}
320 }
321 var goldenResult TargetResult
322 if err := json.Unmarshal(goldenData, &goldenResult); err != nil {
323 return &FileTestResult{File: file, Status: "ERROR", Message: fmt.Sprintf("Could not parse golden file %s: %v", goldenFile, err)}
324 }
325
326 targetResult, err := compileAndRun(*targetCompiler, strings.Fields(*targetArgs), file, tempDir, fileHash)
327 if err != nil {
328 return &FileTestResult{
329 File: file,
330 Status: "FAIL",
331 Message: "Target compiler failed to compile, but golden file expected success.",
332 Diff: fmt.Sprintf("Target Compiler STDERR:\n%s", targetResult.Compile.Stderr),
333 Reference: &goldenResult,
334 Target: targetResult,
335 }
336 }
337
338 return compareRuntimeResults(file, &goldenResult, targetResult)
339}
340
341func testWithReferenceCompiler(file, tempDir, fileHash string) *FileTestResult {
342 var refResult, targetResult *TargetResult
343 var refErr, targetErr error
344 var wg sync.WaitGroup
345
346 wg.Add(2)
347 go func() {
348 defer wg.Done()
349 refResult, refErr = compileAndRun(*refCompiler, strings.Fields(*refArgs), file, tempDir, "ref-"+fileHash)
350 }()
351 go func() {
352 defer wg.Done()
353 targetResult, targetErr = compileAndRun(*targetCompiler, strings.Fields(*targetArgs), file, tempDir, "target-"+fileHash)
354 }()
355 wg.Wait()
356
357 refCompiled := refErr == nil
358 targetCompiled := targetErr == nil
359
360 if !refCompiled && !targetCompiled {
361 return &FileTestResult{File: file, Status: "PASS", Message: "Both compilers failed to compile as expected", Reference: refResult, Target: targetResult}
362 }
363 if !targetCompiled && refCompiled {
364 return &FileTestResult{
365 File: file,
366 Status: "FAIL",
367 Message: "Target compiler failed, but reference compiler succeeded",
368 Diff: fmt.Sprintf("Target Compiler STDERR:\n%s", targetResult.Compile.Stderr),
369 Reference: refResult,
370 Target: targetResult,
371 }
372 }
373 if targetCompiled && !refCompiled {
374 return &FileTestResult{
375 File: file,
376 Status: "FAIL",
377 Message: "Target compiler succeeded, but reference compiler failed",
378 Diff: fmt.Sprintf("Reference Compiler STDERR:\n%s", refResult.Compile.Stderr),
379 Reference: refResult,
380 Target: targetResult,
381 }
382 }
383
384 return compareRuntimeResults(file, refResult, targetResult)
385}
386
387func compareRuntimeResults(file string, refResult, targetResult *TargetResult) *FileTestResult {
388 var diffs strings.Builder
389 var failed bool
390
391 targetRuns := make(map[string]TestRun)
392 for _, run := range targetResult.Runs {
393 targetRuns[run.Name] = run
394 }
395
396 sort.Slice(refResult.Runs, func(i, j int) bool {
397 return refResult.Runs[i].Name < refResult.Runs[j].Name
398 })
399
400 ignoredSubstrings := []string{}
401 if *ignoreLines != "" {
402 ignoredSubstrings = strings.Split(*ignoreLines, ",")
403 }
404
405 for _, refRun := range refResult.Runs {
406 targetRun, ok := targetRuns[refRun.Name]
407 if !ok {
408 failed = true
409 diffs.WriteString(fmt.Sprintf("Test run '%s' missing in target results.\n", refRun.Name))
410 continue
411 }
412
413 if refRun.Result.UnstableOutput != targetRun.Result.UnstableOutput {
414 failed = true
415 diffs.WriteString(fmt.Sprintf("Run '%s' Output Stability Mismatch:\n - Ref: %v\n - Target: %v\n", refRun.Name, refRun.Result.UnstableOutput, targetRun.Result.UnstableOutput))
416 }
417
418 if refRun.Result.ExitCode != targetRun.Result.ExitCode {
419 failed = true
420 diffs.WriteString(fmt.Sprintf("Run '%s' Exit Code mismatch:\n - Ref: %d\n - Target: %d\n", refRun.Name, refRun.Result.ExitCode, targetRun.Result.ExitCode))
421 }
422
423 // Filter output based on --ignore-lines first
424 refStdout := filterOutput(refRun.Result.Stdout, ignoredSubstrings)
425 targetStdout := filterOutput(targetRun.Result.Stdout, ignoredSubstrings)
426 refStderr := filterOutput(refRun.Result.Stderr, ignoredSubstrings)
427 targetStderr := filterOutput(targetRun.Result.Stderr, ignoredSubstrings)
428
429 // Normalize by replacing binary paths (argv[0]) if they exist.
430 // This handles cases where a program prints its own name.
431 // We replace both the full path and the basename with a generic placeholder.
432 const binaryPlaceholder = "__BINARY__"
433 if refResult.BinaryPath != "" {
434 refStdout = strings.ReplaceAll(refStdout, refResult.BinaryPath, binaryPlaceholder)
435 refStderr = strings.ReplaceAll(refStderr, refResult.BinaryPath, binaryPlaceholder)
436 refBase := filepath.Base(refResult.BinaryPath)
437 refStdout = strings.ReplaceAll(refStdout, refBase, binaryPlaceholder)
438 refStderr = strings.ReplaceAll(refStderr, refBase, binaryPlaceholder)
439 }
440 if targetResult.BinaryPath != "" {
441 targetStdout = strings.ReplaceAll(targetStdout, targetResult.BinaryPath, binaryPlaceholder)
442 targetStderr = strings.ReplaceAll(targetStderr, targetResult.BinaryPath, binaryPlaceholder)
443 targetBase := filepath.Base(targetResult.BinaryPath)
444 targetStdout = strings.ReplaceAll(targetStdout, targetBase, binaryPlaceholder)
445 targetStderr = strings.ReplaceAll(targetStderr, targetBase, binaryPlaceholder)
446 }
447
448 if refStdout != targetStdout {
449 failed = true
450 // Show the diff of the original, unmodified output for clarity.
451 diffs.WriteString(fmt.Sprintf("Run '%s' STDOUT mismatch:\n%s", refRun.Name, cmp.Diff(refRun.Result.Stdout, targetRun.Result.Stdout)))
452 }
453
454 if refStderr != targetStderr {
455 failed = true
456 // Show the diff of the original, unmodified output for clarity.
457 diffs.WriteString(fmt.Sprintf("Run '%s' STDERR mismatch:\n%s", refRun.Name, cmp.Diff(refRun.Result.Stderr, targetRun.Result.Stderr)))
458 }
459 }
460
461 if failed {
462 return &FileTestResult{
463 File: file,
464 Status: "FAIL",
465 Message: "Runtime output or exit code mismatch",
466 Diff: diffs.String(),
467 Reference: refResult,
468 Target: targetResult,
469 }
470 }
471
472 return &FileTestResult{
473 File: file,
474 Status: "PASS",
475 Message: "All test cases passed",
476 Reference: refResult,
477 Target: targetResult,
478 }
479}
480
481// executeCommand runs a command with a timeout and captures its output, optionally piping data to stdin
482func executeCommand(ctx context.Context, command string, stdinData string, args ...string) Execution {
483 startTime := time.Now()
484 cmd := exec.CommandContext(ctx, command, args...)
485 var stdout, stderr bytes.Buffer
486 cmd.Stdout = &stdout
487 cmd.Stderr = &stderr
488
489 if stdinData != "" {
490 cmd.Stdin = strings.NewReader(stdinData)
491 }
492
493 err := cmd.Run()
494 duration := time.Since(startTime)
495
496 execResult := Execution{
497 Stdout: stdout.String(),
498 Stderr: stderr.String(),
499 Duration: duration,
500 }
501
502 if ctx.Err() == context.DeadlineExceeded {
503 execResult.TimedOut = true
504 execResult.ExitCode = -1
505 } else if err != nil {
506 if exitErr, ok := err.(*exec.ExitError); ok {
507 execResult.ExitCode = exitErr.ExitCode()
508 } else {
509 execResult.ExitCode = -2 // Should not happen often
510 execResult.Stderr += "\nExecution error: " + err.Error()
511 }
512 } else {
513 execResult.ExitCode = 0
514 }
515
516 return execResult
517}
518
519func compileAndRun(compiler string, compilerArgs []string, sourceFile, tempDir, binaryHash string) (*TargetResult, error) {
520 ctx, cancel := context.WithTimeout(context.Background(), *timeout)
521 defer cancel()
522
523 // Use the hash for a unique, deterministic binary name
524 binaryPath := filepath.Join(tempDir, binaryHash)
525
526 allArgs := []string{"-o", binaryPath}
527 allArgs = append(allArgs, compilerArgs...)
528 allArgs = append(allArgs, sourceFile)
529
530 compileResult := executeCommand(ctx, compiler, "", allArgs...)
531 if compileResult.ExitCode != 0 || compileResult.TimedOut {
532 return &TargetResult{Compile: compileResult}, fmt.Errorf("compilation failed with exit code %d", compileResult.ExitCode)
533 }
534
535 if _, err := os.Stat(binaryPath); os.IsNotExist(err) {
536 return &TargetResult{Compile: compileResult}, fmt.Errorf("compilation succeeded but binary was not created at %s", binaryPath)
537 }
538
539 // Probe to see if the binary waits for stdin by running it with a very short timeout.
540 // If it times out, it's likely waiting for input.
541 probeCtx, probeCancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
542 defer probeCancel()
543 probeResult := executeCommand(probeCtx, binaryPath, "")
544 readsStdin := probeResult.TimedOut
545
546 testCases := map[string][]string{
547 "no_args": {},
548 "quit": {"q"},
549 "hashTable": {"s foo 10\ns bar 50\ng\ng foo\ng bar\np\nq\n"},
550 "fold": {"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzAB\n"},
551 "fold2": {"ABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWX\n"},
552 "string_arg": {"test"},
553 "numeric_arg_pos": {"5"},
554 "numeric_arg_neg": {"-5"},
555 "numeric_arg_0": {"0"},
556 }
557 var testCaseNames []string
558 for name := range testCases {
559 testCaseNames = append(testCaseNames, name)
560 }
561 sort.Strings(testCaseNames)
562
563 ignoredSubstrings := []string{}
564 if *ignoreLines != "" {
565 ignoredSubstrings = strings.Split(*ignoreLines, ",")
566 }
567
568 runResults := make([]TestRun, 0, len(testCaseNames))
569 for _, name := range testCaseNames {
570 args := testCases[name]
571 var durations []time.Duration
572 var firstRunResult Execution
573 var inputData string
574 var unstableOutput bool
575
576 // If the program is detected to read from stdin, we join the args to form the input string.
577 // Otherwise, we pass them as command-line arguments.
578 if readsStdin {
579 inputData = strings.Join(args, "\n")
580 if len(args) > 0 {
581 inputData += "\n"
582 }
583 args = []string{} // Clear args as they are now used for stdin
584 }
585
586 for i := 0; i < *runs; i++ {
587 runCtx, runCancel := context.WithTimeout(context.Background(), *timeout)
588 runResult := executeCommand(runCtx, binaryPath, inputData, args...)
589 runCancel()
590
591 if i == 0 {
592 firstRunResult = runResult
593 } else {
594 // Compare with the first run's output, after filtering
595 filteredFirstStdout := filterOutput(firstRunResult.Stdout, ignoredSubstrings)
596 filteredCurrentStdout := filterOutput(runResult.Stdout, ignoredSubstrings)
597 filteredFirstStderr := filterOutput(firstRunResult.Stderr, ignoredSubstrings)
598 filteredCurrentStderr := filterOutput(runResult.Stderr, ignoredSubstrings)
599
600 if firstRunResult.ExitCode != runResult.ExitCode || filteredFirstStdout != filteredCurrentStdout || filteredFirstStderr != filteredCurrentStderr {
601 unstableOutput = true
602 // Inconsistent output. We'll use the first run's result but mark it.
603 // We stop iterating because finding the "fastest" run is meaningless
604 // if the output is different each time.
605 break
606 }
607 }
608
609 if runResult.ExitCode != 0 || runResult.TimedOut {
610 firstRunResult = runResult
611 break
612 }
613 durations = append(durations, runResult.Duration)
614 }
615
616 if len(durations) > 0 {
617 sort.Slice(durations, func(i, j int) bool { return durations[i] < durations[j] })
618 firstRunResult.Duration = durations[0]
619 }
620 firstRunResult.UnstableOutput = unstableOutput
621
622 runResults = append(runResults, TestRun{Name: name, Args: args, Input: inputData, Result: firstRunResult})
623 if firstRunResult.ExitCode != 0 || firstRunResult.TimedOut {
624 break
625 }
626 }
627
628 return &TargetResult{Compile: compileResult, Runs: runResults, BinaryPath: binaryPath}, nil
629}
630
631// filterOutput removes lines containing any of the given substrings.
632func filterOutput(output string, ignoredSubstrings []string) string {
633 if len(ignoredSubstrings) == 0 || output == "" {
634 return output
635 }
636 // To preserve original line endings, we can split by \n and rejoin
637 lines := strings.Split(output, "\n")
638 filteredLines := make([]string, 0, len(lines))
639
640 for _, line := range lines {
641 ignore := false
642 for _, sub := range ignoredSubstrings {
643 if sub != "" && strings.Contains(line, sub) {
644 ignore = true
645 break
646 }
647 }
648 if !ignore {
649 filteredLines = append(filteredLines, line)
650 }
651 }
652 return strings.Join(filteredLines, "\n")
653}
654
655func formatDuration(d time.Duration) string {
656 if d < time.Millisecond {
657 return fmt.Sprintf("%6dµs", d.Microseconds())
658 }
659 return fmt.Sprintf("%6dms", d.Milliseconds())
660}
661
662func printSummary(results []*FileTestResult) {
663 var passed, failed, skipped, errored int
664 var totalTargetCompile, totalRefCompile, totalTargetRuntime, totalRefRuntime time.Duration
665 var comparedFileCount, runtimeFileCount int
666
667 // Pre-calculate max lengths for alignment.
668 var maxTestNameLen int
669 targetName := filepath.Base(*targetCompiler)
670 refName := filepath.Base(*refCompiler)
671
672 // Calculate max label length for alignment of performance stats
673 labels := []string{
674 targetName,
675 refName,
676 targetName + "_comp",
677 refName + "_comp",
678 targetName + "_runt",
679 refName + "_runt",
680 }
681 var maxLabelLen int
682 for _, label := range labels {
683 if len(label) > maxLabelLen {
684 maxLabelLen = len(label)
685 }
686 }
687
688 // Only calculate maxTestNameLen if in verbose mode, as it's only used there.
689 if *verbose {
690 for _, result := range results {
691 isBothFailed := result.Message == "Both compilers failed to compile as expected"
692 if result.Status == "PASS" && result.Target != nil && result.Reference != nil && !isBothFailed {
693 for _, run := range result.Target.Runs {
694 if len(run.Name) > maxTestNameLen {
695 maxTestNameLen = len(run.Name)
696 }
697 }
698 }
699 }
700 }
701
702 for _, result := range results {
703 fmt.Println("----------------------------------------------------------------------")
704 fmt.Printf("Testing %s%s%s...\n", cCyan, result.File, cNone)
705
706 switch result.Status {
707 case "PASS":
708 passed++
709 fmt.Printf(" [%sPASS%s] %s\n", cGreen, cNone, result.Message)
710 case "FAIL":
711 failed++
712 fmt.Printf(" [%sFAIL%s] %s\n", cRed, cNone, result.Message)
713 fmt.Println(formatDiff(result.Diff))
714 case "SKIP":
715 skipped++
716 fmt.Printf(" [%sSKIP%s] %s\n", cYellow, cNone, result.Message)
717 case "ERROR":
718 errored++
719 fmt.Printf(" [%sERROR%s] %s\n", cRed, cNone, result.Message)
720 }
721
722 isBothFailed := result.Message == "Both compilers failed to compile as expected"
723
724 if (result.Status == "PASS" || result.Status == "FAIL") && result.Target != nil && result.Reference != nil {
725 comparedFileCount++
726 totalTargetCompile += result.Target.Compile.Duration
727 totalRefCompile += result.Reference.Compile.Duration
728
729 if !isBothFailed {
730 runtimeFileCount++
731 for _, run := range result.Target.Runs {
732 totalTargetRuntime += run.Result.Duration
733 }
734 for _, run := range result.Reference.Runs {
735 totalRefRuntime += run.Result.Duration
736 }
737 }
738 }
739
740 if (result.Status == "PASS") && result.Target != nil && result.Reference != nil {
741 if *verbose && !isBothFailed {
742 refRunsMap := make(map[string]TestRun, len(result.Reference.Runs))
743 for _, run := range result.Reference.Runs {
744 refRunsMap[run.Name] = run
745 }
746 sortedTargetRuns := result.Target.Runs
747 sort.Slice(sortedTargetRuns, func(i, j int) bool {
748 return sortedTargetRuns[i].Name < sortedTargetRuns[j].Name
749 })
750
751 for _, targetRun := range sortedTargetRuns {
752 if refRun, ok := refRunsMap[targetRun.Name]; ok {
753 targetColor, refColor := cNone, cNone
754 if targetRun.Result.Duration < refRun.Result.Duration {
755 targetColor = cMagenta
756 } else if refRun.Result.Duration < targetRun.Result.Duration {
757 refColor = cMagenta
758 }
759 leftPart := fmt.Sprintf("%-*s: %s%s%s", maxLabelLen, targetName, targetColor, formatDuration(targetRun.Result.Duration), cNone)
760 rightPart := fmt.Sprintf("%-*s: %s%s%s", maxLabelLen, refName, refColor, formatDuration(refRun.Result.Duration), cNone)
761
762 fmt.Printf(" [%sPASS%s] %-*s [%s | %s]\n",
763 cGreen, cNone, maxTestNameLen, targetRun.Name,
764 leftPart, rightPart)
765 }
766 }
767 }
768
769 var summaryPadding string
770 if *verbose && maxTestNameLen > 0 {
771 // Aligns the summary block with the performance block in verbose mode.
772 // Prefix is " [PASS] " (8) + name (maxTestNameLen) + " " (1)
773 summaryPadding = strings.Repeat(" ", 8+maxTestNameLen+1)
774 } else {
775 // In non-verbose mode, just indent slightly.
776 summaryPadding = " "
777 }
778
779 refCompileColor, targetCompileColor := cNone, cNone
780 if result.Reference.Compile.Duration < result.Target.Compile.Duration {
781 refCompileColor = cMagenta
782 } else if result.Target.Compile.Duration < result.Reference.Compile.Duration {
783 targetCompileColor = cMagenta
784 }
785 leftCompilePart := fmt.Sprintf("%-*s: %s%s%s", maxLabelLen, targetName+"_comp", targetCompileColor, formatDuration(result.Target.Compile.Duration), cNone)
786 rightCompilePart := fmt.Sprintf("%-*s: %s%s%s", maxLabelLen, refName+"_comp", refCompileColor, formatDuration(result.Reference.Compile.Duration), cNone)
787 fmt.Printf("%s[%s | %s]\n", summaryPadding, leftCompilePart, rightCompilePart)
788
789 if !isBothFailed {
790 var totalTargetRun, totalRefRun time.Duration
791 for _, run := range result.Target.Runs {
792 totalTargetRun += run.Result.Duration
793 }
794 for _, run := range result.Reference.Runs {
795 totalRefRun += run.Result.Duration
796 }
797 refRunColor, targetRunColor := cNone, cNone
798 if totalRefRun < totalTargetRun {
799 refRunColor = cMagenta
800 } else if totalTargetRun < totalRefRun {
801 targetRunColor = cMagenta
802 }
803 leftRuntPart := fmt.Sprintf("%-*s: %s%s%s", maxLabelLen, targetName+"_runt", targetRunColor, formatDuration(totalTargetRun), cNone)
804 rightRuntPart := fmt.Sprintf("%-*s: %s%s%s", maxLabelLen, refName+"_runt", refRunColor, formatDuration(totalRefRun), cNone)
805 fmt.Printf("%s[%s | %s]\n", summaryPadding, leftRuntPart, rightRuntPart)
806 }
807
808 } else if result.Status != "PASS" && result.Target != nil && result.Reference != nil {
809 fmt.Printf(" %s compile: %s, %s compile: %s\n",
810 refName, result.Reference.Compile.Duration,
811 targetName, result.Target.Compile.Duration)
812 }
813 }
814
815 fmt.Println("----------------------------------------------------------------------")
816 fmt.Printf("%sTest Summary:%s %s%d Passed%s, %s%d Failed%s, %s%d Skipped%s, %s%d Errored%s, %d Total\n",
817 cBold, cNone, cGreen, passed, cNone, cRed, failed, cNone, cYellow, skipped, cNone, cRed, errored, cNone, len(results))
818
819 if comparedFileCount > 0 {
820 targetName, refName := filepath.Base(*targetCompiler), filepath.Base(*refCompiler)
821 avgTargetCompile := totalTargetCompile / time.Duration(comparedFileCount)
822 avgRefCompile := totalRefCompile / time.Duration(comparedFileCount)
823
824 fmt.Println("---")
825 if avgTargetCompile > avgRefCompile {
826 if avgRefCompile > 0 {
827 factor := float64(avgTargetCompile) / float64(avgRefCompile)
828 fmt.Printf("On average, %s%s%s was %s%.2fx%s slower to compile than %s.\n", cBold, targetName, cNone, cRed, factor, cNone, refName)
829 }
830 } else if avgRefCompile > avgTargetCompile {
831 if avgTargetCompile > 0 {
832 factor := float64(avgRefCompile) / float64(avgTargetCompile)
833 fmt.Printf("On average, %s%s%s was %s%.2fx%s faster to compile than %s.\n", cBold, targetName, cNone, cGreen, factor, cNone, refName)
834 }
835 }
836
837 if runtimeFileCount > 0 {
838 avgTargetRuntime := totalTargetRuntime / time.Duration(runtimeFileCount)
839 avgRefRuntime := totalRefRuntime / time.Duration(runtimeFileCount)
840 if avgTargetRuntime > avgRefRuntime {
841 if avgRefRuntime > 0 {
842 factor := float64(avgTargetRuntime) / float64(avgRefRuntime)
843 fmt.Printf("Binaries from %s%s%s ran %s%.2fx%s slower than those from %s.\n", cBold, targetName, cNone, cRed, factor, cNone, refName)
844 }
845 } else if avgRefRuntime > avgTargetRuntime {
846 if avgTargetRuntime > 0 {
847 factor := float64(avgRefRuntime) / float64(avgTargetRuntime)
848 fmt.Printf("Binaries from %s%s%s ran %s%.2fx%s faster than those from %s.\n", cBold, targetName, cNone, cGreen, factor, cNone, refName)
849 }
850 }
851 }
852 }
853}
854
855func formatDiff(diff string) string {
856 if diff == "" {
857 return ""
858 }
859 var builder strings.Builder
860 builder.WriteString(" --- Diff ---\n")
861 for _, line := range strings.Split(diff, "\n") {
862 lineWithIndent := " " + line
863 trimmedLine := strings.TrimSpace(line)
864 if strings.HasPrefix(trimmedLine, "-") {
865 builder.WriteString(cRed)
866 } else if strings.HasPrefix(trimmedLine, "+") {
867 builder.WriteString(cGreen)
868 }
869 builder.WriteString(lineWithIndent)
870 builder.WriteString(cNone)
871 builder.WriteString("\n")
872 }
873 return builder.String()
874}
875
876func writeJSONReport(results []*FileTestResult) TestSuiteResults {
877 resultsMap := make(TestSuiteResults, len(results))
878 for _, r := range results {
879 resultsMap[r.File] = r
880 }
881
882 jsonData, err := json.MarshalIndent(resultsMap, "", " ")
883 if err != nil {
884 log.Printf("%s[ERROR]%s Failed to marshal results to JSON: %v\n", cRed, cNone, err)
885 return resultsMap
886 }
887
888 outputFile := *outputJSON
889 if *jsonDir != "" {
890 if err := os.MkdirAll(*jsonDir, 0755); err != nil {
891 log.Printf("%s[ERROR]%s Failed to create dir %s: %v\n", cRed, cNone, *jsonDir, err)
892 }
893 outputFile = filepath.Join(*jsonDir, *outputJSON)
894 }
895
896 if err := os.WriteFile(outputFile, jsonData, 0644); err != nil {
897 log.Printf("%s[ERROR]%s Failed to write JSON report to %s: %v\n", cRed, cNone, outputFile, err)
898 } else {
899 fmt.Printf("Full test report saved to %s\n", outputFile)
900 }
901 return resultsMap
902}
903
904func hasFailures(results TestSuiteResults) bool {
905 for _, result := range results {
906 if result.Status == "FAIL" || result.Status == "ERROR" {
907 return true
908 }
909 }
910 return false
911}
912
913func expandGlobPatterns(patterns string) ([]string, error) {
914 var allFiles []string
915 seen := make(map[string]bool)
916 for _, pattern := range strings.Fields(patterns) {
917 files, err := filepath.Glob(pattern)
918 if err != nil {
919 return nil, fmt.Errorf("bad pattern %s: %w", pattern, err)
920 }
921 for _, file := range files {
922 absFile, err := filepath.Abs(file)
923 if err != nil {
924 continue // Skip files we can't resolve
925 }
926 if !seen[absFile] {
927 if info, err := os.Stat(absFile); err == nil && info.Mode().IsRegular() {
928 allFiles = append(allFiles, absFile)
929 seen[absFile] = true
930 }
931 }
932 }
933 }
934 return allFiles, nil
935}