Language-timings reorg (#2000)

* reorg

* naive

* optimizing

* .gitignore

* more

scripts/perf/multilanguage-timings/naive/.gitignore

@@ -0,0 +1,5 @@
+cutcpp
+cutgo
+cutnim
+cutrs
+cutzig

scripts/perf/multilanguage-timings/naive/Makefile

@@ -0,0 +1,24 @@
+# Cut benchmarks: same CLI for all (step 1-6, comma-separated fields, filenames).
+# Usage: ./cutgo 6 a,b,c [file ...]   # step 6 = full pipeline
+
+binaries: cutgo cutcpp cutnim cutzig cutrs
+
+cutgo: cutgo.go
+	go build -o cutgo cutgo.go
+
+cutcpp: cutcpp.cpp
+	$(CXX) $(CXXFLAGS) -O2 -o cutcpp cutcpp.cpp
+
+cutnim: cutnim.nim
+	nim c -d:release -o:cutnim cutnim.nim
+
+cutzig: cutzig.zig
+	zig build-exe -O ReleaseFast -femit-bin=cutzig cutzig.zig
+
+cutrs: cutrs.rs
+	rustc -C opt-level=3 -o cutrs cutrs.rs
+
+clean:
+	rm -f cutgo cutcpp cutnim cutzig cutrs cutd.o
+
+.PHONY: all clean

scripts/perf/multilanguage-timings/naive/cutcpp.cpp

@@ -0,0 +1,111 @@
+// CLI: step (1-6) arg1, comma-separated field names arg2, filenames arg3+.
+// Step: 1=read, 2=+parse, 3=+select, 4=+build, 5=+newline, 6=+write.
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <string>
+#include <unordered_map>
+#include <vector>
+#include <cstdlib>
+
+static void usage(const char* prog) {
+	std::cerr << "usage: " << prog << " <step 1-6> <field1,field2,...> [file ...]\n";
+	std::exit(1);
+}
+
+static std::vector<std::string> split(const std::string& s, char delim) {
+	std::vector<std::string> out;
+	std::istringstream iss(s);
+	std::string part;
+	while (std::getline(iss, part, delim)) {
+		out.push_back(part);
+	}
+	return out;
+}
+
+static bool splitKeyValue(const std::string& field, std::string& key, std::string& value) {
+	size_t pos = field.find('=');
+	if (pos == std::string::npos) return false;
+	key = field.substr(0, pos);
+	value = field.substr(pos + 1);
+	return true;
+}
+
+static bool handle(const std::string& fileName, int step,
+	const std::vector<std::string>& includeFields) {
+	std::istream* in = &std::cin;
+	std::ifstream fileStream;
+	if (fileName != "-") {
+		fileStream.open(fileName);
+		if (!fileStream) {
+			std::cerr << "open: " << fileName << "\n";
+			return false;
+		}
+		in = &fileStream;
+	}
+
+	std::string line;
+	while (std::getline(*in, line)) {
+		if (step <= 1) continue;
+
+		// Step 2: line to map
+		std::unordered_map<std::string, std::string> mymap;
+		for (const auto& field : split(line, ',')) {
+			std::string k, v;
+			if (splitKeyValue(field, k, v))
+				mymap[k] = v;
+		}
+		if (step <= 2) continue;
+
+		// Step 3: map-to-map transform
+		std::unordered_map<std::string, std::string> newmap;
+		for (const auto& includeField : includeFields) {
+			auto it = mymap.find(includeField);
+			if (it != mymap.end())
+				newmap[it->first] = it->second;
+		}
+		if (step <= 3) continue;
+
+		// Step 4-5: map to string + newline (emit in includeFields order)
+		std::ostringstream buf;
+		bool first = true;
+		for (const auto& k : includeFields) {
+			auto it = newmap.find(k);
+			if (it != newmap.end()) {
+				if (!first) buf << ',';
+				buf << it->first << '=' << it->second;
+				first = false;
+			}
+		}
+		buf << '\n';
+		if (step <= 5) continue;
+
+		// Step 6: write to stdout
+		std::cout << buf.str();
+	}
+
+	return true;
+}
+
+int main(int argc, char* argv[]) {
+	if (argc < 3) usage(argv[0]);
+
+	int step = std::atoi(argv[1]);
+	if (step < 1 || step > 6) {
+		std::cerr << "step must be 1-6, got " << argv[1] << "\n";
+		std::exit(1);
+	}
+
+	std::vector<std::string> includeFields = split(argv[2], ',');
+	std::vector<std::string> filenames;
+	for (int i = 3; i < argc; i++)
+		filenames.push_back(argv[i]);
+	if (filenames.empty())
+		filenames.push_back("-");
+
+	bool ok = true;
+	for (const auto& arg : filenames) {
+		if (!handle(arg, step, includeFields)) ok = false;
+	}
+	return ok ? 0 : 1;
+}

scripts/perf/multilanguage-timings/naive/cutgo.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"log"
+	"os"
+	"strconv"
+	"strings"
+)
+
+// CLI: step (1-6) arg1, comma-separated field names arg2, filenames arg3+.
+// Step controls how far the pipeline runs (for profiling): 1=read, 2=+parse, 3=+select, 4=+build, 5=+newline, 6=+write.
+func main() {
+	if len(os.Args) < 3 {
+		log.Fatalf("usage: %s <step 1-6> <field1,field2,...> [file ...]\n", os.Args[0])
+	}
+	step, err := strconv.Atoi(os.Args[1])
+	if err != nil || step < 1 || step > 6 {
+		log.Fatalf("step must be 1-6, got %q", os.Args[1])
+	}
+	includeFields := strings.Split(os.Args[2], ",")
+	filenames := os.Args[3:]
+	if len(filenames) == 0 {
+		filenames = []string{"-"}
+	}
+
+	ok := true
+	for _, arg := range filenames {
+		ok = handle(arg, step, includeFields) && ok
+	}
+	if ok {
+		os.Exit(0)
+	}
+	os.Exit(1)
+}
+
+func handle(fileName string, step int, includeFields []string) (ok bool) {
+	inputStream := os.Stdin
+	if fileName != "-" {
+		var err error
+		if inputStream, err = os.Open(fileName); err != nil {
+			log.Println(err)
+			return false
+		}
+		defer inputStream.Close()
+	}
+
+	reader := bufio.NewReader(inputStream)
+	eof := false
+
+	for !eof {
+		line, err := reader.ReadString('\n')
+		if err == io.EOF {
+			eof = true
+		} else if err != nil {
+			log.Println(err)
+			return false
+		}
+		if step <= 1 {
+			continue
+		}
+
+		// Step 2: line to map
+		mymap := make(map[string]string)
+		fields := strings.Split(line, ",")
+		for _, field := range fields {
+			kvps := strings.SplitN(field, "=", 2)
+			if len(kvps) >= 2 {
+				mymap[kvps[0]] = kvps[1]
+			}
+		}
+		if step <= 2 {
+			continue
+		}
+
+		// Step 3: map-to-map transform (keep include order for output)
+		newmap := make(map[string]string)
+		for _, includeField := range includeFields {
+			if value, present := mymap[includeField]; present {
+				newmap[includeField] = value
+			}
+		}
+		if step <= 3 {
+			continue
+		}
+
+		// Step 4–5: map to string + newline (iterate includeFields to preserve order)
+		var buffer bytes.Buffer
+		first := true
+		for _, includeField := range includeFields {
+			if value, present := newmap[includeField]; present {
+				if !first {
+					buffer.WriteString(",")
+				}
+				buffer.WriteString(includeField)
+				buffer.WriteString("=")
+				buffer.WriteString(value)
+				first = false
+			}
+		}
+		buffer.WriteString("\n")
+		if step <= 5 {
+			continue
+		}
+
+		// Step 6: write to stdout
+		if _, err := os.Stdout.WriteString(buffer.String()); err != nil {
+			log.Println(err)
+			return false
+		}
+	}
+
+	return true
+}
+

scripts/perf/multilanguage-timings/naive/cutnim.nim

@@ -0,0 +1,77 @@
+# CLI: step (1-6) arg1, comma-separated field names arg2, filenames arg3+.
+# Step: 1=read, 2=+parse, 3=+select, 4=+build, 5=+newline, 6=+write.
+import strutils, tables, os, streams
+
+proc handle(fileName: string, step: int, includeFields: seq[string]): bool =
+  var inputStream: Stream
+  if fileName == "-":
+    inputStream = newFileStream(stdin)
+  else:
+    try:
+      inputStream = newFileStream(fileName)
+    except OSError as e:
+      stderr.writeLine(e.msg)
+      return false
+
+  defer:
+    if fileName != "-":
+      inputStream.close()
+
+  var line: string
+  while inputStream.readLine(line):
+    if step <= 1:
+      continue
+
+    # Step 2: line to map
+    var oldmap = initOrderedTable[string, string]()
+    for word in line.split(','):
+      let pair = word.split('=', 2)
+      if pair.len >= 2:
+        oldmap[pair[0]] = pair[1]
+    if step <= 2:
+      continue
+
+    # Step 3: map-to-map transform
+    var newmap = initOrderedTable[string, string]()
+    for k in includeFields:
+      if k in oldmap:
+        newmap[k] = oldmap[k]
+    if step <= 3:
+      continue
+
+    # Step 4-5: map to string + newline
+    var parts: seq[string]
+    for k, v in newmap:
+      parts.add(k & "=" & v)
+    let outLine = parts.join(",") & "\n"
+    if step <= 5:
+      continue
+
+    # Step 6: write to stdout
+    stdout.write(outLine)
+
+  return true
+
+proc main =
+  if paramCount() < 2:
+    stderr.writeLine("usage: ", getAppFilename(), " <step 1-6> <field1,field2,...> [file ...]")
+    quit(1)
+  let stepStr = paramStr(1)
+  let step = parseInt(stepStr)
+  if step < 1 or step > 6:
+    stderr.writeLine("step must be 1-6, got ", stepStr)
+    quit(1)
+  let includeFields = paramStr(2).split(',')
+  var filenames: seq[string]
+  for i in 3..paramCount():
+    filenames.add(paramStr(i))
+  if filenames.len == 0:
+    filenames = @["-"]
+
+  var ok = true
+  for arg in filenames:
+    if not handle(arg, step, includeFields):
+      ok = false
+  quit(if ok: 0 else: 1)
+
+main()

scripts/perf/multilanguage-timings/naive/cutrs.rs

@@ -0,0 +1,121 @@
+// CLI: step (1-6) arg1, comma-separated field names arg2, filenames arg3+.
+// Step controls how far the pipeline runs (for profiling): 1=read, 2=+parse, 3=+select, 4=+build, 5=+newline, 6=+write.
+
+use std::collections::HashMap;
+use std::env;
+use std::fs::File;
+use std::io::{self, BufRead, BufReader, Read, Write};
+
+fn main() {
+    let args: Vec<String> = env::args().collect();
+    if args.len() < 3 {
+        eprintln!("usage: {} <step 1-6> <field1,field2,...> [file ...]", args[0]);
+        std::process::exit(1);
+    }
+    let step: u32 = match args[1].parse() {
+        Ok(n) if (1..=6).contains(&n) => n,
+        _ => {
+            eprintln!("step must be 1-6, got \"{}\"", args[1]);
+            std::process::exit(1);
+        }
+    };
+    let include_fields: Vec<&str> = args[2].split(',').collect();
+    let filenames: Vec<&str> = if args.len() > 3 {
+        args[3..].iter().map(String::as_str).collect()
+    } else {
+        vec!["-"]
+    };
+
+    let mut ok = true;
+    for name in filenames {
+        ok = handle(name, step, &include_fields) && ok;
+    }
+    std::process::exit(if ok { 0 } else { 1 });
+}
+
+fn handle(file_name: &str, step: u32, include_fields: &[&str]) -> bool {
+    let input: Box<dyn Read> = if file_name == "-" {
+        Box::new(io::stdin())
+    } else {
+        match File::open(file_name) {
+            Ok(f) => Box::new(f),
+            Err(e) => {
+                eprintln!("{}", e);
+                return false;
+            }
+        }
+    };
+
+    let mut reader = BufReader::new(input);
+    let mut line = String::new();
+
+    loop {
+        line.clear();
+        let n = match reader.read_line(&mut line) {
+            Ok(0) => break,
+            Ok(n) => n,
+            Err(e) => {
+                eprintln!("{}", e);
+                return false;
+            }
+        };
+        if n > 0 && !line.ends_with('\n') {
+            // EOF without newline; line still has content
+        }
+
+        if step <= 1 {
+            continue;
+        }
+
+        // Step 2: line to map
+        let mymap: HashMap<String, String> = line
+            .trim_end_matches('\n')
+            .split(',')
+            .filter_map(|field| {
+                let mut kvps = field.splitn(2, '=');
+                let k = kvps.next()?;
+                let v = kvps.next()?;
+                Some((k.to_string(), v.to_string()))
+            })
+            .collect();
+        if step <= 2 {
+            continue;
+        }
+
+        // Step 3: map-to-map transform (newmap for step 4 lookup; order from include_fields)
+        let newmap: HashMap<String, String> = include_fields
+            .iter()
+            .filter_map(|&k| mymap.get(k).map(|v| (k.to_string(), v.clone())))
+            .collect();
+        if step <= 3 {
+            continue;
+        }
+
+        // Step 4–5: map to string + newline (iterate include_fields to preserve order)
+        let mut first = true;
+        let mut out = String::new();
+        for &k in include_fields {
+            if let Some(v) = newmap.get(k) {
+                if !first {
+                    out.push(',');
+                }
+                out.push_str(k);
+                out.push('=');
+                out.push_str(v);
+                first = false;
+            }
+        }
+        out.push('\n');
+        if step <= 5 {
+            continue;
+        }
+
+        // Step 6: write to stdout
+        if let Err(e) = io::stdout().write_all(out.as_bytes()) {
+            eprintln!("{}", e);
+            return false;
+        }
+    }
+
+    true
+}

scripts/perf/multilanguage-timings/naive/cutzig.zig

@@ -0,0 +1,157 @@
+// CLI: step (1-6) arg1, comma-separated field names arg2, filenames arg3+.
+// Step: 1=read, 2=+parse, 3=+select, 4=+build, 5=+newline, 6=+write.
+const std = @import("std");
+
+fn handle(
+    allocator: std.mem.Allocator,
+    filename: []const u8,
+    step: u8,
+    include_fields: []const []const u8,
+) !bool {
+    var file = if (std.mem.eql(u8, filename, "-"))
+        std.fs.File.stdin()
+    else
+        std.fs.cwd().openFile(filename, .{}) catch |err| {
+            std.debug.print("open {s}: {}\n", .{ filename, err });
+            return false;
+        };
+    defer if (!std.mem.eql(u8, filename, "-")) file.close();
+
+    var buf: [256 * 1024]u8 = undefined;
+    var line_buf = try std.ArrayList(u8).initCapacity(allocator, 4096);
+    defer line_buf.deinit(allocator);
+
+    while (true) {
+        const n = file.read(&buf) catch |err| {
+            std.debug.print("read: {}\n", .{err});
+            return false;
+        };
+        if (n == 0) break;
+        var i: usize = 0;
+        while (i < n) {
+            if (buf[i] == '\n') {
+                i += 1;
+                const line = line_buf.items;
+                line_buf.clearRetainingCapacity();
+
+                if (step <= 1) continue;
+
+                // Step 2: line to map
+                var mymap = std.StringHashMap([]const u8).init(allocator);
+                defer {
+                    var it = mymap.iterator();
+                    while (it.next()) |e| {
+                        allocator.free(e.key_ptr.*);
+                        allocator.free(e.value_ptr.*);
+                    }
+                    mymap.deinit();
+                }
+                var iter = std.mem.splitScalar(u8, line, ',');
+                while (iter.next()) |field| {
+                    var kv_iter = std.mem.splitScalar(u8, field, '=');
+                    const k = kv_iter.next() orelse continue;
+                    const v = kv_iter.rest();
+                    const k_dup = allocator.dupe(u8, k) catch continue;
+                    const v_dup = allocator.dupe(u8, v) catch {
+                        allocator.free(k_dup);
+                        continue;
+                    };
+                    if (mymap.getPtr(k_dup)) |existing| {
+                        allocator.free(existing.*);
+                    }
+                    mymap.put(k_dup, v_dup) catch {
+                        allocator.free(k_dup);
+                        allocator.free(v_dup);
+                    };
+                }
+                if (step <= 2) continue;
+
+                // Step 3: map-to-map transform (output in include_fields order)
+                var newmap = std.StringHashMap([]const u8).init(allocator);
+                defer {
+                    var it = newmap.iterator();
+                    while (it.next()) |e| {
+                        allocator.free(e.value_ptr.*);
+                    }
+                    newmap.deinit();
+                }
+                for (include_fields) |inc_k| {
+                    if (mymap.get(inc_k)) |val| {
+                        const val_dup = allocator.dupe(u8, val) catch continue;
+                        newmap.put(inc_k, val_dup) catch allocator.free(val_dup);
+                    }
+                }
+                if (step <= 3) continue;
+
+                // Step 4-5: map to string + newline
+                var out = try std.ArrayList(u8).initCapacity(allocator, 256);
+                defer out.deinit(allocator);
+                var first = true;
+                for (include_fields) |k| {
+                    if (newmap.get(k)) |v| {
+                        if (!first) try out.appendSlice(allocator, ",");
+                        try out.appendSlice(allocator, k);
+                        try out.appendSlice(allocator, "=");
+                        try out.appendSlice(allocator, v);
+                        first = false;
+                    }
+                }
+                try out.append(allocator, '\n');
+                if (step <= 5) continue;
+
+                // Step 6: write to stdout
+                const stdout_file = std.fs.File.stdout();
+                stdout_file.writeAll(out.items) catch |err| {
+                    std.debug.print("write: {}\n", .{err});
+                    return false;
+                };
+            } else {
+                line_buf.append(allocator, buf[i]) catch return false;
+                i += 1;
+            }
+        }
+    }
+
+    return true;
+}
+
+pub fn main() !void {
+    // GPA is slow (debug allocator); page_allocator is ~10–20x faster for allocation-heavy workloads.
+    const allocator = std.heap.page_allocator;
+
+    var args = try std.process.argsAlloc(allocator);
+    defer std.process.argsFree(allocator, args);
+
+    if (args.len < 3) {
+        std.debug.print("usage: {s} <step 1-6> <field1,field2,...> [file ...]\n", .{args[0]});
+        std.process.exit(1);
+    }
+
+    const step_n = std.fmt.parseInt(u8, args[1], 10) catch {
+        std.debug.print("step must be 1-6, got {s}\n", .{args[1]});
+        std.process.exit(1);
+    };
+    if (step_n < 1 or step_n > 6) {
+        std.debug.print("step must be 1-6, got {s}\n", .{args[1]});
+        std.process.exit(1);
+    }
+
+    var include_fields = try std.ArrayList([]const u8).initCapacity(allocator, 8);
+    defer include_fields.deinit();
+    var field_iter = std.mem.splitScalar(u8, args[2], ',');
+    while (field_iter.next()) |f| {
+        if (f.len > 0) include_fields.append(allocator, f) catch {};
+    }
+
+    var ok = true;
+    if (args.len > 3) {
+        for (args[3..]) |arg| {
+            const result = handle(allocator, arg, step_n, include_fields.items) catch false;
+            if (!result) ok = false;
+        }
+    } else {
+        const result = handle(allocator, "-", step_n, include_fields.items) catch false;
+        if (!result) ok = false;
+    }
+    std.process.exit(if (ok) 0 else 1);
+}

scripts/perf/multilanguage-timings/naive/output.txt

@@ -0,0 +1,27 @@
+$ ./run-cut-benchmark.sh 6 color,shape ~/data/small.dkvp
+Cut benchmark: step=6 fields=color,shape files=/Users/kerl/data/small.dkvp (N=5)
+---
+./cutgo   .2720 s (mean of 5 runs)
+./cutcpp  .2792 s (mean of 5 runs)
+./cutnim  .2222 s (mean of 5 runs)
+./cutrs   .2203 s (mean of 5 runs)
+./cutzig  .3034 s (mean of 5 runs)
+
+$ ./run-cut-benchmark.sh 6 color,shape ~/data/medium.dkvp
+Cut benchmark: step=6 fields=color,shape files=/Users/kerl/data/medium.dkvp (N=5)
+---
+./cutgo   .2602 s (mean of 5 runs)
+./cutcpp  .2532 s (mean of 5 runs)
+./cutnim  .2365 s (mean of 5 runs)
+./cutrs   .2405 s (mean of 5 runs)
+./cutzig  .8213 s (mean of 5 runs)
+
+$ ./run-cut-benchmark.sh 6 color,shape ~/data/big.dkvp
+Cut benchmark: step=6 fields=color,shape files=/Users/kerl/data/big.dkvp (N=5)
+---
+./cutgo   2.2477 s (mean of 5 runs)
+./cutcpp  2.8137 s (mean of 5 runs)
+./cutnim  3.0912 s (mean of 5 runs)
+./cutrs   3.1144 s (mean of 5 runs)
+./cutzig  68.8362 s (mean of 5 runs)
+

scripts/perf/multilanguage-timings/naive/run-cut-benchmark.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# Run each cut implementation N times and print mean execution time.
+# Usage: ./run-cut-benchmark.sh [step] [fields] [file ...]
+#   step: 1-6 (default 6 = full pipeline)
+#   fields: comma-separated (default "a,x")
+#   file: input file(s); default stdin (you can pipe or pass a file)
+# Example: ./run-cut-benchmark.sh 6 a,x data.dkvp
+# Example: ./run-cut-benchmark.sh 6 ccode,milex,year,cinc ../c/nmc1.dkvp
+
+set -e
+N=5
+STEP="${1:-6}"
+FIELDS="${2:-a,x}"
+shift 2 2>/dev/null || true
+FILES=("$@")
+if [ ${#FILES[@]} -eq 0 ]; then
+  FILES=("-")
+fi
+
+run_one() {
+  local bin="$1"
+  if [ ! -x "$bin" ]; then
+    echo "$bin: binary not found (build with make)"
+    return
+  fi
+  local sum=0
+  local i
+  for i in $(seq 1 "$N"); do
+    local start end t
+    start=$(python3 -c 'import time; print(time.time())')
+    "$bin" "$STEP" "$FIELDS" "${FILES[@]}" >/dev/null
+    end=$(python3 -c 'import time; print(time.time())')
+    t=$(echo "$end - $start" | bc -l)
+    sum=$(echo "$sum + $t" | bc -l)
+  done
+  local mean
+  mean=$(echo "scale=4; $sum / $N" | bc -l)
+  printf "%-8s  %s s (mean of %d runs)\n" "$bin" "$mean" "$N"
+}
+
+echo "Cut benchmark: step=$STEP fields=$FIELDS files=${FILES[*]} (N=$N)"
+echo "---"
+run_one "./cutgo"
+run_one "./cutcpp"
+run_one "./cutnim"
+run_one "./cutrs"
+run_one "./cutzig"

scripts/perf/multilanguage-timings/optimized/.gitignore

@@ -0,0 +1,5 @@
+cutcpp
+cutgo
+cutnim
+cutrs
+cutzig

scripts/perf/multilanguage-timings/optimized/Makefile

@@ -0,0 +1,24 @@
+# Cut benchmarks: same CLI for all (step 1-6, comma-separated fields, filenames).
+# Usage: ./cutgo 6 a,b,c [file ...]   # step 6 = full pipeline
+
+binaries: cutgo cutcpp cutnim cutzig cutrs
+
+cutgo: cutgo.go
+	go build -o cutgo cutgo.go
+
+cutcpp: cutcpp.cpp
+	$(CXX) $(CXXFLAGS) -O2 -o cutcpp cutcpp.cpp
+
+cutnim: cutnim.nim
+	nim c -d:release --threads:on -o:cutnim cutnim.nim
+
+cutzig: cutzig.zig
+	zig build-exe -O ReleaseFast -femit-bin=cutzig cutzig.zig
+
+cutrs: cutrs.rs
+	rustc -C opt-level=3 -o cutrs cutrs.rs
+
+clean:
+	rm -f cutgo cutcpp cutnim cutzig cutrs cutd.o
+
+.PHONY: all clean

scripts/perf/multilanguage-timings/optimized/cutcpp.cpp

@@ -0,0 +1,199 @@
+// CLI: step (1-6) arg1, comma-separated field names arg2, filenames arg3+.
+// Step: 1=read, 2=+parse, 3=+select, 4=+build, 5=+newline, 6=+write.
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <string>
+#include <unordered_map>
+#include <vector>
+#include <queue>
+#include <thread>
+#include <mutex>
+#include <condition_variable>
+#include <atomic>
+#include <cstdlib>
+
+static const size_t PIPELINE_CAP = 64;
+
+template<typename T>
+struct BoundedQueue {
+	std::queue<T> q;
+	std::mutex m;
+	std::condition_variable not_full;
+	std::condition_variable not_empty;
+	size_t cap;
+	bool closed = false;
+
+	explicit BoundedQueue(size_t capacity) : cap(capacity) {}
+
+	bool push(T item) {
+		std::unique_lock<std::mutex> lock(m);
+		not_full.wait(lock, [this] { return q.size() < cap || closed; });
+		if (closed) return false;
+		q.push(std::move(item));
+		not_empty.notify_one();
+		return true;
+	}
+
+	bool pop(T& out) {
+		std::unique_lock<std::mutex> lock(m);
+		not_empty.wait(lock, [this] { return !q.empty() || closed; });
+		if (q.empty()) return false;
+		out = std::move(q.front());
+		q.pop();
+		not_full.notify_one();
+		return true;
+	}
+
+	void close() {
+		std::lock_guard<std::mutex> lock(m);
+		closed = true;
+		not_full.notify_all();
+		not_empty.notify_all();
+	}
+};
+
+static void usage(const char* prog) {
+	std::cerr << "usage: " << prog << " <step 1-6> <field1,field2,...> [file ...]\n";
+	std::exit(1);
+}
+
+static std::vector<std::string> split(const std::string& s, char delim) {
+	std::vector<std::string> out;
+	std::istringstream iss(s);
+	std::string part;
+	while (std::getline(iss, part, delim)) {
+		out.push_back(part);
+	}
+	return out;
+}
+
+static bool splitKeyValue(const std::string& field, std::string& key, std::string& value) {
+	size_t pos = field.find('=');
+	if (pos == std::string::npos) return false;
+	key = field.substr(0, pos);
+	value = field.substr(pos + 1);
+	return true;
+}
+
+static bool handle(const std::string& fileName, int step,
+	const std::vector<std::string>& includeFields) {
+	std::istream* in = &std::cin;
+	std::ifstream fileStream;
+	static char readBuf[256 * 1024];
+	if (fileName != "-") {
+		fileStream.open(fileName);
+		if (!fileStream) {
+			std::cerr << "open: " << fileName << "\n";
+			return false;
+		}
+		fileStream.rdbuf()->pubsetbuf(readBuf, sizeof readBuf);
+		in = &fileStream;
+	}
+
+	using Job = std::pair<size_t, std::string>;
+	BoundedQueue<Job> readQueue(PIPELINE_CAP);
+	BoundedQueue<Job> writeQueue(PIPELINE_CAP);
+	std::mutex err_mutex;
+	std::string err_msg;
+	std::atomic<bool> has_error{false};
+
+	std::thread reader([in, &readQueue]() {
+		std::string line;
+		line.reserve(4096);
+		size_t index = 0;
+		while (std::getline(*in, line)) {
+			if (!readQueue.push({index, line})) break;
+			index++;
+		}
+		readQueue.close();
+	});
+
+	std::thread processor([step, &includeFields, &readQueue, &writeQueue]() {
+		std::unordered_map<std::string, std::string> mymap;
+		std::unordered_map<std::string, std::string> newmap;
+		newmap.reserve(includeFields.size());
+		std::ostringstream buf;
+		Job job;
+		while (readQueue.pop(job)) {
+			if (step <= 1) continue;
+			mymap.clear();
+			for (const auto& field : split(job.second, ',')) {
+				std::string k, v;
+				if (splitKeyValue(field, k, v))
+					mymap[k] = v;
+			}
+			if (step <= 2) continue;
+			newmap.clear();
+			for (const auto& includeField : includeFields) {
+				auto it = mymap.find(includeField);
+				if (it != mymap.end())
+					newmap[it->first] = it->second;
+			}
+			if (step <= 3) continue;
+			buf.str("");
+			buf.clear();
+			bool first = true;
+			for (const auto& k : includeFields) {
+				auto it = newmap.find(k);
+				if (it != newmap.end()) {
+					if (!first) buf << ',';
+					buf << it->first << '=' << it->second;
+					first = false;
+				}
+			}
+			buf << '\n';
+			if (step <= 5) continue;
+			writeQueue.push({job.first, buf.str()});
+		}
+		writeQueue.close();
+	});
+
+	std::thread writer([&writeQueue, &err_mutex, &err_msg, &has_error]() {
+		Job job;
+		while (writeQueue.pop(job)) {
+			std::cout << job.second;
+			if (!std::cout) {
+				std::lock_guard<std::mutex> lock(err_mutex);
+				err_msg = "write error";
+				has_error.store(true);
+				return;
+			}
+		}
+	});
+
+	reader.join();
+	processor.join();
+	writer.join();
+
+	if (has_error.load()) {
+		std::lock_guard<std::mutex> lock(err_mutex);
+		if (!err_msg.empty())
+			std::cerr << err_msg << "\n";
+		return false;
+	}
+	return true;
+}
+
+int main(int argc, char* argv[]) {
+	if (argc < 3) usage(argv[0]);
+
+	int step = std::atoi(argv[1]);
+	if (step < 1 || step > 6) {
+		std::cerr << "step must be 1-6, got " << argv[1] << "\n";
+		std::exit(1);
+	}
+
+	std::vector<std::string> includeFields = split(argv[2], ',');
+	std::vector<std::string> filenames;
+	for (int i = 3; i < argc; i++)
+		filenames.push_back(argv[i]);
+	if (filenames.empty())
+		filenames.push_back("-");
+
+	bool ok = true;
+	for (const auto& arg : filenames) {
+		if (!handle(arg, step, includeFields)) ok = false;
+	}
+	return ok ? 0 : 1;
+}

scripts/perf/multilanguage-timings/optimized/cutgo.go

@@ -0,0 +1,177 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"log"
+	"os"
+	"strconv"
+	"strings"
+)
+
+const pipelineCap = 64
+
+type lineJob struct {
+	index int
+	line  string
+}
+
+type outJob struct {
+	index int
+	out   string
+}
+
+// CLI: step (1-6) arg1, comma-separated field names arg2, filenames arg3+.
+// Step controls how far the pipeline runs (for profiling): 1=read, 2=+parse, 3=+select, 4=+build, 5=+newline, 6=+write.
+func main() {
+	if len(os.Args) < 3 {
+		log.Fatalf("usage: %s <step 1-6> <field1,field2,...> [file ...]\n", os.Args[0])
+	}
+	step, err := strconv.Atoi(os.Args[1])
+	if err != nil || step < 1 || step > 6 {
+		log.Fatalf("step must be 1-6, got %q", os.Args[1])
+	}
+	includeFields := strings.Split(os.Args[2], ",")
+	filenames := os.Args[3:]
+	if len(filenames) == 0 {
+		filenames = []string{"-"}
+	}
+
+	ok := true
+	for _, arg := range filenames {
+		ok = handle(arg, step, includeFields) && ok
+	}
+	if ok {
+		os.Exit(0)
+	}
+	os.Exit(1)
+}
+
+func handle(fileName string, step int, includeFields []string) (ok bool) {
+	inputStream := os.Stdin
+	if fileName != "-" {
+		var err error
+		if inputStream, err = os.Open(fileName); err != nil {
+			log.Println(err)
+			return false
+		}
+		defer inputStream.Close()
+	}
+
+	readCh := make(chan lineJob, pipelineCap)
+	writeCh := make(chan outJob, pipelineCap)
+	done := make(chan struct{})
+	errCh := make(chan error, 1)
+
+	const readBufSize = 256 * 1024
+	reader := bufio.NewReaderSize(inputStream, readBufSize)
+
+	// Reader goroutine
+	go func() {
+		index := 0
+		for {
+			line, err := reader.ReadString('\n')
+			if err != nil && err != io.EOF {
+				select {
+				case errCh <- err:
+				default:
+				}
+				close(readCh)
+				return
+			}
+			if err == io.EOF && line == "" {
+				close(readCh)
+				return
+			}
+			readCh <- lineJob{index: index, line: line}
+			index++
+			if err == io.EOF {
+				close(readCh)
+				return
+			}
+		}
+	}()
+
+	// Processor goroutine
+	go func() {
+		mymap := make(map[string]string)
+		newmap := make(map[string]string, len(includeFields))
+		var buffer bytes.Buffer
+		for job := range readCh {
+			if step <= 1 {
+				continue
+			}
+			// Step 2: line to map
+			for k := range mymap {
+				delete(mymap, k)
+			}
+			fields := strings.Split(job.line, ",")
+			for _, field := range fields {
+				kvps := strings.SplitN(field, "=", 2)
+				if len(kvps) >= 2 {
+					mymap[kvps[0]] = kvps[1]
+				}
+			}
+			if step <= 2 {
+				continue
+			}
+			// Step 3: map-to-map transform
+			for k := range newmap {
+				delete(newmap, k)
+			}
+			for _, includeField := range includeFields {
+				if value, present := mymap[includeField]; present {
+					newmap[includeField] = value
+				}
+			}
+			if step <= 3 {
+				continue
+			}
+			// Step 4–5: map to string + newline
+			buffer.Reset()
+			first := true
+			for _, includeField := range includeFields {
+				if value, present := newmap[includeField]; present {
+					if !first {
+						buffer.WriteString(",")
+					}
+					buffer.WriteString(includeField)
+					buffer.WriteString("=")
+					buffer.WriteString(value)
+					first = false
+				}
+			}
+			buffer.WriteString("\n")
+			if step <= 5 {
+				continue
+			}
+			// Step 6: send to writer
+			writeCh <- outJob{index: job.index, out: buffer.String()}
+		}
+		close(writeCh)
+	}()
+
+	// Writer goroutine
+	go func() {
+		for job := range writeCh {
+			if _, err := os.Stdout.WriteString(job.out); err != nil {
+				select {
+				case errCh <- err:
+				default:
+				}
+				return
+			}
+		}
+		close(done)
+	}()
+
+	select {
+	case err := <-errCh:
+		log.Println(err)
+		return false
+	case <-done:
+		return true
+	}
+}
+

scripts/perf/multilanguage-timings/optimized/cutnim.nim

@@ -0,0 +1,126 @@
+# CLI: step (1-6) arg1, comma-separated field names arg2, filenames arg3+.
+# Step: 1=read, 2=+parse, 3=+select, 4=+build, 5=+newline, 6=+write.
+import strutils, tables, os, streams, std/typedthreads
+
+const pipelineCap = 64
+
+type
+  LineJob = tuple[index: int, line: string]
+  OutJob = tuple[index: int, data: string]
+  PipelineContext = ref object
+    readChan: Channel[LineJob]
+    writeChan: Channel[OutJob]
+    step: int
+    includeFields: seq[string]
+    inputStream: Stream
+
+proc readerProc(ctx: PipelineContext) {.thread.} =
+  var index = 0
+  var line: string
+  while ctx.inputStream.readLine(line):
+    ctx.readChan.send((index, line))
+    index += 1
+  ctx.readChan.send((-1, ""))
+
+proc processorProc(ctx: PipelineContext) {.thread.} =
+  var oldmap = initOrderedTable[string, string]()
+  var newmap = initOrderedTable[string, string]()
+  while true:
+    let job = ctx.readChan.recv()
+    if job.index < 0:
+      break
+    if ctx.step <= 1:
+      continue
+    oldmap.clear()
+    for word in job.line.split(','):
+      let pair = word.split('=', 2)
+      if pair.len >= 2:
+        oldmap[pair[0]] = pair[1]
+    if ctx.step <= 2:
+      continue
+    newmap.clear()
+    for k in ctx.includeFields:
+      if k in oldmap:
+        newmap[k] = oldmap[k]
+    if ctx.step <= 3:
+      continue
+    var outLine: string
+    for k in ctx.includeFields:
+      if k in newmap:
+        if outLine.len > 0:
+          outLine.add(',')
+        outLine.add(k)
+        outLine.add('=')
+        outLine.add(newmap[k])
+    outLine.add('\n')
+    if ctx.step <= 5:
+      continue
+    ctx.writeChan.send((job.index, outLine))
+  ctx.writeChan.send((-1, ""))
+
+proc writerProc(ctx: PipelineContext) {.thread.} =
+  while true:
+    let job = ctx.writeChan.recv()
+    if job.index < 0:
+      break
+    stdout.write(job.data)
+
+proc handle(fileName: string, step: int, includeFields: seq[string]): bool =
+  var inputStream: Stream
+  if fileName == "-":
+    inputStream = newFileStream(stdin)
+  else:
+    try:
+      inputStream = newFileStream(fileName)
+    except OSError as e:
+      stderr.writeLine(e.msg)
+      return false
+
+  defer:
+    if fileName != "-":
+      inputStream.close()
+
+  var ctx: PipelineContext
+  new ctx
+  ctx.step = step
+  ctx.includeFields = includeFields
+  ctx.inputStream = inputStream
+  ctx.readChan.open(pipelineCap)
+  ctx.writeChan.open(pipelineCap)
+
+  var readerThread: Thread[PipelineContext]
+  var processorThread: Thread[PipelineContext]
+  var writerThread: Thread[PipelineContext]
+  createThread(readerThread, readerProc, ctx)
+  createThread(processorThread, processorProc, ctx)
+  createThread(writerThread, writerProc, ctx)
+  joinThreads(readerThread, processorThread, writerThread)
+
+  ctx.readChan.close()
+  ctx.writeChan.close()
+
+  return true
+
+proc main =
+  if paramCount() < 2:
+    stderr.writeLine("usage: ", getAppFilename(), " <step 1-6> <field1,field2,...> [file ...]")
+    quit(1)
+  let stepStr = paramStr(1)
+  let step = parseInt(stepStr)
+  if step < 1 or step > 6:
+    stderr.writeLine("step must be 1-6, got ", stepStr)
+    quit(1)
+  let includeFields = paramStr(2).split(',')
+  var filenames: seq[string]
+  for i in 3..paramCount():
+    filenames.add(paramStr(i))
+  if filenames.len == 0:
+    filenames = @["-"]
+
+  var ok = true
+  for arg in filenames:
+    if not handle(arg, step, includeFields):
+      ok = false
+  quit(if ok: 0 else: 1)
+
+main()

scripts/perf/multilanguage-timings/optimized/cutrs.rs

@@ -0,0 +1,156 @@
+// CLI: step (1-6) arg1, comma-separated field names arg2, filenames arg3+.
+// Step controls how far the pipeline runs (for profiling): 1=read, 2=+parse, 3=+select, 4=+build, 5=+newline, 6=+write.
+
+use std::collections::HashMap;
+use std::env;
+use std::fs::File;
+use std::io::{self, BufRead, BufReader, Read, Write};
+use std::sync::mpsc;
+use std::sync::Arc;
+use std::thread;
+
+const PIPELINE_CAP: usize = 64;
+
+fn main() {
+    let args: Vec<String> = env::args().collect();
+    if args.len() < 3 {
+        eprintln!("usage: {} <step 1-6> <field1,field2,...> [file ...]", args[0]);
+        std::process::exit(1);
+    }
+    let step: u32 = match args[1].parse() {
+        Ok(n) if (1..=6).contains(&n) => n,
+        _ => {
+            eprintln!("step must be 1-6, got \"{}\"", args[1]);
+            std::process::exit(1);
+        }
+    };
+    let include_fields: Vec<&str> = args[2].split(',').collect();
+    let filenames: Vec<&str> = if args.len() > 3 {
+        args[3..].iter().map(String::as_str).collect()
+    } else {
+        vec!["-"]
+    };
+
+    let mut ok = true;
+    for name in filenames {
+        ok = handle(name, step, &include_fields) && ok;
+    }
+    std::process::exit(if ok { 0 } else { 1 });
+}
+
+fn handle(file_name: &str, step: u32, include_fields: &[&str]) -> bool {
+    let input: Box<dyn Read + Send> = if file_name == "-" {
+        Box::new(io::stdin())
+    } else {
+        match File::open(file_name) {
+            Ok(f) => Box::new(f),
+            Err(e) => {
+                eprintln!("{}", e);
+                return false;
+            }
+        }
+    };
+
+    let (read_tx, read_rx) = mpsc::sync_channel::<(usize, String)>(PIPELINE_CAP);
+    let (write_tx, write_rx) = mpsc::sync_channel::<(usize, String)>(PIPELINE_CAP);
+    let (err_tx, err_rx) = mpsc::sync_channel::<std::io::Error>(1);
+    let err_tx = Arc::new(err_tx);
+
+    const READ_BUF_SIZE: usize = 256 * 1024;
+    let reader = BufReader::with_capacity(READ_BUF_SIZE, input);
+
+    let err_tx_reader = Arc::clone(&err_tx);
+    let h_reader = thread::spawn(move || {
+        let mut reader = reader;
+        let mut line = String::new();
+        let mut index = 0;
+        loop {
+            line.clear();
+            match reader.read_line(&mut line) {
+                Ok(0) => break,
+                Ok(_) => {}
+                Err(e) => {
+                    let _ = err_tx_reader.send(e);
+                    break;
+                }
+            }
+            if read_tx.send((index, line.clone())).is_err() {
+                break;
+            }
+            index += 1;
+        }
+        drop(read_tx);
+    });
+
+    let inc: Vec<String> = include_fields.iter().map(|s| s.to_string()).collect();
+    let h_processor = thread::spawn(move || {
+        let mut mymap: HashMap<String, String> = HashMap::new();
+        let mut newmap: HashMap<String, String> = HashMap::with_capacity(inc.len());
+        let mut out = String::new();
+        while let Ok((idx, line)) = read_rx.recv() {
+            if step <= 1 {
+                continue;
+            }
+            mymap.clear();
+            for field in line.trim_end_matches('\n').split(',') {
+                let mut kvps = field.splitn(2, '=');
+                if let (Some(k), Some(v)) = (kvps.next(), kvps.next()) {
+                    mymap.insert(k.to_string(), v.to_string());
+                }
+            }
+            if step <= 2 {
+                continue;
+            }
+            newmap.clear();
+            for k in &inc {
+                if let Some(v) = mymap.get(k) {
+                    newmap.insert(k.clone(), v.clone());
+                }
+            }
+            if step <= 3 {
+                continue;
+            }
+            out.clear();
+            let mut first = true;
+            for k in &inc {
+                if let Some(v) = newmap.get(k) {
+                    if !first {
+                        out.push(',');
+                    }
+                    out.push_str(k);
+                    out.push('=');
+                    out.push_str(v);
+                    first = false;
+                }
+            }
+            out.push('\n');
+            if step <= 5 {
+                continue;
+            }
+            if write_tx.send((idx, out.clone())).is_err() {
+                break;
+            }
+        }
+        drop(write_tx);
+    });
+
+    let err_tx_writer = Arc::clone(&err_tx);
+    let h_writer = thread::spawn(move || {
+        for (_, out) in write_rx {
+            if let Err(e) = io::stdout().write_all(out.as_bytes()) {
+                let _ = err_tx_writer.send(e);
+                return;
+            }
+        }
+    });
+
+    let _ = h_reader.join();
+    let _ = h_processor.join();
+    let _ = h_writer.join();
+
+    if let Ok(e) = err_rx.try_recv() {
+        eprintln!("{}", e);
+        return false;
+    }
+    true
+}

scripts/perf/multilanguage-timings/optimized/cutzig.zig

@@ -0,0 +1,260 @@
+// CLI: step (1-6) arg1, comma-separated field names arg2, filenames arg3+.
+// Step: 1=read, 2=+parse, 3=+select, 4=+build, 5=+newline, 6=+write.
+const std = @import("std");
+
+const pipeline_cap = 64;
+
+const LineJob = struct { index: usize, line: []const u8 };
+const OutJob = struct { index: usize, out: []const u8 };
+
+fn BoundedQueue(comptime T: type) type {
+    return struct {
+        list: std.ArrayList(T),
+        allocator: std.mem.Allocator,
+        cap: usize,
+        mutex: std.Thread.Mutex = .{},
+        condition: std.Thread.Condition = .{},
+        closed: bool = false,
+
+        const Self = @This();
+        fn init(allocator: std.mem.Allocator, capacity: usize) !Self {
+            return .{
+                .list = try std.ArrayList(T).initCapacity(allocator, capacity),
+                .allocator = allocator,
+                .cap = capacity,
+            };
+        }
+        fn deinit(self: *Self) void {
+            self.list.deinit(self.allocator);
+        }
+        fn push(self: *Self, item: T) bool {
+            self.mutex.lock();
+            defer self.mutex.unlock();
+            while (self.list.items.len >= self.cap and !self.closed) {
+                self.condition.wait(&self.mutex);
+            }
+            if (self.closed) return false;
+            self.list.append(self.allocator, item) catch return false;
+            self.condition.signal();
+            return true;
+        }
+        fn pop(self: *Self) ?T {
+            self.mutex.lock();
+            defer self.mutex.unlock();
+            while (self.list.items.len == 0 and !self.closed) {
+                self.condition.wait(&self.mutex);
+            }
+            if (self.list.items.len == 0) return null;
+            const item = self.list.orderedRemove(0);
+            self.condition.signal();
+            return item;
+        }
+        fn close(self: *Self) void {
+            self.mutex.lock();
+            defer self.mutex.unlock();
+            self.closed = true;
+            self.condition.broadcast();
+        }
+    };
+}
+
+fn readerRun(allocator: std.mem.Allocator, file: std.fs.File, read_queue: *BoundedQueue(LineJob)) void {
+    var buf: [256 * 1024]u8 = undefined;
+    var line_buf = std.ArrayList(u8).initCapacity(allocator, 4096) catch return;
+    defer line_buf.deinit(allocator);
+    var index: usize = 0;
+    while (true) {
+        const n = file.read(&buf) catch break;
+        if (n == 0) break;
+        var i: usize = 0;
+        while (i < n) {
+            if (buf[i] == '\n') {
+                i += 1;
+                const line = allocator.dupe(u8, line_buf.items) catch break;
+                _ = read_queue.push(.{ .index = index, .line = line });
+                index += 1;
+                line_buf.clearRetainingCapacity();
+            } else {
+                line_buf.append(allocator, buf[i]) catch break;
+                i += 1;
+            }
+        }
+    }
+    read_queue.close();
+}
+
+fn processorRun(
+    allocator: std.mem.Allocator,
+    step: u8,
+    include_fields: []const []const u8,
+    read_queue: *BoundedQueue(LineJob),
+    write_queue: *BoundedQueue(OutJob),
+) void {
+    var mymap = std.StringHashMap([]const u8).init(allocator);
+    defer {
+        var it = mymap.iterator();
+        while (it.next()) |e| {
+            allocator.free(e.key_ptr.*);
+            allocator.free(e.value_ptr.*);
+        }
+        mymap.deinit();
+    }
+    var newmap = std.StringHashMap([]const u8).init(allocator);
+    defer {
+        var it = newmap.iterator();
+        while (it.next()) |e| {
+            allocator.free(e.value_ptr.*);
+        }
+        newmap.deinit();
+    }
+    var out = std.ArrayList(u8).initCapacity(allocator, 256) catch return;
+    defer out.deinit(allocator);
+
+    while (read_queue.pop()) |job| {
+        defer allocator.free(job.line);
+        if (step <= 1) continue;
+        {
+            var it = mymap.iterator();
+            while (it.next()) |e| {
+                allocator.free(e.key_ptr.*);
+                allocator.free(e.value_ptr.*);
+            }
+            mymap.clearRetainingCapacity();
+        }
+        var iter = std.mem.splitScalar(u8, job.line, ',');
+        while (iter.next()) |field| {
+            var kv_iter = std.mem.splitScalar(u8, field, '=');
+            const k = kv_iter.next() orelse continue;
+            const v = kv_iter.rest();
+            const k_dup = allocator.dupe(u8, k) catch continue;
+            const v_dup = allocator.dupe(u8, v) catch {
+                allocator.free(k_dup);
+                continue;
+            };
+            if (mymap.getPtr(k_dup)) |existing| {
+                allocator.free(existing.*);
+            }
+            mymap.put(k_dup, v_dup) catch {
+                allocator.free(k_dup);
+                allocator.free(v_dup);
+            };
+        }
+        if (step <= 2) continue;
+        {
+            var it = newmap.iterator();
+            while (it.next()) |e| {
+                allocator.free(e.value_ptr.*);
+            }
+            newmap.clearRetainingCapacity();
+        }
+        for (include_fields) |inc_k| {
+            if (mymap.get(inc_k)) |val| {
+                const val_dup = allocator.dupe(u8, val) catch continue;
+                newmap.put(inc_k, val_dup) catch allocator.free(val_dup);
+            }
+        }
+        if (step <= 3) continue;
+        out.clearRetainingCapacity();
+        var first = true;
+        for (include_fields) |k| {
+            if (newmap.get(k)) |v| {
+                if (!first) out.appendSlice(allocator, ",") catch {};
+                out.appendSlice(allocator, k) catch {};
+                out.appendSlice(allocator, "=") catch {};
+                out.appendSlice(allocator, v) catch {};
+                first = false;
+            }
+        }
+        out.append(allocator, '\n') catch {};
+        if (step <= 5) continue;
+        const out_slice = allocator.dupe(u8, out.items) catch continue;
+        _ = write_queue.push(.{ .index = job.index, .out = out_slice });
+    }
+    write_queue.close();
+}
+
+fn writerRun(allocator: std.mem.Allocator, write_queue: *BoundedQueue(OutJob)) void {
+    const stdout_file = std.fs.File.stdout();
+    while (write_queue.pop()) |job| {
+        defer allocator.free(job.out);
+        stdout_file.writeAll(job.out) catch return;
+    }
+}
+
+fn handle(
+    allocator: std.mem.Allocator,
+    filename: []const u8,
+    step: u8,
+    include_fields: []const []const u8,
+) !bool {
+    var file = if (std.mem.eql(u8, filename, "-"))
+        std.fs.File.stdin()
+    else
+        std.fs.cwd().openFile(filename, .{}) catch |err| {
+            std.debug.print("open {s}: {}\n", .{ filename, err });
+            return false;
+        };
+    defer if (!std.mem.eql(u8, filename, "-")) file.close();
+
+    var read_queue = try BoundedQueue(LineJob).init(allocator, pipeline_cap);
+    defer read_queue.deinit();
+    var write_queue = try BoundedQueue(OutJob).init(allocator, pipeline_cap);
+    defer write_queue.deinit();
+
+    const h_reader = try std.Thread.spawn(.{}, readerRun, .{ allocator, file, &read_queue });
+    const h_processor = try std.Thread.spawn(.{}, processorRun, .{
+        allocator,
+        step,
+        include_fields,
+        &read_queue,
+        &write_queue,
+    });
+    const h_writer = try std.Thread.spawn(.{}, writerRun, .{ allocator, &write_queue });
+
+    h_reader.join();
+    h_processor.join();
+    h_writer.join();
+
+    return true;
+}
+
+pub fn main() !void {
+    // GPA is slow (debug allocator); page_allocator is ~10–20x faster for allocation-heavy workloads.
+    const allocator = std.heap.page_allocator;
+
+    var args = try std.process.argsAlloc(allocator);
+    defer std.process.argsFree(allocator, args);
+
+    if (args.len < 3) {
+        std.debug.print("usage: {s} <step 1-6> <field1,field2,...> [file ...]\n", .{args[0]});
+        std.process.exit(1);
+    }
+
+    const step_n = std.fmt.parseInt(u8, args[1], 10) catch {
+        std.debug.print("step must be 1-6, got {s}\n", .{args[1]});
+        std.process.exit(1);
+    };
+    if (step_n < 1 or step_n > 6) {
+        std.debug.print("step must be 1-6, got {s}\n", .{args[1]});
+        std.process.exit(1);
+    }
+
+    var include_fields = try std.ArrayList([]const u8).initCapacity(allocator, 8);
+    defer include_fields.deinit();
+    var field_iter = std.mem.splitScalar(u8, args[2], ',');
+    while (field_iter.next()) |f| {
+        if (f.len > 0) include_fields.append(allocator, f) catch {};
+    }
+
+    var ok = true;
+    if (args.len > 3) {
+        for (args[3..]) |arg| {
+            const result = handle(allocator, arg, step_n, include_fields.items) catch false;
+            if (!result) ok = false;
+        }
+    } else {
+        const result = handle(allocator, "-", step_n, include_fields.items) catch false;
+        if (!result) ok = false;
+    }
+    std.process.exit(if (ok) 0 else 1);
+}

scripts/perf/multilanguage-timings/optimized/output.txt

@@ -0,0 +1,26 @@
+$ ./run-cut-benchmark.sh 6 color,shape ~/data/small.dkvp
+Cut benchmark: step=6 fields=color,shape files=/Users/kerl/data/small.dkvp (N=5)
+---
+./cutgo   .1663 s (mean of 5 runs)
+./cutcpp  .1657 s (mean of 5 runs)
+./cutnim  .1682 s (mean of 5 runs)
+./cutrs   .1657 s (mean of 5 runs)
+./cutzig  .2239 s (mean of 5 runs)
+
+$ ./run-cut-benchmark.sh 6 color,shape ~/data/medium.dkvp
+Cut benchmark: step=6 fields=color,shape files=/Users/kerl/data/medium.dkvp (N=5)
+---
+./cutgo   .1782 s (mean of 5 runs)
+./cutcpp  .2021 s (mean of 5 runs)
+./cutnim  .2116 s (mean of 5 runs)
+./cutrs   .1824 s (mean of 5 runs)
+./cutzig  .7551 s (mean of 5 runs)
+
+$ ./run-cut-benchmark.sh 6 color,shape ~/data/big.dkvp
+Cut benchmark: step=6 fields=color,shape files=/Users/kerl/data/big.dkvp (N=5)
+---
+./cutgo   1.7837 s (mean of 5 runs)
+./cutcpp  4.0104 s (mean of 5 runs)
+./cutnim  4.7630 s (mean of 5 runs)
+./cutrs   1.9678 s (mean of 5 runs)
+./cutzig  64.1099 s (mean of 5 runs)

scripts/perf/multilanguage-timings/optimized/run-cut-benchmark.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# Run each cut implementation N times and print mean execution time.
+# Usage: ./run-cut-benchmark.sh [step] [fields] [file ...]
+#   step: 1-6 (default 6 = full pipeline)
+#   fields: comma-separated (default "a,x")
+#   file: input file(s); default stdin (you can pipe or pass a file)
+# Example: ./run-cut-benchmark.sh 6 a,x data.dkvp
+# Example: ./run-cut-benchmark.sh 6 ccode,milex,year,cinc ../c/nmc1.dkvp
+
+set -e
+N=5
+STEP="${1:-6}"
+FIELDS="${2:-a,x}"
+shift 2 2>/dev/null || true
+FILES=("$@")
+if [ ${#FILES[@]} -eq 0 ]; then
+  FILES=("-")
+fi
+
+run_one() {
+  local bin="$1"
+  if [ ! -x "$bin" ]; then
+    echo "$bin: binary not found (build with make)"
+    return
+  fi
+  local sum=0
+  local i
+  for i in $(seq 1 "$N"); do
+    local start end t
+    start=$(python3 -c 'import time; print(time.time())')
+    "$bin" "$STEP" "$FIELDS" "${FILES[@]}" >/dev/null
+    end=$(python3 -c 'import time; print(time.time())')
+    t=$(echo "$end - $start" | bc -l)
+    sum=$(echo "$sum + $t" | bc -l)
+  done
+  local mean
+  mean=$(echo "scale=4; $sum / $N" | bc -l)
+  printf "%-8s  %s s (mean of %d runs)\n" "$bin" "$mean" "$N"
+}
+
+echo "Cut benchmark: step=$STEP fields=$FIELDS files=${FILES[*]} (N=$N)"
+echo "---"
+run_one "./cutgo"
+run_one "./cutcpp"
+run_one "./cutnim"
+run_one "./cutrs"
+run_one "./cutzig"