qem 0.6.2

High-performance cross-platform text engine for massive files.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122

param(
    [string]$InputJsonl = "target\perf-matrix.jsonl",
    [string]$OutputMarkdown = "target\perf-matrix-summary.md"
)

$ErrorActionPreference = "Stop"
$InvariantCulture = [System.Globalization.CultureInfo]::InvariantCulture

function Format-InvariantNumber {
    param(
        [double]$Value
    )

    return $Value.ToString("0.000", $InvariantCulture)
}

function Get-StatsRow {
    param(
        [object[]]$Values
    )

    $filtered = @($Values | Where-Object { $_ -ne $null })
    if ($filtered.Count -eq 0) {
        return @{
            median = "-"
            min = "-"
            max = "-"
        }
    }

    $sorted = @($filtered | Sort-Object)
    $count = $sorted.Count
    if ($count % 2 -eq 1) {
        $median = [double]$sorted[[int]($count / 2)]
    }
    else {
        $median = ([double]$sorted[($count / 2) - 1] + [double]$sorted[$count / 2]) / 2.0
    }

    return @{
        median = (Format-InvariantNumber ([double]$median))
        min = (Format-InvariantNumber ([double]$sorted[0]))
        max = (Format-InvariantNumber ([double]$sorted[$count - 1]))
    }
}

$repoRoot = Split-Path -Parent $PSScriptRoot
$inputPath = Join-Path $repoRoot $InputJsonl
$outputPath = Join-Path $repoRoot $OutputMarkdown

if (-not (Test-Path $inputPath)) {
    throw "Input JSONL not found: $inputPath"
}

$rows = Get-Content $inputPath |
    Where-Object { -not [string]::IsNullOrWhiteSpace($_) } |
    ForEach-Object { $_ | ConvertFrom-Json }

if ($rows.Count -eq 0) {
    throw "Input JSONL is empty: $inputPath"
}

$groups = $rows | Group-Object {
    "{0}|{1}|{2}|{3}|{4}" -f $_.matrix_input_label, $_.matrix_state, $_.backing, $_.matrix_label, $_.matrix_viewport_anchor
}

$lines = New-Object System.Collections.Generic.List[string]
$lines.Add("# Perf Matrix Summary")
$lines.Add("")
$lines.Add(('Source: `{0}`' -f $InputJsonl))
$lines.Add("")
$lines.Add('| input | size GiB | label | anchor | state | backing | runs | open ms | viewport ms | next ms | prev ms | find_all ms |')
$lines.Add('| --- | ---: | --- | --- | --- | --- | ---: | --- | --- | --- | --- | --- |')

foreach ($group in ($groups | Sort-Object Name)) {
    $parts = $group.Name.Split('|', 5)
    $inputLabel = $parts[0]
    $state = $parts[1]
    $backing = $parts[2]
    $matrixLabel = $parts[3]
    $viewportAnchor = $parts[4]
    if ([string]::IsNullOrWhiteSpace($matrixLabel)) {
        $matrixLabel = "-"
    }
    if ([string]::IsNullOrWhiteSpace($viewportAnchor)) {
        $viewportAnchor = "middle"
    }

    $firstRow = $group.Group | Select-Object -First 1
    $sizeGiB = Format-InvariantNumber (([double]$firstRow.file_len_bytes) / 1GB)

    $openStats = Get-StatsRow ($group.Group | ForEach-Object { $_.open_ms })
    $viewportStats = Get-StatsRow ($group.Group | ForEach-Object { $_.viewport_ms })
    $nextStats = Get-StatsRow ($group.Group | ForEach-Object { $_.next_ms })
    $prevStats = Get-StatsRow ($group.Group | ForEach-Object { $_.prev_ms })
    $findAllStats = Get-StatsRow ($group.Group | ForEach-Object { $_.find_all_ms })

    $lines.Add(
        ('| {0} | {1} | {2} | {3} | {4} | {5} | {6} | {7} [{8}-{9}] | {10} [{11}-{12}] | {13} [{14}-{15}] | {16} [{17}-{18}] | {19} [{20}-{21}] |' -f
            $inputLabel,
            $sizeGiB,
            $matrixLabel,
            $viewportAnchor,
            $state,
            $backing,
            $group.Count,
            $openStats.median, $openStats.min, $openStats.max,
            $viewportStats.median, $viewportStats.min, $viewportStats.max,
            $nextStats.median, $nextStats.min, $nextStats.max,
            $prevStats.median, $prevStats.min, $prevStats.max,
            $findAllStats.median, $findAllStats.min, $findAllStats.max
        )
    )
}

$outputDir = Split-Path -Parent $outputPath
if (-not (Test-Path $outputDir)) {
    New-Item -ItemType Directory -Path $outputDir | Out-Null
}

$lines | Set-Content -Path $outputPath
Write-Host ("Wrote markdown summary to {0}" -f $outputPath)