sqc 0.4.84

Software Code Quality - CERT C compliance checker
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129

#!/usr/bin/env python3
"""Generate the Juliet benchmark-runtime figure data for the paper.

Pulls per-version scan timing from data/benchmarks.db and emits:
  - data/benchmark_timing.csv      (full series, one row per version)
  - paper/fig_timing.tex           (pgfplots block, \\input-able)

Primary metric is **seconds per CWE** (analysis_s / cwe_count): the sum of
per-CWE sqc subprocess wall time, normalized by corpus size. This is
*parallelism-independent* (immune to the --jobs setting) and *corpus-size
independent* (the Juliet CWE set grew 68->74 over the window), so it is the
only metric comparable across the full version history.

Window: the first stable-methodology version (0.3.25) onward. Versions
0.3.20-0.3.24 are real but reflect an early, ~4-6x slower pre-optimization
era whose outliers crush the y-axis; everything before 0.3.20 is backfill
with synthetic timestamps and no per-CWE durations.

Realworld timing is deliberately excluded: realworld_results.duration_s is
currently a run-level total duplicated across every project row (not
per-project), LOC is mostly unpopulated, and cppcheck/clang-tidy are not
timed -- so it cannot support a version-history or throughput graph yet.
"""
import csv
import sqlite3
from pathlib import Path

REPO = Path(__file__).resolve().parent.parent
DB = REPO / "data" / "benchmarks.db"
CSV_OUT = REPO / "data" / "benchmark_timing.csv"
TEX_OUT = REPO / "paper" / "fig_timing.tex"
START_DATE = "2026-03-20"  # >= first 0.3.25 run; excludes 0.3.20-0.3.24 outliers
N_TICKS = 11               # number of sampled x-axis version labels

QUERY = """
WITH real AS (
  SELECT r.run_id, r.sqc_version, r.started_at,
         (SELECT COUNT(*)        FROM cwe_scans cs WHERE cs.run_id=r.run_id) AS cwes,
         (SELECT SUM(cs.duration_s) FROM cwe_scans cs WHERE cs.run_id=r.run_id) AS analysis_s,
         ROW_NUMBER() OVER (PARTITION BY r.sqc_version ORDER BY r.started_at DESC) AS rn
  FROM runs r
  WHERE r.status='completed' AND r.started_at >= ?
)
SELECT sqc_version, cwes, ROUND(analysis_s,1), substr(started_at,1,10)
FROM real WHERE rn=1 AND analysis_s IS NOT NULL
ORDER BY started_at;
"""


def main() -> None:
    con = sqlite3.connect(DB)
    rows = con.execute(QUERY, (START_DATE,)).fetchall()
    con.close()
    if not rows:
        raise SystemExit("no timing rows found")

    series = [
        {
            "i": i + 1,
            "version": v,
            "cwes": cwes,
            "analysis_s": analysis_s,
            "s_per_cwe": round(analysis_s / cwes, 1),
            "date": date,
        }
        for i, (v, cwes, analysis_s, date) in enumerate(rows)
    ]

    with CSV_OUT.open("w", newline="") as f:
        w = csv.DictWriter(f, fieldnames=["i", "version", "cwes", "analysis_s", "s_per_cwe", "date"])
        w.writeheader()
        w.writerows(series)

    # Sample evenly spaced version labels (always include first and last).
    n = len(series)
    step = max(1, (n - 1) // (N_TICKS - 1))
    tick_idx = list(range(0, n, step))
    if tick_idx[-1] != n - 1:
        tick_idx.append(n - 1)
    xtick = ", ".join(str(series[k]["i"]) for k in tick_idx)
    xlabels = ", ".join("v" + series[k]["version"] for k in tick_idx)
    coords = " ".join(f"({s['i']}, {s['s_per_cwe']})" for s in series)

    ys = [s["s_per_cwe"] for s in series]
    ymax = (int(max(ys)) // 25 + 1) * 25

    tex = f"""% Auto-generated by scripts/benchmark_timing_figure.py -- do not edit by hand.
% Juliet scan time per CWE (parallelism- and corpus-size-independent), v{series[0]['version']}-v{series[-1]['version']}.
\\begin{{figure}}[t]
\\centering
\\begin{{tikzpicture}}
\\begin{{axis}}[
    width=\\columnwidth,
    height=5cm,
    xlabel={{Version}},
    ylabel={{Scan time per CWE (s)}},
    ymin=0, ymax={ymax},
    xtick={{{xtick}}},
    xticklabels={{{xlabels}}},
    xticklabel style={{rotate=45, anchor=east, font=\\footnotesize}},
    ylabel style={{font=\\footnotesize}},
    xlabel style={{font=\\footnotesize}},
    tick label style={{font=\\footnotesize}},
    grid=major,
    grid style={{dashed, gray!30}},
]
\\addplot[color=blue, thick, mark=none] coordinates {{
    {coords}
}};
\\end{{axis}}
\\end{{tikzpicture}}
\\caption{{Juliet scan time per CWE across {n} released versions
(v{series[0]['version']}--v{series[-1]['version']}). The metric is the summed
per-CWE \\texttt{{sqc}} analysis time divided by the number of CWEs scanned,
making it independent of both parallelism (\\texttt{{--jobs}}) and corpus growth
(68--74 CWEs). The sharp drop at v0.3.67 is a single perf optimization
($\\approx$8$\\times$); the subsequent climb is the cost of added
inter-procedural analysis, which still settles below the v0.3.5x peak.}}
\\label{{fig:scan-time}}
\\end{{figure}}
"""
    TEX_OUT.write_text(tex)
    print(f"wrote {CSV_OUT.relative_to(REPO)} ({n} versions)")
    print(f"wrote {TEX_OUT.relative_to(REPO)}")
    print(f"range: {min(ys)}-{max(ys)} s/CWE  (latest v{series[-1]['version']}: {series[-1]['s_per_cwe']})")


if __name__ == "__main__":
    main()