Versions benchmark rework 2

versions/apply-minvers.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+# Normalise the committed result files to the current per-query minimum versions
+# (queries/<ds>.minver). fetch-results.sh pulls RAW data from the sink, which reflects the
+# minvers that were in effect WHEN each version ran -- not the current ones -- so this makes
+# every result consistent with the current minvers regardless of when it was benchmarked:
+#
+#   * a query below its minimum version is nulled (its whole per-query timing array), and
+#   * a dataset every query of which is above the version (i.e. the version is below the
+#     dataset's lowest minimum) has its load_time / data_size dropped too -- such a version
+#     never loads that dataset, so it must not report a size or load time for it.
+#
+# Idempotent: re-running on already-normalised files changes nothing. Edits go through jq so
+# untouched number literals keep their exact formatting (matching fetch-results.sh's jq -cS).
+
+import glob, os, re, subprocess, sys
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+os.chdir(HERE)
+
+# Same fixed order the runner concatenates per-dataset results in (see run-version.sh).
+QUERY_ORDER = ["mgbench", "ssb", "hits", "uk", "ontime", "taxi", "coffeeshop", "tpch", "tpcds", "job"]
+
+def sql_count(ds):
+    with open(f"queries/{ds}.sql") as f:
+        return sum(1 for line in f if line.strip())
+
+def minver_lines(ds):
+    with open(f"queries/{ds}.minver") as f:
+        return [line.strip() for line in f if line.strip() != ""]
+
+# Version key mirroring run-version.sh's version_key: a bare build number is a 1.1.<n>
+# snapshot; dotted versions use their components (missing -> 0).
+def vkey(v):
+    if re.fullmatch(r"\d+", v):
+        return (1, 1, int(v), 0)
+    parts = v.split(".")
+    return tuple(int(parts[i]) if i < len(parts) and parts[i].isdigit() else 0 for i in range(4))
+
+def below(v, m):            # "0" == no minimum
+    return m != "0" and vkey(v) < vkey(m)
+
+# Build the global (concatenated) minver list and each dataset's [start,end) span + minimum.
+global_minver = []
+spans = {}          # ds -> (start, end)
+dmin = {}           # ds -> lowest minimum ("0" if any query runs everywhere)
+for ds in QUERY_ORDER:
+    n = sql_count(ds)
+    mvs = minver_lines(ds)
+    if len(mvs) != n:
+        sys.exit(f"{ds}: {len(mvs)} minver lines != {n} queries")
+    start = len(global_minver)
+    global_minver.extend(mvs)
+    spans[ds] = (start, start + n)
+    dmin[ds] = "0" if any(m == "0" for m in mvs) else min(mvs, key=vkey)
+
+edited = 0
+for f in sorted(glob.glob("results/*.json")):
+    v = subprocess.check_output(["jq", "-r", ".version", f], text=True).strip()
+    idx = [i for i, m in enumerate(global_minver) if below(v, m)]
+    drop = [ds for ds in QUERY_ORDER if below(v, dmin[ds])]
+    if not idx and not drop:
+        continue
+    prog = ("reduce $idx[] as $i (.; .result[$i] |= map(null)) "
+            "| reduce $drop[] as $d (.; del(.load_time[$d], .data_size[$d]))")
+    out = subprocess.check_output(
+        ["jq", "-cS", "--argjson", "idx", str(idx), "--argjson", "drop",
+         "[" + ",".join(f'"{d}"' for d in drop) + "]", prog, f])
+    with open(f, "wb") as fh:
+        fh.write(out)
+    edited += 1
+
+print(f"apply-minvers: normalised {edited} result file(s) to current minvers", file=sys.stderr)

versions/fetch-results.sh

@@ -114,3 +114,8 @@ for f in results/*.json; do
         rm -f "${f}"; echo "  dropped ${v}.json (duplicate of ${av})" >&2
     fi
 done
+
+# Normalise every result to the current per-query minimum versions (queries/*.minver):
+# the sink stores what each version produced under the minvers in effect WHEN it ran, which
+# may predate the current ones, so re-apply them here for consistency (idempotent).
+python3 "${HERE}/apply-minvers.py"

versions/index.html

@@ -704,19 +704,42 @@ <h2>About this benchmark</h2>
         /// and cannot flip the order of two other versions. Geometric mean across enabled
         /// queries (dataset on + query checkbox on).
         metric_noun = 'query time';
-        /// Per-query worst (slowest) selected-run time across the shown versions.
+        /// An absent result is penalized by the MAXIMUM of two estimates of how slow it
+        /// would have been:
+        ///   (A) 2× the worst time for that query across the shown versions (the original
+        ///       rule), and
+        ///   (B) 2× this version's own geometric-mean slowdown (over the queries it DID run,
+        ///       relative to the per-query baseline) applied to this query's baseline.
+        /// (A) alone flattens a very slow version's many absent queries to ~2× the (fast)
+        /// baseline, hiding its true slowness; (B) lifts each absent query to this version's
+        /// typical slowdown, so e.g. a version running 50-200× slower on the queries it ran
+        /// is penalized in that range for the ones it did not.
         const worst_per_query = [...Array(num_queries).keys()].map(i => {
             const vals = filtered_data.map(elem => selectRun(elem.result[i])).filter(x => x !== null);
             return vals.length ? Math.max(...vals) : null;
         });
         summaries = filtered_data.map(elem => {
+            /// This version's geometric-mean slowdown to the per-query baseline, over the
+            /// queries it actually ran (used for estimate B); 1 if it ran nothing.
+            let sacc = 0, sn = 0;
+            for (let i = 0; i < num_queries; ++i) {
+                if (!any_enabled || queryEnabled(i)) {
+                    const base = selectRun(baseline_data[i]), cur = selectRun(elem.result[i]);
+                    if (base !== null && cur !== null) { sacc += Math.log(cur / base); ++sn; }
+                }
+            }
+            const slowdown = sn ? Math.exp(sacc / sn) : 1;
             let accumulator = 0, used = 0;
             for (let i = 0; i < num_queries; ++i) {
                 if (!any_enabled || queryEnabled(i)) {
                     const base = selectRun(baseline_data[i]);
                     if (base === null) { continue; }
-                    const penalty = worst_per_query[i] != null ? missing_result_penalty * worst_per_query[i] : missing_result_time;
-                    const curr = selectRun(elem.result[i]) ?? penalty;
+                    let curr = selectRun(elem.result[i]);
+                    if (curr === null) {
+                        const byWorst = worst_per_query[i] != null ? missing_result_penalty * worst_per_query[i] : missing_result_time;
+                        const bySlowdown = missing_result_penalty * slowdown * base;
+                        curr = Math.max(byWorst, bySlowdown);
+                    }
                     accumulator += Math.log((constant_time_add + curr) / (constant_time_add + base));
                     ++used;
                 }
@@ -782,7 +805,7 @@ <h2>About this benchmark</h2>
         td_number.appendChild(document.createTextNode(value_text(idx)));
 
         /// Horizon (multi-scale) bar, ported from the main ClickBench page: the same
-        /// value is drawn at 1×, 2×, 4×, 8× zoom in four colour bands, so small
+        /// value is drawn at 1×, 4×, 16×, 64× zoom in four colour bands, so small
         /// differences near the fast end stay visible across a wide dynamic range.
         const percentage = (ratio != null && isFinite(max_ratio) && max_ratio > 0) ? ratio / max_ratio * 100 : 0;
         let td_bar = document.createElement('td');
@@ -797,12 +820,12 @@ <h2>About this benchmark</h2>
                 var(--bar-color1) 0%,
                 var(--bar-color1) ${Math.min(100, percentage)}%,
                 var(--bar-color2) ${Math.min(100, percentage)}%,
-                var(--bar-color2) ${Math.min(100, percentage * 2)}%,
-                var(--bar-color3) ${Math.min(100, percentage * 2)}%,
+                var(--bar-color2) ${Math.min(100, percentage * 4)}%,
                 var(--bar-color3) ${Math.min(100, percentage * 4)}%,
-                var(--bar-color4) ${Math.min(100, percentage * 4)}%,
-                var(--bar-color4) ${Math.min(100, percentage * 8)}%,
-                transparent ${Math.min(100, percentage * 8)}%,
+                var(--bar-color3) ${Math.min(100, percentage * 16)}%,
+                var(--bar-color4) ${Math.min(100, percentage * 16)}%,
+                var(--bar-color4) ${Math.min(100, percentage * 64)}%,
+                transparent ${Math.min(100, percentage * 64)}%,
                 transparent 100%)`;
         }
         td_bar.appendChild(bar);
@@ -1121,7 +1144,7 @@ <h2>About this benchmark</h2>
 }
 
 document.getElementById('scale_hint').textContent =
-    'Different colors on the bar chart represent the same values shown at different scales (1x, 2x, 4x, 8x zoom).';
+    'Different colors on the bar chart represent the same values shown at different scales (1x, 4x, 16x, 64x zoom).';
 
 render();
 updateSelectors();

versions/list-versions.sh

@@ -142,12 +142,17 @@ emit() {  # version date  -> resolve provider and print the line
         printf '%s\tclickhouse-built:%s\t%s\n' "${bv}" "${bv}" "${BUILT_DATE[$bv]}"
     done
 
-    # Prehistoric monthly builds (2012-04 .. 2016-02), reconstructed from source and
-    # labeled by month; skip the pre-server months that could not be built.
+    # Prehistoric monthly builds (2012-06 .. 2016-02), reconstructed from source and
+    # labeled by month. Skipped:
+    #  * pre-server months (2012-01..03): no server binary at all.
+    #  * 2012-04/05: the server boots but the era's client has no --query and its
+    #    interactive/HTTP paths can't be scripted, so no query can run -- not benchmarkable.
+    #    (2012-06 is the first month with a --query-capable client.)
     if [ -f "${MONTHLY_BUILT}" ]; then
         while IFS=$'\t' read -r m _sha note; do
             [ -z "${m}" ] && continue
             case "${note}" in pre-server*) continue ;; esac   # no server binary -> not runnable
+            [[ "${m}" < "2012-06-01" ]] && continue           # no scriptable client -> not benchmarkable
             printf '%s\tclickhouse-built:%s\t%s\n' "${m}" "${m}" "${MONTH_DATE[$m]:-${m}}"
         done < "${MONTHLY_BUILT}"
     fi

versions/queries/hits.minver

@@ -16,6 +16,8 @@
 0
 0
 0
+1.1.53991
+1.1.53991
 0
 0
 0
@@ -24,20 +26,18 @@
 0
 0
 0
+0.0.18847
 0
 0
 0
 0
+1.1.53991
 0
 0
+0.0.18847
+0.0.18847
 0
+0.0.18847
+0.0.18847
 0
-0
-0
-0
-0
-0
-0
-0
-0
-0
+0.0.18847

versions/queries/job.minver

@@ -1,20 +1,20 @@
 21.4
-0
-0
-0
-0
-0
-0
+1.1.53991
+1.1.53991
+1.1.53991
+1.1.53991
+1.1.53991
+1.1.53991
 25.1
-0
-0
-0
-0
-0
-0
-0
-0
-0
+1.1.53991
+1.1.53991
+1.1.53991
+1.1.53991
+1.1.53991
+1.1.53991
+1.1.53991
+1.1.53991
+1.1.53991
 25.1
 25.1
 25.1
@@ -33,15 +33,15 @@
 25.1
 25.1
 25.1
-0
+1.1.53991
 20.5
-0
+1.1.53991
 20.5
 21.4
 25.1
 20.5
 21.4
-0
+1.1.53991
 20.5
 21.4
 21.4
@@ -73,9 +73,9 @@
 25.1
 25.1
 25.1
-0
-0
-0
+1.1.53991
+1.1.53991
+1.1.53991
 25.1
 25.1
 25.1
@@ -91,9 +91,9 @@
 25.1
 25.1
 25.1
-0
-0
-0
+1.1.53991
+1.1.53991
+1.1.53991
 25.1
 25.1
 25.1

versions/queries/ontime.minver

@@ -1,11 +1,11 @@
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
+1.1.53991
+1.1.53991
+1.1.53991
+1.1.53991
+1.1.53991
+1.1.53991
+1.1.53991
+1.1.53991
+1.1.53991
+1.1.53991
+1.1.53991

versions/queries/ssb.minver

@@ -1,13 +1,13 @@
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
+1.1.53991
+1.1.53991
+1.1.53991
+1.1.53991
+1.1.53991
+1.1.53991
+1.1.53991
+1.1.53991
+1.1.53991
+1.1.53991
+1.1.53991
+1.1.53991
+1.1.53991

versions/queries/taxi.minver

@@ -1,4 +1,4 @@
-0
-0
-0
-0
+0.0.29410
+0.0.29410
+0.0.29410
+0.0.29410