diff --git a/AGENTS.md b/AGENTS.md
index 9875e8f06f..8604b33646 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -136,6 +136,21 @@ Before changing a serialized type that persists on disk, inspect its `serializer
Edit `.actor.cpp` and `.actor.h` sources, not actorcompiler-generated output under the build directory.
+## Source File Headers
+
+Every new `.cpp` / `.h` / `.actor.cpp` / `.actor.h` file starts with the standard Apache 2.0 license block, with the filename on line 2 and the current year on the copyright line. Copy from any existing file in the tree (e.g. `flow/Knobs.cpp`). Add file-purpose comments *after* the license block, not in place of it.
+
+## Code Review
+
+Unless you have specific instructions to the contrary, when asked to review code (named files or a diff), address all of these explicitly:
+
+- What is it trying to accomplish?
+- Is it correct?
+- Are there bugs?
+- Are there omissions?
+- Are there things that could be done better?
+- Should it be LGTM'd? (clear yes / no / not-yet)
+
## Branching
PRs target `main`. Release branches receive cherry-picks rather than direct PRs — don't open backport PRs without confirming first.
diff --git a/CMakeLists.txt b/CMakeLists.txt
index be9c579d81..e4a50c580b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -83,6 +83,12 @@ if(WITH_ACAC)
add_compile_definitions(WITH_ACAC)
endif()
+option(FDB_MEMORY_TRACKER "Compile in the sampled per-call-site memory tracker (flow/MemoryTracker)" ON)
+if(NOT FDB_MEMORY_TRACKER)
+ message(STATUS "Building FoundationDB with the memory tracker compiled out")
+ add_compile_definitions(FDB_MEMORY_TRACKER=0)
+endif()
+
###############################################################################
# Packages used for bindings
###############################################################################
diff --git a/contrib/mako_ab_binaries.py b/contrib/mako_ab_binaries.py
new file mode 100644
index 0000000000..3679a608ba
--- /dev/null
+++ b/contrib/mako_ab_binaries.py
@@ -0,0 +1,365 @@
+#!/usr/bin/env python3
+#
+# mako_ab_binaries.py
+#
+# This source file is part of the FoundationDB open source project
+#
+# Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+A/B benchmark across two *different fdbserver binaries*:
+
+ A = vanilla main (a build with no memory-tracker code at all)
+ B = this PR's build, with memory tracking turned OFF (sample_inverse=0)
+
+The point is to isolate the cost of the always-compiled memory-tracker hooks
+when they are disabled — i.e. to check the design's claim that the off-state
+overhead is <=1%. Unlike mako_ab_memtracker.py (one binary, two knob settings),
+each arm here runs a different --build.
+
+Only arm B is passed the memory-tracking knob; arm A (main) does not have it and
+would reject an unknown knob. Both arms get the RocksDB direct-I/O-off knobs for
+the rocksdb engine (main has those long-standing knobs too) since the data dir
+is on tmpfs.
+
+Reuses contrib/mako_storage_bench.sh and the chart.js report from
+mako_ab_memtracker.py. Run on the dev pod, e.g.:
+
+ python3 contrib/mako_ab_binaries.py \
+ --build-a /root/build_output4 --build-b /root/build_output5 \
+ --engines redwood rocksdb --warmup 60 --seconds 240 \
+ --outdir /mnt/ram/binab
+"""
+
+import argparse
+import json
+import os
+import re
+import shutil
+import subprocess
+
+from mako_ab_memtracker import parse_run, line_ds, PCTS, clobber_ramdisk
+
+# Found next to this script rather than hard-coded to one workspace.
+_HERE = os.path.dirname(os.path.abspath(__file__))
+BENCH_DEFAULT = os.path.join(_HERE, "mako_storage_bench.sh")
+
+# A = baseline blue, B = warm orange.
+COLOR = {"a": "#4477CC", "b": "#EE7733"}
+
+
+def rocksdb_tmpfs_knobs(engine):
+ """RocksDB opens its DB with O_DIRECT, which tmpfs does not support; both
+ binaries need direct I/O off to run rocksdb on /mnt/ram. redwood needs
+ nothing."""
+ if engine == "rocksdb":
+ return [
+ "--knob_rocksdb_use_direct_reads=0",
+ "--knob_rocksdb_use_direct_io_flush_compaction=0",
+ ]
+ return []
+
+
+def run_arm(bench, build, engine, armkey, extra_knobs, warmup, seconds, rows, outbase):
+ """Run one (engine, arm) via mako_storage_bench.sh with the arm's build."""
+ workdir = os.path.join(outbase, armkey)
+ env = dict(os.environ)
+ env["WORKDIR"] = workdir
+ env["WARMUP_SECONDS"] = str(warmup)
+ env["SECONDS_RUN"] = str(seconds)
+ env["ROWS"] = str(rows)
+ env["KNOBS"] = " ".join(extra_knobs + rocksdb_tmpfs_knobs(engine))
+ print(f"\n=== {engine} / {armkey} build={build} ===", flush=True)
+ print(f" WORKDIR={workdir} KNOBS={env['KNOBS']}", flush=True)
+ subprocess.run(["bash", bench, build, engine], env=env, check=False)
+ return os.path.join(workdir, engine)
+
+
+def source_version(build):
+ """The git source version baked into the binary (fdbserver --version). Used
+ to prove A and B are genuinely different builds, and to record provenance."""
+ fdbserver = os.path.join(build, "bin", "fdbserver")
+ try:
+ out = subprocess.run(
+ [fdbserver, "--version"], capture_output=True, text=True, timeout=60
+ ).stdout
+ except (OSError, subprocess.SubprocessError):
+ return None
+ m = re.search(r"source version (\w+)", out)
+ return m.group(1) if m else None
+
+
+# --------------------------------------------------------------------------
+
+
+def _harvest(rundir, dst):
+ """Copy the small result files off tmpfs to persistent storage before the
+ bulky SS/cluster data is wiped."""
+ os.makedirs(dst, exist_ok=True)
+ for name in ("mako.json", "mako-run.txt"):
+ src = os.path.join(rundir, name)
+ if os.path.exists(src):
+ shutil.copy2(src, os.path.join(dst, name))
+
+
+def collect(bench, build_a, build_b, engines, warmup, seconds, rows, ramdir, outdir):
+ b_knobs = ["--knob_memory_tracking_sample_inverse=0"]
+ data = {}
+ for engine in engines:
+ data[engine] = {}
+ for armkey, build, knobs in (("a", build_a, []), ("b", build_b, b_knobs)):
+ rundir = run_arm(
+ bench, build, engine, armkey, knobs, warmup, seconds, rows, ramdir
+ )
+ dst = os.path.join(outdir, armkey, engine)
+ _harvest(rundir, dst) # save results to /root for persistence + debugging
+ metrics = parse_run(dst)
+ print(f" -> overallTPS={metrics['overallTPS']}", flush=True)
+ data[engine][armkey] = metrics
+ return data
+
+
+def generate_html(data, outpath, meta, warmup, seconds, rows):
+ engines = list(data.keys())
+ la, lb = meta["label_a"], meta["label_b"]
+
+ rows_html = []
+ for eng in engines:
+ a = data[eng].get("a", {})
+ b = data[eng].get("b", {})
+ ta, tb = a.get("overallTPS"), b.get("overallTPS")
+ # B relative to A: negative = B slower (overhead).
+ d = (100.0 * (tb - ta) / ta) if (ta and tb) else None
+ p99a = a.get("latency", {}).get("p99")
+ p99b = b.get("latency", {}).get("p99")
+ dp = (100.0 * (p99b - p99a) / p99a) if (p99a and p99b) else None
+
+ def fmt(v, s=""):
+ return f"{v:,.0f}{s}" if isinstance(v, (int, float)) else "—"
+
+ def dfmt(v):
+ if v is None:
+ return "—"
+ return f"{'+' if v >= 0 else ''}{v:.2f}%"
+
+ rows_html.append(
+ f"| {eng} | {fmt(ta)} | {fmt(tb)} | "
+ f"{dfmt(d)} | "
+ f"{fmt(p99a,' µs')} | {fmt(p99b,' µs')} | "
+ f"{dfmt(dp)} |
"
+ )
+
+ tps_a = [data[e].get("a", {}).get("overallTPS") or 0 for e in engines]
+ tps_b = [data[e].get("b", {}).get("overallTPS") or 0 for e in engines]
+ tps_datasets = json.dumps(
+ [
+ {"label": la, "data": tps_a, "backgroundColor": COLOR["a"]},
+ {"label": lb, "data": tps_b, "backgroundColor": COLOR["b"]},
+ ]
+ )
+
+ blocks = []
+ for i, eng in enumerate(engines):
+ a = data[eng].get("a", {})
+ b = data[eng].get("b", {})
+ ps_datasets = json.dumps(
+ [
+ line_ds(f"{eng} {la}", a.get("persec", []), COLOR["a"]),
+ line_ds(f"{eng} {lb}", b.get("persec", []), COLOR["b"], dashed=True),
+ ]
+ )
+ lat_labels = json.dumps([s for _, s in PCTS])
+ lat_a = [a.get("latency", {}).get(s) for _, s in PCTS]
+ lat_b = [b.get("latency", {}).get(s) for _, s in PCTS]
+ lat_datasets = json.dumps(
+ [
+ {"label": la, "data": lat_a, "backgroundColor": COLOR["a"]},
+ {"label": lb, "data": lat_b, "backgroundColor": COLOR["b"]},
+ ]
+ )
+ n = max(len(a.get("persec", [])), len(b.get("persec", [])), 1)
+ blocks.append(
+ f"""
+ {eng}
+
+
+ """
+ )
+
+ same = meta["ver_a"] and meta["ver_a"] == meta["ver_b"]
+ guard = ""
+ if same:
+ guard = (
+ "WARNING: both builds report the same source "
+ "version — A and B may be the same binary; results are not meaningful.
"
+ )
+
+ html = f"""
+
+
+
+ FDB Memory Tracker A/B: vanilla main vs PR (tracking off)
+
+
+
+
+ FDB Memory Tracker A/B — vanilla main vs PR (tracking OFF)
+ Two different fdbserver binaries on the same host / workload, isolating the
+ cost of the always-compiled memory-tracker code when it is disabled.
+ A = {la} (build {meta['build_a']}, source {meta['ver_a']});
+ B = {lb} (build {meta['build_b']}, source {meta['ver_b']},
+ memory_tracking_sample_inverse=0).
+ Single-host 1/1/1 loopback cluster on /mnt/ram (storage isolated, CPU-bound) via
+ contrib/mako_storage_bench.sh. rows={rows:,}, warmup {warmup}s, run {seconds}s.
+ Δ is B relative to A: a small negative Δ is the off-state overhead; the design
+ target is ≤1%.
+ {guard}
+ Caveat: {meta['drift_note']}
+
+ Summary
+
+ | engine | TPS A ({la}) | TPS B ({lb}) | Δ TPS (B vs A) |
+ p99 lat A | p99 lat B | Δ p99 |
|---|
+ {''.join(rows_html)}
+
+
+
+
+ {''.join(blocks)}
+
+
+"""
+ with open(outpath, "w") as f:
+ f.write(html)
+ return outpath
+
+
+def main():
+ ap = argparse.ArgumentParser(
+ description="Vanilla-main vs PR-tracking-off A/B via mako_storage_bench.sh"
+ )
+ ap.add_argument(
+ "--build-a", default="/root/build_output4", help="vanilla main build (arm A)"
+ )
+ ap.add_argument(
+ "--build-b", default="/root/build_output5", help="this-PR build (arm B)"
+ )
+ ap.add_argument("--label-a", default="main (no tracker)")
+ ap.add_argument("--label-b", default="PR, tracking off")
+ ap.add_argument("--bench", default=BENCH_DEFAULT)
+ ap.add_argument("--engines", nargs="+", default=["redwood", "rocksdb"])
+ ap.add_argument("--warmup", type=int, default=60)
+ ap.add_argument("--seconds", type=int, default=240)
+ ap.add_argument("--rows", type=int, default=100000)
+ ap.add_argument(
+ "--ramdir",
+ default="/mnt/ram/binab",
+ help="tmpfs scratch for SS/cluster data (only SS data lives on /mnt/ram)",
+ )
+ ap.add_argument(
+ "--ram-mount", default="/mnt/ram", help="tmpfs mount clobbered clean at startup"
+ )
+ ap.add_argument(
+ "--outdir",
+ default="/root/binab_results",
+ help="persistent results dir on /root (~1 TB); harvested off tmpfs per arm",
+ )
+ ap.add_argument(
+ "--report",
+ default="/root/src/mako_binab.html",
+ help="HTML output path (syncs to ~/src on the Mac)",
+ )
+ ap.add_argument(
+ "--drift-note",
+ default="A and B may build from slightly different main "
+ "revisions; the delta bundles the PR's off-state cost with any main drift.",
+ )
+ ap.add_argument("--report-only", action="store_true")
+ args = ap.parse_args()
+
+ meta = {
+ "label_a": args.label_a,
+ "label_b": args.label_b,
+ "build_a": args.build_a,
+ "build_b": args.build_b,
+ "ver_a": source_version(args.build_a),
+ "ver_b": source_version(args.build_b),
+ "drift_note": args.drift_note,
+ }
+ print(f"A: {args.build_a} source={meta['ver_a']}")
+ print(f"B: {args.build_b} source={meta['ver_b']}")
+
+ if args.report_only:
+ data = {}
+ for eng in args.engines:
+ data[eng] = {
+ "a": parse_run(os.path.join(args.outdir, "a", eng)),
+ "b": parse_run(os.path.join(args.outdir, "b", eng)),
+ }
+ else:
+ clobber_ramdisk(args.ramdir, args.ram_mount) # clean slate up front; no end-of-run cleanup
+ os.makedirs(args.ramdir, exist_ok=True)
+ os.makedirs(args.outdir, exist_ok=True)
+ data = collect(
+ args.bench,
+ args.build_a,
+ args.build_b,
+ args.engines,
+ args.warmup,
+ args.seconds,
+ args.rows,
+ args.ramdir,
+ args.outdir,
+ )
+
+ generate_html(data, args.report, meta, args.warmup, args.seconds, args.rows)
+ print(f"\nReport written: {args.report}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/contrib/mako_ab_memtracker.py b/contrib/mako_ab_memtracker.py
new file mode 100644
index 0000000000..b7bcf35a41
--- /dev/null
+++ b/contrib/mako_ab_memtracker.py
@@ -0,0 +1,570 @@
+#!/usr/bin/env python3
+#
+# mako_ab_memtracker.py
+#
+# This source file is part of the FoundationDB open source project
+#
+# Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+A/B benchmark: FDB per-call-site memory tracker OFF vs ON (1:100 sampling).
+
+Uses contrib/mako_storage_bench.sh (single-host 1/1/1 loopback cluster, storage
+role isolated on ~one core, CPU-bound on /mnt/ram) to run each engine under both
+arms back-to-back on the same host, then generates a self-contained chart.js
+report (mako_memtracker_ab.html) viewable in any browser.
+
+Both arms are the SAME build; only runtime knobs differ:
+ OFF: --knob_memory_tracking_sample_inverse=0
+ ON : --knob_memory_tracking_sample_inverse=100
+Both also force --knob_memory_tracking_report_interval=30 so the tracker's
+periodic dump fires within the run (equal cadence in both arms -> fair) and so
+we can verify the knob took by reading MemoryTrackerSummary's SampleInverse out
+of the storage process trace.
+
+Run on the dev pod, e.g.:
+ python3 /root/src/fdb5/foundationdb/contrib/mako_ab_memtracker.py --build /root/build_output5 \
+ --engines redwood rocksdb --warmup 60 --seconds 240
+
+Only SS/cluster data goes on tmpfs (--ramdir under /mnt/ram, clobbered clean at
+startup); per-arm metrics are saved under --outdir on /root so the report
+survives the next run's clobber.
+"""
+
+import argparse
+import json
+import os
+import re
+import statistics
+import subprocess
+import sys
+
+# The /root and /mnt/ram defaults below are dev-pod defaults; override via flags
+# (or FDB_BUILD) elsewhere. BENCH is found next to this script rather than hard-coded.
+_HERE = os.path.dirname(os.path.abspath(__file__))
+BENCH_DEFAULT = os.path.join(_HERE, "mako_storage_bench.sh")
+BUILD_DEFAULT = os.environ.get("FDB_BUILD", "/root/build_output5")
+
+# OFF = baseline blue, ON = warm orange.
+COLOR = {"off": "#4477CC", "on": "#EE7733"}
+ARMS = [("off", 0), ("on", 100)] # (label, sample_inverse)
+PCTS = [
+ ("medianLatency", "p50"),
+ ("p95Latency", "p95"),
+ ("p99Latency", "p99"),
+ ("p99.9Latency", "p99.9"),
+]
+
+
+def _assert_safe_scratch_mount(mount):
+ """Refuse to recursively delete anything that isn't clearly a dedicated tmpfs
+ scratch mount. Guards against a typo or a stray --ram-mount (e.g. /mnt, /, or
+ $HOME) wiping unrelated data before the run even starts."""
+ real = os.path.realpath(mount)
+ denylist = {"/", "/mnt", "/tmp", "/root", "/home", os.path.expanduser("~")}
+ if real in denylist or real.count("/") < 2:
+ raise SystemExit(
+ f"refusing to clobber unsafe scratch path: {mount!r} -> {real!r}"
+ )
+ if not os.path.ismount(real):
+ raise SystemExit(
+ f"refusing to clobber {real!r}: not a mountpoint (expected a tmpfs mount)"
+ )
+ fstype = subprocess.run(
+ ["stat", "-f", "-c", "%T", real], capture_output=True, text=True
+ ).stdout.strip()
+ if fstype != "tmpfs":
+ raise SystemExit(
+ f"refusing to clobber {real!r}: filesystem is {fstype!r}, not tmpfs"
+ )
+
+
+def clobber_ramdisk(ramdir, ram_mount):
+ """Clear only this benchmark's own scratch dir under the tmpfs so every run
+ starts clean, without touching anything else sharing the mount (e.g. other
+ processes' data under /dev/shm). The ramdisk (~24 GB) fills fast across runs.
+ Done at startup rather than teardown: a killed run can't be trusted to have
+ cleaned up, and leaving the last run's data in place until the next run keeps
+ it available to inspect when something fails."""
+ if not os.path.isdir(ramdir):
+ return
+ _assert_safe_scratch_mount(ram_mount)
+ real_mount = os.path.realpath(ram_mount)
+ real_ramdir = os.path.realpath(ramdir)
+ # Only ever delete a directory strictly beneath the validated tmpfs mount, so a
+ # stray --ramdir can't wipe the mount root or unrelated data elsewhere.
+ if real_ramdir == real_mount or os.path.commonpath([real_mount, real_ramdir]) != real_mount:
+ raise SystemExit(
+ f"refusing to clobber {ramdir!r} -> {real_ramdir!r}: not strictly under mount {real_mount!r}"
+ )
+ subprocess.run(["rm", "-rf", real_ramdir], check=False)
+ print(f"clobbered benchmark scratch dir {real_ramdir}", flush=True)
+
+
+def save_metrics(dst, metrics):
+ """Persist an arm's computed metrics to /root so the report survives the
+ next run's ramdisk clobber (the traces alloc-rate/knob-verify are derived
+ from live only while the run's tmpfs data exists)."""
+ os.makedirs(dst, exist_ok=True)
+ with open(os.path.join(dst, "metrics.json"), "w") as f:
+ json.dump(metrics, f)
+
+
+def load_metrics(dst):
+ try:
+ with open(os.path.join(dst, "metrics.json")) as f:
+ return json.load(f)
+ except (OSError, json.JSONDecodeError):
+ return {"overallTPS": None, "persec": [], "latency": {}, "ok": False}
+
+
+def run_arm(bench, build, engine, arm, inverse, warmup, seconds, rows, outbase):
+ """Run one (engine, arm) via mako_storage_bench.sh; return its output dir."""
+ workdir = os.path.join(outbase, arm)
+ env = dict(os.environ)
+ env["WORKDIR"] = workdir
+ env["WARMUP_SECONDS"] = str(warmup)
+ env["SECONDS_RUN"] = str(seconds)
+ env["ROWS"] = str(rows)
+ knobs = [
+ f"--knob_memory_tracking_sample_inverse={inverse}",
+ "--knob_memory_tracking_report_interval=30",
+ ]
+ if engine == "rocksdb":
+ # RocksDB opens its DB with O_DIRECT by default; tmpfs (/mnt/ram, where
+ # this harness runs its data dir) does not support direct I/O, so the
+ # storage engine fails to Open and the cluster never configures. Turn
+ # direct I/O off via RocksDB's existing knobs (no new knobs added).
+ knobs += [
+ "--knob_rocksdb_use_direct_reads=0",
+ "--knob_rocksdb_use_direct_io_flush_compaction=0",
+ ]
+ env["KNOBS"] = " ".join(knobs)
+ print(f"\n=== {engine} / {arm} (sample_inverse={inverse}) ===", flush=True)
+ print(f" WORKDIR={workdir} KNOBS={env['KNOBS']}", flush=True)
+ subprocess.run(["bash", bench, build, engine], env=env, check=False)
+ return os.path.join(workdir, engine)
+
+
+def verify_sample_inverse(rundir):
+ """Read MemoryTrackerSummary's SampleInverse from the storage trace.
+ Returns the observed int, or None if no summary event was found."""
+ lc = os.path.join(rundir, "loopback-cluster")
+ observed = None
+ if not os.path.isdir(lc):
+ return None
+ for root, _, files in os.walk(lc):
+ for fn in files:
+ if "trace" not in fn:
+ continue
+ try:
+ with open(os.path.join(root, fn), errors="ignore") as fh:
+ for line in fh:
+ if "MemoryTrackerSummary" in line:
+ m = re.search(r'SampleInverse["\s:=]+(-?\d+)', line)
+ if m:
+ observed = int(m.group(1))
+ except OSError:
+ pass
+ return observed
+
+
+def alloc_rate_from_trace(rundir):
+ """Estimate the storage process's allocation rate from the ON arm's
+ MemoryTrackerSummary events (emitted every report interval, 30s here):
+ rate = delta(EstCumulativeAllocs)/delta(time). Only meaningful when
+ sampling is on. Returns {alloc_per_sec, mb_per_sec, samples_per_sec} or
+ None."""
+ lc = os.path.join(rundir, "loopback-cluster")
+ if not os.path.isdir(lc):
+ return None
+
+ def field(line, key, cast=float):
+ m = re.search(key + r'="?(-?[\d.]+)"?', line)
+ return cast(m.group(1)) if m else None
+
+ rows = []
+ for root, _, files in os.walk(lc):
+ for fn in files:
+ if "trace" not in fn:
+ continue
+ try:
+ for line in open(os.path.join(root, fn), errors="ignore"):
+ if "MemoryTrackerSummary" not in line:
+ continue
+ t = field(line, "Time")
+ ea = field(line, "EstCumulativeAllocs", int)
+ eb = field(line, "EstCumulativeBytes", int)
+ se = field(line, "SamplesEmitted", int)
+ mm = re.search(r'Machine="([^"]+)"', line)
+ rm = re.search(r'Roles="([^"]*)"', line)
+ if t is not None and ea is not None:
+ rows.append(
+ (
+ mm.group(1) if mm else "?",
+ rm.group(1) if rm else "",
+ t,
+ ea,
+ eb or 0,
+ se or 0,
+ )
+ )
+ except OSError:
+ pass
+ if not rows:
+ return None
+ # Pick the storage process: the machine with the highest peak
+ # EstCumulativeAllocs (storage allocates far more than stateless/log). Use
+ # all of that machine's summaries (early role-less ones included) so even
+ # short runs yield >=2 points to difference.
+ by_mach = {}
+ for r in rows:
+ by_mach.setdefault(r[0], []).append(r)
+ best = max(by_mach.values(), key=lambda rs: max(x[3] for x in rs))
+ best.sort(key=lambda r: r[2])
+ if len(best) < 2:
+ return None
+ ar, br, sr = [], [], []
+ for a, b in zip(best, best[1:]):
+ dt = b[2] - a[2]
+ if dt <= 0:
+ continue
+ ar.append((b[3] - a[3]) / dt)
+ br.append((b[4] - a[4]) / dt)
+ sr.append((b[5] - a[5]) / dt)
+ if len(ar) >= 3: # drop the first interval (startup ramp)
+ ar, br, sr = ar[1:], br[1:], sr[1:]
+ if not ar:
+ return None
+ return {
+ "alloc_per_sec": statistics.median(ar),
+ "mb_per_sec": statistics.median(br) / 1e6,
+ "samples_per_sec": statistics.median(sr),
+ }
+
+
+def parse_run(rundir):
+ """Extract metrics from a completed run dir."""
+ out = {"overallTPS": None, "persec": [], "latency": {}, "ok": False}
+ mj = os.path.join(rundir, "mako.json")
+ if os.path.exists(mj):
+ try:
+ d = json.load(open(mj))
+ except (OSError, json.JSONDecodeError):
+ return out
+ res = d.get("results", {})
+ out["overallTPS"] = res.get("overallTPS")
+ out["persec"] = [s["tps"] for s in d.get("samples", []) if "tps" in s]
+ for key, short in PCTS:
+ blk = res.get(key, {})
+ if isinstance(blk, dict) and "TRANSACTION" in blk:
+ out["latency"][short] = blk["TRANSACTION"] # microseconds
+ out["ok"] = out["overallTPS"] is not None
+ # Fallback: Overall TPS from the teed text report.
+ if out["overallTPS"] is None:
+ mt = os.path.join(rundir, "mako-run.txt")
+ if os.path.exists(mt):
+ for line in open(mt, errors="ignore"):
+ m = re.search(r"Overall TPS:\s*([\d.]+)", line)
+ if m:
+ out["overallTPS"] = float(m.group(1))
+ out["ok"] = True
+ out["rate"] = alloc_rate_from_trace(rundir)
+ return out
+
+
+def collect(bench, build, engines, warmup, seconds, rows, ramdir, outdir):
+ data = {} # engine -> arm -> {metrics, verified_inverse}
+ for engine in engines:
+ data[engine] = {}
+ for arm, inverse in ARMS:
+ rundir = run_arm(
+ bench, build, engine, arm, inverse, warmup, seconds, rows, ramdir
+ )
+ metrics = parse_run(rundir) # reads mako.json + alloc-rate off tmpfs
+ observed = verify_sample_inverse(rundir) # reads the trace off tmpfs
+ metrics["verified_inverse"] = observed
+ metrics["expected_inverse"] = inverse
+ ok = "OK" if observed == inverse else f"MISMATCH (saw {observed})"
+ print(
+ f" -> overallTPS={metrics['overallTPS']} "
+ f"knob-verify SampleInverse={observed} expected={inverse} [{ok}]",
+ flush=True,
+ )
+ save_metrics(os.path.join(outdir, arm, engine), metrics) # persist to /root
+ data[engine][arm] = metrics
+ return data
+
+
+# --------------------------------------------------------------------------
+# HTML / chart.js generation
+# --------------------------------------------------------------------------
+
+
+def line_ds(label, series, color, dashed=False):
+ d = {
+ "label": label,
+ "data": series,
+ "borderColor": color,
+ "backgroundColor": color,
+ "pointRadius": 0,
+ "borderWidth": 2,
+ "tension": 0.2,
+ "fill": False,
+ }
+ if dashed:
+ d["borderDash"] = [6, 3]
+ return d
+
+
+def generate_html(data, outpath, warmup, seconds, rows):
+ engines = list(data.keys())
+
+ # ---- Summary rows ----
+ rows_html = []
+ for eng in engines:
+ off = data[eng].get("off", {})
+ on = data[eng].get("on", {})
+ t_off, t_on = off.get("overallTPS"), on.get("overallTPS")
+ dtps = (100.0 * (t_on - t_off) / t_off) if (t_off and t_on) else None
+ p99_off = off.get("latency", {}).get("p99")
+ p99_on = on.get("latency", {}).get("p99")
+ dp99 = (100.0 * (p99_on - p99_off) / p99_off) if (p99_off and p99_on) else None
+
+ def fmt(v, s=""):
+ return f"{v:,.0f}{s}" if isinstance(v, (int, float)) else "—"
+
+ def dfmt(v):
+ if v is None:
+ return "—"
+ sign = "+" if v >= 0 else ""
+ return f"{sign}{v:.2f}%"
+
+ rows_html.append(
+ f"| {eng} | {fmt(t_off)} | {fmt(t_on)} | "
+ f"{dfmt(dtps)} | "
+ f"{fmt(p99_off,' µs')} | {fmt(p99_on,' µs')} | "
+ f"{dfmt(dp99)} | "
+ f"off={off.get('verified_inverse')} / on={on.get('verified_inverse')} |
"
+ )
+
+ # ---- Per-engine charts ----
+ tps_labels = engines
+ tps_off = [data[e].get("off", {}).get("overallTPS") or 0 for e in engines]
+ tps_on = [data[e].get("on", {}).get("overallTPS") or 0 for e in engines]
+ tps_datasets = json.dumps(
+ [
+ {
+ "label": "off (inverse=0)",
+ "data": tps_off,
+ "backgroundColor": COLOR["off"],
+ },
+ {
+ "label": "on (inverse=100)",
+ "data": tps_on,
+ "backgroundColor": COLOR["on"],
+ },
+ ]
+ )
+
+ blocks = []
+ for i, eng in enumerate(engines):
+ off = data[eng].get("off", {})
+ on = data[eng].get("on", {})
+ # per-second time series
+ ps_datasets = json.dumps(
+ [
+ line_ds(f"{eng} off", off.get("persec", []), COLOR["off"]),
+ line_ds(f"{eng} on", on.get("persec", []), COLOR["on"], dashed=True),
+ ]
+ )
+ # latency percentiles
+ lat_labels = json.dumps([s for _, s in PCTS])
+ lat_off = [off.get("latency", {}).get(s) for _, s in PCTS]
+ lat_on = [on.get("latency", {}).get(s) for _, s in PCTS]
+ lat_datasets = json.dumps(
+ [
+ {"label": "off", "data": lat_off, "backgroundColor": COLOR["off"]},
+ {"label": "on", "data": lat_on, "backgroundColor": COLOR["on"]},
+ ]
+ )
+ blocks.append(
+ f"""
+ {eng}
+
+
+ """
+ )
+
+ rate_bits = []
+ for eng in engines:
+ r = data[eng].get("on", {}).get("rate")
+ if r:
+ rate_bits.append(
+ f"{eng} ≈ {r['alloc_per_sec'] / 1e6:.2f} M allocs/s "
+ f"({r['mb_per_sec']:.0f} MB/s, {r['samples_per_sec'] / 1e3:.0f}K samples/s)"
+ )
+ rate_note = (
+ (
+ "Observed storage-process allocation rate (on arm), "
+ "from the tracker's own MemoryTrackerSummary "
+ "(ΔEstCumulativeAllocs / report interval): "
+ + "; ".join(rate_bits)
+ + ". This is the rate the per-free global-lock cost scales with — far above "
+ "a single-threaded microbenchmark's assumed 100K/s, which (with cross-thread lock "
+ "contention) is why the end-to-end overhead here exceeds the μbench estimate.
"
+ )
+ if rate_bits
+ else ""
+ )
+
+ html = f"""
+
+
+
+ FDB Memory Tracker A/B: off vs 1:100 sampling
+
+
+
+
+ FDB Memory Tracker A/B — off vs 1:100 sampling
+ Same build; only the runtime knob differs
+ (memory_tracking_sample_inverse 0 vs 100, report interval forced to 30 s in both).
+ Single-host 1/1/1 loopback cluster on /mnt/ram (storage isolated, CPU-bound) via
+ contrib/mako_storage_bench.sh. Workload g18ui,
+ rows={rows:,}, warmup {warmup}s, run {seconds}s. A TPS drop in the "on" arm means the
+ tracker added CPU cost on the storage hot path; near-parity means no regression.
+
+ Summary
+
+ | engine | TPS off | TPS on | Δ TPS |
+ p99 lat off | p99 lat on | Δ p99 |
+ knob verify (SampleInverse) |
|---|
+ {''.join(rows_html)}
+
+ {rate_note}
+
+
+
+ {''.join(blocks)}
+
+
+"""
+ with open(outpath, "w") as f:
+ f.write(html)
+ return outpath
+
+
+def main():
+ ap = argparse.ArgumentParser(
+ description="Memory-tracker off/on A/B via mako_storage_bench.sh"
+ )
+ ap.add_argument("--build", default=BUILD_DEFAULT)
+ ap.add_argument("--bench", default=BENCH_DEFAULT)
+ ap.add_argument("--engines", nargs="+", default=["redwood", "rocksdb"])
+ ap.add_argument("--warmup", type=int, default=60)
+ ap.add_argument("--seconds", type=int, default=240)
+ ap.add_argument("--rows", type=int, default=100000)
+ ap.add_argument(
+ "--ramdir",
+ default="/mnt/ram/memtracker_ab",
+ help="tmpfs scratch for SS/cluster data (only SS data lives on /mnt/ram)",
+ )
+ ap.add_argument(
+ "--ram-mount", default="/mnt/ram", help="tmpfs mount clobbered clean at startup"
+ )
+ ap.add_argument(
+ "--outdir",
+ default="/root/memtracker_ab_results",
+ help="persistent results dir on /root (~1 TB); per-arm metrics saved here",
+ )
+ ap.add_argument(
+ "--report",
+ default=None,
+ help="HTML output path (default: /root/src/mako_memtracker_ab.html, "
+ "which syncs to ~/src on the Mac)",
+ )
+ ap.add_argument(
+ "--report-only",
+ action="store_true",
+ help="Skip running; regenerate HTML from an existing --outdir",
+ )
+ args = ap.parse_args()
+
+ # Default the report into the okteto sync root (~/src <-> /root/src) so it
+ # lands on the Mac for viewing without a separate copy step.
+ report = args.report or "/root/src/mako_memtracker_ab.html"
+
+ if args.report_only:
+ data = {}
+ for eng in args.engines:
+ data[eng] = {}
+ for arm, inverse in ARMS:
+ data[eng][arm] = load_metrics(os.path.join(args.outdir, arm, eng))
+ else:
+ clobber_ramdisk(args.ramdir, args.ram_mount) # clean slate up front; no end-of-run cleanup
+ os.makedirs(args.ramdir, exist_ok=True)
+ os.makedirs(args.outdir, exist_ok=True)
+ data = collect(
+ args.bench,
+ args.build,
+ args.engines,
+ args.warmup,
+ args.seconds,
+ args.rows,
+ args.ramdir,
+ args.outdir,
+ )
+
+ generate_html(data, report, args.warmup, args.seconds, args.rows)
+ print(f"\nReport written: {report}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/design/memory-tracker.md b/design/memory-tracker.md
new file mode 100644
index 0000000000..033cdf5997
--- /dev/null
+++ b/design/memory-tracker.md
@@ -0,0 +1,1030 @@
+# FDB Memory Tracker — Per-Call-Site Allocation Attribution
+
+## Objective
+
+Make it easier to debug memory leaks and generally to understand the
+principal consumers of memory in FDB. Specifically: add a sampled,
+always-compiled, knob-controlled memory attribution mechanism
+that captures a small return-address backtrace at a configurable
+fraction of allocations, aggregates byte and call counts per call
+site, and periodically emits the aggregates as TraceEvents for offline
+`addr2line` symbolization. The mechanism must be lightweight enough to
+leave on by default in production (~1% sampling) and at higher rates
+(~10%) in simulation, where it must remain deterministic. (That ≤1%
+overhead is a *target*, not a proven bound: the v1 single-lock implementation is
+over it at high allocation rates and the always-compiled off-state cost is not
+confidently measured, so the feature ships sampling-**off** by default and can be
+compiled out entirely via `FDB_MEMORY_TRACKER=OFF` — see Performance and Rollout.)
+
+**What this targets — and what it does not.** The point is to surface memory
+*leaks* and *untuned / oversized allocations* that drive RSS growth. FDB
+effectively never hits a real `malloc` / `operator new` failure: a process is
+killed by `fdbmonitor` when its RSS crosses a configured ceiling (typically
+~12–16 GB against an ~8 GB target), i.e. "OOM" is a self-imposed RSS threshold,
+not an allocator failure. So the interesting regime is memory growth well *short*
+of any allocation failure, and the tracker's behaviour under an actual
+allocation failure is explicitly not a scenario we optimize for — the hooks fail
+open (drop the sample). Reviewers should not over-index on `bad_alloc` paths.
+(The sampled path is nonetheless ordered so its own table growth happens before
+any counter update, so even that never-in-practice case leaves the accounting
+consistent — see "Fail-open" below.)
+
+## Background
+
+Historically memory leaks that happen in FDB production or at scale
+can be challenging to debug, taking O(weeks) to resolve.
+Additionally, lack of granular understanding of memory usage might
+make it more challenging to reduce process memory usage in support of
+ongoing fleet efficiency goals.
+
+FDB has aggregate memory telemetry today — per-`FastAllocator` size-class
+counters, total `Arena` bytes (`/flow/arena/arenaBlockBytesAllocated`), the
+`g_hugeArenaMemory` atomic, and process RSS reported in
+`flow/SystemMonitor.cpp`'s `MemoryMetrics` event — but no per-call-site
+attribution. When memory pressure or a leak shows up in production, the
+trace tells us *what size class* or *how much arena memory* but not *which
+caller*, making root-causing difficult.
+
+A partial framework already exists in `flow/include/flow/FastAlloc.h`
+under the `ALLOC_INSTRUMENTATION` compile flag. It defines the right
+shapes — `memSample` (pointer → backtrace hash + size), `backTraceLookup`
+(hash → aggregate), a `memSample_entered` reentrancy flag — and global
+`operator new`/`delete` overrides (in `fdbserver/GlobalNewDelete.cpp`). But
+it relies on glibc `backtrace(3)` (microseconds per call), is gated behind
+a non-default compile flag, and was designed for offline analysis, not
+production. In practice it is dead code: builds that turn it on are too
+slow to run real workloads.
+
+The infrastructure needed to make a lightweight version cheap is already
+in place:
+
+- Frame pointers are globally enabled
+ (`cmake/ConfigureCompiler.cmake`: `-fno-omit-frame-pointer`), so a
+ manual frame-pointer walk replaces `backtrace(3)` and runs in ~100 ns.
+- `platform::format_backtrace` (`flow/Platform.cpp`) already emits a
+ ready-to-paste `addr2line -e -p -C -f -i 0xADDR ...` command with
+ the PIE load offset subtracted via `dl_iterate_phdr`. Symbolization is
+ fully offline.
+- `flow/SystemMonitor.cpp` already drives periodic memory TraceEvents we
+ can hang the per-site dump off.
+
+## Requirements
+
+R0. **Aim to be cheap enough to leave compiled in by default.** The *target* is
+ ≤1% end-to-end CPU overhead at the production sampling rate on realistic
+ workloads. We do **not** yet have confident measurement that this holds: the
+ µbench and mako A/B put the always-compiled *off-state* cost in the
+ low-single-digit-percent / measurement-noise range, and the enabled
+ single-lock v1 is over the target at high allocation rates. Because the ≤1%
+ figure is a goal rather than a proven guarantee, the feature is (a) shipped
+ sampling-**off** by default, and (b) gated by a compile-time switch
+ (`FDB_MEMORY_TRACKER`, default on — see "Compile-time gate") so a deployment
+ that wants zero always-compiled footprint can exclude the code entirely.
+
+R1. **Un-sampled hot path.** When sampling is disabled, an allocation incurs
+ only one thread-local load (a per-thread "off" flag) and one branch; when
+ sampling is enabled, an un-sampled allocation additionally does one
+ thread-local decrement and branch. A non-sampled free (which has no
+ per-thread counter to gate on) incurs one relaxed read of a global enabled
+ flag and one branch. That flag is written only on an enabled-state change,
+ so it stays in MESI shared state and the read is a cached, uncontended
+ atomic load (~12-15 cycles for an L2 cache hit) — never a contended or
+ uncached atomic. No syscalls, no library calls, no locks on either path.
+
+R2. **Default-on in production and simulation (post-rollout
+ steady state).** Steady-state production default is 1% sampling
+ (one in 100 allocations); simulation runs at 10% (one in ten) so
+ the tracker is exercised under fault injection. The Rollout
+ section describes a phased landing where the production default
+ starts at 0 and is flipped to the steady-state value after
+ baseline performance testing.
+
+R3. **Allocation-path coverage.** All three FDB-owned allocation paths
+ are attributed:
+ - Global `operator new` / `operator delete` (all standard overloads:
+ sized, unsized, array, nothrow, aligned).
+ - `FastAllocator::allocate` / `::release` for every size class.
+ - `Arena` allocations, attributed at the `ArenaBlock` granularity
+ (one entry per block, sized to the block's byte count).
+
+R4. **Out-of-scope allocators.** Direct libc `malloc`/`free`,
+ `posix_memalign`/`aligned_alloc`, `mmap`, and third-party allocators
+ that bypass `operator new` are not attributed in v1. RocksDB
+ allocations *are* covered via the `operator new` override since
+ RocksDB STL containers and internal `Arena` slabs go through global
+ `operator new[]`.
+
+R5. **Determinism in simulation.** Sampling state lives entirely in
+ thread-local variables and never reads `g_random` or
+ `g_network->now()`. Identical workload + seed must produce identical
+ aggregate counts.
+
+R6. **Periodic reporting.** A configurable cadence emits one TraceEvent per
+ qualifying site, where a site qualifies when its estimated live
+ bytes exceed `MEMORY_TRACKING_REPORT_BYTES_THRESHOLD` (see Reporting
+ for rationale). Each site event carries the site's
+ raw return-address frames together with a ready-to-paste `addr2line`
+ command (the `AddrCmd` detail) covering just that site's frames. The dump
+ itself runs in under 5 ms when ~50 sites qualify.
+
+R7. **Zero-cost runtime symbolization.** All address-to-symbol mapping
+ happens offline. The runtime never calls `backtrace_symbols`,
+ `dladdr`, or any DWARF reader.
+
+R8. **Strip-aware.** Reports work against stripped production binaries
+ when separate debug info (the `.debug` sidecar this repo's release
+ build already produces) is available to the offline tooling.
+
+R9. **Off-switch.** With the sample-inverse knob set to 0 (the shipping
+ default) the tracker is inert: the alloc hot path is a TLS load, decrement,
+ and branch; the free hot path is a cached atomic-flag read and a branch. No
+ aggregation work, and in particular no lock acquisition, happens on either
+ path. The knob is read at startup — see Non-requirements.
+
+R10. **Reentrancy safety.** No allocation made by the tracker itself
+ (table grows, dump scratch space) re-enters the tracking path. A
+ thread-local guard prevents re-entry.
+
+R11. **Side-thread coverage.** Allocations from any thread are
+ attributed, not only the network thread. This includes threads
+ spawned by third-party libraries we do not own (RocksDB
+ compaction/flush/iteration, OpenSSL background workers, etc.).
+ The frame-pointer walker must terminate cleanly when it
+ traverses FP-elided code such as glibc's pthread shutdown
+ machinery — see "Side-thread safety" for the empirical repro
+ (joshua-found `IThreadPool` segfaults) that exposed the
+ original walker's crash mode.
+
+## Non-requirements
+
+**Dynamic runtime enable/disable is not supported.** The sampling knobs are read
+at startup; to turn the tracker on or off, or change its sample rate, edit the
+config and restart the process. Toggling `MEMORY_TRACKING_SAMPLE_INVERSE` on a
+running process is not a use case we support — trying to make live allocations
+reconcile across an on↔off transition adds complexity for no benefit.
+
+## Design Overview
+
+Three pieces:
+
+1. **Sampling and capture.** Every allocation site calls a
+ header-inlined `memTrackerOnAlloc(p, size)` (and matching
+ `memTrackerOnFree(p)`). Both check a thread-local reentrancy flag
+ and a thread-local decrementing counter; the un-sampled path is one
+ TLS load + one decrement + one branch. On sample, walk the
+ frame-pointer chain for 4-6 return addresses (4-6 being initial
+ estimate, subject to refinement during development of this
+ feature).
+
+2. **Storage.** Two tables, both backed by `std::malloc` (which bypasses
+ our `operator new` hooks, so the tracker's own bookkeeping cannot recurse
+ into the tracking path):
+ - A fingerprint-keyed *aggregation table* (`fnv64(frames) → per-site
+ counters`): live / peak / cumulative bytes and counts, plus exemplar
+ frames for offline symbolization. Always present.
+ - A pointer-keyed *live-block table* (`void* → {fingerprint, size,
+ weight}`) so `onFree` can find what `onAlloc` recorded — the pointer is
+ the only key available at free time. Gated by the
+ `MEMORY_TRACKING_LIVE_TRACKING` knob (default on); when off, only
+ cumulative per-site stats are kept.
+
+3. **Reporting.** A `memTrackerDump(int64_t bytesThreshold)` walks the
+ aggregation table, emits a TraceEvent per site whose estimated live
+ bytes exceed the threshold — each carrying the site's estimated usage,
+ raw sampled counters, and a ready-to-paste `addr2line` command for its
+ stack — plus one summary event. Called periodically from `SystemMonitor.cpp`.
+
+Hooks:
+
+| Path | Allocate | Free |
+|---|---|---|
+| Global `new`/`delete` | `fdbserver/GlobalNewDelete.cpp` | `fdbserver/GlobalNewDelete.cpp` |
+| `FastAllocator` | `FastAlloc.cpp` (`::allocate`) | `FastAlloc.cpp` (`::release`) |
+| `Arena` | `Arena.cpp` (`ArenaBlock::create`) | `Arena.cpp` (`ArenaBlock::destroyLeaf`) |
+
+## Detailed Design
+
+### Sampling and reentrancy
+
+```cpp
+// MemoryTracker.h — header-inlined hot path
+
+extern thread_local bool gInMemTracker;
+extern thread_local int gMemTrackerCounter;
+extern thread_local size_t gForceSampleBytes; // refreshed periodically from FlowKnobs
+extern thread_local bool gMemTrackerOff; // set once per thread when sampling is off
+
+// Written only on an enabled-state change (rare), so it stays MESI-shared and
+// reads are cached/uncontended. Lives on its own cache line.
+extern std::atomic g_memTrackerEnabled;
+
+inline void memTrackerOnAlloc(void* p, size_t n) {
+ if (gMemTrackerOff) return; // disabled fast path: one TLS load + branch
+ if (gInMemTracker || !p) return;
+ if (--gMemTrackerCounter > 0 && n < gForceSampleBytes) return; // enabled, un-sampled
+ MemTrackerSuppress guard; // sets gInMemTracker; restores on scope exit
+ memTrackerSampleAlloc(p, n); // out-of-line; reseeds counter, sets gMemTrackerOff if off
+}
+
+inline void memTrackerOnFree(void* p) {
+ if (gInMemTracker || !p) return;
+ if (!g_memTrackerEnabled.load(relaxed)) return; // disabled: no lock, no table probe
+ MemTrackerSuppress guard;
+ memTrackerSampleFree(p); // no-op if p not tracked
+}
+```
+
+The alloc gate is a *per-thread* flag, not the global `g_memTrackerEnabled`: the
+counter bootstraps sampling per thread (the first allocation reaches the slow
+path, which reads the knob), so a single global gate set by an early main-thread
+allocation before `FLOW_KNOBS` is ready would wrongly disable worker threads that
+bootstrap later. Once a thread's slow path sees sampling off it sets
+`gMemTrackerOff`, collapsing its disabled cost to one TLS load.
+
+A free has no per-thread sampling counter — every free is a candidate to
+debit a live entry — so it cannot be gated the way an alloc is. Gating it on
+the enabled flag instead keeps the disabled free path lock-free: without this
+gate, every hooked `delete`/`FastAllocator::release`/`ArenaBlock` teardown
+would acquire the global spinlock even with sampling off, defeating the
+off-switch. The flag is published by `memTrackerInit()` — an explicit call
+`fdbserver` makes once, after all knobs are finalized and before any serving role
+starts — which reads `MEMORY_TRACKING_SAMPLE_INVERSE` and, on the calling
+(network) thread, either arms sampling or latches it off. Publishing it from a
+known init point rather than inferring it from the first allocation is what keeps
+an early startup allocation from latching the tracker off before the knobs are
+configured. It stays constant thereafter (the knob is read at startup only), so
+its cache line stays shared across cores and the hot-path read is cheap.
+`memTrackerResetForTest()` performs the same publish-and-arm so unit tests can
+flip the knob and re-initialize deterministically.
+
+The counter is reseeded on every sample to a uniform random integer on
+`[1, 2·INV−1]`, whose mean is exactly `INV`, so the sampling rate averages
+1-in-`INV` without aliasing to fixed allocation patterns. When `INV == 0` the
+thread parks its counter and stays on the fast path; the knob is read at startup
+only (see Non-requirements), so it never needs to re-observe a change.
+
+The counter's initial value (per-thread) is chosen so the first
+allocation a thread sees is always sampled, which guarantees that
+test workloads with very few allocations still exercise the path.
+
+**Force-sample-large.** Any allocation at or above
+`MEMORY_TRACKING_FORCE_SAMPLE_BYTES` (default ~100 KB) is *always* sampled
+regardless of the counter; setting the knob to `-1` disables force-sampling.
+
+Large allocations are rare per second so unconditional
+sampling costs almost nothing in CPU, and they are often the most
+interesting individual allocations regardless of frequency (caches,
+buffers, big arrays). This collapses the byte-rate-vs-count-rate
+distinction for the case it actually matters: rare-but-huge
+allocations are no longer at risk of being missed.
+
+`gForceSampleBytes` is a thread-local cache of
+`FLOW_KNOBS->MEMORY_TRACKING_FORCE_SAMPLE_BYTES`, refreshed inside
+`memTrackerSampleAlloc` (which already runs under sampled-path cost),
+so the hot path reads a TLS slot rather than touching the knobs
+struct.
+
+The aggregation table per-site adds a `forceSampledCount` counter
+incremented on this path; the dump emits it as `ForceSampledCount`.
+Consumers compute `ForceSampledCount / CumulativeAllocs` to tell
+whether a site's stats are predominantly "every alloc above the
+force-sample threshold" vs "1% of allocs" — without that, two sites
+with the same `CumulativeAllocs` value can mean very different
+population rates.
+
+### Stack capture
+
+Capture is a hand-rolled frame-pointer walk (`captureFramesFP`): starting from
+`__builtin_frame_address(0)`, follow the saved-FP links, recording the saved
+return address at each frame, up to `MEMORY_TRACKING_FRAMES` deep. The caller
+(`memTrackerSampleAlloc`) strips the topmost 1–2 tracker frames so the captured
+stack starts at the real allocation site rather than inside the tracker.
+
+The **load-bearing safety invariant** is a check on every iteration: each
+candidate frame pointer must be word-aligned and lie within the current
+thread's stack range — cached once per thread via `pthread_getattr_np` /
+`pthread_attr_getstack`, with a bounded fallback around the initial frame if
+that fails — *before* it is dereferenced. This is what lets the walk terminate
+cleanly instead of chasing a garbage saved-FP into unmapped memory when it
+reaches FP-elided code; see "Side-thread safety" for why that is not
+hypothetical.
+
+The walk relies on `-fno-omit-frame-pointer`, which is already set
+globally (`cmake/ConfigureCompiler.cmake`). On x86_64 and aarch64 each
+frame is two adjacent words (saved FP, saved RA), so the walk performs
+one indirect load per frame — roughly 100 ns for 6 frames.
+
+Caveats and mitigations:
+
+- Code compiled with `-fomit-frame-pointer` (glibc's pthread
+ shutdown / TLS-destructor machinery, third-party static libs not rebuilt
+ with project flags) does not terminate the walk cleanly — the saved-FP slot
+ at the boundary is uninitialized garbage, not `NULL`, and a naive `next <= fp`
+ check is insufficient. The stack-bounds invariant above is the mitigation;
+ see "Side-thread safety" for the empirical repro and full chain analysis.
+- Signal handlers can leave a transiently bad FP chain mid-walk; the same
+ bounds check bails out.
+- ASAN/MSAN builds may instrument the FP chain. The tracker is a
+ no-op-equivalent in those builds (sampling defaults can be flipped to
+ 0 in CMake when sanitizers are on).
+
+### Side-thread safety
+
+The tracker must work on every thread that allocates, not just the
+network thread. RocksDB — FDB's largest dependency — runs
+significant work on its own background threads (compaction, flush,
+iteration), and those code paths certainly allocate. Missing
+side-thread coverage would leave a large blind spot precisely
+where we expect a lot of byte volume. Side-thread coverage is a
+hard requirement (see R11), not a nice-to-have.
+
+This pulled in two non-obvious constraints that the initial draft
+missed:
+
+1. **No assumption of a single thread.** All tracker state that
+ touches the slow path is either thread-local (sampling counter,
+ reentrancy flag, stack-bounds cache, xorshift seed) or guarded
+ by the global spinlock (aggregation table, live-block table).
+ The hot path on every thread is one TLS load + one decrement +
+ one branch — it does not need to know whether it is on the
+ network thread, an I/O thread, a RocksDB compaction thread, or
+ a thread the tracker has never seen before.
+
+2. **Robust frame-pointer walking on threads we did not spawn.**
+ FDB-spawned threads start in FDB code (FP-enabled all the way
+ up). RocksDB-spawned threads start inside RocksDB and end up
+ in libc's `start_thread` machinery, which is FP-elided. At
+ thread *exit*, every thread (FDB- or library-spawned)
+ traverses glibc's TLS-destructor invocation, also FP-elided. A
+ FP-elided frame leaves the saved-FP slot uninitialized; our
+ walker, if it just trusts that slot, follows garbage into
+ unmapped memory and segfaults. **This is not hypothetical — we
+ hit it directly.** Joshua reproduced it with three RandomUnitTests
+ seeds (3288611985, 3731245491, 2219741568): every run segfaulted
+ inside `captureFramesFP` when an `IThreadPool` worker exited and
+ its `FastAllocator::ThreadData` destructor allocated a vector
+ grow during slab return. Symbolizing the crash showed the walker
+ had stepped from `~ThreadData` into glibc's TLS-destructor
+ invocation, where the saved-FP slot was garbage. The
+ stack-bounds check described under "Stack capture" above is what
+ makes this safe: any walk that crosses into FP-elided code
+ terminates at the stack boundary instead of dereferencing
+ garbage.
+
+The same considerations apply to threads that use `setjmp` /
+`longjmp` or coroutine resumption to switch stacks — at the switch
+point, the FP chain may temporarily lead into a different stack
+region. The stack-bounds check correctly terminates those walks
+too (the cached bounds are for the thread's *primary* stack;
+walks that wander into a coroutine stack or sigaltstack simply
+end early). For our use case "early termination on a non-primary
+stack" is the right behavior: the captured prefix is the part on
+the primary stack, which is what a human reading the dump cares
+about.
+
+### Storage
+
+#### Live-block table
+
+Hash table, key `void*`, value `{uint64_t fingerprint, uint64_t size,
+int64_t weight}`. All backing memory comes from `std::malloc`
+directly: this bypasses our `operator new` hooks (we override
+`operator new`, not libc `malloc`), so the tracker's own allocations
+cannot recurse back into the tracking path. The thread-local
+`gInMemTracker` flag is the single line of defense and is sufficient
+on its own; we don't need a private slab pool.
+
+`size` is stored full-width (a narrower field would under-debit `liveBytes`
+on free for allocations ≥ 4 GiB). `weight` is the block's estimate multiplier,
+captured at sample time (≈ `SampleInverse` for a randomly-sampled block, 1 for
+a force-sampled one); storing it per block lets a free debit the *estimated*
+totals by exactly what its alloc credited even if the sampling knob changed in
+between — see "Estimated usage" under Reporting.
+
+The table is created lazily alongside the aggregation table on the first
+sampled allocation; when live-tracking is off it stays allocated-but-empty.
+
+Both tables are allocated once and never torn down — intentionally leaked at
+process exit so `delete`s arriving during static destruction still find valid
+tables, and the tracker installs no `fork` handler since fdbserver does not
+fork after init.
+
+Single global `ThreadSpinLock` for v1. Two things acquire it: sampled
+allocs (~1K/sec at 1% of 100K alloc/sec; uncontended acquire ~20 ns, so
+~0.002% CPU) and **every** free while live-tracking is enabled — a free
+carries no backtrace, so its only key is the pointer and it must probe the
+live table to learn whether that pointer was sampled. At 100K free/sec that
+is ~100K lock+probe/sec, the dominant cost of the enabled steady state. Two
+mitigations: (1) when the tracker is *disabled* the `g_memTrackerEnabled`
+flag short-circuits the free path before the lock, so the off-switch is
+genuinely free of lock work; (2) per-thread or pointer-sharded live tables to
+cut enabled-state contention are deferred to a follow-up, to be sized against
+the phased-rollout performance baseline before the production default is
+flipped on (see Alternatives).
+
+#### Aggregation table
+
+A `std::unordered_map` keyed by `uint64_t` fingerprint. Each entry holds, for one
+call site: the **estimated** population usage (live / peak / cumulative bytes
+and counts, sampling-correction already applied at sample time so consumers read
+them directly), the **raw** sampled counters alongside them (one increment per
+observed sample, for auditing the estimate and gauging its confidence), a
+force-sampled count, and the exemplar return-address frames (bounded by
+`MEMORY_TRACKING_FRAMES`, capped at 10). The exact field set is in `CallSite`
+(`flow/MemoryTracker.h`).
+
+`fingerprint = fnv64(frames)`, computed once at sample time; exemplar frames are
+recorded from the first allocation with that fingerprint and never updated.
+Fingerprint collisions between distinct call paths are very unlikely with 64-bit
+FNV over several frames; if it ever matters we can switch hashes or store all
+observed frames. Live and peak fields are maintained only when live-tracking is
+enabled (they stay 0 in degraded mode); the cumulative fields are always
+maintained.
+
+### Hook sites
+
+#### Global `operator new` / `delete`
+
+`fdbserver/GlobalNewDelete.cpp` defines the ~14 standard global overloads
+— `new`, `new[]`, `delete`, `delete[]`, sized variants, `nothrow_t`
+variants, and the C++17 `std::align_val_t` overloads. Each calls
+`std::malloc` / `std::free` (or the portable `aligned_alloc` / `aligned_free`
+for the aligned variants) and then `memTrackerOnAlloc`/`OnFree`. The throwing
+overloads retry through the installed `std::new_handler` on failure, exactly as
+the default `operator new` does, so an allocation failure still reaches FDB's
+`platform::outOfMemory` handler (`FDB_EXIT_NO_MEM`) instead of throwing past it.
+
+These overloads live in a translation unit compiled directly into the
+`fdbserver` executable — not in the `flow` static library — for two
+reasons:
+
+1. **Correct interposition.** `operator new` / `operator delete` are
+ replaceable functions. A definition sitting in a static archive is
+ only pulled into the link if the linker already needs some other
+ symbol from that same object file; placing the overloads in an
+ executable TU guarantees they win rather than relying on incidental
+ archive pull-in.
+2. **Client isolation.** `flow` is linked into `libfdb_c` and every
+ client binding. A global-`new` override compiled into it would
+ interpose the entire host process's allocator in any application
+ that loads the client. `fdbserver` is a standalone executable that
+ clients never link, so keeping these here confines the interposition
+ to the server. (The `FastAllocator` and `Arena` hooks still compile
+ into `flow`, and hence into the client, but those are ordinary direct
+ calls gated on the sampling knob — not global-symbol interposition —
+ and are inert when sampling is off.)
+
+The same file also hosts the legacy `ALLOC_INSTRUMENTATION`
+`recordAllocation`/`recordDeallocation` overrides; the two
+implementations are selected by a single
+`#if defined(ALLOC_INSTRUMENTATION) …` / `#else` / `#endif`, so exactly
+one set of global operators is ever defined.
+
+
+#### FastAllocator
+
+In `flow/FastAlloc.cpp`, `FastAllocator::allocate` / `::release` get an
+unconditional `memTrackerOnAlloc(p, Size)` / `memTrackerOnFree(ptr)` call added
+next to (not replacing) the existing conditional `ALLOC_INSTRUMENTATION`
+`recordAllocation`/`recordDeallocation` lines, which are left in place. `Size`
+is a compile-time template parameter, so no size lookup is needed.
+
+The global `operator new` / `delete` overrides (both the tracker path
+and the legacy `ALLOC_INSTRUMENTATION` path) live in
+`fdbserver/GlobalNewDelete.cpp`; see the "Global `operator new` /
+`delete`" section above.
+
+#### Arena
+
+`Arena` has no per-allocation free path — only block-level lifetime via
+`ArenaBlock::create` (`flow/Arena.cpp`) and `ArenaBlock::destroyLeaf`
+(`flow/Arena.cpp`). Hooks are placed there:
+
+- `ArenaBlock::create`: after the block is allocated and before return,
+ call `memTrackerOnAlloc(blockPtr, blockSizeBytes)`. The captured stack
+ is whoever in user code triggered the arena to grow — typically the
+ caller doing a large `new (arena) ...` that overflowed the current
+ block.
+- `ArenaBlock::destroyLeaf`: before returning the block to its
+ underlying allocator (`FastAllocator::release` or `free`), call
+ `memTrackerOnFree(blockPtr)`.
+
+This is **block-level attribution**. A single `Arena` containing many
+small `new (arena) Foo` allocations is attributed to whichever calls
+forced new blocks to be created, not to the `new (arena) Foo` calls
+themselves. This is the right granularity for finding "who is making the
+arena grow" — the more interesting question — and avoids the need for an
+allocate-time hook on every bump-pointer call.
+
+Note: The tracker's `memTrackerOnAlloc` runs inside the Arena's
+allocation path, which itself runs inside callers that may already hold
+locks. Because the tracker takes only its own private spinlock and
+doesn't recurse into any FDB-visible state (no `g_network`, no
+`g_random`, no Arena allocations), this is safe.
+
+### Reporting
+
+`memTrackerDump(int64_t bytesThreshold)` is called from
+`flow/SystemMonitor.cpp` next to the existing `MemoryMetrics` event,
+gated by a knob. The default cadence is **once every 10 minutes** in
+production (knob-controlled via `MEMORY_TRACKING_REPORT_INTERVAL`);
+simulation defaults to 30 s for test exercise.
+
+Rather than logging a fixed top-N, the dump emits a `MemoryTrackerSite`
+event for every site whose **estimated** live bytes exceed
+`MEMORY_TRACKING_REPORT_BYTES_THRESHOLD`. The default threshold is
+**80 MB**, chosen as roughly 1% of the target RSS for a production
+fdbserver (~8 GB). Because the threshold is expressed in real bytes,
+it is compared against the estimated (sampling-corrected) live bytes,
+not the raw sampled bytes — comparing 80 MB against ~1/100th-scale
+sampled bytes would qualify almost nothing. Sites smaller than that are
+not load-bearing for RSS-level investigations and would only add log
+volume; sites above it are the ones worth attributing. A pure top-N has
+the wrong shape for this — top-50 against a process that genuinely has
+only three heavyweight sites still emits 47 noise events, while a
+process with hundreds of meaningful sites silently truncates at 50. A
+byte threshold scales naturally to the actual distribution.
+
+**Estimated usage.** The consumer should not have to do sampling math.
+Each site reports `Est*` fields — estimated *population* usage —
+alongside the raw sampled counters. The estimate weights each sampled
+block by its inverse inclusion probability at sample time: a randomly
+sampled block (1-in-`INV`) is weighted `INV`, a force-sampled block
+(captured with certainty) is weighted 1. `EstLiveBytes = Σ size×weight`
+over that site's live blocks, and likewise for cumulative and peak. The
+weight is stored per live block, so a free debits the estimate by exactly what
+its alloc credited (force-sampled blocks carry weight 1, randomly-sampled blocks
+weight `INV`).
+Weighting at sample time (rather than scaling raw totals at report time)
+also makes `EstPeakBytes` well-defined as the running max of
+`EstLiveBytes`. The raw counters remain in the event for auditing the
+estimate and gauging confidence (a site with few `SampledAllocs` is
+noisy); the `MemoryTrackerSummary` event carries an `EstimateBasis`
+caveat string.
+
+**Degraded mode.** When `MEMORY_TRACKING_LIVE_TRACKING` is `false`,
+the live-block side table is not populated, so `onFree` is a no-op and
+the per-site `liveBytes` / `liveCount` / `peakBytes` (and their `est*`
+counterparts) are never incremented — they stay at 0. Only the
+`Cumulative*` / `EstCumulative*` fields are meaningful in this mode, and
+the dump filters and ranks on `estCumulativeBytes` accordingly, so it
+reports "any site that has ever allocated ≥ threshold bytes" rather than
+"any site currently holding ≥ threshold". Operators using this mode
+should read the dump accordingly and ignore the (zero) live/peak fields.
+
+Each dump snapshots the aggregation table under the spinlock into a
+`std::malloc`-backed local (the `gInMemTracker` guard prevents recursion),
+releases the lock, then ranks and filters by the estimated bytes and emits:
+
+- one **`MemoryTrackerSite`** event per qualifying site: its fingerprint, the
+ estimated population fields, the raw sampled counters (for auditing), and an
+ `AddrCmd` detail — a ready-to-paste `addr2line` invocation for just that
+ site's frames (short enough to fit under the TraceEvent string-detail cap, so
+ a consumer pastes one site's `AddrCmd` and gets exactly that site's stack);
+- one **`MemoryTrackerSummary`** event per dump: site counts, estimated and raw
+ population totals, live-block count, samples emitted, an echo of the active
+ knobs (`SampleInverse`, `ForceSampleBytes`, threshold), and an `EstimateBasis`
+ caveat string.
+
+The exact detail keys live in the `TraceEvent` call sites in
+`flow/MemoryTracker.cpp`; a comment there points back to this section. The raw
+running totals are maintained as globals under the same spinlock (cumulative on
+every sample; live/peak totals only when live-tracking is on); the `Est*` totals
+are their sampling-corrected counterparts, accumulated with the per-block weight
+so consumers read them without post-hoc scaling.
+
+### Knobs (FlowKnobs, since clients use Arena/new too)
+
+Authoritative defaults live in `flow/Knobs.cpp`; this table gives the meaning
+and intent, not exact literals.
+
+| Knob | Meaning / intent |
+|---|---|
+| `MEMORY_TRACKING_SAMPLE_INVERSE` | 0 = off, N = sample 1-in-N. Ships off in prod during rollout; ~1% (N=100) is the steady-state target; simulation runs at 1-in-10 to exercise the path. |
+| `MEMORY_TRACKING_FORCE_SAMPLE_BYTES` | Always sample allocations at or above this size (~100 KB); `-1` disables force-sampling. |
+| `MEMORY_TRACKING_LIVE_TRACKING` | On by default. When off, skip the live-block table and report cumulative-only stats. |
+| `MEMORY_TRACKING_REPORT_INTERVAL` | Seconds between dumps (~10 min in prod, ~30 s in sim); 0 disables reporting. |
+| `MEMORY_TRACKING_REPORT_BYTES_THRESHOLD` | Report sites whose estimated bytes exceed this (prod ≈ 1% of an ~8 GB target RSS; lowered in sim so events actually fire). |
+| `MEMORY_TRACKING_FRAMES` | Captured stack depth (1–10). |
+
+Simulation uses a fixed 1-in-10 sample rate; the every-allocation path
+(`inverse==1`) and the sampled/weighted path (`inverse=N>1`) are both
+pinned deterministically by unit tests (`MemoryTrackerTest.cpp`), so no
+buggify of the rate is needed.
+
+### Code layout
+
+- **Tracker core:** `flow/MemoryTracker.{cpp,h}` — hot-path inlines, out-of-line
+ sample/free paths, the two tables, the dump, and `memTrackerInit()`.
+- **Startup init:** `fdbserver/fdbserver.cpp` calls `memTrackerInit()` once, after
+ server knobs are finalized and before any serving role starts, so the enabled
+ state is set from the final knob values rather than inferred from the first
+ allocation.
+- **Global `operator new`/`delete`:** `fdbserver/GlobalNewDelete.cpp` — compiled
+ only into the `fdbserver` executable (client isolation), and also home to the
+ legacy `ALLOC_INSTRUMENTATION` overrides via `#if`/`#else`.
+- **Allocator hooks:** unconditional `memTrackerOnAlloc`/`OnFree` calls added in
+ `flow/FastAlloc.cpp` and `flow/Arena.cpp`, next to the existing
+ `ALLOC_INSTRUMENTATION` lines (which are left untouched — dormant in default
+ builds).
+- **Dump driver / knobs:** `flow/SystemMonitor.cpp` and `flow/Knobs.{cpp,h}`.
+- **Tests / bench:** `fdbserver/MemoryTrackerTest.cpp` (unit tests) and
+ `fdbserver/bench/` (`fdbserver_bench`). The unit tests live under `fdbserver`,
+ not `flow`, so they run in a binary that links the global `operator new`/`delete`
+ override and can assert operator-new attribution end to end (a test in `flow`
+ would exercise the standard-library allocator, which never calls the hook).
+
+Most files are picked up by the source globs; the exceptions are the
+`fdbserver/bench/` subdirectory (its own `CMakeLists.txt` + an `add_subdirectory`
+line) and the `forceLinkMemoryTrackerTests()` stub that `fdbserver/workloads/UnitTests.cpp`
+must call so the test TU is not dropped from the link.
+
+### Determinism in simulation
+
+Sampling state (`gInMemTracker`, `gMemTrackerCounter`, the xorshift
+seed) lives in thread local storage. Sim2 runs single-threaded, so the sequence of
+sampling decisions is fully replayable across runs with the same seed.
+The tracker never reads `g_random` or `g_network->now()`, never
+allocates from FDB-visible heaps, and never takes any lock that
+participates in Sim2 ordering. The sample dump cadence is driven by
+`now()`, but only inside `SystemMonitor.cpp` where time-driven cadence
+is already deterministic in simulation.
+
+## Alternatives Considered
+
+### A1. In-band header (store backtrace in the allocated block)
+
+Original instinct: prepend an 8–16 byte header to each sampled
+allocation containing the fingerprint, look it back up on free. Fast
+free-side lookup (no hash).
+
+Rejected because:
+- `operator new` array allocations have a separate cookie; mixing in a
+ header breaks `delete[]`.
+- C++17 `std::align_val_t` overloads expect specific alignment that a
+ prepended header would invalidate.
+- `FastAllocator` uses fixed size classes; a header changes the
+ effective payload size and would force a different size class for
+ sampled vs. unsampled allocations, complicating `release()`.
+- `Arena` allocations are often 4–16 bytes; an 8-byte header is 50–200%
+ overhead on those allocations.
+
+A side table costs one hash lookup on free in exchange for not
+disturbing layout. At 1% sampling the side table is small and the
+lookup is one cache line in the common case.
+
+### A2. `backtrace(3)` / libunwind for stack capture
+
+The existing `ALLOC_INSTRUMENTATION` framework uses glibc `backtrace(3)`,
+which on first call touches the symbol table and is generally
+microseconds per call. libunwind is faster (sub-microsecond) but still
+slower than frame-pointer walking by an order of magnitude.
+
+Rejected because frame pointers are already enabled globally
+(`-fno-omit-frame-pointer` in `cmake/ConfigureCompiler.cmake`), and
+a hand-rolled FP walk is ~100 ns for 6 frames — fast enough to leave on
+in production. Falling back to libunwind is feasible if we ever need to
+support a build configuration that omits frame pointers.
+
+### A3. `--wrap=malloc` to intercept libc `malloc`/`free`
+
+Linker `--wrap` rewrites direct calls to `malloc` into `__wrap_malloc`,
+catching the long tail of allocations that bypass `operator new`
+(`posix_memalign`, OpenSSL's `OPENSSL_malloc`, RocksDB's cacheline-aligned
+allocators, third-party compression libraries).
+
+Deferred from v1 because:
+- FDB's own code allocates almost exclusively through `operator new`,
+ `FastAllocator`, and `Arena`, all of which are covered by direct
+ hooks.
+- RocksDB's STL containers and internal `Arena` slabs allocate via
+ `operator new[]` and are caught by the operator-new override.
+- The remaining gap (aligned allocations and OpenSSL/compression) is
+ small in byte volume and not currently a known source of mystery
+ memory growth.
+- `--wrap` requires changes at every link line and slightly complicates
+ static-analysis tools that don't model the rename.
+
+Trivial to add later if production data shows we need it.
+
+### A4. Byte-rate sampling vs count-rate sampling
+
+True byte-rate sampling (per-allocation Bernoulli draw with probability
+proportional to size, à la jemalloc/tcmalloc heap profilers) gives
+unbiased per-site byte estimates but adds per-allocation work and
+complicates determinism. The hybrid we ship — count-rate for small
+allocations plus unconditional sampling above
+`MEMORY_TRACKING_FORCE_SAMPLE_BYTES` — captures the case byte-rate
+would have caught (rare-but-huge allocations) without the extra
+hot-path cost.
+
+### A5. Per-thread aggregation tables vs single global lock
+
+Per-thread tables merged on dump avoid all lock contention on the hot
+path. With FDB's threading model (one network thread plus a small
+number of I/O / RocksDB threads), the merge cost is small.
+
+Chose single global spinlock for v1 because:
+- At 1% sampling + 100K alloc/sec the lock fires only 1K times/sec.
+ Uncontended spinlock acquire is ~20 ns. Total CPU cost ~20 µs/sec.
+
+ These figures are back-of-the-envelope, not measured. The 100K
+ alloc/sec is an assumed workload rate (real rates vary widely by
+ process role and load), and the ~20 ns uncontended-acquire is a
+ rough x86 ballpark that will differ across microarchitectures and
+ ISAs (aarch64), and under cache pressure or NUMA effects. Both could
+ be optimistic; treat the conclusion as "cheap on the sampled path
+ under expected conditions," to be confirmed by the microbenchmarks
+ and rollout baseline rather than as a guaranteed bound.
+
+- Per-thread tables require a thread-registration mechanism (the
+ tracker has to know about thread births/deaths so it can merge) and a
+ thread-local pointer in TLS; non-trivial to get right under
+ `pthread_create`-spawned background threads in third-party libraries
+ whose lifecycle FDB doesn't own.
+- Single-lock is dead simple to reason about and easy to retrofit into
+ per-thread later if profiling justifies it.
+
+### A6. Extending `ALLOC_INSTRUMENTATION` vs a new module
+
+The existing framework has the right shapes (`memSample`,
+`backTraceLookup`, `memSample_entered`). Could in principle be
+generalized.
+
+Rejected because:
+- The existing framework is gated behind a compile-time flag and was
+ designed for offline analysis with the slow `backtrace(3)`. Repurposing
+ it requires removing the compile gate, replacing the capture path,
+ changing the lock discipline, replacing the data structures, and
+ adding the dump path — at which point the rewrite is cleaner as a
+ fresh module.
+- Two coexisting paths under different flags are confusing for
+ maintainers.
+
+The new module supersedes it. The dead code stays in place behind its
+`#ifdef` for v1 and is a candidate for removal in a follow-up cleanup PR.
+
+### A7. jemalloc's built-in heap profiler
+
+jemalloc has a mature sampled-stack heap profiler enabled via
+`MALLOC_CONF=prof:true,prof_active:true,lg_prof_sample:14`. Zero
+implementation cost.
+
+Its runtime cost is not a single well-established figure we can cite here.
+jemalloc profiling samples at an average byte interval set by
+`lg_prof_sample` and captures a backtrace per sample, so the overhead
+scales with the allocation rate and the configured interval and is
+generally characterized as low at coarse sampling intervals. We have not
+benchmarked it for FDB's workload, and jemalloc's documentation does not
+give a workload-independent overhead number. We therefore make no specific
+performance claim; the reasons below reject it on coverage and portability
+grounds regardless of its cost.
+
+Rejected as a complete solution because:
+- Covers only `malloc` and `operator new`, not `FastAllocator` or
+ `Arena`. We suspect that most of FDB's allocation byte volume goes through the latter
+ two.
+- Requires a custom-built jemalloc with `--enable-prof` (the FDB
+ `USE_CUSTOM_JEMALLOC` build option). Default Linux distro jemalloc
+ packages don't enable profiling.
+- Not available on macOS / FreeBSD / Windows where jemalloc is
+ optional or absent.
+
+We could enable it *in addition* to the FDB tracker for the libc-malloc
+tail. Out of scope for v1.
+
+### A8. `__FILE__`/`__LINE__` macro instrumentation
+
+An alternative considered: replace every `new` / `arena.allocate` /
+`malloc` call with a macro that captures `__FILE__` / `__LINE__`,
+storing those instead of return-address frames. Avoids the need for
+`addr2line`.
+
+Rejected because:
+- File/line at the *allocator* layer is uninformative
+ (always `Arena.cpp` or `FastAlloc.cpp`). To get useful attribution
+ the macro would have to be applied at every *caller* site —
+ thousands of `new` and `arena.allocate` call sites. This is
+ invasive churn and breaks templates that allocate on behalf of
+ callers (the template body sees its own `__FILE__`, not the
+ instantiation's).
+- Return-address frames give correct multi-frame attribution
+ with no source changes. Symbolization via `addr2line` is offline
+ but already supported in this repo.
+
+## Testing Considerations
+
+### Unit tests
+
+`fdbserver/MemoryTrackerTest.cpp` implements these `TEST_CASE`s (all under
+`/flow/MemoryTracker/`). They live under `fdbserver`, not `flow`, so they run in a
+binary that links the global `operator new`/`delete` override and can exercise it:
+
+- **`coverage`** — sentinel functions drive one allocation each through
+ `operator new`, `FastAllocator`, and `Arena`; confirms a captured call
+ site contains a frame inside each sentinel. Linux-only (the FP walker is
+ a no-op stub elsewhere).
+- **`cumulativeIsMonotonic`** — after allocating then freeing, `liveCount`
+ returns to 0 while `cumulativeAllocs` does not decrement.
+- **`offSwitch`** — with sample inverse 0, no sites are recorded and the
+ `g_memTrackerEnabled` flag is false (so frees skip the lock).
+ `memTrackerResetForTest` arms the off-latch from the knob, matching
+ `memTrackerInit` at startup.
+- **`initEnablesFromKnob`** — drives the production `memTrackerInit()` path
+ directly (not the test-only reset, which the reviewer noted masks the startup
+ race): init publishes the enabled flag and arms/latches this thread from
+ `MEMORY_TRACKING_SAMPLE_INVERSE`, including re-arming a thread a prior
+ off-configuration had already latched off.
+- **`operatorNewHonorsNewHandler`** — a failing `::operator new` invokes the
+ installed `std::new_handler`, so allocation failure reaches FDB's OOM path
+ (item 1) rather than throwing past it.
+- **`operatorNewAccounting`** — allocations via `new` / `delete[]` (through the
+ fdbserver global override) are attributed to the calling site with correct
+ byte/block counts, and `delete` debits the live totals. Only meaningful because
+ the test runs in `fdbserver`, where the override is linked.
+- **`samplingRate`** — at inverse N the observed sampled fraction is ~1-in-N,
+ checking the `[1, 2N-1]` reseed's mean.
+- **`freeOfUntrackedPtrIsNoop`** — `memTrackerOnFree` on a never-sampled
+ pointer (and on `nullptr`) records nothing.
+- **`estimateScaling`** — with a fixed inverse N and no force-sampled
+ blocks, each site's `Est*` value equals N × its raw counter, and freeing
+ debits the estimate symmetrically.
+- **`fastAlloc32Accounting`, `arenaSmallAccounting`,
+ `arenaMediumAccounting`, `arenaHugeAccounting`** — byte/block accounting
+ per allocation path; the "exactly one site carries the sentinel's frames"
+ assertion is the B1 double-tracking regression test (the medium and huge
+ `Arena` paths are the ones that regressed).
+- **`failOpenOnMetadataAllocFailure`** — an injected tracker-metadata allocation
+ failure is swallowed: the underlying `new`/`delete` still succeeds and tracking
+ recovers on the thread afterward (the reentrancy guard is restored, not leaked).
+
+Not yet covered by a dedicated unit test: a frame-pointer-walk-depth test — that
+behavior is exercised indirectly by the sentinel tests above.
+
+### Microbenchmarks
+
+`fdbserver/bench/BenchMemoryTracker.cpp` (Google Benchmark; run
+`bin/fdbserver_bench --benchmark_filter=memtracker`) measures the tracker's
+per-operation cost. Two cases matter, both feeding R0 ("cheap enough to leave
+on?"): the **off-state** cost (always-compiled hooks, sampling disabled) and
+the **enabled-state** cost (hooks at the production 1% rate, and at the pessimal
+every-allocation rate). Each is measured both end-to-end through global
+`operator new[]`/`delete[]` and by calling `memTrackerOnAlloc`/`OnFree` directly
+on a preallocated buffer (which isolates the tracker's own work, with no
+allocation in the loop); a raw `malloc`/`free` loop is the tracker-free
+baseline. The operator-new path only fires the tracker when the global override
+(`fdbserver/GlobalNewDelete.cpp`) is linked into the binary, so this bench lives
+under `fdbserver/` and its CMake compiles that override into `fdbserver_bench`;
+`flow_bench` links only `flow` (no override) and would measure the unhooked
+allocator.
+
+Measured on the dev pod (AMD EPYC 9R14 / Zen 4, 32 vCPUs, 128 GB; `clang -O3`
+release build), ns per alloc+free pair, projected to one second at **~2M
+allocations/sec** — the storage-process allocation rate measured in the mako A/B
+below. These are a point-in-time snapshot on one machine (numbers stable to
+within a few percent across repeated runs), not a portable guarantee:
+
+| Case | ns/op | @2M/s | % of one core |
+|---|---|---|---|
+| `malloc`/`free` (baseline, no tracker) | 7.8 | 15.6 ms/s | 1.56% |
+| hooks, off | 1.6 | 3.3 ms/s | 0.33% |
+| hooks, 1% sampling | 5.1 | 10.2 ms/s | 1.02% |
+| hooks, every alloc (worst case, not a default) | 75 | 150 ms/s | 15% |
+| `operator new`/`delete`, off | 11.4 | 22.8 ms/s | 2.28% |
+| `operator new`/`delete`, 1% sampling | 14.4 | 28.8 ms/s | 2.88% |
+| `operator new`/`delete`, every alloc (worst case, not a default) | 81 | 162 ms/s | 16% |
+
+Takeaways:
+
+- **Disabled**, the tracker adds ~1.6 ns per alloc+free pair — ~0.33% of a core
+ at 2M/s. Small but no longer negligible at this rate; this is the off-switch
+ cost (R9).
+- **At the production 1% rate**, the tracker's own *single-threaded* work is
+ ~5.1 ns/pair — ~1.0% of a core at 2M/s, i.e. right at R0's 1% ceiling, and
+ that is *without* cross-thread lock contention. The cost is dominated by the
+ per-free lock+probe (every free takes the global lock while live-tracking is
+ on), not the 1%-of-allocs slow path.
+- These are single-threaded projections and are a **floor**, not the real cost.
+ The end-to-end mako A/B (storage isolated on one core, real multi-threaded
+ allocation including I/O threads) measures a materially larger hit at 1%
+ sampling — **−16.5% TPS on redwood, −9.9% on rocksdb** (off vs 1:100, live
+ tracking on) — because all those frees contend on the one global `g_mtLock`.
+ The engine with the higher baseline throughput (redwood) takes the larger
+ hit, consistent with a per-free-lock cost that scales with allocation rate.
+- **Conclusion:** even at this worst-case ~2M/s (a CPU-saturated storage core),
+ the current single-global-lock design is *not* comfortably under R0's 1%
+ ceiling when enabled at 1%. This is why the production default stays off
+ (R2's phased rollout) until the live-table lock is sharded / made per-thread
+ (the deferred follow-up in Storage and Alternatives); the µbench and A/B
+ together size that work.
+- The **every-allocation** rows are **not a proposed configuration** — they are
+ an estimated worst case for buggified `inverse=1` simulation runs.
+
+Caveat: 2M/s is a **worst-case** rate — it was measured on a deliberately
+CPU-pegged storage server (the mako bench pins the storage role to ~one core at
+max throughput). A production server not running flat out allocates less, so the
+%-of-core figures here are upper bounds; scale linearly for lower rates or other
+roles. Conversely, the single-threaded µbench excludes cross-thread lock
+contention, so its per-op %-of-core is a *lower* bound — the mako A/B is the
+representative end-to-end number. The real cost is bracketed between the two.
+
+Two mako A/B harnesses drive these end-to-end numbers, both on a single-host
+1/1/1 loopback cluster with the storage role CPU-pegged on tmpfs.
+`contrib/mako_ab_memtracker.py` runs one build at sampling off vs on (the
+−16.5% / −9.9% figures above). `contrib/mako_ab_binaries.py` compares a
+vanilla-`main` fdbserver against this PR built with tracking off, isolating the
+cost of the always-compiled-but-disabled code: on a shared base commit it
+measured **−0.53% (redwood) / −0.10% (rocksdb)**, confirming the off-state
+overhead is well within R0's 1% ceiling. Each emits a self-contained chart.js
+report.
+
+### Strip-aware symbolization spot-check
+
+Build the release binary, strip it, paste an `AddrCmd` field from a
+trace into a shell pointing at the matching `.debug` sidecar, confirm
+symbol resolution works.
+
+### Coverage spot-check via sentinel functions
+
+The `coverage` / `*Accounting` tests verify capture without log parsing or
+`addr2line`: they sample at inverse 1, drive a known number of allocations
+through each path from a dedicated sentinel function, then use the
+`memTrackerForEachSite` introspection API to assert that some captured site's
+exemplar frames fall inside that sentinel's address range with the expected
+allocation count. Comparing raw return addresses to function-pointer values is
+deterministic, fast, and works on stripped builds — it exercises the full
+sampling → capture → fingerprint → aggregation pipeline.
+
+## Observability/Supportability Considerations
+
+The `MemoryTrackerSummary` event (see Reporting) doubles as the tracker's own
+health signal: site counts, live-block count, and samples-emitted answer "is the
+tracker working" and "is it costing us." No separate self-metrics are added.
+
+## Rollout/Migration Considerations
+
+### Phased rollout
+
+1. Land code with `MEMORY_TRACKING_SAMPLE_INVERSE` defaulting to 0
+ (off) but compiled in — overriding the steady-state default in the
+ knobs table for this initial commit only. Verify no regression in
+ cluster-level performance testing.
+2. Flip the production default to 100 (1% sampling — the steady-state
+ value shown in the knobs table) in a follow-up commit.
+3. Confirm `MemoryTrackerSite` events flow through the trace pipeline
+ end-to-end.
+4. On a case by case basis, test in developer-specific performance
+ cluster testing for specific use cases related to storage server
+ memory use.
+
+### Client library implications
+
+`libfdb_c.so` and other client artifacts link `flow/`, so the tracker's
+`FastAllocator` and `Arena` hooks — and the out-of-line sample paths —
+are present in client binaries. The **global `operator new` / `delete`
+overrides are not**: they live in `fdbserver/GlobalNewDelete.cpp`, which
+is compiled only into the `fdbserver` executable (a standalone binary
+clients never link). Implications:
+
+- The global-allocator interposition never reaches an embedding
+ application. This removes the main client risk — a host app that has
+ its own `operator new` override, or is sensitive to which allocator
+ services `new`, is unaffected.
+- The `FastAllocator` / `Arena` hooks that do compile into the client
+ are ordinary direct calls gated on the sampling knob, not global-symbol
+ interposition. Sampling defaults to 0 in client contexts, so their
+ cost is just the `if (gInMemTracker)` / disabled-flag check. We have no
+ plans or requirements to enable sampling for client library users.
+
+### `ALLOC_INSTRUMENTATION` coexistence
+
+The existing framework in `flow/include/flow/FastAlloc.h` and its
+conditional callers in `flow/FastAlloc.cpp` remain in place. Its global
+`operator new` / `delete` overrides now live in the `#if
+defined(ALLOC_INSTRUMENTATION)` branch of `fdbserver/GlobalNewDelete.cpp`,
+mutually exclusive (via `#else`) with the tracker overrides, so exactly
+one set of global operators is defined. No removal is planned.
+
+### Rollback
+
+Setting `MEMORY_TRACKING_SAMPLE_INVERSE=0` and
+`MEMORY_TRACKING_REPORT_INTERVAL=0` in the config and restarting fully disables
+the tracker without a binary rebuild. The hot-path cost reduces to the TLS
+load + branch (`gInMemTracker` check) and is observably zero on
+benchmarks.
+
+If the operator-new overrides themselves need to be rolled back (e.g.,
+they cause issues with a third-party library that has its own global
+new override), removing them is a matter of deleting
+`fdbserver/GlobalNewDelete.cpp`'s tracker branch while keeping the
+FastAlloc and Arena tracking that lives in `flow`.
+
+### Compile-time gate
+
+`FDB_MEMORY_TRACKER` (CMake `option(... ON)`; default on) removes the feature at
+build time. `cmake -DFDB_MEMORY_TRACKER=OFF` (or `-DFDB_MEMORY_TRACKER=0` on the
+compiler line) makes the hooks no-ops via header shims and drops the global
+`operator new`/`delete` override entirely (libc++'s allocator is used, which still
+honors the installed `new_handler`). This is the escape hatch for anyone who wants
+zero always-compiled footprint, and it is how the microbenchmark measures the
+present-but-disabled vs absent cost of each allocation path.
diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt
index b0640a2fe2..dbb0aa45d1 100644
--- a/fdbserver/CMakeLists.txt
+++ b/fdbserver/CMakeLists.txt
@@ -229,3 +229,8 @@ if(NOT OPEN_FOR_IDE)
endif()
target_link_libraries(fdbserver PUBLIC fdbctl)
+
+if(NOT FOUNDATIONDB_CROSS_COMPILING) # FIXME(swift): make this work when
+ # x-compiling.
+ add_subdirectory(bench EXCLUDE_FROM_ALL)
+endif()
diff --git a/fdbserver/GlobalNewDelete.cpp b/fdbserver/GlobalNewDelete.cpp
new file mode 100644
index 0000000000..6953b81a82
--- /dev/null
+++ b/fdbserver/GlobalNewDelete.cpp
@@ -0,0 +1,248 @@
+/*
+ * GlobalNewDelete.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Process-wide replacements for the global operator new / operator delete set,
+// owned by the fdbserver binary.
+//
+// These live here, in a translation unit compiled directly into the fdbserver
+// executable, rather than in the `flow` static library, for two reasons:
+//
+// 1. Correctness of interposition. operator new / operator delete are
+// replaceable functions; a definition sitting in a static archive is only
+// pulled into the link if the linker already needs some other symbol from
+// that same object file. Placing them in an executable TU guarantees the
+// replacements are part of the final link instead of relying on incidental
+// archive pull-in.
+//
+// 2. Client isolation. `flow` is linked into libfdb_c and every client
+// binding; a global-new override compiled into it would interpose the
+// entire host process's allocator in any application that loads the client.
+// fdbserver is a standalone executable that clients never link, so keeping
+// these here confines the interposition to the server.
+//
+// Exactly one implementation is compiled, chosen by the same ALLOC_INSTRUMENTATION
+// flags the legacy accounting framework uses (so the two never define the global
+// operators twice):
+//
+// - ALLOC_INSTRUMENTATION[_STDOUT] on -> legacy FastAlloc accounting hooks.
+// - otherwise -> the sampled per-call-site memory
+// tracker (flow/MemoryTracker.*).
+
+#include
+#include
+
+#include "flow/MemoryTracker.h" // for FDB_MEMORY_TRACKER (default on)
+
+// TODO: the old ALLOC_INSTRUMENTATION doesn't seem to be usable at
+// scale. Consider deleting it.
+#if defined(ALLOC_INSTRUMENTATION) || defined(ALLOC_INSTRUMENTATION_STDOUT)
+
+#include "flow/FastAlloc.h"
+
+void* operator new(std::size_t size) {
+ void* p = malloc(size);
+ if (!p) {
+ throw std::bad_alloc();
+ }
+ recordAllocation(p, size);
+ return p;
+}
+void operator delete(void* ptr) throw() {
+ recordDeallocation(ptr);
+ free(ptr);
+}
+
+void* operator new(std::size_t size, const std::nothrow_t&) throw() {
+ void* p = malloc(size);
+ recordAllocation(p, size);
+ return p;
+}
+void operator delete(void* ptr, const std::nothrow_t&) throw() {
+ recordDeallocation(ptr);
+ free(ptr);
+}
+
+void* operator new[](std::size_t size) {
+ void* p = malloc(size);
+ if (!p) {
+ throw std::bad_alloc();
+ }
+ recordAllocation(p, size);
+ return p;
+}
+void operator delete[](void* ptr) throw() {
+ recordDeallocation(ptr);
+ free(ptr);
+}
+
+void* operator new[](std::size_t size, const std::nothrow_t&) throw() {
+ void* p = malloc(size);
+ recordAllocation(p, size);
+ return p;
+}
+void operator delete[](void* ptr, const std::nothrow_t&) throw() {
+ recordDeallocation(ptr);
+ free(ptr);
+}
+
+#else // sampled memory tracker, see design/memory-tracker.md
+
+#include "flow/Platform.h" // aligned_alloc / aligned_free (portable across MSVC/POSIX)
+
+#if FDB_MEMORY_TRACKER
+
+// NOTE: We (Apple) do not maintain a local facility to build FDB with MSVC on
+// Windows, and CI only *configures* (not compiles) there — so the MSVC-specific
+// pieces below are best-effort and not compile-verified: the exact signatures of
+// the replaceable global operator new/delete set, and the
+// aligned_alloc/aligned_free ↔ _aligned_malloc/_aligned_free pairing routed
+// through flow/Platform.h. This code may have issues on MSVC; community help for
+// the Windows build would be welcome. (See flow/MemoryTracker.cpp for the parallel
+// note on the non-Linux frame walker.)
+
+namespace {
+
+// Retry through the installed std::new_handler on failure, as the default
+// operator new does. fdbserver installs platform::outOfMemory, so an allocation
+// failure (including the tracker's own map growth) reaches FDB's OOM diagnostics
+// and FDB_EXIT_NO_MEM rather than throwing straight past them.
+void* mallocWithNewHandler(std::size_t n) {
+ void* p;
+ while (!(p = std::malloc(n))) {
+ std::new_handler h = std::get_new_handler();
+ if (!h) {
+ throw std::bad_alloc();
+ }
+ h();
+ }
+ return p;
+}
+
+// Same handler loop for over-aligned allocations. C11 aligned_alloc requires the
+// size to be a multiple of the alignment, so round up (harmless over-allocation)
+// to accept arbitrary operator-new sizes.
+void* alignedAllocWithNewHandler(std::size_t alignment, std::size_t n) {
+ std::size_t rounded = (n + alignment - 1) & ~(alignment - 1);
+ if (rounded < n) {
+ throw std::bad_alloc(); // round-up overflowed; the request can't be satisfied
+ }
+ void* p;
+ while (!(p = aligned_alloc(alignment, rounded))) {
+ std::new_handler h = std::get_new_handler();
+ if (!h) {
+ throw std::bad_alloc();
+ }
+ h();
+ }
+ return p;
+}
+
+} // namespace
+
+void* operator new(std::size_t n) {
+ void* p = mallocWithNewHandler(n);
+ memTrackerOnAlloc(p, n);
+ return p;
+}
+void operator delete(void* p) noexcept {
+ memTrackerOnFree(p);
+ std::free(p);
+}
+void operator delete(void* p, std::size_t) noexcept {
+ memTrackerOnFree(p);
+ std::free(p);
+}
+
+void* operator new[](std::size_t n) {
+ void* p = mallocWithNewHandler(n);
+ memTrackerOnAlloc(p, n);
+ return p;
+}
+void operator delete[](void* p) noexcept {
+ memTrackerOnFree(p);
+ std::free(p);
+}
+void operator delete[](void* p, std::size_t) noexcept {
+ memTrackerOnFree(p);
+ std::free(p);
+}
+
+void* operator new(std::size_t n, const std::nothrow_t&) noexcept {
+ try {
+ void* p = mallocWithNewHandler(n);
+ memTrackerOnAlloc(p, n);
+ return p;
+ } catch (...) {
+ return nullptr;
+ }
+}
+void operator delete(void* p, const std::nothrow_t&) noexcept {
+ memTrackerOnFree(p);
+ std::free(p);
+}
+
+void* operator new[](std::size_t n, const std::nothrow_t&) noexcept {
+ try {
+ void* p = mallocWithNewHandler(n);
+ memTrackerOnAlloc(p, n);
+ return p;
+ } catch (...) {
+ return nullptr;
+ }
+}
+void operator delete[](void* p, const std::nothrow_t&) noexcept {
+ memTrackerOnFree(p);
+ std::free(p);
+}
+
+// C++17 over-aligned new/delete. aligned_alloc/aligned_free (flow/Platform.h)
+// keep the alloc and free sides paired on MSVC (_aligned_malloc/_aligned_free).
+void* operator new(std::size_t n, std::align_val_t a) {
+ void* p = alignedAllocWithNewHandler(static_cast(a), n);
+ memTrackerOnAlloc(p, n);
+ return p;
+}
+void operator delete(void* p, std::align_val_t) noexcept {
+ memTrackerOnFree(p);
+ aligned_free(p);
+}
+void operator delete(void* p, std::size_t, std::align_val_t) noexcept {
+ memTrackerOnFree(p);
+ aligned_free(p);
+}
+
+void* operator new[](std::size_t n, std::align_val_t a) {
+ void* p = alignedAllocWithNewHandler(static_cast(a), n);
+ memTrackerOnAlloc(p, n);
+ return p;
+}
+void operator delete[](void* p, std::align_val_t) noexcept {
+ memTrackerOnFree(p);
+ aligned_free(p);
+}
+void operator delete[](void* p, std::size_t, std::align_val_t) noexcept {
+ memTrackerOnFree(p);
+ aligned_free(p);
+}
+
+#else // !FDB_MEMORY_TRACKER — no global operator new/delete override; libc++'s is used.
+#endif // FDB_MEMORY_TRACKER
+
+#endif // ALLOC_INSTRUMENTATION
diff --git a/fdbserver/MemoryTrackerTest.cpp b/fdbserver/MemoryTrackerTest.cpp
new file mode 100644
index 0000000000..8db4fe662f
--- /dev/null
+++ b/fdbserver/MemoryTrackerTest.cpp
@@ -0,0 +1,769 @@
+/*
+ * MemoryTrackerTest.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Unit tests for the per-call-site memory tracker.
+//
+// The "coverage" test uses sentinel functions: each sentinel triggers exactly
+// one allocation path (operator new, FastAllocator, Arena), and the test
+// confirms that some call site in the aggregation table contains a frame
+// inside that sentinel's body. We compare raw return-address values against
+// function-pointer values at runtime, so this works on stripped builds with
+// no symbolization.
+
+#include "flow/Arena.h"
+#include "flow/FastAlloc.h"
+#include "flow/Knobs.h"
+#include "flow/MemoryTracker.h"
+#include "flow/Platform.h"
+#include "flow/UnitTest.h"
+
+#include
+#include
+#include
+#include
+#include
+#include
+
+// Force this TU to link. The TEST_CASE macro registers via a static
+// initializer; in a static library, a TU containing only static initializers
+// gets dropped by the linker because nothing references its symbols.
+// fdbserver/workloads/UnitTests.cpp calls this function to keep the TU.
+void forceLinkMemoryTrackerTests() {}
+
+#if FDB_MEMORY_TRACKER
+
+namespace {
+
+// A sentinel is an out-of-line function that performs exactly one kind of
+// allocation, then returns its own address. We use the returned address to
+// recognize captured stack frames that fell inside the sentinel's body.
+constexpr uintptr_t SENTINEL_FUNC_SIZE = 4096;
+
+// Defeat clang -O3 heap elision (P0593): if the allocated pointer doesn't
+// escape, the compiler is free to drop the new/delete pair entirely, which
+// then never reaches our operator-new override and the test sees zero samples.
+void* volatile gEscapeSink;
+inline void escape(void* p) {
+ gEscapeSink = p;
+}
+
+bool frameInside(void* frame, void* sentinel) {
+ uintptr_t f = reinterpret_cast(frame);
+ uintptr_t s = reinterpret_cast(sentinel);
+ return f >= s && f < s + SENTINEL_FUNC_SIZE;
+}
+
+force_noinline void* triggerOperatorNewSentinel(int n, int k) {
+ for (int i = 0; i < n; i++) {
+ auto* p = new int[k];
+ p[0] = i;
+ escape(p);
+ delete[] p;
+ }
+ return reinterpret_cast(&triggerOperatorNewSentinel);
+}
+
+force_noinline void* triggerFastAllocSentinel(int n) {
+ for (int i = 0; i < n; i++) {
+ void* p = FastAllocator<32>::allocate();
+ escape(p);
+ FastAllocator<32>::release(p);
+ }
+ return reinterpret_cast(&triggerFastAllocSentinel);
+}
+
+force_noinline void* triggerArenaSentinel(int n) {
+ // Force ArenaBlock::create by allocating large enough chunks to exceed
+ // the small-block threshold.
+ for (int i = 0; i < n; i++) {
+ Arena a;
+ // One ~512-byte allocation per arena -> goes through allocateAndMaybeKeepalive
+ // path which has the explicit Arena hook.
+ auto* p = new (a) uint8_t[600];
+ escape(p);
+ }
+ return reinterpret_cast(&triggerArenaSentinel);
+}
+
+// ---------------------------------------------------------------------------
+// Accounting tests: verify byte/block counts come out right per allocation path
+// and there's no double-tracking.
+force_noinline void* allocateArenaMediumSentinel(int n, std::vector& arenas) {
+ for (int i = 0; i < n; i++) {
+ arenas.emplace_back();
+ auto* p = new (arenas.back()) uint8_t[600];
+ escape(p);
+ }
+ return reinterpret_cast(&allocateArenaMediumSentinel);
+}
+
+force_noinline void* allocateArenaHugeSentinel(int n, std::vector& arenas) {
+ for (int i = 0; i < n; i++) {
+ arenas.emplace_back();
+ auto* p = new (arenas.back()) uint8_t[100000];
+ escape(p);
+ }
+ return reinterpret_cast(&allocateArenaHugeSentinel);
+}
+
+force_noinline void* allocateArenaSmallSentinel(int n, std::vector& arenas) {
+ for (int i = 0; i < n; i++) {
+ arenas.emplace_back();
+ auto* p = new (arenas.back()) uint8_t[64];
+ escape(p);
+ }
+ return reinterpret_cast(&allocateArenaSmallSentinel);
+}
+
+force_noinline void* allocateOperatorNewSentinel(int n, int k, std::vector& ptrs) {
+ for (int i = 0; i < n; i++) {
+ auto* p = new int[k];
+ escape(p);
+ ptrs.push_back(p);
+ }
+ return reinterpret_cast(&allocateOperatorNewSentinel);
+}
+
+force_noinline void* allocateFastAlloc32Sentinel(int n, std::vector& ptrs) {
+ for (int i = 0; i < n; i++) {
+ void* p = FastAllocator<32>::allocate();
+ escape(p);
+ ptrs.push_back(p);
+ }
+ return reinterpret_cast(&allocateFastAlloc32Sentinel);
+}
+
+force_noinline void releaseFastAlloc32(std::vector& ptrs) {
+ for (void* p : ptrs) {
+ FastAllocator<32>::release(p);
+ }
+ ptrs.clear();
+}
+
+struct AccountingSummary {
+ int sitesWithSentinelFrames = 0;
+ int64_t cumBytesSentinel = 0;
+ int64_t cumAllocsSentinel = 0;
+ int64_t liveBytesSentinel = 0;
+ int64_t liveCountSentinel = 0;
+ int totalSites = 0;
+};
+
+AccountingSummary collectAccounting(void* sentinel) {
+ AccountingSummary acc;
+ memTrackerForEachSite([&](const MemoryTrackerCallSite& s) {
+ acc.totalSites++;
+ bool touches = false;
+ for (int i = 0; i < s.exemplarFrameCount; i++) {
+ if (frameInside(s.exemplarFrames[i], sentinel)) {
+ touches = true;
+ break;
+ }
+ }
+ if (touches && s.cumulativeBytes > 0) {
+ acc.sitesWithSentinelFrames++;
+ acc.cumBytesSentinel += s.cumulativeBytes;
+ acc.cumAllocsSentinel += s.cumulativeAllocs;
+ acc.liveBytesSentinel += s.liveBytes;
+ acc.liveCountSentinel += s.liveCount;
+ }
+ });
+ return acc;
+}
+
+void dumpSitesForFailure(const char* tag) {
+ fprintf(stderr, "[%s] dumping all tracker sites:\n", tag);
+ memTrackerForEachSite([&](const MemoryTrackerCallSite& s) {
+ fprintf(stderr,
+ " fp=%016llx liveBytes=%lld liveCount=%lld cumBytes=%lld cumAllocs=%lld frames=",
+ (unsigned long long)s.fingerprint,
+ (long long)s.liveBytes,
+ (long long)s.liveCount,
+ (long long)s.cumulativeBytes,
+ (long long)s.cumulativeAllocs);
+ for (int i = 0; i < s.exemplarFrameCount; i++) {
+ fprintf(stderr, "%p ", s.exemplarFrames[i]);
+ }
+ fprintf(stderr, "\n");
+ });
+}
+
+class KnobOverride {
+public:
+ explicit KnobOverride(int inverse = 1) : prevInverse(FLOW_KNOBS->MEMORY_TRACKING_SAMPLE_INVERSE) {
+ auto* k = const_cast(FLOW_KNOBS);
+ k->MEMORY_TRACKING_SAMPLE_INVERSE = inverse;
+ }
+ ~KnobOverride() {
+ auto* k = const_cast(FLOW_KNOBS);
+ k->MEMORY_TRACKING_SAMPLE_INVERSE = prevInverse;
+ }
+
+private:
+ int prevInverse;
+};
+
+} // namespace
+
+TEST_CASE("/flow/MemoryTracker/coverage") {
+#ifndef __linux__
+ // captureFramesFP is a no-op stub on non-Linux (FP walking through libc
+ // can't be made reliable on macOS); tests that inspect captured frames
+ // have nothing to inspect. Skip cleanly. The tracker still compiles
+ // and the non-frame tests (offSwitch, freeOfUntrackedPtrIsNoop) still
+ // run.
+ return Void();
+#endif
+ // Sample everything, reset, run sentinels, check.
+ KnobOverride ko;
+ memTrackerResetForTest();
+
+ void* opNew = triggerOperatorNewSentinel(50, 4);
+ void* fastAlloc = triggerFastAllocSentinel(50);
+ void* arena = triggerArenaSentinel(50);
+
+ bool foundOpNew = false;
+ bool foundFastAlloc = false;
+ bool foundArena = false;
+ int siteCount = 0;
+ memTrackerForEachSite([&](const MemoryTrackerCallSite& s) {
+ siteCount++;
+ for (int i = 0; i < s.exemplarFrameCount; i++) {
+ if (frameInside(s.exemplarFrames[i], opNew)) {
+ foundOpNew = true;
+ }
+ if (frameInside(s.exemplarFrames[i], fastAlloc)) {
+ foundFastAlloc = true;
+ }
+ if (frameInside(s.exemplarFrames[i], arena)) {
+ foundArena = true;
+ }
+ }
+ });
+
+ if (!foundOpNew || !foundFastAlloc || !foundArena) {
+ fprintf(stderr,
+ "MemoryTracker/coverage: sites=%d opNewSentinel=%p fastAllocSentinel=%p arenaSentinel=%p\n",
+ siteCount,
+ opNew,
+ fastAlloc,
+ arena);
+ fprintf(stderr,
+ "MemoryTracker/coverage: foundOpNew=%d foundFastAlloc=%d foundArena=%d\n",
+ foundOpNew,
+ foundFastAlloc,
+ foundArena);
+ memTrackerForEachSite([&](const MemoryTrackerCallSite& s) {
+ fprintf(stderr,
+ " site fp=%016llx liveBytes=%lld cumAllocs=%lld frames=",
+ (unsigned long long)s.fingerprint,
+ (long long)s.liveBytes,
+ (long long)s.cumulativeAllocs);
+ for (int i = 0; i < s.exemplarFrameCount; i++) {
+ fprintf(stderr, "%p ", s.exemplarFrames[i]);
+ }
+ fprintf(stderr, "\n");
+ });
+ }
+
+ ASSERT(foundOpNew);
+ ASSERT(foundFastAlloc);
+ ASSERT(foundArena);
+ return Void();
+}
+
+TEST_CASE("/flow/MemoryTracker/offSwitch") {
+ // With sample inverse 0, no allocations are attributed. memTrackerResetForTest
+ // arms this thread's off-latch straight from the knob (as memTrackerInit does
+ // at startup), so even the first allocation short-circuits.
+ auto* k = const_cast(FLOW_KNOBS);
+ int prev = k->MEMORY_TRACKING_SAMPLE_INVERSE;
+ k->MEMORY_TRACKING_SAMPLE_INVERSE = 0;
+
+ memTrackerResetForTest();
+
+ for (int i = 0; i < 100; i++) {
+ auto* p = new int[4];
+ p[0] = i;
+ delete[] p;
+ }
+
+ int siteCount = 0;
+ memTrackerForEachSite([&](const MemoryTrackerCallSite&) { siteCount++; });
+ ASSERT_EQ(siteCount, 0);
+
+ // The enabled flag gates the free hot path: with sampling off it must be
+ // false, so memTrackerOnFree short-circuits before taking g_mtLock.
+ ASSERT(!g_memTrackerEnabled.value.load(std::memory_order_relaxed));
+
+ k->MEMORY_TRACKING_SAMPLE_INVERSE = prev;
+ return Void();
+}
+
+TEST_CASE("/flow/MemoryTracker/operatorNewHonorsNewHandler") {
+ // The global operator new override (fdbserver/GlobalNewDelete.cpp) must run
+ // the std::new_handler retry loop so an allocation failure reaches FDB's OOM
+ // path instead of throwing straight past it. (Where the override isn't linked,
+ // the standard library's operator new provides the same contract, so this
+ // still passes.)
+ static bool handlerRan;
+ handlerRan = false;
+ std::new_handler prev = std::set_new_handler([]() {
+ handlerRan = true;
+ throw std::bad_alloc(); // break the retry loop
+ });
+
+ bool caught = false;
+ try {
+ // volatile so the compiler can't fold the size and warn (-Walloc-size); malloc
+ // reliably fails for SIZE_MAX, driving the handler loop.
+ volatile std::size_t huge = std::numeric_limits::max();
+ void* p = ::operator new(huge);
+ escape(p);
+ } catch (const std::bad_alloc&) {
+ caught = true;
+ }
+ std::set_new_handler(prev);
+
+ ASSERT(handlerRan);
+ ASSERT(caught);
+ return Void();
+}
+
+TEST_CASE("/flow/MemoryTracker/samplingRate") {
+ // The reseed gap is uniform on [1, 2N-1] (mean N), so at inverse N the sampled
+ // fraction should be ~1/N. Wide bounds keep it non-flaky across RNG state.
+ constexpr int N = 10;
+ constexpr int ALLOCS = 200000;
+ KnobOverride ko(N);
+ memTrackerResetForTest();
+
+ std::vector ptrs;
+ ptrs.reserve(ALLOCS);
+ for (int i = 0; i < ALLOCS; i++) {
+ auto* p = new int[4];
+ escape(p);
+ ptrs.push_back(p);
+ }
+
+ int64_t sampled = 0;
+ memTrackerForEachSite([&](const MemoryTrackerCallSite& s) {
+ if (s.forceSampledCount == 0) {
+ sampled += s.cumulativeAllocs;
+ }
+ });
+
+ for (auto* p : ptrs) {
+ delete[] p;
+ }
+ ptrs.clear();
+
+ double frac = double(sampled) / ALLOCS;
+ ASSERT(frac > 0.06 && frac < 0.15); // expect ~0.1
+ memTrackerResetForTest();
+ return Void();
+}
+
+TEST_CASE("/flow/MemoryTracker/freeOfUntrackedPtrIsNoop") {
+ // memTrackerOnFree on a pointer the tracker never recorded must be a no-op.
+ KnobOverride ko;
+ memTrackerResetForTest();
+
+ int x = 0;
+ memTrackerOnFree(&x); // not in any table
+ memTrackerOnFree(nullptr);
+
+ int siteCount = 0;
+ memTrackerForEachSite([&](const MemoryTrackerCallSite&) { siteCount++; });
+ ASSERT_EQ(siteCount, 0);
+ return Void();
+}
+
+TEST_CASE("/flow/MemoryTracker/cumulativeIsMonotonic") {
+#ifndef __linux__
+ return Void(); // see /coverage for rationale
+#endif
+ // liveCount must return to ~0 after we free everything we allocated;
+ // cumulativeAllocs must NOT decrement.
+ KnobOverride ko;
+ memTrackerResetForTest();
+
+ void* sentinel = triggerOperatorNewSentinel(100, 8);
+
+ int64_t maxCumulative = 0;
+ int64_t finalLive = 0;
+ memTrackerForEachSite([&](const MemoryTrackerCallSite& s) {
+ for (int i = 0; i < s.exemplarFrameCount; i++) {
+ if (frameInside(s.exemplarFrames[i], sentinel)) {
+ if (s.cumulativeAllocs > maxCumulative) {
+ maxCumulative = s.cumulativeAllocs;
+ }
+ finalLive += s.liveCount;
+ }
+ }
+ });
+
+ ASSERT(maxCumulative >= 100);
+ ASSERT_EQ(finalLive, 0); // every alloc was paired with delete
+ return Void();
+}
+
+TEST_CASE("/flow/MemoryTracker/estimateScaling") {
+ // End-to-end estimate check. With a fixed inverse N > 1 and no force-sampled
+ // blocks, every sample at a site carries weight N, so the site's estimated
+ // usage must be *exactly* N times its raw sampled counters. This verifies
+ // the reported Est* numbers without depending on which specific allocations
+ // happened to be sampled. Runs on all platforms (no frame inspection).
+ constexpr int N = 8;
+ KnobOverride ko(N);
+ memTrackerResetForTest();
+
+ // Small allocations far below the force-sample threshold, so none
+ // are force-sampled and every sampled block gets weight N.
+ std::vector ptrs;
+ ptrs.reserve(5000);
+ for (int i = 0; i < 5000; i++) {
+ auto* p = new int[4];
+ escape(p);
+ ptrs.push_back(p);
+ }
+
+ int checked = 0;
+ int64_t rawLiveBefore = 0;
+ memTrackerForEachSite([&](const MemoryTrackerCallSite& s) {
+ if (s.forceSampledCount != 0) {
+ return; // ignore any incidental force-sampled site (weight 1, not N)
+ }
+ ASSERT_EQ(s.estCumulativeBytes, s.cumulativeBytes * N);
+ ASSERT_EQ(s.estCumulativeAllocs, s.cumulativeAllocs * N);
+ ASSERT_EQ(s.estLiveBytes, s.liveBytes * N);
+ ASSERT_EQ(s.estLiveCount, s.liveCount * N);
+ ASSERT_EQ(s.estPeakBytes, s.peakBytes * N);
+ rawLiveBefore += s.liveBytes;
+ checked++;
+ });
+ ASSERT(checked > 0);
+ ASSERT(rawLiveBefore > 0);
+
+ for (auto* p : ptrs) {
+ delete[] p;
+ }
+ ptrs.clear();
+
+ // Symmetric debit: the per-site scaling invariant must still hold after the
+ // frees (each free debits the estimate by exactly weight×size), and the live
+ // total must have dropped. We check the invariant rather than "live == 0"
+ // because incidental still-live allocations (e.g. the ptrs vector's own
+ // backing buffer) legitimately remain tracked.
+ int64_t rawLiveAfter = 0;
+ int64_t estLiveAfter = 0;
+ memTrackerForEachSite([&](const MemoryTrackerCallSite& s) {
+ if (s.forceSampledCount != 0) {
+ return;
+ }
+ ASSERT_EQ(s.estLiveBytes, s.liveBytes * N);
+ rawLiveAfter += s.liveBytes;
+ estLiveAfter += s.estLiveBytes;
+ });
+ ASSERT_EQ(estLiveAfter, rawLiveAfter * N);
+ ASSERT(rawLiveAfter < rawLiveBefore); // the freed blocks were debited
+
+ memTrackerResetForTest();
+ return Void();
+}
+
+// ---------------------------------------------------------------------------
+// Accounting tests. The "sites with the sentinel's frames" assertion is the
+// main check here.
+
+TEST_CASE("/flow/MemoryTracker/fastAlloc32Accounting") {
+#ifndef __linux__
+ return Void(); // see /coverage for rationale
+#endif
+ KnobOverride ko;
+ constexpr int N = 30;
+
+ std::vector ptrs;
+ ptrs.reserve(N);
+
+ memTrackerResetForTest();
+ void* sentinel = allocateFastAlloc32Sentinel(N, ptrs);
+
+ auto pre = collectAccounting(sentinel);
+ if (pre.sitesWithSentinelFrames != 1) {
+ dumpSitesForFailure("fastAlloc32Accounting/post-alloc");
+ }
+ ASSERT_EQ(pre.sitesWithSentinelFrames, 1);
+ ASSERT_EQ(pre.cumAllocsSentinel, N);
+ ASSERT_EQ(pre.liveCountSentinel, N);
+ ASSERT_EQ(pre.cumBytesSentinel, int64_t(N) * 32);
+ ASSERT_EQ(pre.liveBytesSentinel, pre.cumBytesSentinel);
+ // Global totals are intentionally not asserted: at inverse=1 a foreign-thread
+ // allocation in the window would break a strict global equality (flaky at
+ // Joshua scale); the sentinel-scoped checks above pin the regression.
+
+ releaseFastAlloc32(ptrs);
+
+ auto post = collectAccounting(sentinel);
+ ASSERT_EQ(post.liveBytesSentinel, 0);
+ ASSERT_EQ(post.liveCountSentinel, 0);
+ // Global live totals intentionally not asserted (flaky at inverse=1; see above).
+ ASSERT_EQ(post.cumAllocsSentinel, N); // cumulative never decrements
+ return Void();
+}
+
+TEST_CASE("/flow/MemoryTracker/arenaSmallAccounting") {
+#ifndef __linux__
+ return Void(); // see /coverage for rationale
+#endif
+ KnobOverride ko;
+ constexpr int N = 30;
+
+ std::vector arenas;
+ arenas.reserve(N);
+
+ memTrackerResetForTest();
+ void* sentinel = allocateArenaSmallSentinel(N, arenas);
+
+ auto pre = collectAccounting(sentinel);
+ if (pre.sitesWithSentinelFrames != 1) {
+ dumpSitesForFailure("arenaSmallAccounting/post-alloc");
+ }
+ ASSERT_EQ(pre.sitesWithSentinelFrames, 1);
+ ASSERT_EQ(pre.cumAllocsSentinel, N);
+ ASSERT_EQ(pre.liveCountSentinel, N);
+ ASSERT_EQ(pre.liveBytesSentinel, pre.cumBytesSentinel);
+ // Global totals are intentionally not asserted: at inverse=1 a foreign-thread
+ // allocation in the window would break a strict global equality (flaky at
+ // Joshua scale); the sentinel-scoped checks above pin the regression.
+
+ arenas.clear();
+
+ auto post = collectAccounting(sentinel);
+ ASSERT_EQ(post.liveBytesSentinel, 0);
+ ASSERT_EQ(post.liveCountSentinel, 0);
+ // Global live totals intentionally not asserted (flaky at inverse=1; see above).
+ ASSERT_EQ(post.cumAllocsSentinel, N);
+ return Void();
+}
+
+TEST_CASE("/flow/MemoryTracker/arenaMediumAccounting") {
+#ifndef __linux__
+ return Void(); // see /coverage for rationale
+#endif
+ // Make sure arenas aren't counted twice, once due to their direct
+ // instrumentation and a second time due to their use of operator new.
+ KnobOverride ko;
+ constexpr int N = 30;
+
+ std::vector arenas;
+ arenas.reserve(N);
+
+ memTrackerResetForTest();
+ void* sentinel = allocateArenaMediumSentinel(N, arenas);
+
+ auto pre = collectAccounting(sentinel);
+ if (pre.sitesWithSentinelFrames != 1) {
+ dumpSitesForFailure("arenaMediumAccounting/post-alloc");
+ }
+ ASSERT_EQ(pre.sitesWithSentinelFrames, 1);
+ ASSERT_EQ(pre.cumAllocsSentinel, N);
+ ASSERT_EQ(pre.liveCountSentinel, N);
+ ASSERT_EQ(pre.liveBytesSentinel, pre.cumBytesSentinel);
+ // Global totals intentionally not asserted (flaky at inverse=1; see above).
+
+ arenas.clear();
+
+ auto post = collectAccounting(sentinel);
+ ASSERT_EQ(post.liveBytesSentinel, 0);
+ ASSERT_EQ(post.liveCountSentinel, 0);
+ // Global live totals intentionally not asserted (flaky at inverse=1; see above).
+ ASSERT_EQ(post.cumAllocsSentinel, N);
+ return Void();
+}
+
+TEST_CASE("/flow/MemoryTracker/arenaHugeAccounting") {
+#ifndef __linux__
+ return Void(); // see /coverage for rationale
+#endif
+ // Verify the huge Arena-block path (reqSize >= LARGE), including that a block is
+ // tracked once (not double-counted by both the explicit Arena hook and the inner
+ // operator new[]). This is the deepest tracked call chain, and under a real
+ // (non-simulation) network its frame-pointer backtrace is unreliable — the
+ // best-effort walker may, run to run and even alloc to alloc, fail to climb to
+ // the test's frame or attribute the blocks to varying fingerprints. So instead of
+ // the sentinel-frame approach the other *Accounting tests use, we identify the
+ // huge blocks by their unmistakable ~100 KB size signature: the recorded size is
+ // correct regardless of which frames were captured, and no incidental or
+ // foreign-thread allocation comes anywhere near this large.
+ KnobOverride ko;
+ constexpr int N = 10;
+ constexpr int64_t BLOCK = 100000;
+ constexpr int64_t HUGE_MIN = 90000; // mean bytes/alloc of a huge site; nothing else is this big
+
+ std::vector arenas;
+ arenas.reserve(N);
+
+ memTrackerResetForTest();
+ allocateArenaHugeSentinel(N, arenas);
+
+ int64_t cumAllocs = 0, cumBytes = 0, liveBytes = 0, liveCount = 0;
+ memTrackerForEachSite([&](const MemoryTrackerCallSite& s) {
+ if (s.cumulativeAllocs > 0 && s.cumulativeBytes / s.cumulativeAllocs >= HUGE_MIN) {
+ cumAllocs += s.cumulativeAllocs;
+ cumBytes += s.cumulativeBytes;
+ liveBytes += s.liveBytes;
+ liveCount += s.liveCount;
+ }
+ });
+ // Exactly N huge blocks, each tracked once (double-tracking would show as 2N),
+ // all currently live, with live bytes == cumulative bytes (nothing freed yet).
+ ASSERT_EQ(cumAllocs, N);
+ ASSERT_EQ(liveCount, N);
+ ASSERT_EQ(liveBytes, cumBytes);
+ ASSERT(cumBytes >= int64_t(N) * BLOCK);
+
+ arenas.clear();
+
+ int64_t cumAllocsPost = 0, liveBytesPost = 0, liveCountPost = 0;
+ memTrackerForEachSite([&](const MemoryTrackerCallSite& s) {
+ if (s.cumulativeAllocs > 0 && s.cumulativeBytes / s.cumulativeAllocs >= HUGE_MIN) {
+ cumAllocsPost += s.cumulativeAllocs;
+ liveBytesPost += s.liveBytes;
+ liveCountPost += s.liveCount;
+ }
+ });
+ // After freeing, the huge blocks are debited: live returns to 0, cumulative persists.
+ ASSERT_EQ(liveBytesPost, 0);
+ ASSERT_EQ(liveCountPost, 0);
+ ASSERT_EQ(cumAllocsPost, N);
+ return Void();
+}
+
+TEST_CASE("/flow/MemoryTracker/operatorNewAccounting") {
+#ifndef __linux__
+ return Void(); // see /coverage for rationale
+#endif
+ KnobOverride ko;
+ constexpr int N = 30;
+ constexpr int K = 8; // new int[8] -> 32 bytes; int is trivial so no array cookie
+
+ std::vector ptrs;
+ ptrs.reserve(N);
+
+ memTrackerResetForTest();
+ void* sentinel = allocateOperatorNewSentinel(N, K, ptrs);
+
+ auto pre = collectAccounting(sentinel);
+ if (pre.sitesWithSentinelFrames != 1) {
+ dumpSitesForFailure("operatorNewAccounting/post-alloc");
+ }
+ ASSERT_EQ(pre.sitesWithSentinelFrames, 1);
+ ASSERT_EQ(pre.cumAllocsSentinel, N);
+ ASSERT_EQ(pre.liveCountSentinel, N);
+ ASSERT_EQ(pre.cumBytesSentinel, static_cast(N) * K * static_cast(sizeof(int)));
+ ASSERT_EQ(pre.liveBytesSentinel, pre.cumBytesSentinel);
+ // Global totals intentionally not asserted (flaky at inverse=1; see above).
+
+ for (auto* p : ptrs) {
+ delete[] p;
+ }
+ ptrs.clear();
+
+ auto post = collectAccounting(sentinel);
+ ASSERT_EQ(post.liveBytesSentinel, 0);
+ ASSERT_EQ(post.liveCountSentinel, 0);
+ // Global live totals intentionally not asserted (flaky at inverse=1; see above).
+ ASSERT_EQ(post.cumAllocsSentinel, N);
+ return Void();
+}
+
+TEST_CASE("/flow/MemoryTracker/failOpenOnMetadataAllocFailure") {
+ // The tracker must fail open: if its own metadata allocation throws, the
+ // underlying user allocation still succeeds and tracking recovers on this
+ // thread afterward (the reentrancy guard is restored, not leaked). Runs on all
+ // platforms — no frame inspection.
+ KnobOverride ko; // inverse = 1: sample every allocation
+ memTrackerResetForTest();
+
+ // Arm the one-shot; it is consumed by the next sampled allocation, which throws
+ // inside the tracker. new/delete must not observe that exception.
+ memTrackerFailNextSampleForTest();
+ int* p = new int[4];
+ ASSERT(p != nullptr);
+ p[0] = 42;
+ int observed = p[0];
+ delete[] p;
+ ASSERT_EQ(observed, 42);
+
+ // Tracking must still work after the injected failure.
+ memTrackerResetForTest();
+ std::vector ptrs;
+ ptrs.reserve(8);
+ for (int i = 0; i < 8; i++) {
+ auto* q = new int[4];
+ escape(q);
+ ptrs.push_back(q);
+ }
+ int siteCount = 0;
+ memTrackerForEachSite([&](const MemoryTrackerCallSite&) { siteCount++; });
+ for (auto* q : ptrs) {
+ delete[] q;
+ }
+ ASSERT(siteCount > 0);
+ return Void();
+}
+
+TEST_CASE("/flow/MemoryTracker/initEnablesFromKnob") {
+ // Exercises the *production* enablement path (memTrackerInit), not the test-only
+ // reset — the reviewer noted that memTrackerResetForTest masks the startup race.
+ // memTrackerInit must set the global enabled flag and this thread's fast-path
+ // off-latch straight from MEMORY_TRACKING_SAMPLE_INVERSE, and (the regression)
+ // must re-arm a thread that a prior off-configuration had already latched off.
+ auto* k = const_cast(FLOW_KNOBS);
+ int prev = k->MEMORY_TRACKING_SAMPLE_INVERSE;
+
+ // Sampling off: init publishes disabled and latches this thread off. This is the
+ // state an early startup allocation used to get stuck in before init existed.
+ k->MEMORY_TRACKING_SAMPLE_INVERSE = 0;
+ memTrackerInit();
+ ASSERT(!g_memTrackerEnabled.value.load(std::memory_order_relaxed));
+ ASSERT(gMemTrackerOff); // alloc hot path short-circuits
+
+ // Knobs now configured with sampling on: init must re-arm THIS thread. The bug
+ // was that nothing re-armed the network thread once it latched off before the
+ // knobs were ready, so sampling stayed dead for the life of the process.
+ k->MEMORY_TRACKING_SAMPLE_INVERSE = 8;
+ memTrackerInit();
+ ASSERT(g_memTrackerEnabled.value.load(std::memory_order_relaxed));
+ ASSERT(!gMemTrackerOff); // re-armed: alloc hot path now reaches the sampler
+
+ // And the reverse transition: a subsequent off-configuration re-latches it.
+ k->MEMORY_TRACKING_SAMPLE_INVERSE = 0;
+ memTrackerInit();
+ ASSERT(!g_memTrackerEnabled.value.load(std::memory_order_relaxed));
+ ASSERT(gMemTrackerOff);
+
+ k->MEMORY_TRACKING_SAMPLE_INVERSE = prev;
+ memTrackerInit(); // restore tracker state from the baseline knob for later tests
+ return Void();
+}
+
+#endif // FDB_MEMORY_TRACKER
diff --git a/fdbserver/bench/BenchMain.cpp b/fdbserver/bench/BenchMain.cpp
new file mode 100644
index 0000000000..02e2270439
--- /dev/null
+++ b/fdbserver/bench/BenchMain.cpp
@@ -0,0 +1,29 @@
+/*
+ * BenchMain.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Most microbenchmarks belong in flow/bench; this fdbserver-local benchmark
+// binary exists only for benchmarks that must link against fdbserver-only code.
+// See BenchMemoryTracker.cpp for why that one lives here.
+
+#include "flow/BenchMain.h"
+
+int main(int argc, char** argv) {
+ return runBenchmarks(argc, argv);
+}
diff --git a/fdbserver/bench/BenchMemoryTracker.cpp b/fdbserver/bench/BenchMemoryTracker.cpp
new file mode 100644
index 0000000000..11cf424c27
--- /dev/null
+++ b/fdbserver/bench/BenchMemoryTracker.cpp
@@ -0,0 +1,167 @@
+/*
+ * BenchMemoryTracker.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Microbenchmarks for the per-call-site memory tracker (see
+// design/memory-tracker.md, Testing Considerations -> Microbenchmarks).
+//
+// Two questions, both about "is it cheap enough to leave on?" (R0):
+// * off-state cost — the always-compiled hooks with sampling disabled;
+// * enabled-state cost — hooks at the envisioned production 1% rate
+// and the pessimal every-allocation rate, which includes both the
+// sampled-alloc slow path and the per-free lock+probe (the
+// dominant enabled-state cost).
+//
+// Run: bin/fdbserver_bench --benchmark_filter=memtracker
+//
+// This bench lives under fdbserver/ (not flow/) and its CMake compiles
+// fdbserver/GlobalNewDelete.cpp into the executable, so the real global
+// operator new/delete override is active here and bench_memtracker_operator_new
+// actually exercises the tracker. flow_bench links only flow (no override), so
+// the operator-new path could not be measured there.
+//
+// FLOW_KNOBS points at the process-default (non-simulated) bootstrap knobs in
+// fdbserver_bench, so MEMORY_TRACKING_SAMPLE_INVERSE starts at 0 (off); we drive
+// it per benchmark via const_cast, exactly like the unit tests do.
+
+#include "benchmark/benchmark.h"
+
+#include "flow/Arena.h"
+#include "flow/FastAlloc.h"
+#include "flow/Knobs.h"
+#include "flow/MemoryTracker.h"
+
+#include
+
+namespace {
+
+constexpr int kSize = 64;
+
+// Set the sample-inverse knob (0=off, N=1-in-N) and clear tracker state so the
+// run starts clean. Returns the previous inverse for restoration.
+int setInverseAndReset(int inverse) {
+ auto* k = const_cast(FLOW_KNOBS);
+ int prev = k->MEMORY_TRACKING_SAMPLE_INVERSE;
+ k->MEMORY_TRACKING_SAMPLE_INVERSE = inverse;
+ memTrackerResetForTest();
+ return prev;
+}
+
+} // namespace
+
+// Baseline: raw libc malloc/free. std::malloc is NOT hooked (we override
+// operator new, not libc malloc), so this is the tracker-free reference the
+// operator-new benchmark is compared against.
+static void bench_memtracker_malloc_free(benchmark::State& state) {
+ for (auto _ : state) {
+ void* p = std::malloc(kSize);
+ benchmark::DoNotOptimize(p);
+ std::free(p);
+ }
+ state.SetItemsProcessed(state.iterations());
+}
+
+// End-to-end cost of a hooked allocation: global operator new[]/delete[] (which
+// fire memTrackerOnAlloc/OnFree) at sample inverse Arg(0). 0 = off, 100 = prod
+// 1%, 1 = every allocation. Compare Arg(0) against bench_memtracker_malloc_free
+// for the disabled-hook cost, and Arg(100)/Arg(1) against Arg(0) for sampling.
+static void bench_memtracker_operator_new(benchmark::State& state) {
+ int prev = setInverseAndReset(state.range(0));
+ for (auto _ : state) {
+ char* p = new char[kSize];
+ benchmark::DoNotOptimize(p);
+ delete[] p;
+ }
+ state.SetItemsProcessed(state.iterations());
+ setInverseAndReset(prev); // restore so later benchmarks aren't sampled
+}
+
+// Isolated tracker-hook cost: call memTrackerOnAlloc/OnFree directly on one
+// preallocated buffer, with no real allocation in the loop, so only the
+// tracker's own work is measured. At inverse>0 with live-tracking on, every
+// OnFree still takes the global lock and probes the live table (the dominant
+// enabled-state cost), while ~1/inverse of the OnAlloc calls take the sampling
+// slow path (frame walk + table insert).
+//
+// This is doing a stack unwind against the same stack, and is going
+// to hit the same hash table entries each iteration, so this is definitely
+// a best-case estimate.
+static void bench_memtracker_hooks(benchmark::State& state) {
+ int prev = setInverseAndReset(state.range(0));
+ void* p = std::malloc(kSize);
+ for (auto _ : state) {
+ memTrackerOnAlloc(p, kSize);
+ memTrackerOnFree(p);
+ }
+ state.SetItemsProcessed(state.iterations());
+ std::free(p);
+ setInverseAndReset(prev);
+}
+
+BENCHMARK(bench_memtracker_malloc_free);
+BENCHMARK(bench_memtracker_operator_new)->Arg(0)->Arg(100)->Arg(1);
+BENCHMARK(bench_memtracker_hooks)->Arg(0)->Arg(100)->Arg(1);
+
+// --- Per-path unweighted overhead (for the FDB_MEMORY_TRACKER compile gate) ---
+// Plain allocation loops with no knob manipulation: in a default build
+// (FDB_MEMORY_TRACKER=1) they run with the tracker present but sampling off; in a
+// FDB_MEMORY_TRACKER=0 build the tracker code is absent. Diffing the two builds
+// gives each path's always-compiled off-state cost per op, unweighted by how
+// often the path is actually taken at runtime.
+
+// operator new[]/delete[] at size Arg(0): exercises GlobalNewDelete.cpp (our
+// override when FDB_MEMORY_TRACKER, else libc++'s operator new).
+static void bench_path_operator_new(benchmark::State& state) {
+ size_t n = state.range(0);
+ for (auto _ : state) {
+ char* p = new char[n];
+ benchmark::DoNotOptimize(p);
+ delete[] p;
+ }
+ state.SetItemsProcessed(state.iterations());
+}
+
+// FastAllocator allocate/release: exercises the flow/FastAlloc.cpp hook.
+template
+static void bench_path_fastalloc(benchmark::State& state) {
+ for (auto _ : state) {
+ void* p = FastAllocator::allocate();
+ benchmark::DoNotOptimize(p);
+ FastAllocator::release(p);
+ }
+ state.SetItemsProcessed(state.iterations());
+}
+
+// Arena block create/destroy: exercises the flow/Arena.cpp hook. Arg(0) is the
+// user allocation size, which selects the block class (medium vs huge).
+static void bench_path_arena(benchmark::State& state) {
+ int n = state.range(0);
+ for (auto _ : state) {
+ Arena a;
+ uint8_t* p = new (a) uint8_t[n];
+ benchmark::DoNotOptimize(p);
+ }
+ state.SetItemsProcessed(state.iterations());
+}
+
+BENCHMARK(bench_path_operator_new)->Arg(64)->Arg(96)->Arg(256)->Arg(100000);
+BENCHMARK_TEMPLATE(bench_path_fastalloc, 64);
+BENCHMARK_TEMPLATE(bench_path_fastalloc, 96);
+BENCHMARK_TEMPLATE(bench_path_fastalloc, 256);
+BENCHMARK(bench_path_arena)->Arg(600)->Arg(100000);
diff --git a/fdbserver/bench/CMakeLists.txt b/fdbserver/bench/CMakeLists.txt
new file mode 100644
index 0000000000..f505832d9a
--- /dev/null
+++ b/fdbserver/bench/CMakeLists.txt
@@ -0,0 +1,28 @@
+# Most bench stuff is in flow/bench. This is in fdbserver because it links with
+# code we specifically do not want in flow/ (to avoid pulling into clients).
+
+include(FDBBenchmark)
+
+fdb_find_sources(FDBSERVER_BENCH_SRCS)
+
+# Compile fdbserver/GlobalNewDelete.cpp directly into this benchmark executable.
+# It defines the global operator new/delete overrides that route through the
+# memory tracker; as a strong definition in the executable link it wins over
+# libc++'s weak one and is active process-wide here, exactly as in fdbserver.
+# Without it the bench would link libc++'s operator new and the operator-new
+# microbench would measure the unhooked allocator (its Arg(100)/Arg(1) rows
+# would equal Arg(0)). It only needs flow (the tracker hooks it calls), not the
+# rest of the fdbserver dependency graph.
+add_flow_target(EXECUTABLE NAME fdbserver_bench
+ SRCS ${FDBSERVER_BENCH_SRCS}
+ ADDL_SRCS "${CMAKE_CURRENT_SOURCE_DIR}/../GlobalNewDelete.cpp")
+
+fdb_setup_googlebenchmark()
+
+target_include_directories(
+ fdbserver_bench
+ PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}"
+ "${CMAKE_CURRENT_SOURCE_DIR}/.."
+ "${CMAKE_CURRENT_SOURCE_DIR}/include")
+
+target_link_libraries(fdbserver_bench PRIVATE Threads::Threads fdb_google_benchmark flow)
diff --git a/fdbserver/fdbserver.cpp b/fdbserver/fdbserver.cpp
index f969c577c4..626a3be1cb 100644
--- a/fdbserver/fdbserver.cpp
+++ b/fdbserver/fdbserver.cpp
@@ -77,6 +77,7 @@
#include "flow/ProtocolVersion.h"
#include "SimpleOpt/SimpleOpt.h"
#include "flow/SystemMonitor.h"
+#include "flow/MemoryTracker.h"
#include "flow/TLSConfig.h"
#include "fdbclient/Tracing.h"
#include "flow/WriteOnlySet.h"
@@ -705,54 +706,9 @@ static void printUsage(const char* name, bool devhelp) {
extern bool g_crashOnError;
-#if defined(ALLOC_INSTRUMENTATION) || defined(ALLOC_INSTRUMENTATION_STDOUT)
-void* operator new(std::size_t size) {
- void* p = malloc(size);
- if (!p)
- throw std::bad_alloc();
- recordAllocation(p, size);
- return p;
-}
-void operator delete(void* ptr) throw() {
- recordDeallocation(ptr);
- free(ptr);
-}
-
-// scalar, nothrow new and it matching delete
-void* operator new(std::size_t size, const std::nothrow_t&) throw() {
- void* p = malloc(size);
- recordAllocation(p, size);
- return p;
-}
-void operator delete(void* ptr, const std::nothrow_t&) throw() {
- recordDeallocation(ptr);
- free(ptr);
-}
-
-// array throwing new and matching delete[]
-void* operator new[](std::size_t size) {
- void* p = malloc(size);
- if (!p)
- throw std::bad_alloc();
- recordAllocation(p, size);
- return p;
-}
-void operator delete[](void* ptr) throw() {
- recordDeallocation(ptr);
- free(ptr);
-}
-
-// array, nothrow new and matching delete[]
-void* operator new[](std::size_t size, const std::nothrow_t&) throw() {
- void* p = malloc(size);
- recordAllocation(p, size);
- return p;
-}
-void operator delete[](void* ptr, const std::nothrow_t&) throw() {
- recordDeallocation(ptr);
- free(ptr);
-}
-#endif
+// The global operator new / operator delete replacements (both the legacy
+// ALLOC_INSTRUMENTATION accounting hooks and the sampled memory tracker) live in
+// fdbserver/GlobalNewDelete.cpp.
Optional checkBuggifyOverride(const char* testFile) {
std::ifstream ifs;
@@ -1955,6 +1911,13 @@ int main(int argc, char* argv[]) {
// Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs
initializeServerKnobs(Randomize::True, role == ServerRole::Simulation ? IsSimulated::True : IsSimulated::False);
+ // Knobs are now final; initialize the sampled memory tracker from them on
+ // this (soon-to-be network) thread, before any serving role starts. Reading
+ // the sample-inverse knob explicitly here — rather than inferring it from
+ // the first allocation — keeps early startup allocations from latching the
+ // tracker off before the knobs were configured. See design/memory-tracker.md.
+ memTrackerInit();
+
// evictionPolicyStringToEnum will throw an exception if the string is not recognized as a valid
EvictablePageCache::evictionPolicyStringToEnum(FLOW_KNOBS->CACHE_EVICTION_POLICY);
diff --git a/fdbserver/workloads/UnitTests.cpp b/fdbserver/workloads/UnitTests.cpp
index cc7f828501..84caea8e14 100644
--- a/fdbserver/workloads/UnitTests.cpp
+++ b/fdbserver/workloads/UnitTests.cpp
@@ -35,6 +35,7 @@ void forceLinkJsonWebKeySetTests();
void forceLinkVersionVectorTests();
void forceLinkRESTClientTests();
void forceLinkRESTUtilsTests();
+void forceLinkMemoryTrackerTests();
void forceLinkCompressedIntTests();
void forceLinkAtomicTests();
void forceLinkIdempotencyIdTests();
@@ -115,6 +116,7 @@ struct UnitTestWorkload : TestWorkload {
forceLinkVersionVectorTests();
forceLinkRESTClientTests();
forceLinkRESTUtilsTests();
+ forceLinkMemoryTrackerTests();
forceLinkCompressedIntTests();
forceLinkAtomicTests();
forceLinkIdempotencyIdTests();
diff --git a/flow/Arena.cpp b/flow/Arena.cpp
index ed6170c17a..e31c284686 100644
--- a/flow/Arena.cpp
+++ b/flow/Arena.cpp
@@ -19,6 +19,7 @@
*/
#include "flow/Arena.h"
+#include "flow/MemoryTracker.h"
#include "flow/ScopeExit.h"
#include "flow/SimpleCounter.h"
#include "flow/UnitTest.h"
@@ -460,36 +461,55 @@ ArenaBlock* ArenaBlock::create(int dataSize, Reference& next) {
b = (ArenaBlock*)FastAllocator<256>::allocate();
b->bigSize = 256;
INSTRUMENT_ALLOCATE("Arena256");
- } else if (reqSize <= 512) {
- b = (ArenaBlock*)allocateAndMaybeKeepalive(512);
- b->bigSize = 512;
- INSTRUMENT_ALLOCATE("Arena512");
- } else if (reqSize <= 1024) {
- b = (ArenaBlock*)allocateAndMaybeKeepalive(1024);
- b->bigSize = 1024;
- INSTRUMENT_ALLOCATE("Arena1024");
- } else if (reqSize <= 2048) {
- b = (ArenaBlock*)allocateAndMaybeKeepalive(2048);
- b->bigSize = 2048;
- INSTRUMENT_ALLOCATE("Arena2048");
- } else if (reqSize <= 4096) {
- b = (ArenaBlock*)allocateAndMaybeKeepalive(4096);
- b->bigSize = 4096;
- INSTRUMENT_ALLOCATE("Arena4096");
} else {
- b = (ArenaBlock*)allocateAndMaybeKeepalive(8192);
- b->bigSize = 8192;
- INSTRUMENT_ALLOCATE("Arena8192");
+ // Suppress the operator-new[] memory-tracker hook around the
+ // underlying `new uint8_t[]`; the explicit memTrackerOnAlloc
+ // below is the sole hook for these blocks. Without this
+ // guard the same pointer would be tracked twice under two
+ // different fingerprints.
+ MemTrackerSuppress _suppress;
+ if (reqSize <= 512) {
+ b = (ArenaBlock*)allocateAndMaybeKeepalive(512);
+ b->bigSize = 512;
+ INSTRUMENT_ALLOCATE("Arena512");
+ } else if (reqSize <= 1024) {
+ b = (ArenaBlock*)allocateAndMaybeKeepalive(1024);
+ b->bigSize = 1024;
+ INSTRUMENT_ALLOCATE("Arena1024");
+ } else if (reqSize <= 2048) {
+ b = (ArenaBlock*)allocateAndMaybeKeepalive(2048);
+ b->bigSize = 2048;
+ INSTRUMENT_ALLOCATE("Arena2048");
+ } else if (reqSize <= 4096) {
+ b = (ArenaBlock*)allocateAndMaybeKeepalive(4096);
+ b->bigSize = 4096;
+ INSTRUMENT_ALLOCATE("Arena4096");
+ } else {
+ b = (ArenaBlock*)allocateAndMaybeKeepalive(8192);
+ b->bigSize = 8192;
+ INSTRUMENT_ALLOCATE("Arena8192");
+ }
}
b->totalSizeEstimate = b->bigSize;
b->tinySize = b->tinyUsed = NOT_TINY;
b->bigUsed = sizeof(ArenaBlock);
b->secure = 0;
+ // Block-level attribution for >256 sizes (sizes <=256 use FastAllocator,
+ // which fires its own memTrackerOnAlloc hook).
+ if (b->bigSize > 256) {
+ memTrackerOnAlloc(b, b->bigSize);
+ }
} else {
#ifdef ALLOC_INSTRUMENTATION
allocInstr["ArenaHugeKB"].alloc((reqSize + 1023) >> 10);
#endif
- b = (ArenaBlock*)allocateAndMaybeKeepalive(reqSize);
+ {
+ // Suppress the operator-new[] hook so the explicit
+ // memTrackerOnAlloc below is the sole tracker for huge
+ // arena blocks (see comment in the small-block branch).
+ MemTrackerSuppress _suppress;
+ b = (ArenaBlock*)allocateAndMaybeKeepalive(reqSize);
+ }
b->tinySize = b->tinyUsed = NOT_TINY;
b->bigSize = reqSize;
b->totalSizeEstimate = b->bigSize;
@@ -505,6 +525,9 @@ ArenaBlock* ArenaBlock::create(int dataSize, Reference& next) {
}
#endif
g_hugeArenaMemory.fetch_add(reqSize);
+ // Block-level attribution for huge arena blocks. allocateAndMaybeKeepalive
+ // bypasses FastAllocator, so this is the only hook for these blocks.
+ memTrackerOnAlloc(b, reqSize);
// If the new block has less free space than the old block, make the old block depend on it
if (next && !next->isTiny() && next->unused() >= reqSize - dataSize) {
@@ -585,26 +608,50 @@ void ArenaBlock::destroyLeaf() {
FastAllocator<256>::release(this);
INSTRUMENT_RELEASE("Arena256");
} else if (bigSize <= 512) {
- freeOrMaybeKeepalive(this);
+ memTrackerOnFree(this);
+ {
+ MemTrackerSuppress _suppress;
+ freeOrMaybeKeepalive(this);
+ }
INSTRUMENT_RELEASE("Arena512");
} else if (bigSize <= 1024) {
- freeOrMaybeKeepalive(this);
+ memTrackerOnFree(this);
+ {
+ MemTrackerSuppress _suppress;
+ freeOrMaybeKeepalive(this);
+ }
INSTRUMENT_RELEASE("Arena1024");
} else if (bigSize <= 2048) {
- freeOrMaybeKeepalive(this);
+ memTrackerOnFree(this);
+ {
+ MemTrackerSuppress _suppress;
+ freeOrMaybeKeepalive(this);
+ }
INSTRUMENT_RELEASE("Arena2048");
} else if (bigSize <= 4096) {
- freeOrMaybeKeepalive(this);
+ memTrackerOnFree(this);
+ {
+ MemTrackerSuppress _suppress;
+ freeOrMaybeKeepalive(this);
+ }
INSTRUMENT_RELEASE("Arena4096");
} else if (bigSize <= 8192) {
- freeOrMaybeKeepalive(this);
+ memTrackerOnFree(this);
+ {
+ MemTrackerSuppress _suppress;
+ freeOrMaybeKeepalive(this);
+ }
INSTRUMENT_RELEASE("Arena8192");
} else {
#ifdef ALLOC_INSTRUMENTATION
allocInstr["ArenaHugeKB"].dealloc((bigSize + 1023) >> 10);
#endif
g_hugeArenaMemory.fetch_sub(bigSize);
- freeOrMaybeKeepalive(this);
+ memTrackerOnFree(this);
+ {
+ MemTrackerSuppress _suppress;
+ freeOrMaybeKeepalive(this);
+ }
}
}
}
diff --git a/flow/FastAlloc.cpp b/flow/FastAlloc.cpp
index ed51df74b6..173d3d48ea 100644
--- a/flow/FastAlloc.cpp
+++ b/flow/FastAlloc.cpp
@@ -20,6 +20,7 @@
#include "flow/FastAlloc.h"
+#include "flow/MemoryTracker.h"
#include "flow/ThreadPrimitives.h"
#include "flow/Trace.h"
#include "flow/Error.h"
@@ -432,6 +433,7 @@ void* FastAllocator::allocate() {
#if defined(ALLOC_INSTRUMENTATION) || defined(ALLOC_INSTRUMENTATION_STDOUT)
recordAllocation(p, Size);
#endif
+ memTrackerOnAlloc(p, Size);
return p;
}
@@ -509,6 +511,7 @@ void FastAllocator::release(void* ptr) {
#if defined(ALLOC_INSTRUMENTATION) || defined(ALLOC_INSTRUMENTATION_STDOUT)
recordDeallocation(ptr);
#endif
+ memTrackerOnFree(ptr);
}
template
diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp
index 291e276596..27a1aa0a95 100644
--- a/flow/Knobs.cpp
+++ b/flow/Knobs.cpp
@@ -86,6 +86,15 @@ void FlowKnobs::initialize(Randomize randomize, IsSimulated isSimulated) {
init( MEMORY_USAGE_CHECK_INTERVAL, 1.0 );
+ // Per-call-site sampled memory tracker. See design/memory-tracker.md.
+ // Initial rollout: prod default off (=0). Simulation defaults to 1-in-10 sampling so the path is exercised.
+ init( MEMORY_TRACKING_SAMPLE_INVERSE, 0 ); if( isSimulated ) MEMORY_TRACKING_SAMPLE_INVERSE = 10;
+ init( MEMORY_TRACKING_FORCE_SAMPLE_BYTES, 100000 );
+ init( MEMORY_TRACKING_LIVE_TRACKING, true );
+ init( MEMORY_TRACKING_REPORT_INTERVAL, 600.0 ); if( isSimulated ) MEMORY_TRACKING_REPORT_INTERVAL = 30.0;
+ init( MEMORY_TRACKING_REPORT_BYTES_THRESHOLD, 80000000 ); if( isSimulated ) MEMORY_TRACKING_REPORT_BYTES_THRESHOLD = 1000000;
+ init( MEMORY_TRACKING_FRAMES, 6 );
+
// Chaos testing - enabled for simulation by default
init( ENABLE_CHAOS_FEATURES, isSimulated );
init( CHAOS_LOGGING_INTERVAL, 5.0 );
diff --git a/flow/MemoryTracker.cpp b/flow/MemoryTracker.cpp
new file mode 100644
index 0000000000..62a4c7dac1
--- /dev/null
+++ b/flow/MemoryTracker.cpp
@@ -0,0 +1,695 @@
+/*
+ * MemoryTracker.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Implementation of the sampled per-call-site memory tracker.
+// See design/memory-tracker.md and flow/include/flow/MemoryTracker.h.
+//
+// What this is for: finding memory LEAKS and untuned / oversized allocations
+// that drive RSS growth. FDB effectively never sees a real malloc / operator new
+// failure — a process is killed by fdbmonitor when its RSS crosses a configured
+// ceiling (typically ~12-16 GB against an ~8 GB target), i.e. "OOM" here is a
+// self-imposed RSS threshold, not an allocator failure. So the interesting range
+// is memory growth well SHORT of any allocation failure, and the tracker's
+// behaviour under an actual malloc/new failure is not a scenario we optimize for:
+// the hooks fail open (drop the sample; see memTrackerSampleAlloc). The sampled
+// path is nonetheless ordered so its table growth happens before any counter
+// update, so even that never-in-practice case leaves the accounting consistent.
+
+#include "flow/MemoryTracker.h"
+
+#include "flow/Knobs.h"
+#include "flow/Platform.h"
+#include "flow/ThreadPrimitives.h"
+#include "flow/Trace.h"
+#include "flow/flow.h"
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#ifdef __linux__
+#include
+#endif
+
+#if FDB_MEMORY_TRACKER
+
+// Thread-local sampling state.
+// gMemTrackerCounter starts at 1 so the first allocation per thread is
+// sampled (and the slow path then reseeds from the knob).
+// gForceSampleBytes initialized to ~0 so force-sample never fires before we've
+// loaded the knob value at least once.
+thread_local bool gInMemTracker = false;
+thread_local int gMemTrackerCounter = 1;
+thread_local std::size_t gForceSampleBytes = static_cast(-1);
+// Starts false so the first allocation on each thread reaches the slow path to
+// read the knob; set true there if sampling is off (see memTrackerSampleAlloc).
+thread_local bool gMemTrackerOff = false;
+
+// Test-only one-shot: when set, the next sampled allocation throws to simulate a
+// tracker metadata-allocation failure (see memTrackerFailNextSampleForTest).
+static thread_local bool gFailNextSampleForTest = false;
+
+// Definition of the cache-line-isolated enabled flag declared in the header.
+MemTrackerEnabledFlag g_memTrackerEnabled;
+// Same initial seed for every thread; cheap and adequate. Threads in
+// production start at different times and call into the slow path at
+// uncorrelated rates, so any phase correlation washes out within the
+// first handful of samples. If profiling ever shows correlated bursts
+// at startup, mix in a thread-id-derived value here.
+thread_local uint32_t gMemTrackerSeed = 0x9E3779B9u;
+
+namespace {
+
+// FNV-1a 64-bit over the captured frame array.
+uint64_t fnv64(const void* data, std::size_t len) {
+ uint64_t h = 0xcbf29ce484222325ULL;
+ const auto* p = static_cast(data);
+ for (std::size_t i = 0; i < len; i++) {
+ h ^= p[i];
+ h *= 0x100000001b3ULL;
+ }
+ return h;
+}
+
+inline std::uint32_t xorshift32(std::uint32_t& s) {
+ std::uint32_t x = s ? s : 0x9E3779B9u;
+ x ^= x << 13;
+ x ^= x >> 17;
+ x ^= x << 5;
+ s = x;
+ return x;
+}
+
+// Per-thread stack bounds, populated lazily on first use of captureFramesFP.
+// Used by captureFramesFP to terminate the FP walk when it crosses into
+// FP-elided code (notably glibc's pthread shutdown / TLS-destructor
+// machinery). Without this guard, the FP-elided frame leaves an
+// uninitialized saved-FP slot and the walk dereferences garbage. See
+// design/memory-tracker.md, "Side-thread safety".
+//
+// You've heard of optimistic concurrency control in database systems? This is
+// basically *optimistic segfault avoidance* to enable fast stack unwinding.
+// Caveat: the heuristics here aren't perfect. They seem pretty effective
+// so far.
+thread_local uintptr_t gStackLow = 0;
+thread_local uintptr_t gStackHigh = 0;
+
+#ifdef __linux__
+
+void initStackBoundsForThread() {
+ pthread_attr_t attr;
+ if (pthread_getattr_np(pthread_self(), &attr) == 0) {
+ void* base = nullptr;
+ size_t size = 0;
+ if (pthread_attr_getstack(&attr, &base, &size) == 0) {
+ gStackLow = reinterpret_cast(base);
+ gStackHigh = gStackLow + size;
+ }
+ pthread_attr_destroy(&attr);
+ }
+}
+
+// Manual frame-pointer walk. Captures the return-address chain starting at
+// the caller of this function (and up). Relies on -fno-omit-frame-pointer.
+// Annotated noinline + no_instrument_function so the compiler can't fold the
+// frame chain in unexpected ways.
+//
+// Bounds the walk by the current thread's stack range so that crossing into
+// FP-elided code (which leaves the saved-FP slot uninitialized rather than
+// NULL) terminates cleanly instead of dereferencing garbage (see above).
+__attribute__((no_instrument_function, noinline)) int captureFramesFP(void** out, int max) {
+ if (!gStackLow) {
+ initStackBoundsForThread();
+ }
+ void** fp = static_cast(__builtin_frame_address(0));
+ // Fallback for threads where pthread_getattr_np failed: ±8 MB around
+ // the initial frame.
+ // Caveat: this may need to be constrained more tightly to deal with
+ // smaller stacks.
+ uintptr_t lo = gStackLow ? gStackLow : reinterpret_cast(fp);
+ uintptr_t hi = gStackHigh ? gStackHigh : reinterpret_cast(fp) + (8u << 20);
+ int n = 0;
+ while (fp && n < max) {
+ uintptr_t a = reinterpret_cast(fp);
+ // Reject out-of-stack or misaligned fp before dereferencing.
+ if (a < lo || a + 16 > hi) {
+ break;
+ }
+ if (a & (sizeof(void*) - 1)) {
+ break;
+ }
+ void* ra = fp[1];
+ if (!ra) {
+ break;
+ }
+ out[n++] = ra;
+ void** next = static_cast(fp[0]);
+ // Sanity: stack grows down, so each next frame address must be larger.
+ if (next <= fp) {
+ break;
+ }
+ fp = next;
+ }
+ return n;
+}
+
+#else // !__linux__
+
+// NOTE: We (Apple) do not maintain a local facility to build FDB with
+// MSVC on Windows. The code in this file **may** have issues. We are
+// doing a best-effort attempt not to break the build. Support for
+// this memory tracking feature by community users of Windows would be
+// welcome.
+
+// macOS / non-Linux: stack walking is unreliable here (system runtime
+// has -fomit-frame-pointer in places we can't avoid, and pthread_getattr_np
+// is Linux-specific). FDB is required to compile on macOS/Windows but is not run
+// in production there, so we just no-op the walker. The rest of the
+// tracker still compiles and runs; per-call-site reports will simply
+// lack stack attribution.
+force_noinline int captureFramesFP(void**, int) {
+ return 0;
+}
+
+#endif // __linux__
+
+// TODO: when memory tracking is enabled, regardless of the sampling
+// rate, every deallocation has to acquire this mutex. At O(1M)
+// frees/second this is noticeable overhead. This could be sped up by
+// sharding the mutex and the global state protected by the mutex
+// (the global state is the 2 maps and ~10 scalars defined below).
+// Presumably hash of the malloc buffer address itself could direct
+// the sharding. Consumers of the global state would have to merge
+// across shards but that is intended only to be the periodic log
+// reporter so making it do costly work is fine since it only runs
+// O(1/minute).
+
+ThreadSpinLock g_mtLock;
+
+struct LiveEntry {
+ std::uint64_t fingerprint;
+ std::uint64_t size;
+ std::int64_t weight; // inverse inclusion probability at sample time (≈ SampleInverse, or 1 if
+ // force-sampled); estimated contribution of this block is size * weight.
+ // Stored so free debits the estimate by exactly what alloc credited, even
+ // if the sampling knob changed in between.
+};
+
+// Lazily-constructed maps. Allocated under the spinlock the first time we
+// reach the sampled path. Heap allocations from the maps' internals go
+// through our overridden operator new, which short-circuits (gInMemTracker
+// is true on the sampled path) and falls through to std::malloc — so map
+// growth never recurses into tracking.
+std::unordered_map* g_aggMap = nullptr;
+std::unordered_map* g_liveMap = nullptr;
+
+// Sampled-totals (i.e. across what we actually saw, not population estimates).
+std::int64_t g_liveBytesTotal = 0;
+std::int64_t g_liveBlocksTotal = 0;
+std::int64_t g_cumulativeBytesTotal = 0;
+std::int64_t g_cumulativeAllocsTotal = 0;
+std::int64_t g_samplesEmitted = 0;
+
+// Estimated population totals (sampling correction applied; see LiveEntry::weight).
+std::int64_t g_estLiveBytesTotal = 0;
+std::int64_t g_estLiveBlocksTotal = 0;
+std::int64_t g_estCumulativeBytesTotal = 0;
+std::int64_t g_estCumulativeAllocsTotal = 0;
+
+void ensureMaps() {
+ if (!g_aggMap) {
+ g_aggMap = new std::unordered_map();
+ }
+ if (!g_liveMap) {
+ g_liveMap = new std::unordered_map();
+ }
+}
+
+} // namespace
+
+// Publish the enabled flag and arm the calling thread from the current
+// MEMORY_TRACKING_SAMPLE_INVERSE. Shared by memTrackerInit (once, at startup)
+// and memTrackerResetForTest. Reading the knob explicitly here — rather than
+// inferring the enabled state from the first sampled allocation — is what keeps
+// an early main-thread allocation from latching the tracker off before the
+// knobs are configured.
+static void memTrackerArmFromKnobs() {
+ int inverse = FLOW_KNOBS ? FLOW_KNOBS->MEMORY_TRACKING_SAMPLE_INVERSE : 0;
+ bool enabled = (inverse > 0);
+ g_memTrackerEnabled.value.store(enabled, std::memory_order_relaxed);
+ // If enabled, clear this thread's off-latch and force its next allocation onto
+ // the slow path (counter==1) to seed the reseed; if disabled, latch off so the
+ // alloc hot path short-circuits on a single TLS load.
+ gMemTrackerOff = !enabled;
+ gMemTrackerCounter = 1;
+ gForceSampleBytes = FLOW_KNOBS ? static_cast(FLOW_KNOBS->MEMORY_TRACKING_FORCE_SAMPLE_BYTES)
+ : static_cast(-1);
+}
+
+void memTrackerInit() {
+ memTrackerArmFromKnobs();
+}
+
+static void memTrackerSampleAllocImpl(void* p, std::size_t n) {
+ if (gFailNextSampleForTest) {
+ gFailNextSampleForTest = false;
+ throw std::bad_alloc(); // simulate a tracker metadata-allocation failure (test only)
+ }
+ int inverse = 0;
+ int frames = 6;
+ bool liveTracking = true;
+ if (FLOW_KNOBS) {
+ inverse = FLOW_KNOBS->MEMORY_TRACKING_SAMPLE_INVERSE;
+ frames = FLOW_KNOBS->MEMORY_TRACKING_FRAMES;
+ liveTracking = FLOW_KNOBS->MEMORY_TRACKING_LIVE_TRACKING;
+ gForceSampleBytes = static_cast(FLOW_KNOBS->MEMORY_TRACKING_FORCE_SAMPLE_BYTES);
+ }
+ if (frames < 1) {
+ frames = 1;
+ }
+ if (frames > MEMORY_TRACKER_MAX_FRAMES) {
+ frames = MEMORY_TRACKER_MAX_FRAMES;
+ }
+ // Bound the reseed's `2 * inverse` arithmetic to int range for absurd knob
+ // values; 1-in-256M sampling is already effectively off.
+ if (inverse > (1 << 28)) {
+ inverse = 1 << 28;
+ }
+
+ if (inverse <= 0) {
+ // Off (knob is startup-only). Flag this thread so the alloc hot path
+ // short-circuits on one TLS load, without touching the counter again.
+ gMemTrackerOff = true;
+ return;
+ }
+ if (inverse == 1) {
+ // Sample every allocation — keep counter at 1 so the next decrement
+ // drops it to 0 and re-enters the slow path. Bypass the random
+ // reseed below, which would otherwise leave counter==2 half the
+ // time and cause us to miss every other allocation.
+ gMemTrackerCounter = 1;
+ } else if (gMemTrackerCounter <= 0) {
+ // The random countdown actually expired: draw a fresh gap uniformly from
+ // [1, 2*inverse-1], whose mean is exactly `inverse` — so the 1-in-inverse
+ // sampling rate is unbiased and the integer `weight` below is exact.
+ std::uint32_t r = xorshift32(gMemTrackerSeed);
+ gMemTrackerCounter = 1 + static_cast(r % static_cast(2 * inverse - 1));
+ }
+
+ bool isForceSampled = (n >= gForceSampleBytes);
+
+ // Weight = inverse inclusion probability of this sample, i.e. how many
+ // allocations in the population it stands in for. A randomly-sampled block
+ // (1-in-inverse) represents ~inverse allocations; a force-sampled block was
+ // captured with certainty and represents only itself. The reseed draws
+ // uniformly from [1, 2*inverse-1] (mean exactly `inverse`), so this integer
+ // weight matches the true mean sampling gap with no bias.
+ std::int64_t weight = (isForceSampled || inverse <= 1) ? 1 : inverse;
+
+ // Capture frames; skip the topmost two (this function and captureFramesFP
+ // itself) so the recorded stack starts at the caller of memTrackerOnAlloc.
+ // The strip count of 2 assumes memTrackerOnAlloc is inlined into its
+ // caller (it's declared `inline` and the body is trivial). Production
+ // builds run at -O3 and the inliner cooperates; at -O0 the inline hint
+ // can be ignored and the recorded stack starts one frame too deep
+ // (frame 0 = memTrackerOnAlloc body rather than the user's
+ // allocation site). Acceptable: -O0 builds are not load-bearing for
+ // memory attribution; the off-by-one is harmless for that workflow.
+ void* tmp[MEMORY_TRACKER_MAX_FRAMES + 4];
+ int captured = captureFramesFP(tmp, frames + 2);
+ int kept = 0;
+ void* keep[MEMORY_TRACKER_MAX_FRAMES];
+ for (int i = 2; i < captured && kept < frames; i++) {
+ keep[kept++] = tmp[i];
+ }
+
+ std::uint64_t fp = (kept == 0) ? 0 : fnv64(keep, static_cast(kept) * sizeof(void*));
+
+ ThreadSpinLockHolder lk(g_mtLock);
+ ensureMaps();
+
+ auto nBytes = static_cast(n);
+ auto estBytes = nBytes * weight;
+
+ // Do both node-allocating map operations up front, before mutating any
+ // counter, so that if one throws (a bad_alloc while a table grows) the
+ // exception unwinds with all per-site and global totals still consistent and
+ // the fail-open catch in memTrackerSampleAlloc simply drops the sample. FDB
+ // never sees a real malloc/new failure in practice (see the file header), so
+ // this is cheap hygiene rather than a hot path.
+ auto& site = (*g_aggMap)[fp]; // inserts a node for a new fingerprint; may throw
+ if (site.fingerprint == 0 && site.cumulativeAllocs == 0) {
+ site.fingerprint = fp;
+ site.exemplarFrameCount = static_cast(kept);
+ for (int i = 0; i < kept; i++) {
+ site.exemplarFrames[i] = keep[i];
+ }
+ }
+
+ // Reserve the live-block slot before crediting anything. A brand-new key
+ // allocates a node here (may throw); an existing key — a stale entry whose
+ // free was suppressed (e.g. during memTrackerDump or a memTrackerForEachSite
+ // callback) so it was never debited — reuses its slot and cannot throw. We
+ // capture the stale value so it can be debited below; otherwise its live
+ // credit would leak once the address is reused and live totals would creep up.
+ bool hadStale = false;
+ LiveEntry stalePrev{};
+ if (liveTracking) {
+ auto key = reinterpret_cast(p);
+ auto res = g_liveMap->try_emplace(key, LiveEntry{ fp, static_cast(n), weight });
+ if (!res.second) {
+ hadStale = true;
+ stalePrev = res.first->second;
+ res.first->second = LiveEntry{ fp, static_cast(n), weight };
+ }
+ }
+
+ // ---- Nothing below allocates or throws; per-site and global totals move in lockstep. ----
+
+ if (hadStale) {
+ auto oldBytes = static_cast(stalePrev.size);
+ auto oldEst = oldBytes * stalePrev.weight;
+ auto oldSite = g_aggMap->find(stalePrev.fingerprint);
+ if (oldSite != g_aggMap->end()) {
+ oldSite->second.liveBytes -= oldBytes;
+ oldSite->second.liveCount -= 1;
+ oldSite->second.estLiveBytes -= oldEst;
+ oldSite->second.estLiveCount -= stalePrev.weight;
+ }
+ g_liveBytesTotal -= oldBytes;
+ g_liveBlocksTotal -= 1;
+ g_estLiveBytesTotal -= oldEst;
+ g_estLiveBlocksTotal -= stalePrev.weight;
+ }
+
+ site.cumulativeAllocs += 1;
+ site.cumulativeBytes += nBytes;
+ site.estCumulativeAllocs += weight;
+ site.estCumulativeBytes += estBytes;
+ if (isForceSampled) {
+ site.forceSampledCount += 1;
+ }
+
+ if (liveTracking) {
+ site.liveBytes += nBytes;
+ site.liveCount += 1;
+ if (site.liveBytes > site.peakBytes) {
+ site.peakBytes = site.liveBytes;
+ }
+ site.estLiveBytes += estBytes;
+ site.estLiveCount += weight;
+ if (site.estLiveBytes > site.estPeakBytes) {
+ site.estPeakBytes = site.estLiveBytes;
+ }
+ g_liveBytesTotal += nBytes;
+ g_liveBlocksTotal += 1;
+ g_estLiveBytesTotal += estBytes;
+ g_estLiveBlocksTotal += weight;
+ }
+ g_cumulativeBytesTotal += nBytes;
+ g_cumulativeAllocsTotal += 1;
+ g_estCumulativeBytesTotal += estBytes;
+ g_estCumulativeAllocsTotal += weight;
+ g_samplesEmitted += 1;
+}
+
+void memTrackerSampleAlloc(void* p, std::size_t n) {
+ // Fail open: the tracker is a diagnostic; a std::bad_alloc from its own map
+ // growth (or the test injection) must never propagate into the caller's
+ // allocation path, which has already handed out the underlying block.
+ try {
+ memTrackerSampleAllocImpl(p, n);
+ } catch (...) {
+ }
+}
+
+static void memTrackerSampleFreeImpl(void* p) {
+ bool liveTracking = FLOW_KNOBS ? FLOW_KNOBS->MEMORY_TRACKING_LIVE_TRACKING : true;
+ if (!liveTracking) {
+ return;
+ }
+
+ ThreadSpinLockHolder lk(g_mtLock);
+ if (!g_liveMap) {
+ return;
+ }
+ auto it = g_liveMap->find(reinterpret_cast(p));
+ if (it == g_liveMap->end()) {
+ return;
+ }
+
+ LiveEntry e = it->second;
+ g_liveMap->erase(it);
+
+ auto eBytes = static_cast(e.size);
+ auto eEstBytes = eBytes * e.weight;
+ if (g_aggMap) {
+ auto sit = g_aggMap->find(e.fingerprint);
+ if (sit != g_aggMap->end()) {
+ sit->second.liveBytes -= eBytes;
+ sit->second.liveCount -= 1;
+ sit->second.estLiveBytes -= eEstBytes;
+ sit->second.estLiveCount -= e.weight;
+ }
+ }
+ g_liveBytesTotal -= eBytes;
+ g_liveBlocksTotal -= 1;
+ g_estLiveBytesTotal -= eEstBytes;
+ g_estLiveBlocksTotal -= e.weight;
+}
+
+void memTrackerSampleFree(void* p) {
+ // Fail open (see memTrackerSampleAlloc): operator delete is noexcept, so the
+ // tracker must never let an exception escape the free path.
+ try {
+ memTrackerSampleFreeImpl(p);
+ } catch (...) {
+ }
+}
+
+void memTrackerForEachSite(std::function cb) {
+ // Suppress for the entire call so callbacks that allocate (e.g.
+ // fprintf or std::vector growth in test failure paths) don't
+ // re-enter the tracker and pollute the agg map mid-iteration.
+ MemTrackerSuppress _suppress;
+ std::vector snapshot;
+ {
+ ThreadSpinLockHolder lk(g_mtLock);
+ if (g_aggMap) {
+ snapshot.reserve(g_aggMap->size());
+ for (auto& kv : *g_aggMap) {
+ snapshot.push_back(kv.second);
+ }
+ }
+ }
+ for (auto& s : snapshot) {
+ cb(s);
+ }
+}
+
+void memTrackerResetForTest() {
+ MemTrackerSuppress _suppress;
+ {
+ ThreadSpinLockHolder lk(g_mtLock);
+ if (g_aggMap) {
+ g_aggMap->clear();
+ }
+ if (g_liveMap) {
+ g_liveMap->clear();
+ }
+ g_liveBytesTotal = 0;
+ g_liveBlocksTotal = 0;
+ g_cumulativeBytesTotal = 0;
+ g_cumulativeAllocsTotal = 0;
+ g_samplesEmitted = 0;
+ g_estLiveBytesTotal = 0;
+ g_estLiveBlocksTotal = 0;
+ g_estCumulativeBytesTotal = 0;
+ g_estCumulativeAllocsTotal = 0;
+ }
+ // Publish the enabled flag and arm this thread from the current knob value
+ // (a test typically sets MEMORY_TRACKING_SAMPLE_INVERSE via KnobOverride just
+ // before calling this), mirroring memTrackerInit at process startup.
+ gFailNextSampleForTest = false;
+ memTrackerArmFromKnobs();
+}
+
+void memTrackerFailNextSampleForTest() {
+ gFailNextSampleForTest = true;
+}
+
+static void memTrackerDumpImpl(int64_t bytesThreshold) {
+ MemTrackerSuppress _suppress;
+
+ std::vector sites;
+ int aggSize = 0;
+ int liveSize = 0;
+ std::int64_t liveBytesTotalSnap = 0;
+ std::int64_t liveBlocksTotalSnap = 0;
+ std::int64_t cumBytesSnap = 0;
+ std::int64_t cumAllocsSnap = 0;
+ std::int64_t samplesEmittedSnap = 0;
+ std::int64_t estLiveBytesTotalSnap = 0;
+ std::int64_t estLiveBlocksTotalSnap = 0;
+ std::int64_t estCumBytesSnap = 0;
+ std::int64_t estCumAllocsSnap = 0;
+ {
+ ThreadSpinLockHolder lk(g_mtLock);
+ if (g_aggMap) {
+ sites.reserve(g_aggMap->size());
+ for (auto& kv : *g_aggMap) {
+ sites.push_back(kv.second);
+ }
+ aggSize = static_cast(g_aggMap->size());
+ }
+ liveSize = g_liveMap ? static_cast(g_liveMap->size()) : 0;
+ liveBytesTotalSnap = g_liveBytesTotal;
+ liveBlocksTotalSnap = g_liveBlocksTotal;
+ cumBytesSnap = g_cumulativeBytesTotal;
+ cumAllocsSnap = g_cumulativeAllocsTotal;
+ samplesEmittedSnap = g_samplesEmitted;
+ estLiveBytesTotalSnap = g_estLiveBytesTotal;
+ estLiveBlocksTotalSnap = g_estLiveBlocksTotal;
+ estCumBytesSnap = g_estCumulativeBytesTotal;
+ estCumAllocsSnap = g_estCumulativeAllocsTotal;
+ }
+
+ bool liveTracking = FLOW_KNOBS ? FLOW_KNOBS->MEMORY_TRACKING_LIVE_TRACKING : true;
+ // Rank and threshold on the *estimated* usage, since that is the real
+ // per-site cost the report is about; the threshold knob is expressed in
+ // real bytes (~1% of target RSS), not sampled bytes.
+ // std::sort is unstable and unordered_map iteration is bucket-order, so
+ // MemoryTrackerSite events for sites with tied byte values may appear in
+ // different orders across same-seed sim2 runs. The R5 determinism
+ // requirement is on aggregate counts, not event ordering — those are
+ // unaffected — so we don't pay for stable_sort here.
+ auto byLive = [](const MemoryTrackerCallSite& a, const MemoryTrackerCallSite& b) {
+ return a.estLiveBytes > b.estLiveBytes;
+ };
+ auto byCum = [](const MemoryTrackerCallSite& a, const MemoryTrackerCallSite& b) {
+ return a.estCumulativeBytes > b.estCumulativeBytes;
+ };
+ if (liveTracking) {
+ std::sort(sites.begin(), sites.end(), byLive);
+ } else {
+ std::sort(sites.begin(), sites.end(), byCum);
+ }
+
+ // Filter: a site qualifies when its estimated currently-live bytes (or
+ // estimated cumulative bytes in degraded mode) exceed the threshold. Sites
+ // are already sorted descending, so we can stop at the first non-qualifier.
+ std::vector qualifying;
+ qualifying.reserve(sites.size());
+ for (const auto& s : sites) {
+ int64_t v = liveTracking ? s.estLiveBytes : s.estCumulativeBytes;
+ if (v < bytesThreshold) {
+ break;
+ }
+ qualifying.push_back(s);
+ }
+
+ // Build addr2line prefix once per dump. Built directly here rather
+ // than via platform::format_backtrace, which deliberately drops index
+ // 0 of its input (its single-site use case treats that as the helper's
+ // caller); we want every captured frame including the leaf.
+ std::string addrCmdPrefix;
+ uintptr_t pieOffset = 0;
+ if (!qualifying.empty()) {
+ platform::ImageInfo img = platform::getImageInfo();
+#ifdef __clang__
+ const char* addr2lineTool = "/usr/local/bin/llvm-addr2line";
+#else
+ const char* addr2lineTool = "/usr/bin/addr2line";
+#endif
+ addrCmdPrefix = format("%s -e %s -p -C -f -i", addr2lineTool, img.symbolFileName.c_str());
+ pieOffset = reinterpret_cast(img.offset);
+ }
+
+ for (const auto& s : qualifying) {
+ std::string addrCmd = addrCmdPrefix;
+ for (int i = 0; i < s.exemplarFrameCount; i++) {
+ uintptr_t pieRelative = reinterpret_cast(s.exemplarFrames[i]) - pieOffset;
+ addrCmd += format(" 0x%lx", pieRelative);
+ }
+ TraceEvent("MemoryTrackerSite")
+ .detail("Fingerprint", format("%016llx", static_cast(s.fingerprint)))
+ .detail("EstLiveBytes", s.estLiveBytes)
+ .detail("EstLiveCount", s.estLiveCount)
+ .detail("EstPeakBytes", s.estPeakBytes)
+ .detail("EstCumulativeBytes", s.estCumulativeBytes)
+ .detail("EstCumulativeAllocs", s.estCumulativeAllocs)
+ .detail("LiveBytes", s.liveBytes)
+ .detail("LiveCount", s.liveCount)
+ .detail("PeakBytes", s.peakBytes)
+ .detail("CumulativeBytes", s.cumulativeBytes)
+ .detail("CumulativeAllocs", s.cumulativeAllocs)
+ .detail("ForceSampledCount", s.forceSampledCount)
+ .detail("AddrCmd", addrCmd);
+ }
+
+ TraceEvent("MemoryTrackerSummary")
+ .detail("SitesTracked", aggSize)
+ .detail("SitesReported", static_cast(qualifying.size()))
+ .detail("EstLiveBytesTotal", estLiveBytesTotalSnap)
+ .detail("EstLiveBlocksTotal", estLiveBlocksTotalSnap)
+ .detail("EstCumulativeBytes", estCumBytesSnap)
+ .detail("EstCumulativeAllocs", estCumAllocsSnap)
+ .detail("LiveBlocks", liveSize)
+ .detail("LiveBytesTotal", liveBytesTotalSnap)
+ .detail("LiveBlocksTotal", liveBlocksTotalSnap)
+ .detail("CumulativeAllocs", cumAllocsSnap)
+ .detail("CumulativeBytes", cumBytesSnap)
+ .detail("SamplesEmitted", samplesEmittedSnap)
+ .detail("SampleInverse", FLOW_KNOBS ? FLOW_KNOBS->MEMORY_TRACKING_SAMPLE_INVERSE : 0)
+ .detail("ForceSampleBytes",
+ FLOW_KNOBS ? FLOW_KNOBS->MEMORY_TRACKING_FORCE_SAMPLE_BYTES : static_cast(-1))
+ .detail("ReportBytesThreshold", bytesThreshold)
+ // Caveat: Est* values are statistical estimates. Each randomly-sampled block is
+ // scaled by SampleInverse; force-sampled blocks (>= ForceSampleBytes) count once.
+ // Accuracy improves with SamplesEmitted; a site with few samples is noisy.
+ .detail("EstimateBasis", "Est*=sampled*SampleInverse; force-sampled weight 1; statistical estimate");
+}
+
+void memTrackerDump(int64_t bytesThreshold) {
+ // Disabled: nothing is sampled, so skip the dump entirely rather than emit an
+ // empty MemoryTrackerSummary every report interval (the production default is
+ // off, and SystemMonitor calls this on a fixed cadence regardless).
+ if (!g_memTrackerEnabled.value.load(std::memory_order_relaxed)) {
+ return;
+ }
+ // Fail open (see memTrackerSampleAlloc): a diagnostic dump must never crash the
+ // server, e.g. on an allocation failure while building the report.
+ try {
+ memTrackerDumpImpl(bytesThreshold);
+ } catch (...) {
+ }
+}
+
+// The global operator new / operator delete replacements that route through
+// memTrackerOnAlloc/OnFree live in fdbserver/GlobalNewDelete.cpp, not here, so
+// the interposition is confined to the fdbserver executable and never ships in
+// libfdb_c / client bindings. This TU provides only the tracker machinery those
+// overrides (and the FastAllocator / ArenaBlock hooks) call into.
+
+#endif // FDB_MEMORY_TRACKER
diff --git a/flow/SystemMonitor.cpp b/flow/SystemMonitor.cpp
index 88dfd70482..a8761b829d 100644
--- a/flow/SystemMonitor.cpp
+++ b/flow/SystemMonitor.cpp
@@ -22,6 +22,8 @@
#include "flow/Platform.h"
#include "flow/TDMetric.h"
#include "flow/SystemMonitor.h"
+#include "flow/Knobs.h"
+#include "flow/MemoryTracker.h"
#if defined(ALLOC_INSTRUMENTATION) && defined(__linux__)
#include
@@ -489,6 +491,20 @@ SystemStatistics customSystemMonitor(std::string const& eventName, StatisticsSta
#endif
statState->networkMetricsState = g_network->networkInfo.metrics;
statState->networkState = netData;
+
+ // Periodic dump of the per-call-site memory tracker; cadence from the
+ // MEMORY_TRACKING_REPORT_INTERVAL knob (<=0 disables). In simulation the
+ // tracker's tables and this static are shared across all simulated
+ // processes, so one dump fires per interval cluster-wide and its site
+ // numbers blend every process — correct only in a real single-process server.
+ if (FLOW_KNOBS && FLOW_KNOBS->MEMORY_TRACKING_REPORT_INTERVAL > 0) {
+ static double lastMemTrackerDump = 0;
+ if (now() - lastMemTrackerDump >= FLOW_KNOBS->MEMORY_TRACKING_REPORT_INTERVAL) {
+ memTrackerDump(FLOW_KNOBS->MEMORY_TRACKING_REPORT_BYTES_THRESHOLD);
+ lastMemTrackerDump = now();
+ }
+ }
+
return currentStats;
}
diff --git a/flow/include/flow/Knobs.h b/flow/include/flow/Knobs.h
index 042916e279..c003d8905b 100644
--- a/flow/include/flow/Knobs.h
+++ b/flow/include/flow/Knobs.h
@@ -127,6 +127,17 @@ public:
double MEMORY_USAGE_CHECK_INTERVAL;
+ // Per-call-site sampled memory tracker. See design/memory-tracker.md and flow/MemoryTracker.h.
+ // All of these are startup-only: they are read once and not meant to change at runtime
+ // (dynamic enable/disable is a Non-requirement -- edit the config and restart).
+ int MEMORY_TRACKING_SAMPLE_INVERSE; // 0=off, N=1-in-N
+ int64_t MEMORY_TRACKING_FORCE_SAMPLE_BYTES; // always sample allocations >= this many bytes; -1 disables
+ bool MEMORY_TRACKING_LIVE_TRACKING; // when false, skip the pointer-keyed live-block table
+ double MEMORY_TRACKING_REPORT_INTERVAL; // seconds between dumps; 0 disables reporting
+ int64_t MEMORY_TRACKING_REPORT_BYTES_THRESHOLD; // sites with live bytes >= this are reported each dump (~1% of an 8
+ // GB target RSS)
+ int MEMORY_TRACKING_FRAMES; // captured stack depth (1..MEMORY_TRACKER_MAX_FRAMES)
+
// Chaos testing
bool ENABLE_CHAOS_FEATURES;
double CHAOS_LOGGING_INTERVAL;
diff --git a/flow/include/flow/MemoryTracker.h b/flow/include/flow/MemoryTracker.h
new file mode 100644
index 0000000000..038d3e395e
--- /dev/null
+++ b/flow/include/flow/MemoryTracker.h
@@ -0,0 +1,223 @@
+/*
+ * MemoryTracker.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Sampled per-call-site memory attribution.
+//
+// See design/memory-tracker.md for the full design.
+//
+// Hot path: memTrackerOnAlloc is header-inlined; when the feature is disabled
+// (the common production default) it short-circuits on a single per-thread TLS
+// load + branch. memTrackerOnFree is header-inlined, one relaxed read of a
+// cache-line-isolated enabled flag + one branch when disabled -- no lock, no
+// table probe.
+//
+// Sampled path delegates to memTrackerSampleAlloc / memTrackerSampleFree,
+// which take a private spinlock, capture a frame-pointer-walk backtrace, and
+// update two tables (aggregation by fingerprint, and an optional pointer-
+// keyed live-block table).
+//
+// Reentrancy: the gInMemTracker thread-local guard is set to true while the
+// tracker is doing its own work. Any allocator hook called recursively
+// during that window observes the guard and bails out, leaving the
+// underlying allocation un-tracked. Higher-level hooks (e.g. ArenaBlock::create)
+// may also set this guard to suppress an inner allocator hook so the same
+// block is attributed at exactly one level.
+
+#ifndef FLOW_MEMORY_TRACKER_H
+#define FLOW_MEMORY_TRACKER_H
+#pragma once
+
+#include
+#include
+#include
+#include
+
+// FDB_MEMORY_TRACKER gates the whole feature at compile time; on (1) by default.
+// Build with -DFDB_MEMORY_TRACKER=0 (cmake -DFDB_MEMORY_TRACKER=OFF) to compile it
+// out entirely: the hooks become no-ops and the global operator new/delete override
+// (fdbserver/GlobalNewDelete.cpp) is not defined, so libc++'s allocator is used.
+#ifndef FDB_MEMORY_TRACKER
+#define FDB_MEMORY_TRACKER 1
+#endif
+
+#if FDB_MEMORY_TRACKER
+
+// Maximum number of stack frames the tracker can capture per sample.
+// MEMORY_TRACKING_FRAMES knob controls the runtime depth (1..MEMORY_TRACKER_MAX_FRAMES).
+constexpr int MEMORY_TRACKER_MAX_FRAMES = 10;
+
+// Per-site aggregate, exposed for tests via memTrackerForEachSite.
+//
+// Two families of numbers are kept per site:
+// * Est* — the estimated *population* usage, i.e. what the site is really
+// costing. Each sampled block is weighted by its inverse inclusion
+// probability (≈ SampleInverse for randomly-sampled blocks, 1 for
+// force-sampled blocks) at sample time, so these already have the sampling
+// math applied — a consumer (logging, etc) reads them directly, no scaling required.
+// * the raw sampled counters (liveBytes, cumulativeAllocs, …) — the
+// uninterpreted "what we actually observed" numbers, kept for auditing the
+// estimate and gauging its confidence (few samples ⇒ noisy estimate).
+struct MemoryTrackerCallSite {
+ uint64_t fingerprint;
+
+ int64_t estLiveBytes;
+ int64_t estLiveCount;
+ int64_t estPeakBytes;
+ int64_t estCumulativeBytes;
+ int64_t estCumulativeAllocs;
+
+ int64_t liveBytes;
+ int64_t liveCount;
+ int64_t peakBytes;
+ int64_t cumulativeAllocs;
+ int64_t cumulativeBytes;
+ int64_t forceSampledCount;
+
+ void* exemplarFrames[MEMORY_TRACKER_MAX_FRAMES];
+ uint8_t exemplarFrameCount;
+};
+
+extern thread_local bool gInMemTracker;
+extern thread_local int gMemTrackerCounter;
+extern thread_local std::size_t gForceSampleBytes;
+// Set true (per thread) once this thread's slow path observes sampling is off,
+// so the alloc hot path then short-circuits on a single TLS load instead of
+// decrementing the counter and reading gForceSampleBytes every call. Per-thread
+// (not global) so an early main-thread allocation before FLOW_KNOBS is ready
+// can't disable sampling on worker threads that bootstrap later. Cleared by
+// memTrackerResetForTest.
+extern thread_local bool gMemTrackerOff;
+
+// Global "is the tracker enabled" flag, kept in its own cache line. Published
+// once, from the first slow-path visit, and thereafter constant: the sample-
+// inverse knob is read at startup only (dynamic enable/disable is a
+// Non-requirement -- see design/memory-tracker.md). The flag stays in MESI
+// shared state across cores, so the free hot path's relaxed read is cached:
+// when disabled, a free is one read + one branch, no lock. This is the
+// free-path off switch (a free has no per-thread sampling counter to gate on,
+// unlike an alloc).
+//
+// Relaxed ordering is sufficient: a free of a sampled pointer is always
+// preceded (via the pointer handoff that let the freeing thread learn the
+// pointer at all) by the sampling alloc that inserted it, and that alloc set
+// this flag true before inserting -- so the happens-before edge guarantees the
+// freeing thread observes the flag as true.
+struct alignas(64) MemTrackerEnabledFlag {
+ std::atomic value{ false };
+ char pad[64 - sizeof(std::atomic)];
+};
+extern MemTrackerEnabledFlag g_memTrackerEnabled;
+
+// RAII suppressor: while alive, allocator hooks short-circuit. Used by code
+// paths that call into a lower-level allocator (e.g. ArenaBlock wrapping
+// `new uint8_t[]`) and want their explicit memTrackerOnAlloc/OnFree call to
+// be the sole tracker for the block — without this guard the inner
+// allocator's hook fires too and the same pointer is double-tracked under
+// two different fingerprints. Nest-safe: saves and restores prev.
+class MemTrackerSuppress {
+ bool prev;
+
+public:
+ MemTrackerSuppress() : prev(gInMemTracker) { gInMemTracker = true; }
+ ~MemTrackerSuppress() { gInMemTracker = prev; }
+ MemTrackerSuppress(const MemTrackerSuppress&) = delete;
+ MemTrackerSuppress& operator=(const MemTrackerSuppress&) = delete;
+};
+
+// Initialize the tracker from the current knob values. Call once, from process
+// startup, AFTER all knobs are finalized and BEFORE any serving role starts (see
+// fdbserver.cpp). Publishes the enabled state and arms the calling (network)
+// thread from MEMORY_TRACKING_SAMPLE_INVERSE, so early startup allocations on the
+// main thread cannot latch the tracker off before the knob is configured. The
+// sample-inverse knob is read here (and in memTrackerResetForTest) rather than
+// inferred from the first allocation; dynamic runtime enable/disable is a
+// Non-requirement (see design/memory-tracker.md).
+void memTrackerInit();
+
+void memTrackerSampleAlloc(void* p, std::size_t n);
+void memTrackerSampleFree(void* p);
+
+inline void memTrackerOnAlloc(void* p, std::size_t n) {
+ if (gMemTrackerOff) [[likely]] {
+ return;
+ }
+ if (gInMemTracker || !p) {
+ return;
+ }
+ if (--gMemTrackerCounter > 0 && n < gForceSampleBytes) {
+ return;
+ }
+ MemTrackerSuppress _suppress;
+ memTrackerSampleAlloc(p, n);
+}
+
+inline void memTrackerOnFree(void* p) {
+ if (gInMemTracker || !p) {
+ return;
+ }
+ // Cheap cache-line-shared read: when the tracker is disabled there is no
+ // live-block table to debit, so skip all lock/table work.
+ if (!g_memTrackerEnabled.value.load(std::memory_order_relaxed)) {
+ return;
+ }
+ MemTrackerSuppress _suppress;
+ memTrackerSampleFree(p);
+}
+
+// Periodic dump — emits one TraceEvent("MemoryTrackerSite") per site whose
+// estLiveBytes (or estCumulativeBytes when MEMORY_TRACKING_LIVE_TRACKING is off)
+// exceeds bytesThreshold. The threshold is compared against the sampling-corrected
+// estimate, not the raw sampled bytes. Each site event carries an "AddrCmd"
+// detail: a ready-to-paste addr2line invocation covering just that site's frames.
+// A final TraceEvent("MemoryTrackerSummary") reports aggregate totals.
+void memTrackerDump(int64_t bytesThreshold);
+
+// Snapshot iteration for tests. The callback runs while a copy of the
+// aggregation table is held; the spinlock is not held during the callback.
+void memTrackerForEachSite(std::function cb);
+
+// Reset all state. Tests only — not safe for production use.
+void memTrackerResetForTest();
+
+// Test only: arm a one-shot so the next sampled allocation simulates a tracker
+// metadata-allocation failure (the sampled path throws std::bad_alloc, which the
+// tracker swallows). Lets tests verify the tracker fails open — the underlying
+// allocation still succeeds and tracking recovers afterward. Never used in
+// production.
+void memTrackerFailNextSampleForTest();
+
+#else // !FDB_MEMORY_TRACKER — compiled out: hooks are no-ops, no operator new override.
+
+inline void memTrackerInit() {}
+inline void memTrackerOnAlloc(void*, std::size_t) {}
+inline void memTrackerOnFree(void*) {}
+inline void memTrackerDump(int64_t) {}
+inline void memTrackerResetForTest() {}
+class MemTrackerSuppress {
+public:
+ MemTrackerSuppress() {}
+ ~MemTrackerSuppress() {}
+ MemTrackerSuppress(const MemTrackerSuppress&) = delete;
+ MemTrackerSuppress& operator=(const MemTrackerSuppress&) = delete;
+};
+
+#endif // FDB_MEMORY_TRACKER
+
+#endif // FLOW_MEMORY_TRACKER_H
diff --git a/flow/include/flow/Platform.h b/flow/include/flow/Platform.h
index 1d5f41b509..bf40534f99 100644
--- a/flow/include/flow/Platform.h
+++ b/flow/include/flow/Platform.h
@@ -95,6 +95,20 @@
#error Missing force inline
#endif
+// Keep a function un-inlined and (on GCC) un-cloned so its address stays stable
+// for return-address matching in tests. GCC -O3 IPA cloning (.constprop/.isra)
+// otherwise runs the code under a synthetic clone symbol at a different address
+// than &fn, so a captured frame won't fall in [&fn, &fn+size); noclone disables
+// it. clang lacks noclone and doesn't clone this way, so it gets noinline only;
+// empty elsewhere (e.g. MSVC, which we cannot compile-test).
+#if defined(__clang__)
+#define force_noinline __attribute__((noinline))
+#elif defined(__GNUC__)
+#define force_noinline __attribute__((noinline, noclone))
+#else
+#define force_noinline
+#endif
+
/*
* Visual Studio (.NET 2003 and beyond) has an __assume compiler
* intrinsic to hint to the compiler that a given condition is true