diff --git a/CMakeLists.txt b/CMakeLists.txt index 29643264fc..0e67235020 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -438,6 +438,7 @@ set(SeQuant_eval_src SeQuant/core/eval/eval_expr.hpp SeQuant/core/eval/eval_node.hpp SeQuant/core/eval/eval_node_compare.hpp + SeQuant/core/eval/node_batch_annotation.hpp SeQuant/core/eval/result.cpp SeQuant/core/eval/result.hpp SeQuant/core/eval/fwd.hpp diff --git a/SeQuant/core/batch_policy.hpp b/SeQuant/core/batch_policy.hpp index 48c3c5cd45..482ec8faf8 100644 --- a/SeQuant/core/batch_policy.hpp +++ b/SeQuant/core/batch_policy.hpp @@ -3,6 +3,7 @@ #include #include +#include namespace sequant { @@ -12,7 +13,39 @@ class Tensor; /// One batchability policy shared by the single-term optimizer and the runtime /// batched evaluator (make_evaluator, Task A3). All predicates default empty. struct BatchPolicy { - std::function is_batchable_index = {}; + /// Spaces batchable in the CONTRACTED role: a mode of such a space is + /// batchable where it is summed. Companion to \ref + /// is_batchable_external_index (the EXTERNAL role). Splitting batchability by + /// role lets a caller admit a space only where batching it is meaningful -- + /// e.g. a space batchable only as an external spectator contributes none of + /// its contracted occurrences to the optimizer's 2^m search. Building block; + /// the derived "batchable in any role" query is \ref is_batchable_index(). + /// Defaults to decline every index; a caller opts spaces in explicitly. + std::function is_batchable_contracted_index = + [](Index const&) { return false; }; + /// Spaces batchable in the EXTERNAL role: a mode of such a space is batchable + /// where it is open on the term root (a spectator carried to the result), not + /// where it is contracted. Building block; declared adjacent to its + /// contracted companion. Defaults to decline every index; a caller that wants + /// external batching sets this predicate explicitly (there is no fallback to + /// the contracted role). + std::function is_batchable_external_index = + [](Index const&) { return false; }; + + /// Derived "batchable in ANY role": the union of the two building-block + /// predicates. This is NEVER a settable field -- it is computed from + /// \ref is_batchable_contracted_index and \ref is_batchable_external_index. + /// The runtime batched evaluator's accept predicate is this union (a mode is + /// accepted at runtime if it is batchable in either role); the factorizer's + /// role filters instead consume the individual building blocks. The building + /// blocks default-decline, so both are always callable here. + std::function is_batchable_index() const { + auto contracted = is_batchable_contracted_index; + auto external = is_batchable_external_index; + return [contracted, external](Index const& ix) { + return contracted(ix) || external(ix); + }; + } /// Per-index per-batch slice size (in elements) for a batchable index -- an /// UPPER BOUND, not a goal. Both the single-term optimizer and the runtime /// batched evaluator treat it as a ceiling: the realized whole-tile batch is @@ -21,6 +54,41 @@ struct BatchPolicy { std::function batch_target_size = {}; std::function is_volatile_leaf = {}; + /// If true, an external/spectator index -- open on the whole network's result + /// yet contracted at no node -- is eligible for batching; its per-slice size + /// comes from \c batch_target_size(ix) like any batchable index. Default + /// false = no spectator batching (byte-identical to non-spectator behavior). + /// Necessary but not sufficient: spectator axes are emitted only under a + /// TIME-FIRST objective (DenseTimeSpaceBatched) and only when the selected + /// root's modeled peak exceeds \c peak_threshold. Spectator batching is + /// therefore currently unavailable under the space-first objectives. + bool batch_spectator_indices = false; + + /// Enable the order-aware multilevel recompute cost model (resident-scan peak + /// + ordered-key flops recompute). SELECTION knob ONLY: it makes the DP + /// charge recompute realistically and thus pick a different (better-batching) + /// factorization. It does NOT control external-mode EMISSION -- that is the + /// independent \ref node_level_placement. Consulted only by the batched + /// objectives (threaded via CostParams). Default TRUE: the recompute-aware + /// model is the more realistic cost for selection. This is SAFE precisely + /// because it is now selection-only -- the node-level emission it used to + /// force is separately gated by \ref node_level_placement (default off), so + /// the emission stays the correct, cheap root-level forest seed. (Before the + /// decouple, defaulting this true forced the node-level runtime regression.) + bool order_aware_recompute = true; + + /// Emission-placement knob for external (spectator) modes, INDEPENDENT of the + /// order-aware cost model. Only meaningful with \ref batch_spectator_indices. + /// If true, the emit uses node-level placement (per-node External stamps); if + /// false (default) it uses the root-level forest seed (one global spectator + /// loop). Node-level placement is currently a net runtime REGRESSION -- ~6x + /// wall time and ~8x batch scopes on water-8, and it produces a wrong + /// residual on water-20 -- because it nests a batch scope at every carrying + /// node and the batched evaluator replays each. It stays OFF by default until + /// that is fixed; the root-seed emission is correct and cheap regardless of + /// order_aware_recompute. + bool node_level_placement = false; + /// If true, restrict batching to persistent (amplitude-independent) subtrees, /// declining to batch any subtree that contains a volatile leaf. If false /// (the default), batch ACROSS THE BOARD: slicing the batch axis shrinks any @@ -40,6 +108,57 @@ struct BatchPolicy { /// accumulator + contribution co-residency of a node that contracts a /// batchable index. double accumulation_factor = 0.0; + + /// Coexistence switch (Task 6 of the whole-scope batched DAG execution + /// design, `doc/dev/specs/2026-08-10-whole-scope-batched-dag-execution- + /// design.md`) between the two RUNTIME EXECUTION MODELS: forest descent + /// (default false -- one tree at a time, `sequant::evaluate(Nodes const&, + /// ...)`, unchanged) and whole-scope descent (true -- one fused scope-tree + /// walk over the whole forest, `sequant::eval::evaluate_whole_scope`, so a + /// value shared across trees is built once per home block and reused, + /// rather than rebuilt per tree). Consulted by the `sequant::evaluate( + /// Nodes const&, BatchPolicy const&, ...)` driver overload + /// (`scope_executor.hpp`) to select the driver, and by + /// `sequant::eval::dryrun::cost_profile()` to select the matching peak + /// model: the co-residency oracle (`peak_profile_sweep` over `home_modes`) + /// when true, since that model is what predicts the whole-scope realized + /// peak, vs the batched-scratch replay high-watermark (models forest + /// descent) when false. Purely additive: false reproduces today's behavior + /// on both call sites byte-for-byte. + bool whole_scope_execution = false; + + /// SP3 gating switch (`doc/dev/specs/2026-08-05-dryrun-wetrun-schedule- + /// equivalence-design.md` follow-on, the ordered-scope batched-eval + /// design): between forest descent / whole-scope descent (both selected + /// above via \ref whole_scope_execution) and the new ORDERED executor + /// (true -- `sequant::eval::evaluate_ordered_schedule`, driven by the SP2 + /// `eval::OrderedSchedule` IR rather than the narrow `ScopeSchedule` scope + /// tree). Consulted by the `sequant::evaluate(Nodes const&, BatchPolicy + /// const&, ...)` driver overload (`scope_executor.hpp`) BEFORE \ref + /// whole_scope_execution, so it takes priority when both are set (setting + /// both is well-defined: the ordered executor wins) and the two pre- + /// existing dispatch arms (forest descent / whole-scope descent) are + /// reached, byte-identically, only when this flag is false. Default false + /// reproduces today's dispatch byte-for-byte on every existing caller. + bool ordered_schedule_execution = false; + + /// Peak-memory budget in BYTES for the batched objectives. Its meaning + /// DIFFERS between them: + /// + /// - SPACE-FIRST (DenseSpaceTimeBatched): a hard feasibility gate. The + /// single-term optimizer minimizes flops among schedules whose modeled peak + /// is <= peak_threshold, falling back to min-peak (best effort) when none + /// fit. Default +infinity => every schedule feasible => min flops => no + /// batching, i.e. here a finite value is the *enable* trigger for batching. + /// + /// - TIME-FIRST (DenseTimeSpaceBatched): NOT a feasibility gate. Root + /// selection ignores it entirely (peak breaks exact flop ties only), so it + /// can neither constrain the schedule's peak nor enable CONTRACTED-axis + /// batching (which is emitted regardless). Its ONLY effect is to trigger + /// EXTERNAL (spectator) axis emission, together with + /// \c batch_spectator_indices: axes are emitted iff the selected root's + /// modeled peak exceeds this threshold. + double peak_threshold = std::numeric_limits::infinity(); }; } // namespace sequant diff --git a/SeQuant/core/eval/backends/dryrun/cost_model_object.hpp b/SeQuant/core/eval/backends/dryrun/cost_model_object.hpp new file mode 100644 index 0000000000..e73e7d259d --- /dev/null +++ b/SeQuant/core/eval/backends/dryrun/cost_model_object.hpp @@ -0,0 +1,262 @@ +#ifndef SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_COST_MODEL_OBJECT_HPP +#define SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_COST_MODEL_OBJECT_HPP + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace sequant::eval::dryrun { + +/// +/// \brief Opt-in accumulator for the REPLAY-tallied (recompute-aware) cost. +/// +/// The static per-node cost walk in \c cost_profile() reports order-/batching- +/// blind DP-model quantities (\c CostProfile::model_flops etc.): each internal +/// node is priced ONCE, so the walk never sees the per-occ-block REPLAY +/// recompute the batched evaluator incurs at runtime. This sink is the +/// replay-side counterpart: when a non-null \c CostSink is attached to the +/// \c CostModel shared by every dry-run \c Result token, each ACTUAL product-op +/// execution during the \c Trace::On replay folds its own SLICED-extent cost +/// here (see \c CostModel::tally_op and \c DryRunOps::prod). Because a sliced, +/// occ-DEPENDENT op executed N times does ~1/N work each pass, its sliced-cost +/// sum is work-neutral (~= its unsliced cost); only the occ-INDEPENDENT work +/// re-executed at full size once per block inflates -- so the totals here +/// isolate the recompute the model walk cannot. +/// +/// Mirrors \c sequant::eval::PeakSink (eval.hpp): an OPTIONAL sink, defaulting +/// off, so the production runtime path (which never constructs a dry-run \c +/// CostModel) is byte-identical. The atomics let a fold from a concurrent +/// evaluator stay correct, though \c cost_profile() itself is single-threaded. +/// +/// Per-node AVOIDABLE-recompute tally, keyed by the LABEL signature (result + +/// operand indices). Avoidable recompute is measured in FLOPs against the +/// BATCHING-FREE (unlimited-memory) ideal, where each distinct value is built +/// ONCE at full extent and reused: \c total_flops accumulates the actual +/// (possibly sliced) FLOPs over every build of this value; \c full_flops is the +/// FLOPs to build it once at FULL extent (constant per label). The rollup takes +/// avoidable = max(0, total_flops - full_flops) -- the arithmetic batching +/// repeats beyond building the value once, which is exactly the recompute +/// hoisting exists to avoid. +/// +/// FLOPs (unlike roofline exec) is LINEAR in extents, hence ADDITIVE across +/// slices: disjoint slices that tile the full value sum to exactly \c +/// full_flops +/// => 0 avoidable (tiling repeats no arithmetic), while a value rebuilt full +/// once per block sums to N*full => (N-1)*full avoidable. That additivity is +/// why no slice-context bucketing is needed and why the pathological >100% +/// roofline-spread of the exec-weighted metric cannot arise. +/// +/// NOTE: the per-DISTINCT-value avoidable rollup does NOT live here. It is kept +/// by \c CacheManager::recompute_tally(), keyed by the exact cache node +/// identity (TreeNodeHasher + TreeNodeEqualityComparator) so 64-bit hash +/// collisions are not folded; a string-keyed sink here could not reproduce that +/// identity (the node type is kept out of this header by the +/// dryrun/eval_expr.hpp -> cost_model_object.hpp include cycle). This sink +/// carries only the whole-forest scalar totals (flops/exec/n_ops via \c +/// tally_op). +struct CostSink { + std::atomic flops{0.0}; + std::atomic exec{0.0}; + std::atomic n_ops{0}; +}; + +/// Per-index extent OVERRIDE table: narrows specific indices (by identity, so +/// it survives reshaping across prod/sum/permute -- the same shared/ +/// tensor MODE POSITION (0-based, in the value's canon index order) to a +/// runtime-realized element count. Positional -- NOT keyed by Index -- because +/// a DAG value has no intrinsic labels: only an op binds labels to it, so the +/// only stable handle on a mode across ops is its position. Populated by +/// Result::slice_mode()/mode_batches() call sites (see result.hpp); empty => +/// no override, the regime's nominal extent applies. This table -- not a +/// second cost model -- is what lets a zero-data DryRun Result report the +/// REALIZED (possibly runtime-sliced) size rather than always the full +/// regime extent, which is exactly the signal Task 6's replay witnesses. +/// Consumers that operate in LABEL space (flops, keyed by the op's annotation) +/// resolve positions to labels through that annotation first. +using ExtentOverrides = container::map; + +/// +/// \brief Bundles the optimizer's own cost closures (memsize/flops/roofline) +/// behind one value type so dry-run Results report MODEL size (not an +/// allocated size), and the harness can additionally read FLOPs and +/// projected execution cost per operation. +/// +/// This is a thin wrapper: all arithmetic is delegated verbatim to +/// \c sequant::opt::detail::memsize_counter / \c flops_counter / \c +/// roofline_op_cost (see \c core/optimize/single_term_detail.hpp and \c +/// core/optimize/cost_model.hpp) -- no parallel cost model is implemented +/// here. The only thing this class adds is the ExtentOverrides indirection: +/// each query builds a fresh (cheap; no heap allocation beyond the closure +/// itself) index-to-extent callable that consults \p overrides before +/// falling back to the SizeRegime's nominal extent, then hands that callable +/// to the counter. +/// +class CostModel { + public: + explicit CostModel(SizeRegime regime, RooflineParams roofline = {}) + : regime_{std::move(regime)}, roofline_{roofline} {} + + /// + /// \brief Bytes for a tensor with these (literal, canon-order) indices, + /// honoring any per-index extent override (a runtime slice_mode()/ + /// mode_batches() narrowing). + /// + /// Delegates the extent-product / composite-moment math to \c + /// memsize_counter, invoked with \p idxset as the sole (`lhs`) operand and + /// empty `rhs`/`result` -- an empty operand's tot_indices() split + /// accumulates the starting product of 1.0, which memsize_counter itself + /// special-cases to contribute zero bytes, so this reproduces exactly the + /// single-operand byte count \c memsize_counter is designed to report per + /// operand. + /// + [[nodiscard]] std::size_t memsize( + container::svector const& idxset, + ExtentOverrides const& overrides = {}) const { + // Resolve the POSITIONAL overrides against THIS index list: override at + // mode position `pos` applies to `idxset[pos]`, whatever its label. Build a + // per-call Index->extent map so make_extent_fn's atom lookup finds it (the + // counter revisits idxset[pos] by identity, incl. as a composite proto). + // Named local: make_extent_fn captures it by reference, so it must outlive + // `ext` (a temporary here would dangle). + auto const resolved = resolve_overrides(idxset, overrides); + auto const ext = make_extent_fn(resolved); + auto const mc = + sequant::opt::detail::memsize_counter(ext, regime_.inner_pow_fn()); + double const elems = + mc(idxset, container::svector{}, container::svector{}); + return static_cast(elems * numeric_size_); + } + + /// + /// \brief Multiply-add count for a contraction whose free (result) indices + /// are \p out and whose contracted (summed-over) indices are + /// \p contracted. + /// + /// Delegates to \c flops_counter, which prices the union of its (lhs, rhs, + /// result) arguments; passing (\p out, \p contracted, {}) makes that union + /// exactly `out U contracted` -- the full index set touched by the + /// contraction, since by construction `contracted` holds precisely the + /// indices present in both operands but absent from the result. + /// + /// \p label_extents maps an ANNOTATION label (an Index appearing in \p out or + /// \p contracted) to its runtime-realized (sliced) extent. Unlike the value's + /// positional \c ExtentOverrides, this is keyed by Index because \p out / + /// \p contracted ARE labels -- the op's annotation is the sole source of + /// labels. \c DryRunOps::prod builds it from each operand's positional + /// overrides via that operand's annotation (see \c extents_by_label). + [[nodiscard]] double flops( + container::svector const& out, + container::svector const& contracted, + container::map const& label_extents = {}) const { + auto const ext = make_extent_fn(label_extents); + auto const fc = + sequant::opt::detail::flops_counter(ext, regime_.inner_pow_fn()); + return fc(out, contracted, container::svector{}); + } + + /// + /// \brief Roofline-projected execution cost of one contraction (see + /// \c sequant::opt::detail::roofline_op_cost). + /// + /// \p left_bytes / \p right_bytes are operand footprints in BYTES (as + /// reported by \c Result::size_in_bytes()); converted to elements (the + /// counter's native unit) via \c numeric_size before delegating. + /// + [[nodiscard]] double exec_cost(double flops_count, std::size_t left_bytes, + std::size_t right_bytes) const { + double const traffic_elems = + static_cast(left_bytes + right_bytes) / numeric_size_; + return sequant::opt::detail::roofline_op_cost( + flops_count, traffic_elems, roofline_.machine_balance, + roofline_.fast_mem_elems, roofline_.block_tiles, + roofline_.block_prefactor); + } + + [[nodiscard]] SizeRegime const& regime() const noexcept { return regime_; } + + /// + /// \brief Attach (or detach with nullptr) the optional replay cost sink. + /// + /// Const because the \c CostModel is shared as \c shared_ptr + /// by every dry-run \c Result token; \c cost_profile() sets this on its one + /// shared model just before the \c Trace::On replay so each product op can + /// fold into it. The pointee (a \c CostSink) is external and owns the mutable + /// state; this only records where to fold. Off by default => no fold => the + /// dry-run backend is byte-identical when unused. + /// + void set_cost_sink(CostSink* sink) const noexcept { sink_ = sink; } + + /// + /// \brief Fold one product op's SLICED-extent \p flops_count / \p exec into + /// the attached sink (no-op when none is attached). + /// + /// Called at each actual product execution in the replay, so a contraction + /// re-executed once per occ block is tallied once per block at its sliced + /// size -- exactly the recompute signal (see \c CostSink). + /// + void tally_op(double flops_count, double exec) const noexcept { + if (!sink_) return; + sink_->flops.fetch_add(flops_count, std::memory_order_relaxed); + sink_->exec.fetch_add(exec, std::memory_order_relaxed); + sink_->n_ops.fetch_add(1, std::memory_order_relaxed); + } + + private: + // Index-to-extent callable consulting `overrides` first, else the + // regime's nominal extent. The returned std::function captures `overrides` + // (and `this`) BY REFERENCE and is only ever used -- never stored -- + // within the (memsize/flops) call that constructs it, so the reference + // stays valid for its entire lifetime. Explicit (non-deduced) return type + // so this can be called from memsize()/flops(), which appear earlier in + // the class body (a deduced `auto` return type would require the + // definition to precede every use, even within the same class). + [[nodiscard]] std::function make_extent_fn( + container::map const& overrides) const { + return [this, &overrides](Index const& ix) -> std::size_t { + if (auto it = overrides.find(ix); it != overrides.end()) + return it->second; + return regime_.extent(ix); + }; + } + + // Resolve a value's POSITIONAL overrides against its own index list: override + // at mode position `pos` binds to `idxset[pos]`. Yields an Index-keyed map so + // make_extent_fn's per-atom lookup finds the sliced extent wherever that + // Index recurs in idxset (including as a composite's outer proto), exactly as + // the pre-positional Index-keyed table did -- but now the key is derived from + // THIS list, not carried from a producer's labels. + [[nodiscard]] static container::map resolve_overrides( + container::svector const& idxset, + ExtentOverrides const& overrides) { + container::map out; + for (auto const& [pos, w] : overrides) + if (pos < idxset.size()) out.emplace(idxset[pos], w); + return out; + } + + SizeRegime regime_; + RooflineParams roofline_; + // Optional replay cost sink (see set_cost_sink/tally_op). Mutable so it can + // be (de)attached on a shared_ptr; a raw non-owning pointer + // to caller-owned state. nullptr (default) => tally_op is a no-op. + mutable CostSink* sink_ = nullptr; + // sizeof(double); see doc/dev/plans/2026-07-04-dryrun-eval-backend.md Task 2 + // note on OptimizeOptions::numeric_size (hardcoded here, matching the C60 + // trace's real-only CSV-CCk path; complex CSV-CCk is out of scope, see the + // plan's carried-minor N4). + double numeric_size_ = 8.0; +}; + +} // namespace sequant::eval::dryrun + +#endif // SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_COST_MODEL_OBJECT_HPP diff --git a/SeQuant/core/eval/backends/dryrun/cost_profile.hpp b/SeQuant/core/eval/backends/dryrun/cost_profile.hpp new file mode 100644 index 0000000000..b826d0d7cb --- /dev/null +++ b/SeQuant/core/eval/backends/dryrun/cost_profile.hpp @@ -0,0 +1,625 @@ +#ifndef SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_COST_PROFILE_HPP +#define SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_COST_PROFILE_HPP + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace sequant::eval::dryrun { + +/// Configuration for a faithful (gated) dry-run cache: the same footprint gate +/// and cross-occurrence batch-variant veto the real batched eval loop applies, +/// so a batch-variant giant (a `mu~`/`K`-carrying DF intermediate) is NOT +/// cached whole but recomputed sliced under each consumer's batch trigger. +/// +/// The element types mirror the gated \c sequant::cache_manager overload +/// (\c cache_manager.hpp): \c is_volatile is invoked on every \c TreeNode +/// (deduced as \c EvalNodeDryRun for the dry-run backend). +/// +/// This struct lives here (rather than only in the test) because Task 4's +/// \c cost_profile() entry point consumes it. +struct CacheConfig { + /// Footprint gate (bytes): a node whose result footprint exceeds this is not + /// cached. 0 (default) disables the gate. + double max_footprint = 0.; + /// Minimum non-persistent repeats to cache an internal node (CSE rule). + std::size_t min_repeats = 2; + /// `bool(EvalNodeDryRun const&)`: true if the node is intrinsically volatile + /// (typically the amplitude leaves). Empty => nothing is volatile. + std::function is_volatile; +}; + +/// Builds a gated dry-run cache from an eval-node range, a \p cfg, and a +/// \p regime that supplies the moment-aware node-size model used for the +/// footprint gate. +/// +/// The footprint functor sizes a node's result (its \c canon_indices()) with +/// the SAME moment-aware counter the DryRun \c Result uses +/// (\c memsize_counter over \c regime.idx_to_extent()/inner_pow_fn()), scaled +/// to bytes, so the gate compares like-for-like against \c cfg.max_footprint. +/// +/// Unlike the SIMPLE \c cache_manager(nodes) factory the ad-hoc dry-run test +/// sites use, this routes through the GATED overload so free-batchable-axis +/// giants are vetoed (matching the real run). The returned cache is used across +/// the WHOLE forest without a per-summand reset (matching a real solve's +/// whole-iteration cache scope; \c cost_profile relies on the lifetime mask to +/// release each value after its last cross-term use, so cross-summand-shared +/// values are reused, not rebuilt). +/// +/// \param nodes the evaluation forest (a range of \c EvalNodeDryRun). +/// \param cfg footprint/repeat/volatility/batchability configuration. +/// \param regime the size regime supplying extents and CSV moment tables. +/// \return a \c CacheManager over \c EvalNodeDryRun. +template +auto build_dryrun_cache(NodeRange const& nodes, CacheConfig const& cfg, + SizeRegime const& regime) { + auto memsize = sequant::opt::detail::memsize_counter(regime.idx_to_extent(), + regime.inner_pow_fn()); + + // Footprint (bytes) of a node's RESULT: canon_indices() fed to the + // moment-aware counter (as the counter's `result` slot; the empty lhs/rhs + // contribute nothing) times 8 bytes/element. Same arithmetic as the DryRun + // Result::size_in_bytes(), so the gate is faithful. + auto footprint_of = + [memsize = std::move(memsize)](EvalNodeDryRun const& n) -> double { + std::vector const result(n->canon_indices().begin(), + n->canon_indices().end()); + return memsize(std::vector{}, std::vector{}, result) * 8.0; + }; + + // Default the volatility predicate so the gated factory never invokes an + // empty std::function (nothing volatile leaves that gate inert, matching the + // factory's own default). + std::function is_volatile = + cfg.is_volatile ? cfg.is_volatile + : std::function( + [](EvalNodeDryRun const&) { return false; }); + + // Note: the gated sequant::cache_manager() overload below stamps the + // cross-occurrence lifetime mask on `nodes` itself before its DAG walk / + // veto (cache_manager.hpp), so this call site does not need to do so. + return sequant::cache_manager(nodes, std::move(is_volatile), cfg.min_repeats, + std::move(footprint_of), cfg.max_footprint); +} + +/// One recomputed value's avoidable-recompute breakdown (see +/// \c CostProfile::avoidable_nodes). \c label is the value's signature (a +/// `result;lhs;rhs` full-label string built in \c DryRunOps::prod). \c flops is +/// the avoidable recompute in FLOPs -- `total_flops - full_flops`, the +/// arithmetic the batched replay repeated beyond building this value ONCE at +/// full extent (the batching-free / unlimited-memory ideal). \c count is the +/// equivalent number of extra full rebuilds (`total_flops/full_flops - 1`): 0 +/// when the builds tile the value once (disjoint slices), ~N-1 when the value +/// is rebuilt full N times (an un-hoisted invariant). Only values with a +/// positive avoidable FLOP count are recorded. +struct AvoidableNode { + std::string label; + double count = 0; + double flops = 0; +}; + +/// Roll a replay's per-DISTINCT-value build tally (a \c CacheManager's +/// \c recompute_tally(), populated by \c CacheManager::tally_build from the +/// eval loop's product-build site during a \c Trace::On replay) into the +/// avoidable-recompute breakdown: per distinct value, avoidable FLOPs = +/// max(0, total_flops - full_flops), sorted by avoidable FLOPs descending, +/// keeping only values with a positive amount. The tally is keyed by the EXACT +/// cache identity (TreeNodeHasher + TreeNodeEqualityComparator = topological +/// hash bin + Bliss connectivity 3-way cmp + recursive child compare), so two +/// topologically-distinct nodes sharing a 64-bit hash are NOT folded (a +/// hash-string key folds them, inventing avoidable recompute; a space/arity +/// structural key can't separate same-shape different-connectivity nodes +/// either), and per-block / alpha-renamed builds of ONE value ARE folded. +/// Shared by \c cost_profile() (whole-forest rollup) and any caller that drives +/// its own tally-enabled replay (e.g. the schedule-dump test). \c label is the +/// value's topological hash as a string (the join key the IR and run-event +/// nodes carry). Single-threaded caller expected: the replay has finished. +template +inline std::vector avoidable_nodes_from_tally( + TallyMap const& tally) { + // Per DISTINCT value, rolled up over its SLICES (see BuildTally): for each + // slice, total += builds*cost and build_once += cost, so avoidable (the + // arithmetic the replay repeated beyond building each distinct slice once) is + // sum over slices of (builds-1)*cost. A value tiled over DISTINCT slices has + // builds==1 per slice => 0 avoidable (tiling, even non-uniform); a value + // rebuilt at the SAME slice (e.g. an invariant rebuilt every block of a loop + // it does not carry) has builds>1 there => that slice's (builds-1)*cost is + // avoidable. No full-extent denominator is used; every number is actual + // replay FLOPs. + auto roll = [](auto const& t) { + double total = 0, once = 0, extra_builds = 0; + for (auto const& [sig, bc] : t.slices) { + total += bc.count * bc.flops; + once += bc.flops; + extra_builds += static_cast(bc.count - 1); + } + return std::tuple{total, once, extra_builds}; + }; + std::vector out; + for (auto const& [node, t] : tally) { + auto const [total, once, extra_builds] = roll(t); + double const avoidable = total - once; + if (avoidable <= 0.0) continue; + out.push_back( + {std::to_string(node->hash_value()), extra_builds, avoidable}); + } + std::sort(out.begin(), out.end(), + [](AvoidableNode const& a, AvoidableNode const& b) { + return a.flops > b.flops; + }); + + // DIAGNOSTIC (SEQUANT_AVOIDABLE_DEBUG): per-value FLOP accounting, worst + // first. total == build_once => every slice built once (tiling, 0 avoidable); + // total >> build_once => some slice rebuilt (invariant recompute). + if (std::getenv("SEQUANT_AVOIDABLE_DEBUG")) { + double sum_total = 0, sum_avoid = 0; + for (auto const& [node, t] : tally) { + auto const [total, once, extra] = roll(t); + (void)extra; + sum_total += total; + if (total > once) sum_avoid += total - once; + } + std::fprintf(stderr, + "[avoidable-debug] dryrun_flops=%.6g avoidable_flops=%.6g " + "frac=%.4f n_values=%zu\n", + sum_total, sum_avoid, + sum_total > 0 ? sum_avoid / sum_total : 0, tally.size()); + // Worst offenders by avoidable flops: builds = total builds over slices, + // slices = distinct slices, so builds>>slices is genuine same-slice + // recompute (an invariant rebuilt every block), builds==slices is pure + // tiling. + std::vector> + ranked; // {avoid, builds, slices, total, once} + for (auto const& [node, t] : tally) { + auto const [total, once, extra] = roll(t); + (void)extra; + if (total <= once) continue; + std::size_t builds = 0; + for (auto const& [sig, bc] : t.slices) builds += bc.count; + ranked.emplace_back(total - once, builds, t.slices.size(), total, once); + } + std::sort(ranked.begin(), ranked.end(), [](auto const& a, auto const& b) { + return std::get<0>(a) > std::get<0>(b); + }); + std::size_t shown = 0; + for (auto const& [av, builds, nslices, total, once] : ranked) { + if (shown++ >= 20) break; + std::fprintf(stderr, + " builds=%zu slices=%zu total=%.4g build_once=%.4g " + "avoid=%.4g\n", + builds, nslices, total, once, av); + } + } + return out; +} + +/// Summary of the modeled cost of a factorized dry-run eval forest, as produced +/// by \c cost_profile(). All quantities are summed/maxed over every summand +/// tree in the forest. +struct CostProfile { + /// Predicted peak working-set (bytes). Task 6 (whole-scope batched DAG + /// execution design) makes this model SELECTED by \c + /// BatchPolicy::whole_scope_execution, since the two runtime drivers + /// realize different co-residency: + /// + /// - \p policy.whole_scope_execution == false (default, forest descent): + /// the max over summands of the batched-scratch high-watermark folded by + /// the Task-3 \c PeakSink and the outer gated cache's \c + /// working_set_hwmark() -- unchanged from before this field's Task-6 + /// selection existed. See the paragraphs below for its accounting detail. + /// - \p policy.whole_scope_execution == true (whole-scope descent): the + /// CO-RESIDENCY oracle (\c eval::peak_profile_sweep over \c + /// eval::compute_dag_path's \c home_modes-based footprints), computed + /// ONCE over the whole fused forest rather than per-summand. Per the + /// design's "paradox resolved" section, this is the model that MATCHES + /// the realized whole-scope peak (forest descent never co-resides + /// cross-tree, so the batched-scratch replay watermark below is the wrong + /// oracle once execution actually routes through \c + /// eval::evaluate_whole_scope). + /// + /// The remainder of this doc comment describes the flag-OFF (default) + /// accounting; it is unaffected by the flag. + /// + /// This ACCOUNTS FOR co-resident residency across the scope chain, rather + /// than being the max-of-independent-hwmarks lower bound it was before: each + /// per-op hwmark folded into a cache's \c working_set_hwmark_ (in eval.hpp) + /// already adds \c CacheManager::chain_residency() of that cache's + /// scope-chain ancestors at the instant of the op, so a scratch cache's + /// high-watermark is the max over its life of (scratch residency + + /// everything alive up its parent chain at that instant) -- the co-resident + /// sum when a persistent cross-term cache entry is alive at the same instant + /// as a batched-inner transient. The outer (root) cache has no parent, so + /// its own hwmark is unaffected (added term is 0); the \c max() fold here + /// (over \c peak.load() and \c cache.working_set_hwmark()) is then correct in + /// both regimes -- batched (peak.load() already carries the co-resident + /// scratch peak) and unbatched (peak.load() == 0, the outer hwmark alone is + /// the peak). + /// + /// Each live buffer is counted exactly once: the per-op operand guards skip + /// an operand whose buffer any cache on the scope chain already holds + /// (\c CacheManager::chain_holds(), pointer identity), so a value read full + /// from an ANCESTOR cache is counted only via \c chain_residency() and not + /// again as an operand, while a sliced/permuted/phase-shifted read (a + /// distinct buffer) is correctly added. + /// + /// One known deviation remains -- an UNDER-count that keeps this a lower + /// bound in exactly one place: the external-scatter accumulator (\c dest in + /// eval.hpp) is a plain local, not a cache entry, so it is invisible to + /// \c chain_residency() even though it co-resides with every inner block's + /// working set. Pre-existing, outside this field's scope. + double peak_bytes = 0; + /// STATIC per-node DP-MODEL FLOPs: summed unweighted static contraction FLOPs + /// over all internal nodes, from the order-/batching-blind static walk. NOT + /// CSE-aware across summands (a cross-term shared intermediate is walked, and + /// its FLOPs counted, once per occurrence, not once overall), and NOT + /// replay-aware: each node is priced exactly once, so this never reflects the + /// per-occ-block recompute the batched replay incurs. Compare against \c + /// dryrun_flops: the two are ~equal when no batching engages, and \c + /// dryrun_flops exceeds this by the recompute factor when it does. + double model_flops = 0; + /// STATIC per-node DP-MODEL roofline exec cost, summed over all internal + /// nodes. Same per-occurrence (not CSE-deduplicated), batching-blind caveat + /// as \c model_flops. + double model_exec = 0; + /// Number of internal (contraction) nodes across the forest (static count). + std::size_t model_n_ops = 0; + /// REPLAY-tallied (recompute-aware) FLOPs: the sum, over every ACTUAL product + /// op executed in the \c Trace::On replay, of that op's SLICED-extent flops + /// (folded via the \c CostSink attached to the shared \c CostModel; see + /// result.hpp \c DryRunOps::prod). A batched op re-executed once per occ + /// block is charged once per block at its sliced size, so occ-DEPENDENT work + /// stays ~work-neutral while occ-INDEPENDENT recompute (persistent + /// intermediates / leaf re-materializations re-run at full size per block) + /// inflates -- making this the metric that PREDICTS batched-replay + /// overcompute (e.g. occ-batching being ~Nx aux-only). Equal to \c + /// model_flops (up to the product-op-only tally) when no batching engages. + double dryrun_flops = 0; + /// REPLAY-tallied roofline exec cost (traffic-dominated, the better wall-time + /// proxy), summed per product-op execution. Same recompute semantics as \c + /// dryrun_flops. + double dryrun_exec = 0; + /// Number of product-op EXECUTIONS in the replay (counts re-executions per + /// batch block), so it grows with recompute -- unlike \c model_n_ops. + std::size_t dryrun_n_ops = 0; + + /// Per-value avoidable-recompute breakdown: one entry per DISTINCT value + /// whose batched replay repeated arithmetic beyond building it ONCE at full + /// extent + /// (`total_flops > full_flops`) -- the recompute a hoist would have avoided. + /// Sorted by avoidable FLOPs descending. Empty when batching repeats no + /// arithmetic (every value is built at most once-worth, e.g. disjoint slices + /// tiling it). See \c DryRunOps::prod (per-build tally) and the post-replay + /// rollup in \c cost_profile(). + std::vector avoidable_nodes; + /// DIAGNOSTIC: per-value (label signature) build-once FLOPs from the replay, + /// so a caller can join per node (by the SAME signature the schedule Build + /// event carries) against an independent per-node model and localize any + /// per-node flops disagreement. Populated from the CostSink's per_node map. + std::map sig_full_flops; + /// Total avoidable recompute in FLOPs (sum of \c avoidable_nodes[i].flops): + /// arithmetic the batched replay repeated beyond the build-once ideal. + /// Compare against \c dryrun_flops for the avoidable FRACTION (see \c + /// avoidable_time()). Zero when batching repeats no arithmetic. + double avoidable_flops = 0; + /// Total avoidable recompute expressed as equivalent extra full rebuilds (sum + /// of \c avoidable_nodes[i].count). + double avoidable_ops = 0; + + /// Avoidable FRACTION of replay arithmetic: \c avoidable_flops / \c + /// dryrun_flops (0 when no ops ran), in [0, 1] by construction. The + /// single-number "how much of the batched replay's arithmetic was repeated + /// recompute vs. the unlimited-memory ideal" summary. (FLOPs, not roofline + /// exec: recompute is repeated WORK, and FLOPs -- being linear in extents -- + /// makes disjoint slicing exactly free and keeps this bounded.) + [[nodiscard]] double avoidable_time() const { + return dryrun_flops > 0 ? avoidable_flops / dryrun_flops : 0.0; + } +}; + +/// Replays a factorized eval forest zero-data through the real eval loop -- +/// with a gated cache built from \p cfg (Task 2) and a \c PeakSink threaded +/// through the batched evaluator (Task 3) -- and, alongside, does a static walk +/// of the forest to accumulate FLOPs / roofline exec cost / op count. This is +/// the single reusable entry point both SeQuant tests and MPQC call. +/// +/// \par The printing gate +/// \c CacheManager::working_set_hwmark() only accumulates while +/// \c sequant::eval::log::printing() is true (the hwmark update sits on the +/// trace-printing path). This routine therefore FORCES the eval logger's level +/// > 0 around the replay -- discarding the narrow trace to a null sink when no +/// \p trace is requested -- and restores the previous logger state afterward, +/// so \c peak_bytes is non-zero even with no trace stream. +/// +/// \par Global state / threading +/// This routine mutates the process-global \c Logger::instance().eval state +/// (\c level and \c stream) for the duration of the replay (restored on every +/// exit path, including exceptions). Because that state is a singleton shared +/// by the whole process, \c cost_profile() MUST be called single-threaded -- +/// e.g. as a pre-flight step before, or a post-hoc step after, the real +/// multi-threaded eval -- never concurrently with other code that reads or +/// writes \c Logger::instance().eval (including another concurrent +/// \c cost_profile() call). +/// +/// \par FLOPs / exec accounting (model vs dryrun) +/// \c CostProfile::model_flops and \c CostProfile::model_exec are accumulated +/// by a STATIC walk that sums a contribution per BINARIZED internal node of the +/// forest; they are NOT CSE-aware across summands (a shared intermediate that +/// recurs across summand trees, or multiple times within one, is counted once +/// per occurrence) and NOT replay-aware (each node is priced exactly once, +/// blind to order/batching). \c CostProfile::dryrun_flops / \c dryrun_exec / +/// \c dryrun_n_ops are the recompute-aware counterparts, tallied from the +/// \c Trace::On replay below: every ACTUAL product-op execution folds its +/// SLICED-extent cost into a \c CostSink attached to the shared \c CostModel, +/// so an op re-executed once per batch block is counted once per block. When no +/// batching engages the two agree (up to the product-op-only dryrun tally); +/// when it does, \c dryrun_* exceeds \c model_* by the recompute factor -- a +/// TIME (arithmetic) cost that \c peak_bytes, a SPACE (co-resident working +/// set) measure, does not express. +/// +/// \param forest per-summand optimized+binarized eval forest (the real IR). +/// \param policy the batch policy driving the replay evaluator; its accept is +/// the derived role union \c policy.is_batchable_index(). +/// \param cfg gated-cache config (footprint gate, volatile, repeats). +/// \param regime the size regime supplying extents and CSV moment tables; +/// the internal \c CostModel and \c DryRunLeafEvaluator are built from +/// it. +/// \param trace optional per-op trace sink (nullptr = no trace). When +/// non-null, the eval loop's narrow trace is transcoded (UTF-8) into it. +/// \param router optional placement router (see \c placement_router.hpp), +/// attached to the replay cache right after it is built. Every current +/// caller omits this (nullptr, the default), which leaves the router +/// seam in \c evaluate() inert -- byte-identical to before this +/// parameter existed. Phase 2 wires a router only from test call sites. +/// \return the accumulated \c CostProfile. +inline CostProfile cost_profile( + std::vector const& forest, BatchPolicy const& policy, + CacheConfig const& cfg, SizeRegime const& regime, + std::wostream* trace = nullptr, + PlacementRouter const* router = nullptr, + sequant::eval::ScheduleSink* schedule_sink = nullptr) { + CostProfile profile; + + auto cm = std::make_shared(regime); + DryRunLeafEvaluator const leaf{cm}; + + // ---- static cost walk (independent of the replay) -------------------- + // For every internal node: flops = flops_counter(left, right, result); the + // roofline exec cost uses the left operand's footprint as the transferred + // bytes and the arena convention (4096) the [dryrun-costmodel] test fixes. + auto const flops_of = sequant::opt::detail::flops_counter( + regime.idx_to_extent(), regime.inner_pow_fn()); + std::function walk = + [&](EvalNodeDryRun const& n) { + if (n.leaf()) return; + profile.model_n_ops += 1; + double const node_flops = + flops_of(n.left()->canon_indices(), n.right()->canon_indices(), + n->canon_indices()); + profile.model_flops += node_flops; + container::svector const left(n.left()->canon_indices().begin(), + n.left()->canon_indices().end()); + profile.model_exec += + cm->exec_cost(node_flops, cm->memsize(left), 4096); + walk(n.left()); + walk(n.right()); + }; + for (auto const& root : forest) walk(root); + + // ---- peak replay through the real eval loop -------------------------- + // The cache's batch-variant veto is driven by the cross-occurrence lifetime + // mask (stamped inside build_dryrun_cache -> cache_manager); the replay + // EVALUATOR's accept is the derived role union, applied inside + // make_evaluator(policy) via policy.is_batchable_index(). + auto cache = build_dryrun_cache(forest, cfg, regime); + // Enable the per-DISTINCT-value recompute tally on the (root) cache: the eval + // loop's product-build site records each build against the node's identity + // here (CacheManager::tally_build), keyed by the exact cache identity, for + // the avoidable rollup below. Off by default so the wet eval path never + // populates it; only this costing replay opts in. + cache.set_recompute_tally_enabled(true); + // Null (default) => every existing caller's replay is unaffected, since + // set_placement_router(nullptr) is exactly the cache's own default. + cache.set_placement_router(router); + // Route this replay's SCHEDULE_RUN_EVENT records to the caller's sink (if + // any). Null (default) => no dump, byte-identical. The wet-run sets an + // equivalent sink on its own eval cache, so the two batched schedules can be + // captured and diffed for structural equivalence. + cache.set_schedule_sink(schedule_sink); + + auto& logger = Logger::instance(); + // RAII guard restoring the process-global Logger::eval state on EVERY exit + // path from this point on -- normal return, early return, or an exception + // unwinding out of the replay loop below -- not just the two trailing + // assignments a plain save/restore would rely on. Without this, a throw + // from anything in the loop OTHER than evaluate (e.g. + // std::bad_alloc from make_evaluator/set_custom_evaluator/ + // working_set_hwmark/cache.reset()) would unwind past the local + // `trace_capture` destructor while `logger.eval.stream` still points at it, + // leaving a dangling pointer in the process-global singleton with + // level == 2 still set. + struct LoggerEvalGuard { + decltype(logger.eval)& eval; + std::size_t const prev_level; + std::ostream* const prev_stream; + ~LoggerEvalGuard() { + eval.level = prev_level; + eval.stream = prev_stream; + } + } logger_eval_guard{logger.eval, logger.eval.level, logger.eval.stream}; + + // Force printing() on so working_set_hwmark() accumulates. The eval logger + // stream is narrow; capture into a narrow buffer only when a (wide) trace + // sink was requested, else discard to a null stream. + std::ostringstream trace_capture; + logger.eval.level = 2; + logger.eval.stream = trace ? &trace_capture : nullptr; + + // Attach a replay cost sink to the shared CostModel so each product op + // executed in the Trace::On replay below folds its SLICED-extent cost here + // (DryRunOps::prod -> CostModel::tally_op). Only DryRun Results built from + // this same `cm` fold in, and only while the sink is attached, so this + // records exactly the replay recompute for THIS forest. Detached on every + // exit path by the guard below (the model outlives the results, but leaving a + // dangling sink pointer set would be a latent hazard if `cm` were reused). + CostSink costsink; + cm->set_cost_sink(&costsink); + struct CostSinkGuard { + CostModel const& cm; + ~CostSinkGuard() { cm.set_cost_sink(nullptr); } + } costsink_guard{*cm}; + + std::atomic peak{0.0}; + for (auto const& root : forest) { + cache.set_custom_evaluator(sequant::make_evaluator( + policy, leaf, sequant::make_no_scope_guard{}, &peak)); + try { + (void)sequant::evaluate(root, leaf, cache); + } catch (std::exception const&) { + // A zero-data DryRun sizing throw must not mask the peak read. + } + // Fold the outer cached residency BEFORE reset() (which zeroes the + // hwmark). `peak` folds every batched scratch high-watermark across all + // summands via std::max, so its running load() is the global scratch peak. + // Only feeds profile.peak_bytes when the co-residency oracle (below) is + // NOT selected -- see CostProfile::peak_bytes's doc comment (Task 6): + // this replay watermark models forest descent, the co-residency oracle + // models whole-scope descent, and the two are mutually exclusive + // predictors, not folded together. + if (!policy.whole_scope_execution) + profile.peak_bytes = std::max({profile.peak_bytes, peak.load(), + double(cache.working_set_hwmark())}); + // NO per-term reset. A real solve's cache spans the whole iteration (all + // summands + equations) and reuses cross-summand values, evicting only by + // the lifetime mask stamped over the whole forest. Resetting between terms + // would instead drop non-persistent scratch after each summand, REBUILDING + // a cross-summand-shared value in every summand -- over-counting recompute + // (and mis-estimating peak) vs any real run. So the replay keeps the shared + // cache across the whole forest; the lifetime mask releases each value + // after its last cross-term use. (This makes the dry-run schedule match the + // wet run's; see doc/dev/specs/2026-08-05-...schedule-equivalence.) + } + + // Task 6 (whole-scope batched DAG execution design): under + // policy.whole_scope_execution, replace the per-summand replay watermark + // folded above (skipped, see the guard inside the loop) with the + // CO-RESIDENCY oracle computed ONCE over the WHOLE fused forest -- the + // model that matches the peak sequant::eval::evaluate_whole_scope actually + // realizes (see CostProfile::peak_bytes's doc comment). block_of mirrors + // the batch-partition source the whole-scope driver itself uses (see + // sequant::evaluate(Nodes const&, BatchPolicy const&, ...), + // scope_executor.hpp): policy.batch_target_size, guarded the same way + // (empty => decline batching, size 1) so an unset policy never throws + // std::bad_function_call out of compute_dag_path. + if (policy.whole_scope_execution) { + std::function const block_of = + policy.batch_target_size + ? policy.batch_target_size + : std::function( + [](Index const&) -> std::size_t { return 1; }); + auto const dag = compute_dag_path(forest, *cm, block_of); + profile.peak_bytes = peak_profile_sweep(dag).peak_bytes; + } + + // Read the replay-tallied (recompute-aware) totals the sink accumulated over + // every product op of every summand's Trace::On replay. (costsink_guard + // detaches the sink from `cm` on function exit.) + profile.dryrun_flops = costsink.flops.load(std::memory_order_relaxed); + profile.dryrun_exec = costsink.exec.load(std::memory_order_relaxed); + profile.dryrun_n_ops = costsink.n_ops.load(std::memory_order_relaxed); + + // Per-value avoidable-recompute rollup (shared with the schedule-dump + // emitter): for each DISTINCT value the replay built, avoidable FLOPs = the + // actual replay FLOPs of every slice rebuilt beyond once (see + // avoidable_nodes_from_tally() and CacheManager::BuildTally). + auto const& tally = cache.recompute_tally(); + profile.avoidable_nodes = avoidable_nodes_from_tally(tally); + for (auto const& an : profile.avoidable_nodes) { + profile.avoidable_flops += an.flops; + profile.avoidable_ops += an.count; + } + // DIAGNOSTIC: per-DISTINCT-value build-once flops (sum over its DISTINCT + // slices of one build's cost), for external per-node join and the build-once + // identity check (sum == dryrun_flops - avoidable_flops). Keyed by a unique + // running index prefixed to the node hash so two nodes that share a 64-bit + // hash (the case this whole tally keying exists to separate) still get + // distinct map entries and the sum stays exact. + { + std::size_t idx = 0; + for (auto const& [node, t] : tally) { + double once = 0.0; + for (auto const& [sig, bc] : t.slices) once += bc.flops; + profile.sig_full_flops.emplace( + std::to_string(idx++) + ":" + std::to_string(node->hash_value()), + once); + } + } + + // logger_eval_guard's destructor restores logger.eval.{level,stream} at + // function exit (see above); no manual restore needed here. + + // If a wide trace sink was requested, transcode the captured narrow (UTF-8) + // eval trace into it (the eval loop writes only to the narrow logger stream; + // index labels such as mu~/K are multi-byte, so a plain widen would corrupt + // them -- decode UTF-8 to code points instead). + if (trace) { + std::string const s = trace_capture.str(); + std::wstring w; + w.reserve(s.size()); + for (std::size_t i = 0; i < s.size();) { + unsigned char const c = static_cast(s[i]); + char32_t cp; + std::size_t len; + if (c < 0x80) { + cp = c; + len = 1; + } else if ((c >> 5) == 0x6) { + cp = c & 0x1Fu; + len = 2; + } else if ((c >> 4) == 0xE) { + cp = c & 0x0Fu; + len = 3; + } else if ((c >> 3) == 0x1E) { + cp = c & 0x07u; + len = 4; + } else { + cp = c; // invalid lead byte: pass through + len = 1; + } + for (std::size_t k = 1; k < len && i + k < s.size(); ++k) + cp = (cp << 6) | (static_cast(s[i + k]) & 0x3Fu); + w.push_back(static_cast(cp)); + i += len; + } + *trace << w; + } + + return profile; +} + +} // namespace sequant::eval::dryrun + +#endif // SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_COST_PROFILE_HPP diff --git a/SeQuant/core/eval/backends/dryrun/eval_expr.hpp b/SeQuant/core/eval/backends/dryrun/eval_expr.hpp new file mode 100644 index 0000000000..a7bcad51f9 --- /dev/null +++ b/SeQuant/core/eval/backends/dryrun/eval_expr.hpp @@ -0,0 +1,87 @@ +#ifndef SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_EVAL_EXPR_HPP +#define SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_EVAL_EXPR_HPP + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +namespace sequant::eval::dryrun { + +/// +/// \brief Extends EvalExpr with an annot() method so DryRun eval nodes can be +/// evaluated. +/// +/// Unlike \c EvalExprTAPP (opaque \c int64_t hashes of index labels -- see +/// \c backends/tapp/eval_expr.hpp), DryRun's annotation IS the plain literal +/// (canon-order) index list itself: \c Result::prod/sum/permute need each +/// index's actual space/extent (via \c CostModel), not just its identity, to +/// compute a modeled size. +/// +class EvalExprDryRun final : public EvalExpr { + public: + using annot_t = dryrun::annot_t; // container::svector + + template >> + explicit EvalExprDryRun(Args&&... args) + : EvalExpr{std::forward(args)...} { + annot_ = canon_indices() | ranges::to; + } + + /// + /// \return Annotation (container::svector) for DryRun tensors. + /// + [[nodiscard]] annot_t const& annot() const noexcept { return annot_; } + + private: + annot_t annot_; +}; + +/// Type alias for DryRun evaluation nodes +using EvalNodeDryRun = EvalNode; + +static_assert(meta::eval_node); +static_assert(meta::can_evaluate); + +/// +/// \brief Leaf yielder: turns each IR leaf (a tensor/constant/variable node) +/// into a zero-data DryRun Result. This is the `F` in +/// \c evaluate(node, layout, F, cache). +/// +/// A tensor leaf's literal (canon-order) index list decides flat vs nested: +/// \c make_dryrun_result builds a flat \c ResultDryRun if none of the leaf's +/// indices are proto-indexed, or a nested \c ResultDryRunNested (a CSV/PNO +/// amplitude or coefficient) if any are -- and threads that SAME literal list +/// through as the nested result's canon-order position map, so a later +/// \c slice_mode()/\c mode_batches() call (which the batched runtime only +/// ever issues against a LEAF's result) resolves its positional `mode` +/// argument correctly regardless of the leaf's flat/nested-ness. +/// +struct DryRunLeafEvaluator { + std::shared_ptr cm; + + [[nodiscard]] ResultPtr operator()(EvalNodeDryRun const& leaf) const { + SEQUANT_ASSERT(leaf.leaf()); + if (!leaf->is_tensor()) { + // Constant / Variable leaf: a bare scalar. No real numeric value is + // ever tracked by this zero-data backend (only sizes/costs), so 1.0 is + // a placeholder never meant to be read as a physical result. + return eval_result>(1.0); + } + container::svector idx = leaf->canon_indices() | ranges::to; + return make_dryrun_result(std::move(idx), cm); + } +}; + +} // namespace sequant::eval::dryrun + +#endif // SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_EVAL_EXPR_HPP diff --git a/SeQuant/core/eval/backends/dryrun/meter.hpp b/SeQuant/core/eval/backends/dryrun/meter.hpp new file mode 100644 index 0000000000..dbafeabef3 --- /dev/null +++ b/SeQuant/core/eval/backends/dryrun/meter.hpp @@ -0,0 +1,363 @@ +#ifndef SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_METER_HPP +#define SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_METER_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace sequant::eval::dryrun { + +/// +/// \brief One value's build-vs-home fidelity witness for a \c MeterReport +/// (see \c assemble_report): how many times a distinct value was built (over +/// its whole recompute tally) versus WHERE it is homed and WHERE it is used, +/// read off the matching \c RichSchedule::ValueCell (looked up by hash). +/// +struct HomeFidelity { + std::string label; ///< value signature "idx:hash" (idx disambiguates a + ///< 64-bit hash collision between distinct nodes; + ///< see cost_profile.hpp's sig_full_flops) + std::size_t hash = 0; ///< the value's EvalExpr::hash_value() + std::size_t builds = 0; ///< total builds across slices (recompute-aware) + std::string home; ///< dag-scope of home_modes ("" == root {} -- a + ///< whole-nest invariant) + std::string uses; ///< dag-scope list of the value's occurrences +}; + +/// +/// \brief Summary of one metered dry-run (or wet) replay: the hierarchy-wide +/// peak (from a wired \c PeakMonitor), the persistent/volatile FLOPs and +/// CostModel exec-time split (rolled up from a \c CacheManager's recompute +/// tally, classified by \c compute_volatility), the total build count, and +/// the per-value build-vs-home fidelity list (see \c HomeFidelity). +/// +struct MeterReport { + double peak_bytes = 0; ///< PeakMonitor high-water (dense on dry + ///< run / sparse on wet run) + std::size_t peak_op_hash = 0; ///< location (op hash) of the peak + + double flops_persistent = 0, flops_volatile = 0; ///< dense model; dry-only + double cost_persistent = 0, cost_volatile = 0; ///< CostModel exec; dry-only + + std::size_t builds_total = 0; + + std::vector home_fidelity; ///< sorted: builds desc + + bool whole_scope = false; ///< which executor this report describes +}; + +/// +/// \brief Bottom-up memoized volatility over an evaluation \p forest: a node +/// is volatile iff \p is_volatile flags it directly, or (for an internal +/// node) either child is volatile -- the SAME rule the gated +/// \c sequant::cache_manager factory applies while building its NV/V +/// frontier (see cache_manager.hpp's DAG walk). Keyed by +/// \c TreeNode::hash_value() (rather than the node identity itself) so a +/// caller can classify a \c CacheManager::recompute_tally() entry -- keyed by +/// the SAME node identity but not necessarily the SAME node object -- by its +/// hash. +/// +/// \param forest the evaluation forest (a range of eval nodes). +/// \param is_volatile `bool(TreeNode const&)`: true if the node is +/// intrinsically volatile. Only its value on leaves matters in +/// practice (volatility propagates up), but it is consulted on every +/// node, matching \c cache_manager's gated factory. +/// \return a map from \c hash_value() to whether that value is volatile. +/// +template +std::unordered_map compute_volatility( + Forest const& forest, IsVolatile const& is_volatile) { + using Node = std::ranges::range_value_t; + + std::unordered_map volatile_of; + + auto visit = [&](auto&& self, Node const& n) -> bool { + std::size_t const h = n->hash_value(); + if (auto it = volatile_of.find(h); it != volatile_of.end()) + return it->second; + bool v; + if (n.leaf()) { + v = is_volatile(n); + } else { + bool const vl = self(self, n.left()); + bool const vr = self(self, n.right()); + v = is_volatile(n) || vl || vr; + } + volatile_of.emplace(h, v); + return v; + }; + + for (auto const& tree : forest) visit(visit, tree); + return volatile_of; +} + +/// +/// \brief Assemble a \c MeterReport from a metered replay: a walked +/// \p cache (its \c recompute_tally() populated by \c CacheManager:: +/// tally_build over the replay), the hierarchy-wide \p mon (\c PeakMonitor), +/// the \p rich linearized schedule (\c compute_dag_boulevard over the SAME +/// \p forest, supplying each value's home/use dag-scope), and \p is_volatile +/// (fed to \c compute_volatility to classify each distinct value). +/// +/// Per distinct value (one \c cache.recompute_tally() entry): \c builds is +/// the sum, over its slices, of each slice's build count; \c node_flops / +/// \c node_exec are the sum, over its slices, of build-count times that +/// slice's actual (flops, exec). The value is classified persistent/volatile +/// by \c compute_volatility's verdict for its hash and folded into the +/// matching \c MeterReport::flops_*/cost_* accumulator. Its \c HomeFidelity +/// entry's \c home/uses are read off the \p rich cell sharing its hash (empty +/// if the value has no matching cell, e.g. a leaf never realized as its own +/// distinct product build). +/// +/// \param cache the (root) cache whose \c recompute_tally() was populated by +/// a \c Trace::On metered replay. +/// \param mon the \c PeakMonitor wired onto \p cache's scope chain during the +/// replay. +/// \param rich the linearized schedule (\c compute_dag_boulevard) over the +/// SAME forest the replay walked. +/// \param forest the evaluation forest (fed to \c compute_volatility). +/// \param is_volatile `bool(TreeNode const&)`: intrinsic volatility +/// predicate, as for \c compute_volatility. +/// \param whole_scope which executor this report describes (stashed verbatim +/// into \c MeterReport::whole_scope). +/// \return the assembled \c MeterReport. +/// +template +MeterReport assemble_report(Cache const& cache, PeakMonitor const& mon, + RichSchedule const& rich, Forest const& forest, + IsVolatile const& is_volatile, bool whole_scope) { + MeterReport report; + report.whole_scope = whole_scope; + report.peak_bytes = static_cast(mon.hwmark_bytes); + report.peak_op_hash = mon.peak.op_hash; + + auto const volatility = compute_volatility(forest, is_volatile); + + // hash -> ValueCell* lookup, mirroring make_node_meta's map build + // (scope_executor.hpp): rich.cells is a flat vector, not keyed by hash. + std::unordered_map cell_by_hash; + cell_by_hash.reserve(rich.cells.size()); + for (auto const& cell : rich.cells) cell_by_hash.emplace(cell.hash, &cell); + + // dag-scope formatting: comma-joined IndexSpace base_keys, no trailing + // comma -- the same convention make_node_meta uses (scope_executor.hpp). + auto const dag_scope = [](auto const& modes) { + std::string s; + for (auto const& m : modes) { + if (!s.empty()) s += ","; + s += toUtf8(m.space().base_key()); + } + return s; + }; + + std::size_t idx = 0; + for (auto const& [node, tally] : cache.recompute_tally()) { + std::size_t builds = 0; + double node_flops = 0.0, node_exec = 0.0; + for (auto const& [sig, rec] : tally.slices) { + builds += rec.count; + node_flops += static_cast(rec.count) * rec.flops; + node_exec += static_cast(rec.count) * rec.exec; + } + report.builds_total += builds; + + std::size_t const hash = node->hash_value(); + bool const is_vol = [&] { + auto it = volatility.find(hash); + return it != volatility.end() && it->second; + }(); + + if (is_vol) { + report.flops_volatile += node_flops; + report.cost_volatile += node_exec; + } else { + report.flops_persistent += node_flops; + report.cost_persistent += node_exec; + } + + HomeFidelity hf; + hf.label = std::to_string(idx++) + ":" + std::to_string(hash); + hf.hash = hash; + hf.builds = builds; + if (auto it = cell_by_hash.find(hash); it != cell_by_hash.end()) { + auto const* cell = it->second; + hf.home = dag_scope(cell->home_modes); + container::svector uses_modes; + for (auto const& occ : cell->occurrences) + for (auto const& [mode, range] : occ.ectx) uses_modes.push_back(mode); + hf.uses = dag_scope(uses_modes); + } + report.home_fidelity.push_back(std::move(hf)); + } + + std::sort(report.home_fidelity.begin(), report.home_fidelity.end(), + [](HomeFidelity const& a, HomeFidelity const& b) { + return a.builds > b.builds; + }); + + return report; +} + +/// +/// \brief Runs the policy-selected executor (whole-scope or per-tree forest +/// descent, per \p policy.whole_scope_execution) over \p forest through the +/// DryRun sizing backend, metering the replay with a fresh, \c PeakMonitor +/// -wired, build-tallying cache, and returns the assembled \c MeterReport. +/// +/// Mirrors MPQC's wet dispatch: this drives the SAME Task-6 coexistence entry +/// point (\c sequant::evaluate(Nodes const&, BatchPolicy const&, layout, F, +/// CacheManager&, mode_order, ScopeGuardFactory), \c scope_executor.hpp) a +/// real solve would use under \p policy -- whole-scope and forest descent are +/// selected by the SAME flag, not two independently maintained code paths -- +/// so the metered replay is exactly the run \p policy describes, not a +/// hand-rolled proxy of it. Non-throwing wrapper (if desired) is the +/// caller's responsibility; an exception from the replay propagates out of +/// this call, but the RAII logger-state guard still restores +/// \c Logger::instance().eval on the way out. +/// +/// \param forest the evaluation forest (a range of \c EvalNodeDryRun). +/// \param policy the batch policy driving the coexistence entry -- in +/// particular \c whole_scope_execution (executor selection) and +/// \c batch_target_size (the batch-partition source; also the source +/// of the \c block_of function this call builds its OWN \c rich +/// schedule with, for \c assemble_report -- the coexistence entry +/// builds an independent, internal \c RichSchedule of its own from +/// the SAME \p policy.batch_target_size to drive the executor). +/// \param regime the size regime supplying the DryRun \c CostModel. +/// \param cfg cache configuration (footprint gate, min repeats, volatility) +/// for the metered cache, built exactly as \c build_dryrun_cache does +/// (same footprint arithmetic, same is_volatile default) -- NOT via +/// that builder directly, since its is_volatile default (substituted +/// for an empty \c cfg.is_volatile) is internal to it and would +/// otherwise be invisible to \c assemble_report below, which also +/// needs a callable predicate (an empty \c cfg.is_volatile passed to +/// it directly throws \c std::bad_function_call from +/// \c compute_volatility). The SAME locally-defaulted predicate is +/// used for both. +/// \param router optional placement override, installed on the metered cache +/// when non-null. +/// \param trace optional sink for the eval trace; when non-null, +/// \c Logger::instance().eval.stream is redirected there for the +/// duration of the call (restored on exit, along with the elevated +/// \c eval.level and the installed \c eval.node_meta). +/// \return the assembled \c MeterReport (peak, persistent/volatile +/// FLOPs+time, build-vs-home fidelity), stamped with +/// \p policy.whole_scope_execution. +/// +inline MeterReport meter( + std::vector const& forest, BatchPolicy const& policy, + SizeRegime const& regime, CacheConfig const& cfg, + PlacementRouter const* router = nullptr, + std::ostream* trace = nullptr) { + auto cm = std::make_shared(regime); + DryRunLeafEvaluator yield{cm}; + + // Default is_volatile the SAME way build_dryrun_cache does (an empty + // cfg.is_volatile means nothing is volatile) -- but keep the defaulted + // function LOCAL rather than routing through that builder, so the exact + // same predicate can also be threaded to assemble_report below. + std::function const is_volatile = + cfg.is_volatile ? cfg.is_volatile + : std::function( + [](EvalNodeDryRun const&) { return false; }); + + // Footprint (bytes) of a node's RESULT, identical to build_dryrun_cache's + // footprint_of (cost_profile.hpp): the moment-aware memsize counter over + // canon_indices(), scaled to bytes, so cfg.max_footprint gates like-for-like. + auto memsize = sequant::opt::detail::memsize_counter(regime.idx_to_extent(), + regime.inner_pow_fn()); + auto footprint_of = + [memsize = std::move(memsize)](EvalNodeDryRun const& n) -> double { + std::vector const result(n->canon_indices().begin(), + n->canon_indices().end()); + return memsize(std::vector{}, std::vector{}, result) * 8.0; + }; + + auto cache = + sequant::cache_manager(forest, is_volatile, cfg.min_repeats, + std::move(footprint_of), cfg.max_footprint); + cache.set_recompute_tally_enabled(true); + if (router) cache.set_placement_router(router); + + PeakMonitor mon; + cache.set_peak_monitor(&mon); + + // The SAME block_of source the coexistence entry itself derives from + // policy.batch_target_size (scope_executor.hpp's evaluate(Nodes const&, + // BatchPolicy const&, ...)) -- an empty batch_target_size means "no + // batching", guarded identically so compute_dag_boulevard never invokes an + // empty std::function. + std::function const block_of = + policy.batch_target_size + ? policy.batch_target_size + : std::function( + [](Index const&) -> std::size_t { return 1; }); + RichSchedule const rich = compute_dag_boulevard(forest, *cm, block_of); + + // RAII save/restore of every Logger::eval field this call touches, so an + // exception from the replay below still leaves the process-wide Singleton + // exactly as this call found it. + auto& logger = Logger::instance(); + struct LoggerStateGuard { + Logger& l; + std::size_t prev_level; + std::ostream* prev_stream; + std::function prev_node_meta; + ~LoggerStateGuard() { + l.eval.level = prev_level; + l.eval.stream = prev_stream; + l.eval.node_meta = std::move(prev_node_meta); + } + } guard{logger, logger.eval.level, logger.eval.stream, logger.eval.node_meta}; + + // Ensure printing() so DryRunOps::prod records flops/exec (feeding + // cache.tally_build) and note_working_set() actually observes the + // PeakMonitor -- without raising the level any HIGHER than a caller who + // already wants a louder trace. + logger.eval.level = std::max(logger.eval.level, 1); + if (trace) logger.eval.stream = trace; + logger.eval.node_meta = make_node_meta(rich); + + // Forest descent (whole_scope_execution == false) needs the SAME batched + // custom evaluator MPQC's wet forest path installs (cck.ipp's `else` + // branch, `cache.set_custom_evaluator(sequant::make_evaluator(ctx. + // batch_policy, yielder, make_scope_guard))`): without it, plain + // sequant::evaluate(Nodes const&, ...) ignores every batched_here() + // stamp and runs an unbatched, no-schedule single pass -- an infidelity + // vs. the wet run this meter is supposed to mirror. Installed ONLY on + // this branch: evaluate_impl consults cache.custom_evaluator() on every + // non-leaf node, so installing it unconditionally would also fire on the + // whole-scope path's invariant-root evaluate_impl calls (evaluate_whole_ + // scope, not the batched custom evaluator, must drive those). + if (!policy.whole_scope_execution) + cache.set_custom_evaluator( + sequant::make_evaluator(policy, yield, sequant::make_no_scope_guard{})); + + (void)sequant::evaluate(forest, policy, std::wstring{}, yield, + cache, {}, sequant::make_no_scope_guard{}); + + return assemble_report(cache, mon, rich, forest, is_volatile, + policy.whole_scope_execution); +} + +} // namespace sequant::eval::dryrun + +#endif // SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_METER_HPP diff --git a/SeQuant/core/eval/backends/dryrun/result.hpp b/SeQuant/core/eval/backends/dryrun/result.hpp new file mode 100644 index 0000000000..dc95309275 --- /dev/null +++ b/SeQuant/core/eval/backends/dryrun/result.hpp @@ -0,0 +1,713 @@ +#ifndef SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_RESULT_HPP +#define SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_RESULT_HPP + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace sequant::eval::dryrun { + +/// +/// \brief Annotation type DryRun's Result ops decode from the eval engine's +/// std::any [l,r,res] / [pre,post] triples/pairs. +/// +/// Unlike \c EvalExprTAPP (opaque \c int64_t index-label hashes -- see +/// \c backends/tapp/eval_expr.hpp), DryRun's annotation IS the plain literal +/// (canon-order) index list itself: \c Result::prod/sum/permute need each +/// index's actual space/extent (via \c CostModel), not just an opaque +/// identity, to compute a modeled size. +/// +using annot_t = container::svector; + +/// Per-mode assembled element coverage recorded by write_into_slice(): maps an +/// outer mode position to the contiguous `[lo, hi)` element range filled so far +/// by scattered blocks. Lets a zero-data DryRun destination report the REALIZED +/// (assembled) size along a partitioned mode and detect gaps/overlaps between +/// blocks -- the assemble-side analogue of ExtentOverrides for slice_mode(). +using AssembledCoverage = + container::map>; + +class ResultDryRun; +class ResultDryRunNested; + +/// +/// \brief Builds whichever concrete DryRun Result type matches \p idx's +/// content: a nested \c ResultDryRunNested if any index in \p idx is +/// proto-indexed (a CSV/PNO composite leg, e.g. a CSV amplitude's PNO +/// domain leg `a_1`), otherwise a flat \c ResultDryRun. +/// +/// Dispatch is by CONTENT of the decoded result annotation, not by either +/// operand's concrete type -- exactly mirroring how the real eval engine +/// itself decides tensor-of-tensor-ness (\c EvalExpr::tot(), from the same +/// proto-indexed-leg criterion). This is what lets \c prod()/sum() freely +/// combine a flat operand (e.g. a bare 3-center DF integral) with a nested +/// one (e.g. a CSV/PNO coefficient), exactly as real CSV-CCSD terms do, +/// without either side needing to know the other's concrete type. +/// +[[nodiscard]] inline ResultPtr make_dryrun_result( + container::svector idx, std::shared_ptr cm, + ExtentOverrides overrides = {}); + +namespace detail { + +[[nodiscard]] inline bool has_proto(container::svector const& idx) { + return std::any_of(idx.begin(), idx.end(), + [](Index const& ix) { return ix.has_proto_indices(); }); +} + +[[nodiscard]] inline ExtentOverrides merge_overrides(ExtentOverrides const& a, + ExtentOverrides const& b) { + ExtentOverrides out = a; + for (auto const& [pos, n] : b) out[pos] = n; + return out; +} + +// Remap positional overrides from one annotation's mode positions to another's +// by matching labels: position p (labeled `from[p]`) -> the position of that +// same label in `to`. A position whose label is absent from `to` (e.g. an index +// this op contracts away) is dropped -- it has no mode in the result. This is +// how a sliced mode's width survives prod/sum/permute now that the value itself +// carries no labels: the op's annotation is the only place labels live, so two +// positional maps from different operands can only be combined after both are +// projected onto a COMMON annotation (the result's). +[[nodiscard]] inline ExtentOverrides remap_overrides_by_annot( + ExtentOverrides const& ov, annot_t const& from, annot_t const& to) { + ExtentOverrides out; + for (auto const& [pos, w] : ov) { + if (pos >= from.size()) continue; + auto const it = std::find(to.begin(), to.end(), from[pos]); + if (it != to.end()) + out.emplace(static_cast(it - to.begin()), w); + } + return out; +} + +// Project a value's positional overrides into LABEL space via its annotation: +// position p -> (annot[p] -> width). For the flops call, whose out/contracted +// index sets ARE annotation labels (see CostModel::flops). +[[nodiscard]] inline container::map extents_by_label( + ExtentOverrides const& ov, annot_t const& annot) { + container::map out; + for (auto const& [pos, w] : ov) + if (pos < annot.size()) out.emplace(annot[pos], w); + return out; +} + +// Uniform read access to a DryRun Result's (index list, overrides, cost +// model) regardless of which concrete DryRun type `r` is. `is()`/`as()` +// are public Result methods, so no friendship is needed; declared here (and +// defined below, after both concrete classes) purely because their bodies +// need the concrete classes' definitions. +[[nodiscard]] container::svector indices_of(Result const& r); +[[nodiscard]] ExtentOverrides overrides_of(Result const& r); + +/// +/// \brief Shared op bodies for the two DryRun Result concrete types. +/// +/// Both \c ResultDryRun and \c ResultDryRunNested carry exactly an (index +/// list, ExtentOverrides, CostModel) triple and differ only in what they +/// additionally expose (\c ResultDryRunNested splits its index list into +/// outer()/inner() views for CSV-composite-aware inspection/testing). +/// Implemented once here so the two classes' prod/sum/permute/slice_mode/ +/// mode_batches bodies are one-line forwards, not near-duplicated logic. +/// +struct DryRunOps { + [[nodiscard]] static ResultPtr sum(container::svector const& idx, + ExtentOverrides const& ov, + std::shared_ptr const& cm, + Result const& other, + std::array const& annot) { + auto const a = Annot{annot}; + // Both summands share the result's index set (possibly reordered); project + // each operand's positional slice widths onto the result annotation by + // label before merging (position k is a different mode in each operand). + auto merged = merge_overrides( + remap_overrides_by_annot(ov, a.lannot, a.this_annot), + remap_overrides_by_annot(overrides_of(other), a.rannot, a.this_annot)); + return make_dryrun_result( + container::svector(a.this_annot.begin(), a.this_annot.end()), cm, + std::move(merged)); + } + + [[nodiscard]] static ResultPtr prod( + container::svector const& idx, ExtentOverrides const& ov, + std::shared_ptr const& cm, Result const& other, + std::array const& annot) { + if (other.is>()) { + // Scalar * tensor: shape (and any accumulated slicing) unchanged. + return make_dryrun_result(idx, cm, ov); + } + auto const a = Annot{annot}; + auto const other_ov = overrides_of(other); + // RESULT overrides: each operand's positional slice widths are positional + // against ITS OWN annotation (== its canon index order); project both onto + // the result's annotation by label (dropping any contracted-away mode), + // then merge. Merging the two raw positional maps directly would be wrong + // -- position k means a different mode in each operand. + auto merged = merge_overrides( + remap_overrides_by_annot(ov, a.lannot, a.this_annot), + remap_overrides_by_annot(other_ov, a.rannot, a.this_annot)); + + // Emit the cost model's OWN flops / roofline exec_cost for THIS op into the + // eval trace (gated on the eval log level), interleaved right before the + // generic engine's `Eval | Product` line for the same op. This lets trace + // post-processing weight avoidable recomputation by MODELLED TIME without + // re-deriving the cost downstream (which would silently drift from the + // model). The (out, contracted) index sets and the realized (sliced) extent + // overrides feed the same CostModel closures the static cost_profile() walk + // uses, so per-op costs are consistent with the whole-forest totals. + if (Logger::instance().eval.level > 0) { + // Cost THIS op in the einsum ANNOTATION label space (lannot/rannot/ + // this_annot), NOT the operands' stored `indices_` (idx / indices_of( + // other)). A DAG VALUE HAS NO INTRINSIC LABELS -- only ops bind labels to + // it, meaningful only within that op; a value's `indices_` merely holds + // whatever labels its PRODUCER used, which say nothing about how a + // CONSUMER binds it. The two diverge for every CSE-shared value used + // under a different binding: e.g. the (g.C)(g.C) legs are the SAME cached + // value (its `indices_` reads [i_1 i_4 K_2 a_4] from its producer) yet + // THIS op binds it as lannot=[i_2 i_3 K_2 a_3] and rannot=[i_2 i_1 K_2 + // a_1]. Deriving `contracted` from the producer-labeled stored indices + // instead of this op's annotations unions modes from different label + // contexts and exploded the flops (6.65e16 vs the correct ~1.3e13). out U + // (lannot & rannot) == lannot U rannot == the real contraction volume. + container::svector out(a.this_annot.begin(), a.this_annot.end()); + container::svector contracted; + for (auto const& ix : a.lannot) + if (std::find(a.rannot.begin(), a.rannot.end(), ix) != a.rannot.end()) + contracted.push_back(ix); + // Slice widths in LABEL space for the flops call: project each operand's + // positional overrides through its annotation. A batched label shared by + // both operands (e.g. a contracted, batched aux index) carries the same + // width from either side, so the first insertion wins harmlessly. + auto label_extents = extents_by_label(ov, a.lannot); + for (auto const& [lbl, w] : extents_by_label(other_ov, a.rannot)) + label_extents.emplace(lbl, w); + double const flops = cm->flops(out, contracted, label_extents); + sequant::eval::detail::last_op_flops() = flops; // for the Build event + double const exec = cm->exec_cost(flops, cm->memsize(idx, ov), 4096); + sequant::eval::detail::last_op_exec() = exec; // for the Build event + write_log(Logger::instance(), "OpCost", std::format(" | {}", flops), + std::format(" | {}", exec), '\n'); + // Fold this op's SLICED-extent cost into the replay cost sink, if one is + // attached (cost_profile()'s recompute-aware tally). merged/ov carry the + // runtime slicing, so a contraction re-executed once per occ block is + // charged once per block at its sliced size -- the same numbers already + // logged above, now summed. No-op (byte-identical) when unattached. + cm->tally_op(flops, exec); + // last_op_flops (set above) is THIS build's ACTUAL realized-extent cost. + // The eval loop's build choke point reads it and records it against the + // node's IDENTITY, at the (value, SLICE) granularity, in the (root) + // cache's recompute tally -- so avoidable recompute is the actual FLOPs a + // slice was rebuilt beyond once, with no ill-defined "full extent" + // denominator (slicing is non-uniform). prod cannot form the node + // identity here (it has no node, and the include cycle + // dryrun/eval_expr.hpp -> cost_model_object.hpp keeps the node type out + // of the CostSink), so the rollup is done there. See + // CacheManager::tally_build / recompute_tally(). + } + + if (a.this_annot.empty()) { + // Full contraction -> scalar. No real numeric value is ever tracked by + // this zero-data backend (only sizes/costs), so the placeholder 0.0 + // is never meant to be read as a physical result. + return eval_result>(0.0); + } + return make_dryrun_result( + container::svector(a.this_annot.begin(), a.this_annot.end()), cm, + std::move(merged)); + } + + [[nodiscard]] static ResultPtr permute( + container::svector const& idx, ExtentOverrides const& ov, + std::shared_ptr const& cm, + std::array const& ann) { + auto const post = std::any_cast(ann[1]); + // Reordering modes moves each mode's position, so the positional overrides + // must move with them: `ov` is positional against `idx` (the pre-permute + // canon order); project it onto `post` by label. + return make_dryrun_result( + container::svector(post.begin(), post.end()), cm, + remap_overrides_by_annot(ov, idx, post)); + } + + [[nodiscard]] static ResultPtr slice_mode( + container::svector const& idx, ExtentOverrides const& ov, + std::shared_ptr const& cm, std::size_t mode, + std::size_t elem_lo, std::size_t elem_hi) { + SEQUANT_ASSERT(mode < idx.size()); + auto merged = ov; + merged[mode] = elem_hi - elem_lo; // positional: mode `mode`, any label + return make_dryrun_result(idx, cm, std::move(merged)); + } + + /// Scatter \p block into the `[block_lo, block_hi)` element slice of the + /// destination's mode \p mode -- the inverse of slice_mode(). Zero-data: + /// updates only the destination's modelled size and assembled-coverage + /// bookkeeping. \p ov and \p cov are the destination's (mutated in place). + static void write_into_slice(container::svector const& idx, + ExtentOverrides& ov, AssembledCoverage& cov, + std::shared_ptr const& cm, + Result const& block, std::size_t mode, + std::size_t block_lo, std::size_t block_hi) { + SEQUANT_ASSERT(mode < idx.size()); + SEQUANT_ASSERT(block_lo < block_hi); + Index const& mix = idx[mode]; + // Tile/width consistency: the block's own modelled extent on the shared + // mode index must equal the slice width it is being written into. The + // block's overrides are positional against ITS OWN index list, so locate + // the shared index there (its mode need not equal the dest's `mode`). + auto const bov = overrides_of(block); + auto const bidx = indices_of(block); + std::size_t const block_extent = [&] { + auto const it = std::find(bidx.begin(), bidx.end(), mix); + if (it != bidx.end()) { + auto const bpos = static_cast(it - bidx.begin()); + if (auto ov_it = bov.find(bpos); ov_it != bov.end()) + return ov_it->second; + } + return cm->regime().extent(mix); + }(); + SEQUANT_ASSERT(block_extent == block_hi - block_lo); + // Merge the block's range into the assembled coverage, requiring + // contiguity: a block that neither appends after nor prepends before the + // filled range would leave a gap or overlap another block (a + // double-count). This is what makes disjoint gap-free tiling the only + // accepted assembly. + if (auto it = cov.find(mode); it == cov.end()) { + cov.emplace(mode, + std::pair{block_lo, block_hi}); + } else { + auto& lohi = it->second; + bool const append = block_lo == lohi.second; + bool const prepend = block_hi == lohi.first; + SEQUANT_ASSERT(append || prepend); + if (append) + lohi.second = block_hi; + else + lohi.first = block_lo; + } + // Reflect the assembled element width (hi - lo, lobound preserved) as the + // realized extent of the batch mode so size_in_bytes() tracks the + // reconstructed footprint. + auto const& lohi = cov.at(mode); + ov[mode] = lohi.second - lohi.first; // positional: dest mode `mode` + } + + /// Build a zero-data destination shaped like \p idx but with mode \p mode's + /// index widened to \p axis_src's FULL (unsliced) extent at \p + /// axis_src_mode -- the dry-run analogue of \c ResultTensorTA:: and \c + /// ResultTensorOfTensorTA::pre_sized_zeros_over_mode's outer-\c + /// TiledRange1 swap. \p axis_src is the unsliced carrier leaf's token (its + /// OWN recorded override for the axis index, if any, else the CostModel + /// regime's natural extent, is the "full" extent -- mirroring how the TA + /// backend reads the swapped-in dimension straight off \p axis_src rather + /// than assuming a fixed default). The width recorded here is a structural + /// (shape) fact, queryable via \c size_in_bytes()/overrides() immediately + /// -- exactly as the TA test \c batched_scratch_tot_presize_scatter checks + /// \c trange().dim(mode) right after presizing, before any block is + /// written. Once the scatter loop's \c write_into_slice() calls begin, the + /// AssembledCoverage bookkeeping there takes over the mode's reported + /// extent (the REALIZED/assembled width so far); by the time every + /// disjoint block has been written the assembled width converges back to + /// this same full extent. + [[nodiscard]] static ResultPtr pre_sized_zeros_over_mode( + container::svector const& idx, ExtentOverrides const& ov, + std::shared_ptr const& cm, std::size_t mode, + Result const& axis_src, std::size_t axis_src_mode) { + SEQUANT_ASSERT(mode < idx.size()); + auto const src_idx = indices_of(axis_src); + SEQUANT_ASSERT(axis_src_mode < src_idx.size()); + Index const& axis_ix = src_idx[axis_src_mode]; + auto const src_ov = overrides_of(axis_src); + std::size_t const full_extent = [&] { + // src_ov is positional against axis_src's own index list (src_idx). + if (auto it = src_ov.find(axis_src_mode); it != src_ov.end()) + return it->second; + return cm->regime().extent(axis_ix); + }(); + auto merged = ov; + merged[mode] = full_extent; // positional: dest mode `mode` + return make_dryrun_result(idx, cm, std::move(merged)); + } + + [[nodiscard]] static container::svector> + mode_batches(container::svector const& idx, ExtentOverrides const& ov, + std::shared_ptr const& cm, std::size_t mode, + std::size_t target_batch_size) { + SEQUANT_ASSERT(mode < idx.size()); + Index const& ix = idx[mode]; // for the regime extent / slice partition + std::size_t extent; + if (auto it = ov.find(mode); it != ov.end()) // positional override + extent = it->second; + else + extent = cm->regime().extent(ix); + + container::svector> out; + if (target_batch_size == 0 || extent == 0) { + out.push_back({0, extent}); + return out; + } + // CALLER-SUPPLIED PARTITION: if the mode's space has a recorded batch + // partition (SizeRegime::space_slice_extents), the wet backend slices this + // axis along whole TILES (mode_batches_of_trange1 reads the operand's real + // TiledRange1), so a batch boundary always falls on a tile edge and a + // (sub)range spanning N whole tiles yields N batches -- NOT extent/target + // uniform blocks. The partition slices ARE those target-grouped tile edges + // (batch_slice_extents_from_tiles applied the target once, at harvest), so + // we emit the PREFIX of partition slices that sums to `extent`: + // - outer call (ov absent, extent == full axis) => all slices, the full + // partition (e.g. aux 672 -> [168,168,168,168], 4 batches); + // - nested call (ov narrowed the axis to one outer batch, extent < full) + // => the prefix reaching that extent, so a single-tile sub-range (168) + // is ONE atomic batch, matching the wet backend, instead of being + // re-sliced into ceil(168/64)=3 uniform blocks (which then cascade). + // `target_batch_size` is not re-applied here: the partition already encodes + // it. If `extent` does not land on a partition boundary (not tile-aligned), + // fall through to uniform blocks. The dry-run stays model-agnostic -- it + // only reads slice extents; the caller decided the tiling. + auto const& slices = cm->regime().slice_extents(ix); + if (!slices.empty()) { + std::size_t lo = 0; + for (std::size_t const s : slices) { + if (lo >= extent) break; + out.push_back({lo, lo + s}); + lo += s; + } + if (lo == extent) return out; // extent tile-aligned to the partition + out.clear(); // not aligned -> uniform fallback below + } + // Fallback: uniform target_batch_size blocks (no partition recorded). + for (std::size_t lo = 0; lo < extent; lo += target_batch_size) + out.push_back({lo, std::min(extent, lo + target_batch_size)}); + return out; + } +}; + +} // namespace detail + +/// +/// \brief Flat (non-CSV) zero-data tensor token. +/// +/// Carries only its own literal outer index list (canon order -- the same +/// order \c EvalExpr::canon_indices()/annot() use, so \c slice_mode()/ +/// \c mode_batches()'s positional `mode` argument indexes it correctly), an +/// \c ExtentOverrides table recording any runtime \c slice_mode()/ +/// \c mode_batches() narrowing (keyed by Index so it survives reshaping +/// across prod/sum/permute), and a shared \c CostModel. No tensor data is +/// ever allocated or copied; every op is index-set bookkeeping plus a +/// CostModel query. Mirrors \c ResultTensorTAPP's structure +/// (backends/tapp/result.hpp) with every real-tensor line replaced by that +/// bookkeeping. +/// +class ResultDryRun final : public Result { + public: + using Result::id_t; + + ResultDryRun(container::svector idxset, + std::shared_ptr cm, + ExtentOverrides overrides = {}) + : Result{Payload{}}, + indices_{std::move(idxset)}, + cm_{std::move(cm)}, + overrides_{std::move(overrides)} {} + + [[nodiscard]] container::svector const& indices() const noexcept { + return indices_; + } + [[nodiscard]] ExtentOverrides const& overrides() const noexcept { + return overrides_; + } + + /// The contiguous `[lo, hi)` element range of outer mode \p mode assembled so + /// far by write_into_slice() (empty `{0, 0}` if nothing written). + [[nodiscard]] std::pair assembled_range( + std::size_t mode) const { + if (auto it = assembled_.find(mode); it != assembled_.end()) + return it->second; + return {0, 0}; + } + + private: + struct Payload {}; + + [[nodiscard]] id_t type_id() const noexcept override { + return id_for_type(); + } + + [[nodiscard]] ResultPtr sum( + Result const& other, + std::array const& annot) const override { + return detail::DryRunOps::sum(indices_, overrides_, cm_, other, annot); + } + + [[nodiscard]] ResultPtr prod(Result const& other, + std::array const& annot, + DeNest /*DeNestFlag*/) const override { + return detail::DryRunOps::prod(indices_, overrides_, cm_, other, annot); + } + + [[nodiscard]] ResultPtr permute( + std::array const& ann) const override { + return detail::DryRunOps::permute(indices_, overrides_, cm_, ann); + } + + [[nodiscard]] ResultPtr adjoint( + std::array const& ann) const override { + return detail::DryRunOps::permute(indices_, overrides_, cm_, ann); + } + + [[nodiscard]] ResultPtr slice_mode(std::size_t mode, std::size_t elem_lo, + std::size_t elem_hi) const override { + return detail::DryRunOps::slice_mode(indices_, overrides_, cm_, mode, + elem_lo, elem_hi); + } + + [[nodiscard]] container::svector> + mode_batches(std::size_t mode, std::size_t target_batch_size) const override { + return detail::DryRunOps::mode_batches(indices_, overrides_, cm_, mode, + target_batch_size); + } + + void write_into_slice(Result const& block, std::size_t mode, + std::size_t block_lo, std::size_t block_hi) override { + detail::DryRunOps::write_into_slice(indices_, overrides_, assembled_, cm_, + block, mode, block_lo, block_hi); + } + + [[nodiscard]] ResultPtr pre_sized_zeros_over_mode( + std::size_t mode, Result const& axis_src, + std::size_t axis_src_mode) const override { + return detail::DryRunOps::pre_sized_zeros_over_mode( + indices_, overrides_, cm_, mode, axis_src, axis_src_mode); + } + + void add_inplace(Result const& other) override { + SEQUANT_ASSERT(other.is() || other.is()); + overrides_ = + detail::merge_overrides(overrides_, detail::overrides_of(other)); + } + + [[nodiscard]] ResultPtr symmetrize() const override { + return eval_result(indices_, cm_, overrides_); + } + + [[nodiscard]] ResultPtr antisymmetrize(size_t /*bra_rank*/) const override { + return eval_result(indices_, cm_, overrides_); + } + + [[nodiscard]] ResultPtr mult_by_phase(std::int8_t /*factor*/) const override { + return eval_result(indices_, cm_, overrides_); + } + + [[nodiscard]] std::size_t size_in_bytes() const final { + return cm_->memsize(indices_, overrides_); + } + + container::svector indices_; + std::shared_ptr cm_; + ExtentOverrides overrides_; + AssembledCoverage assembled_; +}; + +/// +/// \brief CSV/PNO tensor-of-tensor zero-data token. +/// +/// Like \c ResultDryRun, but additionally exposes an outer()/inner() split of +/// its (canon-order) index list -- inner = the proto-indexed (composite) +/// legs, e.g. a CSV amplitude's PNO domain leg `a_1`; outer = every +/// other (plain) leg, e.g. the PAO index `mu~_1`. The split is purely an +/// observability/testing convenience: \c size_in_bytes()'s arithmetic is +/// IDENTICAL to \c ResultDryRun's (\c CostModel::memsize already routes any +/// index list containing a proto-indexed entry through the moment-aware +/// `inner_pow` path internally, via \c tot_indices/inner_aware_volume -- +/// content-driven, not type-driven), so tests that want to confirm "this used +/// the k-th moment, not extent^k" can inspect inner() directly. +/// +/// Position semantics for \c slice_mode()/\c mode_batches(): the `mode` +/// argument the runtime passes is always resolved against the FULL +/// canon-order list (an optional trailing constructor argument, defaulting to +/// `outer ++ inner` when the caller does not need position accuracy, e.g. a +/// hand-built test instance); the \c DryRunLeafEvaluator (eval_expr.hpp) +/// always supplies the leaf's true \c canon_indices() order there, since only +/// LEAF-constructed instances are ever sliced by the runtime (\c slice_mode() +/// is invoked only inside the batched evaluator's leaf-wrapping closure, never +/// on a prod()/sum()-produced intermediate). +/// +class ResultDryRunNested final : public Result { + public: + using Result::id_t; + + ResultDryRunNested(container::svector outer, + container::svector inner, + std::shared_ptr cm, + ExtentOverrides overrides = {}, + container::svector canon_order = {}) + : Result{Payload{}}, + outer_{std::move(outer)}, + inner_{std::move(inner)}, + indices_{canon_order.empty() + ? [this] { + container::svector c = outer_; + c.insert(c.end(), inner_.begin(), inner_.end()); + return c; + }() + : std::move(canon_order)}, + cm_{std::move(cm)}, + overrides_{std::move(overrides)} {} + + [[nodiscard]] container::svector const& outer() const noexcept { + return outer_; + } + [[nodiscard]] container::svector const& inner() const noexcept { + return inner_; + } + [[nodiscard]] container::svector const& indices() const noexcept { + return indices_; + } + [[nodiscard]] ExtentOverrides const& overrides() const noexcept { + return overrides_; + } + + /// The contiguous `[lo, hi)` element range of outer mode \p mode assembled so + /// far by write_into_slice() (empty `{0, 0}` if nothing written). + [[nodiscard]] std::pair assembled_range( + std::size_t mode) const { + if (auto it = assembled_.find(mode); it != assembled_.end()) + return it->second; + return {0, 0}; + } + + private: + struct Payload {}; + + [[nodiscard]] id_t type_id() const noexcept override { + return id_for_type(); + } + + [[nodiscard]] ResultPtr sum( + Result const& other, + std::array const& annot) const override { + return detail::DryRunOps::sum(indices_, overrides_, cm_, other, annot); + } + + [[nodiscard]] ResultPtr prod(Result const& other, + std::array const& annot, + DeNest /*DeNestFlag*/) const override { + return detail::DryRunOps::prod(indices_, overrides_, cm_, other, annot); + } + + [[nodiscard]] ResultPtr permute( + std::array const& ann) const override { + return detail::DryRunOps::permute(indices_, overrides_, cm_, ann); + } + + [[nodiscard]] ResultPtr adjoint( + std::array const& ann) const override { + return detail::DryRunOps::permute(indices_, overrides_, cm_, ann); + } + + [[nodiscard]] ResultPtr slice_mode(std::size_t mode, std::size_t elem_lo, + std::size_t elem_hi) const override { + return detail::DryRunOps::slice_mode(indices_, overrides_, cm_, mode, + elem_lo, elem_hi); + } + + [[nodiscard]] container::svector> + mode_batches(std::size_t mode, std::size_t target_batch_size) const override { + return detail::DryRunOps::mode_batches(indices_, overrides_, cm_, mode, + target_batch_size); + } + + void write_into_slice(Result const& block, std::size_t mode, + std::size_t block_lo, std::size_t block_hi) override { + detail::DryRunOps::write_into_slice(indices_, overrides_, assembled_, cm_, + block, mode, block_lo, block_hi); + } + + [[nodiscard]] ResultPtr pre_sized_zeros_over_mode( + std::size_t mode, Result const& axis_src, + std::size_t axis_src_mode) const override { + return detail::DryRunOps::pre_sized_zeros_over_mode( + indices_, overrides_, cm_, mode, axis_src, axis_src_mode); + } + + void add_inplace(Result const& other) override { + SEQUANT_ASSERT(other.is() || other.is()); + overrides_ = + detail::merge_overrides(overrides_, detail::overrides_of(other)); + } + + [[nodiscard]] ResultPtr symmetrize() const override { + return eval_result(outer_, inner_, cm_, overrides_, + indices_); + } + + [[nodiscard]] ResultPtr antisymmetrize(size_t /*bra_rank*/) const override { + return eval_result(outer_, inner_, cm_, overrides_, + indices_); + } + + [[nodiscard]] ResultPtr mult_by_phase(std::int8_t /*factor*/) const override { + return eval_result(outer_, inner_, cm_, overrides_, + indices_); + } + + [[nodiscard]] std::size_t size_in_bytes() const final { + return cm_->memsize(indices_, overrides_); + } + + container::svector outer_; + container::svector inner_; + container::svector indices_; // canon order; outer_++inner_ content + std::shared_ptr cm_; + ExtentOverrides overrides_; + AssembledCoverage assembled_; +}; + +[[nodiscard]] inline ResultPtr make_dryrun_result( + container::svector idx, std::shared_ptr cm, + ExtentOverrides overrides) { + if (!detail::has_proto(idx)) + return eval_result(std::move(idx), std::move(cm), + std::move(overrides)); + container::svector outer, inner; + for (auto const& ix : idx) + (ix.has_proto_indices() ? inner : outer).push_back(ix); + return eval_result(std::move(outer), std::move(inner), + std::move(cm), std::move(overrides), + std::move(idx)); +} + +namespace detail { + +[[nodiscard]] inline container::svector indices_of(Result const& r) { + if (r.is()) return r.as().indices(); + SEQUANT_ASSERT(r.is()); + return r.as().indices(); +} + +[[nodiscard]] inline ExtentOverrides overrides_of(Result const& r) { + if (r.is()) return r.as().overrides(); + SEQUANT_ASSERT(r.is()); + return r.as().overrides(); +} + +} // namespace detail + +} // namespace sequant::eval::dryrun + +#endif // SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_RESULT_HPP diff --git a/SeQuant/core/eval/backends/dryrun/size_regime.hpp b/SeQuant/core/eval/backends/dryrun/size_regime.hpp new file mode 100644 index 0000000000..ad8657912c --- /dev/null +++ b/SeQuant/core/eval/backends/dryrun/size_regime.hpp @@ -0,0 +1,136 @@ +#ifndef SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_SIZE_REGIME_HPP +#define SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_SIZE_REGIME_HPP + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace sequant::eval::dryrun { + +/// Per-space extents and per-rank CSV moment tables that define one size +/// regime for a dry-run replay. Extents are element counts; CSV moments are +/// power means over occupied pairs (PNO) or singles (OSV). +struct SizeRegime { + std::map space_extent; + + /// OPTIONAL per-space BATCH PARTITION: the element extent of each realized + /// batch slice along the space's batch axis, keyed by space base_key, in + /// order. Empty (default) => the dry-run batches a mode into UNIFORM + /// target_batch_size blocks (backend-model-agnostic fallback). When present + /// for a batch axis, ResultDryRun::mode_batches uses THIS partition directly + /// (accumulated to [lo,hi) ranges), so the dry-run's batch COUNT -- hence its + /// recompute -- matches whatever the wet backend realizes, even when a tile + /// is coarser than target_batch_size. + /// + /// The dry-run backend deliberately does NOT know how these were derived: the + /// CALLER converts its backend's structure into slice extents and supplies + /// them here, so a new backend model is supported without touching dry-run + /// eval internals. For a TILE-based wet backend, \c batch_slice_extents_from_ + /// tiles is the ready-made converter. The extents must sum to space_extent. + std::map> space_slice_extents; + + // csv_pno_moment[k] / csv_osv_moment[k] hold the k-th POWER MEAN + // M_k = (mean_over_pairs d^k)^(1/k) of the per-pair PNO / per-orbital OSV + // domain size d, for k in [1,4] (index 0 is unused, set to 1). inner_pow() + // returns M_k so that inner_aware_volume's per-member product over a + // k-composite group is M_k^k = mean(d^k), and outer_nocc^N * M_k^k equals + // the true block-sparse volume Sum_pairs d^k. Do NOT store raw moments + // mean(d^k) here: that would over-count k-composite groups by a further + // power of k. For a constant domain d, M_k = d for all k. + std::array csv_pno_moment{1.0, 1.0, 1.0, 1.0, 1.0}; + std::array csv_osv_moment{1.0, 1.0, 1.0, 1.0, 1.0}; + + // Moment tables for CSV cluster ranks >= 3 (CSV-CCSDT triples and beyond), + // keyed by cluster rank (= number of proto indices). csv_moment_by_rank[r][k] + // is the k-th power mean of the rank-r cluster domain. A rank not present + // falls back to csv_pno_moment (the rank-2 table) in inner_pow(), preserving + // the pre-rank-general behavior where every proto-rank >= 2 used the PNO + // table. Ranks 1 and 2 are held by csv_osv_moment / csv_pno_moment above and + // are NOT expected here (an entry for 1 or 2 is ignored by inner_pow()). + std::map> csv_moment_by_rank; + + /// \return the flat extent of \p ix's space; throws \c std::out_of_range + /// if the space is not present in \c space_extent (fail loud rather + /// than silently defaulting to 1). + [[nodiscard]] std::size_t extent(Index const& ix) const { + return space_extent.at(std::wstring{ix.space().base_key()}); + } + + /// \return \p ix's space batch-slice-extent sequence, or an empty span if the + /// space has no partition recorded (=> the caller falls back to + /// uniform target_batch_size blocks). Never throws. + [[nodiscard]] container::svector const& slice_extents( + Index const& ix) const { + static const container::svector empty; + auto const it = + space_slice_extents.find(std::wstring{ix.space().base_key()}); + return it != space_slice_extents.end() ? it->second : empty; + } + + /// \return the k-th power-mean moment for a proto-indexed CSV/PNO composite + /// index (\p k clamped to 0..4), or \c pow(extent, k) for a plain + /// (non-composite) index. Rank is determined by the number of proto + /// indices: 1 => OSV (occupied single), 2 => PNO (occupied pair), + /// >= 3 => the rank-specific csv_moment_by_rank table if present, + /// else the PNO (rank-2) table. + [[nodiscard]] double inner_pow(Index const& composite, std::size_t k) const { + if (k > 4) k = 4; + auto const& protos = composite.proto_indices(); + if (protos.empty()) + return std::pow(static_cast(extent(composite)), + static_cast(k)); + auto const rank = protos.size(); + if (rank <= 1) return csv_osv_moment[k]; + if (rank == 2) return csv_pno_moment[k]; + auto const it = csv_moment_by_rank.find(rank); + return (it != csv_moment_by_rank.end()) ? it->second[k] : csv_pno_moment[k]; + } + + [[nodiscard]] std::function idx_to_extent() const { + return [this](Index const& ix) { return extent(ix); }; + } + + [[nodiscard]] std::function inner_pow_fn() + const { + return [this](Index const& ix, std::size_t k) { return inner_pow(ix, k); }; + } +}; + +/// Convert a TILE-extent sequence into a BATCH-slice-extent sequence +/// (SizeRegime::space_slice_extents) by the SAME whole-tile grouping the wet +/// batched evaluator uses (mode_batches_of_trange1, tiledarray/result.hpp): +/// accumulate consecutive tiles into a slice until appending the next would +/// push the slice over \p target_batch_size, then start a new slice; a lone +/// tile larger than the target still forms its own slice. Slice boundaries fall +/// on tile edges. This is a convenience converter for a TILE-based caller; the +/// dry-run backend never calls it -- it only READS the resulting slice extents, +/// so any other backend model can populate space_slice_extents differently +/// without touching dry-run internals. +[[nodiscard]] inline container::svector +batch_slice_extents_from_tiles( + container::svector const& tile_extents, + std::size_t target_batch_size) { + container::svector slices; + std::size_t const target = std::max(target_batch_size, 1); + std::size_t acc = 0; + for (std::size_t const tsz : tile_extents) { + if (acc > 0 && acc + tsz > target) { + slices.push_back(acc); + acc = 0; + } + acc += tsz; + } + if (acc > 0) slices.push_back(acc); + return slices; +} + +} // namespace sequant::eval::dryrun + +#endif // SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_SIZE_REGIME_HPP diff --git a/SeQuant/core/eval/backends/tiledarray/eval_context.hpp b/SeQuant/core/eval/backends/tiledarray/eval_context.hpp index fd5310a978..ef9b1440a9 100644 --- a/SeQuant/core/eval/backends/tiledarray/eval_context.hpp +++ b/SeQuant/core/eval/backends/tiledarray/eval_context.hpp @@ -6,10 +6,13 @@ #include #include #include +#include #include +#include #include +#include #include #include diff --git a/SeQuant/core/eval/backends/tiledarray/result.hpp b/SeQuant/core/eval/backends/tiledarray/result.hpp index 6c640cf1cc..de62af4b1f 100644 --- a/SeQuant/core/eval/backends/tiledarray/result.hpp +++ b/SeQuant/core/eval/backends/tiledarray/result.hpp @@ -329,6 +329,27 @@ template return TA::TiledRange(dims.begin(), dims.end()); } +/// Map a contiguous element range `[elem_lo, elem_hi)` on a mode's TiledRange1 +/// to the tile range `[tile_lo, tile_hi)` it must coincide with. A tiled +/// backend can only cut or scatter whole tiles, so the element bounds must be +/// in-range and fall on tile boundaries; this asserts both (mode_batches() +/// yields exactly such tile-aligned ranges). Shared by slice_mode() (GATHER a +/// block out) and write_into_slice() (SCATTER a block in) so both agree on the +/// element-to-tile contract and its alignment preconditions. +[[nodiscard]] inline std::pair slice_bounds_to_tiles( + TA::TiledRange1 const& tr1, std::size_t elem_lo, std::size_t elem_hi) { + SEQUANT_ASSERT(elem_lo >= tr1.elements_range().first && elem_lo < elem_hi && + elem_hi <= tr1.elements_range().second); + std::size_t const tile_lo = tr1.element_to_tile(elem_lo); + SEQUANT_ASSERT(tr1.tile(tile_lo).first == elem_lo); // lo on a tile boundary + std::size_t const tile_hi = (elem_hi >= tr1.elements_range().second) + ? tr1.tile_extent() + : tr1.element_to_tile(elem_hi); + SEQUANT_ASSERT(elem_hi >= tr1.elements_range().second || + tr1.tile(tile_hi).first == elem_hi); // hi on a tile boundary + return {tile_lo, tile_hi}; +} + } // namespace detail /// TA::Tensor memory use logger @@ -353,6 +374,14 @@ template TA::DistArray const& arr, std::size_t mode, std::size_t tile_lo, std::size_t tile_hi); +// defined below; declared here so the result classes' write_into_slice() +// overrides can call it. The scatter inverse of slice_array_over_mode(). +template +void write_array_into_mode(TA::DistArray& dest, + TA::DistArray const& block, + std::size_t mode, std::size_t tile_lo, + std::size_t tile_hi); + /// Partition a TiledRange1 into contiguous, tile-aligned element-range batches, /// each covering at most \p target_batch_size elements: whole tiles are /// appended to a batch until the next tile would push it over the target, so \p @@ -434,22 +463,8 @@ class ResultTensorTA final : public Result { [[nodiscard]] ResultPtr slice_mode(std::size_t mode, std::size_t elem_lo, std::size_t elem_hi) const override { - auto const& tr1 = get().trange().dim(mode); - // slice_mode takes element bounds, but a tiled backend can only cut on tile - // boundaries; mode_batches() returns exactly such (tile-aligned, in-range) - // bounds. Assert the precondition so misuse is caught rather than silently - // producing an over- or under-sized slice (which would break batched sums). - SEQUANT_ASSERT(elem_lo >= tr1.elements_range().first && elem_lo < elem_hi && - elem_hi <= tr1.elements_range().second); - std::size_t const tile_lo = tr1.element_to_tile(elem_lo); - SEQUANT_ASSERT(tr1.tile(tile_lo).first == - elem_lo); // lo on a tile boundary - std::size_t const tile_hi = (elem_hi >= tr1.elements_range().second) - ? tr1.tile_extent() - : tr1.element_to_tile(elem_hi); - SEQUANT_ASSERT(elem_hi >= tr1.elements_range().second || - tr1.tile(tile_hi).first == - elem_hi); // hi on a tile boundary + auto const [tile_lo, tile_hi] = detail::slice_bounds_to_tiles( + get().trange().dim(mode), elem_lo, elem_hi); return eval_result( slice_array_over_mode(get(), mode, tile_lo, tile_hi)); } @@ -460,6 +475,38 @@ class ResultTensorTA final : public Result { target_batch_size); } + void write_into_slice(Result const& block, std::size_t mode, + std::size_t block_lo, std::size_t block_hi) override { + SEQUANT_ASSERT(block.is()); + auto& dest = get(); + auto const [tile_lo, tile_hi] = detail::slice_bounds_to_tiles( + dest.trange().dim(mode), block_lo, block_hi); + write_array_into_mode(dest, block.get(), mode, tile_lo, tile_hi); + } + + [[nodiscard]] ResultPtr pre_sized_zeros_over_mode( + std::size_t mode, Result const& axis_src, + std::size_t axis_src_mode) const override { + SEQUANT_ASSERT(axis_src.is()); + auto const& self = get(); + auto const& src = axis_src.get(); + auto const rank = self.trange().rank(); + SEQUANT_ASSERT(mode < rank); + SEQUANT_ASSERT(axis_src_mode < src.trange().rank()); + // Take *this's outer trange but swap in the external axis's FULL tiling + // (from axis_src's mode axis_src_mode). Every other mode of a block partial + // is already full extent, so only the sliced axis needs widening. + std::vector dims; + dims.reserve(rank); + for (std::size_t d = 0; d < rank; ++d) dims.push_back(self.trange().dim(d)); + dims[mode] = src.trange().dim(axis_src_mode); + ArrayT dest(self.world(), TA::TiledRange(dims.begin(), dims.end())); + dest.fill_local(numeric_type(0)); + dest.world().gop.fence(); + log_ta_tensor_host_memory_use(); + return eval_result(std::move(dest)); + } + [[nodiscard]] ResultPtr prod(Result const& other, std::array const& annot, DeNest DeNestFlag) const override { @@ -639,22 +686,8 @@ class ResultTensorOfTensorTA final : public Result { [[nodiscard]] ResultPtr slice_mode(std::size_t mode, std::size_t elem_lo, std::size_t elem_hi) const override { - auto const& tr1 = get().trange().dim(mode); - // slice_mode takes element bounds, but a tiled backend can only cut on tile - // boundaries; mode_batches() returns exactly such (tile-aligned, in-range) - // bounds. Assert the precondition so misuse is caught rather than silently - // producing an over- or under-sized slice (which would break batched sums). - SEQUANT_ASSERT(elem_lo >= tr1.elements_range().first && elem_lo < elem_hi && - elem_hi <= tr1.elements_range().second); - std::size_t const tile_lo = tr1.element_to_tile(elem_lo); - SEQUANT_ASSERT(tr1.tile(tile_lo).first == - elem_lo); // lo on a tile boundary - std::size_t const tile_hi = (elem_hi >= tr1.elements_range().second) - ? tr1.tile_extent() - : tr1.element_to_tile(elem_hi); - SEQUANT_ASSERT(elem_hi >= tr1.elements_range().second || - tr1.tile(tile_hi).first == - elem_hi); // hi on a tile boundary + auto const [tile_lo, tile_hi] = detail::slice_bounds_to_tiles( + get().trange().dim(mode), elem_lo, elem_hi); return eval_result( slice_array_over_mode(get(), mode, tile_lo, tile_hi)); } @@ -665,6 +698,57 @@ class ResultTensorOfTensorTA final : public Result { target_batch_size); } + void write_into_slice(Result const& block, std::size_t mode, + std::size_t block_lo, std::size_t block_hi) override { + SEQUANT_ASSERT(block.is()); + auto& dest = get(); + auto const [tile_lo, tile_hi] = detail::slice_bounds_to_tiles( + dest.trange().dim(mode), block_lo, block_hi); + write_array_into_mode(dest, block.get(), mode, tile_lo, tile_hi); + } + + [[nodiscard]] ResultPtr pre_sized_zeros_over_mode( + std::size_t mode, Result const& axis_src, + std::size_t axis_src_mode) const override { + auto const& self = get(); + auto const rank = self.trange().rank(); + SEQUANT_ASSERT(mode < rank); + // The axis-carrying leaf supplying K's FULL tiling for mode `mode` may be + // nested (this_type) or flat (that_type, e.g. an integral over the external + // occ index): read the widened axis TiledRange1 from whichever kind. Only + // this one OUTER TiledRange1 is needed; every other mode of a block partial + // is already at full extent, so *this's own outer tiling supplies them. + TA::TiledRange1 const axis_dim = [&]() -> TA::TiledRange1 { + if (axis_src.is()) { + auto const& src = axis_src.get(); + SEQUANT_ASSERT(axis_src_mode < src.trange().rank()); + return src.trange().dim(axis_src_mode); + } + SEQUANT_ASSERT(axis_src.is()); + auto const& src = axis_src.get(); + SEQUANT_ASSERT(axis_src_mode < src.trange().rank()); + return src.trange().dim(axis_src_mode); + }(); + std::vector dims; + dims.reserve(rank); + for (std::size_t d = 0; d < rank; ++d) dims.push_back(self.trange().dim(d)); + dims[mode] = axis_dim; + // A zero ToT is represented with empty inner tiles (tot_inner_rank() == 0): + // build the widened OUTER trange, then give every local outer tile a + // well-formed (empty-inner) outer tile over its range -- exactly the zero + // ToT that slice_array_over_mode() emits, and a valid destination that the + // ToT write_array_into_mode() block-assignment overwrites per scatter. The + // batches tile the widened `mode` axis with no gaps, so every outer tile is + // subsequently overwritten by some block's real inner tensors. + using value_type = typename ArrayT::value_type; + ArrayT dest(self.world(), TA::TiledRange(dims.begin(), dims.end())); + for (auto it = dest.begin(); it != dest.end(); ++it) + if (dest.is_local(it.index())) *it = value_type{it.make_range()}; + dest.world().gop.fence(); + log_ta_tensor_host_memory_use(); + return eval_result(std::move(dest)); + } + [[nodiscard]] ResultPtr prod(Result const& other, std::array const& annot, DeNest DeNestFlag) const override { @@ -889,6 +973,67 @@ template return out; } +/// \brief Scatter a per-block DistArray into a contiguous tile range of one +/// mode of a pre-sized destination -- the inverse of +/// slice_array_over_mode(). +/// +/// Writes \p block into tiles `[tile_lo, tile_hi)` of \p dest's mode \p mode, +/// leaving every other tile of \p dest untouched. \p dest must already be +/// allocated over its full TiledRange (the caller sizes the whole shape), and +/// \p block's TiledRange must equal \p dest's sub-block over `[tile_lo, +/// tile_hi)` (as produced by slice_array_over_mode() for the same mode/range). +/// Implemented with TA's block() on the assignment LHS, so only the addressed +/// sub-block is written and block-sparse shape is preserved. Every mode's +/// element lobound is preserved (via TA's `preserve_lobound`), exactly as the +/// lobound-preserving GATHER in slice_array_over_mode(): the destination +/// sub-block and the source share element coordinates, so a spectator index +/// carrying a nonzero lobound (e.g. a frozen-core offset) lands at its true +/// offset rather than being rebased to 0. Reconstructs a whole result from a +/// disjoint, gap-free tiling of one mode: scattering each block of a partition +/// reproduces the array `slice_array_over_mode()` would gather back out. +template +void write_array_into_mode(TA::DistArray& dest, + TA::DistArray const& block, + std::size_t mode, std::size_t tile_lo, + std::size_t tile_hi) { + using ranges::views::iota; + auto const rank = dest.trange().rank(); + SEQUANT_ASSERT(mode < rank); + SEQUANT_ASSERT(tile_lo < tile_hi && + tile_hi <= dest.trange().dim(mode).tile_extent()); + container::svector lo(rank, 0), hi(rank); + for (std::size_t d = 0; d < rank; ++d) + hi[d] = dest.trange().dim(d).tile_extent(); + lo[mode] = tile_lo; + hi[mode] = tile_hi; + // For a tensor-of-tensor array the annotation must label an inner block + // ("outer;inner"); a flat annotation trips DistArray's is_tot_index() check. + // The block() is over outer modes only, so both sides share one annotation. + using value_type = typename TA::DistArray::value_type; + std::string annot; + if constexpr (TA::detail::is_tensor_of_tensor_v) { + auto const inner_rank = detail::tot_inner_rank(block); + if (inner_rank == 0) { + // block has all-empty inner tiles (tot_inner_rank() == 0): it represents + // zero and there is no inner rank to form the ToT annotation block() + // needs. A zero contribution leaves the pre-sized destination slice as + // it was, so skip the scatter entirely -- mirroring the zero-ToT early + // return in slice_array_over_mode(). + return; + } + annot = TA::detail::dummy_annotation(static_cast(rank), + static_cast(inner_rank)); + } else { + annot = detail::ords_to_annot(iota(std::size_t{0}, rank)); + } + // preserve_lobound: address the destination sub-block in its original element + // coordinates (keeping every mode's lobound) so it matches the source block, + // which slice_array_over_mode() also gathered with preserve_lobound. Plain + // block() would rebase the sub-block to 0 and mismatch the source trange. + dest(annot).block(lo, hi, TA::preserve_lobound) = block(annot); + TA::DistArray::wait_for_lazy_cleanup(dest.world()); +} + /// \brief Compute the result's OUTER TiledRange for a binary product from the /// type-erased operands and the [left, right, result] annotations. /// diff --git a/SeQuant/core/eval/cache_manager.hpp b/SeQuant/core/eval/cache_manager.hpp index 92fcdb8052..354c316d5c 100644 --- a/SeQuant/core/eval/cache_manager.hpp +++ b/SeQuant/core/eval/cache_manager.hpp @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include #include @@ -17,11 +19,92 @@ #include #include #include +#include +#include #include +#include #include #include +#include +#include #include #include +#include + +namespace sequant::eval { + +// Forward declaration only (not a full include of placement_router.hpp): +// PlacementRouter's BatchContext alias needs CacheManager's full definition, +// so placement_router.hpp includes this header; CacheManager only needs a +// non-owning pointer to the (incomplete) PlacementRouter type, so the +// forward declaration here avoids the include cycle. +template +class PlacementRouter; + +/// \brief Destination for the structured schedule dump (SCHEDULE_RUN_EVENT +/// records) emitted by `evaluate()`. A caller sets one on the (root) cache to +/// capture one evaluation's batched schedule to \c os; \c fired is a fire-once +/// latch the caller raises after the first capture so a later re-entry (e.g. a +/// subsequent CC iteration) does not re-dump. Null sink / null \c os => no dump +/// (the default, byte-identical to before this seam). Non-owning: \c os must +/// outlive the cache. +struct ScheduleSink { + std::ostream* os = nullptr; + bool fired = false; +}; + +/// \brief DIAGNOSTIC (analysis-only, OFF by default): a global monotonic +/// "access clock" stamped on every genuine cache READ, plus a +/// per-value (keyed by canonical node hash) record of the LAST clock at +/// which that value was read. +/// +/// Home (root-homed) cache entries are pinned (life_c == SIZE_MAX), so their +/// lifetime counter never drains and cannot be used to infer their genuine last +/// use. This clock records the real thing: \c CacheManager::access_at and +/// \c access_at_hops (the ONLY genuine consumer-read paths -- the store-return +/// \c entry::access() bypasses both) call \c tick() and stamp the read value's +/// hash into \c last_access_map on every hit when \c enabled(). The record is a +/// GLOBAL map keyed by node hash (not a per-entry field) deliberately: a +/// batch-loop tier-B value lives on a per-block scratch cache that is destroyed +/// at block close, so a per-entry field would be lost; the global map keeps the +/// value's FINAL last-read clock across the whole run regardless of which +/// (root or transient scratch) scope held it. Reset by the harness before a +/// measured run. Single-threaded dry-run only. When \c enabled() is false every +/// stamp site is a no-op and the eval path stays byte-identical. +struct AccessClock { + /// One-shot env gate (SEQUANT_UT_ACCESS_CLOCK). Read once; when unset every + /// stamp site below is inert. + static bool enabled() noexcept { + static bool const on = std::getenv("SEQUANT_UT_ACCESS_CLOCK") != nullptr; + return on; + } + static std::size_t& counter() noexcept { + static std::size_t c = 0; + return c; + } + /// hash -> final (max) clock at which a value with that hash was read. + static std::unordered_map& + last_access_map() noexcept { + static std::unordered_map m; + return m; + } + /// Advance and return the clock (one genuine read == one tick). + static std::size_t tick() noexcept { return ++counter(); } + /// Current clock value WITHOUT advancing (used to timestamp the peak). + static std::size_t now() noexcept { return counter(); } + /// Record a genuine read of the value with hash @p h at a fresh clock tick. + static void stamp(std::size_t h) noexcept { + if (!enabled()) return; + last_access_map()[h] = tick(); + } + /// Clear the clock and the per-value record before a measured run. + static void reset() noexcept { + counter() = 0; + last_access_map().clear(); + } +}; + +} // namespace sequant::eval namespace sequant { @@ -70,6 +153,76 @@ class CacheManager { std::any const& node, Result const& left, Result const& right, std::array const& annot)>; + /// A whole-scope driver type. When set, the forest-range + /// `evaluate(Nodes const&, layout, leaf, cache)` entrypoint (eval.hpp) routes + /// the WHOLE forest through this driver instead of its per-tree descent, + /// passing the forest and the result layout. The driver type-erases the + /// `sequant::evaluate(forest, BatchPolicy, layout, leaf, cache, mode_order, + /// make_scope_guard)` overload (scope_executor.hpp) -- which builds the + /// batched-DAG schedule and calls `eval::evaluate_whole_scope` -- so that + /// eval.hpp need not include scope_executor.hpp (that would be a cycle: + /// scope_executor.hpp includes eval.hpp). The captured leaf evaluator, + /// BatchPolicy, mode_order and scope-guard factory live inside the closure, + /// which is built at the call site that owns those concrete types (e.g. + /// MPQC's batched CSV-CCk residual install). The forest is a + /// `std::vector` and the layout a `std::string` (the residual + /// annotation); other evaluate() instantiations never match and stay on the + /// standard scheme. Empty (default) => no whole-scope routing; behavior is + /// byte-identical. + using whole_scope_driver_type = + std::function const& forest, + std::string const& layout, CacheManager&)>; + + /// The batch context: an ordered stack (outermost-first) of the enclosing + /// realized batch loops, one entry per loop, `{axis K, {block_lo, block_hi}}` + /// (element range). Set on the per-block scratch by the batched evaluator + /// before it re-enters evaluate(); read by the Enter-stage slice-on-use so a + /// cached intermediate fetched from an ancestor scope is sliced to the modes + /// of the loops the fetch crossed (see eval.hpp). Empty (default) => no + /// enclosing batch loop, so slice-on-use is inert and behavior is + /// byte-identical to the pre-slice-on-use path. + using BatchContext = + container::svector>>; + + /// Result of access_at(): the fetched pointer plus the hop distance (number + /// of parent links crossed) to the scope that held it. hops == 0 means a + /// local hit; a null ptr carries hops == 0. + struct AccessResult { + ResultPtr ptr; + std::size_t hops; + }; + + /// DIAGNOSTIC (dry-run costing): per-DISTINCT-value build tally for the + /// avoidable-recompute rollup, keyed by the SAME node identity the cache + /// dedups on (TreeNodeHasher + TreeNodeEqualityComparator = topological hash + /// bin + Bliss connectivity 3-way cmp + recursive child compare), so two + /// topologically-distinct nodes sharing a 64-bit hash are NOT folded and + /// per-block / alpha-renamed builds of ONE value ARE folded. + /// + /// Recompute is measured with ACTUAL replay FLOPs, deduped at the (value, + /// SLICE) granularity -- NOT against a build-once "full extent" denominator, + /// which is ill-defined when slicing is non-uniform. \c slices maps a SLICE + /// signature -- the enclosing batch context PROJECTED onto the modes THIS + /// value actually carries (empty for a value invariant to every live loop) -- + /// to that slice's {build count, one build's actual cost}. Then: + /// total = sum over slices of builds*cost (== the replay's dryrun + /// sum) build-once = sum over slices of cost (each DISTINCT slice + /// once) avoidable = sum over slices of (builds-1)*cost. + /// A value tiled over DISTINCT slices (different blocks) has builds==1 per + /// slice -> 0 avoidable (tiling is not recompute, even if the blocks are + /// unequal). A value rebuilt at the SAME slice -- e.g. a node invariant to an + /// enclosing loop, whose projected signature is identical every block -- has + /// builds>1 at one slice -> (builds-1)*cost avoidable. Costs need not be + /// uniform across slices; each slice carries its own realized cost. + struct BuildRecord { + std::size_t count = 0; // number of builds of this exact (value, slice) + double flops = 0; // this slice's actual realized-extent FLOPs + double exec = 0; // this slice's actual roofline exec-cost estimate + }; + struct BuildTally { + std::unordered_map slices; + }; + private: using hasher_type = TreeNodeHasher; using comparator_type = TreeNodeEqualityComparator; @@ -143,6 +296,46 @@ class CacheManager { [[nodiscard]] bool alive() const noexcept { return data_p ? true : false; } + /// \return true iff this entry currently holds the SAME buffer (pointer + /// identity) as @p other. A sliced/permuted/phase-shifted read of + /// this entry is a DISTINCT buffer, so it compares unequal. Used by + /// the peak trace to detect an operand that aliases a cached buffer + /// (whose bytes are then already counted, and must not be added + /// again). + [[nodiscard]] bool holds(ResultPtr const& other) const noexcept { + return data_p && data_p.get() == other.get(); + } + + /// Upgrade this entry to an unbounded (resident-until-reset) non-persistent + /// life, preserving any currently stored data and its cached size. Used to + /// re-home an existing finite-life CSE entry at a scope where the value + /// must survive ALL of its (possibly per-block-repeated) reads within one + /// evaluation, so a partially drained entry becomes resident again rather + /// than being freed and rebuilt by a later consumer. See \c + /// CacheManager::ensure_home_slot. + void make_resident() noexcept { + max_life = std::numeric_limits::max(); + life_c = std::numeric_limits::max(); + } + + /// Set (or reset) this entry's bounded life to exactly @p count uses, + /// preserving any currently stored data. Used by \c + /// CacheManager::ensure_home_slot(key, use_count, persistent) to home a + /// value with a genuine use-count-bounded lifetime -- released at its + /// count-th access -- rather than \c make_resident's unconditional + /// unbounded pin. + void set_life(size_t count) noexcept { + max_life = count; + life_c = count; + } + + /// Upgrade this entry to persistent (never drained on access, survives + /// reset()), preserving any currently stored data. Used by \c + /// CacheManager::ensure_home_slot(key, use_count, persistent) to promote + /// an existing entry when a later caller discovers the key is actually + /// iteration-invariant. + void make_persistent() noexcept { persistent_ = true; } + private: [[nodiscard]] int decay() noexcept { return life_c > 0 ? static_cast(--life_c) : 0; @@ -157,6 +350,32 @@ class CacheManager { std::unordered_map cache_map_; + /// DIAGNOSTIC: per-DISTINCT-value build tally (see BuildTally), keyed by the + /// same node identity as cache_map_. Populated by tally_build() from the eval + /// loop's build choke point (eval.hpp finish_phase_b) for EVERY product + /// build, whether that value is a cache entry, a footprint-gated recompute, + /// or a per-batch rebuild -- so the rollup is complete. Held only on the + /// scope- chain ROOT (tally_build routes there); scratch caches never + /// populate it, and reset() does NOT clear it (the tally spans the whole + /// forest replay). + std::unordered_map + recompute_tally_; + + /// Gate for tally_build(): false (default) => tally_build is a no-op, so the + /// wet (TA) eval path never populates recompute_tally_ and stays byte- + /// identical. The dry-run costing replay (cost_profile) sets this true on the + /// root cache before the replay. Held on the root only (tally_build routes + /// there and checks it there). + bool recompute_tally_enabled_ = false; + + /// Parent cache for the scope chain (loop-nest visibility). A batch scratch + /// sets this to the cache one level up; access() delegates on a local miss + /// so a loop-invariant node stored once at an ancestor level is found by + /// every inner body without copy-down. Null (default) => standalone cache, + /// byte-identical to pre-scope-chain behavior. Non-owning; the parent must + /// outlive this cache. + CacheManager* parent_ = nullptr; + /// Running high-water mark (bytes) of the eval engine's live working set, /// updated by note_working_set() and cleared by reset(). Held here rather /// than in the recursive evaluate() so it persists across the whole @@ -169,6 +388,44 @@ class CacheManager { shaped_product_hook_type shaped_product_hook_{}; + /// Optional whole-scope driver consulted by the forest-range evaluate() (see + /// whole_scope_driver_type). Empty => no whole-scope routing. + whole_scope_driver_type whole_scope_driver_{}; + + /// Enclosing realized batch loops for slice-on-use (see BatchContext). Empty + /// (default) => no enclosing batch loop; the batched evaluator sets it on the + /// per-block scratch before each re-entry. Not cleared by reset() (it is + /// per-loop-iteration structural, re-set each block by the evaluator). + BatchContext batch_context_{}; + + /// Non-owning placement router (see \c placement_router.hpp). Null + /// (default) => no override wired; \c placement_router() falls through to + /// \c parent_ (only the root cache is wired in practice). The pointee must + /// outlive this cache. + eval::PlacementRouter const* placement_router_ = nullptr; + + /// Non-owning hierarchy-wide co-resident high-water tracker (see + /// \c eval::PeakMonitor). Null (default) => \c note_working_set() only + /// updates this cache's own \c working_set_hwmark_; \c peak_monitor() falls + /// through to \c parent_ (only the root cache is wired in practice). The + /// pointee must outlive this cache. + eval::PeakMonitor* peak_monitor_ = nullptr; + + /// Optional OWNING backing for \c placement_router_. A router built by a + /// pre-pass (e.g. the remat placement pass) is a local at the build site; a + /// CacheManager returned BY VALUE from such a builder must carry the router + /// alive with it. \c adopt_placement_router stores it here (shared, so + /// CacheManager stays copyable) and points \c placement_router_ at it. The + /// forward-declared \c PlacementRouter is fine in a \c shared_ptr member (its + /// deleter is type-erased at construction, where the type is complete). + std::shared_ptr const> owned_router_{}; + + /// Non-owning schedule-dump sink (see \c eval::ScheduleSink). Null (default) + /// => `evaluate()` emits no SCHEDULE_RUN_EVENT records; falls through to + /// \c parent_ (only the root cache is wired in practice). The pointee must + /// outlive this cache. + eval::ScheduleSink* schedule_sink_ = nullptr; + public: /// Sets the custom evaluator (see custom_evaluator_type). Pass an empty /// std::function to clear it. @@ -193,6 +450,147 @@ class CacheManager { return shaped_product_hook_; } + /// Sets the whole-scope driver (see whole_scope_driver_type). Pass an empty + /// std::function to clear it. + void set_whole_scope_driver(whole_scope_driver_type fn) noexcept { + whole_scope_driver_ = std::move(fn); + } + + /// \return the whole-scope driver (empty if none is set). + [[nodiscard]] whole_scope_driver_type const& whole_scope_driver() + const noexcept { + return whole_scope_driver_; + } + + /// Sets the batch context (see batch_context_). Pass an empty context to + /// clear it. + void set_batch_context(BatchContext c) noexcept { + batch_context_ = std::move(c); + } + + /// \return the batch context (empty if none is set). + [[nodiscard]] BatchContext const& batch_context() const noexcept { + return batch_context_; + } + + /// Sets the scope-chain parent (see parent_). Pass nullptr to detach. + void set_parent(CacheManager* p) noexcept { parent_ = p; } + + /// \return the scope-chain parent (see parent_), or nullptr if this is a + /// standalone / chain-root cache. Used by the batched evaluator to + /// walk up to a target ancestor level when hoisting an invariant. + [[nodiscard]] CacheManager* parent() const noexcept { return parent_; } + + /// Sets the local placement router (see placement_router_). Pass nullptr + /// to detach. Non-owning; the pointee must outlive this cache. + void set_placement_router(eval::PlacementRouter const* r) noexcept { + placement_router_ = r; + } + + /// Takes OWNERSHIP of a placement router (see owned_router_) and wires it as + /// the local router. Use this when the router is a build-site local and the + /// CacheManager is returned by value: the shared_ptr keeps the router alive + /// for the cache's whole lifetime. Pass an empty shared_ptr to detach both. + void adopt_placement_router( + std::shared_ptr const> r) noexcept { + owned_router_ = std::move(r); + placement_router_ = owned_router_.get(); + } + + /// \return the local router if set, else the one inherited from parent_ + /// (only the root cache is wired in practice); nullptr if none is + /// wired anywhere along the chain. Non-owning. + [[nodiscard]] eval::PlacementRouter const* placement_router() + const noexcept { + return placement_router_ ? placement_router_ + : parent_ ? parent_->placement_router() + : nullptr; + } + + /// Sets the local peak monitor (see peak_monitor_). Pass nullptr to detach. + /// Non-owning; the pointee must outlive this cache. + void set_peak_monitor(eval::PeakMonitor* m) noexcept { peak_monitor_ = m; } + + /// \return the local peak monitor if set, else the one inherited from + /// \c parent_ (only the root cache is wired in practice); nullptr + /// if none is wired anywhere along the chain. Non-owning. + [[nodiscard]] eval::PeakMonitor* peak_monitor() const noexcept { + return peak_monitor_ ? peak_monitor_ + : parent_ ? parent_->peak_monitor() + : nullptr; + } + + /// Sets the schedule-dump sink (see schedule_sink_). Pass nullptr to detach. + /// Non-owning; the pointee (and its \c os) must outlive this cache. + void set_schedule_sink(eval::ScheduleSink* s) noexcept { schedule_sink_ = s; } + + /// \return the local schedule sink if set, else the one inherited from + /// \c parent_ (only the root cache is wired in practice); nullptr if + /// none is wired anywhere along the chain. Non-owning. + [[nodiscard]] eval::ScheduleSink* schedule_sink() const noexcept { + return schedule_sink_ ? schedule_sink_ + : parent_ ? parent_->schedule_sink() + : nullptr; + } + + /// Ensure a scope-hoist slot exists for @p key so a loop-invariant + /// intermediate can be stored here (store() is a no-op for an unregistered + /// key). The slot is NON-persistent with an effectively unbounded life, so it + /// is never drained by access() and lives until the next reset() -- per-batch + /// for a batch scratch (rebuilt for the next batch of the loop it is scoped + /// to), per-term for the real cache (rebuilt for the next term). Idempotent: + /// an existing entry (with any stored data) is left untouched. The unbounded + /// life -- rather than the emitted effective_count -- is deliberate: a + /// whole-nest invariant's escaped-outer set is empty, so its emitted + /// effective_count is 1, which as a life would drain the entry on first use; + /// reset() is the correct lifetime boundary for a hoisted invariant. + void ensure_hoist_slot(key_type const& key) { + cache_map_.try_emplace( + key, entry{std::numeric_limits::max(), /*persistent=*/false}); + } + + /// Ensure @p key has a RESIDENT home slot at THIS cache: an unbounded + /// non-persistent life (like \c ensure_hoist_slot -- lives until the next + /// \c reset()), so the value, once stored, is read by every consumer rather + /// than drained and rebuilt. Unlike \c ensure_hoist_slot (which is a no-op + /// on an existing entry), this UPGRADES an existing finite-life CSE entry to + /// the same unbounded life via \c entry::make_resident, preserving any + /// stored data. Used by the ordered executor (\c ordered_executor.hpp) to + /// home a root-scope \c BuildStep value (its \c CellLegality::home_floor is + /// empty -- a whole-nest invariant) at the root cache so it is built ONCE + /// and read by every consumer, including a block-internal consumer reaching + /// it through the scope chain -- the "de-alias the composite to root" + /// property the ordered schedule's per-value homing exists to provide, and + /// which the plain per-forest CSE life (drained after its unbatched use + /// count) does not, since a realized batch loop reads a root-homed invariant + /// once per block. + void ensure_home_slot(key_type const& key) { + auto [it, inserted] = cache_map_.try_emplace( + key, entry{std::numeric_limits::max(), /*persistent=*/false}); + if (!inserted) it->second.make_resident(); + } + + /// Home @p key with a bounded (use-count) or persistent lifetime, instead + /// of \c ensure_home_slot(key)'s unconditional \c make_resident pin. A + /// persistent slot survives \c reset() (iteration-invariant); a + /// non-persistent slot is released at its @p use_count-th access (its + /// genuine last use) rather than living unbounded until the next reset(). + /// Idempotent like \c ensure_home_slot(key): an already-present entry is + /// upgraded in place (to persistent, or to the new bounded life) rather + /// than replaced, preserving any stored data. + void ensure_home_slot(key_type const& key, std::size_t use_count, + bool persistent) { + auto [it, inserted] = cache_map_.try_emplace( + key, entry{persistent ? std::numeric_limits::max() : use_count, + persistent}); + if (!inserted) { + if (persistent) + it->second.make_persistent(); + else + it->second.set_life(use_count); + } + } + /// Default persistence classifier: every entry is non-persistent (NP). struct all_non_persistent { bool operator()(key_type const&) const noexcept { return false; } @@ -224,9 +622,57 @@ class CacheManager { /// Fold the per-op live working set @p current_bytes into the running /// high-water mark and return the updated mark. Reported as `hw=` in the - /// per-op eval trace; monotonically non-decreasing until reset(). - size_t note_working_set(size_t current_bytes) noexcept { + /// per-op eval trace; monotonically non-decreasing until reset(). @p op_hash + /// (default 0) identifies the op node being evaluated at the call site (0 + /// when no node is in scope there); forwarded to \c peak_monitor()'s + /// \c observe() so a wired \c PeakMonitor can report WHERE its hierarchy- + /// wide high-water was observed. + size_t note_working_set(size_t current_bytes, size_t op_hash = 0) noexcept { working_set_hwmark_ = std::max(working_set_hwmark_, current_bytes); + if (auto* m = peak_monitor()) { + // DIAGNOSTIC (analysis-only): if a live-set capture hook is installed, + // enumerate the chain's alive entries BEFORE observe() advances the mark, + // on each real high-water advance. Gated on on_peak_liveset being set, so + // the default path is byte-identical (no enumeration). + if (m->on_peak_liveset && current_bytes > m->hwmark_bytes) { + std::vector live; + for (CacheManager const* c = this; c; c = c->parent_) + for (auto const& [k, e] : c->cache_map_) + if (e.alive()) live.push_back({k->hash_value(), e.size_in_bytes()}); + m->on_peak_liveset(current_bytes, live); + } + m->observe(current_bytes, op_hash); + } + // DIAGNOSTIC (SEQUANT_UT_PEAK_COMPOSE): on each new GLOBAL max working set, + // print what composes it -- the co-resident cache chain vs the single + // transient result/scratch being formed (current_bytes - chain_residency), + // plus the largest single alive entry. Answers whether the realized peak is + // cache-co-residency-bound or transient-working-set-bound. Env-gated, off + // by default; harmless (a fprintf on monotone maxima only). + static bool const compose = + std::getenv("SEQUANT_UT_PEAK_COMPOSE") != nullptr; + if (compose) { + static size_t g_max = 0; + if (current_bytes > g_max) { + g_max = current_bytes; + size_t const chain = chain_residency(); + size_t const transient = + current_bytes > chain ? current_bytes - chain : 0; + size_t max_entry = 0, n_alive = 0; + for (CacheManager const* c = this; c; c = c->parent_) + for (auto const& [k, e] : c->cache_map_) + if (e.alive()) { + ++n_alive; + max_entry = std::max(max_entry, e.size_in_bytes()); + } + std::fprintf(stderr, + "[peak-compose] max=%.1f GB = cache_chain %.1f + " + "transient(result) %.1f | n_alive_chain=%zu " + "max_single_alive=%.1f GB\n", + current_bytes / 1e9, chain / 1e9, transient / 1e9, n_alive, + max_entry / 1e9); + } + } return working_set_hwmark_; } @@ -235,14 +681,135 @@ class CacheManager { return working_set_hwmark_; } + /// DIAGNOSTIC: record one product build of @p key at slice @p slice_sig + /// costing @p flops (this build's actual, realized-extent cost). @p slice_sig + /// is the enclosing batch context projected onto the modes @p key carries + /// (empty when the value is invariant to every live loop), so repeats of ONE + /// slice fold (recompute) while distinct slices stay separate (tiling). + /// Routes to the scope-chain ROOT so every build -- from any per-batch + /// scratch -- accumulates in ONE map keyed by node identity (see + /// recompute_tally_). Called only in the dry-run costing replay. + void tally_build(key_type const& key, std::string const& slice_sig, + double flops, double exec) noexcept { + if (parent_) { + parent_->tally_build(key, slice_sig, flops, exec); + return; + } + if (!recompute_tally_enabled_) return; // wet path: no-op + auto& slice = recompute_tally_[key].slices[slice_sig]; + slice.count += 1; // one more build of this exact (value, slice) + slice.flops = flops; // this slice's actual cost (same for repeats) + slice.exec = exec; // this slice's actual exec-cost (same for repeats) + } + + /// Enable/disable the per-node recompute tally (see + /// recompute_tally_enabled_). Set on the root cache by the dry-run costing + /// replay; left false everywhere else so tally_build() is a no-op on the wet + /// eval path. + void set_recompute_tally_enabled(bool on) noexcept { + recompute_tally_enabled_ = on; + } + + /// \return the per-DISTINCT-value build tally accumulated by tally_build() + /// on this (root) cache (see recompute_tally_). Read after the replay + /// to roll up avoidable recompute per node identity. + [[nodiscard]] std::unordered_map const& + recompute_tally() const noexcept { + return recompute_tally_; + } + + /// Sum over ALIVE entries of this cache's own residency (bytes). Unlike + /// working_set_hwmark() (a high-water MAX over time), this is the CURRENT + /// live residency at the instant of the call. + [[nodiscard]] size_t current_residency() const noexcept { + size_t s = 0; + for (auto const& [k, e] : cache_map_) + if (e.alive()) s += e.size_in_bytes(); + return s; + } + /// current_residency() of this cache plus every ancestor along the scope + /// chain (parent_): the total live residency visible at this scope at one + /// instant. + [[nodiscard]] size_t chain_residency() const noexcept { + return current_residency() + (parent_ ? parent_->chain_residency() : 0); + } + + /// \return true iff some ALIVE entry on this cache or any ancestor along the + /// scope chain physically holds @p value (pointer identity). + /// Read-only: unlike access_at() it decays no lifetime. The peak + /// trace uses it to skip an operand whose bytes are already counted + /// -- locally in \c bytes(cache,...) or up-chain in \c + /// chain_residency() + /// -- because the operand aliases that resident buffer. A sliced (or + /// permuted, or phase-shifted) read of a resident value is a DISTINCT + /// buffer with a different pointer, so it is correctly NOT skipped. + [[nodiscard]] bool chain_holds(ResultPtr const& value) const noexcept { + if (!value) return false; + for (auto const& [k, e] : cache_map_) + if (e.holds(value)) return true; + return parent_ ? parent_->chain_holds(value) : false; + } + /// /// @brief Access cached data. /// /// @param key The key that identifies the cached data. - /// @return ResultPtr to Result - ResultPtr access(key_type const& key) noexcept { + /// @return the fetched pointer plus the hop distance (number of parent links + /// crossed) to the scope that held it; {nullptr, 0} on a total miss. + /// + /// A local entry only "hits" if it is currently holding data; a key + /// registered locally but never (yet) stored here -- e.g. a hoisted + /// loop-invariant node whose value lives only at an ancestor level -- is a + /// local miss just like an unregistered key, and must fall through the + /// same way. Standalone (parent_ == nullptr) behavior is unchanged: a total + /// miss returns {nullptr, 0}. The hop distance surfaces the value's lifetime + /// scope so the caller (Enter-stage slice-on-use) can slice it to exactly the + /// batch loops the fetch crossed. + [[nodiscard]] AccessResult access_at(key_type const& key) noexcept { if (auto found = cache_map_.find(key); found != cache_map_.end()) - return found->second.access(); + if (auto data = found->second.access(); data) { + // DIAGNOSTIC (SEQUANT_UT_ACCESS_CLOCK): stamp this genuine local-hit + // read into the global access clock. No-op when the gate is off. + eval::AccessClock::stamp(found->first->hash_value()); + return {data, 0}; + } + if (!parent_) return {nullptr, 0}; + auto up = parent_->access_at(key); + return {up.ptr, up.hops + 1}; // count the link we just crossed + } + + /// @param key The key that identifies the cached data. + /// @return ResultPtr to Result. Thin forwarder to access_at() that drops the + /// hop distance, for the non-batched callers that do not slice. + ResultPtr access(key_type const& key) noexcept { return access_at(key).ptr; } + + /// Fetch @p key from EXACTLY @p hops scopes up the chain (walk @p hops + /// parent links, then one LOCAL entry::access() there), rather than + /// searching the chain like access_at() does. Used by a router-directed + /// read that already knows the target scope (e.g. from a HomeTarget's + /// home_depth) and wants to enforce that home rather than fall through to + /// whatever scope happens to hold the value. + /// + /// @param key The key that identifies the cached data. + /// @param hops The exact number of parent links to walk before accessing. + /// @return the fetched pointer, decaying that scope's entry the same single + /// lifetime step as access_at() (entry::access()); nullptr if the + /// walk runs off the root before @p hops links, or if the target + /// scope does not currently hold @p key. + [[nodiscard]] ResultPtr access_at_hops(key_type const& key, + std::size_t hops) noexcept { + CacheManager* c = this; + for (std::size_t i = 0; i < hops && c; ++i) c = c->parent_; + if (!c) return nullptr; + if (auto found = c->cache_map_.find(key); found != c->cache_map_.end()) { + auto data = found->second.access(); + // DIAGNOSTIC (SEQUANT_UT_ACCESS_CLOCK): stamp this genuine + // router-directed read into the global access clock. No-op when the gate + // is off. + if (data) eval::AccessClock::stamp(found->first->hash_value()); + return data; + } return nullptr; } @@ -306,6 +873,20 @@ class CacheManager { return iter != cache_map_.end() && iter->second.alive(); } + /// \return true iff @p key is alive (holding data) at THIS cache or ANY + /// ancestor scope up the parent chain. Non-decrementing (unlike \c + /// access_at): a pure residency probe. Used by \c + /// make_batched_scratch to decide that a batch-invariant value + /// already resident at its home is read from there each batch (the + /// parent-chain fall-through), so it is neither registered nor + /// rebuilt in the per-batch scratch. + [[nodiscard]] bool resident_in_chain(key_type const& key) const noexcept { + if (auto iter = cache_map_.find(key); + iter != cache_map_.end() && iter->second.alive()) + return true; + return parent_ ? parent_->resident_in_chain(key) : false; + } + /// \return true iff the key is registered for caching and classified /// persistent (P: never released on access, survives reset()). [[nodiscard]] bool persistent(key_type const& key) const noexcept { @@ -436,13 +1017,6 @@ struct zero_footprint { double operator()(auto const&) const noexcept { return 0.; } }; -/// Default batchability predicate for cache_manager: no index is batchable, so -/// the free-batchable-axis caching veto is inert (preserves the pre-batch -/// behavior for callers that do not pass a predicate). -struct never_batchable { - bool operator()(auto const&) const noexcept { return false; } -}; - /// \param nodes the evaluation forest. /// \param is_volatile `bool(TreeNode const&)`: true if the node is /// intrinsically volatile. Only its value on leaves matters in practice @@ -458,26 +1032,22 @@ struct never_batchable { /// of huge intermediates that carry a free large-space index (e.g. a /// half-transformed DF integral with a free projected-AO index), at the /// cost of recomputation. 0 (default) disables the gate. -/// \param is_batchable_index `bool(Index const&)`: an index the runtime batched -/// evaluator slices over (typically the DF/RI auxiliary). A node whose -/// *result* (canonical) indices contain such an index carries a -/// batchable axis FREE: the evaluator slices it per batch and the -/// single-term optimizer prices it sliced, so caching it whole would -/// hold an intermediate both other components mean to slice. Such nodes -/// are NOT cached (neither NP repeat nor P frontier) -- recomputed -/// (sliced under each consumer's batch trigger) instead of materialized -/// whole and held. This is the structural counterpart of \p -/// max_footprint: the batch axis, not a byte threshold, identifies the -/// free-large-index intermediates. The default never_batchable accepts -/// nothing, leaving the veto inert. +/// +/// A node is also refused run-scope residence when it is BATCH-VARIANT: its +/// cross-occurrence lifetime mask is non-empty (\c !EvalExpr::mask_all_full(); +/// this builder itself calls \c stamp_lifetime_masks over \p nodes before the +/// DAG walk below, so the mask is always current here regardless of caller -- +/// see \c lifetime_mask.hpp). Such a node is sliced by some enclosing External +/// batch mode in every occurrence, so its value differs per batch of that mode +/// -- caching it whole at run scope would serve a wrong-batch value to a deeper +/// consumer on cache fall-through (the F1 hazard). Only an all-full node (empty +/// mask, including every node on the OFF path) is admitted. /// \see CacheManager, cache_manager template + typename FootprintOf = zero_footprint> auto cache_manager(meta::eval_node_range auto const& nodes, auto&& is_volatile, size_t min_repeats = 2, FootprintOf footprint_of = {}, - double max_footprint = 0., - IsBatchableIndex is_batchable_index = {}) + double max_footprint = 0.) requires requires( std::ranges::range_value_t> const& n) { @@ -485,6 +1055,17 @@ auto cache_manager(meta::eval_node_range auto const& nodes, auto&& is_volatile, { footprint_of(n) } -> std::convertible_to; } { + // Stamp the cross-occurrence lifetime mask on this SAME forest before the + // DAG walk / veto below reads it (the batch-variant veto reads + // EvalExpr::mask_all_full()). Doing this here -- rather than leaving it to + // each caller -- makes "mask is current for the veto" an invariant of this + // builder instead of a per-caller obligation: every caller of this overload + // (SeQuant's build_dryrun_cache, mpqc's build_cache_manager) is covered + // uniformly. Unconditional and idempotent; a no-op when the forest carries + // no External batched_here() stamps (every mask stays empty/all-full), so + // this never changes behavior on the OFF path. + sequant::stamp_lifetime_masks(nodes); + using TreeNode = std::ranges::range_value_t>; using Hasher = TreeNodeHasher; @@ -523,23 +1104,35 @@ auto cache_manager(meta::eval_node_range auto const& nodes, auto&& is_volatile, // Footprint gate: a node whose result is larger than max_footprint is never // cached (so it is recomputed by each consumer rather than materialized whole // and held), bounding the footprint of huge free-large-index intermediates. - // Free-batchable-axis veto: a node whose result carries an index the runtime - // slices over (is_batchable_index) is, by construction, a free-large-index - // intermediate the evaluator builds one batch-slice at a time and the - // optimizer prices sliced. Caching it -- as an NP repeat or an NV/V-frontier - // P node -- would materialize and hold it whole, contradicting both. Veto its - // caching (the structural form of the max_footprint gate) so each consumer - // recomputes it sliced under its own batch trigger. + // Batch-variant veto ("a batched node cannot be run-scope"): this builder + // populates the outermost / persistent (run-scope) cache, so it must refuse + // any node that is batch-VARIANT -- one whose cached value would depend on + // which batch is live. Such a node is refused for two reasons: caching it + // whole contradicts the runtime slicing it (and the optimizer pricing it + // sliced), and -- the F1 safety invariant -- a child batch scratch that + // misses locally falls through to this cache for ANY key, so a batch-variant + // final left here could be served full (wrong-batch) to an inner body. A node + // is batch-variant iff its cross-occurrence lifetime mask is non-empty (\c + // !n->mask_all_full(), \c lifetime_mask.hpp) -- some External batch mode (of + // this node or an enclosing ancestor, over ALL its occurrences under the + // canonical meet) slices it, so its value differs per batch of that mode even + // if it slices nothing itself. + // A node that is invariant to every batched mode is NOT vetoed and stays + // cacheable at run scope -- this is where a hoisted loop-invariant + // intermediate (all-full mask; or an External-only / no batched_here entry, + // e.g. gC) lands. OFF path (no order-aware annotations, hence no \c + // stamp_lifetime_masks External stamps): every mask is empty (all-full, + // \c EvalExpr::sliced_modes_ default-constructed), so the veto never fires + // and admits exactly what it did before -- byte-identical. std::unordered_map filtered; for (auto&& [n, c] : counts) { if (!(c >= min_repeats || persistent.contains(n))) continue; - bool free_batchable_axis = false; - for (auto const& ix : n->canon_indices()) - if (is_batchable_index(ix)) { - free_batchable_axis = true; - break; - } - if (free_batchable_axis || + // Batch-variant: a node whose cross-occurrence lifetime mask is non-empty + // is sliced by some enclosing external mode in every occurrence => its + // value differs per batch => refused run-scope residence. all-full (empty + // mask; incl. the OFF path) is admitted. + bool const batch_variant = !n->mask_all_full(); + if (batch_variant || (max_footprint > 0. && footprint_of(n) > max_footprint)) { persistent.erase(n); // keep is_persistent consistent with what is cached continue; diff --git a/SeQuant/core/eval/eval.hpp b/SeQuant/core/eval/eval.hpp index 96e3ffbea9..6818ab1422 100644 --- a/SeQuant/core/eval/eval.hpp +++ b/SeQuant/core/eval/eval.hpp @@ -7,11 +7,16 @@ #include #include #include +#include +#include #include +#include +#include #include #include #include #include +#include #include #include @@ -19,7 +24,9 @@ #include #include +#include #include +#include #include #include #include @@ -199,7 +206,7 @@ enum struct TermMode { Begin, End }; /// One log record per eval op. Line format: /// // clang-format off -/// Eval | |