diff --git a/onnxoptimizer/optimize.cc b/onnxoptimizer/optimize.cc index 1e6d055d6..ab63ad614 100644 --- a/onnxoptimizer/optimize.cc +++ b/onnxoptimizer/optimize.cc @@ -46,16 +46,18 @@ ModelProto OptimizeFixed( void OptimizeGraph( Graph& graph, const std::vector& names, - std::map* report) { + std::map* report, + bool clear_tensor_digest_cache) { Optimizer current_opt(names, false); - current_opt.optimize(graph, report); + current_opt.optimize(graph, report, clear_tensor_digest_cache); } void OptimizeGraphFixed( Graph& graph, const std::vector& names, - std::map* report) { + std::map* report, + bool clear_tensor_digest_cache) { Optimizer current_opt(names, true); - current_opt.optimize(graph, report); + current_opt.optimize(graph, report, clear_tensor_digest_cache); } #ifdef ONNX_IR_PB_CONVERTER_HAS_CONSUMING_OVERLOADS ModelProto Optimize( diff --git a/onnxoptimizer/optimize.h b/onnxoptimizer/optimize.h index 364a9374f..ba9626a74 100644 --- a/onnxoptimizer/optimize.h +++ b/onnxoptimizer/optimize.h @@ -10,10 +10,10 @@ #include "onnx/common/ir.h" #include "onnx/common/ir_pb_converter.h" #include "onnx/proto_utils.h" - #include "onnxoptimizer/pass_manager.h" #include "onnxoptimizer/pass_registry.h" - +#include "onnxoptimizer/passes/cse_util.h" +#include "onnxoptimizer/passes/tensor_content_hash.h" #include "vector" namespace ONNX_NAMESPACE { @@ -38,8 +38,28 @@ struct Optimizer { // If `report` is non-null it is filled with a map from pass name to the // total number of positive transforms that pass applied to the graph, // matching the ModelProto-based optimize() below. + // + // If `clear_tensor_digest_cache` is true (the default, and correct for + // essentially every caller), the two tensor-hash caches consulted by + // eliminate_duplicate_initializer and eliminate_common_subexpression -- + // TensorContentDigest's (tensor_content_hash.h, the typed-field path) and + // CSETensorHash's raw_data-branch cache (cse_util.h's g_raw_hash_cache, + // the common path for real exported models) -- are cleared before + // running the passes, bounding their memory to the tensors this one + // optimize() call touches. Pass false only if the caller itself manages + // those caches' lifetime across several optimize() calls on the *same* + // resident Graph (e.g. onnxsim's OptAndShape fixed point, which calls + // this once per round but wants hashes computed in an earlier round to + // stay cached in a later one) -- see ClearTensorContentDigestCache's + // header comment for why that's safe to do explicitly (the same + // reasoning applies to ClearRawHashCache). void optimize(Graph &graph, - std::map *report = nullptr) { + std::map *report = nullptr, + bool clear_tensor_digest_cache = true) { + if (clear_tensor_digest_cache) { + ClearTensorContentDigestCache(); + ClearRawHashCache(); + } auto analysis = this->pass_manager->run(graph); if (report != nullptr && analysis != nullptr) { *report = analysis->transform_counts; @@ -50,7 +70,7 @@ struct Optimizer { // total number of positive transforms that pass applied to the graph. ModelProto optimize(const ModelProto &_mp_in, std::map *report = nullptr) { - const ModelProto* mp_in = &_mp_in; + const ModelProto *mp_in = &_mp_in; std::unique_ptr copy_in; if (mp_in->ir_version() == 3) { // Upgrade ir_version to 4 so that initializer can be not in input @@ -129,8 +149,8 @@ struct Optimizer { void AddFunctionsToModel(const ModelProto &original_model, ModelProto &output_model) { - for (const auto& function_proto : original_model.functions()) { - auto* p_f = output_model.add_functions(); + for (const auto &function_proto : original_model.functions()) { + auto *p_f = output_model.add_functions(); p_f->CopyFrom(function_proto); } } @@ -192,18 +212,22 @@ ModelProto OptimizeFixed(const ModelProto &mp_in, // ModelProto at all, so they work identically whether this library is // linked against onnxsim's onnx fork or onnxruntime's bundled, unpatched // onnx copy. +// `clear_tensor_digest_cache`: see Optimizer::optimize(Graph&, ...)'s doc +// comment above -- the default (true) is correct for essentially every +// caller. void OptimizeGraph(Graph &graph, const std::vector &names, - std::map *report = nullptr); + std::map *report = nullptr, + bool clear_tensor_digest_cache = true); void OptimizeGraphFixed(Graph &graph, const std::vector &names, - std::map *report = nullptr); + std::map *report = nullptr, + bool clear_tensor_digest_cache = true); #ifdef ONNX_IR_PB_CONVERTER_HAS_CONSUMING_OVERLOADS // Consuming overloads: see Optimizer::optimize(ModelProto&, ...)'s doc // comment. Only call these when `mp_in` is about to be discarded or // overwritten by the caller. -ModelProto Optimize(ModelProto &mp_in, - const std::vector &names, +ModelProto Optimize(ModelProto &mp_in, const std::vector &names, std::map *report = nullptr); ModelProto OptimizeFixed(ModelProto &mp_in, diff --git a/onnxoptimizer/pass.cc b/onnxoptimizer/pass.cc index f800d59b9..9955230fe 100644 --- a/onnxoptimizer/pass.cc +++ b/onnxoptimizer/pass.cc @@ -2,17 +2,92 @@ // // SPDX-License-Identifier: Apache-2.0 -#include "onnx/common/assertions.h" - #include "onnxoptimizer/pass.h" +#include + +#include "onnx/common/assertions.h" + namespace ONNX_NAMESPACE { namespace optimization { -Pass::Pass( - PassType pass_type, - PassEfficiency pass_efficiency, - PassOptimizationType pass_optimization_type) { +namespace { +bool g_pass_phase_profiling_enabled = false; +std::unordered_map g_pass_phase_timings; +std::unordered_map g_pass_total_timings; +CSEPassTiming g_cse_pass_timing; +DeadendPassTiming g_deadend_pass_timing; +} // namespace + +void SetPassPhaseProfilingEnabled(bool enabled) { + g_pass_phase_profiling_enabled = enabled; +} + +bool GetPassPhaseProfilingEnabled() { + return g_pass_phase_profiling_enabled; +} + +const std::unordered_map& GetPassPhaseTimings() { + return g_pass_phase_timings; +} + +void ResetPassPhaseTimings() { + g_pass_phase_timings.clear(); +} + +void RecordPassTotalTime(const std::string& pass_name, double ms) { + PassTotalTiming& t = g_pass_total_timings[pass_name]; + t.calls++; + t.total_ms += ms; +} + +const std::unordered_map& GetPassTotalTimings() { + return g_pass_total_timings; +} + +void ResetPassTotalTimings() { + g_pass_total_timings.clear(); +} + +void RecordCSEPassTiming(uint64_t nodes_seen, uint64_t nodes_filtered_out, + uint64_t nodes_replaced, double filter_ms, + double lookup_ms, double replace_ms) { + g_cse_pass_timing.calls++; + g_cse_pass_timing.nodes_seen += nodes_seen; + g_cse_pass_timing.nodes_filtered_out += nodes_filtered_out; + g_cse_pass_timing.nodes_replaced += nodes_replaced; + g_cse_pass_timing.filter_ms += filter_ms; + g_cse_pass_timing.lookup_ms += lookup_ms; + g_cse_pass_timing.replace_ms += replace_ms; +} + +const CSEPassTiming& GetCSEPassTiming() { + return g_cse_pass_timing; +} + +void ResetCSEPassTiming() { + g_cse_pass_timing = CSEPassTiming(); +} + +void RecordDeadendPassTiming(uint64_t nodes_seen, uint64_t nodes_removed, + double has_uses_ms, double destroy_ms) { + g_deadend_pass_timing.calls++; + g_deadend_pass_timing.nodes_seen += nodes_seen; + g_deadend_pass_timing.nodes_removed += nodes_removed; + g_deadend_pass_timing.has_uses_ms += has_uses_ms; + g_deadend_pass_timing.destroy_ms += destroy_ms; +} + +const DeadendPassTiming& GetDeadendPassTiming() { + return g_deadend_pass_timing; +} + +void ResetDeadendPassTiming() { + g_deadend_pass_timing = DeadendPassTiming(); +} + +Pass::Pass(PassType pass_type, PassEfficiency pass_efficiency, + PassOptimizationType pass_optimization_type) { this->pass_type = pass_type; this->pass_efficiency = pass_efficiency; this->pass_optimization_type = pass_optimization_type; @@ -21,8 +96,7 @@ Pass::Pass( Pass::~Pass() {} unsigned int Pass::DescendOnGraphAttributesAndCount( - Node* n, - std::function fn) { + Node* n, std::function fn) { unsigned int num_changes = 0; for (auto name : n->attributeNames()) { auto kind = n->kindOf(name); @@ -39,8 +113,7 @@ unsigned int Pass::DescendOnGraphAttributesAndCount( } void Pass::DescendOnGraphAttributesUnconstrained( - Node* n, - std::function fn) { + Node* n, std::function fn) { for (auto name : n->attributeNames()) { auto kind = n->kindOf(name); if (kind == AttributeKind::g) { @@ -58,13 +131,39 @@ PredicateBasedPass::~PredicateBasedPass() {} unsigned int PredicateBasedPass::_runPassInternal(Graph& graph) { unsigned int num_changes = false; + // Only touches g_pass_phase_timings when profiling is on, so the lookup + // (once per call, not once per node) and the two std::chrono reads per + // node below are the only cost this diagnostic imposes when enabled. + const bool profiling = g_pass_phase_profiling_enabled; + PassPhaseTiming* timing = + profiling ? &g_pass_phase_timings[this->getPassName()] : nullptr; for (auto it = graph.begin(); it != graph.end(); ++it) { auto* n = *it; num_changes += this->DescendOnGraphAttributesAndCount( n, [this](Graph& g) { return _runPassInternal(g); }); - if (this->patternMatchPredicate(n)) { + bool matched; + if (profiling) { + const auto t0 = std::chrono::steady_clock::now(); + matched = this->patternMatchPredicate(n); + const auto t1 = std::chrono::steady_clock::now(); + timing->match_calls++; + timing->match_ms += + std::chrono::duration(t1 - t0).count(); + } else { + matched = this->patternMatchPredicate(n); + } + if (matched) { NodeDestroyType destroy_type = NodeDestroyType::DestroyZero; - num_changes += this->runTransform(n, graph, destroy_type); + if (profiling) { + const auto t0 = std::chrono::steady_clock::now(); + num_changes += this->runTransform(n, graph, destroy_type); + const auto t1 = std::chrono::steady_clock::now(); + timing->transform_calls++; + timing->transform_ms += + std::chrono::duration(t1 - t0).count(); + } else { + num_changes += this->runTransform(n, graph, destroy_type); + } if (destroy_type == NodeDestroyType::DestroyOne) { it.destroyCurrent(); @@ -88,9 +187,7 @@ std::shared_ptr PredicateBasedPass::runPass(Graph& graph) { } CountBasedPassAnalysis::CountBasedPassAnalysis( - Pass* pass, - unsigned int num_positive_transforms, - bool initialization_done, + Pass* pass, unsigned int num_positive_transforms, bool initialization_done, bool finalization_done) { this->pass = pass; this->num_positive_transforms = num_positive_transforms; @@ -100,5 +197,5 @@ CountBasedPassAnalysis::CountBasedPassAnalysis( FullGraphBasedPass::~FullGraphBasedPass() {} -} // namespace optimization -} // namespace ONNX_NAMESPACE +} // namespace optimization +} // namespace ONNX_NAMESPACE diff --git a/onnxoptimizer/pass.h b/onnxoptimizer/pass.h index f3179335b..e36c88004 100644 --- a/onnxoptimizer/pass.h +++ b/onnxoptimizer/pass.h @@ -9,7 +9,10 @@ #pragma once +#include #include +#include + #include "onnx/common/ir.h" #include "onnx/onnx_pb.h" @@ -21,6 +24,85 @@ struct PostPassAnalysis { virtual ~PostPassAnalysis() = default; }; +// Exploratory diagnostic: per-pass-name timing split between +// PredicateBasedPass's two phases -- "matching" (patternMatchPredicate, +// scanning nodes for rewrite candidates) and "modifying" (runTransform, +// actually rewriting a matched node) -- written for onnxsim issue #633's +// investigation into where OptimizeGraphFixed's ~50-round fixed point +// actually spends its time. Covers PredicateBasedPass-derived passes only +// (the majority of the default suite: fuse_*, most eliminate_*); the +// smaller number of FullGraphBasedPass passes (eliminate_duplicate_ +// initializer, eliminate_common_subexpression, DCE, ...) implement their +// own single-phase runPass() and aren't split by this. +// +// Off by default (SetPassPhaseProfilingEnabled(true) to turn on) so normal +// runs pay zero std::chrono overhead. Not thread-safe to toggle +// concurrently with a running pass, matching this library's other global +// toggles (e.g. tensor_content_hash.h's SetTrustTensorContentHash). +struct PassPhaseTiming { + uint64_t match_calls = 0; + double match_ms = 0.0; + uint64_t transform_calls = 0; + double transform_ms = 0.0; +}; +void SetPassPhaseProfilingEnabled(bool enabled); +bool GetPassPhaseProfilingEnabled(); +const std::unordered_map &GetPassPhaseTimings(); +void ResetPassPhaseTimings(); + +// Companion to PassPhaseTiming, at coarser granularity: total wall time +// inside each pass's runPass(Graph&) call (FixedPointPassManager::run's +// call sites), covering BOTH pass kinds uniformly -- PredicateBasedPass's +// per-node loop overhead that PassPhaseTiming's match/transform timers don't +// capture (iterator traversal, DescendOnGraphAttributesAndCount, ...) and +// FullGraphBasedPass passes (eliminate_duplicate_initializer, eliminate_ +// common_subexpression, DCE, ...) that don't have a matching/modifying split +// at all. Shares SetPassPhaseProfilingEnabled's on/off toggle. +struct PassTotalTiming { + uint64_t calls = 0; + double total_ms = 0.0; +}; +void RecordPassTotalTime(const std::string &pass_name, double ms); +const std::unordered_map &GetPassTotalTimings(); +void ResetPassTotalTimings(); + +// Internal breakdown of EliminateCommonSubexpressions's own per-node loop +// (eliminate_common_subexpression.h), beyond what cse_util.h's CSENodeHash/ +// CSEEqual instrumentation already measures inside the hash-map lookup +// itself. `lookup_ms` covers the whole `hash_map.emplace()` call (hashing +// plus, on a bucket collision, CSEEqual), so it overlaps with cse_util.h's +// node_hash_ms/node_equal_ms -- the two are complementary views of the same +// work, not additive. Shares SetPassPhaseProfilingEnabled's on/off toggle. +struct CSEPassTiming { + uint64_t calls = 0; + uint64_t nodes_seen = 0; + uint64_t nodes_filtered_out = 0; + uint64_t nodes_replaced = 0; + double filter_ms = 0.0; + double lookup_ms = 0.0; + double replace_ms = 0.0; +}; +void RecordCSEPassTiming(uint64_t nodes_seen, uint64_t nodes_filtered_out, + uint64_t nodes_replaced, double filter_ms, + double lookup_ms, double replace_ms); +const CSEPassTiming &GetCSEPassTiming(); +void ResetCSEPassTiming(); + +// Internal breakdown of EliminateDead's own reverse-order sweep +// (eliminate_deadend.h). Shares SetPassPhaseProfilingEnabled's on/off +// toggle. +struct DeadendPassTiming { + uint64_t calls = 0; + uint64_t nodes_seen = 0; + uint64_t nodes_removed = 0; + double has_uses_ms = 0.0; + double destroy_ms = 0.0; +}; +void RecordDeadendPassTiming(uint64_t nodes_seen, uint64_t nodes_removed, + double has_uses_ms, double destroy_ms); +const DeadendPassTiming &GetDeadendPassTiming(); +void ResetDeadendPassTiming(); + // Enum that represents the type of optimization it is. enum PassType { // Class of optimizations that fuses operations. diff --git a/onnxoptimizer/pass_manager.cc b/onnxoptimizer/pass_manager.cc index 7331aa9b3..a93c7eaa8 100644 --- a/onnxoptimizer/pass_manager.cc +++ b/onnxoptimizer/pass_manager.cc @@ -3,11 +3,34 @@ // SPDX-License-Identifier: Apache-2.0 #include "onnxoptimizer/pass_manager.h" + +#include + #include "onnxoptimizer/passes/logging.h" namespace ONNX_NAMESPACE { namespace optimization { +namespace { +// Times a single pass->runPass(graph) call when pass-phase profiling +// (pass.h's SetPassPhaseProfilingEnabled) is on; a plain passthrough +// otherwise. See PassTotalTiming's comment for what this captures beyond +// PredicateBasedPass's own matching/modifying timers. +std::shared_ptr RunPassTimed( + const std::shared_ptr& pass, Graph& graph) { + if (!GetPassPhaseProfilingEnabled()) { + return pass->runPass(graph); + } + const auto t0 = std::chrono::steady_clock::now(); + auto analysis = pass->runPass(graph); + const auto t1 = std::chrono::steady_clock::now(); + RecordPassTotalTime( + pass->getPassName(), + std::chrono::duration(t1 - t0).count()); + return analysis; +} +} // namespace + PassManager::PassManager() {} PassManager::~PassManager() {} @@ -40,7 +63,7 @@ std::shared_ptr FixedPointPassManager::run(Graph& graph) { do { fixed_point_optimization_done = false; for (const std::shared_ptr& pass : this->passes) { - std::shared_ptr analysis = pass->runPass(graph); + std::shared_ptr analysis = RunPassTimed(pass, graph); if (pass->getPassAnalysisType() == PassAnalysisType::Empty) { continue; } @@ -49,16 +72,18 @@ std::shared_ptr FixedPointPassManager::run(Graph& graph) { report->transform_counts[pass->getPassName()] += count_analysis->num_positive_transforms; if (count_analysis->num_positive_transforms != 0) { - VLOG(1) << "Pass " << pass->getPassName() << " transformed " << count_analysis->num_positive_transforms; + VLOG(1) << "Pass " << pass->getPassName() << " transformed " + << count_analysis->num_positive_transforms; } while (count_analysis->fixedPointOptimizationNeeded()) { count_analysis = std::static_pointer_cast( - pass->runPass(graph)); + RunPassTimed(pass, graph)); report->transform_counts[pass->getPassName()] += count_analysis->num_positive_transforms; if (count_analysis->num_positive_transforms != 0) { - VLOG(1) << "Pass " << pass->getPassName() << " transformed " << count_analysis->num_positive_transforms; + VLOG(1) << "Pass " << pass->getPassName() << " transformed " + << count_analysis->num_positive_transforms; } fixed_point_optimization_done = true; } @@ -67,5 +92,5 @@ std::shared_ptr FixedPointPassManager::run(Graph& graph) { return report; } -} // namespace optimization -} // namespace ONNX_NAMESPACE +} // namespace optimization +} // namespace ONNX_NAMESPACE diff --git a/onnxoptimizer/passes/cse_util.h b/onnxoptimizer/passes/cse_util.h index e5912fc91..db4482f68 100644 --- a/onnxoptimizer/passes/cse_util.h +++ b/onnxoptimizer/passes/cse_util.h @@ -8,9 +8,13 @@ #pragma once #include +#include +#include #include #include #include +#include +#include #include "onnx/onnx_pb.h" #include "onnxoptimizer/pass.h" @@ -22,6 +26,105 @@ namespace ONNX_NAMESPACE { namespace optimization { +// Exploratory diagnostic (onnxsim issue #633): splits CSETensorHash/ +// CSETensorCompare's cost by which of their two branches did the work -- +// raw_data (a cheap byte hash / a single memcmp, the common case for real +// exported models) vs typed-field (a BLAKE3-digest-backed path, rarer). Off +// by default; shares pass.h's SetPassPhaseProfilingEnabled toggle so it +// switches on/off together with the pass-phase timers. +struct CSEHashCompareTiming { + uint64_t raw_hash_calls = 0; + double raw_hash_ms = 0.0; + // Of raw_hash_calls above, how many were served from g_raw_hash_cache + // below instead of recomputed. On real (raw_data-heavy) exported models, + // measured 98%+ on onnxsim issue #633's repro -- most initializers are + // unchanged, unmutated Tensor objects from one OptAndShape round to the + // next, so their hash need only ever be computed once. + uint64_t raw_hash_cache_hits = 0; + double raw_hash_cache_hit_ms = 0.0; + uint64_t raw_hash_cache_misses = 0; + double raw_hash_cache_miss_ms = 0.0; + uint64_t typed_hash_calls = 0; + double typed_hash_ms = 0.0; + uint64_t raw_compare_calls = 0; + double raw_compare_ms = 0.0; + uint64_t typed_compare_calls = 0; + double typed_compare_ms = 0.0; + // CSENodeHash/CSEEqual (below): the whole-node hash/equality machinery + // that eliminate_common_subexpression's hash_map keys on. Their cost is + // separate from raw_hash_*/typed_hash_* above -- those only fire for + // nodes with a tensor-valued (t/ts) attribute (e.g. Constant), while + // node_hash_ms/node_equal_ms cover every CSE-eligible node. + uint64_t node_hash_calls = 0; + double node_hash_ms = 0.0; + // Of node_hash_ms, time spent specifically in attributeNames() (an + // allocating std::vector return by value, see ir.h) plus sorting + // it -- isolated separately since it is the one obviously allocation-heavy + // step in an otherwise cheap hash. + uint64_t node_hash_attrsort_calls = 0; + double node_hash_attrsort_ms = 0.0; + uint64_t node_equal_calls = 0; + double node_equal_ms = 0.0; + // Same attributeNames()+sort isolation as node_hash_attrsort_*, but + // CSEEqual does it twice per call (once per side). + uint64_t node_equal_attrsort_calls = 0; + double node_equal_attrsort_ms = 0.0; +}; +inline CSEHashCompareTiming g_cse_hash_compare_timing; +inline void ResetCSEHashCompareTiming() { + g_cse_hash_compare_timing = {}; +} +inline const CSEHashCompareTiming& GetCSEHashCompareTiming() { + return g_cse_hash_compare_timing; +} + +// The actual cache: memoizes CSETensorHash's raw_data-branch seed by +// Tensor::tensor_id() (never reused across distinct content -- fresh on +// every construction/assignment, see tensor.h), so an unmutated tensor's +// hash is computed once and reused on every later lookup instead of +// rescanning its raw bytes from scratch each time. This is +// eliminate_duplicate_initializer's dominant cost on raw_data-heavy models +// (see onnxsim issue #633) -- CSETensorCompare's own raw_data fast path +// (a single memcmp on a hash-bucket hit) was already cheap and is +// unaffected. +// +// Cleared via ClearRawHashCache(), with the same lifetime rules as +// tensor_content_hash.h's ClearTensorContentDigestCache (see that +// function's header comment for the full rationale): no onnx-optimizer +// pass mutates a retained tensor's content in place, so this safely +// outlives a single pass call, up to whatever scope the caller (see +// Optimizer::optimize(Graph&, ...)'s clear_tensor_digest_cache parameter) +// chooses to clear it at. +inline std::unordered_map g_raw_hash_cache; +inline void ClearRawHashCache() { + g_raw_hash_cache.clear(); +} + +// RAII: adds the scope's elapsed wall time to *ms and increments *calls on +// destruction, only when profiling is on -- a no-op pair of branches +// otherwise. +class ScopedCSETiming { + public: + ScopedCSETiming(uint64_t* calls, double* ms) + : enabled_(GetPassPhaseProfilingEnabled()), calls_(calls), ms_(ms) { + if (enabled_) + start_ = std::chrono::steady_clock::now(); + } + ~ScopedCSETiming() { + if (enabled_) { + const auto end = std::chrono::steady_clock::now(); + (*calls_)++; + *ms_ += std::chrono::duration(end - start_).count(); + } + } + + private: + bool enabled_; + uint64_t* calls_; + double* ms_; + std::chrono::steady_clock::time_point start_; +}; + /// https://stackoverflow.com/questions/2590677/how-do-i-combine-hash-values-in-c0x inline void hash_combine(std::size_t& seed) {} @@ -64,6 +167,8 @@ inline bool CSETensorCompare(const Tensor* lhs, const Tensor* rhs) { // tensor_content_hash.h's header comment -- it does NOT go through // TensorContentDigest: BLAKE3 would only add cost here, not save any // (most real models are raw_data-heavy, so this is the hot path). + ScopedCSETiming _t(&g_cse_hash_compare_timing.raw_compare_calls, + &g_cse_hash_compare_timing.raw_compare_ms); return lhs->raw() == rhs->raw(); } @@ -79,6 +184,8 @@ inline bool CSETensorCompare(const Tensor* lhs, const Tensor* rhs) { // hit, not a fresh BLAKE3 pass. See GetTrustTensorContentHash's own // comment for the (practically negligible) tradeoff, and how to // disable it. + ScopedCSETiming _t(&g_cse_hash_compare_timing.typed_compare_calls, + &g_cse_hash_compare_timing.typed_compare_ms); return TensorContentDigest(*lhs) == TensorContentDigest(*rhs); } @@ -161,11 +268,44 @@ struct CSETensorHash { if (tensor->is_raw_data()) { // Cheap byte hash, matching CSETensorCompare's raw_data fast path - // above -- no BLAKE3 here, see that comment for why. + // above -- no BLAKE3 here, see that comment for why. Memoized by + // tensor_id() in g_raw_hash_cache: see that cache's own comment for + // why this is normally a hit, not a fresh scan of tensor->raw(). + const bool profiling = GetPassPhaseProfilingEnabled(); + std::chrono::steady_clock::time_point t0; + if (profiling) + t0 = std::chrono::steady_clock::now(); + + const uint64_t id = tensor->tensor_id(); + auto cached = g_raw_hash_cache.find(id); + if (cached != g_raw_hash_cache.end()) { + if (profiling) { + const auto t1 = std::chrono::steady_clock::now(); + const double ms = + std::chrono::duration(t1 - t0).count(); + g_cse_hash_compare_timing.raw_hash_calls++; + g_cse_hash_compare_timing.raw_hash_ms += ms; + g_cse_hash_compare_timing.raw_hash_cache_hits++; + g_cse_hash_compare_timing.raw_hash_cache_hit_ms += ms; + } + return cached->second; + } + std::size_t seed = 0; hash_combine(seed, std::hash(), elem_type); hash_combine(seed, CSEContainerHash(), tensor->sizes()); hash_combine(seed, std::hash(), tensor->raw()); + g_raw_hash_cache.emplace(id, seed); + + if (profiling) { + const auto t1 = std::chrono::steady_clock::now(); + const double ms = + std::chrono::duration(t1 - t0).count(); + g_cse_hash_compare_timing.raw_hash_calls++; + g_cse_hash_compare_timing.raw_hash_ms += ms; + g_cse_hash_compare_timing.raw_hash_cache_misses++; + g_cse_hash_compare_timing.raw_hash_cache_miss_ms += ms; + } return seed; } @@ -179,6 +319,8 @@ struct CSETensorHash { // check on any bucket collision, so using this hash can never by // itself cause two distinct tensors to be merged, whichever way that // toggle is set. + ScopedCSETiming _t(&g_cse_hash_compare_timing.typed_hash_calls, + &g_cse_hash_compare_timing.typed_hash_ms); return std::hash()(TensorContentDigest(*tensor)); } @@ -207,6 +349,8 @@ struct CSEContainerHash { struct CSENodeHash { std::size_t operator()(const Node* n) const { + ScopedCSETiming _t(&g_cse_hash_compare_timing.node_hash_calls, + &g_cse_hash_compare_timing.node_hash_ms); ONNX_ASSERT(n); std::size_t seed = 0; const auto inputs = n->inputs(); @@ -218,9 +362,15 @@ struct CSENodeHash { for (const auto& input : inputs) { hash_combine(seed, string_hasher, input->uniqueName()); } - auto attribute_names = n->attributeNames(); - SymbolCompare cmp; - std::sort(attribute_names.begin(), attribute_names.end(), cmp); + std::vector attribute_names; + { + ScopedCSETiming _attr_t( + &g_cse_hash_compare_timing.node_hash_attrsort_calls, + &g_cse_hash_compare_timing.node_hash_attrsort_ms); + attribute_names = n->attributeNames(); + SymbolCompare cmp; + std::sort(attribute_names.begin(), attribute_names.end(), cmp); + } for (const auto& name : attribute_names) { hash_combine(seed, sym_hasher, name); auto kind = n->kindOf(name); @@ -262,6 +412,8 @@ struct CSENodeHash { struct CSEEqual { bool operator()(const Node* lhs, const Node* rhs) const { + ScopedCSETiming _t(&g_cse_hash_compare_timing.node_equal_calls, + &g_cse_hash_compare_timing.node_equal_ms); if (!lhs) { return !rhs; } else if (!rhs) { @@ -272,11 +424,18 @@ struct CSEEqual { auto inputs_r = rhs->inputs(); auto outputs_l = lhs->outputs(); auto outputs_r = rhs->outputs(); - auto attr_names_l = lhs->attributeNames(); - auto attr_names_r = rhs->attributeNames(); - SymbolCompare cmp; - std::sort(attr_names_l.begin(), attr_names_l.end(), cmp); - std::sort(attr_names_r.begin(), attr_names_r.end(), cmp); + std::vector attr_names_l; + std::vector attr_names_r; + { + ScopedCSETiming _attr_t( + &g_cse_hash_compare_timing.node_equal_attrsort_calls, + &g_cse_hash_compare_timing.node_equal_attrsort_ms); + attr_names_l = lhs->attributeNames(); + attr_names_r = rhs->attributeNames(); + SymbolCompare cmp; + std::sort(attr_names_l.begin(), attr_names_l.end(), cmp); + std::sort(attr_names_r.begin(), attr_names_r.end(), cmp); + } if (lhs->kind() != rhs->kind() || inputs_l.size() != inputs_r.size() || outputs_l.size() != outputs_r.size() || attr_names_l != attr_names_r) { return false; diff --git a/onnxoptimizer/passes/eliminate_common_subexpression.h b/onnxoptimizer/passes/eliminate_common_subexpression.h index f056b882d..5ca7058af 100644 --- a/onnxoptimizer/passes/eliminate_common_subexpression.h +++ b/onnxoptimizer/passes/eliminate_common_subexpression.h @@ -6,6 +6,7 @@ // Adventurous users should note that the APIs will probably change. #pragma once +#include #include #include "onnx/defs/tensor_util.h" @@ -14,7 +15,6 @@ #include "onnxoptimizer/passes/logging.h" #include "onnxoptimizer/passes/pass_util.h" #include "onnxoptimizer/passes/string_utils.h" -#include "onnxoptimizer/passes/tensor_content_hash.h" namespace ONNX_NAMESPACE { namespace optimization { @@ -31,37 +31,75 @@ struct EliminateCommonSubexpression final : public FullGraphBasedPass { } unsigned int EliminateCommonSubexpressions(Graph &graph) { - // Scoped to this call: see ClearTensorContentDigestCache's header - // comment for why it's safe here and must not be skipped. - ClearTensorContentDigestCache(); - + // No longer cleared here: see EliminateDuplicateInitializer's identical + // change for why (TensorContentDigest's cache now outlives a single pass + // call; clearing it is Optimizer::optimize(Graph&, ...)'s job). + const bool profiling = GetPassPhaseProfilingEnabled(); auto node_list = graph.nodes(); unsigned int cse_removed = 0; + uint64_t nodes_seen = 0; + uint64_t nodes_filtered_out = 0; + uint64_t nodes_replaced = 0; + double filter_ms = 0.0; + double lookup_ms = 0.0; + double replace_ms = 0.0; std::unordered_map hash_map; for (auto it = node_list.begin(); it != node_list.end(); ++it) { auto node = *it; auto kind = node->kind(); - if (!node->hasUses() || !IsSupportedByCSE(node)) { + nodes_seen++; + bool skip; + if (profiling) { + const auto t0 = std::chrono::steady_clock::now(); + skip = !node->hasUses() || !IsSupportedByCSE(node); + const auto t1 = std::chrono::steady_clock::now(); + filter_ms += std::chrono::duration(t1 - t0).count(); + } else { + skip = !node->hasUses() || !IsSupportedByCSE(node); + } + if (skip) { + nodes_filtered_out++; continue; } VLOG(2) << Str("kind: ", kind.toString(), ", ", node->name(), " is processing"); // A single emplace instead of find()-then-[]/at(): see - // EliminateDuplicateInitializer's identical fix for why. + // EliminateDuplicateInitializer's identical fix for why. Note this + // (and thus lookup_ms below) internally calls CSENodeHash and, on a + // bucket collision, CSEEqual -- see cse_util.h's node_hash_ms/ + // node_equal_ms for that same work's own breakdown. + std::chrono::steady_clock::time_point t0; + if (profiling) t0 = std::chrono::steady_clock::now(); auto insertion = hash_map.emplace(node, node); + if (profiling) { + const auto t1 = std::chrono::steady_clock::now(); + lookup_ms += std::chrono::duration(t1 - t0).count(); + } if (!insertion.second) { auto other = insertion.first->second; auto outputs = other->outputs(); auto replaced_outputs = node->outputs(); + std::chrono::steady_clock::time_point t2; + if (profiling) t2 = std::chrono::steady_clock::now(); for (int i = 0; i < outputs.size(); ++i) { if (tryReplacingAllUsesWith(replaced_outputs[i], outputs[i])) { VLOG(1) << Str("kind: ", kind.toString(), ", ", node->name(), " [", i, "] output has been replaced by ", other->name()); cse_removed++; + nodes_replaced++; } } + if (profiling) { + const auto t3 = std::chrono::steady_clock::now(); + replace_ms += + std::chrono::duration(t3 - t2).count(); + } } } + if (profiling) { + RecordCSEPassTiming(nodes_seen, nodes_filtered_out, nodes_replaced, + filter_ms, lookup_ms, replace_ms); + } return cse_removed; } diff --git a/onnxoptimizer/passes/eliminate_deadend.h b/onnxoptimizer/passes/eliminate_deadend.h index dafd4d363..19dfeb4d5 100644 --- a/onnxoptimizer/passes/eliminate_deadend.h +++ b/onnxoptimizer/passes/eliminate_deadend.h @@ -5,6 +5,8 @@ // ATTENTION: The code in this file is highly EXPERIMENTAL. // Adventurous users should note that the APIs will probably change. #pragma once +#include + #include "onnxoptimizer/pass.h" namespace ONNX_NAMESPACE { namespace optimization { @@ -19,15 +21,42 @@ struct EliminateDeadEnd final : public FullGraphBasedPass { return PassAnalysisType::CountBased; } unsigned int EliminateDead(Graph& graph) { + const bool profiling = GetPassPhaseProfilingEnabled(); unsigned int nodes_removed = 0; + uint64_t nodes_seen = 0; + double has_uses_ms = 0.0; + double destroy_ms = 0.0; auto nodes = graph.nodes().reverse(); for (auto it = nodes.begin(); it != nodes.end(); it++) { auto node = *it; - if (!node->hasUses()) { + nodes_seen++; + bool has_uses; + if (profiling) { + const auto t0 = std::chrono::steady_clock::now(); + has_uses = node->hasUses(); + const auto t1 = std::chrono::steady_clock::now(); + has_uses_ms += + std::chrono::duration(t1 - t0).count(); + } else { + has_uses = node->hasUses(); + } + if (!has_uses) { nodes_removed++; - it.destroyCurrent(); + if (profiling) { + const auto t0 = std::chrono::steady_clock::now(); + it.destroyCurrent(); + const auto t1 = std::chrono::steady_clock::now(); + destroy_ms += + std::chrono::duration(t1 - t0).count(); + } else { + it.destroyCurrent(); + } } } + if (profiling) { + RecordDeadendPassTiming(nodes_seen, nodes_removed, has_uses_ms, + destroy_ms); + } return nodes_removed; } std::shared_ptr runPass(Graph& graph) override { diff --git a/onnxoptimizer/passes/eliminate_duplicate_initializer.h b/onnxoptimizer/passes/eliminate_duplicate_initializer.h index 2788bb65d..cd2727658 100644 --- a/onnxoptimizer/passes/eliminate_duplicate_initializer.h +++ b/onnxoptimizer/passes/eliminate_duplicate_initializer.h @@ -33,7 +33,6 @@ #include "onnx/defs/tensor_util.h" #include "onnxoptimizer/pass.h" #include "onnxoptimizer/passes/cse_util.h" -#include "onnxoptimizer/passes/tensor_content_hash.h" namespace ONNX_NAMESPACE { namespace optimization { @@ -60,10 +59,14 @@ struct EliminateDuplicateInitializer final : public FullGraphBasedPass { } unsigned int EliminateInitializer(Graph &graph) { - // Scoped to this call: see ClearTensorContentDigestCache's header - // comment for why it's safe here and must not be skipped. - ClearTensorContentDigestCache(); - + // No longer cleared here: TensorContentDigest's cache is keyed by + // Tensor::tensor_id(), which stays valid across pass calls and rounds + // (see tensor_content_hash.h's header comment) -- clearing it is now the + // caller's responsibility (Optimizer::optimize(Graph&, ...)'s + // clear_tensor_digest_cache parameter), so a longer-lived caller (e.g. + // onnxsim's OptAndShape) can opt out and keep entries warm across many + // rounds instead of paying this pass's full tensor-hashing cost on every + // one of them. unsigned int initializers_removed = 0; const std::vector> &initializers = graph.initializers(); diff --git a/onnxoptimizer/passes/tensor_content_hash.cc b/onnxoptimizer/passes/tensor_content_hash.cc index c81046de9..1d662ab7f 100644 --- a/onnxoptimizer/passes/tensor_content_hash.cc +++ b/onnxoptimizer/passes/tensor_content_hash.cc @@ -16,10 +16,9 @@ namespace { bool g_trust_tensor_content_hash = true; -// See ClearTensorContentDigestCache's header comment for the validity scope -// of this cache (one EliminateInitializer/EliminateCommonSubexpressions -// call). -std::unordered_map g_digest_cache; +// Keyed by Tensor::tensor_id(), not by Tensor*: see ClearTensorContentDigestCache's +// header comment for why, and for this cache's validity scope. +std::unordered_map g_digest_cache; std::string ComputeTensorContentDigest(const Tensor& tensor); @@ -171,11 +170,12 @@ std::string ComputeTensorContentDigest(const Tensor& tensor) { } // namespace std::string TensorContentDigest(const Tensor& tensor) { - auto it = g_digest_cache.find(&tensor); + const uint64_t id = tensor.tensor_id(); + auto it = g_digest_cache.find(id); if (it != g_digest_cache.end()) { return it->second; } - return g_digest_cache.emplace(&tensor, ComputeTensorContentDigest(tensor)) + return g_digest_cache.emplace(id, ComputeTensorContentDigest(tensor)) .first->second; } diff --git a/onnxoptimizer/passes/tensor_content_hash.h b/onnxoptimizer/passes/tensor_content_hash.h index 2895a5bc0..32302ed65 100644 --- a/onnxoptimizer/passes/tensor_content_hash.h +++ b/onnxoptimizer/passes/tensor_content_hash.h @@ -45,21 +45,31 @@ namespace optimization { // reaching this). std::string TensorContentDigest(const Tensor& tensor); -// TensorContentDigest is memoized per Tensor pointer (a full BLAKE3 pass is -// too expensive to redo on every hash-bucket lookup and every equality -// check against that bucket's candidates -- see cse_util.h's CSETensorHash/ -// CSETensorCompare, both of which call it for the same tensor within a -// single pass invocation). The cache is valid ONLY within one call to -// EliminateDuplicateInitializer::EliminateInitializer or -// EliminateCommonSubexpression::EliminateCommonSubexpressions, since a -// tensor's content is never mutated in place *during* either pass (only -// nodes/edges are rewired, and any tensor a pass drops stays alive, -// unmutated, for the rest of that same call) -- neither pass mutates a -// retained tensor's bytes mid-call, but a tensor pointer CAN be reused by -// an unrelated, differently-contented tensor once freed between calls (a -// later optimizer pass, a later FixedPointFn round, or an entirely -// different graph), so each pass clears this cache at entry rather than -// relying on any cross-call invariant. +// TensorContentDigest is memoized per Tensor::tensor_id() (a full BLAKE3 +// pass is too expensive to redo on every hash-bucket lookup and every +// equality check against that bucket's candidates -- see cse_util.h's +// CSETensorHash/CSETensorCompare, both of which call it for the same tensor +// possibly many times per pass invocation). Keyed by tensor_id() rather than +// by `&tensor`: a `Tensor*` can be freed and its memory reused by an +// unrelated, differently-contented tensor (e.g. after +// Graph::eraseInitializer, or a Node attribute being replaced) within the +// cache's validity window, which would silently alias a stale digest onto +// the new tensor if keyed by address -- tensor_id() can't collide this way, +// since Tensor mints a fresh one on every construction and every +// (re)assignment (see tensor.h), so the cache naturally misses instead of +// aliasing. +// +// Because of that, this cache's validity is NOT scoped to a single pass +// call the way it once was: no onnx-optimizer pass mutates a *retained* +// tensor's content in place (only nodes/edges are rewired, or a tensor is +// dropped and replaced wholesale by a fresh one, which mints its own +// tensor_id() and simply misses the cache) -- so entries for tensors that +// are still alive and unchanged stay valid, and correct, across many pass +// calls and many onnxsim OptAndShape/FixedPointFn rounds within the same +// Optimizer::optimize(Graph&) call. See that function's +// `clear_tensor_digest_cache` parameter for how a caller opts out of the +// default per-call clear to extend this further, across its own repeated +// optimize() calls on one resident Graph. void ClearTensorContentDigestCache(); // See cse_util.h's CSETensorHash/CSETensorCompare for how this is diff --git a/third_party/onnx b/third_party/onnx index 781592ff8..c3b180f3d 160000 --- a/third_party/onnx +++ b/third_party/onnx @@ -1 +1 @@ -Subproject commit 781592ff8a532d0d20c0ada117c679002f50dffd +Subproject commit c3b180f3dfe417e38d1d9057cce9005f921ac01b