Perf: Reduce allocations while canonicaliziong external_constraints

compiler/rustc_middle/src/traits/solve.rs

@@ -3,10 +3,7 @@ use rustc_macros::StableHash;
 use rustc_type_ir as ir;
 pub use rustc_type_ir::solve::*;
 
-use crate::ty::{
-    self, FallibleTypeFolder, Ty, TyCtxt, TypeFoldable, TypeFolder, TypeVisitable, TypeVisitor,
-    try_visit,
-};
+use crate::ty::{self, Ty, TyCtxt, TypeVisitable, TypeVisitor, try_visit};
 
 pub type Goal<'tcx, P> = ir::solve::Goal<TyCtxt<'tcx>, P>;
 pub type QueryInput<'tcx, P> = ir::solve::QueryInput<TyCtxt<'tcx>, P>;
@@ -36,53 +33,6 @@ impl<'tcx> std::ops::Deref for ExternalConstraints<'tcx> {
     }
 }
 
-// FIXME: Having to clone `region_constraints` for folding feels bad and
-// probably isn't great wrt performance.
-//
-// Not sure how to fix this, maybe we should also intern `opaque_types` and
-// `region_constraints` here or something.
-impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for ExternalConstraints<'tcx> {
-    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
-        self,
-        folder: &mut F,
-    ) -> Result<Self, F::Error> {
-        // Perf testing has found that this check is slightly faster than
-        // folding and re-interning an empty `ExternalConstraintsData`.
-        // See: <https://github.com/rust-lang/rust/pull/142430>.
-        if self.is_empty() {
-            return Ok(self);
-        }
-
-        Ok(FallibleTypeFolder::cx(folder).mk_external_constraints(ExternalConstraintsData {
-            region_constraints: self.region_constraints.clone().try_fold_with(folder)?,
-            opaque_types: self
-                .opaque_types
-                .iter()
-                .map(|opaque| opaque.try_fold_with(folder))
-                .collect::<Result<_, F::Error>>()?,
-            normalization_nested_goals: self
-                .normalization_nested_goals
-                .clone()
-                .try_fold_with(folder)?,
-        }))
-    }
-
-    fn fold_with<F: TypeFolder<TyCtxt<'tcx>>>(self, folder: &mut F) -> Self {
-        // Perf testing has found that this check is slightly faster than
-        // folding and re-interning an empty `ExternalConstraintsData`.
-        // See: <https://github.com/rust-lang/rust/pull/142430>.
-        if self.is_empty() {
-            return self;
-        }
-
-        TypeFolder::cx(folder).mk_external_constraints(ExternalConstraintsData {
-            region_constraints: self.region_constraints.clone().fold_with(folder),
-            opaque_types: self.opaque_types.iter().map(|opaque| opaque.fold_with(folder)).collect(),
-            normalization_nested_goals: self.normalization_nested_goals.clone().fold_with(folder),
-        })
-    }
-}
-
 impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for ExternalConstraints<'tcx> {
     fn visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(&self, visitor: &mut V) -> V::Result {
         let ExternalConstraintsData {

compiler/rustc_middle/src/ty/context/impl_interner.rs

@@ -11,8 +11,8 @@ use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_span::{DUMMY_SP, Span, Symbol};
 use rustc_type_ir::lang_items::{SolverAdtLangItem, SolverProjectionLangItem, SolverTraitLangItem};
 use rustc_type_ir::{
-    BoundVar, CollectAndApply, DebruijnIndex, Interner, TypeFoldable, Unnormalized, VisitorResult,
-    search_graph, try_visit,
+    BoundVar, CollectAndApply, DebruijnIndex, Interner, TypeFoldable, TypeVisitableExt,
+    Unnormalized, VisitorResult, search_graph, try_visit,
 };
 
 use crate::dep_graph::{DepKind, DepNodeIndex};

compiler/rustc_next_trait_solver/src/canonical/mod.rs

@@ -28,7 +28,7 @@ use crate::delegate::SolverDelegate;
 use crate::solve::{
     CanonicalInput, CanonicalResponse, Certainty, ExternalConstraintsData,
     ExternalRegionConstraints, Goal, NestedNormalizationGoals, QueryInput, Response,
-    VisibleForLeakCheck, inspect,
+    ResponseAndExternalConstraints, VisibleForLeakCheck, inspect,
 };
 
 pub mod canonicalizer;
@@ -43,6 +43,12 @@ impl<I: Interner> ResponseT<I> for Response<I> {
     }
 }
 
+impl<I: Interner> ResponseT<I> for ResponseAndExternalConstraints<I> {
+    fn var_values(&self) -> CanonicalVarValues<I> {
+        self.response.var_values
+    }
+}
+
 impl<I: Interner, T> ResponseT<I> for inspect::State<I, T> {
     fn var_values(&self) -> CanonicalVarValues<I> {
         self.var_values
@@ -76,17 +82,25 @@ where
     (orig_values, query_input)
 }
 
-pub(super) fn canonicalize_response<D, I, T>(
+pub(super) fn canonicalize_response<D, I>(
     delegate: &D,
     max_input_universe: ty::UniverseIndex,
-    value: T,
-) -> ty::Canonical<I, T>
+    response: Response<I>,
+    external_constraints: ExternalConstraintsData<I>,
+) -> CanonicalResponse<I>
 where
     D: SolverDelegate<Interner = I>,
     I: Interner,
-    T: TypeFoldable<I>,
 {
-    Canonicalizer::canonicalize_response(delegate, max_input_universe, value)
+    Canonicalizer::canonicalize_response(
+        delegate,
+        max_input_universe,
+        (response, external_constraints),
+    )
+    .unchecked_map(|(response, external_constraints)| ResponseAndExternalConstraints {
+        response,
+        external_constraints: delegate.cx().mk_external_constraints(external_constraints),
+    })
 }
 
 /// After calling a canonical query, we apply the constraints returned
@@ -111,13 +125,18 @@ where
     let instantiation =
         compute_query_response_instantiation_values(delegate, &original_values, &response, span);
 
-    let Response { var_values, external_constraints, certainty } =
-        delegate.instantiate_canonical(response, instantiation);
+    let (Response { certainty, var_values }, external_constraints) = delegate
+        .instantiate_canonical(
+            response.unchecked_map(|response| {
+                (response.response, (&*response.external_constraints).clone())
+            }),
+            instantiation,
+        );
 
     unify_query_var_values(delegate, param_env, &original_values, var_values, span);
 
     let ExternalConstraintsData { region_constraints, opaque_types, normalization_nested_goals } =
-        &*external_constraints;
+        external_constraints;
 
     match region_constraints {
         ExternalRegionConstraints::Old(r) => register_region_constraints(
@@ -137,7 +156,7 @@ where
             delegate.register_solver_region_constraint(r.clone(), span)
         }
     };
-    register_new_opaque_types(delegate, opaque_types, span);
+    register_new_opaque_types(delegate, &opaque_types, span);
 
     (normalization_nested_goals.clone(), certainty)
 }
@@ -621,12 +640,14 @@ pub fn response_no_constraints_raw<I: Interner>(
     ty::Canonical {
         max_universe,
         var_kinds,
-        value: Response {
-            var_values: ty::CanonicalVarValues::make_identity(cx, var_kinds),
-            // FIXME: maybe we should store the "no response" version in cx, like
-            // we do for cx.types and stuff.
+        value: ResponseAndExternalConstraints {
+            response: Response {
+                var_values: ty::CanonicalVarValues::make_identity(cx, var_kinds),
+                // FIXME: maybe we should store the "no response" version in cx, like
+                // we do for cx.types and stuff.
+                certainty,
+            },
             external_constraints: cx.mk_external_constraints(ExternalConstraintsData::new(cx)),
-            certainty,
         },
     }
 }

compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs

@@ -393,7 +393,7 @@ fn maybe_evaluate_root_goal_for_proof_tree_with_higher_recursion_limit<D, I>(
     let goal_evaluation = &initial_result.1;
     match goal_evaluation.result {
         Err(_) => return,
-        Ok(response) if !response.value.certainty.is_overflow() => return,
+        Ok(response) if !response.value.response.certainty.is_overflow() => return,
         Ok(_) => {}
     }
 
@@ -405,7 +405,10 @@ fn maybe_evaluate_root_goal_for_proof_tree_with_higher_recursion_limit<D, I>(
             delegate.cx().recursion_limit() * 2,
         );
 
-        if new_goal_evaluation.result.is_ok_and(|response| response.value.certainty.is_overflow()) {
+        if new_goal_evaluation
+            .result
+            .is_ok_and(|response| response.value.response.certainty.is_overflow())
+        {
             Err(())
         } else {
             Ok((new_result, new_goal_evaluation))
@@ -1611,11 +1614,8 @@ where
         let canonical = canonicalize_response(
             self.delegate,
             self.max_input_universe,
-            Response {
-                var_values,
-                certainty,
-                external_constraints: self.cx().mk_external_constraints(external_constraints),
-            },
+            Response { var_values, certainty },
+            external_constraints,
         );
 
         Ok(canonical)

compiler/rustc_next_trait_solver/src/solve/mod.rs

@@ -54,27 +54,25 @@ pub enum HasChanged {
 
 // FIXME(trait-system-refactor-initiative#117): we don't detect whether a response
 // ended up pulling down any universes.
-fn has_no_inference_or_external_constraints<I: Interner>(
-    response: ty::Canonical<I, Response<I>>,
-) -> bool {
+fn has_no_inference_or_external_constraints<I: Interner>(response: CanonicalResponse<I>) -> bool {
     let ExternalConstraintsData {
         ref region_constraints,
         ref opaque_types,
         ref normalization_nested_goals,
     } = *response.value.external_constraints;
-    response.value.var_values.is_identity()
+    response.value.response.var_values.is_identity()
         && region_constraints.is_empty()
         && opaque_types.is_empty()
         && normalization_nested_goals.is_empty()
 }
 
-fn has_only_region_constraints<I: Interner>(response: ty::Canonical<I, Response<I>>) -> bool {
+fn has_only_region_constraints<I: Interner>(response: CanonicalResponse<I>) -> bool {
     let ExternalConstraintsData {
         region_constraints: _,
         ref opaque_types,
         ref normalization_nested_goals,
     } = *response.value.external_constraints;
-    response.value.var_values.is_identity_modulo_regions()
+    response.value.response.var_values.is_identity_modulo_regions()
         && opaque_types.is_empty()
         && normalization_nested_goals.is_empty()
 }
@@ -308,7 +306,7 @@ where
         }
 
         let always_applicable = candidates.iter().enumerate().find(|(_, candidate)| {
-            candidate.result.value.certainty == Certainty::Yes
+            candidate.result.value.response.certainty == Certainty::Yes
                 && has_no_inference_or_external_constraints(candidate.result)
         });
         if let Some((i, c)) = always_applicable {
@@ -329,7 +327,7 @@ where
             // We pull down the certainty of `Certainty::Yes` to ambiguity when combining
             // these responses, b/c we're combining more than one response and this we
             // don't know which one applies.
-            match candidate.result.value.certainty {
+            match candidate.result.value.response.certainty {
                 Certainty::Yes => maybe,
                 Certainty::Maybe(cand_maybe) => maybe.or(cand_maybe),
             }

compiler/rustc_next_trait_solver/src/solve/search_graph.rs

@@ -86,9 +86,9 @@ where
         match result.0 {
             Ok(response) => {
                 if has_no_inference_or_external_constraints(response) {
-                    if response.value.certainty == Certainty::Yes {
+                    if response.value.response.certainty == Certainty::Yes {
                         return Some(PathKind::Coinductive);
-                    } else if response.value.certainty == Certainty::overflow(false) {
+                    } else if response.value.response.certainty == Certainty::overflow(false) {
                         return Some(PathKind::Unknown);
                     }
                 }
@@ -117,9 +117,9 @@ where
     fn is_ambiguous_result(result: (QueryResult<I>, AccessedOpaques<I>)) -> Option<Certainty> {
         result.0.ok().and_then(|response| {
             if has_no_inference_or_external_constraints(response)
-                && matches!(response.value.certainty, Certainty::Maybe { .. })
+                && matches!(response.value.response.certainty, Certainty::Maybe { .. })
             {
-                Some(response.value.certainty)
+                Some(response.value.response.certainty)
             } else {
                 None
             }

compiler/rustc_trait_selection/src/solve/inspect/analyse.rs

@@ -57,7 +57,7 @@ impl<'a, 'tcx> InspectCandidate<'a, 'tcx> {
     }
 
     pub fn result(&self) -> Result<Certainty, NoSolution> {
-        self.result.map(|c| c.value.certainty)
+        self.result.map(|c| c.value.response.certainty)
     }
 
     pub fn goal(&self) -> &'a InspectGoal<'a, 'tcx> {
@@ -354,7 +354,7 @@ impl<'a, 'tcx> InspectGoal<'a, 'tcx> {
         } = root;
         // If there's a normalizes-to goal, AND the evaluation result with the result of
         // constraining the normalizes-to RHS and computing the nested goals.
-        let result = result.map(|ok| ok.value.certainty);
+        let result = result.map(|ok| ok.value.response.certainty);
 
         InspectGoal {
             infcx,

compiler/rustc_type_ir/src/interner.rs

@@ -152,7 +152,6 @@ pub trait Interner:
         + Debug
         + Hash
         + Eq
-        + TypeFoldable<Self>
         + Deref<Target = ExternalConstraintsData<Self>>;
     fn mk_external_constraints(
         self,

compiler/rustc_type_ir/src/solve/mod.rs

@@ -24,7 +24,7 @@ use crate::{
 
 pub type CanonicalInput<I, T = <I as Interner>::Predicate> =
     ty::CanonicalQueryInput<I, QueryInput<I, T>>;
-pub type CanonicalResponse<I> = Canonical<I, Response<I>>;
+pub type CanonicalResponse<I> = Canonical<I, ResponseAndExternalConstraints<I>>;
 /// The result of evaluating a canonical query.
 ///
 /// FIXME: We use a different type than the existing canonical queries. This is because
@@ -598,11 +598,19 @@ pub enum FetchEligibleAssocItemResponse<I: Interner> {
 pub struct Response<I: Interner> {
     pub certainty: Certainty,
     pub var_values: CanonicalVarValues<I>,
+}
+
+impl<I: Interner> Eq for Response<I> {}
+
+#[derive_where(Clone, Copy, Hash, PartialEq, Debug; I: Interner)]
+#[cfg_attr(feature = "nightly", derive(StableHash_NoContext))]
+pub struct ResponseAndExternalConstraints<I: Interner> {
+    pub response: Response<I>,
     /// Additional constraints returned by this query.
     pub external_constraints: I::ExternalConstraints,
 }
 
-impl<I: Interner> Eq for Response<I> {}
+impl<I: Interner> Eq for ResponseAndExternalConstraints<I> {}
 
 #[derive_where(Clone, Hash, PartialEq, Debug; I: Interner)]
 #[derive(TypeVisitable_Generic, GenericTypeVisitable, TypeFoldable_Generic)]