Split critical edges before borrowck.

Author: cjgillot

Cargo.lock

@@ -3795,6 +3795,7 @@ dependencies = [
  "rustc_trait_selection",
  "rustc_traits",
  "smallvec",
+ "thin-vec",
  "tracing",
 ]
 

compiler/rustc_borrowck/Cargo.toml

@@ -24,5 +24,6 @@ rustc_span = { path = "../rustc_span" }
 rustc_trait_selection = { path = "../rustc_trait_selection" }
 rustc_traits = { path = "../rustc_traits" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
+thin-vec = "0.2.18"
 tracing = "0.1"
 # tidy-alphabetical-end

compiler/rustc_borrowck/src/renumber.rs

@@ -1,9 +1,10 @@
-use rustc_index::IndexSlice;
+use rustc_index::{IndexSlice, IndexVec};
 use rustc_infer::infer::NllRegionVariableOrigin;
 use rustc_middle::mir::visit::{MutVisitor, TyContext};
-use rustc_middle::mir::{Body, ConstOperand, Location, Promoted};
+use rustc_middle::mir::*;
 use rustc_middle::ty::{self, GenericArgsRef, Ty, TyCtxt, TypeFoldable, fold_regions};
 use rustc_span::Symbol;
+use thin_vec::ThinVec;
 use tracing::{debug, instrument};
 
 use crate::BorrowckInferCtxt;
@@ -21,12 +22,68 @@ pub(crate) fn renumber_mir<'tcx>(
     let mut renumberer = RegionRenumberer { infcx };
 
     for body in promoted.iter_mut() {
+        split_critical_edges(body);
         renumberer.visit_body_preserves_cfg(body);
     }
 
+    split_critical_edges(body);
     renumberer.visit_body_preserves_cfg(body);
 }
 
+#[instrument(skip(body), level = "debug")]
+fn split_critical_edges(body: &mut Body<'_>) {
+    let predecessors: IndexVec<BasicBlock, _> =
+        body.basic_blocks.predecessors().iter().map(|preds| preds.len()).collect();
+    debug!(?predecessors);
+
+    let mut new_blocks = vec![];
+    for bb in predecessors.indices() {
+        let term = body.basic_blocks[bb].terminator();
+        if term.successors().count() <= 1 {
+            continue;
+        }
+        if term.successors().all(|s| predecessors[s] <= 1) {
+            continue;
+        }
+
+        debug!(
+            "{bb:?} has critical edges: {:?}",
+            term.successors().map(|s| (s, predecessors[s])).collect::<Vec<_>>(),
+        );
+
+        let original_succ: Vec<_> = term.successors().collect();
+        new_blocks.push((bb, original_succ));
+    }
+
+    if new_blocks.is_empty() {
+        return;
+    }
+
+    debug!(?new_blocks);
+    let basic_blocks = body.basic_blocks.as_mut();
+    for (bb, successors) in new_blocks.iter_mut() {
+        let source_info = basic_blocks[*bb].terminator().source_info;
+        for target in successors.iter_mut() {
+            if predecessors[*target] <= 1 {
+                continue;
+            }
+
+            let is_cleanup = basic_blocks[*target].is_cleanup;
+            let terminator = Terminator {
+                source_info,
+                kind: TerminatorKind::Goto { target: *target },
+                attributes: ThinVec::new(),
+            };
+            *target = basic_blocks.push(BasicBlockData::new(Some(terminator), is_cleanup))
+        }
+    }
+
+    for (bb, new_succ) in new_blocks {
+        let mut new_succ = new_succ.into_iter();
+        basic_blocks[bb].terminator_mut().successors_mut(|succ| *succ = new_succ.next().unwrap());
+    }
+}
+
 // The fields are used only for debugging output in `sccs_info`.
 #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)]
 pub(crate) enum RegionCtxt {

tests/ui/borrowck/borrowck-mut-borrow-linear-errors.stderr

@@ -2,30 +2,31 @@ error[E0499]: cannot borrow `x` as mutable more than once at a time
   --> $DIR/borrowck-mut-borrow-linear-errors.rs:10:30
    |
 LL |             1 => { addr.push(&mut x); }
-   |                              ^^^^^^ second mutable borrow occurs here
-LL |             2 => { addr.push(&mut x); }
-LL |             _ => { addr.push(&mut x); }
-   |                    ----      ------ first mutable borrow occurs here
+   |                    ----      ^^^^^^ second mutable borrow occurs here
    |                    |
    |                    first borrow later used here
+LL |             2 => { addr.push(&mut x); }
+LL |             _ => { addr.push(&mut x); }
+   |                              ------ first mutable borrow occurs here
 
 error[E0499]: cannot borrow `x` as mutable more than once at a time
   --> $DIR/borrowck-mut-borrow-linear-errors.rs:11:30
    |
+LL |             1 => { addr.push(&mut x); }
+   |                    ---- first borrow later used here
 LL |             2 => { addr.push(&mut x); }
    |                              ^^^^^^ second mutable borrow occurs here
 LL |             _ => { addr.push(&mut x); }
-   |                    ----      ------ first mutable borrow occurs here
-   |                    |
-   |                    first borrow later used here
+   |                              ------ first mutable borrow occurs here
 
 error[E0499]: cannot borrow `x` as mutable more than once at a time
   --> $DIR/borrowck-mut-borrow-linear-errors.rs:12:30
    |
+LL |             1 => { addr.push(&mut x); }
+   |                    ---- first borrow used here, in later iteration of loop
+LL |             2 => { addr.push(&mut x); }
 LL |             _ => { addr.push(&mut x); }
-   |                    ----      ^^^^^^ `x` was mutably borrowed here in the previous iteration of the loop
-   |                    |
-   |                    first borrow used here, in later iteration of loop
+   |                              ^^^^^^ `x` was mutably borrowed here in the previous iteration of the loop
 
 error: aborting due to 3 previous errors