Complex layout phase 2

compiler/rustc_abi/src/layout/ty.rs

@@ -7,7 +7,7 @@ use rustc_macros::StableHash;
 
 use crate::layout::{FieldIdx, VariantIdx};
 use crate::{
-    AbiAlign, Align, BackendRepr, FieldsShape, Float, HasDataLayout, LayoutData, Niche,
+    AbiAlign, Align, BackendRepr, FieldsShape, Float, HasDataLayout, LayoutData, Niche, Numeric,
     PointeeInfo, Primitive, Size, Variants,
 };
 
@@ -345,6 +345,17 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
         }
     }
 
+    pub fn complex_number<C>(&self, cx: &C) -> Option<Numeric>
+    where
+        Ty: TyAbiInterface<'a, C> + Copy,
+    {
+        match self.complex_number_primitive(cx)? {
+            Primitive::Int(i, sign) => Some(Numeric::Int(i, sign)),
+            Primitive::Float(f) => Some(Numeric::Float(f)),
+            Primitive::Pointer(_) => None,
+        }
+    }
+
     /// Whether this type/layout has any padding that is dependent on a variant, i.e. has bytes that
     /// are padding for some, but not all, valid values of this type.
     pub fn has_variant_dependent_padding<C>(&self, cx: &C) -> bool

compiler/rustc_abi/src/lib.rs

@@ -1443,6 +1443,31 @@ impl Float {
     }
 }
 
+/// Numeric primitives.
+#[derive(Copy, Clone, PartialEq, Eq, Hash)]
+#[cfg_attr(feature = "nightly", derive(StableHash))]
+pub enum Numeric {
+    /// The `bool` is the signedness of the `Integer` type.
+    Int(Integer, bool),
+    Float(Float),
+}
+
+impl Numeric {
+    pub fn size(self) -> Size {
+        match self {
+            Numeric::Int(integer, _) => integer.size(),
+            Numeric::Float(float) => float.size(),
+        }
+    }
+
+    pub fn reg_kind(self) -> RegKind {
+        match self {
+            Numeric::Int(_, _) => RegKind::Integer,
+            Numeric::Float(_) => RegKind::Float,
+        }
+    }
+}
+
 /// Fundamental unit of memory access and layout.
 #[derive(Copy, Clone, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "nightly", derive(StableHash))]

compiler/rustc_target/src/callconv/mips.rs

@@ -1,16 +1,36 @@
-use rustc_abi::{HasDataLayout, Size, TyAbiInterface};
+use rustc_abi::{Float, HasDataLayout, Integer, Numeric, Reg, RegKind, Size, TyAbiInterface};
 
-use crate::callconv::{ArgAbi, FnAbi, Reg, Uniform};
+use crate::callconv::{ArgAbi, CastTarget, FnAbi, Uniform};
 
-fn classify_ret<Ty, C>(cx: &C, ret: &mut ArgAbi<'_, Ty>, offset: &mut Size)
+fn classify_ret<'a, Ty, C>(cx: &C, ret: &mut ArgAbi<'a, Ty>, offset: &mut Size)
 where
+    Ty: TyAbiInterface<'a, C> + Copy,
     C: HasDataLayout,
 {
-    if !ret.layout.is_aggregate() {
-        ret.extend_integer_width_to(32);
-    } else {
+    let dl = cx.data_layout();
+    let size = ret.layout.size;
+
+    if let Some(component) = ret.layout.complex_number(cx) {
+        match component {
+            Numeric::Float(Float::F128) => {
+                // Same as an aggregate.
+                ret.make_indirect();
+                *offset += dl.pointer_size();
+            }
+            Numeric::Int(Integer::I8 | Integer::I16, _) => {
+                // Pack Complex<{integer}> into a single register if that fits.
+                ret.cast_to(Reg { kind: RegKind::Integer, size });
+            }
+            _ => {
+                let reg = Reg { kind: component.reg_kind(), size: component.size() };
+                ret.cast_to(CastTarget::pair(reg, reg));
+            }
+        }
+    } else if ret.layout.is_aggregate() {
         ret.make_indirect();
-        *offset += cx.data_layout().pointer_size();
+        *offset += dl.pointer_size();
+    } else {
+        ret.extend_integer_width_to(32);
     }
 }
 

compiler/rustc_target/src/callconv/mips64.rs

@@ -1,10 +1,13 @@
 use arrayvec::ArrayVec;
 use rustc_abi::{
-    BackendRepr, FieldsShape, Float, HasDataLayout, Primitive, Reg, Size, TyAbiInterface,
+    BackendRepr, FieldsShape, Float, HasDataLayout, Integer, Numeric, Primitive, Reg, RegKind,
+    Size, TyAbiInterface,
 };
 
 use crate::callconv::{ArgAbi, ArgAttribute, ArgExtension, CastTarget, FnAbi, PassMode, Uniform};
 
+const NUM_ARG_SLOTS: u64 = 8;
+
 fn extend_integer_width_mips<Ty>(arg: &mut ArgAbi<'_, Ty>, bits: u64) {
     // Always sign extend u32 values on 64-bit mips
     if let BackendRepr::Scalar(scalar) = arg.layout.backend_repr
@@ -55,6 +58,22 @@ where
     let size = ret.layout.size;
     let bits = size.bits();
     if bits <= 128 {
+        if let Some(component) = ret.layout.complex_number(cx) {
+            match component {
+                Numeric::Int(Integer::I8 | Integer::I16 | Integer::I32, _) => {
+                    // Return a Complex<{integer}> packed into a single register when that fits.
+                    // We match GCC, not Clang, see https://github.com/llvm/llvm-project/issues/212109.
+                    ret.cast_to(Reg { kind: RegKind::Integer, size });
+                }
+                _ => {
+                    // Otherwise pass in 2 registers.
+                    let reg = Reg { kind: component.reg_kind(), size: component.size() };
+                    ret.cast_to(CastTarget::pair(reg, reg));
+                }
+            }
+            return;
+        }
+
         // Unlike other architectures which return aggregates in registers, MIPS n64 limits the
         // use of float registers to structures (not unions) containing exactly one or two
         // float fields.
@@ -102,6 +121,39 @@ where
         extend_integer_width_mips(arg, 64);
     } else if arg.layout.pass_indirectly_in_non_rustic_abis(cx) {
         arg.make_indirect();
+    } else if let Some(component) = arg.layout.complex_number(cx) {
+        let slot = dl.pointer_size();
+        let curr_offset = offset.align_to(align);
+
+        match component {
+            Numeric::Float(Float::F128) => {
+                // Complex<f128> is passed in 4 GPRs, but aligned to 16 so may need padding.
+                let reg = Reg { kind: RegKind::Float, size: arg.layout.field(cx, 0).size };
+                arg.cast_to_and_pad_i32(CastTarget::pair(reg, reg), pad_i32);
+            }
+            Numeric::Float(_) => {
+                // Only pass a Complex<f32>/Complex<f64> in FPRs when two argument slots are free,
+                if curr_offset.bytes() / slot.bytes() + 2 <= NUM_ARG_SLOTS {
+                    // The default `PassMode::Pair` already passes one component per register. Both
+                    // components claim a slot, even a Complex<f32> which could fit into one slot.
+                    *offset = curr_offset + slot * 2;
+                    return;
+                }
+
+                // Otherwise pack it into GPRs (or the stack) like an integer of the same size.
+                arg.cast_to_and_pad_i32(Uniform::new(Reg::i64(), size), pad_i32);
+            }
+            Numeric::Int(Integer::I8 | Integer::I16 | Integer::I32, _) => {
+                // Cast Complex<i8> into i16, Complex<i16> to i32, etc.
+                let cast_target = CastTarget::from(Reg { kind: RegKind::Integer, size });
+                // The inreg attribute makes the bits land in the right (upper) bits on BE targets.
+                arg.cast_to(cast_target.with_attrs(ArgAttribute::InReg.into()));
+            }
+            Numeric::Int(Integer::I64 | Integer::I128, _) => {
+                // Complex<i64> is passed as 2 separate arguments, which is what the default
+                // `PassMode::Pair` already does.
+            }
+        }
     } else {
         match arg.layout.fields {
             FieldsShape::Primitive => unreachable!(),

compiler/rustc_target/src/callconv/powerpc.rs

@@ -1,17 +1,45 @@
-use rustc_abi::TyAbiInterface;
+use rustc_abi::{BackendRepr, Primitive, Reg, RegKind, TyAbiInterface};
 
-use crate::callconv::{ArgAbi, FnAbi};
+use crate::callconv::{ArgAbi, CastTarget, FnAbi, Uniform};
 use crate::spec::{Env, HasTargetSpec, Os};
 
-fn classify_ret<Ty>(ret: &mut ArgAbi<'_, Ty>) {
-    if ret.layout.is_aggregate() {
+const NUM_ARG_GPRS: u32 = 8; // r3..=r10
+
+/// How to cast `Complex<T>` so that we match the GCC ABI.
+fn complex_cast_target<Ty>(arg: &ArgAbi<'_, Ty>) -> CastTarget {
+    let size = arg.layout.size;
+
+    if size.bytes() <= 4 {
+        // Coerce to an integer for `Complex<i8>` and `Complex<i16>`.
+        CastTarget::from(Reg { kind: RegKind::Integer, size })
+    } else if size.bytes() == 8 {
+        // Coerce to a single `i64` for `Complex<f32>` and `Complex<i32>`, which has the correct
+        // register alignment of 8 bytes.
+        //
+        // NOTE: clang uses a vector (e.g. <2 x f32>) here, but if we try that we run into
+        // ABI issues because vectors require the altivec target feature.
+        CastTarget::from(Reg::i64())
+    } else {
+        // Coerce to an array `[N x i32]` for everything wider. An array of i32 gives the correct
+        // 4-byte register alignment.
+        CastTarget::from(Uniform::new(Reg::i32(), size))
+    }
+}
+
+fn classify_ret<'a, Ty, C>(cx: &C, ret: &mut ArgAbi<'a, Ty>)
+where
+    Ty: TyAbiInterface<'a, C> + Copy,
+{
+    if ret.layout.is_complex_number(cx) {
+        ret.cast_to(complex_cast_target(ret));
+    } else if ret.layout.is_aggregate() {
         ret.make_indirect();
     } else {
         ret.extend_integer_width_to(32);
     }
 }
 
-fn classify_arg<'a, Ty, C: HasTargetSpec>(cx: &C, arg: &mut ArgAbi<'a, Ty>)
+fn classify_arg<'a, Ty, C: HasTargetSpec>(cx: &C, arg: &mut ArgAbi<'a, Ty>, arg_gprs_left: &mut u32)
 where
     Ty: TyAbiInterface<'a, C> + Copy,
 {
@@ -25,22 +53,82 @@ where
         }
         return;
     }
-    if arg.layout.pass_indirectly_in_non_rustic_abis(cx) || arg.layout.is_aggregate() {
-        arg.make_indirect();
+
+    let default = |arg: &mut ArgAbi<'a, Ty>| {
+        if arg.layout.pass_indirectly_in_non_rustic_abis(cx) || arg.layout.is_aggregate() {
+            arg.make_indirect();
+        } else {
+            arg.extend_integer_width_to(32);
+        }
+    };
+
+    let is_complex = arg.layout.is_complex_number(cx);
+    let is_float = match arg.layout.backend_repr {
+        BackendRepr::Scalar(scalar) => matches!(scalar.primitive(), Primitive::Float(_)),
+        _ => false,
+    };
+
+    // Arguments that are not relevant for the GPR budget: floats go in the FPRs, and once the GPRs
+    // are exhausted everything lands on the stack anyway. Complex<T> always needs custom handling.
+    if (*arg_gprs_left == 0 || is_float) && !is_complex {
+        return default(arg);
+    }
+
+    let size = arg.layout.size;
+    let regs_needed = size.bytes().div_ceil(4) as u32; // 32-bit registers
+
+    if arg.layout.is_aggregate() && !is_complex {
+        // Non-complex aggregates are passed indirectly, and consume one GPR.
+        *arg_gprs_left -= 1;
     } else {
-        arg.extend_integer_width_to(32);
+        let mut padding = 0;
+
+        // The powerpc ABI in GCC hardcodes a special rule for values of size 8. It remarks
+        //
+        // > V.4 wants long longs and doubles to be double word aligned. Just
+        // > testing the mode size is a boneheaded way to do this as it means
+        // > that other types such as complex int are also double word aligned.
+        // > However, we're stuck with this because changing the ABI might break
+        // > existing library interfaces.
+        //
+        // An eight-byte value must start in an even-numbered GPR. The `i64` it is coerced to
+        // already makes LLVM skip an odd register, so only account for it in the budget.
+        if size.bytes() == 8 && !arg_gprs_left.is_multiple_of(2) {
+            *arg_gprs_left -= 1;
+        }
+
+        if regs_needed <= *arg_gprs_left {
+            // Everything fits, great!
+            *arg_gprs_left -= regs_needed;
+        } else if is_complex {
+            // Never split a Complex<T> across the GPRs and the stack.
+            //
+            // The full complex value is passed via the stack, and the remaining GPRs are consumed,
+            // so all subsequent arguments will also be passed via the stack. Use the padding value
+            // to fill up the remaining GPRs.
+            padding += *arg_gprs_left;
+            *arg_gprs_left = 0;
+        }
+
+        if is_complex {
+            arg.cast_to_and_pad_i32(complex_cast_target(arg), padding as u8);
+            return;
+        }
     }
+
+    default(arg)
 }
 
 pub(crate) fn compute_abi_info<'a, Ty, C: HasTargetSpec>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>)
 where
     Ty: TyAbiInterface<'a, C> + Copy,
 {
     if !fn_abi.ret.is_ignore() {
-        classify_ret(&mut fn_abi.ret);
+        classify_ret(cx, &mut fn_abi.ret);
     }
 
+    let mut arg_gprs_left = NUM_ARG_GPRS;
     for arg in fn_abi.args.iter_mut() {
-        classify_arg(cx, arg);
+        classify_arg(cx, arg, &mut arg_gprs_left);
     }
 }

compiler/rustc_target/src/callconv/powerpc64.rs

@@ -4,7 +4,7 @@
 
 use rustc_abi::{HasDataLayout, TyAbiInterface};
 
-use crate::callconv::{Align, ArgAbi, FnAbi, Reg, RegKind, Uniform};
+use crate::callconv::{Align, ArgAbi, CastTarget, FnAbi, Reg, RegKind, Uniform};
 use crate::spec::{HasTargetSpec, LlvmAbi, Os};
 
 #[derive(Debug, Clone, Copy, PartialEq)]
@@ -60,6 +60,13 @@ where
         arg.extend_integer_width_to(64);
         return;
     }
+    if let Some(component) = arg.layout.complex_number(cx) {
+        if is_ret {
+            let reg = Reg { kind: component.reg_kind(), size: component.size() };
+            arg.cast_to(CastTarget::pair(reg, reg));
+        }
+        return;
+    }
 
     // The AIX ABI expect byval for aggregates
     // See https://github.com/llvm/llvm-project/blob/main/clang/lib/CodeGen/Targets/PPC.cpp.