Skip to content

Codegen Overloaded LLVM intrinsics based on their name #157145

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sayantn wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Codegen Overloaded LLVM intrinsics based on their name #157145

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1ac3d44
Codegen overloaded LLVM intrinsics using their name
sayantn Dec 29, 2025
36e3bbc
Remove all uses of typed pointers
sayantn May 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 114 additions & 4 deletions compiler/rustc_codegen_llvm/src/intrinsic.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1244,12 +1244,123 @@ fn autocast<'ll>(
}
}

fn parse_integer(string: &mut &[u8]) -> Option<u64> {
let mut number = 0;
let mut position = 0;
while let Some(&digit @ b'0'..=b'9') = string.get(position) {
number = (10 * number) + (digit - b'0') as u64;
position += 1;
}

if position != number.checked_ilog10().unwrap_or(0) as usize + 1 {
return None;
}

*string = &string[position..];
Some(number)
}

fn strip_off_prefix(slice: &mut &[u8], prefix: &[u8]) -> bool {
slice.strip_prefix(prefix).map(|remainder| *slice = remainder).is_some()
}

fn demangle_type_str<'ll>(cx: &CodegenCx<'ll, '_>, slice: &mut &[u8]) -> Option<&'ll Type> {
Some(if strip_off_prefix(slice, b"isVoid") {
cx.type_void()
} else if strip_off_prefix(slice, b"f16") {
cx.type_f16()
} else if strip_off_prefix(slice, b"bf16") {
cx.type_bf16()
} else if strip_off_prefix(slice, b"f32") {
cx.type_f32()
} else if strip_off_prefix(slice, b"f64") {
cx.type_f64()
} else if strip_off_prefix(slice, b"f128") {
cx.type_f128()
} else if strip_off_prefix(slice, b"i") {
let width = parse_integer(slice)?;
cx.type_ix(width)
} else if strip_off_prefix(slice, b"p") {
let address_space = parse_integer(slice)?;
cx.type_ptr_ext(AddressSpace(address_space as u32))
} else if strip_off_prefix(slice, b"v") {
let length = parse_integer(slice)?;
let element_type = demangle_type_str(cx, slice)?;
cx.type_vector(element_type, length)
} else if strip_off_prefix(slice, b"nxv") {
let length = parse_integer(slice)?;
let element_type = demangle_type_str(cx, slice)?;
cx.type_scalable_vector(element_type, length)
} else if strip_off_prefix(slice, b"a") {
let length = parse_integer(slice)?;
let element_type = demangle_type_str(cx, slice)?;
cx.type_array(element_type, length)
} else if strip_off_prefix(slice, b"sl_") {
let mut elements = Vec::new();

loop {
if let Some(remainder) = slice.strip_prefix(b"s")
&& !remainder.starts_with(b"_")
&& !remainder.starts_with(b"l_")
{
*slice = remainder;
break cx.type_struct(&elements, true);
}
elements.push(demangle_type_str(cx, slice)?);
}
} else if strip_off_prefix(slice, b"f_") {
let return_type = demangle_type_str(cx, slice)?;
let mut arguments = Vec::new();

loop {
if let Some(remainder) = slice.strip_prefix(b"f")
&& !remainder.starts_with(b"_")
{
*slice = remainder;
break cx.type_func(&arguments, return_type);
}
if strip_off_prefix(slice, b"varargf") {
break cx.type_variadic_func(&arguments, return_type);
}
arguments.push(demangle_type_str(cx, slice)?);
}
} else {
return None;
})
}

fn parse_type_parameters<'ll, 'tcx>(
cx: &CodegenCx<'ll, 'tcx>,
intrinsic: llvm::Intrinsic,
name: &str,
) -> Option<Vec<&'ll Type>> {
let base_name: &'ll [u8] = intrinsic.base_name();

let slice = &mut name.as_bytes().strip_prefix(base_name).unwrap();

if !intrinsic.is_overloaded() {
return slice.is_empty().then(|| Vec::new());
}

let mut type_params = Vec::new();

while !slice.is_empty() {
if !strip_off_prefix(slice, b".") {
return None;
}

type_params.push(demangle_type_str(cx, slice)?);
}

Some(type_params)
}

fn intrinsic_fn<'ll, 'tcx>(
bx: &Builder<'_, 'll, 'tcx>,
name: &str,
rust_return_ty: &'ll Type,
rust_argument_tys: Vec<&'ll Type>,
instance: ty::Instance<'tcx>,
instance: Instance<'tcx>,
) -> &'ll Value {
let tcx = bx.tcx;

Expand All @@ -1275,10 +1386,9 @@ fn intrinsic_fn<'ll, 'tcx>(
}

if let Some(intrinsic) = intrinsic
&& !intrinsic.is_overloaded()
&& let Some(type_params) = parse_type_parameters(bx.cx, intrinsic, name)
{
// FIXME: also do this for overloaded intrinsics
let llfn = intrinsic.get_declaration(bx.llmod, &[]);
let llfn = intrinsic.get_declaration(bx.llmod, &type_params);
let llvm_fn_ty = bx.get_type_of_global(llfn);

let llvm_return_ty = bx.get_return_type(llvm_fn_ty);
Expand Down
4 changes: 4 additions & 0 deletions compiler/rustc_codegen_llvm/src/llvm/ffi.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1130,6 +1130,10 @@ unsafe extern "C" {
NewFn: &mut Option<&'a Value>,
) -> bool;
pub(crate) fn LLVMRustIsTargetIntrinsic(ID: NonZero<c_uint>) -> bool;
pub(crate) fn LLVMRustIntrinsicGetBaseName(
ID: NonZero<c_uint>,
NameLength: &mut size_t,
) -> *const c_char;

// Operations on parameters
pub(crate) fn LLVMIsAArgument(Val: &Value) -> Option<&Value>;
Expand Down
8 changes: 7 additions & 1 deletion compiler/rustc_codegen_llvm/src/llvm/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@

use std::ffi::{CStr, CString};
use std::num::NonZero;
use std::ptr;
use std::string::FromUtf8Error;
use std::{ptr, slice};

use libc::c_uint;
use rustc_abi::{AddressSpace, Align, Size, WrappingRange};
Expand Down Expand Up @@ -355,6 +355,12 @@ impl Intrinsic {
LLVMGetIntrinsicDeclaration(llmod, self.id, type_params.as_ptr(), type_params.len())
}
}

pub(crate) fn base_name<'ll>(self) -> &'ll [u8] {
let mut length = 0;
let ptr = unsafe { LLVMRustIntrinsicGetBaseName(self.id, &mut length) };
unsafe { slice::from_raw_parts(ptr.cast(), length) }
}
}

/// Safe wrapper for `LLVMSetValueName2` from a byte slice
Expand Down
4 changes: 2 additions & 2 deletions compiler/rustc_codegen_ssa/src/mir/rvalue.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
let start = dest.val.llval;
let size = bx.const_usize(dest.layout.size.bytes());

// Use llvm.memset.p0i8.* to initialize all same byte arrays
// Use llvm.memset.p0.* to initialize all same byte arrays
if let Some(int) = bx.cx().const_to_opt_u128(v, false)
&& let bytes = &int.to_le_bytes()[..cg_elem.layout.size.bytes_usize()]
&& let Ok(&byte) = bytes.iter().all_equal_value()
Expand All @@ -151,7 +151,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
return true;
}

// Use llvm.memset.p0i8.* to initialize byte arrays
// Use llvm.memset.p0.* to initialize byte arrays
let v = bx.from_immediate(v);
if bx.cx().val_ty(v) == bx.cx().type_i8() {
bx.memset(start, v, size, dest.val.align, MemFlags::empty());
Expand Down
7 changes: 7 additions & 0 deletions compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1852,6 +1852,13 @@ extern "C" bool LLVMRustIsTargetIntrinsic(unsigned ID) {
return Intrinsic::isTargetIntrinsic(ID);
}

extern "C" const char *LLVMRustIntrinsicGetBaseName(unsigned ID,
size_t *NameLength) {
auto baseName = Intrinsic::getBaseName(ID);
*NameLength = baseName.size();
return baseName.data();
}

extern "C" LLVMValueRef LLVMRustConstPtrAuth(LLVMValueRef Ptr, uint32_t Key,
uint64_t Disc,
LLVMValueRef AddrDiversity,
Expand Down
6 changes: 3 additions & 3 deletions library/core/src/intrinsics/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -953,7 +953,7 @@ pub const unsafe fn slice_get_unchecked<
#[rustc_intrinsic]
pub fn ptr_mask<T>(ptr: *const T, mask: usize) -> *const T;

/// Equivalent to the appropriate `llvm.memcpy.p0i8.0i8.*` intrinsic, with
/// Equivalent to the appropriate `llvm.memcpy.p0.p0.*` intrinsic, with
/// a size of `count` * `size_of::<T>()` and an alignment of `align_of::<T>()`.
///
/// This intrinsic does not have a stable counterpart.
Expand All @@ -967,7 +967,7 @@ pub fn ptr_mask<T>(ptr: *const T, mask: usize) -> *const T;
#[rustc_intrinsic]
#[rustc_nounwind]
pub unsafe fn volatile_copy_nonoverlapping_memory<T>(dst: *mut T, src: *const T, count: usize);
/// Equivalent to the appropriate `llvm.memmove.p0i8.0i8.*` intrinsic, with
/// Equivalent to the appropriate `llvm.memmove.p0.p0.*` intrinsic, with
/// a size of `count * size_of::<T>()` and an alignment of `align_of::<T>()`.
///
/// The volatile parameter is set to `true`, so it will not be optimized out
Expand All @@ -977,7 +977,7 @@ pub unsafe fn volatile_copy_nonoverlapping_memory<T>(dst: *mut T, src: *const T,
#[rustc_intrinsic]
#[rustc_nounwind]
pub unsafe fn volatile_copy_memory<T>(dst: *mut T, src: *const T, count: usize);
/// Equivalent to the appropriate `llvm.memset.p0i8.*` intrinsic, with a
/// Equivalent to the appropriate `llvm.memset.p0.*` intrinsic, with a
/// size of `count * size_of::<T>()` and an alignment of `align_of::<T>()`.
///
/// This intrinsic does not have a stable counterpart.
Expand Down
2 changes: 1 addition & 1 deletion src/tools/rustfmt/tests/target/format_strings/issue-202.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ fn compile_empty_program() {
let expected = "; ModuleID = \'foo\'

; Function Attrs: nounwind
declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i32, i1) #0
declare void @llvm.memset.p0.i32(i8* nocapture, i8, i32, i32, i1) #0

declare i32 @write(i32, i8*, i32)

Expand Down
2 changes: 1 addition & 1 deletion src/tools/rustfmt/tests/target/format_strings/issue-3263.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ fn compile_empty_program() {
let expected = "; ModuleID = \'foo\'

; Function Attrs: nounwind
declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i32, i1) #0
declare void @llvm.memset.p0.i32(i8* nocapture, i8, i32, i32, i1) #0

declare i32 @write(i32, i8*, i32)

Expand Down
2 changes: 1 addition & 1 deletion src/tools/rustfmt/tests/target/string-lit-custom.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ fn main() {
let expected = "; ModuleID = \'foo\'

; Function Attrs: nounwind
declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i32, i1) #0
declare void @llvm.memset.p0.i32(i8* nocapture, i8, i32, i32, i1) #0

declare i32 @write(i32, i8*, i32)

Expand Down
16 changes: 16 additions & 0 deletions tests/codegen-llvm/inject-autocast.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,20 @@ pub unsafe fn amx_autocast(m: u16, n: u16, k: u16, a: Tile, b: Tile, c: Tile) ->
foo(m, n, k, a, b, c)
}

// CHECK-LABEL: @overloaded_bf16_autocast
#[no_mangle]
pub unsafe fn overloaded_bf16_autocast(a: i16x8) -> i16x8 {
extern "unadjusted" {
#[link_name = "llvm.sqrt.v8bf16"]
fn foo(a: i16x8) -> i16x8;

@RalfJung RalfJung Aug 27, 2026 •
edited by rustbot
Loading

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I find this extremely confusing. What is going on here? The intrinsic actually has type <8 x bfloat> but we pass it an i16x8? Why should we allow such code...?

View changes since the review

@nikic nikic Aug 27, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the main motivation for this entire approach was ability to insert magic type casts for types that Rust does not support.

Of course, this is kind of moot with f16b being introduced in #160859.

(FWIW, I don't think what this PR does is a good idea, but I don't have time to fight it.)

@RalfJung RalfJung Aug 27, 2026 •
edited
Loading

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Of course, this is kind of moot with f16b being introduced in #160859.

Yeah that seems to remove this part of the motivation. OTOH the old referenced PR also mentions

LLVM intrinsics that have types in their signature that can't be accessed from Rust (notable examples are the AMX intrinsics that have the x86amx type, and (almost) all intrinsics that have vectors of i1 types) can't be linked to at all. This is a (major?) roadblock in the AMX and AVX512 support in stdarch.

So the question is, will we have native types for those as well? We might have to, if they are relevant for ABIs we need to implement. But if they are only used for LLVM intrinsics, not for extern "C", then this PR may save us from adding that type. I don't know anything about x86amx, or whether there's a better way to deal with vectors of i1.

Another motivation seems to be dealing with LLVM intrinsics that change their signature -- how is stdarch supposed to import these? But this does works today so it seems the LLVM auto-upgrade for this is enough?

And then there's "finding signatures that are just wrong". But it seems that already landed in #140763.

On the list of downsides we have "keeping the long mangled LLVM names even after LLVM itself doesn't use them any more", if I understood correctly the plan for how to deal with @nikic's work on getting rid of the name mangling in LLVM. That seems like a silly end state and is clearly path-dependent -- if LLVM had removed the mangling 5 years ago, I doubt we'd accept a PR like this to add magic name mangling in the link_name string to indicate the "real" signature. So likewise I don't think we want to be in a state where we are keeping the name mangling alive long after LLVM dropped it.

I also couldn't find an MCP for this change. Should it have one? It's very backend-internal, but OTOH one of our main backend experts disagrees so it may be good to have a bit of a wider discussion. @dianqk seems fine with it.

@dianqk dianqk Aug 30, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The intrinsics name mangling has been in Rust for a long time, before #140763. IMO, this PR just helps handwriting intrinsics for people. This is fine with me.
I think we may want to find an approach to get rid of the name mangling after LLVM changed, even if we are not going to merge the PR.

But I'm curious about alternative implementation. Can we write a cargo out-of-tree plugin for the intrinsics name mangling? IIUC, since this is mostly used for stdarch, a standalone checking tool would be great.

@RalfJung RalfJung Aug 30, 2026 •
edited
Loading

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The intrinsics name mangling has been in Rust for a long time, before #140763

demangle_type_str is new in this PR, so I am not sure what you mean. So far these mangled names are just forwarded opaquely to LLVM.

@nikic nikic Aug 30, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To clarify my position on this: I consider name mangling to be something of an implementation detail of LLVM, that Rust would ideally not need to expose. Instead of declaring something like #[link_name = "llvm.sqrt.v8bf16"], it is sufficient to use #[link_name = "llvm.sqrt"] and infer the mangled name from the function signature. This is how intrinsics are written in textual LLVM IR nowadays, you don't actually write out the mangling suffix, the parser infers it.

The motivation for inverting this, and instead make the mangling suffix (rather than the function signature) the source of truth is ability to insert automatic casts for types that Rust does not support. This is no longer relevant for bfloat, but still relevant for x86_amx. IMHO the proper way to handle that is to represent this as a special type in Rust, which has special ABI rules when passed to LLVM intrinsics. This does not have to be full support for the type like what is happening with f16b right now, but just the minimum to adjust its ABI (which I assume is something like a lang item?)

@RalfJung RalfJung Aug 30, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

IMHO the proper way to handle that is to represent this as a special type in Rust, which has special ABI rules when passed to LLVM intrinsics. This does not have to be full support for the type like what is happening with f16b right now, but just the minimum to adjust its ABI (which I assume is something like a lang item?)

rust-lang/rfcs#4003 does this for two more types.

For types that don't get publicly exposed but only used internally, I think this doesn't need an RFC. Just a t-compiler MCP.

@dianqk dianqk Aug 31, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The intrinsics name mangling has been in Rust for a long time, before #140763

demangle_type_str is new in this PR, so I am not sure what you mean. So far these mangled names are just forwarded opaquely to LLVM.

I misunderstood something. Yes, you are right.

@dianqk dianqk Aug 31, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree with @nikic if we can define several special types in Rust for LLVM. Sometimes I am only thinking about how to implement it in rustc without changing other parts.

}

// CHECK: [[A:%[0-9]+]] = bitcast <8 x i16> {{.*}} to <8 x bfloat>
// CHECK: [[B:%[0-9]+]] = call <8 x bfloat> @llvm.sqrt.v8bf16(<8 x bfloat> [[A]])
// CHECK: bitcast <8 x bfloat> [[B]] to <8 x i16>
foo(a)
}

// CHECK: declare { i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @llvm.x86.encodekey128(i32, <2 x i64>)

// CHECK: declare { <2 x i1>, <2 x i1> } @llvm.x86.avx512.vp2intersect.q.128(<2 x i64>, <2 x i64>)
Expand All @@ -116,3 +130,5 @@ pub unsafe fn amx_autocast(m: u16, n: u16, k: u16, a: Tile, b: Tile, c: Tile) ->
// CHECK: declare x86_amx @llvm.x86.cast.vector.to.tile.v1024i8(<1024 x i8>)

// CHECK: declare <1024 x i8> @llvm.x86.cast.tile.to.vector.v1024i8(x86_amx)

// CHECK: declare <8 x bfloat> @llvm.sqrt.v8bf16(<8 x bfloat>)
2 changes: 1 addition & 1 deletion tests/codegen-llvm/intrinsics/mask.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,6 @@
#[no_mangle]
pub fn mask_ptr(ptr: *const u16, mask: usize) -> *const u16 {
// CHECK: call
// CHECK-SAME: @llvm.ptrmask.{{p0|p0i8}}.[[WORD]](ptr {{%ptr|%1}}, [[WORD]] %mask)
// CHECK-SAME: @llvm.ptrmask.p0.[[WORD]](ptr {{%ptr|%1}}, [[WORD]] %mask)
core::intrinsics::ptr_mask(ptr, mask)
}
4 changes: 2 additions & 2 deletions tests/ui/abi/arm-unadjusted-intrinsic.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ impl Copy for int8x16x4_t {}
pub unsafe fn vld1q_s8_x4(a: *const i8) -> int8x16x4_t {
#[allow(improper_ctypes)]
extern "unadjusted" {
#[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x4.v16i8.p0i8")]
#[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x4.v16i8.p0i8")]
#[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x4.v16i8.p0")]
#[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x4.v16i8.p0")]
fn vld1q_s8_x4_(a: *const i8) -> int8x16x4_t;
}
vld1q_s8_x4_(a)
Expand Down
Loading