From 5c571dbf4ae68ce1144aa95c23af577126d09747 Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Wed, 18 Nov 2026 08:21:57 +0000 Subject: [PATCH] lower `unbounded_shl`/`unbounded_shr` without branching The `core` implementation of these methods branches on whether the shift amount is less than the bit width, which turns into structured control flow with an `OpPhi`. It also goes through `unchecked_shl`/`unchecked_shr`, whose precondition check forces callers to be inlined because of the panic path. Intercept calls to the integer `unbounded_shl`/`unbounded_shr` methods from `core` and emit the shift followed by an `OpSelect` on whether the shift amount is in range, since SPIR-V leaves shifts by the bit width or more undefined. Signed `unbounded_shr` clamps the shift amount to `BITS - 1` instead, which fills the result with the sign bit. --- .../src/builder/builder_methods.rs | 46 ++++++++++++ .../src/codegen_cx/declare.rs | 14 ++++ .../rustc_codegen_spirv/src/codegen_cx/mod.rs | 6 ++ crates/rustc_codegen_spirv/src/symbols.rs | 4 + tests/compiletests/ui/dis/unbounded_shifts.rs | 25 +++++++ .../ui/dis/unbounded_shifts.stderr | 33 +++++++++ tests/difftests/tests/Cargo.lock | 17 +++++ tests/difftests/tests/Cargo.toml | 2 + .../unbounded-shift-cpu/Cargo.toml | 10 +++ .../unbounded-shift-cpu/src/main.rs | 16 ++++ .../unbounded-shift-rust/Cargo.toml | 15 ++++ .../unbounded-shift-rust/src/lib.rs | 73 +++++++++++++++++++ .../unbounded-shift-rust/src/main.rs | 58 +++++++++++++++ 13 files changed, 319 insertions(+) create mode 100644 tests/compiletests/ui/dis/unbounded_shifts.rs create mode 100644 tests/compiletests/ui/dis/unbounded_shifts.stderr create mode 100644 tests/difftests/tests/lang/core/ops/unbounded_shift/unbounded-shift-cpu/Cargo.toml create mode 100644 tests/difftests/tests/lang/core/ops/unbounded_shift/unbounded-shift-cpu/src/main.rs create mode 100644 tests/difftests/tests/lang/core/ops/unbounded_shift/unbounded-shift-rust/Cargo.toml create mode 100644 tests/difftests/tests/lang/core/ops/unbounded_shift/unbounded-shift-rust/src/lib.rs create mode 100644 tests/difftests/tests/lang/core/ops/unbounded_shift/unbounded-shift-rust/src/main.rs diff --git a/crates/rustc_codegen_spirv/src/builder/builder_methods.rs b/crates/rustc_codegen_spirv/src/builder/builder_methods.rs index 53b7ee1f215..589e19ece24 100644 --- a/crates/rustc_codegen_spirv/src/builder/builder_methods.rs +++ b/crates/rustc_codegen_spirv/src/builder/builder_methods.rs @@ -342,6 +342,44 @@ fn memset_dynamic_scalar( } impl<'a, 'tcx> Builder<'a, 'tcx> { + /// Lowers `unbounded_shl`/`unbounded_shr` without branching. Shifting by the bit width or more + /// is undefined in SPIR-V, so the result is selected based on whether `shift` is in range. + fn unbounded_shift( + &mut self, + is_left: bool, + value: SpirvValue, + shift: SpirvValue, + ) -> SpirvValue { + let (width, signed) = match self.lookup_type(value.ty) { + SpirvType::Integer(width, signed) => (width, signed), + other => self.fatal(format!( + "cannot shift non-integer type: {}", + other.debug(value.ty, self) + )), + }; + let int_size = self.constant_int(shift.ty, width.into()); + let in_range = self.icmp(IntPredicate::IntULT, shift, int_size); + + if !is_left && signed { + // Shifting by `width - 1` fills the result with the sign bit, same as an unbounded shift + let max_shift = self.constant_int(shift.ty, (width - 1).into()); + let shift = self.select(in_range, shift, max_shift); + return self.ashr(value, shift); + } + + let zero = self.constant_int(value.ty, 0); + // Avoid emitting (or constant folding) a shift that is out of range + if let Some(ConstValue::Bool(false)) = self.try_get_const_value(in_range) { + return zero; + } + let shifted = if is_left { + self.shl(value, shift) + } else { + self.lshr(value, shift) + }; + self.select(in_range, shifted, zero) + } + #[instrument(level = "trace", skip(self))] fn ordering_to_semantics_def(&mut self, ordering: AtomicOrdering) -> SpirvValue { let mut invalid_seq_cst = false; @@ -3414,6 +3452,14 @@ impl<'a, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'tcx> { .is_some_and(|def_id| self.panic_entry_points.borrow().contains(&def_id)); let from_trait_impl = instance_def_id.and_then(|def_id| self.from_trait_impls.borrow().get(&def_id).copied()); + let unbounded_shift_is_left = + instance_def_id.and_then(|def_id| self.unbounded_shifts.borrow().get(&def_id).copied()); + + if let Some(is_left) = unbounded_shift_is_left + && let &[value, shift] = args + { + return self.unbounded_shift(is_left, value, shift); + } if let Some(libm_intrinsic) = libm_intrinsic { let result = self.call_libm_intrinsic(libm_intrinsic, result_type, args); diff --git a/crates/rustc_codegen_spirv/src/codegen_cx/declare.rs b/crates/rustc_codegen_spirv/src/codegen_cx/declare.rs index 3f71392ba3b..21d367268e3 100644 --- a/crates/rustc_codegen_spirv/src/codegen_cx/declare.rs +++ b/crates/rustc_codegen_spirv/src/codegen_cx/declare.rs @@ -205,6 +205,20 @@ impl<'tcx> CodegenCx<'tcx> { } } + // Inherent `unbounded_shl`/`unbounded_shr` methods of integer types in `core` + if self.tcx.crate_name(def_id.krate) == rustc_span::sym::core + && let Some(impl_def_id) = self.tcx.impl_of_assoc(def_id) + && self.tcx.impl_opt_trait_ref(impl_def_id).is_none() + && fn_abi.ret.layout.ty.is_integral() + { + let item_name = self.tcx.item_name(def_id); + if item_name == self.sym.unbounded_shl || item_name == self.sym.unbounded_shr { + self.unbounded_shifts + .borrow_mut() + .insert(def_id, item_name == self.sym.unbounded_shl); + } + } + if [ self.tcx.lang_items().panic_fn(), self.tcx.lang_items().panic_fmt(), diff --git a/crates/rustc_codegen_spirv/src/codegen_cx/mod.rs b/crates/rustc_codegen_spirv/src/codegen_cx/mod.rs index 91a319b9bdf..a2d0b3ced56 100644 --- a/crates/rustc_codegen_spirv/src/codegen_cx/mod.rs +++ b/crates/rustc_codegen_spirv/src/codegen_cx/mod.rs @@ -102,6 +102,11 @@ pub struct CodegenCx<'tcx> { /// Used to optimize constant conversions like `u32::from(42u8)` to avoid creating the source type. pub from_trait_impls: RefCell, Ty<'tcx>)>>, + /// Maps `DefId`s of `unbounded_shl`/`unbounded_shr` integer methods from `core` to whether + /// they shift left. Their library implementation branches on the shift amount, which can be + /// replaced with a branchless `OpSelect`. + pub unbounded_shifts: RefCell>, + /// Some runtimes (e.g. intel-compute-runtime) disallow atomics on i8 and i16, even though it's allowed by the spec. /// This enables/disables them. pub i8_i16_atomics_allowed: bool, @@ -233,6 +238,7 @@ impl<'tcx> CodegenCx<'tcx> { buffer_load_intrinsics: Default::default(), buffer_store_intrinsics: Default::default(), from_trait_impls: Default::default(), + unbounded_shifts: Default::default(), i8_i16_atomics_allowed: false, codegen_args, } diff --git a/crates/rustc_codegen_spirv/src/symbols.rs b/crates/rustc_codegen_spirv/src/symbols.rs index badba9a8502..e58fb18b231 100644 --- a/crates/rustc_codegen_spirv/src/symbols.rs +++ b/crates/rustc_codegen_spirv/src/symbols.rs @@ -18,6 +18,8 @@ pub struct Symbols { pub vector: Symbol, pub v1: Symbol, pub libm: Symbol, + pub unbounded_shl: Symbol, + pub unbounded_shr: Symbol, pub entry_point_name: Symbol, pub spv_khr_vulkan_memory_model: Symbol, @@ -421,6 +423,8 @@ impl Symbols { vector: Symbol::intern("vector"), v1: Symbol::intern("v1"), libm: Symbol::intern("libm"), + unbounded_shl: Symbol::intern("unbounded_shl"), + unbounded_shr: Symbol::intern("unbounded_shr"), entry_point_name: Symbol::intern("entry_point_name"), spv_khr_vulkan_memory_model: Symbol::intern("SPV_KHR_vulkan_memory_model"), diff --git a/tests/compiletests/ui/dis/unbounded_shifts.rs b/tests/compiletests/ui/dis/unbounded_shifts.rs new file mode 100644 index 00000000000..1a279a4cde0 --- /dev/null +++ b/tests/compiletests/ui/dis/unbounded_shifts.rs @@ -0,0 +1,25 @@ +// build-pass +// compile-flags: -C target-feature=+Int64 +// compile-flags: -C llvm-args=--disassemble-fn=unbounded_shifts::shifts + +use spirv_std::spirv; + +// `unbounded_shl`/`unbounded_shr` must lower to `OpSelect`s without any branches. +#[inline(never)] +fn shifts(a: u32, b: i32, c: u64, shift: u32) -> u64 { + let a = a.unbounded_shl(shift) ^ a.unbounded_shr(shift); + let b = b.unbounded_shr(shift); + let c = c.unbounded_shl(shift); + (a ^ b as u32) as u64 ^ c +} + +#[spirv(fragment)] +pub fn main( + #[spirv(flat)] a: u32, + #[spirv(flat)] b: i32, + #[spirv(flat)] c: u64, + #[spirv(flat)] shift: u32, + out: &mut u64, +) { + *out = shifts(a, b, c, shift); +} diff --git a/tests/compiletests/ui/dis/unbounded_shifts.stderr b/tests/compiletests/ui/dis/unbounded_shifts.stderr new file mode 100644 index 00000000000..0ea26a0685b --- /dev/null +++ b/tests/compiletests/ui/dis/unbounded_shifts.stderr @@ -0,0 +1,33 @@ + %1 = OpFunction %2 DontInline %3 + %4 = OpFunctionParameter %5 + %6 = OpFunctionParameter %7 + %8 = OpFunctionParameter %2 + %9 = OpFunctionParameter %5 + %10 = OpLabel + OpLine %11 10 14 + %12 = OpULessThan %13 %9 %14 + %15 = OpShiftLeftLogical %5 %4 %9 + %16 = OpSelect %5 %12 %15 %17 + OpLine %11 10 39 + %18 = OpULessThan %13 %9 %14 + %19 = OpShiftRightLogical %5 %4 %9 + %20 = OpSelect %5 %18 %19 %17 + OpLine %11 10 12 + %21 = OpBitwiseXor %5 %16 %20 + OpLine %11 11 14 + %22 = OpULessThan %13 %9 %14 + %23 = OpSelect %5 %22 %9 %24 + %25 = OpShiftRightArithmetic %7 %6 %23 + OpLine %11 12 14 + %26 = OpULessThan %13 %9 %27 + %28 = OpShiftLeftLogical %2 %8 %9 + %29 = OpSelect %2 %26 %28 %30 + OpLine %11 13 9 + %31 = OpBitcast %5 %25 + OpLine %11 13 4 + %32 = OpBitwiseXor %5 %21 %31 + %33 = OpUConvert %2 %32 + %34 = OpBitwiseXor %2 %33 %29 + OpNoLine + OpReturnValue %34 + OpFunctionEnd diff --git a/tests/difftests/tests/Cargo.lock b/tests/difftests/tests/Cargo.lock index b25543635b9..169ef471718 100644 --- a/tests/difftests/tests/Cargo.lock +++ b/tests/difftests/tests/Cargo.lock @@ -2265,6 +2265,23 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" +[[package]] +name = "unbounded-shift-cpu" +version = "0.0.0" +dependencies = [ + "difftest", + "unbounded-shift-rust", +] + +[[package]] +name = "unbounded-shift-rust" +version = "0.0.0" +dependencies = [ + "bytemuck", + "difftest", + "spirv-std", +] + [[package]] name = "unicode-ident" version = "1.0.26" diff --git a/tests/difftests/tests/Cargo.toml b/tests/difftests/tests/Cargo.toml index 7435155d325..7d878edb6b4 100644 --- a/tests/difftests/tests/Cargo.toml +++ b/tests/difftests/tests/Cargo.toml @@ -53,6 +53,8 @@ members = [ "lang/core/intrinsics/rotate/rotate-cpu", "lang/core/ops/casts/casts-cpu", "lang/core/ops/casts/casts-gpu", + "lang/core/ops/unbounded_shift/unbounded-shift-rust", + "lang/core/ops/unbounded_shift/unbounded-shift-cpu", ] [workspace.package] diff --git a/tests/difftests/tests/lang/core/ops/unbounded_shift/unbounded-shift-cpu/Cargo.toml b/tests/difftests/tests/lang/core/ops/unbounded_shift/unbounded-shift-cpu/Cargo.toml new file mode 100644 index 00000000000..bcf2d9c9fe3 --- /dev/null +++ b/tests/difftests/tests/lang/core/ops/unbounded_shift/unbounded-shift-cpu/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "unbounded-shift-cpu" +edition.workspace = true + +[lints] +workspace = true + +[dependencies] +difftest.workspace = true +unbounded-shift-rust = { path = "../unbounded-shift-rust" } diff --git a/tests/difftests/tests/lang/core/ops/unbounded_shift/unbounded-shift-cpu/src/main.rs b/tests/difftests/tests/lang/core/ops/unbounded_shift/unbounded-shift-cpu/src/main.rs new file mode 100644 index 00000000000..74119751f66 --- /dev/null +++ b/tests/difftests/tests/lang/core/ops/unbounded_shift/unbounded-shift-cpu/src/main.rs @@ -0,0 +1,16 @@ +use difftest::config::{Config, TestMetadata}; +use unbounded_shift_rust::{test_data, unbounded_shift_case}; + +fn main() { + let config = Config::from_path(std::env::args().nth(1).unwrap()).unwrap(); + + let output: Vec = test_data() + .iter() + .flat_map(|&[value, amount]| unbounded_shift_case(value, amount as u32)) + .collect(); + + config.write_result(&output).unwrap(); + config + .write_metadata(&TestMetadata::u32()) + .expect("Failed to write metadata"); +} diff --git a/tests/difftests/tests/lang/core/ops/unbounded_shift/unbounded-shift-rust/Cargo.toml b/tests/difftests/tests/lang/core/ops/unbounded_shift/unbounded-shift-rust/Cargo.toml new file mode 100644 index 00000000000..57204eb1342 --- /dev/null +++ b/tests/difftests/tests/lang/core/ops/unbounded_shift/unbounded-shift-rust/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "unbounded-shift-rust" +edition.workspace = true + +[lints] +workspace = true + +# GPU deps +[dependencies] +spirv-std.workspace = true + +# CPU deps (for the test harness) +[target.'cfg(not(target_arch = "spirv"))'.dependencies] +difftest.workspace = true +bytemuck.workspace = true diff --git a/tests/difftests/tests/lang/core/ops/unbounded_shift/unbounded-shift-rust/src/lib.rs b/tests/difftests/tests/lang/core/ops/unbounded_shift/unbounded-shift-rust/src/lib.rs new file mode 100644 index 00000000000..e69708c6602 --- /dev/null +++ b/tests/difftests/tests/lang/core/ops/unbounded_shift/unbounded-shift-rust/src/lib.rs @@ -0,0 +1,73 @@ +#![no_std] + +use spirv_std::spirv; + +pub const WORDS_PER_CASE: usize = 24; + +// The second value is negative for every signed integer width +const VALUES: [u64; 2] = [0x9ABC_DEF0_1234_5678, 0x8000_0000_8000_8081]; + +const AMOUNTS: [u32; 15] = [0, 1, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65, u32::MAX]; + +pub const NUM_CASES: usize = VALUES.len() * AMOUNTS.len(); + +pub fn test_data() -> [[u64; 2]; NUM_CASES] { + let mut data = [[0; 2]; NUM_CASES]; + for (i, case) in data.iter_mut().enumerate() { + *case = [VALUES[i / AMOUNTS.len()], AMOUNTS[i % AMOUNTS.len()] as u64]; + } + data +} + +pub fn unbounded_shift_case(value: u64, amount: u32) -> [u32; WORDS_PER_CASE] { + let shl_64 = value.unbounded_shl(amount); + let shr_64 = value.unbounded_shr(amount); + let shl_i64 = (value as i64).unbounded_shl(amount) as u64; + let shr_i64 = (value as i64).unbounded_shr(amount) as u64; + // Amounts that are constants in the shader, in and out of range + let shl_64_const = value.unbounded_shl(40); + let shr_64_const = value.unbounded_shr(64); + let shr_i64_const = (value as i64).unbounded_shr(64) as u64; + [ + shl_64 as u32, + (shl_64 >> 32) as u32, + shr_64 as u32, + (shr_64 >> 32) as u32, + shl_i64 as u32, + (shl_i64 >> 32) as u32, + shr_i64 as u32, + (shr_i64 >> 32) as u32, + (value as u32).unbounded_shl(amount), + (value as u32).unbounded_shr(amount), + (value as i32).unbounded_shl(amount) as u32, + (value as i32).unbounded_shr(amount) as u32, + (value as u16).unbounded_shl(amount) as u32, + (value as u16).unbounded_shr(amount) as u32, + (value as i16).unbounded_shr(amount) as u32, + (value as u8).unbounded_shl(amount) as u32, + (value as u8).unbounded_shr(amount) as u32, + (value as i8).unbounded_shr(amount) as u32, + shl_64_const as u32, + (shl_64_const >> 32) as u32, + shr_64_const as u32, + shr_i64_const as u32, + (value as i8).unbounded_shr(8) as u32, + (value as u8).unbounded_shl(4) as u32, + ] +} + +#[spirv(compute(threads(1)))] +pub fn main_cs( + #[spirv(storage_buffer, descriptor_set = 0, binding = 0)] input: &[u64], + #[spirv(storage_buffer, descriptor_set = 0, binding = 1)] output: &mut [u32], + #[spirv(global_invocation_id)] global_id: spirv_std::glam::UVec3, +) { + let tid = global_id.x as usize; + + if tid < input.len() { + let words = unbounded_shift_case(input[2 * tid], input[2 * tid + 1] as u32); + for i in 0..WORDS_PER_CASE { + output[tid * WORDS_PER_CASE + i] = words[i]; + } + } +} diff --git a/tests/difftests/tests/lang/core/ops/unbounded_shift/unbounded-shift-rust/src/main.rs b/tests/difftests/tests/lang/core/ops/unbounded_shift/unbounded-shift-rust/src/main.rs new file mode 100644 index 00000000000..8160f324b63 --- /dev/null +++ b/tests/difftests/tests/lang/core/ops/unbounded_shift/unbounded-shift-rust/src/main.rs @@ -0,0 +1,58 @@ +#[cfg(not(target_arch = "spirv"))] +fn main() { + use difftest::config::{Config, TestMetadata}; + use difftest::scaffold::compute::{ + AshBackend, BufferConfig, BufferUsage, ComputeShaderTest, RustComputeShader, + }; + use difftest::spirv_builder::Capability; + use unbounded_shift_rust::{NUM_CASES, WORDS_PER_CASE, test_data}; + + let config = Config::from_path(std::env::args().nth(1).unwrap()).unwrap(); + + // Skip on macOS due to Vulkan/MoltenVK configuration issues + #[cfg(target_os = "macos")] + { + use difftest::scaffold::Skip; + + let skip = Skip::new("Ash tests are skipped on macOS due to MoltenVK configuration issues"); + skip.run_test(&config).unwrap(); + return; + } + + #[cfg(not(target_os = "macos"))] + { + let input_bytes: Vec = bytemuck::cast_slice(&test_data()).to_vec(); + let output_size = (NUM_CASES * WORDS_PER_CASE * std::mem::size_of::()) as u64; + + let buffers = vec![ + BufferConfig { + size: input_bytes.len() as u64, + usage: BufferUsage::StorageReadOnly, + initial_data: Some(input_bytes), + }, + BufferConfig { + size: output_size, + usage: BufferUsage::Storage, + initial_data: None, + }, + ]; + + // Use Ash backend since wgpu/naga doesn't support Int8, Int16 and Int64 + let shader = RustComputeShader::default() + .with_capability(Capability::Int8) + .with_capability(Capability::Int16) + .with_capability(Capability::Int64); + let num_workgroups = NUM_CASES as u32; + let test = ComputeShaderTest::::new(shader, [num_workgroups, 1, 1], buffers) + .unwrap(); + + config + .write_metadata(&TestMetadata::u32()) + .expect("Failed to write metadata"); + + test.run_test(&config).unwrap(); + } +} + +#[cfg(target_arch = "spirv")] +fn main() {}