Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions crates/rustc_codegen_spirv/src/builder/builder_methods.rs
Original file line number Diff line number Diff line change
Expand Up @@ -342,6 +342,44 @@ fn memset_dynamic_scalar(
}

impl<'a, 'tcx> Builder<'a, 'tcx> {
/// Lowers `unbounded_shl`/`unbounded_shr` without branching. Shifting by the bit width or more
/// is undefined in SPIR-V, so the result is selected based on whether `shift` is in range.
fn unbounded_shift(
&mut self,
is_left: bool,
value: SpirvValue,
shift: SpirvValue,
) -> SpirvValue {
let (width, signed) = match self.lookup_type(value.ty) {
SpirvType::Integer(width, signed) => (width, signed),
other => self.fatal(format!(
"cannot shift non-integer type: {}",
other.debug(value.ty, self)
)),
};
let int_size = self.constant_int(shift.ty, width.into());
let in_range = self.icmp(IntPredicate::IntULT, shift, int_size);

if !is_left && signed {
// Shifting by `width - 1` fills the result with the sign bit, same as an unbounded shift
let max_shift = self.constant_int(shift.ty, (width - 1).into());
let shift = self.select(in_range, shift, max_shift);
return self.ashr(value, shift);
}

let zero = self.constant_int(value.ty, 0);
// Avoid emitting (or constant folding) a shift that is out of range
if let Some(ConstValue::Bool(false)) = self.try_get_const_value(in_range) {
return zero;
}
let shifted = if is_left {
self.shl(value, shift)
} else {
self.lshr(value, shift)
};
self.select(in_range, shifted, zero)
}

#[instrument(level = "trace", skip(self))]
fn ordering_to_semantics_def(&mut self, ordering: AtomicOrdering) -> SpirvValue {
let mut invalid_seq_cst = false;
Expand Down Expand Up @@ -3414,6 +3452,14 @@ impl<'a, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'tcx> {
.is_some_and(|def_id| self.panic_entry_points.borrow().contains(&def_id));
let from_trait_impl =
instance_def_id.and_then(|def_id| self.from_trait_impls.borrow().get(&def_id).copied());
let unbounded_shift_is_left =
instance_def_id.and_then(|def_id| self.unbounded_shifts.borrow().get(&def_id).copied());

if let Some(is_left) = unbounded_shift_is_left
&& let &[value, shift] = args
{
return self.unbounded_shift(is_left, value, shift);
}

if let Some(libm_intrinsic) = libm_intrinsic {
let result = self.call_libm_intrinsic(libm_intrinsic, result_type, args);
Expand Down
14 changes: 14 additions & 0 deletions crates/rustc_codegen_spirv/src/codegen_cx/declare.rs
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,20 @@ impl<'tcx> CodegenCx<'tcx> {
}
}

// Inherent `unbounded_shl`/`unbounded_shr` methods of integer types in `core`
if self.tcx.crate_name(def_id.krate) == rustc_span::sym::core
&& let Some(impl_def_id) = self.tcx.impl_of_assoc(def_id)
&& self.tcx.impl_opt_trait_ref(impl_def_id).is_none()
&& fn_abi.ret.layout.ty.is_integral()
{
let item_name = self.tcx.item_name(def_id);
if item_name == self.sym.unbounded_shl || item_name == self.sym.unbounded_shr {
self.unbounded_shifts
.borrow_mut()
.insert(def_id, item_name == self.sym.unbounded_shl);
}
}

if [
self.tcx.lang_items().panic_fn(),
self.tcx.lang_items().panic_fmt(),
Expand Down
6 changes: 6 additions & 0 deletions crates/rustc_codegen_spirv/src/codegen_cx/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,11 @@ pub struct CodegenCx<'tcx> {
/// Used to optimize constant conversions like `u32::from(42u8)` to avoid creating the source type.
pub from_trait_impls: RefCell<FxHashMap<DefId, (Ty<'tcx>, Ty<'tcx>)>>,

/// Maps `DefId`s of `unbounded_shl`/`unbounded_shr` integer methods from `core` to whether
/// they shift left. Their library implementation branches on the shift amount, which can be
/// replaced with a branchless `OpSelect`.
pub unbounded_shifts: RefCell<FxHashMap<DefId, bool>>,

/// Some runtimes (e.g. intel-compute-runtime) disallow atomics on i8 and i16, even though it's allowed by the spec.
/// This enables/disables them.
pub i8_i16_atomics_allowed: bool,
Expand Down Expand Up @@ -233,6 +238,7 @@ impl<'tcx> CodegenCx<'tcx> {
buffer_load_intrinsics: Default::default(),
buffer_store_intrinsics: Default::default(),
from_trait_impls: Default::default(),
unbounded_shifts: Default::default(),
i8_i16_atomics_allowed: false,
codegen_args,
}
Expand Down
4 changes: 4 additions & 0 deletions crates/rustc_codegen_spirv/src/symbols.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ pub struct Symbols {
pub vector: Symbol,
pub v1: Symbol,
pub libm: Symbol,
pub unbounded_shl: Symbol,
pub unbounded_shr: Symbol,
pub entry_point_name: Symbol,
pub spv_khr_vulkan_memory_model: Symbol,

Expand Down Expand Up @@ -421,6 +423,8 @@ impl Symbols {
vector: Symbol::intern("vector"),
v1: Symbol::intern("v1"),
libm: Symbol::intern("libm"),
unbounded_shl: Symbol::intern("unbounded_shl"),
unbounded_shr: Symbol::intern("unbounded_shr"),
entry_point_name: Symbol::intern("entry_point_name"),
spv_khr_vulkan_memory_model: Symbol::intern("SPV_KHR_vulkan_memory_model"),

Expand Down
25 changes: 25 additions & 0 deletions tests/compiletests/ui/dis/unbounded_shifts.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
// build-pass
// compile-flags: -C target-feature=+Int64
// compile-flags: -C llvm-args=--disassemble-fn=unbounded_shifts::shifts

use spirv_std::spirv;

// `unbounded_shl`/`unbounded_shr` must lower to `OpSelect`s without any branches.
#[inline(never)]
fn shifts(a: u32, b: i32, c: u64, shift: u32) -> u64 {
let a = a.unbounded_shl(shift) ^ a.unbounded_shr(shift);
let b = b.unbounded_shr(shift);
let c = c.unbounded_shl(shift);
(a ^ b as u32) as u64 ^ c
}

#[spirv(fragment)]
pub fn main(
#[spirv(flat)] a: u32,
#[spirv(flat)] b: i32,
#[spirv(flat)] c: u64,
#[spirv(flat)] shift: u32,
out: &mut u64,
) {
*out = shifts(a, b, c, shift);
}
33 changes: 33 additions & 0 deletions tests/compiletests/ui/dis/unbounded_shifts.stderr
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
%1 = OpFunction %2 DontInline %3
%4 = OpFunctionParameter %5
%6 = OpFunctionParameter %7
%8 = OpFunctionParameter %2
%9 = OpFunctionParameter %5
%10 = OpLabel
OpLine %11 10 14
%12 = OpULessThan %13 %9 %14
%15 = OpShiftLeftLogical %5 %4 %9
%16 = OpSelect %5 %12 %15 %17
OpLine %11 10 39
%18 = OpULessThan %13 %9 %14
%19 = OpShiftRightLogical %5 %4 %9
%20 = OpSelect %5 %18 %19 %17
OpLine %11 10 12
%21 = OpBitwiseXor %5 %16 %20
OpLine %11 11 14
%22 = OpULessThan %13 %9 %14
%23 = OpSelect %5 %22 %9 %24
%25 = OpShiftRightArithmetic %7 %6 %23
OpLine %11 12 14
%26 = OpULessThan %13 %9 %27
%28 = OpShiftLeftLogical %2 %8 %9
%29 = OpSelect %2 %26 %28 %30
OpLine %11 13 9
%31 = OpBitcast %5 %25
OpLine %11 13 4
%32 = OpBitwiseXor %5 %21 %31
%33 = OpUConvert %2 %32
%34 = OpBitwiseXor %2 %33 %29
OpNoLine
OpReturnValue %34
OpFunctionEnd
17 changes: 17 additions & 0 deletions tests/difftests/tests/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions tests/difftests/tests/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ members = [
"lang/core/intrinsics/rotate/rotate-cpu",
"lang/core/ops/casts/casts-cpu",
"lang/core/ops/casts/casts-gpu",
"lang/core/ops/unbounded_shift/unbounded-shift-rust",
"lang/core/ops/unbounded_shift/unbounded-shift-cpu",
]

[workspace.package]
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
[package]
name = "unbounded-shift-cpu"
edition.workspace = true

[lints]
workspace = true

[dependencies]
difftest.workspace = true
unbounded-shift-rust = { path = "../unbounded-shift-rust" }
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
use difftest::config::{Config, TestMetadata};
use unbounded_shift_rust::{test_data, unbounded_shift_case};

fn main() {
let config = Config::from_path(std::env::args().nth(1).unwrap()).unwrap();

let output: Vec<u32> = test_data()
.iter()
.flat_map(|&[value, amount]| unbounded_shift_case(value, amount as u32))
.collect();

config.write_result(&output).unwrap();
config
.write_metadata(&TestMetadata::u32())
.expect("Failed to write metadata");
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
[package]
name = "unbounded-shift-rust"
edition.workspace = true

[lints]
workspace = true

# GPU deps
[dependencies]
spirv-std.workspace = true

# CPU deps (for the test harness)
[target.'cfg(not(target_arch = "spirv"))'.dependencies]
difftest.workspace = true
bytemuck.workspace = true
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
#![no_std]

use spirv_std::spirv;

pub const WORDS_PER_CASE: usize = 24;

// The second value is negative for every signed integer width
const VALUES: [u64; 2] = [0x9ABC_DEF0_1234_5678, 0x8000_0000_8000_8081];

const AMOUNTS: [u32; 15] = [0, 1, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65, u32::MAX];

pub const NUM_CASES: usize = VALUES.len() * AMOUNTS.len();

pub fn test_data() -> [[u64; 2]; NUM_CASES] {
let mut data = [[0; 2]; NUM_CASES];
for (i, case) in data.iter_mut().enumerate() {
*case = [VALUES[i / AMOUNTS.len()], AMOUNTS[i % AMOUNTS.len()] as u64];
}
data
}

pub fn unbounded_shift_case(value: u64, amount: u32) -> [u32; WORDS_PER_CASE] {
let shl_64 = value.unbounded_shl(amount);
let shr_64 = value.unbounded_shr(amount);
let shl_i64 = (value as i64).unbounded_shl(amount) as u64;
let shr_i64 = (value as i64).unbounded_shr(amount) as u64;
// Amounts that are constants in the shader, in and out of range
let shl_64_const = value.unbounded_shl(40);
let shr_64_const = value.unbounded_shr(64);
let shr_i64_const = (value as i64).unbounded_shr(64) as u64;
[
shl_64 as u32,
(shl_64 >> 32) as u32,
shr_64 as u32,
(shr_64 >> 32) as u32,
shl_i64 as u32,
(shl_i64 >> 32) as u32,
shr_i64 as u32,
(shr_i64 >> 32) as u32,
(value as u32).unbounded_shl(amount),
(value as u32).unbounded_shr(amount),
(value as i32).unbounded_shl(amount) as u32,
(value as i32).unbounded_shr(amount) as u32,
(value as u16).unbounded_shl(amount) as u32,
(value as u16).unbounded_shr(amount) as u32,
(value as i16).unbounded_shr(amount) as u32,
(value as u8).unbounded_shl(amount) as u32,
(value as u8).unbounded_shr(amount) as u32,
(value as i8).unbounded_shr(amount) as u32,
shl_64_const as u32,
(shl_64_const >> 32) as u32,
shr_64_const as u32,
shr_i64_const as u32,
(value as i8).unbounded_shr(8) as u32,
(value as u8).unbounded_shl(4) as u32,
]
}

#[spirv(compute(threads(1)))]
pub fn main_cs(
#[spirv(storage_buffer, descriptor_set = 0, binding = 0)] input: &[u64],
#[spirv(storage_buffer, descriptor_set = 0, binding = 1)] output: &mut [u32],
#[spirv(global_invocation_id)] global_id: spirv_std::glam::UVec3,
) {
let tid = global_id.x as usize;

if tid < input.len() {
let words = unbounded_shift_case(input[2 * tid], input[2 * tid + 1] as u32);
for i in 0..WORDS_PER_CASE {
output[tid * WORDS_PER_CASE + i] = words[i];
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
#[cfg(not(target_arch = "spirv"))]
fn main() {
use difftest::config::{Config, TestMetadata};
use difftest::scaffold::compute::{
AshBackend, BufferConfig, BufferUsage, ComputeShaderTest, RustComputeShader,
};
use difftest::spirv_builder::Capability;
use unbounded_shift_rust::{NUM_CASES, WORDS_PER_CASE, test_data};

let config = Config::from_path(std::env::args().nth(1).unwrap()).unwrap();

// Skip on macOS due to Vulkan/MoltenVK configuration issues
#[cfg(target_os = "macos")]
{
use difftest::scaffold::Skip;

let skip = Skip::new("Ash tests are skipped on macOS due to MoltenVK configuration issues");
skip.run_test(&config).unwrap();
return;
}

#[cfg(not(target_os = "macos"))]
{
let input_bytes: Vec<u8> = bytemuck::cast_slice(&test_data()).to_vec();
let output_size = (NUM_CASES * WORDS_PER_CASE * std::mem::size_of::<u32>()) as u64;

let buffers = vec![
BufferConfig {
size: input_bytes.len() as u64,
usage: BufferUsage::StorageReadOnly,
initial_data: Some(input_bytes),
},
BufferConfig {
size: output_size,
usage: BufferUsage::Storage,
initial_data: None,
},
];

// Use Ash backend since wgpu/naga doesn't support Int8, Int16 and Int64
let shader = RustComputeShader::default()
.with_capability(Capability::Int8)
.with_capability(Capability::Int16)
.with_capability(Capability::Int64);
let num_workgroups = NUM_CASES as u32;
let test = ComputeShaderTest::<AshBackend, _>::new(shader, [num_workgroups, 1, 1], buffers)
.unwrap();

config
.write_metadata(&TestMetadata::u32())
.expect("Failed to write metadata");

test.run_test(&config).unwrap();
}
}

#[cfg(target_arch = "spirv")]
fn main() {}
Loading