1use super::*;
2
3pick! {
4 if #[cfg(target_feature="sse2")] {
5 #[derive(Default, Clone, Copy, PartialEq, Eq)]
12 #[repr(C, align(16))]
13 pub struct i32x4 { pub(crate) sse: m128i }
14 } else if #[cfg(target_feature="simd128")] {
15 use core::arch::wasm32::*;
16
17 #[derive(Clone, Copy)]
24 #[repr(transparent)]
25 pub struct i32x4 { pub(crate) simd: v128 }
26
27 impl Default for i32x4 {
28 fn default() -> Self {
29 Self::splat(0)
30 }
31 }
32
33 impl PartialEq for i32x4 {
34 fn eq(&self, other: &Self) -> bool {
35 u32x4_all_true(i32x4_eq(self.simd, other.simd))
36 }
37 }
38
39 impl Eq for i32x4 { }
40 } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
41 use core::arch::aarch64::*;
42
43 #[repr(C)]
50 #[derive(Copy, Clone)]
51 pub struct i32x4 { pub(crate) neon : int32x4_t }
52
53 impl Default for i32x4 {
54 #[inline]
55 fn default() -> Self {
56 Self::splat(0)
57 }
58 }
59
60 impl PartialEq for i32x4 {
61 #[inline]
62 fn eq(&self, other: &Self) -> bool {
63 unsafe { vminvq_u32(vceqq_s32(self.neon, other.neon))==u32::MAX }
64 }
65 }
66
67 impl Eq for i32x4 { }
68 } else {
69 #[derive(Default, Clone, Copy, PartialEq, Eq)]
76 #[repr(C, align(16))]
77 pub struct i32x4 { pub(crate) arr: [i32;4] }
78 }
79}
80
81impl_simd_int! {
82 unsafe {
83 T = i32,
84 N = 4,
85 Simd = i32x4,
86 UintSimd = u32x4,
87 T_BITS = 32,
88 T_BITS_MUL_2 = 64,
89 BitmaskType = u32,
90 [0, 1, 2, 3],
91 optional_type_x86_inner { X86Inner = __m128i },
92 optional_type_arm_inner { ArmInner = int32x4_t },
93 optional_type_wasm_inner { WasmInner = v128 },
94 }
95
96 #[inline]
97 fn simd_lt(self, rhs: Self) -> Self::Output {
98 pick! {
99 if #[cfg(target_feature="sse2")] {
100 Self { sse: cmp_lt_mask_i32_m128i(self.sse, rhs.sse) }
101 } else if #[cfg(target_feature="simd128")] {
102 Self { simd: i32x4_lt(self.simd, rhs.simd) }
103 } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
104 unsafe {Self { neon: vreinterpretq_s32_u32(vcltq_s32(self.neon, rhs.neon)) }}
105 } else {
106 Self { arr: [
107 if self.arr[0] < rhs.arr[0] { -1 } else { 0 },
108 if self.arr[1] < rhs.arr[1] { -1 } else { 0 },
109 if self.arr[2] < rhs.arr[2] { -1 } else { 0 },
110 if self.arr[3] < rhs.arr[3] { -1 } else { 0 },
111 ]}
112 }
113 }
114 }
115
116 #[inline]
117 fn simd_gt(self, rhs: Self) -> Self::Output {
118 pick! {
119 if #[cfg(target_feature="sse2")] {
120 Self { sse: cmp_gt_mask_i32_m128i(self.sse, rhs.sse) }
121 } else if #[cfg(target_feature="simd128")] {
122 Self { simd: i32x4_gt(self.simd, rhs.simd) }
123 } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
124 unsafe {Self { neon: vreinterpretq_s32_u32(vcgtq_s32(self.neon, rhs.neon)) }}
125 } else {
126 Self { arr: [
127 if self.arr[0] > rhs.arr[0] { -1 } else { 0 },
128 if self.arr[1] > rhs.arr[1] { -1 } else { 0 },
129 if self.arr[2] > rhs.arr[2] { -1 } else { 0 },
130 if self.arr[3] > rhs.arr[3] { -1 } else { 0 },
131 ]}
132 }
133 }
134 }
135
136 #[inline]
137 fn simd_le(self, rhs: Self) -> Self::Output {
138 pick! {
139 if #[cfg(target_feature="sse2")] {
140 !self.simd_gt(rhs)
141 } else if #[cfg(target_feature="simd128")] {
142 Self { simd: i32x4_le(self.simd, rhs.simd) }
143 } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
144 !self.simd_gt(rhs)
145 } else {
146 Self { arr: [
147 if self.arr[0] <= rhs.arr[0] { -1 } else { 0 },
148 if self.arr[1] <= rhs.arr[1] { -1 } else { 0 },
149 if self.arr[2] <= rhs.arr[2] { -1 } else { 0 },
150 if self.arr[3] <= rhs.arr[3] { -1 } else { 0 },
151 ]}
152 }
153 }
154 }
155
156 #[inline]
157 fn simd_ge(self, rhs: Self) -> Self::Output {
158 pick! {
159 if #[cfg(target_feature="sse2")] {
160 !self.simd_lt(rhs)
161 } else if #[cfg(target_feature="simd128")] {
162 Self { simd: i32x4_ge(self.simd, rhs.simd) }
163 } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
164 !self.simd_lt(rhs)
165 } else {
166 Self { arr: [
167 if self.arr[0] >= rhs.arr[0] { -1 } else { 0 },
168 if self.arr[1] >= rhs.arr[1] { -1 } else { 0 },
169 if self.arr[2] >= rhs.arr[2] { -1 } else { 0 },
170 if self.arr[3] >= rhs.arr[3] { -1 } else { 0 },
171 ]}
172 }
173 }
174 }
175
176 #[inline]
177 fn shr(self, rhs: u32x4) -> Self::Output {
178 pick! {
179 if #[cfg(target_feature="avx2")] {
180 let shift_by = bitand_m128i(rhs.sse, set_splat_i32_m128i(31));
182 Self { sse: shr_each_i32_m128i(self.sse, shift_by) }
183 } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
184 unsafe {
185 let shift_by = vnegq_s32(vreinterpretq_s32_u32(vandq_u32(rhs.neon, vmovq_n_u32(31))));
188 Self { neon: vshlq_s32(self.neon, shift_by) }
189 }
190 } else {
191 let arr: [i32; 4] = cast(self);
192 let rhs: [u32; 4] = cast(rhs);
193 cast([
194 arr[0].wrapping_shr(rhs[0]),
195 arr[1].wrapping_shr(rhs[1]),
196 arr[2].wrapping_shr(rhs[2]),
197 arr[3].wrapping_shr(rhs[3]),
198 ])
199 }
200 }
201 }
202
203 #[inline]
204 fn shr(self, rhs: u32) -> Self::Output {
205 pick! {
206 if #[cfg(target_feature="sse2")] {
207 #[expect(clippy::suspicious_arithmetic_impl)]
209 let shift = cast([rhs as u64 & 31, 0]);
210 Self { sse: shr_all_i32_m128i(self.sse, shift) }
211 } else if #[cfg(target_feature="simd128")] {
212 Self { simd: i32x4_shr(self.simd, rhs) }
213 } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
214 #[expect(clippy::suspicious_arithmetic_impl)]
216 unsafe {Self { neon: vshlq_s32(self.neon, vmovq_n_s32( -(rhs as i32 & 31))) }}
217 } else {
218 Self { arr: [
219 self.arr[0].wrapping_shr(rhs),
220 self.arr[1].wrapping_shr(rhs),
221 self.arr[2].wrapping_shr(rhs),
222 self.arr[3].wrapping_shr(rhs),
223 ]}
224 }
225 }
226 }
227
228 #[inline]
229 pub fn max(self, rhs: Self) -> Self {
230 pick! {
231 if #[cfg(target_feature="sse4.1")] {
232 Self { sse: max_i32_m128i(self.sse, rhs.sse) }
233 } else if #[cfg(target_feature="simd128")] {
234 Self { simd: i32x4_max(self.simd, rhs.simd) }
235 } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
236 unsafe {Self { neon: vmaxq_s32(self.neon, rhs.neon) }}
237 } else {
238 self.simd_lt(rhs).select(rhs, self)
239 }
240 }
241 }
242
243 #[inline]
244 pub fn min(self, rhs: Self) -> Self {
245 pick! {
246 if #[cfg(target_feature="sse4.1")] {
247 Self { sse: min_i32_m128i(self.sse, rhs.sse) }
248 } else if #[cfg(target_feature="simd128")] {
249 Self { simd: i32x4_min(self.simd, rhs.simd) }
250 } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
251 unsafe {Self { neon: vminq_s32(self.neon, rhs.neon) }}
252 } else {
253 self.simd_lt(rhs).select(self, rhs)
254 }
255 }
256 }
257
258 #[inline]
259 pub fn reduce_max(self) -> i32 {
260 let arr: [i32; 4] = cast(self);
261 arr[0].max(arr[1]).max(arr[2].max(arr[3]))
262 }
263
264 #[inline]
265 pub fn reduce_min(self) -> i32 {
266 let arr: [i32; 4] = cast(self);
267 arr[0].min(arr[1]).min(arr[2].min(arr[3]))
268 }
269
270 #[inline]
271 pub fn unbounded_shr(self, rhs: u32x4) -> Self {
272 pick! {
273 if #[cfg(target_feature="avx2")] {
274 Self { sse: shr_each_i32_m128i(self.sse, rhs.sse) }
275 } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
276 unsafe {
277 Self { neon: vshlq_s32(self.neon, vnegq_s32(vreinterpretq_s32_u32(rhs.min(u32x4::splat(32)).neon))) }
280 }
281 } else {
282 let self_array = self.to_array();
283 let rhs_array = rhs.to_array();
284
285 Self::new([
286 self_array[0].unbounded_shr(rhs_array[0]),
287 self_array[1].unbounded_shr(rhs_array[1]),
288 self_array[2].unbounded_shr(rhs_array[2]),
289 self_array[3].unbounded_shr(rhs_array[3]),
290 ])
291 }
292 }
293 }
294
295 #[inline]
296 pub fn unbounded_shr_scalar(self, rhs: u32) -> Self {
297 pick! {
298 if #[cfg(target_feature="sse2")] {
299 Self { sse: shr_all_i32_m128i(self.sse, cast([rhs as u64, 0])) }
300 } else if #[cfg(target_feature="simd128")] {
301 if rhs < 32 { Self { simd: i32x4_shr(self.simd, rhs) } } else { self.is_negative() }
302 } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
303 unsafe {
304 Self { neon: vshlq_s32(self.neon, vmovq_n_s32(-rhs.min(32).cast_signed())) }
307 }
308 } else {
309 Self {
310 arr: [
311 self.arr[0].unbounded_shr(rhs),
312 self.arr[1].unbounded_shr(rhs),
313 self.arr[2].unbounded_shr(rhs),
314 self.arr[3].unbounded_shr(rhs),
315 ]
316 }
317 }
318 }
319 }
320
321 #[inline]
322 pub fn saturating_add(self, rhs: Self) -> Self {
323 pick! {
324 if #[cfg(any(target_feature="sse2", target_feature="simd128"))] {
325 let result = self + rhs;
326 let overflow = (!(self ^ rhs) & (self ^ result)).is_negative();
327 let negative = self.is_negative();
328
329 overflow.select(Self::MAX ^ negative, result)
331 } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
332 unsafe { Self { neon: vqaddq_s32(self.neon, rhs.neon) } }
333 } else {
334 Self {
335 arr: [
336 self.arr[0].saturating_add(rhs.arr[0]),
337 self.arr[1].saturating_add(rhs.arr[1]),
338 self.arr[2].saturating_add(rhs.arr[2]),
339 self.arr[3].saturating_add(rhs.arr[3]),
340 ],
341 }
342 }
343 }
344 }
345
346 #[inline]
347 pub fn saturating_sub(self, rhs: Self) -> Self {
348 pick! {
349 if #[cfg(any(target_feature="sse2", target_feature="simd128"))] {
350 let result = self - rhs;
351 let overflow = ((self ^ rhs) & (self ^ result)).is_negative();
352 let negative = self.is_negative();
353
354 overflow.select(Self::MAX ^ negative, result)
356 } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
357 unsafe { Self { neon: vqsubq_s32(self.neon, rhs.neon) } }
358 } else {
359 Self {
360 arr: [
361 self.arr[0].saturating_sub(rhs.arr[0]),
362 self.arr[1].saturating_sub(rhs.arr[1]),
363 self.arr[2].saturating_sub(rhs.arr[2]),
364 self.arr[3].saturating_sub(rhs.arr[3]),
365 ],
366 }
367 }
368 }
369 }
370
371 #[inline]
372 pub fn overflowing_mul(self, rhs: Self) -> (Self, Self) {
373 let (low, high) = self.mul_keep_low_high(rhs);
374 let low = cast::<u32x4, i32x4>(low);
375
376 let overflow = high.simd_ne(low.is_negative());
377 (low, overflow)
378 }
379
380 optional_fn_widening_mul {
381 #[inline]
382 pub fn widening_mul(self, rhs: Self) -> i64x4 {
383 pick! {
384 if #[cfg(target_feature="avx2")] {
385 let a = convert_to_i64_m256i_from_i32_m128i(self.sse);
386 let b = convert_to_i64_m256i_from_i32_m128i(rhs.sse);
387 cast(mul_i64_low_bits_m256i(a, b))
388 } else if #[cfg(target_feature="sse4.1")] {
389 let evenp = mul_widen_i32_odd_m128i(self.sse, rhs.sse);
390
391 let oddp = mul_widen_i32_odd_m128i(
392 shr_imm_u64_m128i::<32>(self.sse),
393 shr_imm_u64_m128i::<32>(rhs.sse));
394
395 i64x4 {
396 a: i64x2 { sse: unpack_low_i64_m128i(evenp, oddp)},
397 b: i64x2 { sse: unpack_high_i64_m128i(evenp, oddp)}
398 }
399 } else if #[cfg(target_feature="simd128")] {
400 i64x4 {
401 a: i64x2 { simd: i64x2_extmul_low_i32x4(self.simd, rhs.simd) },
402 b: i64x2 { simd: i64x2_extmul_high_i32x4(self.simd, rhs.simd) },
403 }
404 } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))] {
405 unsafe {
406 i64x4 { a: i64x2 { neon: vmull_s32(vget_low_s32(self.neon), vget_low_s32(rhs.neon)) },
407 b: i64x2 { neon: vmull_s32(vget_high_s32(self.neon), vget_high_s32(rhs.neon)) } }
408 }
409 } else {
410 let a = self.as_array();
411 let b = rhs.as_array();
412
413 cast([
414 i64::from(a[0]) * i64::from(b[0]),
415 i64::from(a[1]) * i64::from(b[1]),
416 i64::from(a[2]) * i64::from(b[2]),
417 i64::from(a[3]) * i64::from(b[3]),
418 ])
419 }
420 }
421 }
422 }
423
424 #[inline]
425 pub fn mul_keep_low_high(self, rhs: Self) -> (u32x4, i32x4) {
426 pick! {
427 if #[cfg(target_feature="sse4.1")] {
428 let even_wide_mul = mul_widen_i32_odd_m128i(self.sse, rhs.sse);
429 let odd_wide_mul = mul_widen_i32_odd_m128i(
430 shuffle_ai_f32_all_m128i::<0b_00_11_00_01>(self.sse),
431 shuffle_ai_f32_all_m128i::<0b_00_11_00_01>(rhs.sse),
432 );
433 let ll_hh_1 = unpack_low_i32_m128i(even_wide_mul, odd_wide_mul);
434 let ll_hh_2 = unpack_high_i32_m128i(even_wide_mul, odd_wide_mul);
435
436 (
437 u32x4 { sse: unpack_low_i64_m128i(ll_hh_1, ll_hh_2) },
438 i32x4 { sse: unpack_high_i64_m128i(ll_hh_1, ll_hh_2) },
439 )
440 } else if #[cfg(target_feature="simd128")] {
441 let low_wide_mul = i64x2_extmul_low_i32x4(self.simd, rhs.simd);
442 let high_wide_mul = i64x2_extmul_high_i32x4(self.simd, rhs.simd);
443
444 (
445 u32x4 { simd: i32x4_shuffle::<0, 2, 4, 6>(low_wide_mul, high_wide_mul) },
446 i32x4 { simd: i32x4_shuffle::<1, 3, 5, 7>(low_wide_mul, high_wide_mul) },
447 )
448 } else if #[cfg(all(target_feature="neon", target_arch="aarch64"))] {
449 unsafe {
450 let low_wide_mul = vreinterpretq_s32_s64(
451 vmull_s32(vget_low_s32(self.neon), vget_low_s32(rhs.neon)),
452 );
453 let high_wide_mul = vreinterpretq_s32_s64(
454 vmull_s32(vget_high_s32(self.neon), vget_high_s32(rhs.neon)),
455 );
456 let low_high = vuzpq_s32(low_wide_mul, high_wide_mul);
457
458 (
459 u32x4 { neon: vreinterpretq_u32_s32(low_high.0) },
460 i32x4 { neon: low_high.1 },
461 )
462 }
463 } else {
464 let self_array = self.to_array();
468 let rhs_array = rhs.to_array();
469
470 let widening_mul = [
471 (self_array[0] as i64).wrapping_mul(rhs_array[0] as i64),
472 (self_array[1] as i64).wrapping_mul(rhs_array[1] as i64),
473 (self_array[2] as i64).wrapping_mul(rhs_array[2] as i64),
474 (self_array[3] as i64).wrapping_mul(rhs_array[3] as i64),
475 ];
476
477 (
478 u32x4::new([
479 widening_mul[0] as u32,
480 widening_mul[1] as u32,
481 widening_mul[2] as u32,
482 widening_mul[3] as u32,
483 ]),
484 i32x4::new([
485 (widening_mul[0] >> 32) as i32,
486 (widening_mul[1] >> 32) as i32,
487 (widening_mul[2] >> 32) as i32,
488 (widening_mul[3] >> 32) as i32,
489 ]),
490 )
491 }
492 }
493 }
494
495 #[inline]
496 pub fn mul_keep_high(self, rhs: Self) -> Self {
497 pick! {
498 if #[cfg(target_feature="sse4.1")] {
499 let even_wide_mul = mul_widen_i32_odd_m128i(self.sse, rhs.sse);
500 let odd_wide_mul = mul_widen_i32_odd_m128i(
501 shuffle_ai_f32_all_m128i::<0b_00_11_00_01>(self.sse),
502 shuffle_ai_f32_all_m128i::<0b_00_11_00_01>(rhs.sse),
503 );
504 let ll_hh_1 = unpack_low_i32_m128i(even_wide_mul, odd_wide_mul);
505 let ll_hh_2 = unpack_high_i32_m128i(even_wide_mul, odd_wide_mul);
506
507 Self { sse: unpack_high_i64_m128i(ll_hh_1, ll_hh_2) }
508 } else if #[cfg(target_feature="simd128")] {
509 let low_wide_mul = i64x2_extmul_low_i32x4(self.simd, rhs.simd);
510 let high_wide_mul = i64x2_extmul_high_i32x4(self.simd, rhs.simd);
511
512 Self { simd: i32x4_shuffle::<1, 3, 5, 7>(low_wide_mul, high_wide_mul) }
513 } else if #[cfg(all(target_feature="neon", target_arch="aarch64"))] {
514 unsafe {
515 let low_wide_mul = vreinterpretq_s32_s64(
516 vmull_s32(vget_low_s32(self.neon), vget_low_s32(rhs.neon)),
517 );
518 let high_wide_mul = vreinterpretq_s32_s64(
519 vmull_s32(vget_high_s32(self.neon), vget_high_s32(rhs.neon)),
520 );
521
522 Self { neon: vuzpq_s32(low_wide_mul, high_wide_mul).1 }
523 }
524 } else {
525 let self_array = self.to_array();
526 let rhs_array = rhs.to_array();
527
528 Self::new([
529 ((self_array[0] as i64).wrapping_mul(rhs_array[0] as i64) >> 32) as i32,
530 ((self_array[1] as i64).wrapping_mul(rhs_array[1] as i64) >> 32) as i32,
531 ((self_array[2] as i64).wrapping_mul(rhs_array[2] as i64) >> 32) as i32,
532 ((self_array[3] as i64).wrapping_mul(rhs_array[3] as i64) >> 32) as i32,
533 ])
534 }
535 }
536 }
537
538 #[inline]
539 pub fn abs(self) -> Self {
540 pick! {
541 if #[cfg(target_feature="ssse3")] {
542 Self { sse: abs_i32_m128i(self.sse) }
543 } else if #[cfg(target_feature="simd128")] {
544 Self { simd: i32x4_abs(self.simd) }
545 } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
546 unsafe {Self { neon: vabsq_s32(self.neon) }}
547 } else {
548 let arr: [i32; 4] = cast(self);
549 cast([
550 arr[0].wrapping_abs(),
551 arr[1].wrapping_abs(),
552 arr[2].wrapping_abs(),
553 arr[3].wrapping_abs(),
554 ])
555 }
556 }
557 }
558
559 #[inline]
560 pub fn is_positive(self) -> Self {
561 pick! {
562 if #[cfg(all(target_feature="neon", target_arch="aarch64"))] {
563 Self { neon: unsafe { vreinterpretq_s32_u32(vcgtzq_s32(self.neon)) } }
564 } else {
565 self.simd_gt(Self::ZERO)
566 }
567 }
568 }
569
570 #[inline]
571 pub fn is_negative(self) -> Self {
572 pick! {
573 if #[cfg(all(target_feature="neon", target_arch="aarch64"))] {
574 Self { neon: unsafe { vreinterpretq_s32_u32(vcltzq_s32(self.neon)) } }
575 } else {
576 self.simd_lt(Self::ZERO)
577 }
578 }
579 }
580
581 optional_fn_deserialize {}
582}
583
584impl i32x4 {
587 #[inline]
589 #[must_use]
590 pub fn round_float(self) -> f32x4 {
591 pick! {
592 if #[cfg(target_feature="sse2")] {
593 cast(convert_to_m128_from_i32_m128i(self.sse))
594 } else if #[cfg(target_feature="simd128")] {
595 cast(Self { simd: f32x4_convert_i32x4(self.simd) })
596 } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
597 cast(unsafe {Self { neon: vreinterpretq_s32_f32(vcvtq_f32_s32(self.neon)) }})
598 } else {
599 let arr: [i32; 4] = cast(self);
600 cast([
601 arr[0] as f32,
602 arr[1] as f32,
603 arr[2] as f32,
604 arr[3] as f32,
605 ])
606 }
607 }
608 }
609
610 #[inline]
619 #[must_use]
620 #[deprecated(since = "1.6.0", note = "renamed to `widening_mul`")]
621 pub fn mul_widen(self, rhs: Self) -> i64x4 {
622 self.widening_mul(rhs)
623 }
624}