wide/
i16x8_.rs

1use super::*;
2
3pick! {
4  if #[cfg(target_feature="sse2")] {
5    /// A SIMD vector with eight elements of type [`i16`].
6    ///
7    /// See the [crate level documentation] for more information about SIMD
8    /// vectors.
9    ///
10    /// [crate level documentation]: crate
11    #[derive(Default, Clone, Copy, PartialEq, Eq)]
12    #[repr(C, align(16))]
13    pub struct i16x8 { pub(crate) sse: m128i }
14  } else if #[cfg(target_feature="simd128")] {
15    use core::arch::wasm32::*;
16
17    /// A SIMD vector with eight elements of type [`i16`].
18    ///
19    /// See the [crate level documentation] for more information about SIMD
20    /// vectors.
21    ///
22    /// [crate level documentation]: crate
23    #[derive(Clone, Copy)]
24    #[repr(transparent)]
25    pub struct i16x8 { pub(crate) simd: v128 }
26
27    impl Default for i16x8 {
28      fn default() -> Self {
29        Self::splat(0)
30      }
31    }
32
33    impl PartialEq for i16x8 {
34      fn eq(&self, other: &Self) -> bool {
35        u16x8_all_true(i16x8_eq(self.simd, other.simd))
36      }
37    }
38
39    impl Eq for i16x8 { }
40  } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
41    use core::arch::aarch64::*;
42
43    /// A SIMD vector with eight elements of type [`i16`].
44    ///
45    /// See the [crate level documentation] for more information about SIMD
46    /// vectors.
47    ///
48    /// [crate level documentation]: crate
49    #[repr(C)]
50    #[derive(Copy, Clone)]
51    pub struct i16x8 { pub(crate) neon : int16x8_t }
52
53    impl Default for i16x8 {
54      #[inline]
55      fn default() -> Self {
56        Self::splat(0)
57      }
58    }
59
60    impl PartialEq for i16x8 {
61      #[inline]
62      fn eq(&self, other: &Self) -> bool {
63        unsafe { vminvq_u16(vceqq_s16(self.neon, other.neon))==u16::MAX }
64      }
65    }
66
67    impl Eq for i16x8 { }
68  } else {
69    /// A SIMD vector with eight elements of type [`i16`].
70    ///
71    /// See the [crate level documentation] for more information about SIMD
72    /// vectors.
73    ///
74    /// [crate level documentation]: crate
75    #[derive(Default, Clone, Copy, PartialEq, Eq)]
76    #[repr(C, align(16))]
77    pub struct i16x8 { pub(crate) arr: [i16;8] }
78  }
79}
80
81impl_simd_int! {
82  unsafe {
83    T = i16,
84    N = 8,
85    Simd = i16x8,
86    UintSimd = u16x8,
87    T_BITS = 16,
88    T_BITS_MUL_2 = 32,
89    BitmaskType = u32,
90    [0, 1, 2, 3, 4, 5, 6, 7],
91    optional_type_x86_inner { X86Inner = __m128i },
92    optional_type_arm_inner { ArmInner = int16x8_t },
93    optional_type_wasm_inner { WasmInner = v128 },
94  }
95
96  #[inline]
97  fn simd_lt(self, rhs: Self) -> Self::Output {
98    pick! {
99      if #[cfg(target_feature="sse2")] {
100        Self { sse: cmp_lt_mask_i16_m128i(self.sse, rhs.sse) }
101      } else if #[cfg(target_feature="simd128")] {
102        Self { simd: i16x8_lt(self.simd, rhs.simd) }
103      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
104        unsafe {Self { neon: vreinterpretq_s16_u16(vcltq_s16(self.neon, rhs.neon)) }}
105      } else {
106        Self { arr: [
107          if self.arr[0] < rhs.arr[0] { -1 } else { 0 },
108          if self.arr[1] < rhs.arr[1] { -1 } else { 0 },
109          if self.arr[2] < rhs.arr[2] { -1 } else { 0 },
110          if self.arr[3] < rhs.arr[3] { -1 } else { 0 },
111          if self.arr[4] < rhs.arr[4] { -1 } else { 0 },
112          if self.arr[5] < rhs.arr[5] { -1 } else { 0 },
113          if self.arr[6] < rhs.arr[6] { -1 } else { 0 },
114          if self.arr[7] < rhs.arr[7] { -1 } else { 0 },
115        ]}
116      }
117    }
118  }
119
120  #[inline]
121  fn simd_gt(self, rhs: Self) -> Self::Output {
122    pick! {
123      if #[cfg(target_feature="sse2")] {
124        Self { sse: cmp_gt_mask_i16_m128i(self.sse, rhs.sse) }
125      } else if #[cfg(target_feature="simd128")] {
126        Self { simd: i16x8_gt(self.simd, rhs.simd) }
127      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
128        unsafe {Self { neon: vreinterpretq_s16_u16(vcgtq_s16(self.neon, rhs.neon)) }}
129      } else {
130        Self { arr: [
131          if self.arr[0] > rhs.arr[0] { -1 } else { 0 },
132          if self.arr[1] > rhs.arr[1] { -1 } else { 0 },
133          if self.arr[2] > rhs.arr[2] { -1 } else { 0 },
134          if self.arr[3] > rhs.arr[3] { -1 } else { 0 },
135          if self.arr[4] > rhs.arr[4] { -1 } else { 0 },
136          if self.arr[5] > rhs.arr[5] { -1 } else { 0 },
137          if self.arr[6] > rhs.arr[6] { -1 } else { 0 },
138          if self.arr[7] > rhs.arr[7] { -1 } else { 0 },
139        ]}
140      }
141    }
142  }
143
144  #[inline]
145  fn simd_le(self, rhs: Self) -> Self::Output {
146    pick! {
147      if #[cfg(target_feature="sse2")] {
148        !self.simd_gt(rhs)
149      } else if #[cfg(target_feature="simd128")] {
150        Self { simd: i16x8_le(self.simd, rhs.simd) }
151      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
152        !self.simd_gt(rhs)
153      } else {
154        Self { arr: [
155          if self.arr[0] <= rhs.arr[0] { -1 } else { 0 },
156          if self.arr[1] <= rhs.arr[1] { -1 } else { 0 },
157          if self.arr[2] <= rhs.arr[2] { -1 } else { 0 },
158          if self.arr[3] <= rhs.arr[3] { -1 } else { 0 },
159          if self.arr[4] <= rhs.arr[4] { -1 } else { 0 },
160          if self.arr[5] <= rhs.arr[5] { -1 } else { 0 },
161          if self.arr[6] <= rhs.arr[6] { -1 } else { 0 },
162          if self.arr[7] <= rhs.arr[7] { -1 } else { 0 },
163        ]}
164      }
165    }
166  }
167
168  #[inline]
169  fn simd_ge(self, rhs: Self) -> Self::Output {
170    pick! {
171      if #[cfg(target_feature="sse2")] {
172        !self.simd_lt(rhs)
173      } else if #[cfg(target_feature="simd128")] {
174        Self { simd: i16x8_ge(self.simd, rhs.simd) }
175      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
176        !self.simd_lt(rhs)
177      } else {
178        Self { arr: [
179          if self.arr[0] >= rhs.arr[0] { -1 } else { 0 },
180          if self.arr[1] >= rhs.arr[1] { -1 } else { 0 },
181          if self.arr[2] >= rhs.arr[2] { -1 } else { 0 },
182          if self.arr[3] >= rhs.arr[3] { -1 } else { 0 },
183          if self.arr[4] >= rhs.arr[4] { -1 } else { 0 },
184          if self.arr[5] >= rhs.arr[5] { -1 } else { 0 },
185          if self.arr[6] >= rhs.arr[6] { -1 } else { 0 },
186          if self.arr[7] >= rhs.arr[7] { -1 } else { 0 },
187        ]}
188      }
189    }
190  }
191
192  #[inline]
193  fn shr(self, rhs: u16x8) -> Self::Output {
194    pick! {
195      if #[cfg(all(target_feature="avx512bw", target_feature="avx512vl"))] {
196        #[cfg(target_arch = "x86")]
197        use core::arch::x86::_mm_srav_epi16;
198        #[cfg(target_arch = "x86_64")]
199        use core::arch::x86_64::_mm_srav_epi16;
200
201        // Mask `rhs` to 15 to match `wrapping_shr`.
202        let rhs = bitand_m128i(rhs.sse, set_splat_i16_m128i(15));
203        // TODO(safe_arch): Add `_mm_srav_epi16`.
204        cast(unsafe { _mm_srav_epi16(self.sse.0, rhs.0) })
205      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))] {
206        unsafe {
207          // Mask `rhs` to 15 to match `wrapping_shr`, and negate it because
208          // there is no shift-right intrinsic.
209          let neg_rhs = vnegq_s16(vreinterpretq_s16_u16(vandq_u16(rhs.neon, vmovq_n_u16(15))));
210          Self { neon: vshlq_s16(self.neon, neg_rhs) }
211        }
212      } else {
213        let self_array = self.to_array();
214        let rhs_array = rhs.to_array();
215
216        Self::new([
217          self_array[0].wrapping_shr(rhs_array[0] as u32),
218          self_array[1].wrapping_shr(rhs_array[1] as u32),
219          self_array[2].wrapping_shr(rhs_array[2] as u32),
220          self_array[3].wrapping_shr(rhs_array[3] as u32),
221          self_array[4].wrapping_shr(rhs_array[4] as u32),
222          self_array[5].wrapping_shr(rhs_array[5] as u32),
223          self_array[6].wrapping_shr(rhs_array[6] as u32),
224          self_array[7].wrapping_shr(rhs_array[7] as u32),
225        ])
226      }
227    }
228  }
229
230  #[inline]
231  fn shr(self, rhs: u32) -> Self::Output {
232    pick! {
233      if #[cfg(target_feature="sse2")] {
234        // Use `rhs % 16` to perform wrapping shift and not unbounded shift.
235        #[expect(clippy::suspicious_arithmetic_impl)]
236        let shift = cast([rhs as u64 & 15, 0]);
237        Self { sse: shr_all_i16_m128i(self.sse, shift) }
238      } else if #[cfg(target_feature="simd128")] {
239        Self { simd: i16x8_shr(self.simd, rhs) }
240      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
241        // Use `rhs % 16` to perform wrapping shift and not unbounded shift.
242        #[expect(clippy::suspicious_arithmetic_impl)]
243        unsafe {Self { neon: vshlq_s16(self.neon, vmovq_n_s16( -(rhs as i16 & 15))) }}
244      } else {
245        Self { arr: [
246          self.arr[0].wrapping_shr(rhs),
247          self.arr[1].wrapping_shr(rhs),
248          self.arr[2].wrapping_shr(rhs),
249          self.arr[3].wrapping_shr(rhs),
250          self.arr[4].wrapping_shr(rhs),
251          self.arr[5].wrapping_shr(rhs),
252          self.arr[6].wrapping_shr(rhs),
253          self.arr[7].wrapping_shr(rhs),
254        ]}
255      }
256    }
257  }
258
259  #[inline]
260  pub fn max(self, rhs: Self) -> Self {
261    pick! {
262      if #[cfg(target_feature="sse2")] {
263        Self { sse: max_i16_m128i(self.sse, rhs.sse) }
264      } else if #[cfg(target_feature="simd128")] {
265        Self { simd: i16x8_max(self.simd, rhs.simd) }
266      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
267        unsafe {Self { neon: vmaxq_s16(self.neon, rhs.neon) }}
268      } else {
269        self.simd_lt(rhs).select(rhs, self)
270      }
271    }
272  }
273
274  #[inline]
275  pub fn min(self, rhs: Self) -> Self {
276    pick! {
277      if #[cfg(target_feature="sse2")] {
278        Self { sse: min_i16_m128i(self.sse, rhs.sse) }
279      } else if #[cfg(target_feature="simd128")] {
280        Self { simd: i16x8_min(self.simd, rhs.simd) }
281      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
282        unsafe {Self { neon: vminq_s16(self.neon, rhs.neon) }}
283      } else {
284        self.simd_lt(rhs).select(self, rhs)
285      }
286    }
287  }
288
289  #[inline]
290  pub fn reduce_max(self) -> i16 {
291    pick! {
292        if #[cfg(target_feature="sse2")] {
293          let hi64 = shuffle_ai_f32_all_m128i::<0b01_00_11_10>(self.sse);
294          let sum64 = max_i16_m128i(self.sse, hi64);
295          let hi32 = shuffle_ai_f32_all_m128i::<0b11_10_00_01>(sum64);
296          let sum32 = max_i16_m128i(sum64, hi32);
297          let lo16 = shr_imm_u32_m128i::<16>(sum32);
298          let sum16 = max_i16_m128i(sum32, lo16);
299          extract_i16_as_i32_m128i::<0>(sum16) as i16
300        } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
301          unsafe { vmaxvq_s16(self.neon) }
302        } else {
303        let arr: [i16; 8] = cast(self);
304
305        // most boring implementation possible so optimizer doesn't overthink this
306        let mut r = arr[0];
307        r = r.max(arr[1]);
308        r = r.max(arr[2]);
309        r = r.max(arr[3]);
310        r = r.max(arr[4]);
311        r = r.max(arr[5]);
312        r = r.max(arr[6]);
313        r.max(arr[7])
314      }
315    }
316  }
317
318  #[inline]
319  pub fn reduce_min(self) -> i16 {
320    pick! {
321        if #[cfg(target_feature="sse2")] {
322          let hi64 = shuffle_ai_f32_all_m128i::<0b01_00_11_10>(self.sse);
323          let sum64 = min_i16_m128i(self.sse, hi64);
324          let hi32 = shuffle_ai_f32_all_m128i::<0b11_10_00_01>(sum64);
325          let sum32 = min_i16_m128i(sum64, hi32);
326          let lo16 = shr_imm_u32_m128i::<16>(sum32);
327          let sum16 = min_i16_m128i(sum32, lo16);
328          extract_i16_as_i32_m128i::<0>(sum16) as i16
329        } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
330          unsafe { vminvq_s16(self.neon) }
331        } else {
332        let arr: [i16; 8] = cast(self);
333
334        // most boring implementation possible so optimizer doesn't overthink this
335        let mut r = arr[0];
336        r = r.min(arr[1]);
337        r = r.min(arr[2]);
338        r = r.min(arr[3]);
339        r = r.min(arr[4]);
340        r = r.min(arr[5]);
341        r = r.min(arr[6]);
342        r.min(arr[7])
343      }
344    }
345  }
346
347  #[inline]
348  pub fn unbounded_shr(self, rhs: u16x8) -> Self {
349    pick! {
350      if #[cfg(all(target_feature="avx512bw", target_feature="avx512vl"))] {
351        #[cfg(target_arch = "x86")]
352        use core::arch::x86::_mm_srav_epi16;
353        #[cfg(target_arch = "x86_64")]
354        use core::arch::x86_64::_mm_srav_epi16;
355
356        // TODO(safe_arch): Add `_mm_srav_epi16`.
357        cast(unsafe { _mm_srav_epi16(self.sse.0, rhs.sse.0) })
358      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))] {
359        unsafe {
360          // Negate `rhs` because there is no direct shift-right intrinsic, and
361          // restrict it to prevent overflow.
362          Self { neon: vshlq_s16(self.neon, vnegq_s16(vreinterpretq_s16_u16(rhs.min(u16x8::splat(16)).neon))) }
363        }
364      } else {
365        let self_array = self.to_array();
366        let rhs_array = rhs.to_array();
367
368        Self::new([
369          self_array[0].unbounded_shr(rhs_array[0] as u32),
370          self_array[1].unbounded_shr(rhs_array[1] as u32),
371          self_array[2].unbounded_shr(rhs_array[2] as u32),
372          self_array[3].unbounded_shr(rhs_array[3] as u32),
373          self_array[4].unbounded_shr(rhs_array[4] as u32),
374          self_array[5].unbounded_shr(rhs_array[5] as u32),
375          self_array[6].unbounded_shr(rhs_array[6] as u32),
376          self_array[7].unbounded_shr(rhs_array[7] as u32),
377        ])
378      }
379    }
380  }
381
382  #[inline]
383  pub fn unbounded_shr_scalar(self, rhs: u32) -> Self {
384    pick! {
385      if #[cfg(target_feature="sse2")] {
386        Self { sse: shr_all_i16_m128i(self.sse, cast([rhs as u64, 0])) }
387      } else if #[cfg(target_feature="simd128")] {
388        if rhs < 16 { Self { simd: i16x8_shr(self.simd, rhs) } } else { self.is_negative() }
389      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
390        unsafe {
391          // Negate `rhs` because there is no direct shift-right intrinsic, and
392          // restrict it to prevent overflow.
393          Self { neon: vshlq_s16(self.neon, vmovq_n_s16(-rhs.min(16).cast_signed() as i16)) }
394        }
395      } else {
396        Self {
397          arr: [
398            self.arr[0].unbounded_shr(rhs),
399            self.arr[1].unbounded_shr(rhs),
400            self.arr[2].unbounded_shr(rhs),
401            self.arr[3].unbounded_shr(rhs),
402            self.arr[4].unbounded_shr(rhs),
403            self.arr[5].unbounded_shr(rhs),
404            self.arr[6].unbounded_shr(rhs),
405            self.arr[7].unbounded_shr(rhs),
406          ]
407        }
408      }
409    }
410  }
411
412  #[inline]
413  pub fn saturating_add(self, rhs: Self) -> Self {
414    pick! {
415      if #[cfg(target_feature="sse2")] {
416        Self { sse: add_saturating_i16_m128i(self.sse, rhs.sse) }
417      } else if #[cfg(target_feature="simd128")] {
418        Self { simd: i16x8_add_sat(self.simd, rhs.simd) }
419      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
420        unsafe {Self { neon: vqaddq_s16(self.neon, rhs.neon) }}
421      } else {
422        Self { arr: [
423          self.arr[0].saturating_add(rhs.arr[0]),
424          self.arr[1].saturating_add(rhs.arr[1]),
425          self.arr[2].saturating_add(rhs.arr[2]),
426          self.arr[3].saturating_add(rhs.arr[3]),
427          self.arr[4].saturating_add(rhs.arr[4]),
428          self.arr[5].saturating_add(rhs.arr[5]),
429          self.arr[6].saturating_add(rhs.arr[6]),
430          self.arr[7].saturating_add(rhs.arr[7]),
431        ]}
432      }
433    }
434  }
435
436  #[inline]
437  pub fn saturating_sub(self, rhs: Self) -> Self {
438    pick! {
439      if #[cfg(target_feature="sse2")] {
440        Self { sse: sub_saturating_i16_m128i(self.sse, rhs.sse) }
441      } else if #[cfg(target_feature="simd128")] {
442        Self { simd: i16x8_sub_sat(self.simd, rhs.simd) }
443      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
444        unsafe { Self { neon: vqsubq_s16(self.neon, rhs.neon) } }
445      } else {
446        Self { arr: [
447          self.arr[0].saturating_sub(rhs.arr[0]),
448          self.arr[1].saturating_sub(rhs.arr[1]),
449          self.arr[2].saturating_sub(rhs.arr[2]),
450          self.arr[3].saturating_sub(rhs.arr[3]),
451          self.arr[4].saturating_sub(rhs.arr[4]),
452          self.arr[5].saturating_sub(rhs.arr[5]),
453          self.arr[6].saturating_sub(rhs.arr[6]),
454          self.arr[7].saturating_sub(rhs.arr[7]),
455        ]}
456      }
457    }
458  }
459
460  #[inline]
461  pub fn overflowing_mul(self, rhs: Self) -> (Self, Self) {
462    let (low, high) = self.mul_keep_low_high(rhs);
463    let low = cast::<u16x8, i16x8>(low);
464
465    let overflow = high.simd_ne(low.is_negative());
466    (low, overflow)
467  }
468
469  optional_fn_widening_mul {
470    #[inline]
471    pub fn widening_mul(self, rhs: Self) -> i32x8 {
472      pick! {
473        if #[cfg(target_feature="avx2")] {
474          let a = convert_to_i32_m256i_from_i16_m128i(self.sse);
475          let b = convert_to_i32_m256i_from_i16_m128i(rhs.sse);
476          i32x8 { avx2: mul_i32_keep_low_m256i(a,b) }
477        } else if #[cfg(target_feature="sse2")] {
478          let low = mul_i16_keep_low_m128i(self.sse, rhs.sse);
479          let high = mul_i16_keep_high_m128i(self.sse, rhs.sse);
480          i32x8 {
481            a: i32x4 { sse:unpack_low_i16_m128i(low, high) },
482            b: i32x4 { sse:unpack_high_i16_m128i(low, high) }
483          }
484        } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))] {
485          let lhs_low = unsafe { vget_low_s16(self.neon) };
486          let rhs_low = unsafe { vget_low_s16(rhs.neon) };
487
488          let lhs_high = unsafe { vget_high_s16(self.neon) };
489          let rhs_high = unsafe { vget_high_s16(rhs.neon) };
490
491          let low = unsafe { vmull_s16(lhs_low, rhs_low) };
492          let high = unsafe { vmull_s16(lhs_high, rhs_high) };
493
494          i32x8 { a: i32x4 { neon: low }, b: i32x4 {neon: high } }
495        } else {
496          let a = self.as_array();
497          let b = rhs.as_array();
498
499          i32x8::new([
500            i32::from(a[0]) * i32::from(b[0]),
501            i32::from(a[1]) * i32::from(b[1]),
502            i32::from(a[2]) * i32::from(b[2]),
503            i32::from(a[3]) * i32::from(b[3]),
504            i32::from(a[4]) * i32::from(b[4]),
505            i32::from(a[5]) * i32::from(b[5]),
506            i32::from(a[6]) * i32::from(b[6]),
507            i32::from(a[7]) * i32::from(b[7]),
508          ])
509        }
510      }
511    }
512  }
513
514  #[inline]
515  pub fn mul_keep_low_high(self, rhs: Self) -> (u16x8, i16x8) {
516    pick! {
517      if #[cfg(target_feature="simd128")] {
518        let low_wide_mul = i32x4_extmul_low_i16x8(self.simd, rhs.simd);
519        let high_wide_mul = i32x4_extmul_high_i16x8(self.simd, rhs.simd);
520        (
521          u16x8 { simd: i16x8_shuffle::<0, 2, 4, 6, 8, 10, 12, 14>(low_wide_mul, high_wide_mul) },
522          i16x8 { simd: i16x8_shuffle::<1, 3, 5, 7, 9, 11, 13, 15>(low_wide_mul, high_wide_mul) },
523        )
524      } else if #[cfg(all(target_feature="neon", target_arch="aarch64"))] {
525        unsafe {
526          let low_wide_mul = vreinterpretq_s16_s32(
527            vmull_s16(vget_low_s16(self.neon), vget_low_s16(rhs.neon)),
528          );
529          let high_wide_mul = vreinterpretq_s16_s32(
530            vmull_s16(vget_high_s16(self.neon), vget_high_s16(rhs.neon)),
531          );
532          let low_high = vuzpq_s16(low_wide_mul, high_wide_mul);
533          (
534            u16x8 { neon: vreinterpretq_u16_s16(low_high.0) },
535            i16x8 { neon: low_high.1 },
536          )
537        }
538      } else {
539        // TODO(perf): This implementation looks quite bad. Is there a better
540        // one?
541
542        let self_array = self.to_array();
543        let rhs_array = rhs.to_array();
544
545        let widening_mul = [
546          (self_array[0] as i32).wrapping_mul(rhs_array[0] as i32),
547          (self_array[1] as i32).wrapping_mul(rhs_array[1] as i32),
548          (self_array[2] as i32).wrapping_mul(rhs_array[2] as i32),
549          (self_array[3] as i32).wrapping_mul(rhs_array[3] as i32),
550          (self_array[4] as i32).wrapping_mul(rhs_array[4] as i32),
551          (self_array[5] as i32).wrapping_mul(rhs_array[5] as i32),
552          (self_array[6] as i32).wrapping_mul(rhs_array[6] as i32),
553          (self_array[7] as i32).wrapping_mul(rhs_array[7] as i32),
554        ];
555
556        (
557          u16x8::new([
558            widening_mul[0] as u16,
559            widening_mul[1] as u16,
560            widening_mul[2] as u16,
561            widening_mul[3] as u16,
562            widening_mul[4] as u16,
563            widening_mul[5] as u16,
564            widening_mul[6] as u16,
565            widening_mul[7] as u16,
566          ]),
567          i16x8::new([
568            (widening_mul[0] >> 16) as i16,
569            (widening_mul[1] >> 16) as i16,
570            (widening_mul[2] >> 16) as i16,
571            (widening_mul[3] >> 16) as i16,
572            (widening_mul[4] >> 16) as i16,
573            (widening_mul[5] >> 16) as i16,
574            (widening_mul[6] >> 16) as i16,
575            (widening_mul[7] >> 16) as i16,
576          ]),
577        )
578      }
579    }
580  }
581
582  #[inline]
583  pub fn mul_keep_high(self, rhs: Self) -> Self {
584    pick! {
585      if #[cfg(target_feature="sse2")] {
586        Self { sse: mul_i16_keep_high_m128i(self.sse, rhs.sse) }
587      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))] {
588        let lhs_low = unsafe { vget_low_s16(self.neon) };
589        let rhs_low = unsafe { vget_low_s16(rhs.neon) };
590
591        let lhs_high = unsafe { vget_high_s16(self.neon) };
592        let rhs_high = unsafe { vget_high_s16(rhs.neon) };
593
594        let low = unsafe { vmull_s16(lhs_low, rhs_low) };
595        let high = unsafe { vmull_s16(lhs_high, rhs_high) };
596
597        Self {
598          neon: unsafe {
599            vreinterpretq_s16_u16(
600              vuzpq_u16(vreinterpretq_u16_s32(low),
601              vreinterpretq_u16_s32(high)).1
602            )
603          }
604        }
605      } else if #[cfg(target_feature="simd128")] {
606        let low =  i32x4_extmul_low_i16x8(self.simd, rhs.simd);
607        let high = i32x4_extmul_high_i16x8(self.simd, rhs.simd);
608
609        Self { simd: i16x8_shuffle::<1, 3, 5, 7, 9, 11, 13, 15>(low, high) }
610      } else {
611        i16x8::new([
612          ((i32::from(rhs.as_array()[0]) * i32::from(self.as_array()[0])) >> 16) as i16,
613          ((i32::from(rhs.as_array()[1]) * i32::from(self.as_array()[1])) >> 16) as i16,
614          ((i32::from(rhs.as_array()[2]) * i32::from(self.as_array()[2])) >> 16) as i16,
615          ((i32::from(rhs.as_array()[3]) * i32::from(self.as_array()[3])) >> 16) as i16,
616          ((i32::from(rhs.as_array()[4]) * i32::from(self.as_array()[4])) >> 16) as i16,
617          ((i32::from(rhs.as_array()[5]) * i32::from(self.as_array()[5])) >> 16) as i16,
618          ((i32::from(rhs.as_array()[6]) * i32::from(self.as_array()[6])) >> 16) as i16,
619          ((i32::from(rhs.as_array()[7]) * i32::from(self.as_array()[7])) >> 16) as i16,
620        ])
621      }
622    }
623  }
624
625  #[inline]
626  pub fn abs(self) -> Self {
627    pick! {
628      if #[cfg(target_feature="sse2")] {
629        let mask = shr_imm_i16_m128i::<15>(self.sse);
630        Self { sse: bitxor_m128i(add_i16_m128i(self.sse, mask), mask) }
631      } else if #[cfg(target_feature="ssse3")] {
632        Self { sse: abs_i16_m128i(self.sse) }
633      } else if #[cfg(target_feature="simd128")] {
634        Self { simd: i16x8_abs(self.simd) }
635      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
636        unsafe {Self { neon: vabsq_s16(self.neon) }}
637      } else {
638        let arr: [i16; 8] = cast(self);
639        cast(
640          [
641            arr[0].wrapping_abs(),
642            arr[1].wrapping_abs(),
643            arr[2].wrapping_abs(),
644            arr[3].wrapping_abs(),
645            arr[4].wrapping_abs(),
646            arr[5].wrapping_abs(),
647            arr[6].wrapping_abs(),
648            arr[7].wrapping_abs(),
649          ])
650      }
651    }
652  }
653
654  #[inline]
655  pub fn is_positive(self) -> Self {
656    pick! {
657      if #[cfg(all(target_feature="neon", target_arch="aarch64"))] {
658        Self { neon: unsafe { vreinterpretq_s16_u16(vcgtzq_s16(self.neon)) } }
659      } else {
660        self.simd_gt(Self::ZERO)
661      }
662    }
663  }
664
665  #[inline]
666  pub fn is_negative(self) -> Self {
667    pick! {
668      if #[cfg(all(target_feature="neon", target_arch="aarch64"))] {
669        Self { neon: unsafe { vreinterpretq_s16_u16(vcltzq_s16(self.neon)) } }
670      } else {
671        self.simd_lt(Self::ZERO)
672      }
673    }
674  }
675
676  optional_fn_deserialize {}
677}
678
679/// The following functionality exists only for [`i16x8`], or only for
680/// particular types inconsistently.
681impl i16x8 {
682  /// Converts the lower eight elements of `u` from [`u8`] to [`i16`], dropping
683  /// the higher eight elements.
684  #[inline]
685  #[must_use]
686  pub fn from_u8x16_low(u: u8x16) -> Self {
687    pick! {
688      if #[cfg(target_feature="sse2")] {
689        Self{ sse: unpack_low_i8_m128i(u.sse, m128i::zeroed()) }
690      } else {
691        let u_arr: [u8; 16] = cast(u);
692        cast([
693          u_arr[0] as u16 as i16,
694          u_arr[1] as u16 as i16,
695          u_arr[2] as u16 as i16,
696          u_arr[3] as u16 as i16,
697          u_arr[4] as u16 as i16,
698          u_arr[5] as u16 as i16,
699          u_arr[6] as u16 as i16,
700          u_arr[7] as u16 as i16,
701        ])
702      }
703    }
704  }
705
706  /// Converts the higher eight elements of `u` from [`u8`] to [`i16`], dropping
707  /// the lower eight elements.
708  #[inline]
709  #[must_use]
710  pub fn from_u8x16_high(u: u8x16) -> Self {
711    pick! {
712      if #[cfg(target_feature="sse2")] {
713        Self{ sse: unpack_high_i8_m128i(u.sse, m128i::zeroed()) }
714      } else {
715        let u_arr: [u8; 16] = cast(u);
716        cast([
717          u_arr[8] as u16 as i16,
718          u_arr[9] as u16 as i16,
719          u_arr[10] as u16 as i16,
720          u_arr[11] as u16 as i16,
721          u_arr[12] as u16 as i16,
722          u_arr[13] as u16 as i16,
723          u_arr[14] as u16 as i16,
724          u_arr[15] as u16 as i16,
725        ])
726      }
727    }
728  }
729
730  /// Converts each element from [`i32`] to [`i16`], saturating out of range
731  /// values.
732  #[inline]
733  #[must_use]
734  pub fn from_i32x8_saturate(v: i32x8) -> Self {
735    pick! {
736      if #[cfg(target_feature="avx2")] {
737        i16x8 { sse: pack_i32_to_i16_m128i( extract_m128i_from_m256i::<0>(v.avx2), extract_m128i_from_m256i::<1>(v.avx2))  }
738      } else if #[cfg(target_feature="sse2")] {
739        i16x8 { sse: pack_i32_to_i16_m128i( v.a.sse, v.b.sse ) }
740      } else if #[cfg(target_feature="simd128")] {
741        use core::arch::wasm32::*;
742
743        i16x8 { simd: i16x8_narrow_i32x4(v.a.simd, v.b.simd) }
744      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))] {
745        use core::arch::aarch64::*;
746
747        unsafe {
748          i16x8 { neon: vcombine_s16(vqmovn_s32(v.a.neon), vqmovn_s32(v.b.neon)) }
749        }
750      } else {
751        fn clamp(a : i32) -> i16 {
752            if a < i16::MIN as i32 {
753                i16::MIN
754            }
755            else if a > i16::MAX as i32 {
756                i16::MAX
757            } else {
758                a as i16
759            }
760        }
761
762        i16x8::new([
763          clamp(v.as_array()[0]),
764          clamp(v.as_array()[1]),
765          clamp(v.as_array()[2]),
766          clamp(v.as_array()[3]),
767          clamp(v.as_array()[4]),
768          clamp(v.as_array()[5]),
769          clamp(v.as_array()[6]),
770          clamp(v.as_array()[7]),
771        ])
772      }
773    }
774  }
775
776  /// Converts each element from [`i32`] to [`i16`], truncating out of range
777  /// values (behaves like [`as`] casting).
778  ///
779  /// [`as`]: https://doc.rust-lang.org/stable/reference/expressions/operator-expr.html#r-expr.as.numeric
780  #[inline]
781  #[must_use]
782  pub fn from_i32x8_truncate(v: i32x8) -> Self {
783    pick! {
784      if #[cfg(target_feature="avx2")] {
785        let a = v.avx2.bitand(set_splat_i32_m256i(0xffff));
786        i16x8 { sse: pack_i32_to_u16_m128i( extract_m128i_from_m256i::<0>(a), extract_m128i_from_m256i::<1>(a) ) }
787      } else if #[cfg(target_feature="sse2")] {
788        let a = shr_imm_i32_m128i::<16>(shl_imm_u32_m128i::<16>(v.a.sse));
789        let b = shr_imm_i32_m128i::<16>(shl_imm_u32_m128i::<16>(v.b.sse));
790
791        i16x8 { sse: pack_i32_to_i16_m128i( a, b)  }
792      } else {
793      i16x8::new([
794        v.as_array()[0] as i16,
795        v.as_array()[1] as i16,
796        v.as_array()[2] as i16,
797        v.as_array()[3] as i16,
798        v.as_array()[4] as i16,
799        v.as_array()[5] as i16,
800        v.as_array()[6] as i16,
801        v.as_array()[7] as i16,
802      ])
803      }
804    }
805  }
806
807  /// Converts a slice to a SIMD vector, ignoring elements beyond the first 8.
808  ///
809  /// # Panics
810  ///
811  /// Panics if `input` has less than 8 elements.
812  #[inline]
813  #[must_use]
814  pub fn from_slice_unaligned(input: &[i16]) -> Self {
815    assert!(input.len() >= 8);
816
817    pick! {
818      if #[cfg(target_feature="sse2")] {
819        unsafe { Self { sse: load_unaligned_m128i( &*(input.as_ptr() as * const [u8;16]) ) } }
820      } else if #[cfg(target_feature="simd128")] {
821        unsafe { Self { simd: v128_load(input.as_ptr() as *const v128 ) } }
822      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
823        unsafe { Self { neon: vld1q_s16( input.as_ptr() as *const i16 ) } }
824      } else {
825        // 2018 edition doesn't have try_into
826        unsafe { Self::new( *(input.as_ptr() as * const [i16;8]) ) }
827      }
828    }
829  }
830
831  /// Partially computes the dot product.
832  ///
833  /// First this multiplies the input 16-bit integers, producing intermediate
834  /// 32-bit integers. Then this horizontally adds adjacent pairs, resulting in
835  /// four 32-bit integers.
836  #[inline]
837  #[must_use]
838  pub fn dot(self, rhs: Self) -> i32x4 {
839    pick! {
840      if #[cfg(target_feature="sse2")] {
841        i32x4 { sse:  mul_i16_horizontal_add_m128i(self.sse, rhs.sse) }
842      } else if #[cfg(target_feature="simd128")] {
843        i32x4 { simd: i32x4_dot_i16x8(self.simd, rhs.simd) }
844      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
845        unsafe {
846          let pl = vmull_s16(vget_low_s16(self.neon),  vget_low_s16(rhs.neon));
847          let ph = vmull_high_s16(self.neon, rhs.neon);
848          i32x4 { neon: vpaddq_s32(pl, ph) }
849        }
850      } else {
851        i32x4 { arr: [
852          (i32::from(self.arr[0]) * i32::from(rhs.arr[0])) + (i32::from(self.arr[1]) * i32::from(rhs.arr[1])),
853          (i32::from(self.arr[2]) * i32::from(rhs.arr[2])) + (i32::from(self.arr[3]) * i32::from(rhs.arr[3])),
854          (i32::from(self.arr[4]) * i32::from(rhs.arr[4])) + (i32::from(self.arr[5]) * i32::from(rhs.arr[5])),
855          (i32::from(self.arr[6]) * i32::from(rhs.arr[6])) + (i32::from(self.arr[7]) * i32::from(rhs.arr[7])),
856        ] }
857      }
858    }
859  }
860
861  /// Multiply and scale equivalent to `((self * rhs) + 0x4000) >> 15` on each
862  /// lane, effectively multiplying by a 16 bit fixed point number between `-1`
863  /// and `1`. This corresponds to the following instructions:
864  /// - `vqrdmulhq_s16` instruction on neon
865  /// - `i16x8_q15mulr_sat` on simd128
866  /// - `_mm_mulhrs_epi16` on ssse3
867  /// - emulated via `mul_i16_*` on sse2
868  #[inline]
869  #[must_use]
870  pub fn mul_scale_round(self, rhs: Self) -> Self {
871    pick! {
872      if #[cfg(target_feature="ssse3")] {
873        Self { sse:  mul_i16_scale_round_m128i(self.sse, rhs.sse) }
874      } else if #[cfg(target_feature="sse2")] {
875        // unfortunately mul_i16_scale_round_m128i only got added in sse3
876        let hi = mul_i16_keep_high_m128i(self.sse, rhs.sse);
877        let lo = mul_i16_keep_low_m128i(self.sse, rhs.sse);
878        let mut v1 = unpack_low_i16_m128i(lo, hi);
879        let mut v2 = unpack_high_i16_m128i(lo, hi);
880        let a = set_splat_i32_m128i(0x4000);
881        v1 = shr_imm_i32_m128i::<15>(add_i32_m128i(v1, a));
882        v2 = shr_imm_i32_m128i::<15>(add_i32_m128i(v2, a));
883        let s = pack_i32_to_i16_m128i(v1, v2);
884        Self { sse: s }
885      } else if #[cfg(target_feature="simd128")] {
886        Self { simd: i16x8_q15mulr_sat(self.simd, rhs.simd) }
887      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
888        unsafe { Self { neon: vqrdmulhq_s16(self.neon, rhs.neon) } }
889      } else {
890        // compiler does a surprisingly good job of vectorizing this
891        Self { arr: [
892          ((i32::from(self.arr[0]) * i32::from(rhs.arr[0]) + 0x4000) >> 15) as i16,
893          ((i32::from(self.arr[1]) * i32::from(rhs.arr[1]) + 0x4000) >> 15) as i16,
894          ((i32::from(self.arr[2]) * i32::from(rhs.arr[2]) + 0x4000) >> 15) as i16,
895          ((i32::from(self.arr[3]) * i32::from(rhs.arr[3]) + 0x4000) >> 15) as i16,
896          ((i32::from(self.arr[4]) * i32::from(rhs.arr[4]) + 0x4000) >> 15) as i16,
897          ((i32::from(self.arr[5]) * i32::from(rhs.arr[5]) + 0x4000) >> 15) as i16,
898          ((i32::from(self.arr[6]) * i32::from(rhs.arr[6]) + 0x4000) >> 15) as i16,
899          ((i32::from(self.arr[7]) * i32::from(rhs.arr[7]) + 0x4000) >> 15) as i16,
900        ]}
901      }
902    }
903  }
904
905  #[inline]
906  #[must_use]
907  /// Multiply and scale, equivalent to `((self * rhs) + 0x4000) >> 15` on each
908  /// lane, effectively multiplying by a 16 bit fixed point number between `-1`
909  /// and `1`. This corresponds to the following instructions:
910  /// - `vqrdmulhq_n_s16` instruction on neon
911  /// - `i16x8_q15mulr_sat` on simd128
912  /// - `_mm_mulhrs_epi16` on ssse3
913  /// - emulated via `mul_i16_*` on sse2
914  pub fn mul_scale_round_n(self, rhs: i16) -> Self {
915    pick! {
916      if #[cfg(target_feature="ssse3")] {
917        Self { sse:  mul_i16_scale_round_m128i(self.sse, set_splat_i16_m128i(rhs)) }
918      } else if #[cfg(target_feature="sse2")] {
919        // unfortunately mul_i16_scale_round_m128i only got added in sse3
920        let r = set_splat_i16_m128i(rhs);
921        let hi = mul_i16_keep_high_m128i(self.sse, r);
922        let lo = mul_i16_keep_low_m128i(self.sse, r);
923        let mut v1 = unpack_low_i16_m128i(lo, hi);
924        let mut v2 = unpack_high_i16_m128i(lo, hi);
925        let a = set_splat_i32_m128i(0x4000);
926        v1 = shr_imm_i32_m128i::<15>(add_i32_m128i(v1, a));
927        v2 = shr_imm_i32_m128i::<15>(add_i32_m128i(v2, a));
928        let s = pack_i32_to_i16_m128i(v1, v2);
929        Self { sse: s }
930      } else if #[cfg(target_feature="simd128")] {
931        Self { simd: i16x8_q15mulr_sat(self.simd, i16x8_splat(rhs)) }
932      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
933        unsafe { Self { neon: vqrdmulhq_n_s16(self.neon, rhs) } }
934      } else {
935        // compiler does a surprisingly good job of vectorizing this
936        Self { arr: [
937          ((i32::from(self.arr[0]) * i32::from(rhs) + 0x4000) >> 15) as i16,
938          ((i32::from(self.arr[1]) * i32::from(rhs) + 0x4000) >> 15) as i16,
939          ((i32::from(self.arr[2]) * i32::from(rhs) + 0x4000) >> 15) as i16,
940          ((i32::from(self.arr[3]) * i32::from(rhs) + 0x4000) >> 15) as i16,
941          ((i32::from(self.arr[4]) * i32::from(rhs) + 0x4000) >> 15) as i16,
942          ((i32::from(self.arr[5]) * i32::from(rhs) + 0x4000) >> 15) as i16,
943          ((i32::from(self.arr[6]) * i32::from(rhs) + 0x4000) >> 15) as i16,
944          ((i32::from(self.arr[7]) * i32::from(rhs) + 0x4000) >> 15) as i16,
945        ]}
946      }
947    }
948  }
949
950  /// Widening multiplication. Computes `self * rhs`, widening to a SIMD
951  /// vector of larger integers.
952  ///
953  /// The returned value is always exact and can never overflow.
954  ///
955  /// This function has been renamed to [`widening_mul`].
956  ///
957  /// [`widening_mul`]: Self::widening_mul
958  #[inline]
959  #[must_use]
960  #[deprecated(since = "1.6.0", note = "renamed to `widening_mul`")]
961  pub fn mul_widen(self, rhs: Self) -> i32x8 {
962    self.widening_mul(rhs)
963  }
964}
wide/i16x8_.rs

wide/
i16x8_.rs
