wide/
i8x16_.rs

1use super::*;
2
3pick! {
4  if #[cfg(target_feature="sse2")] {
5    /// A SIMD vector with 16 elements of type [`i8`].
6    ///
7    /// See the [crate level documentation] for more information about SIMD
8    /// vectors.
9    ///
10    /// [crate level documentation]: crate
11    #[derive(Default, Clone, Copy, PartialEq, Eq)]
12    #[repr(C, align(16))]
13    pub struct i8x16 { pub(crate) sse: m128i }
14  } else if #[cfg(target_feature="simd128")] {
15    use core::arch::wasm32::*;
16
17    /// A SIMD vector with 16 elements of type [`i8`].
18    ///
19    /// See the [crate level documentation] for more information about SIMD
20    /// vectors.
21    ///
22    /// [crate level documentation]: crate
23    #[derive(Clone, Copy)]
24    #[repr(transparent)]
25    pub struct i8x16 { pub(crate) simd: v128 }
26
27    impl Default for i8x16 {
28      fn default() -> Self {
29        Self::splat(0)
30      }
31    }
32
33    impl PartialEq for i8x16 {
34      fn eq(&self, other: &Self) -> bool {
35        u8x16_all_true(i8x16_eq(self.simd, other.simd))
36      }
37    }
38
39    impl Eq for i8x16 { }
40  } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
41    use core::arch::aarch64::*;
42
43    /// A SIMD vector with 16 elements of type [`i8`].
44    ///
45    /// See the [crate level documentation] for more information about SIMD
46    /// vectors.
47    ///
48    /// [crate level documentation]: crate
49    #[repr(C)]
50    #[derive(Copy, Clone)]
51    pub struct i8x16 { pub(crate) neon : int8x16_t }
52
53    impl Default for i8x16 {
54      #[inline]
55      fn default() -> Self {
56        Self::splat(0)
57      }
58    }
59
60    impl PartialEq for i8x16 {
61      #[inline]
62      fn eq(&self, other: &Self) -> bool {
63        unsafe { vminvq_u8(vceqq_s8(self.neon, other.neon))==u8::MAX }
64      }
65    }
66
67    impl Eq for i8x16 { }
68  } else {
69    /// A SIMD vector with 16 elements of type [`i8`].
70    ///
71    /// See the [crate level documentation] for more information about SIMD
72    /// vectors.
73    ///
74    /// [crate level documentation]: crate
75    #[derive(Default, Clone, Copy, PartialEq, Eq)]
76    #[repr(C, align(16))]
77    pub struct i8x16 { arr: [i8;16] }
78  }
79}
80
81impl_simd_int! {
82  unsafe {
83    T = i8,
84    N = 16,
85    Simd = i8x16,
86    UintSimd = u8x16,
87    T_BITS = 8,
88    T_BITS_MUL_2 = 16,
89    BitmaskType = u32,
90    [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
91    optional_type_x86_inner { X86Inner = __m128i },
92    optional_type_arm_inner { ArmInner = int8x16_t },
93    optional_type_wasm_inner { WasmInner = v128 },
94  }
95
96  #[inline]
97  fn simd_lt(self, rhs: Self) -> Self::Output {
98    pick! {
99      if #[cfg(target_feature="sse2")] {
100        Self { sse: cmp_lt_mask_i8_m128i(self.sse, rhs.sse) }
101      } else if #[cfg(target_feature="simd128")] {
102        Self { simd: i8x16_lt(self.simd, rhs.simd) }
103      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
104        unsafe {Self { neon: vreinterpretq_s8_u8(vcltq_s8(self.neon, rhs.neon)) }}
105      } else {
106        Self { arr: [
107          if self.arr[0] < rhs.arr[0] { -1 } else { 0 },
108          if self.arr[1] < rhs.arr[1] { -1 } else { 0 },
109          if self.arr[2] < rhs.arr[2] { -1 } else { 0 },
110          if self.arr[3] < rhs.arr[3] { -1 } else { 0 },
111          if self.arr[4] < rhs.arr[4] { -1 } else { 0 },
112          if self.arr[5] < rhs.arr[5] { -1 } else { 0 },
113          if self.arr[6] < rhs.arr[6] { -1 } else { 0 },
114          if self.arr[7] < rhs.arr[7] { -1 } else { 0 },
115          if self.arr[8] < rhs.arr[8] { -1 } else { 0 },
116          if self.arr[9] < rhs.arr[9] { -1 } else { 0 },
117          if self.arr[10] < rhs.arr[10] { -1 } else { 0 },
118          if self.arr[11] < rhs.arr[11] { -1 } else { 0 },
119          if self.arr[12] < rhs.arr[12] { -1 } else { 0 },
120          if self.arr[13] < rhs.arr[13] { -1 } else { 0 },
121          if self.arr[14] < rhs.arr[14] { -1 } else { 0 },
122          if self.arr[15] < rhs.arr[15] { -1 } else { 0 },
123        ]}
124      }
125    }
126  }
127
128  #[inline]
129  fn simd_gt(self, rhs: Self) -> Self::Output {
130    pick! {
131      if #[cfg(target_feature="sse2")] {
132        Self { sse: cmp_gt_mask_i8_m128i(self.sse, rhs.sse) }
133      } else if #[cfg(target_feature="simd128")] {
134        Self { simd: i8x16_gt(self.simd, rhs.simd) }
135      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
136        unsafe {Self { neon: vreinterpretq_s8_u8(vcgtq_s8(self.neon, rhs.neon)) }}
137      } else {
138        Self { arr: [
139          if self.arr[0] > rhs.arr[0] { -1 } else { 0 },
140          if self.arr[1] > rhs.arr[1] { -1 } else { 0 },
141          if self.arr[2] > rhs.arr[2] { -1 } else { 0 },
142          if self.arr[3] > rhs.arr[3] { -1 } else { 0 },
143          if self.arr[4] > rhs.arr[4] { -1 } else { 0 },
144          if self.arr[5] > rhs.arr[5] { -1 } else { 0 },
145          if self.arr[6] > rhs.arr[6] { -1 } else { 0 },
146          if self.arr[7] > rhs.arr[7] { -1 } else { 0 },
147          if self.arr[8] > rhs.arr[8] { -1 } else { 0 },
148          if self.arr[9] > rhs.arr[9] { -1 } else { 0 },
149          if self.arr[10] > rhs.arr[10] { -1 } else { 0 },
150          if self.arr[11] > rhs.arr[11] { -1 } else { 0 },
151          if self.arr[12] > rhs.arr[12] { -1 } else { 0 },
152          if self.arr[13] > rhs.arr[13] { -1 } else { 0 },
153          if self.arr[14] > rhs.arr[14] { -1 } else { 0 },
154          if self.arr[15] > rhs.arr[15] { -1 } else { 0 },
155        ]}
156      }
157    }
158  }
159
160  #[inline]
161  fn simd_le(self, rhs: Self) -> Self::Output {
162    pick! {
163      if #[cfg(target_feature="sse2")] {
164        !self.simd_gt(rhs)
165      } else if #[cfg(target_feature="simd128")] {
166        Self { simd: i8x16_le(self.simd, rhs.simd) }
167      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
168        !self.simd_gt(rhs)
169      } else {
170        Self { arr: [
171          if self.arr[0] <= rhs.arr[0] { -1 } else { 0 },
172          if self.arr[1] <= rhs.arr[1] { -1 } else { 0 },
173          if self.arr[2] <= rhs.arr[2] { -1 } else { 0 },
174          if self.arr[3] <= rhs.arr[3] { -1 } else { 0 },
175          if self.arr[4] <= rhs.arr[4] { -1 } else { 0 },
176          if self.arr[5] <= rhs.arr[5] { -1 } else { 0 },
177          if self.arr[6] <= rhs.arr[6] { -1 } else { 0 },
178          if self.arr[7] <= rhs.arr[7] { -1 } else { 0 },
179          if self.arr[8] <= rhs.arr[8] { -1 } else { 0 },
180          if self.arr[9] <= rhs.arr[9] { -1 } else { 0 },
181          if self.arr[10] <= rhs.arr[10] { -1 } else { 0 },
182          if self.arr[11] <= rhs.arr[11] { -1 } else { 0 },
183          if self.arr[12] <= rhs.arr[12] { -1 } else { 0 },
184          if self.arr[13] <= rhs.arr[13] { -1 } else { 0 },
185          if self.arr[14] <= rhs.arr[14] { -1 } else { 0 },
186          if self.arr[15] <= rhs.arr[15] { -1 } else { 0 },
187        ]}
188      }
189    }
190  }
191
192  #[inline]
193  fn simd_ge(self, rhs: Self) -> Self::Output {
194    pick! {
195      if #[cfg(target_feature="sse2")] {
196        !self.simd_lt(rhs)
197      } else if #[cfg(target_feature="simd128")] {
198        Self { simd: i8x16_ge(self.simd, rhs.simd) }
199      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
200        !self.simd_lt(rhs)
201      } else {
202        Self { arr: [
203          if self.arr[0] >= rhs.arr[0] { -1 } else { 0 },
204          if self.arr[1] >= rhs.arr[1] { -1 } else { 0 },
205          if self.arr[2] >= rhs.arr[2] { -1 } else { 0 },
206          if self.arr[3] >= rhs.arr[3] { -1 } else { 0 },
207          if self.arr[4] >= rhs.arr[4] { -1 } else { 0 },
208          if self.arr[5] >= rhs.arr[5] { -1 } else { 0 },
209          if self.arr[6] >= rhs.arr[6] { -1 } else { 0 },
210          if self.arr[7] >= rhs.arr[7] { -1 } else { 0 },
211          if self.arr[8] >= rhs.arr[8] { -1 } else { 0 },
212          if self.arr[9] >= rhs.arr[9] { -1 } else { 0 },
213          if self.arr[10] >= rhs.arr[10] { -1 } else { 0 },
214          if self.arr[11] >= rhs.arr[11] { -1 } else { 0 },
215          if self.arr[12] >= rhs.arr[12] { -1 } else { 0 },
216          if self.arr[13] >= rhs.arr[13] { -1 } else { 0 },
217          if self.arr[14] >= rhs.arr[14] { -1 } else { 0 },
218          if self.arr[15] >= rhs.arr[15] { -1 } else { 0 },
219        ]}
220      }
221    }
222  }
223
224  #[inline]
225  fn shr(self, rhs: u8x16) -> Self::Output {
226    // For x86, this technically can be done explicitly by converting
227    // to `i16` or `i32` then converting back after multiplication, but that may
228    // not actually be faster than auto-vectorization.
229    pick! {
230      if #[cfg(all(target_feature="neon", target_arch="aarch64"))] {
231        unsafe {
232          // Mask `rhs` to 7 to match `wrapping_shr`, and negate it because
233          // there is no shift-right intrinsic.
234          let neg_rhs = vnegq_s8(vreinterpretq_s8_u8(vandq_u8(rhs.neon, vmovq_n_u8(7))));
235          Self { neon: vshlq_s8(self.neon, neg_rhs) }
236        }
237      } else {
238        let self_array: [i8; 16] = cast(self);
239        let rhs_array: [u8; 16] = cast(rhs);
240
241        Self::new([
242          self_array[0].wrapping_shr(rhs_array[0] as u32),
243          self_array[1].wrapping_shr(rhs_array[1] as u32),
244          self_array[2].wrapping_shr(rhs_array[2] as u32),
245          self_array[3].wrapping_shr(rhs_array[3] as u32),
246          self_array[4].wrapping_shr(rhs_array[4] as u32),
247          self_array[5].wrapping_shr(rhs_array[5] as u32),
248          self_array[6].wrapping_shr(rhs_array[6] as u32),
249          self_array[7].wrapping_shr(rhs_array[7] as u32),
250          self_array[8].wrapping_shr(rhs_array[8] as u32),
251          self_array[9].wrapping_shr(rhs_array[9] as u32),
252          self_array[10].wrapping_shr(rhs_array[10] as u32),
253          self_array[11].wrapping_shr(rhs_array[11] as u32),
254          self_array[12].wrapping_shr(rhs_array[12] as u32),
255          self_array[13].wrapping_shr(rhs_array[13] as u32),
256          self_array[14].wrapping_shr(rhs_array[14] as u32),
257          self_array[15].wrapping_shr(rhs_array[15] as u32),
258        ])
259      }
260    }
261  }
262
263  #[inline]
264  fn shr(self, rhs: u32) -> Self::Output {
265    // For x86, this technically can be done explicitly by converting
266    // to `i16` or `i32` then converting back after multiplication, but that
267    // may not actually be faster than auto-vectorization.
268    pick! {
269      if #[cfg(target_feature="simd128")] {
270        // Mask `rhs` to 7 to match `wrapping_shr`.
271        Self { simd: i8x16_shr(self.simd, rhs & 7) }
272      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
273        // Mask `rhs` to 7 to match `wrapping_shr`, and negate it because
274        // there is no shift-right intrinsic.
275        unsafe { Self { neon: vshlq_s8(self.neon, vmovq_n_s8(-(rhs as i8 & 7))) } }
276      } else {
277        let self_array = self.to_array();
278
279        cast([
280          self_array[0].wrapping_shr(rhs),
281          self_array[1].wrapping_shr(rhs),
282          self_array[2].wrapping_shr(rhs),
283          self_array[3].wrapping_shr(rhs),
284          self_array[4].wrapping_shr(rhs),
285          self_array[5].wrapping_shr(rhs),
286          self_array[6].wrapping_shr(rhs),
287          self_array[7].wrapping_shr(rhs),
288          self_array[8].wrapping_shr(rhs),
289          self_array[9].wrapping_shr(rhs),
290          self_array[10].wrapping_shr(rhs),
291          self_array[11].wrapping_shr(rhs),
292          self_array[12].wrapping_shr(rhs),
293          self_array[13].wrapping_shr(rhs),
294          self_array[14].wrapping_shr(rhs),
295          self_array[15].wrapping_shr(rhs),
296        ])
297      }
298    }
299  }
300
301  #[inline]
302  pub fn max(self, rhs: Self) -> Self {
303    pick! {
304      if #[cfg(target_feature="sse4.1")] {
305        Self { sse: max_i8_m128i(self.sse, rhs.sse) }
306      } else if #[cfg(target_feature="simd128")] {
307        Self { simd: i8x16_max(self.simd, rhs.simd) }
308      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
309        unsafe {Self { neon: vmaxq_s8(self.neon, rhs.neon) }}
310      } else {
311        self.simd_lt(rhs).select(rhs, self)
312      }
313    }
314  }
315
316  #[inline]
317  pub fn min(self, rhs: Self) -> Self {
318    pick! {
319      if #[cfg(target_feature="sse4.1")] {
320        Self { sse: min_i8_m128i(self.sse, rhs.sse) }
321      } else if #[cfg(target_feature="simd128")] {
322        Self { simd: i8x16_min(self.simd, rhs.simd) }
323      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
324        unsafe {Self { neon: vminq_s8(self.neon, rhs.neon) }}
325      } else {
326        self.simd_lt(rhs).select(self, rhs)
327      }
328    }
329  }
330
331  #[inline]
332  pub fn reduce_max(self) -> i8 {
333    #[allow(dead_code)]
334    const SHUFFLE_1: [i8; 16] =
335      [8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0];
336    #[allow(dead_code)]
337    const SHUFFLE_2: [i8; 16] =
338      [4, 5, 6, 7, 0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0];
339    #[allow(dead_code)]
340    const SHUFFLE_3: [i8; 16] =
341      [2, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
342    #[allow(dead_code)]
343    const SHUFFLE_4: [i8; 16] =
344      [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
345
346    pick! {
347      if #[cfg(all(target_feature="ssse3", target_feature="sse4.1"))] {
348        let rhs = shuffle_av_i8z_all_m128i(self.sse, m128i::from(SHUFFLE_1));
349        let max = max_i8_m128i(self.sse, rhs);
350        let rhs = shuffle_av_i8z_all_m128i(max, m128i::from(SHUFFLE_2));
351        let max = max_i8_m128i(max, rhs);
352        let rhs = shuffle_av_i8z_all_m128i(max, m128i::from(SHUFFLE_3));
353        let max = max_i8_m128i(max, rhs);
354        let rhs = shuffle_av_i8z_all_m128i(max, m128i::from(SHUFFLE_4));
355        let max = max_i8_m128i(max, rhs);
356        get_i32_from_m128i_s(max) as i8
357      } else if #[cfg(target_feature="simd128")] {
358        let rhs = i8x16_shuffle::<8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7>(self.simd, self.simd);
359        let max = i8x16_max(self.simd, rhs);
360        let rhs = i8x16_shuffle::<4, 5, 6, 7, 0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0>(max, max);
361        let max = i8x16_max(max, rhs);
362        let rhs = i8x16_shuffle::<2, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>(max, max);
363        let max = i8x16_max(max, rhs);
364        let rhs = i8x16_shuffle::<1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>(max, max);
365        let max = i8x16_max(max, rhs);
366        i8x16_extract_lane::<0>(max)
367      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
368        unsafe {
369          // Use `transmute` instead of `cast` because `int8x16_t` does not
370          // implement `bytemuck::Pod`.
371          let rhs = vqtbl1q_s8(self.neon, core::mem::transmute(SHUFFLE_1));
372          let max = vmaxq_s8(self.neon, rhs);
373          let rhs = vqtbl1q_s8(max, core::mem::transmute(SHUFFLE_2));
374          let max = vmaxq_s8(max, rhs);
375          let rhs = vqtbl1q_s8(max, core::mem::transmute(SHUFFLE_3));
376          let max = vmaxq_s8(max, rhs);
377          let rhs = vqtbl1q_s8(max, core::mem::transmute(SHUFFLE_4));
378          let max = vmaxq_s8(max, rhs);
379          vgetq_lane_s8(max, 0)
380        }
381      } else {
382        let array: [i8; 16] = cast(self);
383        array.into_iter().reduce(i8::max).unwrap()
384      }
385    }
386  }
387
388  #[inline]
389  pub fn reduce_min(self) -> i8 {
390    #[allow(dead_code)]
391    const SHUFFLE_1: [i8; 16] =
392      [8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0];
393    #[allow(dead_code)]
394    const SHUFFLE_2: [i8; 16] =
395      [4, 5, 6, 7, 0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0];
396    #[allow(dead_code)]
397    const SHUFFLE_3: [i8; 16] =
398      [2, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
399    #[allow(dead_code)]
400    const SHUFFLE_4: [i8; 16] =
401      [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
402
403    pick! {
404      if #[cfg(all(target_feature="ssse3", target_feature="sse4.1"))] {
405        let rhs = shuffle_av_i8z_all_m128i(self.sse, m128i::from(SHUFFLE_1));
406        let min = min_i8_m128i(self.sse, rhs);
407        let rhs = shuffle_av_i8z_all_m128i(min, m128i::from(SHUFFLE_2));
408        let min = min_i8_m128i(min, rhs);
409        let rhs = shuffle_av_i8z_all_m128i(min, m128i::from(SHUFFLE_3));
410        let min = min_i8_m128i(min, rhs);
411        let rhs = shuffle_av_i8z_all_m128i(min, m128i::from(SHUFFLE_4));
412        let min = min_i8_m128i(min, rhs);
413        get_i32_from_m128i_s(min) as i8
414      } else if #[cfg(target_feature="simd128")] {
415        let rhs = i8x16_shuffle::<8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7>(self.simd, self.simd);
416        let min = i8x16_min(self.simd, rhs);
417        let rhs = i8x16_shuffle::<4, 5, 6, 7, 0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0>(min, min);
418        let min = i8x16_min(min, rhs);
419        let rhs = i8x16_shuffle::<2, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>(min, min);
420        let min = i8x16_min(min, rhs);
421        let rhs = i8x16_shuffle::<1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>(min, min);
422        let min = i8x16_min(min, rhs);
423        i8x16_extract_lane::<0>(min)
424      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
425        unsafe {
426          // Use `transmute` instead of `cast` because `int8x16_t` does not
427          // implement `bytemuck::Pod`.
428          let rhs = vqtbl1q_s8(self.neon, core::mem::transmute(SHUFFLE_1));
429          let min = vminq_s8(self.neon, rhs);
430          let rhs = vqtbl1q_s8(min, core::mem::transmute(SHUFFLE_2));
431          let min = vminq_s8(min, rhs);
432          let rhs = vqtbl1q_s8(min, core::mem::transmute(SHUFFLE_3));
433          let min = vminq_s8(min, rhs);
434          let rhs = vqtbl1q_s8(min, core::mem::transmute(SHUFFLE_4));
435          let min = vminq_s8(min, rhs);
436          vgetq_lane_s8(min, 0)
437        }
438      } else {
439        let array: [i8; 16] = cast(self);
440        array.into_iter().reduce(i8::min).unwrap()
441      }
442    }
443  }
444
445  #[inline]
446  pub fn unbounded_shr(self, rhs: u8x16) -> Self {
447    // For x86, this technically can be done explicitly by converting
448    // to `i16` or `i32` then converting back after multiplication, but that may
449    // not actually be faster than auto-vectorization.
450    pick! {
451      if #[cfg(all(target_feature="neon", target_arch="aarch64"))] {
452        unsafe {
453          // Negate `rhs` because there is no direct shift-right intrinsic, and
454          // restrict it to prevent overflow.
455          let neg_rhs = vnegq_s8(vreinterpretq_s8_u8(rhs.min(u8x16::splat(8)).neon));
456          Self { neon: vshlq_s8(self.neon, neg_rhs) }
457        }
458      } else {
459        let self_array = self.to_array();
460        let rhs_array = rhs.to_array();
461
462        Self::new([
463          self_array[0].unbounded_shr(rhs_array[0] as u32),
464          self_array[1].unbounded_shr(rhs_array[1] as u32),
465          self_array[2].unbounded_shr(rhs_array[2] as u32),
466          self_array[3].unbounded_shr(rhs_array[3] as u32),
467          self_array[4].unbounded_shr(rhs_array[4] as u32),
468          self_array[5].unbounded_shr(rhs_array[5] as u32),
469          self_array[6].unbounded_shr(rhs_array[6] as u32),
470          self_array[7].unbounded_shr(rhs_array[7] as u32),
471          self_array[8].unbounded_shr(rhs_array[8] as u32),
472          self_array[9].unbounded_shr(rhs_array[9] as u32),
473          self_array[10].unbounded_shr(rhs_array[10] as u32),
474          self_array[11].unbounded_shr(rhs_array[11] as u32),
475          self_array[12].unbounded_shr(rhs_array[12] as u32),
476          self_array[13].unbounded_shr(rhs_array[13] as u32),
477          self_array[14].unbounded_shr(rhs_array[14] as u32),
478          self_array[15].unbounded_shr(rhs_array[15] as u32),
479        ])
480      }
481    }
482  }
483
484  #[inline]
485  pub fn unbounded_shr_scalar(self, rhs: u32) -> Self {
486    // For x86, this technically can be done explicitly by converting
487    // to `i16` or `i32` then converting back after multiplication, but that
488    // may not actually be faster than auto-vectorization.
489    pick! {
490      if #[cfg(target_feature="simd128")] {
491        if rhs < 8 { Self { simd: i8x16_shr(self.simd, rhs) } } else { self.is_negative() }
492      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
493        unsafe {
494          // Negate `rhs` because there is no direct shift-right intrinsic, and
495          // restrict it to prevent overflow.
496          Self { neon: vshlq_s8(self.neon, vmovq_n_s8(-rhs.min(8).cast_signed() as i8)) }
497        }
498      } else {
499        let self_array = self.to_array();
500
501        cast([
502          self_array[0].unbounded_shr(rhs),
503          self_array[1].unbounded_shr(rhs),
504          self_array[2].unbounded_shr(rhs),
505          self_array[3].unbounded_shr(rhs),
506          self_array[4].unbounded_shr(rhs),
507          self_array[5].unbounded_shr(rhs),
508          self_array[6].unbounded_shr(rhs),
509          self_array[7].unbounded_shr(rhs),
510          self_array[8].unbounded_shr(rhs),
511          self_array[9].unbounded_shr(rhs),
512          self_array[10].unbounded_shr(rhs),
513          self_array[11].unbounded_shr(rhs),
514          self_array[12].unbounded_shr(rhs),
515          self_array[13].unbounded_shr(rhs),
516          self_array[14].unbounded_shr(rhs),
517          self_array[15].unbounded_shr(rhs),
518        ])
519      }
520    }
521  }
522
523  #[inline]
524  pub fn saturating_add(self, rhs: Self) -> Self {
525    pick! {
526      if #[cfg(target_feature="sse2")] {
527        Self { sse: add_saturating_i8_m128i(self.sse, rhs.sse) }
528      } else if #[cfg(target_feature="simd128")] {
529        Self { simd: i8x16_add_sat(self.simd, rhs.simd) }
530      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
531        unsafe {Self { neon: vqaddq_s8(self.neon, rhs.neon) }}
532      } else {
533        Self { arr: [
534          self.arr[0].saturating_add(rhs.arr[0]),
535          self.arr[1].saturating_add(rhs.arr[1]),
536          self.arr[2].saturating_add(rhs.arr[2]),
537          self.arr[3].saturating_add(rhs.arr[3]),
538          self.arr[4].saturating_add(rhs.arr[4]),
539          self.arr[5].saturating_add(rhs.arr[5]),
540          self.arr[6].saturating_add(rhs.arr[6]),
541          self.arr[7].saturating_add(rhs.arr[7]),
542          self.arr[8].saturating_add(rhs.arr[8]),
543          self.arr[9].saturating_add(rhs.arr[9]),
544          self.arr[10].saturating_add(rhs.arr[10]),
545          self.arr[11].saturating_add(rhs.arr[11]),
546          self.arr[12].saturating_add(rhs.arr[12]),
547          self.arr[13].saturating_add(rhs.arr[13]),
548          self.arr[14].saturating_add(rhs.arr[14]),
549          self.arr[15].saturating_add(rhs.arr[15]),
550        ]}
551      }
552    }
553  }
554
555  #[inline]
556  pub fn saturating_sub(self, rhs: Self) -> Self {
557    pick! {
558      if #[cfg(target_feature="sse2")] {
559        Self { sse: sub_saturating_i8_m128i(self.sse, rhs.sse) }
560      } else if #[cfg(target_feature="simd128")] {
561        Self { simd: i8x16_sub_sat(self.simd, rhs.simd) }
562      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
563        unsafe { Self { neon: vqsubq_s8(self.neon, rhs.neon) } }
564      } else {
565        Self { arr: [
566          self.arr[0].saturating_sub(rhs.arr[0]),
567          self.arr[1].saturating_sub(rhs.arr[1]),
568          self.arr[2].saturating_sub(rhs.arr[2]),
569          self.arr[3].saturating_sub(rhs.arr[3]),
570          self.arr[4].saturating_sub(rhs.arr[4]),
571          self.arr[5].saturating_sub(rhs.arr[5]),
572          self.arr[6].saturating_sub(rhs.arr[6]),
573          self.arr[7].saturating_sub(rhs.arr[7]),
574          self.arr[8].saturating_sub(rhs.arr[8]),
575          self.arr[9].saturating_sub(rhs.arr[9]),
576          self.arr[10].saturating_sub(rhs.arr[10]),
577          self.arr[11].saturating_sub(rhs.arr[11]),
578          self.arr[12].saturating_sub(rhs.arr[12]),
579          self.arr[13].saturating_sub(rhs.arr[13]),
580          self.arr[14].saturating_sub(rhs.arr[14]),
581          self.arr[15].saturating_sub(rhs.arr[15]),
582        ]}
583      }
584    }
585  }
586
587  #[inline]
588  pub fn overflowing_mul(self, rhs: Self) -> (Self, Self) {
589    let (low, high) = self.mul_keep_low_high(rhs);
590    let low = cast::<u8x16, i8x16>(low);
591
592    let overflow = high.simd_ne(low.is_negative());
593    (low, overflow)
594  }
595
596  optional_fn_widening_mul {
597    #[inline]
598    pub fn widening_mul(self, rhs: Self) -> i16x16 {
599      pick! {
600        if #[cfg(all(target_feature="neon", target_arch="aarch64"))] {
601          unsafe {
602            let low_wide_mul = vmull_s8(vget_low_s8(self.neon), vget_low_s8(rhs.neon));
603            let high_wide_mul = vmull_s8(vget_high_s8(self.neon), vget_high_s8(rhs.neon));
604
605            i16x16 {
606              a: i16x8 { neon: low_wide_mul },
607              b: i16x8 { neon: high_wide_mul },
608            }
609          }
610        } else {
611          let self_array = self.to_array();
612          let rhs_array = rhs.to_array();
613
614          i16x16::new([
615            (self_array[0] as i16).wrapping_mul(rhs_array[0] as i16),
616            (self_array[1] as i16).wrapping_mul(rhs_array[1] as i16),
617            (self_array[2] as i16).wrapping_mul(rhs_array[2] as i16),
618            (self_array[3] as i16).wrapping_mul(rhs_array[3] as i16),
619            (self_array[4] as i16).wrapping_mul(rhs_array[4] as i16),
620            (self_array[5] as i16).wrapping_mul(rhs_array[5] as i16),
621            (self_array[6] as i16).wrapping_mul(rhs_array[6] as i16),
622            (self_array[7] as i16).wrapping_mul(rhs_array[7] as i16),
623            (self_array[8] as i16).wrapping_mul(rhs_array[8] as i16),
624            (self_array[9] as i16).wrapping_mul(rhs_array[9] as i16),
625            (self_array[10] as i16).wrapping_mul(rhs_array[10] as i16),
626            (self_array[11] as i16).wrapping_mul(rhs_array[11] as i16),
627            (self_array[12] as i16).wrapping_mul(rhs_array[12] as i16),
628            (self_array[13] as i16).wrapping_mul(rhs_array[13] as i16),
629            (self_array[14] as i16).wrapping_mul(rhs_array[14] as i16),
630            (self_array[15] as i16).wrapping_mul(rhs_array[15] as i16),
631          ])
632        }
633      }
634    }
635  }
636
637  #[inline]
638  pub fn mul_keep_low_high(self, rhs: Self) -> (u8x16, i8x16) {
639    pick! {
640      if #[cfg(all(target_feature="neon", target_arch="aarch64"))] {
641        unsafe {
642          let low_wide_mul = vreinterpretq_s8_s16(
643            vmull_s8(vget_low_s8(self.neon), vget_low_s8(rhs.neon)),
644          );
645          let high_wide_mul = vreinterpretq_s8_s16(
646            vmull_s8(vget_high_s8(self.neon), vget_high_s8(rhs.neon)),
647          );
648          let low_high = vuzpq_s8(low_wide_mul, high_wide_mul);
649
650          (
651            u8x16 { neon: vreinterpretq_u8_s8(low_high.0) },
652            i8x16 { neon: low_high.1 },
653          )
654        }
655      } else {
656        // TODO(perf): This implementation looks quite bad. Is there a better
657        // one?
658
659        let self_array = self.to_array();
660        let rhs_array = rhs.to_array();
661
662        let widening_mul = [
663          (self_array[0] as i16).wrapping_mul(rhs_array[0] as i16),
664          (self_array[1] as i16).wrapping_mul(rhs_array[1] as i16),
665          (self_array[2] as i16).wrapping_mul(rhs_array[2] as i16),
666          (self_array[3] as i16).wrapping_mul(rhs_array[3] as i16),
667          (self_array[4] as i16).wrapping_mul(rhs_array[4] as i16),
668          (self_array[5] as i16).wrapping_mul(rhs_array[5] as i16),
669          (self_array[6] as i16).wrapping_mul(rhs_array[6] as i16),
670          (self_array[7] as i16).wrapping_mul(rhs_array[7] as i16),
671          (self_array[8] as i16).wrapping_mul(rhs_array[8] as i16),
672          (self_array[9] as i16).wrapping_mul(rhs_array[9] as i16),
673          (self_array[10] as i16).wrapping_mul(rhs_array[10] as i16),
674          (self_array[11] as i16).wrapping_mul(rhs_array[11] as i16),
675          (self_array[12] as i16).wrapping_mul(rhs_array[12] as i16),
676          (self_array[13] as i16).wrapping_mul(rhs_array[13] as i16),
677          (self_array[14] as i16).wrapping_mul(rhs_array[14] as i16),
678          (self_array[15] as i16).wrapping_mul(rhs_array[15] as i16),
679        ];
680
681        (
682          u8x16::new([
683            widening_mul[0] as u8,
684            widening_mul[1] as u8,
685            widening_mul[2] as u8,
686            widening_mul[3] as u8,
687            widening_mul[4] as u8,
688            widening_mul[5] as u8,
689            widening_mul[6] as u8,
690            widening_mul[7] as u8,
691            widening_mul[8] as u8,
692            widening_mul[9] as u8,
693            widening_mul[10] as u8,
694            widening_mul[11] as u8,
695            widening_mul[12] as u8,
696            widening_mul[13] as u8,
697            widening_mul[14] as u8,
698            widening_mul[15] as u8,
699          ]),
700          i8x16::new([
701            (widening_mul[0] >> 8) as i8,
702            (widening_mul[1] >> 8) as i8,
703            (widening_mul[2] >> 8) as i8,
704            (widening_mul[3] >> 8) as i8,
705            (widening_mul[4] >> 8) as i8,
706            (widening_mul[5] >> 8) as i8,
707            (widening_mul[6] >> 8) as i8,
708            (widening_mul[7] >> 8) as i8,
709            (widening_mul[8] >> 8) as i8,
710            (widening_mul[9] >> 8) as i8,
711            (widening_mul[10] >> 8) as i8,
712            (widening_mul[11] >> 8) as i8,
713            (widening_mul[12] >> 8) as i8,
714            (widening_mul[13] >> 8) as i8,
715            (widening_mul[14] >> 8) as i8,
716            (widening_mul[15] >> 8) as i8,
717          ])
718        )
719      }
720    }
721  }
722
723  #[inline]
724  pub fn mul_keep_high(self, rhs: Self) -> Self {
725    pick! {
726      if #[cfg(all(target_feature="neon", target_arch="aarch64"))] {
727        unsafe {
728          let low_wide_mul = vreinterpretq_s8_s16(
729            vmull_s8(vget_low_s8(self.neon), vget_low_s8(rhs.neon)),
730          );
731          let high_wide_mul = vreinterpretq_s8_s16(
732            vmull_s8(vget_high_s8(self.neon), vget_high_s8(rhs.neon)),
733          );
734
735          Self { neon: vuzpq_s8(low_wide_mul, high_wide_mul).1 }
736        }
737      } else {
738        let self_array = self.to_array();
739        let rhs_array = rhs.to_array();
740
741        Self::new([
742          ((self_array[0] as i16).wrapping_mul(rhs_array[0] as i16) >> 8) as i8,
743          ((self_array[1] as i16).wrapping_mul(rhs_array[1] as i16) >> 8) as i8,
744          ((self_array[2] as i16).wrapping_mul(rhs_array[2] as i16) >> 8) as i8,
745          ((self_array[3] as i16).wrapping_mul(rhs_array[3] as i16) >> 8) as i8,
746          ((self_array[4] as i16).wrapping_mul(rhs_array[4] as i16) >> 8) as i8,
747          ((self_array[5] as i16).wrapping_mul(rhs_array[5] as i16) >> 8) as i8,
748          ((self_array[6] as i16).wrapping_mul(rhs_array[6] as i16) >> 8) as i8,
749          ((self_array[7] as i16).wrapping_mul(rhs_array[7] as i16) >> 8) as i8,
750          ((self_array[8] as i16).wrapping_mul(rhs_array[8] as i16) >> 8) as i8,
751          ((self_array[9] as i16).wrapping_mul(rhs_array[9] as i16) >> 8) as i8,
752          ((self_array[10] as i16).wrapping_mul(rhs_array[10] as i16) >> 8) as i8,
753          ((self_array[11] as i16).wrapping_mul(rhs_array[11] as i16) >> 8) as i8,
754          ((self_array[12] as i16).wrapping_mul(rhs_array[12] as i16) >> 8) as i8,
755          ((self_array[13] as i16).wrapping_mul(rhs_array[13] as i16) >> 8) as i8,
756          ((self_array[14] as i16).wrapping_mul(rhs_array[14] as i16) >> 8) as i8,
757          ((self_array[15] as i16).wrapping_mul(rhs_array[15] as i16) >> 8) as i8,
758        ])
759      }
760    }
761  }
762
763  #[inline]
764  pub fn abs(self) -> Self {
765    pick! {
766      if #[cfg(target_feature="ssse3")] {
767        Self { sse: abs_i8_m128i(self.sse) }
768      } else if #[cfg(target_feature="simd128")] {
769        Self { simd: i8x16_abs(self.simd) }
770      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
771        unsafe {Self { neon: vabsq_s8(self.neon) }}
772      } else {
773        let arr: [i8; 16] = cast(self);
774        cast([
775          arr[0].wrapping_abs(),
776          arr[1].wrapping_abs(),
777          arr[2].wrapping_abs(),
778          arr[3].wrapping_abs(),
779          arr[4].wrapping_abs(),
780          arr[5].wrapping_abs(),
781          arr[6].wrapping_abs(),
782          arr[7].wrapping_abs(),
783          arr[8].wrapping_abs(),
784          arr[9].wrapping_abs(),
785          arr[10].wrapping_abs(),
786          arr[11].wrapping_abs(),
787          arr[12].wrapping_abs(),
788          arr[13].wrapping_abs(),
789          arr[14].wrapping_abs(),
790          arr[15].wrapping_abs(),
791        ])
792      }
793    }
794  }
795
796  #[inline]
797  pub fn is_positive(self) -> Self {
798    pick! {
799      if #[cfg(all(target_feature="neon", target_arch="aarch64"))] {
800        Self { neon: unsafe { vreinterpretq_s8_u8(vcgtzq_s8(self.neon)) } }
801      } else {
802        self.simd_gt(Self::ZERO)
803      }
804    }
805  }
806
807  #[inline]
808  pub fn is_negative(self) -> Self {
809    pick! {
810      if #[cfg(all(target_feature="neon", target_arch="aarch64"))] {
811        Self { neon: unsafe { vreinterpretq_s8_u8(vcltzq_s8(self.neon)) } }
812      } else {
813        self.simd_lt(Self::ZERO)
814      }
815    }
816  }
817
818  optional_fn_deserialize {}
819}
820
821/// The following functionality exists only for [`i8x16`], or only for
822/// particular types inconsistently.
823impl i8x16 {
824  /// Converts each element from [`i16`] to [`i8`], saturating out of range
825  /// values.
826  #[inline]
827  #[must_use]
828  pub fn from_i16x16_saturate(v: i16x16) -> i8x16 {
829    pick! {
830      if #[cfg(target_feature="avx2")] {
831        i8x16 { sse: pack_i16_to_i8_m128i( extract_m128i_from_m256i::<0>(v.avx2), extract_m128i_from_m256i::<1>(v.avx2))  }
832      } else if #[cfg(target_feature="sse2")] {
833        i8x16 { sse: pack_i16_to_i8_m128i( v.a.sse, v.b.sse ) }
834      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))] {
835        use core::arch::aarch64::*;
836
837        unsafe {
838          i8x16 { neon: vcombine_s8(vqmovn_s16(v.a.neon), vqmovn_s16(v.b.neon)) }
839        }
840      } else if #[cfg(target_feature="simd128")] {
841        use core::arch::wasm32::*;
842
843        i8x16 { simd: i8x16_narrow_i16x8(v.a.simd, v.b.simd) }
844      } else {
845        fn clamp(a : i16) -> i8 {
846            if a < i8::MIN as i16 {
847              i8::MIN
848            }
849            else if a > i8::MAX as i16 {
850              i8::MAX
851            } else {
852                a as i8
853            }
854        }
855
856        i8x16::new([
857          clamp(v.as_array()[0]),
858          clamp(v.as_array()[1]),
859          clamp(v.as_array()[2]),
860          clamp(v.as_array()[3]),
861          clamp(v.as_array()[4]),
862          clamp(v.as_array()[5]),
863          clamp(v.as_array()[6]),
864          clamp(v.as_array()[7]),
865          clamp(v.as_array()[8]),
866          clamp(v.as_array()[9]),
867          clamp(v.as_array()[10]),
868          clamp(v.as_array()[11]),
869          clamp(v.as_array()[12]),
870          clamp(v.as_array()[13]),
871          clamp(v.as_array()[14]),
872          clamp(v.as_array()[15]),
873        ])
874      }
875    }
876  }
877
878  /// Converts each element from [`i16`] to [`i8`], truncating out of range
879  /// values (behaves like [`as`] casting).
880  ///
881  /// [`as`]: https://doc.rust-lang.org/stable/reference/expressions/operator-expr.html#r-expr.as.numeric
882  #[inline]
883  #[must_use]
884  pub fn from_i16x16_truncate(v: i16x16) -> i8x16 {
885    pick! {
886      if #[cfg(target_feature="avx2")] {
887        let a = v.avx2.bitand(set_splat_i16_m256i(0xff));
888        i8x16 { sse: pack_i16_to_u8_m128i( extract_m128i_from_m256i::<0>(a), extract_m128i_from_m256i::<1>(a))  }
889      } else if #[cfg(target_feature="sse2")] {
890        let mask = set_splat_i16_m128i(0xff);
891        i8x16 { sse: pack_i16_to_u8_m128i( v.a.sse.bitand(mask), v.b.sse.bitand(mask) ) }
892      } else {
893        // no super good intrinsics on other platforms... plain old codegen does a reasonable job
894        i8x16::new([
895          v.as_array()[0] as i8,
896          v.as_array()[1] as i8,
897          v.as_array()[2] as i8,
898          v.as_array()[3] as i8,
899          v.as_array()[4] as i8,
900          v.as_array()[5] as i8,
901          v.as_array()[6] as i8,
902          v.as_array()[7] as i8,
903          v.as_array()[8] as i8,
904          v.as_array()[9] as i8,
905          v.as_array()[10] as i8,
906          v.as_array()[11] as i8,
907          v.as_array()[12] as i8,
908          v.as_array()[13] as i8,
909          v.as_array()[14] as i8,
910          v.as_array()[15] as i8,
911        ])
912      }
913    }
914  }
915
916  /// Converts a slice to a SIMD vector, ignoring elements beyond the first 16.
917  ///
918  /// # Panics
919  ///
920  /// Panics if `input` has less than 16 elements.
921  #[inline]
922  #[must_use]
923  pub fn from_slice_unaligned(input: &[i8]) -> Self {
924    assert!(input.len() >= 16);
925
926    pick! {
927      if #[cfg(target_feature="sse2")] {
928        unsafe { Self { sse: load_unaligned_m128i( &*(input.as_ptr() as * const [u8;16]) ) } }
929      } else if #[cfg(target_feature="simd128")] {
930        unsafe { Self { simd: v128_load(input.as_ptr() as *const v128 ) } }
931      } else if #[cfg(all(target_feature="neon",target_arch="aarch64"))]{
932        unsafe { Self { neon: vld1q_s8( input.as_ptr() as *const i8 ) } }
933      } else {
934        // 2018 edition doesn't have try_into
935        unsafe { Self::new( *(input.as_ptr() as * const [i8;16]) ) }
936      }
937    }
938  }
939
940  /// Returns a new vector where each element is based on the index values in
941  /// `rhs`.
942  ///
943  /// * Index values in the range `[0, 15]` select the i-th element of `self`.
944  /// * Index values that are out of range will cause that output lane to be
945  ///   `0`.
946  ///
947  /// This function has been deprecated and replaced with [`shuffle_zeroing`].
948  ///
949  /// [`shuffle_zeroing`]: Self::shuffle_zeroing
950  #[inline]
951  #[deprecated(since = "1.7.0", note = "replaced with `shuffle_zeroing`")]
952  pub fn swizzle(self, rhs: i8x16) -> i8x16 {
953    self.shuffle_zeroing(rhs.cast_unsigned())
954  }
955
956  /// Works like [`swizzle`](Self::swizzle) with the following additional
957  /// details
958  ///
959  /// * Indices in the range `[0, 15]` will select the i-th element of `self`.
960  /// * If the high bit of any index is set (meaning that the index is
961  ///   negative), then the corresponding output lane is guaranteed to be zero.
962  /// * Otherwise the output lane is either `0` or `self[rhs[i] % 16]`,
963  ///   depending on the implementation.
964  ///
965  /// This function has been deprecated and replaced with [`shuffle`].
966  ///
967  /// [`shuffle`]: Self::shuffle
968  #[inline]
969  #[deprecated(since = "1.7.0", note = "replaced with `shuffle`")]
970  pub fn swizzle_relaxed(self, rhs: i8x16) -> i8x16 {
971    self.shuffle(rhs.cast_unsigned())
972  }
973}
wide/i8x16_.rs

wide/
i8x16_.rs
