libstdc++
simd_fixed_size.h
1 // Simd fixed_size ABI specific implementations -*- C++ -*-
2 
3 // Copyright (C) 2020-2025 Free Software Foundation, Inc.
4 //
5 // This file is part of the GNU ISO C++ Library. This library is free
6 // software; you can redistribute it and/or modify it under the
7 // terms of the GNU General Public License as published by the
8 // Free Software Foundation; either version 3, or (at your option)
9 // any later version.
10 
11 // This library is distributed in the hope that it will be useful,
12 // but WITHOUT ANY WARRANTY; without even the implied warranty of
13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 // GNU General Public License for more details.
15 
16 // Under Section 7 of GPL version 3, you are granted additional
17 // permissions described in the GCC Runtime Library Exception, version
18 // 3.1, as published by the Free Software Foundation.
19 
20 // You should have received a copy of the GNU General Public License and
21 // a copy of the GCC Runtime Library Exception along with this program;
22 // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23 // <http://www.gnu.org/licenses/>.
24 
25 /*
26  * The fixed_size ABI gives the following guarantees:
27  * - simd objects are passed via the stack
28  * - memory layout of `simd<_Tp, _Np>` is equivalent to `array<_Tp, _Np>`
29  * - alignment of `simd<_Tp, _Np>` is `_Np * sizeof(_Tp)` if _Np is __a
30  * power-of-2 value, otherwise `std::__bit_ceil(_Np * sizeof(_Tp))` (Note:
31  * if the alignment were to exceed the system/compiler maximum, it is bounded
32  * to that maximum)
33  * - simd_mask objects are passed like bitset<_Np>
34  * - memory layout of `simd_mask<_Tp, _Np>` is equivalent to `bitset<_Np>`
35  * - alignment of `simd_mask<_Tp, _Np>` is equal to the alignment of
36  * `bitset<_Np>`
37  */
38 
39 #ifndef _GLIBCXX_EXPERIMENTAL_SIMD_FIXED_SIZE_H_
40 #define _GLIBCXX_EXPERIMENTAL_SIMD_FIXED_SIZE_H_
41 
42 #if __cplusplus >= 201703L
43 
44 #include <array>
45 
46 _GLIBCXX_SIMD_BEGIN_NAMESPACE
47 
48 // __simd_tuple_element {{{
49 template <size_t _I, typename _Tp>
50  struct __simd_tuple_element;
51 
52 template <typename _Tp, typename _A0, typename... _As>
53  struct __simd_tuple_element<0, _SimdTuple<_Tp, _A0, _As...>>
54  { using type = simd<_Tp, _A0>; };
55 
56 template <size_t _I, typename _Tp, typename _A0, typename... _As>
57  struct __simd_tuple_element<_I, _SimdTuple<_Tp, _A0, _As...>>
58  { using type = typename __simd_tuple_element<_I - 1, _SimdTuple<_Tp, _As...>>::type; };
59 
60 template <size_t _I, typename _Tp>
61  using __simd_tuple_element_t = typename __simd_tuple_element<_I, _Tp>::type;
62 
63 // }}}
64 // __simd_tuple_concat {{{
65 
66 template <typename _Tp, typename... _A0s, typename... _A1s>
67  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple<_Tp, _A0s..., _A1s...>
68  __simd_tuple_concat(const _SimdTuple<_Tp, _A0s...>& __left,
69  const _SimdTuple<_Tp, _A1s...>& __right)
70  {
71  if constexpr (sizeof...(_A0s) == 0)
72  return __right;
73  else if constexpr (sizeof...(_A1s) == 0)
74  return __left;
75  else
76  return {__left.first, __simd_tuple_concat(__left.second, __right)};
77  }
78 
79 template <typename _Tp, typename _A10, typename... _A1s>
80  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple<_Tp, simd_abi::scalar, _A10, _A1s...>
81  __simd_tuple_concat(const _Tp& __left, const _SimdTuple<_Tp, _A10, _A1s...>& __right)
82  { return {__left, __right}; }
83 
84 // }}}
85 // __simd_tuple_pop_front {{{
86 // Returns the next _SimdTuple in __x that has _Np elements less.
87 // Precondition: _Np must match the number of elements in __first (recursively)
88 template <size_t _Np, typename _Tp>
89  _GLIBCXX_SIMD_INTRINSIC constexpr decltype(auto)
90  __simd_tuple_pop_front(_Tp&& __x)
91  {
92  if constexpr (_Np == 0)
93  return static_cast<_Tp&&>(__x);
94  else
95  {
96  using _Up = __remove_cvref_t<_Tp>;
97  static_assert(_Np >= _Up::_S_first_size);
98  return __simd_tuple_pop_front<_Np - _Up::_S_first_size>(__x.second);
99  }
100  }
101 
102 // }}}
103 // __get_simd_at<_Np> {{{1
104 struct __as_simd {};
105 
106 struct __as_simd_tuple {};
107 
108 template <typename _Tp, typename _A0, typename... _Abis>
109  _GLIBCXX_SIMD_INTRINSIC constexpr simd<_Tp, _A0>
110  __simd_tuple_get_impl(__as_simd, const _SimdTuple<_Tp, _A0, _Abis...>& __t, _SizeConstant<0>)
111  { return {__private_init, __t.first}; }
112 
113 template <typename _Tp, typename _A0, typename... _Abis>
114  _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
115  __simd_tuple_get_impl(__as_simd_tuple, const _SimdTuple<_Tp, _A0, _Abis...>& __t,
116  _SizeConstant<0>)
117  { return __t.first; }
118 
119 template <typename _Tp, typename _A0, typename... _Abis>
120  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
121  __simd_tuple_get_impl(__as_simd_tuple, _SimdTuple<_Tp, _A0, _Abis...>& __t, _SizeConstant<0>)
122  { return __t.first; }
123 
124 template <typename _R, size_t _Np, typename _Tp, typename... _Abis>
125  _GLIBCXX_SIMD_INTRINSIC constexpr auto
126  __simd_tuple_get_impl(_R, const _SimdTuple<_Tp, _Abis...>& __t, _SizeConstant<_Np>)
127  { return __simd_tuple_get_impl(_R(), __t.second, _SizeConstant<_Np - 1>()); }
128 
129 template <size_t _Np, typename _Tp, typename... _Abis>
130  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
131  __simd_tuple_get_impl(__as_simd_tuple, _SimdTuple<_Tp, _Abis...>& __t, _SizeConstant<_Np>)
132  { return __simd_tuple_get_impl(__as_simd_tuple(), __t.second, _SizeConstant<_Np - 1>()); }
133 
134 template <size_t _Np, typename _Tp, typename... _Abis>
135  _GLIBCXX_SIMD_INTRINSIC constexpr auto
136  __get_simd_at(const _SimdTuple<_Tp, _Abis...>& __t)
137  { return __simd_tuple_get_impl(__as_simd(), __t, _SizeConstant<_Np>()); }
138 
139 // }}}
140 // __get_tuple_at<_Np> {{{
141 template <size_t _Np, typename _Tp, typename... _Abis>
142  _GLIBCXX_SIMD_INTRINSIC constexpr auto
143  __get_tuple_at(const _SimdTuple<_Tp, _Abis...>& __t)
144  { return __simd_tuple_get_impl(__as_simd_tuple(), __t, _SizeConstant<_Np>()); }
145 
146 template <size_t _Np, typename _Tp, typename... _Abis>
147  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
148  __get_tuple_at(_SimdTuple<_Tp, _Abis...>& __t)
149  { return __simd_tuple_get_impl(__as_simd_tuple(), __t, _SizeConstant<_Np>()); }
150 
151 // __tuple_element_meta {{{1
152 template <typename _Tp, typename _Abi, size_t _Offset>
153  struct __tuple_element_meta : public _Abi::_SimdImpl
154  {
155  static_assert(is_same_v<typename _Abi::_SimdImpl::abi_type,
156  _Abi>); // this fails e.g. when _SimdImpl is an
157  // alias for _SimdImplBuiltin<_DifferentAbi>
158  using value_type = _Tp;
159  using abi_type = _Abi;
160  using _Traits = _SimdTraits<_Tp, _Abi>;
161  using _MaskImpl = typename _Abi::_MaskImpl;
162  using _MaskMember = typename _Traits::_MaskMember;
163  using simd_type = simd<_Tp, _Abi>;
164  static constexpr size_t _S_offset = _Offset;
165  static constexpr size_t _S_size() { return simd_size<_Tp, _Abi>::value; }
166  static constexpr _MaskImpl _S_mask_impl = {};
167 
168  template <size_t _Np, bool _Sanitized>
169  _GLIBCXX_SIMD_INTRINSIC static constexpr auto
170  _S_submask(_BitMask<_Np, _Sanitized> __bits)
171  { return __bits.template _M_extract<_Offset, _S_size()>(); }
172 
173  template <size_t _Np, bool _Sanitized>
174  _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember
175  _S_make_mask(_BitMask<_Np, _Sanitized> __bits)
176  {
177  return _MaskImpl::template _S_convert<_Tp>(
178  __bits.template _M_extract<_Offset, _S_size()>()._M_sanitized());
179  }
180 
181  _GLIBCXX_SIMD_INTRINSIC static constexpr _ULLong
182  _S_mask_to_shifted_ullong(_MaskMember __k)
183  { return _MaskImpl::_S_to_bits(__k).to_ullong() << _Offset; }
184  };
185 
186 template <size_t _Offset, typename _Tp, typename _Abi, typename... _As>
187  _GLIBCXX_SIMD_INTRINSIC constexpr
188  __tuple_element_meta<_Tp, _Abi, _Offset>
189  __make_meta(const _SimdTuple<_Tp, _Abi, _As...>&)
190  { return {}; }
191 
192 // }}}1
193 // _WithOffset wrapper class {{{
194 template <size_t _Offset, typename _Base>
195  struct _WithOffset : public _Base
196  {
197  static inline constexpr size_t _S_offset = _Offset;
198 
199  _GLIBCXX_SIMD_INTRINSIC char*
200  _M_as_charptr()
201  { return reinterpret_cast<char*>(this) + _S_offset * sizeof(typename _Base::value_type); }
202 
203  _GLIBCXX_SIMD_INTRINSIC const char*
204  _M_as_charptr() const
205  { return reinterpret_cast<const char*>(this) + _S_offset * sizeof(typename _Base::value_type); }
206  };
207 
208 // make _WithOffset<_WithOffset> ill-formed to use:
209 template <size_t _O0, size_t _O1, typename _Base>
210  struct _WithOffset<_O0, _WithOffset<_O1, _Base>> {};
211 
212 template <size_t _Offset, typename _Tp>
213  _GLIBCXX_SIMD_INTRINSIC
214  decltype(auto)
215  __add_offset(_Tp& __base)
216  { return static_cast<_WithOffset<_Offset, __remove_cvref_t<_Tp>>&>(__base); }
217 
218 template <size_t _Offset, typename _Tp>
219  _GLIBCXX_SIMD_INTRINSIC
220  decltype(auto)
221  __add_offset(const _Tp& __base)
222  { return static_cast<const _WithOffset<_Offset, __remove_cvref_t<_Tp>>&>(__base); }
223 
224 template <size_t _Offset, size_t _ExistingOffset, typename _Tp>
225  _GLIBCXX_SIMD_INTRINSIC
226  decltype(auto)
227  __add_offset(_WithOffset<_ExistingOffset, _Tp>& __base)
228  { return static_cast<_WithOffset<_Offset + _ExistingOffset, _Tp>&>(static_cast<_Tp&>(__base)); }
229 
230 template <size_t _Offset, size_t _ExistingOffset, typename _Tp>
231  _GLIBCXX_SIMD_INTRINSIC
232  decltype(auto)
233  __add_offset(const _WithOffset<_ExistingOffset, _Tp>& __base)
234  {
235  return static_cast<const _WithOffset<_Offset + _ExistingOffset, _Tp>&>(
236  static_cast<const _Tp&>(__base));
237  }
238 
239 template <typename _Tp>
240  constexpr inline size_t __offset = 0;
241 
242 template <size_t _Offset, typename _Tp>
243  constexpr inline size_t __offset<_WithOffset<_Offset, _Tp>>
244  = _WithOffset<_Offset, _Tp>::_S_offset;
245 
246 template <typename _Tp>
247  constexpr inline size_t __offset<const _Tp> = __offset<_Tp>;
248 
249 template <typename _Tp>
250  constexpr inline size_t __offset<_Tp&> = __offset<_Tp>;
251 
252 template <typename _Tp>
253  constexpr inline size_t __offset<_Tp&&> = __offset<_Tp>;
254 
255 // }}}
256 // _SimdTuple specializations {{{1
257 // empty {{{2
258 template <typename _Tp>
259  struct _SimdTuple<_Tp>
260  {
261  using value_type = _Tp;
262  static constexpr size_t _S_tuple_size = 0;
263  static constexpr size_t _S_size() { return 0; }
264  };
265 
266 // _SimdTupleData {{{2
267 template <typename _FirstType, typename _SecondType>
268  struct _SimdTupleData
269  {
270  _FirstType first;
271  _SecondType second;
272 
273  _GLIBCXX_SIMD_INTRINSIC
274  constexpr bool
275  _M_is_constprop() const
276  {
277  if constexpr (is_class_v<_FirstType>)
278  return first._M_is_constprop() && second._M_is_constprop();
279  else
280  return __builtin_constant_p(first) && second._M_is_constprop();
281  }
282  };
283 
284 template <typename _FirstType, typename _Tp>
285  struct _SimdTupleData<_FirstType, _SimdTuple<_Tp>>
286  {
287  _FirstType first;
288  static constexpr _SimdTuple<_Tp> second = {};
289 
290  _GLIBCXX_SIMD_INTRINSIC
291  constexpr bool
292  _M_is_constprop() const
293  {
294  if constexpr (is_class_v<_FirstType>)
295  return first._M_is_constprop();
296  else
297  return __builtin_constant_p(first);
298  }
299  };
300 
301 // 1 or more {{{2
302 template <typename _Tp, typename _Abi0, typename... _Abis>
303  struct _SimdTuple<_Tp, _Abi0, _Abis...>
304  : _SimdTupleData<typename _SimdTraits<_Tp, _Abi0>::_SimdMember,
305  _SimdTuple<_Tp, _Abis...>>
306  {
307  static_assert(!__is_fixed_size_abi_v<_Abi0>);
308  using value_type = _Tp;
309  using _FirstType = typename _SimdTraits<_Tp, _Abi0>::_SimdMember;
310  using _FirstAbi = _Abi0;
311  using _SecondType = _SimdTuple<_Tp, _Abis...>;
312  static constexpr size_t _S_tuple_size = sizeof...(_Abis) + 1;
313 
314  static constexpr size_t _S_size()
315  { return simd_size_v<_Tp, _Abi0> + _SecondType::_S_size(); }
316 
317  static constexpr size_t _S_first_size = simd_size_v<_Tp, _Abi0>;
318  static constexpr bool _S_is_homogeneous = (is_same_v<_Abi0, _Abis> && ...);
319 
320  using _Base = _SimdTupleData<typename _SimdTraits<_Tp, _Abi0>::_SimdMember,
321  _SimdTuple<_Tp, _Abis...>>;
322  using _Base::first;
323  using _Base::second;
324 
325  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple() = default;
326  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple(const _SimdTuple&) = default;
327  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple& operator=(const _SimdTuple&)
328  = default;
329 
330  template <typename _Up>
331  _GLIBCXX_SIMD_INTRINSIC constexpr
332  _SimdTuple(_Up&& __x)
333  : _Base{static_cast<_Up&&>(__x)} {}
334 
335  template <typename _Up, typename _Up2>
336  _GLIBCXX_SIMD_INTRINSIC constexpr
337  _SimdTuple(_Up&& __x, _Up2&& __y)
338  : _Base{static_cast<_Up&&>(__x), static_cast<_Up2&&>(__y)} {}
339 
340  template <typename _Up>
341  _GLIBCXX_SIMD_INTRINSIC constexpr
342  _SimdTuple(_Up&& __x, _SimdTuple<_Tp>)
343  : _Base{static_cast<_Up&&>(__x)} {}
344 
345  _GLIBCXX_SIMD_INTRINSIC char*
346  _M_as_charptr()
347  { return reinterpret_cast<char*>(this); }
348 
349  _GLIBCXX_SIMD_INTRINSIC const char*
350  _M_as_charptr() const
351  { return reinterpret_cast<const char*>(this); }
352 
353  template <size_t _Np>
354  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
355  _M_at()
356  {
357  if constexpr (_Np == 0)
358  return first;
359  else
360  return second.template _M_at<_Np - 1>();
361  }
362 
363  template <size_t _Np>
364  _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
365  _M_at() const
366  {
367  if constexpr (_Np == 0)
368  return first;
369  else
370  return second.template _M_at<_Np - 1>();
371  }
372 
373  template <size_t _Np>
374  _GLIBCXX_SIMD_INTRINSIC constexpr auto
375  _M_simd_at() const
376  {
377  if constexpr (_Np == 0)
378  return simd<_Tp, _Abi0>(__private_init, first);
379  else
380  return second.template _M_simd_at<_Np - 1>();
381  }
382 
383  template <size_t _Offset = 0, typename _Fp>
384  _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple
385  _S_generate(_Fp&& __gen, _SizeConstant<_Offset> = {})
386  {
387  auto&& __first = __gen(__tuple_element_meta<_Tp, _Abi0, _Offset>());
388  if constexpr (_S_tuple_size == 1)
389  return {__first};
390  else
391  return {__first,
392  _SecondType::_S_generate(
393  static_cast<_Fp&&>(__gen),
394  _SizeConstant<_Offset + simd_size_v<_Tp, _Abi0>>())};
395  }
396 
397  template <size_t _Offset = 0, typename _Fp, typename... _More>
398  _GLIBCXX_SIMD_INTRINSIC _SimdTuple
399  _M_apply_wrapped(_Fp&& __fun, const _More&... __more) const
400  {
401  auto&& __first
402  = __fun(__make_meta<_Offset>(*this), first, __more.first...);
403  if constexpr (_S_tuple_size == 1)
404  return {__first};
405  else
406  return {
407  __first,
408  second.template _M_apply_wrapped<_Offset + simd_size_v<_Tp, _Abi0>>(
409  static_cast<_Fp&&>(__fun), __more.second...)};
410  }
411 
412  template <typename _Tup>
413  _GLIBCXX_SIMD_INTRINSIC constexpr decltype(auto)
414  _M_extract_argument(_Tup&& __tup) const
415  {
416  using _TupT = typename __remove_cvref_t<_Tup>::value_type;
417  if constexpr (is_same_v<_SimdTuple, __remove_cvref_t<_Tup>>)
418  return __tup.first;
419  else if (__builtin_is_constant_evaluated())
420  return __fixed_size_storage_t<_TupT, _S_first_size>::_S_generate(
421  [&](auto __meta) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
422  return __meta._S_generator(
423  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
424  return __tup[__i];
425  }, static_cast<_TupT*>(nullptr));
426  });
427  else
428  return [&]() { // not always_inline; allow the compiler to decide
429  __fixed_size_storage_t<_TupT, _S_first_size> __r;
430  __builtin_memcpy(__r._M_as_charptr(), __tup._M_as_charptr(),
431  sizeof(__r));
432  return __r;
433  }();
434  }
435 
436  template <typename _Tup>
437  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
438  _M_skip_argument(_Tup&& __tup) const
439  {
440  static_assert(_S_tuple_size > 1);
441  using _Up = __remove_cvref_t<_Tup>;
442  constexpr size_t __off = __offset<_Up>;
443  if constexpr (_S_first_size == _Up::_S_first_size && __off == 0)
444  return __tup.second;
445  else if constexpr (_S_first_size > _Up::_S_first_size
446  && _S_first_size % _Up::_S_first_size == 0
447  && __off == 0)
448  return __simd_tuple_pop_front<_S_first_size>(__tup);
449  else if constexpr (_S_first_size + __off < _Up::_S_first_size)
450  return __add_offset<_S_first_size>(__tup);
451  else if constexpr (_S_first_size + __off == _Up::_S_first_size)
452  return __tup.second;
453  else
454  __assert_unreachable<_Tup>();
455  }
456 
457  template <size_t _Offset, typename... _More>
458  _GLIBCXX_SIMD_INTRINSIC constexpr void
459  _M_assign_front(const _SimdTuple<_Tp, _Abi0, _More...>& __x) &
460  {
461  static_assert(_Offset == 0);
462  first = __x.first;
463  if constexpr (sizeof...(_More) > 0)
464  {
465  static_assert(sizeof...(_Abis) >= sizeof...(_More));
466  second.template _M_assign_front<0>(__x.second);
467  }
468  }
469 
470  template <size_t _Offset>
471  _GLIBCXX_SIMD_INTRINSIC constexpr void
472  _M_assign_front(const _FirstType& __x) &
473  {
474  static_assert(_Offset == 0);
475  first = __x;
476  }
477 
478  template <size_t _Offset, typename... _As>
479  _GLIBCXX_SIMD_INTRINSIC constexpr void
480  _M_assign_front(const _SimdTuple<_Tp, _As...>& __x) &
481  {
482  __builtin_memcpy(_M_as_charptr() + _Offset * sizeof(value_type),
483  __x._M_as_charptr(),
484  sizeof(_Tp) * _SimdTuple<_Tp, _As...>::_S_size());
485  }
486 
487  /*
488  * Iterate over the first objects in this _SimdTuple and call __fun for each
489  * of them. If additional arguments are passed via __more, chunk them into
490  * _SimdTuple or __vector_type_t objects of the same number of values.
491  */
492  template <typename _Fp, typename... _More>
493  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple
494  _M_apply_per_chunk(_Fp&& __fun, _More&&... __more) const
495  {
496  if constexpr ((...
497  || conjunction_v<
498  is_lvalue_reference<_More>,
499  negation<is_const<remove_reference_t<_More>>>>) )
500  {
501  // need to write back at least one of __more after calling __fun
502  auto&& __first = [&](auto... __args) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
503  auto __r = __fun(__tuple_element_meta<_Tp, _Abi0, 0>(), first,
504  __args...);
505  [[maybe_unused]] auto&& __ignore_me = {(
506  [](auto&& __dst, const auto& __src) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
507  if constexpr (is_assignable_v<decltype(__dst),
508  decltype(__dst)>)
509  {
510  __dst.template _M_assign_front<__offset<decltype(__dst)>>(
511  __src);
512  }
513  }(static_cast<_More&&>(__more), __args),
514  0)...};
515  return __r;
516  }(_M_extract_argument(__more)...);
517  if constexpr (_S_tuple_size == 1)
518  return {__first};
519  else
520  return {__first,
521  second._M_apply_per_chunk(static_cast<_Fp&&>(__fun),
522  _M_skip_argument(__more)...)};
523  }
524  else
525  {
526  auto&& __first = __fun(__tuple_element_meta<_Tp, _Abi0, 0>(), first,
527  _M_extract_argument(__more)...);
528  if constexpr (_S_tuple_size == 1)
529  return {__first};
530  else
531  return {__first,
532  second._M_apply_per_chunk(static_cast<_Fp&&>(__fun),
533  _M_skip_argument(__more)...)};
534  }
535  }
536 
537  template <typename _R = _Tp, typename _Fp, typename... _More>
538  _GLIBCXX_SIMD_INTRINSIC constexpr auto
539  _M_apply_r(_Fp&& __fun, const _More&... __more) const
540  {
541  auto&& __first = __fun(__tuple_element_meta<_Tp, _Abi0, 0>(), first,
542  __more.first...);
543  if constexpr (_S_tuple_size == 1)
544  return __first;
545  else
546  return __simd_tuple_concat<_R>(
547  __first, second.template _M_apply_r<_R>(static_cast<_Fp&&>(__fun),
548  __more.second...));
549  }
550 
551  template <typename _Fp, typename... _More>
552  _GLIBCXX_SIMD_INTRINSIC constexpr friend _SanitizedBitMask<_S_size()>
553  _M_test(const _Fp& __fun, const _SimdTuple& __x, const _More&... __more)
554  {
555  const _SanitizedBitMask<_S_first_size> __first
556  = _Abi0::_MaskImpl::_S_to_bits(
557  __fun(__tuple_element_meta<_Tp, _Abi0, 0>(), __x.first,
558  __more.first...));
559  if constexpr (_S_tuple_size == 1)
560  return __first;
561  else
562  return _M_test(__fun, __x.second, __more.second...)
563  ._M_prepend(__first);
564  }
565 
566  template <typename _Up, _Up _I>
567  _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
568  operator[](integral_constant<_Up, _I>) const noexcept
569  {
570  if constexpr (_I < simd_size_v<_Tp, _Abi0>)
571  return _M_subscript_read(_I);
572  else
573  return second[integral_constant<_Up, _I - simd_size_v<_Tp, _Abi0>>()];
574  }
575 
576  _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
577  operator[](size_t __i) const noexcept
578  {
579  if constexpr (_S_tuple_size == 1)
580  return _M_subscript_read(__i);
581 #ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
582  else if (not __builtin_is_constant_evaluated())
583  return reinterpret_cast<const __may_alias<_Tp>*>(this)[__i];
584 #endif
585  else if constexpr (__is_scalar_abi<_Abi0>())
586  {
587  const _Tp* ptr = &first;
588  return ptr[__i];
589  }
590  else
591  return __i < simd_size_v<_Tp, _Abi0> ? _M_subscript_read(__i)
592  : second[__i - simd_size_v<_Tp, _Abi0>];
593  }
594 
595  _GLIBCXX_SIMD_INTRINSIC constexpr void
596  _M_set(size_t __i, _Tp __val) noexcept
597  {
598  if constexpr (_S_tuple_size == 1)
599  return _M_subscript_write(__i, __val);
600 #ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
601  else if (not __builtin_is_constant_evaluated())
602  reinterpret_cast<__may_alias<_Tp>*>(this)[__i] = __val;
603 #endif
604  else if (__i < simd_size_v<_Tp, _Abi0>)
605  _M_subscript_write(__i, __val);
606  else
607  second._M_set(__i - simd_size_v<_Tp, _Abi0>, __val);
608  }
609 
610  private:
611  // _M_subscript_read/_write {{{
612  _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
613  _M_subscript_read([[maybe_unused]] size_t __i) const noexcept
614  {
615  if constexpr (__is_vectorizable_v<_FirstType>)
616  return first;
617  else
618  return first[__i];
619  }
620 
621  _GLIBCXX_SIMD_INTRINSIC constexpr void
622  _M_subscript_write([[maybe_unused]] size_t __i, _Tp __y) noexcept
623  {
624  if constexpr (__is_vectorizable_v<_FirstType>)
625  first = __y;
626  else
627  first._M_set(__i, __y);
628  }
629 
630  // }}}
631  };
632 
633 // __make_simd_tuple {{{1
634 template <typename _Tp, typename _A0>
635  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple<_Tp, _A0>
636  __make_simd_tuple(simd<_Tp, _A0> __x0)
637  { return {__data(__x0)}; }
638 
639 template <typename _Tp, typename _A0, typename... _As>
640  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple<_Tp, _A0, _As...>
641  __make_simd_tuple(const simd<_Tp, _A0>& __x0, const simd<_Tp, _As>&... __xs)
642  { return {__data(__x0), __make_simd_tuple(__xs...)}; }
643 
644 template <typename _Tp, typename _A0>
645  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple<_Tp, _A0>
646  __make_simd_tuple(const typename _SimdTraits<_Tp, _A0>::_SimdMember& __arg0)
647  { return {__arg0}; }
648 
649 template <typename _Tp, typename _A0, typename _A1, typename... _Abis>
650  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple<_Tp, _A0, _A1, _Abis...>
651  __make_simd_tuple(
652  const typename _SimdTraits<_Tp, _A0>::_SimdMember& __arg0,
653  const typename _SimdTraits<_Tp, _A1>::_SimdMember& __arg1,
654  const typename _SimdTraits<_Tp, _Abis>::_SimdMember&... __args)
655  { return {__arg0, __make_simd_tuple<_Tp, _A1, _Abis...>(__arg1, __args...)}; }
656 
657 // __to_simd_tuple {{{1
658 template <typename _Tp, size_t _Np, typename _V, size_t _NV, typename... _VX>
659  _GLIBCXX_SIMD_INTRINSIC constexpr __fixed_size_storage_t<_Tp, _Np>
660  __to_simd_tuple(const array<_V, _NV>& __from, const _VX... __fromX);
661 
662 template <typename _Tp, size_t _Np,
663  size_t _Offset = 0, // skip this many elements in __from0
664  typename _R = __fixed_size_storage_t<_Tp, _Np>, typename _V0,
665  typename _V0VT = _VectorTraits<_V0>, typename... _VX>
666  _GLIBCXX_SIMD_INTRINSIC _R constexpr __to_simd_tuple(const _V0 __from0, const _VX... __fromX)
667  {
668  static_assert(is_same_v<typename _V0VT::value_type, _Tp>);
669  static_assert(_Offset < _V0VT::_S_full_size);
670  using _R0 = __vector_type_t<_Tp, _R::_S_first_size>;
671  if constexpr (_R::_S_tuple_size == 1)
672  {
673  if constexpr (_Np == 1)
674  return _R{__from0[_Offset]};
675  else if constexpr (_Offset == 0 && _V0VT::_S_full_size >= _Np)
676  return _R{__intrin_bitcast<_R0>(__from0)};
677  else if constexpr (_Offset * 2 == _V0VT::_S_full_size
678  && _V0VT::_S_full_size / 2 >= _Np)
679  return _R{__intrin_bitcast<_R0>(__extract_part<1, 2>(__from0))};
680  else if constexpr (_Offset * 4 == _V0VT::_S_full_size
681  && _V0VT::_S_full_size / 4 >= _Np)
682  return _R{__intrin_bitcast<_R0>(__extract_part<1, 4>(__from0))};
683  else
684  __assert_unreachable<_Tp>();
685  }
686  else
687  {
688  if constexpr (1 == _R::_S_first_size)
689  { // extract one scalar and recurse
690  if constexpr (_Offset + 1 < _V0VT::_S_full_size)
691  return _R{__from0[_Offset],
692  __to_simd_tuple<_Tp, _Np - 1, _Offset + 1>(__from0,
693  __fromX...)};
694  else
695  return _R{__from0[_Offset],
696  __to_simd_tuple<_Tp, _Np - 1, 0>(__fromX...)};
697  }
698 
699  // place __from0 into _R::first and recurse for __fromX -> _R::second
700  else if constexpr (_V0VT::_S_full_size == _R::_S_first_size
701  && _Offset == 0)
702  return _R{__from0,
703  __to_simd_tuple<_Tp, _Np - _R::_S_first_size>(__fromX...)};
704 
705  // place lower part of __from0 into _R::first and recurse with _Offset
706  else if constexpr (_V0VT::_S_full_size > _R::_S_first_size
707  && _Offset == 0)
708  return _R{__intrin_bitcast<_R0>(__from0),
709  __to_simd_tuple<_Tp, _Np - _R::_S_first_size,
710  _R::_S_first_size>(__from0, __fromX...)};
711 
712  // place lower part of second quarter of __from0 into _R::first and
713  // recurse with _Offset
714  else if constexpr (_Offset * 4 == _V0VT::_S_full_size
715  && _V0VT::_S_full_size >= 4 * _R::_S_first_size)
716  return _R{__intrin_bitcast<_R0>(__extract_part<2, 4>(__from0)),
717  __to_simd_tuple<_Tp, _Np - _R::_S_first_size,
718  _Offset + _R::_S_first_size>(__from0,
719  __fromX...)};
720 
721  // place lower half of high half of __from0 into _R::first and recurse
722  // with _Offset
723  else if constexpr (_Offset * 2 == _V0VT::_S_full_size
724  && _V0VT::_S_full_size >= 4 * _R::_S_first_size)
725  return _R{__intrin_bitcast<_R0>(__extract_part<2, 4>(__from0)),
726  __to_simd_tuple<_Tp, _Np - _R::_S_first_size,
727  _Offset + _R::_S_first_size>(__from0,
728  __fromX...)};
729 
730  // place high half of __from0 into _R::first and recurse with __fromX
731  else if constexpr (_Offset * 2 == _V0VT::_S_full_size
732  && _V0VT::_S_full_size / 2 >= _R::_S_first_size)
733  return _R{__intrin_bitcast<_R0>(__extract_part<1, 2>(__from0)),
734  __to_simd_tuple<_Tp, _Np - _R::_S_first_size, 0>(
735  __fromX...)};
736 
737  // ill-formed if some unforseen pattern is needed
738  else
739  __assert_unreachable<_Tp>();
740  }
741  }
742 
743 template <typename _Tp, size_t _Np, typename _V, size_t _NV, typename... _VX>
744  _GLIBCXX_SIMD_INTRINSIC constexpr __fixed_size_storage_t<_Tp, _Np>
745  __to_simd_tuple(const array<_V, _NV>& __from, const _VX... __fromX)
746  {
747  if constexpr (is_same_v<_Tp, _V>)
748  {
749  static_assert(
750  sizeof...(_VX) == 0,
751  "An array of scalars must be the last argument to __to_simd_tuple");
752  return __call_with_subscripts(
753  __from, make_index_sequence<_NV>(),
754  [&](const auto... __args) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
755  return __simd_tuple_concat(
756  _SimdTuple<_Tp, simd_abi::scalar>{__args}..., _SimdTuple<_Tp>());
757  });
758  }
759  else
760  return __call_with_subscripts(
761  __from, make_index_sequence<_NV>(),
762  [&](const auto... __args) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
763  return __to_simd_tuple<_Tp, _Np>(__args..., __fromX...);
764  });
765  }
766 
767 template <size_t, typename _Tp>
768  using __to_tuple_helper = _Tp;
769 
770 template <typename _Tp, typename _A0, size_t _NOut, size_t _Np,
771  size_t... _Indexes>
772  _GLIBCXX_SIMD_INTRINSIC __fixed_size_storage_t<_Tp, _NOut>
773  __to_simd_tuple_impl(index_sequence<_Indexes...>,
774  const array<__vector_type_t<_Tp, simd_size_v<_Tp, _A0>>, _Np>& __args)
775  {
776  return __make_simd_tuple<_Tp, __to_tuple_helper<_Indexes, _A0>...>(
777  __args[_Indexes]...);
778  }
779 
780 template <typename _Tp, typename _A0, size_t _NOut, size_t _Np,
781  typename _R = __fixed_size_storage_t<_Tp, _NOut>>
782  _GLIBCXX_SIMD_INTRINSIC _R
783  __to_simd_tuple_sized(
784  const array<__vector_type_t<_Tp, simd_size_v<_Tp, _A0>>, _Np>& __args)
785  {
786  static_assert(_Np * simd_size_v<_Tp, _A0> >= _NOut);
787  return __to_simd_tuple_impl<_Tp, _A0, _NOut>(
788  make_index_sequence<_R::_S_tuple_size>(), __args);
789  }
790 
791 // __optimize_simd_tuple {{{1
792 template <typename _Tp>
793  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple<_Tp>
794  __optimize_simd_tuple(const _SimdTuple<_Tp>)
795  { return {}; }
796 
797 template <typename _Tp, typename _Ap>
798  _GLIBCXX_SIMD_INTRINSIC constexpr const _SimdTuple<_Tp, _Ap>&
799  __optimize_simd_tuple(const _SimdTuple<_Tp, _Ap>& __x)
800  { return __x; }
801 
802 template <typename _Tp, typename _A0, typename _A1, typename... _Abis,
803  typename _R = __fixed_size_storage_t<
804  _Tp, _SimdTuple<_Tp, _A0, _A1, _Abis...>::_S_size()>>
805  _GLIBCXX_SIMD_INTRINSIC constexpr _R
806  __optimize_simd_tuple(const _SimdTuple<_Tp, _A0, _A1, _Abis...>& __x)
807  {
808  using _Tup = _SimdTuple<_Tp, _A0, _A1, _Abis...>;
809  if constexpr (is_same_v<_R, _Tup>)
810  return __x;
811  else if constexpr (is_same_v<typename _R::_FirstType,
812  typename _Tup::_FirstType>)
813  return {__x.first, __optimize_simd_tuple(__x.second)};
814  else if constexpr (__is_scalar_abi<_A0>()
815  || _A0::template _S_is_partial<_Tp>)
816  return {__generate_from_n_evaluations<_R::_S_first_size,
817  typename _R::_FirstType>(
818  [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; }),
819  __optimize_simd_tuple(
820  __simd_tuple_pop_front<_R::_S_first_size>(__x))};
821  else if constexpr (is_same_v<_A0, _A1>
822  && _R::_S_first_size == simd_size_v<_Tp, _A0> + simd_size_v<_Tp, _A1>)
823  return {__concat(__x.template _M_at<0>(), __x.template _M_at<1>()),
824  __optimize_simd_tuple(__x.second.second)};
825  else if constexpr (sizeof...(_Abis) >= 2
826  && _R::_S_first_size == (4 * simd_size_v<_Tp, _A0>)
827  && simd_size_v<_Tp, _A0> == __simd_tuple_element_t<
828  (sizeof...(_Abis) >= 2 ? 3 : 0), _Tup>::size())
829  return {
830  __concat(__concat(__x.template _M_at<0>(), __x.template _M_at<1>()),
831  __concat(__x.template _M_at<2>(), __x.template _M_at<3>())),
832  __optimize_simd_tuple(__x.second.second.second.second)};
833  else
834  {
835  static_assert(sizeof(_R) == sizeof(__x));
836  _R __r;
837  __builtin_memcpy(__r._M_as_charptr(), __x._M_as_charptr(),
838  sizeof(_Tp) * _R::_S_size());
839  return __r;
840  }
841  }
842 
843 // __for_each(const _SimdTuple &, Fun) {{{1
844 template <size_t _Offset = 0, typename _Tp, typename _A0, typename _Fp>
845  _GLIBCXX_SIMD_INTRINSIC constexpr void
846  __for_each(const _SimdTuple<_Tp, _A0>& __t, _Fp&& __fun)
847  { static_cast<_Fp&&>(__fun)(__make_meta<_Offset>(__t), __t.first); }
848 
849 template <size_t _Offset = 0, typename _Tp, typename _A0, typename _A1,
850  typename... _As, typename _Fp>
851  _GLIBCXX_SIMD_INTRINSIC constexpr void
852  __for_each(const _SimdTuple<_Tp, _A0, _A1, _As...>& __t, _Fp&& __fun)
853  {
854  __fun(__make_meta<_Offset>(__t), __t.first);
855  __for_each<_Offset + simd_size<_Tp, _A0>::value>(__t.second,
856  static_cast<_Fp&&>(__fun));
857  }
858 
859 // __for_each(_SimdTuple &, Fun) {{{1
860 template <size_t _Offset = 0, typename _Tp, typename _A0, typename _Fp>
861  _GLIBCXX_SIMD_INTRINSIC constexpr void
862  __for_each(_SimdTuple<_Tp, _A0>& __t, _Fp&& __fun)
863  { static_cast<_Fp&&>(__fun)(__make_meta<_Offset>(__t), __t.first); }
864 
865 template <size_t _Offset = 0, typename _Tp, typename _A0, typename _A1,
866  typename... _As, typename _Fp>
867  _GLIBCXX_SIMD_INTRINSIC constexpr void
868  __for_each(_SimdTuple<_Tp, _A0, _A1, _As...>& __t, _Fp&& __fun)
869  {
870  __fun(__make_meta<_Offset>(__t), __t.first);
871  __for_each<_Offset + simd_size<_Tp, _A0>::value>(__t.second,
872  static_cast<_Fp&&>(__fun));
873  }
874 
875 // __for_each(_SimdTuple &, const _SimdTuple &, Fun) {{{1
876 template <size_t _Offset = 0, typename _Tp, typename _A0, typename _Fp>
877  _GLIBCXX_SIMD_INTRINSIC constexpr void
878  __for_each(_SimdTuple<_Tp, _A0>& __a, const _SimdTuple<_Tp, _A0>& __b, _Fp&& __fun)
879  { static_cast<_Fp&&>(__fun)(__make_meta<_Offset>(__a), __a.first, __b.first); }
880 
881 template <size_t _Offset = 0, typename _Tp, typename _A0, typename _A1,
882  typename... _As, typename _Fp>
883  _GLIBCXX_SIMD_INTRINSIC constexpr void
884  __for_each(_SimdTuple<_Tp, _A0, _A1, _As...>& __a,
885  const _SimdTuple<_Tp, _A0, _A1, _As...>& __b, _Fp&& __fun)
886  {
887  __fun(__make_meta<_Offset>(__a), __a.first, __b.first);
888  __for_each<_Offset + simd_size<_Tp, _A0>::value>(__a.second, __b.second,
889  static_cast<_Fp&&>(__fun));
890  }
891 
892 // __for_each(const _SimdTuple &, const _SimdTuple &, Fun) {{{1
893 template <size_t _Offset = 0, typename _Tp, typename _A0, typename _Fp>
894  _GLIBCXX_SIMD_INTRINSIC constexpr void
895  __for_each(const _SimdTuple<_Tp, _A0>& __a, const _SimdTuple<_Tp, _A0>& __b, _Fp&& __fun)
896  { static_cast<_Fp&&>(__fun)(__make_meta<_Offset>(__a), __a.first, __b.first); }
897 
898 template <size_t _Offset = 0, typename _Tp, typename _A0, typename _A1,
899  typename... _As, typename _Fp>
900  _GLIBCXX_SIMD_INTRINSIC constexpr void
901  __for_each(const _SimdTuple<_Tp, _A0, _A1, _As...>& __a,
902  const _SimdTuple<_Tp, _A0, _A1, _As...>& __b, _Fp&& __fun)
903  {
904  __fun(__make_meta<_Offset>(__a), __a.first, __b.first);
905  __for_each<_Offset + simd_size<_Tp, _A0>::value>(__a.second, __b.second,
906  static_cast<_Fp&&>(__fun));
907  }
908 
909 // }}}1
910 // __extract_part(_SimdTuple) {{{
911 template <int _Index, int _Total, int _Combine, typename _Tp, typename _A0, typename... _As>
912  _GLIBCXX_SIMD_INTRINSIC constexpr auto // __vector_type_t or _SimdTuple
913  __extract_part(const _SimdTuple<_Tp, _A0, _As...>& __x)
914  {
915  // worst cases:
916  // (a) 4, 4, 4 => 3, 3, 3, 3 (_Total = 4)
917  // (b) 2, 2, 2 => 3, 3 (_Total = 2)
918  // (c) 4, 2 => 2, 2, 2 (_Total = 3)
919  using _Tuple = _SimdTuple<_Tp, _A0, _As...>;
920  static_assert(_Index + _Combine <= _Total && _Index >= 0 && _Total >= 1);
921  constexpr size_t _Np = _Tuple::_S_size();
922  static_assert(_Np >= _Total && _Np % _Total == 0);
923  constexpr size_t __values_per_part = _Np / _Total;
924  [[maybe_unused]] constexpr size_t __values_to_skip
925  = _Index * __values_per_part;
926  constexpr size_t __return_size = __values_per_part * _Combine;
927  using _RetAbi = simd_abi::deduce_t<_Tp, __return_size>;
928 
929  // handle (optimize) the simple cases
930  if constexpr (__return_size == 1)
931  return __x[integral_constant<size_t, __values_to_skip>()];
932  else if constexpr (_Index == 0 && _Tuple::_S_first_size == __return_size)
933  return __x.first._M_data;
934  else if constexpr (_Index == 0 && _Total == _Combine)
935  return __x;
936  else if constexpr (_Index == 0 && _Tuple::_S_first_size >= __return_size)
937  return __intrin_bitcast<__vector_type_t<_Tp, __return_size>>(
938  __as_vector(__x.first));
939 
940  // recurse to skip unused data members at the beginning of _SimdTuple
941  else if constexpr (__values_to_skip >= _Tuple::_S_first_size)
942  { // recurse
943  if constexpr (_Tuple::_S_first_size % __values_per_part == 0)
944  {
945  constexpr int __parts_in_first
946  = _Tuple::_S_first_size / __values_per_part;
947  return __extract_part<_Index - __parts_in_first,
948  _Total - __parts_in_first, _Combine>(
949  __x.second);
950  }
951  else
952  return __extract_part<__values_to_skip - _Tuple::_S_first_size,
953  _Np - _Tuple::_S_first_size, __return_size>(
954  __x.second);
955  }
956 
957  // extract from multiple _SimdTuple data members
958  else if constexpr (__return_size > _Tuple::_S_first_size - __values_to_skip)
959  {
960 #ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
961  const __may_alias<_Tp>* const element_ptr
962  = reinterpret_cast<const __may_alias<_Tp>*>(&__x) + __values_to_skip;
963  return __as_vector(simd<_Tp, _RetAbi>(element_ptr, element_aligned));
964 #else
965  [[maybe_unused]] constexpr size_t __offset = __values_to_skip;
966  return __as_vector(simd<_Tp, _RetAbi>(
967  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
968  constexpr _SizeConstant<__i + __offset> __k;
969  return __x[__k];
970  }));
971 #endif
972  }
973 
974  // all of the return values are in __x.first
975  else if constexpr (_Tuple::_S_first_size % __values_per_part == 0)
976  return __extract_part<_Index, _Tuple::_S_first_size / __values_per_part,
977  _Combine>(__x.first);
978  else
979  return __extract_part<__values_to_skip, _Tuple::_S_first_size,
980  _Combine * __values_per_part>(__x.first);
981  }
982 
983 // }}}
984 // __fixed_size_storage_t<_Tp, _Np>{{{
985 template <typename _Tp, int _Np, typename _Tuple,
986  typename _Next = simd<_Tp, _AllNativeAbis::_BestAbi<_Tp, _Np>>,
987  int _Remain = _Np - int(_Next::size())>
988  struct __fixed_size_storage_builder;
989 
990 template <typename _Tp, int _Np>
991  struct __fixed_size_storage
992  : public __fixed_size_storage_builder<_Tp, _Np, _SimdTuple<_Tp>> {};
993 
994 template <typename _Tp, int _Np, typename... _As, typename _Next>
995  struct __fixed_size_storage_builder<_Tp, _Np, _SimdTuple<_Tp, _As...>, _Next,
996  0>
997  { using type = _SimdTuple<_Tp, _As..., typename _Next::abi_type>; };
998 
999 template <typename _Tp, int _Np, typename... _As, typename _Next, int _Remain>
1000  struct __fixed_size_storage_builder<_Tp, _Np, _SimdTuple<_Tp, _As...>, _Next,
1001  _Remain>
1002  {
1003  using type = typename __fixed_size_storage_builder<
1004  _Tp, _Remain, _SimdTuple<_Tp, _As..., typename _Next::abi_type>>::type;
1005  };
1006 
1007 // }}}
1008 // __autocvt_to_simd {{{
1009 template <typename _Tp, bool = is_arithmetic_v<__remove_cvref_t<_Tp>>>
1010  struct __autocvt_to_simd
1011  {
1012  _Tp _M_data;
1013  using _TT = __remove_cvref_t<_Tp>;
1014 
1015  _GLIBCXX_SIMD_INTRINSIC constexpr
1016  operator _TT()
1017  { return _M_data; }
1018 
1019  _GLIBCXX_SIMD_INTRINSIC constexpr
1020  operator _TT&()
1021  {
1022  static_assert(is_lvalue_reference<_Tp>::value, "");
1023  static_assert(!is_const<_Tp>::value, "");
1024  return _M_data;
1025  }
1026 
1027  _GLIBCXX_SIMD_INTRINSIC constexpr
1028  operator _TT*()
1029  {
1030  static_assert(is_lvalue_reference<_Tp>::value, "");
1031  static_assert(!is_const<_Tp>::value, "");
1032  return &_M_data;
1033  }
1034 
1035  _GLIBCXX_SIMD_INTRINSIC constexpr
1036  __autocvt_to_simd(_Tp dd) : _M_data(dd) {}
1037 
1038  template <typename _Abi>
1039  _GLIBCXX_SIMD_INTRINSIC constexpr
1040  operator simd<typename _TT::value_type, _Abi>()
1041  { return {__private_init, _M_data}; }
1042 
1043  template <typename _Abi>
1044  _GLIBCXX_SIMD_INTRINSIC constexpr
1045  operator simd<typename _TT::value_type, _Abi>&()
1046  { return *reinterpret_cast<simd<typename _TT::value_type, _Abi>*>(&_M_data); }
1047 
1048  template <typename _Abi>
1049  _GLIBCXX_SIMD_INTRINSIC constexpr
1050  operator simd<typename _TT::value_type, _Abi>*()
1051  { return reinterpret_cast<simd<typename _TT::value_type, _Abi>*>(&_M_data); }
1052  };
1053 
1054 template <typename _Tp>
1055  __autocvt_to_simd(_Tp &&) -> __autocvt_to_simd<_Tp>;
1056 
1057 template <typename _Tp>
1058  struct __autocvt_to_simd<_Tp, true>
1059  {
1060  using _TT = __remove_cvref_t<_Tp>;
1061  _Tp _M_data;
1062  fixed_size_simd<_TT, 1> _M_fd;
1063 
1064  _GLIBCXX_SIMD_INTRINSIC
1065  constexpr __autocvt_to_simd(_Tp dd) : _M_data(dd), _M_fd(_M_data) {}
1066 
1067  _GLIBCXX_SIMD_INTRINSIC
1068  ~__autocvt_to_simd()
1069  { _M_data = __data(_M_fd).first; }
1070 
1071  _GLIBCXX_SIMD_INTRINSIC constexpr
1072  operator fixed_size_simd<_TT, 1>()
1073  { return _M_fd; }
1074 
1075  _GLIBCXX_SIMD_INTRINSIC constexpr
1076  operator fixed_size_simd<_TT, 1> &()
1077  {
1078  static_assert(is_lvalue_reference<_Tp>::value, "");
1079  static_assert(!is_const<_Tp>::value, "");
1080  return _M_fd;
1081  }
1082 
1083  _GLIBCXX_SIMD_INTRINSIC constexpr
1084  operator fixed_size_simd<_TT, 1> *()
1085  {
1086  static_assert(is_lvalue_reference<_Tp>::value, "");
1087  static_assert(!is_const<_Tp>::value, "");
1088  return &_M_fd;
1089  }
1090  };
1091 
1092 // }}}
1093 
1094 struct _CommonImplFixedSize;
1095 template <int _Np, typename = __detail::__odr_helper> struct _SimdImplFixedSize;
1096 template <int _Np, typename = __detail::__odr_helper> struct _MaskImplFixedSize;
1097 // simd_abi::_Fixed {{{
1098 template <int _Np>
1099  struct simd_abi::_Fixed
1100  {
1101  template <typename _Tp> static constexpr size_t _S_size = _Np;
1102  template <typename _Tp> static constexpr size_t _S_full_size = _Np;
1103  // validity traits {{{
1104  struct _IsValidAbiTag : public __bool_constant<(_Np > 0)> {};
1105 
1106  template <typename _Tp>
1107  struct _IsValidSizeFor
1108  : __bool_constant<(_Np <= simd_abi::max_fixed_size<_Tp>)> {};
1109 
1110  template <typename _Tp>
1111  struct _IsValid : conjunction<_IsValidAbiTag, __is_vectorizable<_Tp>,
1112  _IsValidSizeFor<_Tp>> {};
1113 
1114  template <typename _Tp>
1115  static constexpr bool _S_is_valid_v = _IsValid<_Tp>::value;
1116 
1117  // }}}
1118  // _S_masked {{{
1119  _GLIBCXX_SIMD_INTRINSIC static constexpr _SanitizedBitMask<_Np>
1120  _S_masked(_BitMask<_Np> __x)
1121  { return __x._M_sanitized(); }
1122 
1123  _GLIBCXX_SIMD_INTRINSIC static constexpr _SanitizedBitMask<_Np>
1124  _S_masked(_SanitizedBitMask<_Np> __x)
1125  { return __x; }
1126 
1127  // }}}
1128  // _*Impl {{{
1129  using _CommonImpl = _CommonImplFixedSize;
1130  using _SimdImpl = _SimdImplFixedSize<_Np>;
1131  using _MaskImpl = _MaskImplFixedSize<_Np>;
1132 
1133  // }}}
1134  // __traits {{{
1135  template <typename _Tp, bool = _S_is_valid_v<_Tp>>
1136  struct __traits : _InvalidTraits {};
1137 
1138  template <typename _Tp>
1139  struct __traits<_Tp, true>
1140  {
1141  using _IsValid = true_type;
1142  using _SimdImpl = _SimdImplFixedSize<_Np>;
1143  using _MaskImpl = _MaskImplFixedSize<_Np>;
1144 
1145  // simd and simd_mask member types {{{
1146  using _SimdMember = __fixed_size_storage_t<_Tp, _Np>;
1147  using _MaskMember = _SanitizedBitMask<_Np>;
1148 
1149  static constexpr size_t _S_simd_align
1150  = std::__bit_ceil(_Np * sizeof(_Tp));
1151 
1152  static constexpr size_t _S_mask_align = alignof(_MaskMember);
1153 
1154  // }}}
1155  // _SimdBase / base class for simd, providing extra conversions {{{
1156  struct _SimdBase
1157  {
1158  // The following ensures, function arguments are passed via the stack.
1159  // This is important for ABI compatibility across TU boundaries
1160  _GLIBCXX_SIMD_ALWAYS_INLINE constexpr
1161  _SimdBase(const _SimdBase&) {}
1162 
1163  _SimdBase() = default;
1164 
1165  _GLIBCXX_SIMD_ALWAYS_INLINE constexpr explicit
1166  operator const _SimdMember &() const
1167  { return static_cast<const simd<_Tp, _Fixed>*>(this)->_M_data; }
1168 
1169  _GLIBCXX_SIMD_ALWAYS_INLINE constexpr explicit
1170  operator array<_Tp, _Np>() const
1171  {
1172  array<_Tp, _Np> __r;
1173  // _SimdMember can be larger because of higher alignment
1174  static_assert(sizeof(__r) <= sizeof(_SimdMember), "");
1175  __builtin_memcpy(__r.data(), &static_cast<const _SimdMember&>(*this),
1176  sizeof(__r));
1177  return __r;
1178  }
1179  };
1180 
1181  // }}}
1182  // _MaskBase {{{
1183  // empty. The bitset interface suffices
1184  struct _MaskBase {};
1185 
1186  // }}}
1187  // _SimdCastType {{{
1188  struct _SimdCastType
1189  {
1190  _GLIBCXX_SIMD_ALWAYS_INLINE constexpr
1191  _SimdCastType(const array<_Tp, _Np>&);
1192 
1193  _GLIBCXX_SIMD_ALWAYS_INLINE constexpr
1194  _SimdCastType(const _SimdMember& dd) : _M_data(dd) {}
1195 
1196  _GLIBCXX_SIMD_ALWAYS_INLINE constexpr explicit
1197  operator const _SimdMember &() const { return _M_data; }
1198 
1199  private:
1200  const _SimdMember& _M_data;
1201  };
1202 
1203  // }}}
1204  // _MaskCastType {{{
1205  class _MaskCastType
1206  {
1207  _MaskCastType() = delete;
1208  };
1209  // }}}
1210  };
1211  // }}}
1212  };
1213 
1214 // }}}
1215 // _CommonImplFixedSize {{{
1216 struct _CommonImplFixedSize
1217 {
1218  // _S_store {{{
1219  template <typename _Tp, typename... _As>
1220  _GLIBCXX_SIMD_INTRINSIC static void
1221  _S_store(const _SimdTuple<_Tp, _As...>& __x, void* __addr)
1222  {
1223  constexpr size_t _Np = _SimdTuple<_Tp, _As...>::_S_size();
1224  __builtin_memcpy(__addr, &__x, _Np * sizeof(_Tp));
1225  }
1226 
1227  // }}}
1228 };
1229 
1230 // }}}
1231 // _SimdImplFixedSize {{{1
1232 // fixed_size should not inherit from _SimdMathFallback in order for
1233 // specializations in the used _SimdTuple Abis to get used
1234 template <int _Np, typename>
1235  struct _SimdImplFixedSize
1236  {
1237  // member types {{{2
1238  using _MaskMember = _SanitizedBitMask<_Np>;
1239 
1240  template <typename _Tp>
1241  using _SimdMember = __fixed_size_storage_t<_Tp, _Np>;
1242 
1243  template <typename _Tp>
1244  static constexpr size_t _S_tuple_size = _SimdMember<_Tp>::_S_tuple_size;
1245 
1246  template <typename _Tp>
1247  using _Simd = simd<_Tp, simd_abi::fixed_size<_Np>>;
1248 
1249  template <typename _Tp>
1250  using _TypeTag = _Tp*;
1251 
1252  // broadcast {{{2
1253  template <typename _Tp>
1254  _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdMember<_Tp>
1255  _S_broadcast(_Tp __x) noexcept
1256  {
1257  return _SimdMember<_Tp>::_S_generate(
1258  [&](auto __meta) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1259  return __meta._S_broadcast(__x);
1260  });
1261  }
1262 
1263  // _S_generator {{{2
1264  template <typename _Fp, typename _Tp>
1265  _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdMember<_Tp>
1266  _S_generator(_Fp&& __gen, _TypeTag<_Tp>)
1267  {
1268  return _SimdMember<_Tp>::_S_generate(
1269  [&__gen](auto __meta) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1270  return __meta._S_generator(
1271  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1272  return __i < _Np ? __gen(_SizeConstant<__meta._S_offset + __i>())
1273  : 0;
1274  },
1275  _TypeTag<_Tp>());
1276  });
1277  }
1278 
1279  // _S_load {{{2
1280  template <typename _Tp, typename _Up>
1281  _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdMember<_Tp>
1282  _S_load(const _Up* __mem, _TypeTag<_Tp>) noexcept
1283  {
1284  return _SimdMember<_Tp>::_S_generate(
1285  [&](auto __meta) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1286  return __meta._S_load(&__mem[__meta._S_offset], _TypeTag<_Tp>());
1287  });
1288  }
1289 
1290  // _S_masked_load {{{2
1291  template <typename _Tp, typename... _As, typename _Up>
1292  _GLIBCXX_SIMD_INTRINSIC static _SimdTuple<_Tp, _As...>
1293  _S_masked_load(const _SimdTuple<_Tp, _As...>& __old,
1294  const _MaskMember __bits, const _Up* __mem) noexcept
1295  {
1296  auto __merge = __old;
1297  __for_each(__merge, [&](auto __meta, auto& __native) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1298  if (__meta._S_submask(__bits).any())
1299 #pragma GCC diagnostic push
1300  // Dereferencing __mem + __meta._S_offset could be UB ([expr.add]/4.3).
1301  // It is the responsibility of the caller of the masked load (via the mask's value) to
1302  // avoid UB. Consequently, the compiler may assume this branch is unreachable, if the
1303  // pointer arithmetic is UB.
1304 #pragma GCC diagnostic ignored "-Warray-bounds"
1305  __native
1306  = __meta._S_masked_load(__native, __meta._S_make_mask(__bits),
1307  __mem + __meta._S_offset);
1308 #pragma GCC diagnostic pop
1309  });
1310  return __merge;
1311  }
1312 
1313  // _S_store {{{2
1314  template <typename _Tp, typename _Up>
1315  _GLIBCXX_SIMD_INTRINSIC static constexpr void
1316  _S_store(const _SimdMember<_Tp>& __v, _Up* __mem, _TypeTag<_Tp>) noexcept
1317  {
1318  __for_each(__v, [&](auto __meta, auto __native) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1319  __meta._S_store(__native, &__mem[__meta._S_offset], _TypeTag<_Tp>());
1320  });
1321  }
1322 
1323  // _S_masked_store {{{2
1324  template <typename _Tp, typename... _As, typename _Up>
1325  _GLIBCXX_SIMD_INTRINSIC static void
1326  _S_masked_store(const _SimdTuple<_Tp, _As...>& __v, _Up* __mem,
1327  const _MaskMember __bits) noexcept
1328  {
1329  __for_each(__v, [&](auto __meta, auto __native) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1330  if (__meta._S_submask(__bits).any())
1331 #pragma GCC diagnostic push
1332  // __mem + __mem._S_offset could be UB ([expr.add]/4.3, but it punts
1333  // the responsibility for avoiding UB to the caller of the masked
1334  // store via the mask. Consequently, the compiler may assume this
1335  // branch is unreachable, if the pointer arithmetic is UB.
1336 #pragma GCC diagnostic ignored "-Warray-bounds"
1337  __meta._S_masked_store(__native, __mem + __meta._S_offset,
1338  __meta._S_make_mask(__bits));
1339 #pragma GCC diagnostic pop
1340  });
1341  }
1342 
1343  // negation {{{2
1344  template <typename _Tp, typename... _As>
1345  static constexpr inline _MaskMember
1346  _S_negate(const _SimdTuple<_Tp, _As...>& __x) noexcept
1347  {
1348  _MaskMember __bits = 0;
1349  __for_each(
1350  __x, [&__bits](auto __meta, auto __native) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1351  __bits
1352  |= __meta._S_mask_to_shifted_ullong(__meta._S_negate(__native));
1353  });
1354  return __bits;
1355  }
1356 
1357  // reductions {{{2
1358  template <typename _Tp, typename _BinaryOperation>
1359  static constexpr inline _Tp _S_reduce(const _Simd<_Tp>& __x,
1360  const _BinaryOperation& __binary_op)
1361  {
1362  using _Tup = _SimdMember<_Tp>;
1363  const _Tup& __tup = __data(__x);
1364  if constexpr (_Tup::_S_tuple_size == 1)
1365  return _Tup::_FirstAbi::_SimdImpl::_S_reduce(
1366  __tup.template _M_simd_at<0>(), __binary_op);
1367  else if constexpr (_Tup::_S_tuple_size == 2 && _Tup::_S_size() > 2
1368  && _Tup::_SecondType::_S_size() == 1)
1369  {
1370  return __binary_op(simd<_Tp, simd_abi::scalar>(
1371  reduce(__tup.template _M_simd_at<0>(),
1372  __binary_op)),
1373  __tup.template _M_simd_at<1>())[0];
1374  }
1375  else if constexpr (_Tup::_S_tuple_size == 2 && _Tup::_S_size() > 4
1376  && _Tup::_SecondType::_S_size() == 2)
1377  {
1378  return __binary_op(
1379  simd<_Tp, simd_abi::scalar>(
1380  reduce(__tup.template _M_simd_at<0>(), __binary_op)),
1381  simd<_Tp, simd_abi::scalar>(
1382  reduce(__tup.template _M_simd_at<1>(), __binary_op)))[0];
1383  }
1384  else
1385  {
1386  const auto& __x2 = __call_with_n_evaluations<
1387  __div_roundup(_Tup::_S_tuple_size, 2)>(
1388  [](auto __first_simd, auto... __remaining) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1389  if constexpr (sizeof...(__remaining) == 0)
1390  return __first_simd;
1391  else
1392  {
1393  using _Tup2
1394  = _SimdTuple<_Tp,
1395  typename decltype(__first_simd)::abi_type,
1396  typename decltype(__remaining)::abi_type...>;
1397  return fixed_size_simd<_Tp, _Tup2::_S_size()>(
1398  __private_init,
1399  __make_simd_tuple(__first_simd, __remaining...));
1400  }
1401  },
1402  [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1403  auto __left = __tup.template _M_simd_at<2 * __i>();
1404  if constexpr (2 * __i + 1 == _Tup::_S_tuple_size)
1405  return __left;
1406  else
1407  {
1408  auto __right = __tup.template _M_simd_at<2 * __i + 1>();
1409  using _LT = decltype(__left);
1410  using _RT = decltype(__right);
1411  if constexpr (_LT::size() == _RT::size())
1412  return __binary_op(__left, __right);
1413  else
1414  {
1415  _GLIBCXX_SIMD_USE_CONSTEXPR_API
1416  typename _LT::mask_type __k(
1417  __private_init,
1418  [](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1419  return __j < _RT::size();
1420  });
1421  _LT __ext_right = __left;
1422  where(__k, __ext_right)
1423  = __proposed::resizing_simd_cast<_LT>(__right);
1424  where(__k, __left) = __binary_op(__left, __ext_right);
1425  return __left;
1426  }
1427  }
1428  });
1429  return reduce(__x2, __binary_op);
1430  }
1431  }
1432 
1433  // _S_min, _S_max {{{2
1434  template <typename _Tp, typename... _As>
1435  _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple<_Tp, _As...>
1436  _S_min(const _SimdTuple<_Tp, _As...>& __a, const _SimdTuple<_Tp, _As...>& __b)
1437  {
1438  return __a._M_apply_per_chunk(
1439  [](auto __impl, auto __aa, auto __bb) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1440  return __impl._S_min(__aa, __bb);
1441  },
1442  __b);
1443  }
1444 
1445  template <typename _Tp, typename... _As>
1446  _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple<_Tp, _As...>
1447  _S_max(const _SimdTuple<_Tp, _As...>& __a, const _SimdTuple<_Tp, _As...>& __b)
1448  {
1449  return __a._M_apply_per_chunk(
1450  [](auto __impl, auto __aa, auto __bb) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1451  return __impl._S_max(__aa, __bb);
1452  },
1453  __b);
1454  }
1455 
1456  // _S_complement {{{2
1457  template <typename _Tp, typename... _As>
1458  _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple<_Tp, _As...>
1459  _S_complement(const _SimdTuple<_Tp, _As...>& __x) noexcept
1460  {
1461  return __x._M_apply_per_chunk(
1462  [](auto __impl, auto __xx) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1463  return __impl._S_complement(__xx);
1464  });
1465  }
1466 
1467  // _S_unary_minus {{{2
1468  template <typename _Tp, typename... _As>
1469  _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple<_Tp, _As...>
1470  _S_unary_minus(const _SimdTuple<_Tp, _As...>& __x) noexcept
1471  {
1472  return __x._M_apply_per_chunk(
1473  [](auto __impl, auto __xx) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1474  return __impl._S_unary_minus(__xx);
1475  });
1476  }
1477 
1478  // arithmetic operators {{{2
1479 
1480 #define _GLIBCXX_SIMD_FIXED_OP(name_, op_) \
1481  template <typename _Tp, typename... _As> \
1482  _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple<_Tp, _As...> name_( \
1483  const _SimdTuple<_Tp, _As...>& __x, const _SimdTuple<_Tp, _As...>& __y) \
1484  { \
1485  return __x._M_apply_per_chunk( \
1486  [](auto __impl, auto __xx, auto __yy) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { \
1487  return __impl.name_(__xx, __yy); \
1488  }, \
1489  __y); \
1490  }
1491 
1492  _GLIBCXX_SIMD_FIXED_OP(_S_plus, +)
1493  _GLIBCXX_SIMD_FIXED_OP(_S_minus, -)
1494  _GLIBCXX_SIMD_FIXED_OP(_S_multiplies, *)
1495  _GLIBCXX_SIMD_FIXED_OP(_S_divides, /)
1496  _GLIBCXX_SIMD_FIXED_OP(_S_modulus, %)
1497  _GLIBCXX_SIMD_FIXED_OP(_S_bit_and, &)
1498  _GLIBCXX_SIMD_FIXED_OP(_S_bit_or, |)
1499  _GLIBCXX_SIMD_FIXED_OP(_S_bit_xor, ^)
1500  _GLIBCXX_SIMD_FIXED_OP(_S_bit_shift_left, <<)
1501  _GLIBCXX_SIMD_FIXED_OP(_S_bit_shift_right, >>)
1502 #undef _GLIBCXX_SIMD_FIXED_OP
1503 
1504  template <typename _Tp, typename... _As>
1505  _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple<_Tp, _As...>
1506  _S_bit_shift_left(const _SimdTuple<_Tp, _As...>& __x, int __y)
1507  {
1508  return __x._M_apply_per_chunk(
1509  [__y](auto __impl, auto __xx) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1510  return __impl._S_bit_shift_left(__xx, __y);
1511  });
1512  }
1513 
1514  template <typename _Tp, typename... _As>
1515  _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple<_Tp, _As...>
1516  _S_bit_shift_right(const _SimdTuple<_Tp, _As...>& __x, int __y)
1517  {
1518  return __x._M_apply_per_chunk(
1519  [__y](auto __impl, auto __xx) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1520  return __impl._S_bit_shift_right(__xx, __y);
1521  });
1522  }
1523 
1524  // math {{{2
1525 #define _GLIBCXX_SIMD_APPLY_ON_TUPLE(_RetTp, __name) \
1526  template <typename _Tp, typename... _As, typename... _More> \
1527  static inline __fixed_size_storage_t<_RetTp, _Np> \
1528  _S_##__name(const _SimdTuple<_Tp, _As...>& __x, \
1529  const _More&... __more) \
1530  { \
1531  if constexpr (sizeof...(_More) == 0) \
1532  { \
1533  if constexpr (is_same_v<_Tp, _RetTp>) \
1534  return __x._M_apply_per_chunk( \
1535  [](auto __impl, auto __xx) \
1536  constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA \
1537  { \
1538  using _V = typename decltype(__impl)::simd_type; \
1539  return __data(__name(_V(__private_init, __xx))); \
1540  }); \
1541  else \
1542  return __optimize_simd_tuple( \
1543  __x.template _M_apply_r<_RetTp>( \
1544  [](auto __impl, auto __xx) \
1545  _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA \
1546  { return __impl._S_##__name(__xx); })); \
1547  } \
1548  else if constexpr ( \
1549  is_same_v< \
1550  _Tp, \
1551  _RetTp> && (... && is_same_v<_SimdTuple<_Tp, _As...>, _More>) ) \
1552  return __x._M_apply_per_chunk( \
1553  [](auto __impl, auto __xx, auto... __pack) \
1554  constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA \
1555  { \
1556  using _V = typename decltype(__impl)::simd_type; \
1557  return __data(__name(_V(__private_init, __xx), \
1558  _V(__private_init, __pack)...)); \
1559  }, __more...); \
1560  else if constexpr (is_same_v<_Tp, _RetTp>) \
1561  return __x._M_apply_per_chunk( \
1562  [](auto __impl, auto __xx, auto... __pack) \
1563  constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA \
1564  { \
1565  using _V = typename decltype(__impl)::simd_type; \
1566  return __data(__name(_V(__private_init, __xx), \
1567  __autocvt_to_simd(__pack)...)); \
1568  }, __more...); \
1569  else \
1570  __assert_unreachable<_Tp>(); \
1571  }
1572 
1573  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, acos)
1574  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, asin)
1575  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, atan)
1576  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, atan2)
1577  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, cos)
1578  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, sin)
1579  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, tan)
1580  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, acosh)
1581  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, asinh)
1582  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, atanh)
1583  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, cosh)
1584  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, sinh)
1585  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, tanh)
1586  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, exp)
1587  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, exp2)
1588  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, expm1)
1589  _GLIBCXX_SIMD_APPLY_ON_TUPLE(int, ilogb)
1590  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, log)
1591  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, log10)
1592  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, log1p)
1593  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, log2)
1594  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, logb)
1595  // modf implemented in simd_math.h
1596  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp,
1597  scalbn) // double scalbn(double x, int exp);
1598  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, scalbln)
1599  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, cbrt)
1600  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, abs)
1601  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fabs)
1602  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, pow)
1603  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, sqrt)
1604  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, erf)
1605  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, erfc)
1606  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, lgamma)
1607  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, tgamma)
1608  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, trunc)
1609  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, ceil)
1610  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, floor)
1611  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, nearbyint)
1612 
1613  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, rint)
1614  _GLIBCXX_SIMD_APPLY_ON_TUPLE(long, lrint)
1615  _GLIBCXX_SIMD_APPLY_ON_TUPLE(long long, llrint)
1616 
1617  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, round)
1618  _GLIBCXX_SIMD_APPLY_ON_TUPLE(long, lround)
1619  _GLIBCXX_SIMD_APPLY_ON_TUPLE(long long, llround)
1620 
1621  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, ldexp)
1622  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fmod)
1623  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, remainder)
1624  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, copysign)
1625  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, nextafter)
1626  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fdim)
1627  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fmax)
1628  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fmin)
1629  _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fma)
1630  _GLIBCXX_SIMD_APPLY_ON_TUPLE(int, fpclassify)
1631 #undef _GLIBCXX_SIMD_APPLY_ON_TUPLE
1632 
1633  template <typename _Tp, typename... _Abis>
1634  static inline _SimdTuple<_Tp, _Abis...>
1635  _S_remquo(const _SimdTuple<_Tp, _Abis...>& __x, const _SimdTuple<_Tp, _Abis...>& __y,
1636  __fixed_size_storage_t<int, _SimdTuple<_Tp, _Abis...>::_S_size()>* __z)
1637  {
1638  return __x._M_apply_per_chunk(
1639  [](auto __impl, const auto __xx, const auto __yy, auto& __zz)
1640  _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
1641  { return __impl._S_remquo(__xx, __yy, &__zz); },
1642  __y, *__z);
1643  }
1644 
1645  template <typename _Tp, typename... _As>
1646  static inline _SimdTuple<_Tp, _As...>
1647  _S_frexp(const _SimdTuple<_Tp, _As...>& __x,
1648  __fixed_size_storage_t<int, _Np>& __exp) noexcept
1649  {
1650  return __x._M_apply_per_chunk(
1651  [](auto __impl, const auto& __a, auto& __b) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1652  return __data(frexp(typename decltype(__impl)::simd_type(__private_init, __a),
1653  __autocvt_to_simd(__b)));
1654  }, __exp);
1655  }
1656 
1657 #define _GLIBCXX_SIMD_TEST_ON_TUPLE_(name_) \
1658  template <typename _Tp, typename... _As> \
1659  static inline _MaskMember \
1660  _S_##name_(const _SimdTuple<_Tp, _As...>& __x) noexcept \
1661  { \
1662  return _M_test([] (auto __impl, auto __xx) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { \
1663  return __impl._S_##name_(__xx); \
1664  }, __x); \
1665  }
1666 
1667  _GLIBCXX_SIMD_TEST_ON_TUPLE_(isinf)
1668  _GLIBCXX_SIMD_TEST_ON_TUPLE_(isfinite)
1669  _GLIBCXX_SIMD_TEST_ON_TUPLE_(isnan)
1670  _GLIBCXX_SIMD_TEST_ON_TUPLE_(isnormal)
1671  _GLIBCXX_SIMD_TEST_ON_TUPLE_(signbit)
1672 #undef _GLIBCXX_SIMD_TEST_ON_TUPLE_
1673 
1674  // _S_increment & _S_decrement{{{2
1675  template <typename... _Ts>
1676  _GLIBCXX_SIMD_INTRINSIC static constexpr void
1677  _S_increment(_SimdTuple<_Ts...>& __x)
1678  {
1679  __for_each(
1680  __x, [](auto __meta, auto& native) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1681  __meta._S_increment(native);
1682  });
1683  }
1684 
1685  template <typename... _Ts>
1686  _GLIBCXX_SIMD_INTRINSIC static constexpr void
1687  _S_decrement(_SimdTuple<_Ts...>& __x)
1688  {
1689  __for_each(
1690  __x, [](auto __meta, auto& native) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1691  __meta._S_decrement(native);
1692  });
1693  }
1694 
1695  // compares {{{2
1696 #define _GLIBCXX_SIMD_CMP_OPERATIONS(__cmp) \
1697  template <typename _Tp, typename... _As> \
1698  _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember \
1699  __cmp(const _SimdTuple<_Tp, _As...>& __x, \
1700  const _SimdTuple<_Tp, _As...>& __y) \
1701  { \
1702  return _M_test([](auto __impl, auto __xx, auto __yy) \
1703  constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA \
1704  { return __impl.__cmp(__xx, __yy); }, \
1705  __x, __y); \
1706  }
1707 
1708  _GLIBCXX_SIMD_CMP_OPERATIONS(_S_equal_to)
1709  _GLIBCXX_SIMD_CMP_OPERATIONS(_S_not_equal_to)
1710  _GLIBCXX_SIMD_CMP_OPERATIONS(_S_less)
1711  _GLIBCXX_SIMD_CMP_OPERATIONS(_S_less_equal)
1712  _GLIBCXX_SIMD_CMP_OPERATIONS(_S_isless)
1713  _GLIBCXX_SIMD_CMP_OPERATIONS(_S_islessequal)
1714  _GLIBCXX_SIMD_CMP_OPERATIONS(_S_isgreater)
1715  _GLIBCXX_SIMD_CMP_OPERATIONS(_S_isgreaterequal)
1716  _GLIBCXX_SIMD_CMP_OPERATIONS(_S_islessgreater)
1717  _GLIBCXX_SIMD_CMP_OPERATIONS(_S_isunordered)
1718 #undef _GLIBCXX_SIMD_CMP_OPERATIONS
1719 
1720  // smart_reference access {{{2
1721  template <typename _Tp, typename... _As, typename _Up>
1722  _GLIBCXX_SIMD_INTRINSIC static constexpr void
1723  _S_set(_SimdTuple<_Tp, _As...>& __v, int __i, _Up&& __x) noexcept
1724  { __v._M_set(__i, static_cast<_Up&&>(__x)); }
1725 
1726  // _S_masked_assign {{{2
1727  template <typename _Tp, typename... _As>
1728  _GLIBCXX_SIMD_INTRINSIC static constexpr void
1729  _S_masked_assign(const _MaskMember __bits, _SimdTuple<_Tp, _As...>& __lhs,
1730  const __type_identity_t<_SimdTuple<_Tp, _As...>>& __rhs)
1731  {
1732  __for_each(__lhs, __rhs,
1733  [&](auto __meta, auto& __native_lhs, auto __native_rhs)
1734  constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
1735  {
1736  __meta._S_masked_assign(__meta._S_make_mask(__bits), __native_lhs,
1737  __native_rhs);
1738  });
1739  }
1740 
1741  // Optimization for the case where the RHS is a scalar. No need to broadcast
1742  // the scalar to a simd first.
1743  template <typename _Tp, typename... _As>
1744  _GLIBCXX_SIMD_INTRINSIC static constexpr void
1745  _S_masked_assign(const _MaskMember __bits, _SimdTuple<_Tp, _As...>& __lhs,
1746  const __type_identity_t<_Tp> __rhs)
1747  {
1748  __for_each(
1749  __lhs, [&](auto __meta, auto& __native_lhs) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1750  __meta._S_masked_assign(__meta._S_make_mask(__bits), __native_lhs,
1751  __rhs);
1752  });
1753  }
1754 
1755  // _S_masked_cassign {{{2
1756  template <typename _Op, typename _Tp, typename... _As>
1757  static constexpr inline void
1758  _S_masked_cassign(const _MaskMember __bits, _SimdTuple<_Tp, _As...>& __lhs,
1759  const _SimdTuple<_Tp, _As...>& __rhs, _Op __op)
1760  {
1761  __for_each(__lhs, __rhs,
1762  [&](auto __meta, auto& __native_lhs, auto __native_rhs)
1763  constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
1764  {
1765  __meta.template _S_masked_cassign(__meta._S_make_mask(__bits),
1766  __native_lhs, __native_rhs, __op);
1767  });
1768  }
1769 
1770  // Optimization for the case where the RHS is a scalar. No need to broadcast
1771  // the scalar to a simd first.
1772  template <typename _Op, typename _Tp, typename... _As>
1773  static constexpr inline void
1774  _S_masked_cassign(const _MaskMember __bits, _SimdTuple<_Tp, _As...>& __lhs,
1775  const _Tp& __rhs, _Op __op)
1776  {
1777  __for_each(
1778  __lhs, [&](auto __meta, auto& __native_lhs) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1779  __meta.template _S_masked_cassign(__meta._S_make_mask(__bits),
1780  __native_lhs, __rhs, __op);
1781  });
1782  }
1783 
1784  // _S_masked_unary {{{2
1785  template <template <typename> class _Op, typename _Tp, typename... _As>
1786  static constexpr inline _SimdTuple<_Tp, _As...>
1787  _S_masked_unary(const _MaskMember __bits, const _SimdTuple<_Tp, _As...>& __v)
1788  {
1789  return __v._M_apply_wrapped([&__bits](auto __meta,
1790  auto __native) constexpr {
1791  return __meta.template _S_masked_unary<_Op>(__meta._S_make_mask(
1792  __bits),
1793  __native);
1794  });
1795  }
1796 
1797  // }}}2
1798  };
1799 
1800 // _MaskImplFixedSize {{{1
1801 template <int _Np, typename>
1802  struct _MaskImplFixedSize
1803  {
1804  static_assert(
1805  sizeof(_ULLong) * __CHAR_BIT__ >= _Np,
1806  "The fixed_size implementation relies on one _ULLong being able to store "
1807  "all boolean elements."); // required in load & store
1808 
1809  // member types {{{
1810  using _Abi = simd_abi::fixed_size<_Np>;
1811 
1812  using _MaskMember = _SanitizedBitMask<_Np>;
1813 
1814  template <typename _Tp>
1815  using _FirstAbi = typename __fixed_size_storage_t<_Tp, _Np>::_FirstAbi;
1816 
1817  template <typename _Tp>
1818  using _TypeTag = _Tp*;
1819 
1820  // }}}
1821  // _S_broadcast {{{
1822  template <typename>
1823  _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember
1824  _S_broadcast(bool __x)
1825  { return __x ? ~_MaskMember() : _MaskMember(); }
1826 
1827  // }}}
1828  // _S_load {{{
1829  template <typename>
1830  _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember
1831  _S_load(const bool* __mem)
1832  {
1833  if (__builtin_is_constant_evaluated())
1834  {
1835  _MaskMember __r{};
1836  for (size_t __i = 0; __i < _Np; ++__i)
1837  __r.set(__i, __mem[__i]);
1838  return __r;
1839  }
1840  using _Ip = __int_for_sizeof_t<bool>;
1841  // the following load uses element_aligned and relies on __mem already
1842  // carrying alignment information from when this load function was
1843  // called.
1844  const simd<_Ip, _Abi> __bools(reinterpret_cast<const __may_alias<_Ip>*>(
1845  __mem),
1846  element_aligned);
1847  return __data(__bools != 0);
1848  }
1849 
1850  // }}}
1851  // _S_to_bits {{{
1852  template <bool _Sanitized>
1853  _GLIBCXX_SIMD_INTRINSIC static constexpr _SanitizedBitMask<_Np>
1854  _S_to_bits(_BitMask<_Np, _Sanitized> __x)
1855  {
1856  if constexpr (_Sanitized)
1857  return __x;
1858  else
1859  return __x._M_sanitized();
1860  }
1861 
1862  // }}}
1863  // _S_convert {{{
1864  template <typename _Tp, typename _Up, typename _UAbi>
1865  _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember
1866  _S_convert(simd_mask<_Up, _UAbi> __x)
1867  {
1868  return _UAbi::_MaskImpl::_S_to_bits(__data(__x))
1869  .template _M_extract<0, _Np>();
1870  }
1871 
1872  // }}}
1873  // _S_from_bitmask {{{2
1874  template <typename _Tp>
1875  _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember
1876  _S_from_bitmask(_MaskMember __bits, _TypeTag<_Tp>) noexcept
1877  { return __bits; }
1878 
1879  // _S_load {{{2
1880  static constexpr inline _MaskMember
1881  _S_load(const bool* __mem) noexcept
1882  {
1883  // TODO: _UChar is not necessarily the best type to use here. For smaller
1884  // _Np _UShort, _UInt, _ULLong, float, and double can be more efficient.
1885  _ULLong __r = 0;
1886  using _Vs = __fixed_size_storage_t<_UChar, _Np>;
1887  __for_each(_Vs{}, [&](auto __meta, auto) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1888  __r |= __meta._S_mask_to_shifted_ullong(
1889  __meta._S_mask_impl._S_load(&__mem[__meta._S_offset],
1890  _SizeConstant<__meta._S_size()>()));
1891  });
1892  return __r;
1893  }
1894 
1895  // _S_masked_load {{{2
1896  static constexpr inline _MaskMember
1897  _S_masked_load(_MaskMember __merge, _MaskMember __mask, const bool* __mem) noexcept
1898  {
1899  _BitOps::_S_bit_iteration(__mask.to_ullong(),
1900  [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1901  __merge.set(__i, __mem[__i]);
1902  });
1903  return __merge;
1904  }
1905 
1906  // _S_store {{{2
1907  static constexpr inline void
1908  _S_store(const _MaskMember __bitmask, bool* __mem) noexcept
1909  {
1910  if constexpr (_Np == 1)
1911  __mem[0] = __bitmask[0];
1912  else
1913  _FirstAbi<_UChar>::_CommonImpl::_S_store_bool_array(__bitmask, __mem);
1914  }
1915 
1916  // _S_masked_store {{{2
1917  static constexpr inline void
1918  _S_masked_store(const _MaskMember __v, bool* __mem, const _MaskMember __k) noexcept
1919  {
1920  _BitOps::_S_bit_iteration(
1921  __k, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { __mem[__i] = __v[__i]; });
1922  }
1923 
1924  // logical and bitwise operators {{{2
1925  _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember
1926  _S_logical_and(const _MaskMember& __x, const _MaskMember& __y) noexcept
1927  { return __x & __y; }
1928 
1929  _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember
1930  _S_logical_or(const _MaskMember& __x, const _MaskMember& __y) noexcept
1931  { return __x | __y; }
1932 
1933  _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember
1934  _S_bit_not(const _MaskMember& __x) noexcept
1935  { return ~__x; }
1936 
1937  _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember
1938  _S_bit_and(const _MaskMember& __x, const _MaskMember& __y) noexcept
1939  { return __x & __y; }
1940 
1941  _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember
1942  _S_bit_or(const _MaskMember& __x, const _MaskMember& __y) noexcept
1943  { return __x | __y; }
1944 
1945  _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember
1946  _S_bit_xor(const _MaskMember& __x, const _MaskMember& __y) noexcept
1947  { return __x ^ __y; }
1948 
1949  // smart_reference access {{{2
1950  _GLIBCXX_SIMD_INTRINSIC static constexpr void
1951  _S_set(_MaskMember& __k, int __i, bool __x) noexcept
1952  { __k.set(__i, __x); }
1953 
1954  // _S_masked_assign {{{2
1955  _GLIBCXX_SIMD_INTRINSIC static constexpr void
1956  _S_masked_assign(const _MaskMember __k, _MaskMember& __lhs, const _MaskMember __rhs)
1957  { __lhs = (__lhs & ~__k) | (__rhs & __k); }
1958 
1959  // Optimization for the case where the RHS is a scalar.
1960  _GLIBCXX_SIMD_INTRINSIC static constexpr void
1961  _S_masked_assign(const _MaskMember __k, _MaskMember& __lhs, const bool __rhs)
1962  {
1963  if (__rhs)
1964  __lhs |= __k;
1965  else
1966  __lhs &= ~__k;
1967  }
1968 
1969  // }}}2
1970  // _S_all_of {{{
1971  template <typename _Tp>
1972  _GLIBCXX_SIMD_INTRINSIC static constexpr bool
1973  _S_all_of(simd_mask<_Tp, _Abi> __k)
1974  { return __data(__k).all(); }
1975 
1976  // }}}
1977  // _S_any_of {{{
1978  template <typename _Tp>
1979  _GLIBCXX_SIMD_INTRINSIC static constexpr bool
1980  _S_any_of(simd_mask<_Tp, _Abi> __k)
1981  { return __data(__k).any(); }
1982 
1983  // }}}
1984  // _S_none_of {{{
1985  template <typename _Tp>
1986  _GLIBCXX_SIMD_INTRINSIC static constexpr bool
1987  _S_none_of(simd_mask<_Tp, _Abi> __k)
1988  { return __data(__k).none(); }
1989 
1990  // }}}
1991  // _S_some_of {{{
1992  template <typename _Tp>
1993  _GLIBCXX_SIMD_INTRINSIC static constexpr bool
1994  _S_some_of([[maybe_unused]] simd_mask<_Tp, _Abi> __k)
1995  {
1996  if constexpr (_Np == 1)
1997  return false;
1998  else
1999  return __data(__k).any() && !__data(__k).all();
2000  }
2001 
2002  // }}}
2003  // _S_popcount {{{
2004  template <typename _Tp>
2005  _GLIBCXX_SIMD_INTRINSIC static constexpr int
2006  _S_popcount(simd_mask<_Tp, _Abi> __k)
2007  { return __data(__k).count(); }
2008 
2009  // }}}
2010  // _S_find_first_set {{{
2011  template <typename _Tp>
2012  _GLIBCXX_SIMD_INTRINSIC static constexpr int
2013  _S_find_first_set(simd_mask<_Tp, _Abi> __k)
2014  { return std::__countr_zero(__data(__k).to_ullong()); }
2015 
2016  // }}}
2017  // _S_find_last_set {{{
2018  template <typename _Tp>
2019  _GLIBCXX_SIMD_INTRINSIC static constexpr int
2020  _S_find_last_set(simd_mask<_Tp, _Abi> __k)
2021  { return std::__bit_width(__data(__k).to_ullong()) - 1; }
2022 
2023  // }}}
2024  };
2025 // }}}1
2026 
2027 _GLIBCXX_SIMD_END_NAMESPACE
2028 #endif // __cplusplus >= 201703L
2029 #endif // _GLIBCXX_EXPERIMENTAL_SIMD_FIXED_SIZE_H_
2030 
2031 // vim: foldmethod=marker sw=2 noet ts=8 sts=2 tw=80
constexpr enable_if_t< __and_< __is_duration< _ToDur >, __not_< treat_as_floating_point< typename _ToDur::rep > > >::value, _ToDur > round(const duration< _Rep, _Period > &__d)
Definition: chrono.h:437
complex< _Tp > log10(const complex< _Tp > &)
Return complex base 10 logarithm of z.
Definition: complex:1167
complex< _Tp > sin(const complex< _Tp > &)
Return complex sine of z.
Definition: complex:1197
complex< _Tp > log(const complex< _Tp > &)
Return complex natural logarithm of z.
Definition: complex:1162
complex< _Tp > tan(const complex< _Tp > &)
Return complex tangent of z.
Definition: complex:1298
_Tp abs(const complex< _Tp > &)
Return magnitude of z.
Definition: complex:968
complex< _Tp > exp(const complex< _Tp > &)
Return complex base e exponential of z.
Definition: complex:1135
complex< _Tp > cosh(const complex< _Tp > &)
Return complex hyperbolic cosine of z.
Definition: complex:1109
complex< _Tp > tanh(const complex< _Tp > &)
Return complex hyperbolic tangent of z.
Definition: complex:1326
complex< _Tp > pow(const complex< _Tp > &, int)
Return x to the y'th power.
Definition: complex:1357
complex< _Tp > sinh(const complex< _Tp > &)
Return complex hyperbolic sine of z.
Definition: complex:1227
complex< _Tp > cos(const complex< _Tp > &)
Return complex cosine of z.
Definition: complex:1079
complex< _Tp > sqrt(const complex< _Tp > &)
Return complex square root of z.
Definition: complex:1271
constexpr _Tp reduce(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOperation __binary_op)
Calculate reduction of values in a range.
Definition: numeric:293
_Tp fabs(const std::complex< _Tp > &)
fabs(__z) [8.1.8].
Definition: complex:2525
constexpr auto size(const _Container &__cont) noexcept(noexcept(__cont.size())) -> decltype(__cont.size())
Return the size of a container.
Definition: range_access.h:274
constexpr _Iterator __base(_Iterator __it)