36#if defined(HIGHWAY_HWY_TESTS_TEST_UTIL_INL_H_) == defined(HWY_TARGET_TOGGLE)
38#ifdef HIGHWAY_HWY_TESTS_TEST_UTIL_INL_H_
39#undef HIGHWAY_HWY_TESTS_TEST_UTIL_INL_H_
41#define HIGHWAY_HWY_TESTS_TEST_UTIL_INL_H_
49template <
class D, HWY_IF_FLOAT_D(D)>
53template <
class D, HWY_IF_NOT_FLOAT_NOR_SPECIAL_D(D)>
55 const auto vi =
Iota(
d, 1);
57 Set(
d,
static_cast<TFromD<D>
>(1)));
62template <
class D,
typename First, HWY_IF_NOT_SPECIAL_FLOAT_D(D)>
64 return Iota(
d, first);
67template <
class D,
typename First, HWY_IF_F16_D(D), HWY_IF_LANES_GT_D(D, 1)>
69 return Iota(
d, first);
76 const size_t NW =
Lanes(
d) / 2;
78 const float first2 =
static_cast<float>(first) +
static_cast<float>(NW);
89 const Rebind<float, D> df;
96 const Repartition<float, D> df;
97 const float first1 = ConvertScalarTo<float>(first);
98 const float first2 = first1 +
static_cast<float>(
Lanes(
d) / 2);
106 const Rebind<float, D> df;
119template <
class D,
typename T = TFromD<D>>
121 const char* filename,
const int line) {
122 const size_t N =
Lanes(
d);
123 auto actual_lanes = AllocateAligned<T>(N);
124 Store(actual,
d, actual_lanes.get());
126 const auto info = hwy::detail::MakeTypeInfo<T>();
129 target_name, filename,
line);
134template <
class D,
typename T = TFromD<D>>
136 const char* filename,
int line) {
137 const size_t N =
Lanes(
d);
138 auto expected_lanes = AllocateAligned<T>(N);
139 auto actual_lanes = AllocateAligned<T>(N);
140 Store(expected,
d, expected_lanes.get());
141 Store(actual,
d, actual_lanes.get());
143 const auto info = hwy::detail::MakeTypeInfo<T>();
146 N, target_name, filename,
line);
152 const char* filename,
int line) {
163 const size_t N =
Lanes(
d);
164#if HWY_TARGET == HWY_SCALAR
169 const size_t N8 =
Lanes(d8);
170 auto bits_a = AllocateAligned<uint8_t>(
HWY_MAX(
size_t{8}, N8));
171 auto bits_b = AllocateAligned<uint8_t>(
size_t{
HWY_MAX(8, N8)});
172 memset(bits_a.get(), 0, N8);
173 memset(bits_b.get(), 0, N8);
179 for (; i < N / 8; ++i) {
180 if (bits_a[i] != bits_b[i]) {
181 fprintf(stderr,
"Mismatch in byte %d: %d != %d\n",
static_cast<int>(i),
182 bits_a[i], bits_b[i]);
183 Print(d8,
"expect",
Load(d8, bits_a.get()), 0, N8);
184 Print(d8,
"actual",
Load(d8, bits_b.get()), 0, N8);
185 hwy::Abort(filename,
line,
"Masks not equal");
189 const size_t remainder = N % 8;
190 if (remainder != 0) {
191 const int mask = (1 << remainder) - 1;
192 const int valid_a = bits_a[i] & mask;
193 const int valid_b = bits_b[i] & mask;
194 if (valid_a != valid_b) {
195 fprintf(stderr,
"Mismatch in last byte %d: %d != %d\n",
196 static_cast<int>(i), valid_a, valid_b);
197 Print(d8,
"expect",
Load(d8, bits_a.get()), 0, N8);
198 Print(d8,
"actual",
Load(d8, bits_b.get()), 0, N8);
199 hwy::Abort(filename,
line,
"Masks not equal");
218#define HWY_ASSERT_EQ(expected, actual) \
219 hwy::AssertEqual(expected, actual, hwy::TargetName(HWY_TARGET), __FILE__, \
222#define HWY_ASSERT_ARRAY_EQ(expected, actual, count) \
223 hwy::AssertArrayEqual(expected, actual, count, hwy::TargetName(HWY_TARGET), \
226#define HWY_ASSERT_STRING_EQ(expected, actual) \
227 hwy::AssertStringEqual(expected, actual, hwy::TargetName(HWY_TARGET), \
230#define HWY_ASSERT_VEC_EQ(d, expected, actual) \
231 AssertVecEqual(d, expected, actual, __FILE__, __LINE__)
233#define HWY_ASSERT_MASK_EQ(d, expected, actual) \
234 AssertMaskEqual(d, expected, actual, __FILE__, __LINE__)
246template <
typename T,
size_t kMul,
size_t kMinArg,
class Test,
int kPow2 = 0>
248 static void Do(
size_t min_lanes,
size_t max_lanes) {
252 const size_t lanes =
Lanes(
d);
253 if (lanes < min_lanes)
return;
255 if (lanes <= max_lanes) {
263template <
typename T,
size_t kMinArg,
class Test,
int kPow2>
265 static void Do(
size_t,
size_t) {}
271constexpr int MinPow2() {
279constexpr int MaxPow2() {
280#if HWY_TARGET == HWY_RVV
292template <
typename T,
int kPow2,
int kMaxPow2,
bool val
id,
class Test>
294 static void Do(
size_t min_lanes) {
295 const ScalableTag<T, kPow2> d;
297 static_assert(MinPow2<T>() <= kPow2 && kPow2 <= MaxPow2(),
"");
298 if (Lanes(d) >= min_lanes) {
301 fprintf(stderr,
"%d lanes < %d: T=%d pow=%d\n",
302 static_cast<int>(
Lanes(d)),
static_cast<int>(min_lanes),
303 static_cast<int>(
sizeof(T)), kPow2);
307 ForeachPow2<T, kPow2 + 1, kMaxPow2, (kPow2 + 1) <= kMaxPow2, Test>::Do(
313template <
typename T,
int kPow2,
int kMaxPow2,
class Test>
314struct ForeachPow2<T, kPow2, kMaxPow2, false, Test> {
315 static void Do(
size_t) {}
323template <
typename T,
int kAddMin,
int kSubMax,
class Test>
324using ForeachPow2Trim =
325 ForeachPow2<T, MinPow2<T>() + kAddMin, MaxPow2() - kSubMax,
326 MinPow2<T>() + kAddMin <= MaxPow2() - kSubMax, Test>;
352 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
356 template <
typename T>
360#if HWY_TARGET == HWY_SCALAR
366#if HWY_TARGET == HWY_RVV
368 detail::ForeachPow2Trim<T, 0, 2, Test>::Do(1);
369#elif HWY_HAVE_SCALABLE
370 detail::ForeachPow2Trim<T, 0, 0, Test>::Do(1);
378template <
class Test,
int kPow2 = 1>
385 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
389 template <
typename T>
392 constexpr size_t kMaxCapped =
HWY_LANES(T);
397#if HWY_TARGET == HWY_SCALAR
400 constexpr size_t kMul = kMaxCapped >> kPow2;
401 constexpr size_t kMinArg =
size_t{1} << kPow2;
404 detail::ForeachPow2Trim<T, 0, kPow2, Test>::Do(1);
412template <
class Test,
int kPow2 = 1>
419 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
423 template <
typename T>
426 constexpr size_t kMinLanes =
size_t{1} << kPow2;
427 constexpr size_t kMaxCapped =
HWY_LANES(T);
429 constexpr size_t max_lanes = kMaxCapped;
434#if HWY_TARGET == HWY_SCALAR
436#elif HWY_HAVE_SCALABLE
437 detail::ForeachPow2Trim<T, kPow2, 0, Test>::Do(kMinLanes);
440 kMinLanes, max_lanes);
447template <
size_t kMinBits,
class Test>
454 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
458 template <
typename T>
461 constexpr size_t kMaxCapped =
HWY_LANES(T);
462 constexpr size_t kMinLanes = kMinBits / 8 /
sizeof(T);
464 constexpr size_t max_lanes = kMaxCapped;
466#if HWY_TARGET == HWY_SCALAR
470 kMinLanes, max_lanes);
473 constexpr size_t kRatio = 128 / kMinBits;
474 constexpr int kMinPow2 =
475 kRatio == 0 ? 0 : -
static_cast<int>(
CeilLog2(kRatio));
476 constexpr bool kValid = kMinPow2 <= detail::MaxPow2();
477 detail::ForeachPow2<T, kMinPow2, detail::MaxPow2(), kValid, Test>::Do(
489template <
class Test,
int kPow2 = 1>
496 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
500 template <
typename T>
503 constexpr size_t kFactor =
size_t{1} << kPow2;
504 static_assert(kFactor >= 2 && kFactor *
sizeof(T) <=
sizeof(uint64_t),
"");
505 constexpr size_t kMaxCapped =
HWY_LANES(T);
510#if HWY_TARGET == HWY_SCALAR
513 using DLargestFrom =
CappedTag<T, (kMaxCapped >> kPow2) * kFactor, -kPow2>;
515 "HWY_MAX_LANES_D(DLargestFrom) must be less than or equal to "
516 "(kMaxCapped >> kPow2)");
520 detail::ForeachPow2Trim<T, 0, kPow2, Test>::Do(1);
529template <
class Test,
int kPow2 = 1>
536 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
540 template <
typename T>
547 constexpr int kMinTVecPow2 = detail::MinPow2<T>();
552 constexpr int kMinPow2Adj =
HWY_MAX(-3 - kMinTVecPow2 + kPow2, 0);
554 detail::ForeachPow2Trim<T, kMinPow2Adj, 0, Test>::Do(1);
560 constexpr size_t kMaxCapped =
HWY_LANES(T) >> 1;
566 constexpr size_t kMaxCapped =
HWY_LANES(T);
567 const size_t max_lanes = kMaxCapped;
575template <
class Test,
int kPow2 = 1>
582 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
586 template <
typename T>
589#if HWY_TARGET == HWY_SCALAR
592 constexpr size_t kMinLanes =
size_t{1} << kPow2;
594 constexpr size_t kMaxCapped =
HWY_LANES(T);
596 kMinLanes, kMaxCapped);
600 detail::ForeachPow2Trim<T, kPow2, 0, Test>::Do(kMinLanes);
615 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
619 template <
typename T>
622#if HWY_TARGET == HWY_SCALAR
636 mutable bool called_ =
false;
639 ~ForPartialFixedOrFullScalableVectors() {
641 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
645 template <
typename T>
646 void operator()(T )
const {
648#if HWY_TARGET == HWY_RVV
649 constexpr int kMinPow2 = -3 +
static_cast<int>(
CeilLog2(
sizeof(T)));
650 constexpr int kMaxPow2 = 3;
652 constexpr int kMinPow2 = 0;
653 constexpr int kMaxPow2 = 0;
655 detail::ForeachPow2<T, kMinPow2, kMaxPow2, true, Test>::Do(1);
658#elif HWY_TARGET == HWY_SVE_256 || HWY_TARGET == HWY_SVE2_128
674#if HWY_HAVE_INTEGER64
684#if HWY_HAVE_INTEGER64
807#if HWY_HAVE_INTEGER64
847#if HWY_HAVE_INTEGER64
868 return HWY_MAX(max_reps / 32, 2);
869#elif HWY_IS_DEBUG_BUILD
870 return HWY_MAX(max_reps / 8, 2);
872 return HWY_MAX(max_reps / 4, 2);
873#elif HWY_COMPILER_MSVC
874 return HWY_MAX(max_reps / 2, 2);
884 return HWY_MIN(max_pow2 - 4, max_pow2);
885#elif HWY_IS_DEBUG_BUILD
886 return HWY_MIN(max_pow2 - 1, max_pow2);
888 return HWY_MIN(max_pow2 - 1, max_pow2);
#define HWY_MAX(a, b)
Definition base.h:177
#define HWY_NOINLINE
Definition base.h:103
#define HWY_MIN(a, b)
Definition base.h:176
#define HWY_ABORT(format,...)
Definition base.h:233
#define HWY_INLINE
Definition base.h:101
#define HWY_ASSERT(condition)
Definition base.h:237
Definition test_util-inl.h:530
~ForDemoteVectors()
Definition test_util-inl.h:534
void operator()(T) const
Definition test_util-inl.h:541
bool called_
Definition test_util-inl.h:531
Definition test_util-inl.h:379
void operator()(T) const
Definition test_util-inl.h:390
bool called_
Definition test_util-inl.h:380
~ForExtendableVectors()
Definition test_util-inl.h:383
Definition test_util-inl.h:448
bool called_
Definition test_util-inl.h:449
~ForGEVectors()
Definition test_util-inl.h:452
void operator()(T) const
Definition test_util-inl.h:459
Definition test_util-inl.h:576
~ForHalfVectors()
Definition test_util-inl.h:580
bool called_
Definition test_util-inl.h:577
void operator()(T) const
Definition test_util-inl.h:587
Definition test_util-inl.h:346
~ForMaxPow2()
Definition test_util-inl.h:350
bool called_
Definition test_util-inl.h:347
void operator()(T) const
Definition test_util-inl.h:357
Definition test_util-inl.h:609
bool called_
Definition test_util-inl.h:610
void operator()(T t) const
Definition test_util-inl.h:620
~ForPartialVectors()
Definition test_util-inl.h:613
Definition test_util-inl.h:413
void operator()(T) const
Definition test_util-inl.h:424
bool called_
Definition test_util-inl.h:414
~ForShrinkableVectors()
Definition test_util-inl.h:417
#define HWY_TARGET
Definition detect_targets.h:543
typename D::template Rebind< T > Rebind
Definition ops/shared-inl.h:460
HWY_API VFromD< D > VecFromMask(D d, const MFromD< D > m)
Definition arm_neon-inl.h:2960
constexpr size_t AdjustedReps(size_t max_reps)
Definition test_util-inl.h:866
void ForUIF32(const Func &func)
Definition test_util-inl.h:800
D d
Definition arm_sve-inl.h:1915
void ForI816(const Func &func)
Definition test_util-inl.h:838
HWY_API size_t CountTrue(D, Mask128< T > mask)
Definition arm_neon-inl.h:8358
void ForUI163264(const Func &func)
Definition test_util-inl.h:853
void ForU816(const Func &func)
Definition test_util-inl.h:832
V VecArg
Definition ops/shared-inl.h:69
HWY_INLINE Vec< D > PositiveIota(D d)
Definition test_util-inl.h:50
HWY_API Vec128< T, N > And(const Vec128< T, N > a, const Vec128< T, N > b)
Definition arm_neon-inl.h:2690
void ForUIF3264(const Func &func)
Definition test_util-inl.h:826
void ForUIF163264(const Func &func)
Definition test_util-inl.h:859
HWY_INLINE void AssertVecEqual(D d, const T *expected, Vec< D > actual, const char *filename, const int line)
Definition test_util-inl.h:120
constexpr size_t AdjustedLog2Reps(size_t max_pow2)
Definition test_util-inl.h:881
HWY_API void Store(VFromD< D > v, D d, TFromD< D > *HWY_RESTRICT aligned)
Definition arm_neon-inl.h:3911
void ForU163264(const Func &func)
Definition test_util-inl.h:844
HWY_API Vec128< uint64_t, N > Max(Vec128< uint64_t, N > a, Vec128< uint64_t, N > b)
Definition arm_neon-inl.h:3377
void ForFloatTypesDynamic(const Func &func)
Definition test_util-inl.h:751
void ForUI32(const Func &func)
Definition test_util-inl.h:794
void ForAllTypes(const Func &func)
Definition test_util-inl.h:758
VFromD< D > IotaForSpecial(D d, First first)
Definition test_util-inl.h:63
void ForFloat16TypesDynamic(const Func &func)
Definition test_util-inl.h:717
HWY_API VFromD< D > OrderedDemote2To(D d, V a, V b)
Definition arm_neon-inl.h:7394
void ForFloatTypes(const Func &func)
Definition test_util-inl.h:745
ForGEVectors< HWY_MAX_BYTES *8, Test > ForPartialFixedOrFullScalableVectors
Definition test_util-inl.h:660
HWY_API size_t StoreMaskBits(D d, MFromD< D > mask, uint8_t *bits)
Definition arm_neon-inl.h:8402
HWY_NOINLINE void AssertMaskEqual(D d, VecArg< Mask< D > > a, VecArg< Mask< D > > b, const char *filename, int line)
Definition test_util-inl.h:151
void ForSpecialTypes(const Func &func)
Definition test_util-inl.h:765
HWY_API VFromD< D > Load(D d, const TFromD< D > *HWY_RESTRICT p)
Definition arm_neon-inl.h:3664
void ForIntegerTypes(const Func &func)
Definition test_util-inl.h:690
HWY_API Vec128< uint8_t > Combine(D, Vec64< uint8_t > hi, Vec64< uint8_t > lo)
Definition arm_neon-inl.h:1314
void ForUI8(const Func &func)
Definition test_util-inl.h:776
HWY_API Vec64< uint16_t > DemoteTo(D, Vec128< int32_t > v)
Definition arm_neon-inl.h:4629
typename detail::CappedTagChecker< T, kLimit, kPow2 >::type CappedTag
Definition ops/shared-inl.h:379
void ForUI3264(const Func &func)
Definition test_util-inl.h:820
HWY_API bool AllTrue(D d, Mask128< T > m)
Definition arm_neon-inl.h:8416
HWY_API VFromD< D > Iota(D d, const T2 first)
Definition arm_neon-inl.h:1297
void ForUIF64(const Func &func)
Definition test_util-inl.h:814
void ForUI16(const Func &func)
Definition test_util-inl.h:782
void ForFloat64TypesDynamic(const Func &func)
Definition test_util-inl.h:728
decltype(MaskFromVec(Zero(D()))) Mask
Definition generic_ops-inl.h:52
void ForFloat3264Types(const Func &func)
Definition test_util-inl.h:739
typename detail::ScalableTagChecker< T, kPow2 >::type ScalableTag
Definition ops/shared-inl.h:367
void ForFloat16Types(const Func &func)
Definition test_util-inl.h:696
void ForUI64(const Func &func)
Definition test_util-inl.h:806
HWY_INLINE Vec128< TFromD< D > > Set(D, T t)
Definition arm_neon-inl.h:931
void ForSignedTypes(const Func &func)
Definition test_util-inl.h:670
typename D::Half Half
Definition ops/shared-inl.h:487
HWY_API bool AllFalse(D d, MFromD< D > m)
Definition arm_neon-inl.h:8410
void ForUIF16(const Func &func)
Definition test_util-inl.h:788
decltype(Set(D(), TFromD< D >())) VFromD
Definition arm_neon-inl.h:944
decltype(Zero(D())) Vec
Definition generic_ops-inl.h:46
HWY_API size_t Lanes(D)
Definition rvv-inl.h:598
HWY_API MFromD< D > FirstN(D d, size_t num)
Definition arm_neon-inl.h:3232
HWY_INLINE Mask< D > MaskTrue(const D d)
Definition test_util-inl.h:209
void ForFloat64Types(const Func &func)
Definition test_util-inl.h:705
void ForAllTypesAndSpecial(const Func &func)
Definition test_util-inl.h:770
typename D::template Repartition< T > Repartition
Definition ops/shared-inl.h:471
HWY_API void Print(const D d, const char *caption, V v, size_t lane_u=0, size_t max_lanes=7)
Definition print-inl.h:39
void ForUnsignedTypes(const Func &func)
Definition test_util-inl.h:680
HWY_TEST_DLLEXPORT void AssertArrayEqual(const TypeInfo &info, const void *expected_void, const void *actual_void, size_t N, const char *target_name, const char *filename, int line)
FuncOutput(*)(const void *, FuncInput) Func
Definition nanobenchmark.h:87
HWY_INLINE void AssertEqual(const TExpected texpected, const TActual actual, const char *target_name, const char *filename, int line, size_t lane=0)
Definition test_util.h:194
static HWY_MAYBE_UNUSED const char * TargetName(int64_t target)
Definition targets.h:85
HWY_DLLEXPORT bool HaveFloat64()
constexpr size_t CeilLog2(TI x)
Definition base.h:2669
HWY_DLLEXPORT HWY_NORETURN void int line
Definition base.h:231
HWY_API constexpr T LimitsMax()
Definition base.h:2174
HWY_DLLEXPORT bool HaveFloat16()
#define HWY_IF_F16_D(D)
Definition ops/shared-inl.h:597
#define HWY_IF_BF16_D(D)
Definition ops/shared-inl.h:594
#define HWY_IF_POW2_GT_D(D, pow2)
Definition ops/shared-inl.h:574
#define HWY_IF_POW2_LE_D(D, pow2)
Definition ops/shared-inl.h:573
#define HWY_IF_LANES_GT_D(D, lanes)
Definition ops/shared-inl.h:562
#define HWY_IF_LANES_D(D, lanes)
Definition ops/shared-inl.h:560
#define HWY_IF_SPECIAL_FLOAT_D(D)
Definition ops/shared-inl.h:540
#define HWY_MAX_LANES_D(D)
Definition ops/shared-inl.h:432
#define HWY_LANES(T)
Definition set_macros-inl.h:169
#define HWY_NAMESPACE
Definition set_macros-inl.h:166
static void Do(size_t, size_t)
Definition test_util-inl.h:265
Definition test_util-inl.h:247
static void Do(size_t min_lanes, size_t max_lanes)
Definition test_util-inl.h:248
int VFromD
Definition tuple-inl.h:25