28#ifndef HWY_HIGHWAY_INCLUDED
29#define HWY_HIGHWAY_INCLUDED
45#define HWY_FULL1(T) hwy::HWY_NAMESPACE::ScalableTag<T>
46#define HWY_FULL2(T, LMUL) \
47 hwy::HWY_NAMESPACE::ScalableTag<T, hwy::CeilLog2(HWY_MAX(0, LMUL))>
48#define HWY_3TH_ARG(arg1, arg2, arg3, ...) arg3
50#define HWY_FULL_RECOMPOSER(args_with_paren) HWY_3TH_ARG args_with_paren
52#define HWY_CHOOSE_FULL(...) \
53 HWY_FULL_RECOMPOSER((__VA_ARGS__, HWY_FULL2, HWY_FULL1, ))
54#define HWY_FULL(...) HWY_CHOOSE_FULL(__VA_ARGS__())(__VA_ARGS__)
57#define HWY_CAPPED(T, MAX_N) hwy::HWY_NAMESPACE::CappedTag<T, MAX_N>
73#if HWY_STATIC_TARGET == HWY_SCALAR
74#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SCALAR::FUNC_NAME
75#elif HWY_STATIC_TARGET == HWY_EMU128
76#define HWY_STATIC_DISPATCH(FUNC_NAME) N_EMU128::FUNC_NAME
77#elif HWY_STATIC_TARGET == HWY_RVV
78#define HWY_STATIC_DISPATCH(FUNC_NAME) N_RVV::FUNC_NAME
79#elif HWY_STATIC_TARGET == HWY_WASM_EMU256
80#define HWY_STATIC_DISPATCH(FUNC_NAME) N_WASM_EMU256::FUNC_NAME
81#elif HWY_STATIC_TARGET == HWY_WASM
82#define HWY_STATIC_DISPATCH(FUNC_NAME) N_WASM::FUNC_NAME
83#elif HWY_STATIC_TARGET == HWY_NEON_WITHOUT_AES
84#define HWY_STATIC_DISPATCH(FUNC_NAME) N_NEON_WITHOUT_AES::FUNC_NAME
85#elif HWY_STATIC_TARGET == HWY_NEON
86#define HWY_STATIC_DISPATCH(FUNC_NAME) N_NEON::FUNC_NAME
87#elif HWY_STATIC_TARGET == HWY_SVE
88#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SVE::FUNC_NAME
89#elif HWY_STATIC_TARGET == HWY_SVE2
90#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SVE2::FUNC_NAME
91#elif HWY_STATIC_TARGET == HWY_SVE_256
92#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SVE_256::FUNC_NAME
93#elif HWY_STATIC_TARGET == HWY_SVE2_128
94#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SVE2_128::FUNC_NAME
95#elif HWY_STATIC_TARGET == HWY_PPC8
96#define HWY_STATIC_DISPATCH(FUNC_NAME) N_PPC8::FUNC_NAME
97#elif HWY_STATIC_TARGET == HWY_PPC9
98#define HWY_STATIC_DISPATCH(FUNC_NAME) N_PPC9::FUNC_NAME
99#elif HWY_STATIC_TARGET == HWY_PPC10
100#define HWY_STATIC_DISPATCH(FUNC_NAME) N_PPC10::FUNC_NAME
101#elif HWY_STATIC_TARGET == HWY_Z14
102#define HWY_STATIC_DISPATCH(FUNC_NAME) N_Z14::FUNC_NAME
103#elif HWY_STATIC_TARGET == HWY_Z15
104#define HWY_STATIC_DISPATCH(FUNC_NAME) N_Z15::FUNC_NAME
105#elif HWY_STATIC_TARGET == HWY_SSE2
106#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SSE2::FUNC_NAME
107#elif HWY_STATIC_TARGET == HWY_SSSE3
108#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SSSE3::FUNC_NAME
109#elif HWY_STATIC_TARGET == HWY_SSE4
110#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SSE4::FUNC_NAME
111#elif HWY_STATIC_TARGET == HWY_AVX2
112#define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX2::FUNC_NAME
113#elif HWY_STATIC_TARGET == HWY_AVX3
114#define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX3::FUNC_NAME
115#elif HWY_STATIC_TARGET == HWY_AVX3_DL
116#define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX3_DL::FUNC_NAME
117#elif HWY_STATIC_TARGET == HWY_AVX3_ZEN4
118#define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX3_ZEN4::FUNC_NAME
119#elif HWY_STATIC_TARGET == HWY_AVX3_SPR
120#define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX3_SPR::FUNC_NAME
125#if HWY_TARGETS & HWY_EMU128
126#define HWY_CHOOSE_FALLBACK(FUNC_NAME) &N_EMU128::FUNC_NAME
127#elif HWY_TARGETS & HWY_SCALAR
128#define HWY_CHOOSE_FALLBACK(FUNC_NAME) &N_SCALAR::FUNC_NAME
132#define HWY_CHOOSE_FALLBACK(FUNC_NAME) &HWY_STATIC_DISPATCH(FUNC_NAME)
135#if HWY_TARGETS & HWY_WASM_EMU256
136#define HWY_CHOOSE_WASM_EMU256(FUNC_NAME) &N_WASM_EMU256::FUNC_NAME
138#define HWY_CHOOSE_WASM_EMU256(FUNC_NAME) nullptr
141#if HWY_TARGETS & HWY_WASM
142#define HWY_CHOOSE_WASM(FUNC_NAME) &N_WASM::FUNC_NAME
144#define HWY_CHOOSE_WASM(FUNC_NAME) nullptr
147#if HWY_TARGETS & HWY_RVV
148#define HWY_CHOOSE_RVV(FUNC_NAME) &N_RVV::FUNC_NAME
150#define HWY_CHOOSE_RVV(FUNC_NAME) nullptr
153#if HWY_TARGETS & HWY_NEON_WITHOUT_AES
154#define HWY_CHOOSE_NEON_WITHOUT_AES(FUNC_NAME) &N_NEON_WITHOUT_AES::FUNC_NAME
156#define HWY_CHOOSE_NEON_WITHOUT_AES(FUNC_NAME) nullptr
159#if HWY_TARGETS & HWY_NEON
160#define HWY_CHOOSE_NEON(FUNC_NAME) &N_NEON::FUNC_NAME
162#define HWY_CHOOSE_NEON(FUNC_NAME) nullptr
165#if HWY_TARGETS & HWY_SVE
166#define HWY_CHOOSE_SVE(FUNC_NAME) &N_SVE::FUNC_NAME
168#define HWY_CHOOSE_SVE(FUNC_NAME) nullptr
171#if HWY_TARGETS & HWY_SVE2
172#define HWY_CHOOSE_SVE2(FUNC_NAME) &N_SVE2::FUNC_NAME
174#define HWY_CHOOSE_SVE2(FUNC_NAME) nullptr
177#if HWY_TARGETS & HWY_SVE_256
178#define HWY_CHOOSE_SVE_256(FUNC_NAME) &N_SVE_256::FUNC_NAME
180#define HWY_CHOOSE_SVE_256(FUNC_NAME) nullptr
183#if HWY_TARGETS & HWY_SVE2_128
184#define HWY_CHOOSE_SVE2_128(FUNC_NAME) &N_SVE2_128::FUNC_NAME
186#define HWY_CHOOSE_SVE2_128(FUNC_NAME) nullptr
189#if HWY_TARGETS & HWY_PPC8
190#define HWY_CHOOSE_PPC8(FUNC_NAME) &N_PPC8::FUNC_NAME
192#define HWY_CHOOSE_PPC8(FUNC_NAME) nullptr
195#if HWY_TARGETS & HWY_PPC9
196#define HWY_CHOOSE_PPC9(FUNC_NAME) &N_PPC9::FUNC_NAME
198#define HWY_CHOOSE_PPC9(FUNC_NAME) nullptr
201#if HWY_TARGETS & HWY_PPC10
202#define HWY_CHOOSE_PPC10(FUNC_NAME) &N_PPC10::FUNC_NAME
204#define HWY_CHOOSE_PPC10(FUNC_NAME) nullptr
207#if HWY_TARGETS & HWY_Z14
208#define HWY_CHOOSE_Z14(FUNC_NAME) &N_Z14::FUNC_NAME
210#define HWY_CHOOSE_Z14(FUNC_NAME) nullptr
213#if HWY_TARGETS & HWY_Z15
214#define HWY_CHOOSE_Z15(FUNC_NAME) &N_Z15::FUNC_NAME
216#define HWY_CHOOSE_Z15(FUNC_NAME) nullptr
219#if HWY_TARGETS & HWY_SSE2
220#define HWY_CHOOSE_SSE2(FUNC_NAME) &N_SSE2::FUNC_NAME
222#define HWY_CHOOSE_SSE2(FUNC_NAME) nullptr
225#if HWY_TARGETS & HWY_SSSE3
226#define HWY_CHOOSE_SSSE3(FUNC_NAME) &N_SSSE3::FUNC_NAME
228#define HWY_CHOOSE_SSSE3(FUNC_NAME) nullptr
231#if HWY_TARGETS & HWY_SSE4
232#define HWY_CHOOSE_SSE4(FUNC_NAME) &N_SSE4::FUNC_NAME
234#define HWY_CHOOSE_SSE4(FUNC_NAME) nullptr
237#if HWY_TARGETS & HWY_AVX2
238#define HWY_CHOOSE_AVX2(FUNC_NAME) &N_AVX2::FUNC_NAME
240#define HWY_CHOOSE_AVX2(FUNC_NAME) nullptr
243#if HWY_TARGETS & HWY_AVX3
244#define HWY_CHOOSE_AVX3(FUNC_NAME) &N_AVX3::FUNC_NAME
246#define HWY_CHOOSE_AVX3(FUNC_NAME) nullptr
249#if HWY_TARGETS & HWY_AVX3_DL
250#define HWY_CHOOSE_AVX3_DL(FUNC_NAME) &N_AVX3_DL::FUNC_NAME
252#define HWY_CHOOSE_AVX3_DL(FUNC_NAME) nullptr
255#if HWY_TARGETS & HWY_AVX3_ZEN4
256#define HWY_CHOOSE_AVX3_ZEN4(FUNC_NAME) &N_AVX3_ZEN4::FUNC_NAME
258#define HWY_CHOOSE_AVX3_ZEN4(FUNC_NAME) nullptr
261#if HWY_TARGETS & HWY_AVX3_SPR
262#define HWY_CHOOSE_AVX3_SPR(FUNC_NAME) &N_AVX3_SPR::FUNC_NAME
264#define HWY_CHOOSE_AVX3_SPR(FUNC_NAME) nullptr
271#if HWY_COMPILER_MSVC && HWY_COMPILER_MSVC < 1915
272#define HWY_DISPATCH_WORKAROUND 1
274#define HWY_DISPATCH_WORKAROUND 0
280template <
typename RetType,
typename... Args>
285#if HWY_DISPATCH_WORKAROUND
286 template <FunctionType* const func>
290 return (*func)(args...);
299 template <FunctionType* const table[]>
303 return (table[chosen_target.
GetIndex()])(args...);
309template <
typename RetType,
typename... Args>
314#define HWY_DISPATCH_TABLE(FUNC_NAME) \
315 HWY_CONCAT(FUNC_NAME, HighwayDispatchTable)
346#if HWY_IDE || ((HWY_TARGETS & (HWY_TARGETS - 1)) == 0)
352#define HWY_EXPORT(FUNC_NAME) \
353 HWY_MAYBE_UNUSED static decltype(&HWY_STATIC_DISPATCH(FUNC_NAME)) const \
354 HWY_DISPATCH_TABLE(FUNC_NAME)[1] = {&HWY_STATIC_DISPATCH(FUNC_NAME)}
355#define HWY_DYNAMIC_DISPATCH(FUNC_NAME) HWY_STATIC_DISPATCH(FUNC_NAME)
356#define HWY_DYNAMIC_POINTER(FUNC_NAME) &HWY_STATIC_DISPATCH(FUNC_NAME)
361#if HWY_DISPATCH_WORKAROUND
363#define HWY_EXPORT(FUNC_NAME) \
364 static decltype(&HWY_STATIC_DISPATCH(FUNC_NAME)) const HWY_DISPATCH_TABLE( \
365 FUNC_NAME)[HWY_MAX_DYNAMIC_TARGETS + 2] = { \
368 &decltype(hwy::DeduceFunctionCache(&HWY_STATIC_DISPATCH( \
369 FUNC_NAME)))::ChooseAndCall<&HWY_STATIC_DISPATCH(FUNC_NAME)>, \
370 HWY_CHOOSE_TARGET_LIST(FUNC_NAME), \
371 HWY_CHOOSE_FALLBACK(FUNC_NAME), \
378#define HWY_EXPORT(FUNC_NAME) \
379 static decltype(&HWY_STATIC_DISPATCH(FUNC_NAME)) const HWY_DISPATCH_TABLE( \
380 FUNC_NAME)[HWY_MAX_DYNAMIC_TARGETS + 2] = { \
383 &decltype(hwy::DeduceFunctionCache(&HWY_STATIC_DISPATCH( \
384 FUNC_NAME)))::ChooseAndCall<HWY_DISPATCH_TABLE(FUNC_NAME)>, \
385 HWY_CHOOSE_TARGET_LIST(FUNC_NAME), \
386 HWY_CHOOSE_FALLBACK(FUNC_NAME), \
391#define HWY_DYNAMIC_DISPATCH(FUNC_NAME) \
392 (*(HWY_DISPATCH_TABLE(FUNC_NAME)[hwy::GetChosenTarget().GetIndex()]))
393#define HWY_DYNAMIC_POINTER(FUNC_NAME) \
394 (HWY_DISPATCH_TABLE(FUNC_NAME)[hwy::GetChosenTarget().GetIndex()])
399#define HWY_CAP_INTEGER64 HWY_HAVE_INTEGER64
400#define HWY_CAP_FLOAT16 HWY_HAVE_FLOAT16
401#define HWY_CAP_FLOAT64 HWY_HAVE_FLOAT64
412#if defined(HWY_HIGHWAY_PER_TARGET) == defined(HWY_TARGET_TOGGLE)
413#ifdef HWY_HIGHWAY_PER_TARGET
414#undef HWY_HIGHWAY_PER_TARGET
416#define HWY_HIGHWAY_PER_TARGET
420#if HWY_TARGET == HWY_SSE2 || HWY_TARGET == HWY_SSSE3 || HWY_TARGET == HWY_SSE4
422#elif HWY_TARGET == HWY_AVX2
424#elif HWY_TARGET == HWY_AVX3 || HWY_TARGET == HWY_AVX3_DL || \
425 HWY_TARGET == HWY_AVX3_ZEN4 || HWY_TARGET == HWY_AVX3_SPR
427#elif HWY_TARGET == HWY_Z14 || HWY_TARGET == HWY_Z15 || \
428 HWY_TARGET == HWY_PPC8 || HWY_TARGET == HWY_PPC9 || \
429 HWY_TARGET == HWY_PPC10
431#elif HWY_TARGET == HWY_NEON || HWY_TARGET == HWY_NEON_WITHOUT_AES
433#elif HWY_TARGET == HWY_SVE || HWY_TARGET == HWY_SVE2 || \
434 HWY_TARGET == HWY_SVE_256 || HWY_TARGET == HWY_SVE2_128
436#elif HWY_TARGET == HWY_WASM_EMU256
438#elif HWY_TARGET == HWY_WASM
440#elif HWY_TARGET == HWY_RVV
442#elif HWY_TARGET == HWY_EMU128
444#elif HWY_TARGET == HWY_SCALAR
447#pragma message("HWY_TARGET does not match any known target")
FunctionCache< RetType, Args... > DeduceFunctionCache(RetType(*)(Args...))
Definition highway.h:310
HWY_DLLEXPORT ChosenTarget & GetChosenTarget()
HWY_DLLEXPORT int64_t SupportedTargets()
size_t HWY_INLINE GetIndex() const
Definition targets.h:320
void Update(int64_t targets)
Definition targets.h:301
RetType FunctionType(Args...)
Definition highway.h:283
static RetType ChooseAndCall(Args... args)
Definition highway.h:300