16#ifndef HIGHWAY_HWY_ALIGNED_ALLOCATOR_H_
17#define HIGHWAY_HWY_ALIGNED_ALLOCATOR_H_
26#include <initializer_list>
41#define HWY_ALIGNMENT 128
45 return reinterpret_cast<uintptr_t
>(ptr) % align == 0;
50using AllocPtr =
void* (*)(
void* opaque,
size_t bytes);
51using FreePtr = void (*)(
void* opaque,
void* memory);
59 void* opaque_ptr =
nullptr);
66 FreePtr free_ptr,
void* opaque_ptr);
81 TypedArrayDeleter<T>);
87 size_t elems = size_in_bytes /
sizeof(T);
88 for (
size_t i = 0; i < elems; i++) {
90 (
static_cast<T*
>(ptr) + i)->~T();
117template <
typename T,
typename... Args>
119 void* opaque, Args&&... args) {
127template <
typename T,
typename... Args>
145 static_assert(std::is_integral<V>::value,
146 "AlignedAllocator only supports integer types");
147 static_assert(
sizeof(V) <=
sizeof(std::size_t),
148 "V n must be smaller or equal size_t to avoid overflow");
159template <
class T,
class V>
165template <
class T,
class V>
184 constexpr size_t size =
sizeof(T);
186 constexpr bool is_pow2 = (size & (size - 1)) == 0;
188 static_assert(!is_pow2 || (1ull << bits) == size,
"ShiftCount is incorrect");
190 const size_t bytes = is_pow2 ? items << bits : items * size;
191 const size_t check = is_pow2 ? bytes >> bits : bytes / size;
192 if (check != items) {
204template <
typename T,
typename... Args>
206 size_t items,
AllocPtr alloc,
FreePtr free,
void* opaque, Args&&... args) {
207 T* ptr = detail::AllocateAlignedItems<T>(items, alloc, opaque);
208 if (ptr !=
nullptr) {
209 for (
size_t i = 0; i < items; i++) {
210 new (ptr + i) T(std::forward<Args>(args)...);
216template <
typename T,
typename... Args>
219 items,
nullptr,
nullptr,
nullptr, std::forward<Args>(args)...);
233 template <
typename T>
255 detail::AllocateAlignedItems<T>(items, alloc, opaque),
262 return AllocateAligned<T>(items,
nullptr,
nullptr,
nullptr);
271 template <
typename U>
315template <
typename T,
size_t axes>
317 static_assert(std::is_trivial<T>::value,
318 "AlignedNDArray can only contain trivial types");
376 void truncate(
const std::array<size_t, axes>& new_shape) {
377#if HWY_IS_DEBUG_BUILD
378 for (
size_t axis_index = 0; axis_index < axes; ++axis_index) {
394 size_t Offset(std::array<const size_t, axes - 1> indices)
const {
396 size_t shape_index = 0;
397 for (
const size_t axis_index : indices) {
410 std::array<size_t, axes>
shape) {
411 std::array<size_t, axes + 1> sizes;
412 size_t axis =
shape.size();
416 sizes[axis] = sizes[axis + 1] *
shape[axis];
#define HWY_ALIGNMENT
Definition aligned_allocator.h:41
#define HWY_API
Definition base.h:171
#define HWY_DASSERT(condition)
Definition base.h:290
#define HWY_MAYBE_UNUSED
Definition base.h:113
#define HWY_ASSERT(condition)
Definition base.h:237
Definition aligned_allocator.h:72
void * opaque_ptr_
Definition aligned_allocator.h:104
void operator()(T *aligned_pointer) const
Definition aligned_allocator.h:79
AlignedDeleter(FreePtr free_ptr, void *opaque_ptr)
Definition aligned_allocator.h:75
AlignedDeleter()
Definition aligned_allocator.h:74
void(*)(void *t_ptr, size_t t_size) ArrayDeleter
Definition aligned_allocator.h:96
FreePtr free_
Definition aligned_allocator.h:103
static HWY_DLLEXPORT void DeleteAlignedArray(void *aligned_pointer, FreePtr free_ptr, void *opaque_ptr, ArrayDeleter deleter)
static void TypedArrayDeleter(void *ptr, size_t size_in_bytes)
Definition aligned_allocator.h:86
Definition aligned_allocator.h:224
AlignedFreer()
Definition aligned_allocator.h:229
void operator()(T *aligned_pointer) const
Definition aligned_allocator.h:234
AlignedFreer(FreePtr free_ptr, void *opaque_ptr)
Definition aligned_allocator.h:230
void * opaque_ptr_
Definition aligned_allocator.h:241
FreePtr free_
Definition aligned_allocator.h:240
static void DoNothing(void *, void *)
Definition aligned_allocator.h:227
Definition aligned_allocator.h:316
AlignedNDArray & operator=(AlignedNDArray &&other)=default
AlignedNDArray(std::array< size_t, axes > shape)
Definition aligned_allocator.h:325
Span< T > operator[](std::array< const size_t, axes - 1 > indices)
Definition aligned_allocator.h:338
std::array< size_t, axes+1 > memory_sizes_
Definition aligned_allocator.h:390
const T * data() const
Definition aligned_allocator.h:369
size_t size() const
Definition aligned_allocator.h:359
hwy::AlignedFreeUniquePtr< T[]> buffer_
Definition aligned_allocator.h:391
const std::array< size_t, axes > & shape() const
Definition aligned_allocator.h:351
Span< const T > operator[](std::array< const size_t, axes - 1 > indices) const
Definition aligned_allocator.h:344
T * data()
Definition aligned_allocator.h:366
std::array< size_t, axes+1 > sizes_
Definition aligned_allocator.h:389
void truncate(const std::array< size_t, axes > &new_shape)
Definition aligned_allocator.h:376
AlignedNDArray(AlignedNDArray &&other)=default
const std::array< size_t, axes > & memory_shape() const
Definition aligned_allocator.h:355
static std::array< size_t, axes+1 > ComputeSizes(std::array< size_t, axes > shape)
Definition aligned_allocator.h:409
std::array< size_t, axes > memory_shape_
Definition aligned_allocator.h:388
size_t memory_size() const
Definition aligned_allocator.h:363
std::array< size_t, axes > shape_
Definition aligned_allocator.h:387
size_t Offset(std::array< const size_t, axes - 1 > indices) const
Definition aligned_allocator.h:394
Definition aligned_allocator.h:267
T & operator[](size_t index) const
Definition aligned_allocator.h:290
size_t size() const
Definition aligned_allocator.h:283
T * begin()
Definition aligned_allocator.h:293
Span(U u)
Definition aligned_allocator.h:272
T * data_
Definition aligned_allocator.h:308
Span(T *data, size_t size)
Definition aligned_allocator.h:270
Span< T > & operator=(std::initializer_list< const T > v)
Definition aligned_allocator.h:276
constexpr const T * cbegin() const
Definition aligned_allocator.h:296
Span(std::initializer_list< const T > v)
Definition aligned_allocator.h:273
constexpr const T * cend() const
Definition aligned_allocator.h:304
T * data()
Definition aligned_allocator.h:286
size_t size_
Definition aligned_allocator.h:307
T * end()
Definition aligned_allocator.h:300
T * data() const
Definition aligned_allocator.h:287
#define HWY_DLLEXPORT
Definition highway_export.h:13
static constexpr size_t ShiftCount(size_t n)
Definition aligned_allocator.h:178
T * AllocateAlignedItems(size_t items, AllocPtr alloc_ptr, void *opaque_ptr)
Definition aligned_allocator.h:183
HWY_API void CopyBytes(const From *from, To *to)
Definition base.h:327
HWY_DLLEXPORT size_t VectorBytes()
std::unique_ptr< T, AlignedFreer > AlignedFreeUniquePtr
Definition aligned_allocator.h:247
HWY_API void ZeroBytes(To *to)
Definition base.h:352
std::unique_ptr< T, AlignedDeleter > AlignedUniquePtr
Definition aligned_allocator.h:111
constexpr bool operator!=(const AlignedAllocator< T > &, const AlignedAllocator< V > &) noexcept
Definition aligned_allocator.h:166
AlignedUniquePtr< T[]> MakeUniqueAlignedArrayWithAlloc(size_t items, AllocPtr alloc, FreePtr free, void *opaque, Args &&... args)
Definition aligned_allocator.h:205
constexpr bool operator==(const AlignedAllocator< T > &, const AlignedAllocator< V > &) noexcept
Definition aligned_allocator.h:160
void *(*)(void *opaque, size_t bytes) AllocPtr
Definition aligned_allocator.h:50
void(*)(void *opaque, void *memory) FreePtr
Definition aligned_allocator.h:51
HWY_DLLEXPORT void * AllocateAlignedBytes(size_t payload_size, AllocPtr alloc_ptr=nullptr, void *opaque_ptr=nullptr)
AlignedUniquePtr< T[]> MakeUniqueAlignedArray(size_t items, Args &&... args)
Definition aligned_allocator.h:217
AlignedFreeUniquePtr< T[]> AllocateAligned(const size_t items, AllocPtr alloc, FreePtr free, void *opaque)
Definition aligned_allocator.h:252
HWY_API constexpr bool IsAligned(T *ptr, size_t align=HWY_ALIGNMENT)
Definition aligned_allocator.h:44
HWY_DLLEXPORT void FreeAlignedBytes(const void *aligned_pointer, FreePtr free_ptr, void *opaque_ptr)
AlignedUniquePtr< T > MakeUniqueAlignedWithAlloc(AllocPtr alloc, FreePtr free, void *opaque, Args &&... args)
Definition aligned_allocator.h:118
constexpr size_t RoundUpTo(size_t what, size_t align)
Definition base.h:2490
AlignedUniquePtr< T > MakeUniqueAligned(Args &&... args)
Definition aligned_allocator.h:128
std::vector< T, AlignedAllocator< T > > AlignedVector
Definition aligned_allocator.h:172
Definition aligned_allocator.h:135
void deallocate(value_type *p, HWY_MAYBE_UNUSED V n)
Definition aligned_allocator.h:154
T value_type
Definition aligned_allocator.h:136
value_type * allocate(V n)
Definition aligned_allocator.h:144
AlignedAllocator()=default
AlignedAllocator(const AlignedAllocator< V > &) noexcept
Definition aligned_allocator.h:141