70M&A2c0 0 0 02`7l8J)%+)^+P9{c`'3 $@&Lʆ2y0 qGQA1(BN!g5 ost-increment operator (x++) HWY_CXX14_CONSTEXPR bfloat16_t operator++(int) noexcept { bfloat16_t result = *this; native = static_cast(native + Native{1}); return result; } constexpr bfloat16_t operator-() const noexcept { return bfloat16_t(static_cast(-native)); } constexpr bfloat16_t operator+() const noexcept { return *this; } // Reduce clutter by generating `operator+` and `operator+=` etc. Note that // we cannot token-paste `operator` and `+`, so pass it in as `op_func`. #define HWY_BFLOAT16_BINARY_OP(op, op_func, assign_func) \ constexpr bfloat16_t op_func(const bfloat16_t& rhs) const noexcept { \ return bfloat16_t(static_cast(native op rhs.native)); \ } \ template () (the largest finite F32 value) // It is okay if the F64 to F32 conversion is inexact for F64 values that have // a magnitude that is less than 2^(-126) as the magnitude of a denormal F32 // value is much smaller than 2^(-24) (the smallest positive denormal F16 // value). return F16FromF32( static_cast(BitCastScalar(static_cast( (BitCastScalar(f64) & 0xFFFFFFFFE0000000ULL) | ((BitCastScalar(f64) + 0x000000001FFFFFFFULL) & 0x0000000020000000ULL))))); #endif } // More convenient to define outside float16_t because these may use // F32FromF16, which is defined after the struct. HWY_F16_CONSTEXPR inline bool operator==(float16_t lhs, float16_t rhs) noexcept { #if HWY_HAVE_SCALAR_F16_OPERATORS return lhs.native == rhs.native; #else return F32FromF16(lhs) == F32constexpr operator Native() const noexcept { return native; } #endif #if HWY_HAVE_SCALAR_F16_TYPE static HWY_BITCASTSCALAR_CONSTEXPR float16_t FromBits(uint16_t bits) { return float16_t(BitCastScalar(bits)); } #else private: struct F16FromU16BitsTag {}; constexpr float16_t(F16FromU16BitsTag /*tag*/, uint16_t u16_bits) : bits(u16_bits) {} public: static constexpr float16_t FromBits(uint16_t bits) { return float16_t(F16FromU16BitsTag(), bits); } #endif // When backed by a native type, ensure the wrapper behaves like the native // type by forwarding all operators. Unfortunately it seems difficult to reuse // this code in a base class, so we repeat it in float16_t. #if HWY_HAVE_SCALAR_F16_OPERATORS || HWY_IDE template , float16_t>() && IsConvertible()>* = nullptr> constexpr float16_t(T&& arg) noexcept : native(static_cast(static_cast(arg))) {} tedetail // Match [u]int##_t naming scheme so rvv-inl.h macros can obtain the type name // by concatenating base type and bits. We use a wrapper class instead of a // typedef to the native type to ensure that the same symbols, e.g. for VQSort, // are generated regardless of F16 support; see #1684. struct alignas(2) float16_t { #if HWY_HAVE_SCALAR_F16_TYPE #if HWY_RVV_HAVE_F16_VEC || HWY_SSE2_HAVE_F16_TYPE || __SPIRV_DEVICE__ using Native = _Float16; #elif HWY_NEON_HAVE_F16C using Native = __fp16; #else #error "Logic error: condition should be 'all but NEON_HAVE_F16C'" #endif #elif HWY_IDE using Native = uint16_t; #endif // HWY_HAVE_SCALAR_F16_TYPE union { #if HWY_HAVE_SCALAR_F16_TYPE || HWY_IDE // Accessed via NativeLaneType, and used directly if // HWY_HAVE_SCALAR_F16_OPERATORS. Native native; #endif // Only accessed via NativeLaneType or U16LaneType. uint16_t bits; }; // Default init and copying. float16_t() noexcept = default; constexpr float16_t(const float16_t&) noexeConst>, void>()) || (!IsArray() && (IsSame())>() || !IsSame, RemoveConst>()) && IsSame(0)), hwy::SizeTag<1>>()) }; }; #if HWY_COMPILER_MSVC HWY_DIAGNOSTICS(pop) #endif template HWY_API constexpr bool IsConvertible() { return IsConvertibleT::value; } template class IsStaticCastableT { private: template (DeclVal()))> static hwy::SizeTag<1> TryStaticCastTest(int); template static hwy::SizeTag<0> TryStaticCastTest(Arg); public: enum { value = IsSame(0)), hwy::SizeTag<1>>() }; }; template static constexpr bool IsStaticCastable() { return IsStaticCastableT::value; } #define HWY_IF_CASTABLE(From, To) \ hwy::EnableIf( \ HWY_ASSUME_ALIGNED((ptr), alignof(hwy::RemovePtr))) // Clang and GCC require attributes on each function into which SIMD intrinsics // are inlined. Support both per-function annotation (HWY_ATTR) for lambdas and // automatic annotation via pragmas. #if HWY_COMPILER_ICC // As of ICC 2021.{1-9} the pragma is neither implemented nor required. #define HWY_PUSH_ATTRIBUTES(targets_str) #define HWY_POP_ATTRIBUTES #elif ined(__GLIBC__) && defined(__powerpc64__) asm volatile("mfspr %0, %1" : "=r"(t) : "i"(268)); #elif HWY_ARCH_ARM_A64 && !HWY_COMPILER_MSVC // pmccntr_el0 is privileged but cntvct_el0 is accessible in Linux and QEMU. asm volatile("mrs %0, cntvct_el0" : "=r"(t)); #elif HWY_ARCH_X86 && HWY_COMPILER_MSVC _ReadWriteBarrier(); unsigned aux; t = __rdtscp(&aux); _ReadWriteBarrier(); _mm_lfence(); _ReadWriteBarrier(); #elif HWY_ARCH_X86_64 // Use inline asm because __rdtscp generates code to store TSC_AUX (ecx). asm volatile( "rdtscp\n\t" "shl $32, %%rdx\n\t" "or %%rdx, %0\n\t" "lfence" : "=a"(t) : // "memory" avoids reordering. rcx = TSC_AUX. rdx = TSC >> 32. // "cc" = flags modified by SHL. : "rcx", "rdx", "memory", "cc"); #else t = Start(); #endif return t; } } // namespace timer // Wrapper around Start/Stop that checks whether the CPU supports Stop. class Timer { public: Timer() { char cpu100[100]; have_timer_stop_ = platfns a const iterator pointing just beyond the last element at the // end of this span. constexpr const T* cend() const { return data_ + size_; } private: size_t size_ = 0; T* data_ = nullptr; }; // A multi dimensional array containing an aligned buffer. // // To maintain alignment, the innermost dimension will be padded to ensure all // innermost arrays are aligned. template class AlignedNDArray { static_assert(std::is_trivial::value, "AlignedNDArray can only contain trivial types"); public: AlignedNDArray(AlignedNDArray&& other) = default; AlignedNDArray& operator=(AlignedNDArray&& other) = default; // Constructs an array of the provided shape and fills it with zeros. explicit AlignedNDArray(std::array shape) : shape_(shape) { sizes_ = ComputeSizes(shape_); memory_shape_ = shape_; // Round the innermost dimension up to the number of bytes available for // SIMD operations on this architecture to make sure that ea can be a single // element U or an array of element if T is a U[]. The custom aligned deleter // will call the destructor on U or each element of a U[] in the array case. template using AlignedUniquePtr = std::unique_ptr; // Aligned memory equivalent of make_unique using the custom allocators // alloc/free with the passed `opaque` pointer. This function calls the // constructor with the passed Args... and calls the destructor of the object // when the AlignedUniquePtr is destroyed. template AlignedUniquePtr MakeUniqueAlignedWithAlloc(AllocPtr alloc, FreePtr free, void* opaque, Args&&... args) { T* ptr = static_cast(AllocateAlignedBytes(sizeof(T), alloc, opaque)); if (HWY_UNLIKELY(ptr == nullptr)) { return AlignedUniquePtr(nullptr, AlignedDeleter(free, opaque)); } #ifdef HWY_EXCEPTIONS_ENABLED try { return AlignedUniquePtr(new (ptr) T(std::forward(args)...), ense. #ifndef HIGHWAY_HWY_ROBUST_STATISTICS_H_ #define HIGHWAY_HWY_ROBUST_STATISTICS_H_ #include // std::sort, std::find_if #include #include // std::pair #include #include "hwy/base.h" namespace hwy { namespace robust_statistics { // Sorts integral values in ascending order (e.g. for Mode). About 3x faster // than std::sort for input distributions with very few unique values. template void CountingSort(T* values, size_t num_values) { // Unique values and their frequency (similar to flat_map). using Unique = std::pair; std::vector unique; for (size_t i = 0; i < num_values; ++i) { const T value = values[i]; const auto pos = std::find_if(unique.begin(), unique.end(), [value](const Unique u) { return u.first == value; }); if (pos == unique.end()) { unique.push_back(std::make_pair(value, 1)); } else { ++pos->second; } } // Sort in ascending order of value (pair.first)._)( \ HWY_CONCAT(TABLE_NAME, HighwayDispatchExportsKey), #TABLE_NAME, \ HWY_DISPATCH_TABLE(TABLE_NAME)) // For non-template functions. Not necessarily invoked within a function, hence // we derive the string and variable names from FUNC_NAME, not HWY_FUNCTION. #if HWY_DISPATCH_WORKAROUND #define HWY_EXPORT(FUNC_NAME) HWY_EXPORT_T(FUNC_NAME, FUNC_NAME) #else #define HWY_EXPORT(FUNC_NAME) \ static decltype(&HWY_STATIC_DISPATCH(FUNC_NAME)) const HWY_DISPATCH_TABLE( \ FUNC_NAME)[static_cast(HWY_MAX_DYNAMIC_TARGETS + 2)] = { \ /* The first entry in the table initializes the global cache and \ * calls the appropriate function. */ \ &decltype(hwy::DeduceFunctionCache(&HWY_STATIC_DISPATCH(FUNC_NAME))):: \ template TableChooseAndCall, \ HWY_CHOOSE_TARGET_LIST(FUNC_NAME), ATIC_NAMESPACE N_AVX3_SPR #elif HWY_STATIC_TARGET == HWY_AVX10_2 #define HWY_STATIC_NAMESPACE N_AVX10_2 #endif // `HWY_STATIC_DISPATCH(FUNC_NAME)` is the namespace-qualified FUNC_NAME for // `HWY_STATIC_TARGET`, and can be used to deduce the return type of Choose*. #define HWY_STATIC_DISPATCH(FUNC_NAME) HWY_STATIC_NAMESPACE::FUNC_NAME // `HWY_CHOOSE_*(FUNC_NAME)` expands to the function pointer for that target or // nullptr if that target was not compiled. // `HWY_VISIT_*(VISITOR)` expands to `VISITOR(TARGET, NAMESPACE)` or nothing if // that target was not compiled. #if HWY_TARGETS & HWY_EMU128 #define HWY_CHOOSE_FALLBACK(FUNC_NAME) &N_EMU128::FUNC_NAME #define HWY_VISIT_FALLBACK(VISITOR) VISITOR(HWY_EMU128, N_EMU128) #elif HWY_TARGETS & HWY_SCALAR #define HWY_CHOOSE_FALLBACK(FUNC_NAME) &N_SCALAR::FUNC_NAME #define HWY_VISIT_FALLBACK(VISITOR) VISITOR(HWY_SCALAR, N_SCALAR) #else // When HWY_SCALAR/HWY_EMU128 are not present and other targets were disabled at // runtime, fall back to the baseline with HWY_STMSVC #include // SSE2 #include // _mm_prefetch #elif HWY_ARCH_ARM_A64 #include #endif #endif // HWY_DISABLE_CACHE_CONTROL namespace hwy { // Even if N*sizeof(T) is smaller, Stream may write a multiple of this size. #define HWY_STREAM_MULTIPLE 16 // The following functions may also require an attribute. #if HWY_ARCH_X86 && !defined(HWY_DISABLE_CACHE_CONTROL) && !HWY_COMPILER_MSVC #define HWY_ATTR_CACHE __attribute__((target("sse2"))) #else #define HWY_ATTR_CACHE #endif // Windows.h #defines this, which causes infinite recursion. Temporarily // undefine to avoid conflict with our function. // TODO(janwas): remove when this function is removed. #pragma push_macro("LoadFence") #undef LoadFence // Delays subsequent loads until prior loads are visible. Beware of potentially // differing behavior across architectures and vendors: on Intel but not // AMD CPUs, also serves as a full fence (waits for all prior instructions to // complete). HWY_INLINE HWY_ATTR_CACHE void & HWY_SCALAR) && (HWY_STATIC_TARGET != HWY_SCALAR) #undef HWY_TARGET #define HWY_TARGET HWY_SCALAR #include HWY_TARGET_INCLUDE #ifdef HWY_TARGET_TOGGLE #undef HWY_TARGET_TOGGLE #else #define HWY_TARGET_TOGGLE #endif #endif #endif // !HWY_IDE && (HWY_TARGETS != HWY_STATIC_TARGET) // Now that all but the static target have been generated, re-enable HWY_EXPORT. #undef HWY_ONCE #define HWY_ONCE 1 // If we re-include once per enabled target, the translation unit's // implementation would have to be skipped via #if to avoid redefining symbols. // We instead skip the re-include for HWY_STATIC_TARGET, and generate its // implementation when resuming compilation of the translation unit. #undef HWY_TARGET #define HWY_TARGET HWY_STATIC_TARGET #ifdef HWY_ALREADY_INCLUDED // Revert the previous toggle to prevent redefinitions for the static target. #ifdef HWY_TARGET_TOGGLE #undef HWY_TARGET_TOGGLE #else #define HWY_TARGET_TOGGLE #endif // Force re-inclusion of set_macros-inl.h now that HWY_TARGET is restored. #ifdet after compiling each target, so // the header is again visible. This is done by flipping HWY_TARGET_TOGGLE, // defining it if undefined and vice versa. This macro is initially undefined // so that IDEs don't gray out the contents of each header. #ifdef HWY_TARGET_TOGGLE #error "This macro must not be defined outside foreach_target.h" #endif #ifdef HWY_HIGHWAY_INCLUDED // highway.h include guard // Trigger fixup at the bottom of this header. #define HWY_ALREADY_INCLUDED // The next highway.h must re-include set_macros-inl.h because the first // highway.h chose the static target instead of what we will set below. #undef HWY_SET_MACROS_PER_TARGET #endif // Disable HWY_EXPORT in user code until we have generated all targets. Note // that a subsequent highway.h will not override this definition. #undef HWY_ONCE #define HWY_ONCE (0 || HWY_IDE) // Avoid warnings on #include HWY_TARGET_INCLUDE by hiding them from the IDE; // also skip if only 1 target defined (no re-inclusion will be necessary). #if !HWY_IDE & i < num_values_; ++i) { OnlineNotify(copy[i]); } HWY_DASSERT(IsOnline()); #if SIZE_MAX == 0xFFFFFFFFu (void)padding_; #endif } size_t num_values_ = 0; // size of `values_` <= `kMaxValues` #if SIZE_MAX == 0xFFFFFFFFu uint32_t padding_ = 0; #endif double online_n_ = 0.0; // number of calls to `OnlineNotify`. double values_[kMaxValues]; }; static_assert(sizeof(CostDistribution) == 128, ""); // Implements a counter with wrap-around, plus the ability to skip values. // O(1) time, O(N) space via doubly-linked list of indices. class NextWithSkip { public: NextWithSkip() {} explicit NextWithSkip(size_t num) { links_.reserve(num); for (size_t i = 0; i < num; ++i) { links_.emplace_back(i, num); } } size_t Next(size_t pos) { HWY_DASSERT(pos < links_.size()); HWY_DASSERT(!links_[pos].IsRemoved()); return links_[pos].Next(); } // Must not be called for an already skipped position. Ignores an attempt to // skip the last remaining position. lude #define HWY_HAVE_ASM_HWCAP 0 #endif #endif // HWY_HAVE_ASM_HWCAP #ifndef HWY_HAVE_AUXV // allow override #ifdef TOOLCHAIN_MISS_SYS_AUXV_H #define HWY_HAVE_AUXV 0 // CMake failed to find the header // glibc 2.16 added auxv, but checking for that requires features.h, and we do // not want to include system headers here. Instead check for the header // directly, which has been supported at least since GCC 5.4 and Clang 3. #elif defined(__has_include) // note: wrapper macro fails on Clang ~17 // clang-format off #if __has_include() // clang-format on #define HWY_HAVE_AUXV 1 // header present #else #define HWY_HAVE_AUXV 0 // header not present #endif // __has_include #else // compiler lacks __has_include #define HWY_HAVE_AUXV 0 #endif #endif // HWY_HAVE_AUXV #ifndef HWY_HAVE_RUNTIME_DISPATCH_RVV // allow override // Clang 19+ supports target attributes for RVV intrinsics (resolved in // https://github.com/llvm/llvm-project/issues/56592 and // httbaseline targets are disabled or considered broken." \ "This is typically due to very restrictive HWY_BASELINE_TARGETS, or " \ "too expansive HWY_BROKEN_TARGETS or HWY_DISABLED_TAREGTS. User code " \ "must also check for this and skip any usage of SIMD." #endif // Best baseline, used for static dispatch. This is the least-significant 1-bit // within HWY_ENABLED_BASELINE and lower bit values imply "better". #define HWY_STATIC_TARGET (HWY_ENABLED_BASELINE & -HWY_ENABLED_BASELINE) // Start by assuming static dispatch. If we later use dynamic dispatch, this // will be defined to other targets during the multiple-inclusion, and finally // return to the initial value. Defining this outside begin/end_target ensures // inl headers successfully compile by themselves (required by Bazel). #define HWY_TARGET HWY_STATIC_TARGET //------------------------------------------------------------------------------ // Choose targets for dynamic dispatch according to one of four policies #if 1 < (definHWY_BROKEN_32BIT | HWY_BROKEN_MSVC | \ HWY_BROKEN_AVX10_2 | HWY_BROKEN_AVX3_DL_ZEN4 | HWY_BROKEN_AVX3_SPR | \ HWY_BROKEN_ARM7_BIG_ENDIAN | HWY_BROKEN_ARM7_WITHOUT_VFP4 | \ HWY_BROKEN_NEON_BF16 | HWY_BROKEN_SVE | HWY_BROKEN_SVE2 | \ HWY_BROKEN_SVE2_128 | HWY_BROKEN_PPC10 | HWY_BROKEN_PPC_32BIT | \ HWY_BROKEN_RVV | HWY_BROKEN_LOONGARCH | HWY_BROKEN_Z14) #endif // HWY_BROKEN_TARGETS // Enabled means not disabled nor blocklisted. #define HWY_ENABLED(targets) \ ((targets) & ~((HWY_DISABLED_TARGETS) | (HWY_BROKEN_TARGETS))) // Opt-out for EMU128 (affected by a GCC bug on multiple arches, fixed in 12.3: // see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=106322). An issue still // remains with 13.2, see #1683. This is separate from HWY_BROKEN_TARGETS // because it affects the fallback target, which must always be enabled. If 1, // we instead choose HWY_SCALAR even without HWY_COMPILE_ONLY_SCALAR being set. #if !defined(HWY_BROKEN_EMU128) // alLL << 7) #define HWY_AVX3 (1LL << 8) // HWY_AVX2 plus AVX-512F/BW/CD/DQ/VL #define HWY_AVX2 (1LL << 9) // HWY_SSE4 plus BMI2 + F16 + FMA // Bit 10: reserved #define HWY_SSE4 (1LL << 11) // SSE4.2 plus AES + CLMUL #define HWY_SSSE3 (1LL << 12) // S-SSE3 // Bit 13: reserved for SSE3 #define HWY_SSE2 (1LL << 14) // The highest bit in the HWY_TARGETS mask that a x86 target can have. Used for // dynamic dispatch. All x86 target bits must be lower or equal to // (1 << HWY_HIGHEST_TARGET_BIT_X86) and they can only use // HWY_MAX_DYNAMIC_TARGETS in total. #define HWY_HIGHEST_TARGET_BIT_X86 14 // --------------------------- Arm: 15 targets (+ one fallback) // Bits 15..17 reserved (3 targets) #define HWY_SVE2_128 (1LL << 18) // specialized (e.g. Neoverse V2/N2/N3) #define HWY_SVE_256 (1LL << 19) // specialized (Neoverse V1) // Bits 20-22 reserved for later SVE (3 targets) #define HWY_SVE2 (1LL << 23) #define HWY_SVE (1LL << 24) // Bit 25 reserved for NEON #define HWY_NEON_BF16 (1LL << 26) // fp16/dot/bf16 (e.l_mad. This // prevents infinite loops. size_t max_evals = 9; // Whether to print additional statistics to stdout. bool verbose = true; }; // Measurement result for each unique input. struct Result { FuncInput input; // Robust estimate (mode or median) of duration. float ticks; // Measure of variability (median absolute deviation relative to "ticks"). float variability; }; // Returns a Params struct with customized configuration for benchmarks. // Specifically limits `max_evals` to prevent timeout in tests. static inline Params DefaultBenchmarkParams() { Params p; p.max_evals = HWY_IS_DEBUG_BUILD ? 3 : 4; return p; } // Precisely measures the number of ticks elapsed when calling "func" with the // given inputs, shuffled to ensure realistic branch prediction hit rates. // // "func" returns a 'proof of work' to ensure its computations are not elided. // "arg" is passed to Func, or reserved for internal use by MeasureClosure. // "inputs" is an array of "num_inputs" (not necessarily zone_idx, accumulators.Get(global_idx, zone_idx)); }); // OK to reset even if we have active zones, because we set `visited_zones_` // when exiting the zone. visited_zones_ = ZoneSet(); } private: // 40 bytes: ZoneSet visited_zones_; // Which `zones_` have been active on this worker. uint64_t depth_ = 0; // Current nesting level for active zones. Overheads overheads_; uint64_t t_enter_[kMaxDepth]; // Used to deduct child duration from parent's self time (unless inclusive). // Shifting by one avoids bounds-checks for depth_ = 0 (root zone). uint64_t child_total_[1 + kMaxDepth] = {0}; }; // Enables shift rather than multiplication. static_assert(sizeof(PerWorker) == 256, "Wrong size"); } // namespace profiler class Profiler { public: static HWY_DLLEXPORT Profiler& Get(); // Returns `global_idx` from thread-local storage (0 for the main thread). // Used by `PROFILER_ZONE/PROFILER_FUNC`. It is faster to instead pass the // globalivided by // the number of unique threads seen. This is facilitated by unique `global_idx` // passed in by callers, or taken from thread-local `GlobalIdx()`. // // We also want to support varying thread counts per call site, because the same // function/zone may be called from multiple pools. `EndRootRun` calls // `CountWorkersAndReset` after each top-level `ThreadPool::Run`, which // generates one data point summarized via descriptive statistics. Here we // implement a simpler version of `Stats` because we do not require // geomean/variance/kurtosis/skewness. Because concurrency is a small integer, // we can simply compute sums rather than online moments. There is also only one // instance across all threads, hence we do not require a `Take`. // // Note that subsequently discovered prior work estimates the number of active // and idle processors by updating atomic counters whenever they start/finish a // task: https://homes.cs.washington.edu/~tom/pubs/quartz.pdf and "Effective // performance measurement and ; // Allows other classes such as `ThreadPool` to register/unregister a function // to call during `PrintResults`. This allows us to gather data from the worker // threads without having to wait until they exit, and decouples the profiler // from other modules. Thread-safe. class Funcs { static constexpr auto kAcq = std::memory_order_acquire; static constexpr auto kRel = std::memory_order_release; public: // Can be called concurrently with distinct keys. void Add(intptr_t key, ProfilerFunc func) { HWY_ASSERT(key != 0 && key != kPending); // reserved values HWY_ASSERT(func); // not empty for (size_t i = 0; i < kMaxFuncs; ++i) { intptr_t expected = 0; // Lost a race with a concurrent `Add`, try the next slot. if (!keys_[i].compare_exchange_strong(expected, kPending, kRel)) { continue; } // We own the slot: move func there. funcs_[i] = std::move(func); keys_[i].store(key, kRel); // publishes the `func` write. emplate HWY_API Vec128 AndNot(Vec128 not_mask, Vec128 mask) { return Vec128{wasm_v128_andnot(mask.raw, not_mask.raw)}; } // ------------------------------ Or template HWY_API Vec128 Or(Vec128 a, Vec128 b) { return Vec128{wasm_v128_or(a.raw, b.raw)}; } // ------------------------------ Xor template HWY_API Vec128 Xor(Vec128 a, Vec128 b) { return Vec128{wasm_v128_xor(a.raw, b.raw)}; } // ------------------------------ Or3 template HWY_API Vec128 Or3(Vec128 o1, Vec128 o2, Vec128 o3) { return Or(o1, Or(o2, o3)); } // ------------------------------ OrAnd template HWY_API Vec128 OrAnd(Vec128 o, Vec128 a1, Vec128 a2) { return Or(o, And(a1, a2)); } // ------------------------------ IfVecThenElse template HWY_API Vec128 IfVecThenElse(Vec12t() + 1>(Add(vu, vu))); return RebindMask(d, Lt(exp, Set(di, hwy::MaxExponentField()))); } // ================================================== COMPARE // Comparisons fill a lane with 1-bits if the condition is true, else 0. // Mask and Vec are the same (true = FF..FF). template HWY_API Mask128 MaskFromVec(const Vec128 v) { return Mask128{v.raw}; } template using MFromD = decltype(MaskFromVec(VFromD())); template HWY_API MFromD RebindMask(DTo /* tag */, Mask128 m) { static_assert(sizeof(TFrom) == sizeof(TFromD), "Must have same size"); return MFromD{m.raw}; } template HWY_API Mask128 TestBit(Vec128 v, Vec128 bit) { static_assert(!hwy::IsFloat(), "Only integer vectors supported"); return (v & bit) == bit; } // ------------------------------ Equality // Unsigned template HWY_API Mask128 b) { const auto l = wasm_i64x2_extmul_low_i32x4(a.raw, b.raw); const auto h = wasm_i64x2_extmul_high_i32x4(a.raw, b.raw); // TODO(eustas): shift-right + narrow? return Vec128{wasm_i32x4_shuffle(l, h, 1, 3, 5, 7)}; } template HWY_API Vec128 MulFixedPoint15(Vec128 a, Vec128 b) { return Vec128{wasm_i16x8_q15mulr_sat(a.raw, b.raw)}; } // Multiplies even lanes (0, 2 ..) and returns the double-width result. template HWY_API Vec128, (N + 1) / 2> MulEven(const Vec128 a, const Vec128 b) { const DFromV d; const RepartitionToWide dw; constexpr int kSrcBits = sizeof(T) * 8; const auto ae = ShiftRight(ShiftLeft(ResizeBitCast(dw, a))); const auto be = Min(Vec128 a, Vec128 b) { alignas(16) int64_t min[4]; min[0] = HWY_MIN(wasm_i64x2_extract_lane(a.raw, 0), wasm_i64x2_extract_lane(b.raw, 0)); min[1] = HWY_MIN(wasm_i64x2_extract_lane(a.raw, 1), wasm_i64x2_extract_lane(b.raw, 1)); return Vec128{wasm_v128_load(min)}; } // Float template HWY_API Vec128 Min(Vec128 a, Vec128 b) { // Equivalent to a < b ? a : b (taking into account our swapped arg order, // so that Min(NaN, x) is x to match x86). return Vec128{wasm_f32x4_pmin(b.raw, a.raw)}; } template HWY_API Vec128 Min(Vec128 a, Vec128 b) { // Equivalent to a < b ? a : b (taking into account our swapped arg order, // so that Min(NaN, x) is x to match x86). return Vec128{wasm_f64x2_pmin(b.raw, a.raw)}; } // ------------------------------ Maximum // Unsigned template HWY_API Vec128 Max(Vec128size_t N> HWY_API Vec128 ShiftLeft(const Vec128 v) { return Vec128{wasm_i32x4_shl(v.raw, kBits)}; } template HWY_API Vec128 ShiftLeft(const Vec128 v) { return Vec128{wasm_i64x2_shl(v.raw, kBits)}; } template HWY_API Vec128 ShiftRight(const Vec128 v) { return Vec128{wasm_i32x4_shr(v.raw, kBits)}; } template HWY_API Vec128 ShiftRight(const Vec128 v) { return Vec128{wasm_i64x2_shr(v.raw, kBits)}; } // 8-bit template HWY_API Vec128 ShiftLeft(const Vec128 v) { const DFromV d8; // Use raw instead of BitCast to support N=1. const Vec128 shifted{ShiftLeft(Vec128>{v.raw}).raw}; return kBits == 1 ? (v + v) : (shifted & Set(d8, static_cast((0xFF << kBits) & 0xFF)));TFromD t1, TFromD t2, TFromD t3) { return VFromD{wasm_f32x4_make(t0, t1, t2, t3)}; } template HWY_API VFromD Dup128VecFromValues(D /*d*/, TFromD t0, TFromD t1) { return VFromD{wasm_i64x2_make(t0, t1)}; } template HWY_API VFromD Dup128VecFromValues(D /*d*/, TFromD t0, TFromD t1) { return VFromD{wasm_u64x2_make(t0, t1)}; } template HWY_API VFromD Dup128VecFromValues(D /*d*/, TFromD t0, TFromD t1) { return VFromD{wasm_f64x2_make(t0, t1)}; } // ================================================== ARITHMETIC // ------------------------------ Addition // Unsigned template HWY_API Vec128 operator+(const Vec128 a, const Vec128 b) { return Vec128{wasm_i8x16_add(a.rawASM vectors and operations. // External include guard in highway.h - see comment there. #include #include "hwy/base.h" #include "hwy/ops/shared-inl.h" #ifdef HWY_WASM_OLD_NAMES #define wasm_i8x16_shuffle wasm_v8x16_shuffle #define wasm_i16x8_shuffle wasm_v16x8_shuffle #define wasm_i32x4_shuffle wasm_v32x4_shuffle #define wasm_i64x2_shuffle wasm_v64x2_shuffle #define wasm_u16x8_extend_low_u8x16 wasm_i16x8_widen_low_u8x16 #define wasm_u32x4_extend_low_u16x8 wasm_i32x4_widen_low_u16x8 #define wasm_i32x4_extend_low_i16x8 wasm_i32x4_widen_low_i16x8 #define wasm_i16x8_extend_low_i8x16 wasm_i16x8_widen_low_i8x16 #define wasm_u32x4_extend_high_u16x8 wasm_i32x4_widen_high_u16x8 #define wasm_i32x4_extend_high_i16x8 wasm_i32x4_widen_high_i16x8 #define wasm_i32x4_trunc_sat_f32x4 wasm_i32x4_trunc_saturate_f32x4 #define wasm_i62x2_trunc_sat_f64x2 wasm_i64x2_trunc_saturate_f64x2 #define wasm_u8x16_add_sat wasm_u8x16_add_saturate #define wasm_u8x16_sub_sat wasm_u8x16_sub_saturate #define wasm_u16x8_add_salass FromT> HWY_INLINE ToT CastValueForInRangeF2IConv(FromT val) { // Prevent ubsan errors when converting float to narrower integer using FromTU = MakeUnsigned; constexpr unsigned kMaxExpField = static_cast(MaxExponentField()); constexpr unsigned kExpBias = kMaxExpField >> 1; constexpr unsigned kMinOutOfRangeExpField = static_cast(HWY_MIN( kExpBias + sizeof(ToT) * 8 - static_cast(IsSigned()), kMaxExpField)); // If ToT is signed, compare only the exponent bits of val against // kMinOutOfRangeExpField. // // Otherwise, if ToT is unsigned, compare the sign bit plus exponent bits of // val against kMinOutOfRangeExpField as a negative value is outside of the // range of an unsigned integer type. const FromT val_to_compare = static_cast(IsSigned() ? ScalarAbs(val) : val); // val is within the range of ToT if // (BitCastScalar(val_to_compare) >> MantissaBits()) is less // than kMinOu_EMU128_CONCAT_INLINE VFromD ConcatOdd(D d, VFromD hi, VFromD lo) { const Half dh; VFromD ret; for (size_t i = 0; i < MaxLanes(dh); ++i) { ret.raw[i] = lo.raw[2 * i + 1]; } for (size_t i = 0; i < MaxLanes(dh); ++i) { ret.raw[MaxLanes(dh) + i] = hi.raw[2 * i + 1]; } return ret; } // ------------------------------ CombineShiftRightBytes template HWY_API VFromD CombineShiftRightBytes(D d, VFromD hi, VFromD lo) { VFromD ret; const uint8_t* HWY_RESTRICT lo8 = reinterpret_cast(lo.raw); uint8_t* HWY_RESTRICT ret8 = reinterpret_cast(ret.raw); CopyBytes(lo8 + kBytes, ret8); CopyBytes(hi.raw, ret8 + d.MaxBytes() - kBytes); return ret; } // ------------------------------ ShiftLeftBytes template HWY_API VFromD ShiftLeftBytes(D d, VFromD v) { static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes");tive = v.raw[i] < Float(0.0); Bits bits = BitCastScalar(v.raw[i]); const int exponent = static_cast(((bits >> kMantissaBits) & kExponentMask) - kBias); // Already an integer. if (exponent >= kMantissaBits) continue; // |v| <= 1 => -1 or 0. if (exponent < 0) { v.raw[i] = negative ? Float(-1.0) : Float(0.0); continue; } const Bits mantissa_mask = kMantissaMask >> exponent; // Already an integer if ((bits & mantissa_mask) == 0) continue; // Clear fractional bits and round down if (negative) bits += (kMantissaMask + 1) >> exponent; bits &= ~mantissa_mask; v.raw[i] = BitCastScalar(bits); } return v; } // ------------------------------ Floating-point classification template HWY_API Mask128 IsNaN(Vec128 v) { Mask128 ret; for (size_t i = 0; i < N; ++i) { // std::isnan returns false for 0x7F..FF in clang AVX3 builds, so DIY. ret.bits[i] = Mask128::FromBool(/ Multiplies odd lanes (1, 3, ..) and returns the double-wide result. template HWY_API Vec128, (N + 1) / 2> MulOdd(Vec128 a, Vec128 b) { using TW = MakeWide; Vec128 mul; for (size_t i = 0; i < N; i += 2) { const TW a_wide = a.raw[i + 1]; mul.raw[i / 2] = static_cast(a_wide * b.raw[i + 1]); } return mul; } template HWY_API Vec128 ApproximateReciprocal(Vec128 v) { for (size_t i = 0; i < N; ++i) { // Zero inputs are allowed, but callers are responsible for replacing the // return value with something else (typically using IfThenElse). This check // avoids a ubsan error. The result is arbitrary. v.raw[i] = (ScalarAbs(v.raw[i]) == 0.0f) ? 0.0f : 1.0f / v.raw[i]; } return v; } // generic_ops takes care of integer T. templT, N> operator<<(Vec128 v, Vec128 bits) { for (size_t i = 0; i < N; ++i) { const auto shifted = static_cast>(v.raw[i]) << bits.raw[i]; v.raw[i] = static_cast(shifted); } return v; } template HWY_API Vec128 operator>>(Vec128 v, Vec128 bits) { for (size_t i = 0; i < N; ++i) { v.raw[i] = ScalarShr(v.raw[i], static_cast(bits.raw[i])); } return v; } // ================================================== ARITHMETIC // Tag dispatch instead of SFINAE for MSVC 2017 compatibility namespace detail { template HWY_INLINE Vec128 Add(hwy::NonFloatTag /*tag*/, Vec128 a, Vec128 b) { for (size_t i = 0; i < N; ++i) { const uint64_t a64 = static_cast(a.raw[i]); const uint64_t b64 = static_cast(b.raw[i]); a.raw[i] = static_cast((a64 + b64) & static_cast(~T(0))); } return a; } temnstexpr size_t kToByteLen = sizeof(TTo) * HWY_MAX_LANES_D(D); constexpr size_t kCopyByteLen = HWY_MIN(kFromByteLen, kToByteLen); VFromD to = Zero(d); CopyBytes(&v.raw, &to.raw); return to; } namespace detail { // ResizeBitCast on the HWY_EMU128 target has zero-extending semantics if // VFromD is a larger vector than FromV template HWY_INLINE VFromD ZeroExtendResizeBitCast(FromSizeTag /* from_size_tag */, ToSizeTag /* to_size_tag */, DTo d_to, DFrom /* d_from */, VFromD v) { return ResizeBitCast(d_to, v); } } // namespace detail // ------------------------------ Set template HWY_API VFromD Set(D d, const T2 t) { VFromD v; for (size_t i = 0; i < MaxLanes(d); ++i) { v.raw[i] = ConvertScalarTo>(t); } return v; } //HWY_INSERT template , HWY_NEON_IF_EMULATED_D(D)> HWY_API V InsertLane(const V v, TFromD t) { const D d; const RebindToUnsigned du; const uint16_t tu = BitCastScalar(t); return BitCast(d, InsertLane(BitCast(du, v), tu)); } } // namespace detail // Requires one overload per vector length because InsertLane<3> may be a // compile error. template HWY_API Vec128 InsertLane(const Vec128 v, size_t i, T t) { HWY_DASSERT(i == 0); (void)i; return Set(DFromV(), t); } template HWY_API Vec128 InsertLane(const Vec128 v, size_t i, T t) { #if !HWY_IS_DEBUG_BUILD && HWY_COMPILER_GCC // includes clang if (__builtin_constant_p(i)) { switch (i) { case 0: return detail::InsertLane<0>(v, t); case 1: return detail::InsertLane<1>(v, t); } } #endif const DFromV d; alignas(16) T lanes[2]; Store(v, d, lanes); lanes[i] = t; ret t4, TFromD t5, TFromD t6, TFromD t7) { const RebindToSigned di; return BitCast(d, Dup128VecFromValues( di, BitCastScalar(t0), BitCastScalar(t1), BitCastScalar(t2), BitCastScalar(t3), BitCastScalar(t4), BitCastScalar(t5), BitCastScalar(t6), BitCastScalar(t7))); } #if (HWY_COMPILER_GCC || HWY_COMPILER_CLANGCL) && HWY_NEON_HAVE_F16C && \ HWY_HAVE_SCALAR_F16_TYPE template HWY_API VFromD Dup128VecFromValues(D d, TFromD t0, TFromD t1, TFromD t2, TFromD t3, TFromD /*t4*/, TFromD /*t5*/, TFromD /*t6*/, TFromD /*t7*/) { typedef __fp16 GccF16RawVec t2, TFromD t3, TFromD t4, TFromD t5, TFromD t6, TFromD t7, TFromD /*t8*/, TFromD /*t9*/, TFromD /*t10*/, TFromD /*t11*/, TFromD /*t12*/, TFromD /*t13*/, TFromD /*t14*/, TFromD /*t15*/) { #if HWY_COMPILER_GCC || HWY_COMPILER_CLANGCL typedef int8_t GccI8RawVectType __attribute__((__vector_size__(8))); (void)d; const GccI8RawVectType raw = { static_cast(t0), static_cast(t1), static_cast(t2), static_cast(t3), static_cast(t4), static_cast(t5), static_cast(t6), static_cast(t7)}; return VFromD(reinterpret_cast::Raw>(raw)); #else return ResizeBitCast( d, Set(Full64(), BitCastScalar(detail::Vec64ValsWrapper>{ {t0, t using DFromM = Simd; template using TFromV = typename V::PrivateT; // TODO(janwas): ForDemoteVectors, in convert_test and demote_test, appear to // instantiate this with D = double x 4. The cause is unknown. Previously, // defining this in terms of Set rejected that via SFINAE because only // V_SIZE = 16 and V_SIZE <= 8 overloads were defined. As a workaround, // truncate the lane count to 128 bits. template using VFromD = Vec128, HWY_MIN(16 / sizeof(TFromD), MaxLanes(D()))>; // ------------------------------ BitCast namespace detail { // Converts from Vec128 to Vec128 using the // vreinterpret*_u8_*() set of functions. #define HWY_NEON_BUILD_TPL_HWY_CAST_TO_U8 #define HWY_NEON_BUILD_RET_HWY_CAST_TO_U8(type, size) \ Vec128 #define HWY_NEON_BUILD_PARAM_HWY_CAST_TO_U8(type, size) Vec128 v #define HWY_NEON_BUILD_ARG_HWY_CAST_TO_U8 v.raw // Sx, y) vzip_f32(x, y).val[1] #define vzip2q_s8(x, y) vzipq_s8(x, y).val[1] #define vzip2q_u8(x, y) vzipq_u8(x, y).val[1] #define vzip2q_s16(x, y) vzipq_s16(x, y).val[1] #define vzip2q_u16(x, y) vzipq_u16(x, y).val[1] #define vzip2q_s32(x, y) vzipq_s32(x, y).val[1] #define vzip2q_u32(x, y) vzipq_u32(x, y).val[1] #define vzip2q_f32(x, y) vzipq_f32(x, y).val[1] #endif // Wrappers over uint8x16x2_t etc. so we can define StoreInterleaved2 // overloads for all vector types, even those (bfloat16_t) where the // underlying vector is the same as others (uint16_t). template struct Tuple2; template struct Tuple3; template struct Tuple4; template <> struct Tuple2 { uint8x16x2_t raw; }; template struct Tuple2 { uint8x8x2_t raw; }; template <> struct Tuple2 { int8x16x2_t raw; }; template struct Tuple2 { int8x8x2_t raw; }; template <> struct Tuple2 { uint16x8x2_t ULATED_D(D) HWY_IF_NOT_BF16_D(D) #elif HWY_HAVE_FLOAT16 && HWY_NEON_HAVE_BFLOAT16 // NOTE: hwy::EnableIf()>* = nullptr is used instead of // hwy::EnableIf* = nullptr to avoid compiler errors since // !hwy::IsSame() is always false and as !hwy::IsSame() will cause // SFINAE to occur instead of a hard error due to a dependency on the D template // argument #define HWY_NEON_IF_EMULATED_D(D) hwy::EnableIf()>* = nullptr #define HWY_GENERIC_IF_EMULATED_D(D) \ hwy::EnableIf()>* = nullptr #define HWY_NEON_IF_NOT_EMULATED_D(D) hwy::EnableIf* = nullptr #else #error "Logic error, handled all four cases" #endif // float #define HWY_NEON_DEF_FUNCTION_FLOAT_32(name, prefix, infix, args) \ HWY_NEON_DEF_FUNCTION(float32, 4, name, prefix##q, infix, f32, args) \ HWY_NEON_DEF_FUNCTION(float32, 2, name, prefix, infix, f32, args) \ HWY_NEON_DEF_FUNCTION(float32, 1, name, prefix, infix, f32, args) // double #if HWY_HAVE_FLOAT64 #defreturn InterleaveOdd(d, p_even, p_odd); #else return InterleaveEven(d, p_even, p_odd); #endif } #if !HWY_PPC_HAVE_10 template HWY_API Vec64 MulHigh(Vec64 a, Vec64 b) { T p_hi; Mul128(GetLane(a), GetLane(b), &p_hi); return Set(Full64(), p_hi); } template HWY_API Vec128 MulHigh(Vec128 a, Vec128 b) { const DFromV d; const Half dh; return Combine(d, MulHigh(UpperHalf(dh, a), UpperHalf(dh, b)), MulHigh(LowerHalf(dh, a), LowerHalf(dh, b))); } #endif // !HWY_PPC_HAVE_10 #undef HWY_PPC_IF_MULHIGH_USING_VEC_MULH #undef HWY_PPC_IF_MULHIGH_8_16_32_NOT_USING_VEC_MULH // Multiplies even lanes (0, 2, ..) and places the double-wide result into // even and the upper half into its odd neighbor lane. template HWY_API Vec128, (N + 1) / 2> MulEven(Vec128_IF_T_SIZE_ONE_OF(T, (1 << 1) | (1 << 2) | (1 << 4))> HWY_API Vec128 SaturatedSub(Vec128 a, Vec128 b) { return Vec128{vec_subs(a.raw, b.raw)}; } #endif // HWY_S390X_HAVE_Z14 #if HWY_PPC_HAVE_10 template )> HWY_API V SaturatedSub(V a, V b) { const DFromV d; const auto diff = Sub(a, b); const auto overflow_mask = BroadcastSignBit(detail::TernaryLogic<0x18>(a, b, diff)); const auto overflow_result = Xor(BroadcastSignBit(a), Set(d, LimitsMax())); return IfNegativeThenElse(overflow_mask, overflow_result, diff); } #endif // HWY_PPC_HAVE_10 // ------------------------------ AverageRound // Returns (a + b + 1) / 2 #ifdef HWY_NATIVE_AVERAGE_ROUND_UI32 #undef HWY_NATIVE_AVERAGE_ROUND_UI32 #else #define HWY_NATIVE_AVERAGE_ROUND_UI32 #endif #if HWY_S390X_HAVE_Z14 #ifdef HWY_NATIVE_AVERAGE_ROUND_UI64 #undef HWY_NATIVE_AVERAGE_ROUND_UI64 #else #define HWY_NATIVE_AVERAGE_ROUND_UI64 #endif #define HWY_PPC_IF_AVERAG { return Load(d, p); } // 128-bit SIMD => nothing to duplicate, same as an unaligned load. template > HWY_API VFromD LoadDup128(D d, const T* HWY_RESTRICT p) { return LoadU(d, p); } #if (HWY_PPC_HAVE_9 && HWY_ARCH_PPC_64) || HWY_S390X_HAVE_Z14 #ifdef HWY_NATIVE_LOAD_N #undef HWY_NATIVE_LOAD_N #else #define HWY_NATIVE_LOAD_N #endif template > HWY_API VFromD LoadN(D d, const T* HWY_RESTRICT p, size_t max_lanes_to_load) { #if HWY_COMPILER_GCC && !HWY_IS_DEBUG_BUILD if (__builtin_constant_p(max_lanes_to_load) && max_lanes_to_load == 0) { return Zero(d); } if (__builtin_constant_p(max_lanes_to_load >= HWY_MAX_LANES_D(D)) && max_lanes_to_load >= HWY_MAX_LANES_D(D)) { return LoadU(d, p); } #endif const size_t num_of_bytes_to_load = HWY_MIN(max_lanes_to_load, HWY_MAX_LANES_D(D)) * sizeof(TFromD); const Repartition du8; #if HWY_S390X_HAVE_Z14 return (num_of_ 3,2,1,0 (0 is least-significant). // Shuffle0321 rotates one lane to the right (the previous least-significant // lane is now most-significant). These could also be implemented via // CombineShiftRightBytes but the shuffle_abcd notation is more convenient. // Swap 32-bit halves in 64-bit halves. template HWY_API Vec128 Shuffle2301(Vec128 v) { static_assert(sizeof(T) == 4, "Only for 32-bit lanes"); static_assert(N == 2 || N == 4, "Does not make sense for N=1"); const __vector unsigned char kShuffle = {4, 5, 6, 7, 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11}; return Vec128{vec_perm(v.raw, v.raw, kShuffle)}; } // These are used by generic_ops-inl to implement LoadInterleaved3. As with // Intel's shuffle* intrinsics and InterleaveLower, the lower half of the output // comes from the first argument. namespace detail { template HWY_API Vec32 ShuffleTwo2301(Vec32 a, Vec32 b) { IAL_FLOAT(T)> HWY_API Vec128 Neg(const Vec128 v) { return Xor(v, SignBit(DFromV())); } // ------------------------------ Abs // Returns absolute value, except that LimitsMin() maps to LimitsMax() + 1. template HWY_API Vec128 Abs(Vec128 v) { // If T is a signed integer type, use Max(v, Neg(v)) instead of vec_abs to // avoid undefined behavior in the case where v[i] == LimitsMin(). return Max(v, Neg(v)); } template HWY_API Vec128 Abs(Vec128 v) { return Vec128{vec_abs(v.raw)}; } // ------------------------------ CopySign #if HWY_S390X_HAVE_Z14 template HWY_API V CopySign(const V magn, const V sign) { static_assert(IsFloat>(), "Only makes sense for floating-point"); const DFromV d; const auto msb = SignBit(d); // Truth table for msb, magn, sign | bitwise msb ? sign : mag // 0 0 0 | 0 // w128>::RawT; return VFromD{vec_splats(static_cast(t))}; } template )> HWY_API VFromD Set(D d, TFromD t) { const RebindToUnsigned du; return BitCast(d, Set(du, BitCastScalar>(t))); } // Returns a vector with uninitialized elements. template HWY_API VFromD Undefined(D d) { #if HWY_COMPILER_GCC_ACTUAL // Suppressing maybe-uninitialized both here and at the caller does not work, // so initialize. return Zero(d); #elif HWY_HAS_BUILTIN(__builtin_nondeterministic_value) return VFromD{__builtin_nondeterministic_value(Zero(d).raw)}; #else HWY_DIAGNOSTICS(push) HWY_DIAGNOSTICS_OFF(disable : 4700, ignored "-Wuninitialized") typename detail::Raw128>::type raw; return VFromD{raw}; HWY_DIAGNOSTICS(pop) #endif } // ------------------------------ GetLane // Gets the single value stored in a vector/part. template HWY_API T GetLane(Vec12 License. // 128-bit vectors for VSX/Z14 // External include guard in highway.h - see comment there. #if HWY_TARGET == HWY_Z14 || HWY_TARGET == HWY_Z15 #define HWY_S390X_HAVE_Z14 1 #else #define HWY_S390X_HAVE_Z14 0 #endif #pragma push_macro("vector") #pragma push_macro("pixel") #pragma push_macro("bool") #undef vector #undef pixel #undef bool #if HWY_S390X_HAVE_Z14 #include #else #include #endif #pragma pop_macro("vector") #pragma pop_macro("pixel") #pragma pop_macro("bool") #include "hwy/ops/shared-inl.h" // clang's altivec.h gates some intrinsics behind #ifdef __POWER10_VECTOR__, and // some GCC do the same for _ARCH_PWR10. // This means we can only use POWER10-specific intrinsics in static dispatch // mode (where the -mpower10-vector compiler flag is passed). Same for PPC9. // On other compilers, the usual target check is sufficient. #if !HWY_S390X_HAVE_Z14 && HWY_TARGET <= HWY_PPC9 && \ (defined(_ARCH_PWR9) || defined(__POWER9_VECTOR__)) #define HWY_PPC_HAVE_9 1 #elsUnsignedTag /*to_type_tag*/, hwy::SizeTag /*to_lane_size_tag*/, hwy::UnsignedTag /*from_type_tag*/, D d_to, V v) { #if HWY_IS_LITTLE_ENDIAN // On little-endian targets, need to shift each lane of the bitcasted vector // right by kToLaneSize * 4 bits to get the bits of the odd source lanes into // the lower kToLaneSize * 4 bits of the result. // The right shift below will zero out the upper kToLaneSize * 4 bits of the // result. return ShiftRight(BitCast(d_to, v)); #else // On big-endian targets, the bits of the even source lanes are already // in the lower kToLaneSize * 4 bits of the lanes of the bitcasted vector. // Simply need to zero out the upper bits of each lane of the bitcasted // vector. return And(BitCast(d_to, v), Set(d_to, static_cast>(LimitsMax>()))); #endif } // Unsigned to signed: Same as unsigned->unsigned PromoteEvenTo/PromoteOddTo // followed by BitCast to signed template tuple, VFromD val) { static_assert(kIndex < 4, "Tuple index out of bounds"); if (kIndex == 0) { tuple.v0 = val; } else if (kIndex == 1) { tuple.v1 = val; } else if (kIndex == 2) { tuple.v2 = val; } else { tuple.v3 = val; } return tuple; } #endif // !HWY_HAVE_SCALABLE || HWY_IDE // ------------------------------ Rol/Ror (And, Or, Neg, Shl, Shr) #if (defined(HWY_NATIVE_ROL_ROR_8) == defined(HWY_TARGET_TOGGLE)) #ifdef HWY_NATIVE_ROL_ROR_8 #undef HWY_NATIVE_ROL_ROR_8 #else #define HWY_NATIVE_ROL_ROR_8 #endif template )> HWY_API V Rol(V a, V b) { const DFromV d; const RebindToSigned di; const RebindToUnsigned du; const auto shift_amt_mask = Set(du, uint8_t{7}); const auto shl_amt = And(BitCast(du, b), shift_amt_mask); const auto shr_amt = And(BitCast(du, Neg(BitCast(di, b))), shift_amt_mask); const auto vu = BitCast(du, a); return BitCast(d, Or(Shl(vu, shl_amt), Shr(vu, shr_amt))); } BF16_D(D) HWY_IF_BF16(hwy::HWY_NAMESPACE::TFromD) #define HWY_IF_NOT_BF16_D(D) HWY_IF_NOT_BF16(hwy::HWY_NAMESPACE::TFromD) #define HWY_IF_F16_D(D) HWY_IF_F16(hwy::HWY_NAMESPACE::TFromD) #define HWY_IF_NOT_F16_D(D) HWY_IF_NOT_F16(hwy::HWY_NAMESPACE::TFromD) #define HWY_IF_F32_D(D) HWY_IF_F32(hwy::HWY_NAMESPACE::TFromD) #define HWY_IF_F64_D(D) HWY_IF_F64(hwy::HWY_NAMESPACE::TFromD) #define HWY_V_SIZE_D(D) \ (HWY_MAX_LANES_D(D) * sizeof(hwy::HWY_NAMESPACE::TFromD)) #define HWY_IF_V_SIZE_D(D, bytes) \ HWY_IF_V_SIZE(hwy::HWY_NAMESPACE::TFromD, HWY_MAX_LANES_D(D), bytes) #define HWY_IF_V_SIZE_LE_D(D, bytes) \ HWY_IF_V_SIZE_LE(hwy::HWY_NAMESPACE::TFromD, HWY_MAX_LANES_D(D), bytes) #define HWY_IF_V_SIZE_GT_D(D, bytes) \ HWY_IF_V_SIZE_GT(hwy::HWY_NAMESPACE::TFromD, HWY_MAX_LANES_D(D), bytes) // Same, but with a vector argument. ops/*-inl.h define their own TFromV. #define HWY_IF_UNSIGNED_V(V) HWY_IF_UNSIGNED(hwy::HWY_NAMESPACE::TFromV) #define HWY_IF_NOT_UNSIGNED_V(V) \ H 0, "Does not make sense to have zero lanes"); static_assert(kNumLanes <= HWY_LANES(T), "Too many lanes"); using type = Simd; }; } // namespace detail // ------------------------------ Aliases for Simd<> // Tag describing a full vector (kPow2 == 0: the most common usage, e.g. 1D // loops where the application does not care about the vector size) or a // fraction/multiple of one. Fractions (kPow2 < 0) are useful for arguments or // return values of type promotion and demotion. User-specified kPow2 is // interpreted as `HWY_MIN(kPow2, HWY_MAX_POW2)`. template using ScalableTag = typename detail::ScalableTagChecker::type; // Tag describing a vector with *up to* kLimit active lanes, even on targets // with scalable vectors and HWY_SCALAR. The runtime lane count `Lanes(tag)` may // be less than kLimit, and is 1 on HWY_SCALAR. This alias is typically used for // 1D loops with a relatively low application-defined upper bound, e.g. for 8x8 // DCTs. Howevot work for this include, so hide it from the IDE. #if !HWY_IDE #include #endif #include "hwy/detect_compiler_arch.h" #include "hwy/detect_targets.h" // Separate header because foreach_target.h re-enables its include guard. #include "hwy/ops/set_macros-inl.h" // IWYU pragma: end_exports #if HWY_IS_MSAN #include #endif // We are covered by the highway.h include guard, but generic_ops-inl.h // includes this again #if HWY_IDE. // clang-format off #if defined(HIGHWAY_HWY_OPS_SHARED_TOGGLE) == defined(HWY_TARGET_TOGGLE) // NOLINT // clang-format on #ifdef HIGHWAY_HWY_OPS_SHARED_TOGGLE #undef HIGHWAY_HWY_OPS_SHARED_TOGGLE #else #define HIGHWAY_HWY_OPS_SHARED_TOGGLE #endif HWY_BEFORE_NAMESPACE(); namespace hwy { namespace HWY_NAMESPACE { // NOTE: GCC generates incorrect code for vector arguments to non-inlined // functions in two situations: // - on Windows and GCC 10.3, passing by value crashes due to unaligned loads: // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=aw)}; } } // namespace detail template HWY_API Vec128 IfThenZeroElse(Mask128 mask, Vec128 no) { return detail::IfThenZeroElse(hwy::SizeTag(), mask, no); } template HWY_API Vec128 IfThenZeroElse(Mask128 mask, Vec128 no) { return Vec128{_mm_mask_xor_ps(no.raw, mask.raw, no.raw, no.raw)}; } template HWY_API Vec128 IfThenZeroElse(Mask128 mask, Vec128 no) { return Vec128{_mm_mask_xor_pd(no.raw, mask.raw, no.raw, no.raw)}; } // Generic for all vector lengths. template , HWY_IF_SPECIAL_FLOAT_D(D)> HWY_API V IfThenZeroElse(MFromD mask, V no) { const RebindToUnsigned du; return BitCast(D(), IfThenZeroElse(RebindMask(du, mask), BitCast(du, no))); } // ------------------------------ Mask logical namesecFromMask(const Mask128 v) { return Vec128{_mm_castsi128_pd(_mm_movm_epi64(v.raw))}; } // Generic for all vector lengths. template HWY_API VFromD VecFromMask(D /* tag */, MFromD v) { return VecFromMask(v); } // ------------------------------ RebindMask (MaskFromVec) template HWY_API MFromD RebindMask(DTo /* tag */, Mask128 m) { static_assert(sizeof(TFrom) == sizeof(TFromD), "Must have same size"); return MFromD{m.raw}; } // ------------------------------ IfThenElse namespace detail { template HWY_INLINE Vec128 IfThenElse(hwy::SizeTag<1> /* tag */, Mask128 mask, Vec128 yes, Vec128 no) { return Vec128{_mm_mask_blend_epi8(mask.raw, no.raw, yes.raw)}; } template HWY_INLINE Vec128 IfThenElse(hwy::SizeTag<2> /* ta MFromD CombineMasks(D /*d*/, MFromD> hi, MFromD> lo) { #if HWY_COMPILER_HAS_MASK_INTRINSICS const __mmask16 combined_mask = _mm512_kunpackb( static_cast<__mmask16>(hi.raw), static_cast<__mmask16>(lo.raw)); #else const auto combined_mask = ((static_cast(hi.raw) << 8) | (lo.raw & 0xFFu)); #endif return MFromD{static_cast().raw)>(combined_mask)}; } // ------------------------------ LowerHalfOfMask (MFromD) #ifdef HWY_NATIVE_LOWER_HALF_OF_MASK #undef HWY_NATIVE_LOWER_HALF_OF_MASK #else #define HWY_NATIVE_LOWER_HALF_OF_MASK #endif // Generic for all vector lengths template HWY_API MFromD LowerHalfOfMask(D d, MFromD> m) { using RawM = decltype(MFromD().raw); constexpr size_t kN = MaxLanes(d); constexpr size_t kNumOfBitsInRawMask = sizeof(RawM) * 8; MFromD result_mask{static_cast(m.raw)}; if (kN < kNumOfBitsInRawMask) { result_mask = And(result_mask, MFromIF_THEN_ELSE #endif template // generic for all vector lengths HWY_API V BitwiseIfThenElse(V mask, V yes, V no) { return IfVecThenElse(mask, yes, no); } #endif // HWY_X86_HAVE_TERNARY_LOGIC // ------------------------------ Operator overloads (internal-only if float) template HWY_API Vec128 operator&(const Vec128 a, const Vec128 b) { return And(a, b); } template HWY_API Vec128 operator|(const Vec128 a, const Vec128 b) { return Or(a, b); } template HWY_API Vec128 operator^(const Vec128 a, const Vec128 b) { return Xor(a, b); } // ------------------------------ PopulationCount // 8/16 require BITALG, 32/64 require VPOPCNTDQ. #if HWY_TARGET <= HWY_AVX3_DL #ifdef HWY_NATIVE_POPCNT #undef HWY_NATIVE_POPCNT #else #define HWY_NATIVE_POPCNT #endif namespace detail { template HWY_INLINE Vec128 PopulationCount(hwy::SizeTag<1> /* tag */, tant_p(v[15]); } #if HWY_TARGET <= HWY_AVX2 template static HWY_INLINE HWY_MAYBE_UNUSED bool IsConstantRawX86Vec( hwy::SizeTag<32> /* num_of_lanes_tag*/, RawV v) { return __builtin_constant_p(v[0]) && __builtin_constant_p(v[1]) && __builtin_constant_p(v[2]) && __builtin_constant_p(v[3]) && __builtin_constant_p(v[4]) && __builtin_constant_p(v[5]) && __builtin_constant_p(v[6]) && __builtin_constant_p(v[7]) && __builtin_constant_p(v[8]) && __builtin_constant_p(v[9]) && __builtin_constant_p(v[10]) && __builtin_constant_p(v[11]) && __builtin_constant_p(v[12]) && __builtin_constant_p(v[13]) && __builtin_constant_p(v[14]) && __builtin_constant_p(v[15]) && __builtin_constant_p(v[16]) && __builtin_constant_p(v[17]) && __builtin_constant_p(v[18]) && __builtin_constant_p(v[19]) && __builtin_constant_p(v[20]) && __builtin_constant_p(v[21]) && __builtin_constant_p(v[22]) && __builtin_constant_p(v[23]) && \ (HWY_COMPILER_GCC_ACTUAL >= 1501 || \ (HWY_COMPILER3_CLANG >= 200103 && HWY_COMPILER_CLANG != 2100)) #define HWY_X86_HAVE_AVX10_2_OPS 1 #else #define HWY_X86_HAVE_AVX10_2_OPS 0 #endif template struct Raw128 { using type = __m128i; }; #if HWY_HAVE_FLOAT16 template <> struct Raw128 { using type = __m128h; }; #endif // HWY_HAVE_FLOAT16 template <> struct Raw128 { using type = __m128; }; template <> struct Raw128 { using type = __m128d; }; } // namespace detail template class Vec128 { using Raw = typename detail::Raw128::type; public: using PrivateT = T; // only for DFromV static constexpr size_t kPrivateN = N; // only for DFromV // Compound assignment. Only usable if there is a corresponding non-member // binary operator overload. For example, only f32 and f64 support division. HWY_INLINE Vec128& operator*=(const Vec128 other) { return *this = (*this * other); } // We don't know the lane type, so assume 8-bit. For larger types, this will // de-canonicalize the predicate, i.e. set bits to 1 even though they do not // correspond to the lowest byte in the lane. Arm says such bits are ignored. return svbrka_b_z(HWY_SVE_PTRUE(8), m); } HWY_API svbool_t SetOnlyFirst(svbool_t m) { return svbrka_b_z(m, m); } HWY_API svbool_t SetAtOrAfterFirst(svbool_t m) { return Not(SetBeforeFirst(m)); } // ------------------------------ PromoteMaskTo #ifdef HWY_NATIVE_PROMOTE_MASK_TO #undef HWY_NATIVE_PROMOTE_MASK_TO #else #define HWY_NATIVE_PROMOTE_MASK_TO #endif template ) * 2)> HWY_API svbool_t PromoteMaskTo(DTo /*d_to*/, DFrom /*d_from*/, svbool_t m) { return svunpklo_b(m); } template ) * 2)> HWY_API svbool_t PromoteMaskTo(DTo d_to, DFrom d_from, svbool_t m) { using TFrom = TFromD; using TWFrom = MakeWide< Neg(BitCast(di, bits))); \ } HWY_SVE_FOREACH_UI(HWY_SVE_ROUNDING_SHR, RoundingShr, rshl) #undef HWY_SVE_ROUNDING_SHR template HWY_API V RoundingShiftRightSame(V v, int bits) { const DFromV d; using T = TFromD; return RoundingShr(v, Set(d, static_cast(bits))); } #endif // HWY_SVE_HAVE_2 // ------------------------------ BroadcastSignBit (ShiftRight) template HWY_API V BroadcastSignBit(const V v) { return ShiftRight) * 8 - 1>(v); } // ------------------------------ Abs (ShiftRight, Add, Xor, AndN) // Workaround for incorrect results with `svabs`. #if HWY_COMPILER_CLANG template HWY_API V Abs(V v) { const V sign = BroadcastSignBit(v); return Xor(Add(v, sign), sign); } template HWY_NOINLINE V Abs(V v) { const DFromV d; const RebindToUnsigned du; using TU = MakeUnsigned>; re zero-extended) const svuint64_t lo = svextw_u64_x(pg, BitCast(du64, sums_of_4)); return Add(hi, lo); #endif } HWY_API svint64_t SumsOf8(const svint8_t v) { const ScalableTag di32; const ScalableTag di64; const svbool_t pg = detail::PTrue(di64); const svint32_t sums_of_4 = svdot_n_s32(Zero(di32), v, 1); #if HWY_SVE_HAVE_2 return svadalp_s64_x(pg, Zero(di64), sums_of_4); #else const svint64_t hi = svasr_n_s64_x(pg, BitCast(di64, sums_of_4), 32); // Isolate the lower 32 bits (to be added to the upper 32 and sign-extended) const svint64_t lo = svextw_s64_x(pg, BitCast(di64, sums_of_4)); return Add(hi, lo); #endif } // ------------------------------ SumsOf2 #if HWY_SVE_HAVE_2 namespace detail { HWY_INLINE svint16_t SumsOf2(hwy::SignedTag /*type_tag*/, hwy::SizeTag<1> /*lane_size_tag*/, svint8_t v) { const ScalableTag di16; const svbool_t pg = detail::PTrue(di16); return svadalp_s16_x(pg, Zero(di16), v); } HWY_INLINE svuint16_emplate HWY_API VBF16 Dup128VecFromValues(D d, TFromD t0, TFromD t1, TFromD t2, TFromD t3, TFromD t4, TFromD t5, TFromD t6, TFromD t7) { #if HWY_SVE_HAVE_BF16_FEATURE (void)d; return svdupq_n_bf16(t0, t1, t2, t3, t4, t5, t6, t7); #else const RebindToUnsigned du; return BitCast( d, Dup128VecFromValues( du, BitCastScalar(t0), BitCastScalar(t1), BitCastScalar(t2), BitCastScalar(t3), BitCastScalar(t4), BitCastScalar(t5), BitCastScalar(t6), BitCastScalar(t7))); #endif } template HWY_API svint32_t Dup128VecFromValues(D /*d*/, TFromD t0, TFromD t1, TFromD t2, TFromD t3) { return svdupq_n_s32(t0, t1, t2, t3); } template HWY_API svuint32_SVE_IF_EMULATED_D(DFromV)> HWY_INLINE svuint8_t BitCastToByte(V v) { const RebindToUnsigned> du; return BitCastToByte(BitCast(du, v)); } template HWY_INLINE VFromD BitCastFromByte(D d, svuint8_t v) { const RebindToUnsigned du; return BitCastFromByte(du, v); } #endif // HWY_SVE_HAVE_BF16_FEATURE || HWY_SVE_HAVE_BF16_VEC #undef HWY_SVE_CAST_NOP #undef HWY_SVE_CAST } // namespace detail template HWY_API VFromD BitCast(D d, FromV v) { return detail::BitCastFromByte(d, detail::BitCastToByte(v)); } // ------------------------------ Undefined #define HWY_SVE_UNDEFINED(BASE, CHAR, BITS, HALF, NAME, OP) \ template \ HWY_API HWY_SVE_V(BASE, BITS) \ NAME(HWY_SVE_D(BASE, BITS, N, kPow2) /* d */) { \ return sv##OP##_##CHAR##BITS(); \ } HWY_SVE_FOREACH(HWY_SVE_UNDEFINED, Undefined, undef) #if HScalableTag; \ }; HWY_SVE_FOREACH(HWY_SPECIALIZE, _, _) #if HWY_SVE_HAVE_BF16_FEATURE || HWY_SVE_HAVE_BF16_VEC HWY_SVE_FOREACH_BF16_UNCONDITIONAL(HWY_SPECIALIZE, _, _) #endif #undef HWY_SPECIALIZE // Note: _x (don't-care value for inactive lanes) avoids additional MOVPRFX // instructions, and we anyway only use it when the predicate is ptrue. // vector = f(vector), e.g. Not #define HWY_SVE_RETV_ARGPV(BASE, CHAR, BITS, HALF, NAME, OP) \ HWY_API HWY_SVE_V(BASE, BITS) NAME(HWY_SVE_V(BASE, BITS) v) { \ return sv##OP##_##CHAR##BITS##_x(HWY_SVE_PTRUE(BITS), v); \ } #define HWY_SVE_RETV_ARGV(BASE, CHAR, BITS, HALF, NAME, OP) \ HWY_API HWY_SVE_V(BASE, BITS) NAME(HWY_SVE_V(BASE, BITS) v) { \ return sv##OP##_##CHAR##BITS(v); \ } #define HWY_SVE_RETV_ARGMV_M(BASE, CHAR, BITS, HALF, NAME, OP) \ HWY_API HWY_SVE_V(BASE, BITS) \ NAME(HWY_SVE_V(BASE, BITS) no, svbool_t m, HWY_SVand svcvtnt_bf16_f32_x intrinsics are available, even if the __bf16 type // is disabled #if HWY_SVE_HAVE_BF16_VEC && defined(__ARM_FEATURE_SVE_BF16) #define HWY_SVE_HAVE_F32_TO_BF16C 1 #else #define HWY_SVE_HAVE_F32_TO_BF16C 0 #endif HWY_BEFORE_NAMESPACE(); namespace hwy { namespace HWY_NAMESPACE { template struct DFromV_t {}; // specialized in macros template using DFromV = typename DFromV_t>::type; template using TFromV = TFromD>; // ================================================== MACROS // Generate specializations and function definitions using X macros. Although // harder to read and debug, writing everything manually is too bulky. namespace detail { // for code folding // Args: BASE, CHAR, BITS, HALF, NAME, OP // Unsigned: #define HWY_SVE_FOREACH_U08(X_MACRO, NAME, OP) X_MACRO(uint, u, 8, 8, NAME, OP) #define HWY_SVE_FOREACH_U16(X_MACRO, NAME, OP) X_MACRO(uint, u, 16, 8, NAME, OP) #define HWY_SVE_FOREACH_U32(X_MACRO, NAME, OP) \ X_MACR HWY_INLINE VFromD>> SumsOf2( hwy::UnsignedTag, hwy::SizeTag<4> /*lane_size_tag*/, V v) { return VFromD>>{__lsx_vhaddw_du_wu(v.raw, v.raw)}; } } // namespace detail // ------------------------------ SumsOf8 template HWY_API Vec128 SumsOf8(const Vec128 v) { __m128i temp = __lsx_vhaddw_hu_bu(v.raw, v.raw); temp = __lsx_vhaddw_wu_hu(temp, temp); return Vec128{__lsx_vhaddw_du_wu(temp, temp)}; } template HWY_API Vec128 SumsOf8(const Vec128 v) { __m128i temp = __lsx_vhaddw_h_b(v.raw, v.raw); temp = __lsx_vhaddw_w_h(temp, temp); return Vec128{__lsx_vhaddw_d_w(temp, temp)}; } // ------------------------------ SaturatedAdd // Returns a + b clamped to the destination range. #ifdef HWY_NATIVE_I32_SATURATED_ADDSUB #undef HWY_NATIVE_I32_SATURATED_ADDSUB #else #define HWY_NATIVE_I32_SATURATED_ADDSUB #endif #ifdef HWY_NATIVE_I64_SATURA_INLINE Mask128 Ge(hwy::FloatTag /*tag*/, Vec128 a, Vec128 b) { return Mask128{ reinterpret_cast<__m128d>(__lsx_vfcmp_cle_d(b.raw, a.raw))}; } } // namespace detail template HWY_API Mask128 operator>=(Vec128 a, Vec128 b) { return detail::Ge(hwy::TypeTag(), a, b); } // ------------------------------ Reversed comparisons template HWY_API Mask128 operator<(Vec128 a, Vec128 b) { return b > a; } template HWY_API Mask128 operator<=(Vec128 a, Vec128 b) { return b >= a; } // ------------------------------ Iota (Load) namespace detail { template HWY_INLINE VFromD Iota0(D d) { return Dup128VecFromValues( d, TFromD{0}, TFromD{1}, TFromD{2}, TFromD{3}, TFromD{4}, TFromD{5}, TFromD{6}, TFromD{7}, TFro Vec128 b) { return Mask128{__lsx_vseq_h(a.raw, b.raw)}; } template HWY_API Mask128 operator==(Vec128 a, Vec128 b) { return Mask128{__lsx_vseq_w(a.raw, b.raw)}; } template HWY_API Mask128 operator==(const Vec128 a, const Vec128 b) { return Mask128{__lsx_vseq_d(a.raw, b.raw)}; } // Float template HWY_API Mask128 operator==(Vec128 a, Vec128 b) { return Mask128{ reinterpret_cast<__m128>(__lsx_vfcmp_ceq_s(a.raw, b.raw))}; } template HWY_API Mask128 operator==(Vec128 a, Vec128 b) { return Mask128{ reinterpret_cast<__m128d>(__lsx_vfcmp_ceq_d(a.raw, b.raw))}; } // ------------------------------ Inequality // mMask(d, b))); } // ------------------------------ ExclusiveNeither template HWY_API Mask128 ExclusiveNeither(const Mask128 a, Mask128 b) { const Simd d; return MaskFromVec(AndNot(VecFromMask(d, a), Not(VecFromMask(d, b)))); } // ------------------------------ ShiftLeft template HWY_API Vec128 ShiftLeft(const Vec128 v) { return Vec128{__lsx_vslli_b(v.raw, kBits)}; } template HWY_API Vec128 ShiftLeft(const Vec128 v) { return Vec128{__lsx_vslli_h(v.raw, kBits)}; } template HWY_API Vec128 ShiftLeft(const Vec128 v) { return Vec128{__lsx_vslli_w(v.raw, kBits)}; } template HWY_API Vec128 ShiftLeft(const Vec128 v) { return Vec128{__lsx_vslli_d(v.raw, kBits)}; } template HWY_API Vec128 t1, TFromD t2, TFromD t3) { typedef float GccF32RawVectType __attribute__((__vector_size__(16))); const GccF32RawVectType raw = {t0, t1, t2, t3}; return VFromD{reinterpret_cast<__m128>(raw)}; } template HWY_API VFromD Dup128VecFromValues(D /*d*/, TFromD t0, TFromD t1) { typedef double GccF64RawVectType __attribute__((__vector_size__(16))); const GccF64RawVectType raw = {t0, t1}; return VFromD{reinterpret_cast<__m128d>(raw)}; } // ================================================== LOGICAL // ------------------------------ And template HWY_API Vec128 And(Vec128 a, Vec128 b) { const DFromV d; const RebindToUnsigned du; return BitCast(d, VFromD{ __lsx_vand_v(BitCast(du, a).raw, BitCast(du, b).raw)}); } // ------------------------------ AndNot // Returns ~not_mask & mask. e_t N> HWY_INLINE Vec128 BitCastToByte(Vec128 v) { return Vec128{BitCastToInteger(v.raw)}; } // Cannot rely on function overloading because return types differ. template struct BitCastFromInteger128 { HWY_INLINE __m128i operator()(__m128i v) { return v; } }; template <> struct BitCastFromInteger128 { HWY_INLINE __m128 operator()(__m128i v) { return reinterpret_cast<__m128>(v); } }; template <> struct BitCastFromInteger128 { HWY_INLINE __m128d operator()(__m128i v) { return reinterpret_cast<__m128d>(v); } }; } // namespace detail // ------------------------------ Zero // Use HWY_MAX_LANES_D here because VFromD is defined in terms of Zero. template HWY_API Vec128, HWY_MAX_LANES_D(D)> Zero(D /* tag */) { return Vec128, HWY_MAX_LANES_D(D)>{(__lsx_vreplgr2vr_w(0))}; } template HWY_API void LoadInterleaved3(D d, const TFromD* HWY_RESTRICT unaligned, VFromD& v0, VFromD& v1, VFromD& v2) { VFromD A; // v1[0] v0[0] VFromD B; // v0[1] v2[0] VFromD C; // v2[1] v1[1] detail::LoadTransposedBlocks3(d, unaligned, A, B, C); v0 = OddEven(B, A); v1 = CombineShiftRightBytes)>(d, C, A); v2 = OddEven(C, B); } template , HWY_IF_LANES_D(D, 1)> HWY_API void LoadInterleaved3(D d, const T* HWY_RESTRICT unaligned, VFromD& v0, VFromD& v1, VFromD& v2) { v0 = LoadU(d, unaligned + 0); v1 = LoadU(d, unaligned + 1); v2 = LoadU(d, unaligned + 2); } // ------------------------------ LoadInterleaved4 namespace detail { // Default for <= 128-bit vectors; x86_256 and x86_512 have their own overload. template HWY_IeLookupBytesOr0(B, idx_v2B)); const V v2U = BitCast(d, TableLookupBytesOr0(C, idx_v2C)); v0 = Xor3(v0L, v0M, v0U); v1 = Xor3(v1L, v1M, v1U); v2 = Xor3(v2L, v2M, v2U); } // 8-bit lanes x8 template HWY_API void LoadInterleaved3(D d, const TFromD* HWY_RESTRICT unaligned, VFromD& v0, VFromD& v1, VFromD& v2) { const RebindToUnsigned du; using V = VFromD; using VU = VFromD; V A; // v1[2] v0[2] v2[1] v1[1] v0[1] v2[0] v1[0] v0[0] V B; // v0[5] v2[4] v1[4] v0[4] v2[3] v1[3] v0[3] v2[2] V C; // v2[7] v1[7] v0[7] v2[6] v1[6] v0[6] v2[5] v1[5] detail::LoadTransposedBlocks3(d, unaligned, A, B, C); // Compress all lanes belonging to v0 into consecutive lanes. constexpr uint8_t Z = 0x80; const VU idx_v0A = Dup128VecFromValues(du, 0, 3, 6, Z, Z, Z, Z, Z, 0, 0, 0, 0, 0, 0, 0, 0); const VU idx_v0B = Dup128VecFromValues(du, Z, Z, Z, 1, 4, 7, Z, Z, d, Eq(Add(vu, vu), Set(du, static_cast>(hwy::MaxExponentTimes2())))); } // Returns whether normal/subnormal/zero. template > HWY_API MFromD IsFinite(const V v) { using T = TFromD; const D d; const RebindToUnsigned du; const RebindToSigned di; // cheaper than unsigned comparison const VFromD vu = BitCast(du, v); // 'Shift left' to clear the sign bit. MSVC seems to generate incorrect code // for AVX2 if we instead add vu + vu. #if HWY_COMPILER_MSVC const VFromD shl = ShiftLeft<1>(vu); #else const VFromD shl = Add(vu, vu); #endif // Then shift right so we can compare with the max exponent (cannot compare // with MaxExponentTimes2 directly because it is negative and non-negative // floats would be greater). const VFromD exp = BitCast(di, ShiftRight() + 1>(shl)); return RebindMask(d, Lt(exp, Set(di, hwy::Mturn IfThenElseZero(m, SaturatedSub(a, b)); } template , HWY_IF_I16_D(D)> HWY_API V MaskedMulFixedPoint15(M m, V a, V b) { return IfThenElseZero(m, MulFixedPoint15(a, b)); } template HWY_API V MaskedMulAdd(M m, V mul, V x, V add) { return IfThenElseZero(m, MulAdd(mul, x, add)); } template HWY_API V MaskedNegMulAdd(M m, V mul, V x, V add) { return IfThenElseZero(m, NegMulAdd(mul, x, add)); } template >> HWY_API VFromD MaskedWidenMulPairwiseAdd(D d32, M m, V16 a, V16 b) { return IfThenElseZero(m, WidenMulPairwiseAdd(d32, a, b)); } template HWY_API VFromD MaskedWidenMulPairwiseAdd(DF df, M m, VBF a, VBF b) { return IfThenElseZero(m, WidenMulPairwiseAdd(df, a, b)); } #endif // HWY_NATIVE_ZERO_MASKED_ARITH // ------------------------------ MaskedShift template HWY_API V RotateLeft(V v) { constexpr size_t kSizeInBits = sizeof(TFromV) * 8; static_assert(0 <= kBits && kBits < kSizeInBits, "Invalid shift count"); constexpr int kRotateRightAmt = (kBits == 0) ? 0 : static_cast(kSizeInBits) - kBits; return RotateRight(v); } // ------------------------------ InterleaveWholeLower/InterleaveWholeUpper #if (defined(HWY_TOGGLE_INTERLEAVE_WHOLE) == defined(HWY_TARGET_TOGGLE)) #ifdef HWY_TOGGLE_INTERLEAVE_WHOLE #undef HWY_TOGGLE_INTERLEAVE_WHOLE #else #define HWY_TOGGLE_INTERLEAVE_WHOLE #endif #if HWY_TARGET != HWY_SCALAR || HWY_IDE template HWY_API VFromD InterleaveWholeLower(D d, VFromD a, VFromD b) { // InterleaveWholeLower(d, a, b) is equivalent to InterleaveLower(a, b) if // D().MaxBytes() <= 16 is true return InterleaveLower(d, a, b); } tempte HWY_API Mask> IsNegative(V v) { const DFromV d; const RebindToSigned di; return RebindMask(d, MaskFromVec(BroadcastSignBit(BitCast(di, v)))); } #endif // HWY_NATIVE_IS_NEGATIVE // ------------------------------ MaskFalse #if (defined(HWY_NATIVE_MASK_FALSE) == defined(HWY_TARGET_TOGGLE)) #ifdef HWY_NATIVE_MASK_FALSE #undef HWY_NATIVE_MASK_FALSE #else #define HWY_NATIVE_MASK_FALSE #endif template HWY_API Mask MaskFalse(D d) { return MaskFromVec(Zero(d)); } #endif // HWY_NATIVE_MASK_FALSE // ------------------------------ SetMask #if (defined(HWY_NATIVE_SET_MASK) == defined(HWY_TARGET_TOGGLE)) #ifdef HWY_NATIVE_SET_MASK #undef HWY_NATIVE_SET_MASK #else #define HWY_NATIVE_SET_MASK #endif template HWY_API Mask SetMask(D d, bool val) { const Repartition di32; return MaskFromVec(ResizeBitCast(d, Set(di32, -static_cast(val)))); } #endif // HWY_NATIVE_SET_MASK // --ype, e.g. Vec128 for CappedTag. Useful as the return // type of functions that do not take a vector argument, or as an argument type // if the function only has a template argument for D, or for explicit type // names instead of auto. This may be a built-in type. template using Vec = decltype(Zero(D())); // Mask type. Useful as the return type of functions that do not take a mask // argument, or as an argument type if the function only has a template argument // for D, or for explicit type names instead of auto. template using Mask = decltype(MaskFromVec(Zero(D()))); // Returns the closest value to v within [lo, hi]. template HWY_API V Clamp(const V v, const V lo, const V hi) { return Min(Max(lo, v), hi); } // CombineShiftRightBytes (and -Lanes) are not available for the scalar target, // and RVV has its own implementation of -Lanes. #if (HWY_TARGET != HWY_SCALAR && HWY_TARGET != HWY_RVV) || HWY_IDE template HWY_API VFromD Combintargets); #ifndef HWY_NO_LIBCXX // Return the list of targets in HWY_TARGETS supported by the CPU as a list of // individual HWY_* target macros such as HWY_SCALAR or HWY_NEON. This list // is affected by the current SetSupportedTargetsForTest() mock if any. HWY_INLINE std::vector SupportedAndGeneratedTargets() { std::vector ret; for (int64_t targets = SupportedTargets() & HWY_TARGETS; targets != 0; targets = targets & (targets - 1)) { int64_t current_target = targets & ~(targets - 1); ret.push_back(current_target); } return ret; } #endif // HWY_NO_LIBCXX // Returns a string that satisfies gtest IsValidParamName(). No longer report // targets as "Unknown" if they are for a different architecture, because some // users unconditionally disable targets and we want to see which. static inline HWY_MAYBE_UNUSED const char* TargetName(int64_t target) { switch (target) { case HWY_EMU128: return "EMU128"; case HWY_SCALAR: return "SCALAR"; // X86 WY_HAS_CPP_ATTRIBUTE(name) 0 #endif #ifdef __has_feature #define HWY_HAS_FEATURE(name) __has_feature(name) #else #define HWY_HAS_FEATURE(name) 0 #endif // NOTE: clang ~17 does not correctly handle wrapping __has_include in a macro. #if HWY_COMPILER_MSVC && defined(_MSVC_LANG) && _MSVC_LANG > __cplusplus #define HWY_CXX_LANG _MSVC_LANG #else #define HWY_CXX_LANG __cplusplus #endif // Use instead of constexpr to avoid compiler errors for older compilers. This // macro is for when a constexpr function involves multiple statements and // loops, which is allowed in C++14 but not before. If the compiler does not // support C++14 constexpr, this evaluates to nothing. #if defined(__cpp_constexpr) && __cpp_constexpr >= 201304L #define HWY_CXX14_CONSTEXPR constexpr #else #define HWY_CXX14_CONSTEXPR #endif // Same as above, but for C++17 constexpr, which adds support for lambdas, the // standard library, and capturing *this. // Note that C++17 constexpr still disallows allocating and virtual functions, // which aret(bits.load(kAcq)); } return total; } private: static constexpr size_t kNum64 = DivCeil(kMaxSize, size_t{64}); std::atomic bits_[kNum64] = {}; }; // Two-level bitset for up to `kMaxSize` <= 4096 values. The iterators // (`Any/First/Foreach/Count`) are more efficient than `BitSet` for sparse sets. // This comes at the cost of slightly slower mutators (`Set/Clear`). template class BitSet4096 { static_assert(kMaxSize != 0, "BitSet4096 requires non-zero size"); public: constexpr size_t MaxSize() const { return kMaxSize; } // No harm if `i` is already set. void Set(size_t i) { HWY_DASSERT(i < kMaxSize); const size_t idx = i / 64; const size_t mod = i % 64; bits_[idx].Set(mod); nonzero_.Set(idx); HWY_DASSERT(Get(i)); } // Equivalent to Set(i) for i in [0, 64) where (bits >> i) & 1. This does // not clear any existing bits. void SetNonzeroBitsFrom64(uint64_t bits) { bits_[0].SetNonzeroBitsFrom64(bits); if (bits) nns lowest i such that `!Get(i)`. Caller must first ensure `All()`! size_t First0() const { HWY_DASSERT(!All()); for (size_t idx = 0;; ++idx) { HWY_DASSERT(idx < kNum64); if (!bits_[idx].All()) { const size_t first0 = idx * 64 + bits_[idx].First0(); HWY_DASSERT(first0 < kMaxSize); return first0; } } } // Calls `func(i)` for each `i` in the set. It is safe for `func` to modify // the set, but the current Foreach call is only affected if changing one of // the not yet visited BitSet64. template void Foreach(const Func& func) const { for (size_t idx = 0; idx < kNum64; ++idx) { bits_[idx].Foreach([idx, &func](size_t mod) { func(idx * 64 + mod); }); } } size_t Count() const { size_t total = 0; for (const BitSet64& bits : bits_) { total += bits.Count(); } return total; } private: static constexpr size_t kNum64 = DivCeil(kMaxSize, size_t{64}); BitSet64 bits_[kNum64]; }; // Any number of rry out) // 1[11] + 10 = C0[01] (round up toward even with C=1 carry out) // If |f32| >= 2^-24, f16_ulp_bit_idx is the index of the F32 mantissa bit // that will be shifted down into the ULP bit of the rounded down F16 result // The biased F32 exponent of 2^-14 (the smallest positive normal F16 value) // is 113, and bit 13 of the F32 mantissa will be shifted down to into the ULP // bit of the rounded down F16 result if |f32| >= 2^14 // If |f32| < 2^-24, f16_ulp_bit_idx is equal to 24 as there are 24 mantissa // bits (including the implied 1 bit) in the mantissa of a normal F32 value // and as we want to round up the mantissa if |f32| > 2^-25 && |f32| < 2^-24 const int32_t f16_ulp_bit_idx = HWY_MIN(HWY_MAX(126 - static_cast(biased_exp32), 13), 24); const uint32_t odd_bit = ((mantissa32 | 0x800000u) >> f16_ulp_bit_idx) & 1; const uint32_t rounded = mantissa32 + odd_bit + (uint32_t{1} << (f16_ulp_bit_idx - 1)) - 1u; const bool carry = rounded >= (1u << 23); c HWY_CXX14_CONSTEXPR hwy::float16_t& assign_func(const T& rhs) noexcept( \ noexcept( \ static_cast(DeclVal() op DeclVal()))) { \ native = static_cast(native op rhs); \ return *this; \ } HWY_FLOAT16_BINARY_OP(+, operator+, operator+=) HWY_FLOAT16_BINARY_OP(-, operator-, operator-=) HWY_FLOAT16_BINARY_OP(*, operator*, operator*=) HWY_FLOAT16_BINARY_OP(/, operator/, operator/=) #undef HWY_FLOAT16_BINARY_OP #endif // HWY_HAVE_SCALAR_F16_OPERATORS }; static_assert(sizeof(hwy::float16_t) == 2, "Wrong size of float16_t"); #if HWY_HAVE_SCALAR_F16_TYPE namespace detail { #if HWY_HAVE_SCALAR_F16_OPERATORS template struct SpecialFloatUnwrapArithOpOperandT { using type = hwy::float16_t::Native; }; #endif template struct NativeSpecialFloatToWrapperT>::CastSrcValRef( val)); #endif } template HWY_API HWY_BITCASTSCALAR_CONSTEXPR To BitCastScalar(const From& val) { // If To is hwy::float16_t or hwy::bfloat16_t, first do a BitCastScalar of val // to uint16_t, and then bit cast the uint16_t value to To using To::FromBits // as hwy::float16_t::FromBits and hwy::bfloat16_t::FromBits are guaranteed to // be constexpr if the __builtin_bit_cast intrinsic is available. return To::FromBits(BitCastScalar(val)); } #else template HWY_API HWY_BITCASTSCALAR_CONSTEXPR To BitCastScalar(const From& val) { To result; CopySameSize(&val, &result); return result; } #endif //------------------------------------------------------------------------------ // F16 lane type #pragma pack(push, 1) #ifndef HWY_NEON_HAVE_F16C // allow override // Compiler supports __fp16 and loadIF_NOT_F16(T) HWY_IF_NOT_SAME(T, hwy::float16_t) #define HWY_IF_F32(T) HWY_IF_SAME(T, float) #define HWY_IF_F64(T) HWY_IF_SAME(T, double) // Use instead of HWY_IF_T_SIZE to avoid ambiguity with float16_t/float/double // overloads. #define HWY_IF_UI8(T) HWY_IF_SAME2(T, uint8_t, int8_t) #define HWY_IF_UI16(T) HWY_IF_SAME2(T, uint16_t, int16_t) #define HWY_IF_UI32(T) HWY_IF_SAME2(T, uint32_t, int32_t) #define HWY_IF_UI64(T) HWY_IF_SAME2(T, uint64_t, int64_t) #define HWY_IF_LANES_PER_BLOCK(T, N, LANES) \ hwy::EnableIf* = nullptr // Empty struct used as a size tag type. template struct SizeTag {}; template class DeclValT { private: template static URef TryAddRValRef(int); template static U TryAddRValRef(Arg); public: using type = decltype(TryAddRValRef(0)); enum { kDisableDeclValEvaluation = 1 }; }; // hwy::DeclVal() can only be used in unevaluated contexts such as w#if (!defined(__OPTIMIZE__) && !defined(NDEBUG)) || \ (HWY_IS_ASAN || (HWY_IS_SANITIZER && !HWY_IS_UBSAN)) || \ defined(__clang_analyzer__) #define HWY_IS_DEBUG_BUILD 1 #else #define HWY_IS_DEBUG_BUILD 0 #endif #endif // HWY_IS_DEBUG_BUILD #if HWY_IS_DEBUG_BUILD #define HWY_DASSERT_M(condition, msg) HWY_ASSERT_M(condition, msg) #define HWY_DASSERT(condition) HWY_ASSERT_M(condition, "") #else #define HWY_DASSERT_M(condition, msg) \ do { \ } while (0) #define HWY_DASSERT(condition) \ do { \ } while (0) #endif //------------------------------------------------------------------------------ // CopyBytes / ZeroBytes #if HWY_COMPILER_MSVC #pragma intrinsic(memcpy) #pragma intrinsic(memset) #endif template HWY_API void CopyBytes(const From* HWY_RESTRICT from, To* HWY_RESTRICT to) { #if HWY_COMPILER_MSVC memcpy(to, from, kBytes); #else __builtin_memcpy(to, from, kBytes); #endif } HWY_APne HWY_MUST_USE_RESULT #endif #else #define HWY_FUNCTION __PRETTY_FUNCTION__ // function name + template args #define HWY_RESTRICT __restrict__ // force inlining without optimization enabled creates very inefficient code // that can cause compiler timeout #ifdef __OPTIMIZE__ #define HWY_INLINE inline __attribute__((always_inline)) #else #define HWY_INLINE inline #endif #define HWY_NOINLINE __attribute__((noinline)) #define HWY_FLATTEN __attribute__((flatten)) #define HWY_NORETURN __attribute__((noreturn)) #define HWY_LIKELY(expr) __builtin_expect(!!(expr), 1) #define HWY_UNLIKELY(expr) __builtin_expect(!!(expr), 0) #if HWY_COMPILER_GCC || HWY_HAS_BUILTIN(__builtin_unreachable) #define HWY_UNREACHABLE __builtin_unreachable() #else #define HWY_UNREACHABLE #endif #define HWY_PRAGMA(tokens) _Pragma(#tokens) #define HWY_DIAGNOSTICS(tokens) HWY_PRAGMA(GCC diagnostic tokens) #define HWY_DIAGNOSTICS_OFF(msc, gcc) HWY_DIAGNOSTICS(gcc) // Encountered "attribute list cannot appear here" when using the C++17 // [[maybn (on the order of microseconds). // Uses InvariantTicksPerSecond and the baseline version of timer::Start(). HWY_DLLEXPORT double Now(); // Functions related to `Ticks` below. // Returns whether it is safe to call timer::Stop without executing an illegal // instruction; if false, fills cpu100 (a pointer to a 100 character buffer) // via GetCpuString(). HWY_DLLEXPORT bool HaveTimerStop(char* cpu100); // Returns tick rate, useful for converting timer::Ticks to seconds. Invariant // means the tick counter frequency is independent of CPU throttling or sleep. // This call may be expensive, callers should cache the result. HWY_DLLEXPORT double InvariantTicksPerSecond(); // Returns ticks elapsed in back to back timer calls, i.e. a function of the // timer resolution (minimum measurable difference) and overhead. // This call is expensive, callers should cache the result. HWY_DLLEXPORT uint64_t TimerResolution(); // Returns false if no detailed description is available, otherwise fills // `cpu100` with up to 100 // Helpers for printing vector lanes. #include #include #include "hwy/base.h" #include "hwy/highway_export.h" namespace hwy { namespace detail { // For implementing value comparisons etc. as type-erased functions to reduce // template bloat. struct TypeInfo { size_t sizeof_t; bool is_float; bool is_signed; bool is_bf16; }; template HWY_INLINE TypeInfo MakeTypeInfo() { TypeInfo info; info.sizeof_t = sizeof(T); info.is_float = IsFloat(); info.is_signed = IsSigned(); info.is_bf16 = IsSame(); return info; } HWY_DLLEXPORT void TypeName(const TypeInfo& info, size_t N, char* string100); HWY_DLLEXPORT void ToString(const TypeInfo& info, const void* ptr, char* string100); HWY_DLLEXPORT void PrintArray(const TypeInfo& info, const char* caption, const void* array_void, size_t N, size_t lane_u = 0, size_t max_lanes = 7); } // namespace detail templ return static_cast( AllocateAlignedBytes(count * sizeof(value_type))); } template void deallocate(value_type* p, HWY_MAYBE_UNUSED V n) { return FreeAlignedBytes(p, nullptr, nullptr); } }; template constexpr bool operator==(const AlignedAllocator&, const AlignedAllocator&) noexcept { return true; } template constexpr bool operator!=(const AlignedAllocator&, const AlignedAllocator&) noexcept { return false; } template using AlignedVector = std::vector>; // Helpers for array allocators (avoids overflow) namespace detail { // Returns x such that 1u << x == n (if n is a power of two). static inline constexpr size_t ShiftCount(size_t n) { return (n <= 1) ? 0 : 1 + ShiftCount(n / 2); } template T* AllocateAlignedItems(size_t items, AllocPtr alloc_ptr, void* opaque_ptr) { constexpr size_t kSize = sizeof(T); c" must be in ascending order. This is the // Half Sample Mode estimator proposed by Bickel in "On a fast, robust // estimator of the mode", with complexity O(N log N). The mode is less // affected by outliers in highly-skewed distributions than the median. // The averaging operation below assumes "T" is an unsigned integer type. template T ModeOfSorted(const T* const HWY_RESTRICT sorted, const size_t num_values) { size_t idx_begin = 0; size_t half_count = num_values / 2; while (half_count > 1) { idx_begin = MinRange(sorted, idx_begin, half_count); half_count >>= 1; } const T x = sorted[idx_begin + 0]; if (half_count == 0) { return x; } HWY_ASSERT(half_count == 1); const T average = (x + sorted[idx_begin + 1] + 1) / 2; return average; } // Returns the mode. Side effect: sorts "values". template T Mode(T* values, const size_t num_values) { CountingSort(values, num_values); return ModeOfSorted(values, num_values); } template ame as the key. Because // ChooseAndCall has the type information, it can then cast to the function // pointer type. However, we cannot simply pass the name as a template // argument to ChooseAndCall because this requires char*, which hits the same // linkage problem. We instead hash the table name, which assumes the // function names do not have collisions. #if HWY_DISPATCH_MAP static constexpr uint64_t FNV(const char* name) { return *name ? static_cast(static_cast(*name)) ^ (0x100000001b3ULL * FNV(name + 1)) : 0xcbf29ce484222325ULL; } template struct AddExport { template AddExport(ExportsKey /*exports_key*/, const char* table_name, const FuncPtr* table) { using FuncCache = decltype(DeduceFunctionCache(hwy::DeclVal())); static_assert( hwy::IsSame, typename FuncCache::FuncPtr>(), "FuncPtr should be same type as FuncCaOOSE_AVX3_SPR(FUNC_NAME) nullptr #define HWY_VISIT_AVX3_SPR(VISITOR) #endif #if HWY_TARGETS & HWY_AVX10_2 #define HWY_CHOOSE_AVX10_2(FUNC_NAME) &N_AVX10_2::FUNC_NAME #define HWY_VISIT_AVX10_2(VISITOR) VISITOR(HWY_AVX10_2, N_AVX10_2) #else #define HWY_CHOOSE_AVX10_2(FUNC_NAME) nullptr #define HWY_VISIT_AVX10_2(VISITOR) #endif // MSVC 2017 workaround: the non-type template parameter to ChooseAndCall // apparently cannot be an array. Use a function pointer instead, which has the // disadvantage that we call the static (not best) target on the first call to // any HWY_DYNAMIC_DISPATCH. #if (HWY_COMPILER_MSVC && HWY_COMPILER_MSVC < 1915) || \ (HWY_COMPILER_GCC_ACTUAL && HWY_COMPILER_GCC_ACTUAL < 700) #define HWY_DISPATCH_WORKAROUND 1 #else #define HWY_DISPATCH_WORKAROUND 0 #endif #if HWY_DISPATCH_MAP struct AllExports { template static const FuncPtr*& GetRefToExportsPtr() { static const FuncPtr* s_exports = nullptr; return s_exports; } }; #endiRGET == HWY_RVV #include "hwy/aligned_allocator.h" #endif HWY_BEFORE_NAMESPACE(); namespace hwy { namespace HWY_NAMESPACE { // Prints lanes around `lane`, in memory order. template > HWY_API void Print(const D d, const char* caption, V v, size_t lane_u = 0, size_t max_lanes = 7) { const size_t N = Lanes(d); using T = TFromD; #if HWY_TARGET == HWY_RVV auto storage = AllocateAligned(N); T* HWY_RESTRICT lanes = storage.get(); #else // This works around an SVE compile error on GCC 11 and 12. Calling // AllocateAligned here would seem to require it be marked with HWY_ATTR. HWY_ALIGN T lanes[MaxLanes(d)]; #endif Store(v, d, lanes); const auto info = hwy::detail::MakeTypeInfo(); hwy::detail::PrintArray(info, caption, lanes, N, lane_u, max_lanes); } // NOLINTNEXTLINE(google-readability-namespace-comments) } // namespace HWY_NAMESPACE } // namespace hwy HWY_AFTER_NAMESPACE(); #endif // per-target include guard // Copyright 2020 Googleinstruction with eax=level and ecx=count and returns the result // in abcd array where abcd = {eax, ebx, ecx, edx} (hence the name abcd). static inline void Cpuid(const uint32_t level, const uint32_t count, uint32_t* HWY_RESTRICT abcd) { #if HWY_COMPILER_MSVC || HWY_COMPILER_CLANGCL int regs[4]; __cpuidex(regs, static_cast(level), static_cast(count)); for (int i = 0; i < 4; ++i) { abcd[i] = static_cast(regs[i]); } #else // HWY_COMPILER_MSVC || HWY_COMPILER_CLANGCL uint32_t a; uint32_t b; uint32_t c; uint32_t d; __cpuid_count(level, count, a, b, c, d); abcd[0] = a; abcd[1] = b; abcd[2] = c; abcd[3] = d; #endif // HWY_COMPILER_MSVC || HWY_COMPILER_CLANGCL } static inline bool IsBitSet(const uint32_t reg, const int index) { return (reg & (1U << index)) != 0; } static inline uint32_t MaxLevel() { uint32_t abcd[4]; Cpuid(0, 0, abcd); return abcd[0]; } static inline bool IsAMD() { uint32_t abcd[4]; Cpuid(0, 0, abcd); consurther // measurements if they are sufficiently worse than the current best. // `kMinSamples` can be several dozen when the tasks being tuned take a few // microseconds. Even for longer tasks, it should be at least 2 for some noise // tolerance. After this, there are another `kMinSamples / 2 + 1` rounds before // declaring the winner. template class AutoTune { public: // Returns non-null best configuration if auto-tuning has already finished. // Otherwise, callers continue calling `NextConfig` and `NotifyCost`. // Points into `Candidates()`. const Config* Best() const { return best_; } // If false, caller must call `SetCandidates` before `NextConfig`. // NOTE: also called after Best() is non-null. bool HasCandidates() const { return !candidates_.empty(); } // WARNING: invalidates `Best()`, do not call if that is non-null. void SetCandidates(std::vector candidates) { HWY_DASSERT(!Best() && !HasCandidates()); candidates_.swap(candida #ifdef HWY_AUTOTUNE_STDSORT std::sort(to_sort, to_sort + n); #else // F64 is supported everywhere except Armv7. #if !HWY_ARCH_ARM_V7 VQSort(to_sort, n, SortAscending()); #else // Values are known to be finite and non-negative, hence sorting as U64 is // equivalent. VQSort(reinterpret_cast(to_sort), n, SortAscending()); #endif #endif if (n & 1) return to_sort[n / 2]; // Even length: average of two middle elements. return (to_sort[n / 2] + to_sort[n / 2 - 1]) * 0.5; } static double MAD(const double* values, size_t n, const double median) { double abs_dev[kMaxValues]; for (size_t i = 0; i < n; ++i) { abs_dev[i] = ScalarAbs(values[i] - median); } return Median(abs_dev, n); } // If `num_values_` is large enough, sorts and discards outliers: either via // MAD, or if too many values are equal, by trimming according to skewness. void RemoveOutliers() { if (num_values_ < 3) return; // Not enough to discard two. HWY_DASSERT(num_vaINABLE_S390X 0 #endif #if HWY_ARCH_RISCV && HWY_HAVE_RUNTIME_DISPATCH #define HWY_ATTAINABLE_RISCV HWY_RVV #else #define HWY_ATTAINABLE_RISCV HWY_BASELINE_RVV #endif #if HWY_ARCH_LOONGARCH && HWY_HAVE_RUNTIME_DISPATCH #define HWY_ATTAINABLE_LOONGARCH (HWY_LSX | HWY_LASX) #else #define HWY_ATTAINABLE_LOONGARCH HWY_BASELINE_LOONGARCH #endif #ifndef HWY_ATTAINABLE_TARGETS_X86 // allow override #if HWY_COMPILER_MSVC && defined(HWY_SLOW_MSVC) // Fewer targets for faster builds. #define HWY_ATTAINABLE_TARGETS_X86 \ HWY_ENABLED(HWY_BASELINE_SCALAR | HWY_STATIC_TARGET | HWY_AVX2) #else // !HWY_COMPILER_MSVC #define HWY_ATTAINABLE_TARGETS_X86 \ HWY_ENABLED(HWY_BASELINE_SCALAR | HWY_SSE2 | HWY_SSSE3 | HWY_SSE4 | \ HWY_AVX2 | HWY_AVX3 | HWY_AVX3_DL | HWY_AVX3_ZEN4 | \ HWY_AVX3_SPR | HWY_AVX10_2) #endif // !HWY_COMPILER_MSVC #endif // HWY_ATTAINABLE_TARGETS_X86 // Attainable means enabled and the compiler allows intrinsics (even when not // alS == 256 #define HWY_BASELINE_SVE HWY_SVE_256 #else #define HWY_BASELINE_SVE HWY_SVE #endif // __ARM_FEATURE_SVE_BITS #endif // __ARM_FEATURE_SVE // GCC 4.5.4 only defines __ARM_NEON__; 5.4 defines both. #if defined(__ARM_NEON__) || defined(__ARM_NEON) #undef HWY_BASELINE_NEON #if defined(__ARM_FEATURE_AES) && \ defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && \ defined(__ARM_FEATURE_DOTPROD) && \ defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC) && \ defined(__ARM_FEATURE_MATMUL_INT8) #define HWY_BASELINE_NEON HWY_ALL_NEON #elif defined(__ARM_FEATURE_AES) #define HWY_BASELINE_NEON (HWY_NEON_WITHOUT_AES | HWY_NEON) #else #define HWY_BASELINE_NEON (HWY_NEON_WITHOUT_AES) #endif // __ARM_FEATURE* #endif // __ARM_NEON #endif // HWY_ARCH_ARM // Special handling for MSVC because it has fewer predefined macros: #if HWY_COMPILER_MSVC #if HWY_ARCH_X86_32 #if _M_IX86_FP >= 2 #define HWY_CHECK_SSE2 1 #else #define HWY_CHECK_SSE2 0 #endif #elif HWY_ARCH_X86_64 #define targets using predefined macros // Baseline means the targets for which the compiler is allowed to generate // instructions, implying the target CPU would have to support them. This does // not take the blocklist into account. #if defined(HWY_COMPILE_ONLY_SCALAR) || HWY_BROKEN_EMU128 #define HWY_BASELINE_SCALAR HWY_SCALAR #else #define HWY_BASELINE_SCALAR HWY_EMU128 #endif // Also check HWY_ARCH to ensure that simulating unknown platforms ends up with // HWY_TARGET == HWY_BASELINE_SCALAR. #if HWY_ARCH_WASM && defined(__wasm_simd128__) #if defined(HWY_WANT_WASM2) #define HWY_BASELINE_WASM HWY_WASM_EMU256 #else #define HWY_BASELINE_WASM HWY_WASM #endif // HWY_WANT_WASM2 #else #define HWY_BASELINE_WASM 0 #endif // GCC or Clang. #if HWY_ARCH_PPC && HWY_COMPILER_GCC && defined(__ALTIVEC__) && \ defined(__VSX__) && defined(__POWER8_VECTOR__) && \ (defined(__CRYPTO__) || defined(HWY_DISABLE_PPC8_CRYPTO)) #define HWY_BASELINE_PPC8 HWY_PPC8 #else #define HWY_BASELINE_PPC8 0 #endif #if HWY_BA0 // --------------------------- Emulation: 2 targets #define HWY_EMU128 (1LL << 61) // We do not add/left-shift, so this will not overflow to a negative number. #define HWY_SCALAR (1LL << 62) #define HWY_HIGHEST_TARGET_BIT_SCALAR 62 // Do not use bit 63 - would be confusing to have negative numbers. //------------------------------------------------------------------------------ // Set default blocklists // Disabled means excluded from enabled at user's request. A separate config // macro allows disabling without deactivating the blocklist below. #ifndef HWY_DISABLED_TARGETS #define HWY_DISABLED_TARGETS 0 #endif // Broken means excluded from enabled due to known compiler issues. We define // separate HWY_BROKEN_* and then OR them together (more than one might apply). #ifndef HWY_BROKEN_CLANG6 // allow override // x86 clang-6: we saw multiple AVX2/3 compile errors and in one case invalid // SSE4 codegen (possibly only for msan), so disable all those targets. #if HWY_ARCH_X86 && (HWY_COMPILER_CLANG != , p); } } // namespace hwy #endif // HIGHWAY_HWY_NANOBENCHMARK_H_ // Copyright 2023 Google LLC // SPDX-License-Identifier: Apache-2.0 // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // DEPRECATED, use timer.h instead. #include "hwy/timer.h" #if defined(HIGHWAY_HWY_TIMER_INL_H_) == defined(HWY_TARGET_TOGGLE) #ifdef HIGHWAY_HWY_TIMER_INL_H_ #undef HIGHWAY_HWY_TIMER_INL_H_ #else #define HIGHWAY_HWY_TIMER_INL_H_ #endif #include "hwy/highway.h" HWY_BEFORE_NAMESPACE(); namespace hwy { namespace HWY_NAMESPst uint64_t t_exit = timer::Stop(); profiler_.Exit(t_exit, static_cast(global_idx_), zone_); HWY_FENCE; } private: Profiler& profiler_; uint32_t global_idx_; ZoneHandle zone_; }; } // namespace profiler #else // profiler disabled: stub implementation namespace profiler { struct ZoneHandle {}; } // namespace profiler struct Profiler { static HWY_DLLEXPORT Profiler& Get(); // DEPRECATED: use `GlobalIdx` instead. static size_t Thread() { return 0; } static size_t GlobalIdx() { return 0; } static void SetGlobalIdx(size_t) {} void ReserveWorker(size_t) {} void FreeWorker(size_t) {} void Enter(uint64_t, size_t) {} void Exit(uint64_t, size_t, profiler::ZoneHandle) {} uint64_t GetFirstDurationAndReset(size_t) { return 0; } const char* Name(profiler::ZoneHandle) const { return nullptr; } profiler::ZoneHandle AddZone(const char*, ProfilerFlags = ProfilerFlags::kDefault) { return profiler::ZoneHandle(); } void AddFunc(voidassert(sizeof(Overheads) == 16, "Wrong Overheads size"); class Accumulators { // We generally want to group threads together because they are often // accessed together during a zone, but also want to avoid threads sharing a // cache line. Hence interleave 8 zones per worker. static constexpr size_t kPerLine = HWY_ALIGNMENT / sizeof(Accumulator); public: Accumulator& Get(const size_t global_idx, const size_t zone_idx) { HWY_DASSERT(global_idx < kMaxWorkers); HWY_DASSERT(zone_idx < kMaxZones); const size_t line = zone_idx / kPerLine; const size_t offset = zone_idx % kPerLine; return zones_[(line * kMaxWorkers + global_idx) * kPerLine + offset]; } private: Accumulator zones_[kMaxZones * kMaxWorkers]; }; // Reacts to zone enter/exit events. Builds a stack of active zones and // accumulates self/child duration for each. class PerWorker { public: template static T ClampedSubtract(const T minuend, const T subtrahend) { static_assert(IsUnsigned(), "");st { return bits_ != other.bits_; } size_t ZoneIdx() const { HWY_DASSERT(bits_ != 0); const size_t zone_idx = bits_ >> kNumFlags; HWY_DASSERT(0 != zone_idx && zone_idx < kMaxZones); return zone_idx; } bool IsInclusive() const { HWY_DASSERT(bits_ != 0); return (bits_ & static_cast(ProfilerFlags::kInclusive)) != 0; } // Returns a mask to zero/ignore child totals for inclusive zones. uint64_t ChildTotalMask() const { // With a ternary operator, clang tends to generate a branch. // return IsInclusive() ? 0 : ~uint64_t{0}; const uint32_t bit = bits_ & static_cast(ProfilerFlags::kInclusive); return uint64_t{bit} - 1; } private: uint32_t bits_; }; // Storage for zone names. class Zones { public: // Returns a copy of the `name` passed to `AddZone` that returned the //