Mercurial > hg > sv-dependency-builds

annotate src/fftw-3.3.5/simd-support/simd-avx-128-fma.h @ 84:08ae793730bd

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add null config files
author Chris Cannam
date Mon, 02 Mar 2020 14:03:47 +0000
parents 2cd0e3b3e1fd
children
rev   line source
Chris@42 1 /*
Chris@42 2 * Copyright (c) 2003, 2007-14 Matteo Frigo
Chris@42 3 * Copyright (c) 2003, 2007-14 Massachusetts Institute of Technology
Chris@42 4 *
Chris@42 5 * 128-bit AVX support by Erik Lindahl, 2015.
Chris@42 6 * Erik Lindahl hereby places his modifications in the public domain.
Chris@42 7 *
Chris@42 8 * This program is free software; you can redistribute it and/or modify
Chris@42 9 * it under the terms of the GNU General Public License as published by
Chris@42 10 * the Free Software Foundation; either version 2 of the License, or
Chris@42 11 * (at your option) any later version.
Chris@42 12 *
Chris@42 13 * This program is distributed in the hope that it will be useful,
Chris@42 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
Chris@42 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
Chris@42 16 * GNU General Public License for more details.
Chris@42 17 *
Chris@42 18 * You should have received a copy of the GNU General Public License
Chris@42 19 * along with this program; if not, write to the Free Software
Chris@42 20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
Chris@42 21 *
Chris@42 22 */
Chris@42 23
Chris@42 24 #if defined(FFTW_LDOUBLE) || defined(FFTW_QUAD)
Chris@42 25 #error "AVX only works in single or double precision"
Chris@42 26 #endif
Chris@42 27
Chris@42 28 #ifdef FFTW_SINGLE
Chris@42 29 # define DS(d,s) s /* single-precision option */
Chris@42 30 # define SUFF(name) name ## s
Chris@42 31 #else
Chris@42 32 # define DS(d,s) d /* double-precision option */
Chris@42 33 # define SUFF(name) name ## d
Chris@42 34 #endif
Chris@42 35
Chris@42 36 #define SIMD_SUFFIX _avx_128_fma /* for renaming */
Chris@42 37 #define VL DS(1,2) /* SIMD vector length, in term of complex numbers */
Chris@42 38 #define SIMD_VSTRIDE_OKA(x) DS(1,((x) == 2))
Chris@42 39 #define SIMD_STRIDE_OKPAIR SIMD_STRIDE_OK
Chris@42 40
Chris@42 41 #ifdef _MSC_VER
Chris@42 42 #ifndef inline
Chris@42 43 #define inline __inline
Chris@42 44 #endif
Chris@42 45 #endif
Chris@42 46
Chris@42 47 #include <immintrin.h>
Chris@42 48 #ifdef _MSC_VER
Chris@42 49 # include <intrin.h>
Chris@42 50 #elif defined (__GNUC__)
Chris@42 51 # include <x86intrin.h>
Chris@42 52 #endif
Chris@42 53
Chris@42 54 #if !(defined(__AVX__) && defined(__FMA4__)) /* sanity check */
Chris@42 55 #error "compiling simd-avx-128-fma.h without -mavx or -mfma4"
Chris@42 56 #endif
Chris@42 57
Chris@42 58 typedef DS(__m128d,__m128) V;
Chris@42 59 #define VADD SUFF(_mm_add_p)
Chris@42 60 #define VSUB SUFF(_mm_sub_p)
Chris@42 61 #define VMUL SUFF(_mm_mul_p)
Chris@42 62 #define VXOR SUFF(_mm_xor_p)
Chris@42 63 #define SHUF SUFF(_mm_shuffle_p)
Chris@42 64 #define VPERM1 SUFF(_mm_permute_p)
Chris@42 65 #define UNPCKL SUFF(_mm_unpacklo_p)
Chris@42 66 #define UNPCKH SUFF(_mm_unpackhi_p)
Chris@42 67
Chris@42 68 #define SHUFVALS(fp0,fp1,fp2,fp3) \
Chris@42 69 (((fp3) << 6) | ((fp2) << 4) | ((fp1) << 2) | ((fp0)))
Chris@42 70
Chris@42 71 #define VDUPL(x) DS(_mm_permute_pd(x,0), _mm_moveldup_ps(x))
Chris@42 72 #define VDUPH(x) DS(_mm_permute_pd(x,3), _mm_movehdup_ps(x))
Chris@42 73 #define LOADH(addr, val) _mm_loadh_pi(val, (const __m64 *)(addr))
Chris@42 74 #define LOADL(addr, val) _mm_loadl_pi(val, (const __m64 *)(addr))
Chris@42 75 #define STOREH(a, v) DS(_mm_storeh_pd(a, v), _mm_storeh_pi((__m64 *)(a), v))
Chris@42 76 #define STOREL(a, v) DS(_mm_storel_pd(a, v), _mm_storel_pi((__m64 *)(a), v))
Chris@42 77
Chris@42 78 #define VLIT(x0, x1) DS(_mm_set_pd(x0, x1), _mm_set_ps(x0, x1, x0, x1))
Chris@42 79 #define DVK(var, val) V var = VLIT(val, val)
Chris@42 80 #define LDK(x) x
Chris@42 81
Chris@42 82 static inline V LDA(const R *x, INT ivs, const R *aligned_like)
Chris@42 83 {
Chris@42 84 (void)aligned_like; /* UNUSED */
Chris@42 85 (void)ivs; /* UNUSED */
Chris@42 86 return *(const V *)x;
Chris@42 87 }
Chris@42 88
Chris@42 89 static inline void STA(R *x, V v, INT ovs, const R *aligned_like)
Chris@42 90 {
Chris@42 91 (void)aligned_like; /* UNUSED */
Chris@42 92 (void)ovs; /* UNUSED */
Chris@42 93 *(V *)x = v;
Chris@42 94 }
Chris@42 95
Chris@42 96 #ifdef FFTW_SINGLE
Chris@42 97
Chris@42 98 static inline V LD(const R *x, INT ivs, const R *aligned_like)
Chris@42 99 {
Chris@42 100 V var;
Chris@42 101 #if defined(__ICC) || (__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ > 8)
Chris@42 102 var = LOADL(x, SUFF(_mm_undefined_p)());
Chris@42 103 var = LOADH(x + ivs, var);
Chris@42 104 #else
Chris@42 105 var = LOADL(x, var);
Chris@42 106 var = LOADH(x + ivs, var);
Chris@42 107 #endif
Chris@42 108 return var;
Chris@42 109 }
Chris@42 110
Chris@42 111 # ifdef _MSC_VER
Chris@42 112 # pragma warning(default : 4700)
Chris@42 113 # pragma runtime_checks("u", restore)
Chris@42 114 # endif
Chris@42 115
Chris@42 116 static inline void ST(R *x, V v, INT ovs, const R *aligned_like)
Chris@42 117 {
Chris@42 118 (void)aligned_like; /* UNUSED */
Chris@42 119 /* WARNING: the extra_iter hack depends upon STOREL occurring
Chris@42 120 after STOREH */
Chris@42 121 STOREH(x + ovs, v);
Chris@42 122 STOREL(x, v);
Chris@42 123 }
Chris@42 124
Chris@42 125 #else /* ! FFTW_SINGLE */
Chris@42 126 # define LD LDA
Chris@42 127 # define ST STA
Chris@42 128 #endif
Chris@42 129
Chris@42 130 #define STM2 DS(STA,ST)
Chris@42 131 #define STN2(x, v0, v1, ovs) /* nop */
Chris@42 132
Chris@42 133 #ifdef FFTW_SINGLE
Chris@42 134 # define STM4(x, v, ovs, aligned_like) /* no-op */
Chris@42 135 /* STN4 is a macro, not a function, thanks to Visual C++ developers
Chris@42 136 deciding "it would be infrequent that people would want to pass more
Chris@42 137 than 3 [__m128 parameters] by value." 3 parameters ought to be enough
Chris@42 138 for anybody. */
Chris@42 139 # define STN4(x, v0, v1, v2, v3, ovs) \
Chris@42 140 { \
Chris@42 141 V xxx0, xxx1, xxx2, xxx3; \
Chris@42 142 xxx0 = UNPCKL(v0, v2); \
Chris@42 143 xxx1 = UNPCKH(v0, v2); \
Chris@42 144 xxx2 = UNPCKL(v1, v3); \
Chris@42 145 xxx3 = UNPCKH(v1, v3); \
Chris@42 146 STA(x, UNPCKL(xxx0, xxx2), 0, 0); \
Chris@42 147 STA(x + ovs, UNPCKH(xxx0, xxx2), 0, 0); \
Chris@42 148 STA(x + 2 * ovs, UNPCKL(xxx1, xxx3), 0, 0); \
Chris@42 149 STA(x + 3 * ovs, UNPCKH(xxx1, xxx3), 0, 0); \
Chris@42 150 }
Chris@42 151 #else /* !FFTW_SINGLE */
Chris@42 152 static inline void STM4(R *x, V v, INT ovs, const R *aligned_like)
Chris@42 153 {
Chris@42 154 (void)aligned_like; /* UNUSED */
Chris@42 155 STOREL(x, v);
Chris@42 156 STOREH(x + ovs, v);
Chris@42 157 }
Chris@42 158 # define STN4(x, v0, v1, v2, v3, ovs) /* nothing */
Chris@42 159 #endif
Chris@42 160
Chris@42 161 static inline V FLIP_RI(V x)
Chris@42 162 {
Chris@42 163 return VPERM1(x, DS(1, SHUFVALS(1, 0, 3, 2)));
Chris@42 164 }
Chris@42 165
Chris@42 166
Chris@42 167 static inline V VCONJ(V x)
Chris@42 168 {
Chris@42 169 V pmpm = VLIT(-0.0, 0.0);
Chris@42 170 return VXOR(pmpm, x);
Chris@42 171 }
Chris@42 172
Chris@42 173 static inline V VBYI(V x)
Chris@42 174 {
Chris@42 175 x = VCONJ(x);
Chris@42 176 x = FLIP_RI(x);
Chris@42 177 return x;
Chris@42 178 }
Chris@42 179
Chris@42 180 /* FMA support */
Chris@42 181 #define VFMA(a, b, c) SUFF(_mm_macc_p)(a,b,c)
Chris@42 182 #define VFNMS(a, b, c) SUFF(_mm_nmacc_p)(a,b,c)
Chris@42 183 #define VFMS(a, b, c) SUFF(_mm_msub_p)(a,b,c)
Chris@42 184 #define VFMAI(b, c) SUFF(_mm_addsub_p)(c,FLIP_RI(b))
Chris@42 185 #define VFNMSI(b, c) VSUB(c, VBYI(b))
Chris@42 186 #define VFMACONJ(b,c) VADD(VCONJ(b),c)
Chris@42 187 #define VFMSCONJ(b,c) VSUB(VCONJ(b),c)
Chris@42 188 #define VFNMSCONJ(b,c) SUFF(_mm_addsub_p)(c,b)
Chris@42 189
Chris@42 190 static inline V VZMUL(V tx, V sr)
Chris@42 191 {
Chris@42 192 V tr = VDUPL(tx);
Chris@42 193 V ti = VDUPH(tx);
Chris@42 194 tr = VMUL(tr, sr);
Chris@42 195 ti = VMUL(ti, FLIP_RI(sr));
Chris@42 196 return SUFF(_mm_addsub_p)(tr,ti);
Chris@42 197 }
Chris@42 198
Chris@42 199 static inline V VZMULJ(V tx, V sr)
Chris@42 200 {
Chris@42 201 V tr = VDUPL(tx);
Chris@42 202 V ti = VDUPH(tx);
Chris@42 203 tr = VMUL(tr, sr);
Chris@42 204 sr = VBYI(sr);
Chris@42 205 return VFNMS(ti, sr, tr);
Chris@42 206 }
Chris@42 207
Chris@42 208 static inline V VZMULI(V tx, V sr)
Chris@42 209 {
Chris@42 210 V tr = VDUPL(tx);
Chris@42 211 V ti = VDUPH(tx);
Chris@42 212 ti = VMUL(ti, sr);
Chris@42 213 sr = VBYI(sr);
Chris@42 214 return VFMS(tr, sr, ti);
Chris@42 215 }
Chris@42 216
Chris@42 217 static inline V VZMULIJ(V tx, V sr)
Chris@42 218 {
Chris@42 219 V tr = VDUPL(tx);
Chris@42 220 V ti = VDUPH(tx);
Chris@42 221 ti = VMUL(ti, sr);
Chris@42 222 tr = VMUL(tr, FLIP_RI(sr));
Chris@42 223 return SUFF(_mm_addsub_p)(ti,tr);
Chris@42 224 }
Chris@42 225
Chris@42 226 /* twiddle storage #1: compact, slower */
Chris@42 227 #ifdef FFTW_SINGLE
Chris@42 228 # define VTW1(v,x) \
Chris@42 229 {TW_COS, v, x}, {TW_COS, v+1, x}, {TW_SIN, v, x}, {TW_SIN, v+1, x}
Chris@42 230 static inline V BYTW1(const R *t, V sr)
Chris@42 231 {
Chris@42 232 const V *twp = (const V *)t;
Chris@42 233 V tx = twp[0];
Chris@42 234 V tr = UNPCKL(tx, tx);
Chris@42 235 V ti = UNPCKH(tx, tx);
Chris@42 236 tr = VMUL(tr, sr);
Chris@42 237 ti = VMUL(ti, FLIP_RI(sr));
Chris@42 238 return SUFF(_mm_addsub_p)(tr,ti);
Chris@42 239 }
Chris@42 240 static inline V BYTWJ1(const R *t, V sr)
Chris@42 241 {
Chris@42 242 const V *twp = (const V *)t;
Chris@42 243 V tx = twp[0];
Chris@42 244 V tr = UNPCKL(tx, tx);
Chris@42 245 V ti = UNPCKH(tx, tx);
Chris@42 246 tr = VMUL(tr, sr);
Chris@42 247 sr = VBYI(sr);
Chris@42 248 return VFNMS(ti, sr, tr);
Chris@42 249 }
Chris@42 250 #else /* !FFTW_SINGLE */
Chris@42 251 # define VTW1(v,x) {TW_CEXP, v, x}
Chris@42 252 static inline V BYTW1(const R *t, V sr)
Chris@42 253 {
Chris@42 254 V tx = LD(t, 1, t);
Chris@42 255 return VZMUL(tx, sr);
Chris@42 256 }
Chris@42 257 static inline V BYTWJ1(const R *t, V sr)
Chris@42 258 {
Chris@42 259 V tx = LD(t, 1, t);
Chris@42 260 return VZMULJ(tx, sr);
Chris@42 261 }
Chris@42 262 #endif
Chris@42 263 #define TWVL1 (VL)
Chris@42 264
Chris@42 265 /* twiddle storage #2: twice the space, faster (when in cache) */
Chris@42 266 #ifdef FFTW_SINGLE
Chris@42 267 # define VTW2(v,x) \
Chris@42 268 {TW_COS, v, x}, {TW_COS, v, x}, {TW_COS, v+1, x}, {TW_COS, v+1, x}, \
Chris@42 269 {TW_SIN, v, -x}, {TW_SIN, v, x}, {TW_SIN, v+1, -x}, {TW_SIN, v+1, x}
Chris@42 270 #else /* !FFTW_SINGLE */
Chris@42 271 # define VTW2(v,x) \
Chris@42 272 {TW_COS, v, x}, {TW_COS, v, x}, {TW_SIN, v, -x}, {TW_SIN, v, x}
Chris@42 273 #endif
Chris@42 274 #define TWVL2 (2 * VL)
Chris@42 275 static inline V BYTW2(const R *t, V sr)
Chris@42 276 {
Chris@42 277 const V *twp = (const V *)t;
Chris@42 278 V si = FLIP_RI(sr);
Chris@42 279 V tr = twp[0], ti = twp[1];
Chris@42 280 return VFMA(tr, sr, VMUL(ti, si));
Chris@42 281 }
Chris@42 282 static inline V BYTWJ2(const R *t, V sr)
Chris@42 283 {
Chris@42 284 const V *twp = (const V *)t;
Chris@42 285 V si = FLIP_RI(sr);
Chris@42 286 V tr = twp[0], ti = twp[1];
Chris@42 287 return VFNMS(ti, si, VMUL(tr, sr));
Chris@42 288 }
Chris@42 289
Chris@42 290 /* twiddle storage #3 */
Chris@42 291 #ifdef FFTW_SINGLE
Chris@42 292 # define VTW3(v,x) {TW_CEXP, v, x}, {TW_CEXP, v+1, x}
Chris@42 293 # define TWVL3 (VL)
Chris@42 294 #else
Chris@42 295 # define VTW3(v,x) VTW1(v,x)
Chris@42 296 # define TWVL3 TWVL1
Chris@42 297 #endif
Chris@42 298
Chris@42 299 /* twiddle storage for split arrays */
Chris@42 300 #ifdef FFTW_SINGLE
Chris@42 301 # define VTWS(v,x) \
Chris@42 302 {TW_COS, v, x}, {TW_COS, v+1, x}, {TW_COS, v+2, x}, {TW_COS, v+3, x}, \
Chris@42 303 {TW_SIN, v, x}, {TW_SIN, v+1, x}, {TW_SIN, v+2, x}, {TW_SIN, v+3, x}
Chris@42 304 #else
Chris@42 305 # define VTWS(v,x) \
Chris@42 306 {TW_COS, v, x}, {TW_COS, v+1, x}, {TW_SIN, v, x}, {TW_SIN, v+1, x}
Chris@42 307 #endif
Chris@42 308 #define TWVLS (2 * VL)
Chris@42 309
Chris@42 310 #define VLEAVE() /* nothing */
Chris@42 311
Chris@42 312 #include "simd-common.h"