Chris@10: /*
Chris@10:  * Copyright (c) 2003, 2007-11 Matteo Frigo
Chris@10:  * Copyright (c) 2003, 2007-11 Massachusetts Institute of Technology
Chris@10:  *
Chris@10:  * This program is free software; you can redistribute it and/or modify
Chris@10:  * it under the terms of the GNU General Public License as published by
Chris@10:  * the Free Software Foundation; either version 2 of the License, or
Chris@10:  * (at your option) any later version.
Chris@10:  *
Chris@10:  * This program is distributed in the hope that it will be useful,
Chris@10:  * but WITHOUT ANY WARRANTY; without even the implied warranty of
Chris@10:  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
Chris@10:  * GNU General Public License for more details.
Chris@10:  *
Chris@10:  * You should have received a copy of the GNU General Public License
Chris@10:  * along with this program; if not, write to the Free Software
Chris@10:  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
Chris@10:  *
Chris@10:  */
Chris@10: 
Chris@10: #ifndef FFTW_SINGLE
Chris@10: #error "ALTIVEC only works in single precision"
Chris@10: #endif
Chris@10: 
Chris@10: /* define these unconditionally, because they are used by
Chris@10:    taint.c which is compiled without altivec */
Chris@10: #define SIMD_SUFFIX _altivec  /* for renaming */
Chris@10: #define VL 2            /* SIMD complex vector length */
Chris@10: #define SIMD_VSTRIDE_OKA(x) ((x) == 2)
Chris@10: #define SIMD_STRIDE_OKPAIR SIMD_STRIDE_OKA
Chris@10: 
Chris@10: #if !defined(__VEC__) && !defined(FAKE__VEC__)
Chris@10: #  error "compiling simd-altivec.h requires -maltivec or equivalent"
Chris@10: #endif
Chris@10: 
Chris@10: #ifdef HAVE_ALTIVEC_H
Chris@10: #  include <altivec.h>
Chris@10: #endif
Chris@10: 
Chris@10: typedef vector float V;
Chris@10: #define VLIT(x0, x1, x2, x3) {x0, x1, x2, x3}
Chris@10: #define LDK(x) x
Chris@10: #define DVK(var, val) const V var = VLIT(val, val, val, val)
Chris@10: 
Chris@10: static inline V VADD(V a, V b) { return vec_add(a, b); }
Chris@10: static inline V VSUB(V a, V b) { return vec_sub(a, b); }
Chris@10: static inline V VFMA(V a, V b, V c) { return vec_madd(a, b, c); }
Chris@10: static inline V VFNMS(V a, V b, V c) { return vec_nmsub(a, b, c); }
Chris@10: 
Chris@10: static inline V VMUL(V a, V b)
Chris@10: {
Chris@10:      DVK(zero, -0.0);
Chris@10:      return VFMA(a, b, zero);
Chris@10: }
Chris@10: 
Chris@10: static inline V VFMS(V a, V b, V c) { return VSUB(VMUL(a, b), c); }
Chris@10: 
Chris@10: static inline V LDA(const R *x, INT ivs, const R *aligned_like) 
Chris@10: {
Chris@10:      UNUSED(ivs);
Chris@10:      UNUSED(aligned_like);
Chris@10:      return vec_ld(0, x);
Chris@10: }
Chris@10: 
Chris@10: static inline V LD(const R *x, INT ivs, const R *aligned_like) 
Chris@10: {
Chris@10:      /* common subexpressions */
Chris@10:      const INT fivs = sizeof(R) * ivs;
Chris@10:        /* you are not expected to understand this: */
Chris@10:      const vector unsigned int perm = VLIT(0, 0, 0xFFFFFFFF, 0xFFFFFFFF);
Chris@10:      vector unsigned char ml = vec_lvsr(fivs + 8, aligned_like);
Chris@10:      vector unsigned char mh = vec_lvsl(0, aligned_like);
Chris@10:      vector unsigned char msk = 
Chris@10: 	  (vector unsigned char)vec_sel((V)mh, (V)ml, perm);
Chris@10:      /* end of common subexpressions */
Chris@10: 
Chris@10:      return vec_perm(vec_ld(0, x), vec_ld(fivs, x), msk);
Chris@10: }
Chris@10: 
Chris@10: /* store lower half */
Chris@10: static inline void STH(R *x, V v, R *aligned_like)
Chris@10: {
Chris@10:      v = vec_perm(v, v, vec_lvsr(0, aligned_like));
Chris@10:      vec_ste(v, 0, x);
Chris@10:      vec_ste(v, sizeof(R), x);
Chris@10: }
Chris@10: 
Chris@10: static inline void STL(R *x, V v, INT ovs, R *aligned_like)
Chris@10: {
Chris@10:      const INT fovs = sizeof(R) * ovs;
Chris@10:      v = vec_perm(v, v, vec_lvsr(fovs + 8, aligned_like));
Chris@10:      vec_ste(v, fovs, x);
Chris@10:      vec_ste(v, sizeof(R) + fovs, x);
Chris@10: }
Chris@10: 
Chris@10: static inline void STA(R *x, V v, INT ovs, R *aligned_like) 
Chris@10: {
Chris@10:      UNUSED(ovs);
Chris@10:      UNUSED(aligned_like);
Chris@10:      vec_st(v, 0, x);
Chris@10: }
Chris@10: 
Chris@10: static inline void ST(R *x, V v, INT ovs, R *aligned_like) 
Chris@10: {
Chris@10:      /* WARNING: the extra_iter hack depends upon STH occurring after
Chris@10: 	STL */
Chris@10:      STL(x, v, ovs, aligned_like);
Chris@10:      STH(x, v, aligned_like);
Chris@10: }
Chris@10: 
Chris@10: #define STM2(x, v, ovs, aligned_like) /* no-op */
Chris@10: 
Chris@10: static inline void STN2(R *x, V v0, V v1, INT ovs)
Chris@10: {
Chris@10:      const INT fovs = sizeof(R) * ovs;
Chris@10:      const vector unsigned int even = 
Chris@10: 	  VLIT(0x00010203, 0x04050607, 0x10111213, 0x14151617);
Chris@10:      const vector unsigned int odd = 
Chris@10: 	  VLIT(0x08090a0b, 0x0c0d0e0f, 0x18191a1b, 0x1c1d1e1f);
Chris@10:      vec_st(vec_perm(v0, v1, (vector unsigned char)even), 0, x);
Chris@10:      vec_st(vec_perm(v0, v1, (vector unsigned char)odd), fovs, x);
Chris@10: }
Chris@10: 
Chris@10: #define STM4(x, v, ovs, aligned_like) /* no-op */
Chris@10: 
Chris@10: static inline void STN4(R *x, V v0, V v1, V v2, V v3, INT ovs)
Chris@10: {
Chris@10:      const INT fovs = sizeof(R) * ovs;
Chris@10:      V x0 = vec_mergeh(v0, v2);
Chris@10:      V x1 = vec_mergel(v0, v2);
Chris@10:      V x2 = vec_mergeh(v1, v3);
Chris@10:      V x3 = vec_mergel(v1, v3);
Chris@10:      V y0 = vec_mergeh(x0, x2);
Chris@10:      V y1 = vec_mergel(x0, x2);
Chris@10:      V y2 = vec_mergeh(x1, x3);
Chris@10:      V y3 = vec_mergel(x1, x3);
Chris@10:      vec_st(y0, 0, x);
Chris@10:      vec_st(y1, fovs, x);
Chris@10:      vec_st(y2, 2 * fovs, x);
Chris@10:      vec_st(y3, 3 * fovs, x);
Chris@10: }
Chris@10: 
Chris@10: static inline V FLIP_RI(V x)
Chris@10: {
Chris@10:      const vector unsigned int perm = 
Chris@10: 	  VLIT(0x04050607, 0x00010203, 0x0c0d0e0f, 0x08090a0b);
Chris@10:      return vec_perm(x, x, (vector unsigned char)perm);
Chris@10: }
Chris@10: 
Chris@10: static inline V VCONJ(V x)
Chris@10: {
Chris@10:      const V pmpm = VLIT(0.0, -0.0, 0.0, -0.0);
Chris@10:      return vec_xor(x, pmpm);
Chris@10: }
Chris@10: 
Chris@10: static inline V VBYI(V x)
Chris@10: {
Chris@10:      return FLIP_RI(VCONJ(x));
Chris@10: }
Chris@10: 
Chris@10: static inline V VFMAI(V b, V c)
Chris@10: {
Chris@10:      const V mpmp = VLIT(-1.0, 1.0, -1.0, 1.0);
Chris@10:      return VFMA(FLIP_RI(b), mpmp, c);
Chris@10: }
Chris@10: 
Chris@10: static inline V VFNMSI(V b, V c)
Chris@10: {
Chris@10:      const V mpmp = VLIT(-1.0, 1.0, -1.0, 1.0);
Chris@10:      return VFNMS(FLIP_RI(b), mpmp, c);
Chris@10: }
Chris@10: 
Chris@10: static inline V VFMACONJ(V b, V c)
Chris@10: {
Chris@10:      const V pmpm = VLIT(1.0, -1.0, 1.0, -1.0);
Chris@10:      return VFMA(b, pmpm, c);
Chris@10: }
Chris@10: 
Chris@10: static inline V VFNMSCONJ(V b, V c)
Chris@10: {
Chris@10:      const V pmpm = VLIT(1.0, -1.0, 1.0, -1.0);
Chris@10:      return VFNMS(b, pmpm, c);
Chris@10: }
Chris@10: 
Chris@10: static inline V VFMSCONJ(V b, V c)
Chris@10: {
Chris@10:      return VSUB(VCONJ(b), c);
Chris@10: }
Chris@10: 
Chris@10: static inline V VZMUL(V tx, V sr)
Chris@10: {
Chris@10:      const vector unsigned int real = 
Chris@10: 	  VLIT(0x00010203, 0x00010203, 0x08090a0b, 0x08090a0b);
Chris@10:      const vector unsigned int imag = 
Chris@10: 	  VLIT(0x04050607, 0x04050607, 0x0c0d0e0f, 0x0c0d0e0f);
Chris@10:      V si = VBYI(sr);
Chris@10:      V tr = vec_perm(tx, tx, (vector unsigned char)real);
Chris@10:      V ti = vec_perm(tx, tx, (vector unsigned char)imag);
Chris@10:      return VFMA(ti, si, VMUL(tr, sr));
Chris@10: }
Chris@10: 
Chris@10: static inline V VZMULJ(V tx, V sr)
Chris@10: {
Chris@10:      const vector unsigned int real = 
Chris@10: 	  VLIT(0x00010203, 0x00010203, 0x08090a0b, 0x08090a0b);
Chris@10:      const vector unsigned int imag = 
Chris@10: 	  VLIT(0x04050607, 0x04050607, 0x0c0d0e0f, 0x0c0d0e0f);
Chris@10:      V si = VBYI(sr);
Chris@10:      V tr = vec_perm(tx, tx, (vector unsigned char)real);
Chris@10:      V ti = vec_perm(tx, tx, (vector unsigned char)imag);
Chris@10:      return VFNMS(ti, si, VMUL(tr, sr));
Chris@10: }
Chris@10: 
Chris@10: static inline V VZMULI(V tx, V si)
Chris@10: {
Chris@10:      const vector unsigned int real = 
Chris@10: 	  VLIT(0x00010203, 0x00010203, 0x08090a0b, 0x08090a0b);
Chris@10:      const vector unsigned int imag = 
Chris@10: 	  VLIT(0x04050607, 0x04050607, 0x0c0d0e0f, 0x0c0d0e0f);
Chris@10:      V sr = VBYI(si);
Chris@10:      V tr = vec_perm(tx, tx, (vector unsigned char)real);
Chris@10:      V ti = vec_perm(tx, tx, (vector unsigned char)imag);
Chris@10:      return VFNMS(ti, si, VMUL(tr, sr));
Chris@10: }
Chris@10: 
Chris@10: static inline V VZMULIJ(V tx, V si)
Chris@10: {
Chris@10:      const vector unsigned int real = 
Chris@10: 	  VLIT(0x00010203, 0x00010203, 0x08090a0b, 0x08090a0b);
Chris@10:      const vector unsigned int imag = 
Chris@10: 	  VLIT(0x04050607, 0x04050607, 0x0c0d0e0f, 0x0c0d0e0f);
Chris@10:      V sr = VBYI(si);
Chris@10:      V tr = vec_perm(tx, tx, (vector unsigned char)real);
Chris@10:      V ti = vec_perm(tx, tx, (vector unsigned char)imag);
Chris@10:      return VFMA(ti, si, VMUL(tr, sr));
Chris@10: }
Chris@10: 
Chris@10: /* twiddle storage #1: compact, slower */
Chris@10: #define VTW1(v,x) \
Chris@10:  {TW_COS, v, x}, {TW_COS, v+1, x}, {TW_SIN, v, x}, {TW_SIN, v+1, x}
Chris@10: #define TWVL1 (VL)
Chris@10: 
Chris@10: static inline V BYTW1(const R *t, V sr)
Chris@10: {
Chris@10:      const V *twp = (const V *)t;
Chris@10:      V si = VBYI(sr);
Chris@10:      V tx = twp[0];
Chris@10:      V tr = vec_mergeh(tx, tx);
Chris@10:      V ti = vec_mergel(tx, tx);
Chris@10:      return VFMA(ti, si, VMUL(tr, sr));
Chris@10: }
Chris@10: 
Chris@10: static inline V BYTWJ1(const R *t, V sr)
Chris@10: {
Chris@10:      const V *twp = (const V *)t;
Chris@10:      V si = VBYI(sr);
Chris@10:      V tx = twp[0];
Chris@10:      V tr = vec_mergeh(tx, tx);
Chris@10:      V ti = vec_mergel(tx, tx);
Chris@10:      return VFNMS(ti, si, VMUL(tr, sr));
Chris@10: }
Chris@10: 
Chris@10: /* twiddle storage #2: twice the space, faster (when in cache) */
Chris@10: #define VTW2(v,x)							\
Chris@10:   {TW_COS, v, x}, {TW_COS, v, x}, {TW_COS, v+1, x}, {TW_COS, v+1, x},	\
Chris@10:   {TW_SIN, v, -x}, {TW_SIN, v, x}, {TW_SIN, v+1, -x}, {TW_SIN, v+1, x}
Chris@10: #define TWVL2 (2 * VL)
Chris@10: 
Chris@10: static inline V BYTW2(const R *t, V sr)
Chris@10: {
Chris@10:      const V *twp = (const V *)t;
Chris@10:      V si = FLIP_RI(sr);
Chris@10:      V tr = twp[0], ti = twp[1];
Chris@10:      return VFMA(ti, si, VMUL(tr, sr));
Chris@10: }
Chris@10: 
Chris@10: static inline V BYTWJ2(const R *t, V sr)
Chris@10: {
Chris@10:      const V *twp = (const V *)t;
Chris@10:      V si = FLIP_RI(sr);
Chris@10:      V tr = twp[0], ti = twp[1];
Chris@10:      return VFNMS(ti, si, VMUL(tr, sr));
Chris@10: }
Chris@10: 
Chris@10: /* twiddle storage #3 */
Chris@10: #define VTW3(v,x) {TW_CEXP, v, x}, {TW_CEXP, v+1, x}
Chris@10: #define TWVL3 (VL)
Chris@10: 
Chris@10: /* twiddle storage for split arrays */
Chris@10: #define VTWS(v,x)							\
Chris@10:   {TW_COS, v, x}, {TW_COS, v+1, x}, {TW_COS, v+2, x}, {TW_COS, v+3, x},	\
Chris@10:   {TW_SIN, v, x}, {TW_SIN, v+1, x}, {TW_SIN, v+2, x}, {TW_SIN, v+3, x}
Chris@10: #define TWVLS (2 * VL)
Chris@10: 
Chris@10: #define VLEAVE() /* nothing */
Chris@10: 
Chris@10: #include "simd-common.h"