Chris@42: /* Chris@42: * Copyright (c) 2003, 2007-14 Matteo Frigo Chris@42: * Copyright (c) 2003, 2007-14 Massachusetts Institute of Technology Chris@42: * Chris@42: * This program is free software; you can redistribute it and/or modify Chris@42: * it under the terms of the GNU General Public License as published by Chris@42: * the Free Software Foundation; either version 2 of the License, or Chris@42: * (at your option) any later version. Chris@42: * Chris@42: * This program is distributed in the hope that it will be useful, Chris@42: * but WITHOUT ANY WARRANTY; without even the implied warranty of Chris@42: * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the Chris@42: * GNU General Public License for more details. Chris@42: * Chris@42: * You should have received a copy of the GNU General Public License Chris@42: * along with this program; if not, write to the Free Software Chris@42: * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA Chris@42: * Chris@42: */ Chris@42: Chris@42: /* This file was automatically generated --- DO NOT EDIT */ Chris@42: /* Generated on Sat Jul 30 16:45:05 EDT 2016 */ Chris@42: Chris@42: #include "codelet-dft.h" Chris@42: Chris@42: #ifdef HAVE_FMA Chris@42: Chris@42: /* Generated by: ../../../genfft/gen_twiddle.native -fma -reorder-insns -schedule-for-pipeline -simd -compact -variables 4 -pipeline-latency 8 -n 8 -name t1sv_8 -include ts.h */ Chris@42: Chris@42: /* Chris@42: * This function contains 66 FP additions, 36 FP multiplications, Chris@42: * (or, 44 additions, 14 multiplications, 22 fused multiply/add), Chris@42: * 59 stack variables, 1 constants, and 32 memory accesses Chris@42: */ Chris@42: #include "ts.h" Chris@42: Chris@42: static void t1sv_8(R *ri, R *ii, const R *W, stride rs, INT mb, INT me, INT ms) Chris@42: { Chris@42: DVK(KP707106781, +0.707106781186547524400844362104849039284835938); Chris@42: { Chris@42: INT m; Chris@42: for (m = mb, W = W + (mb * 14); m < me; m = m + (2 * VL), ri = ri + ((2 * VL) * ms), ii = ii + ((2 * VL) * ms), W = W + ((2 * VL) * 14), MAKE_VOLATILE_STRIDE(16, rs)) { Chris@42: V T1, T1m, T1l, T7, TS, Tk, TQ, Te, To, Tr, Tu, T14, TF, Tx, T16; Chris@42: V TL, Tt, TW, Tp, Tq, Tw; Chris@42: { Chris@42: V T3, T6, T2, T5; Chris@42: T1 = LD(&(ri[0]), ms, &(ri[0])); Chris@42: T1m = LD(&(ii[0]), ms, &(ii[0])); Chris@42: T3 = LD(&(ri[WS(rs, 4)]), ms, &(ri[0])); Chris@42: T6 = LD(&(ii[WS(rs, 4)]), ms, &(ii[0])); Chris@42: T2 = LDW(&(W[TWVL * 6])); Chris@42: T5 = LDW(&(W[TWVL * 7])); Chris@42: { Chris@42: V Tg, Tj, Ti, Ta, Td, T1k, T4, T9, Tc, TR, Th, Tf; Chris@42: Tg = LD(&(ri[WS(rs, 6)]), ms, &(ri[0])); Chris@42: Tj = LD(&(ii[WS(rs, 6)]), ms, &(ii[0])); Chris@42: Tf = LDW(&(W[TWVL * 10])); Chris@42: Ti = LDW(&(W[TWVL * 11])); Chris@42: Ta = LD(&(ri[WS(rs, 2)]), ms, &(ri[0])); Chris@42: Td = LD(&(ii[WS(rs, 2)]), ms, &(ii[0])); Chris@42: T1k = VMUL(T2, T6); Chris@42: T4 = VMUL(T2, T3); Chris@42: T9 = LDW(&(W[TWVL * 2])); Chris@42: Tc = LDW(&(W[TWVL * 3])); Chris@42: TR = VMUL(Tf, Tj); Chris@42: Th = VMUL(Tf, Tg); Chris@42: { Chris@42: V TB, TE, TH, TK, TG, TD, TJ, T13, TC, TA, TP, Tb, T15, TI, Tn; Chris@42: TB = LD(&(ri[WS(rs, 7)]), ms, &(ri[WS(rs, 1)])); Chris@42: TE = LD(&(ii[WS(rs, 7)]), ms, &(ii[WS(rs, 1)])); Chris@42: T1l = VFNMS(T5, T3, T1k); Chris@42: T7 = VFMA(T5, T6, T4); Chris@42: TP = VMUL(T9, Td); Chris@42: Tb = VMUL(T9, Ta); Chris@42: TS = VFNMS(Ti, Tg, TR); Chris@42: Tk = VFMA(Ti, Tj, Th); Chris@42: TA = LDW(&(W[TWVL * 12])); Chris@42: TH = LD(&(ri[WS(rs, 3)]), ms, &(ri[WS(rs, 1)])); Chris@42: TK = LD(&(ii[WS(rs, 3)]), ms, &(ii[WS(rs, 1)])); Chris@42: TG = LDW(&(W[TWVL * 4])); Chris@42: TQ = VFNMS(Tc, Ta, TP); Chris@42: Te = VFMA(Tc, Td, Tb); Chris@42: TD = LDW(&(W[TWVL * 13])); Chris@42: TJ = LDW(&(W[TWVL * 5])); Chris@42: T13 = VMUL(TA, TE); Chris@42: TC = VMUL(TA, TB); Chris@42: To = LD(&(ri[WS(rs, 1)]), ms, &(ri[WS(rs, 1)])); Chris@42: T15 = VMUL(TG, TK); Chris@42: TI = VMUL(TG, TH); Chris@42: Tr = LD(&(ii[WS(rs, 1)]), ms, &(ii[WS(rs, 1)])); Chris@42: Tn = LDW(&(W[0])); Chris@42: Tu = LD(&(ri[WS(rs, 5)]), ms, &(ri[WS(rs, 1)])); Chris@42: T14 = VFNMS(TD, TB, T13); Chris@42: TF = VFMA(TD, TE, TC); Chris@42: Tx = LD(&(ii[WS(rs, 5)]), ms, &(ii[WS(rs, 1)])); Chris@42: T16 = VFNMS(TJ, TH, T15); Chris@42: TL = VFMA(TJ, TK, TI); Chris@42: Tt = LDW(&(W[TWVL * 8])); Chris@42: TW = VMUL(Tn, Tr); Chris@42: Tp = VMUL(Tn, To); Chris@42: Tq = LDW(&(W[TWVL * 1])); Chris@42: Tw = LDW(&(W[TWVL * 9])); Chris@42: } Chris@42: } Chris@42: } Chris@42: { Chris@42: V T8, T1g, TM, T1j, TX, Ts, T1n, T1r, T1s, Tl, T1c, T18, TZ, Ty, T1a; Chris@42: V TU; Chris@42: { Chris@42: V TO, T17, T12, TY, Tv, TT; Chris@42: T8 = VADD(T1, T7); Chris@42: TO = VSUB(T1, T7); Chris@42: T17 = VSUB(T14, T16); Chris@42: T1g = VADD(T14, T16); Chris@42: TM = VADD(TF, TL); Chris@42: T12 = VSUB(TF, TL); Chris@42: TY = VMUL(Tt, Tx); Chris@42: Tv = VMUL(Tt, Tu); Chris@42: TT = VSUB(TQ, TS); Chris@42: T1j = VADD(TQ, TS); Chris@42: TX = VFNMS(Tq, To, TW); Chris@42: Ts = VFMA(Tq, Tr, Tp); Chris@42: T1n = VADD(T1l, T1m); Chris@42: T1r = VSUB(T1m, T1l); Chris@42: T1s = VSUB(Te, Tk); Chris@42: Tl = VADD(Te, Tk); Chris@42: T1c = VADD(T12, T17); Chris@42: T18 = VSUB(T12, T17); Chris@42: TZ = VFNMS(Tw, Tu, TY); Chris@42: Ty = VFMA(Tw, Tx, Tv); Chris@42: T1a = VSUB(TO, TT); Chris@42: TU = VADD(TO, TT); Chris@42: } Chris@42: { Chris@42: V T1v, T1t, Tm, T1e, T1o, T1q, TN, T1p, T1d, T1u, T19, T1w, T1i, T1h; Chris@42: { Chris@42: V T10, T1f, Tz, TV, T11, T1b; Chris@42: T1v = VADD(T1s, T1r); Chris@42: T1t = VSUB(T1r, T1s); Chris@42: T10 = VSUB(TX, TZ); Chris@42: T1f = VADD(TX, TZ); Chris@42: Tz = VADD(Ts, Ty); Chris@42: TV = VSUB(Ts, Ty); Chris@42: T11 = VADD(TV, T10); Chris@42: T1b = VSUB(T10, TV); Chris@42: Tm = VADD(T8, Tl); Chris@42: T1e = VSUB(T8, Tl); Chris@42: T1o = VADD(T1j, T1n); Chris@42: T1q = VSUB(T1n, T1j); Chris@42: TN = VADD(Tz, TM); Chris@42: T1p = VSUB(TM, Tz); Chris@42: T1d = VSUB(T1b, T1c); Chris@42: T1u = VADD(T1b, T1c); Chris@42: T19 = VADD(T11, T18); Chris@42: T1w = VSUB(T18, T11); Chris@42: T1i = VADD(T1f, T1g); Chris@42: T1h = VSUB(T1f, T1g); Chris@42: } Chris@42: ST(&(ii[WS(rs, 6)]), VSUB(T1q, T1p), ms, &(ii[0])); Chris@42: ST(&(ri[0]), VADD(Tm, TN), ms, &(ri[0])); Chris@42: ST(&(ri[WS(rs, 4)]), VSUB(Tm, TN), ms, &(ri[0])); Chris@42: ST(&(ii[WS(rs, 1)]), VFMA(LDK(KP707106781), T1u, T1t), ms, &(ii[WS(rs, 1)])); Chris@42: ST(&(ii[WS(rs, 5)]), VFNMS(LDK(KP707106781), T1u, T1t), ms, &(ii[WS(rs, 1)])); Chris@42: ST(&(ri[WS(rs, 3)]), VFMA(LDK(KP707106781), T1d, T1a), ms, &(ri[WS(rs, 1)])); Chris@42: ST(&(ri[WS(rs, 7)]), VFNMS(LDK(KP707106781), T1d, T1a), ms, &(ri[WS(rs, 1)])); Chris@42: ST(&(ii[WS(rs, 3)]), VFMA(LDK(KP707106781), T1w, T1v), ms, &(ii[WS(rs, 1)])); Chris@42: ST(&(ii[WS(rs, 7)]), VFNMS(LDK(KP707106781), T1w, T1v), ms, &(ii[WS(rs, 1)])); Chris@42: ST(&(ri[WS(rs, 1)]), VFMA(LDK(KP707106781), T19, TU), ms, &(ri[WS(rs, 1)])); Chris@42: ST(&(ri[WS(rs, 5)]), VFNMS(LDK(KP707106781), T19, TU), ms, &(ri[WS(rs, 1)])); Chris@42: ST(&(ri[WS(rs, 6)]), VSUB(T1e, T1h), ms, &(ri[0])); Chris@42: ST(&(ii[0]), VADD(T1i, T1o), ms, &(ii[0])); Chris@42: ST(&(ii[WS(rs, 4)]), VSUB(T1o, T1i), ms, &(ii[0])); Chris@42: ST(&(ri[WS(rs, 2)]), VADD(T1e, T1h), ms, &(ri[0])); Chris@42: ST(&(ii[WS(rs, 2)]), VADD(T1p, T1q), ms, &(ii[0])); Chris@42: } Chris@42: } Chris@42: } Chris@42: } Chris@42: VLEAVE(); Chris@42: } Chris@42: Chris@42: static const tw_instr twinstr[] = { Chris@42: VTW(0, 1), Chris@42: VTW(0, 2), Chris@42: VTW(0, 3), Chris@42: VTW(0, 4), Chris@42: VTW(0, 5), Chris@42: VTW(0, 6), Chris@42: VTW(0, 7), Chris@42: {TW_NEXT, (2 * VL), 0} Chris@42: }; Chris@42: Chris@42: static const ct_desc desc = { 8, XSIMD_STRING("t1sv_8"), twinstr, &GENUS, {44, 14, 22, 0}, 0, 0, 0 }; Chris@42: Chris@42: void XSIMD(codelet_t1sv_8) (planner *p) { Chris@42: X(kdft_dit_register) (p, t1sv_8, &desc); Chris@42: } Chris@42: #else /* HAVE_FMA */ Chris@42: Chris@42: /* Generated by: ../../../genfft/gen_twiddle.native -simd -compact -variables 4 -pipeline-latency 8 -n 8 -name t1sv_8 -include ts.h */ Chris@42: Chris@42: /* Chris@42: * This function contains 66 FP additions, 32 FP multiplications, Chris@42: * (or, 52 additions, 18 multiplications, 14 fused multiply/add), Chris@42: * 28 stack variables, 1 constants, and 32 memory accesses Chris@42: */ Chris@42: #include "ts.h" Chris@42: Chris@42: static void t1sv_8(R *ri, R *ii, const R *W, stride rs, INT mb, INT me, INT ms) Chris@42: { Chris@42: DVK(KP707106781, +0.707106781186547524400844362104849039284835938); Chris@42: { Chris@42: INT m; Chris@42: for (m = mb, W = W + (mb * 14); m < me; m = m + (2 * VL), ri = ri + ((2 * VL) * ms), ii = ii + ((2 * VL) * ms), W = W + ((2 * VL) * 14), MAKE_VOLATILE_STRIDE(16, rs)) { Chris@42: V T7, T1e, TH, T19, TF, T13, TR, TU, Ti, T1f, TK, T16, Tu, T12, TM; Chris@42: V TP; Chris@42: { Chris@42: V T1, T18, T6, T17; Chris@42: T1 = LD(&(ri[0]), ms, &(ri[0])); Chris@42: T18 = LD(&(ii[0]), ms, &(ii[0])); Chris@42: { Chris@42: V T3, T5, T2, T4; Chris@42: T3 = LD(&(ri[WS(rs, 4)]), ms, &(ri[0])); Chris@42: T5 = LD(&(ii[WS(rs, 4)]), ms, &(ii[0])); Chris@42: T2 = LDW(&(W[TWVL * 6])); Chris@42: T4 = LDW(&(W[TWVL * 7])); Chris@42: T6 = VFMA(T2, T3, VMUL(T4, T5)); Chris@42: T17 = VFNMS(T4, T3, VMUL(T2, T5)); Chris@42: } Chris@42: T7 = VADD(T1, T6); Chris@42: T1e = VSUB(T18, T17); Chris@42: TH = VSUB(T1, T6); Chris@42: T19 = VADD(T17, T18); Chris@42: } Chris@42: { Chris@42: V Tz, TS, TE, TT; Chris@42: { Chris@42: V Tw, Ty, Tv, Tx; Chris@42: Tw = LD(&(ri[WS(rs, 7)]), ms, &(ri[WS(rs, 1)])); Chris@42: Ty = LD(&(ii[WS(rs, 7)]), ms, &(ii[WS(rs, 1)])); Chris@42: Tv = LDW(&(W[TWVL * 12])); Chris@42: Tx = LDW(&(W[TWVL * 13])); Chris@42: Tz = VFMA(Tv, Tw, VMUL(Tx, Ty)); Chris@42: TS = VFNMS(Tx, Tw, VMUL(Tv, Ty)); Chris@42: } Chris@42: { Chris@42: V TB, TD, TA, TC; Chris@42: TB = LD(&(ri[WS(rs, 3)]), ms, &(ri[WS(rs, 1)])); Chris@42: TD = LD(&(ii[WS(rs, 3)]), ms, &(ii[WS(rs, 1)])); Chris@42: TA = LDW(&(W[TWVL * 4])); Chris@42: TC = LDW(&(W[TWVL * 5])); Chris@42: TE = VFMA(TA, TB, VMUL(TC, TD)); Chris@42: TT = VFNMS(TC, TB, VMUL(TA, TD)); Chris@42: } Chris@42: TF = VADD(Tz, TE); Chris@42: T13 = VADD(TS, TT); Chris@42: TR = VSUB(Tz, TE); Chris@42: TU = VSUB(TS, TT); Chris@42: } Chris@42: { Chris@42: V Tc, TI, Th, TJ; Chris@42: { Chris@42: V T9, Tb, T8, Ta; Chris@42: T9 = LD(&(ri[WS(rs, 2)]), ms, &(ri[0])); Chris@42: Tb = LD(&(ii[WS(rs, 2)]), ms, &(ii[0])); Chris@42: T8 = LDW(&(W[TWVL * 2])); Chris@42: Ta = LDW(&(W[TWVL * 3])); Chris@42: Tc = VFMA(T8, T9, VMUL(Ta, Tb)); Chris@42: TI = VFNMS(Ta, T9, VMUL(T8, Tb)); Chris@42: } Chris@42: { Chris@42: V Te, Tg, Td, Tf; Chris@42: Te = LD(&(ri[WS(rs, 6)]), ms, &(ri[0])); Chris@42: Tg = LD(&(ii[WS(rs, 6)]), ms, &(ii[0])); Chris@42: Td = LDW(&(W[TWVL * 10])); Chris@42: Tf = LDW(&(W[TWVL * 11])); Chris@42: Th = VFMA(Td, Te, VMUL(Tf, Tg)); Chris@42: TJ = VFNMS(Tf, Te, VMUL(Td, Tg)); Chris@42: } Chris@42: Ti = VADD(Tc, Th); Chris@42: T1f = VSUB(Tc, Th); Chris@42: TK = VSUB(TI, TJ); Chris@42: T16 = VADD(TI, TJ); Chris@42: } Chris@42: { Chris@42: V To, TN, Tt, TO; Chris@42: { Chris@42: V Tl, Tn, Tk, Tm; Chris@42: Tl = LD(&(ri[WS(rs, 1)]), ms, &(ri[WS(rs, 1)])); Chris@42: Tn = LD(&(ii[WS(rs, 1)]), ms, &(ii[WS(rs, 1)])); Chris@42: Tk = LDW(&(W[0])); Chris@42: Tm = LDW(&(W[TWVL * 1])); Chris@42: To = VFMA(Tk, Tl, VMUL(Tm, Tn)); Chris@42: TN = VFNMS(Tm, Tl, VMUL(Tk, Tn)); Chris@42: } Chris@42: { Chris@42: V Tq, Ts, Tp, Tr; Chris@42: Tq = LD(&(ri[WS(rs, 5)]), ms, &(ri[WS(rs, 1)])); Chris@42: Ts = LD(&(ii[WS(rs, 5)]), ms, &(ii[WS(rs, 1)])); Chris@42: Tp = LDW(&(W[TWVL * 8])); Chris@42: Tr = LDW(&(W[TWVL * 9])); Chris@42: Tt = VFMA(Tp, Tq, VMUL(Tr, Ts)); Chris@42: TO = VFNMS(Tr, Tq, VMUL(Tp, Ts)); Chris@42: } Chris@42: Tu = VADD(To, Tt); Chris@42: T12 = VADD(TN, TO); Chris@42: TM = VSUB(To, Tt); Chris@42: TP = VSUB(TN, TO); Chris@42: } Chris@42: { Chris@42: V Tj, TG, T1b, T1c; Chris@42: Tj = VADD(T7, Ti); Chris@42: TG = VADD(Tu, TF); Chris@42: ST(&(ri[WS(rs, 4)]), VSUB(Tj, TG), ms, &(ri[0])); Chris@42: ST(&(ri[0]), VADD(Tj, TG), ms, &(ri[0])); Chris@42: { Chris@42: V T15, T1a, T11, T14; Chris@42: T15 = VADD(T12, T13); Chris@42: T1a = VADD(T16, T19); Chris@42: ST(&(ii[0]), VADD(T15, T1a), ms, &(ii[0])); Chris@42: ST(&(ii[WS(rs, 4)]), VSUB(T1a, T15), ms, &(ii[0])); Chris@42: T11 = VSUB(T7, Ti); Chris@42: T14 = VSUB(T12, T13); Chris@42: ST(&(ri[WS(rs, 6)]), VSUB(T11, T14), ms, &(ri[0])); Chris@42: ST(&(ri[WS(rs, 2)]), VADD(T11, T14), ms, &(ri[0])); Chris@42: } Chris@42: T1b = VSUB(TF, Tu); Chris@42: T1c = VSUB(T19, T16); Chris@42: ST(&(ii[WS(rs, 2)]), VADD(T1b, T1c), ms, &(ii[0])); Chris@42: ST(&(ii[WS(rs, 6)]), VSUB(T1c, T1b), ms, &(ii[0])); Chris@42: { Chris@42: V TX, T1g, T10, T1d, TY, TZ; Chris@42: TX = VSUB(TH, TK); Chris@42: T1g = VSUB(T1e, T1f); Chris@42: TY = VSUB(TP, TM); Chris@42: TZ = VADD(TR, TU); Chris@42: T10 = VMUL(LDK(KP707106781), VSUB(TY, TZ)); Chris@42: T1d = VMUL(LDK(KP707106781), VADD(TY, TZ)); Chris@42: ST(&(ri[WS(rs, 7)]), VSUB(TX, T10), ms, &(ri[WS(rs, 1)])); Chris@42: ST(&(ii[WS(rs, 5)]), VSUB(T1g, T1d), ms, &(ii[WS(rs, 1)])); Chris@42: ST(&(ri[WS(rs, 3)]), VADD(TX, T10), ms, &(ri[WS(rs, 1)])); Chris@42: ST(&(ii[WS(rs, 1)]), VADD(T1d, T1g), ms, &(ii[WS(rs, 1)])); Chris@42: } Chris@42: { Chris@42: V TL, T1i, TW, T1h, TQ, TV; Chris@42: TL = VADD(TH, TK); Chris@42: T1i = VADD(T1f, T1e); Chris@42: TQ = VADD(TM, TP); Chris@42: TV = VSUB(TR, TU); Chris@42: TW = VMUL(LDK(KP707106781), VADD(TQ, TV)); Chris@42: T1h = VMUL(LDK(KP707106781), VSUB(TV, TQ)); Chris@42: ST(&(ri[WS(rs, 5)]), VSUB(TL, TW), ms, &(ri[WS(rs, 1)])); Chris@42: ST(&(ii[WS(rs, 7)]), VSUB(T1i, T1h), ms, &(ii[WS(rs, 1)])); Chris@42: ST(&(ri[WS(rs, 1)]), VADD(TL, TW), ms, &(ri[WS(rs, 1)])); Chris@42: ST(&(ii[WS(rs, 3)]), VADD(T1h, T1i), ms, &(ii[WS(rs, 1)])); Chris@42: } Chris@42: } Chris@42: } Chris@42: } Chris@42: VLEAVE(); Chris@42: } Chris@42: Chris@42: static const tw_instr twinstr[] = { Chris@42: VTW(0, 1), Chris@42: VTW(0, 2), Chris@42: VTW(0, 3), Chris@42: VTW(0, 4), Chris@42: VTW(0, 5), Chris@42: VTW(0, 6), Chris@42: VTW(0, 7), Chris@42: {TW_NEXT, (2 * VL), 0} Chris@42: }; Chris@42: Chris@42: static const ct_desc desc = { 8, XSIMD_STRING("t1sv_8"), twinstr, &GENUS, {52, 18, 14, 0}, 0, 0, 0 }; Chris@42: Chris@42: void XSIMD(codelet_t1sv_8) (planner *p) { Chris@42: X(kdft_dit_register) (p, t1sv_8, &desc); Chris@42: } Chris@42: #endif /* HAVE_FMA */