Mercurial > hg > sv-dependency-builds

comparison src/fftw-3.3.3/dft/simd/common/t1sv_32.c @ 10:37bf6b4a2645

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add FFTW3
author Chris Cannam
date Wed, 20 Mar 2013 15:35:50 +0000
parents
children
comparison
equal deleted inserted replaced
9:c0fb53affa76 10:37bf6b4a2645
1 /*
2 * Copyright (c) 2003, 2007-11 Matteo Frigo
3 * Copyright (c) 2003, 2007-11 Massachusetts Institute of Technology
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18 *
19 */
20
21 /* This file was automatically generated --- DO NOT EDIT */
22 /* Generated on Sun Nov 25 07:39:25 EST 2012 */
23
24 #include "codelet-dft.h"
25
26 #ifdef HAVE_FMA
27
28 /* Generated by: ../../../genfft/gen_twiddle.native -fma -reorder-insns -schedule-for-pipeline -simd -compact -variables 4 -pipeline-latency 8 -n 32 -name t1sv_32 -include ts.h */
29
30 /*
31 * This function contains 434 FP additions, 260 FP multiplications,
32 * (or, 236 additions, 62 multiplications, 198 fused multiply/add),
33 * 158 stack variables, 7 constants, and 128 memory accesses
34 */
35 #include "ts.h"
36
37 static void t1sv_32(R *ri, R *ii, const R *W, stride rs, INT mb, INT me, INT ms)
38 {
39 DVK(KP831469612, +0.831469612302545237078788377617905756738560812);
40 DVK(KP980785280, +0.980785280403230449126182236134239036973933731);
41 DVK(KP668178637, +0.668178637919298919997757686523080761552472251);
42 DVK(KP198912367, +0.198912367379658006911597622644676228597850501);
43 DVK(KP923879532, +0.923879532511286756128183189396788286822416626);
44 DVK(KP414213562, +0.414213562373095048801688724209698078569671875);
45 DVK(KP707106781, +0.707106781186547524400844362104849039284835938);
46 {
47 INT m;
48 for (m = mb, W = W + (mb * 62); m < me; m = m + (2 * VL), ri = ri + ((2 * VL) * ms), ii = ii + ((2 * VL) * ms), W = W + ((2 * VL) * 62), MAKE_VOLATILE_STRIDE(64, rs)) {
49 V T8Z, T90;
50 {
51 V T87, T8x, T3w, T8, T3B, T83, Tl, T8y, T6F, Tz, T3J, T5T, T6G, TM, T3Q;
52 V T5U, T46, T5Y, T7D, T6L, T5X, T3Z, T6M, T1f, T4l, T61, T7E, T6R, T60, T4e;
53 V T6O, T1G, T5r, T6c, T78, T7N, T54, T6f, T32, T7b, T4S, T65, T6X, T7I, T4v;
54 V T68, T29, T70, T4x, T2f, T5b, T5s, T7O, T7e, T5t, T5i, T79, T3t, T2h, T2k;
55 V T2j, T2o, T2r, T4H, T2y, T2n, T2q, T4y, T2i;
56 {
57 V T3U, TU, TW, TZ, TY, T13, T16, T12, T15, T3V, TX, T44, T1d;
58 {
59 V T1, T86, T3, T6, T5, Ta, Td, Tg, Tj, Tf, T84, T4, Tc, Ti, T3x;
60 V Tb, T2, T9;
61 T1 = LD(&(ri[0]), ms, &(ri[0]));
62 T86 = LD(&(ii[0]), ms, &(ii[0]));
63 T3 = LD(&(ri[WS(rs, 16)]), ms, &(ri[0]));
64 T6 = LD(&(ii[WS(rs, 16)]), ms, &(ii[0]));
65 T2 = LDW(&(W[TWVL * 30]));
66 T5 = LDW(&(W[TWVL * 31]));
67 Ta = LD(&(ri[WS(rs, 8)]), ms, &(ri[0]));
68 Td = LD(&(ii[WS(rs, 8)]), ms, &(ii[0]));
69 T9 = LDW(&(W[TWVL * 14]));
70 Tg = LD(&(ri[WS(rs, 24)]), ms, &(ri[0]));
71 Tj = LD(&(ii[WS(rs, 24)]), ms, &(ii[0]));
72 Tf = LDW(&(W[TWVL * 46]));
73 T84 = VMUL(T2, T6);
74 T4 = VMUL(T2, T3);
75 Tc = LDW(&(W[TWVL * 15]));
76 Ti = LDW(&(W[TWVL * 47]));
77 T3x = VMUL(T9, Td);
78 Tb = VMUL(T9, Ta);
79 {
80 V Tu, Tx, T3F, Ts, Tt, Tw;
81 {
82 V To, Tr, Tq, T3E, Tp;
83 {
84 V T3y, Te, Tn, T3A, Tk;
85 {
86 V T3z, Th, T85, T7;
87 To = LD(&(ri[WS(rs, 4)]), ms, &(ri[0]));
88 T3z = VMUL(Tf, Tj);
89 Th = VMUL(Tf, Tg);
90 T85 = VFNMS(T5, T3, T84);
91 T7 = VFMA(T5, T6, T4);
92 Tr = LD(&(ii[WS(rs, 4)]), ms, &(ii[0]));
93 T3y = VFNMS(Tc, Ta, T3x);
94 Te = VFMA(Tc, Td, Tb);
95 Tn = LDW(&(W[TWVL * 6]));
96 T3A = VFNMS(Ti, Tg, T3z);
97 Tk = VFMA(Ti, Tj, Th);
98 T87 = VADD(T85, T86);
99 T8x = VSUB(T86, T85);
100 T3w = VSUB(T1, T7);
101 T8 = VADD(T1, T7);
102 }
103 Tq = LDW(&(W[TWVL * 7]));
104 T3E = VMUL(Tn, Tr);
105 Tp = VMUL(Tn, To);
106 T3B = VSUB(T3y, T3A);
107 T83 = VADD(T3y, T3A);
108 Tl = VADD(Te, Tk);
109 T8y = VSUB(Te, Tk);
110 }
111 Tu = LD(&(ri[WS(rs, 20)]), ms, &(ri[0]));
112 Tx = LD(&(ii[WS(rs, 20)]), ms, &(ii[0]));
113 T3F = VFNMS(Tq, To, T3E);
114 Ts = VFMA(Tq, Tr, Tp);
115 Tt = LDW(&(W[TWVL * 38]));
116 Tw = LDW(&(W[TWVL * 39]));
117 }
118 {
119 V TB, TE, TD, TH, TK, T3G, Tv, TG, TJ, T3L, TC, TA;
120 TB = LD(&(ri[WS(rs, 28)]), ms, &(ri[0]));
121 TE = LD(&(ii[WS(rs, 28)]), ms, &(ii[0]));
122 TA = LDW(&(W[TWVL * 54]));
123 TD = LDW(&(W[TWVL * 55]));
124 TH = LD(&(ri[WS(rs, 12)]), ms, &(ri[0]));
125 TK = LD(&(ii[WS(rs, 12)]), ms, &(ii[0]));
126 T3G = VMUL(Tt, Tx);
127 Tv = VMUL(Tt, Tu);
128 TG = LDW(&(W[TWVL * 22]));
129 TJ = LDW(&(W[TWVL * 23]));
130 T3L = VMUL(TA, TE);
131 TC = VMUL(TA, TB);
132 {
133 V T19, T1c, T3P, T3K, T18, T1b, TV, T43, T1a;
134 {
135 V TQ, TT, T3M, TF, TS, T3I, T3D, T3O, TL, T3T, TR;
136 {
137 V T3H, Ty, T3N, TI, TP;
138 TQ = LD(&(ri[WS(rs, 2)]), ms, &(ri[0]));
139 TT = LD(&(ii[WS(rs, 2)]), ms, &(ii[0]));
140 T3H = VFNMS(Tw, Tu, T3G);
141 Ty = VFMA(Tw, Tx, Tv);
142 T3N = VMUL(TG, TK);
143 TI = VMUL(TG, TH);
144 T3M = VFNMS(TD, TB, T3L);
145 TF = VFMA(TD, TE, TC);
146 TP = LDW(&(W[TWVL * 2]));
147 TS = LDW(&(W[TWVL * 3]));
148 T6F = VADD(T3F, T3H);
149 T3I = VSUB(T3F, T3H);
150 Tz = VADD(Ts, Ty);
151 T3D = VSUB(Ts, Ty);
152 T3O = VFNMS(TJ, TH, T3N);
153 TL = VFMA(TJ, TK, TI);
154 T3T = VMUL(TP, TT);
155 TR = VMUL(TP, TQ);
156 }
157 T19 = LD(&(ri[WS(rs, 26)]), ms, &(ri[0]));
158 T1c = LD(&(ii[WS(rs, 26)]), ms, &(ii[0]));
159 T3J = VADD(T3D, T3I);
160 T5T = VSUB(T3I, T3D);
161 T6G = VADD(T3M, T3O);
162 T3P = VSUB(T3M, T3O);
163 TM = VADD(TF, TL);
164 T3K = VSUB(TF, TL);
165 T3U = VFNMS(TS, TQ, T3T);
166 TU = VFMA(TS, TT, TR);
167 T18 = LDW(&(W[TWVL * 50]));
168 T1b = LDW(&(W[TWVL * 51]));
169 }
170 TW = LD(&(ri[WS(rs, 18)]), ms, &(ri[0]));
171 TZ = LD(&(ii[WS(rs, 18)]), ms, &(ii[0]));
172 T3Q = VSUB(T3K, T3P);
173 T5U = VADD(T3K, T3P);
174 TV = LDW(&(W[TWVL * 34]));
175 TY = LDW(&(W[TWVL * 35]));
176 T43 = VMUL(T18, T1c);
177 T1a = VMUL(T18, T19);
178 T13 = LD(&(ri[WS(rs, 10)]), ms, &(ri[0]));
179 T16 = LD(&(ii[WS(rs, 10)]), ms, &(ii[0]));
180 T12 = LDW(&(W[TWVL * 18]));
181 T15 = LDW(&(W[TWVL * 19]));
182 T3V = VMUL(TV, TZ);
183 TX = VMUL(TV, TW);
184 T44 = VFNMS(T1b, T19, T43);
185 T1d = VFMA(T1b, T1c, T1a);
186 }
187 }
188 }
189 }
190 {
191 V T4Z, T2H, T2J, T2M, T2L, T2Q, T2T, T2P, T2S, T5p, T30, T50, T2K;
192 {
193 V T49, T1l, T1n, T1q, T1p, T1u, T1x, T4j, T1E, T1t, T1w, T4a, T1o;
194 {
195 V T1A, T1D, T1C, T4i, T1B, T1m;
196 {
197 V T1h, T1k, T41, T14, T3W, T10, T1g, T1j;
198 T1h = LD(&(ri[WS(rs, 30)]), ms, &(ri[0]));
199 T1k = LD(&(ii[WS(rs, 30)]), ms, &(ii[0]));
200 T41 = VMUL(T12, T16);
201 T14 = VMUL(T12, T13);
202 T3W = VFNMS(TY, TW, T3V);
203 T10 = VFMA(TY, TZ, TX);
204 T1g = LDW(&(W[TWVL * 58]));
205 T1j = LDW(&(W[TWVL * 59]));
206 {
207 V T6J, T3X, T11, T40, T48, T1i, T6K, T45, T1e, T3Y, T1z, T42, T17;
208 T1A = LD(&(ri[WS(rs, 22)]), ms, &(ri[0]));
209 T1D = LD(&(ii[WS(rs, 22)]), ms, &(ii[0]));
210 T42 = VFNMS(T15, T13, T41);
211 T17 = VFMA(T15, T16, T14);
212 T6J = VADD(T3U, T3W);
213 T3X = VSUB(T3U, T3W);
214 T11 = VADD(TU, T10);
215 T40 = VSUB(TU, T10);
216 T48 = VMUL(T1g, T1k);
217 T1i = VMUL(T1g, T1h);
218 T6K = VADD(T42, T44);
219 T45 = VSUB(T42, T44);
220 T1e = VADD(T17, T1d);
221 T3Y = VSUB(T17, T1d);
222 T1z = LDW(&(W[TWVL * 42]));
223 T1C = LDW(&(W[TWVL * 43]));
224 T49 = VFNMS(T1j, T1h, T48);
225 T1l = VFMA(T1j, T1k, T1i);
226 T46 = VADD(T40, T45);
227 T5Y = VSUB(T40, T45);
228 T7D = VADD(T6J, T6K);
229 T6L = VSUB(T6J, T6K);
230 T5X = VADD(T3X, T3Y);
231 T3Z = VSUB(T3X, T3Y);
232 T6M = VSUB(T11, T1e);
233 T1f = VADD(T11, T1e);
234 T4i = VMUL(T1z, T1D);
235 T1B = VMUL(T1z, T1A);
236 }
237 }
238 T1n = LD(&(ri[WS(rs, 14)]), ms, &(ri[0]));
239 T1q = LD(&(ii[WS(rs, 14)]), ms, &(ii[0]));
240 T1m = LDW(&(W[TWVL * 26]));
241 T1p = LDW(&(W[TWVL * 27]));
242 T1u = LD(&(ri[WS(rs, 6)]), ms, &(ri[0]));
243 T1x = LD(&(ii[WS(rs, 6)]), ms, &(ii[0]));
244 T4j = VFNMS(T1C, T1A, T4i);
245 T1E = VFMA(T1C, T1D, T1B);
246 T1t = LDW(&(W[TWVL * 10]));
247 T1w = LDW(&(W[TWVL * 11]));
248 T4a = VMUL(T1m, T1q);
249 T1o = VMUL(T1m, T1n);
250 }
251 {
252 V T2W, T2Z, T6P, T4c, T1s, T4f, T6Q, T4k, T1F, T4d, T2V, T2Y, T5o, T2X, T2I;
253 {
254 V T2D, T2G, T2C, T2F, T4g, T1v, T4b, T1r;
255 T2D = LD(&(ri[WS(rs, 31)]), ms, &(ri[WS(rs, 1)]));
256 T2G = LD(&(ii[WS(rs, 31)]), ms, &(ii[WS(rs, 1)]));
257 T2C = LDW(&(W[TWVL * 60]));
258 T2F = LDW(&(W[TWVL * 61]));
259 T4g = VMUL(T1t, T1x);
260 T1v = VMUL(T1t, T1u);
261 T4b = VFNMS(T1p, T1n, T4a);
262 T1r = VFMA(T1p, T1q, T1o);
263 T2W = LD(&(ri[WS(rs, 23)]), ms, &(ri[WS(rs, 1)]));
264 T2Z = LD(&(ii[WS(rs, 23)]), ms, &(ii[WS(rs, 1)]));
265 {
266 V T4Y, T2E, T4h, T1y;
267 T4Y = VMUL(T2C, T2G);
268 T2E = VMUL(T2C, T2D);
269 T4h = VFNMS(T1w, T1u, T4g);
270 T1y = VFMA(T1w, T1x, T1v);
271 T6P = VADD(T49, T4b);
272 T4c = VSUB(T49, T4b);
273 T1s = VADD(T1l, T1r);
274 T4f = VSUB(T1l, T1r);
275 T4Z = VFNMS(T2F, T2D, T4Y);
276 T2H = VFMA(T2F, T2G, T2E);
277 T6Q = VADD(T4h, T4j);
278 T4k = VSUB(T4h, T4j);
279 T1F = VADD(T1y, T1E);
280 T4d = VSUB(T1y, T1E);
281 T2V = LDW(&(W[TWVL * 44]));
282 }
283 T2Y = LDW(&(W[TWVL * 45]));
284 }
285 T2J = LD(&(ri[WS(rs, 15)]), ms, &(ri[WS(rs, 1)]));
286 T2M = LD(&(ii[WS(rs, 15)]), ms, &(ii[WS(rs, 1)]));
287 T4l = VADD(T4f, T4k);
288 T61 = VSUB(T4f, T4k);
289 T7E = VADD(T6P, T6Q);
290 T6R = VSUB(T6P, T6Q);
291 T60 = VADD(T4c, T4d);
292 T4e = VSUB(T4c, T4d);
293 T6O = VSUB(T1s, T1F);
294 T1G = VADD(T1s, T1F);
295 T5o = VMUL(T2V, T2Z);
296 T2X = VMUL(T2V, T2W);
297 T2I = LDW(&(W[TWVL * 28]));
298 T2L = LDW(&(W[TWVL * 29]));
299 T2Q = LD(&(ri[WS(rs, 7)]), ms, &(ri[WS(rs, 1)]));
300 T2T = LD(&(ii[WS(rs, 7)]), ms, &(ii[WS(rs, 1)]));
301 T2P = LDW(&(W[TWVL * 12]));
302 T2S = LDW(&(W[TWVL * 13]));
303 T5p = VFNMS(T2Y, T2W, T5o);
304 T30 = VFMA(T2Y, T2Z, T2X);
305 T50 = VMUL(T2I, T2M);
306 T2K = VMUL(T2I, T2J);
307 }
308 }
309 {
310 V T4q, T1O, T1Q, T1T, T1S, T1X, T20, T4Q, T27, T1W, T1Z, T4r, T1R;
311 {
312 V T23, T26, T25, T4P, T24, T1P;
313 {
314 V T1K, T1N, T5m, T2R, T1J, T1M, T51, T2N;
315 T1K = LD(&(ri[WS(rs, 1)]), ms, &(ri[WS(rs, 1)]));
316 T1N = LD(&(ii[WS(rs, 1)]), ms, &(ii[WS(rs, 1)]));
317 T5m = VMUL(T2P, T2T);
318 T2R = VMUL(T2P, T2Q);
319 T1J = LDW(&(W[0]));
320 T1M = LDW(&(W[TWVL * 1]));
321 T51 = VFNMS(T2L, T2J, T50);
322 T2N = VFMA(T2L, T2M, T2K);
323 {
324 V T76, T52, T2O, T5l, T77, T5q, T31, T53, T22;
325 T23 = LD(&(ri[WS(rs, 25)]), ms, &(ri[WS(rs, 1)]));
326 T26 = LD(&(ii[WS(rs, 25)]), ms, &(ii[WS(rs, 1)]));
327 {
328 V T5n, T2U, T4p, T1L;
329 T5n = VFNMS(T2S, T2Q, T5m);
330 T2U = VFMA(T2S, T2T, T2R);
331 T4p = VMUL(T1J, T1N);
332 T1L = VMUL(T1J, T1K);
333 T76 = VADD(T4Z, T51);
334 T52 = VSUB(T4Z, T51);
335 T2O = VADD(T2H, T2N);
336 T5l = VSUB(T2H, T2N);
337 T77 = VADD(T5n, T5p);
338 T5q = VSUB(T5n, T5p);
339 T31 = VADD(T2U, T30);
340 T53 = VSUB(T2U, T30);
341 T4q = VFNMS(T1M, T1K, T4p);
342 T1O = VFMA(T1M, T1N, T1L);
343 T22 = LDW(&(W[TWVL * 48]));
344 }
345 T25 = LDW(&(W[TWVL * 49]));
346 T5r = VADD(T5l, T5q);
347 T6c = VSUB(T5l, T5q);
348 T78 = VSUB(T76, T77);
349 T7N = VADD(T76, T77);
350 T54 = VSUB(T52, T53);
351 T6f = VADD(T52, T53);
352 T32 = VADD(T2O, T31);
353 T7b = VSUB(T2O, T31);
354 T4P = VMUL(T22, T26);
355 T24 = VMUL(T22, T23);
356 }
357 }
358 T1Q = LD(&(ri[WS(rs, 17)]), ms, &(ri[WS(rs, 1)]));
359 T1T = LD(&(ii[WS(rs, 17)]), ms, &(ii[WS(rs, 1)]));
360 T1P = LDW(&(W[TWVL * 32]));
361 T1S = LDW(&(W[TWVL * 33]));
362 T1X = LD(&(ri[WS(rs, 9)]), ms, &(ri[WS(rs, 1)]));
363 T20 = LD(&(ii[WS(rs, 9)]), ms, &(ii[WS(rs, 1)]));
364 T4Q = VFNMS(T25, T23, T4P);
365 T27 = VFMA(T25, T26, T24);
366 T1W = LDW(&(W[TWVL * 16]));
367 T1Z = LDW(&(W[TWVL * 17]));
368 T4r = VMUL(T1P, T1T);
369 T1R = VMUL(T1P, T1Q);
370 }
371 {
372 V T56, T38, T3a, T3d, T3c, T3h, T3k, T3g, T3j, T5g, T3r, T57, T3b;
373 {
374 V T3n, T3q, T6V, T4t, T1V, T4M, T6W, T4R, T28, T4u, T3m, T3p, T5f, T3o, T39;
375 {
376 V T34, T37, T33, T36, T4N, T1Y, T4s, T1U;
377 T34 = LD(&(ri[WS(rs, 3)]), ms, &(ri[WS(rs, 1)]));
378 T37 = LD(&(ii[WS(rs, 3)]), ms, &(ii[WS(rs, 1)]));
379 T33 = LDW(&(W[TWVL * 4]));
380 T36 = LDW(&(W[TWVL * 5]));
381 T4N = VMUL(T1W, T20);
382 T1Y = VMUL(T1W, T1X);
383 T4s = VFNMS(T1S, T1Q, T4r);
384 T1U = VFMA(T1S, T1T, T1R);
385 T3n = LD(&(ri[WS(rs, 11)]), ms, &(ri[WS(rs, 1)]));
386 T3q = LD(&(ii[WS(rs, 11)]), ms, &(ii[WS(rs, 1)]));
387 {
388 V T55, T35, T4O, T21;
389 T55 = VMUL(T33, T37);
390 T35 = VMUL(T33, T34);
391 T4O = VFNMS(T1Z, T1X, T4N);
392 T21 = VFMA(T1Z, T20, T1Y);
393 T6V = VADD(T4q, T4s);
394 T4t = VSUB(T4q, T4s);
395 T1V = VADD(T1O, T1U);
396 T4M = VSUB(T1O, T1U);
397 T56 = VFNMS(T36, T34, T55);
398 T38 = VFMA(T36, T37, T35);
399 T6W = VADD(T4O, T4Q);
400 T4R = VSUB(T4O, T4Q);
401 T28 = VADD(T21, T27);
402 T4u = VSUB(T21, T27);
403 T3m = LDW(&(W[TWVL * 20]));
404 }
405 T3p = LDW(&(W[TWVL * 21]));
406 }
407 T3a = LD(&(ri[WS(rs, 19)]), ms, &(ri[WS(rs, 1)]));
408 T3d = LD(&(ii[WS(rs, 19)]), ms, &(ii[WS(rs, 1)]));
409 T4S = VADD(T4M, T4R);
410 T65 = VSUB(T4M, T4R);
411 T6X = VSUB(T6V, T6W);
412 T7I = VADD(T6V, T6W);
413 T4v = VSUB(T4t, T4u);
414 T68 = VADD(T4t, T4u);
415 T29 = VADD(T1V, T28);
416 T70 = VSUB(T1V, T28);
417 T5f = VMUL(T3m, T3q);
418 T3o = VMUL(T3m, T3n);
419 T39 = LDW(&(W[TWVL * 36]));
420 T3c = LDW(&(W[TWVL * 37]));
421 T3h = LD(&(ri[WS(rs, 27)]), ms, &(ri[WS(rs, 1)]));
422 T3k = LD(&(ii[WS(rs, 27)]), ms, &(ii[WS(rs, 1)]));
423 T3g = LDW(&(W[TWVL * 52]));
424 T3j = LDW(&(W[TWVL * 53]));
425 T5g = VFNMS(T3p, T3n, T5f);
426 T3r = VFMA(T3p, T3q, T3o);
427 T57 = VMUL(T39, T3d);
428 T3b = VMUL(T39, T3a);
429 }
430 {
431 V T2u, T2x, T2w, T4G, T2v, T2g;
432 {
433 V T2b, T2e, T5d, T3i, T2a, T2d, T58, T3e, T2t;
434 T2b = LD(&(ri[WS(rs, 5)]), ms, &(ri[WS(rs, 1)]));
435 T2e = LD(&(ii[WS(rs, 5)]), ms, &(ii[WS(rs, 1)]));
436 T5d = VMUL(T3g, T3k);
437 T3i = VMUL(T3g, T3h);
438 T2a = LDW(&(W[TWVL * 8]));
439 T2d = LDW(&(W[TWVL * 9]));
440 T58 = VFNMS(T3c, T3a, T57);
441 T3e = VFMA(T3c, T3d, T3b);
442 T2u = LD(&(ri[WS(rs, 13)]), ms, &(ri[WS(rs, 1)]));
443 T2x = LD(&(ii[WS(rs, 13)]), ms, &(ii[WS(rs, 1)]));
444 {
445 V T5e, T3l, T4w, T2c;
446 T5e = VFNMS(T3j, T3h, T5d);
447 T3l = VFMA(T3j, T3k, T3i);
448 T4w = VMUL(T2a, T2e);
449 T2c = VMUL(T2a, T2b);
450 {
451 V T7c, T59, T3f, T5a;
452 T7c = VADD(T56, T58);
453 T59 = VSUB(T56, T58);
454 T3f = VADD(T38, T3e);
455 T5a = VSUB(T38, T3e);
456 {
457 V T7d, T5h, T3s, T5c;
458 T7d = VADD(T5e, T5g);
459 T5h = VSUB(T5e, T5g);
460 T3s = VADD(T3l, T3r);
461 T5c = VSUB(T3l, T3r);
462 T4x = VFNMS(T2d, T2b, T4w);
463 T2f = VFMA(T2d, T2e, T2c);
464 T5b = VSUB(T59, T5a);
465 T5s = VADD(T5a, T59);
466 T2t = LDW(&(W[TWVL * 24]));
467 T7O = VADD(T7c, T7d);
468 T7e = VSUB(T7c, T7d);
469 T5t = VSUB(T5c, T5h);
470 T5i = VADD(T5c, T5h);
471 T79 = VSUB(T3s, T3f);
472 T3t = VADD(T3f, T3s);
473 }
474 }
475 }
476 T2w = LDW(&(W[TWVL * 25]));
477 T4G = VMUL(T2t, T2x);
478 T2v = VMUL(T2t, T2u);
479 }
480 T2h = LD(&(ri[WS(rs, 21)]), ms, &(ri[WS(rs, 1)]));
481 T2k = LD(&(ii[WS(rs, 21)]), ms, &(ii[WS(rs, 1)]));
482 T2g = LDW(&(W[TWVL * 40]));
483 T2j = LDW(&(W[TWVL * 41]));
484 T2o = LD(&(ri[WS(rs, 29)]), ms, &(ri[WS(rs, 1)]));
485 T2r = LD(&(ii[WS(rs, 29)]), ms, &(ii[WS(rs, 1)]));
486 T4H = VFNMS(T2w, T2u, T4G);
487 T2y = VFMA(T2w, T2x, T2v);
488 T2n = LDW(&(W[TWVL * 56]));
489 T2q = LDW(&(W[TWVL * 57]));
490 T4y = VMUL(T2g, T2k);
491 T2i = VMUL(T2g, T2h);
492 }
493 }
494 }
495 }
496 }
497 {
498 V T4C, T4T, T4U, T4J, T7A, T7w, T7j, T75, T7i, T6U, T8p, T8n, T8v, T8t, T7q;
499 V T7y, T7t, T7z, T7g, T7k;
500 {
501 V T6E, T8j, T6H, T8k, T73, T6Y, T7S, T8i, T8h, T7V;
502 {
503 V T7P, T7Y, T7C, TO, T89, T8e, T3u, T7M, T8d, T1H, T7K, T7X, T2B, T7H;
504 {
505 V T71, T2m, T72, T4I, T2z, T4D, Tm, TN, T2A, T7J;
506 T6E = VSUB(T8, Tl);
507 Tm = VADD(T8, Tl);
508 TN = VADD(Tz, TM);
509 T8j = VSUB(TM, Tz);
510 T7P = VSUB(T7N, T7O);
511 T7Y = VADD(T7N, T7O);
512 {
513 V T82, T4E, T2p, T4z, T2l, T88;
514 T82 = VADD(T6F, T6G);
515 T6H = VSUB(T6F, T6G);
516 T4E = VMUL(T2n, T2r);
517 T2p = VMUL(T2n, T2o);
518 T4z = VFNMS(T2j, T2h, T4y);
519 T2l = VFMA(T2j, T2k, T2i);
520 T8k = VSUB(T87, T83);
521 T88 = VADD(T83, T87);
522 T7C = VSUB(Tm, TN);
523 TO = VADD(Tm, TN);
524 {
525 V T4F, T2s, T4A, T4B;
526 T4F = VFNMS(T2q, T2o, T4E);
527 T2s = VFMA(T2q, T2r, T2p);
528 T71 = VADD(T4x, T4z);
529 T4A = VSUB(T4x, T4z);
530 T2m = VADD(T2f, T2l);
531 T4B = VSUB(T2f, T2l);
532 T89 = VADD(T82, T88);
533 T8e = VSUB(T88, T82);
534 T72 = VADD(T4F, T4H);
535 T4I = VSUB(T4F, T4H);
536 T2z = VADD(T2s, T2y);
537 T4D = VSUB(T2s, T2y);
538 T4C = VSUB(T4A, T4B);
539 T4T = VADD(T4B, T4A);
540 }
541 }
542 T3u = VADD(T32, T3t);
543 T7M = VSUB(T32, T3t);
544 T7J = VADD(T71, T72);
545 T73 = VSUB(T71, T72);
546 T4U = VSUB(T4D, T4I);
547 T4J = VADD(T4D, T4I);
548 T6Y = VSUB(T2z, T2m);
549 T2A = VADD(T2m, T2z);
550 T8d = VSUB(T1G, T1f);
551 T1H = VADD(T1f, T1G);
552 T7K = VSUB(T7I, T7J);
553 T7X = VADD(T7I, T7J);
554 T2B = VADD(T29, T2A);
555 T7H = VSUB(T29, T2A);
556 }
557 {
558 V T1I, T80, T7Q, T7U, T7F, T7L, T7T, T3v, T8b, T8c, T8a, T7W, T81, T7Z;
559 T7W = VSUB(TO, T1H);
560 T1I = VADD(TO, T1H);
561 T7Z = VSUB(T7X, T7Y);
562 T80 = VADD(T7X, T7Y);
563 T7Q = VSUB(T7M, T7P);
564 T7U = VADD(T7M, T7P);
565 T7F = VSUB(T7D, T7E);
566 T81 = VADD(T7D, T7E);
567 T7L = VADD(T7H, T7K);
568 T7T = VSUB(T7K, T7H);
569 T3v = VADD(T2B, T3u);
570 T8b = VSUB(T3u, T2B);
571 ST(&(ri[WS(rs, 24)]), VSUB(T7W, T7Z), ms, &(ri[0]));
572 ST(&(ri[WS(rs, 8)]), VADD(T7W, T7Z), ms, &(ri[0]));
573 T8c = VSUB(T89, T81);
574 T8a = VADD(T81, T89);
575 {
576 V T8f, T8g, T7G, T7R;
577 T7S = VSUB(T7C, T7F);
578 T7G = VADD(T7C, T7F);
579 T7R = VADD(T7L, T7Q);
580 T8i = VSUB(T7Q, T7L);
581 T8h = VSUB(T8e, T8d);
582 T8f = VADD(T8d, T8e);
583 ST(&(ri[0]), VADD(T1I, T3v), ms, &(ri[0]));
584 ST(&(ri[WS(rs, 16)]), VSUB(T1I, T3v), ms, &(ri[0]));
585 T8g = VADD(T7T, T7U);
586 T7V = VSUB(T7T, T7U);
587 ST(&(ii[WS(rs, 16)]), VSUB(T8a, T80), ms, &(ii[0]));
588 ST(&(ii[0]), VADD(T80, T8a), ms, &(ii[0]));
589 ST(&(ii[WS(rs, 24)]), VSUB(T8c, T8b), ms, &(ii[0]));
590 ST(&(ii[WS(rs, 8)]), VADD(T8b, T8c), ms, &(ii[0]));
591 ST(&(ri[WS(rs, 4)]), VFMA(LDK(KP707106781), T7R, T7G), ms, &(ri[0]));
592 ST(&(ri[WS(rs, 20)]), VFNMS(LDK(KP707106781), T7R, T7G), ms, &(ri[0]));
593 ST(&(ii[WS(rs, 20)]), VFNMS(LDK(KP707106781), T8g, T8f), ms, &(ii[0]));
594 ST(&(ii[WS(rs, 4)]), VFMA(LDK(KP707106781), T8g, T8f), ms, &(ii[0]));
595 }
596 }
597 }
598 {
599 V T7f, T7a, T7m, T6I, T7s, T7r, T8r, T8l, T8m, T6T, T8s, T7p;
600 {
601 V T7n, T6N, T6S, T7o, T7u, T7v, T6Z, T74;
602 T7f = VSUB(T7b, T7e);
603 T7u = VADD(T7b, T7e);
604 T7v = VADD(T78, T79);
605 T7a = VSUB(T78, T79);
606 ST(&(ri[WS(rs, 12)]), VFMA(LDK(KP707106781), T7V, T7S), ms, &(ri[0]));
607 ST(&(ri[WS(rs, 28)]), VFNMS(LDK(KP707106781), T7V, T7S), ms, &(ri[0]));
608 ST(&(ii[WS(rs, 28)]), VFNMS(LDK(KP707106781), T8i, T8h), ms, &(ii[0]));
609 ST(&(ii[WS(rs, 12)]), VFMA(LDK(KP707106781), T8i, T8h), ms, &(ii[0]));
610 T7m = VADD(T6E, T6H);
611 T6I = VSUB(T6E, T6H);
612 T7A = VFMA(LDK(KP414213562), T7u, T7v);
613 T7w = VFNMS(LDK(KP414213562), T7v, T7u);
614 T7n = VADD(T6M, T6L);
615 T6N = VSUB(T6L, T6M);
616 T6S = VADD(T6O, T6R);
617 T7o = VSUB(T6O, T6R);
618 T7s = VADD(T6X, T6Y);
619 T6Z = VSUB(T6X, T6Y);
620 T74 = VSUB(T70, T73);
621 T7r = VADD(T70, T73);
622 T8r = VSUB(T8k, T8j);
623 T8l = VADD(T8j, T8k);
624 T8m = VADD(T6N, T6S);
625 T6T = VSUB(T6N, T6S);
626 T7j = VFNMS(LDK(KP414213562), T6Z, T74);
627 T75 = VFMA(LDK(KP414213562), T74, T6Z);
628 T8s = VSUB(T7o, T7n);
629 T7p = VADD(T7n, T7o);
630 }
631 T7i = VFNMS(LDK(KP707106781), T6T, T6I);
632 T6U = VFMA(LDK(KP707106781), T6T, T6I);
633 T8p = VFNMS(LDK(KP707106781), T8m, T8l);
634 T8n = VFMA(LDK(KP707106781), T8m, T8l);
635 T8v = VFNMS(LDK(KP707106781), T8s, T8r);
636 T8t = VFMA(LDK(KP707106781), T8s, T8r);
637 T7q = VFMA(LDK(KP707106781), T7p, T7m);
638 T7y = VFNMS(LDK(KP707106781), T7p, T7m);
639 T7t = VFMA(LDK(KP414213562), T7s, T7r);
640 T7z = VFNMS(LDK(KP414213562), T7r, T7s);
641 T7g = VFNMS(LDK(KP414213562), T7f, T7a);
642 T7k = VFMA(LDK(KP414213562), T7a, T7f);
643 }
644 }
645 {
646 V T5S, T8O, T8N, T5V, T6d, T6g, T66, T4L, T5I, T69, T5y, T4o, T8J, T8L, T5M;
647 V T5Q, T5A, T5w, T5H, T4W, T5O, T5G, T8D, T8F;
648 {
649 V T5C, T3S, T8C, T4n, T8H, T8B, T8I, T5F, T5L, T5k, T5K, T5v, T4V;
650 {
651 V T5D, T47, T4m, T5E, T8z, T8A, T3C, T3R, T5j, T5u, T4K;
652 T5S = VSUB(T3w, T3B);
653 T3C = VADD(T3w, T3B);
654 T3R = VADD(T3J, T3Q);
655 T8O = VSUB(T3Q, T3J);
656 {
657 V T8o, T7B, T7x, T8q;
658 T8o = VADD(T7z, T7A);
659 T7B = VSUB(T7z, T7A);
660 T7x = VADD(T7t, T7w);
661 T8q = VSUB(T7w, T7t);
662 {
663 V T8u, T7l, T7h, T8w;
664 T8u = VSUB(T7k, T7j);
665 T7l = VADD(T7j, T7k);
666 T7h = VSUB(T75, T7g);
667 T8w = VADD(T75, T7g);
668 ST(&(ri[WS(rs, 10)]), VFMA(LDK(KP923879532), T7B, T7y), ms, &(ri[0]));
669 ST(&(ri[WS(rs, 26)]), VFNMS(LDK(KP923879532), T7B, T7y), ms, &(ri[0]));
670 ST(&(ii[WS(rs, 18)]), VFNMS(LDK(KP923879532), T8o, T8n), ms, &(ii[0]));
671 ST(&(ii[WS(rs, 2)]), VFMA(LDK(KP923879532), T8o, T8n), ms, &(ii[0]));
672 ST(&(ii[WS(rs, 26)]), VFNMS(LDK(KP923879532), T8q, T8p), ms, &(ii[0]));
673 ST(&(ii[WS(rs, 10)]), VFMA(LDK(KP923879532), T8q, T8p), ms, &(ii[0]));
674 ST(&(ri[WS(rs, 2)]), VFMA(LDK(KP923879532), T7x, T7q), ms, &(ri[0]));
675 ST(&(ri[WS(rs, 18)]), VFNMS(LDK(KP923879532), T7x, T7q), ms, &(ri[0]));
676 ST(&(ri[WS(rs, 30)]), VFMA(LDK(KP923879532), T7l, T7i), ms, &(ri[0]));
677 ST(&(ri[WS(rs, 14)]), VFNMS(LDK(KP923879532), T7l, T7i), ms, &(ri[0]));
678 ST(&(ii[WS(rs, 22)]), VFNMS(LDK(KP923879532), T8u, T8t), ms, &(ii[0]));
679 ST(&(ii[WS(rs, 6)]), VFMA(LDK(KP923879532), T8u, T8t), ms, &(ii[0]));
680 ST(&(ii[WS(rs, 30)]), VFMA(LDK(KP923879532), T8w, T8v), ms, &(ii[0]));
681 ST(&(ii[WS(rs, 14)]), VFNMS(LDK(KP923879532), T8w, T8v), ms, &(ii[0]));
682 ST(&(ri[WS(rs, 6)]), VFMA(LDK(KP923879532), T7h, T6U), ms, &(ri[0]));
683 ST(&(ri[WS(rs, 22)]), VFNMS(LDK(KP923879532), T7h, T6U), ms, &(ri[0]));
684 T5C = VFMA(LDK(KP707106781), T3R, T3C);
685 T3S = VFNMS(LDK(KP707106781), T3R, T3C);
686 }
687 }
688 T5D = VFMA(LDK(KP414213562), T3Z, T46);
689 T47 = VFNMS(LDK(KP414213562), T46, T3Z);
690 T4m = VFMA(LDK(KP414213562), T4l, T4e);
691 T5E = VFNMS(LDK(KP414213562), T4e, T4l);
692 T8N = VADD(T8y, T8x);
693 T8z = VSUB(T8x, T8y);
694 T8A = VADD(T5T, T5U);
695 T5V = VSUB(T5T, T5U);
696 T6d = VSUB(T5i, T5b);
697 T5j = VADD(T5b, T5i);
698 T5u = VADD(T5s, T5t);
699 T6g = VSUB(T5s, T5t);
700 T66 = VSUB(T4J, T4C);
701 T4K = VADD(T4C, T4J);
702 T8C = VADD(T47, T4m);
703 T4n = VSUB(T47, T4m);
704 T8H = VFNMS(LDK(KP707106781), T8A, T8z);
705 T8B = VFMA(LDK(KP707106781), T8A, T8z);
706 T8I = VSUB(T5E, T5D);
707 T5F = VADD(T5D, T5E);
708 T5L = VFMA(LDK(KP707106781), T5j, T54);
709 T5k = VFNMS(LDK(KP707106781), T5j, T54);
710 T5K = VFMA(LDK(KP707106781), T5u, T5r);
711 T5v = VFNMS(LDK(KP707106781), T5u, T5r);
712 T4L = VFNMS(LDK(KP707106781), T4K, T4v);
713 T5I = VFMA(LDK(KP707106781), T4K, T4v);
714 T4V = VADD(T4T, T4U);
715 T69 = VSUB(T4T, T4U);
716 }
717 T5y = VFNMS(LDK(KP923879532), T4n, T3S);
718 T4o = VFMA(LDK(KP923879532), T4n, T3S);
719 T8J = VFMA(LDK(KP923879532), T8I, T8H);
720 T8L = VFNMS(LDK(KP923879532), T8I, T8H);
721 T5M = VFNMS(LDK(KP198912367), T5L, T5K);
722 T5Q = VFMA(LDK(KP198912367), T5K, T5L);
723 T5A = VFMA(LDK(KP668178637), T5k, T5v);
724 T5w = VFNMS(LDK(KP668178637), T5v, T5k);
725 T5H = VFMA(LDK(KP707106781), T4V, T4S);
726 T4W = VFNMS(LDK(KP707106781), T4V, T4S);
727 T5O = VFNMS(LDK(KP923879532), T5F, T5C);
728 T5G = VFMA(LDK(KP923879532), T5F, T5C);
729 T8D = VFMA(LDK(KP923879532), T8C, T8B);
730 T8F = VFNMS(LDK(KP923879532), T8C, T8B);
731 }
732 {
733 V T6p, T6q, T6o, T5W, T8W, T63;
734 {
735 V T5J, T5P, T5z, T4X, T5Z, T62;
736 T5J = VFMA(LDK(KP198912367), T5I, T5H);
737 T5P = VFNMS(LDK(KP198912367), T5H, T5I);
738 T5z = VFNMS(LDK(KP668178637), T4L, T4W);
739 T4X = VFMA(LDK(KP668178637), T4W, T4L);
740 T6p = VFNMS(LDK(KP414213562), T5X, T5Y);
741 T5Z = VFMA(LDK(KP414213562), T5Y, T5X);
742 T62 = VFNMS(LDK(KP414213562), T61, T60);
743 T6q = VFMA(LDK(KP414213562), T60, T61);
744 {
745 V T8G, T5N, T5R, T8E;
746 T8G = VSUB(T5M, T5J);
747 T5N = VADD(T5J, T5M);
748 T5R = VSUB(T5P, T5Q);
749 T8E = VADD(T5P, T5Q);
750 {
751 V T5B, T8K, T8M, T5x;
752 T5B = VADD(T5z, T5A);
753 T8K = VSUB(T5A, T5z);
754 T8M = VADD(T4X, T5w);
755 T5x = VSUB(T4X, T5w);
756 T6o = VFNMS(LDK(KP707106781), T5V, T5S);
757 T5W = VFMA(LDK(KP707106781), T5V, T5S);
758 T8W = VADD(T5Z, T62);
759 T63 = VSUB(T5Z, T62);
760 ST(&(ii[WS(rs, 25)]), VFNMS(LDK(KP980785280), T8G, T8F), ms, &(ii[WS(rs, 1)]));
761 ST(&(ii[WS(rs, 9)]), VFMA(LDK(KP980785280), T8G, T8F), ms, &(ii[WS(rs, 1)]));
762 ST(&(ri[WS(rs, 1)]), VFMA(LDK(KP980785280), T5N, T5G), ms, &(ri[WS(rs, 1)]));
763 ST(&(ri[WS(rs, 17)]), VFNMS(LDK(KP980785280), T5N, T5G), ms, &(ri[WS(rs, 1)]));
764 ST(&(ri[WS(rs, 9)]), VFMA(LDK(KP980785280), T5R, T5O), ms, &(ri[WS(rs, 1)]));
765 ST(&(ri[WS(rs, 25)]), VFNMS(LDK(KP980785280), T5R, T5O), ms, &(ri[WS(rs, 1)]));
766 ST(&(ii[WS(rs, 17)]), VFNMS(LDK(KP980785280), T8E, T8D), ms, &(ii[WS(rs, 1)]));
767 ST(&(ii[WS(rs, 1)]), VFMA(LDK(KP980785280), T8E, T8D), ms, &(ii[WS(rs, 1)]));
768 ST(&(ri[WS(rs, 29)]), VFMA(LDK(KP831469612), T5B, T5y), ms, &(ri[WS(rs, 1)]));
769 ST(&(ri[WS(rs, 13)]), VFNMS(LDK(KP831469612), T5B, T5y), ms, &(ri[WS(rs, 1)]));
770 ST(&(ii[WS(rs, 21)]), VFNMS(LDK(KP831469612), T8K, T8J), ms, &(ii[WS(rs, 1)]));
771 ST(&(ii[WS(rs, 5)]), VFMA(LDK(KP831469612), T8K, T8J), ms, &(ii[WS(rs, 1)]));
772 ST(&(ii[WS(rs, 29)]), VFMA(LDK(KP831469612), T8M, T8L), ms, &(ii[WS(rs, 1)]));
773 ST(&(ii[WS(rs, 13)]), VFNMS(LDK(KP831469612), T8M, T8L), ms, &(ii[WS(rs, 1)]));
774 ST(&(ri[WS(rs, 5)]), VFMA(LDK(KP831469612), T5x, T4o), ms, &(ri[WS(rs, 1)]));
775 ST(&(ri[WS(rs, 21)]), VFNMS(LDK(KP831469612), T5x, T4o), ms, &(ri[WS(rs, 1)]));
776 }
777 }
778 }
779 {
780 V T6k, T64, T8V, T6r, T8R, T8T, T6y, T6C, T6m, T6i, T6v, T6B, T6l, T6b, T6A;
781 V T6s, T8X;
782 {
783 V T6x, T6e, T6w, T6h, T6u, T67, T6t, T6a, T8P, T8Q;
784 T6k = VFNMS(LDK(KP923879532), T63, T5W);
785 T64 = VFMA(LDK(KP923879532), T63, T5W);
786 T8V = VFNMS(LDK(KP707106781), T8O, T8N);
787 T8P = VFMA(LDK(KP707106781), T8O, T8N);
788 T8Q = VSUB(T6q, T6p);
789 T6r = VADD(T6p, T6q);
790 T6x = VFMA(LDK(KP707106781), T6d, T6c);
791 T6e = VFNMS(LDK(KP707106781), T6d, T6c);
792 T6w = VFMA(LDK(KP707106781), T6g, T6f);
793 T6h = VFNMS(LDK(KP707106781), T6g, T6f);
794 T6u = VFMA(LDK(KP707106781), T66, T65);
795 T67 = VFNMS(LDK(KP707106781), T66, T65);
796 T6t = VFMA(LDK(KP707106781), T69, T68);
797 T6a = VFNMS(LDK(KP707106781), T69, T68);
798 T8R = VFMA(LDK(KP923879532), T8Q, T8P);
799 T8T = VFNMS(LDK(KP923879532), T8Q, T8P);
800 T6y = VFNMS(LDK(KP198912367), T6x, T6w);
801 T6C = VFMA(LDK(KP198912367), T6w, T6x);
802 T6m = VFMA(LDK(KP668178637), T6e, T6h);
803 T6i = VFNMS(LDK(KP668178637), T6h, T6e);
804 T6v = VFMA(LDK(KP198912367), T6u, T6t);
805 T6B = VFNMS(LDK(KP198912367), T6t, T6u);
806 T6l = VFNMS(LDK(KP668178637), T67, T6a);
807 T6b = VFMA(LDK(KP668178637), T6a, T67);
808 }
809 T6A = VFMA(LDK(KP923879532), T6r, T6o);
810 T6s = VFNMS(LDK(KP923879532), T6r, T6o);
811 T8X = VFNMS(LDK(KP923879532), T8W, T8V);
812 T8Z = VFMA(LDK(KP923879532), T8W, T8V);
813 {
814 V T6z, T6D, T8Y, T6n, T8S, T8U, T6j;
815 T6z = VSUB(T6v, T6y);
816 T90 = VADD(T6v, T6y);
817 T6D = VADD(T6B, T6C);
818 T8Y = VSUB(T6C, T6B);
819 T6n = VSUB(T6l, T6m);
820 T8S = VADD(T6l, T6m);
821 T8U = VSUB(T6i, T6b);
822 T6j = VADD(T6b, T6i);
823 ST(&(ri[WS(rs, 7)]), VFMA(LDK(KP980785280), T6z, T6s), ms, &(ri[WS(rs, 1)]));
824 ST(&(ri[WS(rs, 23)]), VFNMS(LDK(KP980785280), T6z, T6s), ms, &(ri[WS(rs, 1)]));
825 ST(&(ii[WS(rs, 23)]), VFNMS(LDK(KP980785280), T8Y, T8X), ms, &(ii[WS(rs, 1)]));
826 ST(&(ii[WS(rs, 7)]), VFMA(LDK(KP980785280), T8Y, T8X), ms, &(ii[WS(rs, 1)]));
827 ST(&(ri[WS(rs, 11)]), VFMA(LDK(KP831469612), T6n, T6k), ms, &(ri[WS(rs, 1)]));
828 ST(&(ri[WS(rs, 27)]), VFNMS(LDK(KP831469612), T6n, T6k), ms, &(ri[WS(rs, 1)]));
829 ST(&(ii[WS(rs, 19)]), VFNMS(LDK(KP831469612), T8S, T8R), ms, &(ii[WS(rs, 1)]));
830 ST(&(ii[WS(rs, 3)]), VFMA(LDK(KP831469612), T8S, T8R), ms, &(ii[WS(rs, 1)]));
831 ST(&(ii[WS(rs, 27)]), VFNMS(LDK(KP831469612), T8U, T8T), ms, &(ii[WS(rs, 1)]));
832 ST(&(ii[WS(rs, 11)]), VFMA(LDK(KP831469612), T8U, T8T), ms, &(ii[WS(rs, 1)]));
833 ST(&(ri[WS(rs, 3)]), VFMA(LDK(KP831469612), T6j, T64), ms, &(ri[WS(rs, 1)]));
834 ST(&(ri[WS(rs, 19)]), VFNMS(LDK(KP831469612), T6j, T64), ms, &(ri[WS(rs, 1)]));
835 ST(&(ri[WS(rs, 31)]), VFMA(LDK(KP980785280), T6D, T6A), ms, &(ri[WS(rs, 1)]));
836 ST(&(ri[WS(rs, 15)]), VFNMS(LDK(KP980785280), T6D, T6A), ms, &(ri[WS(rs, 1)]));
837 }
838 }
839 }
840 }
841 }
842 }
843 ST(&(ii[WS(rs, 31)]), VFMA(LDK(KP980785280), T90, T8Z), ms, &(ii[WS(rs, 1)]));
844 ST(&(ii[WS(rs, 15)]), VFNMS(LDK(KP980785280), T90, T8Z), ms, &(ii[WS(rs, 1)]));
845 }
846 }
847 VLEAVE();
848 }
849
850 static const tw_instr twinstr[] = {
851 VTW(0, 1),
852 VTW(0, 2),
853 VTW(0, 3),
854 VTW(0, 4),
855 VTW(0, 5),
856 VTW(0, 6),
857 VTW(0, 7),
858 VTW(0, 8),
859 VTW(0, 9),
860 VTW(0, 10),
861 VTW(0, 11),
862 VTW(0, 12),
863 VTW(0, 13),
864 VTW(0, 14),
865 VTW(0, 15),
866 VTW(0, 16),
867 VTW(0, 17),
868 VTW(0, 18),
869 VTW(0, 19),
870 VTW(0, 20),
871 VTW(0, 21),
872 VTW(0, 22),
873 VTW(0, 23),
874 VTW(0, 24),
875 VTW(0, 25),
876 VTW(0, 26),
877 VTW(0, 27),
878 VTW(0, 28),
879 VTW(0, 29),
880 VTW(0, 30),
881 VTW(0, 31),
882 {TW_NEXT, (2 * VL), 0}
883 };
884
885 static const ct_desc desc = { 32, XSIMD_STRING("t1sv_32"), twinstr, &GENUS, {236, 62, 198, 0}, 0, 0, 0 };
886
887 void XSIMD(codelet_t1sv_32) (planner *p) {
888 X(kdft_dit_register) (p, t1sv_32, &desc);
889 }
890 #else /* HAVE_FMA */
891
892 /* Generated by: ../../../genfft/gen_twiddle.native -simd -compact -variables 4 -pipeline-latency 8 -n 32 -name t1sv_32 -include ts.h */
893
894 /*
895 * This function contains 434 FP additions, 208 FP multiplications,
896 * (or, 340 additions, 114 multiplications, 94 fused multiply/add),
897 * 96 stack variables, 7 constants, and 128 memory accesses
898 */
899 #include "ts.h"
900
901 static void t1sv_32(R *ri, R *ii, const R *W, stride rs, INT mb, INT me, INT ms)
902 {
903 DVK(KP195090322, +0.195090322016128267848284868477022240927691618);
904 DVK(KP980785280, +0.980785280403230449126182236134239036973933731);
905 DVK(KP555570233, +0.555570233019602224742830813948532874374937191);
906 DVK(KP831469612, +0.831469612302545237078788377617905756738560812);
907 DVK(KP382683432, +0.382683432365089771728459984030398866761344562);
908 DVK(KP923879532, +0.923879532511286756128183189396788286822416626);
909 DVK(KP707106781, +0.707106781186547524400844362104849039284835938);
910 {
911 INT m;
912 for (m = mb, W = W + (mb * 62); m < me; m = m + (2 * VL), ri = ri + ((2 * VL) * ms), ii = ii + ((2 * VL) * ms), W = W + ((2 * VL) * 62), MAKE_VOLATILE_STRIDE(64, rs)) {
913 V Tj, T5F, T7C, T7Q, T35, T4T, T78, T7m, T1Q, T61, T5Y, T6J, T3K, T59, T41;
914 V T56, T2B, T67, T6e, T6O, T4b, T5d, T4s, T5g, TG, T7l, T5I, T73, T3a, T4U;
915 V T3f, T4V, T14, T5N, T5M, T6E, T3m, T4Y, T3r, T4Z, T1r, T5P, T5S, T6F, T3x;
916 V T51, T3C, T52, T2d, T5Z, T64, T6K, T3V, T57, T44, T5a, T2Y, T6f, T6a, T6P;
917 V T4m, T5h, T4v, T5e;
918 {
919 V T1, T76, T6, T75, Tc, T32, Th, T33;
920 T1 = LD(&(ri[0]), ms, &(ri[0]));
921 T76 = LD(&(ii[0]), ms, &(ii[0]));
922 {
923 V T3, T5, T2, T4;
924 T3 = LD(&(ri[WS(rs, 16)]), ms, &(ri[0]));
925 T5 = LD(&(ii[WS(rs, 16)]), ms, &(ii[0]));
926 T2 = LDW(&(W[TWVL * 30]));
927 T4 = LDW(&(W[TWVL * 31]));
928 T6 = VFMA(T2, T3, VMUL(T4, T5));
929 T75 = VFNMS(T4, T3, VMUL(T2, T5));
930 }
931 {
932 V T9, Tb, T8, Ta;
933 T9 = LD(&(ri[WS(rs, 8)]), ms, &(ri[0]));
934 Tb = LD(&(ii[WS(rs, 8)]), ms, &(ii[0]));
935 T8 = LDW(&(W[TWVL * 14]));
936 Ta = LDW(&(W[TWVL * 15]));
937 Tc = VFMA(T8, T9, VMUL(Ta, Tb));
938 T32 = VFNMS(Ta, T9, VMUL(T8, Tb));
939 }
940 {
941 V Te, Tg, Td, Tf;
942 Te = LD(&(ri[WS(rs, 24)]), ms, &(ri[0]));
943 Tg = LD(&(ii[WS(rs, 24)]), ms, &(ii[0]));
944 Td = LDW(&(W[TWVL * 46]));
945 Tf = LDW(&(W[TWVL * 47]));
946 Th = VFMA(Td, Te, VMUL(Tf, Tg));
947 T33 = VFNMS(Tf, Te, VMUL(Td, Tg));
948 }
949 {
950 V T7, Ti, T7A, T7B;
951 T7 = VADD(T1, T6);
952 Ti = VADD(Tc, Th);
953 Tj = VADD(T7, Ti);
954 T5F = VSUB(T7, Ti);
955 T7A = VSUB(T76, T75);
956 T7B = VSUB(Tc, Th);
957 T7C = VSUB(T7A, T7B);
958 T7Q = VADD(T7B, T7A);
959 }
960 {
961 V T31, T34, T74, T77;
962 T31 = VSUB(T1, T6);
963 T34 = VSUB(T32, T33);
964 T35 = VSUB(T31, T34);
965 T4T = VADD(T31, T34);
966 T74 = VADD(T32, T33);
967 T77 = VADD(T75, T76);
968 T78 = VADD(T74, T77);
969 T7m = VSUB(T77, T74);
970 }
971 }
972 {
973 V T1y, T3G, T1O, T3Z, T1D, T3H, T1J, T3Y;
974 {
975 V T1v, T1x, T1u, T1w;
976 T1v = LD(&(ri[WS(rs, 1)]), ms, &(ri[WS(rs, 1)]));
977 T1x = LD(&(ii[WS(rs, 1)]), ms, &(ii[WS(rs, 1)]));
978 T1u = LDW(&(W[0]));
979 T1w = LDW(&(W[TWVL * 1]));
980 T1y = VFMA(T1u, T1v, VMUL(T1w, T1x));
981 T3G = VFNMS(T1w, T1v, VMUL(T1u, T1x));
982 }
983 {
984 V T1L, T1N, T1K, T1M;
985 T1L = LD(&(ri[WS(rs, 25)]), ms, &(ri[WS(rs, 1)]));
986 T1N = LD(&(ii[WS(rs, 25)]), ms, &(ii[WS(rs, 1)]));
987 T1K = LDW(&(W[TWVL * 48]));
988 T1M = LDW(&(W[TWVL * 49]));
989 T1O = VFMA(T1K, T1L, VMUL(T1M, T1N));
990 T3Z = VFNMS(T1M, T1L, VMUL(T1K, T1N));
991 }
992 {
993 V T1A, T1C, T1z, T1B;
994 T1A = LD(&(ri[WS(rs, 17)]), ms, &(ri[WS(rs, 1)]));
995 T1C = LD(&(ii[WS(rs, 17)]), ms, &(ii[WS(rs, 1)]));
996 T1z = LDW(&(W[TWVL * 32]));
997 T1B = LDW(&(W[TWVL * 33]));
998 T1D = VFMA(T1z, T1A, VMUL(T1B, T1C));
999 T3H = VFNMS(T1B, T1A, VMUL(T1z, T1C));
1000 }
1001 {
1002 V T1G, T1I, T1F, T1H;
1003 T1G = LD(&(ri[WS(rs, 9)]), ms, &(ri[WS(rs, 1)]));
1004 T1I = LD(&(ii[WS(rs, 9)]), ms, &(ii[WS(rs, 1)]));
1005 T1F = LDW(&(W[TWVL * 16]));
1006 T1H = LDW(&(W[TWVL * 17]));
1007 T1J = VFMA(T1F, T1G, VMUL(T1H, T1I));
1008 T3Y = VFNMS(T1H, T1G, VMUL(T1F, T1I));
1009 }
1010 {
1011 V T1E, T1P, T5W, T5X;
1012 T1E = VADD(T1y, T1D);
1013 T1P = VADD(T1J, T1O);
1014 T1Q = VADD(T1E, T1P);
1015 T61 = VSUB(T1E, T1P);
1016 T5W = VADD(T3G, T3H);
1017 T5X = VADD(T3Y, T3Z);
1018 T5Y = VSUB(T5W, T5X);
1019 T6J = VADD(T5W, T5X);
1020 }
1021 {
1022 V T3I, T3J, T3X, T40;
1023 T3I = VSUB(T3G, T3H);
1024 T3J = VSUB(T1J, T1O);
1025 T3K = VADD(T3I, T3J);
1026 T59 = VSUB(T3I, T3J);
1027 T3X = VSUB(T1y, T1D);
1028 T40 = VSUB(T3Y, T3Z);
1029 T41 = VSUB(T3X, T40);
1030 T56 = VADD(T3X, T40);
1031 }
1032 }
1033 {
1034 V T2j, T4o, T2z, T49, T2o, T4p, T2u, T48;
1035 {
1036 V T2g, T2i, T2f, T2h;
1037 T2g = LD(&(ri[WS(rs, 31)]), ms, &(ri[WS(rs, 1)]));
1038 T2i = LD(&(ii[WS(rs, 31)]), ms, &(ii[WS(rs, 1)]));
1039 T2f = LDW(&(W[TWVL * 60]));
1040 T2h = LDW(&(W[TWVL * 61]));
1041 T2j = VFMA(T2f, T2g, VMUL(T2h, T2i));
1042 T4o = VFNMS(T2h, T2g, VMUL(T2f, T2i));
1043 }
1044 {
1045 V T2w, T2y, T2v, T2x;
1046 T2w = LD(&(ri[WS(rs, 23)]), ms, &(ri[WS(rs, 1)]));
1047 T2y = LD(&(ii[WS(rs, 23)]), ms, &(ii[WS(rs, 1)]));
1048 T2v = LDW(&(W[TWVL * 44]));
1049 T2x = LDW(&(W[TWVL * 45]));
1050 T2z = VFMA(T2v, T2w, VMUL(T2x, T2y));
1051 T49 = VFNMS(T2x, T2w, VMUL(T2v, T2y));
1052 }
1053 {
1054 V T2l, T2n, T2k, T2m;
1055 T2l = LD(&(ri[WS(rs, 15)]), ms, &(ri[WS(rs, 1)]));
1056 T2n = LD(&(ii[WS(rs, 15)]), ms, &(ii[WS(rs, 1)]));
1057 T2k = LDW(&(W[TWVL * 28]));
1058 T2m = LDW(&(W[TWVL * 29]));
1059 T2o = VFMA(T2k, T2l, VMUL(T2m, T2n));
1060 T4p = VFNMS(T2m, T2l, VMUL(T2k, T2n));
1061 }
1062 {
1063 V T2r, T2t, T2q, T2s;
1064 T2r = LD(&(ri[WS(rs, 7)]), ms, &(ri[WS(rs, 1)]));
1065 T2t = LD(&(ii[WS(rs, 7)]), ms, &(ii[WS(rs, 1)]));
1066 T2q = LDW(&(W[TWVL * 12]));
1067 T2s = LDW(&(W[TWVL * 13]));
1068 T2u = VFMA(T2q, T2r, VMUL(T2s, T2t));
1069 T48 = VFNMS(T2s, T2r, VMUL(T2q, T2t));
1070 }
1071 {
1072 V T2p, T2A, T6c, T6d;
1073 T2p = VADD(T2j, T2o);
1074 T2A = VADD(T2u, T2z);
1075 T2B = VADD(T2p, T2A);
1076 T67 = VSUB(T2p, T2A);
1077 T6c = VADD(T4o, T4p);
1078 T6d = VADD(T48, T49);
1079 T6e = VSUB(T6c, T6d);
1080 T6O = VADD(T6c, T6d);
1081 }
1082 {
1083 V T47, T4a, T4q, T4r;
1084 T47 = VSUB(T2j, T2o);
1085 T4a = VSUB(T48, T49);
1086 T4b = VSUB(T47, T4a);
1087 T5d = VADD(T47, T4a);
1088 T4q = VSUB(T4o, T4p);
1089 T4r = VSUB(T2u, T2z);
1090 T4s = VADD(T4q, T4r);
1091 T5g = VSUB(T4q, T4r);
1092 }
1093 }
1094 {
1095 V To, T36, TE, T3d, Tt, T37, Tz, T3c;
1096 {
1097 V Tl, Tn, Tk, Tm;
1098 Tl = LD(&(ri[WS(rs, 4)]), ms, &(ri[0]));
1099 Tn = LD(&(ii[WS(rs, 4)]), ms, &(ii[0]));
1100 Tk = LDW(&(W[TWVL * 6]));
1101 Tm = LDW(&(W[TWVL * 7]));
1102 To = VFMA(Tk, Tl, VMUL(Tm, Tn));
1103 T36 = VFNMS(Tm, Tl, VMUL(Tk, Tn));
1104 }
1105 {
1106 V TB, TD, TA, TC;
1107 TB = LD(&(ri[WS(rs, 12)]), ms, &(ri[0]));
1108 TD = LD(&(ii[WS(rs, 12)]), ms, &(ii[0]));
1109 TA = LDW(&(W[TWVL * 22]));
1110 TC = LDW(&(W[TWVL * 23]));
1111 TE = VFMA(TA, TB, VMUL(TC, TD));
1112 T3d = VFNMS(TC, TB, VMUL(TA, TD));
1113 }
1114 {
1115 V Tq, Ts, Tp, Tr;
1116 Tq = LD(&(ri[WS(rs, 20)]), ms, &(ri[0]));
1117 Ts = LD(&(ii[WS(rs, 20)]), ms, &(ii[0]));
1118 Tp = LDW(&(W[TWVL * 38]));
1119 Tr = LDW(&(W[TWVL * 39]));
1120 Tt = VFMA(Tp, Tq, VMUL(Tr, Ts));
1121 T37 = VFNMS(Tr, Tq, VMUL(Tp, Ts));
1122 }
1123 {
1124 V Tw, Ty, Tv, Tx;
1125 Tw = LD(&(ri[WS(rs, 28)]), ms, &(ri[0]));
1126 Ty = LD(&(ii[WS(rs, 28)]), ms, &(ii[0]));
1127 Tv = LDW(&(W[TWVL * 54]));
1128 Tx = LDW(&(W[TWVL * 55]));
1129 Tz = VFMA(Tv, Tw, VMUL(Tx, Ty));
1130 T3c = VFNMS(Tx, Tw, VMUL(Tv, Ty));
1131 }
1132 {
1133 V Tu, TF, T5G, T5H;
1134 Tu = VADD(To, Tt);
1135 TF = VADD(Tz, TE);
1136 TG = VADD(Tu, TF);
1137 T7l = VSUB(TF, Tu);
1138 T5G = VADD(T36, T37);
1139 T5H = VADD(T3c, T3d);
1140 T5I = VSUB(T5G, T5H);
1141 T73 = VADD(T5G, T5H);
1142 }
1143 {
1144 V T38, T39, T3b, T3e;
1145 T38 = VSUB(T36, T37);
1146 T39 = VSUB(To, Tt);
1147 T3a = VSUB(T38, T39);
1148 T4U = VADD(T39, T38);
1149 T3b = VSUB(Tz, TE);
1150 T3e = VSUB(T3c, T3d);
1151 T3f = VADD(T3b, T3e);
1152 T4V = VSUB(T3b, T3e);
1153 }
1154 }
1155 {
1156 V TM, T3i, T12, T3p, TR, T3j, TX, T3o;
1157 {
1158 V TJ, TL, TI, TK;
1159 TJ = LD(&(ri[WS(rs, 2)]), ms, &(ri[0]));
1160 TL = LD(&(ii[WS(rs, 2)]), ms, &(ii[0]));
1161 TI = LDW(&(W[TWVL * 2]));
1162 TK = LDW(&(W[TWVL * 3]));
1163 TM = VFMA(TI, TJ, VMUL(TK, TL));
1164 T3i = VFNMS(TK, TJ, VMUL(TI, TL));
1165 }
1166 {
1167 V TZ, T11, TY, T10;
1168 TZ = LD(&(ri[WS(rs, 26)]), ms, &(ri[0]));
1169 T11 = LD(&(ii[WS(rs, 26)]), ms, &(ii[0]));
1170 TY = LDW(&(W[TWVL * 50]));
1171 T10 = LDW(&(W[TWVL * 51]));
1172 T12 = VFMA(TY, TZ, VMUL(T10, T11));
1173 T3p = VFNMS(T10, TZ, VMUL(TY, T11));
1174 }
1175 {
1176 V TO, TQ, TN, TP;
1177 TO = LD(&(ri[WS(rs, 18)]), ms, &(ri[0]));
1178 TQ = LD(&(ii[WS(rs, 18)]), ms, &(ii[0]));
1179 TN = LDW(&(W[TWVL * 34]));
1180 TP = LDW(&(W[TWVL * 35]));
1181 TR = VFMA(TN, TO, VMUL(TP, TQ));
1182 T3j = VFNMS(TP, TO, VMUL(TN, TQ));
1183 }
1184 {
1185 V TU, TW, TT, TV;
1186 TU = LD(&(ri[WS(rs, 10)]), ms, &(ri[0]));
1187 TW = LD(&(ii[WS(rs, 10)]), ms, &(ii[0]));
1188 TT = LDW(&(W[TWVL * 18]));
1189 TV = LDW(&(W[TWVL * 19]));
1190 TX = VFMA(TT, TU, VMUL(TV, TW));
1191 T3o = VFNMS(TV, TU, VMUL(TT, TW));
1192 }
1193 {
1194 V TS, T13, T5K, T5L;
1195 TS = VADD(TM, TR);
1196 T13 = VADD(TX, T12);
1197 T14 = VADD(TS, T13);
1198 T5N = VSUB(TS, T13);
1199 T5K = VADD(T3i, T3j);
1200 T5L = VADD(T3o, T3p);
1201 T5M = VSUB(T5K, T5L);
1202 T6E = VADD(T5K, T5L);
1203 }
1204 {
1205 V T3k, T3l, T3n, T3q;
1206 T3k = VSUB(T3i, T3j);
1207 T3l = VSUB(TX, T12);
1208 T3m = VADD(T3k, T3l);
1209 T4Y = VSUB(T3k, T3l);
1210 T3n = VSUB(TM, TR);
1211 T3q = VSUB(T3o, T3p);
1212 T3r = VSUB(T3n, T3q);
1213 T4Z = VADD(T3n, T3q);
1214 }
1215 }
1216 {
1217 V T19, T3t, T1p, T3A, T1e, T3u, T1k, T3z;
1218 {
1219 V T16, T18, T15, T17;
1220 T16 = LD(&(ri[WS(rs, 30)]), ms, &(ri[0]));
1221 T18 = LD(&(ii[WS(rs, 30)]), ms, &(ii[0]));
1222 T15 = LDW(&(W[TWVL * 58]));
1223 T17 = LDW(&(W[TWVL * 59]));
1224 T19 = VFMA(T15, T16, VMUL(T17, T18));
1225 T3t = VFNMS(T17, T16, VMUL(T15, T18));
1226 }
1227 {
1228 V T1m, T1o, T1l, T1n;
1229 T1m = LD(&(ri[WS(rs, 22)]), ms, &(ri[0]));
1230 T1o = LD(&(ii[WS(rs, 22)]), ms, &(ii[0]));
1231 T1l = LDW(&(W[TWVL * 42]));
1232 T1n = LDW(&(W[TWVL * 43]));
1233 T1p = VFMA(T1l, T1m, VMUL(T1n, T1o));
1234 T3A = VFNMS(T1n, T1m, VMUL(T1l, T1o));
1235 }
1236 {
1237 V T1b, T1d, T1a, T1c;
1238 T1b = LD(&(ri[WS(rs, 14)]), ms, &(ri[0]));
1239 T1d = LD(&(ii[WS(rs, 14)]), ms, &(ii[0]));
1240 T1a = LDW(&(W[TWVL * 26]));
1241 T1c = LDW(&(W[TWVL * 27]));
1242 T1e = VFMA(T1a, T1b, VMUL(T1c, T1d));
1243 T3u = VFNMS(T1c, T1b, VMUL(T1a, T1d));
1244 }
1245 {
1246 V T1h, T1j, T1g, T1i;
1247 T1h = LD(&(ri[WS(rs, 6)]), ms, &(ri[0]));
1248 T1j = LD(&(ii[WS(rs, 6)]), ms, &(ii[0]));
1249 T1g = LDW(&(W[TWVL * 10]));
1250 T1i = LDW(&(W[TWVL * 11]));
1251 T1k = VFMA(T1g, T1h, VMUL(T1i, T1j));
1252 T3z = VFNMS(T1i, T1h, VMUL(T1g, T1j));
1253 }
1254 {
1255 V T1f, T1q, T5Q, T5R;
1256 T1f = VADD(T19, T1e);
1257 T1q = VADD(T1k, T1p);
1258 T1r = VADD(T1f, T1q);
1259 T5P = VSUB(T1f, T1q);
1260 T5Q = VADD(T3t, T3u);
1261 T5R = VADD(T3z, T3A);
1262 T5S = VSUB(T5Q, T5R);
1263 T6F = VADD(T5Q, T5R);
1264 }
1265 {
1266 V T3v, T3w, T3y, T3B;
1267 T3v = VSUB(T3t, T3u);
1268 T3w = VSUB(T1k, T1p);
1269 T3x = VADD(T3v, T3w);
1270 T51 = VSUB(T3v, T3w);
1271 T3y = VSUB(T19, T1e);
1272 T3B = VSUB(T3z, T3A);
1273 T3C = VSUB(T3y, T3B);
1274 T52 = VADD(T3y, T3B);
1275 }
1276 }
1277 {
1278 V T1V, T3R, T20, T3S, T3Q, T3T, T26, T3M, T2b, T3N, T3L, T3O;
1279 {
1280 V T1S, T1U, T1R, T1T;
1281 T1S = LD(&(ri[WS(rs, 5)]), ms, &(ri[WS(rs, 1)]));
1282 T1U = LD(&(ii[WS(rs, 5)]), ms, &(ii[WS(rs, 1)]));
1283 T1R = LDW(&(W[TWVL * 8]));
1284 T1T = LDW(&(W[TWVL * 9]));
1285 T1V = VFMA(T1R, T1S, VMUL(T1T, T1U));
1286 T3R = VFNMS(T1T, T1S, VMUL(T1R, T1U));
1287 }
1288 {
1289 V T1X, T1Z, T1W, T1Y;
1290 T1X = LD(&(ri[WS(rs, 21)]), ms, &(ri[WS(rs, 1)]));
1291 T1Z = LD(&(ii[WS(rs, 21)]), ms, &(ii[WS(rs, 1)]));
1292 T1W = LDW(&(W[TWVL * 40]));
1293 T1Y = LDW(&(W[TWVL * 41]));
1294 T20 = VFMA(T1W, T1X, VMUL(T1Y, T1Z));
1295 T3S = VFNMS(T1Y, T1X, VMUL(T1W, T1Z));
1296 }
1297 T3Q = VSUB(T1V, T20);
1298 T3T = VSUB(T3R, T3S);
1299 {
1300 V T23, T25, T22, T24;
1301 T23 = LD(&(ri[WS(rs, 29)]), ms, &(ri[WS(rs, 1)]));
1302 T25 = LD(&(ii[WS(rs, 29)]), ms, &(ii[WS(rs, 1)]));
1303 T22 = LDW(&(W[TWVL * 56]));
1304 T24 = LDW(&(W[TWVL * 57]));
1305 T26 = VFMA(T22, T23, VMUL(T24, T25));
1306 T3M = VFNMS(T24, T23, VMUL(T22, T25));
1307 }
1308 {
1309 V T28, T2a, T27, T29;
1310 T28 = LD(&(ri[WS(rs, 13)]), ms, &(ri[WS(rs, 1)]));
1311 T2a = LD(&(ii[WS(rs, 13)]), ms, &(ii[WS(rs, 1)]));
1312 T27 = LDW(&(W[TWVL * 24]));
1313 T29 = LDW(&(W[TWVL * 25]));
1314 T2b = VFMA(T27, T28, VMUL(T29, T2a));
1315 T3N = VFNMS(T29, T28, VMUL(T27, T2a));
1316 }
1317 T3L = VSUB(T26, T2b);
1318 T3O = VSUB(T3M, T3N);
1319 {
1320 V T21, T2c, T62, T63;
1321 T21 = VADD(T1V, T20);
1322 T2c = VADD(T26, T2b);
1323 T2d = VADD(T21, T2c);
1324 T5Z = VSUB(T2c, T21);
1325 T62 = VADD(T3R, T3S);
1326 T63 = VADD(T3M, T3N);
1327 T64 = VSUB(T62, T63);
1328 T6K = VADD(T62, T63);
1329 }
1330 {
1331 V T3P, T3U, T42, T43;
1332 T3P = VSUB(T3L, T3O);
1333 T3U = VADD(T3Q, T3T);
1334 T3V = VMUL(LDK(KP707106781), VSUB(T3P, T3U));
1335 T57 = VMUL(LDK(KP707106781), VADD(T3U, T3P));
1336 T42 = VSUB(T3T, T3Q);
1337 T43 = VADD(T3L, T3O);
1338 T44 = VMUL(LDK(KP707106781), VSUB(T42, T43));
1339 T5a = VMUL(LDK(KP707106781), VADD(T42, T43));
1340 }
1341 }
1342 {
1343 V T2G, T4c, T2L, T4d, T4e, T4f, T2R, T4i, T2W, T4j, T4h, T4k;
1344 {
1345 V T2D, T2F, T2C, T2E;
1346 T2D = LD(&(ri[WS(rs, 3)]), ms, &(ri[WS(rs, 1)]));
1347 T2F = LD(&(ii[WS(rs, 3)]), ms, &(ii[WS(rs, 1)]));
1348 T2C = LDW(&(W[TWVL * 4]));
1349 T2E = LDW(&(W[TWVL * 5]));
1350 T2G = VFMA(T2C, T2D, VMUL(T2E, T2F));
1351 T4c = VFNMS(T2E, T2D, VMUL(T2C, T2F));
1352 }
1353 {
1354 V T2I, T2K, T2H, T2J;
1355 T2I = LD(&(ri[WS(rs, 19)]), ms, &(ri[WS(rs, 1)]));
1356 T2K = LD(&(ii[WS(rs, 19)]), ms, &(ii[WS(rs, 1)]));
1357 T2H = LDW(&(W[TWVL * 36]));
1358 T2J = LDW(&(W[TWVL * 37]));
1359 T2L = VFMA(T2H, T2I, VMUL(T2J, T2K));
1360 T4d = VFNMS(T2J, T2I, VMUL(T2H, T2K));
1361 }
1362 T4e = VSUB(T4c, T4d);
1363 T4f = VSUB(T2G, T2L);
1364 {
1365 V T2O, T2Q, T2N, T2P;
1366 T2O = LD(&(ri[WS(rs, 27)]), ms, &(ri[WS(rs, 1)]));
1367 T2Q = LD(&(ii[WS(rs, 27)]), ms, &(ii[WS(rs, 1)]));
1368 T2N = LDW(&(W[TWVL * 52]));
1369 T2P = LDW(&(W[TWVL * 53]));
1370 T2R = VFMA(T2N, T2O, VMUL(T2P, T2Q));
1371 T4i = VFNMS(T2P, T2O, VMUL(T2N, T2Q));
1372 }
1373 {
1374 V T2T, T2V, T2S, T2U;
1375 T2T = LD(&(ri[WS(rs, 11)]), ms, &(ri[WS(rs, 1)]));
1376 T2V = LD(&(ii[WS(rs, 11)]), ms, &(ii[WS(rs, 1)]));
1377 T2S = LDW(&(W[TWVL * 20]));
1378 T2U = LDW(&(W[TWVL * 21]));
1379 T2W = VFMA(T2S, T2T, VMUL(T2U, T2V));
1380 T4j = VFNMS(T2U, T2T, VMUL(T2S, T2V));
1381 }
1382 T4h = VSUB(T2R, T2W);
1383 T4k = VSUB(T4i, T4j);
1384 {
1385 V T2M, T2X, T68, T69;
1386 T2M = VADD(T2G, T2L);
1387 T2X = VADD(T2R, T2W);
1388 T2Y = VADD(T2M, T2X);
1389 T6f = VSUB(T2X, T2M);
1390 T68 = VADD(T4c, T4d);
1391 T69 = VADD(T4i, T4j);
1392 T6a = VSUB(T68, T69);
1393 T6P = VADD(T68, T69);
1394 }
1395 {
1396 V T4g, T4l, T4t, T4u;
1397 T4g = VSUB(T4e, T4f);
1398 T4l = VADD(T4h, T4k);
1399 T4m = VMUL(LDK(KP707106781), VSUB(T4g, T4l));
1400 T5h = VMUL(LDK(KP707106781), VADD(T4g, T4l));
1401 T4t = VSUB(T4h, T4k);
1402 T4u = VADD(T4f, T4e);
1403 T4v = VMUL(LDK(KP707106781), VSUB(T4t, T4u));
1404 T5e = VMUL(LDK(KP707106781), VADD(T4u, T4t));
1405 }
1406 }
1407 {
1408 V T1t, T6X, T7a, T7c, T30, T7b, T70, T71;
1409 {
1410 V TH, T1s, T72, T79;
1411 TH = VADD(Tj, TG);
1412 T1s = VADD(T14, T1r);
1413 T1t = VADD(TH, T1s);
1414 T6X = VSUB(TH, T1s);
1415 T72 = VADD(T6E, T6F);
1416 T79 = VADD(T73, T78);
1417 T7a = VADD(T72, T79);
1418 T7c = VSUB(T79, T72);
1419 }
1420 {
1421 V T2e, T2Z, T6Y, T6Z;
1422 T2e = VADD(T1Q, T2d);
1423 T2Z = VADD(T2B, T2Y);
1424 T30 = VADD(T2e, T2Z);
1425 T7b = VSUB(T2Z, T2e);
1426 T6Y = VADD(T6J, T6K);
1427 T6Z = VADD(T6O, T6P);
1428 T70 = VSUB(T6Y, T6Z);
1429 T71 = VADD(T6Y, T6Z);
1430 }
1431 ST(&(ri[WS(rs, 16)]), VSUB(T1t, T30), ms, &(ri[0]));
1432 ST(&(ii[WS(rs, 16)]), VSUB(T7a, T71), ms, &(ii[0]));
1433 ST(&(ri[0]), VADD(T1t, T30), ms, &(ri[0]));
1434 ST(&(ii[0]), VADD(T71, T7a), ms, &(ii[0]));
1435 ST(&(ri[WS(rs, 24)]), VSUB(T6X, T70), ms, &(ri[0]));
1436 ST(&(ii[WS(rs, 24)]), VSUB(T7c, T7b), ms, &(ii[0]));
1437 ST(&(ri[WS(rs, 8)]), VADD(T6X, T70), ms, &(ri[0]));
1438 ST(&(ii[WS(rs, 8)]), VADD(T7b, T7c), ms, &(ii[0]));
1439 }
1440 {
1441 V T6H, T6T, T7g, T7i, T6M, T6U, T6R, T6V;
1442 {
1443 V T6D, T6G, T7e, T7f;
1444 T6D = VSUB(Tj, TG);
1445 T6G = VSUB(T6E, T6F);
1446 T6H = VADD(T6D, T6G);
1447 T6T = VSUB(T6D, T6G);
1448 T7e = VSUB(T1r, T14);
1449 T7f = VSUB(T78, T73);
1450 T7g = VADD(T7e, T7f);
1451 T7i = VSUB(T7f, T7e);
1452 }
1453 {
1454 V T6I, T6L, T6N, T6Q;
1455 T6I = VSUB(T1Q, T2d);
1456 T6L = VSUB(T6J, T6K);
1457 T6M = VADD(T6I, T6L);
1458 T6U = VSUB(T6L, T6I);
1459 T6N = VSUB(T2B, T2Y);
1460 T6Q = VSUB(T6O, T6P);
1461 T6R = VSUB(T6N, T6Q);
1462 T6V = VADD(T6N, T6Q);
1463 }
1464 {
1465 V T6S, T7d, T6W, T7h;
1466 T6S = VMUL(LDK(KP707106781), VADD(T6M, T6R));
1467 ST(&(ri[WS(rs, 20)]), VSUB(T6H, T6S), ms, &(ri[0]));
1468 ST(&(ri[WS(rs, 4)]), VADD(T6H, T6S), ms, &(ri[0]));
1469 T7d = VMUL(LDK(KP707106781), VADD(T6U, T6V));
1470 ST(&(ii[WS(rs, 4)]), VADD(T7d, T7g), ms, &(ii[0]));
1471 ST(&(ii[WS(rs, 20)]), VSUB(T7g, T7d), ms, &(ii[0]));
1472 T6W = VMUL(LDK(KP707106781), VSUB(T6U, T6V));
1473 ST(&(ri[WS(rs, 28)]), VSUB(T6T, T6W), ms, &(ri[0]));
1474 ST(&(ri[WS(rs, 12)]), VADD(T6T, T6W), ms, &(ri[0]));
1475 T7h = VMUL(LDK(KP707106781), VSUB(T6R, T6M));
1476 ST(&(ii[WS(rs, 12)]), VADD(T7h, T7i), ms, &(ii[0]));
1477 ST(&(ii[WS(rs, 28)]), VSUB(T7i, T7h), ms, &(ii[0]));
1478 }
1479 }
1480 {
1481 V T5J, T7n, T7t, T6n, T5U, T7k, T6x, T6B, T6q, T7s, T66, T6k, T6u, T6A, T6h;
1482 V T6l;
1483 {
1484 V T5O, T5T, T60, T65;
1485 T5J = VSUB(T5F, T5I);
1486 T7n = VADD(T7l, T7m);
1487 T7t = VSUB(T7m, T7l);
1488 T6n = VADD(T5F, T5I);
1489 T5O = VSUB(T5M, T5N);
1490 T5T = VADD(T5P, T5S);
1491 T5U = VMUL(LDK(KP707106781), VSUB(T5O, T5T));
1492 T7k = VMUL(LDK(KP707106781), VADD(T5O, T5T));
1493 {
1494 V T6v, T6w, T6o, T6p;
1495 T6v = VADD(T67, T6a);
1496 T6w = VADD(T6e, T6f);
1497 T6x = VFNMS(LDK(KP382683432), T6w, VMUL(LDK(KP923879532), T6v));
1498 T6B = VFMA(LDK(KP923879532), T6w, VMUL(LDK(KP382683432), T6v));
1499 T6o = VADD(T5N, T5M);
1500 T6p = VSUB(T5P, T5S);
1501 T6q = VMUL(LDK(KP707106781), VADD(T6o, T6p));
1502 T7s = VMUL(LDK(KP707106781), VSUB(T6p, T6o));
1503 }
1504 T60 = VSUB(T5Y, T5Z);
1505 T65 = VSUB(T61, T64);
1506 T66 = VFMA(LDK(KP923879532), T60, VMUL(LDK(KP382683432), T65));
1507 T6k = VFNMS(LDK(KP923879532), T65, VMUL(LDK(KP382683432), T60));
1508 {
1509 V T6s, T6t, T6b, T6g;
1510 T6s = VADD(T5Y, T5Z);
1511 T6t = VADD(T61, T64);
1512 T6u = VFMA(LDK(KP382683432), T6s, VMUL(LDK(KP923879532), T6t));
1513 T6A = VFNMS(LDK(KP382683432), T6t, VMUL(LDK(KP923879532), T6s));
1514 T6b = VSUB(T67, T6a);
1515 T6g = VSUB(T6e, T6f);
1516 T6h = VFNMS(LDK(KP923879532), T6g, VMUL(LDK(KP382683432), T6b));
1517 T6l = VFMA(LDK(KP382683432), T6g, VMUL(LDK(KP923879532), T6b));
1518 }
1519 }
1520 {
1521 V T5V, T6i, T7r, T7u;
1522 T5V = VADD(T5J, T5U);
1523 T6i = VADD(T66, T6h);
1524 ST(&(ri[WS(rs, 22)]), VSUB(T5V, T6i), ms, &(ri[0]));
1525 ST(&(ri[WS(rs, 6)]), VADD(T5V, T6i), ms, &(ri[0]));
1526 T7r = VADD(T6k, T6l);
1527 T7u = VADD(T7s, T7t);
1528 ST(&(ii[WS(rs, 6)]), VADD(T7r, T7u), ms, &(ii[0]));
1529 ST(&(ii[WS(rs, 22)]), VSUB(T7u, T7r), ms, &(ii[0]));
1530 }
1531 {
1532 V T6j, T6m, T7v, T7w;
1533 T6j = VSUB(T5J, T5U);
1534 T6m = VSUB(T6k, T6l);
1535 ST(&(ri[WS(rs, 30)]), VSUB(T6j, T6m), ms, &(ri[0]));
1536 ST(&(ri[WS(rs, 14)]), VADD(T6j, T6m), ms, &(ri[0]));
1537 T7v = VSUB(T6h, T66);
1538 T7w = VSUB(T7t, T7s);
1539 ST(&(ii[WS(rs, 14)]), VADD(T7v, T7w), ms, &(ii[0]));
1540 ST(&(ii[WS(rs, 30)]), VSUB(T7w, T7v), ms, &(ii[0]));
1541 }
1542 {
1543 V T6r, T6y, T7j, T7o;
1544 T6r = VADD(T6n, T6q);
1545 T6y = VADD(T6u, T6x);
1546 ST(&(ri[WS(rs, 18)]), VSUB(T6r, T6y), ms, &(ri[0]));
1547 ST(&(ri[WS(rs, 2)]), VADD(T6r, T6y), ms, &(ri[0]));
1548 T7j = VADD(T6A, T6B);
1549 T7o = VADD(T7k, T7n);
1550 ST(&(ii[WS(rs, 2)]), VADD(T7j, T7o), ms, &(ii[0]));
1551 ST(&(ii[WS(rs, 18)]), VSUB(T7o, T7j), ms, &(ii[0]));
1552 }
1553 {
1554 V T6z, T6C, T7p, T7q;
1555 T6z = VSUB(T6n, T6q);
1556 T6C = VSUB(T6A, T6B);
1557 ST(&(ri[WS(rs, 26)]), VSUB(T6z, T6C), ms, &(ri[0]));
1558 ST(&(ri[WS(rs, 10)]), VADD(T6z, T6C), ms, &(ri[0]));
1559 T7p = VSUB(T6x, T6u);
1560 T7q = VSUB(T7n, T7k);
1561 ST(&(ii[WS(rs, 10)]), VADD(T7p, T7q), ms, &(ii[0]));
1562 ST(&(ii[WS(rs, 26)]), VSUB(T7q, T7p), ms, &(ii[0]));
1563 }
1564 }
1565 {
1566 V T3h, T4D, T7R, T7X, T3E, T7O, T4N, T4R, T46, T4A, T4G, T7W, T4K, T4Q, T4x;
1567 V T4B, T3g, T7P;
1568 T3g = VMUL(LDK(KP707106781), VSUB(T3a, T3f));
1569 T3h = VSUB(T35, T3g);
1570 T4D = VADD(T35, T3g);
1571 T7P = VMUL(LDK(KP707106781), VSUB(T4V, T4U));
1572 T7R = VADD(T7P, T7Q);
1573 T7X = VSUB(T7Q, T7P);
1574 {
1575 V T3s, T3D, T4L, T4M;
1576 T3s = VFNMS(LDK(KP923879532), T3r, VMUL(LDK(KP382683432), T3m));
1577 T3D = VFMA(LDK(KP382683432), T3x, VMUL(LDK(KP923879532), T3C));
1578 T3E = VSUB(T3s, T3D);
1579 T7O = VADD(T3s, T3D);
1580 T4L = VADD(T4b, T4m);
1581 T4M = VADD(T4s, T4v);
1582 T4N = VFNMS(LDK(KP555570233), T4M, VMUL(LDK(KP831469612), T4L));
1583 T4R = VFMA(LDK(KP831469612), T4M, VMUL(LDK(KP555570233), T4L));
1584 }
1585 {
1586 V T3W, T45, T4E, T4F;
1587 T3W = VSUB(T3K, T3V);
1588 T45 = VSUB(T41, T44);
1589 T46 = VFMA(LDK(KP980785280), T3W, VMUL(LDK(KP195090322), T45));
1590 T4A = VFNMS(LDK(KP980785280), T45, VMUL(LDK(KP195090322), T3W));
1591 T4E = VFMA(LDK(KP923879532), T3m, VMUL(LDK(KP382683432), T3r));
1592 T4F = VFNMS(LDK(KP923879532), T3x, VMUL(LDK(KP382683432), T3C));
1593 T4G = VADD(T4E, T4F);
1594 T7W = VSUB(T4F, T4E);
1595 }
1596 {
1597 V T4I, T4J, T4n, T4w;
1598 T4I = VADD(T3K, T3V);
1599 T4J = VADD(T41, T44);
1600 T4K = VFMA(LDK(KP555570233), T4I, VMUL(LDK(KP831469612), T4J));
1601 T4Q = VFNMS(LDK(KP555570233), T4J, VMUL(LDK(KP831469612), T4I));
1602 T4n = VSUB(T4b, T4m);
1603 T4w = VSUB(T4s, T4v);
1604 T4x = VFNMS(LDK(KP980785280), T4w, VMUL(LDK(KP195090322), T4n));
1605 T4B = VFMA(LDK(KP195090322), T4w, VMUL(LDK(KP980785280), T4n));
1606 }
1607 {
1608 V T3F, T4y, T7V, T7Y;
1609 T3F = VADD(T3h, T3E);
1610 T4y = VADD(T46, T4x);
1611 ST(&(ri[WS(rs, 23)]), VSUB(T3F, T4y), ms, &(ri[WS(rs, 1)]));
1612 ST(&(ri[WS(rs, 7)]), VADD(T3F, T4y), ms, &(ri[WS(rs, 1)]));
1613 T7V = VADD(T4A, T4B);
1614 T7Y = VADD(T7W, T7X);
1615 ST(&(ii[WS(rs, 7)]), VADD(T7V, T7Y), ms, &(ii[WS(rs, 1)]));
1616 ST(&(ii[WS(rs, 23)]), VSUB(T7Y, T7V), ms, &(ii[WS(rs, 1)]));
1617 }
1618 {
1619 V T4z, T4C, T7Z, T80;
1620 T4z = VSUB(T3h, T3E);
1621 T4C = VSUB(T4A, T4B);
1622 ST(&(ri[WS(rs, 31)]), VSUB(T4z, T4C), ms, &(ri[WS(rs, 1)]));
1623 ST(&(ri[WS(rs, 15)]), VADD(T4z, T4C), ms, &(ri[WS(rs, 1)]));
1624 T7Z = VSUB(T4x, T46);
1625 T80 = VSUB(T7X, T7W);
1626 ST(&(ii[WS(rs, 15)]), VADD(T7Z, T80), ms, &(ii[WS(rs, 1)]));
1627 ST(&(ii[WS(rs, 31)]), VSUB(T80, T7Z), ms, &(ii[WS(rs, 1)]));
1628 }
1629 {
1630 V T4H, T4O, T7N, T7S;
1631 T4H = VADD(T4D, T4G);
1632 T4O = VADD(T4K, T4N);
1633 ST(&(ri[WS(rs, 19)]), VSUB(T4H, T4O), ms, &(ri[WS(rs, 1)]));
1634 ST(&(ri[WS(rs, 3)]), VADD(T4H, T4O), ms, &(ri[WS(rs, 1)]));
1635 T7N = VADD(T4Q, T4R);
1636 T7S = VADD(T7O, T7R);
1637 ST(&(ii[WS(rs, 3)]), VADD(T7N, T7S), ms, &(ii[WS(rs, 1)]));
1638 ST(&(ii[WS(rs, 19)]), VSUB(T7S, T7N), ms, &(ii[WS(rs, 1)]));
1639 }
1640 {
1641 V T4P, T4S, T7T, T7U;
1642 T4P = VSUB(T4D, T4G);
1643 T4S = VSUB(T4Q, T4R);
1644 ST(&(ri[WS(rs, 27)]), VSUB(T4P, T4S), ms, &(ri[WS(rs, 1)]));
1645 ST(&(ri[WS(rs, 11)]), VADD(T4P, T4S), ms, &(ri[WS(rs, 1)]));
1646 T7T = VSUB(T4N, T4K);
1647 T7U = VSUB(T7R, T7O);
1648 ST(&(ii[WS(rs, 11)]), VADD(T7T, T7U), ms, &(ii[WS(rs, 1)]));
1649 ST(&(ii[WS(rs, 27)]), VSUB(T7U, T7T), ms, &(ii[WS(rs, 1)]));
1650 }
1651 }
1652 {
1653 V T4X, T5p, T7D, T7J, T54, T7y, T5z, T5D, T5c, T5m, T5s, T7I, T5w, T5C, T5j;
1654 V T5n, T4W, T7z;
1655 T4W = VMUL(LDK(KP707106781), VADD(T4U, T4V));
1656 T4X = VSUB(T4T, T4W);
1657 T5p = VADD(T4T, T4W);
1658 T7z = VMUL(LDK(KP707106781), VADD(T3a, T3f));
1659 T7D = VADD(T7z, T7C);
1660 T7J = VSUB(T7C, T7z);
1661 {
1662 V T50, T53, T5x, T5y;
1663 T50 = VFNMS(LDK(KP382683432), T4Z, VMUL(LDK(KP923879532), T4Y));
1664 T53 = VFMA(LDK(KP923879532), T51, VMUL(LDK(KP382683432), T52));
1665 T54 = VSUB(T50, T53);
1666 T7y = VADD(T50, T53);
1667 T5x = VADD(T5d, T5e);
1668 T5y = VADD(T5g, T5h);
1669 T5z = VFNMS(LDK(KP195090322), T5y, VMUL(LDK(KP980785280), T5x));
1670 T5D = VFMA(LDK(KP195090322), T5x, VMUL(LDK(KP980785280), T5y));
1671 }
1672 {
1673 V T58, T5b, T5q, T5r;
1674 T58 = VSUB(T56, T57);
1675 T5b = VSUB(T59, T5a);
1676 T5c = VFMA(LDK(KP555570233), T58, VMUL(LDK(KP831469612), T5b));
1677 T5m = VFNMS(LDK(KP831469612), T58, VMUL(LDK(KP555570233), T5b));
1678 T5q = VFMA(LDK(KP382683432), T4Y, VMUL(LDK(KP923879532), T4Z));
1679 T5r = VFNMS(LDK(KP382683432), T51, VMUL(LDK(KP923879532), T52));
1680 T5s = VADD(T5q, T5r);
1681 T7I = VSUB(T5r, T5q);
1682 }
1683 {
1684 V T5u, T5v, T5f, T5i;
1685 T5u = VADD(T56, T57);
1686 T5v = VADD(T59, T5a);
1687 T5w = VFMA(LDK(KP980785280), T5u, VMUL(LDK(KP195090322), T5v));
1688 T5C = VFNMS(LDK(KP195090322), T5u, VMUL(LDK(KP980785280), T5v));
1689 T5f = VSUB(T5d, T5e);
1690 T5i = VSUB(T5g, T5h);
1691 T5j = VFNMS(LDK(KP831469612), T5i, VMUL(LDK(KP555570233), T5f));
1692 T5n = VFMA(LDK(KP831469612), T5f, VMUL(LDK(KP555570233), T5i));
1693 }
1694 {
1695 V T55, T5k, T7H, T7K;
1696 T55 = VADD(T4X, T54);
1697 T5k = VADD(T5c, T5j);
1698 ST(&(ri[WS(rs, 21)]), VSUB(T55, T5k), ms, &(ri[WS(rs, 1)]));
1699 ST(&(ri[WS(rs, 5)]), VADD(T55, T5k), ms, &(ri[WS(rs, 1)]));
1700 T7H = VADD(T5m, T5n);
1701 T7K = VADD(T7I, T7J);
1702 ST(&(ii[WS(rs, 5)]), VADD(T7H, T7K), ms, &(ii[WS(rs, 1)]));
1703 ST(&(ii[WS(rs, 21)]), VSUB(T7K, T7H), ms, &(ii[WS(rs, 1)]));
1704 }
1705 {
1706 V T5l, T5o, T7L, T7M;
1707 T5l = VSUB(T4X, T54);
1708 T5o = VSUB(T5m, T5n);
1709 ST(&(ri[WS(rs, 29)]), VSUB(T5l, T5o), ms, &(ri[WS(rs, 1)]));
1710 ST(&(ri[WS(rs, 13)]), VADD(T5l, T5o), ms, &(ri[WS(rs, 1)]));
1711 T7L = VSUB(T5j, T5c);
1712 T7M = VSUB(T7J, T7I);
1713 ST(&(ii[WS(rs, 13)]), VADD(T7L, T7M), ms, &(ii[WS(rs, 1)]));
1714 ST(&(ii[WS(rs, 29)]), VSUB(T7M, T7L), ms, &(ii[WS(rs, 1)]));
1715 }
1716 {
1717 V T5t, T5A, T7x, T7E;
1718 T5t = VADD(T5p, T5s);
1719 T5A = VADD(T5w, T5z);
1720 ST(&(ri[WS(rs, 17)]), VSUB(T5t, T5A), ms, &(ri[WS(rs, 1)]));
1721 ST(&(ri[WS(rs, 1)]), VADD(T5t, T5A), ms, &(ri[WS(rs, 1)]));
1722 T7x = VADD(T5C, T5D);
1723 T7E = VADD(T7y, T7D);
1724 ST(&(ii[WS(rs, 1)]), VADD(T7x, T7E), ms, &(ii[WS(rs, 1)]));
1725 ST(&(ii[WS(rs, 17)]), VSUB(T7E, T7x), ms, &(ii[WS(rs, 1)]));
1726 }
1727 {
1728 V T5B, T5E, T7F, T7G;
1729 T5B = VSUB(T5p, T5s);
1730 T5E = VSUB(T5C, T5D);
1731 ST(&(ri[WS(rs, 25)]), VSUB(T5B, T5E), ms, &(ri[WS(rs, 1)]));
1732 ST(&(ri[WS(rs, 9)]), VADD(T5B, T5E), ms, &(ri[WS(rs, 1)]));
1733 T7F = VSUB(T5z, T5w);
1734 T7G = VSUB(T7D, T7y);
1735 ST(&(ii[WS(rs, 9)]), VADD(T7F, T7G), ms, &(ii[WS(rs, 1)]));
1736 ST(&(ii[WS(rs, 25)]), VSUB(T7G, T7F), ms, &(ii[WS(rs, 1)]));
1737 }
1738 }
1739 }
1740 }
1741 VLEAVE();
1742 }
1743
1744 static const tw_instr twinstr[] = {
1745 VTW(0, 1),
1746 VTW(0, 2),
1747 VTW(0, 3),
1748 VTW(0, 4),
1749 VTW(0, 5),
1750 VTW(0, 6),
1751 VTW(0, 7),
1752 VTW(0, 8),
1753 VTW(0, 9),
1754 VTW(0, 10),
1755 VTW(0, 11),
1756 VTW(0, 12),
1757 VTW(0, 13),
1758 VTW(0, 14),
1759 VTW(0, 15),
1760 VTW(0, 16),
1761 VTW(0, 17),
1762 VTW(0, 18),
1763 VTW(0, 19),
1764 VTW(0, 20),
1765 VTW(0, 21),
1766 VTW(0, 22),
1767 VTW(0, 23),
1768 VTW(0, 24),
1769 VTW(0, 25),
1770 VTW(0, 26),
1771 VTW(0, 27),
1772 VTW(0, 28),
1773 VTW(0, 29),
1774 VTW(0, 30),
1775 VTW(0, 31),
1776 {TW_NEXT, (2 * VL), 0}
1777 };
1778
1779 static const ct_desc desc = { 32, XSIMD_STRING("t1sv_32"), twinstr, &GENUS, {340, 114, 94, 0}, 0, 0, 0 };
1780
1781 void XSIMD(codelet_t1sv_32) (planner *p) {
1782 X(kdft_dit_register) (p, t1sv_32, &desc);
1783 }
1784 #endif /* HAVE_FMA */