Chris@10: /*
Chris@10:  * Copyright (c) 2003, 2007-11 Matteo Frigo
Chris@10:  * Copyright (c) 2003, 2007-11 Massachusetts Institute of Technology
Chris@10:  *
Chris@10:  * This program is free software; you can redistribute it and/or modify
Chris@10:  * it under the terms of the GNU General Public License as published by
Chris@10:  * the Free Software Foundation; either version 2 of the License, or
Chris@10:  * (at your option) any later version.
Chris@10:  *
Chris@10:  * This program is distributed in the hope that it will be useful,
Chris@10:  * but WITHOUT ANY WARRANTY; without even the implied warranty of
Chris@10:  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
Chris@10:  * GNU General Public License for more details.
Chris@10:  *
Chris@10:  * You should have received a copy of the GNU General Public License
Chris@10:  * along with this program; if not, write to the Free Software
Chris@10:  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
Chris@10:  *
Chris@10:  */
Chris@10: 
Chris@10: /* out of place 2D copy routines */
Chris@10: #include "ifftw.h"
Chris@10: 
Chris@10: #if defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64)
Chris@10: #  ifdef HAVE_XMMINTRIN_H
Chris@10: #    include <xmmintrin.h>
Chris@10: #    define WIDE_TYPE __m128
Chris@10: #  endif
Chris@10: #endif
Chris@10: 
Chris@10: #ifndef WIDE_TYPE
Chris@10: /* fall back to double, which means that WIDE_TYPE will be unused */
Chris@10: #  define WIDE_TYPE double
Chris@10: #endif
Chris@10: 
Chris@10: void X(cpy2d)(R *I, R *O,
Chris@10: 	      INT n0, INT is0, INT os0,
Chris@10: 	      INT n1, INT is1, INT os1,
Chris@10: 	      INT vl)
Chris@10: {
Chris@10:      INT i0, i1, v;
Chris@10: 
Chris@10:      switch (vl) {
Chris@10: 	 case 1:
Chris@10: 	      for (i1 = 0; i1 < n1; ++i1)
Chris@10: 		   for (i0 = 0; i0 < n0; ++i0) {
Chris@10: 			R x0 = I[i0 * is0 + i1 * is1];
Chris@10: 			O[i0 * os0 + i1 * os1] = x0;
Chris@10: 		   }
Chris@10: 	      break;
Chris@10: 	 case 2:
Chris@10: 	      if (1
Chris@10: 		  && (2 * sizeof(R) == sizeof(WIDE_TYPE))
Chris@10: 		  && (sizeof(WIDE_TYPE) > sizeof(double))
Chris@10: 		  && (((size_t)I) % sizeof(WIDE_TYPE) == 0)
Chris@10: 		  && (((size_t)O) % sizeof(WIDE_TYPE) == 0)
Chris@10: 		  && ((is0 & 1) == 0)
Chris@10: 		  && ((is1 & 1) == 0)
Chris@10: 		  && ((os0 & 1) == 0)
Chris@10: 		  && ((os1 & 1) == 0)) {
Chris@10: 		   /* copy R[2] as WIDE_TYPE if WIDE_TYPE is large
Chris@10: 		      enough to hold R[2], and if the input is
Chris@10: 		      properly aligned.  This is a win when R==double
Chris@10: 		      and WIDE_TYPE is 128 bits. */
Chris@10: 		   for (i1 = 0; i1 < n1; ++i1)
Chris@10: 			for (i0 = 0; i0 < n0; ++i0) {
Chris@10: 			     *(WIDE_TYPE *)&O[i0 * os0 + i1 * os1] =
Chris@10: 				  *(WIDE_TYPE *)&I[i0 * is0 + i1 * is1];
Chris@10: 			}
Chris@10: 	      } else if (1
Chris@10: 		  && (2 * sizeof(R) == sizeof(double))
Chris@10: 		  && (((size_t)I) % sizeof(double) == 0)
Chris@10: 		  && (((size_t)O) % sizeof(double) == 0)
Chris@10: 		  && ((is0 & 1) == 0)
Chris@10: 		  && ((is1 & 1) == 0)
Chris@10: 		  && ((os0 & 1) == 0)
Chris@10: 		  && ((os1 & 1) == 0)) {
Chris@10: 		   /* copy R[2] as double if double is large enough to
Chris@10: 		      hold R[2], and if the input is properly aligned.
Chris@10: 		      This case applies when R==float */
Chris@10: 		   for (i1 = 0; i1 < n1; ++i1)
Chris@10: 			for (i0 = 0; i0 < n0; ++i0) {
Chris@10: 			     *(double *)&O[i0 * os0 + i1 * os1] =
Chris@10: 				  *(double *)&I[i0 * is0 + i1 * is1];
Chris@10: 			}
Chris@10: 	      } else {
Chris@10: 		   for (i1 = 0; i1 < n1; ++i1)
Chris@10: 			for (i0 = 0; i0 < n0; ++i0) {
Chris@10: 			     R x0 = I[i0 * is0 + i1 * is1];
Chris@10: 			     R x1 = I[i0 * is0 + i1 * is1 + 1];
Chris@10: 			     O[i0 * os0 + i1 * os1] = x0;
Chris@10:  			     O[i0 * os0 + i1 * os1 + 1] = x1;
Chris@10: 			}
Chris@10: 	      }
Chris@10: 	      break;
Chris@10: 	 default:
Chris@10: 	      for (i1 = 0; i1 < n1; ++i1)
Chris@10: 		   for (i0 = 0; i0 < n0; ++i0)
Chris@10: 			for (v = 0; v < vl; ++v) {
Chris@10: 			     R x0 = I[i0 * is0 + i1 * is1 + v];
Chris@10: 			     O[i0 * os0 + i1 * os1 + v] = x0;
Chris@10: 			}
Chris@10: 	      break;
Chris@10:      }
Chris@10: }
Chris@10: 
Chris@10: /* like cpy2d, but read input contiguously if possible */
Chris@10: void X(cpy2d_ci)(R *I, R *O,
Chris@10: 		 INT n0, INT is0, INT os0,
Chris@10: 		 INT n1, INT is1, INT os1,
Chris@10: 		 INT vl)
Chris@10: {
Chris@10:      if (IABS(is0) < IABS(is1))	/* inner loop is for n0 */
Chris@10: 	  X(cpy2d) (I, O, n0, is0, os0, n1, is1, os1, vl);
Chris@10:      else
Chris@10: 	  X(cpy2d) (I, O, n1, is1, os1, n0, is0, os0, vl);
Chris@10: }
Chris@10: 
Chris@10: /* like cpy2d, but write output contiguously if possible */
Chris@10: void X(cpy2d_co)(R *I, R *O,
Chris@10: 		 INT n0, INT is0, INT os0,
Chris@10: 		 INT n1, INT is1, INT os1,
Chris@10: 		 INT vl)
Chris@10: {
Chris@10:      if (IABS(os0) < IABS(os1))	/* inner loop is for n0 */
Chris@10: 	  X(cpy2d) (I, O, n0, is0, os0, n1, is1, os1, vl);
Chris@10:      else
Chris@10: 	  X(cpy2d) (I, O, n1, is1, os1, n0, is0, os0, vl);
Chris@10: }
Chris@10: 
Chris@10: 
Chris@10: /* tiled copy routines */
Chris@10: struct cpy2d_closure {
Chris@10:      R *I, *O;
Chris@10:      INT is0, os0, is1, os1, vl;
Chris@10:      R *buf;
Chris@10: };
Chris@10: 
Chris@10: static void dotile(INT n0l, INT n0u, INT n1l, INT n1u, void *args)
Chris@10: {
Chris@10:      struct cpy2d_closure *k = (struct cpy2d_closure *)args;
Chris@10:      X(cpy2d)(k->I + n0l * k->is0 + n1l * k->is1,
Chris@10: 	      k->O + n0l * k->os0 + n1l * k->os1,
Chris@10: 	      n0u - n0l, k->is0, k->os0,
Chris@10: 	      n1u - n1l, k->is1, k->os1,
Chris@10: 	      k->vl);
Chris@10: }
Chris@10: 
Chris@10: static void dotile_buf(INT n0l, INT n0u, INT n1l, INT n1u, void *args)
Chris@10: {
Chris@10:      struct cpy2d_closure *k = (struct cpy2d_closure *)args;
Chris@10: 
Chris@10:      /* copy from I to buf */
Chris@10:      X(cpy2d_ci)(k->I + n0l * k->is0 + n1l * k->is1,
Chris@10: 		 k->buf,
Chris@10: 		 n0u - n0l, k->is0, k->vl,
Chris@10: 		 n1u - n1l, k->is1, k->vl * (n0u - n0l),
Chris@10: 		 k->vl);
Chris@10: 
Chris@10:      /* copy from buf to O */
Chris@10:      X(cpy2d_co)(k->buf,
Chris@10: 		 k->O + n0l * k->os0 + n1l * k->os1,
Chris@10: 		 n0u - n0l, k->vl, k->os0,
Chris@10: 		 n1u - n1l, k->vl * (n0u - n0l), k->os1,
Chris@10: 		 k->vl);
Chris@10: }
Chris@10: 
Chris@10: 
Chris@10: void X(cpy2d_tiled)(R *I, R *O,
Chris@10: 		    INT n0, INT is0, INT os0,
Chris@10: 		    INT n1, INT is1, INT os1, INT vl)
Chris@10: {
Chris@10:      INT tilesz = X(compute_tilesz)(vl,
Chris@10: 				    1 /* input array */
Chris@10: 				    + 1 /* ouput array */);
Chris@10:      struct cpy2d_closure k;
Chris@10:      k.I = I;
Chris@10:      k.O = O;
Chris@10:      k.is0 = is0;
Chris@10:      k.os0 = os0;
Chris@10:      k.is1 = is1;
Chris@10:      k.os1 = os1;
Chris@10:      k.vl = vl;
Chris@10:      k.buf = 0; /* unused */
Chris@10:      X(tile2d)(0, n0, 0, n1, tilesz, dotile, &k);
Chris@10: }
Chris@10: 
Chris@10: void X(cpy2d_tiledbuf)(R *I, R *O,
Chris@10: 		       INT n0, INT is0, INT os0,
Chris@10: 		       INT n1, INT is1, INT os1, INT vl)
Chris@10: {
Chris@10:      R buf[CACHESIZE / (2 * sizeof(R))];
Chris@10:      /* input and buffer in cache, or
Chris@10: 	output and buffer in cache */
Chris@10:      INT tilesz = X(compute_tilesz)(vl, 2);
Chris@10:      struct cpy2d_closure k;
Chris@10:      k.I = I;
Chris@10:      k.O = O;
Chris@10:      k.is0 = is0;
Chris@10:      k.os0 = os0;
Chris@10:      k.is1 = is1;
Chris@10:      k.os1 = os1;
Chris@10:      k.vl = vl;
Chris@10:      k.buf = buf;
Chris@10:      A(tilesz * tilesz * vl * sizeof(R) <= sizeof(buf));
Chris@10:      X(tile2d)(0, n0, 0, n1, tilesz, dotile_buf, &k);
Chris@10: }