cannam@167: /* cannam@167: * Copyright (c) 2003, 2007-14 Matteo Frigo cannam@167: * Copyright (c) 2003, 2007-14 Massachusetts Institute of Technology cannam@167: * cannam@167: * This program is free software; you can redistribute it and/or modify cannam@167: * it under the terms of the GNU General Public License as published by cannam@167: * the Free Software Foundation; either version 2 of the License, or cannam@167: * (at your option) any later version. cannam@167: * cannam@167: * This program is distributed in the hope that it will be useful, cannam@167: * but WITHOUT ANY WARRANTY; without even the implied warranty of cannam@167: * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the cannam@167: * GNU General Public License for more details. cannam@167: * cannam@167: * You should have received a copy of the GNU General Public License cannam@167: * along with this program; if not, write to the Free Software cannam@167: * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA cannam@167: * cannam@167: */ cannam@167: cannam@167: /* out of place 2D copy routines */ cannam@167: #include "kernel/ifftw.h" cannam@167: cannam@167: #if defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64) cannam@167: # ifdef HAVE_XMMINTRIN_H cannam@167: # include cannam@167: # define WIDE_TYPE __m128 cannam@167: # endif cannam@167: #endif cannam@167: cannam@167: #ifndef WIDE_TYPE cannam@167: /* fall back to double, which means that WIDE_TYPE will be unused */ cannam@167: # define WIDE_TYPE double cannam@167: #endif cannam@167: cannam@167: void X(cpy2d)(R *I, R *O, cannam@167: INT n0, INT is0, INT os0, cannam@167: INT n1, INT is1, INT os1, cannam@167: INT vl) cannam@167: { cannam@167: INT i0, i1, v; cannam@167: cannam@167: switch (vl) { cannam@167: case 1: cannam@167: for (i1 = 0; i1 < n1; ++i1) cannam@167: for (i0 = 0; i0 < n0; ++i0) { cannam@167: R x0 = I[i0 * is0 + i1 * is1]; cannam@167: O[i0 * os0 + i1 * os1] = x0; cannam@167: } cannam@167: break; cannam@167: case 2: cannam@167: if (1 cannam@167: && (2 * sizeof(R) == sizeof(WIDE_TYPE)) cannam@167: && (sizeof(WIDE_TYPE) > sizeof(double)) cannam@167: && (((size_t)I) % sizeof(WIDE_TYPE) == 0) cannam@167: && (((size_t)O) % sizeof(WIDE_TYPE) == 0) cannam@167: && ((is0 & 1) == 0) cannam@167: && ((is1 & 1) == 0) cannam@167: && ((os0 & 1) == 0) cannam@167: && ((os1 & 1) == 0)) { cannam@167: /* copy R[2] as WIDE_TYPE if WIDE_TYPE is large cannam@167: enough to hold R[2], and if the input is cannam@167: properly aligned. This is a win when R==double cannam@167: and WIDE_TYPE is 128 bits. */ cannam@167: for (i1 = 0; i1 < n1; ++i1) cannam@167: for (i0 = 0; i0 < n0; ++i0) { cannam@167: *(WIDE_TYPE *)&O[i0 * os0 + i1 * os1] = cannam@167: *(WIDE_TYPE *)&I[i0 * is0 + i1 * is1]; cannam@167: } cannam@167: } else if (1 cannam@167: && (2 * sizeof(R) == sizeof(double)) cannam@167: && (((size_t)I) % sizeof(double) == 0) cannam@167: && (((size_t)O) % sizeof(double) == 0) cannam@167: && ((is0 & 1) == 0) cannam@167: && ((is1 & 1) == 0) cannam@167: && ((os0 & 1) == 0) cannam@167: && ((os1 & 1) == 0)) { cannam@167: /* copy R[2] as double if double is large enough to cannam@167: hold R[2], and if the input is properly aligned. cannam@167: This case applies when R==float */ cannam@167: for (i1 = 0; i1 < n1; ++i1) cannam@167: for (i0 = 0; i0 < n0; ++i0) { cannam@167: *(double *)&O[i0 * os0 + i1 * os1] = cannam@167: *(double *)&I[i0 * is0 + i1 * is1]; cannam@167: } cannam@167: } else { cannam@167: for (i1 = 0; i1 < n1; ++i1) cannam@167: for (i0 = 0; i0 < n0; ++i0) { cannam@167: R x0 = I[i0 * is0 + i1 * is1]; cannam@167: R x1 = I[i0 * is0 + i1 * is1 + 1]; cannam@167: O[i0 * os0 + i1 * os1] = x0; cannam@167: O[i0 * os0 + i1 * os1 + 1] = x1; cannam@167: } cannam@167: } cannam@167: break; cannam@167: default: cannam@167: for (i1 = 0; i1 < n1; ++i1) cannam@167: for (i0 = 0; i0 < n0; ++i0) cannam@167: for (v = 0; v < vl; ++v) { cannam@167: R x0 = I[i0 * is0 + i1 * is1 + v]; cannam@167: O[i0 * os0 + i1 * os1 + v] = x0; cannam@167: } cannam@167: break; cannam@167: } cannam@167: } cannam@167: cannam@167: /* like cpy2d, but read input contiguously if possible */ cannam@167: void X(cpy2d_ci)(R *I, R *O, cannam@167: INT n0, INT is0, INT os0, cannam@167: INT n1, INT is1, INT os1, cannam@167: INT vl) cannam@167: { cannam@167: if (IABS(is0) < IABS(is1)) /* inner loop is for n0 */ cannam@167: X(cpy2d) (I, O, n0, is0, os0, n1, is1, os1, vl); cannam@167: else cannam@167: X(cpy2d) (I, O, n1, is1, os1, n0, is0, os0, vl); cannam@167: } cannam@167: cannam@167: /* like cpy2d, but write output contiguously if possible */ cannam@167: void X(cpy2d_co)(R *I, R *O, cannam@167: INT n0, INT is0, INT os0, cannam@167: INT n1, INT is1, INT os1, cannam@167: INT vl) cannam@167: { cannam@167: if (IABS(os0) < IABS(os1)) /* inner loop is for n0 */ cannam@167: X(cpy2d) (I, O, n0, is0, os0, n1, is1, os1, vl); cannam@167: else cannam@167: X(cpy2d) (I, O, n1, is1, os1, n0, is0, os0, vl); cannam@167: } cannam@167: cannam@167: cannam@167: /* tiled copy routines */ cannam@167: struct cpy2d_closure { cannam@167: R *I, *O; cannam@167: INT is0, os0, is1, os1, vl; cannam@167: R *buf; cannam@167: }; cannam@167: cannam@167: static void dotile(INT n0l, INT n0u, INT n1l, INT n1u, void *args) cannam@167: { cannam@167: struct cpy2d_closure *k = (struct cpy2d_closure *)args; cannam@167: X(cpy2d)(k->I + n0l * k->is0 + n1l * k->is1, cannam@167: k->O + n0l * k->os0 + n1l * k->os1, cannam@167: n0u - n0l, k->is0, k->os0, cannam@167: n1u - n1l, k->is1, k->os1, cannam@167: k->vl); cannam@167: } cannam@167: cannam@167: static void dotile_buf(INT n0l, INT n0u, INT n1l, INT n1u, void *args) cannam@167: { cannam@167: struct cpy2d_closure *k = (struct cpy2d_closure *)args; cannam@167: cannam@167: /* copy from I to buf */ cannam@167: X(cpy2d_ci)(k->I + n0l * k->is0 + n1l * k->is1, cannam@167: k->buf, cannam@167: n0u - n0l, k->is0, k->vl, cannam@167: n1u - n1l, k->is1, k->vl * (n0u - n0l), cannam@167: k->vl); cannam@167: cannam@167: /* copy from buf to O */ cannam@167: X(cpy2d_co)(k->buf, cannam@167: k->O + n0l * k->os0 + n1l * k->os1, cannam@167: n0u - n0l, k->vl, k->os0, cannam@167: n1u - n1l, k->vl * (n0u - n0l), k->os1, cannam@167: k->vl); cannam@167: } cannam@167: cannam@167: cannam@167: void X(cpy2d_tiled)(R *I, R *O, cannam@167: INT n0, INT is0, INT os0, cannam@167: INT n1, INT is1, INT os1, INT vl) cannam@167: { cannam@167: INT tilesz = X(compute_tilesz)(vl, cannam@167: 1 /* input array */ cannam@167: + 1 /* ouput array */); cannam@167: struct cpy2d_closure k; cannam@167: k.I = I; cannam@167: k.O = O; cannam@167: k.is0 = is0; cannam@167: k.os0 = os0; cannam@167: k.is1 = is1; cannam@167: k.os1 = os1; cannam@167: k.vl = vl; cannam@167: k.buf = 0; /* unused */ cannam@167: X(tile2d)(0, n0, 0, n1, tilesz, dotile, &k); cannam@167: } cannam@167: cannam@167: void X(cpy2d_tiledbuf)(R *I, R *O, cannam@167: INT n0, INT is0, INT os0, cannam@167: INT n1, INT is1, INT os1, INT vl) cannam@167: { cannam@167: R buf[CACHESIZE / (2 * sizeof(R))]; cannam@167: /* input and buffer in cache, or cannam@167: output and buffer in cache */ cannam@167: INT tilesz = X(compute_tilesz)(vl, 2); cannam@167: struct cpy2d_closure k; cannam@167: k.I = I; cannam@167: k.O = O; cannam@167: k.is0 = is0; cannam@167: k.os0 = os0; cannam@167: k.is1 = is1; cannam@167: k.os1 = os1; cannam@167: k.vl = vl; cannam@167: k.buf = buf; cannam@167: A(tilesz * tilesz * vl * sizeof(R) <= sizeof(buf)); cannam@167: X(tile2d)(0, n0, 0, n1, tilesz, dotile_buf, &k); cannam@167: }