cannam@167: /*
cannam@167:  * Copyright (c) 2003, 2007-14 Matteo Frigo
cannam@167:  * Copyright (c) 2003, 2007-14 Massachusetts Institute of Technology
cannam@167:  *
cannam@167:  * This program is free software; you can redistribute it and/or modify
cannam@167:  * it under the terms of the GNU General Public License as published by
cannam@167:  * the Free Software Foundation; either version 2 of the License, or
cannam@167:  * (at your option) any later version.
cannam@167:  *
cannam@167:  * This program is distributed in the hope that it will be useful,
cannam@167:  * but WITHOUT ANY WARRANTY; without even the implied warranty of
cannam@167:  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
cannam@167:  * GNU General Public License for more details.
cannam@167:  *
cannam@167:  * You should have received a copy of the GNU General Public License
cannam@167:  * along with this program; if not, write to the Free Software
cannam@167:  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
cannam@167:  *
cannam@167:  */
cannam@167: 
cannam@167: #include "api/api.h"
cannam@167: #include "fftw3-mpi.h"
cannam@167: #include "ifftw-mpi.h"
cannam@167: #include "mpi-transpose.h"
cannam@167: #include "mpi-dft.h"
cannam@167: #include "mpi-rdft.h"
cannam@167: #include "mpi-rdft2.h"
cannam@167: 
cannam@167: /* Convert API flags to internal MPI flags. */
cannam@167: #define MPI_FLAGS(f) ((f) >> 27)
cannam@167: 
cannam@167: /*************************************************************************/
cannam@167: 
cannam@167: static int mpi_inited = 0;
cannam@167: 
cannam@167: static MPI_Comm problem_comm(const problem *p) {
cannam@167:      switch (p->adt->problem_kind) {
cannam@167: 	 case PROBLEM_MPI_DFT:
cannam@167: 	      return ((const problem_mpi_dft *) p)->comm;
cannam@167: 	 case PROBLEM_MPI_RDFT:
cannam@167: 	      return ((const problem_mpi_rdft *) p)->comm;
cannam@167: 	 case PROBLEM_MPI_RDFT2:
cannam@167: 	      return ((const problem_mpi_rdft2 *) p)->comm;
cannam@167: 	 case PROBLEM_MPI_TRANSPOSE:
cannam@167: 	      return ((const problem_mpi_transpose *) p)->comm;
cannam@167: 	 default:
cannam@167: 	      return MPI_COMM_NULL;
cannam@167:      }
cannam@167: }
cannam@167: 
cannam@167: /* used to synchronize cost measurements (timing or estimation)
cannam@167:    across all processes for an MPI problem, which is critical to
cannam@167:    ensure that all processes decide to use the same MPI plans
cannam@167:    (whereas serial plans need not be syncronized). */
cannam@167: static double cost_hook(const problem *p, double t, cost_kind k)
cannam@167: {
cannam@167:      MPI_Comm comm = problem_comm(p);
cannam@167:      double tsum;
cannam@167:      if (comm == MPI_COMM_NULL) return t;
cannam@167:      MPI_Allreduce(&t, &tsum, 1, MPI_DOUBLE, 
cannam@167: 		   k == COST_SUM ? MPI_SUM : MPI_MAX, comm);
cannam@167:      return tsum;
cannam@167: }
cannam@167: 
cannam@167: /* Used to reject wisdom that is not in sync across all processes
cannam@167:    for an MPI problem, which is critical to ensure that all processes
cannam@167:    decide to use the same MPI plans.  (Even though costs are synchronized,
cannam@167:    above, out-of-sync wisdom may result from plans being produced
cannam@167:    by communicators that do not span all processes, either from a
cannam@167:    user-specified communicator or e.g. from transpose-recurse. */
cannam@167: static int wisdom_ok_hook(const problem *p, flags_t flags)
cannam@167: {
cannam@167:      MPI_Comm comm = problem_comm(p);
cannam@167:      int eq_me, eq_all;
cannam@167:      /* unpack flags bitfield, since MPI communications may involve
cannam@167: 	byte-order changes and MPI cannot do this for bit fields */
cannam@167: #if SIZEOF_UNSIGNED_INT >= 4 /* must be big enough to hold 20-bit fields */
cannam@167:      unsigned int f[5];
cannam@167: #else
cannam@167:      unsigned long f[5]; /* at least 32 bits as per C standard */
cannam@167: #endif
cannam@167: 
cannam@167:      if (comm == MPI_COMM_NULL) return 1; /* non-MPI wisdom is always ok */
cannam@167: 
cannam@167:      if (XM(any_true)(0, comm)) return 0; /* some process had nowisdom_hook */
cannam@167: 
cannam@167:      /* otherwise, check that the flags and solver index are identical
cannam@167: 	on all processes in this problem's communicator.
cannam@167: 
cannam@167: 	TO DO: possibly we can relax strict equality, but it is
cannam@167: 	critical to ensure that any flags which affect what plan is
cannam@167: 	created (and whether the solver is applicable) are the same,
cannam@167: 	e.g. DESTROY_INPUT, NO_UGLY, etcetera.  (If the MPI algorithm
cannam@167: 	differs between processes, deadlocks/crashes generally result.) */
cannam@167:      f[0] = flags.l;
cannam@167:      f[1] = flags.hash_info;
cannam@167:      f[2] = flags.timelimit_impatience;
cannam@167:      f[3] = flags.u;
cannam@167:      f[4] = flags.slvndx;
cannam@167:      MPI_Bcast(f, 5, 
cannam@167: 	       SIZEOF_UNSIGNED_INT >= 4 ? MPI_UNSIGNED : MPI_UNSIGNED_LONG,
cannam@167: 	       0, comm);
cannam@167:      eq_me = f[0] == flags.l && f[1] == flags.hash_info
cannam@167: 	  && f[2] == flags.timelimit_impatience
cannam@167: 	  && f[3] == flags.u && f[4] == flags.slvndx;
cannam@167:      MPI_Allreduce(&eq_me, &eq_all, 1, MPI_INT, MPI_LAND, comm);
cannam@167:      return eq_all;
cannam@167: }
cannam@167: 
cannam@167: /* This hook is called when wisdom is not found.  The any_true here
cannam@167:    matches up with the any_true in wisdom_ok_hook, in order to handle
cannam@167:    the case where some processes had wisdom (and called wisdom_ok_hook)
cannam@167:    and some processes didn't have wisdom (and called nowisdom_hook). */
cannam@167: static void nowisdom_hook(const problem *p)
cannam@167: {
cannam@167:      MPI_Comm comm = problem_comm(p);
cannam@167:      if (comm == MPI_COMM_NULL) return; /* nothing to do for non-MPI p */
cannam@167:      XM(any_true)(1, comm); /* signal nowisdom to any wisdom_ok_hook */
cannam@167: }
cannam@167: 
cannam@167: /* needed to synchronize planner bogosity flag, in case non-MPI problems
cannam@167:    on a subset of processes encountered bogus wisdom */
cannam@167: static wisdom_state_t bogosity_hook(wisdom_state_t state, const problem *p)
cannam@167: {
cannam@167:      MPI_Comm comm = problem_comm(p);
cannam@167:      if (comm != MPI_COMM_NULL /* an MPI problem */
cannam@167: 	 && XM(any_true)(state == WISDOM_IS_BOGUS, comm)) /* bogus somewhere */
cannam@167: 	  return WISDOM_IS_BOGUS;
cannam@167:      return state;
cannam@167: }
cannam@167: 
cannam@167: void XM(init)(void)
cannam@167: {
cannam@167:      if (!mpi_inited) {
cannam@167: 	  planner *plnr = X(the_planner)();
cannam@167: 	  plnr->cost_hook = cost_hook;
cannam@167: 	  plnr->wisdom_ok_hook = wisdom_ok_hook;
cannam@167: 	  plnr->nowisdom_hook = nowisdom_hook;
cannam@167: 	  plnr->bogosity_hook = bogosity_hook;
cannam@167:           XM(conf_standard)(plnr);
cannam@167: 	  mpi_inited = 1;	  
cannam@167:      }
cannam@167: }
cannam@167: 
cannam@167: void XM(cleanup)(void)
cannam@167: {
cannam@167:      X(cleanup)();
cannam@167:      mpi_inited = 0;
cannam@167: }
cannam@167: 
cannam@167: /*************************************************************************/
cannam@167: 
cannam@167: static dtensor *mkdtensor_api(int rnk, const XM(ddim) *dims0)
cannam@167: {
cannam@167:      dtensor *x = XM(mkdtensor)(rnk);
cannam@167:      int i;
cannam@167:      for (i = 0; i < rnk; ++i) {
cannam@167: 	  x->dims[i].n = dims0[i].n;
cannam@167: 	  x->dims[i].b[IB] = dims0[i].ib;
cannam@167: 	  x->dims[i].b[OB] = dims0[i].ob;
cannam@167:      }
cannam@167:      return x;
cannam@167: }
cannam@167: 
cannam@167: static dtensor *default_sz(int rnk, const XM(ddim) *dims0, int n_pes,
cannam@167: 			   int rdft2)
cannam@167: {
cannam@167:      dtensor *sz = XM(mkdtensor)(rnk);
cannam@167:      dtensor *sz0 = mkdtensor_api(rnk, dims0);
cannam@167:      block_kind k;
cannam@167:      int i;
cannam@167: 
cannam@167:      for (i = 0; i < rnk; ++i)
cannam@167: 	  sz->dims[i].n = dims0[i].n;
cannam@167: 
cannam@167:      if (rdft2) sz->dims[rnk-1].n = dims0[rnk-1].n / 2 + 1;
cannam@167: 
cannam@167:      for (i = 0; i < rnk; ++i) {
cannam@167: 	  sz->dims[i].b[IB] = dims0[i].ib ? dims0[i].ib : sz->dims[i].n;
cannam@167: 	  sz->dims[i].b[OB] = dims0[i].ob ? dims0[i].ob : sz->dims[i].n;
cannam@167:      }
cannam@167: 
cannam@167:      /* If we haven't used all of the processes yet, and some of the
cannam@167: 	block sizes weren't specified (i.e. 0), then set the
cannam@167: 	unspecified blocks so as to use as many processes as
cannam@167: 	possible with as few distributed dimensions as possible. */
cannam@167:      FORALL_BLOCK_KIND(k) {
cannam@167: 	  INT nb = XM(num_blocks_total)(sz, k);
cannam@167: 	  INT np = n_pes / nb;
cannam@167: 	  for (i = 0; i < rnk && np > 1; ++i)
cannam@167: 	       if (!sz0->dims[i].b[k]) {
cannam@167: 		    sz->dims[i].b[k] = XM(default_block)(sz->dims[i].n, np);
cannam@167: 		    nb *= XM(num_blocks)(sz->dims[i].n, sz->dims[i].b[k]);
cannam@167: 		    np = n_pes / nb;
cannam@167: 	       }
cannam@167:      }
cannam@167: 
cannam@167:      if (rdft2) sz->dims[rnk-1].n = dims0[rnk-1].n;
cannam@167: 
cannam@167:      /* punt for 1d prime */
cannam@167:      if (rnk == 1 && X(is_prime)(sz->dims[0].n))
cannam@167: 	  sz->dims[0].b[IB] = sz->dims[0].b[OB] = sz->dims[0].n;
cannam@167: 
cannam@167:      XM(dtensor_destroy)(sz0);
cannam@167:      sz0 = XM(dtensor_canonical)(sz, 0);
cannam@167:      XM(dtensor_destroy)(sz);
cannam@167:      return sz0;
cannam@167: }
cannam@167: 
cannam@167: /* allocate simple local (serial) dims array corresponding to n[rnk] */
cannam@167: static XM(ddim) *simple_dims(int rnk, const ptrdiff_t *n)
cannam@167: {
cannam@167:      XM(ddim) *dims = (XM(ddim) *) MALLOC(sizeof(XM(ddim)) * rnk,
cannam@167: 						TENSORS);
cannam@167:      int i;
cannam@167:      for (i = 0; i < rnk; ++i)
cannam@167: 	  dims[i].n = dims[i].ib = dims[i].ob = n[i];
cannam@167:      return dims;
cannam@167: }
cannam@167: 
cannam@167: /*************************************************************************/
cannam@167: 
cannam@167: static void local_size(int my_pe, const dtensor *sz, block_kind k,
cannam@167: 		       ptrdiff_t *local_n, ptrdiff_t *local_start)
cannam@167: {
cannam@167:      int i;
cannam@167:      if (my_pe >= XM(num_blocks_total)(sz, k))
cannam@167: 	  for (i = 0; i < sz->rnk; ++i)
cannam@167: 	       local_n[i] = local_start[i] = 0;
cannam@167:      else {
cannam@167: 	  XM(block_coords)(sz, k, my_pe, local_start);
cannam@167: 	  for (i = 0; i < sz->rnk; ++i) {
cannam@167: 	       local_n[i] = XM(block)(sz->dims[i].n, sz->dims[i].b[k],
cannam@167: 				      local_start[i]);
cannam@167: 	       local_start[i] *= sz->dims[i].b[k];
cannam@167: 	  }
cannam@167:      }
cannam@167: }
cannam@167: 
cannam@167: static INT prod(int rnk, const ptrdiff_t *local_n) 
cannam@167: {
cannam@167:      int i;
cannam@167:      INT N = 1;
cannam@167:      for (i = 0; i < rnk; ++i) N *= local_n[i];
cannam@167:      return N;
cannam@167: }
cannam@167: 
cannam@167: ptrdiff_t XM(local_size_guru)(int rnk, const XM(ddim) *dims0,
cannam@167: 			      ptrdiff_t howmany, MPI_Comm comm,
cannam@167: 			      ptrdiff_t *local_n_in,
cannam@167: 			      ptrdiff_t *local_start_in,
cannam@167: 			      ptrdiff_t *local_n_out, 
cannam@167: 			      ptrdiff_t *local_start_out,
cannam@167: 			      int sign, unsigned flags)
cannam@167: {
cannam@167:      INT N;
cannam@167:      int my_pe, n_pes, i;
cannam@167:      dtensor *sz;
cannam@167: 
cannam@167:      if (rnk == 0)
cannam@167: 	  return howmany;
cannam@167: 
cannam@167:      MPI_Comm_rank(comm, &my_pe);
cannam@167:      MPI_Comm_size(comm, &n_pes);
cannam@167:      sz = default_sz(rnk, dims0, n_pes, 0);
cannam@167: 
cannam@167:      /* Now, we must figure out how much local space the user should
cannam@167: 	allocate (or at least an upper bound).  This depends strongly
cannam@167: 	on the exact algorithms we employ...ugh!  FIXME: get this info
cannam@167: 	from the solvers somehow? */
cannam@167:      N = 1; /* never return zero allocation size */
cannam@167:      if (rnk > 1 && XM(is_block1d)(sz, IB) && XM(is_block1d)(sz, OB)) {
cannam@167: 	  INT Nafter;
cannam@167: 	  ddim odims[2];
cannam@167: 
cannam@167: 	  /* dft-rank-geq2-transposed */
cannam@167: 	  odims[0] = sz->dims[0]; odims[1] = sz->dims[1]; /* save */
cannam@167: 	  /* we may need extra space for transposed intermediate data */
cannam@167: 	  for (i = 0; i < 2; ++i)
cannam@167: 	       if (XM(num_blocks)(sz->dims[i].n, sz->dims[i].b[IB]) == 1 &&
cannam@167: 		   XM(num_blocks)(sz->dims[i].n, sz->dims[i].b[OB]) == 1) {
cannam@167: 		    sz->dims[i].b[IB]
cannam@167: 			 = XM(default_block)(sz->dims[i].n, n_pes);
cannam@167: 		    sz->dims[1-i].b[IB] = sz->dims[1-i].n;
cannam@167: 		    local_size(my_pe, sz, IB, local_n_in, local_start_in);
cannam@167: 		    N = X(imax)(N, prod(rnk, local_n_in));
cannam@167: 		    sz->dims[i] = odims[i];
cannam@167: 		    sz->dims[1-i] = odims[1-i];
cannam@167: 		    break;
cannam@167: 	       }
cannam@167: 
cannam@167: 	  /* dft-rank-geq2 */
cannam@167: 	  Nafter = howmany;
cannam@167: 	  for (i = 1; i < sz->rnk; ++i) Nafter *= sz->dims[i].n;
cannam@167: 	  N = X(imax)(N, (sz->dims[0].n
cannam@167: 			  * XM(block)(Nafter, XM(default_block)(Nafter, n_pes),
cannam@167: 				      my_pe) + howmany - 1) / howmany);
cannam@167: 
cannam@167: 	  /* dft-rank-geq2 with dimensions swapped */
cannam@167: 	  Nafter = howmany * sz->dims[0].n;
cannam@167:           for (i = 2; i < sz->rnk; ++i) Nafter *= sz->dims[i].n;
cannam@167:           N = X(imax)(N, (sz->dims[1].n
cannam@167:                           * XM(block)(Nafter, XM(default_block)(Nafter, n_pes),
cannam@167:                                       my_pe) + howmany - 1) / howmany);
cannam@167:      }
cannam@167:      else if (rnk == 1) {
cannam@167: 	  if (howmany >= n_pes && !MPI_FLAGS(flags)) { /* dft-rank1-bigvec */
cannam@167: 	       ptrdiff_t n[2], start[2];
cannam@167: 	       dtensor *sz2 = XM(mkdtensor)(2);
cannam@167: 	       sz2->dims[0] = sz->dims[0];
cannam@167: 	       sz2->dims[0].b[IB] = sz->dims[0].n;
cannam@167: 	       sz2->dims[1].n = sz2->dims[1].b[OB] = howmany;
cannam@167: 	       sz2->dims[1].b[IB] = XM(default_block)(howmany, n_pes);
cannam@167: 	       local_size(my_pe, sz2, IB, n, start);
cannam@167: 	       XM(dtensor_destroy)(sz2);
cannam@167: 	       N = X(imax)(N, (prod(2, n) + howmany - 1) / howmany);
cannam@167: 	  }
cannam@167: 	  else { /* dft-rank1 */
cannam@167: 	       INT r, m, rblock[2], mblock[2];
cannam@167: 
cannam@167: 	       /* Since the 1d transforms are so different, we require
cannam@167: 		  the user to call local_size_1d for this case.  Ugh. */
cannam@167: 	       CK(sign == FFTW_FORWARD || sign == FFTW_BACKWARD);
cannam@167: 
cannam@167: 	       if ((r = XM(choose_radix)(sz->dims[0], n_pes, flags, sign,
cannam@167: 					 rblock, mblock))) {
cannam@167: 		    m = sz->dims[0].n / r;
cannam@167: 		    if (flags & FFTW_MPI_SCRAMBLED_IN)
cannam@167: 			 sz->dims[0].b[IB] = rblock[IB] * m;
cannam@167: 		    else { /* !SCRAMBLED_IN */
cannam@167: 			 sz->dims[0].b[IB] = r * mblock[IB];
cannam@167: 			 N = X(imax)(N, rblock[IB] * m);
cannam@167: 		    }
cannam@167: 		    if (flags & FFTW_MPI_SCRAMBLED_OUT)
cannam@167: 			 sz->dims[0].b[OB] = r * mblock[OB];
cannam@167: 		    else { /* !SCRAMBLED_OUT */
cannam@167: 			 N = X(imax)(N, r * mblock[OB]);
cannam@167: 			 sz->dims[0].b[OB] = rblock[OB] * m;
cannam@167: 		    }
cannam@167: 	       }
cannam@167: 	  }
cannam@167:      }
cannam@167: 
cannam@167:      local_size(my_pe, sz, IB, local_n_in, local_start_in);
cannam@167:      local_size(my_pe, sz, OB, local_n_out, local_start_out);
cannam@167: 
cannam@167:      /* at least, make sure we have enough space to store input & output */
cannam@167:      N = X(imax)(N, X(imax)(prod(rnk, local_n_in), prod(rnk, local_n_out)));
cannam@167: 
cannam@167:      XM(dtensor_destroy)(sz);
cannam@167:      return N * howmany;
cannam@167: }
cannam@167: 
cannam@167: ptrdiff_t XM(local_size_many_transposed)(int rnk, const ptrdiff_t *n,
cannam@167: 					 ptrdiff_t howmany,
cannam@167: 					 ptrdiff_t xblock, ptrdiff_t yblock,
cannam@167: 					 MPI_Comm comm,
cannam@167: 					 ptrdiff_t *local_nx,
cannam@167: 					 ptrdiff_t *local_x_start,
cannam@167: 					 ptrdiff_t *local_ny,
cannam@167: 					 ptrdiff_t *local_y_start)
cannam@167: {
cannam@167:      ptrdiff_t N;
cannam@167:      XM(ddim) *dims; 
cannam@167:      ptrdiff_t *local;
cannam@167: 
cannam@167:      if (rnk == 0) {
cannam@167: 	  *local_nx = *local_ny = 1;
cannam@167: 	  *local_x_start = *local_y_start = 0;
cannam@167: 	  return howmany;
cannam@167:      }
cannam@167: 
cannam@167:      dims = simple_dims(rnk, n);
cannam@167:      local = (ptrdiff_t *) MALLOC(sizeof(ptrdiff_t) * rnk * 4, TENSORS);
cannam@167: 
cannam@167:      /* default 1d block distribution, with transposed output
cannam@167:         if yblock < n[1] */
cannam@167:      dims[0].ib = xblock;
cannam@167:      if (rnk > 1) {
cannam@167: 	  if (yblock < n[1])
cannam@167: 	       dims[1].ob = yblock;
cannam@167: 	  else
cannam@167: 	       dims[0].ob = xblock;
cannam@167:      }
cannam@167:      else
cannam@167: 	  dims[0].ob = xblock; /* FIXME: 1d not really supported here 
cannam@167: 				         since we don't have flags/sign */
cannam@167:      
cannam@167:      N = XM(local_size_guru)(rnk, dims, howmany, comm, 
cannam@167: 			     local, local + rnk,
cannam@167: 			     local + 2*rnk, local + 3*rnk,
cannam@167: 			     0, 0);
cannam@167:      *local_nx = local[0];
cannam@167:      *local_x_start = local[rnk];
cannam@167:      if (rnk > 1) {
cannam@167: 	  *local_ny = local[2*rnk + 1];
cannam@167: 	  *local_y_start = local[3*rnk + 1];
cannam@167:      }
cannam@167:      else {
cannam@167: 	  *local_ny = *local_nx;
cannam@167: 	  *local_y_start = *local_x_start;
cannam@167:      }
cannam@167:      X(ifree)(local);
cannam@167:      X(ifree)(dims);
cannam@167:      return N;
cannam@167: }
cannam@167: 
cannam@167: ptrdiff_t XM(local_size_many)(int rnk, const ptrdiff_t *n,
cannam@167: 			      ptrdiff_t howmany, 
cannam@167: 			      ptrdiff_t xblock,
cannam@167: 			      MPI_Comm comm,
cannam@167: 			      ptrdiff_t *local_nx,
cannam@167: 			      ptrdiff_t *local_x_start)
cannam@167: {
cannam@167:      ptrdiff_t local_ny, local_y_start;
cannam@167:      return XM(local_size_many_transposed)(rnk, n, howmany,
cannam@167: 					   xblock, rnk > 1 
cannam@167: 					   ? n[1] : FFTW_MPI_DEFAULT_BLOCK,
cannam@167: 					   comm,
cannam@167: 					   local_nx, local_x_start,
cannam@167: 					   &local_ny, &local_y_start);
cannam@167: }
cannam@167: 
cannam@167: 
cannam@167: ptrdiff_t XM(local_size_transposed)(int rnk, const ptrdiff_t *n,
cannam@167: 				    MPI_Comm comm,
cannam@167: 				    ptrdiff_t *local_nx,
cannam@167: 				    ptrdiff_t *local_x_start,
cannam@167: 				    ptrdiff_t *local_ny,
cannam@167: 				    ptrdiff_t *local_y_start)
cannam@167: {
cannam@167:      return XM(local_size_many_transposed)(rnk, n, 1,
cannam@167: 					   FFTW_MPI_DEFAULT_BLOCK,
cannam@167: 					   FFTW_MPI_DEFAULT_BLOCK,
cannam@167: 					   comm,
cannam@167: 					   local_nx, local_x_start,
cannam@167: 					   local_ny, local_y_start);
cannam@167: }
cannam@167: 
cannam@167: ptrdiff_t XM(local_size)(int rnk, const ptrdiff_t *n,
cannam@167: 			 MPI_Comm comm,
cannam@167: 			 ptrdiff_t *local_nx,
cannam@167: 			 ptrdiff_t *local_x_start)
cannam@167: {
cannam@167:      return XM(local_size_many)(rnk, n, 1, FFTW_MPI_DEFAULT_BLOCK, comm,
cannam@167: 				local_nx, local_x_start);
cannam@167: }
cannam@167: 
cannam@167: ptrdiff_t XM(local_size_many_1d)(ptrdiff_t nx, ptrdiff_t howmany, 
cannam@167: 				 MPI_Comm comm, int sign, unsigned flags,
cannam@167: 				 ptrdiff_t *local_nx, ptrdiff_t *local_x_start,
cannam@167: 				 ptrdiff_t *local_ny, ptrdiff_t *local_y_start)
cannam@167: {
cannam@167:      XM(ddim) d;
cannam@167:      d.n = nx;
cannam@167:      d.ib = d.ob = FFTW_MPI_DEFAULT_BLOCK;
cannam@167:      return XM(local_size_guru)(1, &d, howmany, comm,
cannam@167: 				local_nx, local_x_start,
cannam@167: 				local_ny, local_y_start, sign, flags);
cannam@167: }
cannam@167: 
cannam@167: ptrdiff_t XM(local_size_1d)(ptrdiff_t nx,
cannam@167: 			    MPI_Comm comm, int sign, unsigned flags,
cannam@167: 			    ptrdiff_t *local_nx, ptrdiff_t *local_x_start,
cannam@167: 			    ptrdiff_t *local_ny, ptrdiff_t *local_y_start)
cannam@167: {
cannam@167:      return XM(local_size_many_1d)(nx, 1, comm, sign, flags,
cannam@167: 				   local_nx, local_x_start,
cannam@167: 				   local_ny, local_y_start);
cannam@167: }
cannam@167: 
cannam@167: ptrdiff_t XM(local_size_2d_transposed)(ptrdiff_t nx, ptrdiff_t ny,
cannam@167: 				       MPI_Comm comm,
cannam@167: 				       ptrdiff_t *local_nx,
cannam@167: 				       ptrdiff_t *local_x_start,
cannam@167: 				       ptrdiff_t *local_ny, 
cannam@167: 				       ptrdiff_t *local_y_start)
cannam@167: {
cannam@167:      ptrdiff_t n[2];
cannam@167:      n[0] = nx; n[1] = ny;
cannam@167:      return XM(local_size_transposed)(2, n, comm,
cannam@167: 				      local_nx, local_x_start,
cannam@167: 				      local_ny, local_y_start);
cannam@167: }
cannam@167: 
cannam@167: ptrdiff_t XM(local_size_2d)(ptrdiff_t nx, ptrdiff_t ny, MPI_Comm comm,
cannam@167: 			       ptrdiff_t *local_nx, ptrdiff_t *local_x_start)
cannam@167: {
cannam@167:      ptrdiff_t n[2];
cannam@167:      n[0] = nx; n[1] = ny;
cannam@167:      return XM(local_size)(2, n, comm, local_nx, local_x_start);
cannam@167: }
cannam@167: 
cannam@167: ptrdiff_t XM(local_size_3d_transposed)(ptrdiff_t nx, ptrdiff_t ny,
cannam@167: 				       ptrdiff_t nz,
cannam@167: 				       MPI_Comm comm,
cannam@167: 				       ptrdiff_t *local_nx,
cannam@167: 				       ptrdiff_t *local_x_start,
cannam@167: 				       ptrdiff_t *local_ny, 
cannam@167: 				       ptrdiff_t *local_y_start)
cannam@167: {
cannam@167:      ptrdiff_t n[3];
cannam@167:      n[0] = nx; n[1] = ny; n[2] = nz;
cannam@167:      return XM(local_size_transposed)(3, n, comm,
cannam@167: 				      local_nx, local_x_start,
cannam@167: 				      local_ny, local_y_start);
cannam@167: }
cannam@167: 
cannam@167: ptrdiff_t XM(local_size_3d)(ptrdiff_t nx, ptrdiff_t ny, ptrdiff_t nz,
cannam@167: 			    MPI_Comm comm,
cannam@167: 			    ptrdiff_t *local_nx, ptrdiff_t *local_x_start)
cannam@167: {
cannam@167:      ptrdiff_t n[3];
cannam@167:      n[0] = nx; n[1] = ny; n[2] = nz;
cannam@167:      return XM(local_size)(3, n, comm, local_nx, local_x_start);
cannam@167: }
cannam@167: 
cannam@167: /*************************************************************************/
cannam@167: /* Transpose API */
cannam@167: 
cannam@167: X(plan) XM(plan_many_transpose)(ptrdiff_t nx, ptrdiff_t ny, 
cannam@167: 				ptrdiff_t howmany,
cannam@167: 				ptrdiff_t xblock, ptrdiff_t yblock,
cannam@167: 				R *in, R *out, 
cannam@167: 				MPI_Comm comm, unsigned flags)
cannam@167: {
cannam@167:      int n_pes;
cannam@167:      XM(init)();
cannam@167: 
cannam@167:      if (howmany < 0 || xblock < 0 || yblock < 0 ||
cannam@167: 	 nx <= 0 || ny <= 0) return 0;
cannam@167: 
cannam@167:      MPI_Comm_size(comm, &n_pes);
cannam@167:      if (!xblock) xblock = XM(default_block)(nx, n_pes);
cannam@167:      if (!yblock) yblock = XM(default_block)(ny, n_pes);
cannam@167:      if (n_pes < XM(num_blocks)(nx, xblock)
cannam@167: 	 || n_pes < XM(num_blocks)(ny, yblock))
cannam@167: 	  return 0;
cannam@167: 
cannam@167:      return 
cannam@167: 	  X(mkapiplan)(FFTW_FORWARD, flags,
cannam@167: 		       XM(mkproblem_transpose)(nx, ny, howmany,
cannam@167: 					       in, out, xblock, yblock,
cannam@167: 					       comm, MPI_FLAGS(flags)));
cannam@167: }
cannam@167: 
cannam@167: X(plan) XM(plan_transpose)(ptrdiff_t nx, ptrdiff_t ny, R *in, R *out, 
cannam@167: 			   MPI_Comm comm, unsigned flags)
cannam@167: 			      
cannam@167: {
cannam@167:      return XM(plan_many_transpose)(nx, ny, 1,
cannam@167: 				    FFTW_MPI_DEFAULT_BLOCK,
cannam@167: 				    FFTW_MPI_DEFAULT_BLOCK,
cannam@167: 				    in, out, comm, flags);
cannam@167: }
cannam@167: 
cannam@167: /*************************************************************************/
cannam@167: /* Complex DFT API */
cannam@167: 
cannam@167: X(plan) XM(plan_guru_dft)(int rnk, const XM(ddim) *dims0,
cannam@167: 			  ptrdiff_t howmany,
cannam@167: 			  C *in, C *out,
cannam@167: 			  MPI_Comm comm, int sign, unsigned flags)
cannam@167: {
cannam@167:      int n_pes, i;
cannam@167:      dtensor *sz;
cannam@167:      
cannam@167:      XM(init)();
cannam@167: 
cannam@167:      if (howmany < 0 || rnk < 1) return 0;
cannam@167:      for (i = 0; i < rnk; ++i)
cannam@167: 	  if (dims0[i].n < 1 || dims0[i].ib < 0 || dims0[i].ob < 0)
cannam@167: 	       return 0;
cannam@167: 
cannam@167:      MPI_Comm_size(comm, &n_pes);
cannam@167:      sz = default_sz(rnk, dims0, n_pes, 0);
cannam@167: 
cannam@167:      if (XM(num_blocks_total)(sz, IB) > n_pes
cannam@167: 	 || XM(num_blocks_total)(sz, OB) > n_pes) {
cannam@167: 	  XM(dtensor_destroy)(sz);
cannam@167: 	  return 0;
cannam@167:      }
cannam@167: 
cannam@167:      return
cannam@167:           X(mkapiplan)(sign, flags,
cannam@167:                        XM(mkproblem_dft_d)(sz, howmany,
cannam@167: 					   (R *) in, (R *) out,
cannam@167: 					   comm, sign, 
cannam@167: 					   MPI_FLAGS(flags)));
cannam@167: }
cannam@167: 
cannam@167: X(plan) XM(plan_many_dft)(int rnk, const ptrdiff_t *n,
cannam@167: 			  ptrdiff_t howmany,
cannam@167: 			  ptrdiff_t iblock, ptrdiff_t oblock,
cannam@167: 			  C *in, C *out,
cannam@167: 			  MPI_Comm comm, int sign, unsigned flags)
cannam@167: {
cannam@167:      XM(ddim) *dims = simple_dims(rnk, n);
cannam@167:      X(plan) pln;
cannam@167: 
cannam@167:      if (rnk == 1) {
cannam@167: 	  dims[0].ib = iblock;
cannam@167: 	  dims[0].ob = oblock;
cannam@167:      }
cannam@167:      else if (rnk > 1) {
cannam@167: 	  dims[0 != (flags & FFTW_MPI_TRANSPOSED_IN)].ib = iblock;
cannam@167: 	  dims[0 != (flags & FFTW_MPI_TRANSPOSED_OUT)].ob = oblock;
cannam@167:      }
cannam@167: 
cannam@167:      pln = XM(plan_guru_dft)(rnk,dims,howmany, in,out, comm, sign, flags);
cannam@167:      X(ifree)(dims);
cannam@167:      return pln;
cannam@167: }
cannam@167: 
cannam@167: X(plan) XM(plan_dft)(int rnk, const ptrdiff_t *n, C *in, C *out,
cannam@167: 		     MPI_Comm comm, int sign, unsigned flags)
cannam@167: {
cannam@167:      return XM(plan_many_dft)(rnk, n, 1,
cannam@167: 			      FFTW_MPI_DEFAULT_BLOCK,
cannam@167: 			      FFTW_MPI_DEFAULT_BLOCK,
cannam@167: 			      in, out, comm, sign, flags);
cannam@167: }
cannam@167: 
cannam@167: X(plan) XM(plan_dft_1d)(ptrdiff_t nx, C *in, C *out,
cannam@167: 			MPI_Comm comm, int sign, unsigned flags)
cannam@167: {
cannam@167:      return XM(plan_dft)(1, &nx, in, out, comm, sign, flags);
cannam@167: }
cannam@167: 
cannam@167: X(plan) XM(plan_dft_2d)(ptrdiff_t nx, ptrdiff_t ny, C *in, C *out,
cannam@167: 			MPI_Comm comm, int sign, unsigned flags)
cannam@167: {
cannam@167:      ptrdiff_t n[2];
cannam@167:      n[0] = nx; n[1] = ny;
cannam@167:      return XM(plan_dft)(2, n, in, out, comm, sign, flags);
cannam@167: }
cannam@167: 
cannam@167: X(plan) XM(plan_dft_3d)(ptrdiff_t nx, ptrdiff_t ny, ptrdiff_t nz,
cannam@167: 			C *in, C *out,
cannam@167: 			MPI_Comm comm, int sign, unsigned flags)
cannam@167: {
cannam@167:      ptrdiff_t n[3];
cannam@167:      n[0] = nx; n[1] = ny; n[2] = nz;
cannam@167:      return XM(plan_dft)(3, n, in, out, comm, sign, flags);
cannam@167: }
cannam@167: 
cannam@167: /*************************************************************************/
cannam@167: /* R2R API */
cannam@167: 
cannam@167: X(plan) XM(plan_guru_r2r)(int rnk, const XM(ddim) *dims0,
cannam@167: 			  ptrdiff_t howmany,
cannam@167: 			  R *in, R *out,
cannam@167: 			  MPI_Comm comm, const X(r2r_kind) *kind,
cannam@167: 			  unsigned flags)
cannam@167: {
cannam@167:      int n_pes, i;
cannam@167:      dtensor *sz;
cannam@167:      rdft_kind *k;
cannam@167:      X(plan) pln;
cannam@167:      
cannam@167:      XM(init)();
cannam@167: 
cannam@167:      if (howmany < 0 || rnk < 1) return 0;
cannam@167:      for (i = 0; i < rnk; ++i)
cannam@167: 	  if (dims0[i].n < 1 || dims0[i].ib < 0 || dims0[i].ob < 0)
cannam@167: 	       return 0;
cannam@167: 
cannam@167:      k = X(map_r2r_kind)(rnk, kind);
cannam@167: 
cannam@167:      MPI_Comm_size(comm, &n_pes);
cannam@167:      sz = default_sz(rnk, dims0, n_pes, 0);
cannam@167: 
cannam@167:      if (XM(num_blocks_total)(sz, IB) > n_pes
cannam@167: 	 || XM(num_blocks_total)(sz, OB) > n_pes) {
cannam@167: 	  XM(dtensor_destroy)(sz);
cannam@167: 	  return 0;
cannam@167:      }
cannam@167: 
cannam@167:      pln = X(mkapiplan)(0, flags,
cannam@167: 			XM(mkproblem_rdft_d)(sz, howmany,
cannam@167: 					     in, out,
cannam@167: 					     comm, k, MPI_FLAGS(flags)));
cannam@167:      X(ifree0)(k);
cannam@167:      return pln;
cannam@167: }
cannam@167: 
cannam@167: X(plan) XM(plan_many_r2r)(int rnk, const ptrdiff_t *n,
cannam@167: 			  ptrdiff_t howmany,
cannam@167: 			  ptrdiff_t iblock, ptrdiff_t oblock,
cannam@167: 			  R *in, R *out,
cannam@167: 			  MPI_Comm comm, const X(r2r_kind) *kind,
cannam@167: 			  unsigned flags)
cannam@167: {
cannam@167:      XM(ddim) *dims = simple_dims(rnk, n);
cannam@167:      X(plan) pln;
cannam@167: 
cannam@167:      if (rnk == 1) {
cannam@167: 	  dims[0].ib = iblock;
cannam@167: 	  dims[0].ob = oblock;
cannam@167:      }
cannam@167:      else if (rnk > 1) {
cannam@167: 	  dims[0 != (flags & FFTW_MPI_TRANSPOSED_IN)].ib = iblock;
cannam@167: 	  dims[0 != (flags & FFTW_MPI_TRANSPOSED_OUT)].ob = oblock;
cannam@167:      }
cannam@167: 
cannam@167:      pln = XM(plan_guru_r2r)(rnk,dims,howmany, in,out, comm, kind, flags);
cannam@167:      X(ifree)(dims);
cannam@167:      return pln;
cannam@167: }
cannam@167: 
cannam@167: X(plan) XM(plan_r2r)(int rnk, const ptrdiff_t *n, R *in, R *out,
cannam@167: 		     MPI_Comm comm, 
cannam@167: 		     const X(r2r_kind) *kind,
cannam@167: 		     unsigned flags)
cannam@167: {
cannam@167:      return XM(plan_many_r2r)(rnk, n, 1,
cannam@167: 			      FFTW_MPI_DEFAULT_BLOCK,
cannam@167: 			      FFTW_MPI_DEFAULT_BLOCK,
cannam@167: 			      in, out, comm, kind, flags);
cannam@167: }
cannam@167: 
cannam@167: X(plan) XM(plan_r2r_2d)(ptrdiff_t nx, ptrdiff_t ny, R *in, R *out,
cannam@167: 			MPI_Comm comm,
cannam@167: 			X(r2r_kind) kindx, X(r2r_kind) kindy,
cannam@167: 			unsigned flags)
cannam@167: {
cannam@167:      ptrdiff_t n[2];
cannam@167:      X(r2r_kind) kind[2];
cannam@167:      n[0] = nx; n[1] = ny;
cannam@167:      kind[0] = kindx; kind[1] = kindy;
cannam@167:      return XM(plan_r2r)(2, n, in, out, comm, kind, flags);
cannam@167: }
cannam@167: 
cannam@167: X(plan) XM(plan_r2r_3d)(ptrdiff_t nx, ptrdiff_t ny, ptrdiff_t nz,
cannam@167: 			R *in, R *out,
cannam@167: 			MPI_Comm comm, 
cannam@167: 			X(r2r_kind) kindx, X(r2r_kind) kindy,
cannam@167: 			X(r2r_kind) kindz,
cannam@167: 			unsigned flags)
cannam@167: {
cannam@167:      ptrdiff_t n[3];
cannam@167:      X(r2r_kind) kind[3];
cannam@167:      n[0] = nx; n[1] = ny; n[2] = nz;
cannam@167:      kind[0] = kindx; kind[1] = kindy; kind[2] = kindz;
cannam@167:      return XM(plan_r2r)(3, n, in, out, comm, kind, flags);
cannam@167: }
cannam@167: 
cannam@167: /*************************************************************************/
cannam@167: /* R2C/C2R API */
cannam@167: 
cannam@167: static X(plan) plan_guru_rdft2(int rnk, const XM(ddim) *dims0,
cannam@167: 			       ptrdiff_t howmany,
cannam@167: 			       R *r, C *c,
cannam@167: 			       MPI_Comm comm, rdft_kind kind, unsigned flags)
cannam@167: {
cannam@167:      int n_pes, i;
cannam@167:      dtensor *sz;
cannam@167:      R *cr = (R *) c;
cannam@167:      
cannam@167:      XM(init)();
cannam@167: 
cannam@167:      if (howmany < 0 || rnk < 2) return 0;
cannam@167:      for (i = 0; i < rnk; ++i)
cannam@167: 	  if (dims0[i].n < 1 || dims0[i].ib < 0 || dims0[i].ob < 0)
cannam@167: 	       return 0;
cannam@167: 
cannam@167:      MPI_Comm_size(comm, &n_pes);
cannam@167:      sz = default_sz(rnk, dims0, n_pes, 1);
cannam@167: 
cannam@167:      sz->dims[rnk-1].n = dims0[rnk-1].n / 2 + 1;
cannam@167:      if (XM(num_blocks_total)(sz, IB) > n_pes
cannam@167: 	 || XM(num_blocks_total)(sz, OB) > n_pes) {
cannam@167: 	  XM(dtensor_destroy)(sz);
cannam@167: 	  return 0;
cannam@167:      }
cannam@167:      sz->dims[rnk-1].n = dims0[rnk-1].n;
cannam@167: 
cannam@167:      if (kind == R2HC)
cannam@167: 	  return X(mkapiplan)(0, flags,
cannam@167: 			      XM(mkproblem_rdft2_d)(sz, howmany,
cannam@167: 						    r, cr, comm, R2HC, 
cannam@167: 						    MPI_FLAGS(flags)));
cannam@167:      else
cannam@167: 	  return X(mkapiplan)(0, flags,
cannam@167: 			      XM(mkproblem_rdft2_d)(sz, howmany,
cannam@167: 						    cr, r, comm, HC2R, 
cannam@167: 						    MPI_FLAGS(flags)));
cannam@167: }
cannam@167: 
cannam@167: X(plan) XM(plan_many_dft_r2c)(int rnk, const ptrdiff_t *n,
cannam@167: 			  ptrdiff_t howmany,
cannam@167: 			  ptrdiff_t iblock, ptrdiff_t oblock,
cannam@167: 			  R *in, C *out,
cannam@167: 			  MPI_Comm comm, unsigned flags)
cannam@167: {
cannam@167:      XM(ddim) *dims = simple_dims(rnk, n);
cannam@167:      X(plan) pln;
cannam@167: 
cannam@167:      if (rnk == 1) {
cannam@167: 	  dims[0].ib = iblock;
cannam@167: 	  dims[0].ob = oblock;
cannam@167:      }
cannam@167:      else if (rnk > 1) {
cannam@167: 	  dims[0 != (flags & FFTW_MPI_TRANSPOSED_IN)].ib = iblock;
cannam@167: 	  dims[0 != (flags & FFTW_MPI_TRANSPOSED_OUT)].ob = oblock;
cannam@167:      }
cannam@167: 
cannam@167:      pln = plan_guru_rdft2(rnk,dims,howmany, in,out, comm, R2HC, flags);
cannam@167:      X(ifree)(dims);
cannam@167:      return pln;
cannam@167: }
cannam@167: 
cannam@167: X(plan) XM(plan_many_dft_c2r)(int rnk, const ptrdiff_t *n,
cannam@167: 			  ptrdiff_t howmany,
cannam@167: 			  ptrdiff_t iblock, ptrdiff_t oblock,
cannam@167: 			  C *in, R *out,
cannam@167: 			  MPI_Comm comm, unsigned flags)
cannam@167: {
cannam@167:      XM(ddim) *dims = simple_dims(rnk, n);
cannam@167:      X(plan) pln;
cannam@167: 
cannam@167:      if (rnk == 1) {
cannam@167: 	  dims[0].ib = iblock;
cannam@167: 	  dims[0].ob = oblock;
cannam@167:      }
cannam@167:      else if (rnk > 1) {
cannam@167: 	  dims[0 != (flags & FFTW_MPI_TRANSPOSED_IN)].ib = iblock;
cannam@167: 	  dims[0 != (flags & FFTW_MPI_TRANSPOSED_OUT)].ob = oblock;
cannam@167:      }
cannam@167: 
cannam@167:      pln = plan_guru_rdft2(rnk,dims,howmany, out,in, comm, HC2R, flags);
cannam@167:      X(ifree)(dims);
cannam@167:      return pln;
cannam@167: }
cannam@167: 
cannam@167: X(plan) XM(plan_dft_r2c)(int rnk, const ptrdiff_t *n, R *in, C *out,
cannam@167: 		     MPI_Comm comm, unsigned flags)
cannam@167: {
cannam@167:      return XM(plan_many_dft_r2c)(rnk, n, 1,
cannam@167: 			      FFTW_MPI_DEFAULT_BLOCK,
cannam@167: 			      FFTW_MPI_DEFAULT_BLOCK,
cannam@167: 			      in, out, comm, flags);
cannam@167: }
cannam@167: 
cannam@167: X(plan) XM(plan_dft_r2c_2d)(ptrdiff_t nx, ptrdiff_t ny, R *in, C *out,
cannam@167: 			MPI_Comm comm, unsigned flags)
cannam@167: {
cannam@167:      ptrdiff_t n[2];
cannam@167:      n[0] = nx; n[1] = ny;
cannam@167:      return XM(plan_dft_r2c)(2, n, in, out, comm, flags);
cannam@167: }
cannam@167: 
cannam@167: X(plan) XM(plan_dft_r2c_3d)(ptrdiff_t nx, ptrdiff_t ny, ptrdiff_t nz,
cannam@167: 			R *in, C *out, MPI_Comm comm, unsigned flags)
cannam@167: {
cannam@167:      ptrdiff_t n[3];
cannam@167:      n[0] = nx; n[1] = ny; n[2] = nz;
cannam@167:      return XM(plan_dft_r2c)(3, n, in, out, comm, flags);
cannam@167: }
cannam@167: 
cannam@167: X(plan) XM(plan_dft_c2r)(int rnk, const ptrdiff_t *n, C *in, R *out,
cannam@167: 		     MPI_Comm comm, unsigned flags)
cannam@167: {
cannam@167:      return XM(plan_many_dft_c2r)(rnk, n, 1,
cannam@167: 			      FFTW_MPI_DEFAULT_BLOCK,
cannam@167: 			      FFTW_MPI_DEFAULT_BLOCK,
cannam@167: 			      in, out, comm, flags);
cannam@167: }
cannam@167: 
cannam@167: X(plan) XM(plan_dft_c2r_2d)(ptrdiff_t nx, ptrdiff_t ny, C *in, R *out,
cannam@167: 			MPI_Comm comm, unsigned flags)
cannam@167: {
cannam@167:      ptrdiff_t n[2];
cannam@167:      n[0] = nx; n[1] = ny;
cannam@167:      return XM(plan_dft_c2r)(2, n, in, out, comm, flags);
cannam@167: }
cannam@167: 
cannam@167: X(plan) XM(plan_dft_c2r_3d)(ptrdiff_t nx, ptrdiff_t ny, ptrdiff_t nz,
cannam@167: 			C *in, R *out, MPI_Comm comm, unsigned flags)
cannam@167: {
cannam@167:      ptrdiff_t n[3];
cannam@167:      n[0] = nx; n[1] = ny; n[2] = nz;
cannam@167:      return XM(plan_dft_c2r)(3, n, in, out, comm, flags);
cannam@167: }
cannam@167: 
cannam@167: /*************************************************************************/
cannam@167: /* New-array execute functions */
cannam@167: 
cannam@167: void XM(execute_dft)(const X(plan) p, C *in, C *out) {
cannam@167:      /* internally, MPI plans are just rdft plans */
cannam@167:      X(execute_r2r)(p, (R*) in, (R*) out);
cannam@167: }
cannam@167: 
cannam@167: void XM(execute_dft_r2c)(const X(plan) p, R *in, C *out) {
cannam@167:      /* internally, MPI plans are just rdft plans */
cannam@167:      X(execute_r2r)(p, in, (R*) out);
cannam@167: }
cannam@167: 
cannam@167: void XM(execute_dft_c2r)(const X(plan) p, C *in, R *out) {
cannam@167:      /* internally, MPI plans are just rdft plans */
cannam@167:      X(execute_r2r)(p, (R*) in, out);
cannam@167: }
cannam@167: 
cannam@167: void XM(execute_r2r)(const X(plan) p, R *in, R *out) {
cannam@167:      /* internally, MPI plans are just rdft plans */
cannam@167:      X(execute_r2r)(p, in, out);
cannam@167: }