Chris@82: /**************************************************************************/
Chris@82: /* NOTE to users: this is the FFTW-MPI self-test and benchmark program.
Chris@82:    It is probably NOT a good place to learn FFTW usage, since it has a
Chris@82:    lot of added complexity in order to exercise and test the full API,
Chris@82:    etcetera.  We suggest reading the manual. */
Chris@82: /**************************************************************************/
Chris@82: 
Chris@82: #include <math.h>
Chris@82: #include <stdio.h>
Chris@82: #include <string.h>
Chris@82: #include "fftw3-mpi.h"
Chris@82: #include "tests/fftw-bench.h"
Chris@82: 
Chris@82: #if defined(BENCHFFT_SINGLE)
Chris@82: #  define BENCH_MPI_TYPE MPI_FLOAT
Chris@82: #elif defined(BENCHFFT_LDOUBLE)
Chris@82: #  define BENCH_MPI_TYPE MPI_LONG_DOUBLE
Chris@82: #elif defined(BENCHFFT_QUAD)
Chris@82: #  error MPI quad-precision type is unknown
Chris@82: #else
Chris@82: #  define BENCH_MPI_TYPE MPI_DOUBLE
Chris@82: #endif
Chris@82: 
Chris@82: #if SIZEOF_PTRDIFF_T == SIZEOF_INT
Chris@82: #  define FFTW_MPI_PTRDIFF_T MPI_INT
Chris@82: #elif SIZEOF_PTRDIFF_T == SIZEOF_LONG
Chris@82: #  define FFTW_MPI_PTRDIFF_T MPI_LONG
Chris@82: #elif SIZEOF_PTRDIFF_T == SIZEOF_LONG_LONG
Chris@82: #  define FFTW_MPI_PTRDIFF_T MPI_LONG_LONG
Chris@82: #else
Chris@82: #  error MPI type for ptrdiff_t is unknown
Chris@82: #  define FFTW_MPI_PTRDIFF_T MPI_LONG
Chris@82: #endif
Chris@82: 
Chris@82: static const char *mkversion(void) { return FFTW(version); }
Chris@82: static const char *mkcc(void) { return FFTW(cc); }
Chris@82: static const char *mkcodelet_optim(void) { return FFTW(codelet_optim); }
Chris@82: static const char *mknproc(void) {
Chris@82:      static char buf[32];
Chris@82:      int ncpus;
Chris@82:      MPI_Comm_size(MPI_COMM_WORLD, &ncpus);
Chris@82: #ifdef HAVE_SNPRINTF
Chris@82:      snprintf(buf, 32, "%d", ncpus);
Chris@82: #else
Chris@82:      sprintf(buf, "%d", ncpus);
Chris@82: #endif
Chris@82:      return buf;
Chris@82: }
Chris@82: 
Chris@82: BEGIN_BENCH_DOC
Chris@82: BENCH_DOC("name", "fftw3_mpi")
Chris@82: BENCH_DOCF("version", mkversion)
Chris@82: BENCH_DOCF("cc", mkcc)
Chris@82: BENCH_DOCF("codelet-optim", mkcodelet_optim)
Chris@82: BENCH_DOCF("nproc", mknproc)
Chris@82: END_BENCH_DOC 
Chris@82: 
Chris@82: static int n_pes = 1, my_pe = 0;
Chris@82: 
Chris@82: /* global variables describing the shape of the data and its distribution */
Chris@82: static int rnk;
Chris@82: static ptrdiff_t vn, iNtot, oNtot;
Chris@82: static ptrdiff_t *local_ni=0, *local_starti=0;
Chris@82: static ptrdiff_t *local_no=0, *local_starto=0;
Chris@82: static ptrdiff_t *all_local_ni=0, *all_local_starti=0; /* n_pes x rnk arrays */
Chris@82: static ptrdiff_t *all_local_no=0, *all_local_starto=0; /* n_pes x rnk arrays */
Chris@82: static ptrdiff_t *istrides = 0, *ostrides = 0;
Chris@82: static ptrdiff_t *total_ni=0, *total_no=0;
Chris@82: static int *isend_cnt = 0, *isend_off = 0; /* for MPI_Scatterv */
Chris@82: static int *orecv_cnt = 0, *orecv_off = 0; /* for MPI_Gatherv */
Chris@82: 
Chris@82: static bench_real *local_in = 0, *local_out = 0;
Chris@82: static bench_real *all_local_in = 0, *all_local_out = 0;
Chris@82: static int all_local_in_alloc = 0, all_local_out_alloc = 0;
Chris@82: static FFTW(plan) plan_scramble_in = 0, plan_unscramble_out = 0;
Chris@82: 
Chris@82: static void alloc_rnk(int rnk_) {
Chris@82:      rnk = rnk_;
Chris@82:      bench_free(local_ni);
Chris@82:      if (rnk == 0)
Chris@82: 	  local_ni = 0;
Chris@82:      else
Chris@82: 	  local_ni = (ptrdiff_t *) bench_malloc(sizeof(ptrdiff_t) * rnk
Chris@82: 						* (8 + n_pes * 4));
Chris@82: 
Chris@82:      local_starti = local_ni + rnk;
Chris@82:      local_no = local_ni + 2 * rnk;
Chris@82:      local_starto = local_ni + 3 * rnk;
Chris@82:      istrides = local_ni + 4 * rnk;
Chris@82:      ostrides = local_ni + 5 * rnk;
Chris@82:      total_ni = local_ni + 6 * rnk;
Chris@82:      total_no = local_ni + 7 * rnk;
Chris@82:      all_local_ni = local_ni + 8 * rnk;
Chris@82:      all_local_starti = local_ni + (8 + n_pes) * rnk;
Chris@82:      all_local_no = local_ni + (8 + 2 * n_pes) * rnk;
Chris@82:      all_local_starto = local_ni + (8 + 3 * n_pes) * rnk;
Chris@82: }
Chris@82: 
Chris@82: static void setup_gather_scatter(void)
Chris@82: {
Chris@82:      int i, j;
Chris@82:      ptrdiff_t off;
Chris@82: 
Chris@82:      MPI_Gather(local_ni, rnk, FFTW_MPI_PTRDIFF_T,
Chris@82: 		all_local_ni, rnk, FFTW_MPI_PTRDIFF_T,
Chris@82: 		0, MPI_COMM_WORLD);
Chris@82:      MPI_Bcast(all_local_ni, rnk*n_pes, FFTW_MPI_PTRDIFF_T, 0, MPI_COMM_WORLD);
Chris@82:      MPI_Gather(local_starti, rnk, FFTW_MPI_PTRDIFF_T,
Chris@82: 		all_local_starti, rnk, FFTW_MPI_PTRDIFF_T,
Chris@82: 		0, MPI_COMM_WORLD);
Chris@82:      MPI_Bcast(all_local_starti, rnk*n_pes, FFTW_MPI_PTRDIFF_T, 0, MPI_COMM_WORLD);
Chris@82: 
Chris@82:      MPI_Gather(local_no, rnk, FFTW_MPI_PTRDIFF_T,
Chris@82: 		all_local_no, rnk, FFTW_MPI_PTRDIFF_T,
Chris@82: 		0, MPI_COMM_WORLD);
Chris@82:      MPI_Bcast(all_local_no, rnk*n_pes, FFTW_MPI_PTRDIFF_T, 0, MPI_COMM_WORLD);
Chris@82:      MPI_Gather(local_starto, rnk, FFTW_MPI_PTRDIFF_T,
Chris@82: 		all_local_starto, rnk, FFTW_MPI_PTRDIFF_T,
Chris@82: 		0, MPI_COMM_WORLD);
Chris@82:      MPI_Bcast(all_local_starto, rnk*n_pes, FFTW_MPI_PTRDIFF_T, 0, MPI_COMM_WORLD);
Chris@82: 
Chris@82:      off = 0;
Chris@82:      for (i = 0; i < n_pes; ++i) {
Chris@82: 	  ptrdiff_t N = vn;
Chris@82: 	  for (j = 0; j < rnk; ++j)
Chris@82: 	       N *= all_local_ni[i * rnk + j];
Chris@82: 	  isend_cnt[i] = N;
Chris@82: 	  isend_off[i] = off;
Chris@82: 	  off += N;
Chris@82:      }
Chris@82:      iNtot = off;
Chris@82:      all_local_in_alloc = 1;
Chris@82: 
Chris@82:      istrides[rnk - 1] = vn;
Chris@82:      for (j = rnk - 2; j >= 0; --j)
Chris@82: 	  istrides[j] = total_ni[j + 1] * istrides[j + 1];
Chris@82: 
Chris@82:      off = 0;
Chris@82:      for (i = 0; i < n_pes; ++i) {
Chris@82: 	  ptrdiff_t N = vn;
Chris@82: 	  for (j = 0; j < rnk; ++j)
Chris@82: 	       N *= all_local_no[i * rnk + j];
Chris@82: 	  orecv_cnt[i] = N;
Chris@82: 	  orecv_off[i] = off;
Chris@82: 	  off += N;
Chris@82:      }
Chris@82:      oNtot = off;
Chris@82:      all_local_out_alloc = 1;
Chris@82: 
Chris@82:      ostrides[rnk - 1] = vn;
Chris@82:      for (j = rnk - 2; j >= 0; --j)
Chris@82: 	  ostrides[j] = total_no[j + 1] * ostrides[j + 1];
Chris@82: }
Chris@82: 
Chris@82: static void copy_block_out(const bench_real *in,
Chris@82: 			   int rnk, ptrdiff_t *n, ptrdiff_t *start, 
Chris@82: 			   ptrdiff_t is, ptrdiff_t *os, ptrdiff_t vn,
Chris@82: 			   bench_real *out)
Chris@82: {
Chris@82:      ptrdiff_t i;
Chris@82:      if (rnk == 0) { 
Chris@82: 	  for (i = 0; i < vn; ++i)
Chris@82: 	       out[i] = in[i];
Chris@82:      }
Chris@82:      else if (rnk == 1) { /* this case is just an optimization */
Chris@82: 	  ptrdiff_t j;
Chris@82: 	  out += start[0] * os[0];
Chris@82: 	  for (j = 0; j < n[0]; ++j) {
Chris@82: 	       for (i = 0; i < vn; ++i)
Chris@82: 		    out[i] = in[i];
Chris@82: 	       in += is;
Chris@82: 	       out += os[0];
Chris@82: 	  }
Chris@82:      }
Chris@82:      else {
Chris@82: 	  /* we should do n[0] for locality, but this way is simpler to code */
Chris@82: 	  for (i = 0; i < n[rnk - 1]; ++i) 
Chris@82: 	       copy_block_out(in + i * is,
Chris@82: 			      rnk - 1, n, start, is * n[rnk - 1], os, vn,
Chris@82: 			      out + (start[rnk - 1] + i) * os[rnk - 1]);
Chris@82:      }
Chris@82: }
Chris@82: 
Chris@82: static void copy_block_in(bench_real *in,
Chris@82: 			  int rnk, ptrdiff_t *n, ptrdiff_t *start, 
Chris@82: 			  ptrdiff_t is, ptrdiff_t *os, ptrdiff_t vn,
Chris@82: 			  const bench_real *out)
Chris@82: {
Chris@82:      ptrdiff_t i;
Chris@82:      if (rnk == 0) { 
Chris@82: 	  for (i = 0; i < vn; ++i)
Chris@82: 	       in[i] = out[i];
Chris@82:      }
Chris@82:      else if (rnk == 1) { /* this case is just an optimization */
Chris@82: 	  ptrdiff_t j;
Chris@82: 	  out += start[0] * os[0];
Chris@82: 	  for (j = 0; j < n[0]; ++j) {
Chris@82: 	       for (i = 0; i < vn; ++i)
Chris@82: 		    in[i] = out[i];
Chris@82: 	       in += is;
Chris@82: 	       out += os[0];
Chris@82: 	  }
Chris@82:      }
Chris@82:      else {
Chris@82: 	  /* we should do n[0] for locality, but this way is simpler to code */
Chris@82: 	  for (i = 0; i < n[rnk - 1]; ++i) 
Chris@82: 	       copy_block_in(in + i * is,
Chris@82: 			     rnk - 1, n, start, is * n[rnk - 1], os, vn,
Chris@82: 			     out + (start[rnk - 1] + i) * os[rnk - 1]);
Chris@82:      }
Chris@82: }
Chris@82: 
Chris@82: static void do_scatter_in(bench_real *in)
Chris@82: {
Chris@82:      bench_real *ali;
Chris@82:      int i;
Chris@82:      if (all_local_in_alloc) {
Chris@82:           bench_free(all_local_in);
Chris@82: 	  all_local_in = (bench_real*) bench_malloc(iNtot*sizeof(bench_real));
Chris@82: 	  all_local_in_alloc = 0;
Chris@82:      }
Chris@82:      ali = all_local_in;
Chris@82:      for (i = 0; i < n_pes; ++i) {
Chris@82: 	  copy_block_in(ali,
Chris@82: 			rnk, all_local_ni + i * rnk, 
Chris@82: 			all_local_starti + i * rnk,
Chris@82: 			vn, istrides, vn,
Chris@82: 			in);
Chris@82: 	  ali += isend_cnt[i];
Chris@82:      }
Chris@82:      MPI_Scatterv(all_local_in, isend_cnt, isend_off, BENCH_MPI_TYPE,
Chris@82: 		  local_in, isend_cnt[my_pe], BENCH_MPI_TYPE,
Chris@82: 		  0, MPI_COMM_WORLD);
Chris@82: }
Chris@82: 
Chris@82: static void do_gather_out(bench_real *out)
Chris@82: {
Chris@82:      bench_real *alo;
Chris@82:      int i;
Chris@82: 
Chris@82:      if (all_local_out_alloc) {
Chris@82:           bench_free(all_local_out);
Chris@82: 	  all_local_out = (bench_real*) bench_malloc(oNtot*sizeof(bench_real));
Chris@82: 	  all_local_out_alloc = 0;
Chris@82:      }
Chris@82:      MPI_Gatherv(local_out, orecv_cnt[my_pe], BENCH_MPI_TYPE,
Chris@82: 		 all_local_out, orecv_cnt, orecv_off, BENCH_MPI_TYPE,
Chris@82: 		 0, MPI_COMM_WORLD);
Chris@82:      MPI_Bcast(all_local_out, oNtot, BENCH_MPI_TYPE, 0, MPI_COMM_WORLD);
Chris@82:      alo = all_local_out;
Chris@82:      for (i = 0; i < n_pes; ++i) {
Chris@82: 	  copy_block_out(alo,
Chris@82: 			 rnk, all_local_no + i * rnk, 
Chris@82: 			 all_local_starto + i * rnk,
Chris@82: 			 vn, ostrides, vn,
Chris@82: 			 out);
Chris@82: 	  alo += orecv_cnt[i];
Chris@82:      }
Chris@82: }
Chris@82: 
Chris@82: static void alloc_local(ptrdiff_t nreal, int inplace)
Chris@82: {
Chris@82:      bench_free(local_in);
Chris@82:      if (local_out != local_in) bench_free(local_out);
Chris@82:      local_in = local_out = 0;
Chris@82:      if (nreal > 0) {
Chris@82: 	  ptrdiff_t i;
Chris@82: 	  local_in = (bench_real*) bench_malloc(nreal * sizeof(bench_real));
Chris@82: 	  if (inplace)
Chris@82: 	       local_out = local_in;
Chris@82: 	  else
Chris@82: 	       local_out = (bench_real*) bench_malloc(nreal * sizeof(bench_real));
Chris@82: 	  for (i = 0; i < nreal; ++i) local_in[i] = local_out[i] = 0.0;
Chris@82:      }
Chris@82: }
Chris@82: 
Chris@82: void after_problem_rcopy_from(bench_problem *p, bench_real *ri)
Chris@82: {
Chris@82:      UNUSED(p);
Chris@82:      do_scatter_in(ri);
Chris@82:      if (plan_scramble_in) FFTW(execute)(plan_scramble_in);
Chris@82: }
Chris@82: 
Chris@82: void after_problem_rcopy_to(bench_problem *p, bench_real *ro)
Chris@82: {
Chris@82:      UNUSED(p);
Chris@82:      if (plan_unscramble_out) FFTW(execute)(plan_unscramble_out);
Chris@82:      do_gather_out(ro);
Chris@82: }
Chris@82: 
Chris@82: void after_problem_ccopy_from(bench_problem *p, bench_real *ri, bench_real *ii)
Chris@82: {
Chris@82:      UNUSED(ii);
Chris@82:      after_problem_rcopy_from(p, ri);
Chris@82: }
Chris@82: 
Chris@82: void after_problem_ccopy_to(bench_problem *p, bench_real *ro, bench_real *io)
Chris@82: {
Chris@82:      UNUSED(io);
Chris@82:      after_problem_rcopy_to(p, ro);
Chris@82: }
Chris@82: 
Chris@82: void after_problem_hccopy_from(bench_problem *p, bench_real *ri, bench_real *ii)
Chris@82: {
Chris@82:      UNUSED(ii);
Chris@82:      after_problem_rcopy_from(p, ri);
Chris@82: }
Chris@82: 
Chris@82: void after_problem_hccopy_to(bench_problem *p, bench_real *ro, bench_real *io)
Chris@82: {
Chris@82:      UNUSED(io);
Chris@82:      after_problem_rcopy_to(p, ro);
Chris@82: }
Chris@82: 
Chris@82: static FFTW(plan) mkplan_transpose_local(ptrdiff_t nx, ptrdiff_t ny, 
Chris@82: 					 ptrdiff_t vn, 
Chris@82: 					 bench_real *in, bench_real *out)
Chris@82: {
Chris@82:      FFTW(iodim64) hdims[3];
Chris@82:      FFTW(r2r_kind) k[3];
Chris@82:      FFTW(plan) pln;
Chris@82: 
Chris@82:      hdims[0].n = nx;
Chris@82:      hdims[0].is = ny * vn;
Chris@82:      hdims[0].os = vn;
Chris@82:      hdims[1].n = ny;
Chris@82:      hdims[1].is = vn;
Chris@82:      hdims[1].os = nx * vn;
Chris@82:      hdims[2].n = vn;
Chris@82:      hdims[2].is = 1;
Chris@82:      hdims[2].os = 1;
Chris@82:      k[0] = k[1] = k[2] = FFTW_R2HC;
Chris@82:      pln = FFTW(plan_guru64_r2r)(0, 0, 3, hdims, in, out, k, FFTW_ESTIMATE);
Chris@82:      BENCH_ASSERT(pln != 0);
Chris@82:      return pln;
Chris@82: }
Chris@82: 
Chris@82: static int tensor_rowmajor_transposedp(bench_tensor *t)
Chris@82: {
Chris@82:      bench_iodim *d;
Chris@82:      int i;
Chris@82: 
Chris@82:      BENCH_ASSERT(BENCH_FINITE_RNK(t->rnk));
Chris@82:      if (t->rnk < 2)
Chris@82: 	  return 0;
Chris@82: 
Chris@82:      d = t->dims;
Chris@82:      if (d[0].is != d[1].is * d[1].n
Chris@82: 	 || d[0].os != d[1].is
Chris@82: 	 || d[1].os != d[0].os * d[0].n)
Chris@82: 	  return 0;
Chris@82:      if (t->rnk > 2 && d[1].is != d[2].is * d[2].n)
Chris@82: 	  return 0;
Chris@82:      for (i = 2; i + 1 < t->rnk; ++i) {
Chris@82:           d = t->dims + i;
Chris@82:           if (d[0].is != d[1].is * d[1].n
Chris@82: 	      || d[0].os != d[1].os * d[1].n)
Chris@82:                return 0;
Chris@82:      }
Chris@82: 
Chris@82:      if (t->rnk > 2 && t->dims[t->rnk-1].is != t->dims[t->rnk-1].os)
Chris@82: 	  return 0;
Chris@82:      return 1;
Chris@82: }
Chris@82: 
Chris@82: static int tensor_contiguousp(bench_tensor *t, int s)
Chris@82: {
Chris@82:      return (t->dims[t->rnk-1].is == s
Chris@82: 	     && ((tensor_rowmajorp(t) && 
Chris@82: 		  t->dims[t->rnk-1].is == t->dims[t->rnk-1].os)
Chris@82: 		 || tensor_rowmajor_transposedp(t)));
Chris@82: }
Chris@82: 
Chris@82: static FFTW(plan) mkplan_complex(bench_problem *p, unsigned flags)
Chris@82: {
Chris@82:      FFTW(plan) pln = 0;
Chris@82:      int i; 
Chris@82:      ptrdiff_t ntot;
Chris@82: 
Chris@82:      vn = p->vecsz->rnk == 1 ? p->vecsz->dims[0].n : 1;
Chris@82: 
Chris@82:      if (p->sz->rnk < 1
Chris@82: 	 || p->split
Chris@82: 	 || !tensor_contiguousp(p->sz, vn)
Chris@82: 	 || tensor_rowmajor_transposedp(p->sz)
Chris@82: 	 || p->vecsz->rnk > 1
Chris@82: 	 || (p->vecsz->rnk == 1 && (p->vecsz->dims[0].is != 1
Chris@82: 				    || p->vecsz->dims[0].os != 1)))
Chris@82: 	  return 0;
Chris@82: 
Chris@82:      alloc_rnk(p->sz->rnk);
Chris@82:      for (i = 0; i < rnk; ++i) {
Chris@82: 	  total_ni[i] = total_no[i] = p->sz->dims[i].n;
Chris@82: 	  local_ni[i] = local_no[i] = total_ni[i];
Chris@82: 	  local_starti[i] = local_starto[i] = 0;
Chris@82:      }
Chris@82:      if (rnk > 1) {
Chris@82: 	  ptrdiff_t n, start, nT, startT;
Chris@82: 	  ntot = FFTW(mpi_local_size_many_transposed)
Chris@82: 	       (p->sz->rnk, total_ni, vn,
Chris@82: 		FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
Chris@82: 		MPI_COMM_WORLD,
Chris@82: 		&n, &start, &nT, &startT);
Chris@82: 	  if  (flags & FFTW_MPI_TRANSPOSED_IN) {
Chris@82: 	       local_ni[1] = nT;
Chris@82: 	       local_starti[1] = startT;
Chris@82: 	  }
Chris@82: 	  else {
Chris@82: 	       local_ni[0] = n;
Chris@82: 	       local_starti[0] = start;
Chris@82: 	  }
Chris@82: 	  if  (flags & FFTW_MPI_TRANSPOSED_OUT) {
Chris@82: 	       local_no[1] = nT;
Chris@82: 	       local_starto[1] = startT;
Chris@82: 	  }
Chris@82: 	  else {
Chris@82: 	       local_no[0] = n;
Chris@82: 	       local_starto[0] = start;
Chris@82: 	  }
Chris@82:      }
Chris@82:      else if (rnk == 1) {
Chris@82: 	  ntot = FFTW(mpi_local_size_many_1d)
Chris@82: 	       (total_ni[0], vn, MPI_COMM_WORLD, p->sign, flags,
Chris@82: 		local_ni, local_starti, local_no, local_starto);
Chris@82:      }
Chris@82:      alloc_local(ntot * 2, p->in == p->out);
Chris@82: 
Chris@82:      pln = FFTW(mpi_plan_many_dft)(p->sz->rnk, total_ni, vn, 
Chris@82: 				   FFTW_MPI_DEFAULT_BLOCK,
Chris@82: 				   FFTW_MPI_DEFAULT_BLOCK,
Chris@82: 				   (FFTW(complex) *) local_in, 
Chris@82: 				   (FFTW(complex) *) local_out,
Chris@82: 				   MPI_COMM_WORLD, p->sign, flags);
Chris@82: 
Chris@82:      vn *= 2;
Chris@82: 
Chris@82:      if (rnk > 1) {
Chris@82: 	  ptrdiff_t nrest = 1;
Chris@82: 	  for (i = 2; i < rnk; ++i) nrest *= p->sz->dims[i].n;
Chris@82: 	  if (flags & FFTW_MPI_TRANSPOSED_IN)
Chris@82: 	       plan_scramble_in = mkplan_transpose_local(
Chris@82: 		    p->sz->dims[0].n, local_ni[1], vn * nrest,
Chris@82: 		    local_in, local_in);
Chris@82: 	  if (flags & FFTW_MPI_TRANSPOSED_OUT)
Chris@82: 	       plan_unscramble_out = mkplan_transpose_local(
Chris@82: 		    local_no[1], p->sz->dims[0].n, vn * nrest,
Chris@82: 		    local_out, local_out);
Chris@82:      }
Chris@82:      
Chris@82:      return pln;
Chris@82: }
Chris@82: 
Chris@82: static int tensor_real_contiguousp(bench_tensor *t, int sign, int s)
Chris@82: {
Chris@82:      return (t->dims[t->rnk-1].is == s
Chris@82: 	     && ((tensor_real_rowmajorp(t, sign, 1) && 
Chris@82: 		  t->dims[t->rnk-1].is == t->dims[t->rnk-1].os)));
Chris@82: }
Chris@82: 
Chris@82: static FFTW(plan) mkplan_real(bench_problem *p, unsigned flags)
Chris@82: {
Chris@82:      FFTW(plan) pln = 0;
Chris@82:      int i; 
Chris@82:      ptrdiff_t ntot;
Chris@82: 
Chris@82:      vn = p->vecsz->rnk == 1 ? p->vecsz->dims[0].n : 1;
Chris@82: 
Chris@82:      if (p->sz->rnk < 2
Chris@82: 	 || p->split
Chris@82: 	 || !tensor_real_contiguousp(p->sz, p->sign, vn)
Chris@82: 	 || tensor_rowmajor_transposedp(p->sz)
Chris@82: 	 || p->vecsz->rnk > 1
Chris@82: 	 || (p->vecsz->rnk == 1 && (p->vecsz->dims[0].is != 1
Chris@82: 				    || p->vecsz->dims[0].os != 1)))
Chris@82: 	  return 0;
Chris@82: 
Chris@82:      alloc_rnk(p->sz->rnk);
Chris@82:      for (i = 0; i < rnk; ++i) {
Chris@82: 	  total_ni[i] = total_no[i] = p->sz->dims[i].n;
Chris@82: 	  local_ni[i] = local_no[i] = total_ni[i];
Chris@82: 	  local_starti[i] = local_starto[i] = 0;
Chris@82:      }
Chris@82:      local_ni[rnk-1] = local_no[rnk-1] = total_ni[rnk-1] = total_no[rnk-1] 
Chris@82: 	  = p->sz->dims[rnk-1].n / 2 + 1;
Chris@82:      {
Chris@82: 	  ptrdiff_t n, start, nT, startT;
Chris@82: 	  ntot = FFTW(mpi_local_size_many_transposed)
Chris@82: 	       (p->sz->rnk, total_ni, vn,
Chris@82: 		FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
Chris@82: 		MPI_COMM_WORLD,
Chris@82: 		&n, &start, &nT, &startT);
Chris@82: 	  if  (flags & FFTW_MPI_TRANSPOSED_IN) {
Chris@82: 	       local_ni[1] = nT;
Chris@82: 	       local_starti[1] = startT;
Chris@82: 	  }
Chris@82: 	  else {
Chris@82: 	       local_ni[0] = n;
Chris@82: 	       local_starti[0] = start;
Chris@82: 	  }
Chris@82: 	  if  (flags & FFTW_MPI_TRANSPOSED_OUT) {
Chris@82: 	       local_no[1] = nT;
Chris@82: 	       local_starto[1] = startT;
Chris@82: 	  }
Chris@82: 	  else {
Chris@82: 	       local_no[0] = n;
Chris@82: 	       local_starto[0] = start;
Chris@82: 	  }
Chris@82:      }
Chris@82:      alloc_local(ntot * 2, p->in == p->out);
Chris@82: 
Chris@82:      total_ni[rnk - 1] = p->sz->dims[rnk - 1].n;
Chris@82:      if (p->sign < 0)
Chris@82: 	  pln = FFTW(mpi_plan_many_dft_r2c)(p->sz->rnk, total_ni, vn, 
Chris@82: 					    FFTW_MPI_DEFAULT_BLOCK,
Chris@82: 					    FFTW_MPI_DEFAULT_BLOCK,
Chris@82: 					    local_in, 
Chris@82: 					    (FFTW(complex) *) local_out,
Chris@82: 					    MPI_COMM_WORLD, flags);
Chris@82:      else
Chris@82: 	  pln = FFTW(mpi_plan_many_dft_c2r)(p->sz->rnk, total_ni, vn, 
Chris@82: 					    FFTW_MPI_DEFAULT_BLOCK,
Chris@82: 					    FFTW_MPI_DEFAULT_BLOCK,
Chris@82: 					    (FFTW(complex) *) local_in, 
Chris@82: 					    local_out,
Chris@82: 					    MPI_COMM_WORLD, flags);
Chris@82: 
Chris@82:      total_ni[rnk - 1] = p->sz->dims[rnk - 1].n / 2 + 1;
Chris@82:      vn *= 2;
Chris@82: 
Chris@82:      {
Chris@82: 	  ptrdiff_t nrest = 1;
Chris@82: 	  for (i = 2; i < rnk; ++i) nrest *= total_ni[i];
Chris@82: 	  if (flags & FFTW_MPI_TRANSPOSED_IN)
Chris@82: 	       plan_scramble_in = mkplan_transpose_local(
Chris@82: 		    total_ni[0], local_ni[1], vn * nrest,
Chris@82: 		    local_in, local_in);
Chris@82: 	  if (flags & FFTW_MPI_TRANSPOSED_OUT)
Chris@82: 	       plan_unscramble_out = mkplan_transpose_local(
Chris@82: 		    local_no[1], total_ni[0], vn * nrest,
Chris@82: 		    local_out, local_out);
Chris@82:      }
Chris@82:      
Chris@82:      return pln;
Chris@82: }
Chris@82: 
Chris@82: static FFTW(plan) mkplan_transpose(bench_problem *p, unsigned flags)
Chris@82: {
Chris@82:      ptrdiff_t ntot, nx, ny;
Chris@82:      int ix=0, iy=1, i;
Chris@82:      const bench_iodim *d = p->vecsz->dims;
Chris@82:      FFTW(plan) pln;
Chris@82: 
Chris@82:      if (p->vecsz->rnk == 3) {
Chris@82: 	  for (i = 0; i < 3; ++i)
Chris@82: 	       if (d[i].is == 1 && d[i].os == 1) {
Chris@82: 		    vn = d[i].n;
Chris@82: 		    ix = (i + 1) % 3;
Chris@82: 		    iy = (i + 2) % 3;
Chris@82: 		    break;
Chris@82: 	       }
Chris@82: 	  if (i == 3) return 0;
Chris@82:      }
Chris@82:      else {
Chris@82: 	  vn = 1;
Chris@82: 	  ix = 0;
Chris@82: 	  iy = 1;
Chris@82:      }
Chris@82: 
Chris@82:      if (d[ix].is == d[iy].n * vn && d[ix].os == vn
Chris@82: 	 && d[iy].os == d[ix].n * vn && d[iy].is == vn) {
Chris@82: 	  nx = d[ix].n;
Chris@82: 	  ny = d[iy].n;
Chris@82:      }
Chris@82:      else if (d[iy].is == d[ix].n * vn && d[iy].os == vn
Chris@82: 	      && d[ix].os == d[iy].n * vn && d[ix].is == vn) {
Chris@82: 	  nx = d[iy].n;
Chris@82: 	  ny = d[ix].n;
Chris@82:      }
Chris@82:      else
Chris@82: 	  return 0;
Chris@82: 
Chris@82:      alloc_rnk(2);
Chris@82:      ntot = vn * FFTW(mpi_local_size_2d_transposed)(nx, ny, MPI_COMM_WORLD,
Chris@82: 						    &local_ni[0], 
Chris@82: 						    &local_starti[0],
Chris@82: 						    &local_no[0], 
Chris@82: 						    &local_starto[0]);
Chris@82:      local_ni[1] = ny;
Chris@82:      local_starti[1] = 0;
Chris@82:      local_no[1] = nx;
Chris@82:      local_starto[1] = 0;
Chris@82:      total_ni[0] = nx; total_ni[1] = ny;
Chris@82:      total_no[1] = nx; total_no[0] = ny;
Chris@82:      alloc_local(ntot, p->in == p->out);
Chris@82: 
Chris@82:      pln = FFTW(mpi_plan_many_transpose)(nx, ny, vn,
Chris@82: 					 FFTW_MPI_DEFAULT_BLOCK,
Chris@82: 					 FFTW_MPI_DEFAULT_BLOCK,
Chris@82: 					 local_in, local_out,
Chris@82: 					 MPI_COMM_WORLD, flags);
Chris@82:      
Chris@82:      if (flags & FFTW_MPI_TRANSPOSED_IN)
Chris@82: 	  plan_scramble_in = mkplan_transpose_local(local_ni[0], ny, vn,
Chris@82: 						    local_in, local_in);
Chris@82:      if (flags & FFTW_MPI_TRANSPOSED_OUT)
Chris@82: 	  plan_unscramble_out = mkplan_transpose_local
Chris@82: 	       (nx, local_no[0], vn, local_out, local_out);
Chris@82:      
Chris@82: #if 0
Chris@82:      if (pln && vn == 1) {
Chris@82: 	  int i, j;
Chris@82: 	  bench_real *ri = (bench_real *) p->in;
Chris@82: 	  bench_real *ro = (bench_real *) p->out;
Chris@82: 	  if (!ri || !ro) return pln;
Chris@82: 	  setup_gather_scatter();
Chris@82: 	  for (i = 0; i < nx * ny; ++i)
Chris@82: 	       ri[i] = i;
Chris@82: 	  after_problem_rcopy_from(p, ri);
Chris@82: 	  FFTW(execute)(pln);
Chris@82: 	  after_problem_rcopy_to(p, ro);
Chris@82: 	  if (my_pe == 0) {
Chris@82: 	       for (i = 0; i < nx; ++i) {
Chris@82: 		    for (j = 0; j < ny; ++j)
Chris@82: 			 printf("  %3g", ro[j * nx + i]);
Chris@82: 		    printf("\n");
Chris@82: 	       }
Chris@82: 	  }
Chris@82:      }
Chris@82: #endif
Chris@82: 
Chris@82:      return pln;
Chris@82: }
Chris@82: 
Chris@82: static FFTW(plan) mkplan_r2r(bench_problem *p, unsigned flags)
Chris@82: {
Chris@82:      FFTW(plan) pln = 0;
Chris@82:      int i; 
Chris@82:      ptrdiff_t ntot;
Chris@82:      FFTW(r2r_kind) *k;
Chris@82: 
Chris@82:      if ((p->sz->rnk == 0 || (p->sz->rnk == 1 && p->sz->dims[0].n == 1))
Chris@82: 	 && p->vecsz->rnk >= 2 && p->vecsz->rnk <= 3)
Chris@82: 	  return mkplan_transpose(p, flags);
Chris@82: 
Chris@82:      vn = p->vecsz->rnk == 1 ? p->vecsz->dims[0].n : 1;
Chris@82: 
Chris@82:      if (p->sz->rnk < 1
Chris@82: 	 || p->split
Chris@82: 	 || !tensor_contiguousp(p->sz, vn)
Chris@82: 	 || tensor_rowmajor_transposedp(p->sz)
Chris@82: 	 || p->vecsz->rnk > 1
Chris@82: 	 || (p->vecsz->rnk == 1 && (p->vecsz->dims[0].is != 1
Chris@82: 				    || p->vecsz->dims[0].os != 1)))
Chris@82: 	  return 0;
Chris@82: 
Chris@82:      alloc_rnk(p->sz->rnk);
Chris@82:      for (i = 0; i < rnk; ++i) {
Chris@82: 	  total_ni[i] = total_no[i] = p->sz->dims[i].n;
Chris@82: 	  local_ni[i] = local_no[i] = total_ni[i];
Chris@82: 	  local_starti[i] = local_starto[i] = 0;
Chris@82:      }
Chris@82:      if (rnk > 1) {
Chris@82: 	  ptrdiff_t n, start, nT, startT;
Chris@82: 	  ntot = FFTW(mpi_local_size_many_transposed)
Chris@82: 	       (p->sz->rnk, total_ni, vn,
Chris@82: 		FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
Chris@82: 		MPI_COMM_WORLD,
Chris@82: 		&n, &start, &nT, &startT);
Chris@82: 	  if  (flags & FFTW_MPI_TRANSPOSED_IN) {
Chris@82: 	       local_ni[1] = nT;
Chris@82: 	       local_starti[1] = startT;
Chris@82: 	  }
Chris@82: 	  else {
Chris@82: 	       local_ni[0] = n;
Chris@82: 	       local_starti[0] = start;
Chris@82: 	  }
Chris@82: 	  if  (flags & FFTW_MPI_TRANSPOSED_OUT) {
Chris@82: 	       local_no[1] = nT;
Chris@82: 	       local_starto[1] = startT;
Chris@82: 	  }
Chris@82: 	  else {
Chris@82: 	       local_no[0] = n;
Chris@82: 	       local_starto[0] = start;
Chris@82: 	  }
Chris@82:      }
Chris@82:      else if (rnk == 1) {
Chris@82: 	  ntot = FFTW(mpi_local_size_many_1d)
Chris@82: 	       (total_ni[0], vn, MPI_COMM_WORLD, p->sign, flags,
Chris@82: 		local_ni, local_starti, local_no, local_starto);
Chris@82:      }
Chris@82:      alloc_local(ntot, p->in == p->out);
Chris@82: 
Chris@82:      k = (FFTW(r2r_kind) *) bench_malloc(sizeof(FFTW(r2r_kind)) * p->sz->rnk);
Chris@82:      for (i = 0; i < p->sz->rnk; ++i)
Chris@82: 	  switch (p->k[i]) {
Chris@82: 	      case R2R_R2HC: k[i] = FFTW_R2HC; break;
Chris@82: 	      case R2R_HC2R: k[i] = FFTW_HC2R; break;
Chris@82: 	      case R2R_DHT: k[i] = FFTW_DHT; break;
Chris@82: 	      case R2R_REDFT00: k[i] = FFTW_REDFT00; break;
Chris@82: 	      case R2R_REDFT01: k[i] = FFTW_REDFT01; break;
Chris@82: 	      case R2R_REDFT10: k[i] = FFTW_REDFT10; break;
Chris@82: 	      case R2R_REDFT11: k[i] = FFTW_REDFT11; break;
Chris@82: 	      case R2R_RODFT00: k[i] = FFTW_RODFT00; break;
Chris@82: 	      case R2R_RODFT01: k[i] = FFTW_RODFT01; break;
Chris@82: 	      case R2R_RODFT10: k[i] = FFTW_RODFT10; break;
Chris@82: 	      case R2R_RODFT11: k[i] = FFTW_RODFT11; break;
Chris@82: 	      default: BENCH_ASSERT(0);
Chris@82: 	  }
Chris@82: 
Chris@82:      pln = FFTW(mpi_plan_many_r2r)(p->sz->rnk, total_ni, vn, 
Chris@82: 				   FFTW_MPI_DEFAULT_BLOCK,
Chris@82: 				   FFTW_MPI_DEFAULT_BLOCK,
Chris@82: 				   local_in, local_out,
Chris@82: 				   MPI_COMM_WORLD, k, flags);
Chris@82:      bench_free(k);
Chris@82: 
Chris@82:      if (rnk > 1) {
Chris@82: 	  ptrdiff_t nrest = 1;
Chris@82: 	  for (i = 2; i < rnk; ++i) nrest *= p->sz->dims[i].n;
Chris@82: 	  if (flags & FFTW_MPI_TRANSPOSED_IN)
Chris@82: 	       plan_scramble_in = mkplan_transpose_local(
Chris@82: 		    p->sz->dims[0].n, local_ni[1], vn * nrest,
Chris@82: 		    local_in, local_in);
Chris@82: 	  if (flags & FFTW_MPI_TRANSPOSED_OUT)
Chris@82: 	       plan_unscramble_out = mkplan_transpose_local(
Chris@82: 		    local_no[1], p->sz->dims[0].n, vn * nrest,
Chris@82: 		    local_out, local_out);
Chris@82:      }
Chris@82:      
Chris@82:      return pln;
Chris@82: }
Chris@82: 
Chris@82: FFTW(plan) mkplan(bench_problem *p, unsigned flags)
Chris@82: {
Chris@82:      FFTW(plan) pln = 0;
Chris@82:      FFTW(destroy_plan)(plan_scramble_in); plan_scramble_in = 0;
Chris@82:      FFTW(destroy_plan)(plan_unscramble_out); plan_unscramble_out = 0;
Chris@82:      if (p->scrambled_in) {
Chris@82: 	  if (p->sz->rnk == 1 && p->sz->dims[0].n != 1) 
Chris@82: 	       flags |= FFTW_MPI_SCRAMBLED_IN;
Chris@82: 	  else
Chris@82: 	       flags |= FFTW_MPI_TRANSPOSED_IN;
Chris@82:      }
Chris@82:      if (p->scrambled_out) {
Chris@82: 	  if (p->sz->rnk == 1 && p->sz->dims[0].n != 1) 
Chris@82: 	       flags |= FFTW_MPI_SCRAMBLED_OUT;
Chris@82: 	  else
Chris@82: 	       flags |= FFTW_MPI_TRANSPOSED_OUT;
Chris@82:      }
Chris@82:      switch (p->kind) {
Chris@82:          case PROBLEM_COMPLEX: 
Chris@82: 	      pln =mkplan_complex(p, flags);
Chris@82: 	      break;
Chris@82:          case PROBLEM_REAL: 
Chris@82: 	      pln = mkplan_real(p, flags);
Chris@82: 	      break;
Chris@82:          case PROBLEM_R2R:
Chris@82: 	      pln = mkplan_r2r(p, flags);
Chris@82: 	      break;
Chris@82:          default: BENCH_ASSERT(0);
Chris@82:      }
Chris@82:      if (pln) setup_gather_scatter();
Chris@82:      return pln;
Chris@82: }
Chris@82: 
Chris@82: void main_init(int *argc, char ***argv)
Chris@82: {
Chris@82: #ifdef HAVE_SMP
Chris@82: # if MPI_VERSION >= 2 /* for MPI_Init_thread */
Chris@82:      int provided;
Chris@82:      MPI_Init_thread(argc, argv, MPI_THREAD_FUNNELED, &provided);
Chris@82:      threads_ok = provided >= MPI_THREAD_FUNNELED;
Chris@82: # else
Chris@82:      MPI_Init(argc, argv);
Chris@82:      threads_ok = 0;
Chris@82: # endif
Chris@82: #else
Chris@82:      MPI_Init(argc, argv);
Chris@82: #endif
Chris@82:      MPI_Comm_rank(MPI_COMM_WORLD, &my_pe);
Chris@82:      MPI_Comm_size(MPI_COMM_WORLD, &n_pes);
Chris@82:      if (my_pe != 0) verbose = -999;
Chris@82:      no_speed_allocation = 1; /* so we can benchmark transforms > memory */
Chris@82:      always_pad_real = 1; /* out-of-place real transforms are padded */
Chris@82:      isend_cnt = (int *) bench_malloc(sizeof(int) * n_pes);
Chris@82:      isend_off = (int *) bench_malloc(sizeof(int) * n_pes);
Chris@82:      orecv_cnt = (int *) bench_malloc(sizeof(int) * n_pes);
Chris@82:      orecv_off = (int *) bench_malloc(sizeof(int) * n_pes);
Chris@82: 
Chris@82:      /* init_threads must be called before any other FFTW function,
Chris@82: 	including mpi_init, because it has to register the threads hooks
Chris@82: 	before the planner is initalized */
Chris@82: #ifdef HAVE_SMP
Chris@82:      if (threads_ok) { BENCH_ASSERT(FFTW(init_threads)()); }
Chris@82: #endif
Chris@82:      FFTW(mpi_init)();
Chris@82: }
Chris@82: 
Chris@82: void initial_cleanup(void)
Chris@82: {
Chris@82:      alloc_rnk(0);
Chris@82:      alloc_local(0, 0);
Chris@82:      bench_free(all_local_in); all_local_in = 0;
Chris@82:      bench_free(all_local_out); all_local_out = 0;
Chris@82:      bench_free(isend_off); isend_off = 0;
Chris@82:      bench_free(isend_cnt); isend_cnt = 0;
Chris@82:      bench_free(orecv_off); orecv_off = 0;
Chris@82:      bench_free(orecv_cnt); orecv_cnt = 0;
Chris@82:      FFTW(destroy_plan)(plan_scramble_in); plan_scramble_in = 0;
Chris@82:      FFTW(destroy_plan)(plan_unscramble_out); plan_unscramble_out = 0;
Chris@82: }
Chris@82: 
Chris@82: void final_cleanup(void)
Chris@82: {
Chris@82:      MPI_Finalize();
Chris@82: }
Chris@82: 
Chris@82: void bench_exit(int status)
Chris@82: {
Chris@82:      MPI_Abort(MPI_COMM_WORLD, status);
Chris@82: }
Chris@82: 
Chris@82: double bench_cost_postprocess(double cost)
Chris@82: {
Chris@82:      double cost_max;
Chris@82:      MPI_Allreduce(&cost, &cost_max, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
Chris@82:      return cost_max;
Chris@82: }
Chris@82: 
Chris@82: 
Chris@82: int import_wisdom(FILE *f)
Chris@82: {
Chris@82:      int success = 1, sall;
Chris@82:      if (my_pe == 0) success = FFTW(import_wisdom_from_file)(f);
Chris@82:      FFTW(mpi_broadcast_wisdom)(MPI_COMM_WORLD);
Chris@82:      MPI_Allreduce(&success, &sall, 1, MPI_INT, MPI_LAND, MPI_COMM_WORLD);
Chris@82:      return sall;
Chris@82: }
Chris@82: 
Chris@82: void export_wisdom(FILE *f)
Chris@82: {
Chris@82:      FFTW(mpi_gather_wisdom)(MPI_COMM_WORLD);
Chris@82:      if (my_pe == 0) FFTW(export_wisdom_to_file)(f);
Chris@82: }