Mercurial > hg > sv-dependency-builds

annotate src/fftw-3.3.8/mpi/mpi-bench.c @ 83:ae30d91d2ffe

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Replace these with versions built using an older toolset (so as to avoid ABI compatibilities when linking on Ubuntu 14.04 for packaging purposes)
author Chris Cannam
date Fri, 07 Feb 2020 11:51:13 +0000
parents d0c2a83c1364
children
rev   line source
Chris@82 1 /**************************************************************************/
Chris@82 2 /* NOTE to users: this is the FFTW-MPI self-test and benchmark program.
Chris@82 3 It is probably NOT a good place to learn FFTW usage, since it has a
Chris@82 4 lot of added complexity in order to exercise and test the full API,
Chris@82 5 etcetera. We suggest reading the manual. */
Chris@82 6 /**************************************************************************/
Chris@82 7
Chris@82 8 #include <math.h>
Chris@82 9 #include <stdio.h>
Chris@82 10 #include <string.h>
Chris@82 11 #include "fftw3-mpi.h"
Chris@82 12 #include "tests/fftw-bench.h"
Chris@82 13
Chris@82 14 #if defined(BENCHFFT_SINGLE)
Chris@82 15 # define BENCH_MPI_TYPE MPI_FLOAT
Chris@82 16 #elif defined(BENCHFFT_LDOUBLE)
Chris@82 17 # define BENCH_MPI_TYPE MPI_LONG_DOUBLE
Chris@82 18 #elif defined(BENCHFFT_QUAD)
Chris@82 19 # error MPI quad-precision type is unknown
Chris@82 20 #else
Chris@82 21 # define BENCH_MPI_TYPE MPI_DOUBLE
Chris@82 22 #endif
Chris@82 23
Chris@82 24 #if SIZEOF_PTRDIFF_T == SIZEOF_INT
Chris@82 25 # define FFTW_MPI_PTRDIFF_T MPI_INT
Chris@82 26 #elif SIZEOF_PTRDIFF_T == SIZEOF_LONG
Chris@82 27 # define FFTW_MPI_PTRDIFF_T MPI_LONG
Chris@82 28 #elif SIZEOF_PTRDIFF_T == SIZEOF_LONG_LONG
Chris@82 29 # define FFTW_MPI_PTRDIFF_T MPI_LONG_LONG
Chris@82 30 #else
Chris@82 31 # error MPI type for ptrdiff_t is unknown
Chris@82 32 # define FFTW_MPI_PTRDIFF_T MPI_LONG
Chris@82 33 #endif
Chris@82 34
Chris@82 35 static const char *mkversion(void) { return FFTW(version); }
Chris@82 36 static const char *mkcc(void) { return FFTW(cc); }
Chris@82 37 static const char *mkcodelet_optim(void) { return FFTW(codelet_optim); }
Chris@82 38 static const char *mknproc(void) {
Chris@82 39 static char buf[32];
Chris@82 40 int ncpus;
Chris@82 41 MPI_Comm_size(MPI_COMM_WORLD, &ncpus);
Chris@82 42 #ifdef HAVE_SNPRINTF
Chris@82 43 snprintf(buf, 32, "%d", ncpus);
Chris@82 44 #else
Chris@82 45 sprintf(buf, "%d", ncpus);
Chris@82 46 #endif
Chris@82 47 return buf;
Chris@82 48 }
Chris@82 49
Chris@82 50 BEGIN_BENCH_DOC
Chris@82 51 BENCH_DOC("name", "fftw3_mpi")
Chris@82 52 BENCH_DOCF("version", mkversion)
Chris@82 53 BENCH_DOCF("cc", mkcc)
Chris@82 54 BENCH_DOCF("codelet-optim", mkcodelet_optim)
Chris@82 55 BENCH_DOCF("nproc", mknproc)
Chris@82 56 END_BENCH_DOC
Chris@82 57
Chris@82 58 static int n_pes = 1, my_pe = 0;
Chris@82 59
Chris@82 60 /* global variables describing the shape of the data and its distribution */
Chris@82 61 static int rnk;
Chris@82 62 static ptrdiff_t vn, iNtot, oNtot;
Chris@82 63 static ptrdiff_t *local_ni=0, *local_starti=0;
Chris@82 64 static ptrdiff_t *local_no=0, *local_starto=0;
Chris@82 65 static ptrdiff_t *all_local_ni=0, *all_local_starti=0; /* n_pes x rnk arrays */
Chris@82 66 static ptrdiff_t *all_local_no=0, *all_local_starto=0; /* n_pes x rnk arrays */
Chris@82 67 static ptrdiff_t *istrides = 0, *ostrides = 0;
Chris@82 68 static ptrdiff_t *total_ni=0, *total_no=0;
Chris@82 69 static int *isend_cnt = 0, *isend_off = 0; /* for MPI_Scatterv */
Chris@82 70 static int *orecv_cnt = 0, *orecv_off = 0; /* for MPI_Gatherv */
Chris@82 71
Chris@82 72 static bench_real *local_in = 0, *local_out = 0;
Chris@82 73 static bench_real *all_local_in = 0, *all_local_out = 0;
Chris@82 74 static int all_local_in_alloc = 0, all_local_out_alloc = 0;
Chris@82 75 static FFTW(plan) plan_scramble_in = 0, plan_unscramble_out = 0;
Chris@82 76
Chris@82 77 static void alloc_rnk(int rnk_) {
Chris@82 78 rnk = rnk_;
Chris@82 79 bench_free(local_ni);
Chris@82 80 if (rnk == 0)
Chris@82 81 local_ni = 0;
Chris@82 82 else
Chris@82 83 local_ni = (ptrdiff_t *) bench_malloc(sizeof(ptrdiff_t) * rnk
Chris@82 84 * (8 + n_pes * 4));
Chris@82 85
Chris@82 86 local_starti = local_ni + rnk;
Chris@82 87 local_no = local_ni + 2 * rnk;
Chris@82 88 local_starto = local_ni + 3 * rnk;
Chris@82 89 istrides = local_ni + 4 * rnk;
Chris@82 90 ostrides = local_ni + 5 * rnk;
Chris@82 91 total_ni = local_ni + 6 * rnk;
Chris@82 92 total_no = local_ni + 7 * rnk;
Chris@82 93 all_local_ni = local_ni + 8 * rnk;
Chris@82 94 all_local_starti = local_ni + (8 + n_pes) * rnk;
Chris@82 95 all_local_no = local_ni + (8 + 2 * n_pes) * rnk;
Chris@82 96 all_local_starto = local_ni + (8 + 3 * n_pes) * rnk;
Chris@82 97 }
Chris@82 98
Chris@82 99 static void setup_gather_scatter(void)
Chris@82 100 {
Chris@82 101 int i, j;
Chris@82 102 ptrdiff_t off;
Chris@82 103
Chris@82 104 MPI_Gather(local_ni, rnk, FFTW_MPI_PTRDIFF_T,
Chris@82 105 all_local_ni, rnk, FFTW_MPI_PTRDIFF_T,
Chris@82 106 0, MPI_COMM_WORLD);
Chris@82 107 MPI_Bcast(all_local_ni, rnk*n_pes, FFTW_MPI_PTRDIFF_T, 0, MPI_COMM_WORLD);
Chris@82 108 MPI_Gather(local_starti, rnk, FFTW_MPI_PTRDIFF_T,
Chris@82 109 all_local_starti, rnk, FFTW_MPI_PTRDIFF_T,
Chris@82 110 0, MPI_COMM_WORLD);
Chris@82 111 MPI_Bcast(all_local_starti, rnk*n_pes, FFTW_MPI_PTRDIFF_T, 0, MPI_COMM_WORLD);
Chris@82 112
Chris@82 113 MPI_Gather(local_no, rnk, FFTW_MPI_PTRDIFF_T,
Chris@82 114 all_local_no, rnk, FFTW_MPI_PTRDIFF_T,
Chris@82 115 0, MPI_COMM_WORLD);
Chris@82 116 MPI_Bcast(all_local_no, rnk*n_pes, FFTW_MPI_PTRDIFF_T, 0, MPI_COMM_WORLD);
Chris@82 117 MPI_Gather(local_starto, rnk, FFTW_MPI_PTRDIFF_T,
Chris@82 118 all_local_starto, rnk, FFTW_MPI_PTRDIFF_T,
Chris@82 119 0, MPI_COMM_WORLD);
Chris@82 120 MPI_Bcast(all_local_starto, rnk*n_pes, FFTW_MPI_PTRDIFF_T, 0, MPI_COMM_WORLD);
Chris@82 121
Chris@82 122 off = 0;
Chris@82 123 for (i = 0; i < n_pes; ++i) {
Chris@82 124 ptrdiff_t N = vn;
Chris@82 125 for (j = 0; j < rnk; ++j)
Chris@82 126 N *= all_local_ni[i * rnk + j];
Chris@82 127 isend_cnt[i] = N;
Chris@82 128 isend_off[i] = off;
Chris@82 129 off += N;
Chris@82 130 }
Chris@82 131 iNtot = off;
Chris@82 132 all_local_in_alloc = 1;
Chris@82 133
Chris@82 134 istrides[rnk - 1] = vn;
Chris@82 135 for (j = rnk - 2; j >= 0; --j)
Chris@82 136 istrides[j] = total_ni[j + 1] * istrides[j + 1];
Chris@82 137
Chris@82 138 off = 0;
Chris@82 139 for (i = 0; i < n_pes; ++i) {
Chris@82 140 ptrdiff_t N = vn;
Chris@82 141 for (j = 0; j < rnk; ++j)
Chris@82 142 N *= all_local_no[i * rnk + j];
Chris@82 143 orecv_cnt[i] = N;
Chris@82 144 orecv_off[i] = off;
Chris@82 145 off += N;
Chris@82 146 }
Chris@82 147 oNtot = off;
Chris@82 148 all_local_out_alloc = 1;
Chris@82 149
Chris@82 150 ostrides[rnk - 1] = vn;
Chris@82 151 for (j = rnk - 2; j >= 0; --j)
Chris@82 152 ostrides[j] = total_no[j + 1] * ostrides[j + 1];
Chris@82 153 }
Chris@82 154
Chris@82 155 static void copy_block_out(const bench_real *in,
Chris@82 156 int rnk, ptrdiff_t *n, ptrdiff_t *start,
Chris@82 157 ptrdiff_t is, ptrdiff_t *os, ptrdiff_t vn,
Chris@82 158 bench_real *out)
Chris@82 159 {
Chris@82 160 ptrdiff_t i;
Chris@82 161 if (rnk == 0) {
Chris@82 162 for (i = 0; i < vn; ++i)
Chris@82 163 out[i] = in[i];
Chris@82 164 }
Chris@82 165 else if (rnk == 1) { /* this case is just an optimization */
Chris@82 166 ptrdiff_t j;
Chris@82 167 out += start[0] * os[0];
Chris@82 168 for (j = 0; j < n[0]; ++j) {
Chris@82 169 for (i = 0; i < vn; ++i)
Chris@82 170 out[i] = in[i];
Chris@82 171 in += is;
Chris@82 172 out += os[0];
Chris@82 173 }
Chris@82 174 }
Chris@82 175 else {
Chris@82 176 /* we should do n[0] for locality, but this way is simpler to code */
Chris@82 177 for (i = 0; i < n[rnk - 1]; ++i)
Chris@82 178 copy_block_out(in + i * is,
Chris@82 179 rnk - 1, n, start, is * n[rnk - 1], os, vn,
Chris@82 180 out + (start[rnk - 1] + i) * os[rnk - 1]);
Chris@82 181 }
Chris@82 182 }
Chris@82 183
Chris@82 184 static void copy_block_in(bench_real *in,
Chris@82 185 int rnk, ptrdiff_t *n, ptrdiff_t *start,
Chris@82 186 ptrdiff_t is, ptrdiff_t *os, ptrdiff_t vn,
Chris@82 187 const bench_real *out)
Chris@82 188 {
Chris@82 189 ptrdiff_t i;
Chris@82 190 if (rnk == 0) {
Chris@82 191 for (i = 0; i < vn; ++i)
Chris@82 192 in[i] = out[i];
Chris@82 193 }
Chris@82 194 else if (rnk == 1) { /* this case is just an optimization */
Chris@82 195 ptrdiff_t j;
Chris@82 196 out += start[0] * os[0];
Chris@82 197 for (j = 0; j < n[0]; ++j) {
Chris@82 198 for (i = 0; i < vn; ++i)
Chris@82 199 in[i] = out[i];
Chris@82 200 in += is;
Chris@82 201 out += os[0];
Chris@82 202 }
Chris@82 203 }
Chris@82 204 else {
Chris@82 205 /* we should do n[0] for locality, but this way is simpler to code */
Chris@82 206 for (i = 0; i < n[rnk - 1]; ++i)
Chris@82 207 copy_block_in(in + i * is,
Chris@82 208 rnk - 1, n, start, is * n[rnk - 1], os, vn,
Chris@82 209 out + (start[rnk - 1] + i) * os[rnk - 1]);
Chris@82 210 }
Chris@82 211 }
Chris@82 212
Chris@82 213 static void do_scatter_in(bench_real *in)
Chris@82 214 {
Chris@82 215 bench_real *ali;
Chris@82 216 int i;
Chris@82 217 if (all_local_in_alloc) {
Chris@82 218 bench_free(all_local_in);
Chris@82 219 all_local_in = (bench_real*) bench_malloc(iNtot*sizeof(bench_real));
Chris@82 220 all_local_in_alloc = 0;
Chris@82 221 }
Chris@82 222 ali = all_local_in;
Chris@82 223 for (i = 0; i < n_pes; ++i) {
Chris@82 224 copy_block_in(ali,
Chris@82 225 rnk, all_local_ni + i * rnk,
Chris@82 226 all_local_starti + i * rnk,
Chris@82 227 vn, istrides, vn,
Chris@82 228 in);
Chris@82 229 ali += isend_cnt[i];
Chris@82 230 }
Chris@82 231 MPI_Scatterv(all_local_in, isend_cnt, isend_off, BENCH_MPI_TYPE,
Chris@82 232 local_in, isend_cnt[my_pe], BENCH_MPI_TYPE,
Chris@82 233 0, MPI_COMM_WORLD);
Chris@82 234 }
Chris@82 235
Chris@82 236 static void do_gather_out(bench_real *out)
Chris@82 237 {
Chris@82 238 bench_real *alo;
Chris@82 239 int i;
Chris@82 240
Chris@82 241 if (all_local_out_alloc) {
Chris@82 242 bench_free(all_local_out);
Chris@82 243 all_local_out = (bench_real*) bench_malloc(oNtot*sizeof(bench_real));
Chris@82 244 all_local_out_alloc = 0;
Chris@82 245 }
Chris@82 246 MPI_Gatherv(local_out, orecv_cnt[my_pe], BENCH_MPI_TYPE,
Chris@82 247 all_local_out, orecv_cnt, orecv_off, BENCH_MPI_TYPE,
Chris@82 248 0, MPI_COMM_WORLD);
Chris@82 249 MPI_Bcast(all_local_out, oNtot, BENCH_MPI_TYPE, 0, MPI_COMM_WORLD);
Chris@82 250 alo = all_local_out;
Chris@82 251 for (i = 0; i < n_pes; ++i) {
Chris@82 252 copy_block_out(alo,
Chris@82 253 rnk, all_local_no + i * rnk,
Chris@82 254 all_local_starto + i * rnk,
Chris@82 255 vn, ostrides, vn,
Chris@82 256 out);
Chris@82 257 alo += orecv_cnt[i];
Chris@82 258 }
Chris@82 259 }
Chris@82 260
Chris@82 261 static void alloc_local(ptrdiff_t nreal, int inplace)
Chris@82 262 {
Chris@82 263 bench_free(local_in);
Chris@82 264 if (local_out != local_in) bench_free(local_out);
Chris@82 265 local_in = local_out = 0;
Chris@82 266 if (nreal > 0) {
Chris@82 267 ptrdiff_t i;
Chris@82 268 local_in = (bench_real*) bench_malloc(nreal * sizeof(bench_real));
Chris@82 269 if (inplace)
Chris@82 270 local_out = local_in;
Chris@82 271 else
Chris@82 272 local_out = (bench_real*) bench_malloc(nreal * sizeof(bench_real));
Chris@82 273 for (i = 0; i < nreal; ++i) local_in[i] = local_out[i] = 0.0;
Chris@82 274 }
Chris@82 275 }
Chris@82 276
Chris@82 277 void after_problem_rcopy_from(bench_problem *p, bench_real *ri)
Chris@82 278 {
Chris@82 279 UNUSED(p);
Chris@82 280 do_scatter_in(ri);
Chris@82 281 if (plan_scramble_in) FFTW(execute)(plan_scramble_in);
Chris@82 282 }
Chris@82 283
Chris@82 284 void after_problem_rcopy_to(bench_problem *p, bench_real *ro)
Chris@82 285 {
Chris@82 286 UNUSED(p);
Chris@82 287 if (plan_unscramble_out) FFTW(execute)(plan_unscramble_out);
Chris@82 288 do_gather_out(ro);
Chris@82 289 }
Chris@82 290
Chris@82 291 void after_problem_ccopy_from(bench_problem *p, bench_real *ri, bench_real *ii)
Chris@82 292 {
Chris@82 293 UNUSED(ii);
Chris@82 294 after_problem_rcopy_from(p, ri);
Chris@82 295 }
Chris@82 296
Chris@82 297 void after_problem_ccopy_to(bench_problem *p, bench_real *ro, bench_real *io)
Chris@82 298 {
Chris@82 299 UNUSED(io);
Chris@82 300 after_problem_rcopy_to(p, ro);
Chris@82 301 }
Chris@82 302
Chris@82 303 void after_problem_hccopy_from(bench_problem *p, bench_real *ri, bench_real *ii)
Chris@82 304 {
Chris@82 305 UNUSED(ii);
Chris@82 306 after_problem_rcopy_from(p, ri);
Chris@82 307 }
Chris@82 308
Chris@82 309 void after_problem_hccopy_to(bench_problem *p, bench_real *ro, bench_real *io)
Chris@82 310 {
Chris@82 311 UNUSED(io);
Chris@82 312 after_problem_rcopy_to(p, ro);
Chris@82 313 }
Chris@82 314
Chris@82 315 static FFTW(plan) mkplan_transpose_local(ptrdiff_t nx, ptrdiff_t ny,
Chris@82 316 ptrdiff_t vn,
Chris@82 317 bench_real *in, bench_real *out)
Chris@82 318 {
Chris@82 319 FFTW(iodim64) hdims[3];
Chris@82 320 FFTW(r2r_kind) k[3];
Chris@82 321 FFTW(plan) pln;
Chris@82 322
Chris@82 323 hdims[0].n = nx;
Chris@82 324 hdims[0].is = ny * vn;
Chris@82 325 hdims[0].os = vn;
Chris@82 326 hdims[1].n = ny;
Chris@82 327 hdims[1].is = vn;
Chris@82 328 hdims[1].os = nx * vn;
Chris@82 329 hdims[2].n = vn;
Chris@82 330 hdims[2].is = 1;
Chris@82 331 hdims[2].os = 1;
Chris@82 332 k[0] = k[1] = k[2] = FFTW_R2HC;
Chris@82 333 pln = FFTW(plan_guru64_r2r)(0, 0, 3, hdims, in, out, k, FFTW_ESTIMATE);
Chris@82 334 BENCH_ASSERT(pln != 0);
Chris@82 335 return pln;
Chris@82 336 }
Chris@82 337
Chris@82 338 static int tensor_rowmajor_transposedp(bench_tensor *t)
Chris@82 339 {
Chris@82 340 bench_iodim *d;
Chris@82 341 int i;
Chris@82 342
Chris@82 343 BENCH_ASSERT(BENCH_FINITE_RNK(t->rnk));
Chris@82 344 if (t->rnk < 2)
Chris@82 345 return 0;
Chris@82 346
Chris@82 347 d = t->dims;
Chris@82 348 if (d[0].is != d[1].is * d[1].n
Chris@82 349 || d[0].os != d[1].is
Chris@82 350 || d[1].os != d[0].os * d[0].n)
Chris@82 351 return 0;
Chris@82 352 if (t->rnk > 2 && d[1].is != d[2].is * d[2].n)
Chris@82 353 return 0;
Chris@82 354 for (i = 2; i + 1 < t->rnk; ++i) {
Chris@82 355 d = t->dims + i;
Chris@82 356 if (d[0].is != d[1].is * d[1].n
Chris@82 357 || d[0].os != d[1].os * d[1].n)
Chris@82 358 return 0;
Chris@82 359 }
Chris@82 360
Chris@82 361 if (t->rnk > 2 && t->dims[t->rnk-1].is != t->dims[t->rnk-1].os)
Chris@82 362 return 0;
Chris@82 363 return 1;
Chris@82 364 }
Chris@82 365
Chris@82 366 static int tensor_contiguousp(bench_tensor *t, int s)
Chris@82 367 {
Chris@82 368 return (t->dims[t->rnk-1].is == s
Chris@82 369 && ((tensor_rowmajorp(t) &&
Chris@82 370 t->dims[t->rnk-1].is == t->dims[t->rnk-1].os)
Chris@82 371 || tensor_rowmajor_transposedp(t)));
Chris@82 372 }
Chris@82 373
Chris@82 374 static FFTW(plan) mkplan_complex(bench_problem *p, unsigned flags)
Chris@82 375 {
Chris@82 376 FFTW(plan) pln = 0;
Chris@82 377 int i;
Chris@82 378 ptrdiff_t ntot;
Chris@82 379
Chris@82 380 vn = p->vecsz->rnk == 1 ? p->vecsz->dims[0].n : 1;
Chris@82 381
Chris@82 382 if (p->sz->rnk < 1
Chris@82 383 || p->split
Chris@82 384 || !tensor_contiguousp(p->sz, vn)
Chris@82 385 || tensor_rowmajor_transposedp(p->sz)
Chris@82 386 || p->vecsz->rnk > 1
Chris@82 387 || (p->vecsz->rnk == 1 && (p->vecsz->dims[0].is != 1
Chris@82 388 || p->vecsz->dims[0].os != 1)))
Chris@82 389 return 0;
Chris@82 390
Chris@82 391 alloc_rnk(p->sz->rnk);
Chris@82 392 for (i = 0; i < rnk; ++i) {
Chris@82 393 total_ni[i] = total_no[i] = p->sz->dims[i].n;
Chris@82 394 local_ni[i] = local_no[i] = total_ni[i];
Chris@82 395 local_starti[i] = local_starto[i] = 0;
Chris@82 396 }
Chris@82 397 if (rnk > 1) {
Chris@82 398 ptrdiff_t n, start, nT, startT;
Chris@82 399 ntot = FFTW(mpi_local_size_many_transposed)
Chris@82 400 (p->sz->rnk, total_ni, vn,
Chris@82 401 FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
Chris@82 402 MPI_COMM_WORLD,
Chris@82 403 &n, &start, &nT, &startT);
Chris@82 404 if (flags & FFTW_MPI_TRANSPOSED_IN) {
Chris@82 405 local_ni[1] = nT;
Chris@82 406 local_starti[1] = startT;
Chris@82 407 }
Chris@82 408 else {
Chris@82 409 local_ni[0] = n;
Chris@82 410 local_starti[0] = start;
Chris@82 411 }
Chris@82 412 if (flags & FFTW_MPI_TRANSPOSED_OUT) {
Chris@82 413 local_no[1] = nT;
Chris@82 414 local_starto[1] = startT;
Chris@82 415 }
Chris@82 416 else {
Chris@82 417 local_no[0] = n;
Chris@82 418 local_starto[0] = start;
Chris@82 419 }
Chris@82 420 }
Chris@82 421 else if (rnk == 1) {
Chris@82 422 ntot = FFTW(mpi_local_size_many_1d)
Chris@82 423 (total_ni[0], vn, MPI_COMM_WORLD, p->sign, flags,
Chris@82 424 local_ni, local_starti, local_no, local_starto);
Chris@82 425 }
Chris@82 426 alloc_local(ntot * 2, p->in == p->out);
Chris@82 427
Chris@82 428 pln = FFTW(mpi_plan_many_dft)(p->sz->rnk, total_ni, vn,
Chris@82 429 FFTW_MPI_DEFAULT_BLOCK,
Chris@82 430 FFTW_MPI_DEFAULT_BLOCK,
Chris@82 431 (FFTW(complex) *) local_in,
Chris@82 432 (FFTW(complex) *) local_out,
Chris@82 433 MPI_COMM_WORLD, p->sign, flags);
Chris@82 434
Chris@82 435 vn *= 2;
Chris@82 436
Chris@82 437 if (rnk > 1) {
Chris@82 438 ptrdiff_t nrest = 1;
Chris@82 439 for (i = 2; i < rnk; ++i) nrest *= p->sz->dims[i].n;
Chris@82 440 if (flags & FFTW_MPI_TRANSPOSED_IN)
Chris@82 441 plan_scramble_in = mkplan_transpose_local(
Chris@82 442 p->sz->dims[0].n, local_ni[1], vn * nrest,
Chris@82 443 local_in, local_in);
Chris@82 444 if (flags & FFTW_MPI_TRANSPOSED_OUT)
Chris@82 445 plan_unscramble_out = mkplan_transpose_local(
Chris@82 446 local_no[1], p->sz->dims[0].n, vn * nrest,
Chris@82 447 local_out, local_out);
Chris@82 448 }
Chris@82 449
Chris@82 450 return pln;
Chris@82 451 }
Chris@82 452
Chris@82 453 static int tensor_real_contiguousp(bench_tensor *t, int sign, int s)
Chris@82 454 {
Chris@82 455 return (t->dims[t->rnk-1].is == s
Chris@82 456 && ((tensor_real_rowmajorp(t, sign, 1) &&
Chris@82 457 t->dims[t->rnk-1].is == t->dims[t->rnk-1].os)));
Chris@82 458 }
Chris@82 459
Chris@82 460 static FFTW(plan) mkplan_real(bench_problem *p, unsigned flags)
Chris@82 461 {
Chris@82 462 FFTW(plan) pln = 0;
Chris@82 463 int i;
Chris@82 464 ptrdiff_t ntot;
Chris@82 465
Chris@82 466 vn = p->vecsz->rnk == 1 ? p->vecsz->dims[0].n : 1;
Chris@82 467
Chris@82 468 if (p->sz->rnk < 2
Chris@82 469 || p->split
Chris@82 470 || !tensor_real_contiguousp(p->sz, p->sign, vn)
Chris@82 471 || tensor_rowmajor_transposedp(p->sz)
Chris@82 472 || p->vecsz->rnk > 1
Chris@82 473 || (p->vecsz->rnk == 1 && (p->vecsz->dims[0].is != 1
Chris@82 474 || p->vecsz->dims[0].os != 1)))
Chris@82 475 return 0;
Chris@82 476
Chris@82 477 alloc_rnk(p->sz->rnk);
Chris@82 478 for (i = 0; i < rnk; ++i) {
Chris@82 479 total_ni[i] = total_no[i] = p->sz->dims[i].n;
Chris@82 480 local_ni[i] = local_no[i] = total_ni[i];
Chris@82 481 local_starti[i] = local_starto[i] = 0;
Chris@82 482 }
Chris@82 483 local_ni[rnk-1] = local_no[rnk-1] = total_ni[rnk-1] = total_no[rnk-1]
Chris@82 484 = p->sz->dims[rnk-1].n / 2 + 1;
Chris@82 485 {
Chris@82 486 ptrdiff_t n, start, nT, startT;
Chris@82 487 ntot = FFTW(mpi_local_size_many_transposed)
Chris@82 488 (p->sz->rnk, total_ni, vn,
Chris@82 489 FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
Chris@82 490 MPI_COMM_WORLD,
Chris@82 491 &n, &start, &nT, &startT);
Chris@82 492 if (flags & FFTW_MPI_TRANSPOSED_IN) {
Chris@82 493 local_ni[1] = nT;
Chris@82 494 local_starti[1] = startT;
Chris@82 495 }
Chris@82 496 else {
Chris@82 497 local_ni[0] = n;
Chris@82 498 local_starti[0] = start;
Chris@82 499 }
Chris@82 500 if (flags & FFTW_MPI_TRANSPOSED_OUT) {
Chris@82 501 local_no[1] = nT;
Chris@82 502 local_starto[1] = startT;
Chris@82 503 }
Chris@82 504 else {
Chris@82 505 local_no[0] = n;
Chris@82 506 local_starto[0] = start;
Chris@82 507 }
Chris@82 508 }
Chris@82 509 alloc_local(ntot * 2, p->in == p->out);
Chris@82 510
Chris@82 511 total_ni[rnk - 1] = p->sz->dims[rnk - 1].n;
Chris@82 512 if (p->sign < 0)
Chris@82 513 pln = FFTW(mpi_plan_many_dft_r2c)(p->sz->rnk, total_ni, vn,
Chris@82 514 FFTW_MPI_DEFAULT_BLOCK,
Chris@82 515 FFTW_MPI_DEFAULT_BLOCK,
Chris@82 516 local_in,
Chris@82 517 (FFTW(complex) *) local_out,
Chris@82 518 MPI_COMM_WORLD, flags);
Chris@82 519 else
Chris@82 520 pln = FFTW(mpi_plan_many_dft_c2r)(p->sz->rnk, total_ni, vn,
Chris@82 521 FFTW_MPI_DEFAULT_BLOCK,
Chris@82 522 FFTW_MPI_DEFAULT_BLOCK,
Chris@82 523 (FFTW(complex) *) local_in,
Chris@82 524 local_out,
Chris@82 525 MPI_COMM_WORLD, flags);
Chris@82 526
Chris@82 527 total_ni[rnk - 1] = p->sz->dims[rnk - 1].n / 2 + 1;
Chris@82 528 vn *= 2;
Chris@82 529
Chris@82 530 {
Chris@82 531 ptrdiff_t nrest = 1;
Chris@82 532 for (i = 2; i < rnk; ++i) nrest *= total_ni[i];
Chris@82 533 if (flags & FFTW_MPI_TRANSPOSED_IN)
Chris@82 534 plan_scramble_in = mkplan_transpose_local(
Chris@82 535 total_ni[0], local_ni[1], vn * nrest,
Chris@82 536 local_in, local_in);
Chris@82 537 if (flags & FFTW_MPI_TRANSPOSED_OUT)
Chris@82 538 plan_unscramble_out = mkplan_transpose_local(
Chris@82 539 local_no[1], total_ni[0], vn * nrest,
Chris@82 540 local_out, local_out);
Chris@82 541 }
Chris@82 542
Chris@82 543 return pln;
Chris@82 544 }
Chris@82 545
Chris@82 546 static FFTW(plan) mkplan_transpose(bench_problem *p, unsigned flags)
Chris@82 547 {
Chris@82 548 ptrdiff_t ntot, nx, ny;
Chris@82 549 int ix=0, iy=1, i;
Chris@82 550 const bench_iodim *d = p->vecsz->dims;
Chris@82 551 FFTW(plan) pln;
Chris@82 552
Chris@82 553 if (p->vecsz->rnk == 3) {
Chris@82 554 for (i = 0; i < 3; ++i)
Chris@82 555 if (d[i].is == 1 && d[i].os == 1) {
Chris@82 556 vn = d[i].n;
Chris@82 557 ix = (i + 1) % 3;
Chris@82 558 iy = (i + 2) % 3;
Chris@82 559 break;
Chris@82 560 }
Chris@82 561 if (i == 3) return 0;
Chris@82 562 }
Chris@82 563 else {
Chris@82 564 vn = 1;
Chris@82 565 ix = 0;
Chris@82 566 iy = 1;
Chris@82 567 }
Chris@82 568
Chris@82 569 if (d[ix].is == d[iy].n * vn && d[ix].os == vn
Chris@82 570 && d[iy].os == d[ix].n * vn && d[iy].is == vn) {
Chris@82 571 nx = d[ix].n;
Chris@82 572 ny = d[iy].n;
Chris@82 573 }
Chris@82 574 else if (d[iy].is == d[ix].n * vn && d[iy].os == vn
Chris@82 575 && d[ix].os == d[iy].n * vn && d[ix].is == vn) {
Chris@82 576 nx = d[iy].n;
Chris@82 577 ny = d[ix].n;
Chris@82 578 }
Chris@82 579 else
Chris@82 580 return 0;
Chris@82 581
Chris@82 582 alloc_rnk(2);
Chris@82 583 ntot = vn * FFTW(mpi_local_size_2d_transposed)(nx, ny, MPI_COMM_WORLD,
Chris@82 584 &local_ni[0],
Chris@82 585 &local_starti[0],
Chris@82 586 &local_no[0],
Chris@82 587 &local_starto[0]);
Chris@82 588 local_ni[1] = ny;
Chris@82 589 local_starti[1] = 0;
Chris@82 590 local_no[1] = nx;
Chris@82 591 local_starto[1] = 0;
Chris@82 592 total_ni[0] = nx; total_ni[1] = ny;
Chris@82 593 total_no[1] = nx; total_no[0] = ny;
Chris@82 594 alloc_local(ntot, p->in == p->out);
Chris@82 595
Chris@82 596 pln = FFTW(mpi_plan_many_transpose)(nx, ny, vn,
Chris@82 597 FFTW_MPI_DEFAULT_BLOCK,
Chris@82 598 FFTW_MPI_DEFAULT_BLOCK,
Chris@82 599 local_in, local_out,
Chris@82 600 MPI_COMM_WORLD, flags);
Chris@82 601
Chris@82 602 if (flags & FFTW_MPI_TRANSPOSED_IN)
Chris@82 603 plan_scramble_in = mkplan_transpose_local(local_ni[0], ny, vn,
Chris@82 604 local_in, local_in);
Chris@82 605 if (flags & FFTW_MPI_TRANSPOSED_OUT)
Chris@82 606 plan_unscramble_out = mkplan_transpose_local
Chris@82 607 (nx, local_no[0], vn, local_out, local_out);
Chris@82 608
Chris@82 609 #if 0
Chris@82 610 if (pln && vn == 1) {
Chris@82 611 int i, j;
Chris@82 612 bench_real *ri = (bench_real *) p->in;
Chris@82 613 bench_real *ro = (bench_real *) p->out;
Chris@82 614 if (!ri || !ro) return pln;
Chris@82 615 setup_gather_scatter();
Chris@82 616 for (i = 0; i < nx * ny; ++i)
Chris@82 617 ri[i] = i;
Chris@82 618 after_problem_rcopy_from(p, ri);
Chris@82 619 FFTW(execute)(pln);
Chris@82 620 after_problem_rcopy_to(p, ro);
Chris@82 621 if (my_pe == 0) {
Chris@82 622 for (i = 0; i < nx; ++i) {
Chris@82 623 for (j = 0; j < ny; ++j)
Chris@82 624 printf(" %3g", ro[j * nx + i]);
Chris@82 625 printf("\n");
Chris@82 626 }
Chris@82 627 }
Chris@82 628 }
Chris@82 629 #endif
Chris@82 630
Chris@82 631 return pln;
Chris@82 632 }
Chris@82 633
Chris@82 634 static FFTW(plan) mkplan_r2r(bench_problem *p, unsigned flags)
Chris@82 635 {
Chris@82 636 FFTW(plan) pln = 0;
Chris@82 637 int i;
Chris@82 638 ptrdiff_t ntot;
Chris@82 639 FFTW(r2r_kind) *k;
Chris@82 640
Chris@82 641 if ((p->sz->rnk == 0 || (p->sz->rnk == 1 && p->sz->dims[0].n == 1))
Chris@82 642 && p->vecsz->rnk >= 2 && p->vecsz->rnk <= 3)
Chris@82 643 return mkplan_transpose(p, flags);
Chris@82 644
Chris@82 645 vn = p->vecsz->rnk == 1 ? p->vecsz->dims[0].n : 1;
Chris@82 646
Chris@82 647 if (p->sz->rnk < 1
Chris@82 648 || p->split
Chris@82 649 || !tensor_contiguousp(p->sz, vn)
Chris@82 650 || tensor_rowmajor_transposedp(p->sz)
Chris@82 651 || p->vecsz->rnk > 1
Chris@82 652 || (p->vecsz->rnk == 1 && (p->vecsz->dims[0].is != 1
Chris@82 653 || p->vecsz->dims[0].os != 1)))
Chris@82 654 return 0;
Chris@82 655
Chris@82 656 alloc_rnk(p->sz->rnk);
Chris@82 657 for (i = 0; i < rnk; ++i) {
Chris@82 658 total_ni[i] = total_no[i] = p->sz->dims[i].n;
Chris@82 659 local_ni[i] = local_no[i] = total_ni[i];
Chris@82 660 local_starti[i] = local_starto[i] = 0;
Chris@82 661 }
Chris@82 662 if (rnk > 1) {
Chris@82 663 ptrdiff_t n, start, nT, startT;
Chris@82 664 ntot = FFTW(mpi_local_size_many_transposed)
Chris@82 665 (p->sz->rnk, total_ni, vn,
Chris@82 666 FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK,
Chris@82 667 MPI_COMM_WORLD,
Chris@82 668 &n, &start, &nT, &startT);
Chris@82 669 if (flags & FFTW_MPI_TRANSPOSED_IN) {
Chris@82 670 local_ni[1] = nT;
Chris@82 671 local_starti[1] = startT;
Chris@82 672 }
Chris@82 673 else {
Chris@82 674 local_ni[0] = n;
Chris@82 675 local_starti[0] = start;
Chris@82 676 }
Chris@82 677 if (flags & FFTW_MPI_TRANSPOSED_OUT) {
Chris@82 678 local_no[1] = nT;
Chris@82 679 local_starto[1] = startT;
Chris@82 680 }
Chris@82 681 else {
Chris@82 682 local_no[0] = n;
Chris@82 683 local_starto[0] = start;
Chris@82 684 }
Chris@82 685 }
Chris@82 686 else if (rnk == 1) {
Chris@82 687 ntot = FFTW(mpi_local_size_many_1d)
Chris@82 688 (total_ni[0], vn, MPI_COMM_WORLD, p->sign, flags,
Chris@82 689 local_ni, local_starti, local_no, local_starto);
Chris@82 690 }
Chris@82 691 alloc_local(ntot, p->in == p->out);
Chris@82 692
Chris@82 693 k = (FFTW(r2r_kind) *) bench_malloc(sizeof(FFTW(r2r_kind)) * p->sz->rnk);
Chris@82 694 for (i = 0; i < p->sz->rnk; ++i)
Chris@82 695 switch (p->k[i]) {
Chris@82 696 case R2R_R2HC: k[i] = FFTW_R2HC; break;
Chris@82 697 case R2R_HC2R: k[i] = FFTW_HC2R; break;
Chris@82 698 case R2R_DHT: k[i] = FFTW_DHT; break;
Chris@82 699 case R2R_REDFT00: k[i] = FFTW_REDFT00; break;
Chris@82 700 case R2R_REDFT01: k[i] = FFTW_REDFT01; break;
Chris@82 701 case R2R_REDFT10: k[i] = FFTW_REDFT10; break;
Chris@82 702 case R2R_REDFT11: k[i] = FFTW_REDFT11; break;
Chris@82 703 case R2R_RODFT00: k[i] = FFTW_RODFT00; break;
Chris@82 704 case R2R_RODFT01: k[i] = FFTW_RODFT01; break;
Chris@82 705 case R2R_RODFT10: k[i] = FFTW_RODFT10; break;
Chris@82 706 case R2R_RODFT11: k[i] = FFTW_RODFT11; break;
Chris@82 707 default: BENCH_ASSERT(0);
Chris@82 708 }
Chris@82 709
Chris@82 710 pln = FFTW(mpi_plan_many_r2r)(p->sz->rnk, total_ni, vn,
Chris@82 711 FFTW_MPI_DEFAULT_BLOCK,
Chris@82 712 FFTW_MPI_DEFAULT_BLOCK,
Chris@82 713 local_in, local_out,
Chris@82 714 MPI_COMM_WORLD, k, flags);
Chris@82 715 bench_free(k);
Chris@82 716
Chris@82 717 if (rnk > 1) {
Chris@82 718 ptrdiff_t nrest = 1;
Chris@82 719 for (i = 2; i < rnk; ++i) nrest *= p->sz->dims[i].n;
Chris@82 720 if (flags & FFTW_MPI_TRANSPOSED_IN)
Chris@82 721 plan_scramble_in = mkplan_transpose_local(
Chris@82 722 p->sz->dims[0].n, local_ni[1], vn * nrest,
Chris@82 723 local_in, local_in);
Chris@82 724 if (flags & FFTW_MPI_TRANSPOSED_OUT)
Chris@82 725 plan_unscramble_out = mkplan_transpose_local(
Chris@82 726 local_no[1], p->sz->dims[0].n, vn * nrest,
Chris@82 727 local_out, local_out);
Chris@82 728 }
Chris@82 729
Chris@82 730 return pln;
Chris@82 731 }
Chris@82 732
Chris@82 733 FFTW(plan) mkplan(bench_problem *p, unsigned flags)
Chris@82 734 {
Chris@82 735 FFTW(plan) pln = 0;
Chris@82 736 FFTW(destroy_plan)(plan_scramble_in); plan_scramble_in = 0;
Chris@82 737 FFTW(destroy_plan)(plan_unscramble_out); plan_unscramble_out = 0;
Chris@82 738 if (p->scrambled_in) {
Chris@82 739 if (p->sz->rnk == 1 && p->sz->dims[0].n != 1)
Chris@82 740 flags |= FFTW_MPI_SCRAMBLED_IN;
Chris@82 741 else
Chris@82 742 flags |= FFTW_MPI_TRANSPOSED_IN;
Chris@82 743 }
Chris@82 744 if (p->scrambled_out) {
Chris@82 745 if (p->sz->rnk == 1 && p->sz->dims[0].n != 1)
Chris@82 746 flags |= FFTW_MPI_SCRAMBLED_OUT;
Chris@82 747 else
Chris@82 748 flags |= FFTW_MPI_TRANSPOSED_OUT;
Chris@82 749 }
Chris@82 750 switch (p->kind) {
Chris@82 751 case PROBLEM_COMPLEX:
Chris@82 752 pln =mkplan_complex(p, flags);
Chris@82 753 break;
Chris@82 754 case PROBLEM_REAL:
Chris@82 755 pln = mkplan_real(p, flags);
Chris@82 756 break;
Chris@82 757 case PROBLEM_R2R:
Chris@82 758 pln = mkplan_r2r(p, flags);
Chris@82 759 break;
Chris@82 760 default: BENCH_ASSERT(0);
Chris@82 761 }
Chris@82 762 if (pln) setup_gather_scatter();
Chris@82 763 return pln;
Chris@82 764 }
Chris@82 765
Chris@82 766 void main_init(int *argc, char ***argv)
Chris@82 767 {
Chris@82 768 #ifdef HAVE_SMP
Chris@82 769 # if MPI_VERSION >= 2 /* for MPI_Init_thread */
Chris@82 770 int provided;
Chris@82 771 MPI_Init_thread(argc, argv, MPI_THREAD_FUNNELED, &provided);
Chris@82 772 threads_ok = provided >= MPI_THREAD_FUNNELED;
Chris@82 773 # else
Chris@82 774 MPI_Init(argc, argv);
Chris@82 775 threads_ok = 0;
Chris@82 776 # endif
Chris@82 777 #else
Chris@82 778 MPI_Init(argc, argv);
Chris@82 779 #endif
Chris@82 780 MPI_Comm_rank(MPI_COMM_WORLD, &my_pe);
Chris@82 781 MPI_Comm_size(MPI_COMM_WORLD, &n_pes);
Chris@82 782 if (my_pe != 0) verbose = -999;
Chris@82 783 no_speed_allocation = 1; /* so we can benchmark transforms > memory */
Chris@82 784 always_pad_real = 1; /* out-of-place real transforms are padded */
Chris@82 785 isend_cnt = (int *) bench_malloc(sizeof(int) * n_pes);
Chris@82 786 isend_off = (int *) bench_malloc(sizeof(int) * n_pes);
Chris@82 787 orecv_cnt = (int *) bench_malloc(sizeof(int) * n_pes);
Chris@82 788 orecv_off = (int *) bench_malloc(sizeof(int) * n_pes);
Chris@82 789
Chris@82 790 /* init_threads must be called before any other FFTW function,
Chris@82 791 including mpi_init, because it has to register the threads hooks
Chris@82 792 before the planner is initalized */
Chris@82 793 #ifdef HAVE_SMP
Chris@82 794 if (threads_ok) { BENCH_ASSERT(FFTW(init_threads)()); }
Chris@82 795 #endif
Chris@82 796 FFTW(mpi_init)();
Chris@82 797 }
Chris@82 798
Chris@82 799 void initial_cleanup(void)
Chris@82 800 {
Chris@82 801 alloc_rnk(0);
Chris@82 802 alloc_local(0, 0);
Chris@82 803 bench_free(all_local_in); all_local_in = 0;
Chris@82 804 bench_free(all_local_out); all_local_out = 0;
Chris@82 805 bench_free(isend_off); isend_off = 0;
Chris@82 806 bench_free(isend_cnt); isend_cnt = 0;
Chris@82 807 bench_free(orecv_off); orecv_off = 0;
Chris@82 808 bench_free(orecv_cnt); orecv_cnt = 0;
Chris@82 809 FFTW(destroy_plan)(plan_scramble_in); plan_scramble_in = 0;
Chris@82 810 FFTW(destroy_plan)(plan_unscramble_out); plan_unscramble_out = 0;
Chris@82 811 }
Chris@82 812
Chris@82 813 void final_cleanup(void)
Chris@82 814 {
Chris@82 815 MPI_Finalize();
Chris@82 816 }
Chris@82 817
Chris@82 818 void bench_exit(int status)
Chris@82 819 {
Chris@82 820 MPI_Abort(MPI_COMM_WORLD, status);
Chris@82 821 }
Chris@82 822
Chris@82 823 double bench_cost_postprocess(double cost)
Chris@82 824 {
Chris@82 825 double cost_max;
Chris@82 826 MPI_Allreduce(&cost, &cost_max, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
Chris@82 827 return cost_max;
Chris@82 828 }
Chris@82 829
Chris@82 830
Chris@82 831 int import_wisdom(FILE *f)
Chris@82 832 {
Chris@82 833 int success = 1, sall;
Chris@82 834 if (my_pe == 0) success = FFTW(import_wisdom_from_file)(f);
Chris@82 835 FFTW(mpi_broadcast_wisdom)(MPI_COMM_WORLD);
Chris@82 836 MPI_Allreduce(&success, &sall, 1, MPI_INT, MPI_LAND, MPI_COMM_WORLD);
Chris@82 837 return sall;
Chris@82 838 }
Chris@82 839
Chris@82 840 void export_wisdom(FILE *f)
Chris@82 841 {
Chris@82 842 FFTW(mpi_gather_wisdom)(MPI_COMM_WORLD);
Chris@82 843 if (my_pe == 0) FFTW(export_wisdom_to_file)(f);
Chris@82 844 }