Mercurial > hg > sv-dependency-builds

annotate src/fftw-3.3.5/mpi/transpose-alltoall.c @ 77:4edcd14160a5 pa_catalina

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Duplicate for patch testing
author Chris Cannam
date Wed, 30 Oct 2019 11:25:10 +0000
parents 2cd0e3b3e1fd
children
rev   line source
Chris@42 1 /*
Chris@42 2 * Copyright (c) 2003, 2007-14 Matteo Frigo
Chris@42 3 * Copyright (c) 2003, 2007-14 Massachusetts Institute of Technology
Chris@42 4 *
Chris@42 5 * This program is free software; you can redistribute it and/or modify
Chris@42 6 * it under the terms of the GNU General Public License as published by
Chris@42 7 * the Free Software Foundation; either version 2 of the License, or
Chris@42 8 * (at your option) any later version.
Chris@42 9 *
Chris@42 10 * This program is distributed in the hope that it will be useful,
Chris@42 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
Chris@42 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
Chris@42 13 * GNU General Public License for more details.
Chris@42 14 *
Chris@42 15 * You should have received a copy of the GNU General Public License
Chris@42 16 * along with this program; if not, write to the Free Software
Chris@42 17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
Chris@42 18 *
Chris@42 19 */
Chris@42 20
Chris@42 21 /* plans for distributed out-of-place transpose using MPI_Alltoall,
Chris@42 22 and which destroy the input array (unless TRANSPOSED_IN is used) */
Chris@42 23
Chris@42 24 #include "mpi-transpose.h"
Chris@42 25 #include <string.h>
Chris@42 26
Chris@42 27 typedef struct {
Chris@42 28 solver super;
Chris@42 29 int copy_transposed_in; /* whether to copy the input for TRANSPOSED_IN,
Chris@42 30 which makes the final transpose out-of-place
Chris@42 31 but costs an extra copy and requires us
Chris@42 32 to destroy the input */
Chris@42 33 } S;
Chris@42 34
Chris@42 35 typedef struct {
Chris@42 36 plan_mpi_transpose super;
Chris@42 37
Chris@42 38 plan *cld1, *cld2, *cld2rest, *cld3;
Chris@42 39
Chris@42 40 MPI_Comm comm;
Chris@42 41 int *send_block_sizes, *send_block_offsets;
Chris@42 42 int *recv_block_sizes, *recv_block_offsets;
Chris@42 43
Chris@42 44 INT rest_Ioff, rest_Ooff;
Chris@42 45
Chris@42 46 int equal_blocks;
Chris@42 47 } P;
Chris@42 48
Chris@42 49 static void apply(const plan *ego_, R *I, R *O)
Chris@42 50 {
Chris@42 51 const P *ego = (const P *) ego_;
Chris@42 52 plan_rdft *cld1, *cld2, *cld2rest, *cld3;
Chris@42 53
Chris@42 54 /* transpose locally to get contiguous chunks */
Chris@42 55 cld1 = (plan_rdft *) ego->cld1;
Chris@42 56 if (cld1) {
Chris@42 57 cld1->apply(ego->cld1, I, O);
Chris@42 58
Chris@42 59 /* transpose chunks globally */
Chris@42 60 if (ego->equal_blocks)
Chris@42 61 MPI_Alltoall(O, ego->send_block_sizes[0], FFTW_MPI_TYPE,
Chris@42 62 I, ego->recv_block_sizes[0], FFTW_MPI_TYPE,
Chris@42 63 ego->comm);
Chris@42 64 else
Chris@42 65 MPI_Alltoallv(O, ego->send_block_sizes, ego->send_block_offsets,
Chris@42 66 FFTW_MPI_TYPE,
Chris@42 67 I, ego->recv_block_sizes, ego->recv_block_offsets,
Chris@42 68 FFTW_MPI_TYPE,
Chris@42 69 ego->comm);
Chris@42 70 }
Chris@42 71 else { /* TRANSPOSED_IN, no need to destroy input */
Chris@42 72 /* transpose chunks globally */
Chris@42 73 if (ego->equal_blocks)
Chris@42 74 MPI_Alltoall(I, ego->send_block_sizes[0], FFTW_MPI_TYPE,
Chris@42 75 O, ego->recv_block_sizes[0], FFTW_MPI_TYPE,
Chris@42 76 ego->comm);
Chris@42 77 else
Chris@42 78 MPI_Alltoallv(I, ego->send_block_sizes, ego->send_block_offsets,
Chris@42 79 FFTW_MPI_TYPE,
Chris@42 80 O, ego->recv_block_sizes, ego->recv_block_offsets,
Chris@42 81 FFTW_MPI_TYPE,
Chris@42 82 ego->comm);
Chris@42 83 I = O; /* final transpose (if any) is in-place */
Chris@42 84 }
Chris@42 85
Chris@42 86 /* transpose locally, again, to get ordinary row-major */
Chris@42 87 cld2 = (plan_rdft *) ego->cld2;
Chris@42 88 if (cld2) {
Chris@42 89 cld2->apply(ego->cld2, I, O);
Chris@42 90 cld2rest = (plan_rdft *) ego->cld2rest;
Chris@42 91 if (cld2rest) { /* leftover from unequal block sizes */
Chris@42 92 cld2rest->apply(ego->cld2rest,
Chris@42 93 I + ego->rest_Ioff, O + ego->rest_Ooff);
Chris@42 94 cld3 = (plan_rdft *) ego->cld3;
Chris@42 95 if (cld3)
Chris@42 96 cld3->apply(ego->cld3, O, O);
Chris@42 97 /* else TRANSPOSED_OUT is true and user wants O transposed */
Chris@42 98 }
Chris@42 99 }
Chris@42 100 }
Chris@42 101
Chris@42 102 static int applicable(const S *ego, const problem *p_,
Chris@42 103 const planner *plnr)
Chris@42 104 {
Chris@42 105 const problem_mpi_transpose *p = (const problem_mpi_transpose *) p_;
Chris@42 106 return (1
Chris@42 107 && p->I != p->O
Chris@42 108 && (!NO_DESTROY_INPUTP(plnr) ||
Chris@42 109 ((p->flags & TRANSPOSED_IN) && !ego->copy_transposed_in))
Chris@42 110 && ((p->flags & TRANSPOSED_IN) || !ego->copy_transposed_in)
Chris@42 111 && ONLY_TRANSPOSEDP(p->flags)
Chris@42 112 );
Chris@42 113 }
Chris@42 114
Chris@42 115 static void awake(plan *ego_, enum wakefulness wakefulness)
Chris@42 116 {
Chris@42 117 P *ego = (P *) ego_;
Chris@42 118 X(plan_awake)(ego->cld1, wakefulness);
Chris@42 119 X(plan_awake)(ego->cld2, wakefulness);
Chris@42 120 X(plan_awake)(ego->cld2rest, wakefulness);
Chris@42 121 X(plan_awake)(ego->cld3, wakefulness);
Chris@42 122 }
Chris@42 123
Chris@42 124 static void destroy(plan *ego_)
Chris@42 125 {
Chris@42 126 P *ego = (P *) ego_;
Chris@42 127 X(ifree0)(ego->send_block_sizes);
Chris@42 128 MPI_Comm_free(&ego->comm);
Chris@42 129 X(plan_destroy_internal)(ego->cld3);
Chris@42 130 X(plan_destroy_internal)(ego->cld2rest);
Chris@42 131 X(plan_destroy_internal)(ego->cld2);
Chris@42 132 X(plan_destroy_internal)(ego->cld1);
Chris@42 133 }
Chris@42 134
Chris@42 135 static void print(const plan *ego_, printer *p)
Chris@42 136 {
Chris@42 137 const P *ego = (const P *) ego_;
Chris@42 138 p->print(p, "(mpi-transpose-alltoall%s%(%p%)%(%p%)%(%p%)%(%p%))",
Chris@42 139 ego->equal_blocks ? "/e" : "",
Chris@42 140 ego->cld1, ego->cld2, ego->cld2rest, ego->cld3);
Chris@42 141 }
Chris@42 142
Chris@42 143 static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
Chris@42 144 {
Chris@42 145 const S *ego = (const S *) ego_;
Chris@42 146 const problem_mpi_transpose *p;
Chris@42 147 P *pln;
Chris@42 148 plan *cld1 = 0, *cld2 = 0, *cld2rest = 0, *cld3 = 0;
Chris@42 149 INT b, bt, vn, rest_Ioff, rest_Ooff;
Chris@42 150 R *I;
Chris@42 151 int *sbs, *sbo, *rbs, *rbo;
Chris@42 152 int pe, my_pe, n_pes;
Chris@42 153 int equal_blocks = 1;
Chris@42 154 static const plan_adt padt = {
Chris@42 155 XM(transpose_solve), awake, print, destroy
Chris@42 156 };
Chris@42 157
Chris@42 158 if (!applicable(ego, p_, plnr))
Chris@42 159 return (plan *) 0;
Chris@42 160
Chris@42 161 p = (const problem_mpi_transpose *) p_;
Chris@42 162 vn = p->vn;
Chris@42 163
Chris@42 164 MPI_Comm_rank(p->comm, &my_pe);
Chris@42 165 MPI_Comm_size(p->comm, &n_pes);
Chris@42 166
Chris@42 167 b = XM(block)(p->nx, p->block, my_pe);
Chris@42 168
Chris@42 169 if (p->flags & TRANSPOSED_IN) { /* I is already transposed */
Chris@42 170 if (ego->copy_transposed_in) {
Chris@42 171 cld1 = X(mkplan_f_d)(plnr,
Chris@42 172 X(mkproblem_rdft_0_d)(X(mktensor_1d)
Chris@42 173 (b * p->ny * vn, 1, 1),
Chris@42 174 I = p->I, p->O),
Chris@42 175 0, 0, NO_SLOW);
Chris@42 176 if (XM(any_true)(!cld1, p->comm)) goto nada;
Chris@42 177 }
Chris@42 178 else
Chris@42 179 I = p->O; /* final transpose is in-place */
Chris@42 180 }
Chris@42 181 else { /* transpose b x ny x vn -> ny x b x vn */
Chris@42 182 cld1 = X(mkplan_f_d)(plnr,
Chris@42 183 X(mkproblem_rdft_0_d)(X(mktensor_3d)
Chris@42 184 (b, p->ny * vn, vn,
Chris@42 185 p->ny, vn, b * vn,
Chris@42 186 vn, 1, 1),
Chris@42 187 I = p->I, p->O),
Chris@42 188 0, 0, NO_SLOW);
Chris@42 189 if (XM(any_true)(!cld1, p->comm)) goto nada;
Chris@42 190 }
Chris@42 191
Chris@42 192 if (XM(any_true)(!XM(mkplans_posttranspose)(p, plnr, I, p->O, my_pe,
Chris@42 193 &cld2, &cld2rest, &cld3,
Chris@42 194 &rest_Ioff, &rest_Ooff),
Chris@42 195 p->comm)) goto nada;
Chris@42 196
Chris@42 197 pln = MKPLAN_MPI_TRANSPOSE(P, &padt, apply);
Chris@42 198
Chris@42 199 pln->cld1 = cld1;
Chris@42 200 pln->cld2 = cld2;
Chris@42 201 pln->cld2rest = cld2rest;
Chris@42 202 pln->rest_Ioff = rest_Ioff;
Chris@42 203 pln->rest_Ooff = rest_Ooff;
Chris@42 204 pln->cld3 = cld3;
Chris@42 205
Chris@42 206 MPI_Comm_dup(p->comm, &pln->comm);
Chris@42 207
Chris@42 208 /* Compute sizes/offsets of blocks to send for all-to-all command. */
Chris@42 209 sbs = (int *) MALLOC(4 * n_pes * sizeof(int), PLANS);
Chris@42 210 sbo = sbs + n_pes;
Chris@42 211 rbs = sbo + n_pes;
Chris@42 212 rbo = rbs + n_pes;
Chris@42 213 b = XM(block)(p->nx, p->block, my_pe);
Chris@42 214 bt = XM(block)(p->ny, p->tblock, my_pe);
Chris@42 215 for (pe = 0; pe < n_pes; ++pe) {
Chris@42 216 INT db, dbt; /* destination block sizes */
Chris@42 217 db = XM(block)(p->nx, p->block, pe);
Chris@42 218 dbt = XM(block)(p->ny, p->tblock, pe);
Chris@42 219 if (db != p->block || dbt != p->tblock)
Chris@42 220 equal_blocks = 0;
Chris@42 221
Chris@42 222 /* MPI requires type "int" here; apparently it
Chris@42 223 has no 64-bit API? Grrr. */
Chris@42 224 sbs[pe] = (int) (b * dbt * vn);
Chris@42 225 sbo[pe] = (int) (pe * (b * p->tblock) * vn);
Chris@42 226 rbs[pe] = (int) (db * bt * vn);
Chris@42 227 rbo[pe] = (int) (pe * (p->block * bt) * vn);
Chris@42 228 }
Chris@42 229 pln->send_block_sizes = sbs;
Chris@42 230 pln->send_block_offsets = sbo;
Chris@42 231 pln->recv_block_sizes = rbs;
Chris@42 232 pln->recv_block_offsets = rbo;
Chris@42 233 pln->equal_blocks = equal_blocks;
Chris@42 234
Chris@42 235 X(ops_zero)(&pln->super.super.ops);
Chris@42 236 if (cld1) X(ops_add2)(&cld1->ops, &pln->super.super.ops);
Chris@42 237 if (cld2) X(ops_add2)(&cld2->ops, &pln->super.super.ops);
Chris@42 238 if (cld2rest) X(ops_add2)(&cld2rest->ops, &pln->super.super.ops);
Chris@42 239 if (cld3) X(ops_add2)(&cld3->ops, &pln->super.super.ops);
Chris@42 240 /* FIXME: should MPI operations be counted in "other" somehow? */
Chris@42 241
Chris@42 242 return &(pln->super.super);
Chris@42 243
Chris@42 244 nada:
Chris@42 245 X(plan_destroy_internal)(cld3);
Chris@42 246 X(plan_destroy_internal)(cld2rest);
Chris@42 247 X(plan_destroy_internal)(cld2);
Chris@42 248 X(plan_destroy_internal)(cld1);
Chris@42 249 return (plan *) 0;
Chris@42 250 }
Chris@42 251
Chris@42 252 static solver *mksolver(int copy_transposed_in)
Chris@42 253 {
Chris@42 254 static const solver_adt sadt = { PROBLEM_MPI_TRANSPOSE, mkplan, 0 };
Chris@42 255 S *slv = MKSOLVER(S, &sadt);
Chris@42 256 slv->copy_transposed_in = copy_transposed_in;
Chris@42 257 return &(slv->super);
Chris@42 258 }
Chris@42 259
Chris@42 260 void XM(transpose_alltoall_register)(planner *p)
Chris@42 261 {
Chris@42 262 int cti;
Chris@42 263 for (cti = 0; cti <= 1; ++cti)
Chris@42 264 REGISTER_SOLVER(p, mksolver(cti));
Chris@42 265 }