Chris@10: /* Chris@10: * Copyright (c) 2003, 2007-11 Matteo Frigo Chris@10: * Copyright (c) 2003, 2007-11 Massachusetts Institute of Technology Chris@10: * Chris@10: * This program is free software; you can redistribute it and/or modify Chris@10: * it under the terms of the GNU General Public License as published by Chris@10: * the Free Software Foundation; either version 2 of the License, or Chris@10: * (at your option) any later version. Chris@10: * Chris@10: * This program is distributed in the hope that it will be useful, Chris@10: * but WITHOUT ANY WARRANTY; without even the implied warranty of Chris@10: * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the Chris@10: * GNU General Public License for more details. Chris@10: * Chris@10: * You should have received a copy of the GNU General Public License Chris@10: * along with this program; if not, write to the Free Software Chris@10: * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA Chris@10: * Chris@10: */ Chris@10: Chris@10: /* plans for distributed out-of-place transpose using MPI_Alltoall, Chris@10: and which destroy the input array (unless TRANSPOSED_IN is used) */ Chris@10: Chris@10: #include "mpi-transpose.h" Chris@10: #include Chris@10: Chris@10: typedef struct { Chris@10: solver super; Chris@10: int copy_transposed_in; /* whether to copy the input for TRANSPOSED_IN, Chris@10: which makes the final transpose out-of-place Chris@10: but costs an extra copy and requires us Chris@10: to destroy the input */ Chris@10: } S; Chris@10: Chris@10: typedef struct { Chris@10: plan_mpi_transpose super; Chris@10: Chris@10: plan *cld1, *cld2, *cld2rest, *cld3; Chris@10: Chris@10: MPI_Comm comm; Chris@10: int *send_block_sizes, *send_block_offsets; Chris@10: int *recv_block_sizes, *recv_block_offsets; Chris@10: Chris@10: INT rest_Ioff, rest_Ooff; Chris@10: Chris@10: int equal_blocks; Chris@10: } P; Chris@10: Chris@10: static void apply(const plan *ego_, R *I, R *O) Chris@10: { Chris@10: const P *ego = (const P *) ego_; Chris@10: plan_rdft *cld1, *cld2, *cld2rest, *cld3; Chris@10: Chris@10: /* transpose locally to get contiguous chunks */ Chris@10: cld1 = (plan_rdft *) ego->cld1; Chris@10: if (cld1) { Chris@10: cld1->apply(ego->cld1, I, O); Chris@10: Chris@10: /* transpose chunks globally */ Chris@10: if (ego->equal_blocks) Chris@10: MPI_Alltoall(O, ego->send_block_sizes[0], FFTW_MPI_TYPE, Chris@10: I, ego->recv_block_sizes[0], FFTW_MPI_TYPE, Chris@10: ego->comm); Chris@10: else Chris@10: MPI_Alltoallv(O, ego->send_block_sizes, ego->send_block_offsets, Chris@10: FFTW_MPI_TYPE, Chris@10: I, ego->recv_block_sizes, ego->recv_block_offsets, Chris@10: FFTW_MPI_TYPE, Chris@10: ego->comm); Chris@10: } Chris@10: else { /* TRANSPOSED_IN, no need to destroy input */ Chris@10: /* transpose chunks globally */ Chris@10: if (ego->equal_blocks) Chris@10: MPI_Alltoall(I, ego->send_block_sizes[0], FFTW_MPI_TYPE, Chris@10: O, ego->recv_block_sizes[0], FFTW_MPI_TYPE, Chris@10: ego->comm); Chris@10: else Chris@10: MPI_Alltoallv(I, ego->send_block_sizes, ego->send_block_offsets, Chris@10: FFTW_MPI_TYPE, Chris@10: O, ego->recv_block_sizes, ego->recv_block_offsets, Chris@10: FFTW_MPI_TYPE, Chris@10: ego->comm); Chris@10: I = O; /* final transpose (if any) is in-place */ Chris@10: } Chris@10: Chris@10: /* transpose locally, again, to get ordinary row-major */ Chris@10: cld2 = (plan_rdft *) ego->cld2; Chris@10: if (cld2) { Chris@10: cld2->apply(ego->cld2, I, O); Chris@10: cld2rest = (plan_rdft *) ego->cld2rest; Chris@10: if (cld2rest) { /* leftover from unequal block sizes */ Chris@10: cld2rest->apply(ego->cld2rest, Chris@10: I + ego->rest_Ioff, O + ego->rest_Ooff); Chris@10: cld3 = (plan_rdft *) ego->cld3; Chris@10: if (cld3) Chris@10: cld3->apply(ego->cld3, O, O); Chris@10: /* else TRANSPOSED_OUT is true and user wants O transposed */ Chris@10: } Chris@10: } Chris@10: } Chris@10: Chris@10: static int applicable(const S *ego, const problem *p_, Chris@10: const planner *plnr) Chris@10: { Chris@10: const problem_mpi_transpose *p = (const problem_mpi_transpose *) p_; Chris@10: return (1 Chris@10: && p->I != p->O Chris@10: && (!NO_DESTROY_INPUTP(plnr) || Chris@10: ((p->flags & TRANSPOSED_IN) && !ego->copy_transposed_in)) Chris@10: && ((p->flags & TRANSPOSED_IN) || !ego->copy_transposed_in) Chris@10: && ONLY_TRANSPOSEDP(p->flags) Chris@10: ); Chris@10: } Chris@10: Chris@10: static void awake(plan *ego_, enum wakefulness wakefulness) Chris@10: { Chris@10: P *ego = (P *) ego_; Chris@10: X(plan_awake)(ego->cld1, wakefulness); Chris@10: X(plan_awake)(ego->cld2, wakefulness); Chris@10: X(plan_awake)(ego->cld2rest, wakefulness); Chris@10: X(plan_awake)(ego->cld3, wakefulness); Chris@10: } Chris@10: Chris@10: static void destroy(plan *ego_) Chris@10: { Chris@10: P *ego = (P *) ego_; Chris@10: X(ifree0)(ego->send_block_sizes); Chris@10: MPI_Comm_free(&ego->comm); Chris@10: X(plan_destroy_internal)(ego->cld3); Chris@10: X(plan_destroy_internal)(ego->cld2rest); Chris@10: X(plan_destroy_internal)(ego->cld2); Chris@10: X(plan_destroy_internal)(ego->cld1); Chris@10: } Chris@10: Chris@10: static void print(const plan *ego_, printer *p) Chris@10: { Chris@10: const P *ego = (const P *) ego_; Chris@10: p->print(p, "(mpi-transpose-alltoall%s%(%p%)%(%p%)%(%p%)%(%p%))", Chris@10: ego->equal_blocks ? "/e" : "", Chris@10: ego->cld1, ego->cld2, ego->cld2rest, ego->cld3); Chris@10: } Chris@10: Chris@10: static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr) Chris@10: { Chris@10: const S *ego = (const S *) ego_; Chris@10: const problem_mpi_transpose *p; Chris@10: P *pln; Chris@10: plan *cld1 = 0, *cld2 = 0, *cld2rest = 0, *cld3 = 0; Chris@10: INT b, bt, vn, rest_Ioff, rest_Ooff; Chris@10: R *I; Chris@10: int *sbs, *sbo, *rbs, *rbo; Chris@10: int pe, my_pe, n_pes; Chris@10: int equal_blocks = 1; Chris@10: static const plan_adt padt = { Chris@10: XM(transpose_solve), awake, print, destroy Chris@10: }; Chris@10: Chris@10: if (!applicable(ego, p_, plnr)) Chris@10: return (plan *) 0; Chris@10: Chris@10: p = (const problem_mpi_transpose *) p_; Chris@10: vn = p->vn; Chris@10: Chris@10: MPI_Comm_rank(p->comm, &my_pe); Chris@10: MPI_Comm_size(p->comm, &n_pes); Chris@10: Chris@10: b = XM(block)(p->nx, p->block, my_pe); Chris@10: Chris@10: if (p->flags & TRANSPOSED_IN) { /* I is already transposed */ Chris@10: if (ego->copy_transposed_in) { Chris@10: cld1 = X(mkplan_f_d)(plnr, Chris@10: X(mkproblem_rdft_0_d)(X(mktensor_1d) Chris@10: (b * p->ny * vn, 1, 1), Chris@10: I = p->I, p->O), Chris@10: 0, 0, NO_SLOW); Chris@10: if (XM(any_true)(!cld1, p->comm)) goto nada; Chris@10: } Chris@10: else Chris@10: I = p->O; /* final transpose is in-place */ Chris@10: } Chris@10: else { /* transpose b x ny x vn -> ny x b x vn */ Chris@10: cld1 = X(mkplan_f_d)(plnr, Chris@10: X(mkproblem_rdft_0_d)(X(mktensor_3d) Chris@10: (b, p->ny * vn, vn, Chris@10: p->ny, vn, b * vn, Chris@10: vn, 1, 1), Chris@10: I = p->I, p->O), Chris@10: 0, 0, NO_SLOW); Chris@10: if (XM(any_true)(!cld1, p->comm)) goto nada; Chris@10: } Chris@10: Chris@10: if (XM(any_true)(!XM(mkplans_posttranspose)(p, plnr, I, p->O, my_pe, Chris@10: &cld2, &cld2rest, &cld3, Chris@10: &rest_Ioff, &rest_Ooff), Chris@10: p->comm)) goto nada; Chris@10: Chris@10: pln = MKPLAN_MPI_TRANSPOSE(P, &padt, apply); Chris@10: Chris@10: pln->cld1 = cld1; Chris@10: pln->cld2 = cld2; Chris@10: pln->cld2rest = cld2rest; Chris@10: pln->rest_Ioff = rest_Ioff; Chris@10: pln->rest_Ooff = rest_Ooff; Chris@10: pln->cld3 = cld3; Chris@10: Chris@10: MPI_Comm_dup(p->comm, &pln->comm); Chris@10: Chris@10: /* Compute sizes/offsets of blocks to send for all-to-all command. */ Chris@10: sbs = (int *) MALLOC(4 * n_pes * sizeof(int), PLANS); Chris@10: sbo = sbs + n_pes; Chris@10: rbs = sbo + n_pes; Chris@10: rbo = rbs + n_pes; Chris@10: b = XM(block)(p->nx, p->block, my_pe); Chris@10: bt = XM(block)(p->ny, p->tblock, my_pe); Chris@10: for (pe = 0; pe < n_pes; ++pe) { Chris@10: INT db, dbt; /* destination block sizes */ Chris@10: db = XM(block)(p->nx, p->block, pe); Chris@10: dbt = XM(block)(p->ny, p->tblock, pe); Chris@10: if (db != p->block || dbt != p->tblock) Chris@10: equal_blocks = 0; Chris@10: Chris@10: /* MPI requires type "int" here; apparently it Chris@10: has no 64-bit API? Grrr. */ Chris@10: sbs[pe] = (int) (b * dbt * vn); Chris@10: sbo[pe] = (int) (pe * (b * p->tblock) * vn); Chris@10: rbs[pe] = (int) (db * bt * vn); Chris@10: rbo[pe] = (int) (pe * (p->block * bt) * vn); Chris@10: } Chris@10: pln->send_block_sizes = sbs; Chris@10: pln->send_block_offsets = sbo; Chris@10: pln->recv_block_sizes = rbs; Chris@10: pln->recv_block_offsets = rbo; Chris@10: pln->equal_blocks = equal_blocks; Chris@10: Chris@10: X(ops_zero)(&pln->super.super.ops); Chris@10: if (cld1) X(ops_add2)(&cld1->ops, &pln->super.super.ops); Chris@10: if (cld2) X(ops_add2)(&cld2->ops, &pln->super.super.ops); Chris@10: if (cld2rest) X(ops_add2)(&cld2rest->ops, &pln->super.super.ops); Chris@10: if (cld3) X(ops_add2)(&cld3->ops, &pln->super.super.ops); Chris@10: /* FIXME: should MPI operations be counted in "other" somehow? */ Chris@10: Chris@10: return &(pln->super.super); Chris@10: Chris@10: nada: Chris@10: X(plan_destroy_internal)(cld3); Chris@10: X(plan_destroy_internal)(cld2rest); Chris@10: X(plan_destroy_internal)(cld2); Chris@10: X(plan_destroy_internal)(cld1); Chris@10: return (plan *) 0; Chris@10: } Chris@10: Chris@10: static solver *mksolver(int copy_transposed_in) Chris@10: { Chris@10: static const solver_adt sadt = { PROBLEM_MPI_TRANSPOSE, mkplan, 0 }; Chris@10: S *slv = MKSOLVER(S, &sadt); Chris@10: slv->copy_transposed_in = copy_transposed_in; Chris@10: return &(slv->super); Chris@10: } Chris@10: Chris@10: void XM(transpose_alltoall_register)(planner *p) Chris@10: { Chris@10: int cti; Chris@10: for (cti = 0; cti <= 1; ++cti) Chris@10: REGISTER_SOLVER(p, mksolver(cti)); Chris@10: }