Chris@19: /*
Chris@19:  * Copyright (c) 2003, 2007-14 Matteo Frigo
Chris@19:  * Copyright (c) 1999-2003, 2007-8 Massachusetts Institute of Technology
Chris@19:  *
Chris@19:  * This program is free software; you can redistribute it and/or modify
Chris@19:  * it under the terms of the GNU General Public License as published by
Chris@19:  * the Free Software Foundation; either version 2 of the License, or
Chris@19:  * (at your option) any later version.
Chris@19:  *
Chris@19:  * This program is distributed in the hope that it will be useful,
Chris@19:  * but WITHOUT ANY WARRANTY; without even the implied warranty of
Chris@19:  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
Chris@19:  * GNU General Public License for more details.
Chris@19:  *
Chris@19:  * You should have received a copy of the GNU General Public License
Chris@19:  * along with this program; if not, write to the Free Software
Chris@19:  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
Chris@19:  *
Chris@19:  */
Chris@19: 
Chris@19: /**********************************************************************/
Chris@19: /* This is a modified and combined version of the sched.c and
Chris@19:    test_sched.c files shipped with FFTW 2, written to implement and
Chris@19:    test various all-to-all communications scheduling patterns.
Chris@19: 
Chris@19:    It is not used in FFTW 3, but I keep it around in case we ever want
Chris@19:    to play with this again or to change algorithms.  In particular, I
Chris@19:    used it to implement and test the fill1_comm_sched routine in
Chris@19:    transpose-pairwise.c, which allows us to create a schedule for one
Chris@19:    process at a time and is much more compact than the FFTW 2 code.
Chris@19: 
Chris@19:    Note that the scheduling algorithm is somewhat modified from that
Chris@19:    of FFTW 2.  Originally, I thought that one "stall" in the schedule
Chris@19:    was unavoidable for odd numbers of processes, since this is the
Chris@19:    case for the soccer-timetabling problem.  However, because of the
Chris@19:    self-communication step, we can use the self-communication to fill
Chris@19:    in the stalls.  (Thanks to Ralf Wildenhues for pointing this out.)
Chris@19:    This greatly simplifies the process re-sorting algorithm. */
Chris@19: 
Chris@19: /**********************************************************************/
Chris@19: 
Chris@19: #include <stdio.h>
Chris@19: #include <stdlib.h>
Chris@19: 
Chris@19: /* This file contains routines to compute communications schedules for
Chris@19:    all-to-all communications (complete exchanges) that are performed
Chris@19:    in-place.  (That is, the block that processor x sends to processor
Chris@19:    y gets replaced on processor x by a block received from processor y.)
Chris@19: 
Chris@19:    A schedule, int **sched, is a two-dimensional array where
Chris@19:    sched[pe][i] is the processor that pe expects to exchange a message
Chris@19:    with on the i-th step of the exchange.  sched[pe][i] == -1 for the
Chris@19:    i after the last exchange scheduled on pe.
Chris@19: 
Chris@19:    Here, processors (pe's, for processing elements), are numbered from
Chris@19:    0 to npes-1.
Chris@19: 
Chris@19:    There are a couple of constraints that a schedule should satisfy
Chris@19:    (besides the obvious one that every processor has to communicate
Chris@19:    with every other processor exactly once).
Chris@19:    
Chris@19:    * First, and most importantly, there must be no deadlocks.
Chris@19:    
Chris@19:    * Second, we would like to overlap communications as much as possible,
Chris@19:    so that all exchanges occur in parallel.  It turns out that perfect
Chris@19:    overlap is possible for all number of processes (npes).
Chris@19: 
Chris@19:    It turns out that this scheduling problem is actually well-studied,
Chris@19:    and good solutions are known.  The problem is known as a
Chris@19:    "time-tabling" problem, and is specifically the problem of
Chris@19:    scheduling a sports competition (where n teams must compete exactly
Chris@19:    once with every other team).  The problem is discussed and
Chris@19:    algorithms are presented in:
Chris@19: 
Chris@19:    [1] J. A. M. Schreuder, "Constructing Timetables for Sport
Chris@19:    Competitions," Mathematical Programming Study 13, pp. 58-67 (1980).
Chris@19: 
Chris@19:    [2] A. Schaerf, "Scheduling Sport Tournaments using Constraint
Chris@19:    Logic Programming," Proc. of 12th Europ. Conf. on
Chris@19:    Artif. Intell. (ECAI-96), pp. 634-639 (Budapest 1996).
Chris@19:    http://hermes.dis.uniromal.it/~aschaerf/publications.html
Chris@19: 
Chris@19:    (These people actually impose a lot of additional constraints that
Chris@19:    we don't care about, so they are solving harder problems. [1] gives
Chris@19:    a simple enough algorithm for our purposes, though.)
Chris@19:    
Chris@19:    In the timetabling problem, N teams can all play one another in N-1
Chris@19:    steps if N is even, and N steps if N is odd.  Here, however,
Chris@19:    there is a "self-communication" step (a team must also "play itself")
Chris@19:    and so we can always make an optimal N-step schedule regardless of N.
Chris@19: 
Chris@19:    However, we have to do more: for a particular processor, the
Chris@19:    communications schedule must be sorted in ascending or descending
Chris@19:    order of processor index.  (This is necessary so that the data
Chris@19:    coming in for the transpose does not overwrite data that will be
Chris@19:    sent later; for that processor the incoming and outgoing blocks are
Chris@19:    of different non-zero sizes.)  Fortunately, because the schedule
Chris@19:    is stall free, each parallel step of the schedule is independent
Chris@19:    of every other step, and we can reorder the steps arbitrarily
Chris@19:    to achieve any desired order on a particular process.
Chris@19: */
Chris@19: 
Chris@19: void free_comm_schedule(int **sched, int npes)
Chris@19: {
Chris@19:      if (sched) {
Chris@19: 	  int i;
Chris@19: 
Chris@19: 	  for (i = 0; i < npes; ++i)
Chris@19: 	       free(sched[i]);
Chris@19: 	  free(sched);
Chris@19:      }
Chris@19: }
Chris@19: 
Chris@19: void empty_comm_schedule(int **sched, int npes)
Chris@19: {
Chris@19:      int i;
Chris@19:      for (i = 0; i < npes; ++i)
Chris@19: 	  sched[i][0] = -1;
Chris@19: }
Chris@19: 
Chris@19: extern void fill_comm_schedule(int **sched, int npes);
Chris@19: 
Chris@19: /* Create a new communications schedule for a given number of processors.
Chris@19:    The schedule is initialized to a deadlock-free, maximum overlap
Chris@19:    schedule.  Returns NULL on an error (may print a message to
Chris@19:    stderr if there is a program bug detected).  */
Chris@19: int **make_comm_schedule(int npes)
Chris@19: {
Chris@19:      int **sched;
Chris@19:      int i;
Chris@19: 
Chris@19:      sched = (int **) malloc(sizeof(int *) * npes);
Chris@19:      if (!sched)
Chris@19: 	  return NULL;
Chris@19: 
Chris@19:      for (i = 0; i < npes; ++i)
Chris@19: 	  sched[i] = NULL;
Chris@19: 
Chris@19:      for (i = 0; i < npes; ++i) {
Chris@19: 	  sched[i] = (int *) malloc(sizeof(int) * 10 * (npes + 1));
Chris@19: 	  if (!sched[i]) {
Chris@19: 	       free_comm_schedule(sched,npes);
Chris@19: 	       return NULL;
Chris@19: 	  }
Chris@19:      }
Chris@19:      
Chris@19:      empty_comm_schedule(sched,npes);
Chris@19:      fill_comm_schedule(sched,npes);
Chris@19: 
Chris@19:      if (!check_comm_schedule(sched,npes)) {
Chris@19: 	  free_comm_schedule(sched,npes);
Chris@19: 	  return NULL;
Chris@19:      }
Chris@19: 
Chris@19:      return sched;
Chris@19: }
Chris@19: 
Chris@19: static void add_dest_to_comm_schedule(int **sched, int pe, int dest)
Chris@19: {
Chris@19:      int i;
Chris@19:      
Chris@19:      for (i = 0; sched[pe][i] != -1; ++i)
Chris@19: 	  ;
Chris@19: 
Chris@19:      sched[pe][i] = dest;
Chris@19:      sched[pe][i+1] = -1;
Chris@19: }
Chris@19: 
Chris@19: static void add_pair_to_comm_schedule(int **sched, int pe1, int pe2)
Chris@19: {
Chris@19:      add_dest_to_comm_schedule(sched, pe1, pe2);
Chris@19:      if (pe1 != pe2)
Chris@19: 	  add_dest_to_comm_schedule(sched, pe2, pe1);
Chris@19: }
Chris@19: 
Chris@19: /* Simplification of algorithm presented in [1] (we have fewer
Chris@19:    constraints).  Produces a perfect schedule (npes steps).  */
Chris@19: 
Chris@19: void fill_comm_schedule(int **sched, int npes)
Chris@19: {
Chris@19:      int pe, i, n;
Chris@19: 
Chris@19:      if (npes % 2 == 0) {
Chris@19: 	  n = npes;
Chris@19: 	  for (pe = 0; pe < npes; ++pe)
Chris@19: 	       add_pair_to_comm_schedule(sched,pe,pe);
Chris@19:      }
Chris@19:      else
Chris@19: 	  n = npes + 1;
Chris@19: 
Chris@19:      for (pe = 0; pe < n - 1; ++pe) {
Chris@19: 	  add_pair_to_comm_schedule(sched, pe, npes % 2 == 0 ? npes - 1 : pe);
Chris@19: 	  
Chris@19: 	  for (i = 1; i < n/2; ++i) {
Chris@19: 	       int pe_a, pe_b;
Chris@19: 
Chris@19: 	       pe_a = pe - i;
Chris@19: 	       if (pe_a < 0)
Chris@19: 		    pe_a += n - 1;
Chris@19: 
Chris@19: 	       pe_b = (pe + i) % (n - 1);
Chris@19: 
Chris@19: 	       add_pair_to_comm_schedule(sched,pe_a,pe_b);
Chris@19: 	  }
Chris@19:      }
Chris@19: }
Chris@19: 
Chris@19: /* given an array sched[npes], fills it with the communications
Chris@19:    schedule for process pe. */
Chris@19: void fill1_comm_sched(int *sched, int which_pe, int npes)
Chris@19: {
Chris@19:      int pe, i, n, s = 0;
Chris@19:      if (npes % 2 == 0) {
Chris@19: 	  n = npes;
Chris@19: 	  sched[s++] = which_pe;
Chris@19:      }
Chris@19:      else
Chris@19: 	  n = npes + 1;
Chris@19:      for (pe = 0; pe < n - 1; ++pe) {
Chris@19: 	  if (npes % 2 == 0) {
Chris@19: 	       if (pe == which_pe) sched[s++] = npes - 1;
Chris@19: 	       else if (npes - 1 == which_pe) sched[s++] = pe;
Chris@19: 	  }
Chris@19: 	  else if (pe == which_pe) sched[s++] = pe;
Chris@19: 
Chris@19: 	  if (pe != which_pe && which_pe < n - 1) {
Chris@19: 	       i = (pe - which_pe + (n - 1)) % (n - 1);
Chris@19: 	       if (i < n/2)
Chris@19: 		    sched[s++] = (pe + i) % (n - 1);
Chris@19: 	       
Chris@19: 	       i = (which_pe - pe + (n - 1)) % (n - 1);
Chris@19: 	       if (i < n/2)
Chris@19: 		    sched[s++] = (pe - i + (n - 1)) % (n - 1);
Chris@19: 	  }
Chris@19:      }
Chris@19:      if (s != npes) {
Chris@19: 	  fprintf(stderr, "bug in fill1_com_schedule (%d, %d/%d)\n", 
Chris@19: 		  s, which_pe, npes);
Chris@19: 	  exit(EXIT_FAILURE);
Chris@19:      }
Chris@19: }
Chris@19: 
Chris@19: /* sort the communication schedule sched for npes so that the schedule
Chris@19:    on process sortpe is ascending or descending (!ascending). */
Chris@19: static void sort1_comm_sched(int *sched, int npes, int sortpe, int ascending)
Chris@19: {
Chris@19:      int *sortsched, i;
Chris@19:      sortsched = (int *) malloc(npes * sizeof(int) * 2);
Chris@19:      fill1_comm_sched(sortsched, sortpe, npes);
Chris@19:      if (ascending)
Chris@19:           for (i = 0; i < npes; ++i)
Chris@19:                sortsched[npes + sortsched[i]] = sched[i];
Chris@19:      else
Chris@19:           for (i = 0; i < npes; ++i)
Chris@19:                sortsched[2*npes - 1 - sortsched[i]] = sched[i];
Chris@19:      for (i = 0; i < npes; ++i)
Chris@19:           sched[i] = sortsched[npes + i];
Chris@19:      free(sortsched);
Chris@19: }
Chris@19: 
Chris@19: /* Below, we have various checks in case of bugs: */
Chris@19: 
Chris@19: /* check for deadlocks by simulating the schedule and looking for
Chris@19:    cycles in the dependency list; returns 0 if there are deadlocks
Chris@19:    (or other errors) */
Chris@19: static int check_schedule_deadlock(int **sched, int npes)
Chris@19: {
Chris@19:      int *step, *depend, *visited, pe, pe2, period, done = 0;
Chris@19:      int counter = 0;
Chris@19: 
Chris@19:      /* step[pe] is the step in the schedule that a given pe is on */
Chris@19:      step = (int *) malloc(sizeof(int) * npes);
Chris@19: 
Chris@19:      /* depend[pe] is the pe' that pe is currently waiting for a message
Chris@19: 	from (-1 if none) */
Chris@19:      depend = (int *) malloc(sizeof(int) * npes);
Chris@19: 
Chris@19:      /* visited[pe] tells whether we have visited the current pe already
Chris@19: 	when we are looking for cycles. */
Chris@19:      visited = (int *) malloc(sizeof(int) * npes);
Chris@19: 
Chris@19:      if (!step || !depend || !visited) {
Chris@19: 	  free(step); free(depend); free(visited);
Chris@19: 	  return 0;
Chris@19:      }
Chris@19: 
Chris@19:      for (pe = 0; pe < npes; ++pe)
Chris@19: 	  step[pe] = 0;
Chris@19: 
Chris@19:      while (!done) {
Chris@19: 	  ++counter;
Chris@19: 
Chris@19: 	  for (pe = 0; pe < npes; ++pe)
Chris@19: 	       depend[pe] = sched[pe][step[pe]];
Chris@19: 	  
Chris@19: 	  /* now look for cycles in the dependencies with period > 2: */
Chris@19: 	  for (pe = 0; pe < npes; ++pe)
Chris@19: 	       if (depend[pe] != -1) {
Chris@19: 		    for (pe2 = 0; pe2 < npes; ++pe2)
Chris@19: 			 visited[pe2] = 0;
Chris@19: 
Chris@19: 		    period = 0;
Chris@19: 		    pe2 = pe;
Chris@19: 		    do {
Chris@19: 			 visited[pe2] = period + 1;
Chris@19: 			 pe2 = depend[pe2];
Chris@19: 			 period++;
Chris@19: 		    } while (pe2 != -1 && !visited[pe2]);
Chris@19: 
Chris@19: 		    if (pe2 == -1) {
Chris@19: 			 fprintf(stderr,
Chris@19: 				 "BUG: unterminated cycle in schedule!\n");
Chris@19: 			 free(step); free(depend);
Chris@19: 			 free(visited);
Chris@19: 			 return 0;
Chris@19: 		    }
Chris@19: 		    if (period - (visited[pe2] - 1) > 2) {
Chris@19: 			 fprintf(stderr,"BUG: deadlock in schedule!\n");
Chris@19: 			 free(step); free(depend);
Chris@19: 			 free(visited);
Chris@19: 			 return 0;
Chris@19: 		    }
Chris@19: 
Chris@19: 		    if (pe2 == pe)
Chris@19: 			 step[pe]++;
Chris@19: 	       }
Chris@19: 
Chris@19: 	  done = 1;
Chris@19: 	  for (pe = 0; pe < npes; ++pe)
Chris@19: 	       if (sched[pe][step[pe]] != -1) {
Chris@19: 		    done = 0;
Chris@19: 		    break;
Chris@19: 	       }
Chris@19:      }
Chris@19: 
Chris@19:      free(step); free(depend); free(visited);
Chris@19:      return (counter > 0 ? counter : 1);
Chris@19: }
Chris@19: 
Chris@19: /* sanity checks; prints message and returns 0 on failure.
Chris@19:    undocumented feature: the return value on success is actually the
Chris@19:    number of steps required for the schedule to complete, counting
Chris@19:    stalls. */
Chris@19: int check_comm_schedule(int **sched, int npes)
Chris@19: {
Chris@19:      int pe, i, comm_pe;
Chris@19:      
Chris@19:      for (pe = 0; pe < npes; ++pe) {
Chris@19: 	  for (comm_pe = 0; comm_pe < npes; ++comm_pe) {
Chris@19: 	       for (i = 0; sched[pe][i] != -1 && sched[pe][i] != comm_pe; ++i)
Chris@19: 		    ;
Chris@19: 	       if (sched[pe][i] == -1) {
Chris@19: 		    fprintf(stderr,"BUG: schedule never sends message from "
Chris@19: 			    "%d to %d.\n",pe,comm_pe);
Chris@19: 		    return 0;  /* never send message to comm_pe */
Chris@19: 	       }
Chris@19: 	  }
Chris@19: 	  for (i = 0; sched[pe][i] != -1; ++i)
Chris@19: 	       ;
Chris@19: 	  if (i != npes) {
Chris@19: 	       fprintf(stderr,"BUG: schedule sends too many messages from "
Chris@19: 		       "%d\n",pe);
Chris@19: 	       return 0;
Chris@19: 	  }
Chris@19:      }
Chris@19:      return check_schedule_deadlock(sched,npes);
Chris@19: }
Chris@19: 
Chris@19: /* invert the order of all the schedules; this has no effect on
Chris@19:    its required properties. */
Chris@19: void invert_comm_schedule(int **sched, int npes)
Chris@19: {
Chris@19:      int pe, i;
Chris@19: 
Chris@19:      for (pe = 0; pe < npes; ++pe)
Chris@19: 	  for (i = 0; i < npes/2; ++i) {
Chris@19: 	       int dummy = sched[pe][i];
Chris@19: 	       sched[pe][i] = sched[pe][npes-1-i];
Chris@19: 	       sched[pe][npes-1-i] = dummy;
Chris@19: 	  }
Chris@19: }
Chris@19: 
Chris@19: /* Sort the schedule for sort_pe in ascending order of processor
Chris@19:    index.  Unfortunately, for odd npes (when schedule has a stall
Chris@19:    to begin with) this will introduce an extra stall due to
Chris@19:    the motion of the self-communication past a stall.  We could
Chris@19:    fix this if it were really important.  Actually, we don't
Chris@19:    get an extra stall when sort_pe == 0 or npes-1, which is sufficient
Chris@19:    for our purposes. */
Chris@19: void sort_comm_schedule(int **sched, int npes, int sort_pe)
Chris@19: {
Chris@19:      int i,j,pe;
Chris@19: 
Chris@19:      /* Note that we can do this sort in O(npes) swaps because we know
Chris@19: 	that the numbers we are sorting are just 0...npes-1.   But we'll
Chris@19: 	just do a bubble sort for simplicity here. */
Chris@19: 
Chris@19:      for (i = 0; i < npes - 1; ++i)
Chris@19: 	  for (j = i + 1; j < npes; ++j)
Chris@19: 	       if (sched[sort_pe][i] > sched[sort_pe][j]) {
Chris@19: 		    for (pe = 0; pe < npes; ++pe) {
Chris@19: 			 int s = sched[pe][i];
Chris@19: 			 sched[pe][i] = sched[pe][j];
Chris@19: 			 sched[pe][j] = s;
Chris@19: 		    }
Chris@19: 	       }
Chris@19: }
Chris@19: 
Chris@19: /* print the schedule (for debugging purposes) */
Chris@19: void print_comm_schedule(int **sched, int npes)
Chris@19: {
Chris@19:      int pe, i, width;
Chris@19: 
Chris@19:      if (npes < 10)
Chris@19: 	  width = 1;
Chris@19:      else if (npes < 100)
Chris@19: 	  width = 2;
Chris@19:      else
Chris@19: 	  width = 3;
Chris@19: 
Chris@19:      for (pe = 0; pe < npes; ++pe) {
Chris@19: 	  printf("pe %*d schedule:", width, pe);
Chris@19: 	  for (i = 0; sched[pe][i] != -1; ++i)
Chris@19: 	       printf("  %*d",width,sched[pe][i]);
Chris@19: 	  printf("\n");
Chris@19:      }
Chris@19: }
Chris@19: 
Chris@19: int main(int argc, char **argv)
Chris@19: {
Chris@19:      int **sched;
Chris@19:      int npes = -1, sortpe = -1, steps, i;
Chris@19: 
Chris@19:      if (argc >= 2) {
Chris@19: 	  npes = atoi(argv[1]);
Chris@19: 	  if (npes <= 0) {
Chris@19: 	       fprintf(stderr,"npes must be positive!");
Chris@19: 	       return 1;
Chris@19: 	  }
Chris@19:      }
Chris@19:      if (argc >= 3) {
Chris@19: 	  sortpe = atoi(argv[2]);
Chris@19: 	  if (sortpe < 0 || sortpe >= npes) {
Chris@19: 	       fprintf(stderr,"sortpe must be between 0 and npes-1.\n");
Chris@19: 	       return 1;
Chris@19: 	  }
Chris@19:      }
Chris@19: 
Chris@19:      if (npes != -1) {
Chris@19: 	  printf("Computing schedule for npes = %d:\n",npes);
Chris@19: 	  sched = make_comm_schedule(npes);
Chris@19: 	  if (!sched) {
Chris@19: 	       fprintf(stderr,"Out of memory!");
Chris@19: 	       return 6;
Chris@19: 	  }
Chris@19: 	  
Chris@19: 	  if (steps = check_comm_schedule(sched,npes))
Chris@19: 	       printf("schedule OK (takes %d steps to complete).\n", steps);
Chris@19: 	  else
Chris@19: 	       printf("schedule not OK.\n");
Chris@19: 
Chris@19: 	  print_comm_schedule(sched, npes);
Chris@19: 	  
Chris@19: 	  if (sortpe != -1) {
Chris@19: 	       printf("\nRe-creating schedule for pe = %d...\n", sortpe);
Chris@19: 	       int *sched1 = (int*) malloc(sizeof(int) * npes);
Chris@19: 	       for (i = 0; i < npes; ++i) sched1[i] = -1;
Chris@19: 	       fill1_comm_sched(sched1, sortpe, npes);
Chris@19: 	       printf("  =");
Chris@19: 	       for (i = 0; i < npes; ++i) 
Chris@19: 		    printf("  %*d", npes < 10 ? 1 : (npes < 100 ? 2 : 3),
Chris@19: 			   sched1[i]);
Chris@19: 	       printf("\n");
Chris@19: 
Chris@19: 	       printf("\nSorting schedule for sortpe = %d...\n", sortpe);
Chris@19: 	       sort_comm_schedule(sched,npes,sortpe);
Chris@19: 	       
Chris@19: 	       if (steps = check_comm_schedule(sched,npes))
Chris@19: 		    printf("schedule OK (takes %d steps to complete).\n", 
Chris@19: 			   steps);
Chris@19: 	       else
Chris@19: 		    printf("schedule not OK.\n");
Chris@19: 
Chris@19: 	       print_comm_schedule(sched, npes);
Chris@19: 
Chris@19: 	       printf("\nInverting schedule...\n");
Chris@19: 	       invert_comm_schedule(sched,npes);
Chris@19: 	       
Chris@19: 	       if (steps = check_comm_schedule(sched,npes))
Chris@19: 		    printf("schedule OK (takes %d steps to complete).\n", 
Chris@19: 			   steps);
Chris@19: 	       else
Chris@19: 		    printf("schedule not OK.\n");
Chris@19: 
Chris@19: 	       print_comm_schedule(sched, npes);
Chris@19: 	       
Chris@19: 	       free_comm_schedule(sched,npes);
Chris@19: 
Chris@19: 	       free(sched1);
Chris@19: 	  }
Chris@19:      }
Chris@19:      else {
Chris@19: 	  printf("Doing infinite tests...\n");
Chris@19: 	  for (npes = 1; ; ++npes) {
Chris@19: 	       int *sched1 = (int*) malloc(sizeof(int) * npes);
Chris@19: 	       printf("npes = %d...",npes);
Chris@19: 	       sched = make_comm_schedule(npes);
Chris@19: 	       if (!sched) {
Chris@19: 		    fprintf(stderr,"Out of memory!\n");
Chris@19: 		    return 5;
Chris@19: 	       }
Chris@19: 	       for (sortpe = 0; sortpe < npes; ++sortpe) {
Chris@19: 		    empty_comm_schedule(sched,npes);
Chris@19: 		    fill_comm_schedule(sched,npes);
Chris@19: 		    if (!check_comm_schedule(sched,npes)) {
Chris@19: 			 fprintf(stderr,
Chris@19: 				 "\n -- fill error for sortpe = %d!\n",sortpe);
Chris@19: 			 return 2;
Chris@19: 		    }
Chris@19: 
Chris@19: 		    for (i = 0; i < npes; ++i) sched1[i] = -1;
Chris@19: 		    fill1_comm_sched(sched1, sortpe, npes);
Chris@19: 		    for (i = 0; i < npes; ++i)
Chris@19: 			 if (sched1[i] != sched[sortpe][i])
Chris@19: 			      fprintf(stderr,
Chris@19: 				      "\n -- fill1 error for pe = %d!\n",
Chris@19: 				      sortpe);
Chris@19: 
Chris@19: 		    sort_comm_schedule(sched,npes,sortpe);
Chris@19: 		    if (!check_comm_schedule(sched,npes)) {
Chris@19: 			 fprintf(stderr,
Chris@19: 				 "\n -- sort error for sortpe = %d!\n",sortpe);
Chris@19: 			 return 3;
Chris@19: 		    }
Chris@19: 		    invert_comm_schedule(sched,npes);
Chris@19: 		    if (!check_comm_schedule(sched,npes)) {
Chris@19: 			 fprintf(stderr,
Chris@19: 				 "\n -- invert error for sortpe = %d!\n",
Chris@19: 				 sortpe);
Chris@19: 			 return 4;
Chris@19: 		    }
Chris@19: 	       }
Chris@19: 	       free_comm_schedule(sched,npes);
Chris@19: 	       printf("OK\n");
Chris@19: 	       if (npes % 50 == 0)
Chris@19: 		    printf("(...Hit Ctrl-C to stop...)\n");
Chris@19: 	       free(sched1);
Chris@19: 	  }
Chris@19:      }
Chris@19: 
Chris@19:      return 0;
Chris@19: }