Chris@42: /*
Chris@42:  * Copyright (c) 2003, 2007-14 Matteo Frigo
Chris@42:  * Copyright (c) 2003, 2007-14 Massachusetts Institute of Technology
Chris@42:  *
Chris@42:  * This program is free software; you can redistribute it and/or modify
Chris@42:  * it under the terms of the GNU General Public License as published by
Chris@42:  * the Free Software Foundation; either version 2 of the License, or
Chris@42:  * (at your option) any later version.
Chris@42:  *
Chris@42:  * This program is distributed in the hope that it will be useful,
Chris@42:  * but WITHOUT ANY WARRANTY; without even the implied warranty of
Chris@42:  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
Chris@42:  * GNU General Public License for more details.
Chris@42:  *
Chris@42:  * You should have received a copy of the GNU General Public License
Chris@42:  * along with this program; if not, write to the Free Software
Chris@42:  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
Chris@42:  *
Chris@42:  */
Chris@42: 
Chris@42: /* threads.c: Portable thread spawning for loops, via the X(spawn_loop)
Chris@42:    function.  The first portion of this file is a set of macros to
Chris@42:    spawn and join threads on various systems. */
Chris@42: 
Chris@42: #include "threads.h"
Chris@42: #include "api.h"
Chris@42: 
Chris@42: #if defined(USING_POSIX_THREADS)
Chris@42: 
Chris@42: #include <pthread.h>
Chris@42: 
Chris@42: #ifdef HAVE_UNISTD_H
Chris@42: #  include <unistd.h>
Chris@42: #endif
Chris@42: 
Chris@42: /* imlementation of semaphores and mutexes: */
Chris@42: #if (defined(_POSIX_SEMAPHORES) && (_POSIX_SEMAPHORES >= 200112L))
Chris@42: 
Chris@42:    /* If optional POSIX semaphores are supported, use them to
Chris@42:       implement both semaphores and mutexes. */
Chris@42: #  include <semaphore.h>
Chris@42: #  include <errno.h>
Chris@42: 
Chris@42:    typedef sem_t os_sem_t;
Chris@42: 
Chris@42:    static void os_sem_init(os_sem_t *s) { sem_init(s, 0, 0); }
Chris@42:    static void os_sem_destroy(os_sem_t *s) { sem_destroy(s); }
Chris@42: 
Chris@42:    static void os_sem_down(os_sem_t *s)
Chris@42:    {
Chris@42: 	int err;
Chris@42: 	do {
Chris@42: 	     err = sem_wait(s);
Chris@42: 	} while (err == -1 && errno == EINTR);
Chris@42: 	CK(err == 0);
Chris@42:    }
Chris@42: 
Chris@42:    static void os_sem_up(os_sem_t *s) { sem_post(s); }
Chris@42: 
Chris@42:    /*
Chris@42:       The reason why we use sem_t to implement mutexes is that I have
Chris@42:       seen mysterious hangs with glibc-2.7 and linux-2.6.22 when using
Chris@42:       pthread_mutex_t, but no hangs with sem_t or with linux >=
Chris@42:       2.6.24.  For lack of better information, sem_t looks like the
Chris@42:       safest choice.
Chris@42:    */
Chris@42:    typedef sem_t os_mutex_t;
Chris@42:    static void os_mutex_init(os_mutex_t *s) { sem_init(s, 0, 1); }
Chris@42:    #define os_mutex_destroy os_sem_destroy
Chris@42:    #define os_mutex_lock os_sem_down
Chris@42:    #define os_mutex_unlock os_sem_up
Chris@42: 
Chris@42: #else
Chris@42: 
Chris@42:    /* If optional POSIX semaphores are not defined, use pthread
Chris@42:       mutexes for mutexes, and simulate semaphores with condition
Chris@42:       variables */
Chris@42:    typedef pthread_mutex_t os_mutex_t;
Chris@42: 
Chris@42:    static void os_mutex_init(os_mutex_t *s) 
Chris@42:    { 
Chris@42: 	pthread_mutex_init(s, (pthread_mutexattr_t *)0);
Chris@42:    }
Chris@42: 
Chris@42:    static void os_mutex_destroy(os_mutex_t *s) { pthread_mutex_destroy(s); }
Chris@42:    static void os_mutex_lock(os_mutex_t *s) { pthread_mutex_lock(s); }
Chris@42:    static void os_mutex_unlock(os_mutex_t *s) { pthread_mutex_unlock(s); }
Chris@42: 
Chris@42:    typedef struct {
Chris@42: 	pthread_mutex_t m;
Chris@42: 	pthread_cond_t c;
Chris@42: 	volatile int x;
Chris@42:    } os_sem_t; 
Chris@42: 
Chris@42:    static void os_sem_init(os_sem_t *s)
Chris@42:    {
Chris@42: 	pthread_mutex_init(&s->m, (pthread_mutexattr_t *)0);
Chris@42: 	pthread_cond_init(&s->c, (pthread_condattr_t *)0);
Chris@42: 
Chris@42: 	/* wrap initialization in lock to exploit the release
Chris@42: 	   semantics of pthread_mutex_unlock() */
Chris@42: 	pthread_mutex_lock(&s->m);
Chris@42: 	s->x = 0;
Chris@42: 	pthread_mutex_unlock(&s->m);
Chris@42:    }
Chris@42: 
Chris@42:    static void os_sem_destroy(os_sem_t *s)
Chris@42:    {
Chris@42: 	pthread_mutex_destroy(&s->m);
Chris@42: 	pthread_cond_destroy(&s->c);
Chris@42:    }
Chris@42: 
Chris@42:    static void os_sem_down(os_sem_t *s)
Chris@42:    {
Chris@42: 	pthread_mutex_lock(&s->m);
Chris@42: 	while (s->x <= 0) 
Chris@42: 	     pthread_cond_wait(&s->c, &s->m);
Chris@42: 	--s->x;
Chris@42: 	pthread_mutex_unlock(&s->m);
Chris@42:    }
Chris@42: 
Chris@42:    static void os_sem_up(os_sem_t *s)
Chris@42:    {
Chris@42: 	pthread_mutex_lock(&s->m);
Chris@42: 	++s->x;
Chris@42: 	pthread_cond_signal(&s->c);
Chris@42: 	pthread_mutex_unlock(&s->m);
Chris@42:    }
Chris@42: 
Chris@42: #endif
Chris@42: 
Chris@42: #define FFTW_WORKER void *
Chris@42: 
Chris@42: static void os_create_thread(FFTW_WORKER (*worker)(void *arg), 
Chris@42: 			     void *arg)
Chris@42: {
Chris@42:      pthread_attr_t attr;
Chris@42:      pthread_t tid;
Chris@42: 
Chris@42:      pthread_attr_init(&attr);
Chris@42:      pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM); 
Chris@42:      pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
Chris@42: 
Chris@42:      pthread_create(&tid, &attr, worker, (void *)arg);
Chris@42:      pthread_attr_destroy(&attr);
Chris@42: }
Chris@42: 
Chris@42: static void os_destroy_thread(void)
Chris@42: {
Chris@42:      pthread_exit((void *)0);
Chris@42: }
Chris@42: 
Chris@42: /* support for static mutexes */
Chris@42: typedef pthread_mutex_t os_static_mutex_t;
Chris@42: #define OS_STATIC_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
Chris@42: static void os_static_mutex_lock(os_static_mutex_t *s) { pthread_mutex_lock(s); }
Chris@42: static void os_static_mutex_unlock(os_static_mutex_t *s) { pthread_mutex_unlock(s); }
Chris@42: 
Chris@42: #elif defined(__WIN32__) || defined(_WIN32) || defined(_WINDOWS)
Chris@42: /* hack: windef.h defines INT for its own purposes and this causes
Chris@42:    a conflict with our own INT in ifftw.h.  Divert the windows
Chris@42:    definition into another name unlikely to cause a conflict */
Chris@42: #define INT magnus_ab_INTegro_seclorum_nascitur_ordo
Chris@42: #include <windows.h>
Chris@42: #include <process.h>
Chris@42: #undef INT
Chris@42: 
Chris@42: typedef HANDLE os_mutex_t;
Chris@42: 
Chris@42: static void os_mutex_init(os_mutex_t *s) 
Chris@42: { 
Chris@42:      *s = CreateMutex(NULL, FALSE, NULL);
Chris@42: }
Chris@42: 
Chris@42: static void os_mutex_destroy(os_mutex_t *s) 
Chris@42: { 
Chris@42:      CloseHandle(*s);
Chris@42: }
Chris@42: 
Chris@42: static void os_mutex_lock(os_mutex_t *s)
Chris@42: { 
Chris@42:      WaitForSingleObject(*s, INFINITE);
Chris@42: }
Chris@42: 
Chris@42: static void os_mutex_unlock(os_mutex_t *s) 
Chris@42: { 
Chris@42:      ReleaseMutex(*s);
Chris@42: }
Chris@42: 
Chris@42: typedef HANDLE os_sem_t;
Chris@42: 
Chris@42: static void os_sem_init(os_sem_t *s) 
Chris@42: {
Chris@42:      *s = CreateSemaphore(NULL, 0, 0x7FFFFFFFL, NULL);
Chris@42: }
Chris@42: 
Chris@42: static void os_sem_destroy(os_sem_t *s) 
Chris@42: { 
Chris@42:      CloseHandle(*s);
Chris@42: }
Chris@42: 
Chris@42: static void os_sem_down(os_sem_t *s) 
Chris@42: { 
Chris@42:      WaitForSingleObject(*s, INFINITE);
Chris@42: }
Chris@42: 
Chris@42: static void os_sem_up(os_sem_t *s) 
Chris@42: {
Chris@42:      ReleaseSemaphore(*s, 1, NULL);
Chris@42: }
Chris@42: 
Chris@42: #define FFTW_WORKER unsigned __stdcall
Chris@42: typedef unsigned (__stdcall *winthread_start) (void *);
Chris@42: 
Chris@42: static void os_create_thread(winthread_start worker,
Chris@42: 			     void *arg)
Chris@42: {
Chris@42:      _beginthreadex((void *)NULL,               /* security attrib */
Chris@42: 		    0,				/* stack size */
Chris@42: 		    worker,                     /* start address */
Chris@42: 		    arg,			/* parameters */
Chris@42: 		    0,				/* creation flags */
Chris@42: 		    (unsigned *)NULL);		/* tid */
Chris@42: }
Chris@42: 
Chris@42: static void os_destroy_thread(void)
Chris@42: {
Chris@42:      _endthreadex(0);
Chris@42: }
Chris@42: 
Chris@42: /* windows does not have statically-initialized mutexes---fake a
Chris@42:    spinlock */
Chris@42: typedef volatile LONG os_static_mutex_t;
Chris@42: #define OS_STATIC_MUTEX_INITIALIZER 0
Chris@42: static void os_static_mutex_lock(os_static_mutex_t *s)
Chris@42: {
Chris@42:      while (InterlockedExchange(s, 1) == 1) {
Chris@42:           YieldProcessor();
Chris@42:           Sleep(0);
Chris@42:      }
Chris@42: }
Chris@42: static void os_static_mutex_unlock(os_static_mutex_t *s)
Chris@42: {
Chris@42:      LONG old = InterlockedExchange(s, 0);
Chris@42:      A(old == 1);
Chris@42: }
Chris@42: #else
Chris@42: #error "No threading layer defined"
Chris@42: #endif
Chris@42: 
Chris@42: /************************************************************************/
Chris@42: 
Chris@42: /* Main code: */
Chris@42: struct worker {
Chris@42:      os_sem_t ready;
Chris@42:      os_sem_t done;
Chris@42:      struct work *w;
Chris@42:      struct worker *cdr;
Chris@42: };
Chris@42: 
Chris@42: static struct worker *make_worker(void)
Chris@42: {
Chris@42:      struct worker *q = (struct worker *)MALLOC(sizeof(*q), OTHER);
Chris@42:      os_sem_init(&q->ready);
Chris@42:      os_sem_init(&q->done);
Chris@42:      return q;
Chris@42: }
Chris@42: 
Chris@42: static void unmake_worker(struct worker *q)
Chris@42: {
Chris@42:      os_sem_destroy(&q->done);
Chris@42:      os_sem_destroy(&q->ready);
Chris@42:      X(ifree)(q);
Chris@42: }
Chris@42: 
Chris@42: struct work {
Chris@42:      spawn_function proc;
Chris@42:      spawn_data d;
Chris@42:      struct worker *q; /* the worker responsible for performing this work */
Chris@42: };
Chris@42: 
Chris@42: static os_mutex_t queue_lock;
Chris@42: static os_sem_t termination_semaphore;
Chris@42: 
Chris@42: static struct worker *worker_queue;
Chris@42: #define WITH_QUEUE_LOCK(what)			\
Chris@42: {						\
Chris@42:      os_mutex_lock(&queue_lock);		\
Chris@42:      what;					\
Chris@42:      os_mutex_unlock(&queue_lock);		\
Chris@42: }
Chris@42: 
Chris@42: static FFTW_WORKER worker(void *arg)
Chris@42: {
Chris@42:      struct worker *ego = (struct worker *)arg;
Chris@42:      struct work *w;
Chris@42: 
Chris@42:      for (;;) {
Chris@42: 	  /* wait until work becomes available */
Chris@42: 	  os_sem_down(&ego->ready);
Chris@42: 
Chris@42: 	  w = ego->w;
Chris@42: 
Chris@42: 	  /* !w->proc ==> terminate worker */
Chris@42: 	  if (!w->proc) break;
Chris@42: 
Chris@42: 	  /* do the work */
Chris@42:           w->proc(&w->d);
Chris@42: 
Chris@42: 	  /* signal that work is done */
Chris@42: 	  os_sem_up(&ego->done);
Chris@42:      }
Chris@42: 
Chris@42:      /* termination protocol */
Chris@42:      os_sem_up(&termination_semaphore);
Chris@42: 
Chris@42:      os_destroy_thread();
Chris@42:      /* UNREACHABLE */
Chris@42:      return 0;
Chris@42: }
Chris@42: 
Chris@42: static void enqueue(struct worker *q)
Chris@42: {
Chris@42:      WITH_QUEUE_LOCK({
Chris@42: 	  q->cdr = worker_queue;
Chris@42: 	  worker_queue = q;
Chris@42:      });
Chris@42: }
Chris@42: 
Chris@42: static struct worker *dequeue(void)
Chris@42: {
Chris@42:      struct worker *q;
Chris@42: 
Chris@42:      WITH_QUEUE_LOCK({
Chris@42: 	  q = worker_queue;
Chris@42: 	  if (q) 
Chris@42: 	       worker_queue = q->cdr;
Chris@42:      });
Chris@42: 
Chris@42:      if (!q) {
Chris@42: 	  /* no worker is available.  Create one */
Chris@42: 	  q = make_worker();
Chris@42: 	  os_create_thread(worker, q);
Chris@42:      }
Chris@42: 
Chris@42:      return q;
Chris@42: }
Chris@42: 
Chris@42: 
Chris@42: static void kill_workforce(void)
Chris@42: {
Chris@42:      struct work w;
Chris@42: 
Chris@42:      w.proc = 0;
Chris@42: 
Chris@42:      THREAD_ON; /* needed for debugging mode: since make_worker
Chris@42: 		   is called from dequeue which is only called in
Chris@42: 		   thread_on mode, we need to unmake_worker in thread_on. */
Chris@42:      WITH_QUEUE_LOCK({
Chris@42: 	  /* tell all workers that they must terminate.  
Chris@42: 
Chris@42: 	     Because workers enqueue themselves before signaling the
Chris@42: 	     completion of the work, all workers belong to the worker queue
Chris@42: 	     if we get here.  Also, all workers are waiting at
Chris@42: 	     os_sem_down(ready), so we can hold the queue lock without
Chris@42: 	     deadlocking */
Chris@42: 	  while (worker_queue) {
Chris@42: 	       struct worker *q = worker_queue;
Chris@42: 	       worker_queue = q->cdr;
Chris@42: 	       q->w = &w;
Chris@42: 	       os_sem_up(&q->ready);
Chris@42: 	       os_sem_down(&termination_semaphore);
Chris@42: 	       unmake_worker(q);
Chris@42: 	  }
Chris@42:      });
Chris@42:      THREAD_OFF;
Chris@42: }
Chris@42: 
Chris@42: static os_static_mutex_t initialization_mutex = OS_STATIC_MUTEX_INITIALIZER;
Chris@42: 
Chris@42: int X(ithreads_init)(void)
Chris@42: {
Chris@42:      os_static_mutex_lock(&initialization_mutex); {
Chris@42:           os_mutex_init(&queue_lock);
Chris@42:           os_sem_init(&termination_semaphore);
Chris@42: 
Chris@42:           WITH_QUEUE_LOCK({
Chris@42:                worker_queue = 0;
Chris@42:           });
Chris@42:      } os_static_mutex_unlock(&initialization_mutex);
Chris@42: 
Chris@42:      return 0; /* no error */
Chris@42: }
Chris@42: 
Chris@42: /* Distribute a loop from 0 to loopmax-1 over nthreads threads.
Chris@42:    proc(d) is called to execute a block of iterations from d->min
Chris@42:    to d->max-1.  d->thr_num indicate the number of the thread
Chris@42:    that is executing proc (from 0 to nthreads-1), and d->data is
Chris@42:    the same as the data parameter passed to X(spawn_loop).
Chris@42: 
Chris@42:    This function returns only after all the threads have completed. */
Chris@42: void X(spawn_loop)(int loopmax, int nthr, spawn_function proc, void *data)
Chris@42: {
Chris@42:      int block_size;
Chris@42:      struct work *r;
Chris@42:      int i;
Chris@42: 
Chris@42:      A(loopmax >= 0);
Chris@42:      A(nthr > 0);
Chris@42:      A(proc);
Chris@42: 
Chris@42:      if (!loopmax) return;
Chris@42: 
Chris@42:      /* Choose the block size and number of threads in order to (1)
Chris@42:         minimize the critical path and (2) use the fewest threads that
Chris@42:         achieve the same critical path (to minimize overhead).
Chris@42:         e.g. if loopmax is 5 and nthr is 4, we should use only 3
Chris@42:         threads with block sizes of 2, 2, and 1. */
Chris@42:      block_size = (loopmax + nthr - 1) / nthr;
Chris@42:      nthr = (loopmax + block_size - 1) / block_size;
Chris@42: 
Chris@42:      THREAD_ON; /* prevent debugging mode from failing under threads */
Chris@42:      STACK_MALLOC(struct work *, r, sizeof(struct work) * nthr);
Chris@42: 	  
Chris@42:      /* distribute work: */
Chris@42:      for (i = 0; i < nthr; ++i) {
Chris@42: 	  struct work *w = &r[i];
Chris@42: 	  spawn_data *d = &w->d;
Chris@42: 
Chris@42: 	  d->max = (d->min = i * block_size) + block_size;
Chris@42: 	  if (d->max > loopmax)
Chris@42: 	       d->max = loopmax;
Chris@42: 	  d->thr_num = i;
Chris@42: 	  d->data = data;
Chris@42: 	  w->proc = proc;
Chris@42: 	   
Chris@42: 	  if (i == nthr - 1) {
Chris@42: 	       /* do the work ourselves */
Chris@42: 	       proc(d);
Chris@42: 	  } else {
Chris@42: 	       /* assign a worker to W */
Chris@42: 	       w->q = dequeue();
Chris@42: 
Chris@42: 	       /* tell worker w->q to do it */
Chris@42: 	       w->q->w = w; /* Dirac could have written this */
Chris@42: 	       os_sem_up(&w->q->ready);
Chris@42: 	  }
Chris@42:      }
Chris@42: 
Chris@42:      for (i = 0; i < nthr - 1; ++i) { 
Chris@42: 	  struct work *w = &r[i];
Chris@42: 	  os_sem_down(&w->q->done);
Chris@42: 	  enqueue(w->q);
Chris@42:      }
Chris@42: 
Chris@42:      STACK_FREE(r);
Chris@42:      THREAD_OFF; /* prevent debugging mode from failing under threads */
Chris@42: }
Chris@42: 
Chris@42: void X(threads_cleanup)(void)
Chris@42: {
Chris@42:      kill_workforce();
Chris@42:      os_mutex_destroy(&queue_lock);
Chris@42:      os_sem_destroy(&termination_semaphore);
Chris@42: }
Chris@42: 
Chris@42: static os_static_mutex_t install_planner_hooks_mutex = OS_STATIC_MUTEX_INITIALIZER;
Chris@42: static os_mutex_t planner_mutex;
Chris@42: static int planner_hooks_installed = 0;
Chris@42: 
Chris@42: static void lock_planner_mutex(void)
Chris@42: {
Chris@42:      os_mutex_lock(&planner_mutex);
Chris@42: }
Chris@42: 
Chris@42: static void unlock_planner_mutex(void)
Chris@42: {
Chris@42:      os_mutex_unlock(&planner_mutex);
Chris@42: }
Chris@42: 
Chris@42: void X(threads_register_planner_hooks)(void)
Chris@42: {
Chris@42:      os_static_mutex_lock(&install_planner_hooks_mutex); {
Chris@42:           if (!planner_hooks_installed) {
Chris@42:                os_mutex_init(&planner_mutex);
Chris@42:                X(set_planner_hooks)(lock_planner_mutex, unlock_planner_mutex);
Chris@42:                planner_hooks_installed = 1;
Chris@42:           }
Chris@42:      } os_static_mutex_unlock(&install_planner_hooks_mutex);
Chris@42: }