cannam@167: /*
cannam@167:  * Copyright (c) 2003, 2007-14 Matteo Frigo
cannam@167:  * Copyright (c) 2003, 2007-14 Massachusetts Institute of Technology
cannam@167:  *
cannam@167:  * This program is free software; you can redistribute it and/or modify
cannam@167:  * it under the terms of the GNU General Public License as published by
cannam@167:  * the Free Software Foundation; either version 2 of the License, or
cannam@167:  * (at your option) any later version.
cannam@167:  *
cannam@167:  * This program is distributed in the hope that it will be useful,
cannam@167:  * but WITHOUT ANY WARRANTY; without even the implied warranty of
cannam@167:  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
cannam@167:  * GNU General Public License for more details.
cannam@167:  *
cannam@167:  * You should have received a copy of the GNU General Public License
cannam@167:  * along with this program; if not, write to the Free Software
cannam@167:  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
cannam@167:  *
cannam@167:  */
cannam@167: 
cannam@167: 
cannam@167: /* FFTW internal header file */
cannam@167: #ifndef __IFFTW_H__
cannam@167: #define __IFFTW_H__
cannam@167: 
cannam@167: #include "config.h"
cannam@167: 
cannam@167: #include <stdlib.h>		/* size_t */
cannam@167: #include <stdarg.h>		/* va_list */
cannam@167: #include <stddef.h>             /* ptrdiff_t */
cannam@167: #include <limits.h>             /* INT_MAX */
cannam@167: 
cannam@167: #if HAVE_SYS_TYPES_H
cannam@167: # include <sys/types.h>
cannam@167: #endif
cannam@167: 
cannam@167: #if HAVE_STDINT_H
cannam@167: # include <stdint.h>             /* uintptr_t, maybe */
cannam@167: #endif
cannam@167: 
cannam@167: #if HAVE_INTTYPES_H
cannam@167: # include <inttypes.h>           /* uintptr_t, maybe */
cannam@167: #endif
cannam@167: 
cannam@167: #ifdef __cplusplus
cannam@167: extern "C"
cannam@167: {
cannam@167: #endif /* __cplusplus */
cannam@167: 
cannam@167: /* Windows annoyances -- since tests/hook.c uses some internal
cannam@167:    FFTW functions, we need to given them the dllexport attribute
cannam@167:    under Windows when compiling as a DLL (see api/fftw3.h). */
cannam@167: #if defined(FFTW_EXTERN)
cannam@167: #  define IFFTW_EXTERN FFTW_EXTERN
cannam@167: #elif (defined(FFTW_DLL) || defined(DLL_EXPORT)) \
cannam@167:  && (defined(_WIN32) || defined(__WIN32__))
cannam@167: #  define IFFTW_EXTERN extern __declspec(dllexport)
cannam@167: #else
cannam@167: #  define IFFTW_EXTERN extern
cannam@167: #endif
cannam@167: 
cannam@167: /* determine precision and name-mangling scheme */
cannam@167: #define CONCAT(prefix, name) prefix ## name
cannam@167: #if defined(FFTW_SINGLE)
cannam@167:   typedef float R;
cannam@167: # define X(name) CONCAT(fftwf_, name)
cannam@167: #elif defined(FFTW_LDOUBLE)
cannam@167:   typedef long double R;
cannam@167: # define X(name) CONCAT(fftwl_, name)
cannam@167: # define TRIGREAL_IS_LONG_DOUBLE
cannam@167: #elif defined(FFTW_QUAD)
cannam@167:   typedef __float128 R;
cannam@167: # define X(name) CONCAT(fftwq_, name)
cannam@167: # define TRIGREAL_IS_QUAD
cannam@167: #else
cannam@167:   typedef double R;
cannam@167: # define X(name) CONCAT(fftw_, name)
cannam@167: #endif
cannam@167: 
cannam@167: /*
cannam@167:   integral type large enough to contain a stride (what ``int'' should
cannam@167:   have been in the first place.
cannam@167: */
cannam@167: typedef ptrdiff_t INT;
cannam@167: 
cannam@167: /* dummy use of unused parameters to silence compiler warnings */
cannam@167: #define UNUSED(x) (void)x
cannam@167: 
cannam@167: #define NELEM(array) ((sizeof(array) / sizeof((array)[0])))
cannam@167: 
cannam@167: #define FFT_SIGN (-1)  /* sign convention for forward transforms */
cannam@167: extern void X(extract_reim)(int sign, R *c, R **r, R **i);
cannam@167: 
cannam@167: #define REGISTER_SOLVER(p, s) X(solver_register)(p, s)
cannam@167: 
cannam@167: #define STRINGIZEx(x) #x
cannam@167: #define STRINGIZE(x) STRINGIZEx(x)
cannam@167: #define CIMPLIES(ante, post) (!(ante) || (post))
cannam@167: 
cannam@167: /* define HAVE_SIMD if any simd extensions are supported */
cannam@167: #if defined(HAVE_SSE) || defined(HAVE_SSE2) || \
cannam@167:       defined(HAVE_AVX) || defined(HAVE_AVX_128_FMA) || \
cannam@167:       defined(HAVE_AVX2) || defined(HAVE_AVX512) || \
cannam@167:       defined(HAVE_KCVI) || \
cannam@167:       defined(HAVE_ALTIVEC) || defined(HAVE_VSX) || \
cannam@167:       defined(HAVE_MIPS_PS) || \
cannam@167:       defined(HAVE_GENERIC_SIMD128) || defined(HAVE_GENERIC_SIMD256)
cannam@167: #define HAVE_SIMD 1
cannam@167: #else
cannam@167: #define HAVE_SIMD 0
cannam@167: #endif
cannam@167: 
cannam@167: extern int X(have_simd_sse2)(void);
cannam@167: extern int X(have_simd_avx)(void);
cannam@167: extern int X(have_simd_avx_128_fma)(void);
cannam@167: extern int X(have_simd_avx2)(void);
cannam@167: extern int X(have_simd_avx2_128)(void);
cannam@167: extern int X(have_simd_avx512)(void);
cannam@167: extern int X(have_simd_altivec)(void);
cannam@167: extern int X(have_simd_vsx)(void);
cannam@167: extern int X(have_simd_neon)(void);
cannam@167: 
cannam@167: /* forward declarations */
cannam@167: typedef struct problem_s problem;
cannam@167: typedef struct plan_s plan;
cannam@167: typedef struct solver_s solver;
cannam@167: typedef struct planner_s planner;
cannam@167: typedef struct printer_s printer;
cannam@167: typedef struct scanner_s scanner;
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* alloca: */
cannam@167: #if HAVE_SIMD
cannam@167: #  if defined(HAVE_KCVI) || defined(HAVE_AVX512)
cannam@167: #    define MIN_ALIGNMENT 64
cannam@167: #  elif defined(HAVE_AVX) || defined(HAVE_AVX2) || defined(HAVE_GENERIC_SIMD256)
cannam@167: #    define MIN_ALIGNMENT 32  /* best alignment for AVX, conservative for
cannam@167: 			       * everything else */
cannam@167: #  else
cannam@167:      /* Note that we cannot use 32-byte alignment for all SIMD.  For
cannam@167: 	example, MacOS X malloc is 16-byte aligned, but there was no
cannam@167: 	posix_memalign in MacOS X until version 10.6. */
cannam@167: #    define MIN_ALIGNMENT 16
cannam@167: #  endif
cannam@167: #endif
cannam@167: 
cannam@167: #if defined(HAVE_ALLOCA) && defined(FFTW_ENABLE_ALLOCA)
cannam@167:    /* use alloca if available */
cannam@167: 
cannam@167: #ifndef alloca
cannam@167: #ifdef __GNUC__
cannam@167: # define alloca __builtin_alloca
cannam@167: #else
cannam@167: # ifdef _MSC_VER
cannam@167: #  include <malloc.h>
cannam@167: #  define alloca _alloca
cannam@167: # else
cannam@167: #  if HAVE_ALLOCA_H
cannam@167: #   include <alloca.h>
cannam@167: #  else
cannam@167: #   ifdef _AIX
cannam@167:  #pragma alloca
cannam@167: #   else
cannam@167: #    ifndef alloca /* predefined by HP cc +Olibcalls */
cannam@167: void *alloca(size_t);
cannam@167: #    endif
cannam@167: #   endif
cannam@167: #  endif
cannam@167: # endif
cannam@167: #endif
cannam@167: #endif
cannam@167: 
cannam@167: #  ifdef MIN_ALIGNMENT
cannam@167: #    define STACK_MALLOC(T, p, n)				\
cannam@167:      {								\
cannam@167:          p = (T)alloca((n) + MIN_ALIGNMENT);			\
cannam@167:          p = (T)(((uintptr_t)p + (MIN_ALIGNMENT - 1)) &	\
cannam@167:                (~(uintptr_t)(MIN_ALIGNMENT - 1)));		\
cannam@167:      }
cannam@167: #    define STACK_FREE(n) 
cannam@167: #  else /* HAVE_ALLOCA && !defined(MIN_ALIGNMENT) */
cannam@167: #    define STACK_MALLOC(T, p, n) p = (T)alloca(n) 
cannam@167: #    define STACK_FREE(n) 
cannam@167: #  endif
cannam@167: 
cannam@167: #else /* ! HAVE_ALLOCA */
cannam@167:    /* use malloc instead of alloca */
cannam@167: #  define STACK_MALLOC(T, p, n) p = (T)MALLOC(n, OTHER)
cannam@167: #  define STACK_FREE(n) X(ifree)(n)
cannam@167: #endif /* ! HAVE_ALLOCA */
cannam@167: 
cannam@167: /* allocation of buffers.  If these grow too large use malloc(), else
cannam@167:    use STACK_MALLOC (hopefully reducing to alloca()). */
cannam@167: 
cannam@167: /* 64KiB ought to be enough for anybody */
cannam@167: #define MAX_STACK_ALLOC ((size_t)64 * 1024)
cannam@167: 
cannam@167: #define BUF_ALLOC(T, p, n)			\
cannam@167: {						\
cannam@167:      if (n < MAX_STACK_ALLOC) {			\
cannam@167: 	  STACK_MALLOC(T, p, n);		\
cannam@167:      } else {					\
cannam@167: 	  p = (T)MALLOC(n, BUFFERS);		\
cannam@167:      }						\
cannam@167: }
cannam@167: 
cannam@167: #define BUF_FREE(p, n)				\
cannam@167: {						\
cannam@167:      if (n < MAX_STACK_ALLOC) {			\
cannam@167: 	  STACK_FREE(p);			\
cannam@167:      } else {					\
cannam@167: 	  X(ifree)(p);				\
cannam@167:      }						\
cannam@167: }
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* define uintptr_t if it is not already defined */
cannam@167: 
cannam@167: #ifndef HAVE_UINTPTR_T
cannam@167: #  if SIZEOF_VOID_P == 0
cannam@167: #    error sizeof void* is unknown!
cannam@167: #  elif SIZEOF_UNSIGNED_INT == SIZEOF_VOID_P
cannam@167:      typedef unsigned int uintptr_t;
cannam@167: #  elif SIZEOF_UNSIGNED_LONG == SIZEOF_VOID_P
cannam@167:      typedef unsigned long uintptr_t;
cannam@167: #  elif SIZEOF_UNSIGNED_LONG_LONG == SIZEOF_VOID_P
cannam@167:      typedef unsigned long long uintptr_t;
cannam@167: #  else
cannam@167: #    error no unsigned integer type matches void* sizeof!
cannam@167: #  endif
cannam@167: #endif
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* We can do an optimization for copying pairs of (aligned) floats
cannam@167:    when in single precision if 2*float = double. */
cannam@167: 
cannam@167: #define FFTW_2R_IS_DOUBLE (defined(FFTW_SINGLE) \
cannam@167:                            && SIZEOF_FLOAT != 0 \
cannam@167:                            && SIZEOF_DOUBLE == 2*SIZEOF_FLOAT)
cannam@167: 
cannam@167: #define DOUBLE_ALIGNED(p) ((((uintptr_t)(p)) % sizeof(double)) == 0)
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* assert.c: */
cannam@167: IFFTW_EXTERN void X(assertion_failed)(const char *s, 
cannam@167: 				      int line, const char *file);
cannam@167: 
cannam@167: /* always check */
cannam@167: #define CK(ex)						 \
cannam@167:       (void)((ex) || (X(assertion_failed)(#ex, __LINE__, __FILE__), 0))
cannam@167: 
cannam@167: #ifdef FFTW_DEBUG
cannam@167: /* check only if debug enabled */
cannam@167: #define A(ex)						 \
cannam@167:       (void)((ex) || (X(assertion_failed)(#ex, __LINE__, __FILE__), 0))
cannam@167: #else
cannam@167: #define A(ex) /* nothing */
cannam@167: #endif
cannam@167: 
cannam@167: extern void X(debug)(const char *format, ...);
cannam@167: #define D X(debug)
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* kalloc.c: */
cannam@167: extern void *X(kernel_malloc)(size_t n);
cannam@167: extern void X(kernel_free)(void *p);
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* alloc.c: */
cannam@167: 
cannam@167: /* objects allocated by malloc, for statistical purposes */
cannam@167: enum malloc_tag {
cannam@167:      EVERYTHING,
cannam@167:      PLANS,
cannam@167:      SOLVERS,
cannam@167:      PROBLEMS,
cannam@167:      BUFFERS,
cannam@167:      HASHT,
cannam@167:      TENSORS,
cannam@167:      PLANNERS,
cannam@167:      SLVDESCS,
cannam@167:      TWIDDLES,
cannam@167:      STRIDES,
cannam@167:      OTHER,
cannam@167:      MALLOC_WHAT_LAST		/* must be last */
cannam@167: };
cannam@167: 
cannam@167: IFFTW_EXTERN void X(ifree)(void *ptr);
cannam@167: extern void X(ifree0)(void *ptr);
cannam@167: 
cannam@167: IFFTW_EXTERN void *X(malloc_plain)(size_t sz);
cannam@167: #define MALLOC(n, what)  X(malloc_plain)(n)
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* low-resolution clock */
cannam@167: 
cannam@167: #ifdef FAKE_CRUDE_TIME
cannam@167:  typedef int crude_time;
cannam@167: #else
cannam@167: # if TIME_WITH_SYS_TIME
cannam@167: #  include <sys/time.h>
cannam@167: #  include <time.h>
cannam@167: # else
cannam@167: #  if HAVE_SYS_TIME_H
cannam@167: #   include <sys/time.h>
cannam@167: #  else
cannam@167: #   include <time.h>
cannam@167: #  endif
cannam@167: # endif
cannam@167: 
cannam@167: # ifdef HAVE_BSDGETTIMEOFDAY
cannam@167: # ifndef HAVE_GETTIMEOFDAY
cannam@167: # define gettimeofday BSDgettimeofday
cannam@167: # define HAVE_GETTIMEOFDAY 1
cannam@167: # endif
cannam@167: # endif
cannam@167: 
cannam@167: # if defined(HAVE_GETTIMEOFDAY)
cannam@167:    typedef struct timeval crude_time;
cannam@167: # else
cannam@167:    typedef clock_t crude_time;
cannam@167: # endif
cannam@167: #endif /* else FAKE_CRUDE_TIME */
cannam@167: 
cannam@167: crude_time X(get_crude_time)(void);
cannam@167: double X(elapsed_since)(const planner *plnr, const problem *p,
cannam@167: 			crude_time t0); /* time in seconds since t0 */
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* ops.c: */
cannam@167: /*
cannam@167:  * ops counter.  The total number of additions is add + fma
cannam@167:  * and the total number of multiplications is mul + fma.
cannam@167:  * Total flops = add + mul + 2 * fma
cannam@167:  */
cannam@167: typedef struct {
cannam@167:      double add;
cannam@167:      double mul;
cannam@167:      double fma;
cannam@167:      double other;
cannam@167: } opcnt;
cannam@167: 
cannam@167: void X(ops_zero)(opcnt *dst);
cannam@167: void X(ops_other)(INT o, opcnt *dst);
cannam@167: void X(ops_cpy)(const opcnt *src, opcnt *dst);
cannam@167: 
cannam@167: void X(ops_add)(const opcnt *a, const opcnt *b, opcnt *dst);
cannam@167: void X(ops_add2)(const opcnt *a, opcnt *dst);
cannam@167: 
cannam@167: /* dst = m * a + b */
cannam@167: void X(ops_madd)(INT m, const opcnt *a, const opcnt *b, opcnt *dst);
cannam@167: 
cannam@167: /* dst += m * a */
cannam@167: void X(ops_madd2)(INT m, const opcnt *a, opcnt *dst);
cannam@167: 
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* minmax.c: */
cannam@167: INT X(imax)(INT a, INT b);
cannam@167: INT X(imin)(INT a, INT b);
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* iabs.c: */
cannam@167: INT X(iabs)(INT a);
cannam@167: 
cannam@167: /* inline version */
cannam@167: #define IABS(x) (((x) < 0) ? (0 - (x)) : (x))
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* md5.c */
cannam@167: 
cannam@167: #if SIZEOF_UNSIGNED_INT >= 4
cannam@167: typedef unsigned int md5uint;
cannam@167: #else
cannam@167: typedef unsigned long md5uint; /* at least 32 bits as per C standard */
cannam@167: #endif
cannam@167: 
cannam@167: typedef md5uint md5sig[4];
cannam@167: 
cannam@167: typedef struct {
cannam@167:      md5sig s; /* state and signature */
cannam@167: 
cannam@167:      /* fields not meant to be used outside md5.c: */
cannam@167:      unsigned char c[64]; /* stuff not yet processed */
cannam@167:      unsigned l;  /* total length.  Should be 64 bits long, but this is
cannam@167: 		     good enough for us */
cannam@167: } md5;
cannam@167: 
cannam@167: void X(md5begin)(md5 *p);
cannam@167: void X(md5putb)(md5 *p, const void *d_, size_t len);
cannam@167: void X(md5puts)(md5 *p, const char *s);
cannam@167: void X(md5putc)(md5 *p, unsigned char c);
cannam@167: void X(md5int)(md5 *p, int i);
cannam@167: void X(md5INT)(md5 *p, INT i);
cannam@167: void X(md5unsigned)(md5 *p, unsigned i);
cannam@167: void X(md5end)(md5 *p);
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* tensor.c: */
cannam@167: #define STRUCT_HACK_KR
cannam@167: #undef STRUCT_HACK_C99
cannam@167: 
cannam@167: typedef struct {
cannam@167:      INT n;
cannam@167:      INT is;			/* input stride */
cannam@167:      INT os;			/* output stride */
cannam@167: } iodim;
cannam@167: 
cannam@167: typedef struct {
cannam@167:      int rnk;
cannam@167: #if defined(STRUCT_HACK_KR)
cannam@167:      iodim dims[1];
cannam@167: #elif defined(STRUCT_HACK_C99)
cannam@167:      iodim dims[];
cannam@167: #else
cannam@167:      iodim *dims;
cannam@167: #endif
cannam@167: } tensor;
cannam@167: 
cannam@167: /*
cannam@167:   Definition of rank -infinity.
cannam@167:   This definition has the property that if you want rank 0 or 1,
cannam@167:   you can simply test for rank <= 1.  This is a common case.
cannam@167:  
cannam@167:   A tensor of rank -infinity has size 0.
cannam@167: */
cannam@167: #define RNK_MINFTY  INT_MAX
cannam@167: #define FINITE_RNK(rnk) ((rnk) != RNK_MINFTY)
cannam@167: 
cannam@167: typedef enum { INPLACE_IS, INPLACE_OS } inplace_kind;
cannam@167: 
cannam@167: tensor *X(mktensor)(int rnk);
cannam@167: tensor *X(mktensor_0d)(void);
cannam@167: tensor *X(mktensor_1d)(INT n, INT is, INT os);
cannam@167: tensor *X(mktensor_2d)(INT n0, INT is0, INT os0,
cannam@167: 		       INT n1, INT is1, INT os1);
cannam@167: tensor *X(mktensor_3d)(INT n0, INT is0, INT os0,
cannam@167: 		       INT n1, INT is1, INT os1,
cannam@167: 		       INT n2, INT is2, INT os2);
cannam@167: tensor *X(mktensor_4d)(INT n0, INT is0, INT os0,
cannam@167: 		       INT n1, INT is1, INT os1,
cannam@167: 		       INT n2, INT is2, INT os2,
cannam@167: 		       INT n3, INT is3, INT os3);
cannam@167: tensor *X(mktensor_5d)(INT n0, INT is0, INT os0,
cannam@167: 		       INT n1, INT is1, INT os1,
cannam@167: 		       INT n2, INT is2, INT os2,
cannam@167: 		       INT n3, INT is3, INT os3,
cannam@167: 		       INT n4, INT is4, INT os4);
cannam@167: INT X(tensor_sz)(const tensor *sz);
cannam@167: void X(tensor_md5)(md5 *p, const tensor *t);
cannam@167: INT X(tensor_max_index)(const tensor *sz);
cannam@167: INT X(tensor_min_istride)(const tensor *sz);
cannam@167: INT X(tensor_min_ostride)(const tensor *sz);
cannam@167: INT X(tensor_min_stride)(const tensor *sz);
cannam@167: int X(tensor_inplace_strides)(const tensor *sz);
cannam@167: int X(tensor_inplace_strides2)(const tensor *a, const tensor *b);
cannam@167: int X(tensor_strides_decrease)(const tensor *sz, const tensor *vecsz,
cannam@167:                                inplace_kind k);
cannam@167: tensor *X(tensor_copy)(const tensor *sz);
cannam@167: int X(tensor_kosherp)(const tensor *x);
cannam@167: 
cannam@167: tensor *X(tensor_copy_inplace)(const tensor *sz, inplace_kind k);
cannam@167: tensor *X(tensor_copy_except)(const tensor *sz, int except_dim);
cannam@167: tensor *X(tensor_copy_sub)(const tensor *sz, int start_dim, int rnk);
cannam@167: tensor *X(tensor_compress)(const tensor *sz);
cannam@167: tensor *X(tensor_compress_contiguous)(const tensor *sz);
cannam@167: tensor *X(tensor_append)(const tensor *a, const tensor *b);
cannam@167: void X(tensor_split)(const tensor *sz, tensor **a, int a_rnk, tensor **b);
cannam@167: int X(tensor_tornk1)(const tensor *t, INT *n, INT *is, INT *os);
cannam@167: void X(tensor_destroy)(tensor *sz);
cannam@167: void X(tensor_destroy2)(tensor *a, tensor *b);
cannam@167: void X(tensor_destroy4)(tensor *a, tensor *b, tensor *c, tensor *d);
cannam@167: void X(tensor_print)(const tensor *sz, printer *p);
cannam@167: int X(dimcmp)(const iodim *a, const iodim *b);
cannam@167: int X(tensor_equal)(const tensor *a, const tensor *b);
cannam@167: int X(tensor_inplace_locations)(const tensor *sz, const tensor *vecsz);
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* problem.c: */
cannam@167: enum { 
cannam@167:      /* a problem that cannot be solved */
cannam@167:      PROBLEM_UNSOLVABLE,
cannam@167: 
cannam@167:      PROBLEM_DFT, 
cannam@167:      PROBLEM_RDFT,
cannam@167:      PROBLEM_RDFT2,
cannam@167: 
cannam@167:      /* for mpi/ subdirectory */
cannam@167:      PROBLEM_MPI_DFT,
cannam@167:      PROBLEM_MPI_RDFT,
cannam@167:      PROBLEM_MPI_RDFT2,
cannam@167:      PROBLEM_MPI_TRANSPOSE,
cannam@167: 
cannam@167:      PROBLEM_LAST 
cannam@167: };
cannam@167: 
cannam@167: typedef struct {
cannam@167:      int problem_kind;
cannam@167:      void (*hash) (const problem *ego, md5 *p);
cannam@167:      void (*zero) (const problem *ego);
cannam@167:      void (*print) (const problem *ego, printer *p);
cannam@167:      void (*destroy) (problem *ego);
cannam@167: } problem_adt;
cannam@167: 
cannam@167: struct problem_s {
cannam@167:      const problem_adt *adt;
cannam@167: };
cannam@167: 
cannam@167: problem *X(mkproblem)(size_t sz, const problem_adt *adt);
cannam@167: void X(problem_destroy)(problem *ego);
cannam@167: problem *X(mkproblem_unsolvable)(void);
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* print.c */
cannam@167: struct printer_s {
cannam@167:      void (*print)(printer *p, const char *format, ...);
cannam@167:      void (*vprint)(printer *p, const char *format, va_list ap);
cannam@167:      void (*putchr)(printer *p, char c);
cannam@167:      void (*cleanup)(printer *p);
cannam@167:      int indent;
cannam@167:      int indent_incr;
cannam@167: };
cannam@167: 
cannam@167: printer *X(mkprinter)(size_t size, 
cannam@167: 		      void (*putchr)(printer *p, char c),
cannam@167: 		      void (*cleanup)(printer *p));
cannam@167: IFFTW_EXTERN void X(printer_destroy)(printer *p);
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* scan.c */
cannam@167: struct scanner_s {
cannam@167:      int (*scan)(scanner *sc, const char *format, ...);
cannam@167:      int (*vscan)(scanner *sc, const char *format, va_list ap);
cannam@167:      int (*getchr)(scanner *sc);
cannam@167:      int ungotc;
cannam@167: };
cannam@167: 
cannam@167: scanner *X(mkscanner)(size_t size, int (*getchr)(scanner *sc));
cannam@167: void X(scanner_destroy)(scanner *sc);
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* plan.c: */
cannam@167: 
cannam@167: enum wakefulness {
cannam@167:      SLEEPY,
cannam@167:      AWAKE_ZERO,
cannam@167:      AWAKE_SQRTN_TABLE,
cannam@167:      AWAKE_SINCOS
cannam@167: };
cannam@167: 
cannam@167: typedef struct {
cannam@167:      void (*solve)(const plan *ego, const problem *p);
cannam@167:      void (*awake)(plan *ego, enum wakefulness wakefulness);
cannam@167:      void (*print)(const plan *ego, printer *p);
cannam@167:      void (*destroy)(plan *ego);
cannam@167: } plan_adt;
cannam@167: 
cannam@167: struct plan_s {
cannam@167:      const plan_adt *adt;
cannam@167:      opcnt ops;
cannam@167:      double pcost;
cannam@167:      enum wakefulness wakefulness; /* used for debugging only */
cannam@167:      int could_prune_now_p;
cannam@167: };
cannam@167: 
cannam@167: plan *X(mkplan)(size_t size, const plan_adt *adt);
cannam@167: void X(plan_destroy_internal)(plan *ego);
cannam@167: IFFTW_EXTERN void X(plan_awake)(plan *ego, enum wakefulness wakefulness);
cannam@167: void X(plan_null_destroy)(plan *ego);
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* solver.c: */
cannam@167: typedef struct {
cannam@167:      int problem_kind;
cannam@167:      plan *(*mkplan)(const solver *ego, const problem *p, planner *plnr);
cannam@167:      void (*destroy)(solver *ego);
cannam@167: } solver_adt;
cannam@167: 
cannam@167: struct solver_s {
cannam@167:      const solver_adt *adt;
cannam@167:      int refcnt;
cannam@167: };
cannam@167: 
cannam@167: solver *X(mksolver)(size_t size, const solver_adt *adt);
cannam@167: void X(solver_use)(solver *ego);
cannam@167: void X(solver_destroy)(solver *ego);
cannam@167: void X(solver_register)(planner *plnr, solver *s);
cannam@167: 
cannam@167: /* shorthand */
cannam@167: #define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt)
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* planner.c */
cannam@167: 
cannam@167: typedef struct slvdesc_s {
cannam@167:      solver *slv;
cannam@167:      const char *reg_nam;
cannam@167:      unsigned nam_hash;
cannam@167:      int reg_id;
cannam@167:      int next_for_same_problem_kind;
cannam@167: } slvdesc;
cannam@167: 
cannam@167: typedef struct solution_s solution; /* opaque */
cannam@167: 
cannam@167: /* interpretation of L and U: 
cannam@167: 
cannam@167:    - if it returns a plan, the planner guarantees that all applicable
cannam@167:      plans at least as impatient as U have been tried, and that each
cannam@167:      plan in the solution is at least as impatient as L.
cannam@167:    
cannam@167:    - if it returns 0, the planner guarantees to have tried all solvers
cannam@167:      at least as impatient as L, and that none of them was applicable.
cannam@167: 
cannam@167:    The structure is packed to fit into 64 bits.
cannam@167: */
cannam@167: 
cannam@167: typedef struct {
cannam@167:      unsigned l:20;
cannam@167:      unsigned hash_info:3;
cannam@167: #    define BITS_FOR_TIMELIMIT 9
cannam@167:      unsigned timelimit_impatience:BITS_FOR_TIMELIMIT;
cannam@167:      unsigned u:20;
cannam@167:      
cannam@167:      /* abstraction break: we store the solver here to pad the
cannam@167: 	structure to 64 bits.  Otherwise, the struct is padded to 64
cannam@167: 	bits anyway, and another word is allocated for slvndx. */
cannam@167: #    define BITS_FOR_SLVNDX 12
cannam@167:      unsigned slvndx:BITS_FOR_SLVNDX;
cannam@167: } flags_t;
cannam@167: 
cannam@167: /* impatience flags  */
cannam@167: enum {
cannam@167:      BELIEVE_PCOST = 0x0001,
cannam@167:      ESTIMATE = 0x0002,
cannam@167:      NO_DFT_R2HC = 0x0004,
cannam@167:      NO_SLOW = 0x0008,
cannam@167:      NO_VRECURSE = 0x0010,
cannam@167:      NO_INDIRECT_OP = 0x0020,
cannam@167:      NO_LARGE_GENERIC = 0x0040,
cannam@167:      NO_RANK_SPLITS = 0x0080,
cannam@167:      NO_VRANK_SPLITS = 0x0100,
cannam@167:      NO_NONTHREADED = 0x0200,
cannam@167:      NO_BUFFERING = 0x0400,
cannam@167:      NO_FIXED_RADIX_LARGE_N = 0x0800,
cannam@167:      NO_DESTROY_INPUT = 0x1000,
cannam@167:      NO_SIMD = 0x2000,
cannam@167:      CONSERVE_MEMORY = 0x4000,
cannam@167:      NO_DHT_R2HC = 0x8000,
cannam@167:      NO_UGLY = 0x10000,
cannam@167:      ALLOW_PRUNING = 0x20000
cannam@167: };
cannam@167: 
cannam@167: /* hashtable information */
cannam@167: enum {
cannam@167:      BLESSING = 0x1u,   /* save this entry */
cannam@167:      H_VALID = 0x2u,    /* valid hastable entry */
cannam@167:      H_LIVE = 0x4u      /* entry is nonempty, implies H_VALID */
cannam@167: };
cannam@167: 
cannam@167: #define PLNR_L(plnr) ((plnr)->flags.l)
cannam@167: #define PLNR_U(plnr) ((plnr)->flags.u)
cannam@167: #define PLNR_TIMELIMIT_IMPATIENCE(plnr) ((plnr)->flags.timelimit_impatience)
cannam@167: 
cannam@167: #define ESTIMATEP(plnr) (PLNR_U(plnr) & ESTIMATE)
cannam@167: #define BELIEVE_PCOSTP(plnr) (PLNR_U(plnr) & BELIEVE_PCOST)
cannam@167: #define ALLOW_PRUNINGP(plnr) (PLNR_U(plnr) & ALLOW_PRUNING)
cannam@167: 
cannam@167: #define NO_INDIRECT_OP_P(plnr) (PLNR_L(plnr) & NO_INDIRECT_OP)
cannam@167: #define NO_LARGE_GENERICP(plnr) (PLNR_L(plnr) & NO_LARGE_GENERIC)
cannam@167: #define NO_RANK_SPLITSP(plnr) (PLNR_L(plnr) & NO_RANK_SPLITS)
cannam@167: #define NO_VRANK_SPLITSP(plnr) (PLNR_L(plnr) & NO_VRANK_SPLITS)
cannam@167: #define NO_VRECURSEP(plnr) (PLNR_L(plnr) & NO_VRECURSE)
cannam@167: #define NO_DFT_R2HCP(plnr) (PLNR_L(plnr) & NO_DFT_R2HC)
cannam@167: #define NO_SLOWP(plnr) (PLNR_L(plnr) & NO_SLOW)
cannam@167: #define NO_UGLYP(plnr) (PLNR_L(plnr) & NO_UGLY)
cannam@167: #define NO_FIXED_RADIX_LARGE_NP(plnr) \
cannam@167:   (PLNR_L(plnr) & NO_FIXED_RADIX_LARGE_N)
cannam@167: #define NO_NONTHREADEDP(plnr) \
cannam@167:   ((PLNR_L(plnr) & NO_NONTHREADED) && (plnr)->nthr > 1)
cannam@167: 
cannam@167: #define NO_DESTROY_INPUTP(plnr) (PLNR_L(plnr) & NO_DESTROY_INPUT)
cannam@167: #define NO_SIMDP(plnr) (PLNR_L(plnr) & NO_SIMD)
cannam@167: #define CONSERVE_MEMORYP(plnr) (PLNR_L(plnr) & CONSERVE_MEMORY)
cannam@167: #define NO_DHT_R2HCP(plnr) (PLNR_L(plnr) & NO_DHT_R2HC)
cannam@167: #define NO_BUFFERINGP(plnr) (PLNR_L(plnr) & NO_BUFFERING)
cannam@167: 
cannam@167: typedef enum { FORGET_ACCURSED, FORGET_EVERYTHING } amnesia;
cannam@167: 
cannam@167: typedef enum { 
cannam@167:      /* WISDOM_NORMAL: planner may or may not use wisdom */
cannam@167:      WISDOM_NORMAL, 
cannam@167: 
cannam@167:      /* WISDOM_ONLY: planner must use wisdom and must avoid searching */
cannam@167:      WISDOM_ONLY, 
cannam@167: 
cannam@167:      /* WISDOM_IS_BOGUS: planner must return 0 as quickly as possible */
cannam@167:      WISDOM_IS_BOGUS,
cannam@167: 
cannam@167:      /* WISDOM_IGNORE_INFEASIBLE: planner ignores infeasible wisdom */
cannam@167:      WISDOM_IGNORE_INFEASIBLE,
cannam@167: 
cannam@167:      /* WISDOM_IGNORE_ALL: planner ignores all */
cannam@167:      WISDOM_IGNORE_ALL
cannam@167: } wisdom_state_t;
cannam@167: 
cannam@167: typedef struct {
cannam@167:      void (*register_solver)(planner *ego, solver *s);
cannam@167:      plan *(*mkplan)(planner *ego, const problem *p);
cannam@167:      void (*forget)(planner *ego, amnesia a);
cannam@167:      void (*exprt)(planner *ego, printer *p); /* ``export'' is a reserved
cannam@167: 						 word in C++. */
cannam@167:      int (*imprt)(planner *ego, scanner *sc);
cannam@167: } planner_adt;
cannam@167: 
cannam@167: /* hash table of solutions */
cannam@167: typedef struct {
cannam@167:      solution *solutions;
cannam@167:      unsigned hashsiz, nelem;
cannam@167: 
cannam@167:      /* statistics */
cannam@167:      int lookup, succ_lookup, lookup_iter;
cannam@167:      int insert, insert_iter, insert_unknown;
cannam@167:      int nrehash;
cannam@167: } hashtab;
cannam@167: 
cannam@167: typedef enum { COST_SUM, COST_MAX } cost_kind;
cannam@167: 
cannam@167: struct planner_s {
cannam@167:      const planner_adt *adt;
cannam@167:      void (*hook)(struct planner_s *plnr, plan *pln, 
cannam@167: 		  const problem *p, int optimalp);
cannam@167:      double (*cost_hook)(const problem *p, double t, cost_kind k);
cannam@167:      int (*wisdom_ok_hook)(const problem *p, flags_t flags);
cannam@167:      void (*nowisdom_hook)(const problem *p);
cannam@167:      wisdom_state_t (*bogosity_hook)(wisdom_state_t state, const problem *p);
cannam@167: 
cannam@167:      /* solver descriptors */
cannam@167:      slvdesc *slvdescs;
cannam@167:      unsigned nslvdesc, slvdescsiz;
cannam@167:      const char *cur_reg_nam;
cannam@167:      int cur_reg_id;
cannam@167:      int slvdescs_for_problem_kind[PROBLEM_LAST];
cannam@167: 
cannam@167:      wisdom_state_t wisdom_state;
cannam@167: 
cannam@167:      hashtab htab_blessed;
cannam@167:      hashtab htab_unblessed;
cannam@167: 
cannam@167:      int nthr;
cannam@167:      flags_t flags;
cannam@167: 
cannam@167:      crude_time start_time;
cannam@167:      double timelimit; /* elapsed_since(start_time) at which to bail out */
cannam@167:      int timed_out; /* whether most recent search timed out */
cannam@167:      int need_timeout_check;
cannam@167: 
cannam@167:      /* various statistics */
cannam@167:      int nplan;    /* number of plans evaluated */
cannam@167:      double pcost, epcost; /* total pcost of measured/estimated plans */
cannam@167:      int nprob;    /* number of problems evaluated */
cannam@167: };
cannam@167: 
cannam@167: planner *X(mkplanner)(void);
cannam@167: void X(planner_destroy)(planner *ego);
cannam@167: 
cannam@167: /*
cannam@167:   Iterate over all solvers.   Read:
cannam@167:  
cannam@167:   @article{ baker93iterators,
cannam@167:   author = "Henry G. Baker, Jr.",
cannam@167:   title = "Iterators: Signs of Weakness in Object-Oriented Languages",
cannam@167:   journal = "{ACM} {OOPS} Messenger",
cannam@167:   volume = "4",
cannam@167:   number = "3",
cannam@167:   pages = "18--25"
cannam@167:   }
cannam@167: */
cannam@167: #define FORALL_SOLVERS(ego, s, p, what)			\
cannam@167: {							\
cannam@167:      unsigned _cnt;					\
cannam@167:      for (_cnt = 0; _cnt < ego->nslvdesc; ++_cnt) {	\
cannam@167: 	  slvdesc *p = ego->slvdescs + _cnt;		\
cannam@167: 	  solver *s = p->slv;				\
cannam@167: 	  what;						\
cannam@167:      }							\
cannam@167: }
cannam@167: 
cannam@167: #define FORALL_SOLVERS_OF_KIND(kind, ego, s, p, what)		\
cannam@167: {								\
cannam@167:      int _cnt = ego->slvdescs_for_problem_kind[kind]; 		\
cannam@167:      while (_cnt >= 0) {					\
cannam@167: 	  slvdesc *p = ego->slvdescs + _cnt;			\
cannam@167: 	  solver *s = p->slv;					\
cannam@167: 	  what;							\
cannam@167: 	  _cnt = p->next_for_same_problem_kind;			\
cannam@167:      }								\
cannam@167: }
cannam@167: 
cannam@167: 
cannam@167: /* make plan, destroy problem */
cannam@167: plan *X(mkplan_d)(planner *ego, problem *p);
cannam@167: plan *X(mkplan_f_d)(planner *ego, problem *p, 
cannam@167: 		    unsigned l_set, unsigned u_set, unsigned u_reset);
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* stride.c: */
cannam@167: 
cannam@167: /* If PRECOMPUTE_ARRAY_INDICES is defined, precompute all strides. */
cannam@167: #if (defined(__i386__) || defined(__x86_64__) || _M_IX86 >= 500) && !defined(FFTW_LDOUBLE)
cannam@167: #define PRECOMPUTE_ARRAY_INDICES
cannam@167: #endif
cannam@167: 
cannam@167: extern const INT X(an_INT_guaranteed_to_be_zero);
cannam@167: 
cannam@167: #ifdef PRECOMPUTE_ARRAY_INDICES
cannam@167: typedef INT *stride;
cannam@167: #define WS(stride, i)  (stride[i])
cannam@167: extern stride X(mkstride)(INT n, INT s);
cannam@167: void X(stride_destroy)(stride p);
cannam@167: /* hackery to prevent the compiler from copying the strides array
cannam@167:    onto the stack */
cannam@167: #define MAKE_VOLATILE_STRIDE(nptr, x) (x) = (x) + X(an_INT_guaranteed_to_be_zero)
cannam@167: #else
cannam@167: 
cannam@167: typedef INT stride;
cannam@167: #define WS(stride, i)  (stride * i)
cannam@167: #define fftwf_mkstride(n, stride) stride
cannam@167: #define fftw_mkstride(n, stride) stride
cannam@167: #define fftwl_mkstride(n, stride) stride
cannam@167: #define fftwf_stride_destroy(p) ((void) p)
cannam@167: #define fftw_stride_destroy(p) ((void) p)
cannam@167: #define fftwl_stride_destroy(p) ((void) p)
cannam@167: 
cannam@167: /* hackery to prevent the compiler from ``optimizing'' induction
cannam@167:    variables in codelet loops.  The problem is that for each K and for
cannam@167:    each expression of the form P[I + STRIDE * K] in a loop, most
cannam@167:    compilers will try to lift an induction variable PK := &P[I + STRIDE * K].
cannam@167:    For large values of K this behavior overflows the
cannam@167:    register set, which is likely worse than doing the index computation
cannam@167:    in the first place.
cannam@167: 
cannam@167:    If we guess that there are more than
cannam@167:    ESTIMATED_AVAILABLE_INDEX_REGISTERS such pointers, we deliberately confuse
cannam@167:    the compiler by setting STRIDE ^= ZERO, where ZERO is a value guaranteed to
cannam@167:    be 0, but the compiler does not know this. 
cannam@167: 
cannam@167:    16 registers ought to be enough for anybody, or so the amd64 and ARM ISA's
cannam@167:    seem to imply.
cannam@167: */
cannam@167: #define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16
cannam@167: #define MAKE_VOLATILE_STRIDE(nptr, x)                   \
cannam@167:      (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ?     \
cannam@167:         0 :                                             \
cannam@167:       ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero)))
cannam@167: #endif /* PRECOMPUTE_ARRAY_INDICES */
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* solvtab.c */
cannam@167: 
cannam@167: struct solvtab_s { void (*reg)(planner *); const char *reg_nam; };
cannam@167: typedef struct solvtab_s solvtab[];
cannam@167: void X(solvtab_exec)(const solvtab tbl, planner *p);
cannam@167: #define SOLVTAB(s) { s, STRINGIZE(s) }
cannam@167: #define SOLVTAB_END { 0, 0 }
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* pickdim.c */
cannam@167: int X(pickdim)(int which_dim, const int *buddies, size_t nbuddies,
cannam@167: 	       const tensor *sz, int oop, int *dp);
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* twiddle.c */
cannam@167: /* little language to express twiddle factors computation */
cannam@167: enum { TW_COS = 0, TW_SIN = 1, TW_CEXP = 2, TW_NEXT = 3, 
cannam@167:        TW_FULL = 4, TW_HALF = 5 };
cannam@167: 
cannam@167: typedef struct {
cannam@167:      unsigned char op;
cannam@167:      signed char v;
cannam@167:      short i;
cannam@167: } tw_instr;
cannam@167: 
cannam@167: typedef struct twid_s {
cannam@167:      R *W;                     /* array of twiddle factors */
cannam@167:      INT n, r, m;                /* transform order, radix, # twiddle rows */
cannam@167:      int refcnt;
cannam@167:      const tw_instr *instr;
cannam@167:      struct twid_s *cdr;
cannam@167:      enum wakefulness wakefulness;
cannam@167: } twid;
cannam@167: 
cannam@167: INT X(twiddle_length)(INT r, const tw_instr *p);
cannam@167: void X(twiddle_awake)(enum wakefulness wakefulness,
cannam@167: 		      twid **pp, const tw_instr *instr, INT n, INT r, INT m);
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* trig.c */
cannam@167: #if defined(TRIGREAL_IS_LONG_DOUBLE)
cannam@167:    typedef long double trigreal;
cannam@167: #elif defined(TRIGREAL_IS_QUAD)
cannam@167:    typedef __float128 trigreal;
cannam@167: #else
cannam@167:    typedef double trigreal;
cannam@167: #endif
cannam@167: 
cannam@167: typedef struct triggen_s triggen;
cannam@167: 
cannam@167: struct triggen_s {
cannam@167:      void (*cexp)(triggen *t, INT m, R *result);
cannam@167:      void (*cexpl)(triggen *t, INT m, trigreal *result);
cannam@167:      void (*rotate)(triggen *p, INT m, R xr, R xi, R *res);
cannam@167: 
cannam@167:      INT twshft;
cannam@167:      INT twradix;
cannam@167:      INT twmsk;
cannam@167:      trigreal *W0, *W1;
cannam@167:      INT n;
cannam@167: };
cannam@167: 
cannam@167: triggen *X(mktriggen)(enum wakefulness wakefulness, INT n);
cannam@167: void X(triggen_destroy)(triggen *p);
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* primes.c: */
cannam@167: 
cannam@167: #define MULMOD(x, y, p) \
cannam@167:    (((x) <= 92681 - (y)) ? ((x) * (y)) % (p) : X(safe_mulmod)(x, y, p))
cannam@167: 
cannam@167: INT X(safe_mulmod)(INT x, INT y, INT p);
cannam@167: INT X(power_mod)(INT n, INT m, INT p);
cannam@167: INT X(find_generator)(INT p);
cannam@167: INT X(first_divisor)(INT n);
cannam@167: int X(is_prime)(INT n);
cannam@167: INT X(next_prime)(INT n);
cannam@167: int X(factors_into)(INT n, const INT *primes);
cannam@167: int X(factors_into_small_primes)(INT n);
cannam@167: INT X(choose_radix)(INT r, INT n);
cannam@167: INT X(isqrt)(INT n);
cannam@167: INT X(modulo)(INT a, INT n);
cannam@167: 
cannam@167: #define GENERIC_MIN_BAD 173 /* min prime for which generic becomes bad */
cannam@167: 
cannam@167: /* thresholds below which certain solvers are considered SLOW.  These are guesses
cannam@167:    believed to be conservative */
cannam@167: #define GENERIC_MAX_SLOW     16
cannam@167: #define RADER_MAX_SLOW       32
cannam@167: #define BLUESTEIN_MAX_SLOW   24
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* rader.c: */
cannam@167: typedef struct rader_tls rader_tl;
cannam@167: 
cannam@167: void X(rader_tl_insert)(INT k1, INT k2, INT k3, R *W, rader_tl **tl);
cannam@167: R *X(rader_tl_find)(INT k1, INT k2, INT k3, rader_tl *t);
cannam@167: void X(rader_tl_delete)(R *W, rader_tl **tl);
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* copy/transposition routines */
cannam@167: 
cannam@167: /* lower bound to the cache size, for tiled routines */
cannam@167: #define CACHESIZE 8192
cannam@167: 
cannam@167: INT X(compute_tilesz)(INT vl, int how_many_tiles_in_cache);
cannam@167: 
cannam@167: void X(tile2d)(INT n0l, INT n0u, INT n1l, INT n1u, INT tilesz,
cannam@167: 	       void (*f)(INT n0l, INT n0u, INT n1l, INT n1u, void *args),
cannam@167: 	       void *args);
cannam@167: void X(cpy1d)(R *I, R *O, INT n0, INT is0, INT os0, INT vl);
cannam@167: void X(zero1d_pair)(R *O0, R *O1, INT n0, INT os0);
cannam@167: void X(cpy2d)(R *I, R *O,
cannam@167: 	      INT n0, INT is0, INT os0,
cannam@167: 	      INT n1, INT is1, INT os1,
cannam@167: 	      INT vl);
cannam@167: void X(cpy2d_ci)(R *I, R *O,
cannam@167: 		 INT n0, INT is0, INT os0,
cannam@167: 		 INT n1, INT is1, INT os1,
cannam@167: 		 INT vl);
cannam@167: void X(cpy2d_co)(R *I, R *O,
cannam@167: 		 INT n0, INT is0, INT os0,
cannam@167: 		 INT n1, INT is1, INT os1,
cannam@167: 		 INT vl);
cannam@167: void X(cpy2d_tiled)(R *I, R *O,
cannam@167: 		    INT n0, INT is0, INT os0,
cannam@167: 		    INT n1, INT is1, INT os1, 
cannam@167: 		    INT vl);
cannam@167: void X(cpy2d_tiledbuf)(R *I, R *O,
cannam@167: 		       INT n0, INT is0, INT os0,
cannam@167: 		       INT n1, INT is1, INT os1, 
cannam@167: 		       INT vl);
cannam@167: void X(cpy2d_pair)(R *I0, R *I1, R *O0, R *O1,
cannam@167: 		   INT n0, INT is0, INT os0,
cannam@167: 		   INT n1, INT is1, INT os1);
cannam@167: void X(cpy2d_pair_ci)(R *I0, R *I1, R *O0, R *O1,
cannam@167: 		      INT n0, INT is0, INT os0,
cannam@167: 		      INT n1, INT is1, INT os1);
cannam@167: void X(cpy2d_pair_co)(R *I0, R *I1, R *O0, R *O1,
cannam@167: 		      INT n0, INT is0, INT os0,
cannam@167: 		      INT n1, INT is1, INT os1);
cannam@167: 
cannam@167: void X(transpose)(R *I, INT n, INT s0, INT s1, INT vl);
cannam@167: void X(transpose_tiled)(R *I, INT n, INT s0, INT s1, INT vl);
cannam@167: void X(transpose_tiledbuf)(R *I, INT n, INT s0, INT s1, INT vl);
cannam@167: 
cannam@167: typedef void (*transpose_func)(R *I, INT n, INT s0, INT s1, INT vl);
cannam@167: typedef void (*cpy2d_func)(R *I, R *O,
cannam@167: 			   INT n0, INT is0, INT os0,
cannam@167: 			   INT n1, INT is1, INT os1,
cannam@167: 			   INT vl);
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* misc stuff */
cannam@167: void X(null_awake)(plan *ego, enum wakefulness wakefulness);
cannam@167: double X(iestimate_cost)(const planner *, const plan *, const problem *);
cannam@167: 
cannam@167: #ifdef FFTW_RANDOM_ESTIMATOR
cannam@167: extern unsigned X(random_estimate_seed);
cannam@167: #endif
cannam@167: 
cannam@167: double X(measure_execution_time)(const planner *plnr, 
cannam@167: 				 plan *pln, const problem *p);
cannam@167: IFFTW_EXTERN int X(ialignment_of)(R *p);
cannam@167: unsigned X(hash)(const char *s);
cannam@167: INT X(nbuf)(INT n, INT vl, INT maxnbuf);
cannam@167: int X(nbuf_redundant)(INT n, INT vl, size_t which, 
cannam@167: 		      const INT *maxnbuf, size_t nmaxnbuf);
cannam@167: INT X(bufdist)(INT n, INT vl);
cannam@167: int X(toobig)(INT n);
cannam@167: int X(ct_uglyp)(INT min_n, INT v, INT n, INT r);
cannam@167: 
cannam@167: #if HAVE_SIMD
cannam@167: R *X(taint)(R *p, INT s);
cannam@167: R *X(join_taint)(R *p1, R *p2);
cannam@167: #define TAINT(p, s) X(taint)(p, s)
cannam@167: #define UNTAINT(p) ((R *) (((uintptr_t) (p)) & ~(uintptr_t)3))
cannam@167: #define TAINTOF(p) (((uintptr_t)(p)) & 3)
cannam@167: #define JOIN_TAINT(p1, p2) X(join_taint)(p1, p2)
cannam@167: #else
cannam@167: #define TAINT(p, s) (p)
cannam@167: #define UNTAINT(p) (p)
cannam@167: #define TAINTOF(p) 0
cannam@167: #define JOIN_TAINT(p1, p2) p1
cannam@167: #endif
cannam@167: 
cannam@167: #define ASSERT_ALIGNED_DOUBLE  /*unused, legacy*/
cannam@167: 
cannam@167: /*-----------------------------------------------------------------------*/
cannam@167: /* macros used in codelets to reduce source code size */
cannam@167: 
cannam@167: typedef R E;  /* internal precision of codelets. */
cannam@167: 
cannam@167: #if defined(FFTW_LDOUBLE)
cannam@167: #  define K(x) ((E) x##L)
cannam@167: #elif defined(FFTW_QUAD)
cannam@167: #  define K(x) ((E) x##Q)
cannam@167: #else
cannam@167: #  define K(x) ((E) x)
cannam@167: #endif
cannam@167: #define DK(name, value) const E name = K(value)
cannam@167: 
cannam@167: /* FMA macros */
cannam@167: 
cannam@167: #if defined(__GNUC__) && (defined(__powerpc__) || defined(__ppc__) || defined(_POWER))
cannam@167: /* The obvious expression a * b + c does not work.  If both x = a * b
cannam@167:    + c and y = a * b - c appear in the source, gcc computes t = a * b,
cannam@167:    x = t + c, y = t - c, thus destroying the fma.
cannam@167: 
cannam@167:    This peculiar coding seems to do the right thing on all of
cannam@167:    gcc-2.95, gcc-3.1, gcc-3.2, and gcc-3.3.  It does the right thing
cannam@167:    on gcc-3.4 -fno-web (because the ``web'' pass splits the variable
cannam@167:    `x' for the single-assignment form).
cannam@167: 
cannam@167:    However, gcc-4.0 is a formidable adversary which succeeds in
cannam@167:    pessimizing two fma's into one multiplication and two additions.
cannam@167:    It does it very early in the game---before the optimization passes
cannam@167:    even start.  The only real workaround seems to use fake inline asm
cannam@167:    such as
cannam@167: 
cannam@167:      asm ("# confuse gcc %0" : "=f"(a) : "0"(a));
cannam@167:      return a * b + c;
cannam@167:      
cannam@167:    in each of the FMA, FMS, FNMA, and FNMS functions.  However, this
cannam@167:    does not solve the problem either, because two equal asm statements
cannam@167:    count as a common subexpression!  One must use *different* fake asm
cannam@167:    statements:
cannam@167: 
cannam@167:    in FMA:
cannam@167:      asm ("# confuse gcc for fma %0" : "=f"(a) : "0"(a));
cannam@167: 
cannam@167:    in FMS:
cannam@167:      asm ("# confuse gcc for fms %0" : "=f"(a) : "0"(a));
cannam@167: 
cannam@167:    etc.
cannam@167: 
cannam@167:    After these changes, gcc recalcitrantly generates the fma that was
cannam@167:    in the source to begin with.  However, the extra asm() cruft
cannam@167:    confuses other passes of gcc, notably the instruction scheduler.
cannam@167:    (Of course, one could also generate the fma directly via inline
cannam@167:    asm, but this confuses the scheduler even more.)
cannam@167: 
cannam@167:    Steven and I have submitted more than one bug report to the gcc
cannam@167:    mailing list over the past few years, to no effect.  Thus, I give
cannam@167:    up.  gcc-4.0 can go to hell.  I'll wait at least until gcc-4.3 is
cannam@167:    out before touching this crap again.
cannam@167: */
cannam@167: static __inline__ E FMA(E a, E b, E c)
cannam@167: {
cannam@167:      E x = a * b;
cannam@167:      x = x + c;
cannam@167:      return x;
cannam@167: }
cannam@167: 
cannam@167: static __inline__ E FMS(E a, E b, E c)
cannam@167: {
cannam@167:      E x = a * b;
cannam@167:      x = x - c;
cannam@167:      return x;
cannam@167: }
cannam@167: 
cannam@167: static __inline__ E FNMA(E a, E b, E c)
cannam@167: {
cannam@167:      E x = a * b;
cannam@167:      x = - (x + c);
cannam@167:      return x;
cannam@167: }
cannam@167: 
cannam@167: static __inline__ E FNMS(E a, E b, E c)
cannam@167: {
cannam@167:      E x = a * b;
cannam@167:      x = - (x - c);
cannam@167:      return x;
cannam@167: }
cannam@167: #else
cannam@167: #define FMA(a, b, c) (((a) * (b)) + (c))
cannam@167: #define FMS(a, b, c) (((a) * (b)) - (c))
cannam@167: #define FNMA(a, b, c) (- (((a) * (b)) + (c)))
cannam@167: #define FNMS(a, b, c) ((c) - ((a) * (b)))
cannam@167: #endif
cannam@167: 
cannam@167: #ifdef __cplusplus
cannam@167: }  /* extern "C" */
cannam@167: #endif /* __cplusplus */
cannam@167: 
cannam@167: #endif /* __IFFTW_H__ */