cannam@95: /*
cannam@95:  * Copyright (c) 2003, 2007-11 Matteo Frigo
cannam@95:  * Copyright (c) 2003, 2007-11 Massachusetts Institute of Technology
cannam@95:  *
cannam@95:  * Permission is hereby granted, free of charge, to any person obtaining
cannam@95:  * a copy of this software and associated documentation files (the
cannam@95:  * "Software"), to deal in the Software without restriction, including
cannam@95:  * without limitation the rights to use, copy, modify, merge, publish,
cannam@95:  * distribute, sublicense, and/or sell copies of the Software, and to
cannam@95:  * permit persons to whom the Software is furnished to do so, subject to
cannam@95:  * the following conditions:
cannam@95:  *
cannam@95:  * The above copyright notice and this permission notice shall be
cannam@95:  * included in all copies or substantial portions of the Software.
cannam@95:  *
cannam@95:  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
cannam@95:  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
cannam@95:  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
cannam@95:  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
cannam@95:  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
cannam@95:  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
cannam@95:  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
cannam@95:  *
cannam@95:  */
cannam@95: 
cannam@95: 
cannam@95: /* machine-dependent cycle counters code. Needs to be inlined. */
cannam@95: 
cannam@95: /***************************************************************************/
cannam@95: /* To use the cycle counters in your code, simply #include "cycle.h" (this
cannam@95:    file), and then use the functions/macros:
cannam@95: 
cannam@95:                  ticks getticks(void);
cannam@95: 
cannam@95:    ticks is an opaque typedef defined below, representing the current time.
cannam@95:    You extract the elapsed time between two calls to gettick() via:
cannam@95: 
cannam@95:                  double elapsed(ticks t1, ticks t0);
cannam@95: 
cannam@95:    which returns a double-precision variable in arbitrary units.  You
cannam@95:    are not expected to convert this into human units like seconds; it
cannam@95:    is intended only for *comparisons* of time intervals.
cannam@95: 
cannam@95:    (In order to use some of the OS-dependent timer routines like
cannam@95:    Solaris' gethrtime, you need to paste the autoconf snippet below
cannam@95:    into your configure.ac file and #include "config.h" before cycle.h,
cannam@95:    or define the relevant macros manually if you are not using autoconf.)
cannam@95: */
cannam@95: 
cannam@95: /***************************************************************************/
cannam@95: /* This file uses macros like HAVE_GETHRTIME that are assumed to be
cannam@95:    defined according to whether the corresponding function/type/header
cannam@95:    is available on your system.  The necessary macros are most
cannam@95:    conveniently defined if you are using GNU autoconf, via the tests:
cannam@95:    
cannam@95:    dnl ---------------------------------------------------------------------
cannam@95: 
cannam@95:    AC_C_INLINE
cannam@95:    AC_HEADER_TIME
cannam@95:    AC_CHECK_HEADERS([sys/time.h c_asm.h intrinsics.h mach/mach_time.h])
cannam@95: 
cannam@95:    AC_CHECK_TYPE([hrtime_t],[AC_DEFINE(HAVE_HRTIME_T, 1, [Define to 1 if hrtime_t is defined in <sys/time.h>])],,[#if HAVE_SYS_TIME_H
cannam@95: #include <sys/time.h>
cannam@95: #endif])
cannam@95: 
cannam@95:    AC_CHECK_FUNCS([gethrtime read_real_time time_base_to_time clock_gettime mach_absolute_time])
cannam@95: 
cannam@95:    dnl Cray UNICOS _rtc() (real-time clock) intrinsic
cannam@95:    AC_MSG_CHECKING([for _rtc intrinsic])
cannam@95:    rtc_ok=yes
cannam@95:    AC_TRY_LINK([#ifdef HAVE_INTRINSICS_H
cannam@95: #include <intrinsics.h>
cannam@95: #endif], [_rtc()], [AC_DEFINE(HAVE__RTC,1,[Define if you have the UNICOS _rtc() intrinsic.])], [rtc_ok=no])
cannam@95:    AC_MSG_RESULT($rtc_ok)
cannam@95: 
cannam@95:    dnl ---------------------------------------------------------------------
cannam@95: */
cannam@95: 
cannam@95: /***************************************************************************/
cannam@95: 
cannam@95: #if TIME_WITH_SYS_TIME
cannam@95: # include <sys/time.h>
cannam@95: # include <time.h>
cannam@95: #else
cannam@95: # if HAVE_SYS_TIME_H
cannam@95: #  include <sys/time.h>
cannam@95: # else
cannam@95: #  include <time.h>
cannam@95: # endif
cannam@95: #endif
cannam@95: 
cannam@95: #define INLINE_ELAPSED(INL) static INL double elapsed(ticks t1, ticks t0) \
cannam@95: {									  \
cannam@95:      return (double)t1 - (double)t0;					  \
cannam@95: }
cannam@95: 
cannam@95: /*----------------------------------------------------------------*/
cannam@95: /* Solaris */
cannam@95: #if defined(HAVE_GETHRTIME) && defined(HAVE_HRTIME_T) && !defined(HAVE_TICK_COUNTER)
cannam@95: typedef hrtime_t ticks;
cannam@95: 
cannam@95: #define getticks gethrtime
cannam@95: 
cannam@95: INLINE_ELAPSED(inline)
cannam@95: 
cannam@95: #define HAVE_TICK_COUNTER
cannam@95: #endif
cannam@95: 
cannam@95: /*----------------------------------------------------------------*/
cannam@95: /* AIX v. 4+ routines to read the real-time clock or time-base register */
cannam@95: #if defined(HAVE_READ_REAL_TIME) && defined(HAVE_TIME_BASE_TO_TIME) && !defined(HAVE_TICK_COUNTER)
cannam@95: typedef timebasestruct_t ticks;
cannam@95: 
cannam@95: static __inline ticks getticks(void)
cannam@95: {
cannam@95:      ticks t;
cannam@95:      read_real_time(&t, TIMEBASE_SZ);
cannam@95:      return t;
cannam@95: }
cannam@95: 
cannam@95: static __inline double elapsed(ticks t1, ticks t0) /* time in nanoseconds */
cannam@95: {
cannam@95:      time_base_to_time(&t1, TIMEBASE_SZ);
cannam@95:      time_base_to_time(&t0, TIMEBASE_SZ);
cannam@95:      return (((double)t1.tb_high - (double)t0.tb_high) * 1.0e9 + 
cannam@95: 	     ((double)t1.tb_low - (double)t0.tb_low));
cannam@95: }
cannam@95: 
cannam@95: #define HAVE_TICK_COUNTER
cannam@95: #endif
cannam@95: 
cannam@95: /*----------------------------------------------------------------*/
cannam@95: /*
cannam@95:  * PowerPC ``cycle'' counter using the time base register.
cannam@95:  */
cannam@95: #if ((((defined(__GNUC__) && (defined(__powerpc__) || defined(__ppc__))) || (defined(__MWERKS__) && defined(macintosh)))) || (defined(__IBM_GCC_ASM) && (defined(__powerpc__) || defined(__ppc__))))  && !defined(HAVE_TICK_COUNTER)
cannam@95: typedef unsigned long long ticks;
cannam@95: 
cannam@95: static __inline__ ticks getticks(void)
cannam@95: {
cannam@95:      unsigned int tbl, tbu0, tbu1;
cannam@95: 
cannam@95:      do {
cannam@95: 	  __asm__ __volatile__ ("mftbu %0" : "=r"(tbu0));
cannam@95: 	  __asm__ __volatile__ ("mftb %0" : "=r"(tbl));
cannam@95: 	  __asm__ __volatile__ ("mftbu %0" : "=r"(tbu1));
cannam@95:      } while (tbu0 != tbu1);
cannam@95: 
cannam@95:      return (((unsigned long long)tbu0) << 32) | tbl;
cannam@95: }
cannam@95: 
cannam@95: INLINE_ELAPSED(__inline__)
cannam@95: 
cannam@95: #define HAVE_TICK_COUNTER
cannam@95: #endif
cannam@95: 
cannam@95: /* MacOS/Mach (Darwin) time-base register interface (unlike UpTime,
cannam@95:    from Carbon, requires no additional libraries to be linked). */
cannam@95: #if defined(HAVE_MACH_ABSOLUTE_TIME) && defined(HAVE_MACH_MACH_TIME_H) && !defined(HAVE_TICK_COUNTER)
cannam@95: #include <mach/mach_time.h>
cannam@95: typedef uint64_t ticks;
cannam@95: #define getticks mach_absolute_time
cannam@95: INLINE_ELAPSED(__inline__)
cannam@95: #define HAVE_TICK_COUNTER
cannam@95: #endif
cannam@95: 
cannam@95: /*----------------------------------------------------------------*/
cannam@95: /*
cannam@95:  * Pentium cycle counter 
cannam@95:  */
cannam@95: #if (defined(__GNUC__) || defined(__ICC)) && defined(__i386__)  && !defined(HAVE_TICK_COUNTER)
cannam@95: typedef unsigned long long ticks;
cannam@95: 
cannam@95: static __inline__ ticks getticks(void)
cannam@95: {
cannam@95:      ticks ret;
cannam@95: 
cannam@95:      __asm__ __volatile__("rdtsc": "=A" (ret));
cannam@95:      /* no input, nothing else clobbered */
cannam@95:      return ret;
cannam@95: }
cannam@95: 
cannam@95: INLINE_ELAPSED(__inline__)
cannam@95: 
cannam@95: #define HAVE_TICK_COUNTER
cannam@95: #define TIME_MIN 5000.0   /* unreliable pentium IV cycle counter */
cannam@95: #endif
cannam@95: 
cannam@95: /* Visual C++ -- thanks to Morten Nissov for his help with this */
cannam@95: #if _MSC_VER >= 1200 && _M_IX86 >= 500 && !defined(HAVE_TICK_COUNTER)
cannam@95: #include <windows.h>
cannam@95: typedef LARGE_INTEGER ticks;
cannam@95: #define RDTSC __asm __emit 0fh __asm __emit 031h /* hack for VC++ 5.0 */
cannam@95: 
cannam@95: static __inline ticks getticks(void)
cannam@95: {
cannam@95:      ticks retval;
cannam@95: 
cannam@95:      __asm {
cannam@95: 	  RDTSC
cannam@95: 	  mov retval.HighPart, edx
cannam@95: 	  mov retval.LowPart, eax
cannam@95:      }
cannam@95:      return retval;
cannam@95: }
cannam@95: 
cannam@95: static __inline double elapsed(ticks t1, ticks t0)
cannam@95: {  
cannam@95:      return (double)t1.QuadPart - (double)t0.QuadPart;
cannam@95: }  
cannam@95: 
cannam@95: #define HAVE_TICK_COUNTER
cannam@95: #define TIME_MIN 5000.0   /* unreliable pentium IV cycle counter */
cannam@95: #endif
cannam@95: 
cannam@95: /*----------------------------------------------------------------*/
cannam@95: /*
cannam@95:  * X86-64 cycle counter
cannam@95:  */
cannam@95: #if (defined(__GNUC__) || defined(__ICC) || defined(__SUNPRO_C)) && defined(__x86_64__)  && !defined(HAVE_TICK_COUNTER)
cannam@95: typedef unsigned long long ticks;
cannam@95: 
cannam@95: static __inline__ ticks getticks(void)
cannam@95: {
cannam@95:      unsigned a, d; 
cannam@95:      asm volatile("rdtsc" : "=a" (a), "=d" (d)); 
cannam@95:      return ((ticks)a) | (((ticks)d) << 32); 
cannam@95: }
cannam@95: 
cannam@95: INLINE_ELAPSED(__inline__)
cannam@95: 
cannam@95: #define HAVE_TICK_COUNTER
cannam@95: #endif
cannam@95: 
cannam@95: /* PGI compiler, courtesy Cristiano Calonaci, Andrea Tarsi, & Roberto Gori.
cannam@95:    NOTE: this code will fail to link unless you use the -Masmkeyword compiler
cannam@95:    option (grrr). */
cannam@95: #if defined(__PGI) && defined(__x86_64__) && !defined(HAVE_TICK_COUNTER) 
cannam@95: typedef unsigned long long ticks;
cannam@95: static ticks getticks(void)
cannam@95: {
cannam@95:     asm(" rdtsc; shl    $0x20,%rdx; mov    %eax,%eax; or     %rdx,%rax;    ");
cannam@95: }
cannam@95: INLINE_ELAPSED(__inline__)
cannam@95: #define HAVE_TICK_COUNTER
cannam@95: #endif
cannam@95: 
cannam@95: /* Visual C++, courtesy of Dirk Michaelis */
cannam@95: #if _MSC_VER >= 1400 && (defined(_M_AMD64) || defined(_M_X64)) && !defined(HAVE_TICK_COUNTER)
cannam@95: 
cannam@95: #include <intrin.h>
cannam@95: #pragma intrinsic(__rdtsc)
cannam@95: typedef unsigned __int64 ticks;
cannam@95: #define getticks __rdtsc
cannam@95: INLINE_ELAPSED(__inline)
cannam@95: 
cannam@95: #define HAVE_TICK_COUNTER
cannam@95: #endif
cannam@95: 
cannam@95: /*----------------------------------------------------------------*/
cannam@95: /*
cannam@95:  * IA64 cycle counter
cannam@95:  */
cannam@95: 
cannam@95: /* intel's icc/ecc compiler */
cannam@95: #if (defined(__EDG_VERSION) || defined(__ECC)) && defined(__ia64__) && !defined(HAVE_TICK_COUNTER)
cannam@95: typedef unsigned long ticks;
cannam@95: #include <ia64intrin.h>
cannam@95: 
cannam@95: static __inline__ ticks getticks(void)
cannam@95: {
cannam@95:      return __getReg(_IA64_REG_AR_ITC);
cannam@95: }
cannam@95:  
cannam@95: INLINE_ELAPSED(__inline__)
cannam@95:  
cannam@95: #define HAVE_TICK_COUNTER
cannam@95: #endif
cannam@95: 
cannam@95: /* gcc */
cannam@95: #if defined(__GNUC__) && defined(__ia64__) && !defined(HAVE_TICK_COUNTER)
cannam@95: typedef unsigned long ticks;
cannam@95: 
cannam@95: static __inline__ ticks getticks(void)
cannam@95: {
cannam@95:      ticks ret;
cannam@95: 
cannam@95:      __asm__ __volatile__ ("mov %0=ar.itc" : "=r"(ret));
cannam@95:      return ret;
cannam@95: }
cannam@95: 
cannam@95: INLINE_ELAPSED(__inline__)
cannam@95: 
cannam@95: #define HAVE_TICK_COUNTER
cannam@95: #endif
cannam@95: 
cannam@95: /* HP/UX IA64 compiler, courtesy Teresa L. Johnson: */
cannam@95: #if defined(__hpux) && defined(__ia64) && !defined(HAVE_TICK_COUNTER)
cannam@95: #include <machine/sys/inline.h>
cannam@95: typedef unsigned long ticks;
cannam@95: 
cannam@95: static inline ticks getticks(void)
cannam@95: {
cannam@95:      ticks ret;
cannam@95: 
cannam@95:      ret = _Asm_mov_from_ar (_AREG_ITC);
cannam@95:      return ret;
cannam@95: }
cannam@95: 
cannam@95: INLINE_ELAPSED(inline)
cannam@95: 
cannam@95: #define HAVE_TICK_COUNTER
cannam@95: #endif
cannam@95: 
cannam@95: /* Microsoft Visual C++ */
cannam@95: #if defined(_MSC_VER) && defined(_M_IA64) && !defined(HAVE_TICK_COUNTER)
cannam@95: typedef unsigned __int64 ticks;
cannam@95: 
cannam@95: #  ifdef __cplusplus
cannam@95: extern "C"
cannam@95: #  endif
cannam@95: ticks __getReg(int whichReg);
cannam@95: #pragma intrinsic(__getReg)
cannam@95: 
cannam@95: static __inline ticks getticks(void)
cannam@95: {
cannam@95:      volatile ticks temp;
cannam@95:      temp = __getReg(3116);
cannam@95:      return temp;
cannam@95: }
cannam@95: 
cannam@95: INLINE_ELAPSED(inline)
cannam@95: 
cannam@95: #define HAVE_TICK_COUNTER
cannam@95: #endif
cannam@95: 
cannam@95: /*----------------------------------------------------------------*/
cannam@95: /*
cannam@95:  * PA-RISC cycle counter 
cannam@95:  */
cannam@95: #if defined(__hppa__) || defined(__hppa) && !defined(HAVE_TICK_COUNTER)
cannam@95: typedef unsigned long ticks;
cannam@95: 
cannam@95: #  ifdef __GNUC__
cannam@95: static __inline__ ticks getticks(void)
cannam@95: {
cannam@95:      ticks ret;
cannam@95: 
cannam@95:      __asm__ __volatile__("mfctl 16, %0": "=r" (ret));
cannam@95:      /* no input, nothing else clobbered */
cannam@95:      return ret;
cannam@95: }
cannam@95: #  else
cannam@95: #  include <machine/inline.h>
cannam@95: static inline unsigned long getticks(void)
cannam@95: {
cannam@95:      register ticks ret;
cannam@95:      _MFCTL(16, ret);
cannam@95:      return ret;
cannam@95: }
cannam@95: #  endif
cannam@95: 
cannam@95: INLINE_ELAPSED(inline)
cannam@95: 
cannam@95: #define HAVE_TICK_COUNTER
cannam@95: #endif
cannam@95: 
cannam@95: /*----------------------------------------------------------------*/
cannam@95: /* S390, courtesy of James Treacy */
cannam@95: #if defined(__GNUC__) && defined(__s390__) && !defined(HAVE_TICK_COUNTER)
cannam@95: typedef unsigned long long ticks;
cannam@95: 
cannam@95: static __inline__ ticks getticks(void)
cannam@95: {
cannam@95:      ticks cycles;
cannam@95:      __asm__("stck 0(%0)" : : "a" (&(cycles)) : "memory", "cc");
cannam@95:      return cycles;
cannam@95: }
cannam@95: 
cannam@95: INLINE_ELAPSED(__inline__)
cannam@95: 
cannam@95: #define HAVE_TICK_COUNTER
cannam@95: #endif
cannam@95: /*----------------------------------------------------------------*/
cannam@95: #if defined(__GNUC__) && defined(__alpha__) && !defined(HAVE_TICK_COUNTER)
cannam@95: /*
cannam@95:  * The 32-bit cycle counter on alpha overflows pretty quickly, 
cannam@95:  * unfortunately.  A 1GHz machine overflows in 4 seconds.
cannam@95:  */
cannam@95: typedef unsigned int ticks;
cannam@95: 
cannam@95: static __inline__ ticks getticks(void)
cannam@95: {
cannam@95:      unsigned long cc;
cannam@95:      __asm__ __volatile__ ("rpcc %0" : "=r"(cc));
cannam@95:      return (cc & 0xFFFFFFFF);
cannam@95: }
cannam@95: 
cannam@95: INLINE_ELAPSED(__inline__)
cannam@95: 
cannam@95: #define HAVE_TICK_COUNTER
cannam@95: #endif
cannam@95: 
cannam@95: /*----------------------------------------------------------------*/
cannam@95: #if defined(__GNUC__) && defined(__sparc_v9__) && !defined(HAVE_TICK_COUNTER)
cannam@95: typedef unsigned long ticks;
cannam@95: 
cannam@95: static __inline__ ticks getticks(void)
cannam@95: {
cannam@95:      ticks ret;
cannam@95:      __asm__ __volatile__("rd %%tick, %0" : "=r" (ret));
cannam@95:      return ret;
cannam@95: }
cannam@95: 
cannam@95: INLINE_ELAPSED(__inline__)
cannam@95: 
cannam@95: #define HAVE_TICK_COUNTER
cannam@95: #endif
cannam@95: 
cannam@95: /*----------------------------------------------------------------*/
cannam@95: #if (defined(__DECC) || defined(__DECCXX)) && defined(__alpha) && defined(HAVE_C_ASM_H) && !defined(HAVE_TICK_COUNTER)
cannam@95: #  include <c_asm.h>
cannam@95: typedef unsigned int ticks;
cannam@95: 
cannam@95: static __inline ticks getticks(void)
cannam@95: {
cannam@95:      unsigned long cc;
cannam@95:      cc = asm("rpcc %v0");
cannam@95:      return (cc & 0xFFFFFFFF);
cannam@95: }
cannam@95: 
cannam@95: INLINE_ELAPSED(__inline)
cannam@95: 
cannam@95: #define HAVE_TICK_COUNTER
cannam@95: #endif
cannam@95: /*----------------------------------------------------------------*/
cannam@95: /* SGI/Irix */
cannam@95: #if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_SGI_CYCLE) && !defined(HAVE_TICK_COUNTER)
cannam@95: typedef struct timespec ticks;
cannam@95: 
cannam@95: static inline ticks getticks(void)
cannam@95: {
cannam@95:      struct timespec t;
cannam@95:      clock_gettime(CLOCK_SGI_CYCLE, &t);
cannam@95:      return t;
cannam@95: }
cannam@95: 
cannam@95: static inline double elapsed(ticks t1, ticks t0)
cannam@95: {
cannam@95:      return ((double)t1.tv_sec - (double)t0.tv_sec) * 1.0E9 +
cannam@95: 	  ((double)t1.tv_nsec - (double)t0.tv_nsec);
cannam@95: }
cannam@95: #define HAVE_TICK_COUNTER
cannam@95: #endif
cannam@95: 
cannam@95: /*----------------------------------------------------------------*/
cannam@95: /* Cray UNICOS _rtc() intrinsic function */
cannam@95: #if defined(HAVE__RTC) && !defined(HAVE_TICK_COUNTER)
cannam@95: #ifdef HAVE_INTRINSICS_H
cannam@95: #  include <intrinsics.h>
cannam@95: #endif
cannam@95: 
cannam@95: typedef long long ticks;
cannam@95: 
cannam@95: #define getticks _rtc
cannam@95: 
cannam@95: INLINE_ELAPSED(inline)
cannam@95: 
cannam@95: #define HAVE_TICK_COUNTER
cannam@95: #endif
cannam@95: 
cannam@95: /*----------------------------------------------------------------*/
cannam@95: /* MIPS ZBus */
cannam@95: #if HAVE_MIPS_ZBUS_TIMER
cannam@95: #if defined(__mips__) && !defined(HAVE_TICK_COUNTER)
cannam@95: #include <sys/mman.h>
cannam@95: #include <unistd.h>
cannam@95: #include <fcntl.h>
cannam@95: 
cannam@95: typedef uint64_t ticks;
cannam@95: 
cannam@95: static inline ticks getticks(void)
cannam@95: {
cannam@95:   static uint64_t* addr = 0;
cannam@95: 
cannam@95:   if (addr == 0)
cannam@95:   {
cannam@95:     uint32_t rq_addr = 0x10030000;
cannam@95:     int fd;
cannam@95:     int pgsize;
cannam@95: 
cannam@95:     pgsize = getpagesize();
cannam@95:     fd = open ("/dev/mem", O_RDONLY | O_SYNC, 0);
cannam@95:     if (fd < 0) {
cannam@95:       perror("open");
cannam@95:       return NULL;
cannam@95:     }
cannam@95:     addr = mmap(0, pgsize, PROT_READ, MAP_SHARED, fd, rq_addr);
cannam@95:     close(fd);
cannam@95:     if (addr == (uint64_t *)-1) {
cannam@95:       perror("mmap");
cannam@95:       return NULL;
cannam@95:     }
cannam@95:   }
cannam@95: 
cannam@95:   return *addr;
cannam@95: }
cannam@95: 
cannam@95: INLINE_ELAPSED(inline)
cannam@95: 
cannam@95: #define HAVE_TICK_COUNTER
cannam@95: #endif
cannam@95: #endif /* HAVE_MIPS_ZBUS_TIMER */
cannam@95: