Chris@42: /* Chris@42: * Copyright (c) 2003, 2007-14 Matteo Frigo Chris@42: * Copyright (c) 2003, 2007-14 Massachusetts Institute of Technology Chris@42: * Chris@42: * Permission is hereby granted, free of charge, to any person obtaining Chris@42: * a copy of this software and associated documentation files (the Chris@42: * "Software"), to deal in the Software without restriction, including Chris@42: * without limitation the rights to use, copy, modify, merge, publish, Chris@42: * distribute, sublicense, and/or sell copies of the Software, and to Chris@42: * permit persons to whom the Software is furnished to do so, subject to Chris@42: * the following conditions: Chris@42: * Chris@42: * The above copyright notice and this permission notice shall be Chris@42: * included in all copies or substantial portions of the Software. Chris@42: * Chris@42: * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, Chris@42: * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF Chris@42: * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND Chris@42: * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE Chris@42: * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION Chris@42: * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION Chris@42: * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Chris@42: * Chris@42: */ Chris@42: Chris@42: Chris@42: /* machine-dependent cycle counters code. Needs to be inlined. */ Chris@42: Chris@42: /***************************************************************************/ Chris@42: /* To use the cycle counters in your code, simply #include "cycle.h" (this Chris@42: file), and then use the functions/macros: Chris@42: Chris@42: ticks getticks(void); Chris@42: Chris@42: ticks is an opaque typedef defined below, representing the current time. Chris@42: You extract the elapsed time between two calls to gettick() via: Chris@42: Chris@42: double elapsed(ticks t1, ticks t0); Chris@42: Chris@42: which returns a double-precision variable in arbitrary units. You Chris@42: are not expected to convert this into human units like seconds; it Chris@42: is intended only for *comparisons* of time intervals. Chris@42: Chris@42: (In order to use some of the OS-dependent timer routines like Chris@42: Solaris' gethrtime, you need to paste the autoconf snippet below Chris@42: into your configure.ac file and #include "config.h" before cycle.h, Chris@42: or define the relevant macros manually if you are not using autoconf.) Chris@42: */ Chris@42: Chris@42: /***************************************************************************/ Chris@42: /* This file uses macros like HAVE_GETHRTIME that are assumed to be Chris@42: defined according to whether the corresponding function/type/header Chris@42: is available on your system. The necessary macros are most Chris@42: conveniently defined if you are using GNU autoconf, via the tests: Chris@42: Chris@42: dnl --------------------------------------------------------------------- Chris@42: Chris@42: AC_C_INLINE Chris@42: AC_HEADER_TIME Chris@42: AC_CHECK_HEADERS([sys/time.h c_asm.h intrinsics.h mach/mach_time.h]) Chris@42: Chris@42: AC_CHECK_TYPE([hrtime_t],[AC_DEFINE(HAVE_HRTIME_T, 1, [Define to 1 if hrtime_t is defined in ])],,[#if HAVE_SYS_TIME_H Chris@42: #include Chris@42: #endif]) Chris@42: Chris@42: AC_CHECK_FUNCS([gethrtime read_real_time time_base_to_time clock_gettime mach_absolute_time]) Chris@42: Chris@42: dnl Cray UNICOS _rtc() (real-time clock) intrinsic Chris@42: AC_MSG_CHECKING([for _rtc intrinsic]) Chris@42: rtc_ok=yes Chris@42: AC_TRY_LINK([#ifdef HAVE_INTRINSICS_H Chris@42: #include Chris@42: #endif], [_rtc()], [AC_DEFINE(HAVE__RTC,1,[Define if you have the UNICOS _rtc() intrinsic.])], [rtc_ok=no]) Chris@42: AC_MSG_RESULT($rtc_ok) Chris@42: Chris@42: dnl --------------------------------------------------------------------- Chris@42: */ Chris@42: Chris@42: /***************************************************************************/ Chris@42: Chris@42: #if TIME_WITH_SYS_TIME Chris@42: # include Chris@42: # include Chris@42: #else Chris@42: # if HAVE_SYS_TIME_H Chris@42: # include Chris@42: # else Chris@42: # include Chris@42: # endif Chris@42: #endif Chris@42: Chris@42: #define INLINE_ELAPSED(INL) static INL double elapsed(ticks t1, ticks t0) \ Chris@42: { \ Chris@42: return (double)t1 - (double)t0; \ Chris@42: } Chris@42: Chris@42: /*----------------------------------------------------------------*/ Chris@42: /* Solaris */ Chris@42: #if defined(HAVE_GETHRTIME) && defined(HAVE_HRTIME_T) && !defined(HAVE_TICK_COUNTER) Chris@42: typedef hrtime_t ticks; Chris@42: Chris@42: #define getticks gethrtime Chris@42: Chris@42: INLINE_ELAPSED(inline) Chris@42: Chris@42: #define HAVE_TICK_COUNTER Chris@42: #endif Chris@42: Chris@42: /*----------------------------------------------------------------*/ Chris@42: /* AIX v. 4+ routines to read the real-time clock or time-base register */ Chris@42: #if defined(HAVE_READ_REAL_TIME) && defined(HAVE_TIME_BASE_TO_TIME) && !defined(HAVE_TICK_COUNTER) Chris@42: typedef timebasestruct_t ticks; Chris@42: Chris@42: static __inline ticks getticks(void) Chris@42: { Chris@42: ticks t; Chris@42: read_real_time(&t, TIMEBASE_SZ); Chris@42: return t; Chris@42: } Chris@42: Chris@42: static __inline double elapsed(ticks t1, ticks t0) /* time in nanoseconds */ Chris@42: { Chris@42: time_base_to_time(&t1, TIMEBASE_SZ); Chris@42: time_base_to_time(&t0, TIMEBASE_SZ); Chris@42: return (((double)t1.tb_high - (double)t0.tb_high) * 1.0e9 + Chris@42: ((double)t1.tb_low - (double)t0.tb_low)); Chris@42: } Chris@42: Chris@42: #define HAVE_TICK_COUNTER Chris@42: #endif Chris@42: Chris@42: /*----------------------------------------------------------------*/ Chris@42: /* Chris@42: * PowerPC ``cycle'' counter using the time base register. Chris@42: */ Chris@42: #if ((((defined(__GNUC__) && (defined(__powerpc__) || defined(__ppc__))) || (defined(__MWERKS__) && defined(macintosh)))) || (defined(__IBM_GCC_ASM) && (defined(__powerpc__) || defined(__ppc__)))) && !defined(HAVE_TICK_COUNTER) Chris@42: typedef unsigned long long ticks; Chris@42: Chris@42: static __inline__ ticks getticks(void) Chris@42: { Chris@42: unsigned int tbl, tbu0, tbu1; Chris@42: Chris@42: do { Chris@42: __asm__ __volatile__ ("mftbu %0" : "=r"(tbu0)); Chris@42: __asm__ __volatile__ ("mftb %0" : "=r"(tbl)); Chris@42: __asm__ __volatile__ ("mftbu %0" : "=r"(tbu1)); Chris@42: } while (tbu0 != tbu1); Chris@42: Chris@42: return (((unsigned long long)tbu0) << 32) | tbl; Chris@42: } Chris@42: Chris@42: INLINE_ELAPSED(__inline__) Chris@42: Chris@42: #define HAVE_TICK_COUNTER Chris@42: #endif Chris@42: Chris@42: /* MacOS/Mach (Darwin) time-base register interface (unlike UpTime, Chris@42: from Carbon, requires no additional libraries to be linked). */ Chris@42: #if defined(HAVE_MACH_ABSOLUTE_TIME) && defined(HAVE_MACH_MACH_TIME_H) && !defined(HAVE_TICK_COUNTER) Chris@42: #include Chris@42: typedef uint64_t ticks; Chris@42: #define getticks mach_absolute_time Chris@42: INLINE_ELAPSED(__inline__) Chris@42: #define HAVE_TICK_COUNTER Chris@42: #endif Chris@42: Chris@42: /*----------------------------------------------------------------*/ Chris@42: /* Chris@42: * Pentium cycle counter Chris@42: */ Chris@42: #if (defined(__GNUC__) || defined(__ICC)) && defined(__i386__) && !defined(HAVE_TICK_COUNTER) Chris@42: typedef unsigned long long ticks; Chris@42: Chris@42: static __inline__ ticks getticks(void) Chris@42: { Chris@42: ticks ret; Chris@42: Chris@42: __asm__ __volatile__("rdtsc": "=A" (ret)); Chris@42: /* no input, nothing else clobbered */ Chris@42: return ret; Chris@42: } Chris@42: Chris@42: INLINE_ELAPSED(__inline__) Chris@42: Chris@42: #define HAVE_TICK_COUNTER Chris@42: #define TIME_MIN 5000.0 /* unreliable pentium IV cycle counter */ Chris@42: #endif Chris@42: Chris@42: /* Visual C++ -- thanks to Morten Nissov for his help with this */ Chris@42: #if _MSC_VER >= 1200 && _M_IX86 >= 500 && !defined(HAVE_TICK_COUNTER) Chris@42: #include Chris@42: typedef LARGE_INTEGER ticks; Chris@42: #define RDTSC __asm __emit 0fh __asm __emit 031h /* hack for VC++ 5.0 */ Chris@42: Chris@42: static __inline ticks getticks(void) Chris@42: { Chris@42: ticks retval; Chris@42: Chris@42: __asm { Chris@42: RDTSC Chris@42: mov retval.HighPart, edx Chris@42: mov retval.LowPart, eax Chris@42: } Chris@42: return retval; Chris@42: } Chris@42: Chris@42: static __inline double elapsed(ticks t1, ticks t0) Chris@42: { Chris@42: return (double)t1.QuadPart - (double)t0.QuadPart; Chris@42: } Chris@42: Chris@42: #define HAVE_TICK_COUNTER Chris@42: #define TIME_MIN 5000.0 /* unreliable pentium IV cycle counter */ Chris@42: #endif Chris@42: Chris@42: /*----------------------------------------------------------------*/ Chris@42: /* Chris@42: * X86-64 cycle counter Chris@42: */ Chris@42: #if (defined(__GNUC__) || defined(__ICC) || defined(__SUNPRO_C)) && defined(__x86_64__) && !defined(HAVE_TICK_COUNTER) Chris@42: typedef unsigned long long ticks; Chris@42: Chris@42: static __inline__ ticks getticks(void) Chris@42: { Chris@42: unsigned a, d; Chris@42: asm volatile("rdtsc" : "=a" (a), "=d" (d)); Chris@42: return ((ticks)a) | (((ticks)d) << 32); Chris@42: } Chris@42: Chris@42: INLINE_ELAPSED(__inline__) Chris@42: Chris@42: #define HAVE_TICK_COUNTER Chris@42: #define TIME_MIN 5000.0 Chris@42: #endif Chris@42: Chris@42: /* PGI compiler, courtesy Cristiano Calonaci, Andrea Tarsi, & Roberto Gori. Chris@42: NOTE: this code will fail to link unless you use the -Masmkeyword compiler Chris@42: option (grrr). */ Chris@42: #if defined(__PGI) && defined(__x86_64__) && !defined(HAVE_TICK_COUNTER) Chris@42: typedef unsigned long long ticks; Chris@42: static ticks getticks(void) Chris@42: { Chris@42: asm(" rdtsc; shl $0x20,%rdx; mov %eax,%eax; or %rdx,%rax; "); Chris@42: } Chris@42: INLINE_ELAPSED(__inline__) Chris@42: #define HAVE_TICK_COUNTER Chris@42: #define TIME_MIN 5000.0 Chris@42: #endif Chris@42: Chris@42: /* Visual C++, courtesy of Dirk Michaelis */ Chris@42: #if _MSC_VER >= 1400 && (defined(_M_AMD64) || defined(_M_X64)) && !defined(HAVE_TICK_COUNTER) Chris@42: Chris@42: #include Chris@42: #pragma intrinsic(__rdtsc) Chris@42: typedef unsigned __int64 ticks; Chris@42: #define getticks __rdtsc Chris@42: INLINE_ELAPSED(__inline) Chris@42: Chris@42: #define HAVE_TICK_COUNTER Chris@42: #define TIME_MIN 5000.0 Chris@42: #endif Chris@42: Chris@42: /*----------------------------------------------------------------*/ Chris@42: /* Chris@42: * IA64 cycle counter Chris@42: */ Chris@42: Chris@42: /* intel's icc/ecc compiler */ Chris@42: #if (defined(__EDG_VERSION) || defined(__ECC)) && defined(__ia64__) && !defined(HAVE_TICK_COUNTER) Chris@42: typedef unsigned long ticks; Chris@42: #include Chris@42: Chris@42: static __inline__ ticks getticks(void) Chris@42: { Chris@42: return __getReg(_IA64_REG_AR_ITC); Chris@42: } Chris@42: Chris@42: INLINE_ELAPSED(__inline__) Chris@42: Chris@42: #define HAVE_TICK_COUNTER Chris@42: #endif Chris@42: Chris@42: /* gcc */ Chris@42: #if defined(__GNUC__) && defined(__ia64__) && !defined(HAVE_TICK_COUNTER) Chris@42: typedef unsigned long ticks; Chris@42: Chris@42: static __inline__ ticks getticks(void) Chris@42: { Chris@42: ticks ret; Chris@42: Chris@42: __asm__ __volatile__ ("mov %0=ar.itc" : "=r"(ret)); Chris@42: return ret; Chris@42: } Chris@42: Chris@42: INLINE_ELAPSED(__inline__) Chris@42: Chris@42: #define HAVE_TICK_COUNTER Chris@42: #endif Chris@42: Chris@42: /* HP/UX IA64 compiler, courtesy Teresa L. Johnson: */ Chris@42: #if defined(__hpux) && defined(__ia64) && !defined(HAVE_TICK_COUNTER) Chris@42: #include Chris@42: typedef unsigned long ticks; Chris@42: Chris@42: static inline ticks getticks(void) Chris@42: { Chris@42: ticks ret; Chris@42: Chris@42: ret = _Asm_mov_from_ar (_AREG_ITC); Chris@42: return ret; Chris@42: } Chris@42: Chris@42: INLINE_ELAPSED(inline) Chris@42: Chris@42: #define HAVE_TICK_COUNTER Chris@42: #endif Chris@42: Chris@42: /* Microsoft Visual C++ */ Chris@42: #if defined(_MSC_VER) && defined(_M_IA64) && !defined(HAVE_TICK_COUNTER) Chris@42: typedef unsigned __int64 ticks; Chris@42: Chris@42: # ifdef __cplusplus Chris@42: extern "C" Chris@42: # endif Chris@42: ticks __getReg(int whichReg); Chris@42: #pragma intrinsic(__getReg) Chris@42: Chris@42: static __inline ticks getticks(void) Chris@42: { Chris@42: volatile ticks temp; Chris@42: temp = __getReg(3116); Chris@42: return temp; Chris@42: } Chris@42: Chris@42: INLINE_ELAPSED(inline) Chris@42: Chris@42: #define HAVE_TICK_COUNTER Chris@42: #endif Chris@42: Chris@42: /*----------------------------------------------------------------*/ Chris@42: /* Chris@42: * PA-RISC cycle counter Chris@42: */ Chris@42: #if (defined(__hppa__) || defined(__hppa)) && !defined(HAVE_TICK_COUNTER) Chris@42: typedef unsigned long ticks; Chris@42: Chris@42: # ifdef __GNUC__ Chris@42: static __inline__ ticks getticks(void) Chris@42: { Chris@42: ticks ret; Chris@42: Chris@42: __asm__ __volatile__("mfctl 16, %0": "=r" (ret)); Chris@42: /* no input, nothing else clobbered */ Chris@42: return ret; Chris@42: } Chris@42: # else Chris@42: # include Chris@42: static inline unsigned long getticks(void) Chris@42: { Chris@42: register ticks ret; Chris@42: _MFCTL(16, ret); Chris@42: return ret; Chris@42: } Chris@42: # endif Chris@42: Chris@42: INLINE_ELAPSED(inline) Chris@42: Chris@42: #define HAVE_TICK_COUNTER Chris@42: #endif Chris@42: Chris@42: /*----------------------------------------------------------------*/ Chris@42: /* S390, courtesy of James Treacy */ Chris@42: #if defined(__GNUC__) && defined(__s390__) && !defined(HAVE_TICK_COUNTER) Chris@42: typedef unsigned long long ticks; Chris@42: Chris@42: static __inline__ ticks getticks(void) Chris@42: { Chris@42: ticks cycles; Chris@42: __asm__("stck 0(%0)" : : "a" (&(cycles)) : "memory", "cc"); Chris@42: return cycles; Chris@42: } Chris@42: Chris@42: INLINE_ELAPSED(__inline__) Chris@42: Chris@42: #define HAVE_TICK_COUNTER Chris@42: #endif Chris@42: /*----------------------------------------------------------------*/ Chris@42: #if defined(__GNUC__) && defined(__alpha__) && !defined(HAVE_TICK_COUNTER) Chris@42: /* Chris@42: * The 32-bit cycle counter on alpha overflows pretty quickly, Chris@42: * unfortunately. A 1GHz machine overflows in 4 seconds. Chris@42: */ Chris@42: typedef unsigned int ticks; Chris@42: Chris@42: static __inline__ ticks getticks(void) Chris@42: { Chris@42: unsigned long cc; Chris@42: __asm__ __volatile__ ("rpcc %0" : "=r"(cc)); Chris@42: return (cc & 0xFFFFFFFF); Chris@42: } Chris@42: Chris@42: INLINE_ELAPSED(__inline__) Chris@42: Chris@42: #define HAVE_TICK_COUNTER Chris@42: #endif Chris@42: Chris@42: /*----------------------------------------------------------------*/ Chris@42: #if defined(__GNUC__) && defined(__sparc_v9__) && !defined(HAVE_TICK_COUNTER) Chris@42: typedef unsigned long ticks; Chris@42: Chris@42: static __inline__ ticks getticks(void) Chris@42: { Chris@42: ticks ret; Chris@42: __asm__ __volatile__("rd %%tick, %0" : "=r" (ret)); Chris@42: return ret; Chris@42: } Chris@42: Chris@42: INLINE_ELAPSED(__inline__) Chris@42: Chris@42: #define HAVE_TICK_COUNTER Chris@42: #endif Chris@42: Chris@42: /*----------------------------------------------------------------*/ Chris@42: #if (defined(__DECC) || defined(__DECCXX)) && defined(__alpha) && defined(HAVE_C_ASM_H) && !defined(HAVE_TICK_COUNTER) Chris@42: # include Chris@42: typedef unsigned int ticks; Chris@42: Chris@42: static __inline ticks getticks(void) Chris@42: { Chris@42: unsigned long cc; Chris@42: cc = asm("rpcc %v0"); Chris@42: return (cc & 0xFFFFFFFF); Chris@42: } Chris@42: Chris@42: INLINE_ELAPSED(__inline) Chris@42: Chris@42: #define HAVE_TICK_COUNTER Chris@42: #endif Chris@42: /*----------------------------------------------------------------*/ Chris@42: /* SGI/Irix */ Chris@42: #if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_SGI_CYCLE) && !defined(HAVE_TICK_COUNTER) Chris@42: typedef struct timespec ticks; Chris@42: Chris@42: static inline ticks getticks(void) Chris@42: { Chris@42: struct timespec t; Chris@42: clock_gettime(CLOCK_SGI_CYCLE, &t); Chris@42: return t; Chris@42: } Chris@42: Chris@42: static inline double elapsed(ticks t1, ticks t0) Chris@42: { Chris@42: return ((double)t1.tv_sec - (double)t0.tv_sec) * 1.0E9 + Chris@42: ((double)t1.tv_nsec - (double)t0.tv_nsec); Chris@42: } Chris@42: #define HAVE_TICK_COUNTER Chris@42: #endif Chris@42: Chris@42: /*----------------------------------------------------------------*/ Chris@42: /* Cray UNICOS _rtc() intrinsic function */ Chris@42: #if defined(HAVE__RTC) && !defined(HAVE_TICK_COUNTER) Chris@42: #ifdef HAVE_INTRINSICS_H Chris@42: # include Chris@42: #endif Chris@42: Chris@42: typedef long long ticks; Chris@42: Chris@42: #define getticks _rtc Chris@42: Chris@42: INLINE_ELAPSED(inline) Chris@42: Chris@42: #define HAVE_TICK_COUNTER Chris@42: #endif Chris@42: Chris@42: /*----------------------------------------------------------------*/ Chris@42: /* MIPS ZBus */ Chris@42: #if HAVE_MIPS_ZBUS_TIMER Chris@42: #if defined(__mips__) && !defined(HAVE_TICK_COUNTER) Chris@42: #include Chris@42: #include Chris@42: #include Chris@42: Chris@42: typedef uint64_t ticks; Chris@42: Chris@42: static inline ticks getticks(void) Chris@42: { Chris@42: static uint64_t* addr = 0; Chris@42: Chris@42: if (addr == 0) Chris@42: { Chris@42: uint32_t rq_addr = 0x10030000; Chris@42: int fd; Chris@42: int pgsize; Chris@42: Chris@42: pgsize = getpagesize(); Chris@42: fd = open ("/dev/mem", O_RDONLY | O_SYNC, 0); Chris@42: if (fd < 0) { Chris@42: perror("open"); Chris@42: return NULL; Chris@42: } Chris@42: addr = mmap(0, pgsize, PROT_READ, MAP_SHARED, fd, rq_addr); Chris@42: close(fd); Chris@42: if (addr == (uint64_t *)-1) { Chris@42: perror("mmap"); Chris@42: return NULL; Chris@42: } Chris@42: } Chris@42: Chris@42: return *addr; Chris@42: } Chris@42: Chris@42: INLINE_ELAPSED(inline) Chris@42: Chris@42: #define HAVE_TICK_COUNTER Chris@42: #endif Chris@42: #endif /* HAVE_MIPS_ZBUS_TIMER */ Chris@42: Chris@42: #if defined(__ARM_ARCH_7A__) && defined(ARMV7A_HAS_CNTCVT) Chris@42: typedef uint64_t ticks; Chris@42: static inline ticks getticks(void) Chris@42: { Chris@42: uint32_t Rt, Rt2 = 0; Chris@42: asm volatile("mrrc p15, 1, %0, %1, c14" : "=r"(Rt), "=r"(Rt2)); Chris@42: return ((uint64_t)Rt) | (((uint64_t)Rt2) << 32); Chris@42: } Chris@42: INLINE_ELAPSED(inline) Chris@42: #define HAVE_TICK_COUNTER Chris@42: #endif Chris@42: Chris@42: #if defined(__aarch64__) && defined(ARMV8_HAS_CNTCVT_EL0) && !defined(HAVE_ARMV8CC) Chris@42: typedef uint64_t ticks; Chris@42: static inline ticks getticks(void) Chris@42: { Chris@42: uint64_t Rt; Chris@42: asm volatile("mrs %0, CNTVCT_EL0" : "=r" (Rt)); Chris@42: return Rt; Chris@42: } Chris@42: INLINE_ELAPSED(inline) Chris@42: #define HAVE_TICK_COUNTER Chris@42: #endif Chris@42: Chris@42: #if defined(__aarch64__) && defined(HAVE_ARMV8CC) Chris@42: typedef uint64_t ticks; Chris@42: static inline ticks getticks(void) Chris@42: { Chris@42: uint64_t cc = 0; Chris@42: asm volatile("mrs %0, PMCCNTR_EL0" : "=r"(cc)); Chris@42: return cc; Chris@42: } Chris@42: INLINE_ELAPSED(inline) Chris@42: #define HAVE_TICK_COUNTER Chris@42: #endif