Chris@102: /*
Chris@102:  * Distributed under the Boost Software License, Version 1.0.
Chris@102:  * (See accompanying file LICENSE_1_0.txt or copy at
Chris@102:  * http://www.boost.org/LICENSE_1_0.txt)
Chris@102:  *
Chris@102:  * Copyright (c) 2009 Helge Bahmann
Chris@102:  * Copyright (c) 2013 Tim Blechmann
Chris@102:  * Copyright (c) 2014 Andrey Semashev
Chris@102:  */
Chris@102: /*!
Chris@102:  * \file   atomic/detail/ops_gcc_ppc.hpp
Chris@102:  *
Chris@102:  * This header contains implementation of the \c operations template.
Chris@102:  */
Chris@102: 
Chris@102: #ifndef BOOST_ATOMIC_DETAIL_OPS_GCC_PPC_HPP_INCLUDED_
Chris@102: #define BOOST_ATOMIC_DETAIL_OPS_GCC_PPC_HPP_INCLUDED_
Chris@102: 
Chris@102: #include <boost/memory_order.hpp>
Chris@102: #include <boost/atomic/detail/config.hpp>
Chris@102: #include <boost/atomic/detail/storage_type.hpp>
Chris@102: #include <boost/atomic/detail/operations_fwd.hpp>
Chris@102: #include <boost/atomic/capabilities.hpp>
Chris@102: 
Chris@102: #ifdef BOOST_HAS_PRAGMA_ONCE
Chris@102: #pragma once
Chris@102: #endif
Chris@102: 
Chris@102: namespace boost {
Chris@102: namespace atomics {
Chris@102: namespace detail {
Chris@102: 
Chris@102: /*
Chris@102:     Refer to: Motorola: "Programming Environments Manual for 32-Bit
Chris@102:     Implementations of the PowerPC Architecture", Appendix E:
Chris@102:     "Synchronization Programming Examples" for an explanation of what is
Chris@102:     going on here (can be found on the web at various places by the
Chris@102:     name "MPCFPE32B.pdf", Google is your friend...)
Chris@102: 
Chris@102:     Most of the atomic operations map to instructions in a relatively
Chris@102:     straight-forward fashion, but "load"s may at first glance appear
Chris@102:     a bit strange as they map to:
Chris@102: 
Chris@102:             lwz %rX, addr
Chris@102:             cmpw %rX, %rX
Chris@102:             bne- 1f
Chris@102:         1:
Chris@102: 
Chris@102:     That is, the CPU is forced to perform a branch that "formally" depends
Chris@102:     on the value retrieved from memory. This scheme has an overhead of
Chris@102:     about 1-2 clock cycles per load, but it allows to map "acquire" to
Chris@102:     the "isync" instruction instead of "sync" uniformly and for all type
Chris@102:     of atomic operations. Since "isync" has a cost of about 15 clock
Chris@102:     cycles, while "sync" hast a cost of about 50 clock cycles, the small
Chris@102:     penalty to atomic loads more than compensates for this.
Chris@102: 
Chris@102:     Byte- and halfword-sized atomic values are realized by encoding the
Chris@102:     value to be represented into a word, performing sign/zero extension
Chris@102:     as appropriate. This means that after add/sub operations the value
Chris@102:     needs fixing up to accurately preserve the wrap-around semantic of
Chris@102:     the smaller type. (Nothing special needs to be done for the bit-wise
Chris@102:     and the "exchange type" operators as the compiler already sees to
Chris@102:     it that values carried in registers are extended appropriately and
Chris@102:     everything falls into place naturally).
Chris@102: 
Chris@102:     The register constraint "b"  instructs gcc to use any register
Chris@102:     except r0; this is sometimes required because the encoding for
Chris@102:     r0 is used to signify "constant zero" in a number of instructions,
Chris@102:     making r0 unusable in this place. For simplicity this constraint
Chris@102:     is used everywhere since I am to lazy to look this up on a
Chris@102:     per-instruction basis, and ppc has enough registers for this not
Chris@102:     to pose a problem.
Chris@102: */
Chris@102: 
Chris@102: // A note about memory_order_consume. Technically, this architecture allows to avoid
Chris@102: // unnecessary memory barrier after consume load since it supports data dependency ordering.
Chris@102: // However, some compiler optimizations may break a seemingly valid code relying on data
Chris@102: // dependency tracking by injecting bogus branches to aid out of order execution.
Chris@102: // This may happen not only in Boost.Atomic code but also in user's code, which we have no
Chris@102: // control of. See this thread: http://lists.boost.org/Archives/boost/2014/06/213890.php.
Chris@102: // For this reason we promote memory_order_consume to memory_order_acquire.
Chris@102: 
Chris@102: struct gcc_ppc_operations_base
Chris@102: {
Chris@102:     static BOOST_FORCEINLINE void fence_before(memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102: #if defined(__powerpc64__)
Chris@102:         if (order == memory_order_seq_cst)
Chris@102:             __asm__ __volatile__ ("sync" ::: "memory");
Chris@102:         else if ((order & memory_order_release) != 0)
Chris@102:             __asm__ __volatile__ ("lwsync" ::: "memory");
Chris@102: #else
Chris@102:         if ((order & memory_order_release) != 0)
Chris@102:             __asm__ __volatile__ ("sync" ::: "memory");
Chris@102: #endif
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE void fence_after(memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         if ((order & (memory_order_consume | memory_order_acquire)) != 0)
Chris@102:             __asm__ __volatile__ ("isync" ::: "memory");
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE void fence_after_store(memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         if (order == memory_order_seq_cst)
Chris@102:             __asm__ __volatile__ ("sync" ::: "memory");
Chris@102:     }
Chris@102: };
Chris@102: 
Chris@102: 
Chris@102: template< bool Signed >
Chris@102: struct operations< 4u, Signed > :
Chris@102:     public gcc_ppc_operations_base
Chris@102: {
Chris@102:     typedef typename make_storage_type< 4u, Signed >::type storage_type;
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         fence_before(order);
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "stw %1, %0\n"
Chris@102:             : "+m" (storage)
Chris@102:             : "r" (v)
Chris@102:         );
Chris@102:         fence_after_store(order);
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         storage_type v;
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "lwz %0, %1\n"
Chris@102:             "cmpw %0, %0\n"
Chris@102:             "bne- 1f\n"
Chris@102:             "1:\n"
Chris@102:             : "=&r" (v)
Chris@102:             : "m" (storage)
Chris@102:             : "cr0"
Chris@102:         );
Chris@102:         fence_after(order);
Chris@102:         return v;
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         storage_type original;
Chris@102:         fence_before(order);
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "1:\n"
Chris@102:             "lwarx %0,%y1\n"
Chris@102:             "stwcx. %2,%y1\n"
Chris@102:             "bne- 1b\n"
Chris@102:             : "=&b" (original), "+Z" (storage)
Chris@102:             : "b" (v)
Chris@102:             : "cr0"
Chris@102:         );
Chris@102:         fence_after(order);
Chris@102:         return original;
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE bool compare_exchange_weak(
Chris@102:         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         int success;
Chris@102:         fence_before(success_order);
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "li %1, 0\n"
Chris@102:             "lwarx %0,%y2\n"
Chris@102:             "cmpw %0, %3\n"
Chris@102:             "bne- 1f\n"
Chris@102:             "stwcx. %4,%y2\n"
Chris@102:             "bne- 1f\n"
Chris@102:             "li %1, 1\n"
Chris@102:             "1:"
Chris@102:             : "=&b" (expected), "=&b" (success), "+Z" (storage)
Chris@102:             : "b" (expected), "b" (desired)
Chris@102:             : "cr0"
Chris@102:         );
Chris@102:         if (success)
Chris@102:             fence_after(success_order);
Chris@102:         else
Chris@102:             fence_after(failure_order);
Chris@102:         return !!success;
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE bool compare_exchange_strong(
Chris@102:         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         int success;
Chris@102:         fence_before(success_order);
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "li %1, 0\n"
Chris@102:             "0: lwarx %0,%y2\n"
Chris@102:             "cmpw %0, %3\n"
Chris@102:             "bne- 1f\n"
Chris@102:             "stwcx. %4,%y2\n"
Chris@102:             "bne- 0b\n"
Chris@102:             "li %1, 1\n"
Chris@102:             "1:"
Chris@102:             : "=&b" (expected), "=&b" (success), "+Z" (storage)
Chris@102:             : "b" (expected), "b" (desired)
Chris@102:             : "cr0"
Chris@102:         );
Chris@102:         if (success)
Chris@102:             fence_after(success_order);
Chris@102:         else
Chris@102:             fence_after(failure_order);
Chris@102:         return !!success;
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         storage_type original, tmp;
Chris@102:         fence_before(order);
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "1:\n"
Chris@102:             "lwarx %0,%y2\n"
Chris@102:             "add %1,%0,%3\n"
Chris@102:             "stwcx. %1,%y2\n"
Chris@102:             "bne- 1b\n"
Chris@102:             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
Chris@102:             : "b" (v)
Chris@102:             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
Chris@102:         );
Chris@102:         fence_after(order);
Chris@102:         return original;
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         storage_type original, tmp;
Chris@102:         fence_before(order);
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "1:\n"
Chris@102:             "lwarx %0,%y2\n"
Chris@102:             "sub %1,%0,%3\n"
Chris@102:             "stwcx. %1,%y2\n"
Chris@102:             "bne- 1b\n"
Chris@102:             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
Chris@102:             : "b" (v)
Chris@102:             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
Chris@102:         );
Chris@102:         fence_after(order);
Chris@102:         return original;
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         storage_type original, tmp;
Chris@102:         fence_before(order);
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "1:\n"
Chris@102:             "lwarx %0,%y2\n"
Chris@102:             "and %1,%0,%3\n"
Chris@102:             "stwcx. %1,%y2\n"
Chris@102:             "bne- 1b\n"
Chris@102:             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
Chris@102:             : "b" (v)
Chris@102:             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
Chris@102:         );
Chris@102:         fence_after(order);
Chris@102:         return original;
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         storage_type original, tmp;
Chris@102:         fence_before(order);
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "1:\n"
Chris@102:             "lwarx %0,%y2\n"
Chris@102:             "or %1,%0,%3\n"
Chris@102:             "stwcx. %1,%y2\n"
Chris@102:             "bne- 1b\n"
Chris@102:             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
Chris@102:             : "b" (v)
Chris@102:             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
Chris@102:         );
Chris@102:         fence_after(order);
Chris@102:         return original;
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         storage_type original, tmp;
Chris@102:         fence_before(order);
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "1:\n"
Chris@102:             "lwarx %0,%y2\n"
Chris@102:             "xor %1,%0,%3\n"
Chris@102:             "stwcx. %1,%y2\n"
Chris@102:             "bne- 1b\n"
Chris@102:             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
Chris@102:             : "b" (v)
Chris@102:             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
Chris@102:         );
Chris@102:         fence_after(order);
Chris@102:         return original;
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         return !!exchange(storage, (storage_type)1, order);
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         store(storage, 0, order);
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE bool is_lock_free(storage_type const volatile&) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         return true;
Chris@102:     }
Chris@102: };
Chris@102: 
Chris@102: 
Chris@102: template< >
Chris@102: struct operations< 1u, false > :
Chris@102:     public operations< 4u, false >
Chris@102: {
Chris@102:     typedef operations< 4u, false > base_type;
Chris@102:     typedef base_type::storage_type storage_type;
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         storage_type original, tmp;
Chris@102:         fence_before(order);
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "1:\n"
Chris@102:             "lwarx %0,%y2\n"
Chris@102:             "add %1,%0,%3\n"
Chris@102:             "rlwinm %1, %1, 0, 0xff\n"
Chris@102:             "stwcx. %1,%y2\n"
Chris@102:             "bne- 1b\n"
Chris@102:             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
Chris@102:             : "b" (v)
Chris@102:             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
Chris@102:         );
Chris@102:         fence_after(order);
Chris@102:         return original;
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         storage_type original, tmp;
Chris@102:         fence_before(order);
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "1:\n"
Chris@102:             "lwarx %0,%y2\n"
Chris@102:             "sub %1,%0,%3\n"
Chris@102:             "rlwinm %1, %1, 0, 0xff\n"
Chris@102:             "stwcx. %1,%y2\n"
Chris@102:             "bne- 1b\n"
Chris@102:             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
Chris@102:             : "b" (v)
Chris@102:             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
Chris@102:         );
Chris@102:         fence_after(order);
Chris@102:         return original;
Chris@102:     }
Chris@102: };
Chris@102: 
Chris@102: template< >
Chris@102: struct operations< 1u, true > :
Chris@102:     public operations< 4u, true >
Chris@102: {
Chris@102:     typedef operations< 4u, true > base_type;
Chris@102:     typedef base_type::storage_type storage_type;
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         storage_type original, tmp;
Chris@102:         fence_before(order);
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "1:\n"
Chris@102:             "lwarx %0,%y2\n"
Chris@102:             "add %1,%0,%3\n"
Chris@102:             "extsb %1, %1\n"
Chris@102:             "stwcx. %1,%y2\n"
Chris@102:             "bne- 1b\n"
Chris@102:             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
Chris@102:             : "b" (v)
Chris@102:             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
Chris@102:         );
Chris@102:         fence_after(order);
Chris@102:         return original;
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         storage_type original, tmp;
Chris@102:         fence_before(order);
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "1:\n"
Chris@102:             "lwarx %0,%y2\n"
Chris@102:             "sub %1,%0,%3\n"
Chris@102:             "extsb %1, %1\n"
Chris@102:             "stwcx. %1,%y2\n"
Chris@102:             "bne- 1b\n"
Chris@102:             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
Chris@102:             : "b" (v)
Chris@102:             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
Chris@102:         );
Chris@102:         fence_after(order);
Chris@102:         return original;
Chris@102:     }
Chris@102: };
Chris@102: 
Chris@102: 
Chris@102: template< >
Chris@102: struct operations< 2u, false > :
Chris@102:     public operations< 4u, false >
Chris@102: {
Chris@102:     typedef operations< 4u, false > base_type;
Chris@102:     typedef base_type::storage_type storage_type;
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         storage_type original, tmp;
Chris@102:         fence_before(order);
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "1:\n"
Chris@102:             "lwarx %0,%y2\n"
Chris@102:             "add %1,%0,%3\n"
Chris@102:             "rlwinm %1, %1, 0, 0xffff\n"
Chris@102:             "stwcx. %1,%y2\n"
Chris@102:             "bne- 1b\n"
Chris@102:             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
Chris@102:             : "b" (v)
Chris@102:             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
Chris@102:         );
Chris@102:         fence_after(order);
Chris@102:         return original;
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         storage_type original, tmp;
Chris@102:         fence_before(order);
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "1:\n"
Chris@102:             "lwarx %0,%y2\n"
Chris@102:             "sub %1,%0,%3\n"
Chris@102:             "rlwinm %1, %1, 0, 0xffff\n"
Chris@102:             "stwcx. %1,%y2\n"
Chris@102:             "bne- 1b\n"
Chris@102:             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
Chris@102:             : "b" (v)
Chris@102:             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
Chris@102:         );
Chris@102:         fence_after(order);
Chris@102:         return original;
Chris@102:     }
Chris@102: };
Chris@102: 
Chris@102: template< >
Chris@102: struct operations< 2u, true > :
Chris@102:     public operations< 4u, true >
Chris@102: {
Chris@102:     typedef operations< 4u, true > base_type;
Chris@102:     typedef base_type::storage_type storage_type;
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         storage_type original, tmp;
Chris@102:         fence_before(order);
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "1:\n"
Chris@102:             "lwarx %0,%y2\n"
Chris@102:             "add %1,%0,%3\n"
Chris@102:             "extsh %1, %1\n"
Chris@102:             "stwcx. %1,%y2\n"
Chris@102:             "bne- 1b\n"
Chris@102:             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
Chris@102:             : "b" (v)
Chris@102:             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
Chris@102:         );
Chris@102:         fence_after(order);
Chris@102:         return original;
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         storage_type original, tmp;
Chris@102:         fence_before(order);
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "1:\n"
Chris@102:             "lwarx %0,%y2\n"
Chris@102:             "sub %1,%0,%3\n"
Chris@102:             "extsh %1, %1\n"
Chris@102:             "stwcx. %1,%y2\n"
Chris@102:             "bne- 1b\n"
Chris@102:             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
Chris@102:             : "b" (v)
Chris@102:             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
Chris@102:         );
Chris@102:         fence_after(order);
Chris@102:         return original;
Chris@102:     }
Chris@102: };
Chris@102: 
Chris@102: 
Chris@102: #if defined(__powerpc64__)
Chris@102: 
Chris@102: template< bool Signed >
Chris@102: struct operations< 8u, Signed > :
Chris@102:     public gcc_ppc_operations_base
Chris@102: {
Chris@102:     typedef typename make_storage_type< 8u, Signed >::type storage_type;
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         fence_before(order);
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "std %1, %0\n"
Chris@102:             : "+m" (storage)
Chris@102:             : "r" (v)
Chris@102:         );
Chris@102:         fence_after_store(order);
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         storage_type v;
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "ld %0, %1\n"
Chris@102:             "cmpd %0, %0\n"
Chris@102:             "bne- 1f\n"
Chris@102:             "1:\n"
Chris@102:             : "=&b" (v)
Chris@102:             : "m" (storage)
Chris@102:             : "cr0"
Chris@102:         );
Chris@102:         fence_after(order);
Chris@102:         return v;
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         storage_type original;
Chris@102:         fence_before(order);
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "1:\n"
Chris@102:             "ldarx %0,%y1\n"
Chris@102:             "stdcx. %2,%y1\n"
Chris@102:             "bne- 1b\n"
Chris@102:             : "=&b" (original), "+Z" (storage)
Chris@102:             : "b" (v)
Chris@102:             : "cr0"
Chris@102:         );
Chris@102:         fence_after(order);
Chris@102:         return original;
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE bool compare_exchange_weak(
Chris@102:         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         int success;
Chris@102:         fence_before(success_order);
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "li %1, 0\n"
Chris@102:             "ldarx %0,%y2\n"
Chris@102:             "cmpd %0, %3\n"
Chris@102:             "bne- 1f\n"
Chris@102:             "stdcx. %4,%y2\n"
Chris@102:             "bne- 1f\n"
Chris@102:             "li %1, 1\n"
Chris@102:             "1:"
Chris@102:             : "=&b" (expected), "=&b" (success), "+Z" (storage)
Chris@102:             : "b" (expected), "b" (desired)
Chris@102:             : "cr0"
Chris@102:         );
Chris@102:         if (success)
Chris@102:             fence_after(success_order);
Chris@102:         else
Chris@102:             fence_after(failure_order);
Chris@102:         return !!success;
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE bool compare_exchange_strong(
Chris@102:         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         int success;
Chris@102:         fence_before(success_order);
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "li %1, 0\n"
Chris@102:             "0: ldarx %0,%y2\n"
Chris@102:             "cmpd %0, %3\n"
Chris@102:             "bne- 1f\n"
Chris@102:             "stdcx. %4,%y2\n"
Chris@102:             "bne- 0b\n"
Chris@102:             "li %1, 1\n"
Chris@102:             "1:"
Chris@102:             : "=&b" (expected), "=&b" (success), "+Z" (storage)
Chris@102:             : "b" (expected), "b" (desired)
Chris@102:             : "cr0"
Chris@102:         );
Chris@102:         if (success)
Chris@102:             fence_after(success_order);
Chris@102:         else
Chris@102:             fence_after(failure_order);
Chris@102:         return !!success;
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         storage_type original, tmp;
Chris@102:         fence_before(order);
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "1:\n"
Chris@102:             "ldarx %0,%y2\n"
Chris@102:             "add %1,%0,%3\n"
Chris@102:             "stdcx. %1,%y2\n"
Chris@102:             "bne- 1b\n"
Chris@102:             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
Chris@102:             : "b" (v)
Chris@102:             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
Chris@102:         );
Chris@102:         fence_after(order);
Chris@102:         return original;
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         storage_type original, tmp;
Chris@102:         fence_before(order);
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "1:\n"
Chris@102:             "ldarx %0,%y2\n"
Chris@102:             "sub %1,%0,%3\n"
Chris@102:             "stdcx. %1,%y2\n"
Chris@102:             "bne- 1b\n"
Chris@102:             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
Chris@102:             : "b" (v)
Chris@102:             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
Chris@102:         );
Chris@102:         fence_after(order);
Chris@102:         return original;
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         storage_type original, tmp;
Chris@102:         fence_before(order);
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "1:\n"
Chris@102:             "ldarx %0,%y2\n"
Chris@102:             "and %1,%0,%3\n"
Chris@102:             "stdcx. %1,%y2\n"
Chris@102:             "bne- 1b\n"
Chris@102:             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
Chris@102:             : "b" (v)
Chris@102:             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
Chris@102:         );
Chris@102:         fence_after(order);
Chris@102:         return original;
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         storage_type original, tmp;
Chris@102:         fence_before(order);
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "1:\n"
Chris@102:             "ldarx %0,%y2\n"
Chris@102:             "or %1,%0,%3\n"
Chris@102:             "stdcx. %1,%y2\n"
Chris@102:             "bne- 1b\n"
Chris@102:             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
Chris@102:             : "b" (v)
Chris@102:             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
Chris@102:         );
Chris@102:         fence_after(order);
Chris@102:         return original;
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         storage_type original, tmp;
Chris@102:         fence_before(order);
Chris@102:         __asm__ __volatile__
Chris@102:         (
Chris@102:             "1:\n"
Chris@102:             "ldarx %0,%y2\n"
Chris@102:             "xor %1,%0,%3\n"
Chris@102:             "stdcx. %1,%y2\n"
Chris@102:             "bne- 1b\n"
Chris@102:             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
Chris@102:             : "b" (v)
Chris@102:             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
Chris@102:         );
Chris@102:         fence_after(order);
Chris@102:         return original;
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         return !!exchange(storage, (storage_type)1, order);
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         store(storage, 0, order);
Chris@102:     }
Chris@102: 
Chris@102:     static BOOST_FORCEINLINE bool is_lock_free(storage_type const volatile&) BOOST_NOEXCEPT
Chris@102:     {
Chris@102:         return true;
Chris@102:     }
Chris@102: };
Chris@102: 
Chris@102: #endif // defined(__powerpc64__)
Chris@102: 
Chris@102: 
Chris@102: BOOST_FORCEINLINE void thread_fence(memory_order order) BOOST_NOEXCEPT
Chris@102: {
Chris@102:     switch (order)
Chris@102:     {
Chris@102:     case memory_order_consume:
Chris@102:     case memory_order_acquire:
Chris@102:         __asm__ __volatile__ ("isync" ::: "memory");
Chris@102:         break;
Chris@102:     case memory_order_release:
Chris@102: #if defined(__powerpc64__)
Chris@102:         __asm__ __volatile__ ("lwsync" ::: "memory");
Chris@102:         break;
Chris@102: #endif
Chris@102:     case memory_order_acq_rel:
Chris@102:     case memory_order_seq_cst:
Chris@102:         __asm__ __volatile__ ("sync" ::: "memory");
Chris@102:         break;
Chris@102:     default:;
Chris@102:     }
Chris@102: }
Chris@102: 
Chris@102: BOOST_FORCEINLINE void signal_fence(memory_order order) BOOST_NOEXCEPT
Chris@102: {
Chris@102:     if (order != memory_order_relaxed)
Chris@102:         __asm__ __volatile__ ("" ::: "memory");
Chris@102: }
Chris@102: 
Chris@102: } // namespace detail
Chris@102: } // namespace atomics
Chris@102: } // namespace boost
Chris@102: 
Chris@102: #endif // BOOST_ATOMIC_DETAIL_OPS_GCC_PPC_HPP_INCLUDED_