Chris@16: /*
Chris@16:  *
Chris@16:  * Copyright (c) 2004
Chris@16:  * John Maddock
Chris@16:  *
Chris@16:  * Use, modification and distribution are subject to the 
Chris@16:  * Boost Software License, Version 1.0. (See accompanying file 
Chris@16:  * LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
Chris@16:  *
Chris@16:  */
Chris@16:  
Chris@16:  /*
Chris@16:   *   LOCATION:    see http://www.boost.org for most recent version.
Chris@16:   *   FILE         unicode_iterator.hpp
Chris@16:   *   VERSION      see <boost/version.hpp>
Chris@16:   *   DESCRIPTION: Iterator adapters for converting between different Unicode encodings.
Chris@16:   */
Chris@16: 
Chris@16: /****************************************************************************
Chris@16: 
Chris@16: Contents:
Chris@16: ~~~~~~~~~
Chris@16: 
Chris@16: 1) Read Only, Input Adapters:
Chris@16: ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Chris@16: 
Chris@16: template <class BaseIterator, class U8Type = ::boost::uint8_t>
Chris@16: class u32_to_u8_iterator;
Chris@16: 
Chris@16: Adapts sequence of UTF-32 code points to "look like" a sequence of UTF-8.
Chris@16: 
Chris@16: template <class BaseIterator, class U32Type = ::boost::uint32_t>
Chris@16: class u8_to_u32_iterator;
Chris@16: 
Chris@16: Adapts sequence of UTF-8 code points to "look like" a sequence of UTF-32.
Chris@16: 
Chris@16: template <class BaseIterator, class U16Type = ::boost::uint16_t>
Chris@16: class u32_to_u16_iterator;
Chris@16: 
Chris@16: Adapts sequence of UTF-32 code points to "look like" a sequence of UTF-16.
Chris@16: 
Chris@16: template <class BaseIterator, class U32Type = ::boost::uint32_t>
Chris@16: class u16_to_u32_iterator;
Chris@16: 
Chris@16: Adapts sequence of UTF-16 code points to "look like" a sequence of UTF-32.
Chris@16: 
Chris@16: 2) Single pass output iterator adapters:
Chris@16: 
Chris@16: template <class BaseIterator>
Chris@16: class utf8_output_iterator;
Chris@16: 
Chris@16: Accepts UTF-32 code points and forwards them on as UTF-8 code points.
Chris@16: 
Chris@16: template <class BaseIterator>
Chris@16: class utf16_output_iterator;
Chris@16: 
Chris@16: Accepts UTF-32 code points and forwards them on as UTF-16 code points.
Chris@16: 
Chris@16: ****************************************************************************/
Chris@16: 
Chris@16: #ifndef BOOST_REGEX_UNICODE_ITERATOR_HPP
Chris@16: #define BOOST_REGEX_UNICODE_ITERATOR_HPP
Chris@16: #include <boost/cstdint.hpp>
Chris@16: #include <boost/assert.hpp>
Chris@16: #include <boost/iterator/iterator_facade.hpp>
Chris@16: #include <boost/static_assert.hpp>
Chris@16: #include <boost/throw_exception.hpp>
Chris@16: #include <stdexcept>
Chris@16: #ifndef BOOST_NO_STD_LOCALE
Chris@16: #include <sstream>
Chris@16: #include <ios>
Chris@16: #endif
Chris@16: #include <limits.h> // CHAR_BIT
Chris@16: 
Chris@16: namespace boost{
Chris@16: 
Chris@16: namespace detail{
Chris@16: 
Chris@16: static const ::boost::uint16_t high_surrogate_base = 0xD7C0u;
Chris@16: static const ::boost::uint16_t low_surrogate_base = 0xDC00u;
Chris@16: static const ::boost::uint32_t ten_bit_mask = 0x3FFu;
Chris@16: 
Chris@16: inline bool is_high_surrogate(::boost::uint16_t v)
Chris@16: {
Chris@16:    return (v & 0xFFFFFC00u) == 0xd800u;
Chris@16: }
Chris@16: inline bool is_low_surrogate(::boost::uint16_t v)
Chris@16: {
Chris@16:    return (v & 0xFFFFFC00u) == 0xdc00u;
Chris@16: }
Chris@16: template <class T>
Chris@16: inline bool is_surrogate(T v)
Chris@16: {
Chris@16:    return (v & 0xFFFFF800u) == 0xd800;
Chris@16: }
Chris@16: 
Chris@16: inline unsigned utf8_byte_count(boost::uint8_t c)
Chris@16: {
Chris@16:    // if the most significant bit with a zero in it is in position
Chris@16:    // 8-N then there are N bytes in this UTF-8 sequence:
Chris@16:    boost::uint8_t mask = 0x80u;
Chris@16:    unsigned result = 0;
Chris@16:    while(c & mask)
Chris@16:    {
Chris@16:       ++result;
Chris@16:       mask >>= 1;
Chris@16:    }
Chris@16:    return (result == 0) ? 1 : ((result > 4) ? 4 : result);
Chris@16: }
Chris@16: 
Chris@16: inline unsigned utf8_trailing_byte_count(boost::uint8_t c)
Chris@16: {
Chris@16:    return utf8_byte_count(c) - 1;
Chris@16: }
Chris@16: 
Chris@16: #ifdef BOOST_MSVC
Chris@16: #pragma warning(push)
Chris@16: #pragma warning(disable:4100)
Chris@16: #endif
Chris@16: inline void invalid_utf32_code_point(::boost::uint32_t val)
Chris@16: {
Chris@16: #ifndef BOOST_NO_STD_LOCALE
Chris@16:    std::stringstream ss;
Chris@16:    ss << "Invalid UTF-32 code point U+" << std::showbase << std::hex << val << " encountered while trying to encode UTF-16 sequence";
Chris@16:    std::out_of_range e(ss.str());
Chris@16: #else
Chris@16:    std::out_of_range e("Invalid UTF-32 code point encountered while trying to encode UTF-16 sequence");
Chris@16: #endif
Chris@16:    boost::throw_exception(e);
Chris@16: }
Chris@16: #ifdef BOOST_MSVC
Chris@16: #pragma warning(pop)
Chris@16: #endif
Chris@16: 
Chris@16: 
Chris@16: } // namespace detail
Chris@16: 
Chris@16: template <class BaseIterator, class U16Type = ::boost::uint16_t>
Chris@16: class u32_to_u16_iterator
Chris@16:    : public boost::iterator_facade<u32_to_u16_iterator<BaseIterator, U16Type>, U16Type, std::bidirectional_iterator_tag, const U16Type>
Chris@16: {
Chris@16:    typedef boost::iterator_facade<u32_to_u16_iterator<BaseIterator, U16Type>, U16Type, std::bidirectional_iterator_tag, const U16Type> base_type;
Chris@16: 
Chris@101: #if !defined(BOOST_NO_STD_ITERATOR_TRAITS)
Chris@16:    typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
Chris@16: 
Chris@16:    BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 32);
Chris@16:    BOOST_STATIC_ASSERT(sizeof(U16Type)*CHAR_BIT == 16);
Chris@16: #endif
Chris@16: 
Chris@16: public:
Chris@16:    typename base_type::reference
Chris@16:       dereference()const
Chris@16:    {
Chris@16:       if(m_current == 2)
Chris@16:          extract_current();
Chris@16:       return m_values[m_current];
Chris@16:    }
Chris@16:    bool equal(const u32_to_u16_iterator& that)const
Chris@16:    {
Chris@16:       if(m_position == that.m_position)
Chris@16:       {
Chris@16:          // Both m_currents must be equal, or both even
Chris@16:          // this is the same as saying their sum must be even:
Chris@16:          return (m_current + that.m_current) & 1u ? false : true;
Chris@16:       }
Chris@16:       return false;
Chris@16:    }
Chris@16:    void increment()
Chris@16:    {
Chris@16:       // if we have a pending read then read now, so that we know whether
Chris@16:       // to skip a position, or move to a low-surrogate:
Chris@16:       if(m_current == 2)
Chris@16:       {
Chris@16:          // pending read:
Chris@16:          extract_current();
Chris@16:       }
Chris@16:       // move to the next surrogate position:
Chris@16:       ++m_current;
Chris@16:       // if we've reached the end skip a position:
Chris@16:       if(m_values[m_current] == 0)
Chris@16:       {
Chris@16:          m_current = 2;
Chris@16:          ++m_position;
Chris@16:       }
Chris@16:    }
Chris@16:    void decrement()
Chris@16:    {
Chris@16:       if(m_current != 1)
Chris@16:       {
Chris@16:          // decrementing an iterator always leads to a valid position:
Chris@16:          --m_position;
Chris@16:          extract_current();
Chris@16:          m_current = m_values[1] ? 1 : 0;
Chris@16:       }
Chris@16:       else
Chris@16:       {
Chris@16:          m_current = 0;
Chris@16:       }
Chris@16:    }
Chris@16:    BaseIterator base()const
Chris@16:    {
Chris@16:       return m_position;
Chris@16:    }
Chris@16:    // construct:
Chris@16:    u32_to_u16_iterator() : m_position(), m_current(0)
Chris@16:    {
Chris@16:       m_values[0] = 0;
Chris@16:       m_values[1] = 0;
Chris@16:       m_values[2] = 0;
Chris@16:    }
Chris@16:    u32_to_u16_iterator(BaseIterator b) : m_position(b), m_current(2)
Chris@16:    {
Chris@16:       m_values[0] = 0;
Chris@16:       m_values[1] = 0;
Chris@16:       m_values[2] = 0;
Chris@16:    }
Chris@16: private:
Chris@16: 
Chris@16:    void extract_current()const
Chris@16:    {
Chris@16:       // begin by checking for a code point out of range:
Chris@16:       ::boost::uint32_t v = *m_position;
Chris@16:       if(v >= 0x10000u)
Chris@16:       {
Chris@16:          if(v > 0x10FFFFu)
Chris@16:             detail::invalid_utf32_code_point(*m_position);
Chris@16:          // split into two surrogates:
Chris@16:          m_values[0] = static_cast<U16Type>(v >> 10) + detail::high_surrogate_base;
Chris@16:          m_values[1] = static_cast<U16Type>(v & detail::ten_bit_mask) + detail::low_surrogate_base;
Chris@16:          m_current = 0;
Chris@16:          BOOST_ASSERT(detail::is_high_surrogate(m_values[0]));
Chris@16:          BOOST_ASSERT(detail::is_low_surrogate(m_values[1]));
Chris@16:       }
Chris@16:       else
Chris@16:       {
Chris@16:          // 16-bit code point:
Chris@16:          m_values[0] = static_cast<U16Type>(*m_position);
Chris@16:          m_values[1] = 0;
Chris@16:          m_current = 0;
Chris@16:          // value must not be a surrogate:
Chris@16:          if(detail::is_surrogate(m_values[0]))
Chris@16:             detail::invalid_utf32_code_point(*m_position);
Chris@16:       }
Chris@16:    }
Chris@16:    BaseIterator m_position;
Chris@16:    mutable U16Type m_values[3];
Chris@16:    mutable unsigned m_current;
Chris@16: };
Chris@16: 
Chris@16: template <class BaseIterator, class U32Type = ::boost::uint32_t>
Chris@16: class u16_to_u32_iterator
Chris@16:    : public boost::iterator_facade<u16_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type>
Chris@16: {
Chris@16:    typedef boost::iterator_facade<u16_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type> base_type;
Chris@16:    // special values for pending iterator reads:
Chris@16:    BOOST_STATIC_CONSTANT(U32Type, pending_read = 0xffffffffu);
Chris@16: 
Chris@101: #if !defined(BOOST_NO_STD_ITERATOR_TRAITS)
Chris@16:    typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
Chris@16: 
Chris@16:    BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 16);
Chris@16:    BOOST_STATIC_ASSERT(sizeof(U32Type)*CHAR_BIT == 32);
Chris@16: #endif
Chris@16: 
Chris@16: public:
Chris@16:    typename base_type::reference
Chris@16:       dereference()const
Chris@16:    {
Chris@16:       if(m_value == pending_read)
Chris@16:          extract_current();
Chris@16:       return m_value;
Chris@16:    }
Chris@16:    bool equal(const u16_to_u32_iterator& that)const
Chris@16:    {
Chris@16:       return m_position == that.m_position;
Chris@16:    }
Chris@16:    void increment()
Chris@16:    {
Chris@16:       // skip high surrogate first if there is one:
Chris@16:       if(detail::is_high_surrogate(*m_position)) ++m_position;
Chris@16:       ++m_position;
Chris@16:       m_value = pending_read;
Chris@16:    }
Chris@16:    void decrement()
Chris@16:    {
Chris@16:       --m_position;
Chris@16:       // if we have a low surrogate then go back one more:
Chris@16:       if(detail::is_low_surrogate(*m_position)) 
Chris@16:          --m_position;
Chris@16:       m_value = pending_read;
Chris@16:    }
Chris@16:    BaseIterator base()const
Chris@16:    {
Chris@16:       return m_position;
Chris@16:    }
Chris@16:    // construct:
Chris@16:    u16_to_u32_iterator() : m_position()
Chris@16:    {
Chris@16:       m_value = pending_read;
Chris@16:    }
Chris@16:    u16_to_u32_iterator(BaseIterator b) : m_position(b)
Chris@16:    {
Chris@16:       m_value = pending_read;
Chris@16:    }
Chris@16:    //
Chris@16:    // Range checked version:
Chris@16:    //
Chris@16:    u16_to_u32_iterator(BaseIterator b, BaseIterator start, BaseIterator end) : m_position(b)
Chris@16:    {
Chris@16:       m_value = pending_read;
Chris@16:       //
Chris@16:       // The range must not start with a low surrogate, or end in a high surrogate,
Chris@16:       // otherwise we run the risk of running outside the underlying input range.
Chris@16:       // Likewise b must not be located at a low surrogate.
Chris@16:       //
Chris@16:       boost::uint16_t val;
Chris@16:       if(start != end)
Chris@16:       {
Chris@16:          if((b != start) && (b != end))
Chris@16:          {
Chris@16:             val = *b;
Chris@16:             if(detail::is_surrogate(val) && ((val & 0xFC00u) == 0xDC00u))
Chris@16:                invalid_code_point(val);
Chris@16:          }
Chris@16:          val = *start;
Chris@16:          if(detail::is_surrogate(val) && ((val & 0xFC00u) == 0xDC00u))
Chris@16:             invalid_code_point(val);
Chris@16:          val = *--end;
Chris@16:          if(detail::is_high_surrogate(val))
Chris@16:             invalid_code_point(val);
Chris@16:       }
Chris@16:    }
Chris@16: private:
Chris@16:    static void invalid_code_point(::boost::uint16_t val)
Chris@16:    {
Chris@16: #ifndef BOOST_NO_STD_LOCALE
Chris@16:       std::stringstream ss;
Chris@16:       ss << "Misplaced UTF-16 surrogate U+" << std::showbase << std::hex << val << " encountered while trying to encode UTF-32 sequence";
Chris@16:       std::out_of_range e(ss.str());
Chris@16: #else
Chris@16:       std::out_of_range e("Misplaced UTF-16 surrogate encountered while trying to encode UTF-32 sequence");
Chris@16: #endif
Chris@16:       boost::throw_exception(e);
Chris@16:    }
Chris@16:    void extract_current()const
Chris@16:    {
Chris@16:       m_value = static_cast<U32Type>(static_cast< ::boost::uint16_t>(*m_position));
Chris@16:       // if the last value is a high surrogate then adjust m_position and m_value as needed:
Chris@16:       if(detail::is_high_surrogate(*m_position))
Chris@16:       {
Chris@16:          // precondition; next value must have be a low-surrogate:
Chris@16:          BaseIterator next(m_position);
Chris@16:          ::boost::uint16_t t = *++next;
Chris@16:          if((t & 0xFC00u) != 0xDC00u)
Chris@16:             invalid_code_point(t);
Chris@16:          m_value = (m_value - detail::high_surrogate_base) << 10;
Chris@16:          m_value |= (static_cast<U32Type>(static_cast< ::boost::uint16_t>(t)) & detail::ten_bit_mask);
Chris@16:       }
Chris@16:       // postcondition; result must not be a surrogate:
Chris@16:       if(detail::is_surrogate(m_value))
Chris@16:          invalid_code_point(static_cast< ::boost::uint16_t>(m_value));
Chris@16:    }
Chris@16:    BaseIterator m_position;
Chris@16:    mutable U32Type m_value;
Chris@16: };
Chris@16: 
Chris@16: template <class BaseIterator, class U8Type = ::boost::uint8_t>
Chris@16: class u32_to_u8_iterator
Chris@16:    : public boost::iterator_facade<u32_to_u8_iterator<BaseIterator, U8Type>, U8Type, std::bidirectional_iterator_tag, const U8Type>
Chris@16: {
Chris@16:    typedef boost::iterator_facade<u32_to_u8_iterator<BaseIterator, U8Type>, U8Type, std::bidirectional_iterator_tag, const U8Type> base_type;
Chris@16:    
Chris@101: #if !defined(BOOST_NO_STD_ITERATOR_TRAITS)
Chris@16:    typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
Chris@16: 
Chris@16:    BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 32);
Chris@16:    BOOST_STATIC_ASSERT(sizeof(U8Type)*CHAR_BIT == 8);
Chris@16: #endif
Chris@16: 
Chris@16: public:
Chris@16:    typename base_type::reference
Chris@16:       dereference()const
Chris@16:    {
Chris@16:       if(m_current == 4)
Chris@16:          extract_current();
Chris@16:       return m_values[m_current];
Chris@16:    }
Chris@16:    bool equal(const u32_to_u8_iterator& that)const
Chris@16:    {
Chris@16:       if(m_position == that.m_position)
Chris@16:       {
Chris@16:          // either the m_current's must be equal, or one must be 0 and 
Chris@16:          // the other 4: which means neither must have bits 1 or 2 set:
Chris@16:          return (m_current == that.m_current)
Chris@16:             || (((m_current | that.m_current) & 3) == 0);
Chris@16:       }
Chris@16:       return false;
Chris@16:    }
Chris@16:    void increment()
Chris@16:    {
Chris@16:       // if we have a pending read then read now, so that we know whether
Chris@16:       // to skip a position, or move to a low-surrogate:
Chris@16:       if(m_current == 4)
Chris@16:       {
Chris@16:          // pending read:
Chris@16:          extract_current();
Chris@16:       }
Chris@16:       // move to the next surrogate position:
Chris@16:       ++m_current;
Chris@16:       // if we've reached the end skip a position:
Chris@16:       if(m_values[m_current] == 0)
Chris@16:       {
Chris@16:          m_current = 4;
Chris@16:          ++m_position;
Chris@16:       }
Chris@16:    }
Chris@16:    void decrement()
Chris@16:    {
Chris@16:       if((m_current & 3) == 0)
Chris@16:       {
Chris@16:          --m_position;
Chris@16:          extract_current();
Chris@16:          m_current = 3;
Chris@16:          while(m_current && (m_values[m_current] == 0))
Chris@16:             --m_current;
Chris@16:       }
Chris@16:       else
Chris@16:          --m_current;
Chris@16:    }
Chris@16:    BaseIterator base()const
Chris@16:    {
Chris@16:       return m_position;
Chris@16:    }
Chris@16:    // construct:
Chris@16:    u32_to_u8_iterator() : m_position(), m_current(0)
Chris@16:    {
Chris@16:       m_values[0] = 0;
Chris@16:       m_values[1] = 0;
Chris@16:       m_values[2] = 0;
Chris@16:       m_values[3] = 0;
Chris@16:       m_values[4] = 0;
Chris@16:    }
Chris@16:    u32_to_u8_iterator(BaseIterator b) : m_position(b), m_current(4)
Chris@16:    {
Chris@16:       m_values[0] = 0;
Chris@16:       m_values[1] = 0;
Chris@16:       m_values[2] = 0;
Chris@16:       m_values[3] = 0;
Chris@16:       m_values[4] = 0;
Chris@16:    }
Chris@16: private:
Chris@16: 
Chris@16:    void extract_current()const
Chris@16:    {
Chris@16:       boost::uint32_t c = *m_position;
Chris@16:       if(c > 0x10FFFFu)
Chris@16:          detail::invalid_utf32_code_point(c);
Chris@16:       if(c < 0x80u)
Chris@16:       {
Chris@16:          m_values[0] = static_cast<unsigned char>(c);
Chris@16:          m_values[1] = static_cast<unsigned char>(0u);
Chris@16:          m_values[2] = static_cast<unsigned char>(0u);
Chris@16:          m_values[3] = static_cast<unsigned char>(0u);
Chris@16:       }
Chris@16:       else if(c < 0x800u)
Chris@16:       {
Chris@16:          m_values[0] = static_cast<unsigned char>(0xC0u + (c >> 6));
Chris@16:          m_values[1] = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
Chris@16:          m_values[2] = static_cast<unsigned char>(0u);
Chris@16:          m_values[3] = static_cast<unsigned char>(0u);
Chris@16:       }
Chris@16:       else if(c < 0x10000u)
Chris@16:       {
Chris@16:          m_values[0] = static_cast<unsigned char>(0xE0u + (c >> 12));
Chris@16:          m_values[1] = static_cast<unsigned char>(0x80u + ((c >> 6) & 0x3Fu));
Chris@16:          m_values[2] = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
Chris@16:          m_values[3] = static_cast<unsigned char>(0u);
Chris@16:       }
Chris@16:       else
Chris@16:       {
Chris@16:          m_values[0] = static_cast<unsigned char>(0xF0u + (c >> 18));
Chris@16:          m_values[1] = static_cast<unsigned char>(0x80u + ((c >> 12) & 0x3Fu));
Chris@16:          m_values[2] = static_cast<unsigned char>(0x80u + ((c >> 6) & 0x3Fu));
Chris@16:          m_values[3] = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
Chris@16:       }
Chris@16:       m_current= 0;
Chris@16:    }
Chris@16:    BaseIterator m_position;
Chris@16:    mutable U8Type m_values[5];
Chris@16:    mutable unsigned m_current;
Chris@16: };
Chris@16: 
Chris@16: template <class BaseIterator, class U32Type = ::boost::uint32_t>
Chris@16: class u8_to_u32_iterator
Chris@16:    : public boost::iterator_facade<u8_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type>
Chris@16: {
Chris@16:    typedef boost::iterator_facade<u8_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type> base_type;
Chris@16:    // special values for pending iterator reads:
Chris@16:    BOOST_STATIC_CONSTANT(U32Type, pending_read = 0xffffffffu);
Chris@16: 
Chris@101: #if !defined(BOOST_NO_STD_ITERATOR_TRAITS)
Chris@16:    typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
Chris@16: 
Chris@16:    BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 8);
Chris@16:    BOOST_STATIC_ASSERT(sizeof(U32Type)*CHAR_BIT == 32);
Chris@16: #endif
Chris@16: 
Chris@16: public:
Chris@16:    typename base_type::reference
Chris@16:       dereference()const
Chris@16:    {
Chris@16:       if(m_value == pending_read)
Chris@16:          extract_current();
Chris@16:       return m_value;
Chris@16:    }
Chris@16:    bool equal(const u8_to_u32_iterator& that)const
Chris@16:    {
Chris@16:       return m_position == that.m_position;
Chris@16:    }
Chris@16:    void increment()
Chris@16:    {
Chris@16:       // We must not start with a continuation character:
Chris@16:       if((static_cast<boost::uint8_t>(*m_position) & 0xC0) == 0x80)
Chris@16:          invalid_sequence();
Chris@16:       // skip high surrogate first if there is one:
Chris@16:       unsigned c = detail::utf8_byte_count(*m_position);
Chris@16:       if(m_value == pending_read)
Chris@16:       {
Chris@16:          // Since we haven't read in a value, we need to validate the code points:
Chris@16:          for(unsigned i = 0; i < c; ++i)
Chris@16:          {
Chris@16:             ++m_position;
Chris@16:             // We must have a continuation byte:
Chris@16:             if((i != c - 1) && ((static_cast<boost::uint8_t>(*m_position) & 0xC0) != 0x80))
Chris@16:                invalid_sequence();
Chris@16:          }
Chris@16:       }
Chris@16:       else
Chris@16:       {
Chris@16:          std::advance(m_position, c);
Chris@16:       }
Chris@16:       m_value = pending_read;
Chris@16:    }
Chris@16:    void decrement()
Chris@16:    {
Chris@16:       // Keep backtracking until we don't have a trailing character:
Chris@16:       unsigned count = 0;
Chris@16:       while((*--m_position & 0xC0u) == 0x80u) ++count;
Chris@16:       // now check that the sequence was valid:
Chris@16:       if(count != detail::utf8_trailing_byte_count(*m_position))
Chris@16:          invalid_sequence();
Chris@16:       m_value = pending_read;
Chris@16:    }
Chris@16:    BaseIterator base()const
Chris@16:    {
Chris@16:       return m_position;
Chris@16:    }
Chris@16:    // construct:
Chris@16:    u8_to_u32_iterator() : m_position()
Chris@16:    {
Chris@16:       m_value = pending_read;
Chris@16:    }
Chris@16:    u8_to_u32_iterator(BaseIterator b) : m_position(b)
Chris@16:    {
Chris@16:       m_value = pending_read;
Chris@16:    }
Chris@16:    //
Chris@16:    // Checked constructor:
Chris@16:    //
Chris@16:    u8_to_u32_iterator(BaseIterator b, BaseIterator start, BaseIterator end) : m_position(b)
Chris@16:    {
Chris@16:       m_value = pending_read;
Chris@16:       //
Chris@16:       // We must not start with a continuation character, or end with a 
Chris@16:       // truncated UTF-8 sequence otherwise we run the risk of going past
Chris@16:       // the start/end of the underlying sequence:
Chris@16:       //
Chris@16:       if(start != end)
Chris@16:       {
Chris@16:          unsigned char v = *start;
Chris@16:          if((v & 0xC0u) == 0x80u)
Chris@16:             invalid_sequence();
Chris@16:          if((b != start) && (b != end) && ((*b & 0xC0u) == 0x80u))
Chris@16:             invalid_sequence();
Chris@16:          BaseIterator pos = end;
Chris@16:          do
Chris@16:          {
Chris@16:             v = *--pos;
Chris@16:          }
Chris@16:          while((start != pos) && ((v & 0xC0u) == 0x80u));
Chris@16:          std::ptrdiff_t extra = detail::utf8_byte_count(v);
Chris@16:          if(std::distance(pos, end) < extra)
Chris@16:             invalid_sequence();
Chris@16:       }
Chris@16:    }
Chris@16: private:
Chris@16:    static void invalid_sequence()
Chris@16:    {
Chris@16:       std::out_of_range e("Invalid UTF-8 sequence encountered while trying to encode UTF-32 character");
Chris@16:       boost::throw_exception(e);
Chris@16:    }
Chris@16:    void extract_current()const
Chris@16:    {
Chris@16:       m_value = static_cast<U32Type>(static_cast< ::boost::uint8_t>(*m_position));
Chris@16:       // we must not have a continuation character:
Chris@16:       if((m_value & 0xC0u) == 0x80u)
Chris@16:          invalid_sequence();
Chris@16:       // see how many extra bytes we have:
Chris@16:       unsigned extra = detail::utf8_trailing_byte_count(*m_position);
Chris@16:       // extract the extra bits, 6 from each extra byte:
Chris@16:       BaseIterator next(m_position);
Chris@16:       for(unsigned c = 0; c < extra; ++c)
Chris@16:       {
Chris@16:          ++next;
Chris@16:          m_value <<= 6;
Chris@16:          // We must have a continuation byte:
Chris@16:          if((static_cast<boost::uint8_t>(*next) & 0xC0) != 0x80)
Chris@16:             invalid_sequence();
Chris@16:          m_value += static_cast<boost::uint8_t>(*next) & 0x3Fu;
Chris@16:       }
Chris@16:       // we now need to remove a few of the leftmost bits, but how many depends
Chris@16:       // upon how many extra bytes we've extracted:
Chris@16:       static const boost::uint32_t masks[4] = 
Chris@16:       {
Chris@16:          0x7Fu,
Chris@16:          0x7FFu,
Chris@16:          0xFFFFu,
Chris@16:          0x1FFFFFu,
Chris@16:       };
Chris@16:       m_value &= masks[extra];
Chris@101:       // check the result is in range:
Chris@16:       if(m_value > static_cast<U32Type>(0x10FFFFu))
Chris@16:          invalid_sequence();
Chris@101:       // The result must not be a surrogate:
Chris@101:       if((m_value >= static_cast<U32Type>(0xD800)) && (m_value <= static_cast<U32Type>(0xDFFF)))
Chris@101:          invalid_sequence();
Chris@101:       // We should not have had an invalidly encoded UTF8 sequence:
Chris@101:       if((extra > 0) && (m_value <= static_cast<U32Type>(masks[extra - 1])))
Chris@101:          invalid_sequence();
Chris@16:    }
Chris@16:    BaseIterator m_position;
Chris@16:    mutable U32Type m_value;
Chris@16: };
Chris@16: 
Chris@16: template <class BaseIterator>
Chris@16: class utf16_output_iterator
Chris@16: {
Chris@16: public:
Chris@16:    typedef void                                   difference_type;
Chris@16:    typedef void                                   value_type;
Chris@16:    typedef boost::uint32_t*                       pointer;
Chris@16:    typedef boost::uint32_t&                       reference;
Chris@16:    typedef std::output_iterator_tag               iterator_category;
Chris@16: 
Chris@16:    utf16_output_iterator(const BaseIterator& b)
Chris@16:       : m_position(b){}
Chris@16:    utf16_output_iterator(const utf16_output_iterator& that)
Chris@16:       : m_position(that.m_position){}
Chris@16:    utf16_output_iterator& operator=(const utf16_output_iterator& that)
Chris@16:    {
Chris@16:       m_position = that.m_position;
Chris@16:       return *this;
Chris@16:    }
Chris@16:    const utf16_output_iterator& operator*()const
Chris@16:    {
Chris@16:       return *this;
Chris@16:    }
Chris@16:    void operator=(boost::uint32_t val)const
Chris@16:    {
Chris@16:       push(val);
Chris@16:    }
Chris@16:    utf16_output_iterator& operator++()
Chris@16:    {
Chris@16:       return *this;
Chris@16:    }
Chris@16:    utf16_output_iterator& operator++(int)
Chris@16:    {
Chris@16:       return *this;
Chris@16:    }
Chris@16:    BaseIterator base()const
Chris@16:    {
Chris@16:       return m_position;
Chris@16:    }
Chris@16: private:
Chris@16:    void push(boost::uint32_t v)const
Chris@16:    {
Chris@16:       if(v >= 0x10000u)
Chris@16:       {
Chris@16:          // begin by checking for a code point out of range:
Chris@16:          if(v > 0x10FFFFu)
Chris@16:             detail::invalid_utf32_code_point(v);
Chris@16:          // split into two surrogates:
Chris@16:          *m_position++ = static_cast<boost::uint16_t>(v >> 10) + detail::high_surrogate_base;
Chris@16:          *m_position++ = static_cast<boost::uint16_t>(v & detail::ten_bit_mask) + detail::low_surrogate_base;
Chris@16:       }
Chris@16:       else
Chris@16:       {
Chris@16:          // 16-bit code point:
Chris@16:          // value must not be a surrogate:
Chris@16:          if(detail::is_surrogate(v))
Chris@16:             detail::invalid_utf32_code_point(v);
Chris@16:          *m_position++ = static_cast<boost::uint16_t>(v);
Chris@16:       }
Chris@16:    }
Chris@16:    mutable BaseIterator m_position;
Chris@16: };
Chris@16: 
Chris@16: template <class BaseIterator>
Chris@16: class utf8_output_iterator
Chris@16: {
Chris@16: public:
Chris@16:    typedef void                                   difference_type;
Chris@16:    typedef void                                   value_type;
Chris@16:    typedef boost::uint32_t*                       pointer;
Chris@16:    typedef boost::uint32_t&                       reference;
Chris@16:    typedef std::output_iterator_tag               iterator_category;
Chris@16: 
Chris@16:    utf8_output_iterator(const BaseIterator& b)
Chris@16:       : m_position(b){}
Chris@16:    utf8_output_iterator(const utf8_output_iterator& that)
Chris@16:       : m_position(that.m_position){}
Chris@16:    utf8_output_iterator& operator=(const utf8_output_iterator& that)
Chris@16:    {
Chris@16:       m_position = that.m_position;
Chris@16:       return *this;
Chris@16:    }
Chris@16:    const utf8_output_iterator& operator*()const
Chris@16:    {
Chris@16:       return *this;
Chris@16:    }
Chris@16:    void operator=(boost::uint32_t val)const
Chris@16:    {
Chris@16:       push(val);
Chris@16:    }
Chris@16:    utf8_output_iterator& operator++()
Chris@16:    {
Chris@16:       return *this;
Chris@16:    }
Chris@16:    utf8_output_iterator& operator++(int)
Chris@16:    {
Chris@16:       return *this;
Chris@16:    }
Chris@16:    BaseIterator base()const
Chris@16:    {
Chris@16:       return m_position;
Chris@16:    }
Chris@16: private:
Chris@16:    void push(boost::uint32_t c)const
Chris@16:    {
Chris@16:       if(c > 0x10FFFFu)
Chris@16:          detail::invalid_utf32_code_point(c);
Chris@16:       if(c < 0x80u)
Chris@16:       {
Chris@16:          *m_position++ = static_cast<unsigned char>(c);
Chris@16:       }
Chris@16:       else if(c < 0x800u)
Chris@16:       {
Chris@16:          *m_position++ = static_cast<unsigned char>(0xC0u + (c >> 6));
Chris@16:          *m_position++ = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
Chris@16:       }
Chris@16:       else if(c < 0x10000u)
Chris@16:       {
Chris@16:          *m_position++ = static_cast<unsigned char>(0xE0u + (c >> 12));
Chris@16:          *m_position++ = static_cast<unsigned char>(0x80u + ((c >> 6) & 0x3Fu));
Chris@16:          *m_position++ = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
Chris@16:       }
Chris@16:       else
Chris@16:       {
Chris@16:          *m_position++ = static_cast<unsigned char>(0xF0u + (c >> 18));
Chris@16:          *m_position++ = static_cast<unsigned char>(0x80u + ((c >> 12) & 0x3Fu));
Chris@16:          *m_position++ = static_cast<unsigned char>(0x80u + ((c >> 6) & 0x3Fu));
Chris@16:          *m_position++ = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
Chris@16:       }
Chris@16:    }
Chris@16:    mutable BaseIterator m_position;
Chris@16: };
Chris@16: 
Chris@16: } // namespace boost
Chris@16: 
Chris@16: #endif // BOOST_REGEX_UNICODE_ITERATOR_HPP
Chris@16: