Chris@16: /*=============================================================================
Chris@16:     Boost.Wave: A Standard compliant C++ preprocessor library
Chris@16: 
Chris@16:     A generic C++ lexer token definition
Chris@16:     
Chris@16:     http://www.boost.org/
Chris@16: 
Chris@16:     Copyright (c) 2001-2012 Hartmut Kaiser. Distributed under the Boost
Chris@16:     Software License, Version 1.0. (See accompanying file
Chris@16:     LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
Chris@16: =============================================================================*/
Chris@16: 
Chris@16: #if !defined(CPP_TOKEN_HPP_53A13BD2_FBAA_444B_9B8B_FCB225C2BBA8_INCLUDED)
Chris@16: #define CPP_TOKEN_HPP_53A13BD2_FBAA_444B_9B8B_FCB225C2BBA8_INCLUDED
Chris@16: 
Chris@16: #include <boost/wave/wave_config.hpp>
Chris@16: #if BOOST_WAVE_SERIALIZATION != 0
Chris@16: #include <boost/serialization/serialization.hpp>
Chris@16: #endif
Chris@16: #include <boost/wave/util/file_position.hpp>
Chris@16: #include <boost/wave/token_ids.hpp>  
Chris@16: #include <boost/wave/language_support.hpp>
Chris@16: 
Chris@16: #include <boost/throw_exception.hpp>
Chris@16: #include <boost/pool/singleton_pool.hpp>
Chris@16: #include <boost/detail/atomic_count.hpp>
Chris@16: 
Chris@16: // this must occur after all of the includes and before any code appears
Chris@16: #ifdef BOOST_HAS_ABI_HEADERS
Chris@16: #include BOOST_ABI_PREFIX
Chris@16: #endif
Chris@16: 
Chris@16: ///////////////////////////////////////////////////////////////////////////////
Chris@16: namespace boost {
Chris@16: namespace wave {
Chris@16: namespace cpplexer {
Chris@16: 
Chris@16: namespace impl {
Chris@16: 
Chris@16: template <typename StringTypeT, typename PositionT>
Chris@16: class token_data
Chris@16: {
Chris@16: public:
Chris@16:     typedef StringTypeT string_type;
Chris@16:     typedef PositionT   position_type;
Chris@16: 
Chris@16:     //  default constructed tokens correspond to EOI tokens
Chris@16:     token_data()
Chris@16:     :   id(T_EOI), refcnt(1)
Chris@16:     {}
Chris@16: 
Chris@16:     //  construct an invalid token
Chris@16:     explicit token_data(int)
Chris@16:     :   id(T_UNKNOWN), refcnt(1)
Chris@16:     {}
Chris@16: 
Chris@16:     token_data(token_id id_, string_type const &value_, position_type const &pos_)
Chris@16:     :   id(id_), value(value_), pos(pos_), refcnt(1)
Chris@16:     {}
Chris@16: 
Chris@16:     token_data(token_data const& rhs)
Chris@16:     :   id(rhs.id), value(rhs.value), pos(rhs.pos), refcnt(1)
Chris@16:     {}
Chris@16: 
Chris@16:     ~token_data()
Chris@16:     {}
Chris@16: 
Chris@16:     std::size_t addref() { return ++refcnt; }
Chris@16:     std::size_t release() { return --refcnt; }
Chris@16:     std::size_t get_refcnt() const { return refcnt; }
Chris@16: 
Chris@16: // accessors
Chris@16:     operator token_id() const { return id; }
Chris@16:     string_type const &get_value() const { return value; }
Chris@16:     position_type const &get_position() const { return pos; }
Chris@16: 
Chris@16:     void set_token_id (token_id id_) { id = id_; }
Chris@16:     void set_value (string_type const &value_) { value = value_; }
Chris@16:     void set_position (position_type const &pos_) { pos = pos_; }
Chris@16: 
Chris@16:     friend bool operator== (token_data const& lhs, token_data const& rhs)
Chris@16:     {
Chris@16:         //  two tokens are considered equal even if they refer to different 
Chris@16:         //  positions
Chris@16:         return (lhs.id == rhs.id && lhs.value == rhs.value) ? true : false;
Chris@16:     }
Chris@16: 
Chris@16:     void init(token_id id_, string_type const &value_, position_type const &pos_)
Chris@16:     {
Chris@16:         BOOST_ASSERT(refcnt == 1);
Chris@16:         id = id_;
Chris@16:         value = value_; 
Chris@16:         pos = pos_;
Chris@16:     }
Chris@16: 
Chris@16:     void init(token_data const& rhs)
Chris@16:     {
Chris@16:         BOOST_ASSERT(refcnt == 1);
Chris@16:         id = rhs.id;
Chris@16:         value = rhs.value; 
Chris@16:         pos = rhs.pos;
Chris@16:     }
Chris@16: 
Chris@16:     static void *operator new(std::size_t size);
Chris@16:     static void operator delete(void *p, std::size_t size);
Chris@16: 
Chris@16: #if defined(BOOST_SPIRIT_DEBUG)
Chris@16: // debug support
Chris@16:     void print (std::ostream &stream) const
Chris@16:     {
Chris@16:         stream << get_token_name(id) << "(";
Chris@16:         for (std::size_t i = 0; i < value.size(); ++i) {
Chris@16:             switch (value[i]) {
Chris@16:             case '\r':  stream << "\\r"; break;
Chris@16:             case '\n':  stream << "\\n"; break;
Chris@16:             default:
Chris@16:                 stream << value[i]; 
Chris@16:                 break;
Chris@16:             }
Chris@16:         }
Chris@16:         stream << ")";
Chris@16:     }
Chris@16: #endif // defined(BOOST_SPIRIT_DEBUG)
Chris@16: 
Chris@16: #if BOOST_WAVE_SERIALIZATION != 0
Chris@16:     friend class boost::serialization::access;
Chris@16:     template<typename Archive>
Chris@16:     void serialize(Archive &ar, const unsigned int version)
Chris@16:     {
Chris@16:         using namespace boost::serialization;
Chris@16:         ar & make_nvp("id", id);
Chris@16:         ar & make_nvp("value", value);
Chris@16:         ar & make_nvp("position", pos);
Chris@16:     }
Chris@16: #endif
Chris@16: 
Chris@16: private:
Chris@16:     token_id id;                // the token id
Chris@16:     string_type value;          // the text, which was parsed into this token
Chris@16:     position_type pos;          // the original file position
Chris@16:     boost::detail::atomic_count refcnt;
Chris@16: };
Chris@16: 
Chris@16: ///////////////////////////////////////////////////////////////////////////////
Chris@16: struct token_data_tag {};
Chris@16: 
Chris@16: template <typename StringTypeT, typename PositionT>
Chris@16: inline void *
Chris@16: token_data<StringTypeT, PositionT>::operator new(std::size_t size)
Chris@16: {
Chris@16:     BOOST_ASSERT(sizeof(token_data<StringTypeT, PositionT>) == size);
Chris@16:     typedef boost::singleton_pool<
Chris@16:             token_data_tag, sizeof(token_data<StringTypeT, PositionT>)
Chris@16:         > pool_type;
Chris@16: 
Chris@16:     void *ret = pool_type::malloc();
Chris@16:     if (0 == ret)
Chris@16:         boost::throw_exception(std::bad_alloc());
Chris@16:     return ret;
Chris@16: }
Chris@16: 
Chris@16: template <typename StringTypeT, typename PositionT>
Chris@16: inline void 
Chris@16: token_data<StringTypeT, PositionT>::operator delete(void *p, std::size_t size)
Chris@16: {
Chris@16:     BOOST_ASSERT(sizeof(token_data<StringTypeT, PositionT>) == size);
Chris@16:     typedef boost::singleton_pool<
Chris@16:             token_data_tag, sizeof(token_data<StringTypeT, PositionT>)
Chris@16:         > pool_type;
Chris@16: 
Chris@16:     if (0 != p) 
Chris@16:         pool_type::free(p);
Chris@16: }
Chris@16: 
Chris@16: } // namespace impl
Chris@16: 
Chris@16: ///////////////////////////////////////////////////////////////////////////////
Chris@16: //  forward declaration of the token type
Chris@16: template <typename PositionT = boost::wave::util::file_position_type>
Chris@16: class lex_token;
Chris@16: 
Chris@16: ///////////////////////////////////////////////////////////////////////////////
Chris@16: //  
Chris@16: //  lex_token
Chris@16: //
Chris@16: ///////////////////////////////////////////////////////////////////////////////
Chris@16: 
Chris@16: template <typename PositionT>
Chris@16: class lex_token 
Chris@16: {
Chris@16: public:
Chris@16:     typedef BOOST_WAVE_STRINGTYPE   string_type;
Chris@16:     typedef PositionT               position_type;
Chris@16: 
Chris@16: private:
Chris@16:     typedef impl::token_data<string_type, position_type> data_type;
Chris@16:     
Chris@16: public:
Chris@16:     //  default constructed tokens correspond to EOI tokens
Chris@16:     lex_token()
Chris@16:     :   data(0)
Chris@16:     {}
Chris@16: 
Chris@16:     //  construct an invalid token
Chris@16:     explicit lex_token(int)
Chris@16:     :   data(new data_type(0))
Chris@16:     {}
Chris@16: 
Chris@16:     lex_token(lex_token const& rhs)
Chris@16:     :   data(rhs.data)
Chris@16:     {
Chris@16:         if (0 != data) 
Chris@16:             data->addref();
Chris@16:     }
Chris@16: 
Chris@16:     lex_token(token_id id_, string_type const &value_, PositionT const &pos_)
Chris@16:     :   data(new data_type(id_, value_, pos_))
Chris@16:     {}
Chris@16: 
Chris@16:     ~lex_token()
Chris@16:     {
Chris@16:         if (0 != data && 0 == data->release()) 
Chris@16:             delete data;
Chris@16:         data = 0;
Chris@16:     }
Chris@16: 
Chris@16:     lex_token& operator=(lex_token const& rhs)
Chris@16:     {
Chris@16:         if (&rhs != this) {
Chris@16:             if (0 != data && 0 == data->release()) 
Chris@16:                 delete data;
Chris@16:             
Chris@16:             data = rhs.data;
Chris@16:             if (0 != data) 
Chris@16:                 data->addref();
Chris@16:         }
Chris@16:         return *this;
Chris@16:     }
Chris@16: 
Chris@16: // accessors
Chris@16:     operator token_id() const { return 0 != data ? token_id(*data) : T_EOI; }
Chris@16:     string_type const &get_value() const { return data->get_value(); }
Chris@16:     position_type const &get_position() const { return data->get_position(); }
Chris@16:     bool is_eoi() const { return 0 == data || token_id(*data) == T_EOI; }
Chris@16:     bool is_valid() const { return 0 != data && token_id(*data) != T_UNKNOWN; }
Chris@16: 
Chris@16:     void set_token_id (token_id id_) { make_unique(); data->set_token_id(id_); }
Chris@16:     void set_value (string_type const &value_) { make_unique(); data->set_value(value_); }
Chris@16:     void set_position (position_type const &pos_) { make_unique(); data->set_position(pos_); }
Chris@16: 
Chris@16:     friend bool operator== (lex_token const& lhs, lex_token const& rhs)
Chris@16:     {
Chris@16:         if (0 == rhs.data)
Chris@16:             return 0 == lhs.data;
Chris@16:         if (0 == lhs.data)
Chris@16:             return false;
Chris@16:         return *(lhs.data) == *(rhs.data);
Chris@16:     }
Chris@16: 
Chris@16: // debug support
Chris@16: #if BOOST_WAVE_DUMP_PARSE_TREE != 0
Chris@16: // access functions for the tree_to_xml functionality
Chris@16:     static int get_token_id(lex_token const &t) 
Chris@16:         { return token_id(t); }
Chris@16:     static string_type get_token_value(lex_token const &t) 
Chris@16:         { return t.get_value(); }
Chris@16: #endif 
Chris@16: 
Chris@16: #if defined(BOOST_SPIRIT_DEBUG)
Chris@16: // debug support
Chris@16:     void print (std::ostream &stream) const
Chris@16:     {
Chris@16:         data->print(stream);
Chris@16:     }
Chris@16: #endif // defined(BOOST_SPIRIT_DEBUG)
Chris@16: 
Chris@16: private:
Chris@16: #if BOOST_WAVE_SERIALIZATION != 0
Chris@16:     friend class boost::serialization::access;
Chris@16:     template<typename Archive>
Chris@16:     void serialize(Archive &ar, const unsigned int version)
Chris@16:     {
Chris@16:         data->serialize(ar, version);
Chris@16:     }
Chris@16: #endif
Chris@16: 
Chris@16:     // make a unique copy of the current object
Chris@16:     void make_unique()
Chris@16:     {
Chris@16:         if (1 == data->get_refcnt())
Chris@16:             return;
Chris@16: 
Chris@16:         data_type* newdata = new data_type(*data) ;
Chris@16: 
Chris@16:         data->release();          // release this reference, can't get zero 
Chris@16:         data = newdata;
Chris@16:     }
Chris@16: 
Chris@16:     data_type* data;
Chris@16: };
Chris@16: 
Chris@16: ///////////////////////////////////////////////////////////////////////////////
Chris@16: //  This overload is needed by the multi_pass/functor_input_policy to 
Chris@16: //  validate a token instance. It has to be defined in the same namespace 
Chris@16: //  as the token class itself to allow ADL to find it.
Chris@16: ///////////////////////////////////////////////////////////////////////////////
Chris@16: template <typename Position>
Chris@16: inline bool 
Chris@16: token_is_valid(lex_token<Position> const& t)
Chris@16: {
Chris@16:     return t.is_valid();
Chris@16: }
Chris@16: 
Chris@16: ///////////////////////////////////////////////////////////////////////////////
Chris@16: #if defined(BOOST_SPIRIT_DEBUG)
Chris@16: template <typename PositionT>
Chris@16: inline std::ostream &
Chris@16: operator<< (std::ostream &stream, lex_token<PositionT> const &object)
Chris@16: {
Chris@16:     object.print(stream);
Chris@16:     return stream;
Chris@16: }
Chris@16: #endif // defined(BOOST_SPIRIT_DEBUG)
Chris@16: 
Chris@16: ///////////////////////////////////////////////////////////////////////////////
Chris@16: }   // namespace cpplexer
Chris@16: }   // namespace wave
Chris@16: }   // namespace boost
Chris@16: 
Chris@16: // the suffix header occurs after all of the code
Chris@16: #ifdef BOOST_HAS_ABI_HEADERS
Chris@16: #include BOOST_ABI_SUFFIX
Chris@16: #endif
Chris@16: 
Chris@16: #endif // !defined(CPP_TOKEN_HPP_53A13BD2_FBAA_444B_9B8B_FCB225C2BBA8_INCLUDED)