Chris@16: // Boost token_functions.hpp  ------------------------------------------------//
Chris@16: 
Chris@16: // Copyright John R. Bandela 2001.
Chris@16: 
Chris@16: // Distributed under the Boost Software License, Version 1.0. (See
Chris@16: // accompanying file LICENSE_1_0.txt or copy at
Chris@16: // http://www.boost.org/LICENSE_1_0.txt)
Chris@16: 
Chris@16: // See http://www.boost.org/libs/tokenizer/ for documentation.
Chris@16: 
Chris@16: // Revision History:
Chris@16: // 01 Oct 2004   Joaquin M Lopez Munoz
Chris@16: //      Workaround for a problem with string::assign in msvc-stlport
Chris@16: // 06 Apr 2004   John Bandela
Chris@16: //      Fixed a bug involving using char_delimiter with a true input iterator
Chris@16: // 28 Nov 2003   Robert Zeh and John Bandela
Chris@16: //      Converted into "fast" functions that avoid using += when
Chris@16: //      the supplied iterator isn't an input_iterator; based on
Chris@16: //      some work done at Archelon and a version that was checked into
Chris@16: //      the boost CVS for a short period of time.
Chris@16: // 20 Feb 2002   John Maddock
Chris@16: //      Removed using namespace std declarations and added
Chris@16: //      workaround for BOOST_NO_STDC_NAMESPACE (the library
Chris@16: //      can be safely mixed with regex).
Chris@16: // 06 Feb 2002   Jeremy Siek
Chris@16: //      Added char_separator.
Chris@16: // 02 Feb 2002   Jeremy Siek
Chris@16: //      Removed tabs and a little cleanup.
Chris@16: 
Chris@16: 
Chris@16: #ifndef BOOST_TOKEN_FUNCTIONS_JRB120303_HPP_
Chris@16: #define BOOST_TOKEN_FUNCTIONS_JRB120303_HPP_
Chris@16: 
Chris@16: #include <vector>
Chris@16: #include <stdexcept>
Chris@16: #include <string>
Chris@16: #include <cctype>
Chris@16: #include <algorithm> // for find_if
Chris@16: #include <boost/config.hpp>
Chris@16: #include <boost/assert.hpp>
Chris@16: #include <boost/detail/workaround.hpp>
Chris@16: #include <boost/mpl/if.hpp>
Chris@101: #include <boost/throw_exception.hpp>
Chris@16: #if !defined(BOOST_NO_CWCTYPE)
Chris@16: #include <cwctype>
Chris@16: #endif
Chris@16: 
Chris@16: //
Chris@16: // the following must not be macros if we are to prefix them
Chris@16: // with std:: (they shouldn't be macros anyway...)
Chris@16: //
Chris@16: #ifdef ispunct
Chris@16: #  undef ispunct
Chris@16: #endif
Chris@16: #ifdef iswpunct
Chris@16: #  undef iswpunct
Chris@16: #endif
Chris@16: #ifdef isspace
Chris@16: #  undef isspace
Chris@16: #endif
Chris@16: #ifdef iswspace
Chris@16: #  undef iswspace
Chris@16: #endif
Chris@16: //
Chris@16: // fix namespace problems:
Chris@16: //
Chris@16: #ifdef BOOST_NO_STDC_NAMESPACE
Chris@16: namespace std{
Chris@16:  using ::ispunct;
Chris@16:  using ::isspace;
Chris@16: #if !defined(BOOST_NO_CWCTYPE)
Chris@16:  using ::iswpunct;
Chris@16:  using ::iswspace;
Chris@16: #endif
Chris@16: }
Chris@16: #endif
Chris@16: 
Chris@16: namespace boost{
Chris@16:   //===========================================================================
Chris@16:   // The escaped_list_separator class. Which is a model of TokenizerFunction
Chris@16:   // An escaped list is a super-set of what is commonly known as a comma
Chris@16:   // separated value (csv) list.It is separated into fields by a comma or
Chris@16:   // other character. If the delimiting character is inside quotes, then it is
Chris@16:   // counted as a regular character.To allow for embedded quotes in a field,
Chris@16:   // there can be escape sequences using the \ much like C.
Chris@16:   // The role of the comma, the quotation mark, and the escape
Chris@16:   // character (backslash \), can be assigned to other characters.
Chris@16: 
Chris@16:   struct escaped_list_error : public std::runtime_error{
Chris@16:     escaped_list_error(const std::string& what_arg):std::runtime_error(what_arg) { }
Chris@16:   };
Chris@16: 
Chris@16: 
Chris@16: // The out of the box GCC 2.95 on cygwin does not have a char_traits class.
Chris@16: // MSVC does not like the following typename
Chris@16:   template <class Char,
Chris@16:     class Traits = BOOST_DEDUCED_TYPENAME std::basic_string<Char>::traits_type >
Chris@16:   class escaped_list_separator {
Chris@16: 
Chris@16:   private:
Chris@16:     typedef std::basic_string<Char,Traits> string_type;
Chris@16:     struct char_eq {
Chris@16:       Char e_;
Chris@16:       char_eq(Char e):e_(e) { }
Chris@16:       bool operator()(Char c) {
Chris@16:         return Traits::eq(e_,c);
Chris@16:       }
Chris@16:     };
Chris@16:     string_type  escape_;
Chris@16:     string_type  c_;
Chris@16:     string_type  quote_;
Chris@16:     bool last_;
Chris@16: 
Chris@16:     bool is_escape(Char e) {
Chris@16:       char_eq f(e);
Chris@16:       return std::find_if(escape_.begin(),escape_.end(),f)!=escape_.end();
Chris@16:     }
Chris@16:     bool is_c(Char e) {
Chris@16:       char_eq f(e);
Chris@16:       return std::find_if(c_.begin(),c_.end(),f)!=c_.end();
Chris@16:     }
Chris@16:     bool is_quote(Char e) {
Chris@16:       char_eq f(e);
Chris@16:       return std::find_if(quote_.begin(),quote_.end(),f)!=quote_.end();
Chris@16:     }
Chris@16:     template <typename iterator, typename Token>
Chris@16:     void do_escape(iterator& next,iterator end,Token& tok) {
Chris@16:       if (++next == end)
Chris@101:         BOOST_THROW_EXCEPTION(escaped_list_error(std::string("cannot end with escape")));
Chris@16:       if (Traits::eq(*next,'n')) {
Chris@16:         tok+='\n';
Chris@16:         return;
Chris@16:       }
Chris@16:       else if (is_quote(*next)) {
Chris@16:         tok+=*next;
Chris@16:         return;
Chris@16:       }
Chris@16:       else if (is_c(*next)) {
Chris@16:         tok+=*next;
Chris@16:         return;
Chris@16:       }
Chris@16:       else if (is_escape(*next)) {
Chris@16:         tok+=*next;
Chris@16:         return;
Chris@16:       }
Chris@16:       else
Chris@101:         BOOST_THROW_EXCEPTION(escaped_list_error(std::string("unknown escape sequence")));
Chris@16:     }
Chris@16: 
Chris@16:     public:
Chris@16: 
Chris@16:     explicit escaped_list_separator(Char  e = '\\',
Chris@16:                                     Char c = ',',Char  q = '\"')
Chris@16:       : escape_(1,e), c_(1,c), quote_(1,q), last_(false) { }
Chris@16: 
Chris@16:     escaped_list_separator(string_type e, string_type c, string_type q)
Chris@16:       : escape_(e), c_(c), quote_(q), last_(false) { }
Chris@16: 
Chris@16:     void reset() {last_=false;}
Chris@16: 
Chris@16:     template <typename InputIterator, typename Token>
Chris@16:     bool operator()(InputIterator& next,InputIterator end,Token& tok) {
Chris@16:       bool bInQuote = false;
Chris@16:       tok = Token();
Chris@16: 
Chris@16:       if (next == end) {
Chris@16:         if (last_) {
Chris@16:           last_ = false;
Chris@16:           return true;
Chris@16:         }
Chris@16:         else
Chris@16:           return false;
Chris@16:       }
Chris@16:       last_ = false;
Chris@16:       for (;next != end;++next) {
Chris@16:         if (is_escape(*next)) {
Chris@16:           do_escape(next,end,tok);
Chris@16:         }
Chris@16:         else if (is_c(*next)) {
Chris@16:           if (!bInQuote) {
Chris@16:             // If we are not in quote, then we are done
Chris@16:             ++next;
Chris@16:             // The last character was a c, that means there is
Chris@16:             // 1 more blank field
Chris@16:             last_ = true;
Chris@16:             return true;
Chris@16:           }
Chris@16:           else tok+=*next;
Chris@16:         }
Chris@16:         else if (is_quote(*next)) {
Chris@16:           bInQuote=!bInQuote;
Chris@16:         }
Chris@16:         else {
Chris@16:           tok += *next;
Chris@16:         }
Chris@16:       }
Chris@16:       return true;
Chris@16:     }
Chris@16:   };
Chris@16: 
Chris@16:   //===========================================================================
Chris@16:   // The classes here are used by offset_separator and char_separator to implement
Chris@16:   // faster assigning of tokens using assign instead of +=
Chris@16: 
Chris@16:   namespace tokenizer_detail {
Chris@16:   //===========================================================================
Chris@16:   // Tokenizer was broken for wide character separators, at least on Windows, since
Chris@16:   // CRT functions isspace etc only expect values in [0, 0xFF]. Debug build asserts
Chris@16:   // if higher values are passed in. The traits extension class should take care of this.
Chris@16:   // Assuming that the conditional will always get optimized out in the function
Chris@16:   // implementations, argument types are not a problem since both forms of character classifiers
Chris@16:   // expect an int.
Chris@16: 
Chris@16: #if !defined(BOOST_NO_CWCTYPE)
Chris@16:   template<typename traits, int N>
Chris@16:   struct traits_extension_details : public traits {
Chris@16:     typedef typename traits::char_type char_type;
Chris@16:     static bool isspace(char_type c)
Chris@16:     {
Chris@16:        return std::iswspace(c) != 0;
Chris@16:     }
Chris@16:     static bool ispunct(char_type c)
Chris@16:     {
Chris@16:        return std::iswpunct(c) != 0;
Chris@16:     }
Chris@16:   };
Chris@16: 
Chris@16:   template<typename traits>
Chris@16:   struct traits_extension_details<traits, 1> : public traits {
Chris@16:     typedef typename traits::char_type char_type;
Chris@16:     static bool isspace(char_type c)
Chris@16:     {
Chris@16:        return std::isspace(c) != 0;
Chris@16:     }
Chris@16:     static bool ispunct(char_type c)
Chris@16:     {
Chris@16:        return std::ispunct(c) != 0;
Chris@16:     }
Chris@16:   };
Chris@16: #endif
Chris@16: 
Chris@16: 
Chris@16:   // In case there is no cwctype header, we implement the checks manually.
Chris@16:   // We make use of the fact that the tested categories should fit in ASCII.
Chris@16:   template<typename traits>
Chris@16:   struct traits_extension : public traits {
Chris@16:     typedef typename traits::char_type char_type;
Chris@16:     static bool isspace(char_type c)
Chris@16:     {
Chris@16: #if !defined(BOOST_NO_CWCTYPE)
Chris@16:       return traits_extension_details<traits, sizeof(char_type)>::isspace(c);
Chris@16: #else
Chris@16:       return static_cast< unsigned >(c) <= 255 && std::isspace(c) != 0;
Chris@16: #endif
Chris@16:     }
Chris@16: 
Chris@16:     static bool ispunct(char_type c)
Chris@16:     {
Chris@16: #if !defined(BOOST_NO_CWCTYPE)
Chris@16:       return traits_extension_details<traits, sizeof(char_type)>::ispunct(c);
Chris@16: #else
Chris@16:       return static_cast< unsigned >(c) <= 255 && std::ispunct(c) != 0;
Chris@16: #endif
Chris@16:     }
Chris@16:   };
Chris@16: 
Chris@16:   // The assign_or_plus_equal struct contains functions that implement
Chris@16:   // assign, +=, and clearing based on the iterator type.  The
Chris@16:   // generic case does nothing for plus_equal and clearing, while
Chris@16:   // passing through the call for assign.
Chris@16:   //
Chris@16:   // When an input iterator is being used, the situation is reversed.
Chris@16:   // The assign method does nothing, plus_equal invokes operator +=,
Chris@16:   // and the clearing method sets the supplied token to the default
Chris@16:   // token constructor's result.
Chris@16:   //
Chris@16: 
Chris@16:   template<class IteratorTag>
Chris@16:   struct assign_or_plus_equal {
Chris@16:     template<class Iterator, class Token>
Chris@16:     static void assign(Iterator b, Iterator e, Token &t) {
Chris@16:       t.assign(b, e);
Chris@16:     }
Chris@16: 
Chris@16:     template<class Token, class Value>
Chris@16:     static void plus_equal(Token &, const Value &) { }
Chris@16: 
Chris@16:     // If we are doing an assign, there is no need for the
Chris@16:     // the clear.
Chris@16:     //
Chris@16:     template<class Token>
Chris@16:     static void clear(Token &) { }
Chris@16:   };
Chris@16: 
Chris@16:   template <>
Chris@16:   struct assign_or_plus_equal<std::input_iterator_tag> {
Chris@16:     template<class Iterator, class Token>
Chris@16:     static void assign(Iterator , Iterator , Token &) { }
Chris@16:     template<class Token, class Value>
Chris@16:     static void plus_equal(Token &t, const Value &v) {
Chris@16:       t += v;
Chris@16:     }
Chris@16:     template<class Token>
Chris@16:     static void clear(Token &t) {
Chris@16:       t = Token();
Chris@16:     }
Chris@16:   };
Chris@16: 
Chris@16: 
Chris@16:   template<class Iterator>
Chris@16:   struct pointer_iterator_category{
Chris@16:     typedef std::random_access_iterator_tag type;
Chris@16:   };
Chris@16: 
Chris@16: 
Chris@16:   template<class Iterator>
Chris@16:   struct class_iterator_category{
Chris@16:     typedef typename Iterator::iterator_category type;
Chris@16:   };
Chris@16: 
Chris@16: 
Chris@16: 
Chris@16:   // This portably gets the iterator_tag without partial template specialization
Chris@16:   template<class Iterator>
Chris@16:     struct get_iterator_category{
Chris@16:     typedef typename mpl::if_<is_pointer<Iterator>,
Chris@16:       pointer_iterator_category<Iterator>,
Chris@16:       class_iterator_category<Iterator>
Chris@16:     >::type cat;
Chris@16: 
Chris@16:     typedef typename cat::type iterator_category;
Chris@16:   };
Chris@16: 
Chris@16: 
Chris@16:   } // namespace tokenizer_detail
Chris@16: 
Chris@16: 
Chris@16:   //===========================================================================
Chris@16:   // The offset_separator class, which is a model of TokenizerFunction.
Chris@16:   // Offset breaks a string into tokens based on a range of offsets
Chris@16: 
Chris@16:   class offset_separator {
Chris@16:   private:
Chris@16: 
Chris@16:     std::vector<int> offsets_;
Chris@16:     unsigned int current_offset_;
Chris@16:     bool wrap_offsets_;
Chris@16:     bool return_partial_last_;
Chris@16: 
Chris@16:   public:
Chris@16:     template <typename Iter>
Chris@16:     offset_separator(Iter begin, Iter end, bool wrap_offsets = true,
Chris@16:                      bool return_partial_last = true)
Chris@16:       : offsets_(begin,end), current_offset_(0),
Chris@16:         wrap_offsets_(wrap_offsets),
Chris@16:         return_partial_last_(return_partial_last) { }
Chris@16: 
Chris@16:     offset_separator()
Chris@16:       : offsets_(1,1), current_offset_(),
Chris@16:         wrap_offsets_(true), return_partial_last_(true) { }
Chris@16: 
Chris@16:     void reset() {
Chris@16:       current_offset_ = 0;
Chris@16:     }
Chris@16: 
Chris@16:     template <typename InputIterator, typename Token>
Chris@16:     bool operator()(InputIterator& next, InputIterator end, Token& tok)
Chris@16:     {
Chris@16:       typedef tokenizer_detail::assign_or_plus_equal<
Chris@16:         BOOST_DEDUCED_TYPENAME tokenizer_detail::get_iterator_category<
Chris@16:           InputIterator
Chris@16:         >::iterator_category
Chris@16:       > assigner;
Chris@16: 
Chris@16:       BOOST_ASSERT(!offsets_.empty());
Chris@16: 
Chris@16:       assigner::clear(tok);
Chris@16:       InputIterator start(next);
Chris@16: 
Chris@16:       if (next == end)
Chris@16:         return false;
Chris@16: 
Chris@16:       if (current_offset_ == offsets_.size())
Chris@16:       {
Chris@16:         if (wrap_offsets_)
Chris@16:           current_offset_=0;
Chris@16:         else
Chris@16:           return false;
Chris@16:       }
Chris@16: 
Chris@16:       int c = offsets_[current_offset_];
Chris@16:       int i = 0;
Chris@16:       for (; i < c; ++i) {
Chris@16:         if (next == end)break;
Chris@16:         assigner::plus_equal(tok,*next++);
Chris@16:       }
Chris@16:       assigner::assign(start,next,tok);
Chris@16: 
Chris@16:       if (!return_partial_last_)
Chris@16:         if (i < (c-1) )
Chris@16:           return false;
Chris@16: 
Chris@16:       ++current_offset_;
Chris@16:       return true;
Chris@16:     }
Chris@16:   };
Chris@16: 
Chris@16: 
Chris@16:   //===========================================================================
Chris@16:   // The char_separator class breaks a sequence of characters into
Chris@16:   // tokens based on the character delimiters (very much like bad old
Chris@16:   // strtok). A delimiter character can either be kept or dropped. A
Chris@16:   // kept delimiter shows up as an output token, whereas a dropped
Chris@16:   // delimiter does not.
Chris@16: 
Chris@16:   // This class replaces the char_delimiters_separator class. The
Chris@16:   // constructor for the char_delimiters_separator class was too
Chris@16:   // confusing and needed to be deprecated. However, because of the
Chris@16:   // default arguments to the constructor, adding the new constructor
Chris@16:   // would cause ambiguity, so instead I deprecated the whole class.
Chris@16:   // The implementation of the class was also simplified considerably.
Chris@16: 
Chris@16:   enum empty_token_policy { drop_empty_tokens, keep_empty_tokens };
Chris@16: 
Chris@16:   // The out of the box GCC 2.95 on cygwin does not have a char_traits class.
Chris@16:   template <typename Char,
Chris@16:     typename Tr = BOOST_DEDUCED_TYPENAME std::basic_string<Char>::traits_type >
Chris@16:   class char_separator
Chris@16:   {
Chris@16:     typedef tokenizer_detail::traits_extension<Tr> Traits;
Chris@16:     typedef std::basic_string<Char,Tr> string_type;
Chris@16:   public:
Chris@16:     explicit
Chris@16:     char_separator(const Char* dropped_delims,
Chris@16:                    const Char* kept_delims = 0,
Chris@16:                    empty_token_policy empty_tokens = drop_empty_tokens)
Chris@16:       : m_dropped_delims(dropped_delims),
Chris@16:         m_use_ispunct(false),
Chris@16:         m_use_isspace(false),
Chris@16:         m_empty_tokens(empty_tokens),
Chris@16:         m_output_done(false)
Chris@16:     {
Chris@16:       // Borland workaround
Chris@16:       if (kept_delims)
Chris@16:         m_kept_delims = kept_delims;
Chris@16:     }
Chris@16: 
Chris@16:                 // use ispunct() for kept delimiters and isspace for dropped.
Chris@16:     explicit
Chris@16:     char_separator()
Chris@16:       : m_use_ispunct(true),
Chris@16:         m_use_isspace(true),
Chris@16:         m_empty_tokens(drop_empty_tokens) { }
Chris@16: 
Chris@16:     void reset() { }
Chris@16: 
Chris@16:     template <typename InputIterator, typename Token>
Chris@16:     bool operator()(InputIterator& next, InputIterator end, Token& tok)
Chris@16:     {
Chris@16:       typedef tokenizer_detail::assign_or_plus_equal<
Chris@16:         BOOST_DEDUCED_TYPENAME tokenizer_detail::get_iterator_category<
Chris@16:           InputIterator
Chris@16:         >::iterator_category
Chris@16:       > assigner;
Chris@16: 
Chris@16:       assigner::clear(tok);
Chris@16: 
Chris@16:       // skip past all dropped_delims
Chris@16:       if (m_empty_tokens == drop_empty_tokens)
Chris@16:         for (; next != end  && is_dropped(*next); ++next)
Chris@16:           { }
Chris@16: 
Chris@16:       InputIterator start(next);
Chris@16: 
Chris@16:       if (m_empty_tokens == drop_empty_tokens) {
Chris@16: 
Chris@16:         if (next == end)
Chris@16:           return false;
Chris@16: 
Chris@16: 
Chris@16:         // if we are on a kept_delims move past it and stop
Chris@16:         if (is_kept(*next)) {
Chris@16:           assigner::plus_equal(tok,*next);
Chris@16:           ++next;
Chris@16:         } else
Chris@16:           // append all the non delim characters
Chris@16:           for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next)
Chris@16:             assigner::plus_equal(tok,*next);
Chris@16:       }
Chris@16:       else { // m_empty_tokens == keep_empty_tokens
Chris@16: 
Chris@16:         // Handle empty token at the end
Chris@16:         if (next == end)
Chris@16:         {
Chris@16:           if (m_output_done == false)
Chris@16:           {
Chris@16:             m_output_done = true;
Chris@16:             assigner::assign(start,next,tok);
Chris@16:             return true;
Chris@16:           }
Chris@16:           else
Chris@16:             return false;
Chris@16:         }
Chris@16: 
Chris@16:         if (is_kept(*next)) {
Chris@16:           if (m_output_done == false)
Chris@16:             m_output_done = true;
Chris@16:           else {
Chris@16:             assigner::plus_equal(tok,*next);
Chris@16:             ++next;
Chris@16:             m_output_done = false;
Chris@16:           }
Chris@16:         }
Chris@16:         else if (m_output_done == false && is_dropped(*next)) {
Chris@16:           m_output_done = true;
Chris@16:         }
Chris@16:         else {
Chris@16:           if (is_dropped(*next))
Chris@16:             start=++next;
Chris@16:           for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next)
Chris@16:             assigner::plus_equal(tok,*next);
Chris@16:           m_output_done = true;
Chris@16:         }
Chris@16:       }
Chris@16:       assigner::assign(start,next,tok);
Chris@16:       return true;
Chris@16:     }
Chris@16: 
Chris@16:   private:
Chris@16:     string_type m_kept_delims;
Chris@16:     string_type m_dropped_delims;
Chris@16:     bool m_use_ispunct;
Chris@16:     bool m_use_isspace;
Chris@16:     empty_token_policy m_empty_tokens;
Chris@16:     bool m_output_done;
Chris@16: 
Chris@16:     bool is_kept(Char E) const
Chris@16:     {
Chris@16:       if (m_kept_delims.length())
Chris@16:         return m_kept_delims.find(E) != string_type::npos;
Chris@16:       else if (m_use_ispunct) {
Chris@16:         return Traits::ispunct(E) != 0;
Chris@16:       } else
Chris@16:         return false;
Chris@16:     }
Chris@16:     bool is_dropped(Char E) const
Chris@16:     {
Chris@16:       if (m_dropped_delims.length())
Chris@16:         return m_dropped_delims.find(E) != string_type::npos;
Chris@16:       else if (m_use_isspace) {
Chris@16:         return Traits::isspace(E) != 0;
Chris@16:       } else
Chris@16:         return false;
Chris@16:     }
Chris@16:   };
Chris@16: 
Chris@16:   //===========================================================================
Chris@16:   // The following class is DEPRECATED, use class char_separators instead.
Chris@16:   //
Chris@16:   // The char_delimiters_separator class, which is a model of
Chris@16:   // TokenizerFunction.  char_delimiters_separator breaks a string
Chris@16:   // into tokens based on character delimiters. There are 2 types of
Chris@16:   // delimiters. returnable delimiters can be returned as
Chris@16:   // tokens. These are often punctuation. nonreturnable delimiters
Chris@16:   // cannot be returned as tokens. These are often whitespace
Chris@16: 
Chris@16:   // The out of the box GCC 2.95 on cygwin does not have a char_traits class.
Chris@16:   template <class Char,
Chris@16:     class Tr = BOOST_DEDUCED_TYPENAME std::basic_string<Char>::traits_type >
Chris@16:   class char_delimiters_separator {
Chris@16:   private:
Chris@16: 
Chris@16:     typedef tokenizer_detail::traits_extension<Tr> Traits;
Chris@16:     typedef std::basic_string<Char,Tr> string_type;
Chris@16:     string_type returnable_;
Chris@16:     string_type nonreturnable_;
Chris@16:     bool return_delims_;
Chris@16:     bool no_ispunct_;
Chris@16:     bool no_isspace_;
Chris@16: 
Chris@16:     bool is_ret(Char E)const
Chris@16:     {
Chris@16:       if (returnable_.length())
Chris@16:         return  returnable_.find(E) != string_type::npos;
Chris@16:       else{
Chris@16:         if (no_ispunct_) {return false;}
Chris@16:         else{
Chris@16:           int r = Traits::ispunct(E);
Chris@16:           return r != 0;
Chris@16:         }
Chris@16:       }
Chris@16:     }
Chris@16:     bool is_nonret(Char E)const
Chris@16:     {
Chris@16:       if (nonreturnable_.length())
Chris@16:         return  nonreturnable_.find(E) != string_type::npos;
Chris@16:       else{
Chris@16:         if (no_isspace_) {return false;}
Chris@16:         else{
Chris@16:           int r = Traits::isspace(E);
Chris@16:           return r != 0;
Chris@16:         }
Chris@16:       }
Chris@16:     }
Chris@16: 
Chris@16:   public:
Chris@16:     explicit char_delimiters_separator(bool return_delims = false,
Chris@16:                                        const Char* returnable = 0,
Chris@16:                                        const Char* nonreturnable = 0)
Chris@16:       : returnable_(returnable ? returnable : string_type().c_str()),
Chris@16:         nonreturnable_(nonreturnable ? nonreturnable:string_type().c_str()),
Chris@16:         return_delims_(return_delims), no_ispunct_(returnable!=0),
Chris@16:         no_isspace_(nonreturnable!=0) { }
Chris@16: 
Chris@16:     void reset() { }
Chris@16: 
Chris@16:   public:
Chris@16: 
Chris@16:      template <typename InputIterator, typename Token>
Chris@16:      bool operator()(InputIterator& next, InputIterator end,Token& tok) {
Chris@16:      tok = Token();
Chris@16: 
Chris@16:      // skip past all nonreturnable delims
Chris@16:      // skip past the returnable only if we are not returning delims
Chris@16:      for (;next!=end && ( is_nonret(*next) || (is_ret(*next)
Chris@16:        && !return_delims_ ) );++next) { }
Chris@16: 
Chris@16:      if (next == end) {
Chris@16:        return false;
Chris@16:      }
Chris@16: 
Chris@16:      // if we are to return delims and we are one a returnable one
Chris@16:      // move past it and stop
Chris@16:      if (is_ret(*next) && return_delims_) {
Chris@16:        tok+=*next;
Chris@16:        ++next;
Chris@16:      }
Chris@16:      else
Chris@16:        // append all the non delim characters
Chris@16:        for (;next!=end && !is_nonret(*next) && !is_ret(*next);++next)
Chris@16:          tok+=*next;
Chris@16: 
Chris@16: 
Chris@16:      return true;
Chris@16:    }
Chris@16:   };
Chris@16: 
Chris@16: 
Chris@16: } //namespace boost
Chris@16: 
Chris@16: #endif