Chris@16: // Copyright (c) 2001-2011 Hartmut Kaiser Chris@16: // Chris@16: // Distributed under the Boost Software License, Version 1.0. (See accompanying Chris@16: // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) Chris@16: Chris@16: #if !defined(BOOST_SPIRIT_LEXER_PARSE_NOV_17_2007_0246PM) Chris@16: #define BOOST_SPIRIT_LEXER_PARSE_NOV_17_2007_0246PM Chris@16: Chris@16: #if defined(_MSC_VER) Chris@16: #pragma once Chris@16: #endif Chris@16: Chris@16: #include Chris@16: #include Chris@16: #include Chris@16: #include Chris@16: #include Chris@16: #include Chris@16: Chris@16: namespace boost { namespace spirit { namespace lex Chris@16: { Chris@16: /////////////////////////////////////////////////////////////////////////// Chris@16: // Import skip_flag enumerator type from Qi namespace Chris@16: using qi::skip_flag; Chris@16: Chris@16: /////////////////////////////////////////////////////////////////////////// Chris@16: // Chris@16: // The tokenize_and_parse() function is one of the main Spirit API Chris@16: // functions. It simplifies using a lexer as the underlying token source Chris@16: // while parsing a given input sequence. Chris@16: // Chris@16: // The function takes a pair of iterators spanning the underlying input Chris@16: // stream to parse, the lexer object (built from the token definitions) Chris@16: // and a parser object (built from the parser grammar definition). Chris@16: // Chris@16: // The second version of this function additionally takes an attribute to Chris@16: // be used as the top level data structure instance the parser should use Chris@16: // to store the recognized input to. Chris@16: // Chris@16: // The function returns true if the parsing succeeded (the given input Chris@16: // sequence has been successfully matched by the given grammar). Chris@16: // Chris@16: // first, last: The pair of iterators spanning the underlying input Chris@16: // sequence to parse. These iterators must at least Chris@16: // conform to the requirements of the std::intput_iterator Chris@16: // category. Chris@16: // On exit the iterator 'first' will be updated to the Chris@16: // position right after the last successfully matched Chris@16: // token. Chris@16: // lex: The lexer object (encoding the token definitions) to be Chris@16: // used to convert the input sequence into a sequence of Chris@16: // tokens. This token sequence is passed to the parsing Chris@16: // process. The LexerExpr type must conform to the Chris@16: // lexer interface described in the corresponding section Chris@16: // of the documentation. Chris@16: // xpr: The grammar object (encoding the parser grammar) to be Chris@16: // used to match the token sequence generated by the lex Chris@16: // object instance. The ParserExpr type must conform to Chris@16: // the grammar interface described in the corresponding Chris@16: // section of the documentation. Chris@16: // attr: The top level attribute passed to the parser. It will Chris@16: // be populated during the parsing of the input sequence. Chris@16: // On exit it will hold the 'parser result' corresponding Chris@16: // to the matched input sequence. Chris@16: // Chris@16: /////////////////////////////////////////////////////////////////////////// Chris@16: template Chris@16: inline bool Chris@16: tokenize_and_parse(Iterator& first, Iterator last, Lexer const& lex, Chris@16: ParserExpr const& xpr) Chris@16: { Chris@16: // Report invalid expression error as early as possible. Chris@16: // If you got an error_invalid_expression error message here, Chris@16: // then the expression (expr) is not a valid spirit qi expression. Chris@16: BOOST_SPIRIT_ASSERT_MATCH(qi::domain, ParserExpr); Chris@16: Chris@16: typename Lexer::iterator_type iter = lex.begin(first, last); Chris@16: return compile(xpr).parse( Chris@16: iter, lex.end(), unused, unused, unused); Chris@16: } Chris@16: Chris@16: /////////////////////////////////////////////////////////////////////////// Chris@16: template Chris@16: inline bool Chris@16: tokenize_and_parse(Iterator& first, Iterator last, Lexer const& lex Chris@16: , ParserExpr const& xpr, Attribute& attr) Chris@16: { Chris@16: // Report invalid expression error as early as possible. Chris@16: // If you got an error_invalid_expression error message here, Chris@16: // then the expression (expr) is not a valid spirit qi expression. Chris@16: BOOST_SPIRIT_ASSERT_MATCH(qi::domain, ParserExpr); Chris@16: Chris@16: typename Lexer::iterator_type iter = lex.begin(first, last); Chris@16: return compile(xpr).parse( Chris@16: iter, lex.end(), unused, unused, attr); Chris@16: } Chris@16: Chris@16: /////////////////////////////////////////////////////////////////////////// Chris@16: // Chris@16: // The tokenize_and_phrase_parse() function is one of the main Spirit API Chris@16: // functions. It simplifies using a lexer as the underlying token source Chris@16: // while phrase parsing a given input sequence. Chris@16: // Chris@16: // The function takes a pair of iterators spanning the underlying input Chris@16: // stream to parse, the lexer object (built from the token definitions) Chris@16: // and a parser object (built from the parser grammar definition). The Chris@16: // additional skipper parameter will be used as the skip parser during Chris@16: // the parsing process. Chris@16: // Chris@16: // The second version of this function additionally takes an attribute to Chris@16: // be used as the top level data structure instance the parser should use Chris@16: // to store the recognized input to. Chris@16: // Chris@16: // The function returns true if the parsing succeeded (the given input Chris@16: // sequence has been successfully matched by the given grammar). Chris@16: // Chris@16: // first, last: The pair of iterators spanning the underlying input Chris@16: // sequence to parse. These iterators must at least Chris@16: // conform to the requirements of the std::intput_iterator Chris@16: // category. Chris@16: // On exit the iterator 'first' will be updated to the Chris@16: // position right after the last successfully matched Chris@16: // token. Chris@16: // lex: The lexer object (encoding the token definitions) to be Chris@16: // used to convert the input sequence into a sequence of Chris@16: // tokens. This token sequence is passed to the parsing Chris@16: // process. The LexerExpr type must conform to the Chris@16: // lexer interface described in the corresponding section Chris@16: // of the documentation. Chris@16: // xpr: The grammar object (encoding the parser grammar) to be Chris@16: // used to match the token sequence generated by the lex Chris@16: // object instance. The ParserExpr type must conform to Chris@16: // the grammar interface described in the corresponding Chris@16: // section of the documentation. Chris@16: // skipper: The skip parser to be used while parsing the given Chris@16: // input sequence. Note, the skip parser will have to Chris@16: // act on the same token sequence as the main parser Chris@16: // 'xpr'. Chris@16: // post_skip: The post_skip flag controls whether the function will Chris@16: // invoke an additional post skip after the main parser Chris@16: // returned. Chris@16: // attr: The top level attribute passed to the parser. It will Chris@16: // be populated during the parsing of the input sequence. Chris@16: // On exit it will hold the 'parser result' corresponding Chris@16: // to the matched input sequence. Chris@16: // Chris@16: /////////////////////////////////////////////////////////////////////////// Chris@16: template Chris@16: inline bool Chris@16: tokenize_and_phrase_parse(Iterator& first, Iterator last Chris@16: , Lexer const& lex, ParserExpr const& xpr, Skipper const& skipper Chris@16: , BOOST_SCOPED_ENUM(skip_flag) post_skip = skip_flag::postskip) Chris@16: { Chris@16: // Report invalid expression error as early as possible. Chris@16: // If you got an error_invalid_expression error message here, Chris@16: // then the expression (expr) is not a valid spirit qi expression. Chris@16: BOOST_SPIRIT_ASSERT_MATCH(qi::domain, ParserExpr); Chris@16: BOOST_SPIRIT_ASSERT_MATCH(qi::domain, Skipper); Chris@16: Chris@16: typedef Chris@16: typename spirit::result_of::compile::type Chris@16: skipper_type; Chris@16: skipper_type const skipper_ = compile(skipper); Chris@16: Chris@16: typename Lexer::iterator_type iter = lex.begin(first, last); Chris@16: typename Lexer::iterator_type end = lex.end(); Chris@16: if (!compile(xpr).parse( Chris@16: iter, end, unused, skipper_, unused)) Chris@16: return false; Chris@16: Chris@16: // do a final post-skip Chris@16: if (post_skip == skip_flag::postskip) Chris@16: qi::skip_over(iter, end, skipper_); Chris@16: return true; Chris@16: } Chris@16: Chris@16: template Chris@16: inline bool Chris@16: tokenize_and_phrase_parse(Iterator& first, Iterator last Chris@16: , Lexer const& lex, ParserExpr const& xpr, Skipper const& skipper Chris@16: , BOOST_SCOPED_ENUM(skip_flag) post_skip, Attribute& attr) Chris@16: { Chris@16: // Report invalid expression error as early as possible. Chris@16: // If you got an error_invalid_expression error message here, Chris@16: // then the expression (expr) is not a valid spirit qi expression. Chris@16: BOOST_SPIRIT_ASSERT_MATCH(qi::domain, ParserExpr); Chris@16: BOOST_SPIRIT_ASSERT_MATCH(qi::domain, Skipper); Chris@16: Chris@16: typedef Chris@16: typename spirit::result_of::compile::type Chris@16: skipper_type; Chris@16: skipper_type const skipper_ = compile(skipper); Chris@16: Chris@16: typename Lexer::iterator_type iter = lex.begin(first, last); Chris@16: typename Lexer::iterator_type end = lex.end(); Chris@16: if (!compile(xpr).parse( Chris@16: iter, end, unused, skipper_, attr)) Chris@16: return false; Chris@16: Chris@16: // do a final post-skip Chris@16: if (post_skip == skip_flag::postskip) Chris@16: qi::skip_over(iter, end, skipper_); Chris@16: return true; Chris@16: } Chris@16: Chris@16: /////////////////////////////////////////////////////////////////////////// Chris@16: template Chris@16: inline bool Chris@16: tokenize_and_phrase_parse(Iterator& first, Iterator last Chris@16: , Lexer const& lex, ParserExpr const& xpr, Skipper const& skipper Chris@16: , Attribute& attr) Chris@16: { Chris@16: return tokenize_and_phrase_parse(first, last, lex, xpr, skipper Chris@16: , skip_flag::postskip, attr); Chris@16: } Chris@16: Chris@16: /////////////////////////////////////////////////////////////////////////// Chris@16: // Chris@16: // The tokenize() function is one of the main Spirit API functions. It Chris@16: // simplifies using a lexer to tokenize a given input sequence. It's main Chris@16: // purpose is to use the lexer to tokenize all the input. Chris@16: // Chris@16: // The second version below discards all generated tokens afterwards. Chris@16: // This is useful whenever all the needed functionality has been Chris@16: // implemented directly inside the lexer semantic actions, which are being Chris@16: // executed while the tokens are matched. Chris@16: // Chris@16: // The function takes a pair of iterators spanning the underlying input Chris@16: // stream to scan, the lexer object (built from the token definitions), Chris@16: // and a (optional) functor being called for each of the generated tokens. Chris@16: // Chris@16: // The function returns true if the scanning of the input succeeded (the Chris@16: // given input sequence has been successfully matched by the given token Chris@16: // definitions). Chris@16: // Chris@16: // first, last: The pair of iterators spanning the underlying input Chris@16: // sequence to parse. These iterators must at least Chris@16: // conform to the requirements of the std::intput_iterator Chris@16: // category. Chris@16: // On exit the iterator 'first' will be updated to the Chris@16: // position right after the last successfully matched Chris@16: // token. Chris@16: // lex: The lexer object (encoding the token definitions) to be Chris@16: // used to convert the input sequence into a sequence of Chris@16: // tokens. The LexerExpr type must conform to the Chris@16: // lexer interface described in the corresponding section Chris@16: // of the documentation. Chris@16: // f: A functor (callable object) taking a single argument of Chris@16: // the token type and returning a bool, indicating whether Chris@16: // the tokenization should be canceled. Chris@16: // initial_state: The name of the state the lexer should start matching. Chris@16: // The default value is zero, causing the lexer to start Chris@16: // in its 'INITIAL' state. Chris@16: // Chris@16: /////////////////////////////////////////////////////////////////////////// Chris@16: namespace detail Chris@16: { Chris@16: template Chris@16: bool tokenize_callback(Token const& t, F f) Chris@16: { Chris@16: return f(t); Chris@16: } Chris@16: Chris@16: template Chris@16: bool tokenize_callback(Token const& t, phoenix::actor const& f) Chris@16: { Chris@16: f(t); Chris@16: return true; Chris@16: } Chris@16: Chris@16: template Chris@16: bool tokenize_callback(Token const& t, void (*f)(Token const&)) Chris@16: { Chris@16: f(t); Chris@16: return true; Chris@16: } Chris@16: Chris@16: template Chris@16: bool tokenize_callback(Token const& t, bool (*f)(Token const&)) Chris@16: { Chris@16: return f(t); Chris@16: } Chris@16: } Chris@16: Chris@16: template Chris@16: inline bool Chris@16: tokenize(Iterator& first, Iterator last, Lexer const& lex, F f Chris@16: , typename Lexer::char_type const* initial_state = 0) Chris@16: { Chris@16: typedef typename Lexer::iterator_type iterator_type; Chris@16: Chris@16: iterator_type iter = lex.begin(first, last, initial_state); Chris@16: iterator_type end = lex.end(); Chris@16: for (/**/; iter != end && token_is_valid(*iter); ++iter) Chris@16: { Chris@16: if (!detail::tokenize_callback(*iter, f)) Chris@16: return false; Chris@16: } Chris@16: return (iter == end) ? true : false; Chris@16: } Chris@16: Chris@16: /////////////////////////////////////////////////////////////////////////// Chris@16: template Chris@16: inline bool Chris@16: tokenize(Iterator& first, Iterator last, Lexer const& lex Chris@16: , typename Lexer::char_type const* initial_state = 0) Chris@16: { Chris@16: typedef typename Lexer::iterator_type iterator_type; Chris@16: Chris@16: iterator_type iter = lex.begin(first, last, initial_state); Chris@16: iterator_type end = lex.end(); Chris@16: Chris@16: while (iter != end && token_is_valid(*iter)) Chris@16: ++iter; Chris@16: Chris@16: return (iter == end) ? true : false; Chris@16: } Chris@16: Chris@16: }}} Chris@16: Chris@16: #endif