From: g0dil Date: Mon, 23 Feb 2009 23:39:46 +0000 (+0000) Subject: Utils/Console: Add position information to Tokens X-Git-Url: http://g0dil.de/git?a=commitdiff_plain;h=2c11dd8e777c365c863af6b34a7db54bdd06b7f6;p=senf.git Utils/Console: Add position information to Tokens git-svn-id: https://svn.berlios.de/svnroot/repos/senf/trunk@1134 270642c3-0616-0410-b53a-bc976706d245 --- diff --git a/Utils/Console/Parse.cc b/Utils/Console/Parse.cc index 65dcaf3..28fa61a 100644 --- a/Utils/Console/Parse.cc +++ b/Utils/Console/Parse.cc @@ -112,6 +112,12 @@ namespace detail { /////////////////////////////////////////////////////////////////////////// // senf::console::Token +prefix_ senf::console::Token::Token(TokenType type, std::string token, + detail::FilePositionWithIndex const & pos) + : type_(type), token_ (token), line_ (pos.line), column_ (pos.column), index_ (pos.index) +{} + + prefix_ std::ostream & senf::console::operator<<(std::ostream & os, Token const & token) { static char const * tokenTypeName[] = { @@ -253,12 +259,43 @@ namespace { "path expected", "')' expected", "'\"' expected" }; - boost::spirit::file_position pos (err.where.get_position()); + senf::console::detail::FilePositionWithIndex pos (err.where.get_position()); throw senf::console::CommandParser::ParserErrorException(msg[err.descriptor]) << "\nat " << pos.file << ":" << pos.line << ":" << pos.column; } + } +namespace boost { +namespace spirit { + + template <> + struct position_policy + : public position_policy + { + typedef position_policy Base; + + void next_line(senf::console::detail::FilePositionWithIndex & pos) + { + Base::next_line(pos); + pos.index ++; + } + + void next_char(senf::console::detail::FilePositionWithIndex & pos) + { + Base::next_char(pos); + pos.index ++; + } + + void tabulation(senf::console::detail::FilePositionWithIndex & pos) + { + Base::tabulation(pos); + pos.index ++; + } + }; + +}} + prefix_ senf::console::CommandParser::CommandParser() : impl_ (new Impl()) {} @@ -273,7 +310,8 @@ template prefix_ Iterator senf::console::CommandParser::parseLoop(Iterator npb, Iterator npe, std::string const & source, Callback cb) { - typedef boost::spirit::position_iterator PositionIterator; + typedef boost::spirit::position_iterator< + Iterator, detail::FilePositionWithIndex> PositionIterator; PositionIterator b (npb, npe, source); PositionIterator e (npe, npe, source); ParseCommandInfo info; @@ -305,7 +343,7 @@ prefix_ Iterator senf::console::CommandParser::parseLoop(Iterator npb, Iterator cb(info); } catch (senf::ExceptionMixin & ex) { - boost::spirit::file_position pos (result.stop.get_position()); + detail::FilePositionWithIndex pos (result.stop.get_position()); ex << "\nat " << pos.file << ":" << pos.line << ":" << pos.column; throw; } @@ -329,7 +367,8 @@ prefix_ void senf::console::CommandParser::parseFile(std::string const & filenam prefix_ void senf::console::CommandParser::parseArguments(std::string const & arguments, ParseCommandInfo & info) { - typedef boost::spirit::position_iterator PositionIterator; + typedef boost::spirit::position_iterator< + std::string::const_iterator, detail::FilePositionWithIndex> PositionIterator; PositionIterator b (arguments.begin(), arguments.end(), std::string("")); PositionIterator e (arguments.end(), arguments.end(), std::string("")); detail::ParseDispatcher::BindInfo bind (impl().dispatcher, info); @@ -343,7 +382,7 @@ prefix_ void senf::console::CommandParser::parseArguments(std::string const & ar throwParserError(ex); } if (! result.full) { - boost::spirit::file_position pos (result.stop.get_position()); + detail::FilePositionWithIndex pos (result.stop.get_position()); throw ParserErrorException("argument expected") << "\nat " << pos.file << ":" << pos.line << ":" << pos.column; } @@ -352,7 +391,8 @@ prefix_ void senf::console::CommandParser::parseArguments(std::string const & ar prefix_ void senf::console::CommandParser::parsePath(std::string const & path, ParseCommandInfo & info) { - typedef boost::spirit::position_iterator PositionIterator; + typedef boost::spirit::position_iterator< + std::string::const_iterator, detail::FilePositionWithIndex> PositionIterator; PositionIterator b (path.begin(), path.end(), std::string("")); PositionIterator e (path.end(), path.end(), std::string("")); detail::ParseDispatcher::BindInfo bind (impl().dispatcher, info); @@ -366,7 +406,7 @@ prefix_ void senf::console::CommandParser::parsePath(std::string const & path, throwParserError(ex); } if (! result.full) { - boost::spirit::file_position pos (result.stop.get_position()); + detail::FilePositionWithIndex pos (result.stop.get_position()); throw ParserErrorException("path expected") << "\nat " << pos.file << ":" << pos.line << ":" << pos.column; } diff --git a/Utils/Console/Parse.cci b/Utils/Console/Parse.cci index 374d366..fcbb81f 100644 --- a/Utils/Console/Parse.cci +++ b/Utils/Console/Parse.cci @@ -47,6 +47,24 @@ prefix_ senf::console::Token::TokenType senf::console::Token::type() return type_; } +prefix_ unsigned senf::console::Token::line() + const +{ + return line_; +} + +prefix_ unsigned senf::console::Token::column() + const +{ + return column_; +} + +prefix_ unsigned senf::console::Token::index() + const +{ + return index_; +} + prefix_ bool senf::console::Token::is(unsigned tokens) const { @@ -66,11 +84,11 @@ prefix_ bool senf::console::Token::operator!=(Token const & other) } prefix_ senf::console::Token::Token() - : type_(None), token_() + : type_ (None), token_ () {} prefix_ senf::console::Token::Token(TokenType type, std::string token) - : type_(type), token_ (token) + : type_ (type), token_ (token), line_ (0), column_ (0), index_ (0) {} prefix_ senf::console::Token senf::console::NoneToken() diff --git a/Utils/Console/Parse.hh b/Utils/Console/Parse.hh index 7afbaa1..691d549 100644 --- a/Utils/Console/Parse.hh +++ b/Utils/Console/Parse.hh @@ -205,6 +205,8 @@ namespace senf { namespace console { + namespace detail { class FilePositionWithIndex; } + namespace detail { struct ParserAccess; } /** \brief Single argument token @@ -253,7 +255,10 @@ namespace console { }; Token(); ///< Create empty token - Token(TokenType type, std::string token); ///< Create token with given type and value + Token(TokenType type, std::string token); + ///< Create token with given type and value + Token(TokenType type, std::string token, detail::FilePositionWithIndex const & pos); + ///< Create token with given type and value std::string const & value() const; ///< String value of token @@ -261,6 +266,10 @@ namespace console { TokenType type() const; ///< Token type + unsigned line() const; ///< Line number of token in source + unsigned column() const; ///< Column number of token in source + unsigned index() const; ///< Index (char count) of token in source + bool is(unsigned tokens) const; ///< Check, whether tokens type matches \a tokens /**< \a tokens is a bit-mask of token types to check. */ @@ -272,6 +281,9 @@ namespace console { private: TokenType type_; std::string token_; + unsigned line_; + unsigned column_; + unsigned index_; }; std::ostream & operator<<(std::ostream & os, Token const & token); diff --git a/Utils/Console/Parse.ih b/Utils/Console/Parse.ih index 4f0569f..d99e168 100644 --- a/Utils/Console/Parse.ih +++ b/Utils/Console/Parse.ih @@ -43,6 +43,30 @@ namespace detail { #ifndef DOXYGEN + struct FilePositionWithIndex + : public boost::spirit::file_position + { + int index; + + FilePositionWithIndex(std::string const & file_ = std::string(), + int line_ = 1, int column_ = 1, int index_ = 0) + : boost::spirit::file_position (file_, line_, column_), index (index_) + {} + + bool operator==(const FilePositionWithIndex & fp) const + { + return boost::spirit::file_position::operator==(fp) && index == fp.index; + } + }; + + struct PositionOf { + template struct result { typedef FilePositionWithIndex type; }; + template FilePositionWithIndex operator()(A1 & a1) { return a1.get_position(); } + FilePositionWithIndex operator()(char const * a1) { return FilePositionWithIndex(); } + }; + + ::phoenix::function const positionOf; + /////////////////////////////////////////////////////////////////////////// // Grammar @@ -64,6 +88,7 @@ namespace detail { std::vector path; char ch; Token token; + FilePositionWithIndex pos; }; Context & context; @@ -122,11 +147,12 @@ namespace detail { using namespace senf::phoenix; typedef ParseDispatcher PD; - actor< variable< char > > ch_ (self.context.ch); - actor< variable< std::string > > str_ (self.context.str); - actor< variable< std::vector > > path_ (self.context.path); - actor< variable< Token > > token_ (self.context.token); - actor< variable< ParseDispatcher > > d_ (self.dispatcher); + actor< variable< char > > ch_ (self.context.ch); + actor< variable< std::string > > str_ (self.context.str); + actor< variable< std::vector > > path_ (self.context.path); + actor< variable< Token > > token_ (self.context.token); + actor< variable< FilePositionWithIndex > > pos_ (self.context.pos); + actor< variable< ParseDispatcher > > d_ (self.dispatcher); assertion end_of_statement_expected (EndOfStatementExpected); assertion path_expected (PathExpected); @@ -224,7 +250,7 @@ namespace detail { ; string // Returns value in context.token - = eps_p [ clear(str_) ] + = eps_p [ pos_ = positionOf(arg1) ][ clear(str_) ] >> lexeme_d [ ch_p('"') @@ -234,17 +260,19 @@ namespace detail { ) >> quote_expected(ch_p('"')) [ token_ = construct_(Token::BasicString, - str_) ] + str_, + pos_) ] ] ; hexstring // Returns value in context.token - = eps_p [ clear(str_) ] + = eps_p [ pos_ = positionOf(arg1) ][ clear(str_) ] >> "x\"" >> * ( hexbyte - ch_p('"') ) >> quote_expected(ch_p('"')) [ token_ = construct_(Token::HexString, - str_) ] + str_, + pos_) ] ; opt_path @@ -270,15 +298,19 @@ namespace detail { ; balanced_tokens - = ch_p('(') [ token_ = construct_( + = eps_p [ pos_ = positionOf(arg1) ] + >> ch_p('(') [ token_ = construct_( Token::ArgumentGroupOpen, - "(") ] + "(", + pos_) ] [ bind(&PD::pushToken)(d_, token_) ] >> * token + >> eps_p [ pos_ = positionOf(arg1) ] >> closing_paren_expected(ch_p(')')) [ token_ = construct_( Token::ArgumentGroupClose, - ")") ] + ")", + pos_) ] [ bind(&PD::pushToken)(d_, token_) ] ; @@ -289,31 +321,37 @@ namespace detail { ; punctuation // Returns value in context.str - = ch_p('/') [ token_ = construct_( - Token::PathSeparator, - "/") ] - | ch_p('{') [ token_ = construct_( - Token::DirectoryGroupOpen, - "{") ] - | ch_p('}') [ token_ = construct_( - Token::DirectoryGroupClose, - "}") ] - | ch_p(';') [ token_ = construct_( - Token::CommandTerminator, - ";") ] - | self.punctuation_p [ token_ = construct_( - Token::OtherPunctuation, - construct_(1u, arg1)) ] + = eps_p [ pos_ = positionOf(arg1) ] + >> ( + ch_p('/') [ token_ = construct_( + Token::PathSeparator, + "/") ] + | ch_p('{') [ token_ = construct_( + Token::DirectoryGroupOpen, + "{") ] + | ch_p('}') [ token_ = construct_( + Token::DirectoryGroupClose, + "}") ] + | ch_p(';') [ token_ = construct_( + Token::CommandTerminator, + ";") ] + | self.punctuation_p [ token_ = construct_( + Token::OtherPunctuation, + construct_(1u, arg1), + pos_) ] + ) ; word // Returns value in context.token - = lexeme_d + = eps_p [ pos_ = positionOf(arg1) ] + >> lexeme_d [ (+ self.word_p) [ str_ = construct_(arg1, arg2) ] ] >> eps_p [ token_ = construct_( Token::Word, - str_) ] + str_, + pos_) ] ; hexbyte diff --git a/Utils/Console/Parse.test.cc b/Utils/Console/Parse.test.cc index 57fb988..962e2e2 100644 --- a/Utils/Console/Parse.test.cc +++ b/Utils/Console/Parse.test.cc @@ -85,7 +85,7 @@ BOOST_AUTO_UNIT_TEST(commandGrammar) { static char text[] = "# Comment\n" - "doo/bii/doo arg/two/three" + "doo / bii / doo arg" " flab::blub" " 123.434>a" " (a,b;c (huhu/{haha}))" @@ -99,7 +99,7 @@ BOOST_AUTO_UNIT_TEST(commandGrammar) grammar.use_parser() ) . full ); BOOST_CHECK_EQUAL( ss.str(), "beginCommand( Word('doo')/Word('bii')/Word('doo') )\n" - "pushToken( Word('arg/two/three') )\n" + "pushToken( Word('arg') )\n" "pushToken( Word('flab::blub') )\n" "pushToken( Word('123.434>a') )\n" "pushToken( ArgumentGroupOpen('(') )\n" @@ -214,6 +214,7 @@ BOOST_AUTO_UNIT_TEST(commandParser) BOOST_CHECK_EQUAL_COLLECTIONS( info.commandPath().begin(), info.commandPath().end(), path, path + sizeof(path)/sizeof(path[0]) ); + BOOST_CHECK_EQUAL( boost::next(info.commandPath().begin())->index(), 16u ); BOOST_CHECK_EQUAL( unsigned(info.tokens().size()), 15u ); char const * tokens[] = { "arg", @@ -243,11 +244,15 @@ BOOST_AUTO_UNIT_TEST(commandParser) BOOST_REQUIRE_EQUAL( args->size(), 8u ); for (unsigned i (0); i<8; ++i) BOOST_CHECK_EQUAL( args->begin()[i].value(), tokens[4+i] ); + BOOST_CHECK_EQUAL( info.tokens().begin()[3].index(), 96u ); + BOOST_CHECK_EQUAL( info.tokens().begin()[5].index(), 98u ); + BOOST_CHECK_EQUAL( info.tokens().begin()[12].index(), 109u ); ++ args; BOOST_REQUIRE( args != info.arguments().end() ); BOOST_REQUIRE_EQUAL( args->size(), 1u ); BOOST_CHECK_EQUAL( args->begin()->value(), tokens[13] ); + BOOST_CHECK_EQUAL( args->begin()->index(), 126u ); ++ args; BOOST_REQUIRE( args != info.arguments().end() );