Console: Parser cleanup
g0dil [Tue, 20 May 2008 08:39:27 +0000 (08:39 +0000)]
git-svn-id: https://svn.berlios.de/svnroot/repos/senf/trunk@848 270642c3-0616-0410-b53a-bc976706d245

Console/Executor.cc
Console/Executor.hh
Console/Node.hh
Console/Parse.cc
Console/Parse.cci
Console/Parse.hh
Console/Parse.ih
Console/Parse.test.cc

index 3a8c228..fc0327d 100644 (file)
@@ -36,7 +36,7 @@ namespace {
 
     struct TraverseTokens {
         typedef std::string const & result_type;
-        result_type operator()(senf::console::ArgumentToken const & token) const {
+        result_type operator()(senf::console::Token const & token) const {
             return token.value();
         }
     };
@@ -59,7 +59,7 @@ prefix_ void senf::console::Executor::execute(std::ostream & output,
         case ParseCommandInfo::NoBuiltin : {
             if (skipping_)
                 break;
-            GenericNode & node ( traverseCommand(command.commandPath()) );
+            GenericNode & node ( traverseNode(command.commandPath()) );
             DirectoryNode * dir ( dynamic_cast<DirectoryNode*>(&node) );
             if ( dir ) {
                 if (autocd_ && command.tokens().empty()) {
@@ -183,20 +183,6 @@ senf::console::Executor::traverseNode(ParseCommandInfo::TokensRange const & path
     }
 }
 
-prefix_ senf::console::GenericNode &
-senf::console::Executor::traverseCommand(ParseCommandInfo::CommandPathRange const & path)
-{
-    try {
-        return traverse(cwd(), path);
-    }
-    catch (std::bad_cast &) {
-        throw InvalidPathException();
-    }
-    catch (UnknownNodeNameException &) {
-        throw InvalidPathException();
-    }
-}
-
 prefix_ senf::console::DirectoryNode &
 senf::console::Executor::traverseDirectory(ParseCommandInfo::TokensRange const & path)
 {
index 2c21ee1..5f1cc46 100644 (file)
@@ -128,7 +128,6 @@ namespace console {
 
     private:
         GenericNode & traverseNode(ParseCommandInfo::TokensRange const & path);
-        GenericNode & traverseCommand(ParseCommandInfo::CommandPathRange const & path); 
         DirectoryNode & traverseDirectory(ParseCommandInfo::TokensRange const & path);
 
         template <class ForwardRange>
index 0bfc594..ca57da1 100644 (file)
@@ -568,7 +568,7 @@ namespace console {
                                              \param[in] output stream where result messages may be
                                                  written to
                                              \param[in] arguments command arguments. This is a
-                                                 range of ranges of ArgumentToken instances. */
+                                                 range of ranges of Token instances. */
 
         void operator()(std::ostream & output, ParseCommandInfo const & command) const;
                                         ///< Execute the command
@@ -576,7 +576,7 @@ namespace console {
                                              \param[in] output stream where result messages may be
                                                  written to
                                              \param[in] arguments command arguments. This is a
-                                                 range of ranges of ArgumentToken instances. */
+                                                 range of ranges of Token instances. */
 
         ptr thisptr();
         cptr thisptr() const;
@@ -592,7 +592,7 @@ namespace console {
                                         /**< \param[in] output stream where result messages may be
                                                  written to
                                              \param[in] arguments command arguments. This is a
-                                                 range of ranges of ArgumentToken instances. */
+                                                 range of ranges of Token instances. */
 
     private:
     };
index 52858e9..57aba80 100644 (file)
@@ -30,7 +30,6 @@
 #include <cerrno>
 #include <boost/iterator/transform_iterator.hpp>
 #include <boost/spirit/iterator/file_iterator.hpp>
-#include "../Utils/String.hh"
 #include "../Utils/Exception.hh"
 
 //#include "Parse.mpp"
@@ -51,14 +50,11 @@ namespace detail {
         static void setBuiltin(ParseCommandInfo & info, ParseCommandInfo::BuiltinCommand builtin)
             { info.setBuiltin(builtin); }
 
-        static void setCommand(ParseCommandInfo & info, std::vector<std::string> & commandPath)
+        static void setCommand(ParseCommandInfo & info, std::vector<Token> & commandPath)
             { info.setCommand(commandPath); }
 
-        static void addToken(ParseCommandInfo & info, ArgumentToken const & token)
+        static void addToken(ParseCommandInfo & info, Token const & token)
             { info.addToken(token); }
-
-        static ArgumentToken makeToken(ArgumentToken::TokenType type, std::string const & token)
-            { return ArgumentToken(type, token); }
     };
 
     struct ParseDispatcher
@@ -74,53 +70,29 @@ namespace detail {
             ParseDispatcher & dispatcher;
         };
 
-        void beginCommand(std::vector<std::string> & command)
+        void beginCommand(std::vector<Token> & command)
             { ParserAccess::init(info_);
               ParserAccess::setCommand(info_, command); }
 
         void endCommand()
             { cb_(info_); }
 
-        void pushArgument(ArgumentToken::TokenType type, std::string const & argument)
-            { ParserAccess::addToken(info_, ParserAccess::makeToken(type, argument)); }
-
-        void openGroup()
-            { pushPunctuation("("); }
-
-        void closeGroup()
-            { pushPunctuation(")"); }
-
-        void pushPunctuation(std::string const & token)
-            {
-                ArgumentToken::TokenType type;
-                switch (token[0]) {
-                case '/' : type = ArgumentToken::PathSeparator; break;
-                case '(' : type = ArgumentToken::ArgumentGroupOpen; break;
-                case ')' : type = ArgumentToken::ArgumentGroupClose; break;
-                case '{' : type = ArgumentToken::DirectoryGroupOpen; break;
-                case '}' : type = ArgumentToken::DirectoryGroupClose; break;
-                case ';' : type = ArgumentToken::CommandTerminator; break;
-                default :  type = ArgumentToken::OtherPunctuation; break;
-                }
-                ParserAccess::addToken(info_, ParserAccess::makeToken(type, token));
-            }
+        void pushToken(Token const & token)
+            { ParserAccess::addToken(info_, token); }
 
-        void pushWord(ArgumentToken::TokenType type, std::string const & token)
-            { ParserAccess::addToken(info_, ParserAccess::makeToken(type, token)); }
-
-        void builtin_cd(std::vector<std::string> & path)
+        void builtin_cd(std::vector<Token> & path)
             { ParserAccess::init(info_);
               ParserAccess::setBuiltin(info_, ParseCommandInfo::BuiltinCD);
               setBuiltinPathArg(path);
               cb_(info_); }
 
-        void builtin_ls(std::vector<std::string> & path)
+        void builtin_ls(std::vector<Token> & path)
             { ParserAccess::init(info_);
               ParserAccess::setBuiltin(info_, ParseCommandInfo::BuiltinLS);
               setBuiltinPathArg(path);
               cb_(info_); }
 
-        void pushDirectory(std::vector<std::string> & path)
+        void pushDirectory(std::vector<Token> & path)
             { ParserAccess::init(info_);
               ParserAccess::setBuiltin(info_, ParseCommandInfo::BuiltinPUSHD);
               setBuiltinPathArg(path);
@@ -136,20 +108,19 @@ namespace detail {
               ParserAccess::setBuiltin(info_, ParseCommandInfo::BuiltinEXIT);
               cb_(info_); }
 
-        void builtin_help(std::vector<std::string> & path)
+        void builtin_help(std::vector<Token> & path)
             { ParserAccess::init(info_);
               ParserAccess::setBuiltin(info_, ParseCommandInfo::BuiltinHELP);
               setBuiltinPathArg(path);
               cb_(info_); }
 
-        void setBuiltinPathArg(std::vector<std::string> & path)
+        void setBuiltinPathArg(std::vector<Token> & path)
             {
-                pushPunctuation("(");
-                for (std::vector<std::string>::const_iterator i (path.begin());
+                pushToken(Token(Token::ArgumentGroupOpen, "("));
+                for (std::vector<Token>::const_iterator i (path.begin());
                      i != path.end(); ++i)
-                    ParserAccess::addToken(info_, 
-                                           ParserAccess::makeToken(ArgumentToken::Word, *i));
-                pushPunctuation(")");
+                    pushToken(*i);
+                pushToken(Token(Token::ArgumentGroupClose, ")"));
             }
     };
 
@@ -158,13 +129,56 @@ namespace detail {
 }}}
 
 ///////////////////////////////////////////////////////////////////////////
+// senf::console::Token
+
+prefix_ std::ostream & senf::console::operator<<(std::ostream & os, Token const & token)
+{
+    static char const * tokenTypeName[] = {
+        "None",
+        "PathSeparator",
+        "ArgumentGroupOpen",
+        "ArgumentGroupClose",
+        "DirectoryGroupOpen",
+        "DirectoryGroupClose",
+        "CommandTerminator",
+        "OtherPunctuation",
+        "BasicString",
+        "HexString",
+        "Word" };
+    // The real table is:
+    //     static const int bitPosition[32] = {
+    //         0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, 
+    //         31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9 };
+    // However, we have replaced all values > sizeof(tokenTypeName) with 0
+    static const int bitPosition[32] = {
+        0, 1, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4, 8, 
+        0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 6, 0, 5, 10, 9 };
+    os << tokenTypeName[ token.type() 
+                         ? bitPosition[((token.type() & -token.type()) * 0x077CB531UL) >> 27]+1
+                         : 0 ]
+       << "('"
+       << token.value()
+       << "')";
+    return os;
+}
+
+///////////////////////////////////////////////////////////////////////////
 // senf::console::ParseCommandInfo
 
 prefix_ std::ostream & senf::console::operator<<(std::ostream & stream,
                                                  ParseCommandInfo const & info)
 {
-    if (info.builtin() == ParseCommandInfo::NoBuiltin) 
-        stream << senf::stringJoin(info.commandPath(), "/");
+    if (info.builtin() == ParseCommandInfo::NoBuiltin) {
+        ParseCommandInfo::TokensRange::const_iterator i (info.commandPath().begin());
+        ParseCommandInfo::TokensRange::const_iterator const i_end (info.commandPath().end());
+        if (i != i_end) {
+            for (;;) {
+                stream << i->value();
+                if ( ++i != i_end ) stream << "/";
+                else                break;
+            }
+        }
+    }
     else {
         char const * builtins[] = { "", "cd", "ls", "pushd", "popd", "exit", "help" };
         stream << "builtin-" << builtins[info.builtin()];
@@ -193,13 +207,13 @@ prefix_ std::ostream & senf::console::operator<<(std::ostream & stream,
 prefix_ void senf::console::ParseCommandInfo::ArgumentIterator::setRange()
     const
 {
-    if (b_->is(ArgumentToken::ArgumentGroupOpen)) {
+    if (b_->is(Token::ArgumentGroupOpen)) {
         unsigned level (0);
         e_ = b_;
         for (;;) {
-            if (e_->is(ArgumentToken::ArgumentGroupOpen))
+            if (e_->is(Token::ArgumentGroupOpen))
                 ++ level;
-            else if (e_->is(ArgumentToken::ArgumentGroupClose)) {
+            else if (e_->is(Token::ArgumentGroupClose)) {
                 -- level;
                 if (level == 0)
                     break;
@@ -214,12 +228,12 @@ prefix_ void senf::console::ParseCommandInfo::ArgumentIterator::decrement()
 {
     e_ = b_;
     --b_;
-    if (b_->is(ArgumentToken::ArgumentGroupClose)) {
+    if (b_->is(Token::ArgumentGroupClose)) {
         unsigned level (0);
         for (;;) {
-            if (b_->is(ArgumentToken::ArgumentGroupClose))
+            if (b_->is(Token::ArgumentGroupClose))
                 ++ level;
-            else if (b_->is(ArgumentToken::ArgumentGroupOpen)) {
+            else if (b_->is(Token::ArgumentGroupOpen)) {
                 -- level;
                 if (level == 0)
                     break;
index 011ff9d..74b54f3 100644 (file)
 ///////////////////////////////cci.p///////////////////////////////////////
 
 ///////////////////////////////////////////////////////////////////////////
-// senf::console::ArgumentToken
+// senf::console::Token
 
-prefix_ std::string const & senf::console::ArgumentToken::value()
+prefix_ std::string const & senf::console::Token::value()
     const
 {
     return token_;
 }
 
-prefix_ senf::console::ArgumentToken::TokenType senf::console::ArgumentToken::type()
+prefix_ senf::console::Token::TokenType senf::console::Token::type()
     const
 {
     return type_;
 }
 
-prefix_ bool senf::console::ArgumentToken::is(unsigned tokens)
+prefix_ bool senf::console::Token::is(unsigned tokens)
     const
 {
     return tokens & type_;
 }
 
-prefix_ senf::console::ArgumentToken::ArgumentToken(TokenType type, std::string token)
+prefix_ bool senf::console::Token::operator==(Token const & other)
+    const
+{
+    return type() == other.type() && value() == other.value();
+}
+
+prefix_ bool senf::console::Token::operator!=(Token const & other)
+    const
+{
+    return ! operator==(other);
+}
+
+prefix_ senf::console::Token::Token()
+    : type_(None), token_()
+{}
+
+prefix_ senf::console::Token::Token(TokenType type, std::string token)
     : type_(type), token_ (token)
 {}
 
@@ -67,7 +83,7 @@ senf::console::ParseCommandInfo::builtin()
     return builtin_;
 }
 
-prefix_ senf::console::ParseCommandInfo::CommandPathRange
+prefix_ senf::console::ParseCommandInfo::TokensRange
 senf::console::ParseCommandInfo::commandPath()
     const
 {
@@ -104,13 +120,13 @@ prefix_ void senf::console::ParseCommandInfo::setBuiltin(BuiltinCommand builtin)
 }
 
 prefix_ void
-senf::console::ParseCommandInfo::setCommand(std::vector<std::string> & commandPath)
+senf::console::ParseCommandInfo::setCommand(std::vector<Token> & commandPath)
 {
     commandPath_.clear();
     commandPath_.swap(commandPath);
 }
 
-prefix_ void senf::console::ParseCommandInfo::addToken(ArgumentToken const & token)
+prefix_ void senf::console::ParseCommandInfo::addToken(Token const & token)
 {
     tokens_.push_back(token);
 }
@@ -131,7 +147,7 @@ senf::console::ParseCommandInfo::ArgumentIterator::dereference()
     const
 {
     if (b_ == e_) setRange();
-    return b_->is(ArgumentToken::ArgumentGroupOpen) 
+    return b_->is(Token::ArgumentGroupOpen) 
         ? boost::make_iterator_range(boost::next(b_), boost::prior(e_))
         : boost::make_iterator_range(b_, e_);
 }
index 892c522..a40bd74 100644 (file)
@@ -209,20 +209,21 @@ namespace console {
     /** \brief Single argument token
 
         All command arguments are split into tokens by the parser. Each token is returned as an
-        ArgumentToken instance. 
+        Token instance. 
 
         \ingroup console_parser
       */
-    class ArgumentToken
+    class Token
     {
     public:
         enum TokenType { 
-            PathSeparator       = 0x0001,
-            ArgumentGroupOpen   = 0x0002,
-            ArgumentGroupClose  = 0x0004,
-            DirectoryGroupOpen  = 0x0008,
-            DirectoryGroupClose = 0x0010,
-            CommandTerminator   = 0x0020,
+            None                = 0,
+            PathSeparator       = 0x0001, // '/'
+            ArgumentGroupOpen   = 0x0002, // '('
+            ArgumentGroupClose  = 0x0004, // ')'
+            DirectoryGroupOpen  = 0x0008, // '{'
+            DirectoryGroupClose = 0x0010, // '}'
+            CommandTerminator   = 0x0020, // ';'
             OtherPunctuation    = 0x0040,
             BasicString         = 0x0080,
             HexString           = 0x0100,
@@ -250,6 +251,9 @@ namespace console {
                                 | HexString
         };
         
+        Token();
+        Token(TokenType type, std::string token);
+
         std::string const & value() const; ///< String value of token
                                         /**< This value is properly unquoted */
 
@@ -258,17 +262,18 @@ namespace console {
         bool is(unsigned tokens) const; ///< Check, whether tokens type matches \a tokens
                                         /**< \a tokens is a bit-mask of token types to check. */
 
+        bool operator==(Token const & other) const;
+        bool operator!=(Token const & other) const;
+
     protected:
 
     private:
-        ArgumentToken(TokenType type, std::string token);
-
         TokenType type_;
         std::string token_;
-
-        friend class detail::ParserAccess;
     };
 
+    std::ostream & operator<<(std::ostream & os, Token const & token);
+
     /** \brief Single parsed console command
 
         Every command parsed is returned in a ParseCommandInfo instance. This information is purely
@@ -278,13 +283,13 @@ namespace console {
         \li the type of command: built-in or normal command represented by a possibly relative path
             into the command tree.
         \li the command
-        \li the arguments. Every argument consists of a range of ArgumentToken instances.
+        \li the arguments. Every argument consists of a range of Token instances.
 
         \ingroup console_parser
       */
     class ParseCommandInfo
     {
-        typedef std::vector<ArgumentToken> Tokens;
+        typedef std::vector<Token> Tokens;
         typedef std::vector<std::string> CommandPath;
 
     public:
@@ -310,7 +315,7 @@ namespace console {
         BuiltinCommand builtin() const; ///< Command type
                                         /**< \returns \c NoBuiltin, if the command is an ordinary
                                              command, otherwise the id of the built-in command */
-        CommandPathRange commandPath() const; ///< Command path
+        TokensRange commandPath() const; ///< Command path
                                         /**< This is the path to the command if it is not a built-in
                                              command. Every element of the returned range
                                              constitutes one path element. If the first element is
@@ -328,12 +333,12 @@ namespace console {
     private:
         void init();
         void setBuiltin(BuiltinCommand builtin);
-        void setCommand(std::vector<std::string> & commandPath);
-        void addToken(ArgumentToken const & token);
+        void setCommand(std::vector<Token> & commandPath);
+        void addToken(Token const & token);
 
         struct MakeRange;
 
-        std::vector<std::string> commandPath_;
+        std::vector<Token> commandPath_;
         BuiltinCommand builtin_;
         Tokens tokens_;
 
index 5129197..79b30b6 100644 (file)
@@ -82,21 +82,18 @@ namespace detail {
         ///////////////////////////////////////////////////////////////////////////
         // The parse context (variables needed while parsing)
 
-        typedef ArgumentToken::TokenType TokenType;
+        typedef Token::TokenType TokenType;
 
         struct Context {
             std::string str;
-            std::vector<std::string> path;
+            std::vector<Token> path;
             char ch;
-            TokenType type;
+            Token token;
             
             // OUCH ... This is sooooo stupid .. push_back_a and assign_a take their 
             // arguments by const-reference and STORE the REFERENCE ... they do NOT accept
             // literal values !!!!!! 
-            static const TokenType BasicString;
-            static const TokenType HexString;
-            static const TokenType Word;
-            static const std::string EmptyString;
+            static const Token EmptyToken;
         };
 
         Context & context;
@@ -133,6 +130,42 @@ namespace detail {
         Dispatch_actor dispatch(Callback cb, Arg1 const & arg1, Arg2 const & arg2) const
             { return Dispatch_actor(boost::bind(cb, boost::ref(dispatcher), arg1, arg2)); }
 
+        struct TokenSetter_actor
+        {
+            TokenSetter_actor(Context & c, TokenType t) : c_ (c), t_ (t) {}
+            
+            void operator()(std::string const & value) const
+                { c_.token = Token(t_, value); }
+            void operator()(char value) const
+                { c_.token = Token(t_, std::string(1,value)); }
+            template <class Iterator> void operator()(Iterator const & f, Iterator const & l) const
+                { c_.token = Token(t_, std::string(f,l)); }
+
+            Context & c_;
+            TokenType t_;
+        };
+
+        struct TokenSetter_value_actor
+            : public TokenSetter_actor
+        {
+            TokenSetter_value_actor(Context & c, TokenType t, std::string & v) 
+                : TokenSetter_actor(c,t), v_ (v) {}
+
+            template <class Value> void operator()(Value const &) const
+                { TokenSetter_actor::operator()(v_); }
+
+            template <class Iterator> void operator()(Iterator const &, Iterator const &) const
+                { TokenSetter_actor::operator()(v_); }
+
+            std::string & v_;
+        };
+
+        TokenSetter_actor set_token_a(TokenType t) const
+            { return TokenSetter_actor(context, t); }
+
+        TokenSetter_value_actor set_token_a(TokenType t, std::string & arg) const
+            { return TokenSetter_value_actor(context, t, arg); }
+
         ///////////////////////////////////////////////////////////////////////////
 
         CommandGrammar(ParseDispatcher & d, Context & c) 
@@ -154,9 +187,9 @@ namespace detail {
                 // Characters with a special meaning within the parser
                 special_p ("/(){};"),
 
-                // Characters which are returned as punctuation tokens
-                // (only allowed within '()')
-                punctuation_p (",=/{};"),
+                // Additional characters which are returned as punctuation tokens
+                // (only allowed within '()').
+                punctuation_p (",="),
 
                 // Whitespace characters
                 space_p (" \t\n\r"),
@@ -176,6 +209,7 @@ namespace detail {
             {
                 using namespace boost::spirit;
                 typedef ParseDispatcher PD;
+                typedef Token AT;
 
                 ///////////////////////////////////////////////////////////////////
                 // Spirit grammar
@@ -256,28 +290,19 @@ namespace detail {
                     ;
 
                 argument
-                    =    simple_argument          [ self.dispatch(&PD::pushArgument, 
-                                                                  boost::ref(self.context.type),
-                                                                  boost::ref(self.context.str)) ]
-                    |    complex_argument
+                    =    simple_argument          [ self.dispatch(&PD::pushToken, 
+                                                                  boost::ref(self.context.token)) ]
+                    |    balanced_tokens
                     ;
                 
-                simple_argument         // All these return their value in context.str
+                simple_argument         // All these return their value in context.token
                     =    string
                     |    hexstring
                     |    word
                     ;
                 
-                complex_argument        // Argument consists of multiple tokens
-                    =    ch_p('(')                [ self.dispatch(&PD::openGroup) ]
-                      >> * token
-                      >> ch_p(')')                [ self.dispatch(&PD::closeGroup) ]
-                    ;
-
-                string                  // Returns value in context.str
+                string                  // Returns value in context.token
                     =    eps_p                    [ clear_a(self.context.str) ]
-                      >> eps_p                    [ assign_a(self.context.type, 
-                                                             self.context.BasicString) ]
                       >> lexeme_d
                          [
                              ch_p('"')
@@ -286,15 +311,16 @@ namespace detail {
                                  )                [ append_a(self.context.str,
                                                              self.context.ch) ] 
                                )
-                          >> ch_p('"')
+                          >> ch_p('"')            [ self.set_token_a(AT::BasicString, 
+                                                                     self.context.str) ]
                          ]
                     ;
 
-                hexstring               // Returns value in context.str
+                hexstring               // Returns value in context.token
                     =    eps_p                    [ clear_a(self.context.str) ]
-                      >> eps_p                    [ assign_a(self.context.type, 
-                                                             self.context.HexString) ]
                       >> confix_p( "x\"", * hexbyte, '"' )
+                                                  [ self.set_token_a(AT::HexString,
+                                                                     self.context.str) ]
                     ;
 
                 path                    // Returns value in context.path
@@ -303,45 +329,54 @@ namespace detail {
                     ;
 
                 relpath
-                    =    (   word                 [ push_back_a(self.context.path) ]
+                    =    (   word                 [ push_back_a(self.context.path,
+                                                                self.context.token) ]
                            % ch_p('/') )
                       >> ( ! ch_p('/')            [ push_back_a(self.context.path,
-                                                                self.context.EmptyString) ] )
+                                                                self.context.EmptyToken) ] )
                     ;
 
                 abspath
                     =    ch_p('/')                [ push_back_a(self.context.path,
-                                                                self.context.EmptyString) ]
+                                                                self.context.EmptyToken) ]
                       >> ( relpath
-                         | eps_p                  [ push_back_a(self.context.path, "") ] )
+                         | eps_p                  [ push_back_a(self.context.path,
+                                                                self.context.EmptyToken) ] )
                     ;
 
                 balanced_tokens 
-                    =    ch_p('(')                [ self.dispatch(&PD::pushPunctuation, "(") ]
+                    =    ch_p('(')                [ self.set_token_a(AT::ArgumentGroupOpen) ]
+                                                  [ self.dispatch(&PD::pushToken, 
+                                                                  boost::ref(self.context.token)) ]
                       >> * token
-                      >> ch_p(')')                [ self.dispatch(&PD::pushPunctuation, ")") ]
+                      >> ch_p(')')                [ self.set_token_a(AT::ArgumentGroupClose) ]
+                                                  [ self.dispatch(&PD::pushToken, 
+                                                                  boost::ref(self.context.token)) ]
                     ;
 
                 token
-                    =    simple_argument          [ self.dispatch(&PD::pushWord,
-                                                                 boost::ref(self.context.type), 
-                                                                  boost::ref(self.context.str)) ]
-                    |    punctuation              [ self.dispatch(&PD::pushPunctuation,
-                                                                  boost::ref(self.context.str)) ]
+                    =    simple_argument          [ self.dispatch(&PD::pushToken,
+                                                                 boost::ref(self.context.token)) ]
+                    |    punctuation              [ self.dispatch(&PD::pushToken,
+                                                                  boost::ref(self.context.token)) ]
                     |    balanced_tokens
                     ;
 
                 punctuation             // Returns value in context.str
-                    =    punctuation_p            [ assign_a(self.context.str) ]
+                    =    ch_p('/')                [ self.set_token_a(AT::PathSeparator) ]
+                    |    ch_p('{')                [ self.set_token_a(AT::DirectoryGroupOpen) ]
+                    |    ch_p('}')                [ self.set_token_a(AT::DirectoryGroupClose) ]
+                    |    ch_p(';')                [ self.set_token_a(AT::CommandTerminator) ]
+                    |    punctuation_p            [ self.set_token_a(AT::OtherPunctuation) ]
                     ;
 
-                word                    // Returns value in context.str
+                word                    // Returns value in context.token
                     =    lexeme_d
                          [
-                             eps_p                [ assign_a(self.context.type,
-                                                            self.context.Word) ]
+                             eps_p
                           >> (+ word_p)           [ assign_a(self.context.str) ]
                          ]
+                      >> eps_p                    [ self.set_token_a(AT::Word, self.context.str) ]
                     ;
 
                 hexbyte
@@ -383,19 +418,7 @@ namespace detail {
     };
 
     template <class ParseDispatcher>
-    ArgumentToken::TokenType const CommandGrammar<ParseDispatcher>::Context::BasicString (
-        ArgumentToken::BasicString);
-
-    template <class ParseDispatcher>
-    ArgumentToken::TokenType const CommandGrammar<ParseDispatcher>::Context::HexString(
-        ArgumentToken::HexString);
-
-    template <class ParseDispatcher>
-    ArgumentToken::TokenType const CommandGrammar<ParseDispatcher>::Context::Word(
-        ArgumentToken::Word);
-
-    template <class ParseDispatcher>
-    std::string const CommandGrammar<ParseDispatcher>::Context::EmptyString;
+    Token const CommandGrammar<ParseDispatcher>::Context::EmptyToken;
 
 #endif
 
index 46ad986..c58a6cc 100644 (file)
@@ -49,36 +49,26 @@ namespace
 
         std::ostream & os_;
 
-        void pushDirectory(std::vector<std::string> const & path)
-            { os_ << "pushDirectory( " << senf::stringJoin(path,"/") << " )\n"; }
+        void pushDirectory(std::vector<senf::console::Token> const & path)
+            { os_ << "pushDirectory( " << senf::stringJoin(path, "/") << " )\n"; }
         void popDirectory()
             { os_ << "popDirectory()\n"; }
 
-        void beginCommand(std::vector<std::string> const & command) 
+        void beginCommand(std::vector<senf::console::Token> const & command) 
             { os_ << "beginCommand( " << senf::stringJoin(command, "/") << " )\n"; }
         void endCommand() 
             { os_ << "endCommand()\n"; }
         
-        void pushArgument(senf::console::ArgumentToken::TokenType type, 
-                          std::string const & argument)
-            { os_ << "pushArgument( " << argument << " )\n"; }
-        void openGroup()
-            { os_ << "openGroup()\n"; }
-        void closeGroup()
-            { os_ << "closeGroup()\n"; }
-        void pushPunctuation(std::string const & token)
-            { os_ << "pushPunctuation( " << token << " )\n"; }
-        void pushWord(senf::console::ArgumentToken::TokenType type, 
-                      std::string const & token)
-            { os_ << "pushWord( " << token << " )\n"; }
-
-        void builtin_cd(std::vector<std::string> const & path)
+        void pushToken(senf::console::Token token)
+            { os_ << "pushToken( " << token << " )\n"; }
+
+        void builtin_cd(std::vector<senf::console::Token> const & path)
             { os_ << "builtin_cd( " << senf::stringJoin(path, "/") << " )\n"; }
-        void builtin_ls(std::vector<std::string> const & path)
+        void builtin_ls(std::vector<senf::console::Token> const & path)
             { os_ << "builtin_ls( " << senf::stringJoin(path, "/") << " )\n"; }
         void builtin_exit()
             { os_ << "builtin_exit()\n"; }
-        void builtin_help(std::vector<std::string> const & path)
+        void builtin_help(std::vector<senf::console::Token> const & path)
             { os_ << "builtin_help( " << senf::stringJoin(path, "/") << " )\n"; }
     };
 }
@@ -112,34 +102,34 @@ BOOST_AUTO_UNIT_TEST(commandGrammar)
                      grammar.use_parser<Grammar::CommandParser>(), 
                      grammar.use_parser<Grammar::SkipParser>() ) . full );
     BOOST_CHECK_EQUAL( ss.str(), 
-                       "beginCommand( doo/bii/doo )\n"
-                       "pushArgument( arg )\n"
-                       "pushArgument( flab::blub )\n"
-                       "pushArgument( 123.434>a )\n"
-                       "openGroup()\n"
-                       "pushWord( a )\n"
-                       "pushPunctuation( , )\n"
-                       "pushWord( b )\n"
-                       "pushPunctuation( ; )\n"
-                       "pushWord( c )\n"
-                       "pushPunctuation( ( )\n"
-                       "pushWord( huhu )\n"
-                       "pushPunctuation( / )\n"
-                       "pushPunctuation( { )\n"
-                       "pushWord( haha )\n"
-                       "pushPunctuation( } )\n"
-                       "pushPunctuation( ) )\n"
-                       "closeGroup()\n"
-                       "pushArgument( foo\"bar )\n"
-                       "pushArgument( \x01\x02\x03\x04 )\n"
+                       "beginCommand( Word('doo')/Word('bii')/Word('doo') )\n"
+                       "pushToken( Word('arg') )\n"
+                       "pushToken( Word('flab::blub') )\n"
+                       "pushToken( Word('123.434>a') )\n"
+                       "pushToken( ArgumentGroupOpen('(') )\n"
+                       "pushToken( Word('a') )\n"
+                       "pushToken( OtherPunctuation(',') )\n"
+                       "pushToken( Word('b') )\n"
+                       "pushToken( CommandTerminator(';') )\n"
+                       "pushToken( Word('c') )\n"
+                       "pushToken( ArgumentGroupOpen('(') )\n"
+                       "pushToken( Word('huhu') )\n"
+                       "pushToken( PathSeparator('/') )\n"
+                       "pushToken( DirectoryGroupOpen('{') )\n"
+                       "pushToken( Word('haha') )\n"
+                       "pushToken( DirectoryGroupClose('}') )\n"
+                       "pushToken( ArgumentGroupClose(')') )\n"
+                       "pushToken( ArgumentGroupClose(')') )\n"
+                       "pushToken( BasicString('foo\"bar') )\n"
+                       "pushToken( HexString('\x01\x02\x03\x04') )\n"
                        "endCommand()\n"
-                       "builtin_ls( /foo/bar )\n"
-                       "builtin_cd( /foo/bar )\n"
+                       "builtin_ls( None('')/Word('foo')/Word('bar') )\n"
+                       "builtin_cd( None('')/Word('foo')/Word('bar') )\n"
                        "builtin_exit()\n"
-                       "pushDirectory( foo/bar/ )\n"
+                       "pushDirectory( Word('foo')/Word('bar')/None('') )\n"
                        "builtin_ls(  )\n"
                        "popDirectory()\n"
-                       "builtin_help( /foo/bar )\n" );
+                       "builtin_help( None('')/Word('foo')/Word('bar') )\n" );
 }
 
 namespace {
@@ -164,7 +154,11 @@ BOOST_AUTO_UNIT_TEST(commandParser)
 
     BOOST_CHECK( parser.parse(text, &setInfo) );
 
-    char const * path[] = { "doo", "bii", "doo" };
+    senf::console::Token path[] = { 
+        senf::console::Token(senf::console::Token::Word, "doo"), 
+        senf::console::Token(senf::console::Token::Word, "bii"),
+        senf::console::Token(senf::console::Token::Word, "doo")
+    };
 
     BOOST_CHECK_EQUAL_COLLECTIONS( info.commandPath().begin(), info.commandPath().end(),
                                    path, path + sizeof(path)/sizeof(path[0]) );