Rules: Added ::tokenizeLine

This commit is contained in:
Paul Beckingham 2016-03-25 23:53:34 -04:00
parent 321bddba74
commit 9403fa4974
2 changed files with 15 additions and 0 deletions

View file

@ -343,3 +343,17 @@ unsigned int Rules::getIndentation (const std::string& line)
} }
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
// Tokenize the line. This loses whitespace information and token types.
std::vector <std::string> Rules::tokenizeLine (const std::string& line)
{
std::vector <std::string> tokens;
std::string token;
Lexer::Type type;
Lexer lexer (line);
while (lexer.token (token, type))
tokens.push_back (token);
return tokens;
}
////////////////////////////////////////////////////////////////////////////////

View file

@ -62,6 +62,7 @@ private:
void parseRuleTag (const std::vector <std::string>&); void parseRuleTag (const std::vector <std::string>&);
unsigned int getIndentation (const std::string&); unsigned int getIndentation (const std::string&);
std::vector <std::string> tokenizeLine (const std::string&);
private: private:
std::string _original_file {}; std::string _original_file {};