www.pudn.com > ICE_1.4.zip > Tokenizer.cpp

 
#include "tokenizer.h" 
 
int Tokenizer::Process(std::string input) { 
    //define parsing modes 
    enum eParseMode { 
      PM_WORD = 0, 
      PM_SPACE 
    }; 
 
    eParseMode parseMode; //keeps track of our current parse mode 
     
    //init parseMode here: 
    if(input[0] != ' ') parseMode = PM_WORD; 
    else parseMode = PM_SPACE; 
     
    int wordStart = 0, wordEnd = 0; 
     
    numTokens = 0; //new input, reset tokens 
     
    for(unsigned int i=0; i= MAX_TOKENS) return std::string(""); 
    return tokens[tokenIdx]; 
}