ArgLexer Class Reference

#include <argsparser.h>

Public Member Functions
	ArgLexer ()
	~ArgLexer ()
void	addLine (const String &line)
bool	nextToken ()
ArgToken *	currentToken () const
ArgTokenType	lookahead () const

Detailed Description

This class will split the input stream into tokens. The tokens will be further used by the Parser to build the syntax tree.

Definition at line 94 of file argsparser.h.

Constructor & Destructor Documentation

ArgLexer()

ArgLexer::ArgLexer

(

)

Constructor

Definition at line 81 of file argsparser.cpp.

{
    line = 0;
    pos = 0;
    pToken = new ArgToken();
    //Initialize reserved words dictionary
    reservedWords["WHERE"] = TOK_WHERE;
    reservedWords["ALIAS"] = TOK_ALIAS;
    reservedWords["OR"] = TOK_OR;
    reservedWords["REQUIRES"] = TOK_REQUIRES;
}

~ArgLexer()

ArgLexer::~ArgLexer

(

)

Destructor

Definition at line 93 of file argsparser.cpp.

{
    delete pToken;
}

Member Function Documentation

addLine()

void ArgLexer::addLine

(

const String &

line

)

Add input lines to the lexer

Definition at line 99 of file argsparser.cpp.

{
    input.push_back(line + " ");
#ifdef DEBUG
    std::cout << input.size() << ": " << line << std::endl;
#endif
}

currentToken()

ArgToken * ArgLexer::currentToken

(

)

const

Definition at line 337 of file argsparser.cpp.

{
    return pToken;
}

lookahead()

ArgTokenType ArgLexer::lookahead

(

)

const

Definition at line 341 of file argsparser.cpp.

{
    return pToken->type;
}

nextToken()

bool ArgLexer::nextToken

(

)

Function to parse a new token. If the token is TOK_END will return false and true otherwise. The current token will be changed.

Definition at line 164 of file argsparser.cpp.

{
    //make it visible by default
    pToken->visibility = 0;
    pToken->starred = false;

    if (line == input.size())
    {
        setupToken(TOK_END);
        return false;
    }

    char c = input[line][pos];

    //Skip all white spaces
    while (line < input.size() && pos < input[line].length() && isspace(c
            = input[line][pos]))
    {
        ++pos;
        if (c == '\0' || c == '\n' || pos == input[line].length()) //Next line
        {
            nextLine();
        }
    }

    if (line == input.size())
        setupToken(TOK_END);
    else if (isalpha(c) || c == '_')
    {
        offset = 0;
        while (isalnum(c) || c == '_')
        {
            ++offset;
            c = input[line][pos + offset];
        }
        setupToken(TOK_ID);
    }
    else if (isalnum(c) || (c == '-' && isdigit(input[line][pos + 1])))
    {
        offset = 1;
        ArgTokenType t = TOK_INT;
        while (isdigit(input[line][pos + offset]))
            ++offset;
        if (input[line][pos + offset] == '.')
        {
            ++offset;
            while (isdigit(input[line][pos + offset]))
                ++offset;
            t = TOK_FLOAT;
        }
        if (input[line][pos + offset] == 'e')
        {
            ++offset;
            if (input[line][pos + offset] != '+' && input[line][pos + offset]
                != '-')
            {
                std::cerr << "ERROR: expected '+' or '-' " << std::endl
                << "at line: " << line + 1 << " pos: " << offset + 1
                << std::endl;
                exit(1);
            }
            ++offset;
            while (isdigit(input[line][pos + offset]))
                ++offset;
            t = TOK_FLOAT;

        }
        setupToken(t);
    }
    else
    {
        offset = 1;
        bool empty;
        switch (c)
        {
        case '<':
            setupToken(TOK_LAN);
            break;
        case '>':
            setupToken(TOK_RAN);
            break;
        case '=':
            if (input[line][pos + offset] == '=')
            {
                pos = input[line].find_first_not_of('=', pos + 1);
                checkVisibility();
                offset = input[line].find_first_of('=', pos + 1);
                offset -= pos;
                setupToken(TOK_SECTION);
                nextLine();
            }
            else
                //simple equal sign '='
                setupToken(TOK_EQ);
            break;
        case '[':
            setupToken(TOK_LBRA);
            break;
        case ']':
            setupToken(TOK_RBRA);
            break;
        case ';':
            setupToken(TOK_SEMI);
            break;
        case ',':
            setupToken(TOK_COMMA);
            break;
        case '+':
            setupToken(TOK_PLUS);
            break;
        case '.':
            if (input[line][pos + offset] == '.' &&
                input[line][pos + offset + 1] == '.')
            {
                offset += 2;
                setupToken(TOK_ETC);
            }
            break;
        case ':':
            ++pos;

            checkVisibility();

            offset = input[line].find_first_of("\n\r", pos);
            offset -= pos;
            setupToken(TOK_COMM);
            nextLine();
            break;
        case '-':
            if (input[line][pos + offset] == '-')
                ++offset;
            empty = true;
            c = input[line][pos + offset];
            if (isalpha(c) || c == '_') //should start with letter or _
            {
                while (isalnum(c) || c == '_')
                {
                    empty = false;
                    ++offset;
                    c = input[line][pos + offset];
                }
            }
            if (empty)
            {
                std::cerr << "ERROR: Params should be of the form -ID or --ID"
                << std::endl;
                exit(1);
            }
            setupToken(TOK_OPT);
            checkVisibility();
            checkIndependent();
            break;
        case '"':
            offset = input[line].find_first_of('"', pos + 1);
            ++pos;
            offset -= pos;
            setupToken(TOK_STR);
            break;
        default:
            std::cerr << "ERROR: Unexpected character '" << c << "'"
            << std::endl << "at line: " << line + 1 << " pos: " << pos
            + 1 << std::endl;
            std::cerr << "WRONG LINE: " << input[line] << std::endl;
            exit(1);
        }
    }

    //ConsolePrinter * cp = new ConsolePrinter();
    //cp->printToken(pToken);

    return true;
}

---

The documentation for this class was generated from the following files:

• xmippCore/core/argsparser.h
• xmippCore/core/argsparser.cpp