diff --git a/src/lexer/lexer.c b/src/lexer/lexer.c index f16088b..e6dcfc7 100644 --- a/src/lexer/lexer.c +++ b/src/lexer/lexer.c @@ -133,13 +133,14 @@ struct token *peek_token(struct lexer_context *ctx) i++; } - struct token *tok = new_token(stream, i); + struct token *tok = new_token(stream, i, ctx->only_digits); // if token is comment, we don't want it if (tok->type == TOKEN_COMMENT) { - // drop current stream - get_next_stream(ctx); + // Find next newline or EOF. + go_end_of_line(ctx); + free_token(&tok); tok = peek_token(ctx); } @@ -170,6 +171,9 @@ struct token *pop_token(struct lexer_context *ctx) if (!update_lexing_mode(stream, i, &lexing_mode) && lexing_mode == LEXER_NORMAL) { + // TODO call here a function + // it must check if is a spe char or an operator + // and sets i accordingly. if (is_special_char(stream[i])) { if (i == 0) // where we create spe_char token @@ -195,7 +199,7 @@ struct token *pop_token(struct lexer_context *ctx) // (this should never happen) if (ctx->current_token == NULL) { - ctx->current_token = new_token(stream, i); + ctx->current_token = new_token(stream, i, ctx->only_digits); } save_state(stream, i, ctx); diff --git a/src/lexer/lexer_utils.c b/src/lexer/lexer_utils.c index 4206c4c..4906fcb 100644 --- a/src/lexer/lexer_utils.c +++ b/src/lexer/lexer_utils.c @@ -132,12 +132,30 @@ static void set_token_word(struct token *tok, char *begin, ssize_t size) } } -struct token *new_token(char *begin, ssize_t size) +/* @brief: Sets the token to an IO number + * Also allocates the data and fills it. + */ +static void set_token_ION(struct token *tok, char *begin, ssize_t size) +{ + if (tok->type == TOKEN_NULL && size != 0) + { + tok->type = TOKEN_IONUMBER; + tok->data = calloc(size + 1, sizeof(char)); + if (tok->data == NULL) + return; + strncpy(tok->data, begin, size); + } +} + +struct token *new_token(char *begin, ssize_t size, bool only_digits) { struct token *tok = calloc(1, sizeof(struct token)); if (tok == NULL) return NULL; + if (only_digits) + set_token_ION(tok, begin, size); + set_token_spechar(tok, begin, size); set_token_keyword(tok, begin, size); set_token_word(tok, begin, size); @@ -186,6 +204,21 @@ void stream_init(struct lexer_context *ctx) ctx->end_previous_token = trimed_stream; } +void go_end_of_line(struct lexer_context *ctx) +{ + if (ctx == NULL || ctx->end_previous_token == NULL) + return; + + ssize_t i = 0; + while (ctx->end_previous_token[i] != '\n' + && ctx->end_previous_token[i] != EOF) + { + i++; + } + ctx->end_previous_token += i; + ctx->remaining_chars -= i; +} + void get_next_stream(struct lexer_context *ctx) { ctx->remaining_chars = 0; diff --git a/src/lexer/lexer_utils.h b/src/lexer/lexer_utils.h index 83bc772..fde696b 100644 --- a/src/lexer/lexer_utils.h +++ b/src/lexer/lexer_utils.h @@ -1,6 +1,7 @@ #ifndef LEXER_UTILS_H #define LEXER_UTILS_H +#include #include #include @@ -9,6 +10,10 @@ struct lexer_context char *end_previous_token; ssize_t remaining_chars; + // usefull to detect IO numbers. + // tells us if we only lexed digits in current token. + bool only_digits; + struct token *previous_token; struct token *current_token; }; @@ -39,17 +44,20 @@ enum token_type TOKEN_GRAVE, TOKEN_SEMICOLON, TOKEN_COMMENT, - TOKEN_PIPE, - TOKEN_AMPERSAND, + TOKEN_STAR, TOKEN_BACKSLASH, TOKEN_DOLLAR, TOKEN_LEFT_PAREN, TOKEN_RIGHT_PAREN, TOKEN_LEFT_BRACKET, TOKEN_RIGHT_BRACKET, + + // redirections TOKEN_LESS, TOKEN_GREATER, - TOKEN_STAR, + TOKEN_PIPE, + TOKEN_AMPERSAND, + TOKEN_IONUMBER, // Keywords TOKEN_IF, @@ -73,7 +81,7 @@ struct token * * @return: NULL on error, a token otherwise. */ -struct token *new_token(char *begin, ssize_t size); +struct token *new_token(char *begin, ssize_t size, bool only_digits); /* @brief: frees the token given in argument */ @@ -90,6 +98,12 @@ void free_token(struct token **tok); */ void stream_init(struct lexer_context *ctx); +/* @brief: finds the next '\n' or EOF character, + * starting at [ctx->end_previous_token], + * and updates the stream and remaining_chars accordingly. + */ +void go_end_of_line(struct lexer_context *ctx); + /* * @brief: drops the current stream and asks IOB for a new one */