diff --git a/src/lexer/lexer.c b/src/lexer/lexer.c index b6ea0ad..89b5f71 100644 --- a/src/lexer/lexer.c +++ b/src/lexer/lexer.c @@ -7,9 +7,9 @@ #include #include -#include "lexer_utils.h" #include "../io_backend/io_backend.h" #include "../utils/string_utils/string_utils.h" +#include "lexer_utils.h" /* @return: true if a special character from the grammar was found, * false otherwise. @@ -90,7 +90,6 @@ static bool update_lexing_mode(char *stream, ssize_t i, return *lexing_mode != mode_before_update; } - struct token *peek_token(struct lexer_context *ctx) { // we already created the upcoming token during the previous call to peek() @@ -99,7 +98,8 @@ struct token *peek_token(struct lexer_context *ctx) return ctx->current_token; } - char *stream = stream_init(ctx); + stream_init(ctx); + char *stream = ctx->end_previous_token; ssize_t i = 0; // Usefull to know if we are inside a quote or double quote @@ -138,7 +138,10 @@ struct token *peek_token(struct lexer_context *ctx) if (tok->type == TOKEN_COMMENT) { - // tok = peek_token(); + // drop current stream + get_next_stream(ctx); + free_token(&tok); + tok = peek_token(ctx); } update_current_token(tok, ctx); return tok; @@ -153,7 +156,8 @@ struct token *pop_token(struct lexer_context *ctx) free_token(&ctx->current_token); return NULL; } - char *stream = stream_init(ctx); + stream_init(ctx); + char *stream = ctx->end_previous_token; ssize_t i = 0; // Usefull to know if we are inside a quote or double quote diff --git a/src/lexer/lexer.h b/src/lexer/lexer.h index 0d9ed32..e06bec0 100644 --- a/src/lexer/lexer.h +++ b/src/lexer/lexer.h @@ -1,9 +1,9 @@ #ifndef LEXER_H #define LEXER_H -#include "lexer_utils.h" - #include + +#include "lexer_utils.h" /* * @brief: returns the next (newly allocated) token without consuming it. * if end of input is reached, returns a token of type TOKEN_EOF. diff --git a/src/lexer/lexer_utils.c b/src/lexer/lexer_utils.c index a956cb9..f954676 100644 --- a/src/lexer/lexer_utils.c +++ b/src/lexer/lexer_utils.c @@ -162,11 +162,11 @@ void free_token(struct token **tok) *tok = NULL; } -char *stream_init(struct lexer_context *ctx) +void stream_init(struct lexer_context *ctx) { char *stream; - if (ctx->previous_token == NULL) // at the begining + if (ctx->remaining_chars == 0) // at the begining { ctx->remaining_chars = stream_read(&stream); } @@ -177,7 +177,12 @@ char *stream_init(struct lexer_context *ctx) char *trimed_stream = trim_blank_left(stream); ctx->remaining_chars -= trimed_stream - stream; - stream = trimed_stream; - return stream; + ctx->end_previous_token = trimed_stream; +} + +void get_next_stream(struct lexer_context *ctx) +{ + ctx->remaining_chars = 0; + stream_init(ctx); } diff --git a/src/lexer/lexer_utils.h b/src/lexer/lexer_utils.h index 503c1ec..729e912 100644 --- a/src/lexer/lexer_utils.h +++ b/src/lexer/lexer_utils.h @@ -1,8 +1,8 @@ #ifndef LEXER_UTILS_H #define LEXER_UTILS_H -#include #include +#include struct lexer_context { @@ -80,12 +80,17 @@ void free_token(struct token **tok); /* * @brief: checks if the stream used for the last token creation is empty. * If it is, it calls stream_read() from IO_backend, - * and sets [remaing_chars]. + * and sets [remaining_chars]. * If not, it starts from the end of the last token. * Also trims left blanks before returning. * * @return: char* stream from which we tokenise. */ -char *stream_init(struct lexer_context *ctx); +void stream_init(struct lexer_context *ctx); + +/* + * @brief: drops the current stream and asks IOB for a new one + */ +void get_next_stream(struct lexer_context *ctx); #endif /* LEXER_UTILS_H */