feat(lexer): comment handling + healthy changes

src/lexer/lexer.c

@@ -7,9 +7,9 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "lexer_utils.h"
 #include "../io_backend/io_backend.h"
 #include "../utils/string_utils/string_utils.h"
+#include "lexer_utils.h"
 
 /* @return: true if a special character from the grammar was found,
  *          false otherwise.
@@ -90,7 +90,6 @@ static bool update_lexing_mode(char *stream, ssize_t i,
     return *lexing_mode != mode_before_update;
 }
 
-
 struct token *peek_token(struct lexer_context *ctx)
 {
     // we already created the upcoming token during the previous call to peek()
@@ -99,7 +98,8 @@ struct token *peek_token(struct lexer_context *ctx)
         return ctx->current_token;
     }
 
-    char *stream = stream_init(ctx);
+    stream_init(ctx);
+    char *stream = ctx->end_previous_token;
     ssize_t i = 0;
 
     // Usefull to know if we are inside a quote or double quote
@@ -138,7 +138,10 @@ struct token *peek_token(struct lexer_context *ctx)
 
     if (tok->type == TOKEN_COMMENT)
     {
-        // tok = peek_token();
+        // drop current stream
+        get_next_stream(ctx);
+        free_token(&tok);
+        tok = peek_token(ctx);
     }
     update_current_token(tok, ctx);
     return tok;
@@ -153,7 +156,8 @@ struct token *pop_token(struct lexer_context *ctx)
         free_token(&ctx->current_token);
         return NULL;
     }
-    char *stream = stream_init(ctx);
+    stream_init(ctx);
+    char *stream = ctx->end_previous_token;
     ssize_t i = 0;
 
     // Usefull to know if we are inside a quote or double quote

src/lexer/lexer.h

@@ -1,9 +1,9 @@
 #ifndef LEXER_H
 #define LEXER_H
 
-#include "lexer_utils.h"
-
 #include <sys/types.h>
+
+#include "lexer_utils.h"
 /*
  * @brief: returns the next (newly allocated) token without consuming it.
  *         if end of input is reached, returns a token of type TOKEN_EOF.

src/lexer/lexer_utils.c

@@ -162,11 +162,11 @@ void free_token(struct token **tok)
     *tok = NULL;
 }
 
-char *stream_init(struct lexer_context *ctx)
+void stream_init(struct lexer_context *ctx)
 {
     char *stream;
 
-    if (ctx->previous_token == NULL) // at the begining
+    if (ctx->remaining_chars == 0) // at the begining
     {
         ctx->remaining_chars = stream_read(&stream);
     }
@@ -177,7 +177,12 @@ char *stream_init(struct lexer_context *ctx)
 
     char *trimed_stream = trim_blank_left(stream);
     ctx->remaining_chars -= trimed_stream - stream;
-    stream = trimed_stream;
 
-    return stream;
+    ctx->end_previous_token = trimed_stream;
+}
+
+void get_next_stream(struct lexer_context *ctx)
+{
+    ctx->remaining_chars = 0;
+    stream_init(ctx);
 }

src/lexer/lexer_utils.h

@@ -1,8 +1,8 @@
 #ifndef LEXER_UTILS_H
 #define LEXER_UTILS_H
 
-#include <sys/types.h>
 #include <stddef.h>
+#include <sys/types.h>
 
 struct lexer_context
 {
@@ -80,12 +80,17 @@ void free_token(struct token **tok);
 /*
  * @brief: checks if the stream used for the last token creation is empty.
  *         If it is, it calls stream_read() from IO_backend,
- *         and sets [remaing_chars].
+ *         and sets [remaining_chars].
  *         If not, it starts from the end of the last token.
  *         Also trims left blanks before returning.
  *
  * @return: char* stream from which we tokenise.
  */
-char *stream_init(struct lexer_context *ctx);
+void stream_init(struct lexer_context *ctx);
+
+/*
+ * @brief: drops the current stream and asks IOB for a new one
+ */
+void get_next_stream(struct lexer_context *ctx);
 
 #endif /* LEXER_UTILS_H */