fix(parser + lexer): interaction -- WIP

src/lexer/lexer.c

@@ -13,16 +13,40 @@
 static char *end_last_token;
 static ssize_t remaining_chars;
 static bool at_beginning = true;
+static struct token *last_token;
+static struct token *current_token;
 
-/* @brief: saves state for the next call the the lexer.
+
+/* @brief: sets the current_token to [tok].
+ *        this function is called by token_peek().
+ */
+static void update_current_token(struct token* tok)
+{
+    current_token = tok;
+}
+
+/* @brief: frees the last token and sets it to [tok].
+ *        Also sets current_token to NULL.
+ *        this function is called by token_pop().
  *
  */
-static void save_state(char *stream, ssize_t i)
+static void update_last_token(struct token* tok)
+{
+    free_token(&last_token);
+    last_token = tok;
+}
+
+/* @brief: saves state for the next call to the the lexer.
+ *        this function is called by token_pop().
+ *
+ */
+static void save_state(char *stream, ssize_t i, struct token *tok)
 {
     remaining_chars -= i;
     end_last_token = stream + i;
     at_beginning = false;
-    return;
+
+    update_last_token(tok);
 }
 
 /* @return: true if a special character from the grammar was found,
@@ -67,7 +91,7 @@ static void set_token_spechar(struct token *tok, char *begin, ssize_t size)
  */
 static void set_token_keyword(struct token *tok, char *begin, ssize_t size)
 {
-    if (tok->type != TOKEN_NULL)
+    if (tok->type != TOKEN_NULL || size == 0)
         return;
     if (strncmp(begin, "if", size) == 0)
     {
@@ -101,7 +125,7 @@ static void set_token_keyword(struct token *tok, char *begin, ssize_t size)
  */
 static void set_token_word(struct token *tok, char *begin, ssize_t size)
 {
-    if (tok->type == TOKEN_NULL)
+    if (tok->type == TOKEN_NULL && size != 0)
     {
         tok->type = TOKEN_WORD;
         tok->data = calloc(size + 1, sizeof(char));
@@ -124,13 +148,14 @@ struct token *new_token(char *begin, ssize_t size)
     return tok;
 }
 
-void free_token(struct token *tok)
+void free_token(struct token **tok)
 {
-    if (tok == NULL)
+    if (tok == NULL || *tok == NULL)
         return;
-    if (tok->data != NULL)
-        free(tok->data);
-    free(tok);
+    if ((*tok)->data != NULL)
+        free((*tok)->data);
+    free(*tok);
+    *tok = NULL;
 }
 
 char *stream_init(void)
@@ -156,6 +181,12 @@ char *stream_init(void)
 
 struct token *peek_token(void)
 {
+    // EOF looping mode
+    if (current_token != NULL && current_token->type == TOKEN_EOF)
+    {
+        return current_token;
+    }
+
     char *stream = stream_init();
 
     ssize_t i = 0;
@@ -175,11 +206,18 @@ struct token *peek_token(void)
         i++;
     }
 
-    return new_token(stream, i);
+    struct token *tok = new_token(stream, i);
+    update_current_token(tok);
+    return tok;
 }
 
 struct token *pop_token(void)
 {
+    if (last_token != NULL && last_token->type == TOKEN_EOF)
+    {
+        free_token(&last_token);
+        return NULL;
+    }
     char *stream = stream_init();
 
     ssize_t i = 0;
@@ -199,7 +237,8 @@ struct token *pop_token(void)
         i++;
     }
 
-    save_state(stream, i);
+    struct token *tok = new_token(stream, i);
+    save_state(stream, i, tok);
 
-    return new_token(stream, i);
+    return tok;
 }

src/lexer/lexer.h

@@ -26,7 +26,8 @@ struct token
 
 /*
  * @brief: returns the next (newly allocated) token without consuming it.
- *         if end of input is reached, returns a token of type TOKEN_EOF.
+ *         if end of input is reached, enters in EOF looping node,
+ *         returning only the same token of type TOKEN_EOF.
  *
  */
 struct token *peek_token(void);
@@ -34,6 +35,10 @@ struct token *peek_token(void);
 /*
  * @brief: returns the next (newly allocated) token and consumes it.
  *         if end of input is reached, returns a token of type TOKEN_EOF.
+ *         It also frees the last token created if there was one.
+ * @warning: if the last returned token was a token EOF, it frees it
+ *          and returns NULL. This means that after peeking a token EOF
+ *          in the parser, there must be EXACTLY ONE call to pop_token().
  *
  */
 struct token *pop_token(void);
@@ -58,7 +63,7 @@ struct token *new_token(char *begin, ssize_t size);
 /* @brief: frees the token given in argument
  *
  */
-void free_token(struct token *tok);
+void free_token(struct token **tok);
 
 /*
  * @brief: checks if the stream used for the last token creation is empty.