feat(lexer): comments done + operator WIP

src/lexer/lexer.c

@@ -133,13 +133,14 @@ struct token *peek_token(struct lexer_context *ctx)
         i++;
     }
 
-    struct token *tok = new_token(stream, i);
+    struct token *tok = new_token(stream, i, ctx->only_digits);
     // if token is comment, we don't want it
 
     if (tok->type == TOKEN_COMMENT)
     {
-        // drop current stream
-        get_next_stream(ctx);
+        // Find next newline or EOF.
+        go_end_of_line(ctx);
+
         free_token(&tok);
         tok = peek_token(ctx);
     }
@@ -170,6 +171,9 @@ struct token *pop_token(struct lexer_context *ctx)
         if (!update_lexing_mode(stream, i, &lexing_mode)
             && lexing_mode == LEXER_NORMAL)
         {
+            // TODO call here a function
+            // it must check if is a spe char or an operator
+            // and sets i accordingly.
             if (is_special_char(stream[i]))
             {
                 if (i == 0) // where we create spe_char token
@@ -195,7 +199,7 @@ struct token *pop_token(struct lexer_context *ctx)
     // (this should never happen)
     if (ctx->current_token == NULL)
     {
-        ctx->current_token = new_token(stream, i);
+        ctx->current_token = new_token(stream, i, ctx->only_digits);
     }
     save_state(stream, i, ctx);
 

src/lexer/lexer_utils.c

@@ -132,12 +132,30 @@ static void set_token_word(struct token *tok, char *begin, ssize_t size)
     }
 }
 
-struct token *new_token(char *begin, ssize_t size)
+/* @brief: Sets the token to an IO number
+ *         Also allocates the data and fills it.
+ */
+static void set_token_ION(struct token *tok, char *begin, ssize_t size)
+{
+    if (tok->type == TOKEN_NULL && size != 0)
+    {
+        tok->type = TOKEN_IONUMBER;
+        tok->data = calloc(size + 1, sizeof(char));
+        if (tok->data == NULL)
+            return;
+        strncpy(tok->data, begin, size);
+    }
+}
+
+struct token *new_token(char *begin, ssize_t size, bool only_digits)
 {
     struct token *tok = calloc(1, sizeof(struct token));
     if (tok == NULL)
         return NULL;
 
+    if (only_digits)
+        set_token_ION(tok, begin, size);
+
     set_token_spechar(tok, begin, size);
     set_token_keyword(tok, begin, size);
     set_token_word(tok, begin, size);
@@ -186,6 +204,21 @@ void stream_init(struct lexer_context *ctx)
     ctx->end_previous_token = trimed_stream;
 }
 
+void go_end_of_line(struct lexer_context *ctx)
+{
+    if (ctx == NULL || ctx->end_previous_token == NULL)
+        return;
+
+    ssize_t i = 0;
+    while (ctx->end_previous_token[i] != '\n'
+           && ctx->end_previous_token[i] != EOF)
+    {
+        i++;
+    }
+    ctx->end_previous_token += i;
+    ctx->remaining_chars -= i;
+}
+
 void get_next_stream(struct lexer_context *ctx)
 {
     ctx->remaining_chars = 0;

src/lexer/lexer_utils.h

@@ -1,6 +1,7 @@
 #ifndef LEXER_UTILS_H
 #define LEXER_UTILS_H
 
+#include <stdbool.h>
 #include <stddef.h>
 #include <sys/types.h>
 
@@ -9,6 +10,10 @@ struct lexer_context
     char *end_previous_token;
     ssize_t remaining_chars;
 
+    // usefull to detect IO numbers.
+    // tells us if we only lexed digits in current token.
+    bool only_digits;
+
     struct token *previous_token;
     struct token *current_token;
 };
@@ -39,17 +44,20 @@ enum token_type
     TOKEN_GRAVE,
     TOKEN_SEMICOLON,
     TOKEN_COMMENT,
-    TOKEN_PIPE,
-    TOKEN_AMPERSAND,
+    TOKEN_STAR,
     TOKEN_BACKSLASH,
     TOKEN_DOLLAR,
     TOKEN_LEFT_PAREN,
     TOKEN_RIGHT_PAREN,
     TOKEN_LEFT_BRACKET,
     TOKEN_RIGHT_BRACKET,
+
+    // redirections
     TOKEN_LESS,
     TOKEN_GREATER,
-    TOKEN_STAR,
+    TOKEN_PIPE,
+    TOKEN_AMPERSAND,
+    TOKEN_IONUMBER,
 
     // Keywords
     TOKEN_IF,
@@ -73,7 +81,7 @@ struct token
  *
  * @return: NULL on error, a token otherwise.
  */
-struct token *new_token(char *begin, ssize_t size);
+struct token *new_token(char *begin, ssize_t size, bool only_digits);
 
 /* @brief: frees the token given in argument
  */
@@ -90,6 +98,12 @@ void free_token(struct token **tok);
  */
 void stream_init(struct lexer_context *ctx);
 
+/* @brief: finds the next '\n' or EOF character,
+ * starting at [ctx->end_previous_token],
+ * and updates the stream and remaining_chars accordingly.
+ */
+void go_end_of_line(struct lexer_context *ctx);
+
 /*
  * @brief: drops the current stream and asks IOB for a new one
  */