feat(lexer): comments done + operator WIP

src/lexer/lexer_utils.c

@@ -132,12 +132,30 @@ static void set_token_word(struct token *tok, char *begin, ssize_t size)
     }
 }
 
-struct token *new_token(char *begin, ssize_t size)
+/* @brief: Sets the token to an IO number
+ *         Also allocates the data and fills it.
+ */
+static void set_token_ION(struct token *tok, char *begin, ssize_t size)
+{
+    if (tok->type == TOKEN_NULL && size != 0)
+    {
+        tok->type = TOKEN_IONUMBER;
+        tok->data = calloc(size + 1, sizeof(char));
+        if (tok->data == NULL)
+            return;
+        strncpy(tok->data, begin, size);
+    }
+}
+
+struct token *new_token(char *begin, ssize_t size, bool only_digits)
 {
     struct token *tok = calloc(1, sizeof(struct token));
     if (tok == NULL)
         return NULL;
 
+    if (only_digits)
+        set_token_ION(tok, begin, size);
+
     set_token_spechar(tok, begin, size);
     set_token_keyword(tok, begin, size);
     set_token_word(tok, begin, size);
@@ -186,6 +204,21 @@ void stream_init(struct lexer_context *ctx)
     ctx->end_previous_token = trimed_stream;
 }
 
+void go_end_of_line(struct lexer_context *ctx)
+{
+    if (ctx == NULL || ctx->end_previous_token == NULL)
+        return;
+
+    ssize_t i = 0;
+    while (ctx->end_previous_token[i] != '\n'
+           && ctx->end_previous_token[i] != EOF)
+    {
+        i++;
+    }
+    ctx->end_previous_token += i;
+    ctx->remaining_chars -= i;
+}
+
 void get_next_stream(struct lexer_context *ctx)
 {
     ctx->remaining_chars = 0;