feat(lexer): rework with struct token done

src/lexer/lexer.c

@@ -31,42 +31,92 @@ static bool is_special_char(char c)
     return c == '\'' || c == '\n' || c == ';' || c == 'EOF';
 }
 
-static void new_token_keyword(struct token *tok, char *begin, ssize_t size)
+
+/* @brief: if a special character is found at [begin],
+ * [tok->token_type] is set accordingly
+ *
+ */
+static void set_token_spechar(struct token *tok, char *begin, ssize_t size)
 {
+    if (size != 1)
+        return;
+    if (begin[0] == 'EOF')
+    {
+        tok->type = TOKEN_EOF;
+    }
+    else if (begin[0] == ';')
+    {
+        tok->type = TOKEN_NEWLINE;
+    }
+    else if (begin[0] == '\'')
+    {
+        tok->type = TOKEN_QUOTE;
+    }
+    else if (begin[0] == ';')
+    {
+        tok->type = TOKEN_SEMICOLON;
+    }
+}
+
+
+/* @brief: if a keyword is found at [begin],
+ * [tok->token_type] is set accordingly
+ *
+ */
+static void set_token_keyword(struct token *tok, char *begin, ssize_t size)
+{
+    if (tok->type != TOKEN_NULL)
+        return;
     if (strncmp(begin, "if", size) == 0)
     {
         tok->type = TOKEN_IF;
     }
-    if (strncmp(begin, "fi", size) == 0)
+    else if (strncmp(begin, "fi", size) == 0)
     {
         tok->type = TOKEN_FI;
     }
-    if (strncmp(begin, "then", size) == 0)
+    else if (strncmp(begin, "then", size) == 0)
     {
         tok->type = TOKEN_THEN;
     }
-    if (strncmp(begin, "else", size) == 0)
+    else if (strncmp(begin, "else", size) == 0)
     {
         tok->type = TOKEN_ELSE;
     }
 }
 
-struct token *new_token(char *begin, ssize_t size)
+/* @brief: if token_type has not yet been set, then it is a TOKEN_WORD
+ *         Also allocates the data and fills it.
+ */
+static void set_token_word(struct token *tok, char *begin, ssize_t size)
+{
+    if (tok->token_type == TOKEN_NULL)
     {
-    struct token *res = calloc(1, sizeof(struct token));
-    if (res == NULL)
-        return NULL;
-
-    // checks which type of token
-
-    // is special char
-
-    // is keyword
-
-    // otherwise -> WORD
         char *token_data = calloc(size + 1, sizeof(char));
         strncpy(res, begin, size);
-    return res;
+    }
+}
+
+struct token *new_token(char *begin, ssize_t size)
+{
+    struct token *tok = calloc(1, sizeof(struct token));
+    if (tok == NULL)
+        return NULL;
+
+    set_token_spechar(tok, begin, size);
+    set_token_keyword(tok, begin, size);
+    set_token_word(tok, begin, size);
+
+    return tok;
+}
+
+void free_token(struct token *tok)
+{
+    if (tok == NULL)
+        return;
+    if (tok->data != NULL)
+        free(tok->data);
+    free(tok);
 }
 
 char *stream_init(void)

src/lexer/lexer.h

@@ -5,6 +5,7 @@
 
 enum token_type
 {
+    TOKEN_NULL = 0,
     TOKEN_EOF,
     TOKEN_WORD,
     TOKEN_NEWLINE,