feat(lexer): rework using a struct token and functions associated - unstable

src/lexer/lexer.h

@@ -3,19 +3,38 @@
 
 #include <sys/types.h>
 
+enum token_type
+{
+    TOKEN_EOF,
+    TOKEN_WORD,
+    TOKEN_NEWLINE,
+    TOKEN_QUOTE,
+    TOKEN_SEMICOLON,
+    TOKEN_IF,
+    TOKEN_THEN,
+    TOKEN_ELSE,
+    TOKEN_FI
+};
+
+struct token
+{
+    enum token_type type;
+    char* data;
+};
+
 /*
  * @brief: returns the next (newly allocated) token without consuming it.
- *         if end of input is reached, returns EOF.
+ *         if end of input is reached, returns a token of type TOKEN_EOF.
  *
  */
-char *peek_token(void);
+struct token *peek_token(void);
 
 /*
  * @brief: returns the next (newly allocated) token and consumes it.
- *         if end of input is reached, returns EOF.
+ *         if end of input is reached, returns a token of type TOKEN_EOF.
  *
  */
-char *pop_token(void);
+struct token *pop_token(void);
 
 /*
  * @warning: NOT IMPLEMENTED.
@@ -23,16 +42,21 @@ char *pop_token(void);
  * @note: maybe usefull for subshells.
  */
 
-char *get_token_str(void);
+struct token *get_token_str(void);
 
 /*
- * @brief: return a newly allocated token.
- *         This token contains [size] chars, starting from [begin].
+ * @brief: return a newly allocated token, with the corresponding type.
+ *         The data contains [size] char, starting from [begin].
  *
- * @return: NULL on error, null-terminated char* otherwise.
+ * @return: NULL on error, a token otherwise.
  *
  */
-char *new_token(char *begin, ssize_t size);
+struct token *new_token(char *begin, ssize_t size);
+
+/* @brief: frees the token given in argument
+ *
+ */
+void free_token(struct token* tok);
 
 /*
  * @brief: checks if the stream used for the last token creation is empty.