new token types

src/lexer/lexer.h

@@ -5,12 +5,29 @@
 
 enum token_type
 {
+    // Special characters
     TOKEN_NULL = 0,
     TOKEN_EOF,
     TOKEN_WORD,
     TOKEN_NEWLINE,
     TOKEN_QUOTE,
+    TOKEN_DOUBLE_QUOTE,
+    TOKEN_GRAVE,
     TOKEN_SEMICOLON,
+    TOKEN_COMMENT,
+    TOKEN_PIPE,
+    TOKEN_AMPERSAND,
+    TOKEN_BACKSLASH,
+    TOKEN_DOLLAR,
+    TOKEN_LEFT_PAREN,
+    TOKEN_RIGHT_PAREN,
+    TOKEN_LEFT_BRACKET,
+    TOKEN_RIGHT_BRACKET,
+    TOKEN_LESS,
+    TOKEN_GREATER,
+    TOKEN_STAR,
+    
+    // Keywords
     TOKEN_IF,
     TOKEN_THEN,
     TOKEN_ELSE,
@@ -27,21 +44,18 @@ struct token
 /*
  * @brief: returns the next (newly allocated) token without consuming it.
  *         if end of input is reached, returns a token of type TOKEN_EOF.
- *
  */
 struct token *peek_token(void);
 
 /*
  * @brief: returns the next (newly allocated) token and consumes it.
  *         if end of input is reached, returns a token of type TOKEN_EOF.
- *
  */
 struct token *pop_token(void);
 
-/*
- * @warning: NOT IMPLEMENTED.
+/* @note: maybe usefull for subshells. 
  *
- * @note: maybe usefull for subshells.
+ * @warning: NOT IMPLEMENTED.
  */
 
 struct token *get_token_str(void);
@@ -51,12 +65,10 @@ struct token *get_token_str(void);
  *         The data contains [size] char, starting from [begin].
  *
  * @return: NULL on error, a token otherwise.
- *
  */
 struct token *new_token(char *begin, ssize_t size);
 
 /* @brief: frees the token given in argument
- *
  */
 void free_token(struct token *tok);