Merge branch 'lexer' into dev

src/lexer/lexer.c

@@ -77,7 +77,7 @@ char *stream_init(void)
     return stream;
 }
 
-char *get_token(void)
+char *peek_token(void)
 {
     char *stream = stream_init();
 
@@ -103,7 +103,14 @@ char *get_token(void)
         i++;
     }
 
-    save_state(stream, i);
-
     return new_token(stream, i);
 }
+
+char *pop_token(void)
+{
+    char *token = peek_token();
+
+    save_state(stream, i);
+
+    return token;
+}

src/lexer/lexer.h

@@ -3,10 +3,19 @@
 
 #include <sys/types.h>
 
-/* @return: char*, the next token
+/*
+ * @brief: returns the next (newly allocated) token without consuming it.
+ *         if end of input is reached, returns EOF.
  *
  */
-char *get_token(void);
+char *peek_token(void);
+
+/*
+ * @brief: returns the next (newly allocated) token and consumes it.
+ *         if end of input is reached, returns EOF.
+ *
+ */
+char *pop_token(void);
 
 /*
  * @warning: NOT IMPLEMENTED.