fix(lexer): pop_token

src/lexer/lexer.c

@@ -108,9 +108,31 @@ char *peek_token(void)
 
 char *pop_token(void)
 {
-    char *token = peek_token();
+    char *stream = stream_init();
+
+    ssize_t i = 0;
+
+    while (i < remaining_chars)
+    {
+        if (is_special_char(stream[i]))
+        {
+            if (i == 0) // where we create spe_char token
+                i++;
+            break;
+        }
+        if (isblank(stream[i]))
+        {
+            break;
+        }
+        else if (is_keyword(stream, i))
+        {
+            i++;
+            break;
+        }
+        i++;
+    }
 
     save_state(stream, i);
 
-    return token;
+    return new_token(stream, i);
 }