guillm/42sh
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'lexer' into dev

Browse source
This commit is contained in:
matteo 2026-01-17 16:26:30 +01:00
commit e80058a765
1 changed files with 82 additions and 71 deletions
Download patch file Download diff file Expand all files Collapse all files

153
src/lexer/lexer.c
View file

@ -12,7 +12,6 @@
static char *end_last_token; static char *end_last_token;
static ssize_t remaining_chars; static ssize_t remaining_chars;
static bool at_beginning = true;
static struct token *last_token; static struct token *last_token;
static struct token *current_token; static struct token *current_token;
@ -34,17 +33,19 @@ static void update_last_token(struct token *tok)
last_token = tok; last_token = tok;
} }
/* @brief: saves state for the next call to the the lexer. /* @brief: updates the current position in the stream.
* this function is called by token_pop(). * [stream] += [i]
* * Also frees the last sent token, and sets it to current_token.
* Current token is then set to NULL.
* This function is called by token_pop().
*/ */
static void save_state(char *stream, ssize_t i, struct token *tok) static void save_state(char *stream, ssize_t i)
{ {
remaining_chars -= i; remaining_chars -= i;
end_last_token = stream + i; end_last_token = stream + i;
at_beginning = false;
update_last_token(tok); update_last_token(current_token);
update_current_token(NULL);
} }
/* @return: true if a special character from the grammar was found, /* @return: true if a special character from the grammar was found,
@ -68,62 +69,62 @@ static void set_token_spechar(struct token *tok, char *begin, ssize_t size)
return; return;
switch (begin[0]) switch (begin[0])
{ {
case EOF: case EOF:
tok->type = TOKEN_EOF; tok->type = TOKEN_EOF;
break; break;
case ';': case ';':
tok->type = TOKEN_SEMICOLON; tok->type = TOKEN_SEMICOLON;
break; break;
case '\n': case '\n':
tok->type = TOKEN_NEWLINE; tok->type = TOKEN_NEWLINE;
break; break;
case '\'': case '\'':
tok->type = TOKEN_QUOTE; tok->type = TOKEN_QUOTE;
break; break;
case '"': case '"':
tok->type = TOKEN_DOUBLE_QUOTE; tok->type = TOKEN_DOUBLE_QUOTE;
break; break;
case '`': case '`':
tok->type = TOKEN_GRAVE; tok->type = TOKEN_GRAVE;
break; break;
case '#': case '#':
tok->type = TOKEN_COMMENT; tok->type = TOKEN_COMMENT;
break; break;
case '|': case '|':
tok->type = TOKEN_PIPE; tok->type = TOKEN_PIPE;
break; break;
case '&': case '&':
tok->type = TOKEN_AMPERSAND; tok->type = TOKEN_AMPERSAND;
break; break;
case '\\': case '\\':
tok->type = TOKEN_BACKSLASH; tok->type = TOKEN_BACKSLASH;
break; break;
case '$': case '$':
tok->type = TOKEN_DOLLAR; tok->type = TOKEN_DOLLAR;
break; break;
case '(': case '(':
tok->type = TOKEN_LEFT_PAREN; tok->type = TOKEN_LEFT_PAREN;
break; break;
case ')': case ')':
tok->type = TOKEN_RIGHT_PAREN; tok->type = TOKEN_RIGHT_PAREN;
break; break;
case '{': case '{':
tok->type = TOKEN_LEFT_BRACKET; tok->type = TOKEN_LEFT_BRACKET;
break; break;
case '}': case '}':
tok->type = TOKEN_RIGHT_BRACKET; tok->type = TOKEN_RIGHT_BRACKET;
break; break;
case '<': case '<':
tok->type = TOKEN_LESS; tok->type = TOKEN_LESS;
break; break;
case '>': case '>':
tok->type = TOKEN_GREATER; tok->type = TOKEN_GREATER;
break; break;
case '*': case '*':
tok->type = TOKEN_STAR; tok->type = TOKEN_STAR;
break; break;
default: default:
break; break;
} }
} }
@ -155,6 +156,10 @@ static void set_token_keyword(struct token *tok, char *begin, ssize_t size)
tok->type = TOKEN_ELIF; tok->type = TOKEN_ELIF;
} }
// no keywords found.
if (tok->type == TOKEN_NULL)
return;
tok->data = calloc(size + 1, sizeof(char)); tok->data = calloc(size + 1, sizeof(char));
if (tok->data == NULL) if (tok->data == NULL)
return; return;
@ -203,10 +208,9 @@ char *stream_init(void)
{ {
char *stream; char *stream;
if (at_beginning) if (last_token == NULL) // at the begining
{ {
remaining_chars = stream_read(&stream); remaining_chars = stream_read(&stream);
// at_beginning = true;
} }
else else
{ {
@ -222,8 +226,8 @@ char *stream_init(void)
struct token *peek_token(void) struct token *peek_token(void)
{ {
// EOF looping mode // we already created the upcoming token during the previous call to peek()
if (current_token != NULL && current_token->type == TOKEN_EOF) if (current_token != NULL)
{ {
return current_token; return current_token;
} }
@ -254,9 +258,11 @@ struct token *peek_token(void)
struct token *pop_token(void) struct token *pop_token(void)
{ {
if (last_token != NULL && last_token->type == TOKEN_EOF) if (current_token != NULL && current_token->type == TOKEN_EOF)
{ {
// we reached end of input, frees all the token still allocated.
free_token(&last_token); free_token(&last_token);
free_token(&current_token);
return NULL; return NULL;
} }
char *stream = stream_init(); char *stream = stream_init();
@ -278,8 +284,13 @@ struct token *pop_token(void)
i++; i++;
} }
struct token *tok = new_token(stream, i); // just in case peek() was not called before poping.
save_state(stream, i, tok); // (this should never happen)
if (current_token == NULL)
{
current_token = new_token(stream, i);
}
save_state(stream, i);
return tok; return last_token;
} }