guillm/42sh
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

rebased lexer on dev

Browse source
This commit is contained in:
matteo 2026-01-17 10:43:23 +01:00 • committed by william.valenduc
parent dadbebfceb
commit a4fcce530c
2 changed files with 82 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

82
src/lexer/lexer.c
View file

@ -16,11 +16,10 @@ static bool at_beginning = true;
static struct token *last_token;
static struct token *current_token;
/* @brief: sets the current_token to [tok].
* this function is called by token_peek().
*/
static void update_current_token(struct token* tok)
static void update_current_token(struct token *tok)
{
current_token = tok;
}
@ -30,7 +29,7 @@ static void update_current_token(struct token* tok)
* this function is called by token_pop().
*
*/
static void update_last_token(struct token* tok)
static void update_last_token(struct token *tok)
{
free_token(&last_token);
last_token = tok;
@ -51,43 +50,86 @@ static void save_state(char *stream, ssize_t i, struct token *tok)
/* @return: true if a special character from the grammar was found,
* false otherwise.
*
*/
static bool is_special_char(char c)
{
return c == '\'' || c == '\n' || c == ';' || c == EOF;
if (c == EOF)
return true;
char special_chars[] = "\n'\"`;#|&\\$(){}<>*";
return strchr(special_chars, c) != NULL;
}
/* @brief: if a special character is found at [begin],
* [tok->token_type] is set accordingly
*
*/
static void set_token_spechar(struct token *tok, char *begin, ssize_t size)
{
if (size != 1)
return;
if (begin[0] == EOF)
switch (begin[0])
{
case EOF:
tok->type = TOKEN_EOF;
remaining_chars = 0;
}
else if (begin[0] == ';')
{
tok->type = TOKEN_NEWLINE;
}
else if (begin[0] == '\'')
{
tok->type = TOKEN_QUOTE;
}
else if (begin[0] == ';')
{
break;
case ';':
tok->type = TOKEN_SEMICOLON;
break;
case '\n':
tok->type = TOKEN_NEWLINE;
break;
case '\'':
tok->type = TOKEN_QUOTE;
break;
case '"':
tok->type = TOKEN_DOUBLE_QUOTE;
break;
case '`':
tok->type = TOKEN_GRAVE;
break;
case '#':
tok->type = TOKEN_COMMENT;
break;
case '|':
tok->type = TOKEN_PIPE;
break;
case '&':
tok->type = TOKEN_AMPERSAND;
break;
case '\\':
tok->type = TOKEN_BACKSLASH;
break;
case '$':
tok->type = TOKEN_DOLLAR;
break;
case '(':
tok->type = TOKEN_LEFT_PAREN;
break;
case ')':
tok->type = TOKEN_RIGHT_PAREN;
break;
case '{':
tok->type = TOKEN_LEFT_BRACKET;
break;
case '}':
tok->type = TOKEN_RIGHT_BRACKET;
break;
case '<':
tok->type = TOKEN_LESS;
break;
case '>':
tok->type = TOKEN_GREATER;
break;
case '*':
tok->type = TOKEN_STAR;
break;
default:
break;
}
}
/* @brief: if a keyword is found at [begin],
* [tok->token_type] is set accordingly
*
*/
static void set_token_keyword(struct token *tok, char *begin, ssize_t size)
{

26
src/lexer/lexer.h
View file

@ -5,12 +5,29 @@
enum token_type
{
// Special characters
TOKEN_NULL = 0,
TOKEN_EOF,
TOKEN_WORD,
TOKEN_NEWLINE,
TOKEN_QUOTE,
TOKEN_DOUBLE_QUOTE,
TOKEN_GRAVE,
TOKEN_SEMICOLON,
TOKEN_COMMENT,
TOKEN_PIPE,
TOKEN_AMPERSAND,
TOKEN_BACKSLASH,
TOKEN_DOLLAR,
TOKEN_LEFT_PAREN,
TOKEN_RIGHT_PAREN,
TOKEN_LEFT_BRACKET,
TOKEN_RIGHT_BRACKET,
TOKEN_LESS,
TOKEN_GREATER,
TOKEN_STAR,
// Keywords
TOKEN_IF,
TOKEN_THEN,
TOKEN_ELSE,
@ -28,7 +45,6 @@ struct token
* @brief: returns the next (newly allocated) token without consuming it.
* if end of input is reached, enters in EOF looping node,
* returning only the same token of type TOKEN_EOF.
*
*/
struct token *peek_token(void);
@ -36,6 +52,7 @@ struct token *peek_token(void);
* @brief: returns the next (newly allocated) token and consumes it.
* if end of input is reached, returns a token of type TOKEN_EOF.
* It also frees the last token created if there was one.
*
* @warning: if the last returned token was a token EOF, it frees it
* and returns NULL. This means that after peeking a token EOF
* in the parser, there must be EXACTLY ONE call to pop_token().
@ -43,10 +60,9 @@ struct token *peek_token(void);
*/
struct token *pop_token(void);
/*
* @warning: NOT IMPLEMENTED.
/* @note: maybe usefull for subshells.
*
* @note: maybe usefull for subshells.
* @warning: NOT IMPLEMENTED.
*/
struct token *get_token_str(void);
@ -56,12 +72,10 @@ struct token *get_token_str(void);
* The data contains [size] char, starting from [begin].
*
* @return: NULL on error, a token otherwise.
*
*/
struct token *new_token(char *begin, ssize_t size);
/* @brief: frees the token given in argument
*
*/
void free_token(struct token **tok);