From e7784b8df1350f55ab3413186deb54b9f44c911c Mon Sep 17 00:00:00 2001 From: Matteo Flebus Date: Thu, 8 Jan 2026 15:58:24 +0100 Subject: [PATCH 1/3] feat(tests): mini testsuite for token creation --- tests/unit/lexer/lexer_tests.c | 61 ++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 tests/unit/lexer/lexer_tests.c diff --git a/tests/unit/lexer/lexer_tests.c b/tests/unit/lexer/lexer_tests.c new file mode 100644 index 0000000..0a3ccf7 --- /dev/null +++ b/tests/unit/lexer/lexer_tests.c @@ -0,0 +1,61 @@ +#include +#include +#include +#include +#include + +#include "lexer/lexer.h" + +TestSuite(token_creation); + +Test(token_creation, basic) +{ + char *input = "Hello World"; + + char *actual = new_token(input, 5); + char *expected = "Hello"; + cr_expect(eq(str, actual, expected)); + free(actual); +} + +Test(token_creation, nul) +{ + char *input = NULL; + + char *actual = new_token(input, 5); + char *expected = NULL; + cr_expect(eq(str, actual, expected)); + free(actual); +} + +Test(token_creation, too_large) +{ + char *input = "Hel"; + + char *actual = new_token(input, 5); + char *expected = NULL; + cr_expect(eq(str, actual, expected)); + free(actual); +} + +Test(token_creation, empty) +{ + char *input = ""; + + char *actual = new_token(input, 5); + char *expected = NULL; + cr_expect(eq(str, actual, expected)); + free(actual); +} + +Test(token_creation, basic_long) +{ + char *input = "Hello World! This project is a mini shell, I love BIG G."; + + char *actual = new_token(input, 42); + char *expected = calloc(42 + 1, sizeof(char)); + strncpy(input, expected, 42); + cr_expect(eq(str, actual, expected)); + free(actual); + free(expected); +} From 453a8ab0da0b9b29e95d8efc638bb25aca61a64d Mon Sep 17 00:00:00 2001 From: Matteo Flebus Date: Thu, 8 Jan 2026 16:32:48 +0100 Subject: [PATCH 2/3] feat(lexer): usefull functions --- src/lexer/lexer.c | 67 +++++++++++++++++++++++++++++++++++++++++++++++ src/lexer/lexer.h | 36 +++++++++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/src/lexer/lexer.c b/src/lexer/lexer.c index e69de29..5b03c18 100644 --- a/src/lexer/lexer.c +++ b/src/lexer/lexer.c @@ -0,0 +1,67 @@ +#include "lexer.h" + +#include + +#include "io_backend/io_backend.h" + +static char *end_last_token; +static ssize_t remaining_chars; + +char *new_token(char *begin, size_t size) +{ + char *res = calloc(size + 1, sizeof(char)); + if (res == NULL) + return NULL; + strncpy(res, begin, size); + return res; +} + +char *stream_init(void) +{ + char *stream; + + if (remaining_chars == 0) + { + remaining_chars = stream_read(&stream); + } + else + { + stream = end_last_token; + } + + return stream; +} + +char *get_token(void) +{ + char *stream = stream_init(); + + bool inquotes = false; + ssize_t i = 0; + + while (i < remaining_chars) + { + switch (stream[i]) + { + case '\'': + inquotes = !inquotes; + break; + + case ' ' | '\n' | '\t': + if (inquotes) + break; + else + { + // token creation + // skip blank char + // exit from loop + char *token = new_token(stream, i); + } + default: + break; + } + i++; + } + + remaining_chars -= i; +} diff --git a/src/lexer/lexer.h b/src/lexer/lexer.h index e69de29..d9a484c 100644 --- a/src/lexer/lexer.h +++ b/src/lexer/lexer.h @@ -0,0 +1,36 @@ +#ifndef LEXER_H +#define LEXER_H + +/* @return: char*, the next token + * + */ +char *get_token(void); + +/* + * @warning: NOT IMPLEMENTED. + * + * @note: maybe usefull for subshells. + */ + +char *get_token_str(void); + +/* + * @brief: return a newly allocated token. + * This token contains [size] chars, starting from [begin]. + * + * @return: NULL on error, null-terminated char* otherwise. + * + */ +char *new_token(char *begin, ssize_t size); + +/* + * @brief: checks if the stream used for the last token creation is empty. + * If it is, it calls stream_read() from IO_backend, + * and sets [remaing_chars]. + * If not, it starts from the end of the last token. + * + * @return: char* stream from which we tokenise. + */ +char *stream_init(void); + +#endif /* LEXER_H */ From af1af49ae177e60d04abda494589bdefc0ef04d8 Mon Sep 17 00:00:00 2001 From: Matteo Flebus Date: Thu, 8 Jan 2026 17:03:52 +0100 Subject: [PATCH 3/3] feat(utils): string utils + corresponding tests --- src/utils/string_utils/string_utils.c | 16 +++++ src/utils/string_utils/string_utils.h | 10 +++ tests/unit/lexer/lexer_tests.c | 39 +++++------ tests/unit/utils/utils_tests.c | 97 +++++++++++++++++++++++++++ 4 files changed, 140 insertions(+), 22 deletions(-) create mode 100644 src/utils/string_utils/string_utils.c create mode 100644 src/utils/string_utils/string_utils.h create mode 100644 tests/unit/utils/utils_tests.c diff --git a/src/utils/string_utils/string_utils.c b/src/utils/string_utils/string_utils.c new file mode 100644 index 0000000..6e07f7e --- /dev/null +++ b/src/utils/string_utils/string_utils.c @@ -0,0 +1,16 @@ +#include "utils/string_utils/string_utils.h" + +ssize_t skip_blanks(char **str) +{ + if (str == NULL || *str == NULL) + { + return 0; + } + ssize_t skipped = 0; + while (str[skipped] != '\0' && !isblank(str[skipped])) + { + skipped++; + } + *str += skipped; + return skipped; +} diff --git a/src/utils/string_utils/string_utils.h b/src/utils/string_utils/string_utils.h new file mode 100644 index 0000000..3fee923 --- /dev/null +++ b/src/utils/string_utils/string_utils.h @@ -0,0 +1,10 @@ +#ifndef STRING_UTILS_H +#define STRING_UTILS_H + +/* + * @brief: skips blank characters at the beginning of [str]. + * @return: number of characters skipped. + */ +ssize_t skip_blanks(char **str); + +#endif /* STRING_UTILS_H */ diff --git a/tests/unit/lexer/lexer_tests.c b/tests/unit/lexer/lexer_tests.c index 0a3ccf7..d9046d4 100644 --- a/tests/unit/lexer/lexer_tests.c +++ b/tests/unit/lexer/lexer_tests.c @@ -1,8 +1,9 @@ #include #include -#include +#include #include #include +#include #include "lexer/lexer.h" @@ -10,50 +11,44 @@ TestSuite(token_creation); Test(token_creation, basic) { - char *input = "Hello World"; + char input[] = "Hello World"; - char *actual = new_token(input, 5); - char *expected = "Hello"; + char actual[] = new_token(input, 5); + char expected[] = "Hello"; cr_expect(eq(str, actual, expected)); free(actual); } Test(token_creation, nul) { - char *input = NULL; + char input[] = NULL; - char *actual = new_token(input, 5); - char *expected = NULL; - cr_expect(eq(str, actual, expected)); - free(actual); + char actual[] = new_token(input, 5); + cr_expect(actual == NULL); } Test(token_creation, too_large) { - char *input = "Hel"; + char input[] = "Hel"; - char *actual = new_token(input, 5); - char *expected = NULL; - cr_expect(eq(str, actual, expected)); - free(actual); + char actual[] = new_token(input, 5); + cr_expect(actual == NULL); } Test(token_creation, empty) { - char *input = ""; + char input[] = ""; - char *actual = new_token(input, 5); - char *expected = NULL; - cr_expect(eq(str, actual, expected)); - free(actual); + char actual[] = new_token(input, 5); + cr_expect(actual == NULL); } Test(token_creation, basic_long) { - char *input = "Hello World! This project is a mini shell, I love BIG G."; + char input[] = "Hello World! This project is a mini shell, I love BIG G."; - char *actual = new_token(input, 42); - char *expected = calloc(42 + 1, sizeof(char)); + char actual[] = new_token(input, 42); + char expected[] = calloc(42 + 1, sizeof(char)); strncpy(input, expected, 42); cr_expect(eq(str, actual, expected)); free(actual); diff --git a/tests/unit/utils/utils_tests.c b/tests/unit/utils/utils_tests.c new file mode 100644 index 0000000..1076aab --- /dev/null +++ b/tests/unit/utils/utils_tests.c @@ -0,0 +1,97 @@ +#include +#include +#include +#include +#include +#include + +#include "utils/string_utils/string_utils.h" + +TestSuite(string_utils); + +Test(string_utils, skipblank_basic) +{ + char input[] = " Hello World"; + char expected_str[] = "Hello World"; + + ssize_t actual = skip_blanks(input); + ssize_t expected = 2; + cr_expect(eq(str, input, expected_str)); + cr_expect(actual == expected); +} + +Test(string_utils, skipblank_noblank) +{ + char input[] = "Hello World"; + char expected_str[] = "Hello World"; + + ssize_t actual = skip_blanks(input); + ssize_t expected = 0; + cr_expect(eq(str, input, expected_str)); + cr_expect(actual == expected); +} + +Test(string_utils, skipblank_tab) +{ + char input[] = "\tHello World"; + char expected_str[] = "Hello World"; + + ssize_t actual = skip_blanks(input); + ssize_t expected = 1; + cr_expect(eq(str, input, expected_str)); + cr_expect(actual == expected); +} + +Test(string_utils, skipblank_space_tab) +{ + char input[] = " \tHello World"; + char expected_str[] = "Hello World"; + + ssize_t actual = skip_blanks(input); + ssize_t expected = 2; + cr_expect(eq(str, input, expected_str)); + cr_expect(actual == expected); +} + +Test(string_utils, skipblank_2tab_1space) +{ + char input[] = "\t \tHello World"; + char expected_str[] = "Hello World"; + + ssize_t actual = skip_blanks(input); + ssize_t expected = 3; + cr_expect(eq(str, input, expected_str)); + cr_expect(actual == expected); +} + +Test(string_utils, skipblank_a_lot) +{ + char input[] = "\t \t \tHello World"; + char expected_str[] = "Hello World"; + + ssize_t actual = skip_blanks(input); + ssize_t expected = 8; + cr_expect(eq(str, input, expected_str)); + cr_expect(actual == expected); +} + +Test(string_utils, skipblank_newline) +{ + char input[] = "\nHello World"; + char expected_str[] = "\nHello World"; + + ssize_t actual = skip_blanks(input); + ssize_t expected = 0; + cr_expect(eq(str, input, expected_str)); + cr_expect(actual == expected); +} + +Test(string_utils, skipblank_nul) +{ + char *input = NULL; + + ssize_t actual = skip_blanks(input); + ssize_t expected = 0; + cr_expect(input == NULL); + cr_expect(actual == expected); +}