token.jou

import "stdlib/io.jou"
import "stdlib/str.jou"
import "stdlib/mem.jou"
import "./errors_and_warnings.jou"

# TODO: move to stdlib
declare isprint(b: int) -> int


@public
enum TokenKind:
    Short
    Int
    Long
    Float
    Double
    Byte    # example: 'a' is 97 as a byte
    String
    Name
    Keyword
    Decorator
    Newline
    Indent
    Dedent
    Operator
    EndOfFile  # Marks the end of an array of tokens.

@public
class Token:
    kind: TokenKind
    location: Location

    union:
        short_value: short      # Short
        int_value: int          # Int
        long_value: long        # Long
        byte_value: byte        # Byte
        indentation_level: int  # Newline (indicates how many spaces there are after the newline)
        short_string: byte[100] # Name, Keyword, Operator
        long_string: byte*      # String

    def print(self) -> None:
        match self->kind:
            case TokenKind.Byte:
                printf("byte %#02x", self->byte_value)
                if isprint(self->byte_value) != 0:
                    printf(" '%c'", self->byte_value)
                printf("\n")
            case TokenKind.Short:
                printf("short %hd\n", self->short_value)
            case TokenKind.Int:
                printf("integer %d\n", self->int_value)
            case TokenKind.Long:
                printf("long %lld\n", self->long_value)
            case TokenKind.Float:
                printf("float %s\n", self->short_string)
            case TokenKind.Double:
                printf("double %s\n", self->short_string)
            case TokenKind.EndOfFile:
                printf("end of file\n")
            case TokenKind.Operator:
                printf("operator '%s'\n", self->short_string)
            case TokenKind.Decorator:
                printf("decorator %s\n", self->short_string)
            case TokenKind.Name:
                printf("name \"%s\"\n", self->short_string)
            case TokenKind.Keyword:
                printf("keyword \"%s\"\n", self->short_string)
            case TokenKind.Newline:
                printf("newline token (next line has %d spaces of indentation)\n", self->indentation_level)
            case TokenKind.String:
                printf("string \"")
                for s = self->long_string; *s != 0; s++:
                    if isprint(*s) != 0:
                        putchar(*s)
                    elif *s == '\n':
                        printf("\\n")
                    else:
                        printf("\\x%02x", *s)
                printf("\"\n")
            case TokenKind.Indent:
                printf("indent (+4 spaces)\n")
            case TokenKind.Dedent:
                printf("dedent (-4 spaces)\n")

    def is_keyword(self, kw: byte*) -> bool:
        return self->kind == TokenKind.Keyword and strcmp(self->short_string, kw) == 0

    def is_operator(self, op: byte*) -> bool:
        return self->kind == TokenKind.Operator and strcmp(self->short_string, op) == 0

    def is_comparison(self) -> bool:
        return (
            self->is_operator("==")
            or self->is_operator("!=")
            or self->is_operator("<")
            or self->is_operator(">")
            or self->is_operator("<=")
            or self->is_operator(">=")
        )

    def is_open_paren(self) -> bool:
        return self->is_operator("(") or self->is_operator("[") or self->is_operator("{")

    def is_close_paren(self) -> bool:
        return self->is_operator(")") or self->is_operator("]") or self->is_operator("}")

    def fail_expected_got(self, what_was_expected_instead: byte*) -> None:
        got: byte[100]
        match self->kind:
            case TokenKind.Short:
                got = "a short"
            case TokenKind.Int:
                got = "an integer"
            case TokenKind.Long:
                got = "a long integer"
            case TokenKind.Float:
                got = "a float constant"
            case TokenKind.Double:
                got = "a double constant"
            case TokenKind.Byte:
                got = "a byte literal"
            case TokenKind.String:
                got = "a string"
            case TokenKind.Decorator:
                got = "a decorator"
            case TokenKind.Name:
                snprintf(got, sizeof got, "a variable name '%s'", self->short_string)
            case TokenKind.Keyword:
                snprintf(got, sizeof got, "the '%s' keyword", self->short_string)
            case TokenKind.Newline:
                got = "end of line"
            case TokenKind.Indent:
                got = "more indentation"
            case TokenKind.Dedent:
                got = "less indentation"
            case TokenKind.Operator:
                snprintf(got, sizeof got, "'%s'", self->short_string)
            case TokenKind.EndOfFile:
                got = "end of file"

        message: byte* = malloc(strlen(what_was_expected_instead) + 500)
        sprintf(message, "expected %s, got %s", what_was_expected_instead, got)
        fail(self->location, message)


@public
def print_tokens(tokens: Token*) -> None:
    printf("===== Tokens for file \"%s\" =====\n", tokens->location.path)
    t = tokens
    current_lineno = -1

    while True:
        if t->location.lineno != current_lineno:
            current_lineno = t->location.lineno
            printf("\nLine %d:\n", current_lineno)

        printf("  ")
        t->print()

        if t->kind == TokenKind.EndOfFile:
            break
        t++

    printf("\n")


@public
def free_tokens(tokenlist: Token*) -> None:
    for t = tokenlist; t->kind != TokenKind.EndOfFile; t++:
        if t->kind == TokenKind.String:
            free(t->long_string)
    free(tokenlist)