127 lines
1.8 KiB
C
127 lines
1.8 KiB
C
|
#ifndef TOKENIZER_H
|
||
|
|
#define TOKENIZER_H
|
||
|
|
|
||
|
|
#include "common.h"
|
||
|
|
|
||
|
|
typedef enum TokenType
|
||
|
|
{
|
||
|
|
// Single char tokens
|
||
|
|
TOKEN_LEFT_PAREN,
|
||
|
|
TOKEN_RIGHT_PAREN,
|
||
|
|
TOKEN_LEFT_BRACE,
|
||
|
|
TOKEN_RIGHT_BRACE,
|
||
|
|
TOKEN_TILDE,
|
||
|
|
TOKEN_SLASH,
|
||
|
|
TOKEN_MINUS,
|
||
|
|
// Literals
|
||
|
|
TOKEN_IDENTIFIER,
|
||
|
|
TOKEN_STRING,
|
||
|
|
TOKEN_FLOAT,
|
||
|
|
TOKEN_LIST,
|
||
|
|
TOKEN_ERROR,
|
||
|
|
TOKEN_FALSE,
|
||
|
|
TOKEN_TRUE,
|
||
|
|
TOKEN_PI,
|
||
|
|
TOKEN_E,
|
||
|
|
TOKEN_EOF,
|
||
|
|
// Keywords
|
||
|
|
TOKEN_POP,
|
||
|
|
TOKEN_DUP,
|
||
|
|
TOKEN_EXCH,
|
||
|
|
TOKEN_CLEAR,
|
||
|
|
TOKEN_REMEMBER,
|
||
|
|
TOKEN_FORGET,
|
||
|
|
TOKEN_DUMP,
|
||
|
|
TOKEN_NAME,
|
||
|
|
TOKEN_SET,
|
||
|
|
TOKEN_IFYES,
|
||
|
|
TOKEN_IFNO,
|
||
|
|
TOKEN_CHOOSE,
|
||
|
|
TOKEN_EVAL,
|
||
|
|
TOKEN_ESCAPE,
|
||
|
|
TOKEN_REPEAT,
|
||
|
|
TOKEN_SPLIT,
|
||
|
|
TOKEN_CONS,
|
||
|
|
TOKEN_SHATTER,
|
||
|
|
TOKEN_EMPTY,
|
||
|
|
TOKEN_COMPOSE,
|
||
|
|
TOKEN_STREQ,
|
||
|
|
TOKEN_STRCUT,
|
||
|
|
TOKEN_STRMEASURE,
|
||
|
|
TOKEN_STRTIE,
|
||
|
|
TOKEN_EXPLODE,
|
||
|
|
TOKEN_ADD,
|
||
|
|
TOKEN_SUB,
|
||
|
|
TOKEN_MUL,
|
||
|
|
TOKEN_DIV,
|
||
|
|
TOKEN_IDIV,
|
||
|
|
TOKEN_MOD,
|
||
|
|
TOKEN_POW,
|
||
|
|
TOKEN_SQRT,
|
||
|
|
TOKEN_ADD1,
|
||
|
|
TOKEN_SUB1,
|
||
|
|
TOKEN_SIN,
|
||
|
|
TOKEN_COS,
|
||
|
|
TOKEN_TAN,
|
||
|
|
TOKEN_ATAN,
|
||
|
|
TOKEN_LN,
|
||
|
|
TOKEN_LOG,
|
||
|
|
TOKEN_LOG3,
|
||
|
|
TOKEN_CLIP,
|
||
|
|
TOKEN_SMOOTH,
|
||
|
|
TOKEN_HOWMUCH,
|
||
|
|
TOKEN_SETRAND,
|
||
|
|
TOKEN_RAND,
|
||
|
|
TOKEN_INT,
|
||
|
|
TOKEN_NUMBERIZE,
|
||
|
|
TOKEN_ISOLATE,
|
||
|
|
TOKEN_MIX,
|
||
|
|
TOKEN_CONTRADICT,
|
||
|
|
TOKEN_COMPL,
|
||
|
|
TOKEN_SHIFTRIGHT,
|
||
|
|
TOKEN_SHIFTLEFT,
|
||
|
|
TOKEN_GT,
|
||
|
|
TOKEN_LT,
|
||
|
|
TOKEN_EQ,
|
||
|
|
TOKEN_GE,
|
||
|
|
TOKEN_LE,
|
||
|
|
TOKEN_NE,
|
||
|
|
TOKEN_NULL,
|
||
|
|
TOKEN_NEGATIVE,
|
||
|
|
TOKEN_ISNULL,
|
||
|
|
TOKEN_ISINT,
|
||
|
|
TOKEN_ISNUMBER,
|
||
|
|
TOKEN_AND,
|
||
|
|
TOKEN_OR,
|
||
|
|
TOKEN_XOR,
|
||
|
|
TOKEN_DISP,
|
||
|
|
TOKEN_LISTEN,
|
||
|
|
TOKEN_COMPLAIN,
|
||
|
|
TOKEN_TIME,
|
||
|
|
TOKEN_GARBAGE_COLLECT,
|
||
|
|
} TokenType;
|
||
|
|
|
||
|
|
typedef struct Token Token;
|
||
|
|
struct Token
|
||
|
|
{
|
||
|
|
TokenType type;
|
||
|
|
char *start;
|
||
|
|
int32_t length;
|
||
|
|
int32_t line;
|
||
|
|
};
|
||
|
|
|
||
|
|
typedef struct SourceCode SourceCode;
|
||
|
|
struct SourceCode
|
||
|
|
{
|
||
|
|
size_t size;
|
||
|
|
char *buf;
|
||
|
|
int count;
|
||
|
|
};
|
||
|
|
|
||
|
|
void initTokenizer (char *src);
|
||
|
|
Token nextToken ();
|
||
|
|
void initMap ();
|
||
|
|
void debug_printToken (Token t);
|
||
|
|
|
||
|
|
#endif
|