1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
|
#ifndef TOKEN_H
#define TOKEN_H
#include <sys/types.h>
/*
* This describes the pure lexical elements (tokens), with
* no semantic meaning. In other words, an identifier doesn't
* have a type or meaning, it is only a specific string in
* the input stream.
*
* Semantic meaning is handled elsewhere.
*/
struct stream {
int fd;
const char *name;
/* Use these to check for "already parsed" */
int constant;
dev_t dev;
ino_t ino;
};
extern int input_stream_nr;
extern struct stream *input_streams;
extern int ident_hit, ident_miss;
struct ident {
struct ident *next; /* Hash chain of identifiers */
struct symbol *symbols; /* Pointer to semantic meaning list */
unsigned char len; /* Length of identifier name */
char name[]; /* Actual identifier */
};
enum token_type {
TOKEN_EOF,
TOKEN_ERROR,
TOKEN_IDENT,
TOKEN_INTEGER,
TOKEN_FP,
TOKEN_CHAR,
TOKEN_STRING,
TOKEN_SPECIAL,
TOKEN_STREAMBEGIN,
TOKEN_STREAMEND,
};
/* Combination tokens */
#define COMBINATION_STRINGS { \
"+=", "++", \
"-=", "--", "->", \
"*=", \
"/=", "/*", "//", \
"%=", \
"..", "...", \
"<=", "<<", "<<=", \
">=", ">>", ">>=", \
"==", "!=", \
"&&", "&=", \
"||", "|=", \
"^=", "##", \
" @ ", \
}
enum special_token {
SPECIAL_BASE = 256,
SPECIAL_ADD_ASSIGN = 256,
SPECIAL_INCREMENT,
SPECIAL_MINUS_ASSIGN,
SPECIAL_DECREMENT,
SPECIAL_DEREFERENCE,
SPECIAL_TIMES_ASSIGN,
SPECIAL_DIV_ASSIGN,
SPECIAL_COMMENT,
SPECIAL_CPPCOMMENT,
SPECIAL_MOD_ASSIGN,
SPECIAL_DOTDOT,
SPECIAL_ELLIPSIS,
SPECIAL_LTE,
SPECIAL_LEFTSHIFT,
SPECIAL_SHL_ASSIGN,
SPECIAL_GTE,
SPECIAL_RIGHTSHIFT,
SPECIAL_SHR_ASSIGN,
SPECIAL_EQUAL,
SPECIAL_NOTEQUAL,
SPECIAL_LOGICAL_AND,
SPECIAL_AND_ASSIGN,
SPECIAL_LOGICAL_OR,
SPECIAL_OR_ASSIGN,
SPECIAL_XOR_ASSIGN,
SPECIAL_HASHHASH,
SPECIAL_ARG_SEPARATOR
};
struct string {
unsigned int length;
char data[];
};
/*
* This is a very common data structure, it should be kept
* as small as humanly possible. Big (rare) types go as
* pointers.
*/
struct token {
unsigned int type:8,
stream:8,
pos:14,
newline:1,
whitespace:1;
unsigned int line;
struct token *next;
union {
char *integer;
char *fp;
struct ident *ident;
unsigned int special;
struct string *string;
int character;
};
};
/*
* Last token in the stream - points to itself.
* This allows us to not test for NULL pointers
* when following the token->next chain..
*/
extern struct token eof_token_entry;
#define eof_token(x) ((x) == &eof_token_entry)
extern int init_stream(const char *, int fd);
extern struct ident *hash_ident(struct ident *);
extern struct ident *built_in_ident(const char *);
extern struct token *built_in_token(int, const char *);
extern const char *show_special(int);
extern const char *show_ident(const struct ident *);
extern const char *show_token(const struct token *);
extern struct token * tokenize(const char *, int, struct token *end);
extern void die(const char *, ...);
extern void show_identifier_stats(void);
extern struct token *preprocess(struct token *);
static inline int match_op(struct token *token, int op)
{
return token->type == TOKEN_SPECIAL && token->special == op;
}
static inline int match_ident(struct token *token, struct ident *id)
{
return token->type == TOKEN_IDENT && token->ident == id;
}
#endif
|
GitWeb