Skip to content

Commit 516d277

Browse files
guilliamxavierpetk
guilliamxavier
authored and
petk
committed
Fix tokenizer_data_gen.sh for non-posix bison
And run it to update tokenizer_data.c after recent changes in zend_language_parser.y that reordered some tokens
1 parent 6b362e0 commit 516d277

File tree

2 files changed

+22
-22
lines changed

2 files changed

+22
-22
lines changed

‎ext/tokenizer/tokenizer_data.c

+16-16
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929
void tokenizer_register_constants(INIT_FUNC_ARGS) {
3030
REGISTER_LONG_CONSTANT("T_INCLUDE", T_INCLUDE, CONST_CS | CONST_PERSISTENT);
3131
REGISTER_LONG_CONSTANT("T_INCLUDE_ONCE", T_INCLUDE_ONCE, CONST_CS | CONST_PERSISTENT);
32-
REGISTER_LONG_CONSTANT("T_EVAL", T_EVAL, CONST_CS | CONST_PERSISTENT);
3332
REGISTER_LONG_CONSTANT("T_REQUIRE", T_REQUIRE, CONST_CS | CONST_PERSISTENT);
3433
REGISTER_LONG_CONSTANT("T_REQUIRE_ONCE", T_REQUIRE_ONCE, CONST_CS | CONST_PERSISTENT);
3534
REGISTER_LONG_CONSTANT("T_LOGICAL_OR", T_LOGICAL_OR, CONST_CS | CONST_PERSISTENT);
@@ -79,13 +78,6 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
7978
REGISTER_LONG_CONSTANT("T_CLONE", T_CLONE, CONST_CS | CONST_PERSISTENT);
8079
REGISTER_LONG_CONSTANT("T_ELSEIF", T_ELSEIF, CONST_CS | CONST_PERSISTENT);
8180
REGISTER_LONG_CONSTANT("T_ELSE", T_ELSE, CONST_CS | CONST_PERSISTENT);
82-
REGISTER_LONG_CONSTANT("T_ENDIF", T_ENDIF, CONST_CS | CONST_PERSISTENT);
83-
REGISTER_LONG_CONSTANT("T_STATIC", T_STATIC, CONST_CS | CONST_PERSISTENT);
84-
REGISTER_LONG_CONSTANT("T_ABSTRACT", T_ABSTRACT, CONST_CS | CONST_PERSISTENT);
85-
REGISTER_LONG_CONSTANT("T_FINAL", T_FINAL, CONST_CS | CONST_PERSISTENT);
86-
REGISTER_LONG_CONSTANT("T_PRIVATE", T_PRIVATE, CONST_CS | CONST_PERSISTENT);
87-
REGISTER_LONG_CONSTANT("T_PROTECTED", T_PROTECTED, CONST_CS | CONST_PERSISTENT);
88-
REGISTER_LONG_CONSTANT("T_PUBLIC", T_PUBLIC, CONST_CS | CONST_PERSISTENT);
8981
REGISTER_LONG_CONSTANT("T_LNUMBER", T_LNUMBER, CONST_CS | CONST_PERSISTENT);
9082
REGISTER_LONG_CONSTANT("T_DNUMBER", T_DNUMBER, CONST_CS | CONST_PERSISTENT);
9183
REGISTER_LONG_CONSTANT("T_STRING", T_STRING, CONST_CS | CONST_PERSISTENT);
@@ -95,8 +87,10 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
9587
REGISTER_LONG_CONSTANT("T_CONSTANT_ENCAPSED_STRING", T_CONSTANT_ENCAPSED_STRING, CONST_CS | CONST_PERSISTENT);
9688
REGISTER_LONG_CONSTANT("T_STRING_VARNAME", T_STRING_VARNAME, CONST_CS | CONST_PERSISTENT);
9789
REGISTER_LONG_CONSTANT("T_NUM_STRING", T_NUM_STRING, CONST_CS | CONST_PERSISTENT);
90+
REGISTER_LONG_CONSTANT("T_EVAL", T_EVAL, CONST_CS | CONST_PERSISTENT);
9891
REGISTER_LONG_CONSTANT("T_EXIT", T_EXIT, CONST_CS | CONST_PERSISTENT);
9992
REGISTER_LONG_CONSTANT("T_IF", T_IF, CONST_CS | CONST_PERSISTENT);
93+
REGISTER_LONG_CONSTANT("T_ENDIF", T_ENDIF, CONST_CS | CONST_PERSISTENT);
10094
REGISTER_LONG_CONSTANT("T_ECHO", T_ECHO, CONST_CS | CONST_PERSISTENT);
10195
REGISTER_LONG_CONSTANT("T_DO", T_DO, CONST_CS | CONST_PERSISTENT);
10296
REGISTER_LONG_CONSTANT("T_WHILE", T_WHILE, CONST_CS | CONST_PERSISTENT);
@@ -125,6 +119,12 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
125119
REGISTER_LONG_CONSTANT("T_USE", T_USE, CONST_CS | CONST_PERSISTENT);
126120
REGISTER_LONG_CONSTANT("T_INSTEADOF", T_INSTEADOF, CONST_CS | CONST_PERSISTENT);
127121
REGISTER_LONG_CONSTANT("T_GLOBAL", T_GLOBAL, CONST_CS | CONST_PERSISTENT);
122+
REGISTER_LONG_CONSTANT("T_STATIC", T_STATIC, CONST_CS | CONST_PERSISTENT);
123+
REGISTER_LONG_CONSTANT("T_ABSTRACT", T_ABSTRACT, CONST_CS | CONST_PERSISTENT);
124+
REGISTER_LONG_CONSTANT("T_FINAL", T_FINAL, CONST_CS | CONST_PERSISTENT);
125+
REGISTER_LONG_CONSTANT("T_PRIVATE", T_PRIVATE, CONST_CS | CONST_PERSISTENT);
126+
REGISTER_LONG_CONSTANT("T_PROTECTED", T_PROTECTED, CONST_CS | CONST_PERSISTENT);
127+
REGISTER_LONG_CONSTANT("T_PUBLIC", T_PUBLIC, CONST_CS | CONST_PERSISTENT);
128128
REGISTER_LONG_CONSTANT("T_VAR", T_VAR, CONST_CS | CONST_PERSISTENT);
129129
REGISTER_LONG_CONSTANT("T_UNSET", T_UNSET, CONST_CS | CONST_PERSISTENT);
130130
REGISTER_LONG_CONSTANT("T_ISSET", T_ISSET, CONST_CS | CONST_PERSISTENT);
@@ -170,7 +170,6 @@ char *get_token_type_name(int token_type)
170170

171171
case T_INCLUDE: return "T_INCLUDE";
172172
case T_INCLUDE_ONCE: return "T_INCLUDE_ONCE";
173-
case T_EVAL: return "T_EVAL";
174173
case T_REQUIRE: return "T_REQUIRE";
175174
case T_REQUIRE_ONCE: return "T_REQUIRE_ONCE";
176175
case T_LOGICAL_OR: return "T_LOGICAL_OR";
@@ -220,13 +219,6 @@ char *get_token_type_name(int token_type)
220219
case T_CLONE: return "T_CLONE";
221220
case T_ELSEIF: return "T_ELSEIF";
222221
case T_ELSE: return "T_ELSE";
223-
case T_ENDIF: return "T_ENDIF";
224-
case T_STATIC: return "T_STATIC";
225-
case T_ABSTRACT: return "T_ABSTRACT";
226-
case T_FINAL: return "T_FINAL";
227-
case T_PRIVATE: return "T_PRIVATE";
228-
case T_PROTECTED: return "T_PROTECTED";
229-
case T_PUBLIC: return "T_PUBLIC";
230222
case T_LNUMBER: return "T_LNUMBER";
231223
case T_DNUMBER: return "T_DNUMBER";
232224
case T_STRING: return "T_STRING";
@@ -236,8 +228,10 @@ char *get_token_type_name(int token_type)
236228
case T_CONSTANT_ENCAPSED_STRING: return "T_CONSTANT_ENCAPSED_STRING";
237229
case T_STRING_VARNAME: return "T_STRING_VARNAME";
238230
case T_NUM_STRING: return "T_NUM_STRING";
231+
case T_EVAL: return "T_EVAL";
239232
case T_EXIT: return "T_EXIT";
240233
case T_IF: return "T_IF";
234+
case T_ENDIF: return "T_ENDIF";
241235
case T_ECHO: return "T_ECHO";
242236
case T_DO: return "T_DO";
243237
case T_WHILE: return "T_WHILE";
@@ -266,6 +260,12 @@ char *get_token_type_name(int token_type)
266260
case T_USE: return "T_USE";
267261
case T_INSTEADOF: return "T_INSTEADOF";
268262
case T_GLOBAL: return "T_GLOBAL";
263+
case T_STATIC: return "T_STATIC";
264+
case T_ABSTRACT: return "T_ABSTRACT";
265+
case T_FINAL: return "T_FINAL";
266+
case T_PRIVATE: return "T_PRIVATE";
267+
case T_PROTECTED: return "T_PROTECTED";
268+
case T_PUBLIC: return "T_PUBLIC";
269269
case T_VAR: return "T_VAR";
270270
case T_UNSET: return "T_UNSET";
271271
case T_ISSET: return "T_ISSET";

‎ext/tokenizer/tokenizer_data_gen.sh

+6-6
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,8 @@ echo '/*
4444

4545
echo 'void tokenizer_register_constants(INIT_FUNC_ARGS) {' >> $OUTFILE
4646
$AWK '
47-
/^#define T_(NOELSE|ERROR)/ { next }
48-
/^#define T_/ { print " REGISTER_LONG_CONSTANT(\"" $2 "\", " $2 ", CONST_CS | CONST_PERSISTENT);" }
47+
/^ T_(NOELSE|ERROR)/ { next }
48+
/^ T_/ { print " REGISTER_LONG_CONSTANT(\"" $1 "\", " $1 ", CONST_CS | CONST_PERSISTENT);" }
4949
' < $INFILE >> $OUTFILE
5050
echo ' REGISTER_LONG_CONSTANT("T_DOUBLE_COLON", T_PAAMAYIM_NEKUDOTAYIM, CONST_CS | CONST_PERSISTENT);' >> $OUTFILE
5151
echo '}' >> $OUTFILE
@@ -58,13 +58,13 @@ char *get_token_type_name(int token_type)
5858
' >> $OUTFILE
5959

6060
$AWK '
61-
/^#define T_PAAMAYIM_NEKUDOTAYIM/ {
61+
/^ T_PAAMAYIM_NEKUDOTAYIM/ {
6262
print " case T_PAAMAYIM_NEKUDOTAYIM: return \"T_DOUBLE_COLON\";"
6363
next
6464
}
65-
/^#define T_(NOELSE|ERROR)/ { next }
66-
/^#define T_/ {
67-
print " case " $2 ": return \"" $2 "\";"
65+
/^ T_(NOELSE|ERROR)/ { next }
66+
/^ T_/ {
67+
print " case " $1 ": return \"" $1 "\";"
6868
}
6969
' < $INFILE >> $OUTFILE
7070

0 commit comments

Comments
 (0)