dnrops
/
radare2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update tree-sitter to 0.16.8 (#16991) 

* Fix CentOS6 CI by specifying PKG_CONFIG_PATH
* Do not define var inside for-loop
This commit is contained in:
Riccardo Schirone 2020-06-03 15:01:47 +02:00 committed by GitHub
parent 1fef151d37
commit a14f5c0777
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 2200 additions and 2182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.github/workflows/centos6.yml vendored
View File

@ -32,3 +32,8 @@ jobs:
- name: Run tests
run: cd test/unit && make
working-directory: radare2
env:
# `make install` installs, for some unknown reasons, pkgconfig files in
# /usr/lib and not in /usr/lib64, thus pkg-config cannot find the right
# .pc files if the right path is not specified
PKG_CONFIG_PATH: /usr/lib/pkgconfig

2
shlr/Makefile
View File

@ -29,7 +29,7 @@ endif
# NOTE: when you update TS_TIP or TS_BRA, also update them in shlr/meson.build
TS_URL=https://github.com/tree-sitter/tree-sitter.git
TS_BRA=master
TS_TIP=f049ba350f3f6019ce9a1cbb0975ebd154ef7ad3
TS_TIP=9a82dcc666d06617cbab3061467075019fae0b0d
ifeq ($(CS_RELEASE),1)
CS_VER=4.0.1

2
shlr/meson.build
View File

@ -244,7 +244,7 @@ if not tree_sitter_dep.found() or not get_option('use_sys_tree_sitter')
endif
# NOTE: when you update TS_TIP or TS_BRA, also update them in shlr/Makefile
TS_TIP = 'f049ba350f3f6019ce9a1cbb0975ebd154ef7ad3'
TS_TIP = '9a82dcc666d06617cbab3061467075019fae0b0d'
TS_BRA = 'master'
message('Deleting existing directories @0@ and @1@'.format(tree_sitter_vc_path, tree_sitter_path))

6
shlr/radare2-shell-parser/package-lock.json generated
View File

@ -10,9 +10,9 @@
"integrity": "sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg=="
},
"tree-sitter-cli": {
"version": "0.16.4",
"resolved": "https://registry.npmjs.org/tree-sitter-cli/-/tree-sitter-cli-0.16.4.tgz",
"integrity": "sha512-akCVeK7oOZD+frizRbBx3h6OBlVBxOCNtfpt9nz3zvOdRuJTwoyJUshzF28J+hfcuvQ+yfoZx9/R+2S7NZE2TA==",
"version": "0.16.8",
"resolved": "https://registry.npmjs.org/tree-sitter-cli/-/tree-sitter-cli-0.16.8.tgz",
"integrity": "sha512-gMVZ0tpSCk1DQoyrSXlRaFZ4HztyVTKqYBTXitYKHmM+mrEIznDGV70ycqsCQYDjCnE461naMViGOItDGJDIHw==",
"dev": true
}
}

2
shlr/radare2-shell-parser/package.json
View File

@ -12,6 +12,6 @@
"nan": "^2.14.0"
},
"devDependencies": {
"tree-sitter-cli": "^0.16.4"
"tree-sitter-cli": "^0.16.8"
}
}

2912
shlr/radare2-shell-parser/src/parser.c generated
View File

File diff suppressed because it is too large Load Diff

50
shlr/radare2-shell-parser/src/tree_sitter/parser.h
View File

@ -62,13 +62,13 @@ typedef struct {
TSStateId state;
bool extra : 1;
bool repetition : 1;
};
} shift;
struct {
TSSymbol symbol;
int16_t dynamic_precedence;
uint8_t child_count;
uint8_t production_id;
};
} reduce;
} params;
TSParseActionType type : 4;
} TSParseAction;
@ -83,7 +83,7 @@ typedef union {
struct {
uint8_t count;
bool reusable : 1;
};
} entry;
} TSParseActionEntry;
struct TSLanguage {
@ -167,22 +167,28 @@ struct TSLanguage {
#define ACTIONS(id) id
#define SHIFT(state_value) \
{ \
{ \
.type = TSParseActionTypeShift, \
.params = {.state = state_value}, \
} \
#define SHIFT(state_value) \
{ \
{ \
.params = { \
.shift = { \
.state = state_value \
} \
}, \
.type = TSParseActionTypeShift \
} \
}
#define SHIFT_REPEAT(state_value) \
{ \
{ \
.type = TSParseActionTypeShift, \
.params = { \
.state = state_value, \
.repetition = true \
.shift = { \
.state = state_value, \
.repetition = true \
} \
}, \
.type = TSParseActionTypeShift \
} \
}
@ -194,20 +200,26 @@ struct TSLanguage {
#define SHIFT_EXTRA() \
{ \
{ \
.type = TSParseActionTypeShift, \
.params = {.extra = true} \
.params = { \
.shift = { \
.extra = true \
} \
}, \
.type = TSParseActionTypeShift \
} \
}
#define REDUCE(symbol_val, child_count_val, ...) \
{ \
{ \
.type = TSParseActionTypeReduce, \
.params = { \
.symbol = symbol_val, \
.child_count = child_count_val, \
__VA_ARGS__ \
} \
.reduce = { \
.symbol = symbol_val, \
.child_count = child_count_val, \
__VA_ARGS__ \
}, \
}, \
.type = TSParseActionTypeReduce \
} \
}

50
shlr/tree-sitter/lib/include/tree_sitter/parser.h
View File

@ -62,13 +62,13 @@ typedef struct {
TSStateId state;
bool extra : 1;
bool repetition : 1;
};
} shift;
struct {
TSSymbol symbol;
int16_t dynamic_precedence;
uint8_t child_count;
uint8_t production_id;
};
} reduce;
} params;
TSParseActionType type : 4;
} TSParseAction;
@ -83,7 +83,7 @@ typedef union {
struct {
uint8_t count;
bool reusable : 1;
};
} entry;
} TSParseActionEntry;
struct TSLanguage {
@ -167,22 +167,28 @@ struct TSLanguage {
#define ACTIONS(id) id
#define SHIFT(state_value) \
{ \
{ \
.type = TSParseActionTypeShift, \
.params = {.state = state_value}, \
} \
#define SHIFT(state_value) \
{ \
{ \
.params = { \
.shift = { \
.state = state_value \
} \
}, \
.type = TSParseActionTypeShift \
} \
}
#define SHIFT_REPEAT(state_value) \
{ \
{ \
.type = TSParseActionTypeShift, \
.params = { \
.state = state_value, \
.repetition = true \
.shift = { \
.state = state_value, \
.repetition = true \
} \
}, \
.type = TSParseActionTypeShift \
} \
}
@ -194,20 +200,26 @@ struct TSLanguage {
#define SHIFT_EXTRA() \
{ \
{ \
.type = TSParseActionTypeShift, \
.params = {.extra = true} \
.params = { \
.shift = { \
.extra = true \
} \
}, \
.type = TSParseActionTypeShift \
} \
}
#define REDUCE(symbol_val, child_count_val, ...) \
{ \
{ \
.type = TSParseActionTypeReduce, \
.params = { \
.symbol = symbol_val, \
.child_count = child_count_val, \
__VA_ARGS__ \
} \
.reduce = { \
.symbol = symbol_val, \
.child_count = child_count_val, \
__VA_ARGS__ \
}, \
}, \
.type = TSParseActionTypeReduce \
} \
}

28
shlr/tree-sitter/lib/src/array.h
View File

@ -126,12 +126,28 @@ static inline void array__splice(VoidArray *self, size_t element_size,
array__reserve(self, element_size, new_size);
char *contents = (char *)self->contents;
if (self->size > old_end)
memmove(contents + new_end * element_size, contents + old_end * element_size,
(self->size - old_end) * element_size);
if (new_count > 0)
memcpy((contents + index * element_size), elements,
new_count * element_size);
if (self->size > old_end) {
memmove(
contents + new_end * element_size,
contents + old_end * element_size,
(self->size - old_end) * element_size
);
}
if (new_count > 0) {
if (elements) {
memcpy(
(contents + index * element_size),
elements,
new_count * element_size
);
} else {
memset(
(contents + index * element_size),
0,
new_count * element_size
);
}
}
self->size += new_count - old_count;
}

10
shlr/tree-sitter/lib/src/language.c
View File

@ -33,8 +33,8 @@ void ts_language_table_entry(
assert(symbol < self->token_count);
uint32_t action_index = ts_language_lookup(self, state, symbol);
const TSParseActionEntry *entry = &self->parse_actions[action_index];
result->action_count = entry->count;
result->is_reusable = entry->reusable;
result->action_count = entry->entry.count;
result->is_reusable = entry->entry.reusable;
result->actions = (const TSParseAction *)(entry + 1);
}
}
@ -72,8 +72,10 @@ const char *ts_language_symbol_name(
return "ERROR";
} else if (symbol == ts_builtin_sym_error_repeat) {
return "_ERROR";
} else {
} else if (symbol < ts_language_symbol_count(self)) {
return self->symbol_names[symbol];
} else {
return NULL;
}
}
@ -119,7 +121,7 @@ const char *ts_language_field_name_for_id(
TSFieldId id
) {
uint32_t count = ts_language_field_count(self);
if (count) {
if (count && id <= count) {
return self->field_names[id];
} else {
return NULL;

14
shlr/tree-sitter/lib/src/language.h
View File

@ -29,10 +29,12 @@ static inline bool ts_language_is_symbol_external(const TSLanguage *self, TSSymb
return 0 < symbol && symbol < self->external_token_count + 1;
}
static inline const TSParseAction *ts_language_actions(const TSLanguage *self,
TSStateId state,
TSSymbol symbol,
uint32_t *count) {
static inline const TSParseAction *ts_language_actions(
const TSLanguage *self,
TSStateId state,
TSSymbol symbol,
uint32_t *count
) {
TableEntry entry;
ts_language_table_entry(self, state, symbol, &entry);
*count = entry.action_count;
@ -90,8 +92,8 @@ static inline TSStateId ts_language_next_state(const TSLanguage *self,
const TSParseAction *actions = ts_language_actions(self, state, symbol, &count);
if (count > 0) {
TSParseAction action = actions[count - 1];
if (action.type == TSParseActionTypeShift || action.type == TSParseActionTypeRecover) {
return action.params.state;
if (action.type == TSParseActionTypeShift) {
return action.params.shift.extra ? state : action.params.shift.state;
}
}
return 0;

4
shlr/tree-sitter/lib/src/node.c
View File

@ -150,7 +150,9 @@ static inline TSNode ts_node__child(
while (ts_node_child_iterator_next(&iterator, &child)) {
if (ts_node__is_relevant(child, include_anonymous)) {
if (index == child_index) {
ts_tree_set_cached_parent(self.tree, &child, &self);
if (ts_node__is_relevant(self, true)) {
ts_tree_set_cached_parent(self.tree, &child, &self);
}
return child;
}
index++;

46
shlr/tree-sitter/lib/src/parser.c
View File

@ -324,6 +324,12 @@ static bool ts_parser__can_reuse_first_leaf(
TSStateId leaf_state = ts_subtree_leaf_parse_state(tree);
TSLexMode leaf_lex_mode = self->language->lex_modes[leaf_state];
// At the end of a non-terminal extra node, the lexer normally returns
// NULL, which indicates that the parser should look for a reduce action
// at symbol `0`. Avoid reusing tokens in this situation to ensure that
// the same thing happens when incrementally reparsing.
if (current_lex_mode.lex_state == (uint16_t)(-1)) return false;
// If the token was created in a state with the same set of lookaheads, it is reusable.
if (
table_entry->action_count > 0 &&
@ -593,6 +599,10 @@ static Subtree ts_parser__reuse_node(
uint32_t byte_offset = reusable_node_byte_offset(&self->reusable_node);
uint32_t end_byte_offset = byte_offset + ts_subtree_total_bytes(result);
// Do not reuse an EOF node if the included ranges array has changes
// later on in the file.
if (ts_subtree_is_eof(result)) end_byte_offset = UINT32_MAX;
if (byte_offset > position) {
LOG("before_reusable_node symbol:%s", TREE_NAME(result));
break;
@ -941,15 +951,15 @@ static bool ts_parser__do_all_potential_reductions(
switch (action.type) {
case TSParseActionTypeShift:
case TSParseActionTypeRecover:
if (!action.params.extra && !action.params.repetition) has_shift_action = true;
if (!action.params.shift.extra && !action.params.shift.repetition) has_shift_action = true;
break;
case TSParseActionTypeReduce:
if (action.params.child_count > 0)
if (action.params.reduce.child_count > 0)
ts_reduce_action_set_add(&self->reduce_actions, (ReduceAction){
.symbol = action.params.symbol,
.count = action.params.child_count,
.dynamic_precedence = action.params.dynamic_precedence,
.production_id = action.params.production_id,
.symbol = action.params.reduce.symbol,
.count = action.params.reduce.child_count,
.dynamic_precedence = action.params.reduce.dynamic_precedence,
.production_id = action.params.reduce.production_id,
});
default:
break;
@ -1013,7 +1023,9 @@ static void ts_parser__handle_error(
TSStateId state_after_missing_symbol = ts_language_next_state(
self->language, state, missing_symbol
);
if (state_after_missing_symbol == 0) continue;
if (state_after_missing_symbol == 0 || state_after_missing_symbol == state) {
continue;
}
if (ts_language_has_reduce_action(
self->language,
@ -1238,7 +1250,7 @@ static void ts_parser__recover(
// be counted in error cost calculations.
unsigned n;
const TSParseAction *actions = ts_language_actions(self->language, 1, ts_subtree_symbol(lookahead), &n);
if (n > 0 && actions[n - 1].type == TSParseActionTypeShift && actions[n - 1].params.extra) {
if (n > 0 && actions[n - 1].type == TSParseActionTypeShift && actions[n - 1].params.shift.extra) {
MutableSubtree mutable_lookahead = ts_subtree_make_mut(&self->tree_pool, lookahead);
ts_subtree_set_extra(&mutable_lookahead);
lookahead = ts_subtree_from_mut(mutable_lookahead);
@ -1367,9 +1379,9 @@ static bool ts_parser__advance(
switch (action.type) {
case TSParseActionTypeShift: {
if (action.params.repetition) break;
if (action.params.shift.repetition) break;
TSStateId next_state;
if (action.params.extra) {
if (action.params.shift.extra) {
// TODO: remove when TREE_SITTER_LANGUAGE_VERSION 9 is out.
if (state == ERROR_STATE) continue;
@ -1377,7 +1389,7 @@ static bool ts_parser__advance(
next_state = state;
LOG("shift_extra");
} else {
next_state = action.params.state;
next_state = action.params.shift.state;
LOG("shift state:%u", next_state);
}
@ -1386,7 +1398,7 @@ static bool ts_parser__advance(
next_state = ts_language_next_state(self->language, state, ts_subtree_symbol(lookahead));
}
ts_parser__shift(self, version, next_state, lookahead, action.params.extra);
ts_parser__shift(self, version, next_state, lookahead, action.params.shift.extra);
if (did_reuse) reusable_node_advance(&self->reusable_node);
return true;
}
@ -1394,10 +1406,10 @@ static bool ts_parser__advance(
case TSParseActionTypeReduce: {
bool is_fragile = table_entry.action_count > 1;
bool is_extra = lookahead.ptr == NULL;
LOG("reduce sym:%s, child_count:%u", SYM_NAME(action.params.symbol), action.params.child_count);
LOG("reduce sym:%s, child_count:%u", SYM_NAME(action.params.reduce.symbol), action.params.reduce.child_count);
StackVersion reduction_version = ts_parser__reduce(
self, version, action.params.symbol, action.params.child_count,
action.params.dynamic_precedence, action.params.production_id,
self, version, action.params.reduce.symbol, action.params.reduce.child_count,
action.params.reduce.dynamic_precedence, action.params.reduce.production_id,
is_fragile, is_extra
);
if (reduction_version != STACK_VERSION_NONE) {
@ -1603,8 +1615,8 @@ static unsigned ts_parser__condense_stack(TSParser *self) {
static bool ts_parser_has_outstanding_parse(TSParser *self) {
return (
self->lexer.current_position.bytes > 0 ||
ts_stack_state(self->stack, 0) != 1
ts_stack_state(self->stack, 0) != 1 ||
ts_stack_node_count_since_error(self->stack, 0) != 0
);
}

1224
shlr/tree-sitter/lib/src/query.c
View File

File diff suppressed because it is too large Load Diff

12
shlr/tree-sitter/lib/src/subtree.c
View File

@ -21,7 +21,7 @@ typedef struct {
#define TS_MAX_INLINE_TREE_LENGTH UINT8_MAX
#define TS_MAX_TREE_POOL_SIZE 32
static const ExternalScannerState empty_state = {.length = 0, .short_data = {0}};
static const ExternalScannerState empty_state = {{.short_data = {0}}, .length = 0};
// ExternalScannerState
@ -208,7 +208,7 @@ Subtree ts_subtree_new_leaf(
.has_external_tokens = has_external_tokens,
.is_missing = false,
.is_keyword = is_keyword,
.first_leaf = {.symbol = 0, .parse_state = 0},
{{.first_leaf = {.symbol = 0, .parse_state = 0}}}
};
return (Subtree) {.ptr = data};
}
@ -464,15 +464,17 @@ MutableSubtree ts_subtree_new_node(SubtreePool *pool, TSSymbol symbol,
*data = (SubtreeHeapData) {
.ref_count = 1,
.symbol = symbol,
.production_id = production_id,
.visible = metadata.visible,
.named = metadata.named,
.has_changes = false,
.fragile_left = fragile,
.fragile_right = fragile,
.is_keyword = false,
.node_count = 0,
.first_leaf = {.symbol = 0, .parse_state = 0},
{{
.node_count = 0,
.production_id = production_id,
.first_leaf = {.symbol = 0, .parse_state = 0},
}}
};
MutableSubtree result = {.ptr = data};
ts_subtree_set_children(result, children->contents, children->size, language);

15
test/unit/test_list.c
View File

@ -239,20 +239,19 @@ bool test_r_list_mergesort_pint() {
-80, -76};
RList* list = r_list_new();
for (size_t i = 0; i < R_ARRAY_SIZE(data); ++i) {
r_list_append (list, (void*)&data[i]);
size_t i;
for (i = 0; i < R_ARRAY_SIZE (data); i++) {
r_list_append (list, (void *)&data[i]);
}
// invoke sorting
r_list_sort (list, (RListComparator)pintcmp);
// assert the list is sorted as expected
size_t i = 0;
RListIter* iter = list->head;
do {
mu_assert_eq(*(int*)iter->data, expected[i], "array content mismatch");
iter = iter->n;
} while (++i < R_ARRAY_SIZE(expected));
RListIter* iter;
for (i = 0, iter = list->head; i < R_ARRAY_SIZE (expected); i++, iter = iter->n) {
mu_assert_eq (*(int *)iter->data, expected[i], "array content mismatch");
}
r_list_free(list);
mu_end;