From be1cb9db59c5e4d3c383d71db3b0e861bab1ce9b Mon Sep 17 00:00:00 2001 From: snowyukitty <270071858+snowyukitty@users.noreply.github.com> Date: Mon, 20 Jul 2026 08:08:35 +0900 Subject: [PATCH 1/3] fix: handle a backreference to a non-participating capture group FindStringSubmatchIndex returns -1 offsets for a capture group that did not participate in the match, such as the inner group of `(x)*` when it repeats zero times. Slicing the input with those offsets panicked with "slice bounds out of range [:-1]" for any rule that has an action. Record a capture as its text plus whether the group participated, and expand a backreference to a group that did not participate into an expression that can never match rather than into nothing: - groups[n] stays capture group n, so a backreference in a pushed state still resolves to the group its pattern names; - "\1" cannot match when group 1 did not participate, as in engines with real backreferences, while "\1?" stays optional; - a group that participated and matched the empty string still expands to an empty match, which is a different thing. An expansion is now always a single group, so that a quantified or alternated backreference still parses when the capture is empty. "\1?" previously expanded to "?" and failed to compile. Fixes #324 --- lexer/stateful.go | 88 ++++++++++++++++++++++++++++++++++++------ lexer/stateful_test.go | 75 +++++++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+), 12 deletions(-) diff --git a/lexer/stateful.go b/lexer/stateful.go index 89207d29..c3959f9e 100644 --- a/lexer/stateful.go +++ b/lexer/stateful.go @@ -127,10 +127,20 @@ type compiledRule struct { // compiledRules grouped by name. type compiledRules map[string][]compiledRule +// capture is the text matched by a single regex capture group. +// +// A group that did not participate in the match at all is distinct from one +// that matched the empty string: a backreference to the former cannot match, +// while a backreference to the latter matches the empty string. +type capture struct { + text string + present bool +} + // A Action is applied when a rule matches. type Action interface { // Actions are responsible for validating the match. ie. if they consumed any input. - applyAction(lexer *StatefulLexer, groups []string) error + applyAction(lexer *StatefulLexer, groups []capture) error } // RulesAction is an optional interface that Actions can implement. @@ -147,8 +157,8 @@ type validatingRule interface { // ActionPop pops to the previous state when the Rule matches. type ActionPop struct{} -func (p ActionPop) applyAction(lexer *StatefulLexer, groups []string) error { - if groups[0] == "" { +func (p ActionPop) applyAction(lexer *StatefulLexer, groups []capture) error { + if groups[0].text == "" { return errors.New("did not consume any input") } lexer.stack = lexer.stack[:len(lexer.stack)-1] @@ -173,8 +183,8 @@ type ActionPush struct { State string `json:"state"` } -func (p ActionPush) applyAction(lexer *StatefulLexer, groups []string) error { - if groups[0] == "" { +func (p ActionPush) applyAction(lexer *StatefulLexer, groups []capture) error { + if groups[0].text == "" { return errors.New("did not consume any input") } lexer.stack = append(lexer.stack, lexerState{name: p.State, groups: groups}) @@ -200,7 +210,7 @@ type include struct { State string `json:"state"` } -func (i include) applyAction(_ *StatefulLexer, _ []string) error { +func (i include) applyAction(_ *StatefulLexer, _ []capture) error { panic("should not be called") } @@ -351,7 +361,7 @@ func (d *StatefulDefinition) Symbols() map[string]TokenType { // lexerState stored when switching states in the lexer. type lexerState struct { name string - groups []string + groups []capture } // StatefulLexer implementation. @@ -402,9 +412,15 @@ next: } if rule.Action != nil { - groups := make([]string, 0, len(match)/2) + // A capture group that did not participate in the match has an + // offset of -1. Keep its slot, so that groups[n] is always group n + // for backreferences to resolve against, and record that it was + // absent so that a backreference to it cannot match. + groups := make([]capture, len(match)/2) for i := 0; i < len(match); i += 2 { - groups = append(groups, l.data[match[i]:match[i+1]]) + if match[i] >= 0 { + groups[i/2] = capture{text: l.data[match[i]:match[i+1]], present: true} + } } if err := rule.Action.applyAction(l, groups); err != nil { return Token{}, errorf(l.pos, "lexer: rule %q: %s", rule.Name, err) @@ -440,12 +456,29 @@ func (l *StatefulLexer) getPattern(candidate compiledRule) (*regexp.Regexp, erro } // We don't have a compiled RE. This means there are back-references // that need to be substituted first. - return BackrefRegex(&l.def.backrefCache, candidate.Pattern, l.stack[len(l.stack)-1].groups) + return backrefRegex(&l.def.backrefCache, candidate.Pattern, l.stack[len(l.stack)-1].groups) } // BackrefRegex returns a compiled regular expression with backreferences replaced by groups. func BackrefRegex(backrefCache *sync.Map, input string, groups []string) (*regexp.Regexp, error) { - key := input + "\000" + strings.Join(groups, "\000") + captures := make([]capture, len(groups)) + for i, group := range groups { + captures[i] = capture{text: group, present: true} + } + return backrefRegex(backrefCache, input, captures) +} + +// neverMatch is a zero-width expression that can never match, because a +// position cannot be both a word boundary and not a word boundary. It stands in +// for a backreference to a capture group that did not participate in the match, +// so that the expression around it still composes: "\1" cannot match, while +// "\1?" remains optional. +const neverMatch = `(?:\b\B)` + +// backrefRegex returns a compiled regular expression with backreferences +// replaced by the parent's captures. +func backrefRegex(backrefCache *sync.Map, input string, groups []capture) (*regexp.Regexp, error) { + key := backrefCacheKey(input, groups) cached, ok := backrefCache.Load(key) if ok { return cached.(*regexp.Regexp), nil @@ -467,7 +500,13 @@ func BackrefRegex(backrefCache *sync.Map, input string, groups []string) (*regex return s } // concatenate the leading \\\\ which are already escaped to the quoted match. - return rematch[1][:len(rematch[1])-1] + regexp.QuoteMeta(groups[n]) + // The expansion is always a single group, so that a quantified or + // alternated backreference still parses when the capture is empty. + prefix := rematch[1][:len(rematch[1])-1] + if !groups[n].present { + return prefix + neverMatch + } + return prefix + "(?:" + regexp.QuoteMeta(groups[n].text) + ")" }) if err == nil { re, err = regexp.Compile("^(?:" + pattern + ")") @@ -478,3 +517,28 @@ func BackrefRegex(backrefCache *sync.Map, input string, groups []string) (*regex backrefCache.Store(key, re) return re, nil } + +// backrefCacheKey builds a key identifying a pattern together with the parent +// captures it is expanded against. Lengths are included because the captures +// themselves may contain the separator. +func backrefCacheKey(input string, groups []capture) string { + size := len(input) + 8 + for _, group := range groups { + size += len(group.text) + 8 + } + key := strings.Builder{} + key.Grow(size) + key.WriteString(strconv.Itoa(len(input))) + key.WriteString(":") + key.WriteString(input) + for _, group := range groups { + key.WriteString("\000") + if !group.present { + continue + } + key.WriteString(strconv.Itoa(len(group.text))) + key.WriteString(":") + key.WriteString(group.text) + } + return key.String() +} diff --git a/lexer/stateful_test.go b/lexer/stateful_test.go index 4c147ff2..56de1a0d 100644 --- a/lexer/stateful_test.go +++ b/lexer/stateful_test.go @@ -41,6 +41,24 @@ func TestMarshalUnmarshal(t *testing.T) { require.Equal(t, interpolatedRules, unmarshalledRules) } +// A capture group that did not participate in the match ("Opt" with no "a") is +// distinct from one that matched the empty string ("Star" with no "a"): "\1>" +// cannot match at all in the former, and matches ">" in the latter. +var backrefParticipationRules = lexer.Rules{ + "Root": { + {"Opt", `OPT`, lexer.Push("Opt")}, + {"Star", `STAR`, lexer.Push("Star")}, + }, + "Opt": {{"OptOpen", `<(a)?`, lexer.Push("Body")}, lexer.Return()}, + "Star": {{"StarOpen", `<(a*)`, lexer.Push("Body")}, lexer.Return()}, + "Body": { + {"Close", `\1>`, lexer.Pop()}, + {"Text", `[a-z]+`, nil}, + {"Abort", `>!`, lexer.Pop()}, + {"Loose", `\1?#`, lexer.Pop()}, + }, +} + func TestStatefulLexer(t *testing.T) { tests := []struct { name string @@ -174,6 +192,63 @@ func TestStatefulLexer(t *testing.T) { input: "hello", err: "1:1: lexer: rule \"NoMatch\": did not consume any input", }, + {name: "NonParticipatingGroup", + rules: lexer.Rules{ + "Root": { + {"Percent", `%`, lexer.Push("Percent")}, + {"whitespace", ` +`, nil}, + }, + "Percent": { + {"Ident", `[[:alpha:]]([[:alnum:]])*`, lexer.Pop()}, + }, + }, + input: `%x %ab`, + tokens: []string{"%", "x", "%", "ab"}, + }, + {name: "BackrefToNonParticipatingGroup", + rules: backrefParticipationRules, + input: `OPT!`, + tokens: []string{"OPT", "<", "zz", ">!"}, + }, + {name: "BackrefToEmptyParticipatingGroup", + rules: backrefParticipationRules, + input: `STAR<>`, + tokens: []string{"STAR", "<", ">"}, + }, + {name: "OptionalBackrefToNonParticipatingGroup", + rules: backrefParticipationRules, + input: `OPT!STAR<>`, + tokens: []string{"OPT", "<", "zz", ">!", "STAR", "<", ">"}, + }, + {name: "BackrefAfterNonParticipatingGroup", + rules: lexer.Rules{ + "Root": { + {"Heredoc", `<<(-)?(\w+)@(\w+)\b`, lexer.Push("Heredoc")}, + lexer.Include("Common"), + }, + "Heredoc": { + {"End", `\b\2\b`, lexer.Pop()}, + {"Line", `[^\n]+`, nil}, + lexer.Include("Common"), + }, + "Common": { + {"Whitespace", `\s+`, nil}, + {"Ident", `\w+`, nil}, + }, + }, + input: "< Date: Mon, 7 Sep 2026 18:41:02 +0900 Subject: [PATCH 2/3] fix: expand a backreference inside a character class as members The grouping this change added so that `\1?` and `a|\1` still parse when the capture is empty is not grouping inside a character class. `[\1]` capturing `a` expanded to `[(?:a)]`, a class that also matches `(`, `?`, `:` and `)`, and `[[:alpha:]\1]` capturing `-` expanded to `[[:alpha:](?:-)]`, which does not compile at all because `:-)` is read as a range. The expansion now depends on where the backreference sits. Outside a class it is unchanged. Inside one it is the quoted capture alone, as it was before this branch, and a capture that did not participate contributes no members rather than a zero-width expression that a class cannot hold. A class left with nothing fails to compile, which is what an empty capture in that position has always done. `characterClassSpans` finds those positions in one pass. It has to know about escapes, `\Q...\E` runs, and POSIX names such as `[:alpha:]`, whose `]` closes the name rather than the class. --- lexer/stateful.go | 107 ++++++++++++++++++++++++++++++++++++----- lexer/stateful_test.go | 49 +++++++++++++++++++ 2 files changed, 143 insertions(+), 13 deletions(-) diff --git a/lexer/stateful.go b/lexer/stateful.go index c3959f9e..b1a71d55 100644 --- a/lexer/stateful.go +++ b/lexer/stateful.go @@ -488,26 +488,32 @@ func backrefRegex(backrefCache *sync.Map, input string, groups []capture) (*rege re *regexp.Regexp err error ) - pattern := backrefReplace.ReplaceAllStringFunc(input, func(s string) string { - var rematch = backrefReplace.FindStringSubmatch(s) - n, nerr := strconv.ParseInt(rematch[2], 10, 64) + inClass := characterClassSpans(input) + expanded := strings.Builder{} + last := 0 + for _, match := range backrefReplace.FindAllStringSubmatchIndex(input, -1) { + start, end := match[0], match[1] + slashes := input[match[2]:match[3]] + expanded.WriteString(input[last:start]) + last = end + + n, nerr := strconv.ParseInt(input[match[4]:match[5]], 10, 64) if nerr != nil { err = nerr - return s + expanded.WriteString(input[start:end]) + continue } if len(groups) == 0 || int(n) >= len(groups) { err = fmt.Errorf("invalid group %d from parent with %d groups", n, len(groups)) - return s + expanded.WriteString(input[start:end]) + continue } // concatenate the leading \\\\ which are already escaped to the quoted match. - // The expansion is always a single group, so that a quantified or - // alternated backreference still parses when the capture is empty. - prefix := rematch[1][:len(rematch[1])-1] - if !groups[n].present { - return prefix + neverMatch - } - return prefix + "(?:" + regexp.QuoteMeta(groups[n].text) + ")" - }) + expanded.WriteString(slashes[:len(slashes)-1]) + expanded.WriteString(expandBackref(groups[n], inClass[start])) + } + expanded.WriteString(input[last:]) + pattern := expanded.String() if err == nil { re, err = regexp.Compile("^(?:" + pattern + ")") } @@ -518,6 +524,81 @@ func backrefRegex(backrefCache *sync.Map, input string, groups []capture) (*rege return re, nil } +// expandBackref returns the text a backreference expands to, which depends on +// where it sits: a character class holds members, everywhere else holds a +// subexpression. +func expandBackref(group capture, inCharacterClass bool) string { + if inCharacterClass { + // Grouping syntax is not grouping syntax inside a class. Wrapping here + // would add "(", "?", ":" and ")" to the set, so "[\1]" capturing "a" + // would also match a parenthesis. An absent capture contributes no + // members at all, which leaves the rest of the class to decide, or + // fails to compile when there is no rest -- the same as the empty + // capture this expansion has always produced there. + if !group.present { + return "" + } + return regexp.QuoteMeta(group.text) + } + if !group.present { + return neverMatch + } + // A single group, so that a quantified or alternated backreference still + // parses when the capture is empty. + return "(?:" + regexp.QuoteMeta(group.text) + ")" +} + +// characterClassSpans reports for each byte of input whether it lies inside a +// character class. RE2 has no nested classes, so one flag is enough; what it +// does have is escapes, \Q...\E literal runs, and POSIX names such as +// [:alpha:] whose brackets neither open nor close a class. +func characterClassSpans(input string) []bool { + inClass := make([]bool, len(input)) + open, literal := false, false + for i := 0; i < len(input); i++ { + inClass[i] = open + switch { + case literal: + if input[i] == '\\' && i+1 < len(input) && input[i+1] == 'E' { + literal = false + i++ + inClass[i] = open + } + case input[i] == '\\': + if i+1 < len(input) { + literal = input[i+1] == 'Q' + i++ + inClass[i] = open + } + case input[i] == '[': + if !open { + open = true + } else if end := posixClassEnd(input, i); end >= 0 { + for ; i <= end; i++ { + inClass[i] = true + } + i-- + } + case input[i] == ']': + open = false + } + } + return inClass +} + +// posixClassEnd returns the index of the "]" closing a POSIX name such as +// [:alpha:] opening at open, or -1 when that is not what starts there. +func posixClassEnd(input string, open int) int { + if open+1 >= len(input) || input[open+1] != ':' { + return -1 + } + end := strings.Index(input[open+2:], ":]") + if end < 0 { + return -1 + } + return open + 2 + end + 1 +} + // backrefCacheKey builds a key identifying a pattern together with the parent // captures it is expanded against. Lengths are included because the captures // themselves may contain the separator. diff --git a/lexer/stateful_test.go b/lexer/stateful_test.go index 56de1a0d..320201ce 100644 --- a/lexer/stateful_test.go +++ b/lexer/stateful_test.go @@ -59,6 +59,40 @@ var backrefParticipationRules = lexer.Rules{ }, } +// A backreference inside a character class contributes members to the set +// rather than a subexpression, so it is expanded without the grouping the +// contexts above need: "[^\1]" holding "(?:)" would exclude a parenthesis, a +// question mark and a colon along with the delimiter. +var backrefCharacterClassRules = lexer.Rules{ + "Root": { + {"Open", `<(.)`, lexer.Push("Body")}, + }, + "Body": { + {"Close", `\1`, lexer.Pop()}, + {"Chunk", `[^\1]+`, nil}, + }, +} + +// The same, for a class the backreference is a member of rather than excluded +// from, and for one that also holds a POSIX name -- whose "]" closes the name +// and not the class. +var backrefInClassRules = lexer.Rules{ + "Root": { + {"Fill", `<(.)`, lexer.Push("Fill")}, + {"Word", `>(.)`, lexer.Push("Word")}, + }, + "Fill": { + {"Run", `[\1]+`, nil}, + {"Paren", `\(`, nil}, + {"FillEnd", `!`, lexer.Pop()}, + }, + "Word": { + {"Letters", `[[:alpha:]\1]+`, nil}, + {"Paren", `\(`, nil}, + {"WordEnd", `!`, lexer.Pop()}, + }, +} + func TestStatefulLexer(t *testing.T) { tests := []struct { name string @@ -249,6 +283,21 @@ func TestStatefulLexer(t *testing.T) { input: "<-a-b(c!`, + tokens: []string{">-", "a-b", "(", "c", "!"}, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { From 291bfb43da03c8fd20228c052608da5791f7c680 Mon Sep 17 00:00:00 2001 From: snowyukitty <270071858+snowyukitty@users.noreply.github.com> Date: Tue, 8 Sep 2026 01:18:51 +0900 Subject: [PATCH 3/3] fix: read a leading "]" in a character class as a member The previous commit decided where a backreference sits by scanning for character classes, and that scanner closed a class on the first "]" it found. RE2 does not: a "]" that is the first member of a class -- right after "[" or "[^" -- stands for itself, so "[]a]" is the two-member class it accepts rather than an empty class it rejects. So "[]\1]" put the backreference outside a class as far as the scanner was concerned, and it was expanded as a subexpression again: capturing "a" it became "[](?:a)]", a class that also matches "(", "?", ":" and ")". That is the same defect the previous commit fixed for "[\1]", surviving in the one class shape that shape does not cover. firstClassMember recognises those two positions, and the scanner leaves the class open across them. --- lexer/stateful.go | 28 ++++++++++++++++++++++++---- lexer/stateful_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/lexer/stateful.go b/lexer/stateful.go index b1a71d55..ed944dee 100644 --- a/lexer/stateful.go +++ b/lexer/stateful.go @@ -550,11 +550,16 @@ func expandBackref(group capture, inCharacterClass bool) string { // characterClassSpans reports for each byte of input whether it lies inside a // character class. RE2 has no nested classes, so one flag is enough; what it -// does have is escapes, \Q...\E literal runs, and POSIX names such as -// [:alpha:] whose brackets neither open nor close a class. +// does have is escapes, \Q...\E literal runs, POSIX names such as [:alpha:] +// whose brackets neither open nor close a class, and a "]" that is the first +// member of a class, which is a literal rather than the delimiter: "[]a]" is +// the two-member class RE2 accepts, not an empty class it rejects. func characterClassSpans(input string) []bool { inClass := make([]bool, len(input)) open, literal := false, false + // Index of the "[" that opened the current class, so the first member can + // be recognised. -1 while no class is open. + start := -1 for i := 0; i < len(input); i++ { inClass[i] = open switch { @@ -572,7 +577,7 @@ func characterClassSpans(input string) []bool { } case input[i] == '[': if !open { - open = true + open, start = true, i } else if end := posixClassEnd(input, i); end >= 0 { for ; i <= end; i++ { inClass[i] = true @@ -580,12 +585,27 @@ func characterClassSpans(input string) []bool { i-- } case input[i] == ']': - open = false + if !firstClassMember(input, start, i) { + open, start = false, -1 + } } } return inClass } +// firstClassMember reports whether the byte at i is the first member of the +// class opened at start, where a "]" stands for itself instead of closing. +// That is the position right after "[", or right after "[^". +func firstClassMember(input string, start, i int) bool { + if start < 0 { + return false + } + if i == start+1 { + return true + } + return i == start+2 && input[start+1] == '^' +} + // posixClassEnd returns the index of the "]" closing a POSIX name such as // [:alpha:] opening at open, or -1 when that is not what starts there. func posixClassEnd(input string, open int) int { diff --git a/lexer/stateful_test.go b/lexer/stateful_test.go index 320201ce..76e9bbe8 100644 --- a/lexer/stateful_test.go +++ b/lexer/stateful_test.go @@ -93,6 +93,28 @@ var backrefInClassRules = lexer.Rules{ }, } +// A "]" that is the first member of a class stands for itself instead of +// closing it, so "[]\1]" is a two-member class and the backreference is inside +// one. Reading that "]" as the delimiter puts the backreference outside, where +// it is expanded as a subexpression and the parenthesis, question mark and +// colon join the class. +var backrefLeadingBracketRules = lexer.Rules{ + "Root": { + {"In", `<(.)`, lexer.Push("In")}, + {"Not", `>(.)`, lexer.Push("Not")}, + }, + "In": { + {"InRun", `[]\1]+`, nil}, + {"Paren", `\(`, nil}, + {"InEnd", `!`, lexer.Pop()}, + }, + "Not": { + {"NotRun", `[^]\1]+`, nil}, + {"Bracket", `\]`, nil}, + {"NotEnd", `!`, lexer.Pop()}, + }, +} + func TestStatefulLexer(t *testing.T) { tests := []struct { name string @@ -298,6 +320,16 @@ func TestStatefulLexer(t *testing.T) { input: `>-a-b(c!`, tokens: []string{">-", "a-b", "(", "c", "!"}, }, + {name: "BackrefAfterLeadingBracketInCharacterClass", + rules: backrefLeadingBracketRules, + input: `abc(d]!`, + tokens: []string{">a", "bc(d", "]", "!"}, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) {