fix: handle a backreference to a non-participating capture group

lexer/stateful.go

@@ -127,10 +127,20 @@ type compiledRule struct {
 // compiledRules grouped by name.
 type compiledRules map[string][]compiledRule
 
+// capture is the text matched by a single regex capture group.
+//
+// A group that did not participate in the match at all is distinct from one
+// that matched the empty string: a backreference to the former cannot match,
+// while a backreference to the latter matches the empty string.
+type capture struct {
+	text    string
+	present bool
+}
+
 // A Action is applied when a rule matches.
 type Action interface {
 	// Actions are responsible for validating the match. ie. if they consumed any input.
-	applyAction(lexer *StatefulLexer, groups []string) error
+	applyAction(lexer *StatefulLexer, groups []capture) error
 }
 
 // RulesAction is an optional interface that Actions can implement.
@@ -147,8 +157,8 @@ type validatingRule interface {
 // ActionPop pops to the previous state when the Rule matches.
 type ActionPop struct{}
 
-func (p ActionPop) applyAction(lexer *StatefulLexer, groups []string) error {
-	if groups[0] == "" {
+func (p ActionPop) applyAction(lexer *StatefulLexer, groups []capture) error {
+	if groups[0].text == "" {
 		return errors.New("did not consume any input")
 	}
 	lexer.stack = lexer.stack[:len(lexer.stack)-1]
@@ -173,8 +183,8 @@ type ActionPush struct {
 	State string `json:"state"`
 }
 
-func (p ActionPush) applyAction(lexer *StatefulLexer, groups []string) error {
-	if groups[0] == "" {
+func (p ActionPush) applyAction(lexer *StatefulLexer, groups []capture) error {
+	if groups[0].text == "" {
 		return errors.New("did not consume any input")
 	}
 	lexer.stack = append(lexer.stack, lexerState{name: p.State, groups: groups})
@@ -200,7 +210,7 @@ type include struct {
 	State string `json:"state"`
 }
 
-func (i include) applyAction(_ *StatefulLexer, _ []string) error {
+func (i include) applyAction(_ *StatefulLexer, _ []capture) error {
 	panic("should not be called")
 }
 
@@ -351,7 +361,7 @@ func (d *StatefulDefinition) Symbols() map[string]TokenType {
 // lexerState stored when switching states in the lexer.
 type lexerState struct {
 	name   string
-	groups []string
+	groups []capture
 }
 
 // StatefulLexer implementation.
@@ -402,9 +412,15 @@ next:
 		}
 
 		if rule.Action != nil {
-			groups := make([]string, 0, len(match)/2)
+			// A capture group that did not participate in the match has an
+			// offset of -1. Keep its slot, so that groups[n] is always group n
+			// for backreferences to resolve against, and record that it was
+			// absent so that a backreference to it cannot match.
+			groups := make([]capture, len(match)/2)
 			for i := 0; i < len(match); i += 2 {
-				groups = append(groups, l.data[match[i]:match[i+1]])
+				if match[i] >= 0 {
+					groups[i/2] = capture{text: l.data[match[i]:match[i+1]], present: true}
+				}
 			}
 			if err := rule.Action.applyAction(l, groups); err != nil {
 				return Token{}, errorf(l.pos, "lexer: rule %q: %s", rule.Name, err)
@@ -440,12 +456,29 @@ func (l *StatefulLexer) getPattern(candidate compiledRule) (*regexp.Regexp, erro
 	}
 	// We don't have a compiled RE. This means there are back-references
 	// that need to be substituted first.
-	return BackrefRegex(&l.def.backrefCache, candidate.Pattern, l.stack[len(l.stack)-1].groups)
+	return backrefRegex(&l.def.backrefCache, candidate.Pattern, l.stack[len(l.stack)-1].groups)
 }
 
 // BackrefRegex returns a compiled regular expression with backreferences replaced by groups.
 func BackrefRegex(backrefCache *sync.Map, input string, groups []string) (*regexp.Regexp, error) {
-	key := input + "\000" + strings.Join(groups, "\000")
+	captures := make([]capture, len(groups))
+	for i, group := range groups {
+		captures[i] = capture{text: group, present: true}
+	}
+	return backrefRegex(backrefCache, input, captures)
+}
+
+// neverMatch is a zero-width expression that can never match, because a
+// position cannot be both a word boundary and not a word boundary. It stands in
+// for a backreference to a capture group that did not participate in the match,
+// so that the expression around it still composes: "\1" cannot match, while
+// "\1?" remains optional.
+const neverMatch = `(?:\b\B)`
+
+// backrefRegex returns a compiled regular expression with backreferences
+// replaced by the parent's captures.
+func backrefRegex(backrefCache *sync.Map, input string, groups []capture) (*regexp.Regexp, error) {
+	key := backrefCacheKey(input, groups)
 	cached, ok := backrefCache.Load(key)
 	if ok {
 		return cached.(*regexp.Regexp), nil
@@ -455,20 +488,32 @@ func BackrefRegex(backrefCache *sync.Map, input string, groups []string) (*regex
 		re  *regexp.Regexp
 		err error
 	)
-	pattern := backrefReplace.ReplaceAllStringFunc(input, func(s string) string {
-		var rematch = backrefReplace.FindStringSubmatch(s)
-		n, nerr := strconv.ParseInt(rematch[2], 10, 64)
+	inClass := characterClassSpans(input)
+	expanded := strings.Builder{}
+	last := 0
+	for _, match := range backrefReplace.FindAllStringSubmatchIndex(input, -1) {
+		start, end := match[0], match[1]
+		slashes := input[match[2]:match[3]]
+		expanded.WriteString(input[last:start])
+		last = end
+
+		n, nerr := strconv.ParseInt(input[match[4]:match[5]], 10, 64)
 		if nerr != nil {
 			err = nerr
-			return s
+			expanded.WriteString(input[start:end])
+			continue
 		}
 		if len(groups) == 0 || int(n) >= len(groups) {
 			err = fmt.Errorf("invalid group %d from parent with %d groups", n, len(groups))
-			return s
+			expanded.WriteString(input[start:end])
+			continue
 		}
 		// concatenate the leading \\\\ which are already escaped to the quoted match.
-		return rematch[1][:len(rematch[1])-1] + regexp.QuoteMeta(groups[n])
-	})
+		expanded.WriteString(slashes[:len(slashes)-1])
+		expanded.WriteString(expandBackref(groups[n], inClass[start]))
+	}
+	expanded.WriteString(input[last:])
+	pattern := expanded.String()
 	if err == nil {
 		re, err = regexp.Compile("^(?:" + pattern + ")")
 	}
@@ -478,3 +523,123 @@ func BackrefRegex(backrefCache *sync.Map, input string, groups []string) (*regex
 	backrefCache.Store(key, re)
 	return re, nil
 }
+
+// expandBackref returns the text a backreference expands to, which depends on
+// where it sits: a character class holds members, everywhere else holds a
+// subexpression.
+func expandBackref(group capture, inCharacterClass bool) string {
+	if inCharacterClass {
+		// Grouping syntax is not grouping syntax inside a class. Wrapping here
+		// would add "(", "?", ":" and ")" to the set, so "[\1]" capturing "a"
+		// would also match a parenthesis. An absent capture contributes no
+		// members at all, which leaves the rest of the class to decide, or
+		// fails to compile when there is no rest -- the same as the empty
+		// capture this expansion has always produced there.
+		if !group.present {
+			return ""
+		}
+		return regexp.QuoteMeta(group.text)
+	}
+	if !group.present {
+		return neverMatch
+	}
+	// A single group, so that a quantified or alternated backreference still
+	// parses when the capture is empty.
+	return "(?:" + regexp.QuoteMeta(group.text) + ")"
+}
+
+// characterClassSpans reports for each byte of input whether it lies inside a
+// character class. RE2 has no nested classes, so one flag is enough; what it
+// does have is escapes, \Q...\E literal runs, POSIX names such as [:alpha:]
+// whose brackets neither open nor close a class, and a "]" that is the first
+// member of a class, which is a literal rather than the delimiter: "[]a]" is
+// the two-member class RE2 accepts, not an empty class it rejects.
+func characterClassSpans(input string) []bool {
+	inClass := make([]bool, len(input))
+	open, literal := false, false
+	// Index of the "[" that opened the current class, so the first member can
+	// be recognised. -1 while no class is open.
+	start := -1
+	for i := 0; i < len(input); i++ {
+		inClass[i] = open
+		switch {
+		case literal:
+			if input[i] == '\\' && i+1 < len(input) && input[i+1] == 'E' {
+				literal = false
+				i++
+				inClass[i] = open
+			}
+		case input[i] == '\\':
+			if i+1 < len(input) {
+				literal = input[i+1] == 'Q'
+				i++
+				inClass[i] = open
+			}
+		case input[i] == '[':
+			if !open {
+				open, start = true, i
+			} else if end := posixClassEnd(input, i); end >= 0 {
+				for ; i <= end; i++ {
+					inClass[i] = true
+				}
+				i--
+			}
+		case input[i] == ']':
+			if !firstClassMember(input, start, i) {
+				open, start = false, -1
+			}
+		}
+	}
+	return inClass
+}
+
+// firstClassMember reports whether the byte at i is the first member of the
+// class opened at start, where a "]" stands for itself instead of closing.
+// That is the position right after "[", or right after "[^".
+func firstClassMember(input string, start, i int) bool {
+	if start < 0 {
+		return false
+	}
+	if i == start+1 {
+		return true
+	}
+	return i == start+2 && input[start+1] == '^'
+}
+
+// posixClassEnd returns the index of the "]" closing a POSIX name such as
+// [:alpha:] opening at open, or -1 when that is not what starts there.
+func posixClassEnd(input string, open int) int {
+	if open+1 >= len(input) || input[open+1] != ':' {
+		return -1
+	}
+	end := strings.Index(input[open+2:], ":]")
+	if end < 0 {
+		return -1
+	}
+	return open + 2 + end + 1
+}
+
+// backrefCacheKey builds a key identifying a pattern together with the parent
+// captures it is expanded against. Lengths are included because the captures
+// themselves may contain the separator.
+func backrefCacheKey(input string, groups []capture) string {
+	size := len(input) + 8
+	for _, group := range groups {
+		size += len(group.text) + 8
+	}
+	key := strings.Builder{}
+	key.Grow(size)
+	key.WriteString(strconv.Itoa(len(input)))
+	key.WriteString(":")
+	key.WriteString(input)
+	for _, group := range groups {
+		key.WriteString("\000")
+		if !group.present {
+			continue
+		}
+		key.WriteString(strconv.Itoa(len(group.text)))
+		key.WriteString(":")
+		key.WriteString(group.text)
+	}
+	return key.String()
+}