diff --git a/cmd/participle/gen_lexer_cmd.go b/cmd/participle/gen_lexer_cmd.go index d71e7eea..8308ccdf 100644 --- a/cmd/participle/gen_lexer_cmd.go +++ b/cmd/participle/gen_lexer_cmd.go @@ -153,6 +153,8 @@ func generateRegexMatch(w io.Writer, lexerName, name, pattern string) error { } ids := map[string]int{} idn := 0 + seq := 0 + questSkips := map[*syntax.Regexp]bool{} reid := func(re *syntax.Regexp) int { key := re.Op.String() + ":" + re.String() id, ok := ids[key] @@ -174,6 +176,21 @@ func generateRegexMatch(w io.Writer, lexerName, name, pattern string) error { fmt.Fprintf(w, "func match%s%s(s string, p int, backrefs []string) (groups [%d]int) {\n", lexerName, name, 2*re.MaxCap()+2) flattened := flatten(re) + // Pre-mark quest closures that appear as direct sub-expressions of a + // concat: emitConcatWithQuest inlines their inner expression rather than + // calling the quest closure, so emitting the quest closure would be dead + // code. Done before the flatten loop because flatten is post-order and + // emits the quest before its enclosing concat. + for _, fr := range flattened { + if fr.Op == syntax.OpConcat { + for _, sub := range fr.Sub { + if sub.Op == syntax.OpQuest { + questSkips[sub] = true + } + } + } + } + // Fast-path a single literal. if len(flattened) == 1 && re.Op == syntax.OpLiteral { n := utf8.RuneCountInString(string(re.Rune)) @@ -197,6 +214,14 @@ func generateRegexMatch(w io.Writer, lexerName, name, pattern string) error { if exists(re) { continue } + // Quest closures are not emitted when the quest appears as a direct + // sub-expression of a concat: concat backtracking (emitConcatWithQuest) + // inlines the quest's inner expression, so a standalone quest closure + // would be dead code. Concats are walked first because flatten is + // post-order and emits the quest before its enclosing concat. + if re.Op == syntax.OpQuest && questSkips[re] { + continue + } fmt.Fprintf(w, "// %s (%s)\n", re, re.Op) fmt.Fprintf(w, "l%d := func(s string, p int) int {\n", reid(re)) if re.Flags&syntax.NonGreedy != 0 { @@ -358,10 +383,15 @@ func generateRegexMatch(w io.Writer, lexerName, name, pattern string) error { panic("??") case syntax.OpConcat: // matches concatenation of Subs + // Pre-mark direct quest sub-expressions so their closures are + // skipped (emitConcatWithQuest inlines them), since this concat is + // emitted after its flatten-ordered quest children. for _, sub := range re.Sub { - fmt.Fprintf(w, "if p = l%d(s, p); p == -1 { return -1 }\n", reid(sub)) + if sub.Op == syntax.OpQuest { + questSkips[sub] = true + } } - fmt.Fprintf(w, "return p\n") + emitConcatWithQuest(w, re, reid, &seq, questSkips) case syntax.OpAlternate: // matches alternation of Subs for _, sub := range re.Sub { @@ -382,6 +412,77 @@ func generateRegexMatch(w io.Writer, lexerName, name, pattern string) error { return nil } +// emitConcatWithQuest emits the concatenation of re.Sub, adding backtracking +// for OpQuest sub-expressions so a quest "gives back" if the remainder would +// otherwise fail (issue #276). +// +// For "head (X)? tail" the generated code is: +// +// // p advanced; on failure goto onFail +// altN := p +// if np := lX(s, p); np != -1 { // greedy: take the quest +// p = np +// // tail failure -> goto fallbackN +// return p +// } +// +// fallbackN: +// +// // quest not taken +// p = altN +// +// return p +// +// Tails that themselves contain quests recurse into this function, nesting the +// backtracking without allocations (closures keep returning a single int). +// seq provides unique suffixes for labels and variables and must persist +// across sibling concats in the same generated function. +func emitConcatWithQuest(w io.Writer, re *syntax.Regexp, reid func(re *syntax.Regexp) int, seq *int, questSkips map[*syntax.Regexp]bool) { + var emit func(subs []*syntax.Regexp, onFail string) + emit = func(subs []*syntax.Regexp, onFail string) { + fail := func() { + if onFail == "" { + fmt.Fprintf(w, "return -1\n") + } else { + fmt.Fprintf(w, "goto %s\n", onFail) + } + } + for i, sub := range subs { + if sub.Op != syntax.OpQuest { + fmt.Fprintf(w, "if p = l%d(s, p); p == -1 { ", reid(sub)) + fail() + fmt.Fprintf(w, " }\n") + continue + } + tail := subs[i+1:] + *seq++ + n := *seq + questSkips[sub] = true + // The prefix subs[:i] has already been emitted by the loop above. + fmt.Fprintf(w, "alt%d := p\n", n) + fmt.Fprintf(w, "if np := l%d(s, p); np != -1 {\n", reid(sub.Sub0[0])) + w.Write([]byte("p = np\n")) + if len(tail) > 0 { + emit(tail, fmt.Sprintf("fallback%d", n)) + } else { + fmt.Fprintf(w, "return p\n") + } + w.Write([]byte("}\n")) + fmt.Fprintf(w, "fallback%d:\n", n) + w.Write([]byte("// quest not taken\n")) + w.Write([]byte("p = alt" + fmt.Sprint(n) + "\n")) + if len(tail) > 0 { + emit(tail, onFail) + } else { + fmt.Fprintf(w, "return p\n") + } + return + } + fmt.Fprintf(w, "return p\n") + } + emit(re.Sub, "") +} + // This exists because of https://github.com/golang/go/issues/31666 func decodeRune(w io.Writer, offset string, rn string, n string) { fmt.Fprintf(w, "if s[%s] < utf8.RuneSelf {\n", offset) diff --git a/lexer/internal/basiclexer.go b/lexer/internal/basiclexer.go index 48e118f2..ef98efef 100644 --- a/lexer/internal/basiclexer.go +++ b/lexer/internal/basiclexer.go @@ -1,4 +1,5 @@ // Code generated by Participle. DO NOT EDIT. + package internal import ( @@ -236,15 +237,8 @@ func matchGeneratedBasicNumber(s string, p int, backrefs []string) (groups [4]in } return -1 } - // [\+\-]? (Quest) - l1 := func(s string, p int) int { - if np := l0(s, p); np != -1 { - return np - } - return p - } // [0-9] (CharClass) - l2 := func(s string, p int) int { + l1 := func(s string, p int) int { if len(s) <= p { return -1 } @@ -256,9 +250,9 @@ func matchGeneratedBasicNumber(s string, p int, backrefs []string) (groups [4]in return -1 } // [0-9]* (Star) - l3 := func(s string, p int) int { + l2 := func(s string, p int) int { for len(s) > p { - if np := l2(s, p); np == -1 { + if np := l1(s, p); np == -1 { return p } else { p = np @@ -267,45 +261,38 @@ func matchGeneratedBasicNumber(s string, p int, backrefs []string) (groups [4]in return p } // \. (Literal) - l4 := func(s string, p int) int { + l3 := func(s string, p int) int { if p < len(s) && s[p] == '.' { return p + 1 } return -1 } // [0-9]*\. (Concat) - l5 := func(s string, p int) int { - if p = l3(s, p); p == -1 { + l4 := func(s string, p int) int { + if p = l2(s, p); p == -1 { return -1 } - if p = l4(s, p); p == -1 { + if p = l3(s, p); p == -1 { return -1 } return p } // ([0-9]*\.) (Capture) - l6 := func(s string, p int) int { - np := l5(s, p) + l5 := func(s string, p int) int { + np := l4(s, p) if np != -1 { groups[2] = p groups[3] = np } return np } - // ([0-9]*\.)? (Quest) - l7 := func(s string, p int) int { - if np := l6(s, p); np != -1 { - return np - } - return p - } // [0-9]+ (Plus) - l8 := func(s string, p int) int { - if p = l2(s, p); p == -1 { + l6 := func(s string, p int) int { + if p = l1(s, p); p == -1 { return -1 } for len(s) > p { - if np := l2(s, p); np == -1 { + if np := l1(s, p); np == -1 { return p } else { p = np @@ -314,19 +301,46 @@ func matchGeneratedBasicNumber(s string, p int, backrefs []string) (groups [4]in return p } // [\+\-]?([0-9]*\.)?[0-9]+ (Concat) - l9 := func(s string, p int) int { - if p = l1(s, p); p == -1 { - return -1 - } - if p = l7(s, p); p == -1 { - return -1 + l7 := func(s string, p int) int { + alt1 := p + if np := l0(s, p); np != -1 { + p = np + alt2 := p + if np := l5(s, p); np != -1 { + p = np + if p = l6(s, p); p == -1 { + goto fallback2 + } + return p + } + fallback2: + // quest not taken + p = alt2 + if p = l6(s, p); p == -1 { + goto fallback1 + } + return p + } + fallback1: + // quest not taken + p = alt1 + alt3 := p + if np := l5(s, p); np != -1 { + p = np + if p = l6(s, p); p == -1 { + goto fallback3 + } + return p } - if p = l8(s, p); p == -1 { + fallback3: + // quest not taken + p = alt3 + if p = l6(s, p); p == -1 { return -1 } return p } - np := l9(s, p) + np := l7(s, p) if np == -1 { return } @@ -453,7 +467,7 @@ func matchGeneratedBasicEOL(s string, p int, backrefs []string) (groups [2]int) return } -// (?i:REM)[^\n]*(?i:\n) +// (?i:REM[^\n]*\n) func matchGeneratedBasicComment(s string, p int, backrefs []string) (groups [2]int) { // (?i:REM) (Literal) l0 := func(s string, p int) int { @@ -495,14 +509,14 @@ func matchGeneratedBasicComment(s string, p int, backrefs []string) (groups [2]i } return p } - // (?i:\n) (Literal) + // \n (Literal) l3 := func(s string, p int) int { if p < len(s) && s[p] == '\n' { return p + 1 } return -1 } - // (?i:REM)[^\n]*(?i:\n) (Concat) + // (?i:REM[^\n]*\n) (Concat) l4 := func(s string, p int) int { if p = l0(s, p); p == -1 { return -1 diff --git a/lexer/internal/conformance/conformance_test.go b/lexer/internal/conformance/conformance_test.go index 63481656..fef54100 100644 --- a/lexer/internal/conformance/conformance_test.go +++ b/lexer/internal/conformance/conformance_test.go @@ -19,6 +19,7 @@ var conformanceLexer = lexer.MustStateful(lexer.Rules{ {"ExprTest", `EXPRTEST:`, lexer.Push("ExprTest")}, {"LiteralTest", `LITTEST:`, lexer.Push("LiteralTest")}, {"CaseInsensitiveTest", `CITEST:`, lexer.Push("CaseInsensitiveTest")}, + {"QuestTest", `QTEST:`, lexer.Push("QuestTest")}, // Use this to test \b at very start of the string! {"WordBoundaryTest", `\bWBTEST:`, lexer.Push("WordBoundaryTest")}, }, @@ -60,6 +61,15 @@ var conformanceLexer = lexer.MustStateful(lexer.Rules{ {"Ident", `\w+`, nil}, {"Whitespace", `\s+`, nil}, }, + // Quest in a concatenation must "give back" for the trailing literal to + // match (issue #276): [A-Z][A-Z][A-Z]?T matches "EST" with the optional + // [A-Z] not consuming the final "T". + "QuestTest": { + {`QuestWord`, `[A-Z][A-Z][A-Z]?T`, nil}, + {`QuestNum`, `[\+\-]?([0-9]*\.)?[0-9]+`, nil}, + {"Ident", `\w+`, nil}, + {"Whitespace", `\s+`, nil}, + }, "WordBoundaryTest": { {`WBKeyword`, `\b(?:abc|xyz)\b`, nil}, {`WBGroupKeyword`, `(?:90|0)\b`, nil}, @@ -204,6 +214,20 @@ func testLexer(t *testing.T, lex lexer.Definition) { {"Whitespace", " "}, {"Ident", "world"}, }}, + // The optional [A-Z] must "give back" so the trailing "T" matches + // (issue #276): [A-Z][A-Z][A-Z]?T on "EST" must be one token. + {"QuestGiveBack", `QTEST:EST`, []token{ + {"QuestWord", "EST"}, + }}, + {"QuestNumBoth", `QTEST:12 -12.5 -.5 +.25`, []token{ + {"QuestNum", "12"}, + {"Whitespace", " "}, + {"QuestNum", "-12.5"}, + {"Whitespace", " "}, + {"QuestNum", "-.5"}, + {"Whitespace", " "}, + {"QuestNum", "+.25"}, + }}, } symbols := lexer.SymbolsByRune(lex) for _, test := range tests {