// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // The testing package implements a simple regular expression library. // It is a reduced version of the regular expression package suitable // for use in tests; it avoids many dependencies. // // The syntax of the regular expressions accepted is: // // regexp: // concatenation { '|' concatenation } // concatenation: // { closure } // closure: // term [ '*' | '+' | '?' ] // term: // '^' // '$' // '.' // character // '[' [ '^' ] character-ranges ']' // '(' regexp ')' // package testing import ( "utf8" ) var debug = false // Error codes returned by failures to parse an expression. var ( ErrInternal = "internal error" ErrUnmatchedLpar = "unmatched ''" ErrUnmatchedRpar = "unmatched ''" ErrUnmatchedLbkt = "unmatched '['" ErrUnmatchedRbkt = "unmatched ']'" ErrBadRange = "bad range in character class" ErrExtraneousBackslash = "extraneous backslash" ErrBadClosure = "repeated closure **, ++, etc." ErrBareClosure = "closure applies to nothing" ErrBadBackslash = "illegal backslash escape" ) // An instruction executed by the NFA type instr interface { kind() int // the type of this instruction: _CHAR, _ANY, etc. next() instr // the instruction to execute after this one setNext(i instr) index() int setIndex(i int) print() } // Fields and methods common to all instructions type common struct { _next instr _index int } func (c *common) next() instr { return c._next } func (c *common) setNext(i instr) { c._next = i } func (c *common) index() int { return c._index } func (c *common) setIndex(i int) { c._index = i } // The representation of a compiled regular expression. // The public interface is entirely through methods. type Regexp struct { expr string // the original expression inst []instr start instr nbra int // number of brackets in expression, for subexpressions } const ( _START = iota // beginning of program _END // end of program: success _BOT // '^' beginning of text _EOT // '$' end of text _CHAR // 'a' regular character _CHARCLASS // [a-z] character class _ANY // '.' any character including newline _NOTNL // [^\n] special case: any character but newline _BRA // '(' parenthesized expression _EBRA // ')'; end of '(' parenthesized expression _ALT // '|' alternation _NOP // do nothing; makes it easy to link without patching ) // --- START start of program type _Start struct { common } func (start *_Start) kind() int { return _START } func (start *_Start) print() { print("start") } // --- END end of program type _End struct { common } func (end *_End) kind() int { return _END } func (end *_End) print() { print("end") } // --- BOT beginning of text type _Bot struct { common } func (bot *_Bot) kind() int { return _BOT } func (bot *_Bot) print() { print("bot") } // --- EOT end of text type _Eot struct { common } func (eot *_Eot) kind() int { return _EOT } func (eot *_Eot) print() { print("eot") } // --- CHAR a regular character type _Char struct { common char int } func (char *_Char) kind() int { return _CHAR } func (char *_Char) print() { print("char ", string(char.char)) } func newChar(char int) *_Char { c := new(_Char) c.char = char return c } // --- CHARCLASS [a-z] type _CharClass struct { common char int negate bool // is character class negated? ([^a-z]) // stored pairwise: [a-z] is (a,z); x is (x,x): ranges []int } func (cclass *_CharClass) kind() int { return _CHARCLASS } func (cclass *_CharClass) print() { print("charclass") if cclass.negate { print(" (negated)") } for i := 0; i < len(cclass.ranges); i += 2 { l := cclass.ranges[i] r := cclass.ranges[i+1] if l == r { print(" [", string(l), "]") } else { print(" [", string(l), "-", string(r), "]") } } } func (cclass *_CharClass) addRange(a, b int) { // range is a through b inclusive n := len(cclass.ranges) if n >= cap(cclass.ranges) { nr := make([]int, n, 2*n) for i, j := range nr { nr[i] = j } cclass.ranges = nr } cclass.ranges = cclass.ranges[0 : n+2] cclass.ranges[n] = a n++ cclass.ranges[n] = b n++ } func (cclass *_CharClass) matches(c int) bool { for i := 0; i < len(cclass.ranges); i = i + 2 { min := cclass.ranges[i] max := cclass.ranges[i+1] if min <= c && c <= max { return !cclass.negate } } return cclass.negate } func newCharClass() *_CharClass { c := new(_CharClass) c.ranges = make([]int, 0, 20) return c } // --- ANY any character type _Any struct { common } func (any *_Any) kind() int { return _ANY } func (any *_Any) print() { print("any") } // --- NOTNL any character but newline type _NotNl struct { common } func (notnl *_NotNl) kind() int { return _NOTNL } func (notnl *_NotNl) print() { print("notnl") } // --- BRA parenthesized expression type _Bra struct { common n int // subexpression number } func (bra *_Bra) kind() int { return _BRA } func (bra *_Bra) print() { print("bra", bra.n) } // --- EBRA end of parenthesized expression type _Ebra struct { common n int // subexpression number } func (ebra *_Ebra) kind() int { return _EBRA } func (ebra *_Ebra) print() { print("ebra ", ebra.n) } // --- ALT alternation type _Alt struct { common left instr // other branch } func (alt *_Alt) kind() int { return _ALT } func (alt *_Alt) print() { print("alt(", alt.left.index(), ")") } // --- NOP no operation type _Nop struct { common } func (nop *_Nop) kind() int { return _NOP } func (nop *_Nop) print() { print("nop") } func (re *Regexp) add(i instr) instr { n := len(re.inst) i.setIndex(len(re.inst)) if n >= cap(re.inst) { ni := make([]instr, n, 2*n) for i, j := range re.inst { ni[i] = j } re.inst = ni } re.inst = re.inst[0 : n+1] re.inst[n] = i return i } type parser struct { re *Regexp error string nlpar int // number of unclosed lpars pos int ch int } const endOfFile = -1 func (p *parser) c() int { return p.ch } func (p *parser) nextc() int { if p.pos >= len(p.re.expr) { p.ch = endOfFile } else { c, w := utf8.DecodeRuneInString(p.re.expr[p.pos:]) p.ch = c p.pos += w } return p.ch } func newParser(re *Regexp) *parser { p := new(parser) p.re = re p.nextc() // load p.ch return p } func special(c int) bool { s := `\.+*?()|[]^$` for i := 0; i < len(s); i++ { if c == int(s[i]) { return true } } return false } func specialcclass(c int) bool { s := `\-[]` for i := 0; i < len(s); i++ { if c == int(s[i]) { return true } } return false } func (p *parser) charClass() instr { cc := newCharClass() if p.c() == '^' { cc.negate = true p.nextc() } left := -1 for { switch c := p.c(); c { case ']', endOfFile: if left >= 0 { p.error = ErrBadRange return nil } // Is it [^\n]? if cc.negate && len(cc.ranges) == 2 && cc.ranges[0] == '\n' && cc.ranges[1] == '\n' { nl := new(_NotNl) p.re.add(nl) return nl } p.re.add(cc) return cc case '-': // do this before backslash processing p.error = ErrBadRange return nil case '\\': c = p.nextc() switch { case c == endOfFile: p.error = ErrExtraneousBackslash return nil case c == 'n': c = '\n' case specialcclass(c): // c is as delivered default: p.error = ErrBadBackslash return nil } fallthrough default: p.nextc() switch { case left < 0: // first of pair if p.c() == '-' { // range p.nextc() left = c } else { // single char cc.addRange(c, c) } case left <= c: // second of pair cc.addRange(left, c) left = -1 default: p.error = ErrBadRange return nil } } } return nil } func (p *parser) term() (start, end instr) { // term() is the leaf of the recursion, so it's sufficient to pick off the // error state here for early exit. // The other functions (closure(), concatenation() etc.) assume // it's safe to recur to here. if p.error != "" { return } switch c := p.c(); c { case '|', endOfFile: return nil, nil case '*', '+': p.error = ErrBareClosure return case ')': if p.nlpar == 0 { p.error = ErrUnmatchedRpar return } return nil, nil case ']': p.error = ErrUnmatchedRbkt return case '^': p.nextc() start = p.re.add(new(_Bot)) return start, start case '$': p.nextc() start = p.re.add(new(_Eot)) return start, start case '.': p.nextc() start = p.re.add(new(_Any)) return start, start case '[': p.nextc() start = p.charClass() if p.error != "" { return } if p.c() != ']' { p.error = ErrUnmatchedLbkt return } p.nextc() return start, start case '(': p.nextc() p.nlpar++ p.re.nbra++ // increment first so first subexpr is \1 nbra := p.re.nbra start, end = p.regexp() if p.c() != ')' { p.error = ErrUnmatchedLpar return } p.nlpar-- p.nextc() bra := new(_Bra) p.re.add(bra) ebra := new(_Ebra) p.re.add(ebra) bra.n = nbra ebra.n = nbra if start == nil { if end == nil { p.error = ErrInternal return } start = ebra } else { end.setNext(ebra) } bra.setNext(start) return bra, ebra case '\\': c = p.nextc() switch { case c == endOfFile: p.error = ErrExtraneousBackslash return case c == 'n': c = '\n' case special(c): // c is as delivered default: p.error = ErrBadBackslash return } fallthrough default: p.nextc() start = newChar(c) p.re.add(start) return start, start } panic("unreachable") } func (p *parser) closure() (start, end instr) { start, end = p.term() if start == nil || p.error != "" { return } switch p.c() { case '*': // (start,end)*: alt := new(_Alt) p.re.add(alt) end.setNext(alt) // after end, do alt alt.left = start // alternate brach: return to start start = alt // alt becomes new (start, end) end = alt case '+': // (start,end)+: alt := new(_Alt) p.re.add(alt) end.setNext(alt) // after end, do alt alt.left = start // alternate brach: return to start end = alt // start is unchanged; end is alt case '?': // (start,end)?: alt := new(_Alt) p.re.add(alt) nop := new(_Nop) p.re.add(nop) alt.left = start // alternate branch is start alt.setNext(nop) // follow on to nop end.setNext(nop) // after end, go to nop start = alt // start is now alt end = nop // end is nop pointed to by both branches default: return } switch p.nextc() { case '*', '+', '?': p.error = ErrBadClosure } return } func (p *parser) concatenation() (start, end instr) { for { nstart, nend := p.closure() if p.error != "" { return } switch { case nstart == nil: // end of this concatenation if start == nil { // this is the empty string nop := p.re.add(new(_Nop)) return nop, nop } return case start == nil: // this is first element of concatenation start, end = nstart, nend default: end.setNext(nstart) end = nend } } panic("unreachable") } func (p *parser) regexp() (start, end instr) { start, end = p.concatenation() if p.error != "" { return } for { switch p.c() { default: return case '|': p.nextc() nstart, nend := p.concatenation() if p.error != "" { return } alt := new(_Alt) p.re.add(alt) alt.left = start alt.setNext(nstart) nop := new(_Nop) p.re.add(nop) end.setNext(nop) nend.setNext(nop) start, end = alt, nop } } panic("unreachable") } func unNop(i instr) instr { for i.kind() == _NOP { i = i.next() } return i } func (re *Regexp) eliminateNops() { for i := 0; i < len(re.inst); i++ { inst := re.inst[i] if inst.kind() == _END { continue } inst.setNext(unNop(inst.next())) if inst.kind() == _ALT { alt := inst.(*_Alt) alt.left = unNop(alt.left) } } } func (re *Regexp) doParse() string { p := newParser(re) start := new(_Start) re.add(start) s, e := p.regexp() if p.error != "" { return p.error } start.setNext(s) re.start = start e.setNext(re.add(new(_End))) re.eliminateNops() return p.error } // CompileRegexp parses a regular expression and returns, if successful, a Regexp // object that can be used to match against text. func CompileRegexp(str string) (regexp *Regexp, error string) { regexp = new(Regexp) regexp.expr = str regexp.inst = make([]instr, 0, 20) error = regexp.doParse() return } // MustCompileRegexp is like CompileRegexp but panics if the expression cannot be parsed. // It simplifies safe initialization of global variables holding compiled regular // expressions. func MustCompile(str string) *Regexp { regexp, error := CompileRegexp(str) if error != "" { panicln(`regexp: compiling "`, str, `": `, error) } return regexp } type state struct { inst instr // next instruction to execute match []int // pairs of bracketing submatches. 0th is start,end } // Append new state to to-do list. Leftmost-longest wins so avoid // adding a state that's already active. func addState(s []state, inst instr, match []int) []state { index := inst.index() l := len(s) pos := match[0] // TODO: Once the state is a vector and we can do insert, have inputs always // go in order correctly and this "earlier" test is never necessary, for i := 0; i < l; i++ { if s[i].inst.index() == index && // same instruction s[i].match[0] < pos { // earlier match already going; lefmost wins return s } } if l == cap(s) { s1 := make([]state, 2*l)[0:l] for i := 0; i < l; i++ { s1[i] = s[i] } s = s1 } s = s[0 : l+1] s[l].inst = inst s[l].match = match return s } // Accepts either string or bytes - the logic is identical either way. // If bytes == nil, scan str. func (re *Regexp) doExecute(str string, bytes []byte, pos int) []int { var s [2][]state // TODO: use a vector when state values (not ptrs) can be vector elements s[0] = make([]state, 10)[0:0] s[1] = make([]state, 10)[0:0] in, out := 0, 1 var final state found := false end := len(str) if bytes != nil { end = len(bytes) } for pos <= end { if !found { // prime the pump if we haven't seen a match yet match := make([]int, 2*(re.nbra+1)) for i := 0; i < len(match); i++ { match[i] = -1 // no match seen; catches cases like "a(b)?c" on "ac" } match[0] = pos s[out] = addState(s[out], re.start.next(), match) } in, out = out, in // old out state is new in state s[out] = s[out][0:0] // clear out state if len(s[in]) == 0 { // machine has completed break } charwidth := 1 c := endOfFile if pos < end { if bytes == nil { c, charwidth = utf8.DecodeRuneInString(str[pos:end]) } else { c, charwidth = utf8.DecodeRune(bytes[pos:end]) } } for i := 0; i < len(s[in]); i++ { st := s[in][i] switch s[in][i].inst.kind() { case _BOT: if pos == 0 { s[in] = addState(s[in], st.inst.next(), st.match) } case _EOT: if pos == end { s[in] = addState(s[in], st.inst.next(), st.match) } case _CHAR: if c == st.inst.(*_Char).char { s[out] = addState(s[out], st.inst.next(), st.match) } case _CHARCLASS: if st.inst.(*_CharClass).matches(c) { s[out] = addState(s[out], st.inst.next(), st.match) } case _ANY: if c != endOfFile { s[out] = addState(s[out], st.inst.next(), st.match) } case _NOTNL: if c != endOfFile && c != '\n' { s[out] = addState(s[out], st.inst.next(), st.match) } case _BRA: n := st.inst.(*_Bra).n st.match[2*n] = pos s[in] = addState(s[in], st.inst.next(), st.match) case _EBRA: n := st.inst.(*_Ebra).n st.match[2*n+1] = pos s[in] = addState(s[in], st.inst.next(), st.match) case _ALT: s[in] = addState(s[in], st.inst.(*_Alt).left, st.match) // give other branch a copy of this match vector s1 := make([]int, 2*(re.nbra+1)) for i := 0; i < len(s1); i++ { s1[i] = st.match[i] } s[in] = addState(s[in], st.inst.next(), s1) case _END: // choose leftmost longest if !found || // first st.match[0] < final.match[0] || // leftmost (st.match[0] == final.match[0] && pos > final.match[1]) { // longest final = st final.match[1] = pos } found = true default: st.inst.print() panic("unknown instruction in execute") } } pos += charwidth } return final.match } // ExecuteString matches the Regexp against the string s. // The return value is an array of integers, in pairs, identifying the positions of // substrings matched by the expression. // s[a[0]:a[1]] is the substring matched by the entire expression. // s[a[2*i]:a[2*i+1]] for i > 0 is the substring matched by the ith parenthesized subexpression. // A negative value means the subexpression did not match any element of the string. // An empty array means "no match". func (re *Regexp) ExecuteString(s string) (a []int) { return re.doExecute(s, nil, 0) } // Execute matches the Regexp against the byte slice b. // The return value is an array of integers, in pairs, identifying the positions of // subslices matched by the expression. // b[a[0]:a[1]] is the subslice matched by the entire expression. // b[a[2*i]:a[2*i+1]] for i > 0 is the subslice matched by the ith parenthesized subexpression. // A negative value means the subexpression did not match any element of the slice. // An empty array means "no match". func (re *Regexp) Execute(b []byte) (a []int) { return re.doExecute("", b, 0) } // MatchString returns whether the Regexp matches the string s. // The return value is a boolean: true for match, false for no match. func (re *Regexp) MatchString(s string) bool { return len(re.doExecute(s, nil, 0)) > 0 } // Match returns whether the Regexp matches the byte slice b. // The return value is a boolean: true for match, false for no match. func (re *Regexp) Match(b []byte) bool { return len(re.doExecute("", b, 0)) > 0 } // MatchStrings matches the Regexp against the string s. // The return value is an array of strings matched by the expression. // a[0] is the substring matched by the entire expression. // a[i] for i > 0 is the substring matched by the ith parenthesized subexpression. // An empty array means ``no match''. func (re *Regexp) MatchStrings(s string) (a []string) { r := re.doExecute(s, nil, 0) if r == nil { return nil } a = make([]string, len(r)/2) for i := 0; i < len(r); i += 2 { if r[i] != -1 { // -1 means no match for this subexpression a[i/2] = s[r[i]:r[i+1]] } } return } // MatchSlices matches the Regexp against the byte slice b. // The return value is an array of subslices matched by the expression. // a[0] is the subslice matched by the entire expression. // a[i] for i > 0 is the subslice matched by the ith parenthesized subexpression. // An empty array means ``no match''. func (re *Regexp) MatchSlices(b []byte) (a [][]byte) { r := re.doExecute("", b, 0) if r == nil { return nil } a = make([][]byte, len(r)/2) for i := 0; i < len(r); i += 2 { if r[i] != -1 { // -1 means no match for this subexpression a[i/2] = b[r[i]:r[i+1]] } } return } // MatchString checks whether a textual regular expression // matches a string. More complicated queries need // to use Compile and the full Regexp interface. func MatchString(pattern string, s string) (matched bool, error string) { re, err := CompileRegexp(pattern) if err != "" { return false, err } return re.MatchString(s), "" } // Match checks whether a textual regular expression // matches a byte slice. More complicated queries need // to use Compile and the full Regexp interface. func Match(pattern string, b []byte) (matched bool, error string) { re, err := CompileRegexp(pattern) if err != "" { return false, err } return re.Match(b), "" }