github.com/x04/go/src@v0.0.0-20200202162449-3d481ceb3525/regexp/backtrack.go (about) 1 // Copyright 2015 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // backtrack is a regular expression search with submatch 6 // tracking for small regular expressions and texts. It allocates 7 // a bit vector with (length of input) * (length of prog) bits, 8 // to make sure it never explores the same (character position, instruction) 9 // state multiple times. This limits the search to run in time linear in 10 // the length of the test. 11 // 12 // backtrack is a fast replacement for the NFA code on small 13 // regexps when onepass cannot be used. 14 15 package regexp 16 17 import ( 18 "github.com/x04/go/src/regexp/syntax" 19 "github.com/x04/go/src/sync" 20 ) 21 22 // A job is an entry on the backtracker's job stack. It holds 23 // the instruction pc and the position in the input. 24 type job struct { 25 pc uint32 26 arg bool 27 pos int 28 } 29 30 const ( 31 visitedBits = 32 32 maxBacktrackProg = 500 // len(prog.Inst) <= max 33 maxBacktrackVector = 256 * 1024 // bit vector size <= max (bits) 34 ) 35 36 // bitState holds state for the backtracker. 37 type bitState struct { 38 end int 39 cap []int 40 matchcap []int 41 jobs []job 42 visited []uint32 43 44 inputs inputs 45 } 46 47 var bitStatePool sync.Pool 48 49 func newBitState() *bitState { 50 b, ok := bitStatePool.Get().(*bitState) 51 if !ok { 52 b = new(bitState) 53 } 54 return b 55 } 56 57 func freeBitState(b *bitState) { 58 b.inputs.clear() 59 bitStatePool.Put(b) 60 } 61 62 // maxBitStateLen returns the maximum length of a string to search with 63 // the backtracker using prog. 64 func maxBitStateLen(prog *syntax.Prog) int { 65 if !shouldBacktrack(prog) { 66 return 0 67 } 68 return maxBacktrackVector / len(prog.Inst) 69 } 70 71 // shouldBacktrack reports whether the program is too 72 // long for the backtracker to run. 73 func shouldBacktrack(prog *syntax.Prog) bool { 74 return len(prog.Inst) <= maxBacktrackProg 75 } 76 77 // reset resets the state of the backtracker. 78 // end is the end position in the input. 79 // ncap is the number of captures. 80 func (b *bitState) reset(prog *syntax.Prog, end int, ncap int) { 81 b.end = end 82 83 if cap(b.jobs) == 0 { 84 b.jobs = make([]job, 0, 256) 85 } else { 86 b.jobs = b.jobs[:0] 87 } 88 89 visitedSize := (len(prog.Inst)*(end+1) + visitedBits - 1) / visitedBits 90 if cap(b.visited) < visitedSize { 91 b.visited = make([]uint32, visitedSize, maxBacktrackVector/visitedBits) 92 } else { 93 b.visited = b.visited[:visitedSize] 94 for i := range b.visited { 95 b.visited[i] = 0 96 } 97 } 98 99 if cap(b.cap) < ncap { 100 b.cap = make([]int, ncap) 101 } else { 102 b.cap = b.cap[:ncap] 103 } 104 for i := range b.cap { 105 b.cap[i] = -1 106 } 107 108 if cap(b.matchcap) < ncap { 109 b.matchcap = make([]int, ncap) 110 } else { 111 b.matchcap = b.matchcap[:ncap] 112 } 113 for i := range b.matchcap { 114 b.matchcap[i] = -1 115 } 116 } 117 118 // shouldVisit reports whether the combination of (pc, pos) has not 119 // been visited yet. 120 func (b *bitState) shouldVisit(pc uint32, pos int) bool { 121 n := uint(int(pc)*(b.end+1) + pos) 122 if b.visited[n/visitedBits]&(1<<(n&(visitedBits-1))) != 0 { 123 return false 124 } 125 b.visited[n/visitedBits] |= 1 << (n & (visitedBits - 1)) 126 return true 127 } 128 129 // push pushes (pc, pos, arg) onto the job stack if it should be 130 // visited. 131 func (b *bitState) push(re *Regexp, pc uint32, pos int, arg bool) { 132 // Only check shouldVisit when arg is false. 133 // When arg is true, we are continuing a previous visit. 134 if re.prog.Inst[pc].Op != syntax.InstFail && (arg || b.shouldVisit(pc, pos)) { 135 b.jobs = append(b.jobs, job{pc: pc, arg: arg, pos: pos}) 136 } 137 } 138 139 // tryBacktrack runs a backtracking search starting at pos. 140 func (re *Regexp) tryBacktrack(b *bitState, i input, pc uint32, pos int) bool { 141 longest := re.longest 142 143 b.push(re, pc, pos, false) 144 for len(b.jobs) > 0 { 145 l := len(b.jobs) - 1 146 // Pop job off the stack. 147 pc := b.jobs[l].pc 148 pos := b.jobs[l].pos 149 arg := b.jobs[l].arg 150 b.jobs = b.jobs[:l] 151 152 // Optimization: rather than push and pop, 153 // code that is going to Push and continue 154 // the loop simply updates ip, p, and arg 155 // and jumps to CheckAndLoop. We have to 156 // do the ShouldVisit check that Push 157 // would have, but we avoid the stack 158 // manipulation. 159 goto Skip 160 CheckAndLoop: 161 if !b.shouldVisit(pc, pos) { 162 continue 163 } 164 Skip: 165 166 inst := re.prog.Inst[pc] 167 168 switch inst.Op { 169 default: 170 panic("bad inst") 171 case syntax.InstFail: 172 panic("unexpected InstFail") 173 case syntax.InstAlt: 174 // Cannot just 175 // b.push(inst.Out, pos, false) 176 // b.push(inst.Arg, pos, false) 177 // If during the processing of inst.Out, we encounter 178 // inst.Arg via another path, we want to process it then. 179 // Pushing it here will inhibit that. Instead, re-push 180 // inst with arg==true as a reminder to push inst.Arg out 181 // later. 182 if arg { 183 // Finished inst.Out; try inst.Arg. 184 arg = false 185 pc = inst.Arg 186 goto CheckAndLoop 187 } else { 188 b.push(re, pc, pos, true) 189 pc = inst.Out 190 goto CheckAndLoop 191 } 192 193 case syntax.InstAltMatch: 194 // One opcode consumes runes; the other leads to match. 195 switch re.prog.Inst[inst.Out].Op { 196 case syntax.InstRune, syntax.InstRune1, syntax.InstRuneAny, syntax.InstRuneAnyNotNL: 197 // inst.Arg is the match. 198 b.push(re, inst.Arg, pos, false) 199 pc = inst.Arg 200 pos = b.end 201 goto CheckAndLoop 202 } 203 // inst.Out is the match - non-greedy 204 b.push(re, inst.Out, b.end, false) 205 pc = inst.Out 206 goto CheckAndLoop 207 208 case syntax.InstRune: 209 r, width := i.step(pos) 210 if !inst.MatchRune(r) { 211 continue 212 } 213 pos += width 214 pc = inst.Out 215 goto CheckAndLoop 216 217 case syntax.InstRune1: 218 r, width := i.step(pos) 219 if r != inst.Rune[0] { 220 continue 221 } 222 pos += width 223 pc = inst.Out 224 goto CheckAndLoop 225 226 case syntax.InstRuneAnyNotNL: 227 r, width := i.step(pos) 228 if r == '\n' || r == endOfText { 229 continue 230 } 231 pos += width 232 pc = inst.Out 233 goto CheckAndLoop 234 235 case syntax.InstRuneAny: 236 r, width := i.step(pos) 237 if r == endOfText { 238 continue 239 } 240 pos += width 241 pc = inst.Out 242 goto CheckAndLoop 243 244 case syntax.InstCapture: 245 if arg { 246 // Finished inst.Out; restore the old value. 247 b.cap[inst.Arg] = pos 248 continue 249 } else { 250 if inst.Arg < uint32(len(b.cap)) { 251 // Capture pos to register, but save old value. 252 b.push(re, pc, b.cap[inst.Arg], true) // come back when we're done. 253 b.cap[inst.Arg] = pos 254 } 255 pc = inst.Out 256 goto CheckAndLoop 257 } 258 259 case syntax.InstEmptyWidth: 260 flag := i.context(pos) 261 if !flag.match(syntax.EmptyOp(inst.Arg)) { 262 continue 263 } 264 pc = inst.Out 265 goto CheckAndLoop 266 267 case syntax.InstNop: 268 pc = inst.Out 269 goto CheckAndLoop 270 271 case syntax.InstMatch: 272 // We found a match. If the caller doesn't care 273 // where the match is, no point going further. 274 if len(b.cap) == 0 { 275 return true 276 } 277 278 // Record best match so far. 279 // Only need to check end point, because this entire 280 // call is only considering one start position. 281 if len(b.cap) > 1 { 282 b.cap[1] = pos 283 } 284 if old := b.matchcap[1]; old == -1 || (longest && pos > 0 && pos > old) { 285 copy(b.matchcap, b.cap) 286 } 287 288 // If going for first match, we're done. 289 if !longest { 290 return true 291 } 292 293 // If we used the entire text, no longer match is possible. 294 if pos == b.end { 295 return true 296 } 297 298 // Otherwise, continue on in hope of a longer match. 299 continue 300 } 301 } 302 303 return longest && len(b.matchcap) > 1 && b.matchcap[1] >= 0 304 } 305 306 // backtrack runs a backtracking search of prog on the input starting at pos. 307 func (re *Regexp) backtrack(ib []byte, is string, pos int, ncap int, dstCap []int) []int { 308 startCond := re.cond 309 if startCond == ^syntax.EmptyOp(0) { // impossible 310 return nil 311 } 312 if startCond&syntax.EmptyBeginText != 0 && pos != 0 { 313 // Anchored match, past beginning of text. 314 return nil 315 } 316 317 b := newBitState() 318 i, end := b.inputs.init(nil, ib, is) 319 b.reset(re.prog, end, ncap) 320 321 // Anchored search must start at the beginning of the input 322 if startCond&syntax.EmptyBeginText != 0 { 323 if len(b.cap) > 0 { 324 b.cap[0] = pos 325 } 326 if !re.tryBacktrack(b, i, uint32(re.prog.Start), pos) { 327 freeBitState(b) 328 return nil 329 } 330 } else { 331 332 // Unanchored search, starting from each possible text position. 333 // Notice that we have to try the empty string at the end of 334 // the text, so the loop condition is pos <= end, not pos < end. 335 // This looks like it's quadratic in the size of the text, 336 // but we are not clearing visited between calls to TrySearch, 337 // so no work is duplicated and it ends up still being linear. 338 width := -1 339 for ; pos <= end && width != 0; pos += width { 340 if len(re.prefix) > 0 { 341 // Match requires literal prefix; fast search for it. 342 advance := i.index(re, pos) 343 if advance < 0 { 344 freeBitState(b) 345 return nil 346 } 347 pos += advance 348 } 349 350 if len(b.cap) > 0 { 351 b.cap[0] = pos 352 } 353 if re.tryBacktrack(b, i, uint32(re.prog.Start), pos) { 354 // Match must be leftmost; done. 355 goto Match 356 } 357 _, width = i.step(pos) 358 } 359 freeBitState(b) 360 return nil 361 } 362 363 Match: 364 dstCap = append(dstCap, b.matchcap...) 365 freeBitState(b) 366 return dstCap 367 }