0001"""
0002Iterator based sre token scanner
0003"""
0004import re
0005from re import VERBOSE, MULTILINE, DOTALL
0006import sre_parse
0007import sre_compile
0008import sre_constants
0009from sre_constants import BRANCH, SUBPATTERN
0010
0011__all__ = ['Scanner', 'pattern']
0012
0013FLAGS = (VERBOSE | MULTILINE | DOTALL)
0014
0015class Scanner(object):
0016 def __init__(self, lexicon, flags=FLAGS):
0017 self.actions = [None]
0018
0019 s = sre_parse.Pattern()
0020 s.flags = flags
0021 p = []
0022 for idx, token in enumerate(lexicon):
0023 phrase = token.pattern
0024 try:
0025 subpattern = sre_parse.SubPattern(s,
0026 [(SUBPATTERN, (idx + 1, sre_parse.parse(phrase, flags)))])
0027 except sre_constants.error:
0028 raise
0029 p.append(subpattern)
0030 self.actions.append(token)
0031
0032 s.groups = len(p) + 1
0033 p = sre_parse.SubPattern(s, [(BRANCH, (None, p))])
0034 self.scanner = sre_compile.compile(p)
0035
0036 def iterscan(self, string, idx=0, context=None):
0037 """
0038 Yield match, end_idx for each match
0039 """
0040 match = self.scanner.scanner(string, idx).match
0041 actions = self.actions
0042 lastend = idx
0043 end = len(string)
0044 while True:
0045 m = match()
0046 if m is None:
0047 break
0048 matchbegin, matchend = m.span()
0049 if lastend == matchend:
0050 break
0051 action = actions[m.lastindex]
0052 if action is not None:
0053 rval, next_pos = action(m, context)
0054 if next_pos is not None and next_pos != matchend:
0055
0056 matchend = next_pos
0057 match = self.scanner.scanner(string, matchend).match
0058 yield rval, matchend
0059 lastend = matchend
0060
0061
0062def pattern(pattern, flags=FLAGS):
0063 def decorator(fn):
0064 fn.pattern = pattern
0065 fn.regex = re.compile(pattern, flags)
0066 return fn
0067 return decorator