Source Code for Module epydoc.markup.pyval_repr

  1  # epydoc -- Marked-up Representations for Python Values 
  2  # 
  3  # Copyright (C) 2005 Edward Loper 
  4  # Author: Edward Loper <edloper@loper.org> 
  5  # URL: <http://epydoc.sf.net> 
  6  # 
  7  # $Id: apidoc.py 1448 2007-02-11 00:05:34Z dvarrazzo $ 
  8   
  9  """ 
 10  Syntax highlighter for Python values.  Currently provides special 
 11  colorization support for: 
 12   
 13    - lists, tuples, sets, frozensets, dicts 
 14    - numbers 
 15    - strings 
 16    - compiled regexps 
 17   
 18  The highlighter also takes care of line-wrapping, and automatically 
 19  stops generating repr output as soon as it has exceeded the specified 
 20  number of lines (which should make it faster than pprint for large 
 21  values).  It does I{not} bother to do automatic cycle detection, 
 22  because maxlines is typically around 5, so it's really not worth it. 
 23   
 24  The syntax-highlighted output is encoded using a 
 25  L{ParsedEpytextDocstring}, which can then be used to generate output in 
 26  a variety of formats. 
 27  """ 
 28  __docformat__ = 'epytext en' 
 29   
 30  # Implementation note: we use exact tests for classes (list, etc) 
 31  # rather than using isinstance, because subclasses might override 
 32  # __repr__. 
 33   
 34  import types, re 
 35  import epydoc.apidoc 
 36  from epydoc.util import decode_with_backslashreplace 
 37  from epydoc.util import plaintext_to_html, plaintext_to_latex 
 38  from epydoc.compat import * 
 39  import sre_parse, sre_constants 
 40   
 41  from epydoc.markup.epytext import Element, ParsedEpytextDocstring 
 42   
 43 -def is_re_pattern(pyval): 
 44      return type(pyval).__name__ == 'SRE_Pattern' 
 45   
 46 -class _ColorizerState: 
 47      """ 
 48      An object uesd to keep track of the current state of the pyval 
 49      colorizer.  The L{mark()}/L{restore()} methods can be used to set 
 50      a backup point, and restore back to that backup point.  This is 
 51      used by several colorization methods that first try colorizing 
 52      their object on a single line (setting linebreakok=False); and 
 53      then fall back on a multi-line output if that fails.  The L{score} 
 54      variable is used to keep track of a 'score', reflecting how good 
 55      we think this repr is.  E.g., unhelpful values like '<Foo instance 
 56      at 0x12345>' get low scores.  If the score is too low, we'll use 
 57      the parse-derived repr instead. 
 58      """ 
 59 -    def __init__(self): 
 60          self.result = [] 
 61          self.charpos = 0 
 62          self.lineno = 1 
 63          self.linebreakok = True 
 64           
 65          #: How good this represention is? 
 66          self.score = 0 
 67   
 68 -    def mark(self): 
 69          return (len(self.result), self.charpos, 
 70                  self.lineno, self.linebreakok, self.score) 
 71   
 72 -    def restore(self, mark): 
 73          n, self.charpos, self.lineno, self.linebreakok, self.score = mark 
 74          del self.result[n:] 
 75   
 76 -class _Maxlines(Exception): 
 77      """A control-flow exception that is raised when PyvalColorizer 
 78      exeeds the maximum number of allowed lines.""" 
 79       
 80 -class _Linebreak(Exception): 
 81      """A control-flow exception that is raised when PyvalColorizer 
 82      generates a string containing a newline, but the state object's 
 83      linebreakok variable is False.""" 
 84   
 85 -class ColorizedPyvalRepr(ParsedEpytextDocstring): 
 86      """ 
 87      @ivar score: A score, evaluating how good this repr is. 
 88      @ivar is_complete: True if this colorized repr completely describes 
 89         the object. 
 90      """ 
 91 -    def __init__(self, tree, score, is_complete): 
 92          ParsedEpytextDocstring.__init__(self, tree) 
 93          self.score = score 
 94          self.is_complete = is_complete 
 95   
 96 -def colorize_pyval(pyval, parse_repr=None, min_score=None, 
 97                     linelen=75, maxlines=5, linebreakok=True, sort=True): 
 98      return PyvalColorizer(linelen, maxlines, linebreakok, sort).colorize( 
 99          pyval, parse_repr, min_score) 
100   
101 -class PyvalColorizer: 
102      """ 
103      Syntax highlighter for Python values. 
104      """ 
105   
106 -    def __init__(self, linelen=75, maxlines=5, linebreakok=True, sort=True): 
107          self.linelen = linelen 
108          self.maxlines = maxlines 
109          self.linebreakok = linebreakok 
110          self.sort = sort 
111   
112      #//////////////////////////////////////////////////////////// 
113      # Colorization Tags & other constants 
114      #//////////////////////////////////////////////////////////// 
115   
116      GROUP_TAG = 'variable-group'     # e.g., "[" and "]" 
117      COMMA_TAG = 'variable-op'        # The "," that separates elements 
118      COLON_TAG = 'variable-op'        # The ":" in dictionaries 
119      CONST_TAG = None                 # None, True, False 
120      NUMBER_TAG = None                # ints, floats, etc 
121      QUOTE_TAG = 'variable-quote'     # Quotes around strings. 
122      STRING_TAG = 'variable-string'   # Body of string literals 
123   
124      RE_CHAR_TAG = None 
125      RE_GROUP_TAG = 're-group' 
126      RE_REF_TAG = 're-ref' 
127      RE_OP_TAG = 're-op' 
128      RE_FLAGS_TAG = 're-flags' 
129   
130      ELLIPSIS = Element('code', u'...', style='variable-ellipsis') 
131      LINEWRAP = Element('symbol', u'crarr') 
132      UNKNOWN_REPR = Element('code', u'??', style='variable-unknown') 
133       
134      GENERIC_OBJECT_RE = re.compile(r'^<.* at 0x[0-9a-f]+>$', re.IGNORECASE) 
135   
136      ESCAPE_UNICODE = False # should we escape non-ascii unicode chars? 
137   
138      #//////////////////////////////////////////////////////////// 
139      # Entry Point 
140      #//////////////////////////////////////////////////////////// 
141   
142 -    def colorize(self, pyval, parse_repr=None, min_score=None): 
143          """ 
144          @return: A L{ColorizedPyvalRepr} describing the given pyval. 
145          """ 
146          UNKNOWN = epydoc.apidoc.UNKNOWN 
147          # Create an object to keep track of the colorization. 
148          state = _ColorizerState() 
149          state.linebreakok = self.linebreakok 
150          # Colorize the value.  If we reach maxlines, then add on an 
151          # ellipsis marker and call it a day. 
152          try: 
153              if pyval is not UNKNOWN: 
154                  self._colorize(pyval, state) 
155              elif parse_repr not in (None, UNKNOWN): 
156                  self._output(parse_repr, None, state) 
157              else: 
158                  state.result.append(PyvalColorizer.UNKNOWN_REPR) 
159              is_complete = True 
160          except (_Maxlines, _Linebreak): 
161              if self.linebreakok: 
162                  state.result.append('\n') 
163                  state.result.append(self.ELLIPSIS) 
164              else: 
165                  if state.result[-1] is self.LINEWRAP: 
166                      state.result.pop() 
167                  self._trim_result(state.result, 3) 
168                  state.result.append(self.ELLIPSIS) 
169              is_complete = False 
170          # If we didn't score high enough, then try again. 
171          if (pyval is not UNKNOWN and parse_repr not in (None, UNKNOWN) 
172              and min_score is not None and state.score < min_score): 
173              return self.colorize(UNKNOWN, parse_repr) 
174          # Put it all together. 
175          tree = Element('epytext', *state.result) 
176          return ColorizedPyvalRepr(tree, state.score, is_complete) 
177   
178 -    def _colorize(self, pyval, state): 
179          pyval_type = type(pyval) 
180          state.score += 1 
181           
182          if pyval is None or pyval is True or pyval is False: 
183              self._output(unicode(pyval), self.CONST_TAG, state) 
184          elif pyval_type in (int, float, long, types.ComplexType): 
185              self._output(unicode(pyval), self.NUMBER_TAG, state) 
186          elif pyval_type is str: 
187              self._colorize_str(pyval, state, '', 'string-escape') 
188          elif pyval_type is unicode: 
189              if self.ESCAPE_UNICODE: 
190                  self._colorize_str(pyval, state, 'u', 'unicode-escape') 
191              else: 
192                  self._colorize_str(pyval, state, 'u', None) 
193          elif pyval_type is list: 
194              self._multiline(self._colorize_iter, pyval, state, '[', ']') 
195          elif pyval_type is tuple: 
196              self._multiline(self._colorize_iter, pyval, state, '(', ')') 
197          elif pyval_type is set: 
198              self._multiline(self._colorize_iter, self._sort(pyval), 
199                              state, 'set([', '])') 
200          elif pyval_type is frozenset: 
201              self._multiline(self._colorize_iter, self._sort(pyval), 
202                              state, 'frozenset([', '])') 
203          elif pyval_type is dict: 
204              self._multiline(self._colorize_dict, self._sort(pyval.items()), 
205                              state, '{', '}') 
206          elif is_re_pattern(pyval): 
207              self._colorize_re(pyval, state) 
208          else: 
209              try: 
210                  pyval_repr = repr(pyval) 
211                  if not isinstance(pyval_repr, (str, unicode)): 
212                      pyval_repr = unicode(pyval_repr) 
213                  pyval_repr_ok = True 
214              except KeyboardInterrupt: 
215                  raise 
216              except: 
217                  pyval_repr_ok = False 
218                  state.score -= 100 
219   
220              if pyval_repr_ok: 
221                  if self.GENERIC_OBJECT_RE.match(pyval_repr): 
222                      state.score -= 5 
223                  self._output(pyval_repr, None, state) 
224              else: 
225                  state.result.append(self.UNKNOWN_REPR) 
226   
227 -    def _sort(self, items): 
228          if not self.sort: return items 
229          try: return sorted(items) 
230          except KeyboardInterrupt: raise 
231          except: return items 
232           
233 -    def _trim_result(self, result, num_chars): 
234          while num_chars > 0: 
235              if not result: return  
236              if isinstance(result[-1], Element): 
237                  assert len(result[-1].children) == 1 
238                  trim = min(num_chars, len(result[-1].children[0])) 
239                  result[-1].children[0] = result[-1].children[0][:-trim] 
240                  if not result[-1].children[0]: result.pop() 
241                  num_chars -= trim 
242              else: 
243                  trim = min(num_chars, len(result[-1])) 
244                  result[-1] = result[-1][:-trim] 
245                  if not result[-1]: result.pop() 
246                  num_chars -= trim 
247   
248      #//////////////////////////////////////////////////////////// 
249      # Object Colorization Functions 
250      #//////////////////////////////////////////////////////////// 
251   
252 -    def _multiline(self, func, pyval, state, *args): 
253          """ 
254          Helper for container-type colorizers.  First, try calling 
255          C{func(pyval, state, *args)} with linebreakok set to false; 
256          and if that fails, then try again with it set to true. 
257          """ 
258          linebreakok = state.linebreakok 
259          mark = state.mark() 
260           
261          try: 
262              state.linebreakok = False 
263              func(pyval, state, *args) 
264              state.linebreakok = linebreakok 
265   
266          except _Linebreak: 
267              if not linebreakok: 
268                  raise 
269              state.restore(mark) 
270              func(pyval, state, *args) 
271               
272 -    def _colorize_iter(self, pyval, state, prefix, suffix): 
273          self._output(prefix, self.GROUP_TAG, state) 
274          indent = state.charpos 
275          for i, elt in enumerate(pyval): 
276              if i>=1: 
277                  if state.linebreakok: 
278                      self._output(',', self.COMMA_TAG, state) 
279                      self._output('\n'+' '*indent, None, state) 
280                  else: 
281                      self._output(', ', self.COMMA_TAG, state) 
282              self._colorize(elt, state) 
283          self._output(suffix, self.GROUP_TAG, state) 
284   
285 -    def _colorize_dict(self, items, state, prefix, suffix): 
286          self._output(prefix, self.GROUP_TAG, state) 
287          indent = state.charpos 
288          for i, (key, val) in enumerate(items): 
289              if i>=1: 
290                  if state.linebreakok: 
291                      self._output(',', self.COMMA_TAG, state) 
292                      self._output('\n'+' '*indent, None, state) 
293                  else: 
294                      self._output(', ', self.COMMA_TAG, state) 
295              self._colorize(key, state) 
296              self._output(': ', self.COLON_TAG, state) 
297              self._colorize(val, state) 
298          self._output(suffix, self.GROUP_TAG, state) 
299   
300 -    def _colorize_str(self, pyval, state, prefix, encoding): 
301          # Decide which quote to use. 
302          if '\n' in pyval and state.linebreakok: quote = "'''" 
303          else: quote = "'" 
304          # Divide the string into lines. 
305          if state.linebreakok: 
306              lines = pyval.split('\n') 
307          else: 
308              lines = [pyval] 
309          # Open quote. 
310          self._output(prefix+quote, self.QUOTE_TAG, state) 
311          # Body 
312          for i, line in enumerate(lines): 
313              if i>0: self._output('\n', None, state) 
314              if encoding: line = line.encode(encoding) 
315              self._output(line, self.STRING_TAG, state) 
316          # Close quote. 
317          self._output(quote, self.QUOTE_TAG, state) 
318   
319 -    def _colorize_re(self, pyval, state): 
320          # Extract the flag & pattern from the regexp. 
321          pat, flags = pyval.pattern, pyval.flags 
322          # If the pattern is a string, decode it to unicode. 
323          if isinstance(pat, str): 
324              pat = decode_with_backslashreplace(pat) 
325          # Parse the regexp pattern. 
326          tree = sre_parse.parse(pat, flags) 
327          groups = dict([(num,name) for (name,num) in 
328                         tree.pattern.groupdict.items()]) 
329          # Colorize it! 
330          self._output("re.compile(r'", None, state) 
331          self._colorize_re_flags(tree.pattern.flags, state) 
332          self._colorize_re_tree(tree, state, True, groups) 
333          self._output("')", None, state) 
334   
335 -    def _colorize_re_flags(self, flags, state): 
336          if flags: 
337              flags = [c for (c,n) in sorted(sre_parse.FLAGS.items()) 
338                       if (n&flags)] 
339              flags = '(?%s)' % ''.join(flags) 
340              self._output(flags, self.RE_FLAGS_TAG, state) 
341   
342 -    def _colorize_re_tree(self, tree, state, noparen, groups): 
343          assert noparen in (True, False) 
344          if len(tree) > 1 and not noparen: 
345              self._output('(', self.RE_GROUP_TAG, state) 
346          for elt in tree: 
347              op = elt[0] 
348              args = elt[1] 
349       
350              if op == sre_constants.LITERAL: 
351                  c = unichr(args) 
352                  # Add any appropriate escaping. 
353                  if c in '.^$\\*+?{}[]|()\'': c = '\\'+c 
354                  elif c == '\t': c = '\\t' 
355                  elif c == '\r': c = '\\r' 
356                  elif c == '\n': c = '\\n' 
357                  elif c == '\f': c = '\\f' 
358                  elif c == '\v': c = '\\v' 
359                  elif ord(c) > 0xffff: c = r'\U%08x' % ord(c) 
360                  elif ord(c) > 0xff: c = r'\u%04x' % ord(c) 
361                  elif ord(c)<32 or ord(c)>=127: c = r'\x%02x' % ord(c) 
362                  self._output(c, self.RE_CHAR_TAG, state) 
363               
364              elif op == sre_constants.ANY: 
365                  self._output('.', self.RE_CHAR_TAG, state) 
366                   
367              elif op == sre_constants.BRANCH: 
368                  if args[0] is not None: 
369                      raise ValueError('Branch expected None arg but got %s' 
370                                       % args[0]) 
371                  for i, item in enumerate(args[1]): 
372                      if i > 0: 
373                          self._output('|', self.RE_OP_TAG, state) 
374                      self._colorize_re_tree(item, state, True, groups) 
375                   
376              elif op == sre_constants.IN: 
377                  if (len(args) == 1 and args[0][0] == sre_constants.CATEGORY): 
378                      self._colorize_re_tree(args, state, False, groups) 
379                  else: 
380                      self._output('[', self.RE_GROUP_TAG, state) 
381                      self._colorize_re_tree(args, state, True, groups) 
382                      self._output(']', self.RE_GROUP_TAG, state) 
383                       
384              elif op == sre_constants.CATEGORY: 
385                  if args == sre_constants.CATEGORY_DIGIT: val = r'\d' 
386                  elif args == sre_constants.CATEGORY_NOT_DIGIT: val = r'\D' 
387                  elif args == sre_constants.CATEGORY_SPACE: val = r'\s' 
388                  elif args == sre_constants.CATEGORY_NOT_SPACE: val = r'\S' 
389                  elif args == sre_constants.CATEGORY_WORD: val = r'\w' 
390                  elif args == sre_constants.CATEGORY_NOT_WORD: val = r'\W' 
391                  else: raise ValueError('Unknown category %s' % args) 
392                  self._output(val, self.RE_CHAR_TAG, state) 
393                   
394              elif op == sre_constants.AT: 
395                  if args == sre_constants.AT_BEGINNING_STRING: val = r'\A' 
396                  elif args == sre_constants.AT_BEGINNING: val = r'^' 
397                  elif args == sre_constants.AT_END: val = r'$' 
398                  elif args == sre_constants.AT_BOUNDARY: val = r'\b' 
399                  elif args == sre_constants.AT_NON_BOUNDARY: val = r'\B' 
400                  elif args == sre_constants.AT_END_STRING: val = r'\Z' 
401                  else: raise ValueError('Unknown position %s' % args) 
402                  self._output(val, self.RE_CHAR_TAG, state) 
403                   
404              elif op in (sre_constants.MAX_REPEAT, sre_constants.MIN_REPEAT): 
405                  minrpt = args[0] 
406                  maxrpt = args[1] 
407                  if maxrpt == sre_constants.MAXREPEAT: 
408                      if minrpt == 0:   val = '*' 
409                      elif minrpt == 1: val = '+' 
410                      else: val = '{%d,}' % (minrpt) 
411                  elif minrpt == 0: 
412                      if maxrpt == 1: val = '?' 
413                      else: val = '{,%d}' % (maxrpt) 
414                  elif minrpt == maxrpt: 
415                      val = '{%d}' % (maxrpt) 
416                  else: 
417                      val = '{%d,%d}' % (minrpt, maxrpt) 
418                  if op == sre_constants.MIN_REPEAT: 
419                      val += '?' 
420                       
421                  self._colorize_re_tree(args[2], state, False, groups) 
422                  self._output(val, self.RE_OP_TAG, state) 
423                   
424              elif op == sre_constants.SUBPATTERN: 
425                  if args[0] is None: 
426                      self._output('(?:', self.RE_GROUP_TAG, state) 
427                  elif args[0] in groups: 
428                      self._output('(?P<', self.RE_GROUP_TAG, state) 
429                      self._output(groups[args[0]], self.RE_REF_TAG, state) 
430                      self._output('>', self.RE_GROUP_TAG, state) 
431                  elif isinstance(args[0], (int, long)): 
432                      # This is cheating: 
433                      self._output('(', self.RE_GROUP_TAG, state) 
434                  else: 
435                      self._output('(?P<', self.RE_GROUP_TAG, state) 
436                      self._output(args[0], self.RE_REF_TAG, state) 
437                      self._output('>', self.RE_GROUP_TAG, state) 
438                  self._colorize_re_tree(args[1], state, True, groups) 
439                  self._output(')', self.RE_GROUP_TAG, state) 
440       
441              elif op == sre_constants.GROUPREF: 
442                  self._output('\\%d' % args, self.RE_REF_TAG, state) 
443       
444              elif op == sre_constants.RANGE: 
445                  self._colorize_re_tree( ((sre_constants.LITERAL, args[0]),), 
446                                          state, False, groups ) 
447                  self._output('-', self.RE_OP_TAG, state) 
448                  self._colorize_re_tree( ((sre_constants.LITERAL, args[1]),), 
449                                          state, False, groups ) 
450                   
451              elif op == sre_constants.NEGATE: 
452                  self._output('^', self.RE_OP_TAG, state) 
453       
454              elif op == sre_constants.ASSERT: 
455                  if args[0] > 0: 
456                      self._output('(?=', self.RE_GROUP_TAG, state) 
457                  else: 
458                      self._output('(?<=', self.RE_GROUP_TAG, state) 
459                  self._colorize_re_tree(args[1], state, True, groups) 
460                  self._output(')', self.RE_GROUP_TAG, state) 
461                                  
462              elif op == sre_constants.ASSERT_NOT: 
463                  if args[0] > 0: 
464                      self._output('(?!', self.RE_GROUP_TAG, state) 
465                  else: 
466                      self._output('(?<!', self.RE_GROUP_TAG, state) 
467                  self._colorize_re_tree(args[1], state, True, groups) 
468                  self._output(')', self.RE_GROUP_TAG, state) 
469       
470              elif op == sre_constants.NOT_LITERAL: 
471                  self._output('[^', self.RE_GROUP_TAG, state) 
472                  self._colorize_re_tree( ((sre_constants.LITERAL, args),), 
473                                          state, False, groups ) 
474                  self._output(']', self.RE_GROUP_TAG, state) 
475              else: 
476                  log.error("Error colorizing regexp: unknown elt %r" % elt) 
477          if len(tree) > 1 and not noparen:  
478              self._output(')', self.RE_GROUP_TAG, state) 
479                              
480      #//////////////////////////////////////////////////////////// 
481      # Output function 
482      #//////////////////////////////////////////////////////////// 
483   
484 -    def _output(self, s, tag, state): 
485          """ 
486          Add the string `s` to the result list, tagging its contents 
487          with tag `tag`.  Any lines that go beyond `self.linelen` will 
488          be line-wrapped.  If the total number of lines exceeds 
489          `self.maxlines`, then raise a `_Maxlines` exception. 
490          """ 
491          # Make sure the string is unicode. 
492          if isinstance(s, str): 
493              s = decode_with_backslashreplace(s) 
494           
495          # Split the string into segments.  The first segment is the 
496          # content to add to the current line, and the remaining 
497          # segments are new lines. 
498          segments = s.split('\n') 
499   
500          for i, segment in enumerate(segments): 
501              # If this isn't the first segment, then add a newline to 
502              # split it from the previous segment. 
503              if i > 0: 
504                  if (state.lineno+1) > self.maxlines: 
505                      raise _Maxlines() 
506                  if not state.linebreakok: 
507                      raise _Linebreak() 
508                  state.result.append(u'\n') 
509                  state.lineno += 1 
510                  state.charpos = 0 
511   
512              # If the segment fits on the current line, then just call 
513              # markup to tag it, and store the result. 
514              if state.charpos + len(segment) <= self.linelen: 
515                  state.charpos += len(segment) 
516                  if tag: 
517                      segment = Element('code', segment, style=tag) 
518                  state.result.append(segment) 
519   
520              # If the segment doesn't fit on the current line, then 
521              # line-wrap it, and insert the remainder of the line into 
522              # the segments list that we're iterating over.  (We'll go 
523              # the the beginning of the next line at the start of the 
524              # next iteration through the loop.) 
525              else: 
526                  split = self.linelen-state.charpos 
527                  segments.insert(i+1, segment[split:]) 
528                  segment = segment[:split] 
529                  if tag: 
530                      segment = Element('code', segment, style=tag) 
531                  state.result += [segment, self.LINEWRAP] 
532