|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
printtoken(type,
token,
(srow, scol),
(erow, ecol),
line) |
|
|
|
tokenize(readline,
tokeneater=<function printtoken at 0x40613aac>)
The tokenize() function accepts two parameters: one representing the
input stream, and one providing an output mechanism for tokenize(). |
|
|
|
tokenize_loop(readline,
tokeneater) |
|
|
|
untokenize(iterable)
Transform tokens back into Python source code. |
|
|
|
generate_tokens(readline)
The generate_tokens() generator requires one argment, readline, which
must be a callable object which provides the same interface as the
readline() method of built-in file objects. |
|
|
|
__credits__ = ' GvR, ESR, Tim Peters, Thomas Wouters, Fred Drak ...
|
|
DEDENT = 6
|
|
LPAR = 7
|
|
STAR = 16
|
|
SLASH = 17
|
|
LESS = 20
|
|
SLASHEQUAL = 40
|
|
NUMBER = 2
|
|
RPAR = 8
|
|
CIRCUMFLEX = 33
|
|
NOTEQUAL = 29
|
|
VBAR = 18
|
|
BACKQUOTE = 25
|
|
DOUBLESTAR = 36
|
|
ENDMARKER = 0
|
|
MINUS = 15
|
|
DOT = 23
|
|
RBRACE = 27
|
|
STAREQUAL = 39
|
|
GREATEREQUAL = 31
|
|
MINEQUAL = 38
|
|
LEFTSHIFTEQUAL = 45
|
|
SEMI = 13
|
|
CIRCUMFLEXEQUAL = 44
|
|
NEWLINE = 4
|
|
DOUBLESLASHEQUAL = 49
|
|
COLON = 11
|
|
PERCENTEQUAL = 41
|
|
TILDE = 32
|
|
PLUS = 14
|
|
ERRORTOKEN = 52
|
|
RSQB = 10
|
|
EQEQUAL = 28
|
|
AMPEREQUAL = 42
|
|
RIGHTSHIFT = 35
|
|
STRING = 3
|
|
NT_OFFSET = 256
|
|
PERCENT = 24
|
|
DOUBLESLASH = 48
|
|
DOUBLESTAREQUAL = 47
|
|
EQUAL = 22
|
|
PLUSEQUAL = 37
|
|
AT = 50
|
|
AMPER = 19
|
|
LESSEQUAL = 30
|
|
LSQB = 9
|
|
N_TOKENS = 55
|
|
RIGHTSHIFTEQUAL = 46
|
|
NAME = 1
|
|
LBRACE = 26
|
|
INDENT = 5
|
|
GREATER = 21
|
|
tok_name = { 0: ' ENDMARKER ' , 1: ' NAME ' , 2: ' NUMBER ' , 3: ' STRING ...
|
|
VBAREQUAL = 43
|
|
LEFTSHIFT = 34
|
|
COMMA = 12
|
|
OP = 51
|
|
COMMENT = 53
|
|
NL = 54
|
|
Whitespace = ' [ \\f\\t]* '
|
|
Comment = ' #[^\\r\\n]* '
|
|
Ignore = ' [ \\f\\t]*(\\\\\\r?\\n[ \\f\\t]*)*(#[^\\r\\n]*)? '
|
|
Name = ' [a-zA-Z_]\\w* '
|
|
Hexnumber = ' 0[xX][\\da-fA-F]*[lL]? '
|
|
Octnumber = ' 0[0-7]*[lL]? '
|
|
Decnumber = ' [1-9]\\d*[lL]? '
|
|
Intnumber = ' (0[xX][\\da-fA-F]*[lL]?|0[0-7]*[lL]?|[1-9]\\d*[lL ...
|
|
Exponent = ' [eE][-+]?\\d+ '
|
|
Pointfloat = ' (\\d+\\.\\d*|\\.\\d+)([eE][-+]?\\d+)? '
|
|
Expfloat = ' \\d+[eE][-+]?\\d+ '
|
|
Floatnumber = ' ((\\d+\\.\\d*|\\.\\d+)([eE][-+]?\\d+)?|\\d+[eE] ...
|
|
Imagnumber = ' (\\d+[jJ]|((\\d+\\.\\d*|\\.\\d+)([eE][-+]?\\d+)? ...
|
|
Number = ' ((\\d+[jJ]|((\\d+\\.\\d*|\\.\\d+)([eE][-+]?\\d+)?|\\ ...
|
|
Single = ' [^\'\\\\]*(?:\\\\.[^\'\\\\]*)*\' '
|
|
Double = ' [^"\\\\]*(?:\\\\.[^"\\\\]*)*" '
|
|
Single3 = ' [^\'\\\\]*(?:(?:\\\\.|\'(?!\'\'))[^\'\\\\]*)*\'\'\' '
|
|
Double3 = ' [^"\\\\]*(?:(?:\\\\.|"(?!""))[^"\\\\]*)*""" '
|
|
Triple = ' ([uU]?[rR]?\'\'\'|[uU]?[rR]?""") '
|
|
String = ' ([uU]?[rR]?\'[^\\n\'\\\\]*(?:\\\\.[^\\n\'\\\\]*)*\'| ...
|
|
Operator = ' (\\*\\*=?|>>=?|<<=?|<>|!=|//=?|[+\\-*/%&|^=<>]=?|~) '
|
|
Bracket = ' [][(){}] '
|
|
Special = ' (\\r?\\n|[:;.,`@]) '
|
|
Funny = ' ((\\*\\*=?|>>=?|<<=?|<>|!=|//=?|[+\\-*/%&|^=<>]=?|~)| ...
|
|
PlainToken = ' (((\\d+[jJ]|((\\d+\\.\\d*|\\.\\d+)([eE][-+]?\\d+ ...
|
|
Token = ' [ \\f\\t]*(\\\\\\r?\\n[ \\f\\t]*)*(#[^\\r\\n]*)?(((\\ ...
|
|
ContStr = ' ([uU]?[rR]?\'[^\\n\'\\\\]*(?:\\\\.[^\\n\'\\\\]*)*(\ ...
|
|
PseudoExtras = ' (\\\\\\r?\\n|#[^\\r\\n]*|([uU]?[rR]?\'\'\'|[uU ...
|
|
PseudoToken = ' [ \\f\\t]*((\\\\\\r?\\n|#[^\\r\\n]*|([uU]?[rR]? ...
|
|
endprogs = { ' " ' : re.compile(r'[ ^ "\\] * (?: \\.[ ^ "\\] * ) * "'), ' """ ' ...
|
|
triple_quoted = { ' """ ' : ' """ ' , ' \'\'\' ' : ' \'\'\' ' , ' R""" ' : ' R" ...
|
|
single_quoted = { ' " ' : ' " ' , ' \' ' : ' \' ' , ' R" ' : ' R" ' , ' R\' ' : ' R\' ...
|
|
tabsize = 8
|
|
double3prog = re.compile(r'[ ^ "\\] * (?: (?: \\.| "(?! "") ) [ ^ "\\] * ) * "...
|
|
pseudoprog = re.compile(r'[ \f\t] * ( ( \\\r? \n| #[ ^ \r\n] * | ( [ uU] ? [ r...
|
|
single3prog = re.compile(r'[ ^ \'\\] * (?: (?: \\.| \'(?! \'\') ) [ ^ \'\\...
|
|
t = ' UR" '
|
|
tokenprog = re.compile(r'[ \f\t] * ( \\\r? \n[ \f\t] * ) * ( #[ ^ \r\n] * ) ...
|