OwlCyberSecurity - MANAGER
Edit File: tokenize.cpython-35.opt-2.pyc
���]�l � X @ s4 d Z d Z d d l m Z d d l m Z m Z d d l Z d d l m Z d d l m Z d d l Z d d l Z d d l Te j d e j � Z e j d e j � Z d d l Z e j d d d d d d d g Z [ e Z d e e <e d Z d e e <e d Z d e e <e d 7Z d e d e d e d e d e d e d e d e! d e" d e# d e$ d e% d! e& d" e' d# e( d$ e) d% e* d& e+ d' e, d( e- d) e. d* e/ d+ e0 d, e1 d- e2 d. e3 d/ e4 d0 e5 d1 e6 d2 e7 d3 e8 d4 e9 d5 e: d6 e; d7 e< d8 e= d9 e> d: e? d; e@ d< eA d= eB d> eC d? eD d@ eE i, ZF GdA d � d e jG d dB � � ZH dC dD � ZI dE dF � ZJ dG dH � ZK dI ZL dJ ZM eL eJ dK eL � eK eM � ZN dL ZO dM ZP dN ZQ dO ZR dP ZS eI eP eQ eR eS � ZT dQ ZU eI dR dS � eK eU � ZV dT eU ZW eI eV eW � ZX eI dU eX dV � ZY eI eY eX eT � ZZ dW Z[ dX Z\ dY Z] dZ Z^ d[ Z_ eI e[ d\ e[ d] � Z` eI e[ d^ e[ d_ � Za eI d` da db d* dc dd de d- � Zb df Zc eI dg dh di � Zd eI eb ec ed � Ze eI eZ ee ea eO � Zf eN ef Zg eI e[ dj eI dk dK � e[ dl eI dm dK � � Zh eI dn eM e` � Zi eL eI ei eZ ee eh eO � Zj do dp � Zk dk e\ dm e] d\ e^ d] e_ dq e^ dr e_ ds e^ dt e_ du e^ dv e_ dw e^ dx e_ dy e^ dz e_ d{ e^ d| e_ d} e^ d~ e_ d e^ d� e_ d� e^ d� e_ d� e^ d� e_ d� e^ d� e_ d� e^ d� e_ d� e^ d� e_ d� e^ d� e_ d� d d� d d� d d� d d� d d� d i& Zl i Zm x d� D] Zn en em en <qCWi Zo x d� D] Zn en eo en <qdWd� Zp Gd� d� � d� eq � Zr Gd� d� � d� eq � Zs Gd� d� � d� � Zt d� d � Zu d� d� � Zv d� d � Zw d� d� � Z d� d � Zx d� d� � Zy d� d� � Zz d� d� � Z{ e| d� k r0e{ � d S)�zKa-Ping Yee <ping@lfw.org>zpGvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger, Trent Nelson, Michael Foord� )�open)�lookup�BOM_UTF8N)� TextIOWrapper)�chain)�*z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$)�COMMENT�tokenize�detect_encoding�NL� untokenize�ENCODING� TokenInfo� � � �(�)�[�]�:�,�;�+�-r �/�|�&�<�>�=�.�%�{�}z==z!=z<=z>=�~�^z<<z>>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=�@z@=c @ s. e Z d Z d d � Z e d d � � Z d S)r c C s. d | j t | j f } d | j d | � S)Nz%d (%s)z8TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)�type)r( �tok_name�_replace)�self�annotated_type� r- �-/opt/alt/python35/lib64/python3.5/tokenize.py�__repr__c s zTokenInfo.__repr__c C s4 | j t k r) | j t k r) t | j S| j Sd S)N)r( �OP�string�EXACT_TOKEN_TYPES)r+ r- r- r. � exact_typeh s zTokenInfo.exact_typeN)�__name__� __module__� __qualname__r/ �propertyr3 r- r- r- r. r b s ztype string start end linec G s d d j | � d S)Nr r r )�join)�choicesr- r- r. �groupo s r: c G s t | � d S)Nr )r: )r9 r- r- r. �anyp s r; c G s t | � d S)N�?)r: )r9 r- r- r. �maybeq s r= z[ \f\t]*z #[^\r\n]*z\\\r?\nz\w+z0[xX][0-9a-fA-F]+z 0[bB][01]+z 0[oO][0-7]+z(?:0+|[1-9][0-9]*)z[eE][-+]?[0-9]+z[0-9]+\.[0-9]*z\.[0-9]+z[0-9]+z [0-9]+[jJ]z[jJ]z (?:[bB][rR]?|[rR][bB]?|[uU])?z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""�'''�"""z '[^\n'\\]*(?:\\.[^\n'\\]*)*'z "[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z//=?z->z[+\-*/%&@|^=<>]=?z[][(){}]z\r?\nz\.\.\.z[:;.,@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*�'z"[^\n"\\]*(?:\\.[^\n"\\]*)*�"z \\\r?\n|\Zc C s t j | t j � S)N)�re�compile�UNICODE)�exprr- r- r. �_compile� s rF �r'''�r"""�b'''�b"""�R'''�R"""�B'''�B"""�br'''�br"""�bR'''�bR"""�Br'''�Br"""�BR'''�BR"""�rb'''�rb"""�Rb'''�Rb"""�rB'''�rB"""�RB'''�RB"""�u'''�u"""�U'''�U"""�r�R�b�B�u�U�r'�r"�R'�R"�b'�b"�B'�B"�br'�br"�Br'�Br"�bR'�bR"�BR'�BR"�rb'�rb"�rB'�rB"�Rb'�Rb"�RB'�RB"�u'�u"�U'�U"� c @ s e Z d Z d S)� TokenErrorN)r4 r5 r6 r- r- r- r. r� � s r� c @ s e Z d Z d S)�StopTokenizingN)r4 r5 r6 r- r- r- r. r� � s r� c @ s@ e Z d Z d d � Z d d � Z d d � Z d d � Z d S) � Untokenizerc C s( g | _ d | _ d | _ d | _ d S)Nr r )�tokens�prev_row�prev_col�encoding)r+ r- r- r. �__init__� s zUntokenizer.__init__c C s� | \ } } | | j k s9 | | j k r] | | j k r] t d j | | | j | j � � � | | j } | r� | j j d | � d | _ | | j } | r� | j j d | � d S)Nz+start ({},{}) precedes previous end ({},{})z\ r � )r� r� � ValueError�formatr� �append)r+ �start�row�col� row_offset� col_offsetr- r- r. �add_whitespace� s - zUntokenizer.add_whitespacec C s� t | � } g } d } x}| D]u} t | � d k rH | j | | � P| \ } } } } } | t k ru | | _ q | t k r� P| t k r� | j | � q n� | t k r� | j � | \ | _ | _ q nl | t t f k r� d } nQ | r;| r;| d } | d t | � k r5| j j | � t | � | _ d } | j | � | j j | � | \ | _ | _ | t t f k r | j d 7_ d | _ q Wd j | j � S)NFr Tr r � ���)�iter�len�compatr r� � ENDMARKER�INDENTr� �DEDENT�popr� r� �NEWLINEr r� r� r8 ) r+ �iterable�it�indents� startline�t�tok_type�tokenr� �end�line�indentr- r- r. r � sF zUntokenizer.untokenizec C sJ g } | j j } | d t t f k } d } xt | g | � D]} | d d � \ } } | t k ru | | _ qA | t t t t f k r� | d 7} | t k r� | r� d | } d } n d } | t k r� | j | � qA nW | t k r� | j � qA n; | t t f k rd } n | r8| r8| | d � d } | | � qA Wd S)Nr Fr r� Tr r� )r� r� r� r r r r� �NAME�NUMBER�ASYNC�AWAIT�STRINGr� r� r� ) r+ r� r� r� � toks_appendr� � prevstring�tok�toknum�tokvalr- r- r. r� s8 zUntokenizer.compatN)r4 r5 r6 r� r� r r� r- r- r- r. r� � s %r� c C s= t � } | j | � } | j d k r9 | j | j � } | S)N)r� r r� �encode)r� �ut�outr- r- r. r ? s c C sd | d d � j � j d d � } | d k s= | j d � rA d S| d k s\ | j d � r` d S| S)N� �_r zutf-8zutf-8-�latin-1� iso-8859-1� iso-latin-1�latin-1-� iso-8859-1-� iso-latin-1-)r� r� r� )r� r� r� )�lower�replace� startswith)�orig_enc�encr- r- r. �_get_normal_nameZ s " r� c s1 y � j j � Wn t k r* d � Yn Xd � d } d } � f d d � } � � f d d � } | � } | j t � r� d � | d d � } d } | s� | g f S| | � } | r� | | g f St j | � s� | | g f S| � } | s� | | g f S| | � } | r!| | | g f S| | | g f S) NFzutf-8c s( y � � SWn t k r# d SYn Xd S)N� )� StopIterationr- )�readliner- r. � read_or_stop} s z%detect_encoding.<locals>.read_or_stopc s2 y | j d � } WnB t k rW d } � d k rG d j | � � } t | � � Yn Xt j | � } | sq d St | j d � � } y t | � } WnI t k r� � d k r� d | } n d j � | � } t | � � Yn X� r.| d k r$� d k r d } n d j � � } t | � � | d 7} | S) Nzutf-8z'invalid or missing encoding declarationz {} for {!r}r zunknown encoding: z unknown encoding for {!r}: {}zencoding problem: utf-8z encoding problem for {!r}: utf-8z-sig) �decode�UnicodeDecodeErrorr� � SyntaxError� cookie_re�matchr� r: r � LookupError)r� � line_string�msgr� r� �codec)� bom_found�filenamer- r. � find_cookie� s6 z$detect_encoding.<locals>.find_cookieTr z utf-8-sig)�__self__�name�AttributeErrorr� r �blank_rer� )r� r� �defaultr� r� �first�secondr- )r� r� r� r. r e s8 & c C ss t | d � } yH t | j � \ } } | j d � t | | d d �} d | _ | SWn | j � � Yn Xd S)N�rbr �line_bufferingTrc )� _builtin_openr r� �seekr �mode�close)r� �bufferr� �lines�textr- r- r. r � s r c C s_ d d l m } m } t | � \ } } t | d � } | d � } t | | | | � j | � S)Nr )r �repeatr� )� itertoolsr r� r r� � _tokenize�__next__)r� r r� r� �consumed�rl_gen�emptyr- r- r. r � s c c s� d } } } d } d! \ } } d } d g } d } d } d } d } | d k r| | d k re d } t t | d" d# d � Vx�y | � } Wn t k r� d } Yn X| d k r� | j | � } | d 7} d t | � } } | r�| s� t d | � � | j | � } | rf| j d � } } t t | | d | � | | | f | | � Vd$ \ } } d } q�| r�| d% d � d k r�| d& d � d k r�t t | | | | t | � f | � Vd } d } q q�| | } | | } q n�| d k r�| r�| sPd } xv | | k r�| | d k r7| d 7} n? | | d k r\| t d t } n | | d k rud } n P| d 7} qW| | k r�P| | d k r�| | d k r=| | d � j d � } | t | � } t t | | | f | | t | � f | � Vt t | | d � | | f | t | � f | � Vq t t t f | | d k | | d � | | f | t | � f | � Vq | | d' k r�| j | � t t | d | � | d f | | f | � Vx� | | d( k ri| | k r t d d | | | f � � | d d) � } | rC| | d* k rCd } d } d } t t d | | f | | f | � Vq�W| r�| r�| | d+ k r�d } d } d } n! | s�t d | d f � � d } xo| | k r- t t � j | | � } | r�| j d � \ } } | | f | | f | } } } | | k r-q�| | | � | | } } | | k sx| d k r�| d k r�| d k r�t t | | | | � Vq* | d k r�| r�| Vd } | d k r�t t | | | | � Vq�t t | | | | � V| r�d } q* | d k r2| r| Vd } t t | | | | � Vq* | t k r�t t | � } | j | | � } | r�| j d � } | | | � } t t | | | | f | � Vq�| | f } | | d � } | } Pq* | t k s| d d � t k s| d d � t k r�| d, d k rl| | f } t t | pEt | d pEt | d � } | | d � d } } | } Pq�t t | | | | � Vq* | j � r�| d- k r�| r�t | d k r�t n t | | | | � Vq�t t | | | | � } | d k r| r| } q�| d k rm| rm| j t k rm| j d k rmd } | d. } t t | j | j | j | j! � Vd } | r~| Vd } | Vq* | d k r�d } q* | d k r�| d 7} n | d k r�| d 8} | r�| Vd } t t" | | | | � Vq�t t | | | | f | | d f | � V| d 7} q�Wq W| rC | Vd } x; | d d � D]) } t t d | d f | d f d � VqT Wt t# d | d f | d f d � Vd S)/Nr � 0123456789r� Fz utf-8-sigzutf-8r� r zEOF in multi-line stringr z\ r z\ r� � � z# �#z z3unindent does not match any outer indentation levelz <tokenize>zEOF in multi-line statementr! z...T� �async�await�def�\z([{z)]})r� r )r r )r r )r� r ������r� r� r� r� r� r� )r� r� r� )$r r r� r� r� r� r� r� r� � ERRORTOKEN�tabsize�rstripr r r� r� �IndentationErrorr� rF � PseudoToken�spanr� r� � triple_quoted�endpats� single_quoted� isidentifierr� r� r� r( r1 r� r� r0 r� ) r� r� �lnum�parenlev� continued�numchars�contstr�needcont�contliner� �stashed� async_def�async_def_indent� async_def_nlr� �pos�max�strstart�endprog�endmatchr� �column� comment_token�nl_pos� pseudomatchr� �spos�eposr� �initialr� r� r- r- r. r� � sh 2 $#' - ' $ 'r� c C s t | d � S)N)r� )r� r- r- r. �generate_tokens� s r c s� d d l } d d � � d d � f d d � } | j d d � } | j d d d d d d d d � | j d d d d d d d d �| j � } y� | j r� | j } t | d � � } t t | j � � } Wd QRXn d } t t j j d � } x[ | D]S } | j } | j r| j } d | j | j } t d | t | | j f � q� WWnbt k r�} z? | j d d d � \ } } | | j d | | | f � WYd d } ~ Xnt k r} z5 | j d \ } } | | j d | | | f � WYd d } ~ Xn� t k r8} z | | | � WYd d } ~ Xn t k rf} z | | � WYd d } ~ XnQ t k r�t d � Yn6 t k r�} z � d | � � WYd d } ~ Xn Xd S) Nr c S s t | d t j �d S)N�file)�print�sys�stderr)�messager- r- r. �perror� s zmain.<locals>.perrorc sg | r+ | f | | f } � d | � n+ | rH � d | | f � n � d | � t j d � d S)Nz%s:%d:%d: error: %sz %s: error: %sz error: %sr )r �exit)r r� �location�args)r r- r. �error� s zmain.<locals>.error�progzpython -m tokenize�destr� �nargsr< �metavarz filename.py�helpz'the file to tokenize; defaults to stdinz-ez--exact�exact�action� store_truez(display token names using the exact typer� z<stdin>z %d,%d-%d,%d:z%-20s%-15s%-15rr r z interrupted zunexpected error: %s)�argparse�ArgumentParser� add_argument� parse_argsr� r� �listr r� r� r �stdinr( r* r3 r� r� r r) r1 r� r# r� r� �OSError�KeyboardInterrupt� Exception) r- r$ �parserr# r� �fr� r� � token_type� token_range�errr� r r- )r r. �main� sN // r; �__main__) r> r? rG rH rK rL rI rJ rM rN rO rP rS rT rQ rR rU rV rW rX r[ r\ rY rZ r] r^ r_ r` ra rb ) r@ rA ri rj rk rl rm rn ro rp rq rr rs rt ru rv rw rx ry rz r{ r| r} r~ r r� r� r� r� r� )}� __author__� __credits__�builtinsr r� �codecsr r � collections�ior r� r rB r r� rC �ASCIIr� r� �__all__�N_TOKENSr r) r r �LPAR�RPAR�LSQB�RSQB�COLON�COMMA�SEMI�PLUS�MINUS�STAR�SLASH�VBAR�AMPER�LESS�GREATER�EQUAL�DOT�PERCENT�LBRACE�RBRACE�EQEQUAL�NOTEQUAL� LESSEQUAL� GREATEREQUAL�TILDE� CIRCUMFLEX� LEFTSHIFT� RIGHTSHIFT� DOUBLESTAR� PLUSEQUAL�MINEQUAL� STAREQUAL� SLASHEQUAL� PERCENTEQUAL� AMPEREQUAL� VBAREQUAL�CIRCUMFLEXEQUAL�LEFTSHIFTEQUAL�RIGHTSHIFTEQUAL�DOUBLESTAREQUAL� DOUBLESLASH�DOUBLESLASHEQUAL�AT�ATEQUALr2 � namedtupler r: r; r= � Whitespace�Comment�Ignore�Name� Hexnumber� Binnumber� Octnumber� Decnumber� Intnumber�Exponent� Pointfloat�Expfloat� Floatnumber� Imagnumber�Number� StringPrefix�Single�Double�Single3�Double3�Triple�String�Operator�Bracket�Special�Funny� PlainToken�Token�ContStr� PseudoExtrasr� rF r r� r� r r� r5 r� r� r� r r� r r r� r r; r4 r- r- r- r. �<module> sB " _ ] � <