o
    JAfR                  
   @   s  	 d Z dZddlZddlZddlmZmZ ddlT ddlm	Z	 dd	 e
e	D g d
 Z[	ze W n ey<   eZY nw dd Zdd Zdd Zdd ZdZdZeede  ee ZdZdZdZdZeddZeeeeeZdZeddee Zde ZeeeZed ed! Z ee eeZ!d"Z"d#Z#d$Z$d%Z%d&Z&ee&d' e&d( Z'ee&d) e&d* Z(ed+d,d-d.d/d0d1d2d3	Z)d4Z*ed5d6d7Z+ee)e*e+Z,ee!e,e(eZ-ee- Z.ee&d8 ed9d e&d: ed;d Z/edee'Z0eee0e!e,e/e Z1e2ej3e.e1e$e%f\Z4Z5Z6Z7ed<d=d>d?ed<d=d@dAB h dBB Z8e3e"e3e#e6e7dCdDdE e8D dFdE e8D dGdE e8D Z9d'd(hdHdI e8D B dJdI e8D B Z:d9d;hdKdI e8D B dLdI e8D B Z;dMZ<G dNdO dOe=Z>G dPdQ dQe=Z?dRdS Z@e@fdTdUZAdVdW ZBG dXdY dYZCe3dZejDZEe3d[ejDZFd\d] ZGd^d_ ZHd`da ZIdbdc ZJeKddkrddlLZLeMeLjNdkreAeOeLjNd jP dS eAeLjQjP dS dS )ezKa-Ping Yee <ping@lfw.org>z@GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro    N)BOM_UTF8lookup)*   )tokenc                 C   s   g | ]
}|d  dkr|qS )r   _ ).0xr   r   -/usr/lib/python3.10/lib2to3/pgen2/tokenize.py
<listcomp>%   s    r   )tokenizegenerate_tokens
untokenizec                  G   s   dd |  d S )N(|))joinchoicesr   r   r   group0       r   c                  G      t |  d S )Nr   r   r   r   r   r   any1       r   c                  G   r   )N?r   r   r   r   r   maybe2   r   r   c                     s   t  fdd D S )Nc                 3   s6    | ]} d  D ]}|  |  kr|| V  qqdS )) N)casefold)r	   r
   ylr   r   	<genexpr>4   s     z _combinations.<locals>.<genexpr>)setr!   r   r!   r   _combinations3   s   r%   z[ \f\t]*z	#[^\r\n]*z\\\r?\nz\w+z0[bB]_?[01]+(?:_[01]+)*z(0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?z0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?z[1-9]\d*(?:_\d+)*[lL]?z0[lL]?z[eE][-+]?\d+(?:_\d+)*z\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?z\.\d+(?:_\d+)*z\d+(?:_\d+)*z\d+(?:_\d+)*[jJ]z[jJ]z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'(?:[uUrRbBfF]|[rR][fFbB]|[fFbBuU][rR])?'''"""z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z<>z!=z//=?z->z[+\-*/%&@|^=<>]=?~z[][(){}]z\r?\nz:=z[:;.,`@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"rRfFbB>   ZuRUuZUrZURur)r)   r*   r&   r'   c                 C      i | ]}| d t qS r&   )single3progr	   prefixr   r   r   
<dictcomp>y       r9   c                 C   r4   r'   )double3progr7   r   r   r   r9   z   r:   c                 C   s   i | ]}|d qS Nr   r7   r   r   r   r9   {   r   c                 C      h | ]}| d qS r5   r   r7   r   r   r   	<setcomp>       r?   c                 C   r>   r;   r   r7   r   r   r   r?      r@   c                 C   r>   )r)   r   r7   r   r   r   r?      r@   c                 C   r>   )r*   r   r7   r   r   r   r?      r@      c                   @      e Zd ZdS )
TokenErrorN__name__
__module____qualname__r   r   r   r   rC      r   rC   c                   @   rB   )StopTokenizingNrD   r   r   r   r   rH      r   rH   c           	   	   C   s4   |\}}|\}}t d||||t|  t|f  d S )Nz%d,%d-%d,%d:	%s	%s)printtok_namerepr)	typer   Zxxx_todo_changemeZxxx_todo_changeme1lineZsrowZscolZerowZecolr   r   r   
printtoken   s
   
rN   c                 C   s(   	 zt | | W d S  ty   Y d S w r=   )tokenize_looprH   )readline
tokeneaterr   r   r   r      s   r   c                 C   s   t | D ]}||  qd S r=   )r   )rP   rQ   Z
token_infor   r   r   rO      s   
rO   c                   @   s,   e Zd Zdd Zdd Zdd Zdd Zd	S )
Untokenizerc                 C   s   g | _ d| _d| _d S )Nr   r   )tokensprev_rowprev_col)selfr   r   r   __init__   s   
zUntokenizer.__init__c                 C   s.   |\}}|| j  }|r| jd|  d S d S )N )rU   rS   append)rV   startrowcol
col_offsetr   r   r   add_whitespace   s
   
zUntokenizer.add_whitespacec                 C   s   |D ]8}t |dkr| ||  n)|\}}}}}| | | j| |\| _| _|ttfv r:|  jd7  _d| _qd	| jS )N   r   r   r   )
lencompatr^   rS   rY   rT   rU   NEWLINENLr   )rV   iterablettok_typer   rZ   endrM   r   r   r   r      s   
zUntokenizer.untokenizec           	      C   s   d}g }| j j}|\}}|ttfv r|d7 }|ttfv rd}|D ]B}|d d \}}|ttttfv r6|d7 }|tkr@|| q |t	krI|
  q |ttfv rRd}n|r^|r^||d  d}|| q d S )NFrX   Tr_   )rS   rY   NAMENUMBERrb   rc   ASYNCAWAITINDENTDEDENTpop)	rV   r   rd   	startlineindentstoks_appendtoknumtokvaltokr   r   r   ra      s2   

zUntokenizer.compatN)rE   rF   rG   rW   r^   r   ra   r   r   r   r   rR      s
    rR   z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s   ^[ \t\f]*(?:[#\r\n]|$)c                 C   sJ   	 | d d   dd}|dks|drdS |dv s!|dr#dS | S )	N   r   -utf-8zutf-8-)zlatin-1
iso-8859-1ziso-latin-1)zlatin-1-ziso-8859-1-ziso-latin-1-ry   )lowerreplace
startswith)orig_encencr   r   r   _get_normal_name   s   r   c                    s   	 d d }d}fdd} fdd}| }| tr%d |dd  }d	}|s+|g fS ||}|r6||gfS t|s@||gfS | }|sJ||gfS ||}|rV|||gfS |||gfS )
NFrx   c                      s"   z  W S  t y   t  Y S w r=   )StopIterationbytesr   )rP   r   r   read_or_stop  s
   
z%detect_encoding.<locals>.read_or_stopc                    s   z|  d}W n
 ty   Y d S w t|}|sd S t|d}zt|}W n ty5   td| w  rE|j	dkrAtd|d7 }|S )Nasciir   zunknown encoding: rx   zencoding problem: utf-8z-sig)
decodeUnicodeDecodeError	cookie_rematchr   r   r   LookupErrorSyntaxErrorname)rM   line_stringr   encodingcodec)	bom_foundr   r   find_cookie  s&   

z$detect_encoding.<locals>.find_cookieT   z	utf-8-sig)r|   r   blank_rer   )rP   r   defaultr   r   firstsecondr   )r   rP   r   detect_encoding   s2   




r   c                 C   s   	 t  }|| S r=   )rR   r   )rd   utr   r   r   r   :  s   
r   c                 c   sx   	 d } }}d\}}d }dg}d }d}	d}
d}	 z|  }W n t y*   d}Y nw |d }dt|}}|r|s?td|||}|rg|d }}t||d |  |||f|| fV  d\}}d }nN|r|dd  dkr|d	d  d
krt|| ||t|f|fV  d}d }q|| }|| }q|dkr|s|snd}||k r|| dkr|d }n|| dkr|t d t }n
|| dkrd}nn|d }||k s||krِn|r|V  d }|| dv r;|| dkr ||d  d}|t| }t	|||f||t| f|fV  t
||d  ||f|t|f|fV  nt
t	f|| dk ||d  ||f|t|f|fV  q||d krW|| t|d | |df||f|fV  ||d k r||vrltdd|||f|d d }|	r|
|d krd}	d}d}
td||f||f|fV  ||d k s^|	r|r|
|d krd}	d}d}
n|std|dfd}||k rt||}|rv|d\}}||f||f|}}}||| || }}|tjv s|dkr|dkrt||||fV  n|dv r!t}|dkr
t
}n|	rd}|r|V  d }|||||fV  ni|dkr8|r.|V  d }t	||||fV  nR|tv rwt| }|||}|ri|d}||| }|r]|V  d }t||||f|fV  n!||f}||d  }|}n|tv s|d d tv s|d d tv r|d dkr||f}t| pt|d  pt|d  }||d  d}}|}n|r|V  d }t||||fV  n| r8|dv r|	r|dkrtnt||||fV  qt||||f}|dkr|s|}q|dv r,|r,|d tkr,|d dkr,|dkrd}	|d }
t|d |d |d |d fV  d }|r4|V  d }|V  nR|dkrR|rE|V  d }t
||||f|fV  d}n8|d v r\|d }n	|d!v re|d }|rm|V  d }t||||fV  nt|| ||f||d f|fV  |d }||k sq|r|V  d }|dd  D ]}td|df|dfdfV  qtd|df|dfdfV  d S )"Nr   )r   r   Fr   r   zEOF in multi-line stringz\
z\
rX   	z#
#z
rh   z3unindent does not match any outer indentation levelz
<tokenize>zEOF in multi-line statement.Tr_   r   
)asyncawaitr   )defforr      \z([{z)]})r   r`   rC   r   rg   STRING
ERRORTOKENtabsizerstripCOMMENTrc   rY   rm   IndentationErrorrn   
pseudoprogspanstringdigitsrj   rb   triple_quotedendprogssingle_quotedisidentifierrk   rl   ri   OP	ENDMARKER)rP   lnumparenlev	continuedcontstrneedcontcontlinerq   ZstashedZ	async_defZasync_def_indentZasync_def_nlrM   posmaxstrstartendprogendmatchrg   columncomment_tokenZnl_pospseudomatchrZ   sposeposr   initialnewlineru   indentr   r   r   r   O  s  


$
 















 @r   __main__)R
__author____credits__r   recodecsr   r   Zlib2to3.pgen2.tokenr   r   dir__all__r   	NameErrorstrr   r   r   r%   
WhitespaceCommentIgnoreName	Binnumber	Hexnumber	Octnumber	Decnumber	IntnumberExponent
PointfloatExpfloatFloatnumber
ImagnumberNumberSingleDoubleSingle3Double3Z
_litprefixTripleStringZOperatorZBracketSpecialFunny
PlainTokenTokenContStrPseudoExtrasPseudoTokenmapcompileZ	tokenprogr   r6   r<   Z_strprefixesr   r   r   r   	ExceptionrC   rH   rN   r   rO   rR   ASCIIr   r   r   r   r   r   rE   sysr`   argvopenrP   stdinr   r   r   r   <module>   s   

8I 
c(