add-determinism 0.7.3

RPM buildroot helper to strip nondeterministic bits in files
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115

a

Z��^?Rc
@s�dZdZddlZddlZddlmZmZddlTddlm	Z	dd	�e
e	�D�gd
�Z[	zeWne
yveZYn0dd�Zd
d�Zdd�Zdd�ZdZdZeede�ee�ZdZdZdZdZedd�Zeeeee�ZdZedd�ee�ZdeZeee�Zed ed!�Z ee ee�Z!d"Z"d#Z#d$Z$d%Z%d&Z&ee&d'e&d(�Z'ee&d)e&d*�Z(ed+d,d-d.d/d0d1d2d3�	Z)d4Z*ed5d6d7�Z+ee)e*e+�Z,ee!e,e(e�Z-ee-Z.ee&d8ed9d�e&d:ed;d��Z/edee'�Z0eee0e!e,e/e�Z1e2ej3e.e1e$e%f�\Z4Z5Z6Z7ed<d=d>d?�ed<d=d@dA�BhdB�BZ8e�3e"�e�3e#�e6e7dC�dDdE�e8D��dFdE�e8D��dGdE�e8D��Z9d'd(hdHdI�e8D�BdJdI�e8D�BZ:d9d;hdKdI�e8D�BdLdI�e8D�BZ;dMZ<GdNdO�dOe=�Z>GdPdQ�dQe=�Z?dRdS�Z@e@fdTdU�ZAdVdW�ZBGdXdY�dY�ZCe�3dZejD�ZEe�3d[ejD�ZFd\d]�ZGd^d_�ZHd`da�ZIdbdc�ZJeKddk�r�ddlLZLeMeLjN�dk�rxeAeOeLjNd�jP�neAeLjQjP�dS)ezKa-Ping Yee <ping@lfw.org>z@GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro�N)�BOM_UTF8�lookup)�*�)�tokencCsg|]}|ddkr|�qS)r�_�)�.0�xrr�./usr/lib64/python3.9/lib2to3/pgen2/tokenize.py�
<listcomp>%�r)�tokenize�generate_tokens�
untokenizecGsdd�|�dS)Nz(z|z))�join�Zchoicesrrr
�group0rrcG�t|�dS)Nr�rrrrr
�any1rrcGr)Nz?rrrrr
�maybe2rrcst�fdd��D��S)Nc3s4|],}�dD]}|��|��kr||VqqdS))�N)Zcasefold)rr	Zy�Zlrr
Z	<genexpr>4sz _combinations.<locals>.<genexpr>)Zsetrrrr
�
_combinations3s�rz[ \f\t]*z	#[^\r\n]*z\\\r?\nz\w+z0[bB]_?[01]+(?:_[01]+)*z(0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?z0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?z[1-9]\d*(?:_\d+)*[lL]?z0[lL]?z[eE][-+]?\d+(?:_\d+)*z\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?z\.\d+(?:_\d+)*z\d+(?:_\d+)*z\d+(?:_\d+)*[jJ]z[jJ]z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'(?:[uUrRbBfF]|[rR][fFbB]|[fFbBuU][rR])?�'''�"""z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z<>z!=z//=?z->z[+\-*/%&@|^=<>]=?z~z[][(){}]z\r?\nz:=z[:;.,`@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*�'z"[^\n"\\]*(?:\\.[^\n"\\]*)*�"ZrZRZfZFZbZB>ZuRZurZURZUZuZUr)rrrrcC�i|]}|�d�t�qS�r)�single3prog�rZprefixrrr
�
<dictcomp>yrr"cCr�r)�double3progr!rrr
r"zrcCsi|]
}|d�qS�Nrr!rrr
r"{rcC�h|]}|�d��qSrrr!rrr
�	<setcomp>rr'cCr&r#rr!rrr
r'�rcCr&)rrr!rrr
r'�rcCr&)rrr!rrr
r'�ric@�eZdZdS)�
TokenErrorN��__name__�
__module__�__qualname__rrrr
r)�rr)c@r()�StopTokenizingNr*rrrr
r.�rr.c		Cs4|\}}|\}}td||||t|t|�f�dS)Nz%d,%d-%d,%d:	%s	%s)ZprintZtok_nameZrepr)	ZtyperZxxx_todo_changemeZxxx_todo_changeme1�lineZsrowZscolZerowZecolrrr
�
printtoken�s
�r0cCs&zt||�Wnty Yn0dSr%)�
tokenize_loopr.)�readline�
tokeneaterrrr
r
�s
r
cCst|�D]}||�qdSr%)r)r2r3Z
token_inforrr
r1�sr1c@s,eZdZdd�Zdd�Zdd�Zdd�Zd	S)
�UntokenizercCsg|_d|_d|_dS)Nrr)�tokens�prev_row�prev_col)�selfrrr
�__init__�szUntokenizer.__init__cCs*|\}}||j}|r&|j�d|�dS)N� )r7r5�append)r8�startZrowZcolZ
col_offsetrrr
�add_whitespace�s
zUntokenizer.add_whitespacecCs�|D]p}t|�dkr$|�||�qv|\}}}}}|�|�|j�|�|\|_|_|ttfvr|jd7_d|_qd�	|j�S)N�rrr)
�len�compatr=r5r;r6r7�NEWLINE�NLr)r8�iterableZtZtok_typerr<�endr/rrr
r�s
zUntokenizer.untokenizec	Cs�d}g}|jj}|\}}|ttfvr,|d7}|ttfvr<d}|D]�}|dd�\}}|ttttfvrl|d7}|tkr�|�|�q@n>|t	kr�|�
�q@n*|ttfvr�d}n|r�|r�||d�d}||�q@dS)NFr:Tr>�����)r5r;�NAME�NUMBERrArB�ASYNC�AWAIT�INDENT�DEDENTZpop)	r8rrCZ	startline�indentsZtoks_appendZtoknumZtokval�tokrrr
r@�s0
zUntokenizer.compatN)r+r,r-r9r=rr@rrrr
r4�sr4z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s^[ \t\f]*(?:[#\r\n]|$)cCsH|dd����dd�}|dks*|�d�r.dS|dvs@|�d�rDdS|S)	Nirz-�utf-8zutf-8-)zlatin-1�
iso-8859-1ziso-latin-1)zlatin-1-ziso-8859-1-ziso-latin-1-rO)ZlowerZreplace�
startswith)Zorig_encZencrrr
�_get_normal_name�s�rQcs�d�d}d}�fdd�}�fdd�}|�}|�t�rHd�|dd�}d	}|sT|gfS||�}|rj||gfSt�|�s~||gfS|�}|s�||gfS||�}|r�|||gfS|||gfS)
NFrNcs&z��WSty t�YS0dSr%)�
StopIteration�bytesr)r2rr
�read_or_stopsz%detect_encoding.<locals>.read_or_stopcs�z|�d�}Wnty"YdS0t�|�}|s6dSt|�d��}zt|�}Wntyntd|��Yn0�r�|j	dkr�td��|d7}|S)NZasciirzunknown encoding: rNzencoding problem: utf-8z-sig)
ZdecodeZUnicodeDecodeError�	cookie_re�matchrQrrZLookupErrorZSyntaxErrorZname)r/Zline_stringrV�encodingZcodec)�	bom_foundrr
�find_cookies"

z$detect_encoding.<locals>.find_cookieT�z	utf-8-sig)rPr�blank_rerV)r2rWZdefaultrTrYZfirstZsecondr)rXr2r
�detect_encoding�s0




r\cCst�}|�|�Sr%)r4r)rCZutrrr
r:srccsrd}}}d\}}d}dg}d}d}	d}
d}z
|�}WntyNd}Yn0|d}dt|�}
}|�r0|sztd|��|�|�}|r�|�d�}
}t||d|�|||f||fVd\}}d}nd|�r|dd�dk�r|d	d�d
k�rt||||t|�f|fVd}d}q.n||}||}q.�nB|dk�rZ|�sZ|�sJ�qd}|
|k�r�||
dk�rp|d}n8||
dk�r�|tdt}n||
d
k�r�d}n�q�|
d}
�qN|
|k�rq|�r�|Vd}||
dv�r�||
dk�rR||
d��d�}|
t|�}t	|||
f||
t|�f|fVt
||d�||f|t|�f|fVq.t
t	f||
dk||
d�||
f|t|�f|fVq.||dk�r�|�|�t|d|
�|df||
f|fV||dk�r2||v�r�t
dd||
|f��|dd�}|	�r|
|dk�rd}	d}d}
td||
f||
f|fV�q�|	�rr|�rr|
|dk�rrd}	d}d}
n|�sntd|df��d}|
|kr.t�||
�}|�r�|�d�\}}||f||f|}}}
|||�||}}|tjv�s�|dk�r�|dk�r�t||||fV�q|dv�rHt}|dk�rt
}n
|	�r$d}|�r4|Vd}|||||fV�q|dk�rv|�rb|Vd}t	||||fV�q|tv�r�t|}|�||
�}|�r�|�d�}
|||
�}|�r�|Vd}t||||
f|fVn||f}||d�}|}q.�q|tv�s"|dd�tv�s"|dd�tv�r�|ddk�rv||f}t|�pZt|d�pZt|d}||d�d}}|}q.n |�r�|Vd}t||||fV�q|���rp|dv�r�|	�r�|dk�r�tnt||||fV�qrt||||f}|dk�r�|�s�|}�qr|dk�rX|�rX|dtk�rX|ddk�rXd}	|d}
t|d|d|d|dfVd}|�rh|Vd}|Vnz|dk�r�|�r�|Vd}t
||||
f|fVd}nF|dv�r�|d}n|d v�r�|d}|�r�|Vd}t||||fVn(t||
||
f||
df|fV|
d}
�qrq.|�r*|Vd}|dd�D]}td|df|dfdfV�q6td|df|dfdfVdS)!Nr)rrFrrzEOF in multi-line stringi����z\
i����z\
r:z	zz#
z#z
rEz3unindent does not match any outer indentation levelz
<tokenize>zEOF in multi-line statementZ.Tr>rZZ
)�asyncZawaitr]Zdefiz\z([{z)]})rRr?r)rVrDZSTRINGZ
ERRORTOKEN�tabsizeZrstripZCOMMENTrBr;rJZIndentationErrorrK�
pseudoprogZspan�stringZdigitsrGrA�
triple_quoted�endprogs�
single_quotedZisidentifierrHrIrFZOPZ	ENDMARKER)r2ZlnumZparenlevZ	continuedZcontstrZneedcontZcontlinerLZstashedZ	async_defZasync_def_indentZasync_def_nlr/ZposZmaxZstrstartZendprogZendmatchrDZcolumnZ
comment_tokenZnl_posZpseudomatchr<ZsposZeposrZinitialZnewlinerMZindentrrr
rOs�



�*�

 ���
 

���





��
�

�

�
��
�rZ__main__)RZ
__author__Z__credits__r`ZreZcodecsrrZlib2to3.pgen2.tokenrrZdirZ__all__rSZ	NameErrorZstrrrrrZ
WhitespaceZCommentZIgnoreZNameZ	BinnumberZ	HexnumberZ	OctnumberZ	DecnumberZ	IntnumberZExponentZ
PointfloatZExpfloatZFloatnumberZ
ImagnumberZNumberZSingleZDoubleZSingle3ZDouble3Z
_litprefixZTripleZStringZOperatorZBracketZSpecialZFunnyZ
PlainTokenZTokenZContStrZPseudoExtrasZPseudoTokenZmapZcompileZ	tokenprogr_r r$Z_strprefixesrbrarcr^Z	Exceptionr)r.r0r
r1r4ZASCIIrUr[rQr\rrr+Zsysr?ZargvZopenr2Zstdinrrrr
Z<module>s��


�������������������8Ib
&