f
��_�$ c @ sX d dl Z d dlZd dlZdgZe �dd�ZG dd� d�ZG dd� d�ZG dd � d �Z dS )
� N�RobotFileParser� RequestRatezrequests secondsc @ sn e Zd Zddd�Zdd� Zdd� Zdd � Zd
d � Zd d
� Zdd� Z dd� Z
dd� Z dd� Z dd� Z
dd� ZdS ) r � c C s2 g | _ g | _d | _d| _d| _| �|� d| _d S )NFr )�entries�sitemaps�
default_entry� disallow_all� allow_all�set_url� last_checked��self�url� r �+/usr/lib64/python3.10/urllib/robotparser.py�__init__ s
zRobotFileParser.__init__c C s | j S �N)r
�r r r r �mtime% s zRobotFileParser.mtimec C s dd l }|� � | _d S )Nr )�timer
)r r r r r �modified. s zRobotFileParser.modifiedc C s& || _ tj�|�dd� \| _| _d S )N� i )r
�urllib�parse�urlparseZhost�pathr r r r r 6 s zRobotFileParser.set_urlc
C s� zt j�| j�}W nT t jjyf } z8|jdv r8d| _n|jdkrR|jdk rRd| _W Y d }~n&d }~0 0 |� � }| �
|� d�� � � d S )N)i� i� Ti� i� zutf-8)
r ZrequestZurlopenr
ZerrorZ HTTPErrorZcoder r �readr ZdecodeZ
splitlines)r ZfZerrZrawr r r r ; s
zRobotFileParser.readc C s, d|j v r | jd u r(|| _n | j�|� d S �N�*)�
useragentsr r �append)r �entryr r r �
_add_entryH s
zRobotFileParser._add_entryc C sP d}t � }| �� |D �] }|sP|dkr4t � }d}n |dkrP| �|� t � }d}|�d�}|dkrn|d |� }|�� }|s|q|�dd�}t|�dkr|d �� �� |d< tj �
|d �� �|d< |d dkr�|dkr�| �|� t � }|j � |d � d}q|d dk�r.|dk�r6|j
� t|d d�� d}q|d d k�rb|dk�r6|j
� t|d d
�� d}q|d d k�r�|dk�r6|d �� �� �r�t|d �|_d}q|d d k�r|dk�r6|d �d
�}t|�dk�r|d �� �� �r|d �� �� �rtt|d �t|d ��|_d}q|d dkr| j� |d � q|dk�rL| �|� d S )Nr r i z#z:z
user-agentZdisallowFZallowTz crawl-delayz request-rate�/Zsitemap)�Entryr r! ZfindZstrip�splitZlen�lowerr r �unquoter r � rulelines�RuleLineZisdigitZint�delayr �req_rater )r ZlinesZstater �lineZiZnumbersr r r r Q sj
�
zRobotFileParser.parsec C s� | j r
dS | jrdS | js dS tj�tj�|��}tj�dd|j|j |j
|j f�}tj� |�}|sfd}| j
D ]}|�|�rl|�|� S | jr�| j�|�S dS )NFTr r" )r r r
r r r r& �
urlunparser ZparamsZqueryZfragment�quoter �
applies_to� allowancer )r � useragentr
Z
parsed_urlr r r r � can_fetch� s& �
zRobotFileParser.can_fetchc C �< | � � s d S | jD ]}|�|�r|j S | jr8| jjS d S r )r r r. r) r �r r0 r r r r � crawl_delay� �
zRobotFileParser.crawl_delayc C r2 r )r r r. r* r r3 r r r � request_rate� r5 z RobotFileParser.request_ratec C s | j s
d S | j S r )r r r r r � site_maps� s zRobotFileParser.site_mapsc C s, | j }| jd ur || jg }d�tt|��S )Nz
)r r �join�map�str)r r r r r �__str__� s
zRobotFileParser.__str__N)r )�__name__�
__module__� __qualname__r r r r r r! r r1 r4 r6 r7 r; r r r r r s
I
c @ s$ e Zd Zdd� Zdd� Zdd� ZdS )r( c C s< |dkr|sd}t j�t j�|��}t j�|�| _|| _d S )Nr T)r r r, r r- r r/ )r r r/ r r r r � s
zRuleLine.__init__c C s | j dkp|�| j �S r )r Z
startswith)r �filenamer r r r. � � zRuleLine.applies_toc C s | j r
dndd | j S )NZAllowZDisallowz: )r/ r r r r r r; � r@ zRuleLine.__str__N)r< r= r> r r. r; r r r r r( � s r( c @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S )
r# c C s g | _ g | _d | _d | _d S r )r r' r) r* r r r r r � s zEntry.__init__c C s� g }| j D ]}|�d|� �� q
| jd ur<|�d| j� �� | jd urf| j}|�d|j� d|j� �� |�tt| j �� d�
|�S )Nz User-agent: z
Crawl-delay: zRequest-rate: r" Z
) r r r) r* ZrequestsZsecondsZextendr9 r: r' r8 )r Zret�agentZrater r r r; � s
z
Entry.__str__c C sD |� d�d �� }| jD ]&}|dkr* dS |�� }||v r dS dS )Nr" r r TF)r$ r% r )r r0 rA r r r r. � s
zEntry.applies_toc C s" | j D ]}|�|�r|j S dS )NT)r' r. r/ )r r? r+ r r r r/
s
zEntry.allowanceN)r< r= r> r r; r. r/ r r r r r# � s
r# )
Z collectionsZ urllib.parser Zurllib.requestZ__all__Z
namedtupler r r( r# r r r r Z<module>
s B