Ë
    òT•jÐ$  ã                   óˆ   — d Z ddlZddlZddlZdgZ ej                  dd«      Z G d„ d«      Z G d„ d«      Z	 G d	„ d
«      Z
y)a%   robotparser.py

    Copyright (C) 2000  Bastian Kleineidam

    You can choose between two licenses when using this package:
    1) GNU GPLv2
    2) PSF license for Python 2.2

    The robots.txt Exclusion Protocol is implemented as specified in
    http://www.robotstxt.org/norobots-rfc.txt
é    NÚRobotFileParserÚRequestRatezrequests secondsc                   óZ   — e Zd ZdZdd„Zd„ Zd„ Zd„ Zd„ Zd„ Z	d„ Z
d	„ Zd
„ Zd„ Zd„ Zd„ Zy)r   zs This class provides a set of methods to read, parse and answer
    questions about a single robots.txt file.

    c                 óz   — g | _         g | _        d | _        d| _        d| _        | j                  |«       d| _        y )NFr   )ÚentriesÚsitemapsÚdefault_entryÚdisallow_allÚ	allow_allÚset_urlÚlast_checked©ÚselfÚurls     ú)/usr/lib/python3.12/urllib/robotparser.pyÚ__init__zRobotFileParser.__init__   s;   € ØˆŒØˆŒØ!ˆÔØ!ˆÔØˆŒØ�‰�SÔØˆÕó    c                 ó   — | j                   S )z·Returns the time the robots.txt file was last fetched.

        This is useful for long-running web spiders that need to
        check for new robots.txt files periodically.

        )r   ©r   s    r   ÚmtimezRobotFileParser.mtime%   s   € ð × Ñ Ð r   c                 ó6   — ddl }|j                  «       | _        y)zYSets the time the robots.txt file was last fetched to the
        current time.

        r   N)Útimer   )r   r   s     r   ÚmodifiedzRobotFileParser.modified.   s   € ó
 	Ø ŸI™I›KˆÕr   c                 óp   — || _         t        j                  j                  |«      dd \  | _        | _        y)z,Sets the URL referring to a robots.txt file.é   é   N)r   ÚurllibÚparseÚurlparseÚhostÚpathr   s     r   r   zRobotFileParser.set_url6   s-   € àˆŒÜ%Ÿ|™|×4Ñ4°SÓ9¸!¸AÐ>ÑˆŒ	�4•9r   c                 ó¼  — 	 t         j                  j                  | j                  «      }|j	                  «       }| j                  |j                  d«      j                  «       «       y# t         j                  j                  $ rT}|j                  dv rd| _        n4|j                  dk\  r |j                  dk  rd| _        Y d}~yY d}~yY d}~yY d}~yd}~ww xY w)z4Reads the robots.txt URL and feeds it to the parser.zutf-8)i‘  i“  Ti�  iô  N)r   ÚrequestÚurlopenr   Úreadr   ÚdecodeÚ
splitlinesÚerrorÚ	HTTPErrorÚcoder
   r   )r   ÚfÚrawÚerrs       r   r%   zRobotFileParser.read;   s¤   € ð		9Ü—‘×&Ñ& t§x¡xÓ0ˆAð —&‘&“(ˆCØ�J‰J�s—z‘z 'Ó*×5Ñ5Ó7Õ8øô �|‰|×%Ñ%ò 	&Ø�x‰x˜:Ñ%Ø$(�Õ!Ø—‘˜S’ S§X¡X°¢^Ø!%�—‘ô &4”ô "ûð	&ús   ‚)A* Á*CÂ;CÃCc                 ó€   — d|j                   v r| j                  €|| _        y y | j                  j                  |«       y ©NÚ*)Ú
useragentsr	   r   Úappend)r   Úentrys     r   Ú
_add_entryzRobotFileParser._add_entryH   s=   € Ø�%×"Ñ"Ñ"à×!Ñ!Ð)à%*�Õ"ð *ð �L‰L×Ñ Õ&r   c                 óÒ  — d}t        «       }| j                  «        |D �]®  }|s4|dk(  rt        «       }d}n"|dk(  r| j                  |«       t        «       }d}|j                  d«      }|dk\  r|d| }|j	                  «       }|sŒh|j                  dd«      }t        |«      dk(  sŒ‰|d   j	                  «       j                  «       |d<   t        j                  j                  |d   j	                  «       «      |d<   |d   dk(  rB|dk(  r| j                  |«       t        «       }|j                  j                  |d   «       d}�Œ*|d   dk(  r3|dk7  s�Œ9|j                  j                  t        |d   d	«      «       d}�Œe|d   d
k(  r3|dk7  s�Œt|j                  j                  t        |d   d«      «       d}�Œ |d   dk(  r?|dk7  s�Œ¯|d   j	                  «       j                  «       rt!        |d   «      |_        d}�Œç|d   dk(  r˜|dk7  s�Œö|d   j                  d«      }t        |«      dk(  rk|d   j	                  «       j                  «       rJ|d   j	                  «       j                  «       r)t%        t!        |d   «      t!        |d   «      «      |_        d}�Œ‡|d   dk(  s�Œ‘| j(                  j                  |d   «       �Œ± |dk(  r| j                  |«       yy)z”Parse the input lines from a robots.txt file.

        We allow that a user-agent: line is not preceded by
        one or more blank lines.
        r   r   é   ú#Nú:z
user-agentÚdisallowFÚallowTzcrawl-delayzrequest-rateú/Úsitemap)ÚEntryr   r4   ÚfindÚstripÚsplitÚlenÚlowerr   r   Úunquoter1   r2   Ú	rulelinesÚRuleLineÚisdigitÚintÚdelayr   Úreq_rater   )r   ÚlinesÚstater3   ÚlineÚiÚnumberss          r   r   zRobotFileParser.parseQ   s˜  € ð ˆÜ“ˆà�‰ŒØó 7	2ˆDÙØ˜A’:Ü!›G�EØ‘EØ˜a’ZØ—O‘O EÔ*Ü!›G�EØ�Eà—	‘	˜#“ˆAØ�AŠvØ˜B˜Q�x�Ø—:‘:“<ˆDÙØØ—:‘:˜c 1Ó%ˆDÜ�4‹y˜A‹~Ø˜q™'Ÿ-™-›/×/Ñ/Ó1��Q‘Ü Ÿ,™,×.Ñ.¨t°A©w¯}©}«Ó?��Q‘Ø˜‘7˜lÒ*Ø ’zØŸ™¨Ô.Ü %£˜Ø×$Ñ$×+Ñ+¨D°©GÔ4Ø’EØ˜!‘W 
Ò*Ø ”zØŸ™×.Ñ.¬x¸¸Q¹ÀÓ/GÔHØ !šØ˜!‘W Ò'Ø ”zØŸ™×.Ñ.¬x¸¸Q¹ÀÓ/FÔGØ !šØ˜!‘W Ò-Ø ”zð   ™7Ÿ=™=›?×2Ñ2Ô4Ü*-¨d°1©g«,˜EœKØ !šØ˜!‘W Ò.Ø ”zØ"& q¡'§-¡-°Ó"4˜ä ›L¨AÒ-°'¸!±*×2BÑ2BÓ2D×2LÑ2LÔ2NØ '¨¡
× 0Ñ 0Ó 2× :Ñ :Ô <Ü-8¼¸WÀQ¹Z»Ì#ÈgÐVWÉjË/Ó-Z˜EœNØ !šØ˜!‘W 	Ô)ð
 —M‘M×(Ñ(¨¨a©Ö1ðo7	2ðp �AŠ:Ø�O‰O˜EÕ"ð r   c                 ób  — | j                   ry| j                  ry| j                  syt        j                  j                  t        j                  j                  |«      «      }t        j                  j                  dd|j                  |j                  |j                  |j                  f«      }t        j                  j                  |«      }|sd}| j                  D ]&  }|j                  |«      sŒ|j                  |«      c S  | j                   r| j                   j                  |«      S y)z=using the parsed robots.txt decide if useragent can fetch urlFTÚ r;   )r
   r   r   r   r   r   rC   Ú
urlunparser!   ÚparamsÚqueryÚfragmentÚquoter   Ú
applies_toÚ	allowancer	   )r   Ú	useragentr   Ú
parsed_urlr3   s        r   Ú	can_fetchzRobotFileParser.can_fetchš   sú   € à×ÒØØ�>Š>Øð
 × Ò Øô —\‘\×*Ñ*¬6¯<©<×+?Ñ+?ÀÓ+DÓEˆ
Ü�l‰l×%Ñ% r¨"¨Z¯_©_Ø×Ñ˜j×.Ñ.°
×0CÑ0Cð'Eó Fˆä�l‰l× Ñ  Ó%ˆÙØˆCØ—\‘\ò 	,ˆEØ×Ñ 	Õ*Ø—‘ sÓ+Ò+ð	,ð ×ÒØ×%Ñ%×/Ñ/°Ó4Ð4àr   c                 óÊ   — | j                  «       sy | j                  D ]!  }|j                  |«      sŒ|j                  c S  | j                  r| j                  j                  S y ©N)r   r   rV   rH   r	   ©r   rX   r3   s      r   Úcrawl_delayzRobotFileParser.crawl_delay·   sY   € Ø�z‰zŒ|ØØ—\‘\ò 	#ˆEØ×Ñ 	Õ*Ø—{‘{Ò"ð	#ð ×ÒØ×%Ñ%×+Ñ+Ð+Ør   c                 óÊ   — | j                  «       sy | j                  D ]!  }|j                  |«      sŒ|j                  c S  | j                  r| j                  j                  S y r\   )r   r   rV   rI   r	   r]   s      r   Úrequest_ratezRobotFileParser.request_rateÁ   sY   € Ø�z‰zŒ|ØØ—\‘\ò 	&ˆEØ×Ñ 	Õ*Ø—~‘~Ò%ð	&ð ×ÒØ×%Ñ%×.Ñ.Ð.Ør   c                 ó4   — | j                   sy | j                   S r\   )r   r   s    r   Ú	site_mapszRobotFileParser.site_mapsË   s   € Ø�}Š}ØØ�}‰}Ðr   c                 ó�   — | j                   }| j                  �|| j                  gz   }dj                  t        t        |«      «      S )Nz

)r   r	   ÚjoinÚmapÚstr)r   r   s     r   Ú__str__zRobotFileParser.__str__Ð   s@   € Ø—,‘,ˆØ×ÑÐ)Ø ×!3Ñ!3Ð 4Ñ4ˆGØ�{‰{œ3œs GÓ,Ó-Ð-r   N)rP   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r   r%   r4   r   rZ   r^   r`   rb   rg   © r   r   r   r      sE   „ ñó
ò!ò(ò?ò
9ò'òG#òRò:òòó
.r   c                   ó"   — e Zd ZdZd„ Zd„ Zd„ Zy)rE   zoA rule line is a single "Allow:" (allowance==True) or "Disallow:"
       (allowance==False) followed by a path.c                 óä   — |dk(  r|sd}t         j                  j                  t         j                  j                  |«      «      }t         j                  j	                  |«      | _        || _        y )NrP   T)r   r   rQ   r   rU   r!   rW   )r   r!   rW   s      r   r   zRuleLine.__init__Ú   sP   € Ø�2Š:™iàˆIÜ�|‰|×&Ñ&¤v§|¡|×'<Ñ'<¸TÓ'BÓCˆÜ—L‘L×&Ñ& tÓ,ˆŒ	Ø"ˆ�r   c                 óZ   — | j                   dk(  xs |j                  | j                   «      S r/   )r!   Ú
startswith)r   Úfilenames     r   rV   zRuleLine.applies_toâ   s%   € Ø�y‰y˜CÑÒA 8×#6Ñ#6°t·y±yÓ#AÐAr   c                 óB   — | j                   rdnddz   | j                  z   S )NÚAllowÚDisallowz: )rW   r!   r   s    r   rg   zRuleLine.__str__å   s   € ØŸ>š>‘¨z¸TÑAÀDÇIÁIÑMÐMr   N)rh   ri   rj   rk   r   rV   rg   rl   r   r   rE   rE   ×   s   „ ñ1ò#òBóNr   rE   c                   ó(   — e Zd ZdZd„ Zd„ Zd„ Zd„ Zy)r=   z?An entry has one or more user-agents and zero or more rulelinesc                 ó<   — g | _         g | _        d | _        d | _        y r\   )r1   rD   rH   rI   r   s    r   r   zEntry.__init__ë   s   € ØˆŒØˆŒØˆŒ
Øˆ�r   c                 óž  — g }| j                   D ]  }|j                  d|› �«       Œ | j                  �|j                  d| j                  › �«       | j                  �7| j                  }|j                  d|j                  › d|j
                  › �«       |j                  t        t        | j                  «      «       dj                  |«      S )NzUser-agent: zCrawl-delay: zRequest-rate: r;   ú
)r1   r2   rH   rI   ÚrequestsÚsecondsÚextendre   rf   rD   rd   )r   ÚretÚagentÚrates       r   rg   zEntry.__str__ñ   s§   € ØˆØ—_‘_ò 	/ˆEØ�J‰J˜ e WÐ-Õ.ð	/à�:‰:Ð!Ø�J‰J˜ t§z¡z lÐ3Ô4Ø�=‰=Ð$Ø—=‘=ˆDØ�J‰J˜¨¯© °a¸¿¹°~ÐFÔGØ�
‰
”3”s˜DŸN™NÓ+Ô,Ø�y‰y˜‹~Ðr   c                 ó¤   — |j                  d«      d   j                  «       }| j                  D ]  }|dk(  r y|j                  «       }||v sŒ y y)z2check if this entry applies to the specified agentr;   r   r0   TF)r@   rB   r1   )r   rX   r}   s      r   rV   zEntry.applies_toý   sX   € ð —O‘O CÓ(¨Ñ+×1Ñ1Ó3ˆ	Ø—_‘_ò 	ˆEØ˜Š|áØ—K‘K“MˆEØ˜	Ò!Ùð	ð r   c                 ód   — | j                   D ]!  }|j                  |«      sŒ|j                  c S  y)zZPreconditions:
        - our agent applies to this entry
        - filename is URL decodedT)rD   rV   rW   )r   rq   rL   s      r   rW   zEntry.allowance
  s2   € ð —N‘Nò 	&ˆDØ�‰˜xÕ(Ø—~‘~Ò%ð	&ð r   N)rh   ri   rj   rk   r   rg   rV   rW   rl   r   r   r=   r=   é   s   „ ÙIòò
òór   r=   )rk   ÚcollectionsÚurllib.parser   Úurllib.requestÚ__all__Ú
namedtupler   r   rE   r=   rl   r   r   ú<module>r†      sU   ðñ
ó Û Û àÐ
€à$ˆk×$Ñ$ ]Ð4FÓG€÷~.ñ ~.÷BNñ N÷$(ò (r   