o j6hS @sdZddlZddlZddlZddlZddlZddlZddlZddlZ ddl Z ddl Z ddl mZddlmZddlmZmZmZmZmZmZmZmZmZmZmZmZddlmZm Z ddl!m"Z"ddl#m$Z$m%Z%dd l&m'Z'dd l(m)Z)dd l*m+Z+dd l,m-Z-dd l.m/Z/ddl0m1Z1ddl2m3Z3ddl4m5Z5m6Z6ddl7m8Z8ddl9m:Z:m;Z;mZ=e?e@ZAe jBjCjDZEeeFeFfZGdeFdeeFfddZHGdddeIZJde"ddfddZKGdddeIZLdeFd e-ddfd!d"ZMdeFd e-de"fd#d$ZNd%eGdeeFfd&d'ZOd(eEd)eFdeFfd*d+ZPd,eFdeFfd-d.ZQd,eFdeFfd/d0ZReSd1ejTZUd2eFd3eVdeFfd4d5ZWdeFdeFfd6d7ZXd8eeFeeFfd)eFd9eFdee)fd:d;ZYGdd?d?e=Z[d@e[de[fdAdBZ\dCdDdee)fdEdFZ]e\dCdDdGeVdee)fdHdIZ^GdJdDdDZ_GdKdLdLeZ` d]dMe)dNeeFeIfdOeedPddfdQdRZad^de"dTeVde_fdUdVZb d]dMe)d ee-dedDfdWdXZcGdYdZdZeZdGd[d\d\ZedS)_zO The main purpose of this module is to expose LinkCollector.collect_sources(). N) HTMLParser)Values) TYPE_CHECKINGAnyCallableDictIterableListMutableMapping NamedTupleOptionalSequenceTupleUnion)html5librequests)Response) RetryErrorSSLError)NetworkConnectionError)Link) SearchScope) PipSession)raise_for_status) deprecated)is_archive_file)pairwiseredact_auth_from_url)vcs)CandidatesFromPage LinkSource build_source)ProtocolurlreturncCs6tjD]}||r|t|dvr|SqdS)zgLook for VCS schemes in the URL. Returns the matched VCS scheme, or None if there's no match. z+:N)rschemeslower startswithlen)r$schemer+l/var/www/html/chefvision.cloud.itp360.com/venv/lib/python3.10/site-packages/pip/_internal/index/collector.py_match_vcs_scheme;s r-cs*eZdZdededdffdd ZZS)_NotHTML content_type request_descr%Ncst||||_||_dSN)super__init__r/r0)selfr/r0 __class__r+r,r3Gs z_NotHTML.__init__)__name__ __module__ __qualname__strr3 __classcell__r+r+r5r,r.Fs"r.responsecCs.|jdd}|dst||jjdS)zCheck the Content-Type header to ensure the response contains HTML. Raises `_NotHTML` if the content type is not text/html. Content-Type text/htmlN)headersgetr'r(r.requestmethod)r<r/r+r+r,_ensure_html_headerMsrDc@s eZdZdS)_NotHTTPN)r7r8r9r+r+r+r,rEWsrEsessioncCsFtj|\}}}}}|dvrt|j|dd}t|t|dS)zSend a HEAD request to the URL, and ensure the response contains HTML. Raises `_NotHTTP` if the URL is not available for a HEAD request, or `_NotHTML` if the content type is not text/html. >httphttpsT)allow_redirectsN)urllibparseurlsplitrEheadrrD)r$rFr*netlocpathqueryfragmentrespr+r+r,_ensure_html_response[s  rScCsRtt|jr t||dtdt||j|dddd}t|t ||S)aAccess an HTML page with GET, and return the response. This consists of three parts: 1. If the URL looks suspiciously like an archive, send a HEAD first to check the Content-Type is HTML, to avoid downloading a large file. Raise `_NotHTTP` if the content type cannot be determined, or `_NotHTML` if it is not HTML. 2. Actually perform the request. Raise HTTP exceptions on network failures. 3. Check the Content-Type header to make sure we got HTML, and raise `_NotHTML` otherwise. rFzGetting page %sr?z max-age=0)Acceptz Cache-Control)r@) rrfilenamerSloggerdebugrrArrD)r$rFrRr+r+r,_get_html_responseks rYr@cCs2|rd|vrt|d\}}d|vr|dSdS)z=Determine if we have any encoding information in our headers.r=charsetN)cgi parse_header)r@r/paramsr+r+r,_get_encoding_from_headerss r^documentpage_urlcCs.|dD]}|d}|dur|Sq|S)a$Determine the HTML document's base URL. This looks for a ```` tag in the HTML document. If present, its href attribute denotes the base URL of anchor tags in the document. If there is no such tag (or if it does not have a valid href attribute), the HTML file's URL is used as the base URL. :param document: An HTML document representation. The current implementation expects the result of ``html5lib.parse()``. :param page_url: The URL of the HTML document. TODO: Remove when `html5lib` is dropped. z.//basehrefN)findallrA)r_r`baserar+r+r,_determine_base_urls  rdpartcCtjtj|S)zP Clean a "part" of a URL path (i.e. after splitting on "@" characters). )rJrKquoteunquoterer+r+r,_clean_url_path_partsrjcCrf)z Clean the first part of a URL path that corresponds to a local filesystem path (i.e. the first part after splitting on "@" characters). )rJrB pathname2url url2pathnamerir+r+r,_clean_file_url_paths rmz(@|%2F)rO is_local_pathcCs^|rt}nt}t|}g}tt|dgD]\}}|||||qd |S)z* Clean the path portion of a URL. r>) rmrj_reserved_chars_resplitr itertoolschainappendupperjoin)rOrn clean_funcparts cleaned_partsto_cleanreservedr+r+r,_clean_url_paths  r{cCs6tj|}|j }t|j|d}tj|j|dS)z Make sure a link is fully quoted. For example, if ' ' occurs in the URL, it will be replaced with "%20", and without double-quoting other characters. )rn)rO)rJrKurlparserNr{rO urlunparse_replace)r$resultrnrOr+r+r, _clean_links relement_attribsbase_urlcCsL|d}|s dSttj||}|d}|d}t||||d}|S)zW Convert an anchor element's attributes in a simple repository page to a Link. raNzdata-requires-pythonz data-yanked) comes_fromrequires_python yanked_reason)rArrJrKurljoinr)rr`rrar$ pyrequirerlinkr+r+r,_create_link_from_elements   rc@s6eZdZd ddZdedefdd Zdefd d ZdS) CacheablePageContentpageHTMLPager%NcCs|jsJ||_dSr1)cache_link_parsingr)r4rr+r+r,r3s  zCacheablePageContent.__init__othercCst|t|o|jj|jjkSr1) isinstancetyperr$)r4rr+r+r,__eq__szCacheablePageContent.__eq__cCs t|jjSr1)hashrr$r4r+r+r,__hash__s zCacheablePageContent.__hash__)rrr%N) r7r8r9r3objectboolrintrr+r+r+r,rs rc@s&eZdZdddedeefddZdS) ParseLinksrruse_deprecated_html5libr%cCsdSr1r+)r4rrr+r+r,__call__ szParseLinks.__call__N)r7r8r9rrrrr+r+r+r,rsrfncsXtjdddtdtdttffdd tdd dtdttffd d }|S) z Given a function that parses an Iterable[Link] from an HTMLPage, cache the function's result (keyed by CacheablePageContent), unless the HTMLPage `page` has `page.cache_link_parsing == False`. N)maxsizecacheable_pagerr%cst|j|Sr1)listr)rr)rr+r,wrapper-sz'with_cached_html_pages..wrapperrrcs"|jr t||St||Sr1)rrr)rrrrr+r,wrapper_wrapper3sz/with_cached_html_pages..wrapper_wrapper) functools lru_cacherrr rwraps)rrr+rr,with_cached_html_pages&s "rrrccsZtj|j|jdd}|j}t||}|dD]}t|j||d}|dur'q|VqdS)z~ Parse an HTML document, and yield its anchor elements as Link objects. TODO: Remove when `html5lib` is dropped. F)transport_encodingnamespaceHTMLElementsz.//ar`rN) rrKcontentencodingr$rdrbrattrib)rr_r$ranchorrr+r+r,_parse_links_html5lib<s$ rrc cs|jpd}|s.d|}|jdt|}||dkr.td|jdddddd }|r9t|EdHdSt }| |j||j}|j pL|}|j D]}t |||d } | dur^qP| VqPdS) zP Parse an HTML document, and yield its anchor elements as Link objects. zutf-8zNz The HTML index page being used (z) is not a proper HTML 5 document. This is in violation of PEP 503 which requires these pages to be well-formed HTML 5 documents. Please reach out to the owners of this index page, and ask them to update this index page to a valid HTML 5 document.z22.2iI*)reason replacementgone_inissueTr)rencoderr)decoder'rr$rHTMLLinkParserfeedranchorsr) rrrexpected_doctype actual_startparserr$rrrr+r+r, parse_linksUs<      rc @sDeZdZdZ d dedeedededdf d d Zdefd d Z dS)rz'Represents one page, along with its URLTrrr$rr%NcCs||_||_||_||_dS)am :param encoding: the encoding to decode the given content. :param url: the URL from which the HTML was downloaded. :param cache_link_parsing: whether links parsed from this page's url should be cached. PyPI index urls should have this set to False, for example. N)rrr$r)r4rrr$rr+r+r,r3s zHTMLPage.__init__cCs t|jSr1)rr$rr+r+r,__str__s zHTMLPage.__str__T) r7r8r9__doc__bytesr r:rr3rr+r+r+r,rs cseZdZdZdededdffdd Zdeddfd d Zd ed ee ee efddfd dZ d ee ee efde efddZ dddZ ZS)rzf HTMLParser that keeps the first base HREF and a list of all anchor elements' attributes. argskwargsr%Ncs(tj|i|d|_d|_g|_dS)NF)r2r3 _seen_declrr)r4rrr5r+r,r3s zHTMLLinkParser.__init__declcCs|dkr |d|_dS)Nz doctype htmlT)r' _raise_errorr)r4rr+r+r, handle_decls  zHTMLLinkParser.handle_decltagattrscCs`|js||dkr |jdur ||}|dur||_dSdS|dkr.|jt|dSdS)Nrca)rrrget_hrefrrsdict)r4rrrar+r+r,handle_starttags  zHTMLLinkParser.handle_starttagcCs"|D] \}}|dkr|SqdS)Nrar+)r4rnamevaluer+r+r,rs zHTMLLinkParser.get_hrefcCstd)NzHTML doctype missing or incorrect. Expected . If you believe this error to be incorrect, try passing the command line option --use-deprecated=html5lib and please leave a comment on the pip issue at https://github.com/pypa/pip/issues/10825.) ValueErrorrr+r+r,rszHTMLLinkParser._raise_error)r%N)r7r8r9rrr3r:rr rr rrrr;r+r+r5r,rs&& rrrmeth).NcCs|durtj}|d||dS)Nz%Could not fetch URL %s: %s - skipping)rWrX)rrrr+r+r,_handle_get_page_failsrTrcCst|j}t|j||j|dS)N)rr$r)r^r@rrr$)r<rrr+r+r,_make_html_pages rc Cs |durtd|jddd}t|}|r td||dStj|\}}}}}}|dkrPt j tj |rP|dsC|d7}tj|d}td |zt||d }Wntyhtd |YdSty}ztd ||j|jWYd}~dSd}~wty}z t||WYd}~dSd}~wty}z t||WYd}~dSd}~wty}zd } | t|7} t|| tjdWYd}~dSd}~wtjy}zt|d|WYd}~dSd}~wtjyt|dYdSwt||j dS)Nz?_get_html_page() missing 1 required keyword argument: 'session'#rrzICannot look at %s URL %s because it does not support lookup as web pages.file/z index.htmlz# file: URL is directory, getting %srTz`Skipping page %s because it looks like an archive, and cannot be checked by a HTTP HEAD request.ziSkipping page %s because the %s request got Content-Type: %s.The only supported Content-Type is text/htmlz4There was a problem confirming the ssl certificate: )rzconnection error: z timed out)r)! TypeErrorr$rpr-rWwarningrJrKr|osrOisdirrBrlendswithrrXrYrEr.r0r/rrrrr:inforConnectionErrorTimeoutrr) rrFr$ vcs_schemer*_rOrRexcrr+r+r,_get_html_pagesv       rc@s.eZdZUeeeed<eeeed<dS)CollectedSources find_links index_urlsN)r7r8r9r r r!__annotations__r+r+r+r,rs rc @seZdZdZdededdfddZe dded ed e ddfd d Z e de e fd dZdedeefddZde dedefddZdS) LinkCollectorz Responsible for collecting Link objects from all configured locations, making network requests as needed. The class's main method is its collect_sources() method. rF search_scoper%NcCs||_||_dSr1)rrF)r4rFrr+r+r,r3+s zLinkCollector.__init__Foptionssuppress_no_indexcCs`|jg|j}|jr|stdddd|Dg}|jp g}tj||d}t ||d}|S)z :param session: The Session to use to make requests. :param suppress_no_index: Whether to ignore the --no-index option when constructing the SearchScope object. zIgnoring indexes: %s,css|]}t|VqdSr1)r).0r$r+r+r, Csz'LinkCollector.create..rr)rFr) index_urlextra_index_urlsno_indexrWrXrurrcreater)clsrFrrrrrlink_collectorr+r+r,r3s"  zLinkCollector.createcCs|jjSr1)rrrr+r+r,rTszLinkCollector.find_linkslocationcCst||jdS)z> Fetch an HTML page containing package links. rT)rrF)r4rr+r+r, fetch_pageXszLinkCollector.fetch_page project_namecandidates_from_pagecstfddj|D}tfddjD}ttj rIddt ||D}t |d|dg|}t d|tt|t|d S) Nc3&|]}t|jjdddVqdS)Frpage_validator expand_dirrNr"rFis_secure_originrlocrr4r+r,rd z0LinkCollector.collect_sources..c3r)TrNrrrr+r,rnrcSs*g|]}|dur|jdurd|jqS)Nz* )r)rsr+r+r, zs  z1LinkCollector.collect_sources..z' location(s) to search for versions of : r) collections OrderedDictrget_index_urls_locationsvaluesrrW isEnabledForloggingDEBUGrqrrr)rXrurr)r4rrindex_url_sourcesfind_links_sourceslinesr+rr,collect_sources^s2       zLinkCollector.collect_sources)F)r7r8r9rrrr3 classmethodrrrpropertyr r:rrr rrr rrr+r+r+r,r"s<   rr1r)frr[rrrqr rre urllib.parserJurllib.requestxml.etree.ElementTreexml html.parserroptparsertypingrrrrrr r r r r rr pip._vendorrrpip._vendor.requestsrpip._vendor.requests.exceptionsrrpip._internal.exceptionsrpip._internal.models.linkr!pip._internal.models.search_scoperpip._internal.network.sessionrpip._internal.network.utilsrpip._internal.utils.deprecationrpip._internal.utils.filetypesrpip._internal.utils.miscrrpip._internal.vcsrsourcesr r!r"r#r getLoggerr7rWetree ElementTreeElement HTMLElementr:ResponseHeadersr- Exceptionr.rDrErSrYr^rdrjrmcompile IGNORECASErorr{rrrrrrrrrrrrrrr+r+r+r,s  8              2   /.      ?