^\kdc @ s d Z d d l Z d d l Z d d l Z d d l Z d d l Z d d l Z d d l Z d d l m Z d d l
m Z m Z m
Z
m Z m Z m Z m Z m Z m Z m Z m Z m Z m Z d d l m Z d d l m Z d d l m Z d d l m Z m Z m! Z! m" Z" m# Z# m$ Z$ m% Z% m& Z& m' Z' m( Z( m) Z) m* Z* m+ Z+ m, Z, m- Z- d d l m. Z. d d
l/ m0 Z0 d d l1 m2 Z2 d d l3 m4 Z4 e j5 d
Z6 e j5 d e j7 Z8 e j5 d Z9 e j5 d e j7 j: Z; d j< Z= d d d d g Z> d Z? d Z@ d ZA eB d ZC eB d ZD eB d ZE eB e eB d ZF eB d ZG d ZH e j5 d e j7 ZI eH d ZJ d! e jK d" e d# d$ jK f ZL d% eM f d& YZN d' eN f d( YZO d e f d) YZP e j5 d* jQ ZR d+ ZS d, ZT d- ZU d d. ZV d/ ZW d0 eM f d1 YZX d2 e- j- f d3 YZY e jZ d4 Z[ eV e? e[ Z[ d5 Z\ d6 Z] d S(7 s# PyPI and direct package downloadingiN( t wraps(
t
CHECKOUT_DISTt Distributiont BINARY_DISTt normalize_patht SOURCE_DISTt requiret Environmentt find_distributionst safe_namet safe_versiont to_filenamet Requirementt DEVELOP_DIST( t ssl_support( t log( t DistutilsError( t urllib2t httplibt StringIOt HTTPErrort urlparset
urlunparset unquotet splitusert url2pathnamet name2codepointt unichrt urljoint urlsplitt
urlunsplitt ConfigParser( t filterfalse( t translate( t strip_fragment( t get_all_headerss ^egg=([-A-Za-z0-9_.]+)$s href\s*=\s*['"]?([^'"> ]+)s ([^<]+)
\s+\(md5\)s ([-+.a-z0-9]{2,}):s .tar.gz .tar.bz2 .tar .zip .tgzt PackageIndext distros_for_urlt parse_bdist_wininstt interpret_distro_namei c C s | j } d \ } } } | j d r | j d rL | d } d } q | j d d r~ | d d !} | d } d } q | j d r | d
} d } q | j d d
r | d d !} | d
} d } q n | | | f S( s= Return (base,pyversion) or (None,None) for possible .exe names .exes
.win32.exeit win32s .win32-pyiiis .win-amd64.exeis win-amd64s
.win-amd64-pyiN( NNN( t lowert Nonet endswitht
startswith( t nameR) t baset py_vert plat( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR& . s$
c C s t | \ } } } } } } t | j d d } | d k rk | d k rk t | j d d } n d | k r | j d d \ } } n | | f S( Nt /is sourceforge.nett downloadit #i ( R R t split( t urlt schemet servert patht
parameterst queryt fragmentR. ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyt egg_info_for_urlF s c c s t | \ } } x t | | | D] } | Vq% W| r t j | } | r x1 t | | j d | d t D] } | Vqq Wq n d S( sE Yield egg or source distribution objects that might be found at a URLi t
precedenceN( R< t distros_for_locationt EGG_FRAGMENTt matchR' t groupR ( R5 t metadataR. R; t distR@ ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR% N s "c C s | j d r | d } n | j d rM d | k rM t j | | | g S| j d r t | \ } } } | d k r t | | | | t | Sn x> t D]6 } | j | r | t | } t | | | Sq Wg S( s: Yield egg or source distribution objects based on basenames .egg.zipis .eggt -s .exeN( R+ R t
from_locationR& R* R' R t
EXTENSIONSt len( t locationt basenameRB t win_baseR/ t platformt ext( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR> Z s
c C s" t t | t j j | | S( sE Yield possible egg or source distribution objects based on a filename( R> R t osR8 RI ( t filenameRB ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyt distros_for_filenameo s c c s | j d } | s^ xF t | d D]1 \ } } t | d k r& | j d r& d Sq& Wn xa t d t | d D]F } t | | d j | | d j | | d | d | d | Vqx Wd S(
s Generate alternative interpretations of a source distro name
Note: if `location` is a filesystem filename, you should call
``pkg_resources.normalize_path()`` on it before passing it to this
routine!
RD i i s py2.Ni t
py_versionR= RK ( R4 t enumerateRG R, t rangeR t join( RH RI RB RP R= RK t partst it p( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR' v s ! ) c c s t } | j } | d k rM xg t | j | D] } | | | Vq1 Wn; x8 | D]0 } | | } | | k rT | | | VqT qT Wd S( sH List unique elements, preserving order. Remember all elements ever seen.N( t sett addR* R t __contains__( t iterablet keyt seent seen_addt elementt k( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyt unique_everseen s
c s t f d } | S( ss
Wrap a function returning an iterable such that the resulting iterable
only ever yields unique items.
c s t | | S( N( R` ( t argst kwargs( t func( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyt wrapper s ( R ( Rc Rd ( ( Rc sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyt
unique_values s s( <([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>c c s x t j | D] } | j \ } } t t t j | j j d } d | k sg d | k r x7 t j | D]# } t
| t | j d Vqw Wq q Wxd d D]\ } | j
| } | d k r t j | | } | rt
| t | j d Vqq q Wd S(
sE Find rel="homepage" and rel="download" links in `page`, yielding URLst ,t homepageR2 i s
| Home Pages | Download URLiN( s
| Home Pages | Download URL( t RELt finditert groupsRW t mapt strt stripR) R4 t HREFR t
htmldecodeRA t findt search( R5 t pageR@ t tagt relt relst pos( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyt find_external_links s '(
s Python-urllib/%s setuptools/%si t
setuptoolsi t ContentCheckerc B s) e Z d Z d Z d Z d Z RS( sP
A null content checker that defines the interface for checking content
c C s d S( s3
Feed a block of data to the hash.
N( ( t selft block( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyt feed s c C s t S( sC
Check the hash. Return False if validation fails.
( t True( Rz ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyt is_valid s c C s d S( su
Call reporter with information about the checker (hash name)
substituted into the template.
N( ( Rz t reportert template( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyt report s ( t __name__t
__module__t __doc__R| R~ R ( ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyRy s t HashCheckerc B sJ e Z e j d Z d Z e d Z d Z d Z d Z
RS( sK (?Psha1|sha224|sha384|sha256|sha512|md5)=(?P[a-f0-9]+)c C s( | | _ t j | | _ | | _ d S( N( t hash_namet hashlibt newt hasht expected( Rz R R ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyt __init__ s c C sL t | d } | s t S| j j | } | s< t S| | j S( s5 Construct a (possibly null) ContentChecker from a URLi( R Ry t patternRq t groupdict( t clsR5 R; R@ ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyt from_url s c C s | j j | d S( N( R t update( Rz R{ ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR| s c C s | j j | j k S( N( R t hexdigestR ( Rz ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR~ s c C s | | j } | | S( N( R ( Rz R R t msg( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR s
( R R t ret compileR R t classmethodR R| R~ R ( ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR s c B sv e Z d Z d d% d& e d Z e d Z e d Z e d Z d Z
d Z d Z d
Z
d& d Z d Z d& d
Z d Z d Z d Z d Z d Z e e e d& d Z e e d Z d Z d Z d Z d Z d& d Z d Z d Z d Z d Z d Z! e" e d Z# d Z$ d! Z% d" Z& d# Z' d$ Z( RS(' s; A distribution index that scans web pages for download URLss https://pypi.python.org/simplet *c O s t j | | | | d | j d | _ i | _ i | _ i | _ t j d j t
t | j | _
g | _ | r t j r | s t j r t j | | _ n t j | _ d S( NR1 t |( R R R+ t index_urlt scanned_urlst fetched_urlst
package_pagesR R RS Rk R! R@ t allowst to_scanR t is_availablet find_ca_bundlet
opener_fort openerR t urlopen( Rz R t hostst ca_bundlet
verify_sslRa t kw( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR s ' !c
C sU | | j k r | r d St | j | %si i i ( RA ( t m( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyt s R N(
Rn Ri R Ro RA t
ValueErrorRw R< R+ t need_version_infot scan_urlt PYPI_MD5t sub(
Rz R5 Rr R R@ R R t new_urlR. t frag( ( Rz sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR s s$ &
c C s | j d | d S( NsP Page at %s links to .py file(s) without version info; an index scan is required.( t scan_all( Rz R5 ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR s c G sO | j | j k r; | r+ | j | | n | j d n | j | j d S( Ns6 Scanning index of all packages (this may take a while)( R R R R R ( Rz R Ra ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR s
c C s | j | j | j d | j j | j sN | j | j | j d n | j j | j ss | j | n x3 t | j j | j d D] } | j | q Wd S( NR1 ( ( R R t unsafe_nameR R R[ t project_namet not_found_in_indexR ( Rz t requirementR5 ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyt
find_packages s %c C sk | j | j | x8 | | j D]) } | | k r; | S| j d | | q% Wt t | j | | S( Ns %s does not match %s( t prescanR R[ R t superR$ t obtain( Rz R t installerRC ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR s
c C si | j | j d | | j se | j t j | t d | j j t j j
| f n d S( s-
checker is a ContentChecker
s Validating %%s checksum for %ss7 %s validation failed for %s; possible download problem?N( R R R~ R RM t unlinkR R R- R8 RI ( Rz t checkerRN t tfp( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyt
check_hash s
c C sr xk | D]c } | j d k sJ t | sJ | j d sJ t t | rZ | j | q | j j | q Wd S( s; Add `urls` to the list that will be prescanned for searchess file:N( R R* R R, R R% R t append( Rz t urlsR5 ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyt add_find_links s
c C s2 | j r% t t | j | j n d | _ d S( s7 Scan urls scheduled for prescanning (e.g. --find-links)N( R R Rk R R* ( Rz ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR s c C sN | | j r | j d } } n | j d } } | | | j | j d S( Ns# Couldn't retrieve index page for %rs3 Couldn't find index page for %r (maybe misspelled?)( R[ R R R R ( Rz R t methR ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR s
c C s t | t s t | } | r| | j | j d | | } t | \ } } | j d rx | j | | | } n | St j j
| r | Sy t j | } Wq t k
r t
d | f q Xn t | j | | d d S( s Locate and/or download `spec` to `tmpdir`, returning a local path
`spec` may be a ``Requirement`` object, or a string containing a URL,
an existing local filename, or a project/version requirement spec
(i.e. the string form of a ``Requirement`` object). If it is the URL
of a .py file with an unambiguous ``#egg=name-version`` tag (i.e., one
that escapes ``-`` as ``_`` throughout), a trivial ``setup.py`` is
automatically created alongside the downloaded file.
If `spec` is a ``Requirement`` object or a string containing a
project/version requirement spec, this method returns the location of
a matching distribution (possibly after downloading it to `tmpdir`).
If `spec` is a locally existing file or directory name, it is simply
returned unchanged. If `spec` is a URL, it is downloaded to a subpath
of `tmpdir`, and the local filename is returned. Various errors may be
raised if a problem occurs during downloading.
i s .pys1 Not a URL, existing file, or requirement spec: %rRH N( R R R t
_download_urlRA R< R+ t gen_setupRM R8 R t parseR R R t fetch_distributionR* ( Rz t spect tmpdirR6 t foundR. R; ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR2 s"
c sN j d | i d } d f d } | rc j j | | | } n | d k r | p | | | } n | d k r j d k r j n | | } n | d k r | r j | | | } n | d k r j d rd pd | n/ j d | | j d j | j | Sd S( s| Obtain a distribution suitable for fulfilling `requirement`
`requirement` must be a ``pkg_resources.Requirement`` instance.
If necessary, or if the `force_scan` flag is set, the requirement is
searched for in the (online) package index as well as the locally
installed packages. If a distribution matching `requirement` is found,
the returned distribution's ``location`` is the value you would have
gotten from calling the ``download()`` method with the matching
distribution's URL or filename. If no matching distribution is found,
``None`` is returned.
If the `source` flag is set, only source distributions and source
checkout links will be considered. Unless the `develop_ok` flag is
set, development and system eggs (i.e., those using the ``.egg-info``
format) will be ignored.
s Searching for %sc s | d k r } n x | | j D]q } | j t k rn rn | k r# j d | d |
c C s d S( N( ( Rz R5 RN R t blksizeR ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR s c C s | j d r t | Sy t | | j SWnt t j f k
r t j d } d j g | j
D] } t | ^ qf } | r | j | | qt
d | | f n't j k
r t j d } | St j k
r/t j d } | r| j | | j qt
d | | j f n t j k
rt j d } | rk| j | | j qt
d | | j f nS t j k
rt j d } | r| j | | qt
d | | f n Xd S( Ns file:i t s %s %ss Download error for %s: %ss; %s returned a bad status line. The server might be down, %s( R, t
local_opent open_with_authR R R t
InvalidURLt syst exc_infoRS Ra Rl R R R R t URLErrort reasont
BadStatusLinet linet
HTTPException( Rz R5 t warningt vt argR ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR s@
+c C s< t | \ } } | rL x7 d | k rH | j d d j d d } q Wn d } | j d rn | d } n t j j | | } | d k s | j d r | j | | S| d
k s | j d r | j | | S| j d r | j | | S| d
k rt
t | d S| j | t
| j | | Sd S( Ns ..t .s \t _t __downloaded__s .egg.zipit svns svn+t gits git+s hg+R i ( R< t replaceR+ RM R8 RS R, t
_download_svnt
_download_gitt _download_hgR R R R} t _attempt_download( Rz R6 R5 R R- R; RN ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR s$ %
c C s | j | t d S( N( R R} ( Rz R5 ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR s c C sK | j | | } d | j d d j k rC | j | | | S| Sd S( NR s content-typeR ( R R R) t _download_html( Rz R5 RN R ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR0 s c C s t | } xW | D]O } | j r t j d | r^ | j t j | | j | | SPq q W| j t j | t d | d S( Ns ([^- ]+ - )?Revision \d+:s Unexpected HTML page found at ( R Rm R Rq R RM R R- R ( Rz R5 R RN R R" ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR1
s
c
C sT | j d d d } d } | j j d r#d | k r#t | \ } } } } } } | r#| j d r#d | d k r#| d j d d \ } } t | \ }
} |
r d
|
k r |
j d
d \ } }
d | |
f } n
d |
} | } t | | | | | | f } q q#n | j d
| | t j d | | | f | S( NR3 i i R s svn:t @s //R1 i t :s --username=%s --password=%ss --username=s' Doing subversion checkout from %s to %ss svn checkout%s -q %s %s( R4 R) R, R R R R RM t system( Rz R5 RN t credsR6 t netlocR8 RV t qR t autht hostt usert pw( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR- s" ! &
'c C s t | \ } } } } } | j d d d } | j d d d } d } d | k rt | j d d \ } } n t | | | | d f } | | f S( Nt +i iR3 i R2 R ( R R4 R* t rsplitR ( R5 t
pop_prefixR6 R6 R8 R: R t rev( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyt _vcs_split_rev_from_url- s c C s | j d d d } | j | d t \ } } | j d | | t j d | | f | d k r | j d | t j d | | f n | S( NR3 i i R> s Doing git clone from %s to %ss git clone --quiet %s %ss Checking out %ss" (cd %s && git checkout --quiet %s)( R4 R@ R} R RM R4 R* ( Rz R5 RN R? ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR. ? s c C s | j d d d } | j | d t \ } } | j d | | t j d | | f | d k r | j d | t j d | | f n | S( NR3 i i R> s Doing hg clone from %s to %ss hg clone --quiet %s %ss Updating to %ss (cd %s && hg up -C -r %s >&-)( R4 R@ R} R RM R4 R* ( Rz R5 RN R? ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR/ O s c G s t j | | d S( N( R R ( Rz R Ra ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR _ s c G s t j | | d S( N( R R ( Rz R Ra ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR b s c G s t j | | d S( N( R R ( Rz R Ra ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR e s ( R N() R R R R* R} R t FalseR R R R R R R R R R R R R R R2 R R R R R R R R R R0 R1 R- t staticmethodR@ R. R/ R R R ( ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR$ sL
0 * )B
* $ ( s! &(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?c C s3 t | t s | S| d k r) t | St | S( Ni ( R R R t chr( t c( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyt uchrl s
c C s{ | j d } | j d r4 t | d d } n= | j d rV t | d } n t j | | j d } t | S( Ni s #xi i R3 i ( RA R, R R R RE ( R@ t what( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyt
decode_entityr s c C s
t t | S( s' Decode HTML entities in the given text.( t
entity_subRG ( t text( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyRo | s c s f d } | S( Nc s f d } | S( Nc s? t j } t j z | | SWd t j | Xd S( N( t sockett getdefaulttimeoutt setdefaulttimeout( Ra Rb t old_timeout( Rc t timeout( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyt _socket_timeout s
( ( Rc RO ( RN ( Rc sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyRO s ( ( RN RO ( ( RN sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyt socket_timeout s c C sC t | } | j } t j | } | j } | j d d S( sq
A function compatible with Python 2.3-3.3 that will encode
auth from a URL suitable for an HTTP header.
>>> str(_encode_auth('username%3Apassword'))
'dXNlcm5hbWU6cGFzc3dvcmQ='
Long auth strings should not cause a newline to be inserted.
>>> long_auth = 'username:' + 'password'*10
>>> chr(10) in str(_encode_auth(long_auth))
False
s
R ( R t encodet base64t encodestringR R, ( R8 t auth_st
auth_bytest
encoded_bytest encoded( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyt _encode_auth s
t
Credentialc B s) e Z d Z d Z d Z d Z RS( s:
A username/password pair. Use like a namedtuple.
c C s | | _ | | _ d S( N( t usernamet password( Rz RZ R[ ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR s c c s | j V| j Vd S( N( RZ R[ ( Rz ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyt __iter__ s c C s d t | S( Ns %(username)s:%(password)s( t vars( Rz ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyt __str__ s ( R R R R R\ R^ ( ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyRY s t
PyPIConfigc B s2 e Z d Z e d Z d Z d Z RS( c C su t j d d d g d } t j j | | t j j t j j d d } t j j | rq | j | n d S( s%
Load from ~/.pypirc
RZ R[ t
repositoryR t ~s .pypircN(
t dictt fromkeysR R RM R8 RS t
expanduserR R ( Rz t defaultst rc( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR s
!c C sM g | j D]$ } | j | d j r
| ^ q
} t t | j | S( NR` ( t sectionsR Rm Rb Rk t _get_repo_cred( Rz t sectiont sections_with_repositories( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyt creds_by_repository s $c C sO | j | d j } | t | j | d j | j | d j f S( NR` RZ R[ ( R Rm RY ( Rz Ri t repo( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyRh s c C s7 x0 | j j D] \ } } | j | r | Sq Wd S( s
If the URL indicated appears to be a repository defined in this
config, return the credential for that repository.
N( Rk t itemsR, ( Rz R5 R` t cred( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyt find_credential s ( R R R t propertyRk Rh Ro ( ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR_ s c C s t | \ } } } } } } | j d r? t j d n | d k r` t | \ } } n d
} | s t j | }
|
r t |
} |
j | f } t
j d | q n | r d t | } t
| | | | | | f } t j | }
|
j d | n t j | }
|
j d t | |
} | rt | j \ } } } } } } | | k r| | k rt
| | | | | | f | _ qn | S( s4 Open a urllib2 request, handling HTTP authenticationR3 s nonnumeric port: ''t httpt httpss* Authenticating as %s for %s (from .pypirc)s Basic t
Authorizations
User-Agent( s https httpsN( R R+ R R R R* R_ Ro Rl RZ R R RX R R t Requestt
add_headert
user_agentR5 ( R5 R R6 R6 R8 t paramsR: R R8 R9 Rn R R t requestR t s2t h2t path2t param2t query2t frag2( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyR s2 !'c C s | S( N( ( R5 ( ( sF /var/www/testScraper/venv/build/setuptools/setuptools/package_index.pyt
fix_sf_url s c C su t | \ } } } } } } t | } t j j | rI t j | S| j d r=t j j | r=g } x t j | D] } | d k r t
t j j | | d }
|
j } |
j
Pn. t j j t j j | | r | d 7} n | j d | | f q Wd | d d j | } d \ } }
n d \ } }
} i d
d 6} t | | |
| t | S( s7 Read a local path, with special support for directoriesR1 s
index.htmlt rs <a href=%r>%s</a>s <html><head><title>%ss %s |