^\kdc!@`sddlmZmZmZddlmZddlZddlZddlm Z m Z m Z m Z ddlm Z mZddlmZddlmZydd lmZWnek reZnXydd lmZWn'ek rd efd YZnXege D]Zejd ^qZege D]Zejd ^q:Zege D]Zejd ^qbZeeddgBZejdZe dddddddddddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0g Z!ejd1Z"iZ#d2d@d3YZ$de&e&d4Z'd5dAd6YZ(d7e(fd8YZ)d9e*fd:YZ+d;efd<YZ,d=efd>YZ-d?Z.dS(Bi(tabsolute_importtdivisiontunicode_literals(t text_typeNi(tEOFtspaceCharacterst asciiLetterstasciiUppercase(t encodingstReparseException(tutils(tStringIO(tBytesIO(tBufferedIOBaseR cB`seZRS((t__name__t __module__(((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyR suasciit>t Normalized stream from source for use by html5lib. source can be either a file-object, local filename or a string. The optional encoding parameter must be a string that indicates the encoding. If specified, that encoding will be used, regardless of any BOM or later declaration (such as in a meta element) parseMeta - Look for a element containing encoding information u􏿿iu [-]u0([-](?![-])|(?RAtsubtreplacetTrue(RRHR(tlastv((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyRWs0        (   cC`s:x3tttj|D]}|jjdqWdS(Nuinvalid-codepoint(trangeRtinvalid_unicode_retfindallRJR'(RR(R((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyR=s"cC`st}xtj|D]}|r(qnt|j}|j}tj|||d!rtj|||d!}|t kr|j j dnt }q|dkr|dkr|t |dkr|j j dqt}|j j dqWdS(Niuinvalid-codepointiii(RZRatfinditerR[tgrouptstartR tisSurrogatePairtsurrogatePairToCodepointtnon_bmp_invalid_codepointsRJR'R^R(RR(tskiptmatcht codepointRtchar_val((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyRBs    c C`s}yt||f}Wntk rx&|D]}t|dks+tq+Wdjg|D]}dt|^qZ}|sd|}ntjd|}t||f Normalized stream from source for use by html5lib. source can be either a file-object, local filename or a string. The optional encoding parameter must be a string that indicates the encoding. If specified, that encoding will be used, regardless of any BOM or later declaration (such as in a meta element) parseMeta - Look for a element containing encoding information ucertainiidu windows-1252iN( REt rawStreamR5Rt codecNameRDt numBytesMetatnumBytesChardettdefaultEncodingR3tdetectEncodingRG(RR7R8R9R:((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyRxs   cC`s6tj|jd|jd|_tj|dS(Niureplace(tcodecst getreaderRDRyRFR5RG(R((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyRGs cC`sUt|dr|}n t|}y|j|jWnt|}nX|S(uvProduces a file object from source. source can be either a file object, local filename or a string. uread(R1R R RR(RR7R((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyREs  c C`s|j}d}|dkr9|r9|j}d}n|dkr8|r8d}yyddlm}Wn!tk rddlm}nXg}|}x[|js|jj |j }t |t st |sPn|j||j|qW|j|jd}|jjdWq8tk r4q8Xn|dkrVd}|j}nidd6} |j| kr| |j}n||fS(Nucertainu tentativei(tUniversalDetectoruencodingu windows-1252u iso-8859-1(t detectBOMR3tdetectEncodingMetatcharade.universaldetectorRt ImportErrortchardet.universaldetectortdoneRyR$R|R2R#RR'tfeedtclosetresultR R}tlower( RR9R:R8t confidenceRtbufferstdetectorRt encodingSub((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyR~sB             cC`s|jddkstt|}|d kr:d}n|dkrJdS||jdkrv|jddf|_nF|jjd|j|df|_td|jd|fdS( Niucertainuutf-16u utf-16-beu utf-16-leuutf-8iuEncoding changed from %s to %s(uutf-16u utf-16-beu utf-16-le(RDRRzR3RyR RGR (Rt newEncoding((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pytchangeEncodings     cC`sidtj6dtj6dtj6dtj6dtj6}|jjd}t|t s_t |j |d }d}|s|j |}d}|s|j |d }d}qn|jj |r|pd |S( uAttempts to detect at BOM at the start of the stream. If an encoding can be determined from the BOM return the name of the encoding otherwise return Noneuutf-8u utf-16-leu utf-16-beu utf-32-leu utf-32-beiiii( RtBOM_UTF8t BOM_UTF16_LEt BOM_UTF16_BEt BOM_UTF32_LEt BOM_UTF32_BERyR$R2R#RtgetR (RtbomDicttstringR8R ((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyRs   cC`sk|jj|j}t|ts*tt|}|jjd|j}|dkrgd}n|S(u9Report the encoding declared by the meta element iuutf-16u utf-16-beu utf-16-leuutf-8(uutf-16u utf-16-beu utf-16-le( RyR$R{R2R#RtEncodingParserR t getEncoding(RRtparserR8((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyRs    N( RRR0R3R^RRGRER~RRR(((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyR6ps(  -  t EncodingBytescB`seZdZdZdZdZdZdZdZdZ dZ e e e Z d Z e e Zed Zd Zd Zd ZRS(uString-like object with an associated position and various extra methods If the position is ever greater than the string length then an exception is raisedcC`s+t|tsttj||jS(N(R2R#Rt__new__R(Rtvalue((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyR&scC`s d|_dS(Ni(RT(RR((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyR*scC`s|S(N((R((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyt__iter__-scC`sS|jd}|_|t|kr/tn|dkrDtn|||d!S(Nii(RTRt StopIterationR4(Rtp((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyt__next__0s    cC`s |jS(N(R(R((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pytnext8scC`sY|j}|t|kr$tn|dkr9tn|d|_}|||d!S(Nii(RTRRR4(RR((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pytprevious<s    cC`s+|jt|krtn||_dS(N(RTRR(RR((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyt setPositionEs cC`s<|jt|krtn|jdkr4|jSdSdS(Ni(RTRRR3(R((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyt getPositionJs  cC`s||j|jd!S(Ni(R(R((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pytgetCurrentByteTscC`sc|j}xJ|t|krU|||d!}||krH||_|S|d7}q W||_dS(uSkip past a list of charactersiN(RRRTR3(RRrRRs((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyRiYs    cC`sc|j}xJ|t|krU|||d!}||krH||_|S|d7}q W||_dS(Ni(RRRTR3(RRrRRs((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyt skipUntiles    cC`sQ|j}|||t|!}|j|}|rM|jt|7_n|S(uLook for a sequence of bytes at the start of a string. If the bytes are found return True and advance the position to the byte after the match. Otherwise return False and leave the position alone(RRt startswith(RR#RR(R+((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyt matchBytesps  cC`sh||jj|}|dkr^|jdkr=d|_n|j|t|d7_tStdS(uLook for the next sequence of bytes matching a given sequence. If a match is found advance the position to the last byte of the matchiiiN(RtfindRTRR^R(RR#t newPosition((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pytjumpTo{s  (RRR0RRRRRRRRtpropertyRRt currentBytetspaceCharactersBytesRiRRR(((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyR"s           RcB`s_eZdZdZdZdZdZdZdZdZ dZ d Z RS( u?Mini parser for detecting character encoding from meta elementscC`st||_d|_dS(u3string - the data to work on for encoding detectionN(RR(R3R8(RR(((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyRscC`sd|jfd|jfd|jfd|jfd|jfd|jff}xv|jD]k}t}xR|D]J\}}|jj|rky|}PWqtk rt }PqXqkqkW|sXPqXqXW|j S(Ns(R(R(R((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyRscC`sK|jjtkrtSt}d}x"trF|j}|dkrGtS|ddkr|ddk}|rC|dk rC||_tSq%|ddkr|d}t|}|dk rC||_tSq%|ddkr%t t |d}|j }|dk rCt|}|dk r@|r4||_tS|}q@qCq%q%WdS(Nis http-equivis content-typetcharsettcontent( R(RRR^RZR3t getAttributeR8RztContentAttrParserRtparse(Rt hasPragmatpendingEncodingtattrttentativeEncodingtcodect contentParser((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyRs:             cC`s |jtS(N(thandlePossibleTagRZ(R((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyRscC`st|j|jtS(N(RR(RR^(R((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyRs cC`s|j}|jtkr9|r5|j|jntS|jt}|dkra|jn+|j}x|dk r|j}qpWtS(NR( R(RtasciiLettersBytesRRR^RtspacesAngleBracketsRR3(RtendTagR(RsR((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyRs      cC`s|jjdS(NR(R(R(R((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyRscC`s|j}|jttdgB}|dksIt|dksIt|d krYdSg}g}xtr |dkr|rPnz|tkr|j}Pn^|d krdj|dfS|t kr|j |j n|dkrdS|j |t |}qhW|dkr7|j dj|dfSt ||j}|d kr|}xtrt |}||krt |dj|dj|fS|t kr|j |j qb|j |qbWn^|dkrdj|dfS|t kr|j |j n|dkr-dS|j |x}trt |}|tkrwdj|dj|fS|t kr|j |j q=|dkrdS|j |q=WdS( u_Return a name,value pair for the next attribute in the stream, if one is found, or Nonet/iRt=tt't"N(RN(RR(RR(R(RiRt frozensetR3RRR^R)tasciiUppercaseBytesR'RRRR(RR(RstattrNamet attrValuet quoteChar((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyRsh $                           ( RRR0RRRRRRRRR(((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyRs    $    RcB`seZdZdZRS(cC`s"t|tst||_dS(N(R2R#RR((RR(((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyR?scC`s:y!|jjd|jjd7_|jj|jjdksHdS|jjd7_|jj|jjdkr|jj}|jjd7_|jj}|jj|r|j||jj!SdSnP|jj}y(|jjt|j||jj!SWntk r|j|SXWntk r5dSXdS(NRiRRR(RR( R(RRRiRR3RRR(Rt quoteMarkt oldPosition((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyRCs.       (RRRR(((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyR>s cC`spt|tr:y|jd}Wq:tk r6dSXn|rhtjd|j}tj |dSdSdS(u{Return the python codec name corresponding to an encoding or None if the string doesn't correspond to a valid encoding.uasciiuN( R2R#tdecodetUnicodeDecodeErrorR3tascii_punctuation_reR\RRR(R8t canonicalName((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pyRzes (((/t __future__RRRtpip._vendor.sixRRR?t constantsRRRRRR RR tioR R RR tobjectRR&tencodeRRRRR@RatsetRhRRmRR3R^R<R5R6R#RRRRz(((sG/var/www/testScraper/venv/build/pip/pip/_vendor/html5lib/inputstream.pytsJ  "   (((Jg'