|`, TdZddlmZmZmZmZddlZddlZddlZddl Z ddl m Z ddl m Z ddlmZddlmZddlmZe j(d kZereecZZerdd lmZdd lmZdd lmZdd lmZddl m!Z!ndd l"mZdd l#mZdd l$mZddl%m!Z!erddl&m'Z'e'dZ(nddl&m(Z(ddl)m*Z*e*jVdZ,ddgZ-d>dZ.GddeZ/d?dZ0edduZ1e1r@e,jedddl3m4Z5ddl6m7Z7e5jpZ8e5jrjtZ:nddl;m5Z5ddlGd$d%e>Z?Gd&d'e>Z@Gd(d)ZAd*ZBd+ZCd,ZDd@d-ZEGd.d/ZFeBeCeDd0ZGeFeGZH edeejd1d2d3d4ZJeFeJZK eLeMeNeOejejejejejejd5 ZVeFeVZW d6ZXdAd7ZYd8ZZd9Z[dBd:Z\d;Z]dAd<Z^e_d=k(re^yy)CaPiculet is a module for scraping XML and HTML documents using XPath queries. It consists of this single source file with no dependencies other than the standard library, which makes it very easy to integrate into applications. It has been tested with Python 2.7, Python 3.4+, PyPy2 5.7+, and PyPy3 5.7+. For more information, please refer to the documentation: https://piculet.readthedocs.io/ )absolute_importdivisionprint_functionunicode_literalsN)ArgumentParser)deque)partial) itemgetter) find_loader)r)escape) HTMLParser)StringIO)name2codepoint)urlopen)contextmanagerc#Ktj|c}t_ ||t_y#|t_wxYww)z3Context manager for temporarily redirecting stdout.N)sysstdout) new_stdout old_stdouts :/usr/lib/python3/dist-packages/imdb/parser/http/piculet.pyredirect_stdoutr>s3"%Z CJ $ #CJCJsA0 A =A)r)loggerpiculets< str :param content: Content of HTML document to decode. :param charset: Character set of the page. :param fallback_charset: Character set to use if it can't be figured out. :return: Decoded content of the document. r"asciizcharset found in "meta": "%s"z'charset not found, using fallback: "%s"zdecoding for charset: "%s") _CHARSET_TAGSfindlendecode_loggerdebug)contentcharsetfallback_charsettagstart charset_start charset_ends r decode_htmlr,Zs 'CLL%Ez %C 0 %ll4? !- <CCGL =wG ' MMCEU V&G MM.8 >>' ""cLeZdZdZhdZ d fd ZdZdZdZdZ dZ xZ S) HTMLNormalizerzZHTML cleaner and XHTML convertor. DOCTYPE declarations and comments are removed. >brhrimglinkmetainputc trtj|nt| d| t |n t |_| t |n t |_t|_t|_ y)zInitialize this normalizer. :sig: (Optional[Iterable[str]], Optional[Iterable[str]]) -> None :param omit_tags: Tags to remove, along with all their content. :param omit_attrs: Attributes to remove. T)convert_charrefsN) PY2r__init__superset omit_tags omit_attrsr _open_tags_open_omitted_tags)selfr<r= __class__s rr9zHTMLNormalizer.__init__sc     % G d  3+4+@Yce-7-C#j/ '"''r-c||jvr1tjd||jj ||js6d|vrt d|zdy|dk(r8|j ddk(r&tjd |jdg}|D]j\}}||jvrtjd ||,|tjd ||d}d |t|d dz}|j |ld|t|dkDrddj|znd||jvrdnddz}t |d||jvr|j j |yyy)z&Process the starting of a new element.zomitting: "%s"@z <%s>endNliz=opened "li" without closing previous "li", adding closing tagzomitting "%s" attribute of "%s"z7no value for "%s" attribute of "%s", adding empty valuez%(name)s="%(value)s"T)quotenamevaluez<%(tag)s%(attrs)s%(slash)s>r z /)r(attrsslash) r<r#r$r?appendprintr> handle_endtagr= html_escaper!joinSELF_CLOSING_TAGS)r@r(rN attributes attr_name attr_valuemarkuplines rhandle_starttagzHTMLNormalizer.handle_starttags $.. MM*C 0  # # * *3 /&&czlS(b1t $//""5"= ]^""4(J). *% :/MM"CYPST%MM"["+S2!#J/%(4@3!!&) *19(>!>B4D $B $000&&s+1;'r-c|js)||jvr|jd}|dk(r+|dk(r&tj d|j d||k(r-t dd|izd|jjy ||jvrtj d |y ||jd k(rptj d ||t dd|izdt dd|izd|jj|jjy y y ||jvr.||jdk(r|jjy y y ) z!Process the ending of an element.rHulrGz:closing "ul" without closing last "li", adding closing tagz r(rDrEz$closing tag "%s" without opening tagz9unexpected closing tag "%s" instead of "%s", closing bothN) r?rUr>r#r$rRrQpopr<)r@r(lasts rrRzHTMLNormalizer.handle_endtagsD&&$000r*4KddlMM"^_&&t,$;,%52>OO'')/MM"H#NDOOB//MM"]"%t-,%6B?,%52>OO'')OO'') 01&T^^##1H1H1L*L  # # ' ' )+M#r-c|js@t|}ttr!t |t r|j dn|dyy)z!Process collected character data.utf-8rDrEN)r?rSrQr8 isinstancebytesr")r@datarZs r handle_datazHTMLNormalizer.handle_datas;&&t$D #*T52I$++g&tY[ \'r-cXtj|}|tdd|izdyy)zProcess an entity reference.Nz &#%(ref)d;refrDrE)rgetrQ)r@rKnums rhandle_entityrefzHTMLNormalizer.handle_entityrefs1  & ? ,%-2 6 r-c(tdd|izdy)zProcess a character reference.z &#%(ref)s;rhrDrEN)rQ)r@rKs rhandle_charrefzHTMLNormalizer.handle_charrefs leT]*3r-NN) __name__ __module__ __qualname____doc__rUr9r[rRrfrkrm __classcell__rAs@rr/r/vs4 E)*&#,J*6]74r-r/ct}t||}t|5|j|ddd|j S#1swY|j SxYw)aDClean HTML and convert to XHTML. :sig: (str, Optional[Iterable[str]], Optional[Iterable[str]]) -> str :param document: HTML document to clean and convert. :param omit_tags: Tags to exclude from the output. :param omit_attrs: Attributes to exclude from the output. :return: Normalized XHTML content. )r<r=N)rr/rfeedgetvalue)documentr<r=out normalizers r html_to_xhtmlr{sS *C) KJ  "!" <<>" <<>s A  A%lxmlz using lxml)etree)Element) ElementTreeceZdZdZdZdZy)XPathzAn XPath expression evaluator. This class is mainly needed to compensate for the lack of ``text()`` and ``@attr`` axis queries in ElementTree XPath support. cfd}fd}d}ddk(rdzjdr |}||_y jdr |}||_y jd}|d d |d }}|jd r(t|dj ||d d  }||_y tt j }||_y )zInitialize this evaluator. :sig: (str) -> None :param path: XPath expression to evaluate. c|jddDcgc]}|jD]}|s| c}}Scc}}w)Ni)findallitertext)elementetpaths r descendantz"XPath.__init__..descendants;#*??49#=Yaqzz|Y!WXYYYYs>>c |jddDcgc]A}|jg|Dcgc]}|jr |jndc}zD]}|r|Cc}}}Scc}wcc}}}w)NirD)rtexttail)rrrcrs rchildzXPath.__init__..child!s{#*??49#=ZZa#$66(PQ-R1affB.F-R"RZWXZZZ-RZsA*!A%A*%A*c|j|Dcgc]}|jj|}}|Dcgc]}|| c}Scc}wcc}wN)rattribri)rsubpathattrrresultrs r attributez!XPath.__init__..attribute&sI6=oog6NO!((,,t,OO#);aQ];;P;s"A AAr/.z//text()z/text()NrHrC)rr)r)endswithsplit startswithr rTr~r_apply) r@rrrrrstepsfrontr`s ` rr9zXPath.__init__s  Z Z  <Aw#~Tz}}Z(#!DKy)!DK 3#CRj%)t??3'$YdSTSUhWF!DK%W__4@F DKr-c$|j|S)zApply this evaluator to an element. :sig: (Element) -> XPathResult :param element: Element to apply this expression to. :return: Elements or strings resulting from the query. )rr@rs r__call__zXPath.__call__>s;;w' 'r-N)rorprqrrr9rr-rrrs % !N (r-rc$t||Sr)r)rps rrGsq!r-c6eZdZdZddZdZddZedZy) Extractorz?Abstract base extractor for getting data out of an XML element.NcH||_ |t||_yd|_y)zInitialize this extractor. :sig: (Optional[Transformer], Optional[str]) -> None :param transform: Function to transform the extracted value. :param foreach: Path to apply for generating a collection of values. N) transformrforeach)r@rrs rr9zExtractor.__init__Ws/#8)0) ExtractedItem :param element: Element to apply this extractor to. :return: Extracted raw data. z.Concrete extractors must implement this method)NotImplementedErrorrs rapplyzExtractor.applyds""RSSr-c~|j|}| |tus|s|S|j|S|j|S)a8Get the processed data from an element using this extractor. :sig: (Element, Optional[bool]) -> Any :param element: Element to extract the data from. :param transform: Whether the transformation will be applied or not. :return: Extracted and optionally transformed data. )r_EMPTYr)r@rrrLs rextractzExtractor.extractmsA 7# Mu L.uIDNN54IIr-c|jd}|d}n"tj|}| td|jd}|jd}|I|jd}|d}n"tj|}| tdt ||||}|S|jd }|D cgc]} t j | } } t| |jd || }|Scc} w) zGenerate an extractor from a description map. :sig: (Mapping[str, Any]) -> Extractor :param item: Extractor description. :return: Extractor object. :raise ValueError: When reducer or transformer names are unknown. rNzUnknown transformerrrreducezUnknown reducerrritemssection)rrr)ri transformers ValueErrorreducersPathRulefrom_mapRules) item transformerrrrreducerr extractorriruless rrzExtractor.from_mapzshh{+  I$((5I  !677((9%xx  hhx(G!g.>$%677T6YPI HHW%E/45!T]]1%5E5eTXXi-@(17DI 6s;C9rn)T) rorprqrrr9rr staticmethodrrr-rrrTs+I CT J##r-rc*eZdZdZdfd ZdZxZS)rz4An extractor for getting text out of an XML element.ctrtj|||nt| ||t ||_ |t j}||_y)aInitialize this extractor. :sig: ( str, Optional[Reducer], Optional[PathTransformer], Optional[str] ) -> None :param path: Path to apply to get the data. :param reduce: Function to reduce selected texts into a single string. :param transform: Function to transform extracted value. :param foreach: Path to apply for generating a collection of data. rN) r8rr9r:rrrconcatr)r@rrrrrAs rr9z Path.__init__sU    ty'  J G y'  B$K 7 >__F Er-cn|j|}t|dk(rd}|S|j|}|S)zApply this extractor to an element. :sig: (Element) -> str :param element: Element to apply this extractor to. :return: Extracted text. rN)rr!r)r@rselectedrLs rrz Path.applys?99W% x=A E  KK)E r-NNNrorprqrrr9rrsrts@rrrs>F8r-rc*eZdZdZdfd ZdZxZS)rz:An extractor for getting data items out of an XML element.ctrtj|||nt| ||||_ |t ||_yd|_y)aInitialize this extractor. :sig: ( Sequence[Rule], str, Optional[MapTransformer], Optional[str] ) -> None :param rules: Rules for generating the data items. :param section: Path for setting the root of this section. :param transform: Function to transform extracted value. :param foreach: Path for generating multiple items. rN)r8rr9r:rrr)r@rrrrrAs rr9zRules.__init__sZ    ty'  J G y'  B 2)0) Mapping[str, Any] :param element: Element to apply the extractor to. :return: Extracted mapping. rzNo section root foundrz.Section path should select exactly one elementzMoving root to %s element) rr!r#r$rrr(rrupdate)r@rsubrootsubrootsrerule extracteds rrz Rules.applys << G||G,H8}! 56 8}q  !QRRqkG MM5w{{ CJJ #D W-I KK " #4y1}t0&0r-rrrts@rrrsDI41r-rc.eZdZdZddZedZdZy)rz?A rule describing how to get a data item out of an XML element.NcX||_ ||_ |t||_yd|_y)a"Initialize this rule. :sig: (Union[str, Extractor], Extractor, Optional[str]) -> None :param key: Name to distinguish this data item. :param extractor: Extractor that will generate this data item. :param foreach: Path for generating multiple items. N)keyrrr)r@rrrs rr9z Rule.__init__ s91":)0) Rule :param item: Item description. :return: Rule object. rrLr)rrr)rcstrrrrri)ritem_keyrrLs rrz Rule.from_mapsT;$Xs3h9K9KH9U""4=1udhhy6IJJr-ci}|j|gn|j|}|D]:}t|jtr |jn|jj |}|H|j j,|j j |}||t ur|||<|j j|Dcgc]}|j j |d!}}|D cgc]} | | t us| } } t| dk(r|j j| n(tt|j j| ||<=|Scc}wcc} w)zExtract data out of an element using this rule. :sig: (Element) -> Mapping[str, Any] :param element: Element to extract the data from. :return: Extracted data. F)rr) rrcrrrrrr!rlistmap) r@rrerrrrLr raw_valuesvvaluess rrz Rule.extract+sY $ 4G9$,,w:O @G)37$((TXX=M=Mg=VC{~~%%-..w7Mu!S (,~~'='=g'FH"##nn44Q%4HH H%/YAMQW!YYv;!#&*nn&>&>&FFT^^55v>?S 1 @6 HYs $E%6E*>E*E*r)rorprqrrr9rrrrr-rrr s%I=" K K$r-rc trtjj}n^|jj d}|A|j Dcic] }|D]}||c}}j }||jd<t||}tjdt||t|dkDr?|D]9}tjd|j||j|;yycc}}w)zRemove selected elements from the tree. :sig: (Element, str) -> None :param root: Root element of the tree. :param path: XPath to select the elements to remove. _get_parentNz%removing %s elements using path: "%s"rzremoving element: "%s") _USE_LXMLr_Element getparentrriiterrr#r$r!r(remove)rootr get_parentrrelementsrs rremove_elementsrRs ))33 [[__]3  (, ?1Q?!Q$?!?CCJ)3DKK &uT{4 H MM93x=$O 8}q 0G MM2GKK @ w  & &w / 0 @sC?c:t||}tjdt|||D]}t |t r|n#t j|j|}|!tjd|j\t |t r|n#t j|j|}|!tjd|jtjd|||j||j|<y)aSet an attribute for selected elements. :sig: ( Element, str, Union[str, Mapping[str, Any]], Union[str, Mapping[str, Any]] ) -> None :param root: Root element of the tree. :param path: XPath to select the elements to set attributes for. :param name: Description for name generation. :param value: Description for value generation. %updating %s elements using path: "%s"Nz,no attribute name generated for "%s" elementz-no attribute value generated for "%s" elementz.setting "%s" attribute to "%s" on "%s" element) rr#r$r!rcrrrrr(r)rrrKrLrrrWrXs rset_element_attrrjsuT{4 H MM93x=$O/&tS1D   t $ , ,W 5    MMH'++ V (4U   u % - -g 6    MMI7;; W  FW[[ :$.y!/r-c0t||}tjdt|||D]`}t |t r|n#t j|j|}tjd||j||_ by)a Set the text for selected elements. :sig: (Element, str, Union[str, Mapping[str, Any]]) -> None :param root: Root element of the tree. :param path: XPath to select the elements to set attributes for. :param text: Description for text generation. rz$setting text to "%s" on "%s" elementN) rr#r$r!rcrrrrr(r)rrrrr element_texts rset_element_textrsuT{4 H MM93x=$O$)$4t   t $ , ,W 5    Element :param document: XML document to build the tree from. :param force_html: Force to parse from HTML without converting. :return: Root element of the XML tree. rbzusing lxml html builderrN) r8encoderr#info lxml.htmlhtml fromstringr)rx force_htmlr%r|s r build_treersQ+.hoog&8GZ ./yy##G,,  ! !' **r-c"eZdZdZdZdZdZy)Registryz$A simple, attribute-based namespace.c:|jj|y)zInitialize this registry. :sig: (Mapping[str, Any]) -> None :param entries: Entries to add to this registry. N)__dict__r)r@entriess rr9zRegistry.__init__s W%r-c8|jj|S)zGet the value of an entry from this registry. :sig: (str) -> Any :param item: Entry to get the value for. :return: Value of entry. )rri)r@rs rriz Registry.gets}}  &&r-c"||j|<y)zRegister a new entry in this registry. :sig: (str, Any) -> None :param key: Key to search the entry in this registry. :param value: Value to store for the entry. N)r)r@rrLs rregisterzRegistry.registers# cr-N)rorprqrrr9rirrr-rrrs.&'#r-r)rset_attrset_textrDctjdddj|jddj S)Nz\s+rMrD )resubrTreplacestripxss rrrs1vsBGGBK,?,?,LMSSUr-ctjdddj|jj ddS)Nz [^a-z0-9_]rDrM_)rrrTlowerrr s rrrs3BFF<RWWR[5F5F5H5P5PQTVY5Z[r-)firstrclean normalize) intfloatboolr!rupper capitalizelstriprstripr c|D]^}|d}|dk(rt||d|dk(rt||d|d|d;|dk(rt||d|d  Utd y ) zProcess a tree before starting extraction. :sig: (Element, Sequence[Mapping[str, Any]]) -> None :param root: Root of tree to process. :param pre: Descriptions for processing operations. oprrrrKrLrJrr)rzUnknown preprocessing operationN)rrrr)rpresteprs r preprocessrs} @ $Z > D$v, / :  T4? ? @r-ct|Dcgc]}tj|c}|}|j|Scc}w)aExtract data from an XML element. :sig: ( Element, Sequence[Mapping[str, Any]], Optional[str] ) -> Mapping[str, Any] :param element: Element to extract the data from. :param items: Descriptions for extracting items. :param section: Path to select the root element for these items. :return: Extracted data. r)rrrr)rrrrrs rrrs8 5944==&97 KE == !!:sAct|}|jd}| t||t||jd|jd}|S)zExtract data from a document after optionally preprocessing it. :sig: (str, Mapping[str, Any]) -> Mapping[str, Any] :param document: Document to scrape. :param spec: Extraction specification. :return: Extracted data. rrrr)rrirr)rxspecrrres rscraper"sO h D ((5/C 4 4'*DHHY4G HD Kr-c||dk(r4tjdtjj }natjdt j j|t|d5}t|j }dddttdy#1swY xYw)zyConvert an HTML file into XHTML and print. :sig: (str) -> None :param source: Path of HTML file to convert. -zreading from stdinzreading from file: "%s"rbNrDrE) r#r$rstdinreadosrabspathopenr,rQr{)sourcer%fs rh2xr--s } *+)).." /1HI &$  ,1!!&&(+G , - b) , ,s 9B22B;ctjdtjj |tjj |ddk(r't d tdddl}|j}ntj}t|5}||j}ddd|jdr;tjd |t|5}|j}dddnXtjd tjj |t|d 5}|j}dddt!}|d k(r tjd t#|}t%|} t'tj(| ddy#1swY xYw#1swYtxYw#1swYxYw)aScrape data from a file path or a URL and print. :sig: (str, str, Optional[str]) -> None :param address: File path or URL of document to scrape. :param spec: Path of spec file. :param content_format: Whether the content is XML or HTML. zloading spec from file: "%s"rHz.yamlyamlNzYAML support not availabler)zhttp://zhttps://zloading url: "%s"zloading file: "%s"r%rz!converting html document to xhtmlT)indent sort_keys)r#r$r(rr)splitextr RuntimeErrorr/loadjsonloadsr*r'rrr,r{r"rQdumps) addressr!content_formatr/ spec_loaderr,spec_mapr%rxres rscrape_documentr==s{ MM0"''//$2GH wwb!W, v  &;< <ii jj d)qqvvx()12 )73 W  ffhG    *BGGOOG,DE '4  AffhG 7#H 9: * (H %D $**T!t 45'))     s$F<,G G<G GGct|}|jddd|jddd |jd d }d |_|j d d}|jdd|j d|j dd}|jdd|jddd d|jddd |j d|S)zBuild a parser for command line arguments. :sig: (str) -> ArgumentParser :param prog: Name of program. :return: Parser for arguments. progz --versionversionz%(prog)s 1.0b7)actionrAz--debug store_truezenable debug messages)rBhelpcommand)metavardestTr-zconvert HTML to XHTML)rDfilezfile to convertc,t|jSr)r-rHas rrzmake_parser..ts3qvv;r-)funcr"zscrape a documentrxz&file path or URL of document to scrapez-sz--specz spec file)requiredrDz--htmlzdocument is in HTML formatclt|j|j|jrdSdS)Nrxml)r:)r=rxr!rrJs rrzmake_parser..zs*o AFFQVV6/AF/r-)r add_argumentadd_subparsersrM add_parser set_defaults)r@parsercommands h2x_parser scrape_parsers r make_parserrXds &F  I?OP  ,=TU$$YY$GHH$$U1H$IJF):;!67''7J'KMz0XYtX;Ox C_`% Mr-c||ntj}td}|j|dd}|jr9t j t jtj d |j|y#t$r:}t|tjtjdYd}~yd}~wwxYw)zEntry point of the command line utility. :sig: (Optional[List[str]]) -> None :param argv: Command line arguments. Nrr?r)levelzrunning in debug mode)rH)rargvrX parse_argsr$logging basicConfigDEBUGr#rL ExceptionrQstderrexit)r[rT argumentsrs rmainrds #4D i (F!!$qr(+I'--0 -.y!  acjj!  s;B C0C  C__main__)Nrbrn)Fr)rO)`rr __future__rrrrr6r(rrargparser collectionsr functoolsr operatorr pkgutilr version_infor8unicoderrdcgir rSrrhtmlentitydefsrurllib2rr html.parseriourllib.request contextlibrrimdb.parser.http.loggingrgetChildr#rr,r/r{rrr|r}r lxml.etreer~rrxpath xml.etreexml.etree.ElementTreerrrrrrrrrr_PREPROCESSORS preprocessorsrT _REDUCERSrrrrr!rrrrrr  _TRANSFORMERSrrrr"r-r=rXrdrorr-rrsZ SR # #JC)%!-*&&)$$++ &//) $D #8m4Zm4l 0  t +   LL)"   E  & &E%-5(5(n %E JJZ/9/d41I41nEEP00 /F$$ + ##>   ( ]chh# U[    I      YY YY..jjjj YY   & @&"$ * * $6N:. zFr-