%$}ghzdZddlZddlZddlmZddlmZddlmZddlm Z m Z ddl m Z m Z mZmZmZddlmZmZd d lmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9d d l:m;Z;mm?Z?m@Z@mAZAmBZBd d lCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLejMeNZOdedePfdZQhdZRhdZShdZThdZUddhZVeUWeVZXhdZYejZdZ[e'e(fe)e*ffZ\dePde?de ePfdZ]dedePde?de ePfdZ^dede?de ePfdZ_d eePd!eePd"eePde?de e ePf d#Z`d$ePd%eePd!eePd"eePde?de e ePf d&Zae e;'d(ebdePde?debfd)Zcdede?de ePfd*Zddede?de ePfd+Zed,e e ePdePfd-Zfd$ePde?de ePfd.Zg dEdc.datedc:date bt:pubdate dc.created dc:created og:pubdate og:regdate dcterms.date publish-date sailthru.datedc.date.issueddcterms.issuedpublished-datearticle.createddc.date.createddcterms.createdog:publish_dateog:datepublishedparsely-pub-datetwt-published-atarticle.publishedarticle:post_datearticle:publishedog:published_timevr:published_timevideo:release_datedc.date.publicationrnews:datepublishedfield-name-post-dateog:article:publishedarticle:published_datearticle:published_timearticle:publicationdateog:article:published_timeog:question:published_timecxenseparse:recs:publishtimedcsext.articlefirstpublishedanalyticsattributes.articledate"shareaholic:article_published_timedatemetapdateptimecreatedgentimepubdatedoc_date rbpubdate timestamp dateposted datecreated displaydate publishdate date_created publish_date publish_time release_date citation_date datepublished publisheddatedate_publishedpublished_datepublished_timepublication_datecontent_create_datearticle_date_originalmediator_published_timeoriginalpublicationdatecitation_publication_date>utimelastmodlastdatemodified lastmodified last-modified> bt:moddate dc.modifiedog:updated_timearticle:modifieddcterms.modifiedog:modified_timearticle:modified_datearticle:modified_timearticle:post_modifiedog:article:modified_time datemodifiedrr updated_time modified_time revision_datemodificationdate>pubyearrrr dateupdate>date-publishedtime published publishedz\D+$textoptionsct|}t|tkrdStd|dt }t ||j|j|j |j S)z'Prepare text and try to extract a date.N) r:lenrNON_DIGITS_REGEXsubrrformat extensiveminmax)rrs rL examine_textrsk T??D 4yyO##t   D)9/)9$: ; ;D  gng/gk  rNtree expressionc||}|rt|tkrdS|D]G}||ddfD]}t ||}|r|ccSHdS)z3Check HTML elements one by one for date expressionsNtitler)xpathrr6 text_contentgetr)rrrelementselemrattempts rLexamine_date_elementsrs zz*%%H s8}}'>>>t&&(($((7B*?*?@  D"411G    4rNct d\}}tt|j|j|j|j}|dD]}|jrd|jvr d|jvrd|jvr|dd }|dkr%t|d|}nP|tvrHt d t|||d}n|tvrmt d t||js ||d}n||d}nd |jvr|d d }|tvs |t"vrnt d t|||d}|&|tvr|js|t"vr |js|}n|}nd |jvr6|d d }|t$vrt dt|||dp|d}|"|t&vr|js|t(vr |js|}n|dkr|t dt|d|jvrKd|ddddg}t-|d|j|jr|}nd|jvrt|dd dkrFt dt|||d}nd|jvr |dd }|dkrmt dt||jr||d}n||d}nr|dkrlt dt||js||d}n||d}|n||t d|}|S)a Parse header elements to find date cues :param tree: LXML parsed tree object :type tree: LXML tree :param options: Options for extraction :type options: Extractor :return: Returns a valid date expression as a string, or None )NN) outputformatextensive_searchmin_datemax_datez.//metacontentrnamerzog:urlzexamining meta name: %spropertyzexamining meta property: %sNitempropzexamining meta itemprop: %s copyrightyear-01%Y-%m-%dearliestlatestr|zexamining meta pubdate: %sz http-equivrvzexamining meta http-equiv: %srz-opting for reserve date with less granularity)rrrrrriterfindattribrlowerrDATE_ATTRIBUTESLOGGERdebugrM NAME_MODIFIEDoriginalPROPERTY_MODIFIEDITEMPROP_ATTRSITEMPROP_ATTRS_ORIGINALITEMPROP_ATTRS_MODIFIEDjoinr@)rr headerdatereservetryfuncr attributers rLexamine_headerrs %J^ * G i((VV   ++$+--  T[ ,,2244IH$$*488I+>+>HHo-- 6 $HHH$WTXXi%8%899 m++ 6 $HHH';!())<)z$select_candidate..sY   SYY1 % %tgk'+   rNrg?Tzno suitable candidate: %s %s)rr6searchnextiter most_commonrrsortedrzipappendallanyindex) rrrr firstselectbestonespatternscountsyearspattern year_match validationmatchs ` rLselect_candidatercs( #k**-DDDt ;1||Dk!2!233444))"--K LL"K000kw/?+?@@@!DH LL***H~Hf E((^^G,,  ( LLA ' ' ' J : !9q ! !LL!--EE 1Xq ! !fQi&)&;c&A&ALL!--EELL!--EE Z Xj&6&6t&<&<=>> 3U1XuQxHHH LrN htmlstringrcbt||||j|j}t||||S)z)Chained candidate filtering and selectionrrrr)rBrrr)rrrrr candidatess rLsearch_patternr sA'{ J Jw @ @@rN)maxsize referenceczt||j|j|j|j}|t |||S|S)z[Compare candidate to current date reference (includes date validation and older/newer test))rrrrrr<)rrrrs rLcompare_referencersGGNG$5w{GKGi'::: rNc|d}dt|cxkr tkrnnd}|D]}}d|jvrv t |dd}n#t $rY=wxYwtd||j r|dks||kr|}r|j s||kr|}|dtvrd|jvr|d}td||j r/t||j |j |j|j}||cS t!|||}|dkrnY(|jrNt|jd kr6td |jt!||j|}t%||pt'|d|Sd S) zTScan the page for abbr elements and check if their content contains an eligible datez.//abbrrz data-utimerzdata-utime found: %sclassrzabbr published-title found: %sNrzabbr published found: %s)findallrr6rrr ValueErrorrrr CLASS_ATTRSrrrrrrrr;r)rrrrr candidatetrytextrs rLexamine_abbr_elementsrs/ ||I&&H3x==2222222222 ' Q' QDt{** #DHH\2$>$> ? ?II!H 3Y???#*a9y;P;P )II )*i).C.C )I'""k11dk))"hhw//GLL!A7KKK'""/##N#-#K#K ###.#*NNN/%6i'$R$R $q==!E)YQ3ty>>B#6#6LL!;TYGGG 1)TY P PI(G<< @U   A A  4s #A// A<;A<ct|d}dt|cxkr tkrnnd}|D]}d}|dd}t|dkrdd|jvr>|ddkr%|jrd}t d |nd |jvr|jrp|d dd s)|d dd rd}t d |nY|js6|d dkrd}t d|nt d||r/t||j |j |j |j }||cSt|||}|jNt|jdkr6t d|jt||j|}t!||SdS)zTScan the page for time elements and check if their content contains an eligible datez.//timerFrrr|Tz#shortcut for time pubdate found: %srz entry-datez entry-timez$shortcut for time/datetime found: %supdatedz,shortcut for updated time/datetime found: %sztime/datetime found: %sNztime/datetime found in text: %s)rrr6rrrrr startswithrrrrrrrr;)rrrrr shortcut_flag datetime_attrrs rLexamine_time_elementsrsi ||I&&H3x==2222222222 1 M1 MD!M HHZ44M=!!A%%,,++y88(9%)MLL!F VVVV ++'"--88FF88GR00;;LII)-  BM%-$((72C2Cy2P2P(,  J) LL!:MJJJ U+%)  G*&+!2)]G T TII&3ty>>A+=+= > JJJ-iGLL )G<<< 4rNrcd|D\}}}t|dkr|ddkrd|nd|}|d|d|S)zoNormalize string output by adding "0" if necessary, and optionally expand the year from two to four digits.c3DK|]}||dVdS)rN)zfill)rgs rL z"normalize_match..8s1@@qa@ @@@@@@rNrr91920r)groupsr)rdaymonthrs rLnormalize_matchr+5ssA@ELLNN@@@C 4yyA~~"1gnn{D{{{+t++ " "U " "S " ""rNc 6 tdd}t|ttt|}|^t |d}t t|ddd|j|j rtd||}tdtD][}t||d|dt|}t||d|j ||j |j|j }||cS\t|tt |j|j }i}|D]0} t#j| } t'| } || || <1t)|}t+|t,t.|}t|t|j ||j |j|j }||St|t0t2t|}t|t0|j ||j |j|j }||St|t4t6|j|j d }i}|D]0} t9j| } t'| } || || <1t)|}t+|t,t.|}t|t4|j ||j |j|j }||Std t|t:t<t|}|tt |dt |d d} |dks | j|krTtA| |j |j|j }|0tdt:|d|d |St|tBtD|j|j |j }i}|D]_} tGj| } | d} tI| dkrd| } d%| d | dg} || || <`t)|}t+|t,t.|}t|tB|j ||j |j|j }||StM|} |dks | r1| j|kr&tA| |j |j|j }||S|dkrRtdtt |dd} | '|j Stdt|tPtt|}|tt |ddd} t | d|j|j rL| j|krAtdtP|d| '|j SdS)a Opportunistically search the HTML text for common text patterns :param htmlstring: The HTML document in string format, potentially cleaned and stripped to the core (much faster) :type htmlstring: string :param options: Define extraction options :type options: Extractor :return: Returns a valid date expression as a string, or None z(looking for copyright/footer informationrNrrrz'copyright year/footer pattern found: %sz 3 componentsr T)rrrr incompletezswitching to two componentsrz#date found for pattern "%s": %s, %s0rrzusing copyright year as defaultzswitching to one componentrzdate found for pattern "%s": %s))rrr rrrr@rrrTHREE_COMP_PATTERNSr=rrrBr%r&r1rr+rrrr'r(r)r*r+r2r,r-rrCr.r/r3rrrstrftimer0)rrcopyear bestmatchrrresultr  replacementitemrr dateobjectr*s rL search_pager7>sc  LL;<<<G I9Q<    T1a $W[     LLBD I I IG LL   ("  QK QK     &  QK    N K K    MMM '"{ JK22"(..#E** !+D!1 I%%J [(GLLI !  F  I"  F '{ JK22"(..#E** !+D!1 I%%J [(GLLI !  F  LL./// Ic)A,//Yq\1B1BAFF a<<:?g55)GNW[F! 9"aLaL   '{# JK22$T**a u::??KKEHHeAht455 !+D!1 I%%J [(GLLI !  F Z((J!|| |z''A'A% W[     M!|| 6777c'llAq11 ""7>222 LL-... Ic)A,//A66 JW[    77** LL1>9Q<   &&w~66 6 4rNTFr htmlobjectr original_daterurlverboserrdeferred_url_extractorc |rtjtjt|} | dS|dkrt |sdSt |t |t|||} d} |,| d} | | d}t|| } | |s| St| | pt| | } | | S|r| | St| | }||S ttt!| t"\}}n,#t$$r| }t&dYnwxYw|rt*t,z}nt.t,z}t1||| p t1|d| pt3|| } | | S t5|dd }n5#t6$r(t5|d d d }YnwxYwt;|t<| pt?|| ptA|| } | | S|rt&!dd}tE|D]N}|#}tHtK|cxkr tLksn=tO||| }OtQ|| }|ptS|| SdS)a Extract dates from HTML documents using markup analysis and text patterns :param htmlobject: Two possibilities: 1. HTML document (e.g. body of HTTP request or .html-file) in text string form or LXML parsed tree or 2. URL string (gets detected automatically) :type htmlobject: string or lxml tree :param extensive_search: Activate pattern-based opportunistic text search :type extensive_search: boolean :param original_date: Look for original date (e.g. publication date) instead of most recent one (e.g. last modified, updated time) :type original_date: boolean :param outputformat: Provide a valid datetime format for the returned string (see datetime.strftime()) :type outputformat: string :param url: Provide an URL manually for pattern-searching in URL (in some cases much faster) :type url: string :param verbose: Set verbosity level for debugging :type verbose: boolean :param min_date: Set the earliest acceptable date manually (ISO 8601 YMD format) :type min_date: datetime, string :param max_date: Set the latest acceptable date manually (ISO 8601 YMD format) :type max_date: datetime, string :param deferred_url_extractor: Use url extractor as backup only to prioritize full expressions, e.g. of the type `%Y-%m-%d %H:%M:%S` :type deferred_url_extractor: boolean :return: Returns a valid date expression as a string, or None )levelNrz.//link[@rel="canonical"]hrefzlxml cleaner errorz.//title|.//h1FrGrH)rIzutf-8ignorezextensive search startedr)*logging basicConfigDEBUGr9rAr7r?r>findrrrrrrr8rr5rrerrorrrrrrr UnicodeDecodeErrordecoderr rrrrrKrrrrr;r7)r8rr9rr:r;rrr<rr url_resulturlelemr3 abbr_result search_tree discarded date_exprrrsegment converteds rL find_daterP(sd1'-0000 Z D |tz!!/,*G*G!tXX GJ {))788  ++f%%C"#w//J&<D' * * Hk$.H.HF  *"8( K+!1 x~~} 5 5" "  YY +++  )*****+ 4 #33  #33      7 !     7 !g 6 6  Yk RRR YYYk>>>EEgxXX Y z#4g>> 6 k7 + + 6 W 5 5   = /000 ,[99 G GGmmooG"S\\CCCCOCCCC))WgFFII-iAA 2D11&EE5G/G:9G:)TFrNFNNF)k__doc__rAre collectionsrcopyrr functoolsrrtypingrr r r Counter_Type lxml.htmlr r extractorsrrrrrrrrrrrrrrrrrr r!r"r#r$r%r&r'r(r)r*r+r,r-r.r/r0r1r2r3settingsr4r5r6utilsr7r8r9r: validatorsr;r<r=r>r?r@rArBrC getLogger__name__rstrrMrrrrrunionrrcompilerr/rrrrr rrrrr+r7bytesboolrPrNrLres) ((((((((KKKKKKKKKKKKKK++++++++&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&NIHHHHHHHHH>>>>>>>>>>>>                        8 $ $M{MsMMMM HHHV *FEE)<8(../FGG??? 2:g&&K +,     c]      c] (u u uc]uuuup4c"4 3<4S\4 4 eCj 4444nAA S\A 3<AS\ A  A eCj AAAA$ :        6 6 6c]6666r> > >c]>>>>B#8E#J/#C####gCg)g ggggX""/3/3#(ooeS+-.ooo o # o  ouXs]+,ouXs]+,o!oc]oooooorN