%$}gq& d Z ddlZddlZddlmZ ddlmZ ddlmZm Z m
Z
m Z m Z ddl
mZmZmZmZmZmZ ddlmZ dd lmZmZ dd
lmZmZ ej e Z ej d Z ej d ej! Z" ej d
Z# ej d Z$ ej d Z% ej d Z& ej d Z' ej d Z( ej d Z)g dZ* G d d Z+dddefde,de e, de-d e.d e/d e e, f d Z0de,d e e, d e-fd!Z1d"e,d e e, fd#Z2d$e e, d"e,d e e, fd%Z3dS )&z#
Deriving link info from sitemaps.
N)islice)sleep)CallableListSetOptionalPattern) clean_urlextract_domain filter_urlsfix_relative_urls get_hostinfo lang_filter )is_similar_domain) fetch_url is_live_page) MAX_LINKSMAX_SITEMAPS_SEENz.(?:)?zzhref=["\'](.+?)["\']zg(?:blogger|blogpost|ghost|hubspot|livejournal|medium|typepad|squarespace|tumblr|weebly|wix|wordpress)\.z ^.{0,5}<\?xml|>> !1228b
d&'''''r* linkc \ || j k rdS t | j | }t || j pd}|rt || j sdS t
|d }| t d| dS | j sRt | s8t | j
| s#t d| j
| dS t | r | j | dS | j | dS )z^Examine a link and determine if it's valid and if it leads to
a sitemap or a web page.Nr% T)fastzcouldn't extract domain: %sz-link discarded, diverging domain names: %s %s)r r
r r
r r r r, errorr WHITELISTED_PLATFORMSsearchr r warningDETECT_SITEMAP_LINKr appendr! )r' r1 newdomains r( handle_linkzSitemapObject.handle_linkX sA 4# # # F 55t /006B ;tT-=>> F"4d333 LL6 = = = F
)00;; &dk9==
NN?i
F % %d + + # $ $T * * * * * I T " " " " "r* regexindexhandlerc " fdt | | j t D D ]
} || t dt
| j t
| j | j dS )zJExtract links from the content using pre-defined regex, index and handler.c 3 ( K | ] }| V
d S )N ).0mr= s r( z.SitemapObject.extract_links..~ s8
AeH
r* z%%s sitemaps and %s links found for %sN)
r finditerr r r, r- lenr r! r )r' r< r= r> matchs ` r(
extract_linkszSitemapObject.extract_linksz s
$U^^DL%A%A9 M M
E
GENNNN 3 ! " " NN
r* c d j vrdS t j d j dt j dt
ddf fd } t d| dS ) z7Extract links corresponding to a given target language.z hreflang=Nzhreflang=[\"'](z.*?|x-default)[\"']attrsr# c | r9t | }|r |d dS dS dS )z!Examine language code attributes.r N)r6 HREFLANG_REGEXr; )rJ
lang_match
lang_regexr' s r( handle_lang_linkzASitemapObject.extract_sitemap_langlinks..handle_lang_link sf ''
4 + 2 25 9 9
4$$Z]33333
4
44 4r* r )r recompiler DOTALLstrrH XHTML_REGEX)r' rO rN s ` @r( extract_sitemap_langlinksz'SitemapObject.extract_sitemap_langlinks s dl * * FZ Dt / D D Dbi
4C 4D 4 4 4 4 4 4 4
;+;<<<<z"sitemap_search.. s: A
A
A
!"w A
A
A
r* c & g | ]
}|j v |S rA )r )rB sr s r( ro z"sitemap_search.. s,
q /D/DA/D/D/Dr* z %s sitemap links found for %s)r r, r7 r endswithrF r r find_robots_sitemapsGUESSESr popr r0 r^ r r r! r- )
rf r r rg rh
domainname urlfilter sitemapurlsrn r s
@@r( sitemap_searchry s . 's++J(#... CSIII I
||.// e s88c'llQ& & &IGZk8TTG
3G < <
A
A
A
A
&-A
A
A
3w|#4#4|#C#C % 2 6 6 8 8
+
w| | + + *
3w|#4#4|#C#C <"7<;;
LL0#gl2C2CZPPP <r* contentsc L |dS t d| } t | r/t |t
r8t | r d|dd v r t
d| dS dS )zLCheck if the sitemap corresponds to an expected format,
i.e. TXT or XML.NFr% z| dd }t | dk rh|d |d< |d dk r-| |d t t | }fd |D }t d
t | |S ) z.Read a robots.txt file and find sitemap links.Ni' #r :r rj r c 4 g | ]}|t | S rA )r
)rB urn s r( ro z+extract_robots_sitemaps..- s) JJJQJ$Wa00JJJr* z %s sitemaps found in robots.txt) rF
splitlinesfindstripsplitr r9 listdictfromkeysr, r- )r rn
candidateslinei
line_partsrx s ` r( r r sO C NNU 2 2 J$$&&
9
9 IIcNN 668Dzz|| ZZQ''
z??a &qM / / 1 1 7 7 9 9JqM!} ))!!*Q-"5"5"7"7888dmmJ//00JJJJJ*JJJK
LL2C 4D4DEEE r* )4rb loggingrP itertoolsr timer typingr r r r r courlanr
r r r
r r
deduplicationr downloadsr r settingsr r getLoggerr_ r, rQ rW rR rT rL r5 r[ r8 r\ r} r rt r rS rd floatre ry rZ rs r rA r* r( r s 9 9 9 9 9 9 9 9 9 9 9 9 9 9 - , , , , , . . . . . . . . 2 2 2 2 2 2 2 2
8 $ $
RZI
J
J
bj+RY77 344"
n =>> bj!?@@ rz.// bj '' BJz** ~% ~% ~% ~% ~% ~% ~% ~%F "&) C C C# C C C
C
#Y
C C C CLc Xc] t *7# 7$s) 7 7 7 7x} s tCy r*