!}gIdZddlmZddlZddlZddlmZmZddl m Z ddl m Z ddl Z ddlZddlmZmZdd lmZmZmZdd lmZejd Zd Ze d GddZGddZeZGddZe ej  d#d$dZ!e ej"dZ"Gdd Z#d%d"Z$dS)&aZ`tldextract` accurately separates a URL's subdomain, domain, and public suffix. It does this via the Public Suffix List (PSL). >>> import tldextract >>> tldextract.extract('http://forums.news.cnn.com/') ExtractResult(subdomain='forums.news', domain='cnn', suffix='com', is_private=False) >>> tldextract.extract('http://forums.bbc.co.uk/') # United Kingdom ExtractResult(subdomain='forums', domain='bbc', suffix='co.uk', is_private=False) >>> tldextract.extract('http://www.worldbank.org.kg/') # Kyrgyzstan ExtractResult(subdomain='www', domain='worldbank', suffix='org.kg', is_private=False) Note subdomain and suffix are _optional_. Not all URL-like inputs have a subdomain or a valid suffix. >>> tldextract.extract('google.com') ExtractResult(subdomain='', domain='google', suffix='com', is_private=False) >>> tldextract.extract('google.notavalidsuffix') ExtractResult(subdomain='google', domain='notavalidsuffix', suffix='', is_private=False) >>> tldextract.extract('http://127.0.0.1:8080/deployed/') ExtractResult(subdomain='', domain='127.0.0.1', suffix='', is_private=False) To rejoin the original hostname, if it was indeed a valid, registered hostname: >>> ext = tldextract.extract('http://forums.bbc.co.uk') >>> ext.registered_domain 'bbc.co.uk' >>> ext.fqdn 'forums.bbc.co.uk' ) annotationsN) CollectionSequence) dataclass)wraps) DiskCache get_cache_dir)lenient_netloc looks_like_iplooks_like_ipv6)get_suffix_listsTLDEXTRACT_CACHE_TIMEOUT)z4https://publicsuffix.org/list/public_suffix_list.datzQhttps://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.datT)orderceZdZUdZded<ded<ded<ded<edd Zedd Zedd Zedd Z d S) ExtractResultzA URL's extracted subdomain, domain, and suffix. Also contains metadata, like a flag that indicates if the URL has a private suffix. str subdomaindomainsuffixbool is_privatereturncD|jr|jr|jd|jSdS)zJoins the domain and suffix fields with a dot, if they're both set. >>> extract('http://forums.bbc.co.uk').registered_domain 'bbc.co.uk' >>> extract('http://localhost:8080').registered_domain '' .)rrselfs U/var/www/py-google-trends/myenv/lib/python3.11/site-packages/tldextract/tldextract.pyregistered_domainzExtractResult.registered_domainHs4 ; 24; 2k11DK11 1rc|jr?|js|jr1dd|j|j|jfDSdS)zReturns a Fully Qualified Domain Name, if there is a proper domain/suffix. >>> extract('http://forums.bbc.co.uk/path/to/file').fqdn 'forums.bbc.co.uk' >>> extract('http://localhost:8080').fqdn '' rc3K|]}||V dSN).0is r z%ExtractResult.fqdn.._s(WW!UVWAWWWWWWr!r)rrrjoinrrs rfqdnzExtractResult.fqdnUsV ; XDK X4? X88WW T['QWWWWW Wrr!cf|jr)|js"|jst|jr|jSdS)aReturns the ipv4 if that is what the presented domain/url is. >>> extract('http://127.0.0.1/path/to/file').ipv4 '127.0.0.1' >>> extract('http://127.0.0.1.1/path/to/file').ipv4 '' >>> extract('http://256.1.1.1').ipv4 '' r)rrrr rs ripv4zExtractResult.ipv4bsE K [ $(N dk**  ; rr!cd}t|j|krP|jddkr?|jddkr.|js'|js |jdd}t |r|SdS)aReturns the ipv6 if that is what the presented domain/url is. >>> extract('http://[aBcD:ef01:2345:6789:aBcD:ef01:127.0.0.1]/path/to/file').ipv6 'aBcD:ef01:2345:6789:aBcD:ef01:127.0.0.1' >>> extract('http://[aBcD:ef01:2345:6789:aBcD:ef01:127.0.0.1.1]/path/to/file').ipv6 '' >>> extract('http://[aBcD:ef01:2345:6789:aBcD:ef01:256.0.0.1]').ipv6 '' r[]rr)lenrrrr )rmin_num_ipv6_chars debracketeds ripv6zExtractResult.ipv6us    2 2 2 A#%% B3&&['$(N'+ad+K{++ #""rr!N)rr) __name__ __module__ __qualname____doc____annotations__propertyr r*r,r5r%r!rrr<s NNNKKKKKK    X    X X$Xr!rceZdZdZeedddefd%dZ d&d'dZ d&d'dZ d&d(dZ d)d*dZ d+d,d Z e d)d-d"Z d)d.d$ZdS)/ TLDExtractzOA callable for extracting, subdomain, domain, and suffix components from a URL.TFr% cache_dir str | Nonesuffix_list_urls Sequence[str]fallback_to_snapshotrinclude_psl_private_domainsextra_suffixescache_fetch_timeoutstr | float | NonerNonec>|pd}td|D|_||_|js|s|jstd||_||_d|_t|trt|n||_ t||_ dS)aConstruct a callable for extracting subdomain, domain, and suffix components from a URL. Upon calling it, it first checks for a JSON in `cache_dir`. By default, the `cache_dir` will live in the tldextract directory. You can disable the caching functionality of this module by setting `cache_dir` to `None`. If the cached version does not exist, such as on the first run, HTTP request the URLs in `suffix_list_urls` in order, and use the first successful response for public suffix definitions. Subsequent, untried URLs are ignored. The default URLs are the latest version of the Mozilla Public Suffix List and its mirror, but any similar document URL could be specified. Local files can be specified by using the `file://` protocol (see `urllib2` documentation). To disable HTTP requests, set this to an empty sequence. If there is no cached version loaded and no data is found from the `suffix_list_urls`, the module will fall back to the included TLD set snapshot. If you do not want this behavior, you may set `fallback_to_snapshot` to False, and an exception will be raised instead. The Public Suffix List includes a list of "private domains" as TLDs, such as blogspot.com. These do not fit `tldextract`'s definition of a suffix, so these domains are excluded by default. If you'd like them included instead, set `include_psl_private_domains` to True. You can specify additional suffixes in the `extra_suffixes` argument. These will be merged into whatever public suffix definitions are already in use by `tldextract`, above. cache_fetch_timeout is passed unmodified to the underlying request object per the requests documentation here: http://docs.python-requests.org/en/master/user/advanced/#timeouts cache_fetch_timeout can also be set to a single value with the environment variable TLDEXTRACT_CACHE_TIMEOUT, like so: TLDEXTRACT_CACHE_TIMEOUT="1.2" When set this way, the same timeout value will be used for both connect and read timeouts r%c3fK|],}||V-dSr$)strip)r&urls rr(z&TLDExtract.__init__..sK& & syy{{& IIKK& & & & & & r!zThe arguments you have provided disable all ways for tldextract to obtain data. Please provide a suffix list data, a cache_dir, or set `fallback_to_snapshot` to `True`.N) tupler@rB ValueErrorrCrD _extractor isinstancerfloatrEr _cache)rr>r@rBrCrDrEs r__init__zTLDExtract.__init__sd,1r %& & #3& & & ! ! %9!%  d6O ;  ,G(,@D-s33 %E% & & &$   ** r!NrKr bool | Nonesessionrequests.Session | Nonerc2||||S)zAlias for `extract_str`.rT) extract_strrrKrCrTs r__call__zTLDExtract.__call__s %@'RRRr!cL|t|||S)aTake a string URL and splits it into its subdomain, domain, and suffix components. I.e. its effective TLD, gTLD, ccTLD, etc. components. >>> extractor = TLDExtract() >>> extractor.extract_str('http://forums.news.cnn.com/') ExtractResult(subdomain='forums.news', domain='cnn', suffix='com', is_private=False) >>> extractor.extract_str('http://forums.bbc.co.uk/') ExtractResult(subdomain='forums', domain='bbc', suffix='co.uk', is_private=False) Allows configuring the HTTP request via the optional `session` parameter. For example, if you need to use a HTTP proxy. See also `requests.Session`. >>> import requests >>> session = requests.Session() >>> # customize your session here >>> with session: ... extractor.extract_str("http://forums.news.cnn.com/", session=session) ExtractResult(subdomain='forums.news', domain='cnn', suffix='com', is_private=False) rW)_extract_netlocr rYs rrXzTLDExtract.extract_strs16## 3  !>> extractor = TLDExtract() >>> extractor.extract_urllib(urllib.parse.urlsplit('http://forums.news.cnn.com/')) ExtractResult(subdomain='forums.news', domain='cnn', suffix='com', is_private=False) >>> extractor.extract_urllib(urllib.parse.urlsplit('http://forums.bbc.co.uk/')) ExtractResult(subdomain='forums', domain='bbc', suffix='co.uk', is_private=False) rW)r\netlocrYs rextract_urllibzTLDExtract.extract_urllibs+(## J3W$   r!r_c(|dddddd}d}t||krB|ddkr6|dd kr*t|d drtd |d d S|d}||||\}}d} |t|cxkr| kr$nn!t|rtd |d |S|t|krd||dnd } |dkr d|d|d z nd } |r ||d z nd } t| | | |S)Nu。ru．u｡r.rr/r0r1rrF)rrW)rC) replacer2r rsplit_get_tld_extractor suffix_indexr r)) rr_rCrTnetloc_with_ascii_dotsr3labelsrfrnum_ipv4_labelsrrrs rr\zTLDExtract._extract_netlocs NN8X . . WXx ( ( WXx ( (   & ' '+= = =&q)S00&r*c115ad;<< W$R)?PUVVVV'--c22#'#:#:$;$ $ ,v;V, W W ! j 3v;; 9 9 9 9/ 9 9 9 9 9m "? ? 9!%;RLL L4@CKK4O4O&/000UWr?r@rArBrrCrrDrArErFrrG)NNrKrrCrSrTrUrr)rKr]rCrSrTrUrrr$)r_rrCrSrTrUrrFN)rjrrTrUrrG)rTrUrrn)rTrUrrr)r6r7r8r9r PUBLIC_SUFFIX_LIST_URLS CACHE_TIMEOUTrRrZrXr`r\rmr;rqrer%r!rr=r=sHYY !. *A%),1(*2?H+H+H+H+H+Z48+/ SSSSS48+/      D48+/      8,0 $D$D$D$D$DNKO55555EEEEXE26!!!!!!!r!r=cHeZdZdZ ddd Ze dddZdddZdS)Triez:Trie for storing eTLDs with their labels in reverse-order.NFmatchesdict[str, Trie] | NoneendrrrrGc8|r|ni|_||_||_dS)zTODO.N)rrr)rrrrs rrRz Trie.__init__|s&#*1wwr $r!public_suffixesCollection[str]private_suffixesCollection[str] | Nonect}|D]}|||g}|D]}||d|S)z?Create a Trie from a list of suffixes and return its root node.NT)r add_suffix)rr root_noders rcreatez Trie.createsm FF % ) )F   ( ( ( (  #! & / /F   . . . .r!rrc|}|d}||D].}||jvrt|j|<|j|}/d|_||_dS)z+Append a suffix's labels to this Trie node.rTN)rdreverserrrr)rrrnoderhlabels rrzTrie.add_suffixstc"" ' 'EDL((&*ff U#<&DD$r!)NFF)rrrrrrrrGr$)rrrrrrF)rrrrrrG)r6r7r8r9rR staticmethodrrr%r!rrrysDD+/ % % % % %48\$ % % % % % % %r!rFrKrrCrSrTrUrc&t|||S)N)rCrT) TLD_EXTRACTOR)rKrCrTs rextractrs#  )Dg   r!c$tj|i|Sr$)rrm)argskwargss rrmrms   0 0 00r!c4eZdZdZ ddd ZdddZ dddZd S)rrz8Wrapper around this project's main algo for PSL lookups.FrvrnrwrxrCrc>||_||_||_t||z|z|_t||z|_t |jt||_t |j|_ dSr$) rCrvrw frozensettlds_incl_privatetlds_excl_privaterrtlds_incl_private_trietlds_excl_private_trie)rrvrwrxrCs rrRz&_PublicSuffixListTLDExtractor.__init__s,G(&(!*;+E +R!S!S!*;+C!D!D&*kk  "Il$;$;' ' #'+kk$2H&I&I###r!NrSrfrozenset[str]c4||j}|r|jn|jS)z,Get the currently filtered list of suffixes.)rCrr)rrCs rrqz"_PublicSuffixListTLDExtractor.tldss. & .*.*J '+ (D " "' r!spltuple[int, bool]c||j}|r|jn|j}t|}|}t |D]~}t |}||jvr|dz}|j|}|jr|}6d|jv}|r=d|z|jv} | r||jdjfcS|dz |jdjfcS||jfS)zReturn the index of the first suffix label, and whether it is private. Returns len(spl) if no suffix is found. Nr*!) rCrrr2reversed_decode_punycoderrr) rrrCrr'jr decoded_label is_wildcardis_wildcard_exceptions rrfz*_PublicSuffixListTLDExtractor.suffix_indexs ' .*.*J '+ -D ' ', HH c]]  E,U33M ,,Q|M28A-K ;(+m(;t|(K%(;dl3/:::::1udl3/::::: $/!!r!r)rvrnrwrnrxrnrCrr$)rCrSrr)rrnrCrSrr)r6r7r8r9rRrqrfr%r!rrrrrsvBB-2 JJJJJ$      JN#"#"#"#"#"#"#"r!rrrc|}|d}|r, tj|S#tt f$rYnwxYw|S)Nzxn--)lower startswithidnadecode UnicodeError IndexError)rloweredlooks_like_punys rrrsikkmmG((00O ;w'' 'j)    D  NsAAAr{rz)rrrr)%r9 __future__ros urllib.parseurllibcollections.abcrr dataclassesr functoolsrrrequestsrtr r remoter r r suffix_listrenvirongetr}r|rr=rrrZrrmrrrr%r!rrsP""H#""""" 00000000!!!!!! ++++++++BBBBBBBBBB)))))) 9::   MMMMMMMM`ffffffffR .%.%.%.%.%.%.%.%b}05'+}111C"C"C"C"C"C"C"C"Lr!