%$}gZ dZddlZddlZddlmZmZmZmZddlm Z ddl m Z m Z ddl mZmZmZmZmZmZmZmZmZddlmZdd lmZdd lmZmZmZmZm Z ej!e"Z#ej$d ej%Z&ej$d ej%Z'ej$d ej%Z( d&de)de*de*dee)de*de*deee)e)ffdZ+ d'dddddddddde)dee)de*de*dee)de*de*de*de*dee)dee)dee)fdZ,ddddddd d!e)dee)d"ee)d#ee d$e*de*de*dee)deee)ee)ffd%Z-dS)(z0 Core functions needed to make the module work. N)ListOptionalSetTuple)RobotFileParser) normalize_url scrub_url) basic_filter domain_filterextension_filteris_navigation_pageis_not_crawlable lang_filter path_filter type_filter validate_url)redirection_test) BLACKLIST)extract_domain get_base_urlfix_relative_urls is_external is_known_linkz ]+?>zhreflang=["\']?([a-z-]+)zhref=["\']?([^ ]+?)(["\' >])FTurlstrictwith_redirectslanguagewith_navtrailing_slashreturnc~ t|dur"td|tt |}|rt |}t |||dur"td|t|6t||||dur"td|tt|\}}|dur"td|tt|j dur"td|tt|j dur"td |t|r>t|j |jdur"td |tt||||}|rt!|t"d }nt!|d }|td |dSn3#t$tf$rtd|YdSwxYw||fS)a(Check links for appropriateness and sanity Args: url: url to check strict: set to True for stricter filtering with_redirects: set to True for redirection test (per HTTP HEAD request) language: set target language (ISO 639-1 codes) with_nav: set to True to include navigation pages instead of discarding them trailing_slash: set to False to trim trailing slashes Returns: A tuple consisting of canonical URL and extracted domain Raises: ValueError, handled in exception. Fzrejected, basic filter: %s)rrzrejected, type filter: %sNzrejected, lang filter: %szrejected, validation test: %szrejected, extension filter: %szrejected, domain name: %szrejected, path filter: %sT) blacklistfast)r$zdiscarded URL: %s)r LOGGERdebug ValueErrorr rrrrr pathr netlocrqueryr rrAttributeError) rrrrrr validation_test parsed_urldomains L/var/www/py-google-trends/myenv/lib/python3.11/site-packages/courlan/core.py check_urlr0*s^4>    % % LL5s ; ; ; nn  ("3''C s6H = = = F F LL4c : : :   C6>BBeKK LL4c : : : '33&7&7# e # # LL8# > > >  JO , , 5 5 LL93 ? ? ?  * + +u 4 4 LL4c : : :   k*/:3CDDMM LL4c : : : J.II  4#C94HHHFF#Cd333F > LL4c : : :4  J ' (#...tt ;sHH,H87H8) no_filterrrr r redirects referencebase_url pagecontent external_boolr1r2r3r4c | rtdt|} |p| }tt} } |s| S| p| } dt|DD]} d| vrd| vr |dur|d| vr{t | }|r^|d|s |dd kr7t| }|r| |dt| }|r| |d| D]} | d st|| } |dur7t| ||||| }|C|d } |t| | d krbt| | rs| | tdt!| t!| | S)azFilter links in a HTML document using a series of heuristics Args: pagecontent: whole page in binary format url: full URL of the original page external_bool: set to True for external links only, False for internal links only no_filter: override settings and bypass checks to return all possible URLs language: set target language (ISO 639-1 codes) strict: set to True for stricter filtering trailing_slash: set to False to trim trailing slashes with_nav: set to True to include navigation pages instead of discarding them redirects: set to True for redirection test (per HTTP HEAD request) reference: provide a host reference for external/internal evaluation Returns: A set containing filtered HTTP links checked for sanity and consistency. Raises: Nothing. ,'base_url' is deprecated, use 'url' instead.c3&K|] }|dV dS)rN).0ms r/ z extract_links..s&FF!1FFFFFFrelnofollowFNhreflangrz x-defaulthttp)rr rrrrT)rr3 ignore_suffixu!%s links found – %s valid links)r'rsetFIND_LINKS_REGEXfinditerHREFLANG_REGEXsearch startswith LINK_REGEXaddrr0rrr%infolen)r5rr6r1rrr rr2r3r4 candidates validlinkslink langmatch linkmatchcheckeds r/ extract_linksrTs\DIGHHHC  H /C UUCEE J %XIGF/88EEFFF-- D==Z4//    ("6:;M;M&--d33I 1! ''11 15>q\[5P5P&--d33 1NN9Q<000#))$//I -y|,,,v&& 0$S$//D   -!(! G1:D IT!!! z * *  t KK3S__c*ooVVV r>)langrulesexternalrrr4 htmlstringrUrVrWc(|rtdgg} }t||||||D]d} t| s||d| s*t | r| | O|| e|| fS)zPFind links in a HTML document, filter and prioritize them for crawling purposes.r8)r5rr6rrrN*)r'rTr can_fetchrappend) rXrrUrVrWrrr4linkslinks_priorityrPs r/ filter_linksr_sIGHHH>E   D ! !   eooc4&@&@   d # #   ! !$ ' ' ' ' LL     .  r>)FFNFT)NF).__doc__loggingretypingrrrrurllib.robotparserrcleanr r filtersr r r rrrrrrnetworkrsettingsrurlutilsrrrrr getLogger__name__r%compileIrErGrJstrboolr0rTr_r:r>r/rps  ------------......++++++++                      &%%%%%  8 $ $2:nbd337>> RZ7 > >  " ZZ Z ZZsm Z  Z  ZeCHoZZZZ~] "#"]]]] #]]  ] sm ] ]]]]}]sm] X]]]]H'+"%!%!%!%! #%! 3- %! O $ %!  %! %!%!sm%! 49d3i  %!%!%!%!%!%!r>