!}gHFdZdZdZdZddlZddlZddlZddlZddlZddl m Z ddl m Z dd l m Z dd l m Z dd l mZdd lmZdd lmZddlmZddlmZddlmZmZmZmZmZddlmZeje Z!Gdde"Z#Gdde$Z%Gdde"Z&dS) newspaperz Lucas Ou-YangMITzCopyright 2014, Lucas Ou-YangN)images)network)nlp)settings)urls)DocumentCleaner) Configuration)ContentExtractor)OutputFormatter) URLHelper RawHelper extend_configget_available_languagesextract_meta_refresh)VideoExtractorceZdZdZdZdZdS)ArticleDownloadStaterrN)__name__ __module__ __qualname__ NOT_STARTEDFAILED_RESPONSESUCCESSQ/var/www/py-google-trends/myenv/lib/python3.11/site-packages/newspaper/article.pyrrsKOGGGrrceZdZdS)ArticleExceptionN)rrrrrr r"r"%sDrr"ceZdZdZd)dZdZd*dZdZd Zd Z d Z d Z d Z dZ dZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZd Zd!Z d"Z!d#Z"d$Z#d%Z$d&Z%d'Z&d(Z'dS)+Articlez9Article objects abstract an online news article page Nc t|tst|trtd|p t|_t |j||_t |j|_|dkr2tj|}|d}|dztj |z}||dkrtd||_ tj ||j |_ ||_ dx|_|_d|_gx|_|_g|_d|_g|_g|_t/|_g|_d|_d|_d|_d|_d|_t>j |_!d|_"d|_#d|_$d|_%i|_&d|_'d|_(d|_)d|_*d|_+i|_,dS)znThe **kwargs argument may be filled with config values, which is added into the config object zmConfiguration object being passed incorrectly as title or source_url! Please verify `Article`s __init__() fn.r%Nhttpz://zinput url bad formatF)- isinstancer r"configrr extractorr get_scheme get_domain source_url prepare_urlurltitletop_img top_imagemeta_imgimgsrmoviestextkeywords meta_keywordssettagsauthors publish_datesummaryhtml article_html is_parsedrrdownload_statedownload_exception_msgmeta_description meta_lang meta_favicon meta_datacanonical_linktop_nodeclean_top_nodedoc clean_docadditional_data)selfr/r0r-r)kwargsschemes r __init__zArticle.__init__,s e] + + G:}55 G"FGG G/  #DK88 )$+66   _S))F~%$/#*>*>>J  r!1!1"#9:: :%#C99 )+* t~ #%$ DK    EE    2>&*#!#! # "rc~|||dS)zBuild a lone article from a URL independent of the source (newspaper). Don't normally call this method b/c it's good to multithread articles on a source (newspaper) level. N)downloadparserrMs r buildz Article.builds/    rrc$| tj|j|j}ns#tjj$rZ}tj|_ t||_ t d|jd|j Yd}~dSd}~wwxYw|}|jjrCt|}|r2|dkr,|tj||dzS||||dS)zDownloads the link's HTML content, don't use if you are batch async downloading articles recursion_counter (currently 1) stops refreshes that are potentially infinite NzDownload failed on URL z because of r) input_htmlrecursion_counter)rget_html_2XX_onlyr/r)requests exceptionsRequestExceptionrrrAstrrBlogdebugfollow_meta_refreshrrRget_htmlset_html set_title)rMrWr0rXr>emeta_refresh_urls r rRzArticle.downloads7   04;GG&7   &:&J#.1!ff+ 888T%@%@BCCC  D ; * =3D99  =$5$9$9}}&/0@AA&7!&;%=== d us$BAB  Bc||j|j|_t j|j|_|jdS| }|j |_ t|j}t|j}|j |j}|||j |j}|||j |j}|||jjr9|j |j||j|j |j}|||j |j}|||j |j|j} || |j |j} || |j |j} |!| |j "|j} |#| |j $|j|j|_%|&|j|_|j '|j|_(|j(tS|j|j(} |*| +|j ,|j(|_(t j|j(|_-|.|j(\}}|/||0||1d|_2|3dS)NT)4throw_if_not_downloaded_verboser) get_parser fromstringr>rJcopydeepcopyrKget_parse_candidate link_hashr rr* get_titlerc get_authors set_authors get_meta_langset_meta_languageuse_meta_languageupdate_languagerD get_faviconset_meta_faviconget_meta_descriptionset_meta_descriptionget_canonical_linkr/set_canonical_link extract_tagsset_tagsget_meta_keywordsset_meta_keywords get_meta_data set_meta_dataget_publishing_dater<cleancalculate_best_noderHr set_movies get_videos post_cleanuprI get_formattedset_article_htmlset_text fetch_imagesr@release_resources)rMparse_candidatedocument_cleaneroutput_formatterr0r;rDrErCrGr:r8rFvideo_extractorr6r?s r rSz Article.parses ,,...;))++66tyAAtx00 8  F2244(2*4;77*4;77((88 u.,,T^<< !!!N00@@  y))) ; ( = N * *4> : : :  , ,T^ < < <~11$.AA  l+++ N / / ? ?  !!"2333:: Hdn&& ///~**4>:: d88 N  }---N00@@  9%%% N>> H N $))$(33::48DD = $,T[$-HHO OOO6688 9 9 9 N77 FFDM"&- ">">D !1!?!? "" D,  ! !, / / / MM$          rc|j|j|j|j}|||j|j|j}|jr||j|||j p| s\|j |j|j }|j j r||n||| s"|j j r|dSdSdSN)rKr*get_meta_img_urlr/ set_meta_img get_img_urlsr3addset_imgsrI has_top_imageget_first_img_urlr)r set_top_imgset_top_img_no_checkset_reddit_top_img)rM meta_img_urlr4 first_imgs r rzArticle.fetch_images sH > %>::$.**L   l + + +>..txHHD} (''' MM$      *43E3E3G3G *88$-//I{' 5  ++++)))444!!## & (@ &  # # % % % % % & & & &rc*|jduo |jdkS)Nr%)r1rTs r rzArticle.has_top_image s|4'>DLB,>>rc4tj|jS)zoPerforms a check on the url of this link to determine if article is a real news article or not )r valid_urlr/rTs r is_valid_urlzArticle.is_valid_url#s~dh'''rcD|jstd|j|j}|jd}|jd}|dkrAt||jj kr$t d|j zdS| s+|js$t d|j zdS|j+t|jdd kr$t d |j zdSt||jj kr$t d |j zdSt||jjkr$t d |j zdS|j |jdkr$t d|j zdSt d|j zdS)zrIf the article's body text is long enough to meet standard article requirements, keep the article z]must parse article before checking if it's body is valid! .articlez%s verified for article and wcTz%s caught for no media no textFNrz%s caught for bad titlez%s caught for word cntz%s caught for sent cntr%z%s caught for no htmlz%s verified for default true)r@r"r* get_meta_typerKr6splitlenr)MIN_WORD_COUNTr^r_r/ is_media_newsr0MIN_SENT_COUNTr>)rM meta_type wordcount sentcounts r is_valid_bodyzArticle.is_valid_body)s~ >"$=>> >N00@@ IOOC(( IOOC((  " "s9~~+(-(- II6A B B B4!!## DI  II6A B B B5 : TZ%5%5c%:%:!;!;a!?!? II/$(: ; ; ;5 y>>DK6 6 6 II.9 : : :5 y>>DK6 6 6 II.9 : : :5 9  R II-8 9 9 95 048;<<<trc0gd}|D]}||jvrdSdS)z^If the article is related heavily to media: gallery, video, big pictures, etc )z/videoz/slidez/galleryz /powerpointz/fashionz/glamourz/clothTF)r/)rM safe_urlsss r rzArticle.is_media_newsPs>777   ADH}}tturc||tj|jt tj|j }t tj|j  }t t||z}| ||jj }tj|j |j|}d|}||dS)z#Keyword extraction wrapper )r0r6 max_sents N)rgthrow_if_not_parsed_verboserload_stopwordsr) get_languagelistr7r6keysr0r9 set_keywordsMAX_SUMMARY_SENT summarizejoin set_summary)rM text_keyws title_keywskeywsr summary_sentsr=s r rz Article.nlp[s ,,... ((*** 4;3355666#,ty11668899 3< 3388::;; Sz12233 %   K0  DJTYR[\\\ ))M** !!!!!rc|jrtj|j|jSt j|jS)zyA parse candidate is a wrapper object holding a link hash of this article and a final_url of the article )r>rget_parsing_candidater/rrTs r rlzArticle.get_parse_candidatems9 9 H248TYGG G.tx888rc|}tj|stj|dSdS)z6Must be called after computing HTML/final URL N)get_resource_pathospathexistsmkdir)rMres_paths r build_resource_pathzArticle.build_resource_pathusK))++w~~h''  HX       rcd}tjtj|}tj|stj|tj|d|jz}|S)zxEvery article object has a special directory to store data in from initialization to garbage collection article_resourcesz%s_)rrrr TOP_DIRECTORYrrrm)rM res_dir_fnresource_directorydir_paths r rzArticle.get_resource_path|sl) W\\(*@*MMw~~011 ) H' ( ( (7<< 2EDN4JKKrc|}tj|D]'} tj|#t$rY$wxYwdSr)rglobrremoveOSError)rMrfnames r rzArticle.release_resourcessl%%''Yt__  E  %          sA A Ac tj|}||dS#t$r}d|jdvrt d|znPd|jdvrt d|zn)t d|zYd}~dSYd}~dSYd}~dSd}~wt$r(}t d|zYd}~dSd}~wwxYw) zWrapper for setting images. Queries known image attributes first, then uses Reddit's image algorithm as a fallback. z1Can't convert 'NoneType' object to str implicitlyrz(No pictures found. Top image not set, %sz timed outz4Download of picture timed out. Top image not set, %szsTypeError other than None type error. Cannot set top image using the Reddit algorithm. Possible error with PIL., %sNzZOther error with setting top image using the Reddit algorithm. Possible error with PIL, %s) rScraperrlargest_image_url TypeErrorargsr^r_critical Exception)rMrrds r rzArticle.set_reddit_top_imgs Nt$$A   Q0022 3 3 3 3 3 L L LBafQiOO DqHIIIIq )) PSTTUUUU GIJKLLLLLLLLL JIIIIIUUUUUU  N N N LLIKLM N N N N N N N N N Ns";? D A7C DDDc@|r|d|jj|_dSdSr)r) MAX_TITLEr0)rM input_titles r rczArticle.set_titles/  =$%;dk&;%;rrrA)rMr>s r rbzArticle.set_htmlsb  ?$&& G{--//@@FFDI"6">D     ? ?rc|r ||_dSdS)z7Sets the HTML of just the article's `top_node` N)r?)rMr?s r rzArticle.set_article_htmls$  - ,D    - -rc>||_||dSr)r3rrMsrc_urls r rzArticle.set_meta_imgs#  !!'*****rc|@tj|}||r||dSdSdSr)rrsatisfies_requirementsr)rMrrs r rzArticle.set_top_imgsY  t$$A''00 3))'22222   3 3rc"||_||_dS)zeProvide 2 APIs for images. One at "top_img", "imgs" and one at "top_image", "images" N)r1r2rs r rzArticle.set_top_img_no_checks  rc"||_||_dS)z{The motive for this method is the same as above, provide APIs for both `article.imgs` and `article.images` N)rr4)rMr4s r rzArticle.set_imgss  rct|tstd|r|d|jj|_dSdS)z'Keys are stored in list format zKeyword input must be list!N)r(rrr) MAX_KEYWORDSr7)rMr7s r rzArticle.set_keywordssU(D)) ;9:: :  @$%>dk&>%>?DMMM @ @rct|tstd|r|d|jj|_dSdS)zKAuthors are in ["firstName lastName", "firstName lastName"] format zauthors input must be list!N)r(rrr) MAX_AUTHORSr;)rMr;s r rpzArticle.set_authorssR'4(( ;9:: :  ="#;DK$;#;z-Article.set_meta_keywords..s JJJAaggiiJJJr,N)rr8)rMr8s r r~zArticle.set_meta_keywordss/KJ1D1DS1I1IJJJrc||_dSr)rE)rMrEs r rvzArticle.set_meta_favicons(rc||_dSr)rC)rMrCs r rxzArticle.set_meta_descriptions 0rc||_dSr)rF)rMrFs r rzArticle.set_meta_datas "rc||_dSr)rG)rMrGs r rzzArticle.set_canonical_links,rc||_dSr)r:)rMr:s r r|zArticle.set_tagss  rc,d|D}||_dS)z*Trim video objects into just urls c.g|]}||j |jSr)src)ros r rz&Article.set_movies.. s&BBBaBAEBaeBBBrN)r5)rM movie_objects movie_urlss r rzArticle.set_moviess#CB]BBB   rc|jtjkrtd|jtjkrtd|jd|jdS)zbParse ArticleDownloadState -> log readable status -> maybe throw ArticleException z'You must `download()` an article first!z!Article `download()` failed with z on URL N)rArrr"rrBr/rTs r rgz'Article.throw_if_not_downloaded_verbose sl  "6"B B B"#LMM M  $8$H H H""...$:;; ;I Hrc2|jstddS)z`Parse `is_parsed` status -> log readable status -> maybe throw ArticleException z$You must `parse()` an article first!N)r@r"rTs r rz#Article.throw_if_not_parsed_verboses*~ K"#IJJ J K Kr)r%r%N)NNr)(rrr__doc__rPrUrRrSrrrrrrrlrrrrrcrrbrrrrrrrprrrr~rvrxrrzr|rrgrrrr r$r$)s\j"j"j"j"X:J!J!J!X&&&,???((( %%%N   """$999   NNN(=== ???--- +++333 !!!@@@===999 +++KKK )))111###---!!! ;;;KKKKKrr$)' __title__ __author__ __license__ __copyright__loggingrjrrrZr%rrrr r cleanersr configurationr extractorsr outputformattersrutilsrrrrrvideos.extractorsr getLoggerrr^objectrrr"r$rrr rs    /  %%%%%%((((((((((((------CCCCCCCCCCCCCC------g!!6      y   rKrKrKrKrKfrKrKrKrKrKr