!}gdZdZdZdZdZddlZddlZddlmZddl m Z d d l m Z d Z ead ZddZdZdZdZdZdZdZdZdZdZdS)zH Anything natural language related should be abstracted into this file. newspaperz Lucas Ou-YangMITzCopyright 2014, Lucas Ou-YangN)path)Counter)settingsg4@cl|dkr tj}n2tjtjd|}t |dd5}ttd| DddddS#1swxYwYdS)zB Loads language-specific stopwords for keyword selection enzstopwords-{}.txtrzutf-8)encodingc6g|]}|S)strip).0ws M/var/www/py-google-trends/myenv/lib/python3.11/site-packages/newspaper/nlp.py z"load_stopwords..%s ???Aaggii???N) rNLP_STOPWORDS_ENrjoin STOPWORDS_DIRformatopen stopwordsupdateset readlines)language stopwordsFilefs rload_stopwordsr!s4 1  ("8"4";";H"E"EGG mS7 3 3 3Bq?????@@AAABBBBBBBBBBBBBBBBBBsAB))B-0B-cF|r|r|dkrgSg}t|}t|}t|}t||||}|D]} || d|dd|DS)Nrc|dS)Nrr)summarys rzsummarize..5s wqzr)keycg|] }|d S)rr)rr&s rrzsummarize..6s 0 0 07GAJ 0 0 0r)split_sentenceskeywords split_wordsscore most_commonappendsort) urltitletext max_sents summaries sentenceskeys titleWordsranksranks r summarizer;(s u Q I%%I D>>DU##J )Z . . : :9 E EE""a!!!! NN11N222 0 0i 0 0 00rct|}t}t|D]\}}t|}t ||}t t|} t |dz|} t||} t||} | | zdz dz} |dz| dzz| dzz| dzzdz }||||f<|S)z0Score sentences based on different features rg@$@??g@) lenr enumerater, title_score length_scoresentence_positionsbsdbs)r6r8r+senSizer9issentence titleFeaturesentenceLengthsentencePosition sbsFeature dbsFeature frequency totalScores rr-r-9s)nnG IIE)$$ # #1q>>":x88 %c(mm44,QUG<<8,, 8,, *,3d: "3&36$S()+;C+?@ADE "q!f Lrcd}t|dkrdS|D]}||vr |||z }dtjt|z |zdz S)Nrr?r=)r@mathfabs)wordsr+r-words rrErEMsi E E aq$$ 8   Xd^ #E $)CJJ'' '% /4 77rct|dkrdSd}g}g}t|D]M\}}||vrD||}|gkr||g}|}||g}|d|dz }||d|dz|dzz z }Ntt|t|dz} d| | dzzz |zS)Nrrr?)r@rArr7 intersection) rVr+summfirstsecondrHrWr-difks rrFrFWs E aq D E FU## < <4 8  TNE{{E E Ah*qF1I-#(;; C  - -c%jj 9 9::Q>A a#g $ &'rc tjdd|}d|DS#t$rYdSwxYw)z'Split a string into array of words z[^\w ]r"c\g|])}|d*S).)rlowerrxs rrzsplit_words..rs.;;; ""$$;;;rN)resubsplit TypeError)r3s rr,r,msXviT**;;djjll;;;; tts36 AAcd}t|}|rt|}d|D}i}|D]}||vr||xxdz cc<d||<t|t|}t|dd}|d|}t d|D}|D])}||d zt |dz }|d zdz||<*t |St S) zGet the top 10 keywords and their frequency scores ignores blacklisted words in stopwords, counts the number of occurrences of each word, and sorts them in reverse natural order (so descending) by number of occurrences. c$g|] }|tv |Srrrds rrzkeywords..s"666a1I#5#5#5#5#5rrc"|d|dfS)Nrrr)res rr'zkeywords..s1qt rT)r(reverseNc3$K|] \}}||fV dS)Nr)rreys r zkeywords..s*4441aA444444rr?r>)r,r@minsorteditemsdictmax) r3 NUM_KEYWORDS num_wordsfreqrWmin_sizer+r_ articleScores rr+r+ws9 L t  D II 664666  Dt||T a T |SYY//$**,,44"&(((IXI&44844444 1 1A#A;,s9a/@/@@L&,q0HQKKH~~vv rcddl}|jd}||}d|D}|S)z(Split a large string into sentences rNztokenizers/punkt/english.picklec`g|]+}t|dk|dd,S)rk r")r@replacerds rrz#split_sentences..s1GGG3q66B;;4$$;;;r) nltk.datadataloadtokenize)r3nltk tokenizerr6s rr*r*sP @AAI""4((IGGiGGGI rcPdtjt|z tz z S)Nr)rTrUideal) sentence_lens rrCrCs! ty-..6 66rc|rEd|D}d}|D]}|tvr ||vr|dz }|tt|dz SdS)Nc$g|] }|tv |Srrmrds rrztitle_score..s"888qQi%7%7%7%7%7rrSr?rr)rrwr@)r2rJcountrWs rrBrBsn 88E888  DI%%$%-- s3u::q))))qrc|dz|z }|dkrdS|dkrdS|dkrdS|dkrdS|dkrd S|d krdS|d krd S|d krdS|dkrdS|dkrdS|dkrdSdS)zdDifferent sentence positions indicate different probability of being an important sentence. r?rg?g333333?g?g{Gz?gffffff?g333333?gQ?g?g?g?g333333?g{Gz?g?gQ?g?gq= ףp?g(\?r)rHsize normalizeds rrDrDsS4JSq s  t s  t s  t s  t s  t s  t s  t s  t s  t q..tqr)r"r"r"r#)__doc__ __title__ __author__ __license__ __copyright__rfrTosr collectionsrr"rrrrr!r;r-rErFr,r+r*rCrBrDrrrrs5    /   CEE BBB$1111"(888(((,D777   r