From e638bea1dadc1c989792c855b9d06caaa92a9c84 Mon Sep 17 00:00:00 2001 From: klein panic Date: Thu, 31 Oct 2024 00:04:22 -0400 Subject: [PATCH] switched to json, added keyword attraction, author extraction,excerpts exctraction, timestamp data, added future placeholder for rudimentary sentiment analysis maybe. Increased the number of articles used, added cap for article's analyzed to save cpu data, added a function to avoid repeat data collection of the same article --- .../oil_news_scraper.cpython-311.pyc | Bin 4411 -> 7069 bytes .../scrapers/backups/oil_news_scraper.py.bak | 118 +++++++++++------- .../WebScraper/scrapers/oil_news_scraper.py | 118 +++++++++++------- 3 files changed, 142 insertions(+), 94 deletions(-) diff --git a/Data-Collection/WebScraper/scrapers/__pycache__/oil_news_scraper.cpython-311.pyc b/Data-Collection/WebScraper/scrapers/__pycache__/oil_news_scraper.cpython-311.pyc index cdf37109cf90a3593c722bc6bd05c14fbc8ac6cb..a61209efb09eb1da9bada2d6e5734b83b9cdacd3 100644 GIT binary patch literal 7069 zcmZ3^%ge>Uz`)?dpq!q>&%p2)#DQTJDC4sUBLl;9h7^Vr#vFzy5Y3dsn9CH!#0X+D z=P>88M6rO`EIF*XY*B1rHfs)hE=Lpxn9Y{MnadT$4Q8|F@Z|DF@q*cmIebxkU~~DS z_`$S5lt2nclwc}fmJrl-h6Tbf7BbDmkjj@O0ux!rz`(E?#$ia|T*kz}u$l=b5G9IY zo)}CVnNAg5#>l|1nh_>eEC9Aa9L7hNXJkm>YGH_yNa1c_h>}d+xO5tr`h>}j> zYhj3zNns3T(Byv!((b3pc#E?7A^Ok}ZwDFz0H8b(ypl9^IiKoB)H85x2Zlo<>eido8;Dwrb~${86M zA{iJN7#Y$Tf*CYftIT}z6H^pYD>92qGV{_EK>k%oE6UGR@Cx?#Q%K9qNmVGxS4b?& z&rDHBDJ{s!Oa_IR-Yv$WU#z7iX}T6G8EXG?{L3f=rGFD`QM61_hUb!mm{QjQreG{p_67%sl<% z{FK!2VEuri{H)aEl45<|%#_69kksUiy!@Q}^vu*^eV4?NL|y0noSf8TP@Ls5=weW0$LFP%7so?I^a?6(34z=Y4|ZH;UV1#p%S93l3=Dz{3=G9n7#J8D z7@o3owv@NmHPubXyvVL_gV)KESky8OIj37RORDA&v2r+D|A3uIz6Mn$W+u&IYa@{5oM`7lJ!Yq#B z><6V79mUy`K_LfA5un5bN;sc+7#SGa*`_m;z!M~9a;arXVQgWjVRB(WEe^WaI@r<} zQ}NS}?$2PTVTk3bWvpRLVUUCd z1_Q$+rXKEKhLw!BSaLG+vR5)_GTma*Gq}Z`SX7dkoRbPo?M0x{@)xH~acW6&er|zX zm3(nxS!#Sqemp3n=-K4tCnx3<+v&j-NQ0863^;jS5Os#273vexXQWNZhwv_lI)4Va z0=ttn7#J9)GEQem2RWJ#-O-&)WsK{2Y8WRn_3#HX6oD)&l4W3E&}1$G`L#%%fq~%` zA3W25d{$fxictlH28dUkte`byXDCB7LzHDoD3!WP-WouR0;=ba#_HEkVUXcKxGD$n+1wW zFuMd)27p-%3|VmdQ#jMmDkO9_nt0#Jo0R0?kj=NdjV)$BFw z#iA&#;|IG=0Mm6781qinu)8o!V2oW_%TdFz0Mvj%u>r(M5lj(6x1~s%6x9nrWeUhB z6r94eMi@1$^B7YYYB_P5ERw>sMikYzMY5!*P7y;C5MaBwaOxIMks!g(SX7hbXKp<1 zkR-_+7^+Ef2TzJr4R;wsk!}q)_Fig=6l#p6NFzdY0lfE$9L=a@Odey3Of4_Tw#brd zOAU7|UkzuGMGaSxSq*oQNe$BiP;&vJ-ic^bjF{fmfX)@koOU_9wE{?B~^2=9%wY3$}@=Nnl;99Fh^b}n4KwVXkQJHy( zppK*7Ev5{kTViOAijObJEXhgLWW2>%TvC)-aEmcflkpZyMp0^7kt?Wm$XIYo6hlvL zYDwZP2@K)XisaOyf|6g1H821F|Np;=FElSZFTXrb!LhU?BfqE!R4@KwRM2F<#gUPk zn39v3ms$jB>VpR$Qg5*(f|cE3hniDl4sw7{acW*kW^QU;Nqk~nVoqgoX7NjqL7GxU zTA)@ks9J@Zms(t01ggbvvAbm!rKaUq++t2oExEZA*d|c7q$D+^2vqal;&S$J3=WR>bM$q+#pzm+ zoC=QpB2c|~iyIVI@x}S2MaikRI6&>3cu+%!1;o0=k)B!-4{qMD78m4XmJ~UIJj9k* zP>`CJ0&dFOVgY&W7He@%YHGnPmcr7^5>5UhE3h)i*w8IbQ0pGt0*6|bSDKrYT6Bvu zH?blfWP9-~#?)IZ#rdTLw>V%40^E@)@&SbpD1eK6K`eg|;RiCC6J%>~Nn&n6F{sW2 zH{}roq)pEc=|;xqXXb#%7K(B}{UlJ+Yauge@Zc_}32b(e#rz73`2`m9r+lKFnLU{e z?l;7x8{Dt3h}`8ByTU8Cp!@=_{zYE>E4=y*ZVv>68a!_Z2{(A(0I?o$iO!I`#HDnR zOX&dzUq4qD*VLQ~!qyjrtyf4x=nKNu7ddRMaM)bnu=&8k#wm1{M{Qs(#LM@Efr*#z1DyErm5o6_@&f}C zrx1d;%OiL}DByth1)%^4y2ulFg(vU=Pv8w6!46jt>~#GB>I$$jaSHwT@#8B8gMjRe zuq(3q8)Pr?+h5_g?_hZ#EI!?KlI;S~3;GTdY%dBsUJ-We;JCpr4z>qk#foZ(`4@x& z;8xt^5quyn|A9e}Q}`o@=y1Q0oOL}p|59@Pg@Td~3=DOQVN4g2^FJ~$q%ejveSuIP z7#JiHnI@D^ubEV{z~zFW=Y*Pz;$Bz8y{?M~UlI?#5EXYpJout`{1x%|3*zw~7#L!i z5}7(YZdg0*x7lTLg84#F*bbYE*5OyI!!HOYPB5QtGs$KF^9q#-HW!8UuL$dJV7nr0 zenHs$LTqA(%Z=ES>#>=aVlyvf<$hpbC};Fzx)7WBk%1wO(I4a>5cPq9K_rq1X3zy4 zXOKxQSA<=z3;SLY_Pr1iaY5MkqHyFD;m8ZZkslZsLYX3&I$V(aTKs{5p_VZm?$-#A z(J71(Odl8+#FLmNq)*S8l(WG2g1+a3oQq;!SH!%oiv?d21N&7h_@Y?+6|wjWVj#c9 zF(ol|xZf}_-(j-BdxQ6ckm*sAq82bujGq#}An1yy#&uD>OQL!gMGdZq8e9-HxZoPv z;daB!a!1I9>boq3rL^RhY%F{5;TKqV-s*!5D?@N-&A&i(>DGmVioA zuu4$UL~U5sG9dRoQLEls#u9F*HgKn;h5=PCXeuC>L6ga^iWgS?=z#`|G}($k1y&KL zEq{w8D?c;u7Ke*th-18qXAro60uMfDGJ!`nic&$P6{xOIP=J(a+(o5%urbo20tN<# z1ds|ySMsia&=mIRJd=3lMl7hmC}4g?z`Vipu7LbS0mUlVxWK$FlD*Ks}icN|@br&Qg6G4@NA&3YC5ukuA z$^)rmOHVCHE>9`S2ML3EHMcl&6SGrOGK-2KLrb7x9&jZF33yO&g2&RpK>`l%UmP~M z`6;D2sdhz`3=9mQ)>5%4BLl++W=2NF8w}hHV0eQ;?gA>h!C-a)72ROazJQ8uFqmIJ zMK>56E})_t46+wc(G3R91~5dj9n5dwe8BJ0z;lC}x5vESrpsmq=M1JfoHM!Sa4!hE z$gOjQTc?5J27I_Y<08BK6?XX#42+CYH(2-^oLamma9m`Oyuu=RfkpBI3o9ev2L?F7 m$Hi#)fdNiPFf*!rV1N@`ri`K=7~n)6BQv8qNE)302P*&s#+Uz`(HTk5ak;7X!m%5C?`?pp4H#j0_CZ8B!Qh7;_kM8KW2(L2RZRrd;MI zW-yyMhb5OaiWSUe$zjW7k75V2S#vmYIitA1Y_=TkT%IT%Fq=JxH;ONXA%#7MKZ-wv zBT68ZKT8m58^Z!27z>$ZVo2rB5{8K^V_;xd4dXDRa4utFU|7ur6NnN)F;5gGj!dVD zEMsI~Sj`9%D;5CTAO_>3%QG^haJ4W*iKlS4FhohDum>|}^1K8o)?~cJS)Q7dQj}Si zTBON%i`~DVBr`v+Sd;MpwUV590`WZ+~d z;YY~UFs87e#_$3Tge-zp0#6?d3|ZiG4U#K?$0-9t7TnGh)-)zmTTyw840*oF3?)(^ z6ChZHp~#_@2_^j4(Dl?XF5rWxMkP_hl98cA9wDD40Ar)7N@j{=NMWdDs$&X<(&dch z3>D0g4CRa^$}l|)3^j~d@EnuEjv7w2%*48=hIs)}=pjr`;YeX!gOKWsG5BV5nuQWv^jhfE0%aT`4>%yy$K# z(k4aq0&#?C2v!Qy8a`AX1{kRnmTHJf2B2WprxGSqMs$<}Zd zN!M@{N!G9~kbrv;iBZE*!&$?XA~^>meN15NVaA?MYFJTis^zZX4rb7l@~hGcPA*C; z$jnPu$j{7C$W1KDPAyT$OD!)}C@ltw2B+qv=4F=V>gnl)fXeE8NLl?-gn@zK7JqVL zNosn2QKe&1NoI0RYL%*MQBi)8m4b7AUP)?Ri9$+dib7s~i9$|(Vv0gRQGP*cQBI{^ z6?aBSZjN3-Vo`Bw(Jkhb%rZ^JTWrZWiN(e7RcvngrFkg|Ra^>*5JQXgZZTyT-4a8y zD?Yv?vm_@~lkpa7aY<2T!7au_O~zX+8AYjSMPSD<7TglW(36{5l32w7b*{ooP|;q+ ztDvjkk(!v2lbM%lrBKD8qu`U7mknXLB$lLFDQGg^;>duixW$r_nU@W&Hc~YuZ?P5> zW#*L>fy#5Jn$+UrB2cApi`^}=C^ap=;udpyYRN4ge@~xyKiBZ!_|PDqA~sO)v6kkQ zWaivra&^8XT~L%-oSK)M8lRsQpPG}Jo0?Y=pOX)Ae@YRkGPuR%?Bf_59Pj7o>w1gR zwIVqc91*uz3QIFfZZQ?4+~RaeEJ<`LO3Y0y0+sc*xC;`~Q{#*CON)|IZ*in$=B30Z z=H%RB&PmO?#hIE{nwwgbSdw~+1tfEeBR#bwz9hAxiY8AHs2;q<2C1fR zF{a*PDb6n~xWxgBmRp>lWD*amK5sE*-eQM}7l9IO5h!yOfvSRAOetx_pgKeW3H(ac z&&bbB)z8jJ&CJtJ&QD1V57rMT%FjwoE-BXc%}hxw4oOYU$ji^kPtQy()(81a*Ev5Y zCp8&V_v?qJCV`V}YLR|1DEX!q73=3`=EQ>?1QpRMsJz7wVaLPdiYyoy7(hjT@iis} zh6aYaESwj(%r3H+Utux7z+(QDje%dRGqWeN!TpA~bc6d97LmKWVpn+O7L;G$)xXHA ze}z}S!R-MHSG!k}S8vTl7MUw7G8b56KCrQ~%H8D$JxAg0;^DZQqMLeph(>J~A_La(!W7;^g`OCw_coW8e|}z`)EZham29@Lu3E zy~ts9g~RLuhuIAdzJBg5?oQqw-VWZo{DM6-opn8R7dT|m(2pNKetckOW|jN#7Vpk-!SHxcAw!Xq`-QaSAg{$4W$-B|F#kaxt0~>>YNPlft?TpY1+BO}v7x`_k z@Y^~hV5njYV!9BY{gHtofialr3xxW>z#tOKG{Jnj z%_N%zEEjZLC)ivRcDo|%c3s&2lCb}Uu&4{d{uhO#uLwt95RU%9z!1q4%hbVf!`yCr z&8C_I&L>hf)Lb+VzG5DHfj_pRyuYTaW`^^El#ZH<{MuLewO6=Z;Wxg(Z+sy#wu9wH zWWx2x)Ju`67t%66FfbG|dNEyyO#R5f5XI;Xau105z`!67$^Gl80K&V$BYK@j<`R$04H=ai5;8X= zWInL7$nkw(V3Fhd$iTwO{{=*R01-dG@-T4m^vF!eyvV6=g;Sxy=>|7%gJ&_Qp>$Bx zkM{kRsza*)Au2T4{Hl0hg@aylahV&aI?2pSNv)_7_s`6M zRD_@sPN6ulEHy=;BwwLQ+b>@MEL5IYtNba|=o; z^)xxaMb0gz{Nh_I1&JjYw^*|BGxKh7xHyJ5#=Ce1-C_kRyv0_MAD>)YRwT{9z@W)= ziyd5&rKH{B1eq5P(pd}&Zv}-Sb5PB}T~wME53#$b$eDqG0fdW77{G-ZpX^0G`73<# z4eoaZgr=}h=b6MaH)28EMFF!b0%i@K54eSTY(Frtu-e|>;P2<_;+s&pg0+M1B8T1; z4!sK;df;Nmcm?MLKI02~#uqtEu5g%KfT5?LGRgWPhs_lZn+q`X85C!llD9aDp^XJd z{ig)u>cM+^dTEd