From 736a7ece960f0109e7def55276e0b66c25aa6e79 Mon Sep 17 00:00:00 2001
From: klein panic <kleinpanice@proton.me>
Date: Wed, 30 Oct 2024 23:47:32 -0400
Subject: [PATCH] oil scraper works

---
 .../oil_news_scraper.cpython-311.pyc          | Bin 2985 -> 4411 bytes
 .../scrapers/backups/oil_news_scraper.py.bak  |  76 ++++++++++++++++++
 .../WebScraper/scrapers/oil_news_scraper.py   |  67 ++++++++-------
 3 files changed, 114 insertions(+), 29 deletions(-)
 create mode 100644 Data-Collection/WebScraper/scrapers/backups/oil_news_scraper.py.bak

diff --git a/Data-Collection/WebScraper/scrapers/__pycache__/oil_news_scraper.cpython-311.pyc b/Data-Collection/WebScraper/scrapers/__pycache__/oil_news_scraper.cpython-311.pyc
index 3639622e1d27508e35b324fc8349830d8241d46e..cdf37109cf90a3593c722bc6bd05c14fbc8ac6cb 100644
GIT binary patch
literal 4411
zcmZ3^%ge>Uz`(HTk5ak;7X!m%5C?`?pp4H#j0_CZ8B!Qh7;_kM8KW2(L2RZRrd;MI
zW-yyMhb5OaiWSUe$zjW7k75V2S#vmYIitA1Y_=TkT%IT%Fq=JxH;ONXA%#7MKZ-wv
zBT68ZKT8m58^Z!27z>$ZVo2rB5{8K^V_;xd4dXDRa4utFU|7ur6NnN)F;5gGj!dVD
zEMsI~Sj`9%D;5CTAO_>3%QG^haJ4W*iKlS4FhohDum>|}^1K8o)?~cJS)Q7dQj}Si
zTBON%i`~DVBr`v+Sd;M<lT)Q8<1OCs)Fc<MY<Oa3i6-MM;na$P)Z~)Xl=$TQyp&9s
zYF?+*#L|+?w9=g5{L%uyD&vfjl7eC@ef|8*oPwgv<W#-n{9JvX#FEtF5?$B4)S~oC
zUBA@wV*T*^qMQ^6^A<}=Vo72$$XzhZ&A`CG%)r3#`3?&x7E9nhW?)ERL=6K*h7yoU
zs7}-{u4PDJs$qzShj9rzR2c&ULl!KhFw`)tVMaBxma&F$0V_f+f>pwUV590`WZ+~d
z;YY~UFs87e#_$3Tge-zp0#6?d3|ZiG4U#K?$0-9t7TnGh)-)zmTTyw840*oF3?)(^
z6ChZHp~#_@2_^j4(Dl?XF5rWxMkP_hl98cA9wDD40Ar)7N@j{=NMWdDs$&X<(&dch
z3>D0g4CRa^$}l|)3^j~d@EnuEjv7w2%*48=hIs)}=pjr`;YeX!gO<N)S<)G5Sy4iQ
zt3(r_r-lVJTxK(*aL+|c7mN%g+PHPNFic>KWsG5BV5nuQWv^jhfE0%aT`4>%yy$K#
z(k4aq0&#?C2v!Qy8a`AX<T0i&)N<f7nLmYTjR0!M7s-;MIz><tZavs8PMo@hQiMtH
zGZxh(`I!qPmPKmVQbem57#L<Vq=?mU%w|Xtui>1{kRnmTHJf2B2WprxGSqMs$<}Zd
zN!M@{N!G9~kbrv;iBZE*!&$?XA~^>meN15NVaA?MYFJTis^zZX4rb7l@~hGcPA*C;
z$jnPu$j{7C$W1KDPAyT$OD!)}C@ltw2B+qv=4F=V>gnl)fXeE8NLl?-gn@zK7JqVL
zNosn2QKe&1NoI0RYL%*MQBi)8m4b7AUP)?Ri9$+dib7s~i9$|(Vv0gRQGP*cQBI{^
z6?aBSZjN3-Vo`Bw(Jkhb%rZ^JTWrZWiN(e7RcvngrFkg|Ra^>*5JQXgZZTyT-4a8y
zD?Yv?vm_@~lkpa7aY<2T!7au_O~zX+8AYjSMPSD<7TglW(36{5l32w7b*{ooP|;q+
ztDvjkk(!v2lbM%lrBKD8qu`U7mknXLB$lLFDQGg^;>duixW$r_nU@W&Hc~YuZ?P5>
zW#*L>fy#5Jn$+UrB2cApi`^}=C^ap=;udpyYRN4ge@~xyKiBZ!_|PDqA~sO)v6kkQ
zWaivra&^8XT~L%-oSK)M8lRsQpPG}Jo0?Y=pOX)Ae@YRkGPuR%?Bf_59Pj7o>w1gR
zwIVqc91*uz3QIFfZZQ?4+~RaeEJ<`LO3Y0y0+sc*xC;`~Q{#*CON)|IZ*in$=B30Z
z=H%RB&PmO?#hIE{nwwgbSdw~+1tfEeBR#bwz9hAx<Q7|EK|yL>iY8AHs2;q<2C1fR
zF{a*PDb6n~xWxgBmRp>lWD*amK5sE*-eQM}7l9IO5h!yOfvSRAOetx_pgKeW3H(ac
z&&bbB)z8jJ&CJtJ&QD1V57rMT%FjwoE-BXc%}hxw4oOYU$ji^kPtQy()(81a*Ev5Y
zCp8&V_v?qJCV`V}YLR|1DEX!q73=3`=EQ>?1QpRMsJz7wVaLPdiYyoy7(hjT@iis}
zh6aYaESwj(%r3H+Utux7z+(QDje%dRGqWeN!TpA~bc6d97LmKWVpn+O7L;G$)xXHA
ze}z}S!R-MHSG!k}S8vTl7MUw7G8b56KCrQ~%H8D<osc#o=^}^X6%NJg9O{=i)bH}}
zUE$GM>$JxAg0;^DZQqMLeph(>J~A_La(!W7;^g`OCw_coW8e|}z`)EZham29@Lu3E
zy~ts9g~RLuhuIAdzJBg5?oQqw-VWZo{DM6-opn8R7dT|m(2pNKetckOW|jN#<HuJn
z1|G>7Vpk-!SHxcAw!Xq`-QaSAg{$4W$-B|F#kaxt0~>>YNPlft?TpY1+BO}v7x`_k
z@Y^<c+z=3I@O%mi9-XPi6DlU!PO-hfr2|11S#+<k=w4vaeJUoufa9W=<`pr`4-EXQ
z{2xI?Cszk!hxvo}wCnNNm*TT8<P>~hV5njYV!9BY{gHtofialr3xxW>z#tOKG{Jnj
z%_N%zEEjZLC)ivRcDo|%c3s&2lCb}Uu&4{d{uhO#uLwt95RU%9z!1q4%hbVf!`yCr
z&8C_I&L>hf)Lb+VzG5DHfj_pRyuYTaW`^^El#ZH<{MuLewO6=Z;Wxg(Z+sy#wu9wH
zWWx2x)Ju`67t%66FfbG|dNEyyO#R5f5XI;Xau105z`!67$^<j$f`&cFB!?^f4%hiT
zFY$X`2ne~r?|G3w^a_9I1^&<v3=Dxxp-detU)dNq`7ZDU&JdhoI8*onU*HA4z!eq{
zdPfL^K4Ai(FLDH3;Rw3G5%htLfk)&611l&03>Gl80K&V$BYK@j<`R$04H=ai5;8X=
zWInL7$nkw(V3Fhd$iTwO{{=*R01-dG@-T4m^vF!eyvV6=g;Sxy=>|7%gJ&_Qp>$Bx
zk<m$<;gC3&vm)~$MN?-P=EGu)&Wg;3WmugRnNga%prQ*@UVVPYfZp6iZC=$fl)xLa
z3=COt_5zS@qzVecECCfl5GJVEjMCkxWvpSy0+niD`C{!Fh6SJk5zIp-Y8X?P)-W$a
z(aVU^&`n{fVVun{7p-9m>M{kRsza*)Au2T4{Hl0hg@aylahV&aI?2pSNv)_7_s`6M
zRD_@sPN6ulEHy=;BwwLQ+b>@MEL5IYtN<xsQuGu;GEx<sgToZkGILTDGK&>ba|=o;
z^)xxaMb0gz{Nh_I1&JjYw^*|BGxKh7xHyJ5#=Ce1-C_kRyv0_MAD>)YRwT{9z@W)=
ziyd5&rKH{B1eq5P(pd}&Zv}-Sb5PB}T~wME53#$b$eDqG0fdW77{G-ZpX^0G`73<#
z4eoaZgr=}h=b6MaH)28EMFF!b0%i@K54eSTY(Frtu-e|>;P2<_;+s&pg0+M1B8T1;
z4!sK;df;Nmcm?MLKI02~#uqtEu5g%KfT5?LGRgWPhs_lZn+q`X85C!llD9aDp^XJd
z{ig)u>cM+^dTEd<Q4iXTfYh5ZSQIAb=jP_;=_OS{YCl;lN{UMh3i69e^hz^}K&j`J
z6mB&|puSg;A}HvXlZs6s6(n0hVqQvOagiDW14EHDhyZ1sA{!9P7NnFdJ+&mcJf+AE
z%t@`tEG{X&#gUtsotlzaR1A(;aNw_GC~^U50!7U)4x8Nkl+v73yP^OF1_n?Aqxc^K
z1H%VqMn=XP400Dx(G3Q(3#jM@gZ2efbc4bC0xG(};BWyI-C&TtfQoJ~2w#Aq8w{Kc
zU<fMiT^SpAK*bMVk9ogMm(2{$8BB9HXL8TsUJ!PXTjvV5P6NjccFvaa_PVCJ2^kmJ
z<*%^Ie_&u_l)Ayf-{92ZJ%QsQi{up+$qOu!A6WDm`93hfi6ll2M%xbzm;^X{0N~9X
A*Z=?k

delta 1741
zcmdn3v{IaJIWI340|Ns?=y&n-X&e*zB#iDcFfdGKNMT4}%wdQE(M&mvxlB<^j372+
z4s#R>SdKM{HHA5fEtNHkeewn-IW{JSRMsrci5tY_S(Y&|Fsx<*nF_*DTp%?d76>ok
zo}9=k9K(>xwTzL0VKpO241|jXzy|Pu*dUB9&&ZI%+QJaUo5I$@5XG0m7|fu_{t~3n
zPm}Q$uTyGbX-Q^UX-;r{X~E>#j2_(V3=9m+3=9mPzc5XH$Ee4X1$R#g+vGYX3AS2>
z8ioZdAjeGp$t);e0(S`mLk&X~E68XNPGLx4YGsm~T+bw_SHcStfnXFh%&0!DWvpRb
zzz#B6ogsxKg>?-ZnmkiFLoIU+lM6%aty-2EmIWLjwc-pZ?5JT<qzRJ~#*kaU3DP6W
zkixNs6IE9pV+unpD@-RAQ>9^Y>KLY`a7jYE$H2e<Hir$SR+=G&JB0_`Cq>#Yd2y`r
z5LasAawU5WYmq_?TajE1dyz~H^8zlApF$XFSW|dw*k&`#WvXFM;hV#Z>OV$?p4%nj
zaAz`P!P86)n#x*^8jfHFO@6;B>B$$_B<l6_^fF&CGcZ(fXO!gT=oKUu6{i;6Vou2{
zyTzZJSdyBaUsUN>RFavTld8#hi!C`PvA8(?7Gq+FCgUyE;*z4wf?F&ZMX71GSc(f0
z^KOZv>4}d|Ni0dN63HkjDJZtm*U!()DJaTJPSs1!&(&nU#gUkinwXN4nU{KtB_}g4
z`xXmGxhC%|)`FtUypmfSMX80Qsl_G5x0usYOK$P_d-}xtxrPVFhX(oF5-3W{EKZG2
z%P)#AE=epYExyH4l3Gzx1WMw!IMOonQsNVHawcEkkdV5?RFHCu(<QMa(XA*kH&v4p
zWO8vqeqQn9pB%CZ?1>N`7V$AKFck5F2muBLhFeT2X~moj3=ETPIGgHK7#J7?85kIf
zw=gm=G%(y{;k>}5evw7v3X8@C7LBiL3<458HFJF~@@rn<*KF{(!NPN$MfMVl>_ry&
zD=hLCSmeL5F|cxVX!PjLV4Wc{gY^oR9I_OT<P3>9vMYEma+_V@HfwPCz{VgT(qG$E
zJ0tXhwoOOv<ojG4^&i+61ce(sKd>=~$b4YnVdV!C9qbQmUG~T9iaDWt(bn&Zt=|RV
zhzaJ?Z6?_)V7@4<az$9>y0GpgVciWb7ld^$3R_+gw!9#0c_AdCgX4yd@p_+CK0Aaj
z>eyb<vAw|W-cjCP(^WIW`69o<6@G;UVpsSzF7Rs{FzH~q5fhnwJtpH)OvZ)G{0|Ha
zm5c#Q7h*C#GBCt41~PqtP#+i=1frRshPYkOcIv3P$nSiG-}ySf&n14J3&G(R_<b(&
zM_l2LxWFIrfq@~MDVnK+WwJYu8mH-sk_!^17bHw4=kdtM{P@blz{S%eGa>ULr@|FZ
zg$Ab^+`J8*pFyd7^HH80Mn;v%Mtr)Af|C>Y6g`wd@x@(KnipT3T$EUlT2y4fz`$S*
zimwU=P<%b*lfB3%e}zxJ!Tl>J6sNFH=b6MaH)28EMFF!b0%qV)<q@8Il+VAuNR@$s
zL6g5o2xKU8Qn3jnO|cau=A|SS7fFK>hz5ud0TG%YLJOpnEj_g)xjdyv8_Y?q$ONTa
zj@-oT)RfGk;v!HA0y}FZ1K5TlkfVQb*yQG?l;)(`6@gp}Dsn}O*E29Md|+l|Wc<L!
iz$nnb4}z1Y3zX}!DKqkYV1N@oj696C9~dwRu=xNYyTAni

diff --git a/Data-Collection/WebScraper/scrapers/backups/oil_news_scraper.py.bak b/Data-Collection/WebScraper/scrapers/backups/oil_news_scraper.py.bak
new file mode 100644
index 0000000..344a3c9
--- /dev/null
+++ b/Data-Collection/WebScraper/scrapers/backups/oil_news_scraper.py.bak
@@ -0,0 +1,76 @@
+from selenium import webdriver
+from selenium.webdriver.firefox.options import Options
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from bs4 import BeautifulSoup
+import pandas as pd
+import os
+
+# URL for OilPrice.com homepage
+OIL_NEWS_URL = "https://oilprice.com/Latest-Energy-News/World-News/"
+
+# Set up the data directory
+DATA_DIR = os.path.join(os.getcwd(), "data")
+if not os.path.exists(DATA_DIR):
+    os.makedirs(DATA_DIR)
+
+def scrape_oil_news():
+    print("Scraping oil market news using Selenium...")
+
+    # Set up Selenium options
+    options = Options()
+    options.headless = True
+    driver = webdriver.Firefox(options=options)
+
+    driver.get(OIL_NEWS_URL)
+
+    # Wait until 'categoryArticle' elements load
+    try:
+        WebDriverWait(driver, 20).until(
+            EC.presence_of_element_located((By.CLASS_NAME, "categoryArticle"))
+        )
+    except Exception as e:
+        print("Error: Content did not load properly.")
+        driver.quit()
+        return pd.DataFrame()
+
+    soup = BeautifulSoup(driver.page_source, "html.parser")
+    driver.quit()
+
+    # Parse the articles
+    articles = soup.find_all('div', class_='categoryArticle')
+    news_data = []
+
+    print(f"Found {len(articles)} articles.")
+
+    for i, article in enumerate(articles):
+        # Extract the title, link, and date using the adjusted structure
+        headline = article.find('h2', class_='categoryArticle__title').get_text(strip=True) if article.find('h2', class_='categoryArticle__title') else None
+        link = article.find('a', href=True)['href'] if article.find('a', href=True) else None
+        date = article.find('p', class_='categoryArticle__meta').get_text(strip=True) if article.find('p', class_='categoryArticle__meta') else None
+
+        # Log each article's details for debugging
+        print(f"Article {i+1} - Headline: {headline}, Link: {link}, Date: {date}")
+
+        # Only add valid entries
+        if headline and link and date:
+            news_data.append({
+                'headline': headline,
+                'link': link,  # Assuming the link is already a full URL
+                'date': date
+            })
+
+    df = pd.DataFrame(news_data)
+    return df
+
+def run_scraper():
+    news_df = scrape_oil_news()
+    file_path = os.path.join(DATA_DIR, 'oil_news.csv')
+
+    if not news_df.empty:
+        news_df.to_csv(file_path, index=False)
+        print(f"Oil news data saved to {file_path}")
+    else:
+        print("No data was scraped. The CSV file is empty.")
+
diff --git a/Data-Collection/WebScraper/scrapers/oil_news_scraper.py b/Data-Collection/WebScraper/scrapers/oil_news_scraper.py
index b0791d2..344a3c9 100644
--- a/Data-Collection/WebScraper/scrapers/oil_news_scraper.py
+++ b/Data-Collection/WebScraper/scrapers/oil_news_scraper.py
@@ -1,6 +1,8 @@
-# scrapers/oil_news_scraper.py
-
-import requests
+from selenium import webdriver
+from selenium.webdriver.firefox.options import Options
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
 from bs4 import BeautifulSoup
 import pandas as pd
 import os
@@ -8,60 +10,67 @@ import os
 # URL for OilPrice.com homepage
 OIL_NEWS_URL = "https://oilprice.com/Latest-Energy-News/World-News/"
 
-# Define the directory to store the scraped data
+# Set up the data directory
 DATA_DIR = os.path.join(os.getcwd(), "data")
 if not os.path.exists(DATA_DIR):
     os.makedirs(DATA_DIR)
 
-# Function to scrape news headlines from OilPrice.com
 def scrape_oil_news():
-    print("Scraping oil market news...")
+    print("Scraping oil market news using Selenium...")
 
-    # Send an HTTP request to the website
-    response = requests.get(OIL_NEWS_URL)
-    response.raise_for_status()
+    # Set up Selenium options
+    options = Options()
+    options.headless = True
+    driver = webdriver.Firefox(options=options)
 
-    # Print the HTML to see what we are working with
-    print(response.text[:1000])  # Print only the first 1000 characters for brevity
+    driver.get(OIL_NEWS_URL)
 
-    # Parse the HTML using BeautifulSoup
-    soup = BeautifulSoup(response.text, "html.parser")
+    # Wait until 'categoryArticle' elements load
+    try:
+        WebDriverWait(driver, 20).until(
+            EC.presence_of_element_located((By.CLASS_NAME, "categoryArticle"))
+        )
+    except Exception as e:
+        print("Error: Content did not load properly.")
+        driver.quit()
+        return pd.DataFrame()
 
-    # Find all news article containers (class names updated)
+    soup = BeautifulSoup(driver.page_source, "html.parser")
+    driver.quit()
+
+    # Parse the articles
     articles = soup.find_all('div', class_='categoryArticle')
-
-    # List to store the scraped data
     news_data = []
 
-    # Loop through each article container
-    for article in articles:
-        # Extract the headline, date, and link
-        headline = article.find('a').get_text(strip=True) if article.find('a') else None
-        link = article.find('a')['href'] if article.find('a') else None
-        date = article.find('span', class_='categoryArticle__date').get_text(strip=True) if article.find('span', class_='categoryArticle__date') else None
+    print(f"Found {len(articles)} articles.")
 
-        # Only append valid data
+    for i, article in enumerate(articles):
+        # Extract the title, link, and date using the adjusted structure
+        headline = article.find('h2', class_='categoryArticle__title').get_text(strip=True) if article.find('h2', class_='categoryArticle__title') else None
+        link = article.find('a', href=True)['href'] if article.find('a', href=True) else None
+        date = article.find('p', class_='categoryArticle__meta').get_text(strip=True) if article.find('p', class_='categoryArticle__meta') else None
+
+        # Log each article's details for debugging
+        print(f"Article {i+1} - Headline: {headline}, Link: {link}, Date: {date}")
+
+        # Only add valid entries
         if headline and link and date:
             news_data.append({
                 'headline': headline,
-                'link': f"https://oilprice.com{link}",
+                'link': link,  # Assuming the link is already a full URL
                 'date': date
             })
 
     df = pd.DataFrame(news_data)
     return df
 
-# Function to run the scraper and save data
 def run_scraper():
-    # Scrape oil news
     news_df = scrape_oil_news()
-
-    # Define the file path for saving the data
     file_path = os.path.join(DATA_DIR, 'oil_news.csv')
 
-    # Save the DataFrame to a CSV file
     if not news_df.empty:
         news_df.to_csv(file_path, index=False)
         print(f"Oil news data saved to {file_path}")
     else:
         print("No data was scraped. The CSV file is empty.")
+