68 lines
2.1 KiB
Python
68 lines
2.1 KiB
Python
# scrapers/oil_news_scraper.py
|
|
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
import pandas as pd
|
|
import os
|
|
|
|
# URL for OilPrice.com homepage
|
|
OIL_NEWS_URL = "https://oilprice.com/Latest-Energy-News/World-News/"
|
|
|
|
# Define the directory to store the scraped data
|
|
DATA_DIR = os.path.join(os.getcwd(), "data")
|
|
if not os.path.exists(DATA_DIR):
|
|
os.makedirs(DATA_DIR)
|
|
|
|
# Function to scrape news headlines from OilPrice.com
|
|
def scrape_oil_news():
|
|
print("Scraping oil market news...")
|
|
|
|
# Send an HTTP request to the website
|
|
response = requests.get(OIL_NEWS_URL)
|
|
response.raise_for_status()
|
|
|
|
# Print the HTML to see what we are working with
|
|
print(response.text[:1000]) # Print only the first 1000 characters for brevity
|
|
|
|
# Parse the HTML using BeautifulSoup
|
|
soup = BeautifulSoup(response.text, "html.parser")
|
|
|
|
# Find all news article containers (class names updated)
|
|
articles = soup.find_all('div', class_='categoryArticle')
|
|
|
|
# List to store the scraped data
|
|
news_data = []
|
|
|
|
# Loop through each article container
|
|
for article in articles:
|
|
# Extract the headline, date, and link
|
|
headline = article.find('a').get_text(strip=True) if article.find('a') else None
|
|
link = article.find('a')['href'] if article.find('a') else None
|
|
date = article.find('span', class_='categoryArticle__date').get_text(strip=True) if article.find('span', class_='categoryArticle__date') else None
|
|
|
|
# Only append valid data
|
|
if headline and link and date:
|
|
news_data.append({
|
|
'headline': headline,
|
|
'link': f"https://oilprice.com{link}",
|
|
'date': date
|
|
})
|
|
|
|
df = pd.DataFrame(news_data)
|
|
return df
|
|
|
|
# Function to run the scraper and save data
|
|
def run_scraper():
|
|
# Scrape oil news
|
|
news_df = scrape_oil_news()
|
|
|
|
# Define the file path for saving the data
|
|
file_path = os.path.join(DATA_DIR, 'oil_news.csv')
|
|
|
|
# Save the DataFrame to a CSV file
|
|
if not news_df.empty:
|
|
news_df.to_csv(file_path, index=False)
|
|
print(f"Oil news data saved to {file_path}")
|
|
else:
|
|
print("No data was scraped. The CSV file is empty.")
|