added Adv Historical Module2 Test

src/MidasV1/tests/AdvHistoricalModule2Test/backups/test_historical_approach.py.bak.1

@@ -0,0 +1,223 @@
+import os
+import sys
+import time
+import json
+import shutil
+import signal
+import logging
+import configparser
+from datetime import datetime, timedelta
+from decimal import Decimal
+import pytz
+import threading
+import csv
+
+from ibapi.client import EClient
+from ibapi.wrapper import EWrapper
+from ibapi.contract import Contract
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+# Load config
+config = configparser.ConfigParser()
+if not os.path.exists('config.ini'):
+    print("config.ini not found. Please create one.")
+    sys.exit(1)
+
+config.read('config.ini')
+HOST = config.get('API', 'host', fallback='127.0.0.1')
+PORT = config.getint('API', 'port', fallback=4002)
+CLIENT_ID = config.getint('API', 'clientId', fallback=1)
+DATA_DIR = config.get('Directories', 'data_dir', fallback='./data')
+TICKER_FILE = config.get('General', 'ticker_file', fallback='ticker_ids.csv')
+MIN_VOLUME = config.getint('Thresholds', 'min_volume', fallback=1000000)
+
+if not os.path.exists(DATA_DIR):
+    os.makedirs(DATA_DIR)
+
+if not os.path.exists(TICKER_FILE):
+    print(f"{TICKER_FILE} not found. Please create one with a list of symbols.")
+    sys.exit(1)
+
+class IBWrapper(EWrapper):
+    def __init__(self):
+        super().__init__()
+        self.current_time = None
+        self.current_time_received = threading.Event()
+
+        self.historical_data_reqId = None
+        self.historical_bars = []
+        self.historical_data_received = threading.Event()
+
+    def error(self, reqId, errorCode, errorString, advancedOrderRejectJson=""):
+        logging.error(f"Error. ReqId: {reqId}, Code: {errorCode}, Msg: {errorString}")
+
+    def currentTime(self, time_):
+        self.current_time = time_
+        self.current_time_received.set()
+
+    def historicalData(self, reqId, bar):
+        if reqId == self.historical_data_reqId:
+            self.historical_bars.append(bar)
+
+    def historicalDataEnd(self, reqId, start, end):
+        if reqId == self.historical_data_reqId:
+            self.historical_data_received.set()
+
+class IBClient(EClient):
+    def __init__(self, wrapper):
+        EClient.__init__(self, wrapper)
+
+
+class IBApp(IBWrapper, IBClient):
+    def __init__(self):
+        IBWrapper.__init__(self)
+        IBClient.__init__(self, wrapper=self)
+        self.connect_error = None
+
+    def connect_app(self):
+        try:
+            self.connect(HOST, PORT, CLIENT_ID)
+        except Exception as e:
+            self.connect_error = e
+
+        thread = threading.Thread(target=self.run, daemon=True)
+        thread.start()
+        # Wait a bit for connection
+        time.sleep(2)
+
+    def disconnect_app(self):
+        if self.isConnected():
+            self.disconnect()
+
+def signal_handler(sig, frame):
+    print("Interrupt received, shutting down...")
+    sys.exit(0)
+
+signal.signal(signal.SIGINT, signal_handler)
+
+def is_trading_hours(now_eastern):
+    # Trading hours: Monday-Friday, 9:30 - 16:00 Eastern
+    if now_eastern.weekday() > 4:  # Saturday=5, Sunday=6
+        return False
+    open_time = now_eastern.replace(hour=9, minute=30, second=0, microsecond=0)
+    close_time = now_eastern.replace(hour=16, minute=0, second=0, microsecond=0)
+    return open_time <= now_eastern <= close_time
+
+def get_symbols_from_file(file_path):
+    symbols = []
+    with open(file_path, 'r') as f:
+        csv_reader = csv.reader(f)
+        for row in csv_reader:
+            # Extend the symbols list with all symbols in the current row
+            symbols.extend([symbol.strip() for symbol in row if symbol.strip()])
+    return symbols
+
+
+def request_historical_data(app, symbol):
+    contract = Contract()
+    contract.symbol = symbol
+    contract.secType = "STK"
+    contract.exchange = "SMART"
+    contract.currency = "USD"
+
+    app.historical_bars = []
+    app.historical_data_received.clear()
+    app.historical_data_reqId = 10000  # arbitrary reqId
+
+    endDateTime = datetime.now().strftime("%Y%m%d %H:%M:%S") + " UTC"
+    try:
+        app.reqHistoricalData(
+            reqId=app.historical_data_reqId,
+            contract=contract,
+            endDateTime=endDateTime,
+            durationStr="2 D",
+            barSizeSetting="1 day",
+            whatToShow="TRADES",
+            useRTH=1,
+            formatDate=1,
+            keepUpToDate=False,
+            chartOptions=[]
+        )
+    except Exception as e:
+        logging.error(f"Error requesting historical data for {symbol}: {e}")
+        return None
+
+    if not app.historical_data_received.wait(timeout=10):
+        logging.warning(f"Timeout waiting for historical data for {symbol}.")
+        return None
+
+    return app.historical_bars
+
+def serialize_decimal(obj):
+    if isinstance(obj, Decimal):
+        return float(obj)
+    raise TypeError(f"Type {type(obj)} not serializable")
+
+def main():
+    app = IBApp()
+    app.connect_app()
+
+    if app.connect_error:
+        logging.error(f"Failed to connect to IB API: {app.connect_error}")
+        sys.exit(1)
+
+    app.reqCurrentTime()
+    if not app.current_time_received.wait(timeout=10):
+        logging.error("Timeout waiting for current time.")
+        app.disconnect_app()
+        sys.exit(1)
+
+    ib_server_time = datetime.utcfromtimestamp(app.current_time)
+    eastern = pytz.timezone("US/Eastern")
+    ib_server_time_eastern = ib_server_time.replace(tzinfo=pytz.utc).astimezone(eastern)
+
+    if is_trading_hours(ib_server_time_eastern):
+        logging.info("It's currently within trading hours. No historical retrieval needed.")
+        app.disconnect_app()
+        sys.exit(0)
+    else:
+        logging.info("Outside trading hours. Proceeding with historical data retrieval.")
+
+    symbols = get_symbols_from_file(TICKER_FILE)
+    logging.info(f"Loaded {len(symbols)} symbols from {TICKER_FILE}.")
+
+    raw_data = []
+    for symbol in symbols:
+        bars = request_historical_data(app, symbol)
+        if bars is None or len(bars) < 2:
+            logging.info(f"Skipping {symbol}: not enough data.")
+        else:
+            last_bar = bars[-1]
+            prev_bar = bars[-2]
+            net_change = last_bar.close - prev_bar.close
+            percent_change = (net_change / prev_bar.close) * 100 if prev_bar.close != 0 else 0.0
+
+            entry = {
+                "symbol": symbol,
+                "date": last_bar.date,
+                "open": float(last_bar.open),
+                "high": float(last_bar.high),
+                "low": float(last_bar.low),
+                "close": float(last_bar.close),
+                "volume": int(last_bar.volume),
+                "net_change": float(net_change),
+                "percent_change": float(percent_change)
+            }
+            raw_data.append(entry)
+
+        time.sleep(0.5)
+
+    now_str = datetime.now().strftime("%Y%m%d_%H%M%S")
+    raw_filename = os.path.join(DATA_DIR, f"raw_stock_info_{now_str}.json")
+    with open(raw_filename, 'w') as f:
+        json.dump(raw_data, f, indent=4, default=serialize_decimal)
+    logging.info(f"Raw data saved to {raw_filename}")
+
+    app.disconnect_app()
+    logging.info("All done.")
+
+if __name__ == "__main__":
+    main()
+

src/MidasV1/tests/AdvHistoricalModule2Test/clean_tickes.py

@@ -0,0 +1,70 @@
+import os
+import pandas as pd
+
+def clean_tickers(data_dir):
+    # Define paths
+    input_file = os.path.join(data_dir, "us_tickers.csv")
+    output_dir = os.path.join(data_dir, "csv")
+    output_cleaned = os.path.join(output_dir, "cleaned_us_tickers.csv")
+    output_refined = os.path.join(output_dir, "refined_us_tickers.csv")
+    
+    # Check if input file exists
+    if not os.path.exists(input_file):
+        print(f"Error: {input_file} not found in {data_dir}. Please ensure the file is in the correct directory.")
+        return
+
+    # Create output directory if it doesn't exist
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+    # Load the CSV file
+    try:
+        tickers_data = pd.read_csv(input_file)
+    except Exception as e:
+        print(f"Error reading the file: {e}")
+        return
+
+    # Step 1: Initial Cleanup
+    relevant_columns = ['Ticker', 'Name', 'Primary Exchange', 'Type']
+    cleaned_data = tickers_data[relevant_columns]
+
+    # Filter rows with specific types (e.g., CS for common stock, ETF for exchange-traded fund)
+    valid_types = ['CS', 'ETF']
+    cleaned_data = cleaned_data[cleaned_data['Type'].isin(valid_types)]
+
+    # Drop rows with missing values in critical columns
+    cleaned_data = cleaned_data.dropna(subset=['Ticker', 'Name', 'Primary Exchange'])
+
+    # Save the cleaned data to a new file
+    cleaned_data.to_csv(output_cleaned, index=False)
+    print(f"Cleaned data saved to {output_cleaned}.")
+
+    # Step 2: Ask for further refinement
+    refine = input("Do you want to refine further to keep only Ticker IDs (formatted as CSV)? (yes/no): ").strip().lower()
+    if refine == "yes":
+        # Keep only the Ticker column and format as a single line CSV
+        refined_data = cleaned_data[['Ticker']]
+        ticker_list = refined_data['Ticker'].tolist()
+        
+        # Save as a single line CSV
+        with open(output_refined, 'w') as f:
+            f.write(','.join(ticker_list))
+        
+        print(f"Refined data saved to {output_refined} (formatted as a single-line CSV).")
+    else:
+        print("No further refinement done. Exiting.")
+
+def main():
+    # Define the data directory
+    data_dir = "data"  # Relative path to the data directory
+
+    # Ensure the data directory exists
+    if not os.path.exists(data_dir):
+        os.makedirs(data_dir)
+
+    # Run the cleaning process
+    clean_tickers(data_dir)
+
+if __name__ == "__main__":
+    main()
+

src/MidasV1/tests/AdvHistoricalModule2Test/config.ini

@@ -0,0 +1,17 @@
+[API]
+host=127.0.0.1
+port=4002
+clientId=1
+
+[Directories]
+data_dir=./data
+
+[General]
+ticker_file=ticker_ids.csv
+good_runtime_threshold=120
+
+[Thresholds]
+min_volume = 1000000
+min_percent_change = 0.2
+min_net_change = 0.5
+max_share_price = 1000.0

src/MidasV1/tests/AdvHistoricalModule2Test/polygon_io_retrieval.py

@@ -0,0 +1,76 @@
+import requests
+import csv
+import time
+import logging
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+# Polygon API configuration
+API_KEY = "RumpAnkjWlY69jKygFzMyDm5Chc3luDr"  # Replace with your Polygon.io API key
+BASE_URL = "https://api.polygon.io/v3/reference/tickers"
+OUTPUT_FILE = "data/us_tickers.csv"
+
+# API parameters
+params = {
+    "market": "stocks",  # US stock market
+    "active": "true",    # Only active tickers
+    "limit": 1000,       # Max limit per request
+    "apiKey": API_KEY
+}
+
+def fetch_tickers():
+    tickers = []
+    next_url = BASE_URL
+
+    while next_url:
+        logging.info(f"Fetching data from {next_url}")
+        response = requests.get(next_url, params=params)
+
+        if response.status_code == 200:
+            data = response.json()
+            results = data.get("results", [])
+            tickers.extend(results)
+
+            # Check if there are more pages
+            next_url = data.get("next_url", None)
+            time.sleep(1)  # Rate limit handling (1 second delay)
+        else:
+            logging.error(f"Failed to fetch data: {response.status_code} - {response.text}")
+            break
+
+    return tickers
+
+def save_to_csv(tickers):
+    # Save the tickers to a CSV file
+    with open(OUTPUT_FILE, mode="w", newline="", encoding="utf-8") as file:
+        writer = csv.writer(file)
+        # Write header
+        writer.writerow(["Ticker", "Name", "Market", "Locale", "Primary Exchange", "Type", "Currency"])
+        # Write rows
+        for ticker in tickers:
+            writer.writerow([
+                ticker.get("ticker"),
+                ticker.get("name"),
+                ticker.get("market"),
+                ticker.get("locale"),
+                ticker.get("primary_exchange"),
+                ticker.get("type"),
+                ticker.get("currency")
+            ])
+    logging.info(f"Saved {len(tickers)} tickers to {OUTPUT_FILE}")
+
+def main():
+    logging.info("Starting ticker retrieval program...")
+
+    # Fetch tickers
+    tickers = fetch_tickers()
+    if tickers:
+        logging.info(f"Fetched {len(tickers)} tickers.")
+        save_to_csv(tickers)
+    else:
+        logging.warning("No tickers fetched. Exiting program.")
+
+if __name__ == "__main__":
+    main()
+

src/MidasV1/tests/AdvHistoricalModule2Test/test_historical_approach.py

@@ -0,0 +1,399 @@
+import os
+import sys
+import time
+import json
+import logging
+import configparser
+from datetime import datetime, timedelta
+import csv
+from decimal import Decimal
+import pytz
+import threading
+import yfinance as yf 
+import signal
+
+from ibapi.client import EClient
+from ibapi.wrapper import EWrapper
+from ibapi.contract import Contract
+
+now_str = datetime.now().strftime("%Y%m%d_%H%M%S")
+
+# Color definitions for terminal output
+class Colors:
+    RESET = "\033[0m"
+    RED = "\033[91m"
+    GREEN = "\033[92m"
+    YELLOW = "\033[93m"
+    BLUE = "\033[94m"
+    MAGENTA = "\033[95m"
+
+# Configure logging with colors
+def color_log(level, message, color):
+    print(f"{color}[{level.upper()}]{Colors.RESET} {message}")
+
+# Load config
+config = configparser.ConfigParser()
+if not os.path.exists('config.ini'):
+    color_log("error", "config.ini not found. Please create one.", Colors.RED)
+    sys.exit(1)
+
+config.read('config.ini')
+HOST = config.get('API', 'host', fallback='127.0.0.1')
+PORT = config.getint('API', 'port', fallback=4002)
+CLIENT_ID = config.getint('API', 'clientId', fallback=1)
+DATA_DIR = config.get('Directories', 'data_dir', fallback='./data')
+TICKER_FILE = config.get('General', 'ticker_file', fallback='ticker_ids.csv')
+MIN_VOLUME = config.getint('Thresholds', 'min_volume', fallback=1000000)
+MIN_NET_CHANGE = config.getfloat('Thresholds', 'min_net_change', fallback=0.1)
+MIN_PERCENT_CHANGE = config.getfloat('Thresholds', 'min_percent_change', fallback=0.5)
+MAX_SHARE_PRICE = config.getfloat('Thresholds', 'max_share_price', fallback=500.0)
+GOOD_RUNTIME_THRESHOLD = config.getfloat('Performance', 'good_runtime_threshold', fallback=10.0)
+
+if not os.path.exists(DATA_DIR):
+    os.makedirs(DATA_DIR)
+
+if not os.path.exists(TICKER_FILE):
+    color_log("error", f"{TICKER_FILE} not found in {DATA_DIR}. Please create one with a list of symbols.", Colors.RED)
+    sys.exit(1)
+
+# Debug loaded configuration
+def debug_config():
+    color_log("debug", "Configuration loaded:", Colors.BLUE)
+    for section in config.sections():
+        for key, value in config[section].items():
+            color_log("debug", f"{section.upper()} - {key}: {value}", Colors.BLUE)
+
+debug_config()
+
+class IBWrapper(EWrapper):
+    def __init__(self):
+        super().__init__()
+        self.current_time = None
+        self.current_time_received = threading.Event()
+
+        self.historical_data_reqId = None
+        self.historical_bars = []
+        self.historical_data_received = threading.Event()
+
+    def error(self, reqId, errorCode, errorString, advancedOrderRejectJson=""):
+        if errorCode in [2104, 2106, 2158]:
+            color_log("info", f"System Check: ReqId {reqId}, Code {errorCode}, Msg: {errorString}", Colors.BLUE)
+        else:
+            color_log("error", f"Error. ReqId: {reqId}, Code: {errorCode}, Msg: {errorString}", Colors.RED)
+
+    def currentTime(self, time_):
+        self.current_time = time_
+        self.current_time_received.set()
+
+    def historicalData(self, reqId, bar):
+        if reqId == self.historical_data_reqId:
+            self.historical_bars.append(bar)
+
+    def historicalDataEnd(self, reqId, start, end):
+        if reqId == self.historical_data_reqId:
+            self.historical_data_received.set()
+
+class IBClient(EClient):
+    def __init__(self, wrapper):
+        EClient.__init__(self, wrapper)
+
+class IBApp(IBWrapper, IBClient):
+    def __init__(self):
+        IBWrapper.__init__(self)
+        IBClient.__init__(self, wrapper=self)
+        self.connect_error = None
+
+    def connect_app(self):
+        try:
+            self.connect(HOST, PORT, CLIENT_ID)
+        except Exception as e:
+            self.connect_error = e
+
+        thread = threading.Thread(target=self.run, daemon=True)
+        thread.start()
+        time.sleep(2)
+
+    def disconnect_app(self):
+        if self.isConnected():
+            self.disconnect()
+
+def calculate_start_date():
+    """Calculate the start date for historical data (2 trading days ago)."""
+    current_date = datetime.now()
+    delta_days = 2
+    while delta_days > 0:
+        current_date -= timedelta(days=1)
+        if current_date.weekday() < 5:  # Skip weekends
+            delta_days -= 1
+    return current_date.strftime("%Y%m%d")
+
+def get_symbols_from_file(file_path):
+    symbols = []
+    with open(file_path, 'r') as f:
+        csv_reader = csv.reader(f)
+        for row in csv_reader:
+            symbols.extend([symbol.strip() for symbol in row if symbol.strip()])
+    return symbols
+
+def request_historical_data(app, symbol, start_date):
+    """Request historical data for a given symbol."""
+    contract = Contract()
+    contract.symbol = symbol
+    contract.secType = "STK"
+    contract.exchange = "SMART"
+    contract.currency = "USD"
+
+    app.historical_bars = []
+    app.historical_data_received.clear()
+    app.historical_data_reqId = 10000  # arbitrary reqId
+
+    endDateTime = f"{start_date} 09:30:00 UTC"
+    try:
+        app.reqHistoricalData(
+            reqId=app.historical_data_reqId,
+            contract=contract,
+            endDateTime=endDateTime,
+            durationStr="2 D",
+            barSizeSetting="1 day",
+            whatToShow="TRADES",
+            useRTH=1,
+            formatDate=1,
+            keepUpToDate=False,
+            chartOptions=[]
+        )
+    except Exception as e:
+        color_log("error", f"Error requesting historical data for {symbol}: {e}", Colors.RED)
+        return None
+
+    if not app.historical_data_received.wait(timeout=10):
+        color_log("warning", f"Timeout waiting for historical data for {symbol}.", Colors.YELLOW)
+        return None
+
+    return app.historical_bars
+
+def filter_data(raw_data):
+    """Filter raw data based on thresholds."""
+    filtered_data = [
+        entry for entry in raw_data
+        if entry['volume'] >= MIN_VOLUME and
+           entry['net_change'] >= MIN_NET_CHANGE and
+           entry['percent_change'] >= MIN_PERCENT_CHANGE and
+           entry['close'] <= MAX_SHARE_PRICE
+    ]
+    if not filtered_data:
+        color_log("warning", "No data passed filtering. Check your config or provide a larger data pool.", Colors.YELLOW)
+    return filtered_data
+
+def refine_with_options(processed_data, app):
+    """
+    Refine processed data to include only stocks with associated option contracts.
+    Attempts to use the IBKR API first, then falls back to Yahoo Finance.
+    If IBKR fails persistently, it switches entirely to Yahoo Finance for subsequent checks.
+    """
+    refined_data = []
+    ibkr_persistent_fail = False  # Flag to skip IBKR checks after persistent failures
+
+    def check_with_ibkr(symbol):
+        """Check if a stock has options contracts using IBKR."""
+        nonlocal ibkr_persistent_fail
+
+        if ibkr_persistent_fail:
+            color_log("info", f"Skipping IBKR check for {symbol} due to persistent failures.", Colors.YELLOW)
+            return False
+
+        try:
+            contract = Contract()
+            contract.symbol = symbol
+            contract.secType = "STK"  # Stock type for the underlying security
+            contract.exchange = "SMART"
+            contract.currency = "USD"
+
+            color_log("info", f"IBKR: Requesting options contract for {symbol} with contract parameters: {contract}", Colors.MAGENTA)
+
+            app.historical_data_received.clear()
+            app.reqSecDefOptParams(
+                reqId=1,
+                underlyingSymbol=symbol,
+                futFopExchange="",
+                underlyingSecType="STK",
+                underlyingConId=0
+            )
+
+            if not app.historical_data_received.wait(timeout=5):  # Reduced timeout
+                color_log("warning", f"IBKR: Timeout while querying options for {symbol}.", Colors.YELLOW)
+                return False
+
+            color_log("info", f"IBKR: Successfully queried options contract for {symbol}.", Colors.GREEN)
+            return True
+        except Exception as e:
+            color_log("error", f"IBKR option check failed for {symbol}: {e}", Colors.RED)
+            if "Invalid contract id" in str(e):  # Check for specific persistent error
+                ibkr_persistent_fail = True
+            return False
+
+    def check_with_yfinance(symbol):
+        """Check if a stock has options contracts using Yahoo Finance."""
+        try:
+            stock = yf.Ticker(symbol)
+            if stock.options:
+                color_log("info", f"Yahoo Finance: Options contract found for {symbol}.", Colors.GREEN)
+                return True
+            else:
+                color_log("info", f"Yahoo Finance: No options contract found for {symbol}.", Colors.YELLOW)
+                return False
+        except Exception as e:
+            color_log("warning", f"Yahoo Finance option check failed for {symbol}: {e}", Colors.YELLOW)
+            return False
+
+    # Process each stock in the data
+    for entry in processed_data:
+        symbol = entry['symbol']
+        color_log("info", f"Checking options contracts for {symbol}...", Colors.MAGENTA)
+
+        # Try IBKR first unless persistent failures are detected
+        if check_with_ibkr(symbol):
+            refined_data.append(entry)
+            continue
+
+        # Fallback to Yahoo Finance
+        color_log("info", f"Falling back to Yahoo Finance for {symbol}...", Colors.BLUE)
+        if check_with_yfinance(symbol):
+            refined_data.append(entry)
+
+    if not refined_data:
+        color_log("warning", "No stocks with associated options contracts found.", Colors.YELLOW)
+
+    return refined_data
+
+def save_to_csv(ticker_ids, filename):
+    """Save ticker IDs to a CSV file."""
+    with open(filename, 'w', newline='') as f:
+        writer = csv.writer(f)
+        writer.writerow(ticker_ids)
+    color_log("info", f"Ticker IDs saved to {filename}.", Colors.GREEN)
+
+def save_data(data, filename):
+    """Save data to a JSON file."""
+    if not data:
+        color_log("warning", f"No data to save in {filename}.", Colors.YELLOW)
+    else:
+        with open(filename, 'w') as f:
+            json.dump(data, f, indent=4)
+        color_log("info", f"Data saved to {filename}.", Colors.GREEN)
+
+def handle_sigint(signal_received, frame):
+    """
+    Handle SIGINT (Ctrl+C) to perform cleanup before exiting.
+    Deletes incomplete data files.
+    """
+    color_log("error", "SIGINT received. Cleaning up and exiting...", Colors.RED)
+
+    # List of files to clean up
+    temp_files = [
+        os.path.join(DATA_DIR, f"raw_stock_info_{now_str}.json"),
+        os.path.join(DATA_DIR, f"processed_data_{now_str}.json"),
+        os.path.join(DATA_DIR, f"contract_option_stock_info_{now_str}.json")
+    ]
+
+    for file in temp_files:
+        if os.path.exists(file):
+            os.remove(file)
+            color_log("info", f"Deleted incomplete file: {file}", Colors.YELLOW)
+
+    sys.exit(0)
+
+def main():
+    signal.signal(signal.SIGINT, handle_sigint)
+    start_time = time.time()
+    app = IBApp()
+    app.connect_app()
+
+    if app.connect_error:
+        color_log("error", f"Failed to connect to IB API: {app.connect_error}", Colors.RED)
+        sys.exit(1)
+
+    start_date = calculate_start_date()
+    color_log("info", f"Start date for data retrieval: {start_date}", Colors.BLUE)
+
+    symbols = get_symbols_from_file(TICKER_FILE)
+    color_log("info", f"Loaded {len(symbols)} symbols from {TICKER_FILE}.", Colors.BLUE)
+
+    raw_data = []
+    ibkr_checks = 0
+    yahoo_checks = 0
+
+    for symbol in symbols:
+        color_log("info", f"Retrieving data for {symbol}...", Colors.MAGENTA)
+        bars = request_historical_data(app, symbol, start_date)
+        if bars is None or len(bars) < 2:
+            color_log("warning", f"Skipping {symbol}: not enough data.", Colors.YELLOW)
+        else:
+            last_bar = bars[-1]
+            prev_bar = bars[-2]
+            net_change = last_bar.close - prev_bar.close
+            percent_change = (net_change / prev_bar.close) * 100 if prev_bar.close != 0 else 0.0
+
+            entry = {
+                "symbol": symbol,
+                "date": last_bar.date,
+                "open": float(last_bar.open),
+                "high": float(last_bar.high),
+                "low": float(last_bar.low),
+                "close": float(last_bar.close),
+                "volume": int(last_bar.volume),
+                "net_change": float(net_change),
+                "percent_change": float(percent_change)
+            }
+            raw_data.append(entry)
+
+        time.sleep(0.5)
+
+    now_str = datetime.now().strftime("%Y%m%d_%H%M%S")
+    raw_filename = os.path.join(DATA_DIR, f"raw_stock_info_{now_str}.json")
+    save_data(raw_data, raw_filename)
+
+    # Filter and save processed data
+    processed_data = filter_data(raw_data)
+    processed_filename = os.path.join(DATA_DIR, f"processed_data_{now_str}.json")
+    save_data(processed_data, processed_filename)
+
+    # Track fallback performance metrics
+    color_log("info", "Starting refinement with options contracts...", Colors.BLUE)
+    refined_data = refine_with_options(processed_data, app)
+    refined_filename = os.path.join(DATA_DIR, f"contract_option_stock_info_{now_str}.json")
+    save_data(refined_data, refined_filename)
+
+    # Extract ticker IDs and save to CSV
+    ticker_ids = [entry['symbol'] for entry in refined_data]
+    csv_filename = os.path.join(DATA_DIR, "ticker_ids_with_match.csv")
+    save_to_csv(ticker_ids, csv_filename)
+
+    app.disconnect_app()
+    end_time = time.time()
+    runtime = end_time - start_time
+
+    # Determine the proportion of IBKR and Yahoo Finance checks
+    total_checks = len(processed_data)
+    yahoo_checks = total_checks - ibkr_checks
+
+    # Log detailed performance analysis
+    if runtime < GOOD_RUNTIME_THRESHOLD:
+        color_log("info", f"Program completed in {runtime:.2f} seconds (Good runtime).", Colors.GREEN)
+    elif runtime == GOOD_RUNTIME_THRESHOLD:
+        color_log("warning", f"Program completed in {runtime:.2f} seconds (Runtime threshold met).", Colors.YELLOW)
+    else:
+        color_log("error", f"Program completed in {runtime:.2f} seconds (Bad runtime).", Colors.RED)
+
+    color_log("info", f"Refinement breakdown: IBKR checks = {ibkr_checks}, Yahoo Finance checks = {yahoo_checks}", Colors.MAGENTA)
+    color_log(
+        "info",
+        "Time Complexity: O(k × t_ibkr + (n - k) × t_yahoo), where:\n"
+        f"    k = {ibkr_checks} (IBKR checks)\n"
+        f"    n = {total_checks} (Total symbols processed)\n"
+        f"    t_ibkr = avg. IBKR query time\n"
+        f"    t_yahoo = avg. Yahoo Finance query time",
+        Colors.BLUE
+    )
+
+if __name__ == "__main__":
+    main()