pulls historical data

IB_Gateway/.env

@@ -0,0 +1,2 @@
+jacobmardian
+Griffinisgay123

IB_Gateway/attempttofixgwittdata.py

@@ -0,0 +1,129 @@
+import json
+import datetime
+from zoneinfo import ZoneInfo
+import logging
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import asyncio
+from ib_insync import IB, Future, util
+
+# Logger setup (unchanged from your code)
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+handler = logging.StreamHandler()
+handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s", "%Y-%m-%d %H:%M:%S"))
+logger.addHandler(handler)
+
+TZ = ZoneInfo("US/Eastern")
+CONFIG = {"MAX_WORKERS": 4, "REQUEST_DELAY": 0.2, "BASE_CLIENT_ID": 100}
+
+def get_third_friday(year, month):
+    """Return the third Friday of the given year/month."""
+    fridays = [d for d in range(1, 32) if datetime.date(year, month, d).weekday() == 4 
+               if datetime.date(year, month, d).month == month]
+    dt = datetime.datetime.combine(fridays[2] if len(fridays) >= 3 else fridays[-1], 
+                                   datetime.time(16, 0), tzinfo=TZ)
+    return dt
+
+def get_contract_for_date(date):
+    """Return the MES contract active on the given date."""
+    months = [3, 6, 9, 12]
+    year = date.year
+    month = date.month
+    # Find the next expiration after the date
+    for m in months:
+        exp = get_third_friday(year, m)
+        if exp > date:
+            return f"MES{m:02d}{year % 10}", exp
+    # If past December, roll to next year's March
+    return f"MES03{(year + 1) % 10}", get_third_friday(year + 1, 3)
+
+def get_mes_contract(ib_conn, cm, expiration):
+    """Define and validate an MES contract."""
+    local_symbol = f"MES{cm[-3:]}{cm[:4][-1]}"  # e.g., MESH5
+    contract = Future(
+        symbol='MES',
+        lastTradeDateOrContractMonth=expiration.strftime("%Y%m%d"),
+        localSymbol=local_symbol,
+        exchange='CME',
+        currency='USD',
+        multiplier=5
+    )
+    contract.includeExpired = True
+    details = ib_conn.reqContractDetails(contract)
+    return details[0].contract if details else None
+
+def get_data_chunk(ib_conn, contract, start_dt, end_dt):
+    """Request historical data for a specific date range."""
+    bars = ib_conn.reqHistoricalData(
+        contract,
+        endDateTime=end_dt.strftime("%Y%m%d %H:%M:%S"),
+        durationStr=f"{int((end_dt - start_dt).total_seconds() / 86400)} D",
+        barSizeSetting="5 mins",
+        whatToShow="TRADES",
+        useRTH=False,
+        formatDate=1
+    )
+    return [b for b in bars if b.volume > 0] if bars else []  # Filter zero-volume bars
+
+def process_date_range(start_dt, end_dt, client_id):
+    """Process data for a date range, dynamically selecting contracts."""
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    
+    ib_conn = IB()
+    ib_conn.connect('127.0.0.1', 4002, clientId=client_id)
+    
+    bars_list = []
+    current_dt = start_dt
+    while current_dt < end_dt:
+        cm, expiration = get_contract_for_date(current_dt)
+        contract = get_mes_contract(ib_conn, cm, expiration)
+        if not contract:
+            logger.error(f"Client {client_id}: No contract for {cm}")
+            current_dt = expiration + datetime.timedelta(days=1)
+            continue
+        
+        chunk_end = min(expiration, end_dt)
+        logger.info(f"Client {client_id}: Pulling {contract.localSymbol} from {current_dt} to {chunk_end}")
+        bars = get_data_chunk(ib_conn, contract, current_dt, chunk_end)
+        time.sleep(CONFIG["REQUEST_DELAY"])
+        
+        bars_list.extend({
+            'date': bar.date.astimezone(TZ).strftime("%Y-%m-%d %H:%M:%S %Z"),
+            'open': bar.open,
+            'high': bar.high,
+            'low': bar.low,
+            'close': bar.close,
+            'volume': bar.volume,
+            'contract': contract.localSymbol
+        } for bar in bars if start_dt <= bar.date.astimezone(TZ) <= end_dt)
+        
+        current_dt = expiration + datetime.timedelta(days=1)
+    
+    ib_conn.disconnect()
+    return bars_list
+
+if __name__ == "__main__":
+    start_date = datetime.datetime.now(TZ) - datetime.timedelta(days=3*365)
+    end_date = datetime.datetime.now(TZ)
+    logger.info(f"Retrieving MES data from {start_date} to {end_date}")
+    
+    all_bars = []
+    with ThreadPoolExecutor(max_workers=CONFIG["MAX_WORKERS"]) as executor:
+        chunk_size = (end_date - start_date) / CONFIG["MAX_WORKERS"]
+        futures = [
+            executor.submit(
+                process_date_range,
+                start_date + i * chunk_size,
+                start_date + (i + 1) * chunk_size if i < CONFIG["MAX_WORKERS"] - 1 else end_date,
+                CONFIG["BASE_CLIENT_ID"] + i
+            )
+            for i in range(CONFIG["MAX_WORKERS"])
+        ]
+        for future in as_completed(futures):
+            all_bars.extend(future.result())
+    
+    final_data = sorted(all_bars, key=lambda x: x['date'])
+    with open("mes_5min_data_dynamic.json", "w") as f:
+        json.dump(final_data, f, indent=4)
+    logger.info(f"Saved {len(final_data)} bars to mes_5min_data_dynamic.json")

IB_Gateway/data.py

@@ -0,0 +1,329 @@
+import json
+import datetime
+import logging
+import time
+from zoneinfo import ZoneInfo
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+import asyncio  # For new event loops in threads
+from ib_insync import IB, Future, util
+
+# Try to import tabulate for a nice summary table
+try:
+    from tabulate import tabulate
+    HAS_TABULATE = True
+except ImportError:
+    HAS_TABULATE = False
+
+# --- CONFIGURATION ---
+CONFIG = {
+    "MAX_WORKERS": 4,           # Number of threads (concurrent workers)
+    "REQUEST_DELAY": 0.2,       # Delay in seconds between historical data requests
+    "BASE_CLIENT_ID": 100,      # Starting clientId; each thread gets a unique clientId
+}
+
+# --- Custom Colored Logger Setup ---
+class ColoredFormatter(logging.Formatter):
+    COLORS = {
+        logging.DEBUG: "\033[36m",    # Cyan
+        logging.INFO: "\033[32m",     # Green
+        logging.WARNING: "\033[33m",  # Yellow
+        logging.ERROR: "\033[31m",    # Red
+        logging.CRITICAL: "\033[41m", # Red background
+    }
+    RESET = "\033[0m"
+    
+    def format(self, record):
+        color = self.COLORS.get(record.levelno, self.RESET)
+        record.msg = f"{color}{record.msg}{self.RESET}"
+        return super().format(record)
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+handler = logging.StreamHandler()
+formatter = ColoredFormatter("%(asctime)s [%(levelname)s] %(message)s", "%Y-%m-%d %H:%M:%S")
+handler.setFormatter(formatter)
+logger.addHandler(handler)
+
+# --- Global Time Zone ---
+TZ = ZoneInfo("US/Eastern")
+
+# --- Functions ---
+def get_third_friday(year, month):
+    """Return the third Friday of the given year/month as a datetime with TZ."""
+    fridays = []
+    for day in range(1, 32):
+        try:
+            d = datetime.date(year, month, day)
+        except ValueError:
+            break
+        if d.weekday() == 4:
+            fridays.append(d)
+    if len(fridays) >= 3:
+        dt = datetime.datetime.combine(fridays[2], datetime.time(16, 0))
+    elif fridays:
+        dt = datetime.datetime.combine(fridays[-1], datetime.time(16, 0))
+    else:
+        dt = datetime.datetime(year, month, 1, 16, 0)
+    return dt.replace(tzinfo=TZ)
+
+def generate_contract_months(start_date, end_date):
+    """Generate a sorted list of contract month strings ('YYYYMM') between start_date and end_date."""
+    months = [3, 6, 9, 12]
+    result = []
+    for year in range(start_date.year, end_date.year + 2):
+        for m in months:
+            dt = datetime.datetime(year, m, 1, tzinfo=TZ)
+            if dt <= end_date:
+                result.append(f"{year}{m:02d}")
+    return sorted(set(result))
+
+def get_data_chunk(ib_conn, contract, end_dt, duration_str="1 W"):
+    """Request a chunk of historical data for the given contract ending at end_dt using ib_conn."""
+    try:
+        bars = ib_conn.reqHistoricalData(
+            contract,
+            endDateTime=end_dt.strftime("%Y%m%d %H:%M:%S"),
+            durationStr=duration_str,
+            barSizeSetting="5 mins",
+            whatToShow="TRADES",
+            useRTH=False,
+            formatDate=1
+        )
+        return bars
+    except Exception as e:
+        logger.error(f"Error retrieving data for {contract.localSymbol} ending at {end_dt}: {e}")
+        return None
+
+def getMESContract(ib_conn, cm, contract_expiration):
+    """
+    Try several MES contract definitions for the given contract month (cm) and expiration date.
+    Return a tuple (contract, variant_used) or (None, None) if none match.
+    """
+    expiration_str = contract_expiration.strftime("%Y%m%d")
+    variants = []
+
+    # Variant 1: Full expiration date, exchange GLOBEX.
+    contract1 = Future(
+        symbol='MES',
+        lastTradeDateOrContractMonth=expiration_str,
+        exchange='GLOBEX',
+        currency='USD',
+        multiplier=5
+    )
+    contract1.includeExpired = True
+    variants.append(("Variant 1: full expiration, GLOBEX", contract1))
+
+    # Variant 2: Full expiration date, exchange CME.
+    contract2 = Future(
+        symbol='MES',
+        lastTradeDateOrContractMonth=expiration_str,
+        exchange='CME',
+        currency='USD',
+        multiplier=5
+    )
+    contract2.includeExpired = True
+    variants.append(("Variant 2: full expiration, CME", contract2))
+
+    # Variant 3: Contract month, exchange GLOBEX, add tradingClass.
+    contract3 = Future(
+        symbol='MES',
+        lastTradeDateOrContractMonth=cm,
+        exchange='GLOBEX',
+        currency='USD',
+        multiplier=5,
+        tradingClass='MES'
+    )
+    contract3.includeExpired = True
+    variants.append(("Variant 3: contract month, GLOBEX, tradingClass", contract3))
+
+    # Variant 4: Contract month, exchange CME, add tradingClass.
+    contract4 = Future(
+        symbol='MES',
+        lastTradeDateOrContractMonth=cm,
+        exchange='CME',
+        currency='USD',
+        multiplier=5,
+        tradingClass='MES'
+    )
+    contract4.includeExpired = True
+    variants.append(("Variant 4: contract month, CME, tradingClass", contract4))
+
+    # Variant 5: Contract month, exchange GLOBEX, with computed localSymbol.
+    month_codes = {1:'F', 2:'G', 3:'H', 4:'J', 5:'K', 6:'M', 7:'N', 8:'Q', 9:'U', 10:'V', 11:'X', 12:'Z'}
+    year = int(cm[:4])
+    month = int(cm[4:])
+    local_symbol = f"MES{month_codes.get(month, '')}{str(year)[-1]}"
+    contract5 = Future(
+        symbol='MES',
+        lastTradeDateOrContractMonth=cm,
+        localSymbol=local_symbol,
+        exchange='GLOBEX',
+        currency='USD',
+        multiplier=5
+    )
+    contract5.includeExpired = True
+    variants.append(("Variant 5: contract month, GLOBEX, localSymbol", contract5))
+
+    for variant_desc, contract in variants:
+        logger.info(f"Trying {variant_desc} for {cm} (expiration: {expiration_str})...")
+        details = ib_conn.reqContractDetails(contract)
+        if details:
+            logger.info(f"Success with {variant_desc}: {details[0].contract}")
+            return details[0].contract, variant_desc
+        else:
+            logger.info(f"{variant_desc} did not return any details.")
+    return None, None
+
+def process_month(cm, start_date, end_date, request_delay, client_id):
+    """
+    Process a single contract month:
+      - Create a new event loop and IB connection (with unique client_id).
+      - Retrieve a valid MES contract.
+      - Request historical data in chunks.
+      - Return a summary dict and list of retrieved bars.
+    """
+    # Create and set a new event loop in this thread
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    
+    summary = {
+        "Month": cm,
+        "Status": "",
+        "Variant Used": "",
+        "LocalSymbol": "",
+        "Bars Retrieved": 0,
+        "Reason": ""
+    }
+    bars_list = []
+    try:
+        ib_conn = IB()
+        ib_conn.connect('127.0.0.1', 4002, clientId=client_id)
+    except Exception as e:
+        summary["Status"] = "Skipped"
+        summary["Reason"] = f"Connection error: {e}"
+        logger.error(f"Client {client_id}: Connection error for month {cm}: {e}")
+        return summary, bars_list
+
+    year = int(cm[:4])
+    month = int(cm[4:])
+    contract_expiration = get_third_friday(year, month)
+    contract, variant_used = getMESContract(ib_conn, cm, contract_expiration)
+    if not contract:
+        summary["Status"] = "Skipped"
+        summary["Reason"] = "No valid MES contract found"
+        logger.warning(f"Client {client_id}: No valid MES contract found for {cm}. Skipping.")
+        ib_conn.disconnect()
+        return summary, bars_list
+
+    summary["Status"] = "Processed"
+    summary["Variant Used"] = variant_used
+    summary["LocalSymbol"] = contract.localSymbol
+    logger.info(f"Client {client_id}: Processing contract {contract.localSymbol} for month {cm} (expiration approx: {contract_expiration.strftime('%Y-%m-%d %H:%M:%S %Z')})")
+
+    contract_end = min(end_date, contract_expiration)
+    current_end = contract_end
+    chunk_duration = datetime.timedelta(weeks=1)
+
+    while current_end > start_date:
+        logger.info(f"Client {client_id}: Requesting {contract.localSymbol} data ending at {current_end.strftime('%Y-%m-%d %H:%M:%S %Z')}")
+        bars = get_data_chunk(ib_conn, contract, current_end, duration_str="1 W")
+        time.sleep(request_delay)  # Rate limiting
+        if bars is None:
+            logger.error(f"Client {client_id}: Error retrieving chunk; moving to next week.")
+            current_end -= chunk_duration
+            continue
+        if not bars:
+            logger.info(f"Client {client_id}: No data returned for period ending at {current_end.strftime('%Y-%m-%d %H:%M:%S %Z')}; ending requests.")
+            break
+        for bar in bars:
+            bar_time = bar.date.astimezone(TZ)
+            if start_date <= bar_time <= end_date:
+                bars_list.append({
+                    'date': bar_time.strftime("%Y-%m-%d %H:%M:%S %Z"),
+                    'open': bar.open,
+                    'high': bar.high,
+                    'low': bar.low,
+                    'close': bar.close,
+                    'volume': bar.volume,
+                    'contract': contract.localSymbol
+                })
+        earliest_time = min(bar.date.astimezone(TZ) for bar in bars)
+        new_end = earliest_time - datetime.timedelta(seconds=1)
+        if new_end >= current_end:
+            break
+        current_end = new_end
+
+    summary["Bars Retrieved"] = len(bars_list)
+    if len(bars_list) == 0:
+        summary["Reason"] = "No data returned for this contract period"
+    ib_conn.disconnect()
+    return summary, bars_list
+
+# --- Main Execution ---
+if __name__ == "__main__":
+    overall_start_time = time.time()
+    logger.info("Starting retrieval process...")
+    
+    end_date = datetime.datetime.now(TZ)
+    start_date = end_date - datetime.timedelta(days=3*365)
+    logger.info(f"Retrieving MES data from {start_date.strftime('%Y-%m-%d %H:%M:%S %Z')} to {end_date.strftime('%Y-%m-%d %H:%M:%S %Z')}")
+    
+    contract_months = generate_contract_months(start_date, end_date)
+    logger.info(f"Contract months to process: {contract_months}")
+    
+    all_month_summaries = []
+    all_bars = []
+    
+    with ThreadPoolExecutor(max_workers=CONFIG["MAX_WORKERS"]) as executor:
+        futures = []
+        for i, cm in enumerate(contract_months):
+            client_id = CONFIG["BASE_CLIENT_ID"] + i
+            futures.append(executor.submit(process_month, cm, start_date, end_date, CONFIG["REQUEST_DELAY"], client_id))
+    
+        for future in as_completed(futures):
+            summary, bars = future.result()
+            all_month_summaries.append(summary)
+            all_bars.extend(bars)
+    
+    final_data = sorted(all_bars, key=lambda x: x['date'])
+    expected_bars = int((end_date - start_date).total_seconds() / (5 * 60))
+    
+    output_filename = "mes_5min_data.json"
+    if final_data:
+        try:
+            with open(output_filename, "w") as f:
+                json.dump(final_data, f, indent=4)
+            logger.info(f"Data successfully saved to {output_filename}")
+        except Exception as e:
+            logger.error(f"Error writing to JSON file: {e}")
+    else:
+        logger.info("No data retrieved. File not saved.")
+    
+    # Build final summary table.
+    summary_rows = []
+    for s in all_month_summaries:
+        summary_rows.append([s["Month"], s["Status"], s["Variant Used"], s["LocalSymbol"], s["Bars Retrieved"], s["Reason"]])
+    
+    headers = ["Contract Month", "Status", "Variant Used", "LocalSymbol", "Bars Retrieved", "Reason"]
+    if HAS_TABULATE:
+        table = tabulate(summary_rows, headers=headers, tablefmt="grid")
+    else:
+        header_line = " | ".join(headers)
+        separator = "-" * len(header_line)
+        table = header_line + "\n" + separator + "\n"
+        for row in summary_rows:
+            table += " | ".join(str(item) for item in row) + "\n"
+    
+    logger.info("\n" + table)
+    processed = len([s for s in all_month_summaries if s["Status"] == "Processed"])
+    skipped = len([s for s in all_month_summaries if s["Status"] == "Skipped"])
+    logger.info(f"Total contract months processed: {len(contract_months)}")
+    logger.info(f"  Processed: {processed}   Skipped: {skipped}")
+    logger.info(f"Total bars retrieved: {len(final_data)} (expected ~{expected_bars})")
+    
+    overall_end_time = time.time()
+    runtime_sec = overall_end_time - overall_start_time
+    runtime_str = str(datetime.timedelta(seconds=int(runtime_sec)))
+    logger.info(f"Total runtime: {runtime_str}")
+

IB_Gateway/data.py.bak

@@ -0,0 +1,237 @@
+import json
+import datetime
+from ib_insync import IB, Future, util
+
+def get_third_friday(year, month):
+    """
+    Returns the third Friday of the given year and month as a datetime,
+    which is a common expiration for futures.
+    """
+    fridays = []
+    for day in range(1, 32):
+        try:
+            d = datetime.date(year, month, day)
+        except ValueError:
+            break
+        if d.weekday() == 4:  # Friday
+            fridays.append(d)
+    if len(fridays) >= 3:
+        return datetime.datetime.combine(fridays[2], datetime.time(16, 0))  # 4:00 PM
+    elif fridays:
+        return datetime.datetime.combine(fridays[-1], datetime.time(16, 0))
+    else:
+        return datetime.datetime(year, month, 1, 16, 0)
+
+def generate_contract_months(start_date, end_date):
+    """
+    Generate a sorted list of contract month strings (format "YYYYMM")
+    that might be active between start_date and end_date.
+    MES futures are listed for quarters (Mar, Jun, Sep, Dec).
+    """
+    months = [3, 6, 9, 12]
+    result = []
+    for year in range(start_date.year, end_date.year + 2):
+        for m in months:
+            dt = datetime.datetime(year, m, 1)
+            if dt <= end_date:
+                result.append(f"{year}{m:02d}")
+    return sorted(set(result))
+
+def get_data_chunk(contract, end_dt, duration_str="1 W"):
+    """
+    Request a chunk of historical data for the given contract ending at end_dt.
+    Returns a list of bars (or None on error).
+    """
+    try:
+        bars = ib.reqHistoricalData(
+            contract,
+            endDateTime=end_dt.strftime("%Y%m%d %H:%M:%S"),
+            durationStr=duration_str,
+            barSizeSetting="5 mins",
+            whatToShow="TRADES",
+            useRTH=True,   # Regular trading hours only
+            formatDate=1
+        )
+        return bars
+    except Exception as e:
+        print(f"Error retrieving data chunk for {contract.localSymbol} ending at {end_dt}: {e}")
+        return None
+
+def getMESContract(cm, contract_expiration):
+    """
+    Attempt multiple MES contract definitions for the given contract month (cm) and expiration date.
+    Returns the first valid contract (as returned by reqContractDetails) or None.
+    """
+    expiration_str = contract_expiration.strftime("%Y%m%d")
+    variants = []
+
+    # Variant 1: Use full expiration date (YYYYMMDD), exchange GLOBEX, minimal fields.
+    contract1 = Future(
+        symbol='MES',
+        lastTradeDateOrContractMonth=expiration_str,
+        exchange='GLOBEX',
+        currency='USD',
+        multiplier=5
+    )
+    contract1.includeExpired = True
+    variants.append(("Variant 1: full expiration date, GLOBEX", contract1))
+
+    # Variant 2: Full expiration date, exchange CME.
+    contract2 = Future(
+        symbol='MES',
+        lastTradeDateOrContractMonth=expiration_str,
+        exchange='CME',
+        currency='USD',
+        multiplier=5
+    )
+    contract2.includeExpired = True
+    variants.append(("Variant 2: full expiration date, CME", contract2))
+
+    # Variant 3: Use contract month (YYYYMM), exchange GLOBEX, add tradingClass.
+    contract3 = Future(
+        symbol='MES',
+        lastTradeDateOrContractMonth=cm,
+        exchange='GLOBEX',
+        currency='USD',
+        multiplier=5,
+        tradingClass='MES'
+    )
+    contract3.includeExpired = True
+    variants.append(("Variant 3: contract month, GLOBEX, tradingClass MES", contract3))
+
+    # Variant 4: Use contract month, exchange CME, add tradingClass.
+    contract4 = Future(
+        symbol='MES',
+        lastTradeDateOrContractMonth=cm,
+        exchange='CME',
+        currency='USD',
+        multiplier=5,
+        tradingClass='MES'
+    )
+    contract4.includeExpired = True
+    variants.append(("Variant 4: contract month, CME, tradingClass MES", contract4))
+
+    # Variant 5: Use contract month, exchange GLOBEX, with a computed localSymbol.
+    month_codes = {1:'F', 2:'G', 3:'H', 4:'J', 5:'K', 6:'M', 7:'N', 8:'Q', 9:'U', 10:'V', 11:'X', 12:'Z'}
+    year = int(cm[:4])
+    month = int(cm[4:])
+    local_symbol = f"MES{month_codes.get(month, '')}{str(year)[-1]}"
+    contract5 = Future(
+        symbol='MES',
+        lastTradeDateOrContractMonth=cm,
+        localSymbol=local_symbol,
+        exchange='GLOBEX',
+        currency='USD',
+        multiplier=5
+    )
+    contract5.includeExpired = True
+    variants.append(("Variant 5: contract month, GLOBEX, localSymbol", contract5))
+
+    # Try each variant.
+    for variant_desc, contract in variants:
+        print(f"Trying {variant_desc} for contract month {cm} (expiration: {expiration_str})...")
+        details = ib.reqContractDetails(contract)
+        if details:
+            print(f"Success with {variant_desc}: found contract details: {details[0].contract}")
+            return details[0].contract
+        else:
+            print(f"{variant_desc} did not return contract details.")
+    return None
+
+# --- Main Script ---
+
+# Connect to IB Gateway (ensure your account is active and market data is subscribed)
+ib = IB()
+try:
+    ib.connect('127.0.0.1', 4002, clientId=1)
+except Exception as e:
+    print(f"Connection error: {e}")
+    exit(1)
+
+# Define overall desired time range: last 3 years up until today.
+# We'll use naive datetime objects (local time)
+end_date = datetime.datetime.now()
+start_date = end_date - datetime.timedelta(days=3*365)
+
+# Generate contract month strings (e.g., "202303", "202306", etc.)
+contract_months = generate_contract_months(start_date, end_date)
+print("Contract months to process:", contract_months)
+
+all_data = []
+
+# Process each contract month
+for cm in contract_months:
+    year = int(cm[:4])
+    month = int(cm[4:])
+    # Compute the contract expiration date (using third Friday)
+    contract_expiration = get_third_friday(year, month)
+    
+    # Try to obtain a valid MES contract using our diagnostic function.
+    mes_contract = getMESContract(cm, contract_expiration)
+    if not mes_contract:
+        print(f"*** No valid MES contract found for {cm}. Skipping this month. ***")
+        continue
+
+    print(f"Processing contract {mes_contract.localSymbol} (approx expiration: {contract_expiration})")
+
+    # Determine the effective data period for this contract.
+    contract_end = min(end_date, contract_expiration)
+    current_end = contract_end
+
+    contract_data = []
+    chunk_duration = datetime.timedelta(weeks=1)
+    
+    # Request data in weekly chunks until we reach start_date.
+    while current_end > start_date:
+        print(f"  Requesting {mes_contract.localSymbol} data ending at {current_end}")
+        bars = get_data_chunk(mes_contract, current_end, duration_str="1 W")
+        if bars is None:
+            print("  Error retrieving chunk; moving to next week.")
+            current_end -= chunk_duration
+            continue
+        if len(bars) == 0:
+            print("  No data returned for this period; ending requests for this contract.")
+            break
+        for bar in bars:
+            # Remove timezone info from bar.date so we can compare with naive datetimes.
+            bar_time = bar.date.replace(tzinfo=None)
+            if start_date <= bar_time <= end_date:
+                contract_data.append({
+                    'date': bar_time.strftime("%Y-%m-%d %H:%M:%S"),
+                    'open': bar.open,
+                    'high': bar.high,
+                    'low': bar.low,
+                    'close': bar.close,
+                    'volume': bar.volume,
+                    'contract': mes_contract.localSymbol
+                })
+        earliest_time = min(bar.date.replace(tzinfo=None) for bar in bars)
+        new_end = earliest_time - datetime.timedelta(seconds=1)
+        if new_end >= current_end:
+            break
+        current_end = new_end
+
+    print(f"  Retrieved {len(contract_data)} bars for contract {mes_contract.localSymbol}")
+    all_data.extend(contract_data)
+
+# Remove duplicate bars (if any) based on timestamp.
+unique_data = {d['date']: d for d in all_data}
+final_data = sorted(unique_data.values(), key=lambda x: x['date'])
+
+expected_bars = ((end_date - start_date).total_seconds() / (5 * 60))
+if len(final_data) < expected_bars * 0.9:
+    print("Warning: Retrieved data appears significantly less than expected.")
+
+output_filename = "mes_5min_data.json"
+if final_data:
+    try:
+        with open(output_filename, "w") as f:
+            json.dump(final_data, f, indent=4)
+        print(f"Data successfully saved to {output_filename}")
+    except Exception as e:
+        print(f"Error writing to JSON file: {e}")
+else:
+    print("No data retrieved. File not saved.")
+
+ib.disconnect()
+

IB_Gateway/requirements.txt

@@ -0,0 +1,11 @@
+eventkit==1.0.3
+ib-insync==0.9.86
+ibapi @ file:///home/midas/codeWS/dependencies/IBJts/source/pythonclient
+nest-asyncio==1.6.0
+numpy==2.2.4
+pandas==2.2.3
+python-dateutil==2.9.0.post0
+pytz==2025.2
+six==1.17.0
+tabulate==0.9.0
+tzdata==2025.2

IB_Gateway/test.txt

@@ -0,0 +1,3 @@
+# Test
+
+

IB_Gateway/test3.py

@@ -0,0 +1,273 @@
+import threading
+import time
+import csv
+import json
+from datetime import datetime, timedelta
+from ibapi.client import EClient
+from ibapi.wrapper import EWrapper
+from ibapi.contract import Contract
+
+################################################################################
+# 1) CONTRACT DEFINITION & ROLLOVER
+################################################################################
+
+# Approximate 3rd-Friday expiration dates for quarterly MES futures from June 2022 -> Mar 2025
+quarter_expirations = {
+    "202206": datetime(2022, 6, 17),
+    "202209": datetime(2022, 9, 16),
+    "202212": datetime(2022, 12, 16),
+    "202303": datetime(2023, 3, 17),
+    "202306": datetime(2023, 6, 16),
+    "202309": datetime(2023, 9, 15),
+    "202312": datetime(2023, 12, 15),
+    "202403": datetime(2024, 3, 15),
+    "202406": datetime(2024, 6, 21),
+    "202409": datetime(2024, 9, 20),
+    "202412": datetime(2024, 12, 20),
+    "202503": datetime(2025, 3, 21),
+}
+
+# Build “rolling windows”: from (previous expiration + 1 day) to (this expiration)
+sorted_contracts = sorted(quarter_expirations.items(), key=lambda kv: kv[0])
+rolling_windows = []
+prev_exp = None
+for c_str, exp_date in sorted_contracts:
+    if prev_exp is None:
+        # For the first contract, just assume it might start ~90 days before
+        start_active = exp_date - timedelta(days=90)
+    else:
+        start_active = prev_exp + timedelta(days=1)
+    end_active = exp_date
+    prev_exp = exp_date
+    rolling_windows.append((c_str, start_active, end_active))
+
+# Our overall request window
+overall_start = datetime(2022, 4, 1)
+overall_end   = datetime(2025, 4, 1)
+
+def get_contract_intervals():
+    """
+    Return a list of (contractMonth, interval_start, interval_end)
+    that fall within [overall_start, overall_end].
+    """
+    intervals = []
+    for (c_str, c_start, c_end) in rolling_windows:
+        if c_end <= overall_start:
+            continue
+        if c_start >= overall_end:
+            continue
+        eff_start = max(c_start, overall_start)
+        eff_end   = min(c_end, overall_end)
+        if eff_start < eff_end:
+            intervals.append((c_str, eff_start, eff_end))
+    return intervals
+
+
+################################################################################
+# 2) IB APP & CONNECTION
+################################################################################
+
+class HistoricalApp(EWrapper, EClient):
+    def __init__(self):
+        EClient.__init__(self, self)
+        self.connection_ready = threading.Event()
+        self.data_event = threading.Event()
+        self.current_req_id = None
+        self.historical_data = []
+        self.error_flag = False
+
+    def error(self, reqId, errorCode, errorString, advancedOrderRejectJson=""):
+        # Filter typical farm messages
+        if errorCode in (2104, 2106, 2107, 2108):
+            return
+
+        print(f"ERROR {reqId} {errorCode}: {errorString}")
+        # Connection failure
+        if reqId == -1 and errorCode == 502:
+            self.connection_ready.set()
+            self.error_flag = True
+            return
+        # If it's for our current request, set error_flag
+        if reqId == self.current_req_id:
+            self.error_flag = True
+            self.data_event.set()
+
+    def nextValidId(self, orderId):
+        print(f"Connected. NextValidId={orderId}")
+        self.connection_ready.set()
+
+    def historicalData(self, reqId, bar):
+        if reqId == self.current_req_id:
+            self.historical_data.append(bar)
+
+    def historicalDataEnd(self, reqId, start, end):
+        if reqId == self.current_req_id:
+            print("  Chunk complete.")
+            self.data_event.set()
+
+def ensure_connected(app, timeout=15):
+    if not app.connection_ready.wait(timeout=timeout):
+        raise RuntimeError("Timed out waiting for IB connection.")
+
+
+################################################################################
+# 3) MAIN SCRIPT: FETCH 5-MINUTE RTH BARS, CASTING DECIMALS TO FLOAT, SAVE JSON/CSV
+################################################################################
+
+def main():
+    intervals = get_contract_intervals()
+
+    # Start up the IB API
+    app = HistoricalApp()
+    print("Connecting to IB Gateway (paper port=4002)...")
+    app.connect("127.0.0.1", 4002, clientId=101)
+
+    api_thread = threading.Thread(target=app.run, daemon=True)
+    api_thread.start()
+
+    try:
+        ensure_connected(app, 15)
+        if app.error_flag:
+            print("Connection error. Exiting.")
+            app.disconnect()
+            return
+    except RuntimeError as e:
+        print(e)
+        app.disconnect()
+        return
+
+    all_bars = []
+    req_id = 1
+
+    for (c_month, start_dt, end_dt) in intervals:
+        contract = Contract()
+        contract.symbol = "MES"
+        contract.secType = "FUT"
+        contract.exchange = "CME"    # or GLOBEX if that works better for you
+        contract.currency = "USD"
+        contract.includeExpired = True
+        contract.lastTradeDateOrContractMonth = c_month  # e.g. "202306"
+
+        # Iterate in ~30-day chunks
+        chunk_start = start_dt
+        print(f"\nContract {c_month}, active {start_dt.date()} -> {end_dt.date()} (RTH).")
+
+        while chunk_start < end_dt:
+            chunk_end = chunk_start + timedelta(days=29)
+            if chunk_end > end_dt:
+                chunk_end = end_dt
+
+            # IB UTC format => "YYYYMMDD-HH:mm:ss", e.g. "20230430-00:00:00"
+            end_str = chunk_end.strftime("%Y%m%d-00:00:00")
+            days_count = (chunk_end - chunk_start).days + 1
+            duration_str = f"{days_count} D"
+
+            # Prepare request
+            app.historical_data.clear()
+            app.error_flag = False
+            app.data_event.clear()
+            app.current_req_id = req_id
+
+            print(f" Requesting {c_month} from {chunk_start.date()} to {chunk_end.date()}"
+                  f" -> end='{end_str}', dur='{duration_str}'")
+
+            app.reqHistoricalData(
+                reqId=req_id,
+                contract=contract,
+                endDateTime=end_str,
+                durationStr=duration_str,
+                barSizeSetting="5 mins",
+                whatToShow="TRADES",
+                useRTH=1,  # Only regular trading hours
+                formatDate=1,
+                keepUpToDate=False,
+                chartOptions=[]
+            )
+
+            finished = app.data_event.wait(60)
+            if not finished:
+                print(f"  Timeout on request {req_id}. Breaking out for contract {c_month}.")
+                break
+            if app.error_flag:
+                print(f"  Error on request {req_id}. Aborting {c_month}.")
+                break
+
+            # Store chunk data
+            for bar in app.historical_data:
+                # Convert bar fields from Decimal -> float, volume -> int (if safe)
+                try:
+                    o = float(bar.open)
+                    h = float(bar.high)
+                    l = float(bar.low)
+                    c_ = float(bar.close)
+                    v = float(bar.volume)  # or int(bar.volume) if you prefer
+                except:
+                    # fallback to string if something is weird
+                    o = str(bar.open)
+                    h = str(bar.high)
+                    l = str(bar.low)
+                    c_ = str(bar.close)
+                    v = str(bar.volume)
+
+                all_bars.append({
+                    "contract": c_month,
+                    "barDate": bar.date,  # might include time zone suffix
+                    "open": o,
+                    "high": h,
+                    "low": l,
+                    "close": c_,
+                    "volume": v
+                })
+
+            req_id += 1
+            chunk_start = chunk_end + timedelta(days=1)
+            time.sleep(2)  # pacing
+
+    # Done
+    app.disconnect()
+    api_thread.join(2)
+
+    # We’ll parse bar.date if needed to sort chronologically
+    # Some bars have "YYYYMMDD  HH:MM:SS" or "YYYYMMDD" or "YYYYMMDD HH:MM:SS US/Central"
+    def parse_bar_date(d_str):
+        d_str = " ".join(d_str.strip().split())  # remove double spaces
+        parts = d_str.split()
+        if len(parts) == 1:
+            # daily => "YYYYMMDD"
+            return datetime.strptime(d_str, "%Y%m%d")
+        elif len(parts) == 3:
+            # e.g. "20230810 09:30:00 US/Central"
+            dt_part = " ".join(parts[:2])  # "20230810 09:30:00"
+            return datetime.strptime(dt_part, "%Y%m%d %H:%M:%S")
+        else:
+            # e.g. "20230810 09:30:00"
+            return datetime.strptime(d_str, "%Y%m%d %H:%M:%S")
+
+    all_bars.sort(key=lambda x: (x["contract"], parse_bar_date(x["barDate"])))
+
+    # Save to JSON
+    json_file = "MES_5min_data.json"
+    print(f"\nSaving {len(all_bars)} bars to {json_file} ...")
+    with open(json_file, "w") as jf:
+        json.dump(all_bars, jf, indent=2)  # no error with floats
+
+    # Save to CSV
+    csv_file = "MES_5min_data.csv"
+    print(f"Saving {len(all_bars)} bars to {csv_file} ...")
+    with open(csv_file, "w", newline="") as cf:
+        writer = csv.writer(cf)
+        writer.writerow(["contract","barDate","open","high","low","close","volume"])
+        for row in all_bars:
+            writer.writerow([
+                row["contract"], row["barDate"],
+                row["open"], row["high"],
+                row["low"], row["close"],
+                row["volume"]
+            ])
+
+    print(f"Done! Retrieved {len(all_bars)} total bars.")
+
+
+if __name__ == "__main__":
+    main()
+