initial commit

src/utils/__init__.py

@@ -0,0 +1 @@
+

src/utils/helpers.py

@@ -0,0 +1,46 @@
+"""
+Helper functions for the trading GNN project.
+"""
+
+from datetime import datetime, timedelta
+from typing import List
+
+from config import config
+
+
+def generate_intraday_timestamps(date: str) -> List[str]:
+    """
+    Generate all intraday timestamps for a given trading date.
+
+    Parameters:
+        date: Trading date string (YYYY-MM-DD).
+
+    Returns:
+        List of timestamp strings for the trading day.
+    """
+    market_open = datetime.strptime(config.TRADING_HOURS["start"], "%H:%M").time()
+    market_close = datetime.strptime(config.TRADING_HOURS["end"], "%H:%M").time()
+
+    open_dt = datetime.strptime(f"{date} {market_open}", "%Y-%m-%d %H:%M:%S")
+    close_dt = datetime.strptime(f"{date} {market_close}", "%Y-%m-%d %H:%M:%S")
+
+    if config.TRADING_FREQUENCY == "1min":
+        delta = timedelta(minutes=1)
+    elif config.TRADING_FREQUENCY == "5min":
+        delta = timedelta(minutes=5)
+    elif config.TRADING_FREQUENCY == "15min":
+        delta = timedelta(minutes=15)
+    elif config.TRADING_FREQUENCY == "30min":
+        delta = timedelta(minutes=30)
+    elif config.TRADING_FREQUENCY == "1h":
+        delta = timedelta(hours=1)
+    else:
+        delta = timedelta(minutes=1)
+
+    timestamps = []
+    current = open_dt
+    while current <= close_dt:
+        timestamps.append(current.strftime("%Y-%m-%d %H:%M:%S"))
+        current += delta
+
+    return timestamps

src/utils/memory_manager.py

@@ -0,0 +1,215 @@
+import gc
+import logging
+from typing import Any, Dict, Optional
+
+import torch
+
+from config import config
+
+logger = logging.getLogger(__name__)
+
+
+class MemoryManager:
+    """
+    Memory management for AMD Radeon R9700 AI Pro (32GB)
+    """
+
+    def __init__(self):
+        self.device = torch.device(config.DEVICE)
+        self.max_memory = 0
+        self.memory_limit = 0
+        self._initialize_memory()
+
+    def _initialize_memory(self):
+        """Initialize memory settings"""
+        if config.DEVICE == "cuda":
+            try:
+                # Get total GPU memory
+                self.max_memory = torch.cuda.get_device_properties(0).total_memory
+                self.memory_limit = int(self.max_memory * config.GPU_MEMORY_LIMIT)
+
+                # Set memory limit
+                torch.cuda.set_per_process_memory_fraction(config.GPU_MEMORY_LIMIT, 0)
+
+                logger.info(f"Initialized memory manager for AMD Radeon R9700 AI Pro")
+                logger.info(f"Total GPU Memory: {self.max_memory / 1024**3:.2f}GB")
+                logger.info(
+                    f"Memory Limit: {self.memory_limit / 1024**3:.2f}GB ({config.GPU_MEMORY_LIMIT * 100:.0f}%)"
+                )
+
+            except Exception as e:
+                logger.error(f"Error initializing memory manager: {str(e)}")
+                self.max_memory = 0
+                self.memory_limit = 0
+
+    def empty_cache(self):
+        """Clear the GPU cache and run garbage collection"""
+        if config.DEVICE == "cuda":
+            torch.cuda.empty_cache()
+        gc.collect()
+
+    def check_memory(self) -> Dict[str, Any]:
+        """Check current GPU memory usage"""
+        if config.DEVICE != "cuda":
+            return {
+                "allocated": 0,
+                "max_allocated": 0,
+                "total": 0,
+                "limit": 0,
+                "usage_percent": 0,
+                "free": 0,
+            }
+
+        try:
+            allocated = torch.cuda.memory_allocated(0)
+            max_allocated = torch.cuda.max_memory_allocated(0)
+            free = self.memory_limit - allocated
+
+            return {
+                "allocated": allocated,
+                "max_allocated": max_allocated,
+                "total": self.max_memory,
+                "limit": self.memory_limit,
+                "usage_percent": (allocated / self.memory_limit) * 100,
+                "free": free,
+            }
+        except Exception as e:
+            logger.error(f"Error checking memory: {str(e)}")
+            return {
+                "allocated": 0,
+                "max_allocated": 0,
+                "total": 0,
+                "limit": 0,
+                "usage_percent": 0,
+                "free": 0,
+            }
+
+    def ensure_memory(self, required_memory: int) -> bool:
+        """
+        Ensure there's enough memory for an operation
+
+        Parameters:
+        required_memory: Memory required in bytes
+
+        Returns:
+        True if there's enough memory, False otherwise
+        """
+        if config.DEVICE != "cuda":
+            return True
+
+        memory_info = self.check_memory()
+        if memory_info["allocated"] + required_memory > memory_info["limit"]:
+            # Try to free memory
+            self.empty_cache()
+            memory_info = self.check_memory()
+
+            if memory_info["allocated"] + required_memory > memory_info["limit"]:
+                logger.warning(
+                    f"Not enough GPU memory. Required: {required_memory / 1024**2:.2f}MB, "
+                    f"Available: {memory_info['free'] / 1024**2:.2f}MB"
+                )
+                return False
+
+        return True
+
+    def auto_manage_memory(self, threshold: float = 0.85):
+        """
+        Automatically manage memory based on usage
+
+        Parameters:
+        threshold: Memory usage threshold (0-1) to trigger cleanup
+        """
+        if config.DEVICE != "cuda":
+            return
+
+        memory_info = self.check_memory()
+        if memory_info["usage_percent"] > threshold * 100:
+            logger.info(
+                f"High GPU memory usage: {memory_info['usage_percent']:.2f}%. Clearing cache."
+            )
+            self.empty_cache()
+
+    def get_memory_stats(self) -> str:
+        """Get formatted memory statistics"""
+        memory_info = self.check_memory()
+        return (
+            f"GPU Memory Usage:\n"
+            f"  Allocated: {memory_info['allocated'] / 1024**3:.2f}GB\n"
+            f"  Max Allocated: {memory_info['max_allocated'] / 1024**3:.2f}GB\n"
+            f"  Total: {memory_info['total'] / 1024**3:.2f}GB\n"
+            f"  Limit: {memory_info['limit'] / 1024**3:.2f}GB\n"
+            f"  Usage: {memory_info['usage_percent']:.2f}%\n"
+            f"  Free: {memory_info['free'] / 1024**3:.2f}GB"
+        )
+
+    def estimate_model_memory(self, model: torch.nn.Module) -> int:
+        """
+        Estimate memory required for a model
+
+        Parameters:
+        model: PyTorch model
+
+        Returns:
+        Estimated memory in bytes
+        """
+        if config.DEVICE != "cuda":
+            return 0
+
+        try:
+            # Move model to GPU to get accurate memory estimate
+            model = model.to(config.DEVICE)
+
+            # Get model parameters
+            param_size = 0
+            for param in model.parameters():
+                param_size += param.nelement() * param.element_size()
+
+            # Get model buffers
+            buffer_size = 0
+            for buffer in model.buffers():
+                buffer_size += buffer.nelement() * buffer.element_size()
+
+            # Estimate forward pass memory (activations)
+            # This is a rough estimate - actual memory usage may vary
+            forward_memory = (
+                param_size * 2
+            )  # Activations typically use 2x parameter memory
+
+            # Total memory estimate
+            total_memory = param_size + buffer_size + forward_memory
+
+            # Add some buffer for overhead
+            total_memory = int(total_memory * 1.2)
+
+            return total_memory
+
+        except Exception as e:
+            logger.error(f"Error estimating model memory: {str(e)}")
+            return 0
+
+    def log_memory_usage(self, tag: str = ""):
+        """Log current memory usage"""
+        memory_info = self.check_memory()
+        logger.info(
+            f"Memory Usage {tag}: "
+            f"Allocated={memory_info['allocated'] / 1024**3:.2f}GB, "
+            f"Usage={memory_info['usage_percent']:.2f}%"
+        )
+
+    def monitor_memory(self, interval: float = 60.0):
+        """
+        Monitor memory usage in a background thread
+
+        Parameters:
+        interval: Monitoring interval in seconds
+        """
+        import threading
+
+        def monitor():
+            while True:
+                self.auto_manage_memory()
+                self.log_memory_usage("[Monitor]")
+                time.sleep(interval)
+
+        thread = threading.Thread(target=monitor, daemon=True)
+        thread.start()

src/utils/text_processing.py

@@ -0,0 +1,12 @@
+"""
+Text processing utilities for sentiment and news analysis.
+"""
+
+import re
+
+
+def clean_text(text: str) -> str:
+    """Clean and normalize text."""
+    text = re.sub(r"http\S+", "", text)
+    text = re.sub(r"[^\w\s]", "", text)
+    return text.strip().lower()

src/utils/visualization.py

@@ -0,0 +1,39 @@
+"""
+Visualization tools for the trading GNN project.
+"""
+
+import matplotlib.pyplot as plt
+import pandas as pd
+
+
+def plot_performance(portfolio_values, benchmark_values, filename):
+    """Plot portfolio performance against benchmark."""
+    plt.figure(figsize=(12, 6))
+    plt.plot(portfolio_values.index, portfolio_values.values, label="Portfolio")
+    plt.plot(benchmark_values.index, benchmark_values.values, label="Benchmark")
+    plt.title("Portfolio vs Benchmark Performance")
+    plt.xlabel("Date")
+    plt.ylabel("Value")
+    plt.legend()
+    plt.savefig(filename)
+    plt.close()
+
+
+def plot_trade_log(trade_log, filename):
+    """Plot trade log entries."""
+    # Placeholder implementation
+    plt.figure(figsize=(12, 6))
+    plt.title("Trade Log")
+    plt.xlabel("Date")
+    plt.ylabel("Trade")
+    plt.savefig(filename)
+    plt.close()
+
+
+def plot_feature_importance(feature_importance, filename):
+    """Plot feature importance."""
+    # Placeholder implementation
+    plt.figure(figsize=(10, 6))
+    plt.title("Feature Importance")
+    plt.savefig(filename)
+    plt.close()