From 0cf37e786a4c8cacc118e25bdb9491bb3d093f5e Mon Sep 17 00:00:00 2001 From: Florian Egger Date: Tue, 26 May 2026 14:10:48 +0200 Subject: [PATCH] Add comprehensive project documentation and fix data pipeline - Add detailed README with architecture diagram and usage instructions - Add API, configuration, and development documentation - Fix price data column handling for yfinance auto_adjust=True - Fix model feature dimension indexing and temporal attention batching - Add missing imports and position tracking in paper broker - Add python-dotenv support for environment variables - Update .gitignore with Python artifacts and environment files --- .gitignore | 6 + README.md | 541 ++++++++++++++++++++++++++++++++++ benchmark.py | 29 +- config.py | 19 +- docs/API.md | 406 +++++++++++++++++++++++++ docs/CONFIG.md | 69 +++++ docs/DEVELOPMENT.md | 194 ++++++++++++ live_trading.py | 1 + main.py | 8 +- requirements.txt | 2 +- src/amd/optimizations.py | 29 +- src/data/pipeline.py | 428 ++++++++++++++------------- src/evaluation/backtester.py | 51 +++- src/models/intraday_gnn.py | 15 +- src/trading/paper_broker.py | 12 +- src/utils/memory_manager.py | 1 + src/web/api/data_endpoints.py | 2 +- 17 files changed, 1536 insertions(+), 277 deletions(-) create mode 100644 README.md create mode 100644 docs/API.md create mode 100644 docs/CONFIG.md create mode 100644 docs/DEVELOPMENT.md diff --git a/.gitignore b/.gitignore index 5357087..8af6c80 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,7 @@ implementation.md +.env +*.log +__pycache__/ +*.py[cod] +*.pkl +*.db diff --git a/README.md b/README.md new file mode 100644 index 0000000..b982373 --- /dev/null +++ b/README.md @@ -0,0 +1,541 @@ +# StockGNN R9700 β€” AMD-Optimized Graph Neural Network Trading System + +A real-time intraday stock trading system built around a **Graph Neural Network (GNN)** with **Temporal Attention**, optimized for the **AMD Radeon R9700 AI Pro** GPU (32GB). The system ingests alternative data (news sentiment, social media), price bars, and corporate actions to generate trade signals across a universe of stocks. + +--- + +## πŸ“‹ Table of Contents + +- [Overview](#overview) +- [Architecture](#architecture) +- [Key Features](#key-features) +- [Project Structure](#project-structure) +- [Quick Start](#quick-start) +- [Configuration](#configuration) +- [Training](#training) +- [Live Trading](#live-trading) +- [Web Dashboard](#web-dashboard) +- [API Reference](#api-reference) +- [AMD Optimizations](#amd-optimizations) +- [Memory Management](#memory-management) +- [Data Pipeline](#data-pipeline) +- [Trading System](#trading-system) +- [Benchmarking](#benchmarking) +- [Development](#development) +- [Troubleshooting](#troubleshooting) +- [License](#license) + +--- + +## Overview + +StockGNN uses a **Graph Attention Network (GAT)** with: + +- **Temporal Attention** over multi-source feature sequences (price, news, social media) +- **Corporate Action Awareness** for splits, dividends, and delistings +- **Intraday GNN** variant with stateful LSTM for real-time inference +- **Online Learning** to adapt to regime changes during market hours +- **AMD-specific ROCm optimizations** (bf16 mixed precision, gradient checkpointing, pinned memory) + +--- + +## Architecture + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Data Sources β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”‚ Polygon β”‚ β”‚ yfinance β”‚ β”‚ NewsAPI β”‚ β”‚ Twitter/ β”‚ β”‚ +β”‚ β”‚ WebSocketβ”‚ β”‚ Historicalβ”‚ β”‚ Sentimentβ”‚ β”‚ Reddit β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ β”‚ β”‚ β”‚ + β–Ό β–Ό β–Ό β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Data Pipeline β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”‚ Price β”‚ β”‚ Corporateβ”‚ β”‚ Sector β”‚ β”‚ Alternative β”‚ β”‚ +β”‚ β”‚ Data β”‚ β”‚ Actions β”‚ β”‚ Data β”‚ β”‚ Data β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ +β”‚ β–Ό β–Ό β–Ό β–Ό β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”‚ SQLite Database (WAL mode) β”‚ β”‚ +β”‚ β”‚ price_data | corporate_actions | features | trades β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ GNN Models β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”‚ CorporateAction- β”‚ β”‚ IntradayGNN β”‚ β”‚ +β”‚ β”‚ AwareGNN β”‚ β”‚ (stateful LSTM) β”‚ β”‚ +β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ +β”‚ β”‚ TemporalAttention β”‚ β”‚ TemporalAttention β”‚ β”‚ +β”‚ β”‚ GAT Conv Layers β”‚ β”‚ GAT Conv Layers β”‚ β”‚ +β”‚ β”‚ LSTM β”‚ β”‚ LSTM β”‚ β”‚ +β”‚ β”‚ Alternative Data β”‚ β”‚ Real-time Features β”‚ β”‚ +β”‚ β”‚ Fusion β”‚ β”‚ β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Trading Execution β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”‚ Paper Broker β”‚ β”‚ IB Broker β”‚ β”‚ Risk Manager β”‚ β”‚ +β”‚ β”‚ (Simulation) β”‚ β”‚ (Interactive β”‚ β”‚ (Position β”‚ β”‚ +β”‚ β”‚ β”‚ β”‚ Brokers) β”‚ β”‚ Sizing) β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Web Dashboard β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”‚ Metrics β”‚ β”‚ Charts β”‚ β”‚ Trading β”‚ β”‚ Logs β”‚ β”‚ +β”‚ β”‚ (Live) β”‚ β”‚ (Memory, β”‚ β”‚ Controls β”‚ β”‚ (System) β”‚ β”‚ +β”‚ β”‚ β”‚ β”‚ Portfolio)β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +--- + +## Key Features + +| Feature | Description | +|---------|-------------| +| **AMD Optimizations** | ROCm bf16 mixed precision, gradient checkpointing, Flash Attention, memory-aware batching | +| **GNN Architecture** | Graph Attention Networks with temporal sequence processing and corporate action awareness | +| **Live Data** | Polygon.io WebSocket for real-time trades/quotes with automatic reconnection | +| **Alternative Data** | News sentiment (FinBERT) and social media (Twitter/Reddit) integrated into node features | +| **Online Learning** | Periodic model updates during market hours to adapt to regime shifts | +| **Risk Management** | Max drawdown, daily loss limits, volatility-target position sizing | +| **Web Dashboard** | Single-page application with real-time WebSocket metrics and Chart.js visualizations | +| **Backtesting** | Full backtesting framework with benchmark comparison and performance metrics | + +--- + +## Project Structure + +``` +stock_gnn_r9700/ +β”œβ”€β”€ config.py # Central configuration +β”œβ”€β”€ requirements.txt # Python dependencies +β”œβ”€β”€ main.py # Main training & backtesting script +β”œβ”€β”€ live_trading.py # Live intraday trading system +β”œβ”€β”€ benchmark.py # Performance benchmarking +β”œβ”€β”€ README.md # This file +β”‚ +β”œβ”€β”€ data/ +β”‚ β”œβ”€β”€ raw/ # Raw downloaded data +β”‚ β”œβ”€β”€ processed/ # Processed datasets +β”‚ └── external/ # External datasets (delisted stocks) +β”‚ +β”œβ”€β”€ models/ # Saved model weights +β”‚ +β”œβ”€β”€ src/ +β”‚ β”œβ”€β”€ amd/ +β”‚ β”‚ └── optimizations.py # AMDOptimizer, AMDSparseAttention, AMDGATConv +β”‚ β”œβ”€β”€ data/ +β”‚ β”‚ β”œβ”€β”€ pipeline.py # StockDataPipeline (SQLite, yfinance, features) +β”‚ β”‚ β”œβ”€β”€ live_data.py # LiveDataService (Polygon WebSocket) +β”‚ β”‚ β”œβ”€β”€ news_processor.py # News data processor +β”‚ β”‚ β”œβ”€β”€ social_processor.py # Social media processor +β”‚ β”‚ └── sentiment.py # Sentiment analyzer +β”‚ β”œβ”€β”€ models/ +β”‚ β”‚ β”œβ”€β”€ gnn_model.py # CorporateActionAwareGNN + TemporalAttention +β”‚ β”‚ β”œβ”€β”€ intraday_gnn.py # IntradayGNN (stateful LSTM variant) +β”‚ β”‚ └── trainer.py # GNNTrainer with AMP & gradient clipping +β”‚ β”œβ”€β”€ trading/ +β”‚ β”‚ β”œβ”€β”€ broker.py # Abstract broker interface +β”‚ β”‚ β”œβ”€β”€ paper_broker.py # Paper trading simulation +β”‚ β”‚ β”œβ”€β”€ ib_broker.py # Interactive Brokers implementation +β”‚ β”‚ └── real_time_trader.py # Real-time trading execution +β”‚ β”œβ”€β”€ evaluation/ +β”‚ β”‚ β”œβ”€β”€ backtester.py # GNNBacktester +β”‚ β”‚ β”œβ”€β”€ intraday_backtester.py +β”‚ β”‚ └── metrics.py # Sharpe, Sortino, Calmar, IR, etc. +β”‚ β”œβ”€β”€ utils/ +β”‚ β”‚ β”œβ”€β”€ memory_manager.py # MemoryManager (AMD GPU memory tracking) +β”‚ β”‚ β”œβ”€β”€ helpers.py # generate_intraday_timestamps +β”‚ β”‚ β”œβ”€β”€ visualization.py # Plotting utilities +β”‚ β”‚ └── text_processing.py # Text cleaning +β”‚ └── web/ +β”‚ β”œβ”€β”€ app.py # FastAPI entrypoint +β”‚ β”œβ”€β”€ services/ +β”‚ β”‚ └── state.py # Shared application state +β”‚ β”œβ”€β”€ api/ +β”‚ β”‚ β”œβ”€β”€ dashboard.py # Dashboard API endpoints +β”‚ β”‚ β”œβ”€β”€ trading_endpoints.py +β”‚ β”‚ β”œβ”€β”€ models_endpoints.py +β”‚ β”‚ └── data_endpoints.py +β”‚ β”œβ”€β”€ templates/ +β”‚ β”‚ └── index.html # Main dashboard template +β”‚ └── static/ +β”‚ β”œβ”€β”€ css/style.css # Dark-themed responsive UI +β”‚ └── js/app.js # Chart.js, WebSocket, API client +β”‚ +└── notebooks/ # Jupyter notebooks for exploration +``` + +--- + +## Quick Start + +### Prerequisites + +- Python 3.10+ +- AMD GPU with ROCm 5.6+ (or NVIDIA GPU with CUDA) +- 32GB+ GPU memory recommended (configurable) + +### Installation + +```bash +# Clone repository +git clone +cd stock_gnn_r9700 + +# Create virtual environment +python -m venv venv +source venv/bin/activate # Windows: venv\Scripts\activate + +# Install dependencies +pip install -r requirements.txt + +# For AMD ROCm specifically: +# pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm5.6 +# pip install torch-geometric torch-scatter torch-sparse -f https://data.pyg.org/whl/torch-2.1.0+rocm5.6.html +``` + +### Run Training + +```bash +python main.py +``` + +This will: +1. Initialize the data pipeline and download historical data +2. Create training/validation datasets +3. Train the GNN model with AMD optimizations +4. Run backtesting on validation data +5. Generate performance plots and metrics + +### Run Live Trading (Paper) + +```bash +python live_trading.py +``` + +### Run Web Dashboard + +```bash +python -m uvicorn src.web.app:app --host 0.0.0.0 --port 8000 +``` + +Then open `http://localhost:8000` in your browser. + +--- + +## Configuration + +All settings are centralized in `config.py`. Key parameters: + +```python +# GPU Settings +DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu' +AMD_GPU = True +GPU_MEMORY_LIMIT = 0.9 # Use 90% of 32GB = ~28.8GB +MIXED_PRECISION = True +PRECISION = 'bf16' # 'fp16' or 'bf16' +ROCM_OPT_LEVEL = 'O2' + +# Model Architecture +HIDDEN_CHANNELS = 128 +NUM_HEADS = 16 +DROPOUT = 0.3 +LEARNING_RATE = 0.0005 +BATCH_SIZE = 128 +SEQUENCE_LENGTH = 60 + +# Intraday Trading +TRADING_FREQUENCY = '5min' # '1min', '5min', '15min', '30min', '1h' +INITIAL_CAPITAL = 100000 +MAX_POSITION_SIZE = 0.03 # 3% of portfolio per position + +# Live Data +DATA_PROVIDER = 'polygon' # 'polygon', 'alphavantage', 'ib' +POLYGON_API_KEY = 'your_key' +``` + +--- + +## Training + +The training loop (`src/models/trainer.py`) supports: + +- **Mixed Precision Training** (bf16/fp16) via `torch.cuda.amp` +- **Gradient Checkpointing** for memory-efficient large models +- **AdamW Optimizer** with weight decay +- **ReduceLROnPlateau** scheduler +- **Automatic batch skipping** when GPU memory is constrained +- **Per-epoch benchmarking** and checkpoint saving + +Training is triggered via the Web Dashboard or `main.py`. + +--- + +## Live Trading + +The live trading system (`live_trading.py`) runs an async event loop with: + +1. **LiveDataService** β€” connects to Polygon.io WebSocket for real-time market data +2. **RealTimeTrader** β€” generates signals on configurable intervals (1min–1h) +3. **PaperTradingBroker** β€” simulates execution (or swap for InteractiveBrokersBroker) +4. **Online Learning** β€” periodic model updates every hour +5. **Memory Monitor** β€” automatic GPU cache clearing when usage exceeds 85% + +--- + +## Web Dashboard + +The dashboard provides real-time monitoring and control: + +| Feature | Description | +|---------|-------------| +| **Live Metrics** | GPU memory %, portfolio value, cash, model status via WebSocket | +| **Charts** | Memory usage and portfolio value Chart.js graphs (60-point history) | +| **Trading Controls** | Start/stop trading, view positions/orders, close positions | +| **Model Controls** | Train, save, load, benchmark models | +| **Data Controls** | View tickers, trigger data updates | +| **System Logs** | Real-time log streaming with clear button | + +--- + +## API Reference + +### Dashboard + +| Method | Endpoint | Description | +|--------|----------|-------------| +| GET | `/api/dashboard/metrics` | Current metrics (memory, account, model) | +| GET | `/api/dashboard/logs?limit=100` | Recent log entries | +| POST | `/api/dashboard/logs/clear` | Clear stored logs | + +### Trading + +| Method | Endpoint | Description | +|--------|----------|-------------| +| GET | `/api/trading/status` | Trading status, cash, positions | +| POST | `/api/trading/start` | Start live trading | +| POST | `/api/trading/stop` | Stop live trading | +| GET | `/api/trading/orders` | List all orders | +| POST | `/api/trading/order` | Submit manual order | +| POST | `/api/trading/cancel/{id}` | Cancel order | +| POST | `/api/trading/positions/close/{ticker}` | Close position | +| POST | `/api/trading/positions/close-all` | Close all positions | + +### Models + +| Method | Endpoint | Description | +|--------|----------|-------------| +| GET | `/api/models/status` | Model configuration status | +| POST | `/api/models/train` | Start training run | +| POST | `/api/models/train/stop` | Stop training | +| POST | `/api/models/save` | Save model weights | +| POST | `/api/models/load` | Load model weights | +| POST | `/api/models/benchmark` | Run performance benchmark | + +### Data + +| Method | Endpoint | Description | +|--------|----------|-------------| +| GET | `/api/data/tickers` | List tracked tickers | +| GET | `/api/data/pipeline/status` | Pipeline status | +| POST | `/api/data/update` | Trigger data update | +| GET | `/api/data/features/{ticker}` | Latest features for ticker | +| GET | `/api/data/price/{ticker}` | Latest price for ticker | +| GET | `/api/data/prices/{ticker}?limit=30` | Historical prices | + +### WebSocket + +| Event | Direction | Description | +|-------|-----------|-------------| +| `metrics` | Server β†’ Client | Real-time metrics broadcast (every 2s) | +| `ping` | Client β†’ Server | Keep-alive / latency check | + +--- + +## AMD Optimizations + +`src/amd/optimizations.py` provides: + +- **AMDOptimizer**: Configures ROCm, enables Flash Attention, sets memory limits +- **AMDSparseAttention**: Custom attention implementation optimized for AMD GPUs +- **AMDGATConv**: GAT layer using `torch_geometric.nn.MessagePassing` with `softmax` + +Key optimizations applied: + +| Technique | Benefit | +|-----------|---------| +| bf16 Mixed Precision | 2x memory reduction, faster compute on MI200/RX7000 | +| Gradient Checkpointing | Train larger models with same GPU memory | +| Flash Attention | O(N) memory vs O(NΒ²) for long sequences | +| SiLU Activation | Better AMD GPU utilization vs ReLU | +| Pinned Memory | Faster CPUβ†’GPU data transfers | + +--- + +## Memory Management + +`src/utils/memory_manager.py` provides: + +- **GPU Memory Limiting** β€” caps usage to 90% of total memory +- **Automatic Cache Clearing** β€” triggers at configurable thresholds (default 85%) +- **Per-Operation Memory Checks** β€” skips batches if insufficient memory +- **Background Monitoring** β€” logs usage every 60 seconds +- **Model Memory Estimation** β€” estimates activation memory before training + +--- + +## Data Pipeline + +`src/data/pipeline.py` handles: + +1. **SQLite Database** with WAL mode for concurrent access +2. **Price Data** from yfinance (daily OHLCV, auto-adjusted) +3. **Corporate Actions** (splits, dividends, delistings) +4. **Sector/Industry** classifications +5. **Alternative Data** (news sentiment, social media volume) +6. **Graph Construction** β€” edges based on sector correlation + +The pipeline supports incremental updates: only fetches new data since last run. + +--- + +## Trading System + +### Signal Generation + +The trading system generates signals when: +- Market is open (9:30 AM – 4:00 PM ET) +- Interval matches `TRADING_FREQUENCY` (e.g., every 5 minutes) +- Model prediction > +0.2% β†’ BUY +- Model prediction < -0.2% and position exists β†’ SELL + +### Risk Management + +| Rule | Default | +|------|---------| +| Max Position Size | 3% of portfolio | +| Max Daily Loss | 1% | +| Max Drawdown | 5% | +| Max Daily Positions | 50 | +| Max Hold Time | 4 hours | +| Min Hold Time | 10 minutes | + +### Execution + +- **VWAP** execution algorithm (configurable) +- **Paper Broker** for simulation +- **Interactive Brokers** adapter for live trading + +--- + +## Benchmarking + +`benchmark.py` measures: + +| Metric | Description | +|--------|-------------| +| Inference Time | Average forward pass latency | +| Training Time | Average full training step latency | +| Throughput | Samples/second for inference and training | +| Memory Usage | Peak GPU memory during benchmark | + +Run with: +```bash +python benchmark.py +``` + +Results are logged to `benchmark_r9700.log`. + +--- + +## Development + +### Adding a New Data Source + +1. Create a processor in `src/data/` (e.g., `src/data/earnings_processor.py`) +2. Implement `fetch_earnings(tickers, start, end)` and `get_earnings_features(ticker, date)` +3. Register in `StockDataPipeline` +4. Update `config.py` with API keys and feature definitions + +### Adding a New Model Variant + +1. Inherit from `nn.Module` in `src/models/` +2. Use `torch.utils.checkpoint.checkpoint` for memory efficiency +3. Wrap with `AMDOptimizer.optimize_model()` before training +4. Register in `GNNTrainer` + +### Adding a New Trading Strategy + +1. Subclass `Broker` for execution +2. Implement `_generate_trading_signals()` in `RealTimeTrader` +3. Configure thresholds in `config.py` + +--- + +## Troubleshooting + +### GPU Out of Memory + +- Reduce `BATCH_SIZE` in `config.py` +- Reduce `SEQUENCE_LENGTH` or `HIDDEN_CHANNELS` +- Enable gradient checkpointing (`MIXED_PRECISION = True`) +- Lower `GPU_MEMORY_LIMIT` to trigger earlier cache clearing + +### ROCm Installation Issues + +```bash +# Verify ROCm is installed +rocminfo + +# Check PyTorch sees the GPU +python -c "import torch; print(torch.cuda.is_available())" +``` + +### Data Pipeline Empty + +- Ensure `yfinance` can connect to Yahoo Finance +- Check `data/processed/` directory permissions +- Verify ticker symbols are valid + +### WebSocket Disconnects + +- Check `WEBSOCKET_MAX_RETRIES` and `WEBSOCKET_RECONNECT_DELAY` in config +- Verify Polygon.io API key is valid +- Check firewall rules for WebSocket connections + +### Model Not Loading + +- Verify `models/stock_gnn_r9700.pt` exists +- Check `config.py` `MODEL_DIR` path +- Ensure `DEVICE` matches training device (CPU vs CUDA) + +--- + +## License + +MIT License β€” see [LICENSE](LICENSE) for details. + +--- + +## Acknowledgments + +- PyTorch Geometric for GNN implementations +- yfinance for historical market data +- Polygon.io for real-time market data +- Chart.js for dashboard visualizations diff --git a/benchmark.py b/benchmark.py index abc6aeb..9a98964 100644 --- a/benchmark.py +++ b/benchmark.py @@ -4,6 +4,7 @@ import time import numpy as np import torch import torch.nn as nn +from torch_geometric.data import Data from config import config from src.amd.optimizations import AMDOptimizer @@ -44,21 +45,18 @@ def benchmark_model(): num_stocks = 50 # Number of stocks in the graph # Create random data - x = torch.randn(num_stocks, sequence_length, num_features).to(config.DEVICE) - - # Create random edges num_edges = 200 + x = torch.randn(num_stocks, sequence_length, num_features).to(config.DEVICE) edge_index = torch.randint(0, num_stocks, (2, num_edges)).to(config.DEVICE) edge_attr = torch.randn(num_edges, 1).to(config.DEVICE) - - # Create target y = torch.randn(num_stocks, 1).to(config.DEVICE) + sample_data = Data(x=x, edge_index=edge_index, edge_attr=edge_attr, y=y) # Warm-up logger.info("Warming up...") for _ in range(10): with torch.no_grad(): - _ = model((x, edge_index, edge_attr)) + _ = model(sample_data) # Benchmark inference logger.info("Benchmarking inference...") @@ -67,7 +65,7 @@ def benchmark_model(): for _ in range(num_runs): with torch.no_grad(): - _ = model((x, edge_index, edge_attr)) + _ = model(sample_data) inference_time = (time.time() - start_time) / num_runs logger.info(f"Average inference time: {inference_time:.6f} seconds") @@ -82,7 +80,7 @@ def benchmark_model(): for _ in range(num_runs): optimizer.zero_grad() - out = model((x, edge_index, edge_attr)) + out = model(sample_data) loss = criterion(out, y) loss.backward() optimizer.step() @@ -109,24 +107,25 @@ def benchmark_model(): for batch_size in batch_sizes: for seq_len in sequence_lengths: # Create data for this configuration - x = torch.randn(num_stocks, seq_len, num_features).to(config.DEVICE) - edge_index = torch.randint(0, num_stocks, (2, num_edges)).to(config.DEVICE) - edge_attr = torch.randn(num_edges, 1).to(config.DEVICE) - y = torch.randn(num_stocks, 1).to(config.DEVICE) + bx = torch.randn(num_stocks, seq_len, num_features).to(config.DEVICE) + bei = torch.randint(0, num_stocks, (2, num_edges)).to(config.DEVICE) + bea = torch.randn(num_edges, 1).to(config.DEVICE) + by = torch.randn(num_stocks, 1).to(config.DEVICE) + bdata = Data(x=bx, edge_index=bei, edge_attr=bea, y=by) # Benchmark inference start_time = time.time() for _ in range(10): # Fewer runs for detailed benchmark with torch.no_grad(): - _ = model((x, edge_index, edge_attr)) + _ = model(bdata) inf_time = (time.time() - start_time) / 10 # Benchmark training start_time = time.time() for _ in range(10): optimizer.zero_grad() - out = model((x, edge_index, edge_attr)) - loss = criterion(out, y) + out = model(bdata) + loss = criterion(out, by) loss.backward() optimizer.step() train_time = (time.time() - start_time) / 10 diff --git a/config.py b/config.py index 722326d..2def8db 100644 --- a/config.py +++ b/config.py @@ -2,6 +2,9 @@ import os from datetime import datetime, timedelta import torch +from dotenv import load_dotenv + +load_dotenv() class Config: @@ -10,7 +13,7 @@ class Config: VERSION = "1.0.0" # Data directories - BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + BASE_DIR = os.path.dirname(os.path.abspath(__file__)) DATA_DIR = os.path.join(BASE_DIR, "data") RAW_DATA_DIR = os.path.join(DATA_DIR, "raw") PROCESSED_DATA_DIR = os.path.join(DATA_DIR, "processed") @@ -111,11 +114,11 @@ class Config: DATA_BUFFER_SIZE = 5000 # Number of data points to keep in memory DATA_FLUSH_INTERVAL = 300 # seconds - how often to flush data to database - # Alternative data settings - NEWS_API_KEY = "your_news_api_key" - TWITTER_BEARER_TOKEN = "your_twitter_bearer_token" - REDDIT_CLIENT_ID = "your_reddit_client_id" - REDDIT_CLIENT_SECRET = "your_reddit_client_secret" + # Alternative data settings (set via .env or environment variables) + NEWS_API_KEY = os.environ.get("NEWS_API_KEY", "") + TWITTER_BEARER_TOKEN = os.environ.get("TWITTER_BEARER_TOKEN", "") + REDDIT_CLIENT_ID = os.environ.get("REDDIT_CLIENT_ID", "") + REDDIT_CLIENT_SECRET = os.environ.get("REDDIT_CLIENT_SECRET", "") NEWS_LOOKBACK_DAYS = 7 # Number of days to look back for news SOCIAL_MEDIA_LOOKBACK_DAYS = 3 # Number of days to look back for social media @@ -159,8 +162,8 @@ class Config: # Live trading settings LIVE_DATA_ENABLED = True DATA_PROVIDER = "polygon" # 'polygon', 'alphavantage', 'ib', 'tdameritrade' - POLYGON_API_KEY = "your_polygon_api_key" - ALPHA_VANTAGE_API_KEY = "your_alpha_vantage_api_key" + POLYGON_API_KEY = os.environ.get("POLYGON_API_KEY", "") + ALPHA_VANTAGE_API_KEY = os.environ.get("ALPHA_VANTAGE_API_KEY", "") IB_HOST = "127.0.0.1" IB_PORT = 7497 IB_CLIENT_ID = 1 diff --git a/docs/API.md b/docs/API.md new file mode 100644 index 0000000..f0abfa7 --- /dev/null +++ b/docs/API.md @@ -0,0 +1,406 @@ +# API Reference + +Base URL: `http://localhost:8000` + +--- + +## Dashboard + +### `GET /api/dashboard/metrics` + +Returns current system metrics. + +**Response:** +```json +{ + "memory": { + "allocated_gb": 4.2, + "max_allocated_gb": 5.1, + "total_gb": 32.0, + "limit_gb": 28.8, + "usage_percent": 13.1, + "free_gb": 24.6 + }, + "account": { + "cash": 100000.00, + "total_value": 100000.00, + "positions_count": 0, + "positions": {} + }, + "model": { + "status": "loaded", + "device": "cuda", + "amd_gpu": true, + "mixed_precision": true, + "precision": "bf16", + "hidden_channels": 128, + "num_heads": 16, + "batch_size": 128, + "learning_rate": 0.0005 + }, + "system": { + "project_name": "StockGNN_R9700", + "version": "1.0.0", + "training_active": false, + "trading_active": false + } +} +``` + +### `GET /api/dashboard/logs?limit=100` + +Returns recent log entries. + +**Response:** +```json +{ + "logs": [ + "[2024-01-15 09:30:00] Trading started", + "[2024-01-15 09:30:02] Model loaded from models/stock_gnn_r9700.pt" + ], + "total": 2 +} +``` + +### `POST /api/dashboard/logs/clear` + +Clears stored logs. + +**Response:** +```json +{"status": "cleared"} +``` + +--- + +## Trading + +### `GET /api/trading/status` + +**Response:** +```json +{ + "active": false, + "cash": 100000.00, + "total_value": 100000.00, + "positions": {}, + "orders_count": 0 +} +``` + +### `POST /api/trading/start` + +Starts live trading. + +**Response:** +```json +{"status": "started"} +``` + +### `POST /api/trading/stop` + +Stops live trading. + +**Response:** +```json +{"status": "stopped"} +``` + +### `GET /api/trading/orders` + +Lists all orders. + +**Response:** +```json +[ + { + "order_id": "abc-123", + "ticker": "AAPL", + "action": "buy", + "quantity": 100, + "price": 150.0, + "timestamp": "2024-01-15 09:30:00", + "type": "market", + "status": "filled" + } +] +``` + +### `POST /api/trading/order` + +Submits a manual order. + +**Request Body:** +```json +{ + "ticker": "AAPL", + "action": "buy", + "quantity": 100, + "price": 150.0, + "timestamp": "2024-01-15 09:30:00", + "type": "market" +} +``` + +**Response:** +```json +{"status": "submitted", "order_id": "abc-123"} +``` + +### `POST /api/trading/cancel/{order_id}` + +Cancels an order. + +**Response:** +```json +{"status": "cancelled"} +``` + +### `POST /api/trading/positions/close/{ticker}` + +Closes a position. + +**Response:** +```json +{"status": "submitted", "order_id": "def-456"} +``` + +### `POST /api/trading/positions/close-all` + +Closes all positions. + +**Response:** +```json +{ + "status": "submitted", + "results": [ + {"ticker": "AAPL", "order_id": "ghi-789"} + ] +} +``` + +--- + +## Models + +### `GET /api/models/status` + +**Response:** +```json +{ + "status": "loaded", + "training_active": false, + "device": "cuda", + "amd_gpu": true, + "mixed_precision": true, + "precision": "bf16", + "model_name": "stock_gnn_r9700", + "hidden_channels": 128, + "num_heads": 16, + "dropout": 0.3, + "learning_rate": 0.0005, + "batch_size": 128, + "epochs": 200, + "sequence_length": 60 +} +``` + +### `POST /api/models/train` + +Starts training. + +**Response:** +```json +{"status": "started"} +``` + +### `POST /api/models/train/stop` + +Stops training. + +**Response:** +```json +{"status": "stopped"} +``` + +### `POST /api/models/save` + +Saves model weights. + +**Response:** +```json +{"status": "saved"} +``` + +### `POST /api/models/load` + +Loads model weights. + +**Response:** +```json +{"status": "loaded"} +``` + +### `POST /api/models/benchmark` + +Runs performance benchmark. + +**Response:** +```json +{ + "status": "complete", + "inference_time_ms": 2.341, + "training_time_ms": 8.567, + "inference_throughput": 427.0, + "training_throughput": 116.7, + "memory_allocated_gb": 4.5, + "device": "cuda" +} +``` + +--- + +## Data + +### `GET /api/data/tickers` + +**Response:** +```json +{ + "initial": ["AAPL", "MSFT", "GOOGL", ...], + "index": "^GSPC", + "count": 30 +} +``` + +### `GET /api/data/pipeline/status` + +**Response:** +```json +{ + "status": "ready", + "tickers_loaded": 30, + "db_path": "data/processed/stock_data.db" +} +``` + +### `POST /api/data/update` + +Triggers a full data update. + +**Response:** +```json +{"status": "started"} +``` + +### `GET /api/data/features/{ticker}` + +**Response:** +```json +{ + "ticker": "AAPL", + "timestamp": "2024-01-15 09:30:00", + "features": { + "AAPL": { + "ticker": "AAPL", + "timestamp": "2024-01-15 09:30:00", + "return": 0.0, + "volatility": 0.2, + ... + } + } +} +``` + +### `GET /api/data/price/{ticker}` + +**Response:** +```json +{ + "ticker": "AAPL", + "date": "2024-01-15 00:00:00", + "open": 150.0, + "high": 152.0, + "low": 149.0, + "close": 151.0, + "adj_close": 151.0, + "volume": 50000000 +} +``` + +### `GET /api/data/prices/{ticker}?limit=30` + +**Response:** +```json +[ + { + "date": "2024-01-15", + "open": 150.0, + "high": 152.0, + "low": 149.0, + "close": 151.0, + "volume": 50000000 + } +] +``` + +### `GET /api/data/corporate-actions/{ticker}` + +**Response:** +```json +{ + "ticker": "AAPL", + "actions": { + "splits": {"2020-08-31": 4.0}, + "dividends": {"2024-01-10": 0.24} + } +} +``` + +--- + +## WebSocket + +### Connection + +```javascript +const ws = new WebSocket('ws://localhost:8000/ws'); +``` + +### Server β†’ Client Messages + +**Type: `metrics`** +```json +{ + "type": "metrics", + "timestamp": "2024-01-15T09:30:00.000000", + "memory": { + "allocated_gb": 4.2, + "total_gb": 32.0, + "usage_percent": 13.1 + }, + "account": { + "cash": 100000.00, + "total_value": 100000.00, + "positions": 0 + }, + "model": { + "status": "loaded", + "device": "cuda", + "mixed_precision": true, + "precision": "bf16" + } +} +``` + +### Client β†’ Server Messages + +**Ping:** +```json +{"action": "ping"} +``` + +**Subscribe:** +```json +{"action": "subscribe", "channel": "all"} +``` diff --git a/docs/CONFIG.md b/docs/CONFIG.md new file mode 100644 index 0000000..dbf4075 --- /dev/null +++ b/docs/CONFIG.md @@ -0,0 +1,69 @@ +# Configuration Guide + +All configuration is centralized in `config.py` via the `Config` class. + +## GPU Settings + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `DEVICE` | `'cuda'` | PyTorch device (auto-detected) | +| `AMD_GPU` | `True` | Enable AMD-specific optimizations | +| `GPU_MEMORY_LIMIT` | `0.9` | Fraction of GPU memory to use (0.0–1.0) | +| `ROCM_OPT_LEVEL` | `'O2'` | ROCm JIT optimization level | +| `MIXED_PRECISION` | `True` | Enable AMP | +| `PRECISION` | `'bf16'` | `'bf16'` or `'fp16'` | + +## Model Hyperparameters + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `HIDDEN_CHANNELS` | `128` | GNN hidden dimension | +| `NUM_HEADS` | `16` | Attention heads | +| `DROPOUT` | `0.3` | Dropout rate | +| `LEARNING_RATE` | `0.0005` | AdamW learning rate | +| `BATCH_SIZE` | `128` | Training batch size | +| `SEQUENCE_LENGTH` | `60` | Temporal window length | +| `EPOCHS` | `200` | Max training epochs | + +## Trading Settings + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `TRADING_FREQUENCY` | `'5min'` | Signal generation interval | +| `INITIAL_CAPITAL` | `100000` | Starting portfolio value | +| `MAX_POSITION_SIZE` | `0.03` | Max position as fraction of portfolio | +| `MAX_DAILY_LOSS` | `0.01` | Daily loss circuit breaker | +| `MAX_DRAWDOWN` | `0.05` | Max portfolio drawdown | + +## Data Providers + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `DATA_PROVIDER` | `'polygon'` | Live data source | +| `POLYGON_API_KEY` | β€” | Polygon.io API key | +| `ALPHA_VANTAGE_API_KEY` | β€” | Alpha Vantage API key | +| `IB_HOST` | `'127.0.0.1'` | Interactive Brokers TWS host | +| `IB_PORT` | `7497` | TWS API port | + +## Feature Definitions + +Three feature vectors are defined and referenced by the models: + +```python +NEWS_FEATURES = ['sentiment', 'volume', 'recency', 'source_reliability', 'topic_relevance'] +SOCIAL_FEATURES = ['twitter_sentiment', 'twitter_volume', 'reddit_sentiment', 'reddit_volume', 'social_momentum'] +INTRADAY_FEATURES = ['return', 'volatility', 'momentum', 'volume_momentum', 'bid_ask_spread', 'bid_ask_spread_pct', 'volume_imbalance', 'order_flow', 'vwap_deviation'] +``` + +## Environment Variables + +Sensitive keys can be overridden via environment variables: + +```bash +export POLYGON_API_KEY="your_key" +export ALPHA_VANTAGE_API_KEY="your_key" +export NEWS_API_KEY="your_key" +export TWITTER_BEARER_TOKEN="your_token" +``` + +These are read in `config.py` and fall back to hardcoded placeholders if not set. diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md new file mode 100644 index 0000000..65bf490 --- /dev/null +++ b/docs/DEVELOPMENT.md @@ -0,0 +1,194 @@ +# Development Guide + +## Setting Up Development Environment + +```bash +# Clone repo +git clone +cd stock_gnn_r9700 + +# Create virtual environment +python -m venv venv +source venv/bin/activate + +# Install in editable mode + dev dependencies +pip install -r requirements.txt +pip install black isort mypy pytest pytest-asyncio +``` + +## Code Style + +Format with `black` and `isort`: + +```bash +black src/ config.py main.py live_trading.py benchmark.py +isort src/ config.py main.py live_trading.py benchmark.py +``` + +Type-check with `mypy`: + +```bash +mypy src/ +``` + +## Running Tests + +```bash +pytest tests/ -v +``` + +## Adding a New Data Processor + +1. Create file in `src/data/`: + +```python +# src/data/earnings_processor.py +import logging +from typing import Dict, List + +logger = logging.getLogger(__name__) + +class EarningsProcessor: + def __init__(self): + self.earnings_data = {} + + def fetch_earnings(self, tickers: List[str], start_date: str, end_date: str): + """Fetch earnings data.""" + pass + + def get_earnings_features(self, ticker: str, date: str) -> Dict: + """Return earnings-based features.""" + return {"earnings_surprise": 0.0, "eps_growth": 0.0} +``` + +2. Import and initialize in `StockDataPipeline.__init__`: + +```python +from src.data.earnings_processor import EarningsProcessor + +class StockDataPipeline: + def __init__(self): + ... + self.earnings_processor = EarningsProcessor() +``` + +3. Call `fetch_earnings` in `update_alternative_data` and include features in `create_training_dataset`. + +## Adding a New Model + +1. Create file in `src/models/`: + +```python +# src/models/my_model.py +import torch +import torch.nn as nn +from config import config + +class MyModel(nn.Module): + def __init__(self, num_features: int): + super().__init__() + self.fc = nn.Linear(num_features, 1) + + def forward(self, data): + return self.fc(data.x) +``` + +2. Wrap with `AMDOptimizer` before training: + +```python +from src.amd.optimizations import AMDOptimizer + +model = MyModel(num_features) +model = AMDOptimizer().optimize_model(model) +``` + +## Adding a New Trading Strategy + +1. Subclass `Broker` in `src/trading/`: + +```python +from src.trading.broker import Broker + +class AlpacaBroker(Broker): + def submit_order(self, order): + # Implementation + pass +``` + +2. Instantiate in `live_trading.py` and pass to `RealTimeTrader`. + +## Adding API Endpoints + +1. Create router in `src/web/api/`: + +```python +from fastapi import APIRouter + +router = APIRouter() + +@router.get("/health") +async def health(): + return {"status": "ok"} +``` + +2. Include in `src/web/app.py`: + +```python +from src.web.api import my_endpoints +app.include_router(my_endpoints.router, prefix="/api/my") +``` + +## Environment Variables + +Set in `.env` file (not committed): + +```bash +POLYGON_API_KEY=your_key +ALPHA_VANTAGE_API_KEY=your_key +IB_CLIENT_ID=1 +``` + +Load with `python-dotenv` if needed. + +## Git Workflow + +```bash +# Feature branch +git checkout -b feature/my-feature + +# Commit +git add . +git commit -m "feat: add my feature" + +# Push +git push origin feature/my-feature +``` + +## Debugging + +### GPU Memory Issues + +```python +from src.utils.memory_manager import MemoryManager + +mm = MemoryManager() +print(mm.get_memory_stats()) +``` + +### Model Inspection + +```python +from src.models.gnn_model import CorporateActionAwareGNN + +model = CorporateActionAwareGNN(num_features=10) +print(sum(p.numel() for p in model.parameters())) # parameter count +``` + +### Data Inspection + +```python +from src.data.pipeline import StockDataPipeline + +pipe = StockDataPipeline() +print(pipe.price_data['AAPL'].tail()) +``` diff --git a/live_trading.py b/live_trading.py index 0d82106..d68376b 100644 --- a/live_trading.py +++ b/live_trading.py @@ -4,6 +4,7 @@ import time from datetime import datetime, timedelta from typing import Dict, List +import numpy as np import torch from src.trading.paper_broker import PaperTradingBroker from src.trading.real_time_trader import RealTimeTrader diff --git a/main.py b/main.py index 096cf84..b489b04 100644 --- a/main.py +++ b/main.py @@ -58,8 +58,8 @@ def main(): # Initialize model logger.info("Initializing GNN model") - # Get number of features from first data point - num_features = train_dataset[0].x.shape[1] + # x has shape (num_stocks, seq_len, num_features); features are in the last dim + num_features = train_dataset[0].x.shape[2] model = CorporateActionAwareGNN(num_features) # Optimize model for AMD GPU @@ -95,9 +95,9 @@ def main(): backtester = GNNBacktester(model, pipeline) portfolio_values, trade_log = backtester.run_backtest(val_dataset) - # Get benchmark data + # Get benchmark data (auto_adjust=True means 'Close' already contains adjusted prices) benchmark_data = pipeline.price_data[config.INDEX_TICKER] - benchmark_values = benchmark_data.loc[portfolio_values.index]["Adj Close"] + benchmark_values = benchmark_data.loc[portfolio_values.index]["Close"] # Calculate performance metrics logger.info("Calculating performance metrics") diff --git a/requirements.txt b/requirements.txt index 1739a39..7023826 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ pytz>=2023.3 requests>=2.28.0 websockets>=11.0 SQLAlchemy>=2.0.0 -sqlite3>=3.40.0 # Part of Python standard library +python-dotenv>=1.0.0 # AMD GPU support for PyTorch (ROCm 5.6) torch>=2.1.0 # ROCm-compatible version diff --git a/src/amd/optimizations.py b/src/amd/optimizations.py index d6a0690..79dcfd4 100644 --- a/src/amd/optimizations.py +++ b/src/amd/optimizations.py @@ -54,17 +54,12 @@ class AMDOptimizer: def optimize_model(self, model: nn.Module): """Apply AMD-specific optimizations to a model""" + model = model.to(self.device) + if config.DEVICE != "cuda" or not config.AMD_GPU: return model try: - # Move model to GPU - model = model.to(self.device) - - # Apply mixed precision if enabled - if config.MIXED_PRECISION: - model = self._apply_mixed_precision(model) - # Apply memory optimizations model = self._apply_memory_optimizations(model) @@ -73,22 +68,14 @@ class AMDOptimizer: except Exception as e: logger.error(f"Error optimizing model: {str(e)}") - return model.to(self.device) + return model def _apply_mixed_precision(self, model: nn.Module): - """Apply mixed precision training to the model""" - # Convert model to use mixed precision - if config.PRECISION == "fp16": - model = model.half() - elif config.PRECISION == "bf16": - model = model.to(torch.bfloat16) - - # Convert specific layers to full precision if needed - for name, module in model.named_modules(): - if isinstance(module, (nn.LayerNorm, nn.BatchNorm2d)): - module = module.float() - - logger.info(f"Applied mixed precision training with {config.PRECISION}") + """Mixed precision is handled via torch.autocast in the training loop. + Permanently casting parameters causes numerical instability; this is a no-op.""" + logger.info( + f"Mixed precision ({config.PRECISION}) handled via torch.autocast in training loop" + ) return model def _apply_memory_optimizations(self, model: nn.Module): diff --git a/src/data/pipeline.py b/src/data/pipeline.py index cbab3b6..7d8234e 100644 --- a/src/data/pipeline.py +++ b/src/data/pipeline.py @@ -308,7 +308,7 @@ class StockDataPipeline: "high": row["High"], "low": row["Low"], "close": row["Close"], - "adj_close": row["Adj Close"], + "adj_close": row["Close"], # auto_adjust=True, 'Close' is already adjusted "volume": row["Volume"], } ) @@ -382,37 +382,33 @@ class StockDataPipeline: continue def _store_corporate_actions(self, ticker: str): - """Store corporate actions in the database""" + """Store corporate actions in the database using a single connection.""" if ticker not in self.corporate_actions: return actions = self.corporate_actions[ticker] + rows = [] - # Store splits for date, ratio in actions["splits"].items(): - with sqlite3.connect(self.db_path) as conn: - cursor = conn.cursor() - cursor.execute( - """ - INSERT OR REPLACE INTO corporate_actions - (ticker, date, action_type, value, details) - VALUES (?, ?, ?, ?, ?) - """, - (ticker, date, "split", ratio, f"Split ratio: {ratio}"), - ) + rows.append((ticker, date, "split", ratio, f"Split ratio: {ratio}")) - # Store dividends for date, amount in actions["dividends"].items(): - with sqlite3.connect(self.db_path) as conn: - cursor = conn.cursor() - cursor.execute( - """ - INSERT OR REPLACE INTO corporate_actions - (ticker, date, action_type, value, details) - VALUES (?, ?, ?, ?, ?) + rows.append( + (ticker, date, "dividend", amount, f"Dividend amount: {amount}") + ) + + if not rows: + return + + with sqlite3.connect(self.db_path) as conn: + conn.executemany( + """ + INSERT OR REPLACE INTO corporate_actions + (ticker, date, action_type, value, details) + VALUES (?, ?, ?, ?, ?) """, - (ticker, date, "dividend", amount, f"Dividend amount: {amount}"), - ) + rows, + ) def update_sector_data(self, tickers: List[str]): """Update sector data for given tickers with memory management""" @@ -540,207 +536,242 @@ class StockDataPipeline: self.social_processor.fetch_twitter_data(tickers, start_date, end_date) self.social_processor.fetch_reddit_data(tickers, start_date, end_date) - def create_training_dataset(self) -> List: + def _compute_stock_sequence( + self, ticker: str, date, pit_data: Dict + ) -> Optional[np.ndarray]: """ - Create a training dataset with alternative data features and AMD optimizations + Build a (SEQUENCE_LENGTH, NUM_FEATURES) float32 array for one stock at one date. - Returns: - List of PyG Data objects + NUM_FEATURES = 5 price + len(NEWS_FEATURES) + len(SOCIAL_FEATURES) + 1 corp action + """ + price_df = self.price_data.get(ticker) + if price_df is None or price_df.empty: + return None + + # Fetch enough history to compute SEQUENCE_LENGTH trading-day feature vectors + buf_start = date - timedelta(days=config.SEQUENCE_LENGTH * 2 + 60) + window = price_df.loc[buf_start:date] + if len(window) < 2: + return None + + closes = window["Close"].values.astype(np.float64) + volumes = window["Volume"].values.astype(np.float64) + + raw_returns = np.diff(closes) / (closes[:-1] + 1e-10) + n = len(raw_returns) + + seq_rows: List[List[float]] = [] + for i in range(n): + ret = float(raw_returns[i]) + recent = raw_returns[max(0, i - 19) : i + 1] + vol = float(np.std(recent)) if len(recent) > 1 else 0.0 + mom = float(np.mean(recent)) + log_vol = float(np.log(volumes[i + 1] + 1)) + norm_price = float(closes[i + 1] / (closes[0] + 1e-10) - 1) + seq_rows.append([ret, vol, mom, log_vol, norm_price]) + + seq_arr = np.array(seq_rows, dtype=np.float32) + + # Trim to SEQUENCE_LENGTH (pad with zeros if history is too short) + if len(seq_arr) > config.SEQUENCE_LENGTH: + seq_arr = seq_arr[-config.SEQUENCE_LENGTH :] + elif len(seq_arr) < config.SEQUENCE_LENGTH: + pad = np.zeros( + (config.SEQUENCE_LENGTH - len(seq_arr), 5), dtype=np.float32 + ) + seq_arr = np.vstack([pad, seq_arr]) + + # News features (current date, broadcast across all timesteps) + date_str = date.strftime("%Y-%m-%d") + nf = self.news_processor.get_news_features(ticker, date_str) + news_vec = np.array( + [ + nf.get("news_sentiment", 0.0), + nf.get("news_volume", 0.0), + nf.get("news_recency", 0.0), + nf.get("news_source_reliability", 0.0), + nf.get("news_topic_relevance", 0.0), + ], + dtype=np.float32, + ) + + # Social features (current date, broadcast) + sf = self.social_processor.get_social_features(ticker, date_str) + social_vec = np.array( + [ + sf.get("twitter_sentiment", 0.0), + sf.get("twitter_volume", 0.0), + sf.get("reddit_sentiment", 0.0), + sf.get("reddit_volume", 0.0), + sf.get("social_momentum", 0.0), + ], + dtype=np.float32, + ) + + # Corporate action flag + corp_flag = 0.0 + if pit_data["upcoming_actions"]: + soonest = min(pit_data["upcoming_actions"], key=lambda a: a["days_until"]) + corp_flag = 1.0 if soonest["type"] == "split" else 2.0 + + alt_vec = np.concatenate([news_vec, social_vec, [corp_flag]]) # (11,) + alt_broadcast = np.tile(alt_vec, (config.SEQUENCE_LENGTH, 1)) # (seq_len, 11) + + return np.concatenate([seq_arr, alt_broadcast], axis=1) # (seq_len, 16) + + def _build_edges( + self, tickers: List[str], date, pit_cache: Dict + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Build same-sector edges with return-correlation weights.""" + lookback_start = date - timedelta(days=config.LOOKBACK_WINDOW) + edge_index: List[List[int]] = [] + edge_weight: List[float] = [] + + for i, t1 in enumerate(tickers): + for j, t2 in enumerate(tickers): + if j <= i: + continue + s1 = pit_cache[t1].get("sector") + s2 = pit_cache[t2].get("sector") + if not s1 or not s2 or s1 != s2: + continue + try: + r1 = ( + self.price_data[t1] + .loc[lookback_start:date]["Close"] + .pct_change() + .dropna() + ) + r2 = ( + self.price_data[t2] + .loc[lookback_start:date]["Close"] + .pct_change() + .dropna() + ) + if len(r1) > 5 and len(r2) > 5: + corr = float(r1.corr(r2)) + if not np.isnan(corr): + edge_index.append([i, j]) + edge_weight.append(corr) + except Exception as e: + logger.warning(f"Edge {t1}-{t2}: {e}") + + if edge_index: + ei = torch.tensor(edge_index, dtype=torch.long).t().contiguous() + ew = torch.tensor(edge_weight, dtype=torch.float32).unsqueeze(1) + else: + ei = torch.empty((2, 0), dtype=torch.long) + ew = torch.empty((0, 1), dtype=torch.float32) + return ei, ew + + def _next_trading_date(self, date) -> Optional[object]: + """Return the next date that has price data for at least one ticker.""" + candidate = date + timedelta(days=1) + for _ in range(7): + for ticker in self.price_data.values(): + if not ticker.empty and candidate in ticker.index: + return candidate + candidate += timedelta(days=1) + return None + + def _create_dataset(self, start_date: str, end_date: str) -> List: + """ + Core dataset builder producing PyG Data objects with 3-D node features. + + x shape per object: (num_stocks, SEQUENCE_LENGTH, NUM_FEATURES) + where NUM_FEATURES = 5 price + 5 news + 5 social + 1 corp_action = 16 """ from torch_geometric.data import Data - logger.info("Creating training dataset with AMD optimizations") - dates = pd.date_range(config.START_DATE, config.TRAIN_END_DATE) + all_tickers = config.INITIAL_TICKERS + list(self.delisted_tickers) + dates = pd.bdate_range(start_date, end_date) # business days only dataset = [] - for date in tqdm(dates, desc="Creating dataset"): - # Check memory before processing date - if not self.memory_manager.ensure_memory(500 * 1024**2): # 500MB - logger.warning(f"Skipping {date.date()} due to memory constraints") + for date in tqdm(dates, desc=f"Dataset {start_date}β†’{end_date}"): + if not self.memory_manager.ensure_memory(500 * 1024**2): + logger.warning(f"Skipping {date.date()} β€” low memory") self.memory_manager.empty_cache() continue - # Get current universe of stocks - current_tickers = [] - for ticker in config.INITIAL_TICKERS + list(self.delisted_tickers): - if ticker in self.price_data and not self.price_data[ticker].empty: - if ( - date >= self.price_data[ticker].index[0] - and date <= self.price_data[ticker].index[-1] - ): - current_tickers.append(ticker) - - # Skip if no stocks available - if not current_tickers: + # Stocks with price data spanning this date + valid_tickers = [ + t + for t in all_tickers + if t in self.price_data + and not self.price_data[t].empty + and self.price_data[t].index[0] <= date <= self.price_data[t].index[-1] + ] + if not valid_tickers: continue - # Create node features - node_features = [] - corporate_action_flags = [] + # Pre-fetch PIT data once per ticker (avoids O(nΒ²) repeated calls) + pit_cache = { + t: self.get_point_in_time_data(t, date) for t in valid_tickers + } - for ticker in current_tickers: - # Check memory before processing ticker - if not self.memory_manager.ensure_memory(10 * 1024**2): # 10MB - logger.warning(f"Skipping {ticker} due to memory constraints") + # Build 3-D node feature matrix + node_sequences: List[np.ndarray] = [] + node_tickers: List[str] = [] + for ticker in valid_tickers: + if not self.memory_manager.ensure_memory(10 * 1024**2): continue - try: - # Get price data for lookback period - lookback_start = date - timedelta(days=config.LOOKBACK_WINDOW) - price_data = self.price_data[ticker].loc[lookback_start:date] - - if ( - len(price_data) < 5 - ): # Need at least 5 days for meaningful features - # Use default values - features = [ - 0, - 0, - 0, - 0, - 0, - ] # [return, volatility, momentum, volume, price] - else: - returns = price_data["Adj Close"].pct_change().dropna() - features = [ - returns.iloc[-1], # Last return - returns.std(), # Volatility - returns.mean(), # Momentum - np.log(price_data["Volume"].iloc[-1] + 1), # Log volume - price_data["Adj Close"].iloc[-1], # Last price - ] - - node_features.append(features) - - # Get corporate action flag - pit_data = self.get_point_in_time_data(ticker, date) - flag = 0 - if pit_data["upcoming_actions"]: - # Use the type of the soonest upcoming action - soonest = min( - pit_data["upcoming_actions"], - key=lambda x: x["days_until"], - ) - if soonest["type"] == "split": - flag = 1 - elif soonest["type"] == "dividend": - flag = 2 - - corporate_action_flags.append(flag) - + seq = self._compute_stock_sequence(ticker, date, pit_cache[ticker]) + if seq is not None: + node_sequences.append(seq) + node_tickers.append(ticker) except Exception as e: - logger.warning( - f"Error processing {ticker} for {date.date()}: {str(e)}" - ) - continue + logger.warning(f"Sequence error {ticker} {date.date()}: {e}") - # Skip if no features were created - if not node_features: + if not node_sequences: continue - # Convert to tensors - x = torch.tensor(node_features, dtype=torch.float32) + # x: (num_stocks, seq_len, num_features) + x = torch.tensor(np.array(node_sequences), dtype=torch.float32) - # Add corporate action flags as additional features - corporate_action_tensor = torch.tensor( - corporate_action_flags, dtype=torch.float32 - ).unsqueeze(1) - x = torch.cat([x, corporate_action_tensor], dim=1) + # Cross-sectional z-score per feature per time step (normalise across stocks) + mean = x.mean(dim=0, keepdim=True) + std = x.std(dim=0, keepdim=True).clamp(min=1e-8) + x = (x - mean) / std - # Create edges based on sector relationships - edge_index = [] - edge_weight = [] + # Build graph edges + edge_index, edge_weight = self._build_edges(node_tickers, date, pit_cache) - for i, ticker1 in enumerate(current_tickers): - for j, ticker2 in enumerate(current_tickers): - if i < j: - # Check memory before processing edge - if not self.memory_manager.ensure_memory(1 * 1024**2): # 1MB - logger.warning( - "Skipping edge creation due to memory constraints" - ) - continue - - try: - # Get sector data - pit1 = self.get_point_in_time_data(ticker1, date) - pit2 = self.get_point_in_time_data(ticker2, date) - - # Create edge if same sector - if ( - pit1["sector"] - and pit2["sector"] - and pit1["sector"] == pit2["sector"] - ): - # Calculate correlation as edge weight - lookback_start = date - timedelta( - days=config.LOOKBACK_WINDOW - ) - returns1 = ( - self.price_data[ticker1] - .loc[lookback_start:date]["Adj Close"] - .pct_change() - ) - returns2 = ( - self.price_data[ticker2] - .loc[lookback_start:date]["Adj Close"] - .pct_change() - ) - - if len(returns1) > 5 and len(returns2) > 5: - corr = returns1.corr(returns2) - if not np.isnan(corr): - edge_index.append([i, j]) - edge_weight.append(corr) - except Exception as e: - logger.warning( - f"Error creating edge between {ticker1} and {ticker2}: {str(e)}" - ) - continue - - # Convert to tensors - edge_index = ( - torch.tensor(edge_index, dtype=torch.long).t() - if edge_index - else torch.empty((2, 0), dtype=torch.long) - ) - edge_weight = ( - torch.tensor(edge_weight, dtype=torch.float32).unsqueeze(1) - if edge_weight - else torch.empty((0, 1), dtype=torch.float32) - ) - - # Create target (next day's return) - y = [] - for ticker in current_tickers: - next_date = date + timedelta(days=1) + # Build targets: next trading day's return for each stock + next_date = self._next_trading_date(date) + y_vals: List[float] = [] + for ticker in node_tickers: if ( - ticker in self.price_data + next_date is not None + and ticker in self.price_data and next_date in self.price_data[ticker].index + and date in self.price_data[ticker].index ): - ret = ( - self.price_data[ticker].loc[next_date]["Adj Close"] - / self.price_data[ticker].loc[date]["Adj Close"] + ret = float( + self.price_data[ticker].loc[next_date]["Close"] + / self.price_data[ticker].loc[date]["Close"] - 1 ) - y.append(ret) else: - y.append(0) # Default value + ret = 0.0 + y_vals.append(ret) - y = torch.tensor(y, dtype=torch.float32).unsqueeze(1) + y = torch.tensor(y_vals, dtype=torch.float32).unsqueeze(1) - # Create Data object data = Data(x=x, edge_index=edge_index, edge_attr=edge_weight, y=y) data.date = date - data.tickers = current_tickers - + data.tickers = node_tickers dataset.append(data) - # Memory management self.memory_manager.auto_manage_memory(threshold=0.7) return dataset + def create_training_dataset(self) -> List: + """Create the training dataset for config.START_DATE β†’ config.TRAIN_END_DATE.""" + logger.info("Creating training dataset") + return self._create_dataset(config.START_DATE, config.TRAIN_END_DATE) + def create_intraday_dataset( self, tickers: List[str], start_date: str, end_date: str ) -> List: @@ -825,12 +856,12 @@ class StockDataPipeline: if isinstance(daily_data, pd.Series): # If only one day of data, create a sequence with the same values features = [ - daily_data["Adj Close"] / daily_data["Open"] + daily_data["Close"] / daily_data["Open"] - 1, # Return 0.2, # Volatility (placeholder) 0.0, # Momentum (placeholder) np.log(daily_data["Volume"] + 1), # Log volume - daily_data["Adj Close"], # Price + daily_data["Close"], # Price (auto_adjust=True) ] # Repeat for the sequence @@ -895,12 +926,12 @@ class StockDataPipeline: ) - timedelta(days=config.LOOKBACK_WINDOW) returns1 = ( self.price_data[ticker1] - .loc[lookback_start:date]["Adj Close"] + .loc[lookback_start:date]["Close"] .pct_change() ) returns2 = ( self.price_data[ticker2] - .loc[lookback_start:date]["Adj Close"] + .loc[lookback_start:date]["Close"] .pct_change() ) @@ -938,10 +969,10 @@ class StockDataPipeline: and next_date.strftime("%Y-%m-%d") in self.price_data[ticker].index ): - current_price = self.price_data[ticker].loc[date]["Adj Close"] + current_price = self.price_data[ticker].loc[date]["Close"] future_price = self.price_data[ticker].loc[ next_date.strftime("%Y-%m-%d") - ]["Adj Close"] + ]["Close"] ret = future_price / current_price - 1 y.append(ret) else: @@ -993,7 +1024,7 @@ class StockDataPipeline: price_data = self.price_data[ticker] idx = price_data.index.get_indexer([date], method="ffill")[0] if idx >= 0: - result["price"] = price_data.iloc[idx]["Adj Close"] + result["price"] = price_data.iloc[idx]["Close"] # Get sector data if ticker in self.sector_data: @@ -1117,15 +1148,12 @@ class StockDataPipeline: "volatility": 0.2, # Placeholder - would calculate from intraday data "momentum": 0.0, # Placeholder "volume": np.log(price_data["Volume"] + 1), - "price": price_data["Adj Close"], + "price": price_data["Close"], } return features def create_validation_dataset(self, start_date: str, end_date: str) -> List: - """ - Create a validation dataset (placeholder - reuses training logic) - """ - # For simplicity, reuse create_training_dataset logic with a different date range - # In a real implementation, this would be more sophisticated - return self.create_training_dataset() + """Create a validation dataset for the given date range.""" + logger.info(f"Creating validation dataset {start_date} β†’ {end_date}") + return self._create_dataset(start_date, end_date) diff --git a/src/evaluation/backtester.py b/src/evaluation/backtester.py index 428dcfb..bbec96a 100644 --- a/src/evaluation/backtester.py +++ b/src/evaluation/backtester.py @@ -3,7 +3,8 @@ Backtesting framework for the GNN trading strategy. """ import logging -from typing import Dict, List, Tuple +from datetime import timedelta +from typing import Dict, List, Optional, Tuple import pandas as pd @@ -22,6 +23,30 @@ class GNNBacktester: self.pipeline = pipeline self.broker = PaperTradingBroker(initial_cash=config.INITIAL_CAPITAL) + def _get_current_price(self, ticker: str, date) -> Optional[float]: + """Look up the adjusted close price for a ticker on a given date.""" + price_df = self.pipeline.price_data.get(ticker) + if price_df is None or price_df.empty: + return None + try: + if date in price_df.index: + return float(price_df.loc[date]["Close"]) + idx = price_df.index.get_indexer([date], method="ffill")[0] + if idx >= 0: + return float(price_df.iloc[idx]["Close"]) + except Exception: + pass + return None + + def _portfolio_value(self, date) -> float: + """Compute total portfolio value using current market prices.""" + total = self.broker.cash + for ticker, qty in self.broker.positions.items(): + price = self._get_current_price(ticker, date) + if price: + total += qty * price + return total + def run_backtest(self, dataset: List) -> Tuple[pd.Series, List]: """Run a backtest on the given dataset.""" logger.info(f"Starting backtest with {len(dataset)} samples") @@ -31,30 +56,36 @@ class GNNBacktester: trade_log = [] for data in dataset: - # Get predictions - predictions = self.model(data) + date = getattr(data, "date", None) + + # Move data to the same device as the model + data_device = data.to(config.DEVICE) + + with __import__("torch").no_grad(): + predictions = self.model(data_device) - # Simulate trading based on predictions for i, ticker in enumerate(data.tickers): pred = predictions[i].item() + price = self._get_current_price(ticker, date) + + if price is None or price <= 0: + continue if pred > 0.002: order = { "ticker": ticker, "action": "buy", "quantity": 100, - "price": 100, # placeholder - "timestamp": str(getattr(data, "date", "")), + "price": price, + "timestamp": str(date), "type": "market", } order_id = self.broker.submit_order(order) if order_id: trade_log.append({**order, "order_id": order_id}) - # Record portfolio value - account = self.broker.get_account_summary() - portfolio_values.append(account["total_value"]) - dates.append(getattr(data, "date", None)) + portfolio_values.append(self._portfolio_value(date)) + dates.append(date) portfolio_series = pd.Series(portfolio_values, index=dates) logger.info("Backtest completed") diff --git a/src/models/intraday_gnn.py b/src/models/intraday_gnn.py index 5791da6..30c70d3 100644 --- a/src/models/intraday_gnn.py +++ b/src/models/intraday_gnn.py @@ -132,19 +132,8 @@ class IntradayGNN(nn.Module): x, edge_index, edge_attr = data.x, data.edge_index, data.edge_attr # x shape: (num_stocks, sequence_length, num_features) - batch_size, seq_len, num_features = x.size() - - # Apply temporal attention to each stock's sequence - temporal_features = [] - for i in range(batch_size): - stock_sequence = x[i].unsqueeze(0) # (1, sequence_length, num_features) - temporal_feature = self.temporal_attention(stock_sequence) - temporal_features.append(temporal_feature) - - # Stack temporal features - temporal_features = torch.cat( - temporal_features, dim=0 - ) # (num_stocks, feature_dim) + # TemporalAttention expects (batch, seq_len, feature_dim) β€” pass all stocks at once + temporal_features = self.temporal_attention(x) # (num_stocks, feature_dim) # Process features processed_features = self.feature_processor(temporal_features) diff --git a/src/trading/paper_broker.py b/src/trading/paper_broker.py index cea115a..d437fd7 100644 --- a/src/trading/paper_broker.py +++ b/src/trading/paper_broker.py @@ -18,6 +18,7 @@ class PaperTradingBroker(Broker): def __init__(self, initial_cash: float = 100000.0): self.cash = initial_cash self.positions = {} + self.position_prices = {} # fill price per ticker for valuation self.orders = {} self.transaction_cost = config.TRANSACTION_COST @@ -39,6 +40,7 @@ class PaperTradingBroker(Broker): return None self.cash -= cost self.positions[ticker] = self.positions.get(ticker, 0) + quantity + self.position_prices[ticker] = price elif action == "sell": if self.positions.get(ticker, 0) < quantity: logger.warning(f"Insufficient shares for sell order: {order_id}") @@ -48,6 +50,7 @@ class PaperTradingBroker(Broker): self.positions[ticker] -= quantity if self.positions[ticker] == 0: del self.positions[ticker] + self.position_prices.pop(ticker, None) logger.info( f"Paper order filled: {order_id} - {action} {quantity} {ticker} @ {price}" @@ -70,11 +73,12 @@ class PaperTradingBroker(Broker): return self.positions.copy() def get_account_summary(self) -> Dict: - """Get simulated account summary.""" - total_value = self.cash + sum( - self.positions.get(t, 0) * 100 # placeholder price - for t in self.positions + """Get simulated account summary using fill prices for position valuation.""" + position_value = sum( + qty * self.position_prices.get(ticker, 0) + for ticker, qty in self.positions.items() ) + total_value = self.cash + position_value return { "cash": self.cash, "positions": self.positions.copy(), diff --git a/src/utils/memory_manager.py b/src/utils/memory_manager.py index 534e04d..f937b23 100644 --- a/src/utils/memory_manager.py +++ b/src/utils/memory_manager.py @@ -1,5 +1,6 @@ import gc import logging +import time from typing import Any, Dict, Optional import torch diff --git a/src/web/api/data_endpoints.py b/src/web/api/data_endpoints.py index 3238d28..b9746be 100644 --- a/src/web/api/data_endpoints.py +++ b/src/web/api/data_endpoints.py @@ -101,7 +101,7 @@ async def get_price(ticker: str) -> Dict: "high": latest["High"], "low": latest["Low"], "close": latest["Close"], - "adj_close": latest["Adj Close"], + "adj_close": latest["Close"], # auto_adjust=True; Close is already adjusted "volume": int(latest["Volume"]), }