Add AMD GPU detection, replace LSTM with MLP, and fix signal generation logic
- Detect AMD GPUs via ROCm device name in config - Replace single-timestep LSTM in GNN model with leaner post-GNN MLP - Pass edge_attr through AMDGATConv propagate and use ones for self-loops - Fix live trading to sell existing positions on negative signals instead of skipping them entirely - Use per-file try/except in data pipeline and batch SQLite inserts - Import torch directly in backtester instead of dynamic __import__ - Update AMP autocast import for PyTorch 2.0+ compatibility
This commit is contained in:
+52
-51
@@ -147,25 +147,26 @@ class StockDataPipeline:
|
||||
|
||||
def _load_data(self):
|
||||
"""Load existing data from disk with memory management"""
|
||||
try:
|
||||
# Check memory before loading
|
||||
if not self.memory_manager.ensure_memory(2 * 1024**3): # 2GB
|
||||
logger.warning("Skipping data load due to memory constraints")
|
||||
return
|
||||
if not self.memory_manager.ensure_memory(2 * 1024**3): # 2GB
|
||||
logger.warning("Skipping data load due to memory constraints")
|
||||
return
|
||||
|
||||
self.price_data = self._load_pickle("price_data.pkl")
|
||||
self.corporate_actions = self._load_pickle("corporate_actions.pkl")
|
||||
self.sector_data = self._load_pickle("sector_data.pkl")
|
||||
self.index_composition = self._load_pickle("index_composition.pkl")
|
||||
for attr, filename in [
|
||||
("price_data", "price_data.pkl"),
|
||||
("corporate_actions", "corporate_actions.pkl"),
|
||||
("sector_data", "sector_data.pkl"),
|
||||
("index_composition", "index_composition.pkl"),
|
||||
]:
|
||||
try:
|
||||
setattr(self, attr, self._load_pickle(filename))
|
||||
logger.info(f"Loaded {filename}")
|
||||
except FileNotFoundError:
|
||||
logger.info(f"{filename} not found, using empty store")
|
||||
except Exception as e:
|
||||
logger.error(f"Error loading {filename}: {e}", exc_info=True)
|
||||
self.memory_manager.empty_cache()
|
||||
|
||||
logger.info("Loaded existing data from disk")
|
||||
self.memory_manager.log_memory_usage("[After Data Load]")
|
||||
|
||||
except FileNotFoundError:
|
||||
logger.info("No existing data found. Starting with empty databases.")
|
||||
except Exception as e:
|
||||
logger.error(f"Error loading data: {str(e)}", exc_info=True)
|
||||
self.memory_manager.empty_cache()
|
||||
self.memory_manager.log_memory_usage("[After Data Load]")
|
||||
|
||||
def _save_data(self):
|
||||
"""Save data to disk with memory management"""
|
||||
@@ -497,19 +498,23 @@ class StockDataPipeline:
|
||||
|
||||
def _store_index_composition(self):
|
||||
"""Store index composition in the database"""
|
||||
for index_ticker, composition in self.index_composition.items():
|
||||
for date, members in composition.items():
|
||||
for member in members:
|
||||
with sqlite3.connect(self.db_path) as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT OR REPLACE INTO index_composition
|
||||
(index_ticker, date, member_ticker)
|
||||
VALUES (?, ?, ?)
|
||||
""",
|
||||
(index_ticker, date, member),
|
||||
)
|
||||
rows = [
|
||||
(index_ticker, date, member)
|
||||
for index_ticker, composition in self.index_composition.items()
|
||||
for date, members in composition.items()
|
||||
for member in members
|
||||
]
|
||||
if not rows:
|
||||
return
|
||||
with sqlite3.connect(self.db_path) as conn:
|
||||
conn.executemany(
|
||||
"""
|
||||
INSERT OR REPLACE INTO index_composition
|
||||
(index_ticker, date, member_ticker)
|
||||
VALUES (?, ?, ?)
|
||||
""",
|
||||
rows,
|
||||
)
|
||||
|
||||
def update_alternative_data(self, tickers: List[str]):
|
||||
"""
|
||||
@@ -806,6 +811,20 @@ class StockDataPipeline:
|
||||
self.memory_manager.empty_cache()
|
||||
continue
|
||||
|
||||
# Compute once per day — stock universe and PIT data don't change intraday
|
||||
pd_date = pd.Timestamp(date)
|
||||
current_tickers = [
|
||||
t for t in tickers
|
||||
if t in self.price_data
|
||||
and not self.price_data[t].empty
|
||||
and self.price_data[t].index[0] <= pd_date <= self.price_data[t].index[-1]
|
||||
]
|
||||
if not current_tickers:
|
||||
continue
|
||||
|
||||
date_dt = datetime.strptime(date, "%Y-%m-%d")
|
||||
pit_cache = {t: self.get_point_in_time_data(t, date_dt) for t in current_tickers}
|
||||
|
||||
# Get all timestamps for this trading day
|
||||
timestamps = generate_intraday_timestamps(date)
|
||||
|
||||
@@ -813,20 +832,6 @@ class StockDataPipeline:
|
||||
current_timestamp = timestamps[i]
|
||||
sequence_start = timestamps[i - config.SEQUENCE_LENGTH]
|
||||
|
||||
# Get current universe of stocks
|
||||
current_tickers = []
|
||||
for ticker in tickers:
|
||||
if ticker in self.price_data and not self.price_data[ticker].empty:
|
||||
if (
|
||||
date >= self.price_data[ticker].index[0]
|
||||
and date <= self.price_data[ticker].index[-1]
|
||||
):
|
||||
current_tickers.append(ticker)
|
||||
|
||||
# Skip if no stocks available
|
||||
if not current_tickers:
|
||||
continue
|
||||
|
||||
# Create node features for each stock in the sequence
|
||||
sequence_features = []
|
||||
valid_tickers = []
|
||||
@@ -905,13 +910,9 @@ class StockDataPipeline:
|
||||
continue
|
||||
|
||||
try:
|
||||
# Get sector relationship
|
||||
pit1 = self.get_point_in_time_data(
|
||||
ticker1, datetime.strptime(date, "%Y-%m-%d")
|
||||
)
|
||||
pit2 = self.get_point_in_time_data(
|
||||
ticker2, datetime.strptime(date, "%Y-%m-%d")
|
||||
)
|
||||
# Use pre-fetched PIT data — avoids O(n²) repeated calls
|
||||
pit1 = pit_cache[ticker1]
|
||||
pit2 = pit_cache[ticker2]
|
||||
|
||||
if (
|
||||
pit1["sector"]
|
||||
|
||||
Reference in New Issue
Block a user