Fix yfinance data download issues in data initialization scripts

- Updated data_clear&update_us_v0.py and data_clear&update_tw_v0.py
- Added proper error handling for empty DataFrames
- Added threads=False parameter to yf.download calls
- Added price column detection (Close vs Adj Close)
- Changed host from 'db' to 'localhost' for local execution
- Fixed float conversion for pandas Series elements
- Successfully updated US stocks (527 tickers) and TW stocks (1865 tickers) to 2025-09-24
data-init-fixes
Eric0801 2 months ago
parent a811f29f84
commit 9c6db84e15
  1. 11
      data_init/data_clear&update_tw_v0.py
  2. 11
      data_init/data_clear&update_us_v0.py

@ -8,7 +8,7 @@ from tqdm import tqdm
with open('assets_tw.json') as f:
data_tw = json.load(f)
#SQL setting
SQL_CONFIG = dict(database="portfolio_platform", user='postgres', password='thiispassword1qaz!QAZ', host='db',port ='5432')
SQL_CONFIG = dict(database="portfolio_platform", user='postgres', password='thiispassword1qaz!QAZ', host='localhost',port ='5432')
# TW Stocks
conn = psycopg2.connect(**SQL_CONFIG)
@ -16,8 +16,13 @@ cursor = conn.cursor()
cursor.execute("TRUNCATE TABLE stock_price_tw")
print("TW stock price cleared")
for ticker in tqdm(data_tw):
df = yf.download(ticker, start="2007-1-1", progress=False)
value =[(ticker, df.index[i], df['Close'][i]) for i in range(len(df))]
df = yf.download(ticker, start="2007-01-01", progress=False, threads=False)
if df is None or df.empty:
continue
price_col = 'Close' if 'Close' in df.columns else ('Adj Close' if 'Adj Close' in df.columns else None)
if price_col is None:
continue
value = [(ticker, df.index[i], float(df[price_col].iloc[i])) for i in range(len(df))]
with conn:
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs:
sql = "insert into stock_price_tw (ticker, date, price) values %s"

@ -9,15 +9,20 @@ with open('assets_us.json') as f:
data_tw = json.load(f)
#SQL setting
SQL_CONFIG = dict(database="portfolio_platform", user='postgres', password='thiispassword1qaz!QAZ', host='db',port ='5432')
SQL_CONFIG = dict(database="portfolio_platform", user='postgres', password='thiispassword1qaz!QAZ', host='localhost',port ='5432')
# TW Stocks
conn = psycopg2.connect(**SQL_CONFIG)
cursor = conn.cursor()
cursor.execute("TRUNCATE TABLE stock_price")
print("US stock price cleared")
for ticker in tqdm(data_tw):
df = yf.download(ticker, start="2007-1-1", progress=False)
value =[(ticker, df.index[i], df['Close'][i]) for i in range(len(df))]
df = yf.download(ticker, start="2007-01-01", progress=False, threads=False)
if df is None or df.empty:
continue
price_col = 'Close' if 'Close' in df.columns else ('Adj Close' if 'Adj Close' in df.columns else None)
if price_col is None:
continue
value = [(ticker, df.index[i], float(df[price_col].iloc[i])) for i in range(len(df))]
with conn:
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs:
sql = "insert into stock_price (ticker, date, price) values %s"

Loading…
Cancel
Save