[tpm] fix data init and daily updater (yfinance cols, threads, paths, ports)

data_init/data_init_tw_v0.py

@@ -5,11 +5,11 @@ import numpy as np
 import psycopg2
 from psycopg2.extras import execute_values
 from tqdm import tqdm
-with open('/flask/data_init/assets_tw.json') as f:
+with open('data_init/assets_tw.json') as f:
     data_tw = json.load(f)
 
 #SQL　setting
-SQL_CONFIG = dict(database="portfolio_platform", user='postgres', password='thiispassword1qaz!QAZ', host='db',port ='5432')
+SQL_CONFIG = dict(database="portfolio_platform", user='postgres', password='thiispassword1qaz!QAZ', host='localhost',port ='5432')
 
 
 # TW Stocks
@@ -19,8 +19,13 @@ cursor.execute("SELECT COUNT(*) FROM stock_price_tw;")
 row_count = cursor.fetchone()[0]
 if row_count <= 0 :
     for ticker in tqdm(data_tw):
-        df = yf.download(ticker, start="2007-1-1", progress=False)
-        value =[(ticker, df.index[i], df['Close'][i]) for i in range(len(df))]
+        df = yf.download(ticker, start="2007-01-01", progress=False, threads=False)
+        if df is None or df.empty:
+            continue
+        price_col = 'Close' if 'Close' in df.columns else ('Adj Close' if 'Adj Close' in df.columns else None)
+        if price_col is None:
+            continue
+        value = [(ticker, df.index[i], float(df[price_col].iloc[i])) for i in range(len(df))]
         with conn:
             with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs:
                 sql = "insert into stock_price_tw (ticker, date, price) values %s"

data_init/data_init_us_v0.py

@@ -5,11 +5,11 @@ import numpy as np
 import psycopg2
 from psycopg2.extras import execute_values
 from tqdm import tqdm
-with open('/flask/data_init/assets_us.json') as f:
+with open('data_init/assets_us.json') as f:
     data_tw = json.load(f)
 
 #SQL　setting
-SQL_CONFIG = dict(database="portfolio_platform", user='postgres', password='thiispassword1qaz!QAZ', host='db',port ='5432')
+SQL_CONFIG = dict(database="portfolio_platform", user='postgres', password='thiispassword1qaz!QAZ', host='localhost',port ='5432')
 
 # TW Stocks
 conn = psycopg2.connect(**SQL_CONFIG)
@@ -18,8 +18,13 @@ cursor.execute("SELECT COUNT(*) FROM stock_price;")
 row_count = cursor.fetchone()[0]
 if row_count <= 0 :
     for ticker in tqdm(data_tw):
-        df = yf.download(ticker, start="2007-1-1", progress=False)
-        value =[(ticker, df.index[i], df['Close'][i]) for i in range(len(df))]
+        df = yf.download(ticker, start="2007-01-01", progress=False, threads=False)
+        if df is None or df.empty:
+            continue
+        price_col = 'Close' if 'Close' in df.columns else ('Adj Close' if 'Adj Close' in df.columns else None)
+        if price_col is None:
+            continue
+        value = [(ticker, df.index[i], float(df[price_col].iloc[i])) for i in range(len(df))]
         with conn:
             with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs:
                 sql = "insert into stock_price (ticker, date, price) values %s"

data_init/update_data_daily.py

@@ -27,8 +27,13 @@ def update_data():
         # If the latest date is not today, update the data
         if latest_date < datetime.date.today():
             df = yf.download(ticker, start=latest_date.strftime('%Y-%m-%d'),
-                             end=datetime.date.today().strftime('%Y-%m-%d'), progress=False)
-            value = [(ticker, df.index[i], df['Close'][i]) for i in range(len(df))]
+                             end=datetime.date.today().strftime('%Y-%m-%d'), progress=False, threads=False)
+            if df is None or df.empty:
+                continue
+            price_col = 'Close' if 'Close' in df.columns else ('Adj Close' if 'Adj Close' in df.columns else None)
+            if price_col is None:
+                continue
+            value = [(ticker, df.index[i], float(df[price_col].iloc[i])) for i in range(len(df))]
             with conn:
                 with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs:
                     sql = "insert into stock_price (ticker, date, price) values %s"
@@ -50,8 +55,13 @@ def update_data_tw():
         # If the latest date is not today, update the data
         if latest_date < datetime.date.today():
             df = yf.download(ticker, start=latest_date.strftime('%Y-%m-%d'),
-                             end=datetime.date.today().strftime('%Y-%m-%d'), progress=False)
-            value = [(ticker, df.index[i], df['Close'][i]) for i in range(len(df))]
+                             end=datetime.date.today().strftime('%Y-%m-%d'), progress=False, threads=False)
+            if df is None or df.empty:
+                continue
+            price_col = 'Close' if 'Close' in df.columns else ('Adj Close' if 'Adj Close' in df.columns else None)
+            if price_col is None:
+                continue
+            value = [(ticker, df.index[i], float(df[price_col].iloc[i])) for i in range(len(df))]
             with conn:
                 with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs:
                     sql = "insert into stock_price_tw (ticker, date, price) values %s"

docker-compose.yml

@@ -12,6 +12,8 @@ services:
             - db_data_new:/var/lib/postgresql/data
         networks:
             - common_network
+        ports:
+            - 5432:5432
     redis:
         image: redis:7.0.11-alpine
         container_name: myredis