|
|
import pandas as pd |
|
|
import json |
|
|
import yfinance as yf |
|
|
import numpy as np |
|
|
import psycopg2 |
|
|
from psycopg2.extras import execute_values |
|
|
from tqdm import tqdm |
|
|
import datetime |
|
|
import schedule |
|
|
import time |
|
|
from config import SQL_CONFIG |
|
|
#SQL setting |
|
|
|
|
|
# Quick status: print the latest available dates in DB |
|
|
def print_last_update(): |
|
|
try: |
|
|
conn = psycopg2.connect(**SQL_CONFIG) |
|
|
cursor = conn.cursor() |
|
|
cursor.execute("SELECT MAX(date) FROM stock_price") |
|
|
us_last = cursor.fetchone()[0] |
|
|
cursor.execute("SELECT MAX(date) FROM stock_price_tw") |
|
|
tw_last = cursor.fetchone()[0] |
|
|
print(f"Latest US date in DB: {us_last}" if us_last else "Latest US date in DB: None") |
|
|
print(f"Latest TW date in DB: {tw_last}" if tw_last else "Latest TW date in DB: None") |
|
|
cursor.close() |
|
|
conn.close() |
|
|
except Exception as e: |
|
|
print(f"Check last update failed: {e}") |
|
|
|
|
|
def update_data(): |
|
|
# Connect to the database |
|
|
conn = psycopg2.connect(**SQL_CONFIG) |
|
|
cursor = conn.cursor() |
|
|
print("Stock prices US updating") |
|
|
# Show current latest date in table for quick verification |
|
|
try: |
|
|
cursor.execute("SELECT MAX(date) FROM stock_price") |
|
|
last_overall = cursor.fetchone()[0] |
|
|
print(f"US table current latest date: {last_overall}") |
|
|
except Exception: |
|
|
pass |
|
|
# Get the list of tickers |
|
|
cursor.execute("SELECT DISTINCT ticker FROM stock_price") |
|
|
tickers = [row[0] for row in cursor.fetchall()] |
|
|
|
|
|
for ticker in tqdm(tickers): |
|
|
# Get the latest date for this ticker |
|
|
cursor.execute(f"SELECT MAX(date) FROM stock_price WHERE ticker = '{ticker}'") |
|
|
latest_date = cursor.fetchone()[0] |
|
|
# If the latest date is not today, update the data |
|
|
if latest_date < datetime.date.today(): |
|
|
df = yf.download(ticker, start=latest_date.strftime('%Y-%m-%d'), |
|
|
end=datetime.date.today().strftime('%Y-%m-%d'), progress=False, threads=False) |
|
|
if df is None or df.empty: |
|
|
continue |
|
|
price_col = 'Close' if 'Close' in df.columns else ('Adj Close' if 'Adj Close' in df.columns else None) |
|
|
if price_col is None: |
|
|
continue |
|
|
value = [(ticker, df.index[i], float(df[price_col].iloc[i])) for i in range(len(df))] |
|
|
with conn: |
|
|
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: |
|
|
sql = "insert into stock_price (ticker, date, price) values %s" |
|
|
execute_values(curs, sql, value) |
|
|
print("Stock prices US updated") |
|
|
def update_data_tw(): |
|
|
# Connect to the database |
|
|
conn = psycopg2.connect(**SQL_CONFIG) |
|
|
cursor = conn.cursor() |
|
|
print("Stock prices US updating") |
|
|
# Show current latest date in table for quick verification |
|
|
try: |
|
|
cursor.execute("SELECT MAX(date) FROM stock_price_tw") |
|
|
last_overall = cursor.fetchone()[0] |
|
|
print(f"TW table current latest date: {last_overall}") |
|
|
except Exception: |
|
|
pass |
|
|
# Get the list of tickers |
|
|
cursor.execute("SELECT DISTINCT ticker FROM stock_price_tw") |
|
|
tickers = [row[0] for row in cursor.fetchall()] |
|
|
|
|
|
for ticker in tqdm(tickers): |
|
|
# Get the latest date for this ticker |
|
|
cursor.execute(f"SELECT MAX(date) FROM stock_price_tw WHERE ticker = '{ticker}'") |
|
|
latest_date = cursor.fetchone()[0] |
|
|
# If the latest date is not today, update the data |
|
|
if latest_date < datetime.date.today(): |
|
|
df = yf.download(ticker, start=latest_date.strftime('%Y-%m-%d'), |
|
|
end=datetime.date.today().strftime('%Y-%m-%d'), progress=False, threads=False) |
|
|
if df is None or df.empty: |
|
|
continue |
|
|
price_col = 'Close' if 'Close' in df.columns else ('Adj Close' if 'Adj Close' in df.columns else None) |
|
|
if price_col is None: |
|
|
continue |
|
|
value = [(ticker, row.name.strftime('%Y-%m-%d'), row['Close']) for _, row in df.iterrows()] |
|
|
with conn: |
|
|
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: |
|
|
sql = "insert into stock_price_tw (ticker, date, price) values %s" |
|
|
execute_values(curs, sql, value) |
|
|
print("Stock prices TW updated") |
|
|
# Print once on startup |
|
|
print_last_update() |
|
|
|
|
|
# Schedule the task |
|
|
schedule.every().day.at("17:00").do(update_data) |
|
|
schedule.every().day.at("18:00").do(update_data_tw) |
|
|
# Keep the script running |
|
|
while True: |
|
|
schedule.run_pending() |
|
|
time.sleep(1)
|
|
|
|
