Docs for capital-gains repair; Fix warnings in price-repair
This commit is contained in:
ValueRaider
@@ -111,6 +111,7 @@ Fix errors in dividends:
|
||||
2. duplicate dividend (within 7 days)
|
||||
3. dividend 100x too big/small for the ex-dividend price drop
|
||||
4. ex-div date wrong (price drop is few days/weeks after)
|
||||
5. **NEW: capital gains double-counted**
|
||||
|
||||
Most errors I've seen are on London stock exchange (£/pence mixup), but no exchange is safe.
|
||||
|
||||
@@ -275,3 +276,27 @@ TETY.ST
|
||||
2022-06-21 00:00:00+02:00 71.599998 60.007881 0.0
|
||||
2022-06-20 00:00:00+02:00 71.800003 60.175503 0.0
|
||||
2022-06-17 00:00:00+02:00 71.000000 59.505021 0.0
|
||||
|
||||
|
||||
Capital-gains double-counted
|
||||
----------------------------
|
||||
|
||||
Clue: price drop matches dividend better than dividend+capital gains.
|
||||
|
||||
DODFX
|
||||
|
||||
.. code-block:: text
|
||||
|
||||
# ORIGINAL:
|
||||
Close Adj Close Dividends Capital Gains
|
||||
Date
|
||||
2025-12-18 00:00:00-05:00 16.219999 16.219999 0.837 0.417
|
||||
2025-12-17 00:00:00-05:00 16.920000 15.665999 0.000 0.000
|
||||
|
||||
.. code-block:: text
|
||||
|
||||
# REPAIRED:
|
||||
Close Adj Close Dividends Capital Gains
|
||||
Date
|
||||
2025-12-18 00:00:00-05:00 16.219999 16.219999 0.42 0.417
|
||||
2025-12-17 00:00:00-05:00 16.920000 16.083000 0.00 0.000
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
Date,Open,High,Low,Close,Adj Close,Volume,Dividends,Stock Splits
|
||||
2022-05-30 00:00:00+01:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2022-05-23 00:00:00+01:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2022-05-16 00:00:00+01:00,14.5500,14.5500,14.5500,14.5500,14.5500,532454,0,0
|
||||
2022-05-09 00:00:00+01:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2022-05-02 00:00:00+01:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2022-04-25 00:00:00+01:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2022-04-18 00:00:00+01:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2022-04-11 00:00:00+01:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2022-04-04 00:00:00+01:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2022-03-28 00:00:00+01:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2022-03-21 00:00:00+00:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2022-03-14 00:00:00+00:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2022-03-07 00:00:00+00:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2022-02-28 00:00:00+00:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2022-02-21 00:00:00+00:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2022-02-14 00:00:00+00:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2022-02-07 00:00:00+00:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2022-01-31 00:00:00+00:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2022-01-24 00:00:00+00:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2022-01-17 00:00:00+00:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2022-01-10 00:00:00+00:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2022-01-03 00:00:00+00:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2021-12-27 00:00:00+00:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2021-12-20 00:00:00+00:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2021-12-13 00:00:00+00:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2021-12-06 00:00:00+00:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2021-11-29 00:00:00+00:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2021-11-22 00:00:00+00:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2021-11-15 00:00:00+00:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2021-11-08 00:00:00+00:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2021-11-01 00:00:00+00:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2021-10-25 00:00:00+01:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2021-10-18 00:00:00+01:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2021-10-11 00:00:00+01:00,14.5500,14.5500,14.5500,14.5500,14.5500,0,0,0
|
||||
2021-10-04 00:00:00+01:00,14.8000,15.3400,14.4000,14.5500,14.5500,2171373,0,0
|
||||
2021-09-27 00:00:00+01:00,15.6000,16.0000,14.9000,15.0500,15.0500,3860549,0,0
|
||||
|
@@ -1,37 +0,0 @@
|
||||
Date,Open,High,Low,Close,Adj Close,Volume,Dividends,Stock Splits
|
||||
2022-05-30 00:00:00+01:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2022-05-23 00:00:00+01:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2022-05-16 00:00:00+01:00,14.550000190734863,14.550000190734863,0.14550000429153442,0.14550000429153442,0.14550000429153442,532454,0.0,0.0
|
||||
2022-05-09 00:00:00+01:00,14.550000190734863,14.550000190734863,14.550000190734863,14.550000190734863,14.550000190734863,0,0.0,0.0
|
||||
2022-05-02 00:00:00+01:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2022-04-25 00:00:00+01:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2022-04-18 00:00:00+01:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2022-04-11 00:00:00+01:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2022-04-04 00:00:00+01:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2022-03-28 00:00:00+01:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2022-03-21 00:00:00+00:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2022-03-14 00:00:00+00:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2022-03-07 00:00:00+00:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2022-02-28 00:00:00+00:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2022-02-21 00:00:00+00:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2022-02-14 00:00:00+00:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2022-02-07 00:00:00+00:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2022-01-31 00:00:00+00:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2022-01-24 00:00:00+00:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2022-01-17 00:00:00+00:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2022-01-10 00:00:00+00:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2022-01-03 00:00:00+00:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2021-12-27 00:00:00+00:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2021-12-20 00:00:00+00:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2021-12-13 00:00:00+00:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2021-12-06 00:00:00+00:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2021-11-29 00:00:00+00:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2021-11-22 00:00:00+00:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2021-11-15 00:00:00+00:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2021-11-08 00:00:00+00:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2021-11-01 00:00:00+00:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2021-10-25 00:00:00+01:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2021-10-18 00:00:00+01:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2021-10-11 00:00:00+01:00,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0.14550000429153442,0,0.0,0.0
|
||||
2021-10-04 00:00:00+01:00,14.800000190734863,15.34000015258789,0.14399999380111694,0.14550000429153442,0.14550000429153442,2171373,0.0,0.0
|
||||
2021-09-27 00:00:00+01:00,15.600000381469727,16.0,14.899999618530273,15.050000190734863,15.050000190734863,3860549,0.0,0.0
|
||||
|
@@ -303,10 +303,9 @@ class TestPriceRepair(unittest.TestCase):
|
||||
# Sometimes Yahoo suddenly shifts from cents->$ from some recent date.
|
||||
|
||||
tkrs = ['AET.L', 'SSW.JO']
|
||||
intervals = ['1d', '1wk']
|
||||
# tkrs = ['AET.L']
|
||||
# intervals = ['1wk']
|
||||
# import yfinance as yf ; yf.enable_debug_mode()
|
||||
# intervals = ['1d', '1wk']
|
||||
# Give up repairing 1wk intervals directly. Instead will resample from 1d
|
||||
intervals = ['1d']
|
||||
|
||||
for tkr in tkrs:
|
||||
for interval in intervals:
|
||||
@@ -539,7 +538,6 @@ class TestPriceRepair(unittest.TestCase):
|
||||
false_positives['NVDA'] = {'interval': '1d', 'start': '2001-07-01', 'end': '2007-09-15'}
|
||||
# yf.config.debug.logging = True
|
||||
for tkr, args in false_positives.items():
|
||||
print(f"Testing: {tkr}")
|
||||
interval = args['interval']
|
||||
dat = yf.Ticker(tkr, session=self.session)
|
||||
tz_exchange = dat.fast_info["timezone"]
|
||||
@@ -605,7 +603,6 @@ class TestPriceRepair(unittest.TestCase):
|
||||
|
||||
# Phantom divs
|
||||
bad_tkrs += ['KAP.IL'] # 1x 1d phantom div, and false positives 0.01x in 1wk
|
||||
bad_tkrs += ['SAND']
|
||||
bad_tkrs += ['TEM.L']
|
||||
bad_tkrs += ['TEP.PA']
|
||||
|
||||
|
||||
@@ -1429,7 +1429,7 @@ class PriceHistory:
|
||||
# Consider price drop to decide if Yahoo double-counted -
|
||||
# drop should = true dividend + capital gains
|
||||
# But need to account for normal price volatility:
|
||||
df['Price_Change%'] = df['Close'].pct_change().abs()
|
||||
df['Price_Change%'] = df['Close'].pct_change(fill_method=None).abs()
|
||||
no_distributions = (df['Dividends'] == 0) & (df['Capital Gains'] == 0)
|
||||
price_drop_pct_mean = df.loc[no_distributions, 'Price_Change%'].mean()
|
||||
df = df.drop('Price_Change%', axis=1)
|
||||
@@ -2680,7 +2680,14 @@ class PriceHistory:
|
||||
df_workings = df2.copy()
|
||||
df_workings = df_workings.drop(['Adj Close', 'Dividends', 'Stock Splits', 'Repaired?'], axis=1, errors='ignore')
|
||||
df_workings = df_workings.rename(columns={'Volume': 'Vol'})
|
||||
df_workings['Vol'] = (df_workings['Vol']/1e6).astype('int').astype('str') + 'm'
|
||||
fna = df_workings['Vol'].isna()
|
||||
if fna.any():
|
||||
df_workings['VolStr'] = ''
|
||||
df_workings.loc[fna, 'VolStr'] = 'NaN'
|
||||
df_workings.loc[~fna, 'VolStr'] = (df_workings['Vol'][~fna]/1e6).astype('int').astype('str') + 'm'
|
||||
df_workings['Vol'] = df_workings['VolStr'] ; df_workings.drop('VolStr', axis=1)
|
||||
else:
|
||||
df_workings['Vol'] = (df_workings['Vol']/1e6).astype('int').astype('str') + 'm'
|
||||
debug_cols = ['Close']
|
||||
df_workings = df_workings.drop([c for c in OHLC if c not in debug_cols], axis=1, errors='ignore')
|
||||
|
||||
@@ -2816,7 +2823,8 @@ class PriceHistory:
|
||||
if f_up_shifts.any():
|
||||
nf_up_shifts = ~f_up_shifts
|
||||
flat_indices = np.where(nf_up_shifts)[0]
|
||||
down_dts = df2.index[f_down]
|
||||
f_down_ndims = len(f_down.shape)
|
||||
down_dts = df2.index[f_down if f_down_ndims==1 else f_down.any(axis=1)]
|
||||
for idx in np.where(f_up_shifts)[0]:
|
||||
i = idx-1 # this is when price actually dropped
|
||||
dt = df2.index[i]
|
||||
@@ -2890,9 +2898,10 @@ class PriceHistory:
|
||||
def _calc_volume_zscore(volume, block):
|
||||
# print(f"_calc_volume_zscore(volume={volume})")
|
||||
values = block['Volume'].to_numpy()
|
||||
mean = np.mean(values)
|
||||
std = np.std(values, ddof=1)
|
||||
# print(f"# mean={mean:.0f} std={std:.4f}")
|
||||
if std == 0.0:
|
||||
return 0
|
||||
mean = np.mean(values)
|
||||
z_score = (volume - mean) / std
|
||||
return z_score
|
||||
|
||||
|
||||
Reference in New Issue
Block a user