Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Decoupling _reconcile, from _get_PW_matrices. #123

Merged
merged 3 commits into from
Dec 7, 2022
Merged

Decoupling _reconcile, from _get_PW_matrices. #123

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
cbc2cc7
Decoupled _get_PW_matrices from _reconcile
kdgutier Dec 7, 2022
4534022
Checked constant outputs in examples
kdgutier Dec 7, 2022
325e2c7
Cleaned methods notebook
kdgutier Dec 7, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions hierarchicalforecast/_modidx.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,23 +23,31 @@
'hierarchicalforecast/evaluation.py')},
'hierarchicalforecast.methods': { 'hierarchicalforecast.methods.BottomUp': ( 'methods.html#bottomup',
'hierarchicalforecast/methods.py'),
'hierarchicalforecast.methods.BottomUp._get_PW_matrices': ( 'methods.html#bottomup._get_pw_matrices',
'hierarchicalforecast/methods.py'),
'hierarchicalforecast.methods.BottomUp.reconcile': ( 'methods.html#bottomup.reconcile',
'hierarchicalforecast/methods.py'),
'hierarchicalforecast.methods.ERM': ('methods.html#erm', 'hierarchicalforecast/methods.py'),
'hierarchicalforecast.methods.ERM.__init__': ( 'methods.html#erm.__init__',
'hierarchicalforecast/methods.py'),
'hierarchicalforecast.methods.ERM._get_PW_matrices': ( 'methods.html#erm._get_pw_matrices',
'hierarchicalforecast/methods.py'),
'hierarchicalforecast.methods.ERM.reconcile': ( 'methods.html#erm.reconcile',
'hierarchicalforecast/methods.py'),
'hierarchicalforecast.methods.MiddleOut': ( 'methods.html#middleout',
'hierarchicalforecast/methods.py'),
'hierarchicalforecast.methods.MiddleOut.__init__': ( 'methods.html#middleout.__init__',
'hierarchicalforecast/methods.py'),
'hierarchicalforecast.methods.MiddleOut._get_PW_matrices': ( 'methods.html#middleout._get_pw_matrices',
'hierarchicalforecast/methods.py'),
'hierarchicalforecast.methods.MiddleOut.reconcile': ( 'methods.html#middleout.reconcile',
'hierarchicalforecast/methods.py'),
'hierarchicalforecast.methods.MinTrace': ( 'methods.html#mintrace',
'hierarchicalforecast/methods.py'),
'hierarchicalforecast.methods.MinTrace.__init__': ( 'methods.html#mintrace.__init__',
'hierarchicalforecast/methods.py'),
'hierarchicalforecast.methods.MinTrace._get_PW_matrices': ( 'methods.html#mintrace._get_pw_matrices',
'hierarchicalforecast/methods.py'),
'hierarchicalforecast.methods.MinTrace.reconcile': ( 'methods.html#mintrace.reconcile',
'hierarchicalforecast/methods.py'),
'hierarchicalforecast.methods.OptimalCombination': ( 'methods.html#optimalcombination',
Expand All @@ -50,6 +58,8 @@
'hierarchicalforecast/methods.py'),
'hierarchicalforecast.methods.TopDown.__init__': ( 'methods.html#topdown.__init__',
'hierarchicalforecast/methods.py'),
'hierarchicalforecast.methods.TopDown._get_PW_matrices': ( 'methods.html#topdown._get_pw_matrices',
'hierarchicalforecast/methods.py'),
'hierarchicalforecast.methods.TopDown.reconcile': ( 'methods.html#topdown.reconcile',
'hierarchicalforecast/methods.py'),
'hierarchicalforecast.methods._get_child_nodes': ( 'methods.html#_get_child_nodes',
Expand Down
211 changes: 128 additions & 83 deletions hierarchicalforecast/methods.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,15 @@ class BottomUp:
Economic Review, 58 , 773{787)](http://www.jstor.org/stable/1815532).
"""
insample = False


def _get_PW_matrices(self, S, idx_bottom):
n_hiers, n_bottom = S.shape
P = np.zeros_like(S, dtype=np.float32)
P[idx_bottom] = S[idx_bottom]
P = P.T
W = np.eye(n_hiers, dtype=np.float32)
return P, W

def reconcile(self,
S: np.ndarray,
y_hat: np.ndarray,
Expand All @@ -84,14 +92,10 @@ def reconcile(self,
**Returns:**<br>
`y_tilde`: Reconciliated y_hat using the Bottom Up approach.
"""
n_hiers, n_bottom = S.shape
P = np.zeros_like(S, dtype=np.float32)
P[idx_bottom] = S[idx_bottom]
P = P.T
W = np.eye(n_hiers, dtype=np.float32)
return _reconcile(S, P, W, y_hat, level=level,
sampler=sampler)

self.P, self.W = self._get_PW_matrices(S=S, idx_bottom=idx_bottom)
return _reconcile(S=S, P=self.P, W=self.W, y_hat=y_hat,
level=level, sampler=sampler)

__call__ = reconcile

# %% ../nbs/methods.ipynb 16
Expand Down Expand Up @@ -170,6 +174,33 @@ def __init__(self,
method: str):
self.method = method
self.insample = method in ['average_proportions', 'proportion_averages']

def _get_PW_matrices(self,
S: np.ndarray,
y_hat: np.ndarray,
tags: Dict[str, np.ndarray],
y_insample: Optional[np.ndarray] = None):
if not is_strictly_hierarchical(S, tags):
raise ValueError('Top down reconciliation requires strictly hierarchical structures.')

n_hiers, n_bottom = S.shape
idx_top = int(S.sum(axis=1).argmax())
levels_ = dict(sorted(tags.items(), key=lambda x: len(x[1])))
idx_bottom = levels_[list(levels_)[-1]]

y_top = y_insample[idx_top]
y_btm = y_insample[idx_bottom]
if self.method == 'average_proportions':
prop = np.mean(y_btm / y_top, axis=1)
elif self.method == 'proportion_averages':
prop = np.mean(y_btm, axis=1) / np.mean(y_top)
else:
raise Exception(f'Unknown method {self.method}')

P = np.zeros_like(S, np.float64).T #float 64 if prop is too small, happens with wiki2
P[:, idx_top] = prop
W = np.eye(n_hiers, dtype=np.float32)
return P, W

def reconcile(self,
S: np.ndarray,
Expand All @@ -192,15 +223,9 @@ def reconcile(self,
**Returns:**<br>
`y_tilde`: Reconciliated y_hat using the Top Down approach.
"""
if not is_strictly_hierarchical(S, tags):
raise ValueError('Top down reconciliation requires strictly hierarchical structures.')

n_hiers, n_bottom = S.shape
idx_top = int(S.sum(axis=1).argmax())
levels_ = dict(sorted(tags.items(), key=lambda x: len(x[1])))
idx_bottom = levels_[list(levels_)[-1]]

if self.method == 'forecast_proportions':
idx_top = int(S.sum(axis=1).argmax())
levels_ = dict(sorted(tags.items(), key=lambda x: len(x[1])))
if level is not None:
warnings.warn('Prediction intervals not implement for `forecast_proportions`')
nodes = _get_child_nodes(S=S, tags=levels_)
Expand All @@ -212,25 +237,16 @@ def reconcile(self,
reconciled = np.hstack(reconciled)
return {'mean': reconciled}
else:
y_top = y_insample[idx_top]
y_btm = y_insample[idx_bottom]
if self.method == 'average_proportions':
prop = np.mean(y_btm / y_top, axis=1)
elif self.method == 'proportion_averages':
prop = np.mean(y_btm, axis=1) / np.mean(y_top)
else:
raise Exception(f'Unknown method {self.method}')
P = np.zeros_like(S, np.float64).T #float 64 if prop is too small, happens with wiki2
P[:, idx_top] = prop
W = np.eye(n_hiers, dtype=np.float32)
return _reconcile(S, P, W, y_hat, level=level,
sampler=sampler)
self.P, self.W = self._get_PW_matrices(S=S, y_hat=y_hat,
tags=tags, y_insample=y_insample)
return _reconcile(S=S, P=self.P, W=self.W, y_hat=y_hat,
level=level,sampler=sampler)
__call__ = reconcile

# %% ../nbs/methods.ipynb 27
class MiddleOut:
"""Middle Out Reconciliation Class.

This method is only available for **strictly hierarchical structures**. It anchors the base predictions
in a middle level. The levels above the base predictions use the Bottom-Up approach, while the levels
below use a Top-Down.
Expand All @@ -240,7 +256,7 @@ class MiddleOut:
`top_down_method`: One of `forecast_proportions`, `average_proportions` and `proportion_averages`.<br>

**References:**<br>
- [Hyndman, R.J., & Athanasopoulos, G. (2021). \"Forecasting: principles and practice, 3rd edition:
- [Hyndman, R.J., & Athanasopoulos, G. (2021). \"Forecasting: principles and practice, 3rd edition:
Chapter 11: Forecasting hierarchical and grouped series.\". OTexts: Melbourne, Australia. OTexts.com/fpp3
Accessed on July 2022.](https://otexts.com/fpp3/hierarchical.html)

Expand All @@ -251,7 +267,10 @@ def __init__(self,
self.middle_level = middle_level
self.top_down_method = top_down_method
self.insample = top_down_method in ['average_proportions', 'proportion_averages']


def _get_PW_matrices(self):
raise Exception('Not implemented')

def reconcile(self,
S: np.ndarray,
y_hat: np.ndarray,
Expand Down Expand Up @@ -388,28 +407,12 @@ def __init__(self,
if method == 'mint_shrink':
self.mint_shr_ridge = mint_shr_ridge

def reconcile(self,
def _get_PW_matrices(self,
S: np.ndarray,
y_hat: np.ndarray,
y_insample: Optional[np.ndarray] = None,
y_hat_insample: Optional[np.ndarray] = None,
idx_bottom: Optional[List[int]] = None,
level: Optional[List[int]] = None,
sampler: Optional[Callable] = None):
"""MinTrace Reconciliation Method.

**Parameters:**<br>
`S`: Summing matrix of size (`base`, `bottom`).<br>
`y_hat`: Forecast values of size (`base`, `horizon`).<br>
`y_insample`: Insample values of size (`base`, `insample_size`). Only used by `wls_var`, `mint_cov`, `mint_shrink`<br>
`y_hat_insample`: Insample fitted values of size (`base`, `insample_size`). Only used by `wls_var`, `mint_cov`, `mint_shrink`<br>
`idx_bottom`: Indices corresponding to the bottom level of `S`, size (`bottom`).<br>
`level`: float list 0-100, confidence levels for prediction intervals.<br>
`sampler`: Sampler for prediction intevals, one of Normality(), Bootstrap(), PERMBU().<br>

**Returns:**<br>
`y_tilde`: Reconciliated y_hat using the MinTrace approach.
"""
idx_bottom: Optional[List[int]] = None,):
# shape residuals_insample (n_hiers, obs)
res_methods = ['wls_var', 'mint_cov', 'mint_shrink']
if self.method in res_methods and y_insample is None and y_hat_insample is None:
Expand Down Expand Up @@ -472,8 +475,45 @@ def reconcile(self,
if any(eigenvalues < 1e-8):
raise Exception(f'min_trace ({self.method}) needs covariance matrix to be positive definite.')

W_inv = np.linalg.pinv(W)
if self.nonnegative:
else:
# compute P for free reconciliation
R = S.T @ np.linalg.pinv(W)
P = np.linalg.pinv(R @ S) @ R

return P, W

def reconcile(self,
S: np.ndarray,
y_hat: np.ndarray,
y_insample: Optional[np.ndarray] = None,
y_hat_insample: Optional[np.ndarray] = None,
idx_bottom: Optional[List[int]] = None,
level: Optional[List[int]] = None,
sampler: Optional[Callable] = None):
"""MinTrace Reconciliation Method.

**Parameters:**<br>
`S`: Summing matrix of size (`base`, `bottom`).<br>
`y_hat`: Forecast values of size (`base`, `horizon`).<br>
`y_insample`: Insample values of size (`base`, `insample_size`). Only used by `wls_var`, `mint_cov`, `mint_shrink`<br>
`y_hat_insample`: Insample fitted values of size (`base`, `insample_size`). Only used by `wls_var`, `mint_cov`, `mint_shrink`<br>
`idx_bottom`: Indices corresponding to the bottom level of `S`, size (`bottom`).<br>
`level`: float list 0-100, confidence levels for prediction intervals.<br>
`sampler`: Sampler for prediction intevals, one of Normality(), Bootstrap(), PERMBU().<br>

**Returns:**<br>
`y_tilde`: Reconciliated y_hat using the MinTrace approach.
"""
self.P, self.W = self._get_PW_matrices(S=S, y_hat=y_hat,
y_insample=y_insample, y_hat_insample=y_hat_insample,
idx_bottom=idx_bottom)
if not self.nonnegative:
return _reconcile(S=S, P=self.P, W=self.W, y_hat=y_hat,
level=level, sampler=sampler)

else:
_, n_bottom = S.shape
W_inv = np.linalg.pinv(self.W)
if level is not None and type(sampler).__name__ in ['Bootstrap', 'PERMBU']:
raise Exception('nonnegative reconciliation is not compatible with bootstrap forecasts')
if idx_bottom is None:
Expand All @@ -495,22 +535,14 @@ def reconcile(self,
# the quadratic programming problem
# returns the forecasts of the bottom series
bottom_fcts = np.apply_along_axis(lambda y_hat: solve_qp(G=G, a=a @ y_hat, C=C, b=b)[0],
axis=0,
arr=y_hat)
axis=0, arr=y_hat)
if not np.all(bottom_fcts > -1e-8):
raise Exception('nonnegative optimization failed')
# remove negative values close to zero
bottom_fcts = np.clip(np.float32(bottom_fcts), a_min=0, a_max=None)
y_hat = S @ bottom_fcts
return BottomUp().reconcile(S=S, y_hat=y_hat, idx_bottom=idx_bottom,
return BottomUp().reconcile(S=S, y_hat=y_hat, idx_bottom=idx_bottom,
level=level, sampler=sampler)
else:
# compute P for free reconciliation
R = S.T @ np.linalg.pinv(W)
P = np.linalg.pinv(R @ S) @ R

return _reconcile(S, P, W, y_hat, level=level,
sampler=sampler)

__call__ = reconcile

Expand Down Expand Up @@ -612,28 +644,12 @@ def __init__(self,
self.lambda_reg = lambda_reg
self.insample = True

def reconcile(self,
def _get_PW_matrices(self,
S: np.ndarray,
y_hat: np.ndarray,
y_insample: np.ndarray,
y_hat_insample: np.ndarray,
idx_bottom: np.ndarray,
level: Optional[List[int]] = None,
sampler: Optional[Callable] = None):
"""ERM Reconciliation Method.

**Parameters:**<br>
`S`: Summing matrix of size (`base`, `bottom`).<br>
`y_hat`: Forecast values of size (`base`, `horizon`).<br>
`y_insample`: Train values of size (`base`, `insample_size`).<br>
`y_hat_insample`: Insample train predictions of size (`base`, `insample_size`).<br>
`idx_bottom`: Indices corresponding to the bottom level of `S`, size (`bottom`).<br>
`level`: float list 0-100, confidence levels for prediction intervals.<br>
`sampler`: Sampler for prediction intevals, one of Normality(), Bootstrap(), PERMBU().<br>

**Returns:**<br>
`y_tilde`: Reconciliated y_hat using the ERM approach.
"""
idx_bottom: np.ndarray):
n_hiers, n_bottom = S.shape
# y_hat_insample shape (n_hiers, obs)
# remove obs with nan values
Expand Down Expand Up @@ -670,7 +686,36 @@ def reconcile(self,

W = np.eye(n_hiers, dtype=np.float32)

return _reconcile(S, P, W, y_hat, level=level,
sampler=sampler)
return P, W

def reconcile(self,
S: np.ndarray,
y_hat: np.ndarray,
y_insample: np.ndarray,
y_hat_insample: np.ndarray,
idx_bottom: np.ndarray,
level: Optional[List[int]] = None,
sampler: Optional[Callable] = None):
"""ERM Reconciliation Method.

**Parameters:**<br>
`S`: Summing matrix of size (`base`, `bottom`).<br>
`y_hat`: Forecast values of size (`base`, `horizon`).<br>
`y_insample`: Train values of size (`base`, `insample_size`).<br>
`y_hat_insample`: Insample train predictions of size (`base`, `insample_size`).<br>
`idx_bottom`: Indices corresponding to the bottom level of `S`, size (`bottom`).<br>
`level`: float list 0-100, confidence levels for prediction intervals.<br>
`sampler`: Sampler for prediction intevals, one of Normality(), Bootstrap(), PERMBU().<br>

**Returns:**<br>
`y_tilde`: Reconciliated y_hat using the ERM approach.
"""
self.P, self.W = self._get_PW_matrices(S=S,
y_hat=y_hat,
y_insample=y_insample,
y_hat_insample=y_hat_insample,
idx_bottom=idx_bottom)
return _reconcile(S=S, P=self.P, W=self.W, y_hat=y_hat,
level=level, sampler=sampler)

__call__ = reconcile
Loading