Merge pull request #166 from tgsmith61591/more-issue-140

[MRG+1] Fixes #140 (hopefully)

.circleci/config.yml

@@ -55,9 +55,9 @@ commands:
 
 jobs:
   # For testing PyPy rather than CPython
-  pypy36:
+  pypy:
     docker:
-      - image: pypy:3-6.0.0
+      - image: pypy:3.6-7.1.1
     steps:
       # Download and cache dependencies
       - restore_cache:
@@ -112,7 +112,7 @@ jobs:
     working_directory: ~/pmdarima
     steps:
     - checkout
-    - run: ./build_tools/circle/before_install.sh
+    # - run: ./build_tools/circle/before_install.sh
     - run: ./build_tools/circle/regression_testing.sh
 
   # Simple job that passes when all other tests have passed
@@ -174,7 +174,7 @@ jobs:
     working_directory: ~/pmdarima
     steps:
     - checkout
-    - run: ./build_tools/circle/before_install.sh
+    # - run: ./build_tools/circle/before_install.sh
     - run: make doc-requirements
     - run: make install
     - run: ./build_tools/circle/build_push_doc.sh
@@ -184,7 +184,7 @@ workflows:
   pmdarima-pipeline:
     jobs:
         # All testing jobs
-        - pypy36:
+        - pypy:
             filters: *test-filters
         - cpython35:
             filters: *test-filters
@@ -198,7 +198,7 @@ workflows:
             filters: *test-filters
         - testing-passed:
             requires:
-                - pypy36
+                - pypy
                 - cpython35
                 - cpython36
                 - cpython37

build_tools/circle/build_test_pypy.sh

@@ -16,16 +16,15 @@ source pypy-env/bin/activate
 python --version
 which python
 
-# This is a temporary fix for #93:
-# XXX: numpy version pinning can be reverted once PyPy
-#      compatibility is resolved for numpy v1.6.x. For instance,
-#      when PyPy3 >6.0 is released (see numpy/numpy#12740)
-pip install --extra-index https://antocuni.github.io/pypy-wheels/ubuntu numpy=="1.15.*" Cython pytest
-pip install "scipy>=1.1.0"
-pip install "scikit-learn==0.19.*"
+extra_index_url="https://antocuni.github.io/pypy-wheels/ubuntu"
+
+pip install --extra-index ${extra_index_url} numpy Cython pytest
+pip install --extra-index ${extra_index_url} scipy
+pip install --extra-index ${extra_index_url} scikit-learn
+
 # Pandas has starting throwing issues in Pypy now...
 pip install "pandas==0.23.*" statsmodels matplotlib
-pip install --extra-index https://antocuni.github.io/pypy-wheels/ubuntu pytest-mpl pytest-benchmark
+pip install --extra-index ${extra_index_url} pytest-mpl pytest-benchmark
 
 ccache -M 512M
 export CCACHE_COMPRESS=1

doc/whats_new.rst

@@ -13,6 +13,18 @@ v0.8.1) will document the latest features.
 
 * Adds a new dataset for stock prediction, along with an associated example (``load_msft``)
 
+* Fixes a bug in ``predict_in_sample``, as addressed in `#140 <https://github.com/tgsmith61591/pmdarima/issues/140>`_.
+
+* Numpy 1.16+ is now required
+
+* Statsmodels 0.10.0+ is now required
+
+
+`v1.2.1 <http://alkaline-ml.com/pmdarima/1.2.1/>`_
+--------------------------------------------------
+
+* Pins scipy at 1.2.0 to avoid a statsmodels bug.
+
 
 `v1.2.0 <http://alkaline-ml.com/pmdarima/1.2.0/>`_
 --------------------------------------------------

pmdarima/arima/arima.py

@@ -14,7 +14,6 @@
 
 from statsmodels.tsa.arima_model import ARIMA as _ARIMA
 from statsmodels.tsa.base.tsa_model import TimeSeriesModelResults
-from statsmodels.tsa.tsatools import unintegrate, unintegrate_levels
 from statsmodels import api as sm
 
 from scipy.stats import gaussian_kde, norm
@@ -89,13 +88,6 @@ def _seasonal_prediction_with_confidence(arima_res, start, end, exog, alpha,
     return f, conf_int
 
 
-def _unintegrate_differenced_preds(preds, results_wrapper, d):
-    """If predictions were generated on un-integrate, fix it"""
-    endog = results_wrapper.model.data.endog[-d:]
-    preds = unintegrate(preds, unintegrate_levels(endog, d))[d:]
-    return preds
-
-
 def _uses_legacy_pickling(arima):
     # If the package version is < 1.1.0 it uses legacy pickling behavior, but
     # a later version of the package may actually try to load a legacy model..
@@ -501,7 +493,7 @@ def _check_exog(self, exogenous):
 
     def predict_in_sample(self, exogenous=None, start=None,
                           end=None, dynamic=False, return_conf_int=False,
-                          alpha=0.05):
+                          alpha=0.05, typ='linear'):
         """Generate in-sample predictions from the fit ARIMA model.
 
         Predicts the original training (in-sample) time series values. This can
@@ -539,6 +531,11 @@ def predict_in_sample(self, exogenous=None, start=None,
         alpha : float, optional (default=0.05)
             The confidence intervals for the forecasts are (1 - alpha) %
 
+        typ : str, optional (default='linear')
+            The type of prediction to make. Options are ('linear', 'levels').
+            This is only used when the underlying model is ARIMA (not ARMA or
+            SARIMAX).
+
         Returns
         -------
         preds : array
@@ -555,17 +552,17 @@ def predict_in_sample(self, exogenous=None, start=None,
         d = self.order[1]
         results_wrapper = self.arima_res_
 
+        # The only func that has 'typ' is for ARIMAs, not ARMAs or SARIMAX
+        kwargs = {}
+        if d > 0:
+            kwargs['typ'] = typ
+
         # If not returning the confidence intervals, we have it really easy
         if not return_conf_int:
-            preds = results_wrapper.predict(exog=exogenous, start=start,
-                                            end=end, dynamic=dynamic)
-
-            # Might need to un-integrate for ARIMA
-            if not self._is_seasonal() and d > 0:
-                preds = _unintegrate_differenced_preds(
-                    preds=preds,
-                    results_wrapper=results_wrapper,
-                    d=d)
+            preds = results_wrapper.predict(
+                exog=exogenous, start=start,
+                end=end, dynamic=dynamic,
+                **kwargs)
 
             return preds
 
@@ -587,15 +584,16 @@ def predict_in_sample(self, exogenous=None, start=None,
         else:
             preds = results_wrapper.predict(
                 exog=exogenous, start=start,
-                end=end, dynamic=dynamic)
+                end=end, dynamic=dynamic, **kwargs)
 
             # We only have to un-integrate if there was any differencing
+            # TODO: prob don't need to since this occurs in the predict code
             # (i.e., ARIMA vs. ARMA)
-            if d > 0:
-                preds = _unintegrate_differenced_preds(
-                    preds=preds,
-                    results_wrapper=results_wrapper,
-                    d=d)
+            # if d > 0:
+            #     preds = _unintegrate_differenced_preds(
+            #         preds=preds,
+            #         results_wrapper=results_wrapper,
+            #         d=d)
 
             # get prediction errors
             pred_err = results_wrapper._forecast_error(len(preds))

requirements.txt

@@ -1,6 +1,6 @@
 Cython>=0.29
 joblib>=0.11
-numpy>=1.15
+numpy>=1.16
 pandas>=0.19
 scikit-learn>=0.19
 scipy>=1.3