Merge pull request #4 from mt-jones/refactor

Pull refactor in for testing

.gitignore

@@ -1,6 +1,7 @@
 ## common
 __pycache__
 *.pyc
+*.a
 *.o
 *.so
 *.dylib
@@ -9,18 +10,34 @@ __pycache__
 .vscode
 *.swp
 *.pytest_cache
-build
+DartConfiguration.tcl
+.DS_Store
+
+##build
 htmlcov
+dist/
+cudf.egg-info/
+python/build
+python/cudf/bindings/*.cpp
+
+## Patching
+*.diff
+*.orig
+*.rej
 
 ## libgdf
+CMakeFiles/
 Debug
-/libgdf/thirdparty/googletest/
-/libgdf/include/gdf/ipc/File_generated.h
-/libgdf/include/gdf/ipc/Message_generated.h
-/libgdf/include/gdf/ipc/Schema_generated.h
-/libgdf/include/gdf/ipc/Tensor_generated.h
-/libgdf/python/libgdf_cffi/libgdf_cffi.py
-/libgdf/python/librmm_cffi/librmm_cffi.py
+libgdf/build/
+libgdf/thirdparty/googletest/
+libgdf/include/gdf/ipc/File_generated.h
+libgdf/include/gdf/ipc/Message_generated.h
+libgdf/include/gdf/ipc/Schema_generated.h
+libgdf/include/gdf/ipc/Tensor_generated.h
+libgdf/python/libgdf_cffi/libgdf_cffi.py
+libgdf/python/librmm_cffi/librmm_cffi.py
 
-## eclipse
+## Eclipse IDE
 .project
+.cproject
+.settings

.gitmodules

@@ -1,10 +1,9 @@
-[submodule "thirdparty/cub"]
-	path = libgdf/thirdparty/cub
+[submodule "cpp/thirdparty/cub"]
+	path = cpp/thirdparty/cub
 	url = https://github.com/NVlabs/cub.git
-[submodule "thirdparty/moderngpu"]
-	path = libgdf/thirdparty/moderngpu
+[submodule "cpp/thirdparty/moderngpu"]
+	path = cpp/thirdparty/moderngpu
 	url = https://github.com/moderngpu/moderngpu.git
-[submodule "thirdparty/cnmem"]
-	path = libgdf/thirdparty/cnmem
+[submodule "cpp/thirdparty/cnmem"]
+	path = cpp/thirdparty/cnmem
 	url = https://github.com/NVIDIA/cnmem.git
-

.travis.yml

@@ -14,14 +14,14 @@ env:
 
 matrix:
   include:
-    - env: CUDA=9.2.148_396.37 PYTHON=3.5 LABEL_MAIN=1 BUILD_CUDF=1 BUILD_LIBGDF=1 BUILD_CFFI=1
+    - env: CUDA=9.2.148_396.37 PYTHON=3.5 LABEL_MAIN=1 BUILD_CUDF=1 BUILD_LIBCUDF=1 BUILD_CFFI=1
     - env: CUDA=9.2.148_396.37 PYTHON=3.6 LABEL_MAIN=1 BUILD_CUDF=1 BUILD_CFFI=1
-    - env: CUDA=10.0.130_410.48 PYTHON=3.5 BUILD_LIBGDF=1 BUILD_CFFI=1
+    - env: CUDA=10.0.130_410.48 PYTHON=3.5 BUILD_LIBCUDF=1 BUILD_CFFI=1
     - env: CUDA=10.0.130_410.48 PYTHON=3.6 BUILD_CFFI=1
 
 before_install:
-  # install build environment for libgdf
-  - source ./travisci/libgdf/install-env-libgdf.sh
+  # install build environment for libcudf
+  - source ./travisci/libcudf/install-env-libcudf.sh
   # install miniconda
   - travis_retry wget --progress=dot:mega https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
   - chmod +x miniconda.sh
@@ -38,16 +38,16 @@ script:
   - source activate root
   # check conda versions
   - conda list
-  # build ligdf
-  - source ./travisci/libgdf/build_libgdf.sh
-  # build libgdf_cffi
-  - source ./travisci/libgdf/build_libgdf_cffi.sh
-  # upload libgdf and libgdf_cffi
-  - source ./travisci/libgdf/upload.sh
+  # build libcudf
+  - source ./travisci/libcudf/build_libcudf.sh
+  # build libcudf_cffi
+  - source ./travisci/libcudf/build_libcudf_cffi.sh
+  # upload libcudf and libcudf_cffi
+  - source ./travisci/libcudf/upload.sh
   # build cudf
-  - source ./travisci/build_cudf.sh
+  - source ./travisci/cudf/build_cudf.sh
   - flake8
 
 after_success:
   # upload cudf
-  - source ./travisci/upload-anaconda.sh
+  - source ./travisci/cudf/upload-anaconda.sh

Dockerfile

@@ -33,11 +33,12 @@ ARG NUMPY_VERSION=1.14.3
 # Locked to Pandas 0.20.3 by https://github.com/rapidsai/cudf/issues/118
 ARG PANDAS_VERSION=0.20.3
 ARG PYARROW_VERSION=0.10.0
-RUN conda install -n cudf -y -c numba -c conda-forge -c defaults \
+RUN conda install -n cudf -y -c numba -c conda-forge -c rapidsai -c defaults \
       numba=${NUMBA_VERSION} \
       numpy=${NUMPY_VERSION} \
       pandas=${PANDAS_VERSION} \
       pyarrow=${PYARROW_VERSION} \
+      nvstrings \
       cmake
 
 # Clone cuDF repo

MANIFEST.in

@@ -1,2 +1,2 @@
-include versioneer.py
-include cudf/_version.py
+include python/versioneer.py
+include python/cudf/_version.py

README.md

@@ -17,18 +17,25 @@ You can get a minimal conda installation with [Miniconda](https://conda.io/minic
 You can install and update cuDF using the conda command:
 
 ```bash
-conda install -c numba -c conda-forge -c rapidsai -c defaults cudf=0.2.0
+conda install -c rapidsai -c nvidia -c numba -c conda-forge -c defaults cudf=0.2.0
 ```
 
 Note: This conda installation only applies to Linux and Python versions 3.5/3.6.
 
-You can create and activate a development environment using the conda command:
+You can create and activate a development environment using the conda commands:
 
 ```bash
-conda env create --name cudf --file conda_environments/testing_py35.yml
-source activate cudf
+# create the conda environment (assuming in base `cudf` directory)
+$ conda env create --name cudf_dev --file conda/environments/dev_py35.yml
+# activate the environment
+$ source activate cudf_dev
+# when not using default arrow version 0.10.0, run
+$ conda install -c rapidsai -c nvidia -c numba -c conda-forge -c defaults pyarrow=$ARROW_VERSION
 ```
 
+This installs the required `cmake`, `nvstrings`, `pyarrow` and other
+dependencies into the `cudf_dev` conda environment and activates it.
+
 ### Pip
 
 Support is coming soon, please use conda for the time being.
@@ -39,12 +46,13 @@ The following instructions are tested on Linux Ubuntu 16.04 & 18.04, to enable
 from source builds and development. Other operatings systems may be compatible,
 but are not currently supported.
 
-### Get libgdf Dependencies
+### Get libcudf Dependencies
 
 Compiler requirements:
 
-* `g++` 5.4
-* `cmake` 3.12
+* `gcc`     version 5.4
+* `nvcc`    version 9.2
+* `cmake`   version 3.12
 
 CUDA/GPU requirements:
 
@@ -80,20 +88,41 @@ git clone --recurse-submodules https://github.com/rapidsai/cudf.git
 cd cudf
 ```
 2. Create the conda development environment `cudf` as detailed above
-3. Build and install `libgdf`
+3. Build and install `libcudf`
+```bash
+$ cd /path/to/cudf/cpp                              # navigate to C/C++ CUDA source root directory
+$ mkdir build                                       # make a build directory
+$ cd build                                          # enter the build directory
+$ cmake .. -DCMAKE_INSTALL_PREFIX=/install/path     # configure cmake ... use $CONDA_PREFIX if you're using Anaconda
+$ make -j                                           # compile the libraries librmm.so, libcudf.so ... '-j' will start a parallel job using the number of physical cores available on your system
+$ make install                                      # install the libraries librmm.so, libcudf.so to '/install/path'
+```
+To run tests (Optional):
+
 ```bash
-source activate cudf
-mkdir -p libgdf/build
-cd libgdf/build
-cmake .. -DHASH_JOIN=ON -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX
-make -j install
-make copy_python
-python setup.py install
-```
-4. Build and install `cudf` from the root of the repository
+$ make test
+```
+
+Build and install cffi bindings:
+```bash
+$ make python_cffi                                    # build CFFI bindings for librmm.so, libcudf.so
+$ make install_python                                 # install python bindings into site-packages
+```
+
+4. Build the `cudf` python package, in the `python` folder:
+```bash
+$ cd ../../python
+$ python setup.py build_ext --inplace
+```
+
+To run Python tests (Optional):
+```bash
+$ py.test
+```
+Finally, install the Python package to your Python path:
+
 ```bash
-cd ../..
-python setup.py install
+$ python setup.py install
 ```
 
 ## Automated Build in Docker Container
@@ -116,7 +145,7 @@ After the container is built run the container:
 ```
 docker run --runtime=nvidia -it cudf bash
 ```
-Activate the conda environment `cudf` to use the newly built cuDF and libgdf libraries:
+Activate the conda environment `cudf` to use the newly built cuDF and libcudf libraries:
 ```
 root@3f689ba9c842:/# source activate cudf
 (cudf) root@3f689ba9c842:/# python -c "import cudf"
@@ -151,20 +180,20 @@ This project uses [py.test](https://docs.pytest.org/en/latest/)
 In the source root directory and with the development conda environment activated, run:
 
 ```bash
-py.test --cache-clear --ignore=libgdf
+py.test --cache-clear --ignore=cpp
 ```
 
-### libgdf
+### libcudf
 
-The `libgdf` tests require a GPU and CUDA. CUDA can be installed locally or through the conda packages of `numba` & `cudatoolkit`. For more details on the requirements needed to run these tests see the [libgdf README](libgdf/README.md).
+The `libcudf` tests require a GPU and CUDA. CUDA can be installed locally or through the conda packages of `numba` & `cudatoolkit`. For more details on the requirements needed to run these tests see the [libcudf README](cpp/README.md).
 
-`libgdf` has two testing frameworks `py.test` and GoogleTest:
+`libcudf` has two testing frameworks `py.test` and GoogleTest:
 
 ```bash
-# Run py.test command inside the /libgdf folder
+# Run py.test command inside the /libcudf folder
 py.test
 
-# Run GoogleTest command inside the /libgdf/build folder after cmake
+# Run GoogleTest command inside the /libcudf/build folder after cmake
 make -j test
 ```
 

conda_environments/builddocs_py35.yml → conda/environments/builddocs_py35.yml

@@ -1,18 +1,20 @@
 name: builddocs_py35
 channels:
+- mikewendt
+- nvidia
 - rapidsai
-- conda-forge
 - numba
-- mikewendt
+- conda-forge
 - defaults
 dependencies:
 - python=3.5.*
 - pytest
 - cudatoolkit=9.2
 - nvdriver=396.44
-- libgdf=0.2.0.*
-- libgdf_cffi=0.2.0.*
+- libcudf=0.2.0.*
+- libcudf_cffi=0.2.0.*
 - numba>=0.40.0dev
+- nvstrings
 - pandas=0.20.*
 - pyarrow=0.10.*
 - pip:

conda/environments/dev_py35.yml

@@ -0,0 +1,20 @@
+name: cudf_dev_py35
+channels:
+- nvidia
+- numba
+- conda-forge
+- rapidsai
+- defaults
+dependencies:
+- cmake>=3.12
+- python=3.5.*
+- pytest
+- numba>=0.40.0dev
+- pandas=0.20.3
+- pyarrow=0.10.0
+- notebook>=0.5.0
+- boost
+- nvstrings
+- cffi>=1.10.0   
+- distributed>=1.23.0
+- cython=0.28.*

conda_environments/testing_py35.yml → conda/environments/testing_py35.yml

@@ -1,8 +1,9 @@
-name: pycudf_testing_py35
+name: cudf_testing_py35
 channels:
+- rapidsai
+- nvidia
 - numba
 - conda-forge
-- rapidsai
 - defaults
 dependencies:
 - python=3.5.*
@@ -13,3 +14,6 @@ dependencies:
 - notebook>=0.5.0
 - cmake=3.12.*
 - boost
+- nvstrings
+- distributed>=1.23.0
+

conda/recipes/cudf/build.sh

@@ -0,0 +1,2 @@
+$PYTHON setup.py build_ext --inplace
+$PYTHON setup.py install

conda-recipes/cudf/meta.yaml → conda/recipes/cudf/meta.yaml

@@ -1,33 +1,41 @@
 # Copyright (c) 2018, NVIDIA CORPORATION.
 
 # Usage:
-#   conda build -c defaults -c conda-forge .
+#   conda build -c rapidsai -c nvidia -c numba -c conda-forge -c defaults .
 {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %}
 {% set git_revision_count=environ.get('GIT_DESCRIBE_NUMBER', 0) %}
 package:
   name: cudf
   version: {{ version }}
 
 source:
-  path: ../..
+  path: ../../..
 
 build:
   number: {{ git_revision_count }}
   script_env:
     - CUDF_BUILD_NO_GPU_TEST
 
 requirements:
+  # use channel:
+  #   - rapidsai
+  #   - nvidia
+  #   - numba
+  #   - conda-forge
+  #   - defaults
   build:
     - python
     - setuptools
     - numba>=0.40.0dev
+    - nvstrings
   run:
     - python
     - setuptools
-    - libgdf 0.2.0.*
-    - libgdf_cffi 0.2.0.*
+    - libcudf 0.2.0.*
+    - libcudf_cffi 0.2.0.*
     - pandas 0.20.*
     - numba>=0.40.0dev
+    - nvstrings
 
 test:
   requires:

conda-recipes/libgdf/build.sh → conda/recipes/libcudf/build.sh

@@ -10,7 +10,7 @@ fi
 # Cleanup local git
 git clean -xdf
 # Change directory for build process
-cd libgdf
+cd cpp
 # Use CMake-based build procedure
 mkdir build
 cd build