Introduce Numba target extension to RBC

.conda/environment.yml

@@ -21,4 +21,5 @@ dependencies:
   - clang=10
   - clangxx=10
   - six
+  - cffi
   # - cudatoolkit-dev   # disable temporary

.conda/meta.yaml

@@ -20,6 +20,7 @@ requirements:
   build:
     - python
     - setuptools
+    - cffi
   host:
     - python
     - pytest-runner

.github/workflows/pypi.yml

@@ -6,32 +6,65 @@ on:
     tags:
       - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
 
+
 jobs:
-  pypi-release:
-    runs-on: ubuntu-latest
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
 
     steps:
       - uses: actions/checkout@v2
+
       - name: Set up Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      # https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip
+      - name: Install buildwheels
+        uses: pypa/cibuildwheel@v2.3.1
+        env:  # minimum version is cpython 3.7
+          CIBW_BUILD: "cp37-*64 cp38-*64 cp39-* cp310-*"
+          CIBW_BEFORE_BUILD: python -m pip install cffi
+          CIBW_VERBOSITY: 3
+
+      - uses: actions/upload-artifact@v2
+        with:
+          path: ./wheelhouse/*.whl
+
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Build sdist
+        run: python setup.py sdist
+
+      - uses: actions/upload-artifact@v2
         with:
-          python-version: 3.8
+          path: dist/*.tar.gz
 
-      - uses: conda-incubator/setup-miniconda@v2
+  upload_pypi:
+    needs: [build_sdist, build_wheels]
+    runs-on: ubuntu-latest
+    # upload to PyPI on every tag starting with 'v'
+    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
+    # alternatively, to publish when a GitHub Release is created, use the following rule:
+    # if: github.event_name == 'release' && github.event.action == 'published'
+    steps:
+      - uses: actions/download-artifact@v2
         with:
-          python-version: 3.8
-          mamba-version: "*"
-          channels: conda-forge
-          activate-environment: rbc-env
-          environment-file: .conda/environment.yml
-
-      - name: Build distribution
-        shell: bash -l {0}
-        run: python setup.py sdist bdist_wheel
-
-      - name: Publish package to PyPI
-        uses: pypa/gh-action-pypi-publish@master
+          name: artifact
+          path: dist
+
+      - uses: pypa/gh-action-pypi-publish@master
         with:
+          verify_metadata: true
           verbose: true
           user: __token__
-          password: ${{ secrets.PYPI_API_TOKEN }}
+          password: ${{ secrets.PYPI_API_TOKEN }}
+          # repository_url: https://test.pypi.org/legacy/

.github/workflows/rbc_test.yml

@@ -3,15 +3,18 @@ name: RBC
 on:
   # Trigger the workflow on push or pull request,
   # but only for the main branch
-  push:
-    branches:
-      - master
   pull_request:
     branches:
       - master
 
+# kill any previous running job on a new commit
+concurrency:
+  group: build-and-test-rbc-${{ github.head_ref }}
+  cancel-in-progress: true
+
 jobs:
   lint:
+    if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip-tests') }}
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
@@ -32,11 +35,16 @@ jobs:
     name: ${{ matrix.os }} - Python v${{ matrix.python-version }} - Numba v${{ matrix.numba-version }}
     runs-on: ${{ matrix.os }}
     strategy:
-      fail-fast: true
+      # setting fail-fast=true seems to cause connection issues in remotejit tests
+      fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: [3.9, 3.8, 3.7]
-        numba-version: [0.54, 0.53]
+        python-version: ['3.9', '3.8', '3.7']
+        numba-version: ['0.55', '0.54']
+        include:
+          - os: ubuntu-latest
+            python-version: '3.10'
+            numba-version: '0.55'
 
     needs: [lint, omniscidb]
 
@@ -74,9 +82,15 @@ jobs:
         shell: bash -l {0}
         run: |
           mamba run -n rbc conda list
+
       - name: Develop rbc
         shell: bash -l {0}
         run: |
+          if [ "$RUNNER_OS" == "macOS" ]; then
+              # workaround for
+              # https://github.com/conda-forge/clangdev-feedstock/issues/96
+              export SDKROOT=`xcrun --sdk macosx --show-sdk-path`
+          fi
           mamba run -n rbc python setup.py develop
       - name: Run rbc tests
         shell: bash -l {0}
@@ -94,28 +108,23 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python-version: [3.9, 3.8, 3.7]
-        numba-version: [0.54, 0.53]
-        omniscidb-version: [5.8, 5.7, 5.6]
+        python-version: ['3.9', '3.8', '3.7']
+        numba-version: ['0.55', '0.54']
+        omniscidb-version: ['5.10', '5.9', '5.8', '5.7']
         omniscidb-from: [conda]
         include:
           - os: ubuntu-latest
-            python-version: 3.8
-            numba-version: 0.53
+            python-version: '3.10'
+            numba-version: '0.55'
             omniscidb-version: dev
             docker-image: omnisci/core-os-cpu-dev:latest
             omniscidb-from: docker
           - os: ubuntu-latest
-            python-version: 3.8
-            numba-version: 0.54
+            python-version: '3.9'
+            numba-version: '0.54'
             omniscidb-version: dev
             docker-image: omnisci/core-os-cpu-dev:latest
             omniscidb-from: docker
-          - os: ubuntu-latest
-            python-version: 3.8
-            omniscidb-version: 5.8.0
-            docker-image: omnisci/core-os-cpu:v5.8.0
-            omniscidb-from: docker
 
     needs: lint
 
@@ -168,11 +177,18 @@ jobs:
       - name: Start Omniscidb [conda]
         shell: bash -l {0}
         if: matrix.os == 'ubuntu-latest' && matrix.omniscidb-from == 'conda'
+        env:
+          OMNISCIDB_VERSION: ${{ matrix.omniscidb-version }}
         run: |
           mkdir data
           mamba run -n omniscidb-env omnisci_initdb data -f
           mamba run -n omniscidb-env omnisci_server --version
-          mamba run -n omniscidb-env omnisci_server --enable-runtime-udf --enable-table-functions \> omniscidb-output.txt 2\>\&1 \& echo \$! \> omniscidb.pid
+          RUN_FLAGS="--enable-runtime-udf --enable-table-functions"
+          if [[ ${OMNISCIDB_VERSION} == "5.10" ]]; then
+            RUN_FLAGS="${RUN_FLAGS} --enable-dev-table-functions"
+          fi
+          echo ${RUN_FLAGS}
+          mamba run -n omniscidb-env omnisci_server $RUN_FLAGS \> omniscidb-output.txt 2\>\&1 \& echo \$! \> omniscidb.pid
           sleep 10
           cat omniscidb.pid
 
@@ -222,6 +238,14 @@ jobs:
         run: |
           mamba run -n rbc pytest -sv -r A rbc/tests/ -x -k omnisci
 
+      - name: Show Omniscidb conda logs on failure [conda]
+        shell: bash -l {0}
+        if: failure() && matrix.os == 'ubuntu-latest' && matrix.omniscidb-from == 'conda'
+        run: |
+          mamba run -n omniscidb-env cat data/mapd_log/omnisci_server.INFO
+          mamba run -n omniscidb-env cat data/mapd_log/omnisci_server.WARNING
+          mamba run -n omniscidb-env cat data/mapd_log/omnisci_server.ERROR
+
       - name: Show Omniscidb docker logs on failure [docker]
         shell: bash -l {0}
         if: failure() && matrix.os == 'ubuntu-latest' && matrix.omniscidb-from == 'docker'
@@ -249,3 +273,52 @@ jobs:
         run: |
           mamba run -n docker docker-compose logs --no-color --tail=10000 -f -t \> omniscidb-docker.log
           cat omniscidb-docker.log
+
+  eval-notebooks:
+      runs-on: ubuntu-latest
+      needs: lint
+
+      steps:
+        - uses: actions/checkout@v2
+
+        - name: Set up Python
+          uses: actions/setup-python@v2
+          with:
+            python-version: 3.8
+
+        - uses: conda-incubator/setup-miniconda@v2
+          with:
+            python-version: 3.8
+            mamba-version: "*"
+            channels: conda-forge
+            environment-file: .conda/environment.yml
+
+        - name: Install additional packages
+          shell: bash -l {0}
+          run: |
+            mamba install -c conda-forge omniscidb nbval ibis-omniscidb matplotlib pandas ibis-framework
+
+        - name: Start omniscidb server
+          shell: bash -l {0}
+          run: |
+            mkdir data && omnisci_initdb data -f
+            omnisci_server --version
+            omnisci_server --enable-runtime-udf --enable-table-functions 2>&1 > omniscidb-output.txt &
+            sleep 10
+
+        - name: conda config
+          shell: bash -l {0}
+          run: conda config --show
+
+        - name: conda list
+          shell: bash -l {0}
+          run: |
+            conda list
+
+        - name: Execute pytest
+          shell: bash -l {0}
+          run: |
+            pip install -e .
+            pytest -v -rs --nbval notebooks/ -x
+            pkill -f omnisci_server
+            cat omniscidb-output.txt

README.md

@@ -1,6 +1,6 @@
 # RBC - Remote Backend Compiler
 
-[![Travis CI](https://travis-ci.org/xnd-project/rbc.svg?branch=master)](https://travis-ci.org/xnd-project/rbc) [![Appveyor CI](https://ci.appveyor.com/api/projects/status/i9xbkqkvomhbr8n4/branch/master?svg=true)](https://ci.appveyor.com/project/pearu/rbc-mnh7b/branch/master) [![Documentation Status](https://readthedocs.org/projects/rbc/badge/?version=latest)](https://rbc.readthedocs.io/en/latest/?badge=latest)
+[![GitHub Actions](https://github.com/xnd-project/rbc/actions/workflows/rbc_test.yml/badge.svg)](https://github.com/xnd-project/rbc/actions) [![Documentation Status](https://readthedocs.org/projects/rbc/badge/?version=latest)](https://rbc.readthedocs.io/en/latest/?badge=latest)
 
 ## Introduction - test
 

doc/_templates/autosummary/module.rst

@@ -13,7 +13,7 @@
     {% endfor %}
 {% endif %}
 
-{% if module == "rbc.externals.libdevice" %}
+{% if fullname == "rbc.externals.libdevice" %}
 .. rubric:: Functions
 .. autosummary::
     :toctree:

doc/api.rst

@@ -26,6 +26,19 @@ Top-level functions
     utils
 
 
+Array API
+=========
+
+.. autosummary::
+    :toctree: generated/
+
+    stdlib.datatypes
+    stdlib.constants
+    stdlib.creation_functions
+    stdlib.elementwise_functions
+    stdlib.statistical_functions
+
+
 Externals
 =========