add nbtest job to the GitHub Actions CI

.github/workflows/tests.yml

@@ -0,0 +1,28 @@
+name: notebook-tests
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    services:
+      elasticsearch:
+        image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0
+        env:
+          discovery.type: single-node
+          xpack.security.enabled: false
+          xpack.security.http.ssl.enabled: false
+          xpack.license.self_generated.type: trial
+        ports:
+          - 9200:9200
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - run: sleep 30
+      - run: PATCH_ES=1 FORCE_COLOR=1 make -s test

Makefile

@@ -1,28 +1,34 @@
+# this is the list of notebooks that are integrated with the testing framework
+NOTEBOOKS = \
+	notebooks/search/00-quick-start.ipynb \
+	notebooks/search/01-keyword-querying-filtering.ipynb \
+	notebooks/search/02-hybrid-search.ipynb \
+	notebooks/search/03-ELSER.ipynb \
+	notebooks/search/04-multilingual.ipynb \
+	notebooks/search/05-query-rules.ipynb \
+	notebooks/search/06-synonyms-api.ipynb \
+	notebooks/document-chunking/tokenization.ipynb \
+	notebooks/document-chunking/with-index-pipelines.ipynb \
+	notebooks/document-chunking/with-langchain-splitters.ipynb \
+	notebooks/model-upgrades/upgrading-index-to-use-elser.ipynb \
+	notebooks/langchain/langchain-using-own-model.ipynb \
+	notebooks/langchain/langchain-vector-store-using-elser.ipynb \
+	notebooks/integrations/hugging-face/loading-model-from-hugging-face.ipynb
+
 .PHONY: install pre-commit nbtest test notebooks
 
 test: nbtest notebooks
 
-notebooks: search document-chunking model-upgrades langchain
-
-search:
-	$(MAKE) -C notebooks/search
-
-document-chunking:
-	$(MAKE) -C notebooks/document-chunking
-
-model-upgrades:
-	$(MAKE) -C notebooks/model-upgrades
-
-langchain:
-	$(MAKE) -C notebooks/langchain
+notebooks:
+	bin/nbtest $(NOTEBOOKS)
 
 install: pre-commit nbtest
 
 pre-commit:
 	python -m venv .venv
-	.venv/bin/pip install -r requirements-dev.txt
+	.venv/bin/pip install -qqq -r requirements-dev.txt
 	.venv/bin/pre-commit install
 
 nbtest:
 	python3 -m venv .venv
-	.venv/bin/pip install elastic-nbtest
+	.venv/bin/pip install -qqq elastic-nbtest

bin/mocks.py

@@ -0,0 +1,18 @@
+# This module is provided to nbtest when running under GitHub Actions CI to
+# mock the Elasticsearch client to always connect on localhost, regardless of
+# arguments provided
+
+import elasticsearch
+
+orig_es_init = elasticsearch.Elasticsearch.__init__
+
+
+def patched_es_init(self, *args, **kwargs):
+    if 'cloud_id' in kwargs:
+        assert kwargs['cloud_id'] == 'foo'
+    if 'api_key' in kwargs:
+        assert kwargs['api_key'] == 'bar'
+    return orig_es_init(self, 'http://localhost:9200')
+
+
+elasticsearch.Elasticsearch.__init__ = patched_es_init

bin/mocks/elasticsearch.py

@@ -0,0 +1,41 @@
+import os
+import sys
+
+
+def patch_elasticsearch():
+    # preserve the original import path
+    saved_path = sys.path.copy()
+
+    # remove the path entry that refers to this directory
+    for path in sys.path:
+        if not path.startswith('/'):
+            path = os.path.join(os.getcwd(), path)
+        if __file__ == os.path.join(path, 'elasticsearch.py'):
+            sys.path.remove(path)
+            break
+
+    # remove this module, and import the real one instead
+    del sys.modules['elasticsearch']
+    import elasticsearch
+
+    # restore the import path
+    sys.path = saved_path
+
+    # preserve the original Elasticsearch.__init__ method   
+    orig_es_init = elasticsearch.Elasticsearch.__init__
+
+    # patched version of Elasticsearch.__init__ that connects to self-hosted
+    # regardless of connection arguments given
+    def patched_es_init(self, *args, **kwargs):
+        if 'cloud_id' in kwargs:
+            assert kwargs['cloud_id'] == 'foo'
+        if 'api_key' in kwargs:
+            assert kwargs['api_key'] == 'bar'
+        return orig_es_init(self, 'http://localhost:9200')
+
+    # patch Elasticsearch.__init__
+    elasticsearch.Elasticsearch.__init__ = patched_es_init
+
+
+patch_elasticsearch()
+del patch_elasticsearch

bin/nbtest

@@ -4,5 +4,18 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 if [[ ! -f $SCRIPT_DIR/../.venv/bin/nbtest ]]; then
     make nbtest
 fi
+
+if [[ "$PATCH_ES" != "" ]]; then
+    # here we do some Python dark magic to patch the elasticsearch package to
+    # connect to a locally hosted instance in spite of connection arguments
+    # given
+    export ELASTIC_CLOUD_ID=foo
+    export ELASTIC_API_KEY=bar
+    export PYTHONPATH=$SCRIPT_DIR/mocks
+
+    # ensure elasticsearch is installed so that it can be patched
+    $SCRIPT_DIR/../.venv/bin/pip install -qqq elasticsearch
+fi
+
 source $SCRIPT_DIR/../.venv/bin/activate
 $SCRIPT_DIR/../.venv/bin/nbtest $*

notebooks/integrations/hugging-face/.nbtest.yml

@@ -0,0 +1,2 @@
+masks:
+- 'Score: [0-9]+\.[0-9][0-9]*'

notebooks/integrations/hugging-face/loading-model-from-hugging-face.ipynb

@@ -47,7 +47,7 @@
    },
    "outputs": [],
    "source": [
-    "!python3 -m pip -qU install sentence-transformers eland elasticsearch transformers"
+    "!python3 -m pip install sentence-transformers eland elasticsearch transformers"
    ]
   },
   {
@@ -111,7 +111,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!eland_import_hub_model --cloud-id $ELASTIC_CLOUD_ID --hub-model-id sentence-transformers/all-MiniLM-L6-v2 --task-type text_embedding --es-api-key $ELASTIC_API_KEY --start"
+    "!eland_import_hub_model --cloud-id $ELASTIC_CLOUD_ID --hub-model-id sentence-transformers/all-MiniLM-L6-v2 --task-type text_embedding --es-api-key $ELASTIC_API_KEY --start --clear-previous"
    ]
   },
   {
@@ -423,7 +423,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.6"
+   "version": "3.10.13"
   },
   "vscode": {
    "interpreter": {