feat: parallelise at hole level

.github/workflows/test.yml

@@ -11,6 +11,22 @@ concurrency:
 
 jobs:
   testbed:
+    strategy:
+      matrix:
+        repo:
+          - { name: simple, key: simple, parallel: 8 }
+          - { name: mmaitre314/picklescan, key: picklescan, parallel: 8 }
+          - { name: huggingface/huggingface_hub, key: huggingface_hub, parallel: 8 }
+          - { name: tiangolo/fastapi, key: fastapi, parallel: 8 }
+          - { name: encode/starlette, key: starlette, parallel: 8 }
+          - { name: lancedb/lancedb, key: lancedb, parallel: 2 }
+          - { name: lancedb/lance, key: lance, parallel: 2 }
+          - { name: tkaitchuck/constrandom, key: constrandom, parallel: 8 }
+          - { name: jaemk/cached, key: cached, parallel: 4 }
+          - { name: smol-rs/async-executor, key: async-executor, parallel: 4 }
+          - { name: gcanti/io-ts, key: io-ts, parallel: 8 }
+          - { name: colinhacks/zod, key: zod, parallel: 8 }
+          - { name: helix-editor/helix, key: helix, parallel: 2 }
     runs-on: [self-hosted, intel-cpu, 8-cpu, ci]
     container:
       image: ubuntu:22.04
@@ -60,17 +76,118 @@ jobs:
         run: cargo build -r
 
       - name: Run testbed
-        run: cargo run --bin testbed -r -- --api-token $API_TOKEN -r `pwd`/crates/testbed/repositories-ci.yaml
+        run: 'cargo run --bin testbed -r -- --api-token $API_TOKEN -r `pwd`/crates/testbed/repositories-ci.yaml -f ${{ matrix.repo.name }} -p ${{ matrix.repo.parallel }}'
         if: github.event_name == 'push' || github.event_name == 'pull_request'
         env:
           API_TOKEN: ${{ secrets.API_TOKEN }}
 
       - name: Run testbed
-        run: cargo run --bin testbed -r -- --api-token $API_TOKEN
+        run: 'cargo run --bin testbed -r -- --api-token $API_TOKEN -f ${{ matrix.repo.name }} -p ${{ matrix.repo.parallel }}'
         if: github.event_name == 'workflow_dispatch'
         env:
           API_TOKEN: ${{ secrets.API_TOKEN }}
 
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v1
+        with:
+          name: results-${{ matrix.repo.key }}
+          path: ./results.json
+
+  comment_results:
+    needs: [testbed]
+    runs-on: [self-hosted, intel-cpu, 8-cpu, ci]
+    container:
+      image: ubuntu:22.04
+    steps:
+      - name: Install dependencies
+        run: |
+          apt update
+          apt install -y jq
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: results-simple
+          path: results-simple
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: results-picklescan
+          path: results-picklescan
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: results-huggingface_hub
+          path: results-huggingface_hub
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: results-fastapi
+          path: results-fastapi
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: results-starlette
+          path: results-starlette
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: results-lancedb
+          path: results-lancedb
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: results-lance
+          path: results-lance
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: results-constrandom
+          path: results-constrandom
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: results-cached
+          path: results-cached
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: results-async-executor
+          path: results-async-executor
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: results-io-ts
+          path: results-io-ts
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: results-zod
+          path: results-zod
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: results-helix
+          path: results-helix
+
+      - name: Display structure of downloaded files
+        run: ls -R
+
+      - name: output to markdown
+        run: |
+          cat > results.md  <<EOF
+          | Repository name | Source type | Average hole completion time (s) | Pass percentage |
+          | :-------------- | :---------- | -------------------------------: | --------------: |
+          EOF
+          cat **/results.json | jq -r '"| \(.[0].repo_name) | \(.[0].source_type) | \(.[0].avg_hole_completion_time_ms) | \(.[0].pass_percentage)% |"' >> results.md
+          cat >> results.md <<EOF
+
+          **Note:** The "hole completion time" represents the full process of:
+            - copying files from the setup cache directory
+            - replacing the code from the file with a completion from the model
+            - building the project
+            - running the tests
+          EOF
+
       - name: Find Comment
         uses: peter-evans/find-comment@v2
         id: fc

.gitignore

@@ -3,5 +3,5 @@ dist/
 target/
 .DS_Store
 __pycache__/
-results.md
+results.json
 .pytest_cache/

crates/llm-ls/src/document.rs

@@ -1,7 +1,6 @@
 use ropey::Rope;
 use tower_lsp::jsonrpc::Result;
 use tower_lsp::lsp_types::Range;
-use tracing::info;
 use tree_sitter::{InputEdit, Parser, Point, Tree};
 
 use crate::language_id::LanguageId;

crates/testbed/README.md

@@ -11,9 +11,10 @@ Here is a simplified pseudo code algorithm for testbed:
 read the repositories file
 read the holes file(s)
 for each repository
-  spawn a thread
-  setup the repository
   for each hole
+    spawn a thread
+    setup the repository -- only once for each repository
+    copy files from the setup cache to a new temp dir
     make the hole as specified by the file
     generate completions
     build the code
@@ -37,7 +38,7 @@ Before running testbed, you will need to generate a holes file for each reposito
 
 ### Setup
 
-testbed runs completions for each repository in parallel. It will first create a temporary directory, then copy or download the repository's source files to that location and finally run the setup commands.
+testbed runs hole completions in parallel. It will first, and only once per repository, create a temporary directory, then copy or download the repository's source files to that location and finally run the setup commands. Then for each subsequent completion it will copy the content of the "setup directory" to a new temporary directory so that work can be parallelised.
 
 Setup commands are useful to install dependencies.
 
@@ -60,9 +61,11 @@ build_args: ["-m", "compileall", "-q", "."]
 
 ### Runners
 
-testbed supports two test runners at the moment:
+testbed supports four test runners:
 - cargo
+- jest
 - pytest
+- vitest
 
 To configure your runner, you have the following options:
 ```yaml

crates/testbed/holes/async-executor.json

@@ -1 +1 @@
-[{"cursor":{"line":86,"character":2},"file":"src/lib.rs"},{"cursor":{"line":266,"character":3},"file":"src/lib.rs"},{"cursor":{"line":565,"character":6},"file":"src/lib.rs"},{"cursor":{"line":914,"character":12},"file":"src/lib.rs"},{"cursor":{"line":261,"character":3},"file":"src/lib.rs"},{"cursor":{"line":839,"character":4},"file":"src/lib.rs"},{"cursor":{"line":337,"character":4},"file":"src/lib.rs"},{"cursor":{"line":517,"character":0},"file":"src/lib.rs"},{"cursor":{"line":763,"character":12},"file":"src/lib.rs"},{"cursor":{"line":551,"character":1},"file":"src/lib.rs"},{"cursor":{"line":461,"character":14},"file":"src/lib.rs"},{"cursor":{"line":847,"character":0},"file":"src/lib.rs"},{"cursor":{"line":835,"character":14},"file":"src/lib.rs"},{"cursor":{"line":793,"character":4},"file":"src/lib.rs"},{"cursor":{"line":901,"character":13},"file":"src/lib.rs"},{"cursor":{"line":191,"character":3},"file":"src/lib.rs"},{"cursor":{"line":694,"character":7},"file":"src/lib.rs"},{"cursor":{"line":824,"character":5},"file":"src/lib.rs"},{"cursor":{"line":840,"character":3},"file":"src/lib.rs"},{"cursor":{"line":310,"character":0},"file":"src/lib.rs"},{"cursor":{"line":762,"character":7},"file":"src/lib.rs"},{"cursor":{"line":664,"character":2},"file":"src/lib.rs"},{"cursor":{"line":267,"character":1},"file":"src/lib.rs"},{"cursor":{"line":484,"character":12},"file":"src/lib.rs"},{"cursor":{"line":24,"character":4},"file":"src/lib.rs"},{"cursor":{"line":100,"character":4},"file":"src/lib.rs"},{"cursor":{"line":873,"character":9},"file":"src/lib.rs"},{"cursor":{"line":690,"character":2},"file":"src/lib.rs"},{"cursor":{"line":876,"character":13},"file":"src/lib.rs"},{"cursor":{"line":435,"character":4},"file":"src/lib.rs"},{"cursor":{"line":79,"character":11},"file":"src/lib.rs"},{"cursor":{"line":552,"character":4},"file":"src/lib.rs"},{"cursor":{"line":472,"character":0},"file":"src/lib.rs"},{"cursor":{"line":762,"character":0},"file":"src/lib.rs"},{"cursor":{"line":845,"character":5},"file":"src/lib.rs"},{"cursor":{"line":707,"character":4},"file":"src/lib.rs"},{"cursor":{"line":433,"character":10},"file":"src/lib.rs"},{"cursor":{"line":754,"character":1},"file":"src/lib.rs"},{"cursor":{"line":758,"character":12},"file":"src/lib.rs"},{"cursor":{"line":811,"character":8},"file":"src/lib.rs"},{"cursor":{"line":760,"character":13},"file":"src/lib.rs"},{"cursor":{"line":898,"character":8},"file":"src/lib.rs"},{"cursor":{"line":149,"character":3},"file":"src/lib.rs"},{"cursor":{"line":242,"character":14},"file":"src/lib.rs"},{"cursor":{"line":805,"character":6},"file":"src/lib.rs"},{"cursor":{"line":830,"character":4},"file":"src/lib.rs"},{"cursor":{"line":578,"character":4},"file":"src/lib.rs"},{"cursor":{"line":701,"character":11},"file":"src/lib.rs"},{"cursor":{"line":181,"character":10},"file":"src/lib.rs"},{"cursor":{"line":704,"character":14},"file":"src/lib.rs"},{"cursor":{"line":852,"character":8},"file":"src/lib.rs"},{"cursor":{"line":311,"character":1},"file":"src/lib.rs"},{"cursor":{"line":707,"character":6},"file":"src/lib.rs"},{"cursor":{"line":75,"character":0},"file":"src/lib.rs"},{"cursor":{"line":699,"character":1},"file":"src/lib.rs"},{"cursor":{"line":600,"character":0},"file":"src/lib.rs"},{"cursor":{"line":191,"character":3},"file":"src/lib.rs"},{"cursor":{"line":150,"character":1},"file":"src/lib.rs"},{"cursor":{"line":767,"character":10},"file":"src/lib.rs"},{"cursor":{"line":739,"character":0},"file":"src/lib.rs"},{"cursor":{"line":919,"character":3},"file":"src/lib.rs"},{"cursor":{"line":143,"character":6},"file":"src/lib.rs"},{"cursor":{"line":234,"character":5},"file":"src/lib.rs"},{"cursor":{"line":860,"character":6},"file":"src/lib.rs"},{"cursor":{"line":519,"character":0},"file":"src/lib.rs"},{"cursor":{"line":668,"character":14},"file":"src/lib.rs"},{"cursor":{"line":184,"character":7},"file":"src/lib.rs"},{"cursor":{"line":607,"character":10},"file":"src/lib.rs"},{"cursor":{"line":931,"character":5},"file":"src/lib.rs"},{"cursor":{"line":191,"character":0},"file":"src/lib.rs"},{"cursor":{"line":710,"character":6},"file":"src/lib.rs"},{"cursor":{"line":509,"character":1},"file":"src/lib.rs"},{"cursor":{"line":873,"character":11},"file":"src/lib.rs"},{"cursor":{"line":539,"character":12},"file":"src/lib.rs"},{"cursor":{"line":524,"character":9},"file":"src/lib.rs"},{"cursor":{"line":959,"character":1},"file":"src/lib.rs"},{"cursor":{"line":147,"character":11},"file":"src/lib.rs"},{"cursor":{"line":620,"character":2},"file":"src/lib.rs"},{"cursor":{"line":903,"character":6},"file":"src/lib.rs"},{"cursor":{"line":931,"character":13},"file":"src/lib.rs"},{"cursor":{"line":830,"character":3},"file":"src/lib.rs"},{"cursor":{"line":832,"character":11},"file":"src/lib.rs"},{"cursor":{"line":652,"character":3},"file":"src/lib.rs"},{"cursor":{"line":673,"character":7},"file":"src/lib.rs"},{"cursor":{"line":22,"character":9},"file":"src/lib.rs"},{"cursor":{"line":150,"character":6},"file":"src/lib.rs"},{"cursor":{"line":853,"character":13},"file":"src/lib.rs"},{"cursor":{"line":658,"character":13},"file":"src/lib.rs"},{"cursor":{"line":616,"character":0},"file":"src/lib.rs"},{"cursor":{"line":457,"character":12},"file":"src/lib.rs"},{"cursor":{"line":856,"character":10},"file":"src/lib.rs"},{"cursor":{"line":795,"character":11},"file":"src/lib.rs"},{"cursor":{"line":763,"character":3},"file":"src/lib.rs"},{"cursor":{"line":778,"character":6},"file":"src/lib.rs"},{"cursor":{"line":866,"character":1},"file":"src/lib.rs"},{"cursor":{"line":764,"character":4},"file":"src/lib.rs"},{"cursor":{"line":181,"character":9},"file":"src/lib.rs"},{"cursor":{"line":780,"character":11},"file":"src/lib.rs"},{"cursor":{"line":846,"character":5},"file":"src/lib.rs"},{"cursor":{"line":651,"character":2},"file":"src/lib.rs"}]
+[{"cursor":{"line":875,"character":11},"file":"src/lib.rs"},{"cursor":{"line":730,"character":4},"file":"src/lib.rs"},{"cursor":{"line":716,"character":4},"file":"src/lib.rs"},{"cursor":{"line":37,"character":14},"file":"src/lib.rs"},{"cursor":{"line":906,"character":9},"file":"src/lib.rs"},{"cursor":{"line":322,"character":9},"file":"src/lib.rs"},{"cursor":{"line":914,"character":1},"file":"src/lib.rs"},{"cursor":{"line":757,"character":1},"file":"src/lib.rs"},{"cursor":{"line":37,"character":13},"file":"src/lib.rs"},{"cursor":{"line":314,"character":10},"file":"src/lib.rs"},{"cursor":{"line":927,"character":2},"file":"src/lib.rs"},{"cursor":{"line":694,"character":7},"file":"src/lib.rs"},{"cursor":{"line":490,"character":6},"file":"src/lib.rs"},{"cursor":{"line":159,"character":2},"file":"src/lib.rs"},{"cursor":{"line":494,"character":0},"file":"src/lib.rs"},{"cursor":{"line":690,"character":4},"file":"src/lib.rs"},{"cursor":{"line":602,"character":0},"file":"src/lib.rs"},{"cursor":{"line":940,"character":1},"file":"src/lib.rs"},{"cursor":{"line":190,"character":1},"file":"src/lib.rs"},{"cursor":{"line":655,"character":4},"file":"src/lib.rs"},{"cursor":{"line":843,"character":0},"file":"src/lib.rs"},{"cursor":{"line":802,"character":9},"file":"src/lib.rs"},{"cursor":{"line":781,"character":13},"file":"src/lib.rs"},{"cursor":{"line":748,"character":9},"file":"src/lib.rs"},{"cursor":{"line":233,"character":8},"file":"src/lib.rs"},{"cursor":{"line":781,"character":9},"file":"src/lib.rs"},{"cursor":{"line":864,"character":12},"file":"src/lib.rs"},{"cursor":{"line":26,"character":0},"file":"src/lib.rs"},{"cursor":{"line":822,"character":11},"file":"src/lib.rs"},{"cursor":{"line":902,"character":1},"file":"src/lib.rs"},{"cursor":{"line":898,"character":0},"file":"src/lib.rs"},{"cursor":{"line":753,"character":0},"file":"src/lib.rs"},{"cursor":{"line":583,"character":1},"file":"src/lib.rs"},{"cursor":{"line":250,"character":1},"file":"src/lib.rs"},{"cursor":{"line":745,"character":2},"file":"src/lib.rs"},{"cursor":{"line":88,"character":0},"file":"src/lib.rs"},{"cursor":{"line":519,"character":3},"file":"src/lib.rs"},{"cursor":{"line":747,"character":2},"file":"src/lib.rs"},{"cursor":{"line":264,"character":12},"file":"src/lib.rs"},{"cursor":{"line":656,"character":0},"file":"src/lib.rs"},{"cursor":{"line":872,"character":4},"file":"src/lib.rs"},{"cursor":{"line":189,"character":5},"file":"src/lib.rs"},{"cursor":{"line":470,"character":0},"file":"src/lib.rs"},{"cursor":{"line":906,"character":7},"file":"src/lib.rs"},{"cursor":{"line":378,"character":0},"file":"src/lib.rs"},{"cursor":{"line":871,"character":14},"file":"src/lib.rs"},{"cursor":{"line":378,"character":9},"file":"src/lib.rs"},{"cursor":{"line":850,"character":8},"file":"src/lib.rs"},{"cursor":{"line":683,"character":4},"file":"src/lib.rs"},{"cursor":{"line":925,"character":0},"file":"src/lib.rs"},{"cursor":{"line":504,"character":14},"file":"src/lib.rs"},{"cursor":{"line":460,"character":12},"file":"src/lib.rs"},{"cursor":{"line":79,"character":7},"file":"src/lib.rs"},{"cursor":{"line":241,"character":8},"file":"src/lib.rs"},{"cursor":{"line":605,"character":8},"file":"src/lib.rs"},{"cursor":{"line":741,"character":11},"file":"src/lib.rs"},{"cursor":{"line":875,"character":11},"file":"src/lib.rs"},{"cursor":{"line":854,"character":13},"file":"src/lib.rs"},{"cursor":{"line":800,"character":2},"file":"src/lib.rs"},{"cursor":{"line":502,"character":13},"file":"src/lib.rs"},{"cursor":{"line":316,"character":6},"file":"src/lib.rs"},{"cursor":{"line":711,"character":14},"file":"src/lib.rs"},{"cursor":{"line":675,"character":4},"file":"src/lib.rs"},{"cursor":{"line":531,"character":4},"file":"src/lib.rs"},{"cursor":{"line":147,"character":2},"file":"src/lib.rs"},{"cursor":{"line":90,"character":3},"file":"src/lib.rs"},{"cursor":{"line":334,"character":6},"file":"src/lib.rs"},{"cursor":{"line":336,"character":7},"file":"src/lib.rs"},{"cursor":{"line":461,"character":5},"file":"src/lib.rs"},{"cursor":{"line":895,"character":0},"file":"src/lib.rs"},{"cursor":{"line":38,"character":14},"file":"src/lib.rs"},{"cursor":{"line":823,"character":8},"file":"src/lib.rs"},{"cursor":{"line":845,"character":3},"file":"src/lib.rs"},{"cursor":{"line":856,"character":14},"file":"src/lib.rs"},{"cursor":{"line":525,"character":2},"file":"src/lib.rs"},{"cursor":{"line":919,"character":6},"file":"src/lib.rs"},{"cursor":{"line":462,"character":10},"file":"src/lib.rs"},{"cursor":{"line":215,"character":3},"file":"src/lib.rs"},{"cursor":{"line":389,"character":10},"file":"src/lib.rs"},{"cursor":{"line":76,"character":0},"file":"src/lib.rs"},{"cursor":{"line":844,"character":10},"file":"src/lib.rs"},{"cursor":{"line":563,"character":6},"file":"src/lib.rs"},{"cursor":{"line":529,"character":14},"file":"src/lib.rs"},{"cursor":{"line":659,"character":3},"file":"src/lib.rs"},{"cursor":{"line":286,"character":1},"file":"src/lib.rs"},{"cursor":{"line":525,"character":4},"file":"src/lib.rs"},{"cursor":{"line":669,"character":3},"file":"src/lib.rs"},{"cursor":{"line":622,"character":9},"file":"src/lib.rs"},{"cursor":{"line":280,"character":3},"file":"src/lib.rs"},{"cursor":{"line":916,"character":0},"file":"src/lib.rs"},{"cursor":{"line":574,"character":9},"file":"src/lib.rs"},{"cursor":{"line":795,"character":0},"file":"src/lib.rs"},{"cursor":{"line":246,"character":2},"file":"src/lib.rs"},{"cursor":{"line":878,"character":8},"file":"src/lib.rs"},{"cursor":{"line":785,"character":4},"file":"src/lib.rs"},{"cursor":{"line":757,"character":10},"file":"src/lib.rs"},{"cursor":{"line":921,"character":11},"file":"src/lib.rs"},{"cursor":{"line":647,"character":11},"file":"src/lib.rs"},{"cursor":{"line":154,"character":6},"file":"src/lib.rs"},{"cursor":{"line":558,"character":1},"file":"src/lib.rs"}]