Merge branch 'main' into cv/more-error-codes

.github/workflows/Documenter.yml

@@ -4,7 +4,9 @@ on:
   push:
     branches:
       - main
-    tags: ["*"]  # Triggers from tags ignore the `paths` filter: https://github.com/orgs/community/discussions/26273
+    release:
+      types:
+        - released
     paths:
       - "docs/**"
       - ".github/workflows/Documenter.yml"
@@ -21,6 +23,7 @@ env:
   build_dir: ./build
 jobs:
   docs:
+    if: github.event_name == 'release' && github.event.action == 'published'
     name: Build
     runs-on: ubuntu-latest
     permissions:

.github/workflows/FormatCheck.yml

@@ -18,33 +18,24 @@ concurrency:
   cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
 jobs:
   format-check:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        version:
-          - "1"
-        os:
-          - ubuntu-latest
-        arch:
-          - x64
+    name: Julia
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@v1
         with:
-          version: ${{ matrix.version }}
-          arch: ${{ matrix.arch }}
+          version: "1"
       - uses: julia-actions/cache@v1
         with:
           cache-name: "${{ github.workflow }}-${{ github.job }}-${{ matrix.pkg.name }}-${{ matrix.version }}-${{ matrix.os }}-${{ matrix.arch }}"
           cache-compiled: true
       - name: Install JuliaFormatter
-        shell: julia --color=yes {0}
+        shell: julia --project=@format --color=yes {0}
         run: |
           using Pkg
           Pkg.add(PackageSpec(name="JuliaFormatter", version="1"))
       - name: Check formatting
-        id: check
-        shell: julia --color=yes {0}
+        shell: julia --project=@format --color=yes {0}
         run: |
           using JuliaFormatter
 
@@ -59,15 +50,10 @@ jobs:
           end
           if !formatted
               @error "Found some unformatted files:\n" * read(`git diff --name-only`, String)
+              exit(1)
           end
-          open(ENV["GITHUB_OUTPUT"], "a") do io
-              println(io, "formatted=$formatted")
-          end
+      # Add formatting suggestions to non-draft PRs even if when "Check formatting" fails
       - uses: reviewdog/action-suggester@v1
-        if: ${{ github.event_name == 'pull_request' && github.event.pull_request.draft == false }}
+        if: ${{ !cancelled() && github.event_name == 'pull_request' && github.event.pull_request.draft == false }}
         with:
           tool_name: JuliaFormatter
-          fail_on_error: true
-      - name: Fail if not formatted
-        if: ${{ steps.check.outputs.formatted != 'true' }}
-        run: exit 1

.github/workflows/Pre_release.yml

@@ -1,6 +1,13 @@
-name: "Publish Pre-Release"
+name: Publish Pre-Release
 on:
   workflow_dispatch:
+    inputs:
+      commit:  
+        # Note: The tag itself will not be based off this commit. As we must first commit the 
+        # updates to the Artifacts.toml, then the tag will be based off that commit.
+        description: The commit to branch from when making the tag
+        required: true
+
 # Force a single workflow to run at a time. This ensures we don't accidentally
 # try to perform concurrent publishing of releases.
 concurrency: publish-release
@@ -10,40 +17,34 @@ env:
   tarballs_dir: ./build/tarballs
 jobs:
   release:
-    name: "Publish Pre-Release"
+    name: Publish Pre-Release
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
     steps:
       - uses: actions/checkout@v4
         with:
-          fetch-depth: 0  # Needed determine latest tag
+          fetch-depth: 0  # Needed determine tags
           persist-credentials: true
-          ref: main
-      - name: Get Latest Tag
-        id: latest_tag
-        uses: WyriHaximus/github-action-get-previous-tag@v1
-        # TODO: in future we might need to perform backports, at which point we'll need to check
-        # that the project.version has not already been tagged 
-      - name: Check for updated Project version
+          ref: ${{ inputs.commit }}
+      - name: Check for new Project version
         id: project_toml
         shell: julia --color=yes --project {0}
-        env:
-          LATEST_TAG: ${{ steps.latest_tag.outputs.tag }}
         run: |
           using Pkg.Types
           project = read_project("Project.toml")
-          latest_tag = parse(VersionNumber, ENV["LATEST_TAG"])
-          @info "Project version: project.version"
-          @info "Latest tag: latest_tag"
-          project.version > latest_tag || exit(1)  # error if can't make a tag
+          project_version_tag = "v$(project.version)"
+          @info "Project version: $project_version_tag"
+          tags = readlines(`git tag -l`)
+          project_version_tag in tags && error("Tag $project_version_tag already exists")
           open(ENV["GITHUB_OUTPUT"], "a") do io
-              println(io, "version=$(project.version)")
-              println(io, "tag=v$(project.version)")
+              println(io, "tag=$(project_version_tag)")
           end
       - name: Download artifacts
         id: download-artifact
         uses: dawidd6/action-download-artifact@v2
         with:
-          commit: ${{ github.sha }}  # pull the artifacts for the last commit on main
+          commit: ${{ inputs.commit }}  # pull the artifacts for the given commit
           name: ray_julia_libraries
           path: ${{ env.tarballs_dir }}
           workflow: CI.yml
@@ -70,7 +71,7 @@ jobs:
           git config user.name beacon-buddy
           git config user.email beacon-buddy@beacon.bio
 
-          msg="Update Artifacts.toml to use ${{ steps.project_toml.outputs.version }} assets"
+          msg="Update Artifacts.toml to use ${{ steps.project_toml.outputs.tag }} assets"
           git checkout --detach
           git commit -m "$msg" -- Artifacts.toml
           echo "sha=$(git rev-parse HEAD)" | tee -a "$GITHUB_OUTPUT"
@@ -79,11 +80,13 @@ jobs:
           tag=${{ steps.project_toml.outputs.tag }}
           git tag $tag ${{ steps.commit.outputs.sha }}
           git push origin $tag
+      - name: Build Changelog
+        id: build_changelog
+        uses: mikepenz/release-changelog-builder-action@v4
       - name: Publish Pre-Release
-        uses: softprops/action-gh-release@v1
+        uses: ncipollo/release-action@v1
         with:
           prerelease: true
-          tag_name: ${{ steps.project_toml.outputs.tag }}
-          target_commitish: ${{ steps.commit.outputs.sha }}
-          generate_release_notes: true
-          files: ${{ env.tarballs_dir }}/*.tar.gz
+          tag: ${{ steps.project_toml.outputs.tag }}
+          artifacts: ${{ env.tarballs_dir }}/*.tar.gz
+          body: ${{ steps.build_changelog.outputs.changelog }}

.github/workflows/artifacts_CI.yml → .github/workflows/Release.yml

@@ -1,4 +1,4 @@
-name: Artifacts CI
+name: Publish Release
 on:
   workflow_dispatch:
   push:
@@ -62,13 +62,16 @@ jobs:
           with:
             annotate: true
   release:
-    name: "Publish Release"
+    name: Publish Release
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
     needs: [test]
     steps:
-      - name: Publish Release
-        uses: softprops/action-gh-release@v1
+      - name: Publish Pre-Release
+        uses: ncipollo/release-action@v1
         with:
-          # Any info keys (i.e. prerelease) that are not explicitly set will retain the original
-          # value if there is already an existing release for the tag.
+          allowUpdates: true
+          updateOnlyUnreleased: true
           prerelease: false
+          makeLatest: "legacy"

build/wrapper.cc

@@ -389,7 +389,6 @@ namespace jlcxx
     template<> struct SuperType<rpc::Address> { typedef google::protobuf::Message type; };
     template<> struct SuperType<rpc::JobConfig> { typedef google::protobuf::Message type; };
     template<> struct SuperType<rpc::ObjectReference> { typedef google::protobuf::Message type; };
-    template<> struct SuperType<rpc::GcsNodeInfo> { typedef google::protobuf::Message type; };
     template<> struct SuperType<TaskArgByReference> { typedef TaskArg type; };
     template<> struct SuperType<TaskArgByValue> { typedef TaskArg type; };
 
@@ -700,14 +699,6 @@ JLCXX_MODULE define_julia_module(jlcxx::Module& mod)
     mod.add_type<rpc::ObjectReference>("ObjectReference", jlcxx::julia_base_type<google::protobuf::Message>());
     jlcxx::stl::apply_stl<rpc::ObjectReference>(mod);
 
-    // message GcsNodeInfo
-    // https://github.com/ray-project/ray/blob/ray-2.5.1/src/ray/protobuf/gcs.proto#L286
-    mod.add_type<rpc::GcsNodeInfo>("GcsNodeInfo", jlcxx::julia_base_type<google::protobuf::Message>())
-        .constructor<>()
-        .method("raylet_socket_name", &rpc::GcsNodeInfo::raylet_socket_name)
-        .method("object_store_socket_name", &rpc::GcsNodeInfo::object_store_socket_name)
-        .method("node_manager_port", &rpc::GcsNodeInfo::node_manager_port);
-
     // class RayObject
     // https://github.com/ray-project/ray/blob/ray-2.5.1/src/ray/common/ray_object.h#L28
     mod.add_type<RayObject>("RayObject")
@@ -759,8 +750,7 @@ JLCXX_MODULE define_julia_module(jlcxx::Module& mod)
         .constructor<const gcs::GcsClientOptions&>()
         .method("GetNextJobID", &ray::gcs::GlobalStateAccessor::GetNextJobID)
         .method("Connect", &ray::gcs::GlobalStateAccessor::Connect)
-        .method("Disconnect", &ray::gcs::GlobalStateAccessor::Disconnect)
-        .method("GetNodeToConnectForDriver", &ray::gcs::GlobalStateAccessor::GetNodeToConnectForDriver);
+        .method("Disconnect", &ray::gcs::GlobalStateAccessor::Disconnect);
 
     mod.method("report_error", &report_error);
 

build/wrapper.h

@@ -9,7 +9,6 @@
 #include "ray/core_worker/common.h"
 #include "ray/core_worker/core_worker.h"
 #include "src/ray/protobuf/common.pb.h"
-#include "src/ray/protobuf/gcs.pb.h"
 #include "ray/gcs/gcs_client/gcs_client.h"
 #include "ray/gcs/gcs_client/global_state_accessor.h"
 #include "ray/common/asio/instrumented_io_context.h"

docs/src/building-artifacts.md

@@ -20,5 +20,4 @@ Once the PR has been merged into `main` and the CI has passed you must trigger t
 2. Generate the `Artifacts.toml` bindings and commit them.
 3. Create a new tag and GitHub pre-release which includes the artifacts as assets.
 
-The creation of the tag will trigger the "Artifacts CI" workflow which is responsible for verifying the generated artifacts work. 
-It will then promote the GitHub pre-release to a full release once those checks pass.
+Then you must trigger the ["Release" GitHub Action](https://github.com/beacon-biosignals/Ray.jl/actions/workflows/Release.yml) workflow which is responsible for verifying the generated artifacts work and promoting the aforementioned pre-release to a full release, setting it to "latest" if necessary.

src/constants.jl

@@ -1,49 +1,2 @@
-# NOTES:
-#
-# python function manager maintains a local table of "execution info" with a
-# "function_id" key and values that are named tuples of name/function/max_calls.
-#
-# python remote function sets a UUID4 at construction time:
-# https://github.com/beacon-biosignals/ray/blob/7ad1f47a9c849abf00ca3e8afc7c3c6ee54cda43/python/ray/remote_function.py#L128
-#
-# ...that's used to set the function_hash (???)...
-# https://github.com/beacon-biosignals/ray/blob/7ad1f47a9c849abf00ca3e8afc7c3c6ee54cda43/python/ray/remote_function.py#L263-L265
-#
-# later comment suggests that "ideally" they'd use the hash of the pickled
-# function:
-# https://github.com/beacon-biosignals/ray/blob/7ad1f47a9c849abf00ca3e8afc7c3c6ee54cda43/python/ray/includes/function_descriptor.pxi#L183-L186
-#
-# ...but that it's not stable for some reason.  but.....neither is a random
-# UUID?????
-#
-# the function table key is built like
-# <key type>:<jobid>:key
-
-# function manager holds:
-# local cache of functions (keyed by function id/hash from descriptor)
-# gcs client
-# ~~maybe job id?~~ this is managed by the core worker process
-
-# https://github.com/beacon-biosignals/ray/blob/1c0cddc478fa33d4c244d3c30aba861a77b0def9/python/ray/_private/ray_constants.py#L122-L123
-const FUNCTION_SIZE_WARN_THRESHOLD = 10_000_000  # in bytes
-const FUNCTION_SIZE_ERROR_THRESHOLD = 100_000_000  # in bytes
-
-# python uses "fun" for the namespace: https://github.com/beacon-biosignals/ray/blob/7ad1f47a9c849abf00ca3e8afc7c3c6ee54cda43/python/ray/_private/ray_constants.py#L380
-# so "jlfun" seems reasonable
-const FUNCTION_MANAGER_NAMESPACE = "jlfun"
-
-# env var to control whether logs are sent do stderr or to file.  if "1", sent
-# to stderr; otherwise, will be sent to files in `/tmp/ray/session_latest/logs/`
-# https://github.com/beacon-biosignals/ray/blob/4ceb62daaad05124713ff9d94ffbdad35ee19f86/python/ray/_private/ray_constants.py#L198
-const LOGGING_REDIRECT_STDERR_ENVIRONMENT_VARIABLE = "RAY_LOG_TO_STDERR"
-
-# In ray-2.5.1 this is constant but in later versions it's read from NODE_IP_ADDRESS.json
-# https://github.com/ray-project/ray/blob/a03efd9931128d387649dd48b0e4864b43d3bfb4/python/ray/_private/services.py#L650-L658
-const NODE_IP_ADDRESS = "127.0.0.1"
-
-const DEFAULT_SESSION_DIR = "/tmp/ray/session_latest"
-
-const GCS_ADDRESS_FILE = "/tmp/ray/ray_current_cluster"
-
 # Default `max_retries` option for non-actor tasks. Ray for Python defaults to 3.
 const DEFAULT_TASK_MAX_RETRIES = 0

src/function_manager.jl

@@ -1,5 +1,34 @@
-_mib_string(num_bytes) = string(div(num_bytes, 1024 * 1024), " MiB")
+# NOTES:
+#
+# python function manager maintains a local table of "execution info" with a
+# "function_id" key and values that are named tuples of name/function/max_calls.
+#
+# python remote function sets a UUID4 at construction time:
+# https://github.com/beacon-biosignals/ray/blob/7ad1f47a9c849abf00ca3e8afc7c3c6ee54cda43/python/ray/remote_function.py#L128
+#
+# ...that's used to set the function_hash (???)...
+# https://github.com/beacon-biosignals/ray/blob/7ad1f47a9c849abf00ca3e8afc7c3c6ee54cda43/python/ray/remote_function.py#L263-L265
+#
+# later comment suggests that "ideally" they'd use the hash of the pickled
+# function:
+# https://github.com/beacon-biosignals/ray/blob/7ad1f47a9c849abf00ca3e8afc7c3c6ee54cda43/python/ray/includes/function_descriptor.pxi#L183-L186
+#
+# ...but that it's not stable for some reason.  but.....neither is a random
+# UUID?????
+#
+# the function table key is built like
+# <key type>:<jobid>:key
+
+# function manager holds:
+# local cache of functions (keyed by function id/hash from descriptor)
+# gcs client
+# ~~maybe job id?~~ this is managed by the core worker process
+
+# https://github.com/beacon-biosignals/ray/blob/1c0cddc478fa33d4c244d3c30aba861a77b0def9/python/ray/_private/ray_constants.py#L122-L123
+const FUNCTION_SIZE_WARN_THRESHOLD = 10_000_000  # in bytes
+const FUNCTION_SIZE_ERROR_THRESHOLD = 100_000_000  # in bytes
 
+_mib_string(num_bytes) = string(div(num_bytes, 1024 * 1024), " MiB")
 # https://github.com/beacon-biosignals/ray/blob/1c0cddc478fa33d4c244d3c30aba861a77b0def9/python/ray/_private/utils.py#L744-L746
 const _check_msg = "Check that its definition is not implicitly capturing a large " *
                    "array or other object in scope. Tip: use `Ray.put()` to put large " *
@@ -22,6 +51,10 @@ function check_oversized_function(serialized, function_descriptor)
     return nothing
 end
 
+# python uses "fun" for the namespace: https://github.com/beacon-biosignals/ray/blob/7ad1f47a9c849abf00ca3e8afc7c3c6ee54cda43/python/ray/_private/ray_constants.py#L380
+# so "jlfun" seems reasonable
+const FUNCTION_MANAGER_NAMESPACE = "jlfun"
+
 Base.@kwdef struct FunctionManager
     gcs_client::ray_jll.JuliaGcsClient
     functions::Dict{String,Any}