UCP/RNDV/CUDA: RNDV protocol improvements for CUDA

src/ucp/core/ucp_context.c

@@ -256,6 +256,10 @@ static ucs_config_field_t ucp_config_table[] = {
    "RNDV fragment size \n",
    ucs_offsetof(ucp_config_t, ctx.rndv_frag_size), UCS_CONFIG_TYPE_MEMUNITS},
 
+  {"RNDV_PIPELINE_SEND_THRESH", "inf",
+   "RNDV size threshold to enable sender side pipeline for mem type\n",
+   ucs_offsetof(ucp_config_t, ctx.rndv_pipeline_send_thresh), UCS_CONFIG_TYPE_MEMUNITS},
+
   {"MEMTYPE_CACHE", "y",
    "Enable memory type (cuda/rocm) cache \n",
    ucs_offsetof(ucp_config_t, ctx.enable_memtype_cache), UCS_CONFIG_TYPE_BOOL},

src/ucp/core/ucp_context.h

@@ -61,6 +61,8 @@ typedef struct ucp_context_config {
     size_t                                 seg_size;
     /** RNDV pipeline fragment size */
     size_t                                 rndv_frag_size;
+    /** RNDV pipline send threshold */
+    size_t                                 rndv_pipeline_send_thresh;
     /** Threshold for using tag matching offload capabilities. Smaller buffers
      *  will not be posted to the transport. */
     size_t                                 tm_thresh;

src/ucp/core/ucp_request.h

@@ -155,12 +155,14 @@ struct ucp_request {
                 } proxy;
 
                 struct {
-                    uint64_t             remote_address; /* address of the sender's data buffer */
-                    uintptr_t            remote_request; /* pointer to the sender's request */
-                    ucp_request_t        *rreq;          /* receive request on the recv side */
-                    ucp_rkey_h           rkey;           /* key for remote send buffer */
-                    ucp_lane_map_t       lanes_map;      /* used lanes map */
-                    ucp_lane_index_t     lane_count;     /* number of lanes used in transaction */
+                    uint64_t             remote_address;  /* address of the sender's data buffer */
+                    uintptr_t            remote_request;  /* pointer to the sender's request */
+                    ucp_request_t        *rreq;           /* receive request on the recv side */
+                    ucp_rkey_h           rkey;            /* key for remote send buffer */
+                    ucp_lane_map_t       lanes_map_avail; /* used lanes map */
+                    ucp_lane_map_t       lanes_map_all;   /* actual lanes map */
+                    uint8_t              lanes_count;     /* actual lanes count */
+                    uint8_t              rkey_index[UCP_MAX_LANES];
                 } rndv_get;
 
                 struct {