diff --git a/src/common/mfu_daos.c b/src/common/mfu_daos.c index f15b0820..2dbf8cf5 100644 --- a/src/common/mfu_daos.c +++ b/src/common/mfu_daos.c @@ -308,6 +308,9 @@ int daos_parse_path( } else { strncpy(path, dattr.da_rel_path, path_len); } + } else if (strncmp(path, "daos:", 5) == 0) { + /* Actual error, since we expect a daos path */ + rc = -1; } else { /* If basename does not exist yet then duns_resolve_path will fail even * if dirname is a UNS path */ @@ -354,9 +357,6 @@ int daos_parse_path( snprintf(*pool_str, DAOS_PROP_LABEL_MAX_LEN + 1, "%s", dattr.da_pool); snprintf(*cont_str, DAOS_PROP_LABEL_MAX_LEN + 1, "%s", dattr.da_cont); strncpy(path, dattr.da_rel_path, path_len); - } else if (strncmp(path, "daos:", 5) == 0) { - /* Actual error, since we expect a daos path */ - rc = -1; } else { /* We didn't parse a daos path, * but we weren't necessarily looking for one */ @@ -450,7 +450,7 @@ static int daos_set_paths( return rc; } -static int daos_get_cont_type( +static int mfu_daos_get_cont_type( daos_handle_t coh, enum daos_cont_props* type) { @@ -516,7 +516,7 @@ static int daos_check_cont_status(daos_handle_t coh, bool *status_healthy) * Try to set the file type based on the container type, * using api as a guide. */ -static int daos_set_api_cont_type( +static int mfu_daos_set_api_cont_type( mfu_file_t* mfu_file, enum daos_cont_props cont_type, daos_api_t api) @@ -1013,7 +1013,7 @@ void daos_bcast_handle( } /* connect to DAOS pool, and then open container */ -int daos_connect( +int mfu_daos_connect( int rank, daos_args_t* da, char (*pool)[], @@ -1022,7 +1022,9 @@ int daos_connect( daos_handle_t* coh, bool force_serialize, bool connect_pool, + unsigned int pool_connect_flags, bool create_cont, + unsigned int cont_open_flags, bool require_new_cont, bool preserve, mfu_file_t* mfu_src_file, @@ -1059,12 +1061,12 @@ int daos_connect( } #endif /* Connect to DAOS pool */ - if (connect_pool) { + if (connect_pool && !daos_handle_is_valid(*poh)) { daos_pool_info_t pool_info = {0}; #if DAOS_API_VERSION_MAJOR < 1 - rc = daos_pool_connect(*pool, NULL, NULL, DAOS_PC_RW, poh, &pool_info, NULL); + rc = daos_pool_connect(*pool, NULL, NULL, pool_connect_flags, poh, &pool_info, NULL); #else - rc = daos_pool_connect(*pool, NULL, DAOS_PC_RW, poh, &pool_info, NULL); + rc = daos_pool_connect(*pool, NULL, pool_connect_flags, poh, &pool_info, NULL); #endif if (rc != 0) { MFU_LOG(MFU_LOG_ERR, "Failed to connect to pool: "DF_RC, DP_RC(rc)); @@ -1082,9 +1084,9 @@ int daos_connect( * not generated */ char cont_str[130]; uuid_unparse(da->dst_cont_uuid, cont_str); - rc = daos_cont_open(*poh, cont_str, DAOS_COO_RW, coh, &co_info, NULL); + rc = daos_cont_open(*poh, cont_str, cont_open_flags, coh, &co_info, NULL); } else { - rc = daos_cont_open(*poh, *cont, DAOS_COO_RW, coh, &co_info, NULL); + rc = daos_cont_open(*poh, *cont, cont_open_flags, coh, &co_info, NULL); } if (rc != 0) { if (rc != -DER_NONEXIST || !create_cont) { @@ -1146,7 +1148,7 @@ int daos_connect( /* if nothing is passed in for destination a uuid is always * generated unless user passed one in, destination container * labels are not generated */ - rc = dfs_cont_create(*poh, da->dst_cont_uuid, &attr, NULL, NULL); + rc = dfs_cont_create(*poh, &da->dst_cont_uuid, &attr, NULL, NULL); } if (rc != 0) { MFU_LOG(MFU_LOG_ERR, "Failed to create container: (%d %s)", rc, strerror(rc)); @@ -1156,7 +1158,7 @@ int daos_connect( if (dst_cont_passed && !is_uuid) { rc = daos_cont_create_with_label(*poh, *cont, props, &da->dst_cont_uuid, NULL); } else { - rc = daos_cont_create(*poh, da->dst_cont_uuid, props, NULL); + rc = daos_cont_create(*poh, &da->dst_cont_uuid, props, NULL); } if (rc != 0) { MFU_LOG(MFU_LOG_ERR, "Failed to create container: "DF_RC, DP_RC(rc)); @@ -1174,11 +1176,11 @@ int daos_connect( /* try to open it again */ if (dst_cont_passed && !is_uuid) { - rc = daos_cont_open(*poh, *cont, DAOS_COO_RW, coh, &co_info, NULL); + rc = daos_cont_open(*poh, *cont, cont_open_flags, coh, &co_info, NULL); } else { char cont_str[130]; uuid_unparse(da->dst_cont_uuid, cont_str); - rc = daos_cont_open(*poh, cont_str, DAOS_COO_RW, coh, &co_info, NULL); + rc = daos_cont_open(*poh, cont_str, cont_open_flags, coh, &co_info, NULL); } if (rc != 0) { MFU_LOG(MFU_LOG_ERR, "Failed to open container: "DF_RC, DP_RC(rc)); @@ -1554,8 +1556,7 @@ int daos_setup( } /* Make sure there weren't any errors before continuing. - * Since daos_connect has a collective broadcast. - * we have to make sure same_pool below is valid. */ + * Since mfu_daos_connect has a collective broadcast. */ if (daos_any_error(rank, local_daos_error, flag_daos_args)) { tmp_rc = daos_fini(); return 1; @@ -1569,10 +1570,16 @@ int daos_setup( bool have_src_cont = strlen(da->src_cont) > 0 ? true : false; bool have_dst_pool = strlen(da->dst_pool) > 0 ? true : false; - /* Check if containers are in the same pool */ - bool same_pool = (strcmp(da->src_pool, da->dst_pool) == 0); + /* Need W on destination pools for creating a container */ + unsigned int src_pool_connect_flags = DAOS_PC_RO; + unsigned int dst_pool_connect_flags = DAOS_PC_RW; + + /* For POSIX containers, the source only needs read, but the destination needs read and write. + * For DAOS (object-level), both containers need read and write. + * Open the source with RO first, then elevate to RW if needed below. */ + unsigned int src_cont_open_flags = DAOS_COO_RO; + unsigned int dst_cont_open_flags = DAOS_COO_RW; - bool connect_pool = true; bool create_cont = false; bool require_new_cont = false; bool preserve = false; @@ -1587,28 +1594,52 @@ int daos_setup( local_daos_error = true; goto out; } - tmp_rc = daos_connect(rank, da, &da->src_pool, &da->src_cont, - &da->src_poh, &da->src_coh, false, - connect_pool, create_cont, require_new_cont, - preserve, mfu_src_file, dst_cont_passed); + tmp_rc = mfu_daos_connect(rank, da, &da->src_pool, &da->src_cont, + &da->src_poh, &da->src_coh, false, + true, src_pool_connect_flags, create_cont, src_cont_open_flags, require_new_cont, + preserve, mfu_src_file, dst_cont_passed); if (tmp_rc != 0) { - /* tmp_rc from daos_connect is collective */ + /* tmp_rc from mfu_daos_connect is collective */ local_daos_error = true; goto out; } /* Get the container type */ - tmp_rc = daos_get_cont_type(da->src_coh, &da->src_cont_type); + tmp_rc = mfu_daos_get_cont_type(da->src_coh, &da->src_cont_type); if (tmp_rc != 0) { /* ideally, this should be the same for each process */ local_daos_error = true; goto out; } /* Set the src api based on the container type */ - tmp_rc = daos_set_api_cont_type(mfu_src_file, da->src_cont_type, da->api); + tmp_rc = mfu_daos_set_api_cont_type(mfu_src_file, da->src_cont_type, da->api); if (tmp_rc != 0) { local_daos_error = true; goto out; } + /* If using the DAOS API, we need to elevate permissions to RW for creating a snapshot */ + if (mfu_src_file->type == DAOS) { + MPI_Barrier(MPI_COMM_WORLD); + + tmp_rc = daos_cont_close(da->src_coh, NULL); + if (tmp_rc != 0) { + local_daos_error = true; + goto out; + } + da->src_coh = DAOS_HDL_INVAL; + + /* Re-open with RW */ + src_cont_open_flags = DAOS_COO_RW; + tmp_rc = mfu_daos_connect(rank, da, &da->src_pool, &da->src_cont, + &da->src_poh, &da->src_coh, false, + true, src_pool_connect_flags, create_cont, src_cont_open_flags, require_new_cont, + preserve, mfu_src_file, dst_cont_passed); + if (tmp_rc != 0) { + /* tmp_rc from mfu_daos_connect is collective */ + local_daos_error = true; + goto out; + } + } + /* Sanity check before we create a new container */ if (da->src_cont_type != DAOS_PROP_CO_LAYOUT_POSIX) { if (strcmp(da->src_path, "/") != 0) { @@ -1622,7 +1653,7 @@ int daos_setup( } /* If we're using the DAOS API, the destination container cannot - * exist already, unless overriden by allow_exist_dst_cont. */ + * exist already, unless overridden by allow_exist_dst_cont. */ if (mfu_src_file->type != POSIX && mfu_src_file->type != DFS && !da->allow_exist_dst_cont) { require_new_cont = true; } @@ -1646,11 +1677,11 @@ int daos_setup( create_cont = true; /* do check that src is POSIX and since dst has a pool, * then the dst *should* always be DFS, but this is not set - * on the destintaion until after daos_connect is called, and + * on the destintaion until after mfu_daos_connect is called, and * we should read the properties *before* the container is * created. We do not have the destination container type * here yet which is why extra check is used then passed - * to daos_connect */ + * to mfu_daos_connect */ if (da->daos_preserve) { if (mfu_src_file->type == POSIX) { preserve = true; @@ -1662,26 +1693,17 @@ int daos_setup( goto out; } } - if (same_pool) { - connect_pool = false; - tmp_rc = daos_connect(rank, da, &da->dst_pool, &da->dst_cont, - &da->src_poh, &da->dst_coh, false, - connect_pool, create_cont, require_new_cont, - preserve, mfu_src_file, dst_cont_passed); - } else { - connect_pool = true; - tmp_rc = daos_connect(rank, da, &da->dst_pool, &da->dst_cont, + tmp_rc = mfu_daos_connect(rank, da, &da->dst_pool, &da->dst_cont, &da->dst_poh, &da->dst_coh, false, - connect_pool, create_cont, require_new_cont, - preserve, mfu_src_file, dst_cont_passed); - } + true, dst_pool_connect_flags, create_cont, dst_cont_open_flags, + require_new_cont, preserve, mfu_src_file, dst_cont_passed); if (tmp_rc != 0) { - /* tmp_rc from daos_connect is collective */ + /* tmp_rc from mfu_daos_connect is collective */ local_daos_error = true; goto out; } /* Get the container type */ - tmp_rc = daos_get_cont_type(da->dst_coh, &da->dst_cont_type); + tmp_rc = mfu_daos_get_cont_type(da->dst_coh, &da->dst_cont_type); if (tmp_rc != 0) { /* ideally, this should be the same for each process */ local_daos_error = true; @@ -1689,7 +1711,7 @@ int daos_setup( } if (have_dst) { /* Set the dst api based on the container type */ - tmp_rc = daos_set_api_cont_type(mfu_dst_file, da->dst_cont_type, da->api); + tmp_rc = mfu_daos_set_api_cont_type(mfu_dst_file, da->dst_cont_type, da->api); if (tmp_rc != 0) { local_daos_error = true; goto out; @@ -1716,11 +1738,7 @@ int daos_setup( if (have_dst) { /* Mount destination DFS container */ if (mfu_dst_file->type == DFS) { - if (same_pool) { - tmp_rc = mfu_dfs_mount(mfu_dst_file, &da->src_poh, &da->dst_coh); - } else { - tmp_rc = mfu_dfs_mount(mfu_dst_file, &da->dst_poh, &da->dst_coh); - } + tmp_rc = mfu_dfs_mount(mfu_dst_file, &da->dst_poh, &da->dst_coh); if (tmp_rc != 0) { local_daos_error = true; goto out; @@ -1767,8 +1785,6 @@ int daos_cleanup( int rank; MPI_Comm_rank(MPI_COMM_WORLD, &rank); - bool same_pool = (strcmp(da->src_pool, da->dst_pool) == 0); - /* Destroy source snapshot */ if (rank == 0 && da->src_epc != 0) { tmp_rc = mfu_daos_destroy_snap(da->src_coh, da->src_epc); @@ -1837,7 +1853,7 @@ int daos_cleanup( } /* Close destination pool */ - if (daos_handle_is_valid(da->dst_poh) && !same_pool) { + if (daos_handle_is_valid(da->dst_poh)) { tmp_rc = daos_pool_disconnect(da->dst_poh, NULL); if (tmp_rc != 0) { MFU_LOG(MFU_LOG_ERR, "Failed to disconnect from destination pool "DF_RC, DP_RC(tmp_rc)); diff --git a/src/common/mfu_daos.h b/src/common/mfu_daos.h index 4ebb96bd..ab7ec66a 100644 --- a/src/common/mfu_daos.h +++ b/src/common/mfu_daos.h @@ -301,7 +301,7 @@ int daos_parse_path( /* connect to DAOS pool, * and then open container */ -int daos_connect( +int mfu_daos_connect( int rank, daos_args_t* da, char (*pool)[], @@ -310,7 +310,9 @@ int daos_connect( daos_handle_t* coh, bool force_serialize, bool connect_pool, + unsigned int pool_connect_flags, bool create_cont, + unsigned int cont_open_flags, bool require_new_cont, bool preserve, mfu_file_t* mfu_src_file, diff --git a/src/daos-serialize/daos-serialize.c b/src/daos-serialize/daos-serialize.c index f70f2c78..fc40892d 100644 --- a/src/daos-serialize/daos-serialize.c +++ b/src/daos-serialize/daos-serialize.c @@ -153,10 +153,10 @@ int main(int argc, char** argv) rc = 1; } - tmp_rc = daos_connect(rank, daos_args, &daos_args->src_pool, - &daos_args->src_cont, &daos_args->src_poh, - &daos_args->src_coh, force_serialize, true, - false, false, false, NULL, true); + tmp_rc = mfu_daos_connect(rank, daos_args, &daos_args->src_pool, + &daos_args->src_cont, &daos_args->src_poh, + &daos_args->src_coh, force_serialize, true, DAOS_PC_RW, + false, DAOS_COO_RW, false, false, NULL, true); if (tmp_rc != 0) { daos_fini(); mfu_finalize();