From 958fa224912a70fe802d5d5ca948de82e3cc31a0 Mon Sep 17 00:00:00 2001 From: Igor Shishkin Date: Thu, 22 Aug 2024 19:40:47 +0300 Subject: [PATCH] Add ability to move container from one namespace to another (#174) Related #130 Signed-off-by: Igor Shishkin --- README.md | 44 ++++++++++++++----- cli/service/pb_mock.go | 5 +++ cli/service/service.go | 18 +++++++- cli/service/service_test.go | 7 +++ cmd/cli/main.go | 7 ++- manager/presenter/grpc/handlers.go | 9 ++++ manager/presenter/grpc/handlers_test.go | 11 +++++ manager/presenter/grpc/proto/v1/manager.proto | 8 ++++ .../html/templates/container-list.html | 1 + .../html/testdata/containers.html.sample | 1 + .../cache/metadata/memcache/memcache.go | 4 +- .../cache/metadata/memcache/memcache_test.go | 6 +-- repositories/metadata/metadata.go | 2 +- repositories/metadata/mock/mock.go | 4 +- .../metadata/postgresql/containers.go | 16 ++++++- .../metadata/postgresql/containers_test.go | 13 ++++-- service/mock.go | 5 +++ service/service.go | 11 ++++- service/service_test.go | 19 +++++++- 19 files changed, 164 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 81e9871..eb7eac2 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ To do so archived relies on two storages: metadata and CAS. Metadata is a some kind of database to store all of the things: +* namespaces - group of containers * containers - some kind of directories * versions - immutable version of the data in container * objects - named data BLOBs with some additional metadata @@ -49,7 +50,8 @@ archived is built with microservice architecture containing the following components: * archived-publisher - HTTP server to allow data listing and fetching -* archived-manager - gRPC API to manage containers, versions and objects +* archived-manager - gRPC API to manage namespaces, containers, versions and + objects * archived-exporter - Prometheus metrics exporter for metadata entities * CLI - CLI application to interact with manage component * migrator - metadata migration tool @@ -86,8 +88,8 @@ reference. ## CLI archived-cli provides an CLI interface to operate archived including creating -containers, versions and objects. It works with archived-manager to handle -requests. +namespaces, containers, versions and objects. It works with archived-manager +to handle requests. ```shell usage: archived-cli --endpoint=ENDPOINT [] [ ...] @@ -96,23 +98,42 @@ CLI interface for archived Flags: - --[no-]help Show context-sensitive help (also try --help-long and --help-man). - -d, --[no-]debug Enable debug mode ($ARCHIVED_CLI_DEBUG) - -t, --[no-]trace Enable trace mode (debug mode on steroids) ($ARCHIVED_CLI_TRACE) - -s, --endpoint=ENDPOINT Manage API endpoint address ($ARCHIVED_CLI_ENDPOINT) - --[no-]insecure Do not use TLS for gRPC connection + --[no-]help Show context-sensitive help (also try --help-long and --help-man). + -d, --[no-]debug Enable debug mode ($ARCHIVED_CLI_DEBUG) + -t, --[no-]trace Enable trace mode (debug mode on steroids) ($ARCHIVED_CLI_TRACE) + -s, --endpoint=ENDPOINT Manager API endpoint address ($ARCHIVED_CLI_ENDPOINT) + --[no-]insecure Do not use TLS for gRPC connection --[no-]insecure-skip-verify - Do not perform TLS certificate verification for gRPC connection + Do not perform TLS certificate verification for gRPC connection --cache-dir="~/.cache/archived/cli/objects" - cache directory for objects + Stat-cache directory for objects ($ARCHIVED_CLI_STAT_CACHE_DIR) + -n, --namespace="default" namespace for containers to operate on Commands: help [...] Show help. +namespace create + create new namespace + +namespace rename + rename the given namespace + +namespace delete + delete the given namespace + +namespace list + list namespaces + container create create new container +container move + move container to another namespace + +container rename + rename the given container + container delete delete the given container @@ -142,6 +163,9 @@ object url object delete delete object + +stat-cache show-path + print actual cache path ``` ## How build the project manually diff --git a/cli/service/pb_mock.go b/cli/service/pb_mock.go index 46f87a9..d072ecb 100644 --- a/cli/service/pb_mock.go +++ b/cli/service/pb_mock.go @@ -45,6 +45,11 @@ func (m *protoClientMock) CreateContainer(ctx context.Context, in *v1proto.Creat return &v1proto.CreateContainerResponse{}, args.Error(0) } +func (m *protoClientMock) MoveContainer(ctx context.Context, in *v1proto.MoveContainerRequest, opts ...grpc.CallOption) (*v1proto.MoveContainerResponse, error) { + args := m.Called(in.GetNamespace(), in.GetContainerName(), in.GetDestinationNamespace()) + return &v1proto.MoveContainerResponse{}, args.Error(0) +} + func (m *protoClientMock) RenameContainer(ctx context.Context, in *v1proto.RenameContainerRequest, opts ...grpc.CallOption) (*v1proto.RenameContainerResponse, error) { args := m.Called(in.GetNamespace(), in.GetOldName(), in.GetNewName()) return &v1proto.RenameContainerResponse{}, args.Error(0) diff --git a/cli/service/service.go b/cli/service/service.go index 77e6433..ee81b55 100644 --- a/cli/service/service.go +++ b/cli/service/service.go @@ -31,6 +31,7 @@ type Service interface { DeleteNamespace(namespaceName string) func(ctx context.Context) error CreateContainer(namespaceName, containerName string) func(ctx context.Context) error + MoveContainer(namespaceName, containerName, destinationNamespace string) func(ctx context.Context) error RenameContainer(namespaceName, oldName, newName string) func(ctx context.Context) error ListContainers(namespaceName string) func(ctx context.Context) error DeleteContainer(namespaceName, containerName string) func(ctx context.Context) error @@ -129,6 +130,21 @@ func (s *service) CreateContainer(namespaceName, containerName string) func(ctx } } +func (s *service) MoveContainer(namespaceName, containerName, destinationNamespace string) func(ctx context.Context) error { + return func(ctx context.Context) error { + _, err := s.cli.MoveContainer(ctx, &v1proto.MoveContainerRequest{ + Namespace: namespaceName, + ContainerName: containerName, + DestinationNamespace: destinationNamespace, + }) + if err != nil { + return errors.Wrap(err, "error moving container") + } + fmt.Printf("container `%s` just moved from `%s` to `%s`\n", containerName, namespaceName, destinationNamespace) + return nil + } +} + func (s *service) RenameContainer(namespaceName, oldName, newName string) func(ctx context.Context) error { return func(ctx context.Context) error { _, err := s.cli.RenameContainer(ctx, &v1proto.RenameContainerRequest{ @@ -340,7 +356,7 @@ func (s *service) createVersionFromYUMRepository(ctx context.Context, namespaceN fp, err := lb.Reader(ctx) if err != nil { - return errors.Wrap(err, "error opening package file") + return errors.Wrap(err, "error getting reader for package file") } return uploadBlob(ctx, uploadURL, fp, size) diff --git a/cli/service/service_test.go b/cli/service/service_test.go index 63be278..4a5ac70 100644 --- a/cli/service/service_test.go +++ b/cli/service/service_test.go @@ -29,6 +29,13 @@ func (s *serviceTestSuite) TestCreateContainer() { s.Require().NoError(fn(s.ctx)) } +func (s *serviceTestSuite) TestMoveContainer() { + s.cliMock.On("MoveContainer", defaultNamespace, "test-container", "new-namespace").Return(nil).Once() + + fn := s.svc.MoveContainer(defaultNamespace, "test-container", "new-namespace") + s.Require().NoError(fn(s.ctx)) +} + func (s *serviceTestSuite) TestRenameContainer() { s.cliMock.On("RenameContainer", defaultNamespace, "old-name", "new-name").Return(nil).Once() diff --git a/cmd/cli/main.go b/cmd/cli/main.go index b2a1183..27ef6e3 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -66,7 +66,7 @@ var ( namespace = app.Command("namespace", "namespace operations") namespaceCreate = namespace.Command("create", "create new namespace") - namespaceCreateName = containerCreate.Arg("name", "name of the namespace to create").Required().String() + namespaceCreateName = namespaceCreate.Arg("name", "name of the namespace to create").Required().String() namespaceRename = namespace.Command("rename", "rename the given namespace") namespaceRenameOldName = namespaceRename.Arg("old-name", "the old name of the namespace").Required().String() @@ -81,6 +81,10 @@ var ( containerCreate = container.Command("create", "create new container") containerCreateName = containerCreate.Arg("name", "name of the container to create").Required().String() + containerMove = container.Command("move", "move container to another namespace") + containerMoveName = containerMove.Arg("name", "container namespace to move").Required().String() + containerMoveNamespace = containerMove.Arg("namespace", "destination namespace to move to").Required().String() + containerRename = container.Command("rename", "rename the given container") containerRenameOldName = containerRename.Arg("old-name", "the old name of the container").Required().String() containerRenameNewName = containerRename.Arg("new-name", "the new name of the container").Required().String() @@ -218,6 +222,7 @@ func main() { r.Register(namespaceDelete.FullCommand(), cliSvc.DeleteNamespace(*containerDeleteName)) r.Register(containerCreate.FullCommand(), cliSvc.CreateContainer(*namespaceName, *containerCreateName)) + r.Register(containerMove.FullCommand(), cliSvc.MoveContainer(*namespaceName, *containerMoveName, *containerMoveNamespace)) r.Register(containerRename.FullCommand(), cliSvc.RenameContainer(*namespaceName, *containerRenameOldName, *containerRenameNewName)) r.Register(containerList.FullCommand(), cliSvc.ListContainers(*namespaceName)) r.Register(containerDelete.FullCommand(), cliSvc.DeleteContainer(*namespaceName, *containerDeleteName)) diff --git a/manager/presenter/grpc/handlers.go b/manager/presenter/grpc/handlers.go index db679ab..8a26f27 100644 --- a/manager/presenter/grpc/handlers.go +++ b/manager/presenter/grpc/handlers.go @@ -79,6 +79,15 @@ func (h *handlers) CreateContainer(ctx context.Context, in *v1.CreateContainerRe return &v1.CreateContainerResponse{}, nil } +func (h *handlers) MoveContainer(ctx context.Context, in *v1.MoveContainerRequest) (*v1.MoveContainerResponse, error) { + err := h.svc.MoveContainer(ctx, in.GetNamespace(), in.GetContainerName(), in.GetDestinationNamespace()) + if err != nil { + return nil, mapServiceError(err) + } + + return &v1.MoveContainerResponse{}, nil +} + func (h *handlers) RenameContainer(ctx context.Context, in *v1.RenameContainerRequest) (*v1.RenameContainerResponse, error) { err := h.svc.RenameContainer(ctx, in.GetNamespace(), in.GetOldName(), in.GetNewName()) if err != nil { diff --git a/manager/presenter/grpc/handlers_test.go b/manager/presenter/grpc/handlers_test.go index 4a0bad5..f61a6cf 100644 --- a/manager/presenter/grpc/handlers_test.go +++ b/manager/presenter/grpc/handlers_test.go @@ -63,6 +63,17 @@ func (s *manageHandlersTestSuite) TestCreateContainer() { s.Require().NoError(err) } +func (s *manageHandlersTestSuite) TestMoveContainer() { + s.svcMock.On("MoveContainer", defaultNamespace, "test-container", "new-namespace").Return(nil).Once() + + _, err := s.client.MoveContainer(s.ctx, &v1pb.MoveContainerRequest{ + Namespace: defaultNamespace, + ContainerName: "test-container", + DestinationNamespace: "new-namespace", + }) + s.Require().NoError(err) +} + func (s *manageHandlersTestSuite) TestCreateContainerNotFound() { s.svcMock.On("CreateContainer", defaultNamespace, "test-container").Return(service.ErrNotFound).Once() diff --git a/manager/presenter/grpc/proto/v1/manager.proto b/manager/presenter/grpc/proto/v1/manager.proto index fe97918..d56de8b 100644 --- a/manager/presenter/grpc/proto/v1/manager.proto +++ b/manager/presenter/grpc/proto/v1/manager.proto @@ -32,6 +32,13 @@ message CreateContainerRequest { } message CreateContainerResponse {} +message MoveContainerRequest { + string namespace = 1; + string container_name = 2; + string destination_namespace = 3; +} +message MoveContainerResponse {} + message RenameContainerRequest{ string namespace = 1; string old_name = 2; @@ -136,6 +143,7 @@ service ManageService { rpc ListNamespaces(ListNamespacesRequest) returns (ListNamespacesResponse); rpc CreateContainer(CreateContainerRequest) returns (CreateContainerResponse); + rpc MoveContainer(MoveContainerRequest) returns (MoveContainerResponse); rpc RenameContainer(RenameContainerRequest) returns (RenameContainerResponse); rpc DeleteContainer(DeleteContainerRequest) returns (DeleteContainerResponse); rpc ListContainers(ListContainersRequest) returns (ListContainersResponse); diff --git a/publisher/presenter/html/templates/container-list.html b/publisher/presenter/html/templates/container-list.html index d89548e..15788c4 100644 --- a/publisher/presenter/html/templates/container-list.html +++ b/publisher/presenter/html/templates/container-list.html @@ -1,5 +1,6 @@ {{ $namespace := .Namespace }} {{ template "header" . }} + ..
{{- range $container := .Containers }} {{ $container }}/
{{- else }} diff --git a/publisher/presenter/html/testdata/containers.html.sample b/publisher/presenter/html/testdata/containers.html.sample index 5a139e7..e405178 100644 --- a/publisher/presenter/html/testdata/containers.html.sample +++ b/publisher/presenter/html/testdata/containers.html.sample @@ -9,6 +9,7 @@

Container index (default)

+ ..
test-container-1/
diff --git a/repositories/cache/metadata/memcache/memcache.go b/repositories/cache/metadata/memcache/memcache.go index 2180343..9dd4200 100644 --- a/repositories/cache/metadata/memcache/memcache.go +++ b/repositories/cache/metadata/memcache/memcache.go @@ -93,8 +93,8 @@ func (m *memcache) CreateContainer(ctx context.Context, namespace, name string) return m.repo.CreateContainer(ctx, namespace, name) } -func (m *memcache) RenameContainer(ctx context.Context, namespace, oldName, newName string) error { - return m.repo.RenameContainer(ctx, namespace, oldName, newName) +func (m *memcache) RenameContainer(ctx context.Context, namespace, oldName, newNamespace, newName string) error { + return m.repo.RenameContainer(ctx, namespace, oldName, newNamespace, newName) } func (m *memcache) ListContainers(ctx context.Context, namespace string) ([]string, error) { diff --git a/repositories/cache/metadata/memcache/memcache_test.go b/repositories/cache/metadata/memcache/memcache_test.go index 23c3e35..3b6a687 100644 --- a/repositories/cache/metadata/memcache/memcache_test.go +++ b/repositories/cache/metadata/memcache/memcache_test.go @@ -180,12 +180,12 @@ func (s *memcacheTestSuite) TestCreateContainer() { } func (s *memcacheTestSuite) TestRenameContainer() { - s.repoMock.On("RenameContainer", defaultNamespace, "old-name", "new-name").Return(nil).Twice() + s.repoMock.On("RenameContainer", defaultNamespace, "old-name", "new-namespace", "new-name").Return(nil).Twice() - err := s.cache.RenameContainer(s.ctx, defaultNamespace, "old-name", "new-name") + err := s.cache.RenameContainer(s.ctx, defaultNamespace, "old-name", "new-namespace", "new-name") s.Require().NoError(err) - err = s.cache.RenameContainer(s.ctx, defaultNamespace, "old-name", "new-name") + err = s.cache.RenameContainer(s.ctx, defaultNamespace, "old-name", "new-namespace", "new-name") s.Require().NoError(err) } diff --git a/repositories/metadata/metadata.go b/repositories/metadata/metadata.go index 7d7376f..7f24325 100644 --- a/repositories/metadata/metadata.go +++ b/repositories/metadata/metadata.go @@ -21,7 +21,7 @@ type Repository interface { DeleteNamespace(ctx context.Context, name string) error CreateContainer(ctx context.Context, namespace, name string) error - RenameContainer(ctx context.Context, namespace, oldName, newName string) error + RenameContainer(ctx context.Context, namespace, oldName, newNamespace, newName string) error ListContainers(ctx context.Context, namespace string) ([]string, error) DeleteContainer(ctx context.Context, namespace, name string) error diff --git a/repositories/metadata/mock/mock.go b/repositories/metadata/mock/mock.go index e3147c1..0eb3269 100644 --- a/repositories/metadata/mock/mock.go +++ b/repositories/metadata/mock/mock.go @@ -44,8 +44,8 @@ func (m *Mock) CreateContainer(_ context.Context, namespace, name string) error return args.Error(0) } -func (m *Mock) RenameContainer(_ context.Context, namespace, oldName, newName string) error { - args := m.Called(namespace, oldName, newName) +func (m *Mock) RenameContainer(_ context.Context, namespace, oldName, newNamespace, newName string) error { + args := m.Called(namespace, oldName, newNamespace, newName) return args.Error(0) } diff --git a/repositories/metadata/postgresql/containers.go b/repositories/metadata/postgresql/containers.go index 2b17a44..cc19707 100644 --- a/repositories/metadata/postgresql/containers.go +++ b/repositories/metadata/postgresql/containers.go @@ -48,7 +48,7 @@ func (r *repository) CreateContainer(ctx context.Context, namespace, name string return nil } -func (r *repository) RenameContainer(ctx context.Context, namespace, oldName, newName string) error { +func (r *repository) RenameContainer(ctx context.Context, namespace, oldName, newNamespace, newName string) error { tx, err := r.db.BeginTx(ctx, nil) if err != nil { return mapSQLErrors(err) @@ -68,6 +68,19 @@ func (r *repository) RenameContainer(ctx context.Context, namespace, oldName, ne return mapSQLErrors(err) } + row, err = selectQueryRow(ctx, tx, psql. + Select("id"). + From("namespaces"). + Where(sq.Eq{"name": newNamespace})) + if err != nil { + return mapSQLErrors(err) + } + + var newNamespaceID uint + if err := row.Scan(&newNamespaceID); err != nil { + return mapSQLErrors(err) + } + row, err = selectQueryRow(ctx, tx, psql. Select("id"). From("containers"). @@ -87,6 +100,7 @@ func (r *repository) RenameContainer(ctx context.Context, namespace, oldName, ne _, err = updateQuery(ctx, tx, psql. Update("containers"). Set("name", newName). + Set("namespace_id", newNamespaceID). Where(sq.Eq{ "id": containerID, }), diff --git a/repositories/metadata/postgresql/containers_test.go b/repositories/metadata/postgresql/containers_test.go index cd19ae7..26c2dc0 100644 --- a/repositories/metadata/postgresql/containers_test.go +++ b/repositories/metadata/postgresql/containers_test.go @@ -3,7 +3,10 @@ package postgresql import "github.com/teran/archived/repositories/metadata" func (s *postgreSQLRepositoryTestSuite) TestContainerOperations() { - s.tp.On("Now").Return("2024-01-02T01:02:03Z").Times(3) + s.tp.On("Now").Return("2024-01-02T01:02:03Z").Times(4) + + err := s.repo.CreateNamespace(s.ctx, "new-namespace") + s.Require().NoError(err) list, err := s.repo.ListContainers(s.ctx, defaultNamespace) s.Require().NoError(err) @@ -38,15 +41,19 @@ func (s *postgreSQLRepositoryTestSuite) TestContainerOperations() { "test-container5", }, list) - err = s.repo.RenameContainer(s.ctx, defaultNamespace, "test-container5", "and-then-there-was-the-one") + err = s.repo.RenameContainer(s.ctx, defaultNamespace, "test-container5", "new-namespace", "and-then-there-was-the-one") s.Require().NoError(err) - err = s.repo.RenameContainer(s.ctx, defaultNamespace, "not-existent", "some-name") + err = s.repo.RenameContainer(s.ctx, defaultNamespace, "not-existent", "new-namespace", "some-name") s.Require().Error(err) s.Require().Equal(metadata.ErrNotFound, err) list, err = s.repo.ListContainers(s.ctx, defaultNamespace) s.Require().NoError(err) + s.Require().Equal([]string{}, list) + + list, err = s.repo.ListContainers(s.ctx, "new-namespace") + s.Require().NoError(err) s.Require().Equal([]string{ "and-then-there-was-the-one", }, list) diff --git a/service/mock.go b/service/mock.go index 960235a..8e5c170 100644 --- a/service/mock.go +++ b/service/mock.go @@ -45,6 +45,11 @@ func (m *Mock) CreateContainer(_ context.Context, namespace, name string) error return args.Error(0) } +func (m *Mock) MoveContainer(ctx context.Context, namespace, container, destNamespace string) error { + args := m.Called(namespace, container, destNamespace) + return args.Error(0) +} + func (m *Mock) RenameContainer(_ context.Context, namespace, oldName, newName string) error { args := m.Called(namespace, oldName, newName) return args.Error(0) diff --git a/service/service.go b/service/service.go index c83efcb..6c0634b 100644 --- a/service/service.go +++ b/service/service.go @@ -21,6 +21,7 @@ type Manager interface { DeleteNamespace(ctx context.Context, name string) error CreateContainer(ctx context.Context, namespace, name string) error + MoveContainer(ctx context.Context, namespace, container, destNamespace string) error RenameContainer(ctx context.Context, namespace, oldName, newName string) error DeleteContainer(ctx context.Context, namespace, name string) error @@ -106,8 +107,16 @@ func (s *service) CreateContainer(ctx context.Context, namespace, name string) e return nil } +func (s *service) MoveContainer(ctx context.Context, namespace, container, destNamespace string) error { + err := s.mdRepo.RenameContainer(ctx, namespace, container, destNamespace, container) + if err != nil { + return mapMetadataErrors(err) + } + return nil +} + func (s *service) RenameContainer(ctx context.Context, namespace, oldName, newName string) error { - err := s.mdRepo.RenameContainer(ctx, namespace, oldName, newName) + err := s.mdRepo.RenameContainer(ctx, namespace, oldName, namespace, newName) if err != nil { return mapMetadataErrors(err) } diff --git a/service/service_test.go b/service/service_test.go index 41d1054..d7f957c 100644 --- a/service/service_test.go +++ b/service/service_test.go @@ -87,15 +87,30 @@ func (s *serviceTestSuite) TestCreateContainer() { s.Require().Equal("test error", err.Error()) } +func (s *serviceTestSuite) TestMoveContainer() { + // Happy path + s.mdRepoMock.On("RenameContainer", defaultNamespace, "container", "new-namespace", "container").Return(nil).Once() + + err := s.svc.MoveContainer(s.ctx, defaultNamespace, "container", "new-namespace") + s.Require().NoError(err) + + // return error + s.mdRepoMock.On("RenameContainer", defaultNamespace, "container", "new-namespace", "container").Return(errors.New("test error")).Once() + + err = s.svc.MoveContainer(s.ctx, defaultNamespace, "container", "new-namespace") + s.Require().Error(err) + s.Require().Equal("test error", err.Error()) +} + func (s *serviceTestSuite) TestRenameContainer() { // Happy path - s.mdRepoMock.On("RenameContainer", defaultNamespace, "old-name", "new-name").Return(nil).Once() + s.mdRepoMock.On("RenameContainer", defaultNamespace, "old-name", defaultNamespace, "new-name").Return(nil).Once() err := s.svc.RenameContainer(s.ctx, defaultNamespace, "old-name", "new-name") s.Require().NoError(err) // return error - s.mdRepoMock.On("RenameContainer", defaultNamespace, "old-name", "new-name").Return(errors.New("test error")).Once() + s.mdRepoMock.On("RenameContainer", defaultNamespace, "old-name", defaultNamespace, "new-name").Return(errors.New("test error")).Once() err = s.svc.RenameContainer(s.ctx, defaultNamespace, "old-name", "new-name") s.Require().Error(err)