Refactoring services (#287)

* refactor: remove iternal for Archivista flexibility

Remove the internal in order to enable Archivista source code user to:
- Extend/Modify the existent services (config, storage, etc) within
  their code base

- Allow to use Archivista encapuslated in other services, example API
  service

Signed-off-by: Kairo Araujo <kairo.araujo@testifysec.com>

* refactor: simplify archivista main cmd

Simplifies Archivista main cmd moving all the service logic into the
`pkg/server`.

It allows users to use the Archivista Server instance as an Service
Interface allowing to integrate along to API services.

Signed-off-by: Kairo Araujo <kairo.araujo@testifysec.com>

---------

Signed-off-by: Kairo Araujo <kairo.araujo@testifysec.com>

cmd/archivista/main.go

@@ -21,7 +21,6 @@ package main
 
 import (
 	"context"
-	"fmt"
 	"net"
 	"net/http"
 	"os"
@@ -32,13 +31,7 @@ import (
 
 	nested "github.com/antonfisher/nested-logrus-formatter"
 	"github.com/gorilla/handlers"
-	"github.com/in-toto/archivista/internal/artifactstore"
-	"github.com/in-toto/archivista/internal/config"
-	"github.com/in-toto/archivista/internal/metadatastorage/sqlstore"
-	"github.com/in-toto/archivista/internal/objectstorage/blobstore"
-	"github.com/in-toto/archivista/internal/objectstorage/filestore"
-	"github.com/in-toto/archivista/internal/server"
-	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/in-toto/archivista/pkg/server"
 	"github.com/sirupsen/logrus"
 )
 
@@ -57,76 +50,19 @@ func main() {
 	defer cancel()
 
 	startTime := time.Now()
-	serverOpts := make([]server.Option, 0)
 
-	logrus.Infof("executing phase 1: get config from environment (time since start: %s)", time.Since(startTime))
-	now := time.Now()
-
-	cfg := new(config.Config)
-	if err := cfg.Process(); err != nil {
-		logrus.Fatal(err)
-	}
-
-	level, err := logrus.ParseLevel(cfg.LogLevel)
-	if err != nil {
-		logrus.Fatalf("invalid log level %s", cfg.LogLevel)
-	}
-	logrus.SetLevel(level)
-
-	logrus.WithField("duration", time.Since(now)).Infof("completed phase 1: get config from environment")
-
-	// ********************************************************************************
-	logrus.Infof("executing phase 2: initializing storage clients (time since start: %s)", time.Since(startTime))
-	// ********************************************************************************
-	now = time.Now()
-	fileStore, fileStoreCh, err := initObjectStore(ctx, cfg)
-	if err != nil {
-		logrus.Fatalf("error initializing storage clients: %+v", err)
-	}
-	serverOpts = append(serverOpts, server.WithObjectStore(fileStore))
+	archivistaService := &server.ArchivistaService{Ctx: ctx, Cfg: nil}
 
-	entClient, err := sqlstore.NewEntClient(
-		cfg.SQLStoreBackend,
-		cfg.SQLStoreConnectionString,
-		sqlstore.ClientWithMaxIdleConns(cfg.SQLStoreMaxIdleConnections),
-		sqlstore.ClientWithMaxOpenConns(cfg.SQLStoreMaxOpenConnections),
-		sqlstore.ClientWithConnMaxLifetime(cfg.SQLStoreConnectionMaxLifetime))
+	server, err := archivistaService.Setup()
 	if err != nil {
-		logrus.Fatalf("could not create ent client: %+v", err)
+		logrus.Fatalf("unable to setup archivista service: %+v", err)
 	}
-
-	sqlStore, sqlStoreCh, err := sqlstore.New(ctx, entClient)
-	if err != nil {
-		logrus.Fatalf("error initializing mysql client: %+v", err)
-	}
-	serverOpts = append(serverOpts, server.WithMetadataStore(sqlStore))
-
-	logrus.WithField("duration", time.Since(now)).Infof("completed phase 3: initializing storage clients")
-
 	// ********************************************************************************
-	logrus.Infof("executing phase 3: create and register http service (time since start: %s)", time.Since(startTime))
+	logrus.Infof("executing phase: create and register http service (time since start: %s)", time.Since(startTime))
 	// ********************************************************************************
-	now = time.Now()
-
-	// initialize the artifact store
-	if cfg.EnableArtifactStore {
-		wds, err := artifactstore.New(artifactstore.WithConfigFile(cfg.ArtifactStoreConfig))
-		if err != nil {
-			logrus.Fatalf("could not create the artifact store: %+v", err)
-		}
-
-		serverOpts = append(serverOpts, server.WithArtifactStore(wds))
-	}
-
-	// initialize the server
-	sqlClient := sqlStore.GetClient()
-	serverOpts = append(serverOpts, server.WithEntSqlClient(sqlClient))
-	server, err := server.New(cfg, serverOpts...)
-	if err != nil {
-		logrus.Fatalf("could not create archivista server: %+v", err)
-	}
+	now := time.Now()
 
-	listenAddress := cfg.ListenOn
+	listenAddress := archivistaService.Cfg.ListenOn
 	listenAddress = strings.ToLower(strings.TrimSpace(listenAddress))
 	proto := ""
 	if strings.HasPrefix(listenAddress, "tcp://") {
@@ -143,10 +79,10 @@ func main() {
 	}
 	srv := &http.Server{
 		Handler: handlers.CORS(
-			handlers.AllowedOrigins(cfg.CORSAllowOrigins),
+			handlers.AllowedOrigins(archivistaService.Cfg.CORSAllowOrigins),
 			handlers.AllowedMethods([]string{"GET", "POST", "OPTIONS"}),
 			handlers.AllowedHeaders([]string{"Accept", "Content-Type", "Content-Length", "Accept-Encoding", "X-CSRF-Token", "Authorization"}),
-    )(server.Router()),
+		)(server.Router()),
 		ReadTimeout:  5 * time.Second,
 		WriteTimeout: 10 * time.Second,
 	}
@@ -156,47 +92,12 @@ func main() {
 		}
 	}()
 
-	logrus.WithField("duration", time.Since(now)).Infof("completed phase 5: create and register http service")
+	logrus.WithField("duration", time.Since(now)).Infof("completed phase: create and register http service")
 	logrus.Infof("startup complete (time since start: %s)", time.Since(startTime))
 
 	<-ctx.Done()
-	<-fileStoreCh
-	<-sqlStoreCh
+	<-archivistaService.GetFileStoreCh()
+	<-archivistaService.GetSQLStoreCh()
 
 	logrus.Infof("exiting, uptime: %v", time.Since(startTime))
 }
-
-func initObjectStore(ctx context.Context, cfg *config.Config) (server.StorerGetter, <-chan error, error) {
-	switch strings.ToUpper(cfg.StorageBackend) {
-	case "FILE":
-		return filestore.New(ctx, cfg.FileDir, cfg.FileServeOn)
-
-	case "BLOB":
-		var creds *credentials.Credentials
-		if cfg.BlobStoreCredentialType == "IAM" {
-			creds = credentials.NewIAM("")
-		} else if cfg.BlobStoreCredentialType == "ACCESS_KEY" {
-			creds = credentials.NewStaticV4(cfg.BlobStoreAccessKeyId, cfg.BlobStoreSecretAccessKeyId, "")
-		} else {
-			logrus.Fatalln("invalid blob store credential type: ", cfg.BlobStoreCredentialType)
-		}
-		return blobstore.New(
-			ctx,
-			cfg.BlobStoreEndpoint,
-			creds,
-			cfg.BlobStoreBucketName,
-			cfg.BlobStoreUseTLS,
-		)
-
-	case "":
-		errCh := make(chan error)
-		go func() {
-			<-ctx.Done()
-			close(errCh)
-		}()
-		return nil, errCh, nil
-
-	default:
-		return nil, nil, fmt.Errorf("unknown storage backend: %s", cfg.StorageBackend)
-	}
-}

internal/metadatastorage/attestationcollection/parserstorer.go → pkg/metadatastorage/attestationcollection/parserstorer.go

@@ -20,7 +20,7 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/in-toto/archivista/ent"
-	"github.com/in-toto/archivista/internal/metadatastorage"
+	"github.com/in-toto/archivista/pkg/metadatastorage"
 	"github.com/in-toto/go-witness/attestation"
 )
 

internal/metadatastorage/parserregistry/registry.go → pkg/metadatastorage/parserregistry/registry.go

@@ -15,8 +15,8 @@
 package parserregistry
 
 import (
-	"github.com/in-toto/archivista/internal/metadatastorage"
-	"github.com/in-toto/archivista/internal/metadatastorage/attestationcollection"
+	"github.com/in-toto/archivista/pkg/metadatastorage"
+	"github.com/in-toto/archivista/pkg/metadatastorage/attestationcollection"
 )
 
 var (

internal/metadatastorage/sqlstore/store.go → pkg/metadatastorage/sqlstore/store.go

@@ -25,8 +25,8 @@ import (
 
 	"github.com/digitorus/timestamp"
 	"github.com/in-toto/archivista/ent"
-	"github.com/in-toto/archivista/internal/metadatastorage"
-	"github.com/in-toto/archivista/internal/metadatastorage/parserregistry"
+	"github.com/in-toto/archivista/pkg/metadatastorage"
+	"github.com/in-toto/archivista/pkg/metadatastorage/parserregistry"
 	"github.com/in-toto/go-witness/cryptoutil"
 	"github.com/in-toto/go-witness/dsse"
 	"github.com/in-toto/go-witness/intoto"

internal/objectstorage/filestore/file_test.go → pkg/objectstorage/filestore/file_test.go

@@ -20,7 +20,7 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/in-toto/archivista/internal/objectstorage/filestore"
+	"github.com/in-toto/archivista/pkg/objectstorage/filestore"
 	"github.com/stretchr/testify/suite"
 )
 

internal/server/server.go → pkg/server/server.go

@@ -35,9 +35,9 @@ import (
 	"github.com/in-toto/archivista"
 	_ "github.com/in-toto/archivista/docs"
 	"github.com/in-toto/archivista/ent"
-	"github.com/in-toto/archivista/internal/artifactstore"
-	"github.com/in-toto/archivista/internal/config"
 	"github.com/in-toto/archivista/pkg/api"
+	"github.com/in-toto/archivista/pkg/artifactstore"
+	"github.com/in-toto/archivista/pkg/config"
 	"github.com/sirupsen/logrus"
 	httpSwagger "github.com/swaggo/http-swagger/v2"
 )

internal/server/server_test.go → pkg/server/server_test.go

@@ -27,9 +27,9 @@ import (
 	"testing"
 
 	"github.com/gorilla/mux"
-	"github.com/in-toto/archivista/internal/artifactstore"
-	"github.com/in-toto/archivista/internal/config"
 	"github.com/in-toto/archivista/pkg/api"
+	"github.com/in-toto/archivista/pkg/artifactstore"
+	"github.com/in-toto/archivista/pkg/config"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/suite"
 )