fix: make devnet work for lid (#1375)

* feat: support full addr config in boostd-data

* chore: fix linting for boostd-data

* feat: use addr instead of port for lid

chore: update devnet to work with lid setup

* chore: resolve feedback on lint changes

.gitignore

@@ -2,7 +2,6 @@
 /boostx
 /boostd
 /boostd-data
-/migrate-lid
 /devnet
 /booster-http
 /booster-bitswap

Makefile

@@ -110,6 +110,12 @@ boostd-data:
 .PHONY: boostd-data
 BINS+=boostd-data
 
+boostd-data:
+	$(MAKE) -C ./extern/boostd-data
+	install -C ./extern/boostd-data/boostd-data ./boostd-data
+.PHONY: boostd-data
+BINS+=boostd-data
+
 booster-http: $(BUILD_DEPS)
 	rm -f booster-http
 	$(GOCC) build $(GOFLAGS) -o booster-http ./cmd/booster-http
@@ -158,10 +164,7 @@ install-boost:
 	install -C ./boost /usr/local/bin/boost
 	install -C ./boostd /usr/local/bin/boostd
 	install -C ./boostx /usr/local/bin/boostx
-	install -C ./boostd-data /usr/local/bin/boostd-data
-	install -C ./booster-http /usr/local/bin/booster-http
-	install -C ./booster-bitswap /usr/local/bin/booster-bitswap
-	install -C ./migrate-lid /usr/local/bin/migrate-lid
+	install -C ./extern/boostd-data /usr/local/bin/boostd-data
 
 install-devnet:
 	install -C ./devnet /usr/local/bin/devnet

docker/devnet/Dockerfile.source

@@ -72,7 +72,7 @@ COPY --from=react-builder /src/react/build /go/src/react/build
 
 RUN --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=cache,target=/go/pkg/mod \
-      make debug
+      make debug boostd-data booster-http booster-bitswap
 #########################################################################################
 FROM ubuntu:20.04 as runner
 

docker/devnet/boost/entrypoint.sh

@@ -76,7 +76,7 @@ fi
 
 ## Override config options
 echo Updating config values
-sed -i 's|ServiceApiInfo = ""|ServiceApiInfo = "ws://localhost:8042"|g' $BOOST_PATH/config.toml
+sed -i 's|ServiceApiInfo = ""|ServiceApiInfo = "http://localhost:8042"|g' $BOOST_PATH/config.toml
 
 echo Starting LID service and boost in dev mode...
 trap 'kill %1' SIGINT

extern/boostd-data/cmd/run.go

@@ -3,13 +3,14 @@ package main
 import (
 	"context"
 	"fmt"
+	"net/http"
+
 	"github.com/filecoin-project/boostd-data/couchbase"
 	"github.com/filecoin-project/boostd-data/shared/cliutil"
 	"github.com/filecoin-project/boostd-data/shared/tracing"
 	"github.com/filecoin-project/boostd-data/svc"
 	"github.com/mitchellh/go-homedir"
 	"github.com/urfave/cli/v2"
-	"net/http"
 )
 
 var runCmd = &cli.Command{
@@ -21,10 +22,10 @@ var runCmd = &cli.Command{
 }
 
 var runFlags = []cli.Flag{
-	&cli.UintFlag{
-		Name:  "port",
-		Usage: "the port the boostd-data listens on",
-		Value: 8042,
+	&cli.StringFlag{
+		Name:  "addr",
+		Usage: "the address the boostd-data listens on",
+		Value: "localhost:8042",
 	},
 	&cli.BoolFlag{
 		Name:  "pprof",
@@ -130,14 +131,14 @@ func runAction(cctx *cli.Context, dbType string, store *svc.Service) error {
 	}
 
 	// Start the server
-	port := cctx.Int("port")
-	err = store.Start(ctx, port)
+	addr := cctx.String("addr")
+	err = store.Start(ctx, addr)
 	if err != nil {
 		return fmt.Errorf("starting %s store: %w", dbType, err)
 	}
 
-	log.Infof("Started boostd-data %s service on port %d",
-		dbType, port)
+	log.Infof("Started boostd-data %s service on address %s",
+		dbType, addr)
 
 	// Monitor for shutdown.
 	<-ctx.Done()

extern/boostd-data/couchbase/db.go

@@ -478,7 +478,7 @@ func (db *DB) GetPieceCidToMetadata(ctx context.Context, pieceCid cid.Cid) (Couc
 // GetOffsetSize gets the offset and size of the multihash in the given piece.
 // Note that recordCount is needed in order to determine which shard the multihash is in.
 func (db *DB) GetOffsetSize(ctx context.Context, pieceCid cid.Cid, hash multihash.Multihash, recordCount int) (*model.OffsetSize, error) {
-	ctx, span := tracing.Tracer.Start(ctx, "db.get_offset_size")
+	_, span := tracing.Tracer.Start(ctx, "db.get_offset_size")
 	defer span.End()
 
 	// Get the prefix for the shard that the multihash is in
@@ -1161,7 +1161,7 @@ func (db *DB) RemoveDealForPiece(ctx context.Context, dealId string, pieceCid ci
 		// Remove Metadata if removed deal was last one
 		if len(metadata.Deals) == 0 {
 			if err := db.RemovePieceMetadata(ctx, pieceCid); err != nil {
-				fmt.Errorf("Failed to remove the Metadata after removing the last deal: %w", err)
+				return fmt.Errorf("failed to remove the Metadata after removing the last deal: %w", err)
 			}
 			return nil
 		}

extern/boostd-data/ldb/db.go

@@ -307,7 +307,7 @@ func (db *DB) SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) err
 	return db.SetPieceCidToMetadata(ctx, pieceCid, md)
 }
 
-func (db *DB) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, err error) error {
+func (db *DB) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, sourceErr error) error {
 	ctx, span := tracing.Tracer.Start(ctx, "db.mark_piece_index_errored")
 	defer span.End()
 
@@ -321,8 +321,8 @@ func (db *DB) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, err error)
 		return nil
 	}
 
-	md.Error = err.Error()
-	md.ErrorType = fmt.Sprintf("%T", err)
+	md.Error = sourceErr.Error()
+	md.ErrorType = fmt.Sprintf("%T", sourceErr)
 
 	return db.SetPieceCidToMetadata(ctx, pieceCid, md)
 }
@@ -360,7 +360,7 @@ func (db *DB) AllRecords(ctx context.Context, cursor uint64) ([]model.Record, er
 		kcid := cid.NewCidV1(cid.Raw, m)
 
 		offset, n := binary.Uvarint(r.Value)
-		size, n := binary.Uvarint(r.Value[n:])
+		size, _ := binary.Uvarint(r.Value[n:])
 
 		records = append(records, model.Record{
 			Cid: kcid,
@@ -401,7 +401,7 @@ func (db *DB) GetOffsetSize(ctx context.Context, cursorPrefix string, m multihas
 	}
 
 	offset, n := binary.Uvarint(b)
-	size, n := binary.Uvarint(b[n:])
+	size, _ := binary.Uvarint(b[n:])
 	return &model.OffsetSize{
 		Offset: offset,
 		Size:   size,