diff --git a/extern/boostd-data/bench/cassandra.go b/extern/boostd-data/bench/cassandra.go new file mode 100644 index 000000000..71e7b0722 --- /dev/null +++ b/extern/boostd-data/bench/cassandra.go @@ -0,0 +1,230 @@ +package main + +import ( + "context" + _ "embed" + "fmt" + "github.com/filecoin-project/boostd-data/model" + "github.com/filecoin-project/boostd-data/shared/cliutil" + "github.com/gocql/gocql" + "github.com/ipfs/go-cid" + "github.com/ipld/go-car/v2/index" + "github.com/multiformats/go-multihash" + "github.com/urfave/cli/v2" + "strings" +) + +var cassandraCmd = &cli.Command{ + Name: "cassandra", + Before: before, + Flags: commonFlags, + Action: func(cctx *cli.Context) error { + ctx := cliutil.ReqContext(cctx) + db, err := NewCassandraDB() + if err != nil { + return err + } + return run(ctx, db, runOptsFromCctx(cctx)) + }, + Subcommands: []*cli.Command{ + loadCmd(createCassandra), + bitswapCmd(createCassandra), + graphsyncCmd(createCassandra), + }, +} + +func createCassandra(ctx context.Context, connectString string) (BenchDB, error) { + return NewCassandraDB() +} + +type CassandraDB struct { + session *gocql.Session +} + +func NewCassandraDB() (*CassandraDB, error) { + cluster := gocql.NewCluster("localhost") + cluster.Consistency = gocql.Quorum + session, err := cluster.CreateSession() + if err != nil { + return nil, fmt.Errorf("creating cluster: %w", err) + } + return &CassandraDB{session: session}, nil +} + +func (c *CassandraDB) Name() string { + return "Cassandra DB" +} + +//go:embed create_tables.cql +var createTablesCQL string + +func (c *CassandraDB) Init(ctx context.Context) error { + tables := []string{`PayloadToPieces`, `PieceBlockOffsetSize`} + for _, tbl := range tables { + qry := `drop table if exists bench.` + tbl + log.Debug(qry) + err := c.session.Query(qry).WithContext(ctx).Exec() + if err != nil { + log.Warn(err) + } + } + + createTablesLines := strings.Split(createTablesCQL, ";") + for _, line := range createTablesLines { + line = strings.Trim(line, "\n \t") + if line == "" { + continue + } + log.Debug(line) + err := c.session.Query(line).WithContext(ctx).Exec() + if err != nil { + return fmt.Errorf("creating tables: executing\n%s\n%w", line, err) + } + } + + return nil +} + +func (c *CassandraDB) Cleanup(ctx context.Context) error { + _ = c.session.Query(`drop table if exists bench.PayloadToPieces`).WithContext(ctx).Exec() + _ = c.session.Query(`drop table if exists bench.PieceBlockOffsetSize`).WithContext(ctx).Exec() + _ = c.session.Query(`drop keyspace bench`).WithContext(ctx).Exec() + c.session.Close() + return nil +} + +func (c *CassandraDB) GetBlockSample(ctx context.Context, count int) ([]pieceBlock, error) { + // TODO: randomize order + qry := `SELECT PieceCid, PayloadMultihash FROM bench.PieceBlockOffsetSize LIMIT ?` + iter := c.session.Query(qry, count).WithContext(ctx).Iter() + + var pieceCidBz, payloadMHBz []byte + pbs := make([]pieceBlock, 0, count) + for iter.Scan(&pieceCidBz, &payloadMHBz) { + _, pcid, err := cid.CidFromBytes(pieceCidBz) + if err != nil { + return nil, fmt.Errorf("scanning piece cid: %w", err) + } + _, pmh, err := multihash.MHFromBytes(payloadMHBz) + if err != nil { + return nil, fmt.Errorf("scanning mulithash: %w", err) + } + + pbs = append(pbs, pieceBlock{ + PieceCid: pcid, + PayloadMultihash: pmh, + }) + } + if err := iter.Close(); err != nil { + return nil, err + } + + //log.Debug("got pbs:") + //for _, pb := range pbs { + // log.Debugf(" %s %s", pb.PieceCid, pb.PayloadMultihash) + //} + + return pbs, nil +} + +func (c *CassandraDB) AddIndexRecords(ctx context.Context, pieceCid cid.Cid, recs []model.Record) error { + if len(recs) == 0 { + return nil + } + + batchEntries := make([]gocql.BatchEntry, 0, 2*len(recs)) + + // Add payload to pieces index + for _, rec := range recs { + batchEntries = append(batchEntries, gocql.BatchEntry{ + Stmt: `INSERT INTO bench.PayloadToPieces (PayloadMultihash, PieceCids) VALUES (?, ?)`, + Args: []interface{}{rec.Cid.Hash(), pieceCid.Bytes()}, + Idempotent: true, + }) + } + + // Add piece to block info index + for _, rec := range recs { + batchEntries = append(batchEntries, gocql.BatchEntry{ + Stmt: `INSERT INTO bench.PieceBlockOffsetSize (PieceCid, PayloadMultihash, BlockOffset, BlockSize) VALUES (?, ?, ?, ?)`, + Args: []interface{}{pieceCid.Bytes(), rec.Cid.Hash(), rec.Offset, rec.Size}, + Idempotent: true, + }) + } + + // Cassandra has a 50k limit on batch statements. Keeping batch size small + // makes sure we're under the limit. + const batchSize = 128 + var batchIdx int + var batch *gocql.Batch + for allIdx, entry := range batchEntries { + if batchIdx == 0 { + batch = c.session.NewBatch(gocql.UnloggedBatch).WithContext(ctx) + } + if allIdx == len(batchEntries)-1 || batchIdx == batchSize { + err := c.session.ExecuteBatch(batch) + if err != nil { + return fmt.Errorf("adding index records for piece %s: %w", pieceCid, err) + } + batchIdx = 0 + continue + } + + batch.Entries = append(batch.Entries, entry) + batchIdx++ + } + return nil +} + +func (c *CassandraDB) PiecesContainingMultihash(ctx context.Context, m multihash.Multihash) ([]cid.Cid, error) { + var bz []byte + qry := `SELECT PieceCids FROM bench.PayloadToPieces WHERE PayloadMultihash = ?` + err := c.session.Query(qry, m).WithContext(ctx).Scan(&bz) + if err != nil { + return nil, fmt.Errorf("getting pieces containing multihash: %w", err) + } + + return cidsFromBytes(bz) +} + +func (c *CassandraDB) GetOffsetSize(ctx context.Context, pieceCid cid.Cid, hash multihash.Multihash) (*model.OffsetSize, error) { + var offset, size uint64 + qry := `SELECT BlockOffset, BlockSize FROM bench.PieceBlockOffsetSize WHERE PieceCid = ? AND PayloadMultihash = ?` + err := c.session.Query(qry, pieceCid.Bytes(), hash).WithContext(ctx).Scan(&offset, &size) + if err != nil { + return nil, fmt.Errorf("getting offset / size: %w", err) + } + + return &model.OffsetSize{Offset: offset, Size: size}, nil +} + +func (c *CassandraDB) GetIterableIndex(ctx context.Context, pieceCid cid.Cid) (index.IterableIndex, error) { + qry := `SELECT PayloadMultihash, BlockOffset FROM bench.PieceBlockOffsetSize WHERE PieceCid = ?` + iter := c.session.Query(qry, pieceCid.Bytes()).WithContext(ctx).Iter() + + var records []index.Record + var payloadMHBz []byte + var offset uint64 + for iter.Scan(&payloadMHBz, &offset) { + _, pmh, err := multihash.MHFromBytes(payloadMHBz) + if err != nil { + return nil, fmt.Errorf("scanning mulithash: %w", err) + } + + records = append(records, index.Record{ + Cid: cid.NewCidV1(cid.Raw, pmh), + Offset: offset, + }) + } + if err := iter.Close(); err != nil { + return nil, err + } + + mis := make(index.MultihashIndexSorted) + err := mis.Load(records) + if err != nil { + return nil, err + } + + return &mis, nil +} diff --git a/extern/boostd-data/bench/cidbytes.go b/extern/boostd-data/bench/cidbytes.go new file mode 100644 index 000000000..6194ce511 --- /dev/null +++ b/extern/boostd-data/bench/cidbytes.go @@ -0,0 +1,44 @@ +package main + +import ( + "fmt" + "github.com/ipfs/go-cid" + "math/rand" + "sync" +) + +func cidsFromBytes(bz []byte) ([]cid.Cid, error) { + var bytesIdx int + var cids []cid.Cid + for bytesIdx < len(bz) { + readCount, pcid, err := cid.CidFromBytes(bz[bytesIdx:]) + if err != nil { + return nil, fmt.Errorf("parsing bytes to cid: %w", err) + } + + bytesIdx += readCount + cids = append(cids, pcid) + } + return cids, nil +} + +var rndlk sync.Mutex + +func generateRandomCid(baseCid []byte) (cid.Cid, error) { + buff := make([]byte, len(baseCid)) + copy(buff, baseCid) + + rndlk.Lock() + _, err := rand.Read(buff[len(buff)-8:]) + rndlk.Unlock() + if err != nil { + return cid.Undef, err + } + + _, c, err := cid.CidFromBytes(buff) + if err != nil { + return cid.Undef, fmt.Errorf("generating cid: %w", err) + } + + return c, nil +} diff --git a/extern/boostd-data/bench/common.go b/extern/boostd-data/bench/common.go new file mode 100644 index 000000000..62f6ae5bf --- /dev/null +++ b/extern/boostd-data/bench/common.go @@ -0,0 +1,46 @@ +package main + +import "github.com/urfave/cli/v2" + +var commonFlags = []cli.Flag{ + &cli.IntFlag{ + Name: "piece-add-parallelism", + Value: 2, + }, + &cli.IntFlag{ + Name: "piece-count", + Value: 30, + }, + &cli.IntFlag{ + Name: "blocks-per-piece", + Value: 1024, + }, + &cli.IntFlag{ + Name: "bs-fetch-count", + Value: 100, + }, + &cli.IntFlag{ + Name: "bs-fetch-parallelism", + Value: 10, + }, + &cli.IntFlag{ + Name: "gs-fetch-count", + Value: 10, + }, + &cli.IntFlag{ + Name: "gs-fetch-parallelism", + Value: 3, + }, +} + +func runOptsFromCctx(cctx *cli.Context) runOpts { + return runOpts{ + pieceParallelism: cctx.Int("piece-add-parallelism"), + blocksPerPiece: cctx.Int("blocks-per-piece"), + pieceCount: cctx.Int("piece-count"), + bitswapFetchCount: cctx.Int("bs-fetch-count"), + bitswapFetchParallelism: cctx.Int("bs-fetch-parallelism"), + graphsyncFetchCount: cctx.Int("gs-fetch-count"), + graphsyncFetchParallelism: cctx.Int("gs-fetch-parallelism"), + } +} diff --git a/extern/boostd-data/bench/create_tables.cql b/extern/boostd-data/bench/create_tables.cql new file mode 100644 index 000000000..982059865 --- /dev/null +++ b/extern/boostd-data/bench/create_tables.cql @@ -0,0 +1,14 @@ +create keyspace if not exists bench with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }; + +CREATE TABLE bench.PayloadToPieces ( + PayloadMultihash BLOB PRIMARY KEY, + PieceCids BLOB +); + +CREATE TABLE bench.PieceBlockOffsetSize ( + PieceCid BLOB, + PayloadMultihash BLOB, + BlockOffset BIGINT, + BlockSize BIGINT, + PRIMARY KEY (PieceCid, PayloadMultihash) +); diff --git a/extern/boostd-data/bench/create_tables.sql b/extern/boostd-data/bench/create_tables.sql new file mode 100644 index 000000000..de1c43c93 --- /dev/null +++ b/extern/boostd-data/bench/create_tables.sql @@ -0,0 +1,12 @@ +CREATE TABLE PayloadToPieces ( + PayloadMultihash bytea PRIMARY KEY, + PieceCids bytea +); + +CREATE TABLE PieceBlockOffsetSize ( + PieceCid bytea, + PayloadMultihash bytea, + BlockOffset BIGINT, + BlockSize BIGINT, + PRIMARY KEY (PieceCid, PayloadMultihash) +); diff --git a/extern/boostd-data/bench/foundationdb.go b/extern/boostd-data/bench/foundationdb.go new file mode 100644 index 000000000..0e05ac26f --- /dev/null +++ b/extern/boostd-data/bench/foundationdb.go @@ -0,0 +1,79 @@ +package main + +import ( + "context" + "github.com/filecoin-project/boostd-data/model" + "github.com/filecoin-project/boostd-data/shared/cliutil" + "github.com/ipfs/go-cid" + "github.com/ipld/go-car/v2/index" + "github.com/multiformats/go-multihash" + "github.com/urfave/cli/v2" +) + +var foundationCmd = &cli.Command{ + Name: "foundation", + Before: before, + Flags: commonFlags, + Action: func(cctx *cli.Context) error { + ctx := cliutil.ReqContext(cctx) + db := NewFoundationDB() + return run(ctx, db, runOptsFromCctx(cctx)) + }, +} + +type FoundationDB struct { + //db fdb.Database +} + +func NewFoundationDB() *FoundationDB { + //fdb.MustAPIVersion(720) + //db := fdb.MustOpenDefault() + //return &FoundationDB{db: db} + return nil +} + +func (db *FoundationDB) Name() string { + return "Foundation DB" +} + +func (db *FoundationDB) Init(ctx context.Context) error { + //ret, err := db.db.Transact(func(tr fdb.Transaction) (interface{}, error) { + // tr.Set(fdb.Key("hello"), []byte("world")) + // return tr.Get(fdb.Key("foo")).MustGet(), nil + //}) + //if err != nil { + // return err + //} + // + //fmt.Printf("hello is now world, foo was: %s\n", string(ret.([]byte))) + // + return nil +} + +func (db *FoundationDB) Cleanup(ctx context.Context) error { + return nil +} + +func (db *FoundationDB) GetBlockSample(ctx context.Context, count int) ([]pieceBlock, error) { + //TODO implement me + panic("implement me") +} + +func (db *FoundationDB) AddIndexRecords(ctx context.Context, pieceCid cid.Cid, recs []model.Record) error { + return nil +} + +func (db *FoundationDB) PiecesContainingMultihash(ctx context.Context, m multihash.Multihash) ([]cid.Cid, error) { + //TODO implement me + panic("implement me") +} + +func (db *FoundationDB) GetOffsetSize(ctx context.Context, pieceCid cid.Cid, hash multihash.Multihash) (*model.OffsetSize, error) { + //TODO implement me + panic("implement me") +} + +func (db *FoundationDB) GetIterableIndex(ctx context.Context, pieceCid cid.Cid) (index.IterableIndex, error) { + //TODO implement me + panic("implement me") +} diff --git a/extern/boostd-data/bench/main.go b/extern/boostd-data/bench/main.go new file mode 100644 index 000000000..f224529b2 --- /dev/null +++ b/extern/boostd-data/bench/main.go @@ -0,0 +1,42 @@ +package main + +import ( + "github.com/filecoin-project/boostd-data/shared/cliutil" + logging "github.com/ipfs/go-log/v2" + "github.com/urfave/cli/v2" + "os" +) + +var log = logging.Logger("bench") + +func main() { + app := &cli.App{ + Name: "bench", + Usage: "Benchmark LID databases", + EnableBashCompletion: true, + Flags: []cli.Flag{ + cliutil.FlagVeryVerbose, + }, + Commands: []*cli.Command{ + cassandraCmd, + foundationCmd, + postgresCmd, + }, + } + app.Setup() + + if err := app.Run(os.Args); err != nil { + log.Errorf("Error: %s", err.Error()) + os.Exit(1) + } +} + +func before(cctx *cli.Context) error { + _ = logging.SetLogLevel("bench", "info") + + if cliutil.IsVeryVerbose { + _ = logging.SetLogLevel("bench", "debug") + } + + return nil +} diff --git a/extern/boostd-data/bench/postgres.go b/extern/boostd-data/bench/postgres.go new file mode 100644 index 000000000..dc33b642f --- /dev/null +++ b/extern/boostd-data/bench/postgres.go @@ -0,0 +1,285 @@ +package main + +import ( + "context" + "database/sql" + _ "embed" + "fmt" + "github.com/filecoin-project/boostd-data/model" + "github.com/filecoin-project/boostd-data/shared/cliutil" + "github.com/ipfs/go-cid" + "github.com/ipld/go-car/v2/index" + _ "github.com/lib/pq" + "github.com/multiformats/go-multihash" + "github.com/urfave/cli/v2" + "strings" +) + +var postgresCmd = &cli.Command{ + Name: "postgres", + Before: before, + Flags: append(commonFlags, &cli.StringFlag{ + Name: "connect-string", + Value: "postgresql://postgres:postgres@localhost?sslmode=disable", + }), + Action: func(cctx *cli.Context) error { + ctx := cliutil.ReqContext(cctx) + db, err := NewPostgresDB(cctx.String("connect-string")) + if err != nil { + return err + } + return run(ctx, db, runOptsFromCctx(cctx)) + }, + Subcommands: []*cli.Command{ + loadCmd(createPostgres), + bitswapCmd(createPostgres), + graphsyncCmd(createPostgres), + }, +} + +func createPostgres(ctx context.Context, connectString string) (BenchDB, error) { + db, err := NewPostgresDB(connectString) + if err != nil { + return nil, err + } + err = db.connect(ctx) + if err != nil { + return nil, err + } + return db, err +} + +type Postgres struct { + defDb *sql.DB + db *sql.DB + connectString string +} + +func NewPostgresDB(connectString string) (*Postgres, error) { + defDb, err := sql.Open("postgres", connectString) + if err != nil { + return nil, fmt.Errorf("connecting to default database: %w", err) + } + + return &Postgres{defDb: defDb, connectString: connectString}, nil +} + +func (db *Postgres) Name() string { + return "Postgres DB" +} + +//go:embed create_tables.sql +var createTables string + +func (db *Postgres) Init(ctx context.Context) error { + // Drop the db in case it didn't get cleaned up last time the program ran + _, _ = db.defDb.ExecContext(ctx, `DROP database bench`) + + _, err := db.defDb.ExecContext(ctx, `CREATE DATABASE bench`) + if err != nil { + return fmt.Errorf("creating database bench: %w", err) + } + + err = db.connect(ctx) + if err != nil { + return fmt.Errorf("connecting to db: %w", err) + } + + _, err = db.db.ExecContext(ctx, createTables) + if err != nil { + return fmt.Errorf("creating tables: %w", err) + } + + return nil +} + +func (db *Postgres) connect(ctx context.Context) error { + benchConnStr, err := getConnStringWithDb(db.connectString, "bench") + if err != nil { + return err + } + benchDb, err := sql.Open("postgres", benchConnStr) + if err != nil { + return fmt.Errorf("connecting to database Bench: %w", err) + } + db.db = benchDb + + return nil +} + +func (db *Postgres) Cleanup(ctx context.Context) error { + err := db.db.Close() + if err != nil { + return err + } + + _, err = db.defDb.ExecContext(ctx, `DROP database bench`) + return err +} + +func (db *Postgres) GetBlockSample(ctx context.Context, count int) ([]pieceBlock, error) { + qry := `SELECT PieceCid, PayloadMultihash FROM PieceBlockOffsetSize ORDER BY RANDOM() LIMIT $1` + rows, err := db.db.QueryContext(ctx, qry, count) + if err != nil { + return nil, err + } + defer rows.Close() + + pbs := make([]pieceBlock, 0, count) + for rows.Next() { + var pieceCidBz, payloadMHBz []byte + err := rows.Scan(&pieceCidBz, &payloadMHBz) + if err != nil { + return nil, fmt.Errorf("scanning row: %w", err) + } + + _, pcid, err := cid.CidFromBytes(pieceCidBz) + if err != nil { + return nil, fmt.Errorf("scanning piece cid: %w", err) + } + _, pmh, err := multihash.MHFromBytes(payloadMHBz) + if err != nil { + return nil, fmt.Errorf("scanning mulithash: %w", err) + } + + pbs = append(pbs, pieceBlock{ + PieceCid: pcid, + PayloadMultihash: pmh, + }) + } + if err := rows.Err(); err != nil { + return nil, err + } + + return pbs, nil +} + +func (db *Postgres) AddIndexRecords(ctx context.Context, pieceCid cid.Cid, recs []model.Record) error { + if len(recs) == 0 { + return nil + } + + tx, err := db.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Commit() + + // Add payload to pieces index + vals := "" + args := make([]interface{}, 0, len(recs)*2) + for i, rec := range recs { + if i > 0 { + vals = vals + "," + } + vals = vals + fmt.Sprintf("($%d,$%d)", (i*2)+1, (i*2)+2) + args = append(args, rec.Cid.Hash(), pieceCid.Bytes()) + } + _, err = tx.ExecContext(ctx, `INSERT INTO PayloadToPieces (PayloadMultihash, PieceCids) VALUES `+vals, args...) + if err != nil { + return fmt.Errorf("executing insert: %w", err) + } + + // Add piece to block info index + vals = "" + args = make([]interface{}, 0, len(recs)*4) + for i, rec := range recs { + if i > 0 { + vals = vals + "," + } + vals = vals + fmt.Sprintf("($%d,$%d,$%d,$%d)", (i*4)+1, (i*4)+2, (i*4)+3, (i*4)+4) + args = append(args, pieceCid.Bytes(), rec.Cid.Hash(), rec.Offset, rec.Size) + } + _, err = tx.ExecContext(ctx, `INSERT INTO PieceBlockOffsetSize (PieceCid, PayloadMultihash, BlockOffset, BlockSize) VALUES `+vals, args...) + if err != nil { + return fmt.Errorf("executing insert: %w", err) + } + + return nil +} + +func (db *Postgres) PiecesContainingMultihash(ctx context.Context, m multihash.Multihash) ([]cid.Cid, error) { + var bz []byte + qry := `SELECT PieceCids FROM PayloadToPieces WHERE PayloadMultihash = $1` + err := db.db.QueryRowContext(ctx, qry, m).Scan(&bz) + if err != nil { + return nil, err + } + + return cidsFromBytes(bz) +} + +func (db *Postgres) GetOffsetSize(ctx context.Context, pieceCid cid.Cid, hash multihash.Multihash) (*model.OffsetSize, error) { + var offset, size uint64 + qry := `SELECT BlockOffset, BlockSize FROM PieceBlockOffsetSize WHERE PieceCid = $1 AND PayloadMultihash = $2` + err := db.db.QueryRowContext(ctx, qry, pieceCid.Bytes(), hash).Scan(&offset, &size) + if err != nil { + return nil, err + } + + return &model.OffsetSize{Offset: offset, Size: size}, nil +} + +func (db *Postgres) GetIterableIndex(ctx context.Context, pieceCid cid.Cid) (index.IterableIndex, error) { + qry := `SELECT PayloadMultihash, BlockOffset FROM PieceBlockOffsetSize WHERE PieceCid = $1` + rows, err := db.db.QueryContext(ctx, qry, pieceCid.Bytes()) + if err != nil { + return nil, err + } + defer rows.Close() + + var records []index.Record + for rows.Next() { + var payloadMHBz []byte + var offset uint64 + err := rows.Scan(&payloadMHBz, &offset) + if err != nil { + return nil, fmt.Errorf("scanning row: %w", err) + } + + _, pmh, err := multihash.MHFromBytes(payloadMHBz) + if err != nil { + return nil, fmt.Errorf("scanning mulithash: %w", err) + } + + records = append(records, index.Record{ + Cid: cid.NewCidV1(cid.Raw, pmh), + Offset: offset, + }) + } + if err := rows.Err(); err != nil { + return nil, err + } + + mis := make(index.MultihashIndexSorted) + err = mis.Load(records) + if err != nil { + return nil, err + } + + return &mis, nil +} + +func getConnStringWithDb(connString string, dbName string) (string, error) { + // "postgresql://postgres:postgres@localhost/dbname?sslmode=disable" + prefixEnd := strings.Index(connString, "://") + restStart := prefixEnd + 3 + if prefixEnd == -1 || len(connString) <= restStart { + return "", fmt.Errorf("connect string %s is missing protocol prefix", connString) + } + + rest := connString[restStart:] + slashIdx := strings.Index(rest, "/") + questionIdx := strings.Index(rest, "?") + if questionIdx != -1 { + query := rest[questionIdx:] + if slashIdx != -1 { + return connString[:restStart] + rest[:slashIdx] + "/" + dbName + query, nil + } + return connString[:restStart] + rest[:questionIdx] + "/" + dbName + query, nil + } else if slashIdx != -1 { + return connString[:restStart] + rest[:slashIdx] + "/" + dbName, nil + } + + return connString + "/" + dbName, nil +} diff --git a/extern/boostd-data/bench/postgres_test.go b/extern/boostd-data/bench/postgres_test.go new file mode 100644 index 000000000..a04d697c9 --- /dev/null +++ b/extern/boostd-data/bench/postgres_test.go @@ -0,0 +1,54 @@ +package main + +import "testing" + +func TestGetConnStringWithDb(t *testing.T) { + type args struct { + connString string + dbName string + } + tests := []struct { + name string + args args + expect string + }{{ + name: "connect string with no db or args", + args: args{ + connString: "postgresql://postgres:postgres@localhost", + dbName: "bench", + }, + expect: "postgresql://postgres:postgres@localhost/bench", + }, { + name: "connect string with args but no db", + args: args{ + connString: "postgresql://postgres:postgres@localhost?sslmode=disable", + dbName: "bench", + }, + expect: "postgresql://postgres:postgres@localhost/bench?sslmode=disable", + }, { + name: "connect string with db but no args", + args: args{ + connString: "postgresql://postgres:postgres@localhost/somedb", + dbName: "bench", + }, + expect: "postgresql://postgres:postgres@localhost/bench", + }, { + name: "connect string with db and args", + args: args{ + connString: "postgresql://postgres:postgres@localhost/somedb?sslmode=disable", + dbName: "bench", + }, + expect: "postgresql://postgres:postgres@localhost/bench?sslmode=disable", + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getConnStringWithDb(tt.args.connString, tt.args.dbName) + if err != nil { + t.Error(err) + } + if got != tt.expect { + t.Errorf("getConnStringWithDb(%v, %v)\n returns %v\n expected %v", tt.args.connString, tt.args.dbName, got, tt.expect) + } + }) + } +} diff --git a/extern/boostd-data/bench/run.go b/extern/boostd-data/bench/run.go new file mode 100644 index 000000000..f60e864ff --- /dev/null +++ b/extern/boostd-data/bench/run.go @@ -0,0 +1,291 @@ +package main + +import ( + "context" + "github.com/filecoin-project/boost/testutil" + "github.com/filecoin-project/boostd-data/model" + "github.com/filecoin-project/boostd-data/shared/cliutil" + "github.com/ipfs/go-cid" + carindex "github.com/ipld/go-car/v2/index" + mh "github.com/multiformats/go-multihash" + "github.com/urfave/cli/v2" + "golang.org/x/sync/errgroup" + "math/rand" + "time" +) + +const sectorSize = 32 * 1024 * 1024 * 1024 + +type pieceBlock struct { + PieceCid cid.Cid + PayloadMultihash mh.Multihash +} + +type BenchDB interface { + Name() string + Init(ctx context.Context) error + AddIndexRecords(ctx context.Context, pieceCid cid.Cid, recs []model.Record) error + Cleanup(ctx context.Context) error + GetBlockSample(ctx context.Context, count int) ([]pieceBlock, error) + PiecesContainingMultihash(ctx context.Context, m mh.Multihash) ([]cid.Cid, error) + GetOffsetSize(ctx context.Context, pieceCid cid.Cid, hash mh.Multihash) (*model.OffsetSize, error) + GetIterableIndex(ctx context.Context, pieceCid cid.Cid) (carindex.IterableIndex, error) +} + +type runOpts struct { + pieceParallelism int + blocksPerPiece int + pieceCount int + bitswapFetchCount int + bitswapFetchParallelism int + graphsyncFetchCount int + graphsyncFetchParallelism int +} + +func run(ctx context.Context, db BenchDB, opts runOpts) error { + log.Infof("Running benchmark for %s", db.Name()) + log.Infof("Initializing...") + if err := db.Init(ctx); err != nil { + return err + } + log.Infof("Initialized") + + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := db.Cleanup(ctx); err != nil { + log.Errorf("cleaning up database: %w", err) + } + }() + + // Add sample data to the database + if err := addPieces(ctx, db, opts.pieceParallelism, opts.pieceCount, opts.blocksPerPiece); err != nil { + return err + } + + // Run bitswap fetch simulation + if err := bitswapFetch(ctx, db, opts.bitswapFetchCount, opts.bitswapFetchParallelism); err != nil { + return err + } + + // Run graphsync fetch simulation + if err := graphsyncFetch(ctx, db, opts.graphsyncFetchCount, opts.graphsyncFetchParallelism); err != nil { + return err + } + + return nil +} + +func loadCmd(createDB func(context.Context, string) (BenchDB, error)) *cli.Command { + return &cli.Command{ + Name: "load", + Before: before, + Action: func(cctx *cli.Context) error { + ctx := cliutil.ReqContext(cctx) + db, err := createDB(ctx, cctx.String("connect-string")) + if err != nil { + return err + } + + opts := runOptsFromCctx(cctx) + return addPieces(ctx, db, opts.pieceParallelism, opts.pieceCount, opts.blocksPerPiece) + }, + } +} + +func addPieces(ctx context.Context, db BenchDB, parallelism int, pieceCount int, blocksPerPiece int) error { + log.Infof("Adding %d pieces with %d blocks per piece...", pieceCount, blocksPerPiece) + + queue := make(chan struct{}, pieceCount) + for i := 0; i < pieceCount; i++ { + queue <- struct{}{} + } + close(queue) + + var totalCreateRecs, totalAddRecs time.Duration + addStart := time.Now() + var eg errgroup.Group + baseCid := testutil.GenerateCid().Bytes() + for i := 0; i < parallelism; i++ { + eg.Go(func() error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + case _, ok := <-queue: + if !ok { + return nil + } + + // Create block records + createRecsStart := time.Now() + + recs := make([]model.Record, 0, blocksPerPiece) + for p := 0; p < blocksPerPiece; p++ { + c, err := generateRandomCid(baseCid) + if err != nil { + return err + } + + recs = append(recs, model.Record{ + Cid: c, + OffsetSize: model.OffsetSize{ + Offset: uint64(rand.Intn(sectorSize)), + Size: uint64(rand.Intn(sectorSize)), + }, + }) + } + totalCreateRecs += time.Since(createRecsStart) + + // Add the records to the db + addRecsStart := time.Now() + pcid := testutil.GenerateCid() + err := db.AddIndexRecords(ctx, pcid, recs) + if err != nil { + return err + } + totalAddRecs += time.Since(addRecsStart) + } + } + }) + } + + err := eg.Wait() + if err != nil { + return err + } + + log.Infof("Added %d pieces in %s", pieceCount, time.Since(addStart)) + log.Debugf("total CPU time %s", totalCreateRecs+totalAddRecs) + log.Debugf(" created records in %s", totalCreateRecs) + log.Debugf(" added records in %s", totalAddRecs) + return nil +} + +func bitswapCmd(createDB func(context.Context, string) (BenchDB, error)) *cli.Command { + return &cli.Command{ + Name: "bitswap", + Before: before, + Action: func(cctx *cli.Context) error { + ctx := cliutil.ReqContext(cctx) + db, err := createDB(ctx, cctx.String("connect-string")) + if err != nil { + return err + } + + opts := runOptsFromCctx(cctx) + return bitswapFetch(ctx, db, opts.bitswapFetchCount, opts.bitswapFetchParallelism) + }, + } +} + +func bitswapFetch(ctx context.Context, db BenchDB, count int, parallelism int) error { + log.Infof("Bitswap simulation: fetching %d random blocks with parallelism %d...", count, parallelism) + + fetchStart := time.Now() + var mhLookupTotal, getOffsetSizeTotal time.Duration + err := executeFetch(ctx, db, count, parallelism, func(sample pieceBlock) error { + mhLookupStart := time.Now() + _, err := db.PiecesContainingMultihash(ctx, sample.PayloadMultihash) + if err != nil { + return err + } + mhLookupTotal += time.Since(mhLookupStart) + + getIdxStart := time.Now() + _, err = db.GetOffsetSize(ctx, sample.PieceCid, sample.PayloadMultihash) + getOffsetSizeTotal += time.Since(getIdxStart) + return err + }) + if err != nil { + return err + } + + log.Infof("Bitswap simulation completed in %s", time.Since(fetchStart)) + log.Debugf("total CPU time: %s", mhLookupTotal+getOffsetSizeTotal) + log.Debugf(" multihash lookup total time: %s", mhLookupTotal) + log.Debugf(" get offset / size total time: %s", getOffsetSizeTotal) + return nil +} + +func graphsyncCmd(createDB func(context.Context, string) (BenchDB, error)) *cli.Command { + return &cli.Command{ + Name: "graphsync", + Before: before, + Action: func(cctx *cli.Context) error { + ctx := cliutil.ReqContext(cctx) + db, err := createDB(ctx, cctx.String("connect-string")) + if err != nil { + return err + } + + opts := runOptsFromCctx(cctx) + return graphsyncFetch(ctx, db, opts.graphsyncFetchCount, opts.graphsyncFetchParallelism) + }, + } +} + +func graphsyncFetch(ctx context.Context, db BenchDB, count int, parallelism int) error { + log.Infof("Graphsync simulation: fetching %d random blocks with parallelism %d...", count, parallelism) + + fetchStart := time.Now() + var mhLookupTotal, getIdxTotal time.Duration + err := executeFetch(ctx, db, count, parallelism, func(sample pieceBlock) error { + mhLookupStart := time.Now() + _, err := db.PiecesContainingMultihash(ctx, sample.PayloadMultihash) + if err != nil { + return err + } + mhLookupTotal += time.Since(mhLookupStart) + + getIdxStart := time.Now() + _, err = db.GetIterableIndex(ctx, sample.PieceCid) + getIdxTotal += time.Since(getIdxStart) + return err + }) + if err != nil { + return err + } + + log.Infof("Graphsync simulation completed in %s", time.Since(fetchStart)) + log.Debugf("total CPU time: %s", mhLookupTotal+getIdxTotal) + log.Debugf(" multihash lookup total time: %s", mhLookupTotal) + log.Debugf(" get index total time: %s", getIdxTotal) + return nil +} + +func executeFetch(ctx context.Context, db BenchDB, count int, parallelism int, processSample func(pieceBlock) error) error { + samples, err := db.GetBlockSample(ctx, count) + if err != nil { + return err + } + + sampleChan := make(chan pieceBlock, len(samples)) + for _, sample := range samples { + sampleChan <- sample + } + close(sampleChan) + + var eg errgroup.Group + for i := 0; i < parallelism; i++ { + eg.Go(func() error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + case sample, ok := <-sampleChan: + if !ok { + return nil + } + + err = processSample(sample) + if err != nil { + return err + } + } + } + }) + } + + return eg.Wait() +} diff --git a/extern/boostd-data/go.mod b/extern/boostd-data/go.mod index 9d1c890d0..583777c1e 100644 --- a/extern/boostd-data/go.mod +++ b/extern/boostd-data/go.mod @@ -7,17 +7,20 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/docker/docker v20.10.7+incompatible github.com/docker/go-connections v0.4.0 + github.com/dustin/go-humanize v1.0.0 github.com/filecoin-project/boost v1.4.0 github.com/filecoin-project/go-address v1.0.0 github.com/filecoin-project/go-jsonrpc v0.1.8 github.com/filecoin-project/go-state-types v0.1.10 + github.com/gocql/gocql v1.3.1 github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 github.com/ipfs/go-cid v0.3.2 github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ds-leveldb v0.5.0 github.com/ipfs/go-log/v2 v2.5.1 - github.com/ipld/go-car/v2 v2.4.2-0.20220707083113-89de8134e58e + github.com/ipld/go-car/v2 v2.5.1 + github.com/lib/pq v1.10.4 github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-multicodec v0.6.0 github.com/multiformats/go-multihash v0.2.1 @@ -51,6 +54,7 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/gorilla/websocket v1.5.0 // indirect + github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-block-format v0.0.3 // indirect @@ -79,9 +83,7 @@ require ( github.com/jbenet/goprocess v0.1.4 // indirect github.com/klauspost/cpuid/v2 v2.1.1 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect - github.com/libp2p/go-libp2p v0.23.2 // indirect github.com/libp2p/go-libp2p-core v0.16.1 // indirect - github.com/libp2p/go-libp2p-testing v0.12.0 // indirect github.com/libp2p/go-openssl v0.1.0 // indirect github.com/mattn/go-isatty v0.0.16 // indirect github.com/mattn/go-pointer v0.0.1 // indirect @@ -121,6 +123,7 @@ require ( google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.4.0 // indirect lukechampine.com/blake3 v1.1.7 // indirect diff --git a/extern/boostd-data/go.sum b/extern/boostd-data/go.sum index cf22ef94e..275104884 100644 --- a/extern/boostd-data/go.sum +++ b/extern/boostd-data/go.sum @@ -128,9 +128,13 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bep/debounce v1.2.0/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/briandowns/spinner v1.11.1/go.mod h1:QOuQk7x+EaDASo80FEXwlwiA+j/PPIcX3FScO+3/ZPQ= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= @@ -289,6 +293,7 @@ github.com/drand/kyber v1.1.7/go.mod h1:UkHLsI4W6+jT5PvNxmc0cvQAgppjTUpX+XCsN9TX github.com/drand/kyber-bls12381 v0.2.0/go.mod h1:zQip/bHdeEB6HFZSU3v+d3cQE0GaBVQw9aR2E7AdoeI= github.com/drand/kyber-bls12381 v0.2.1/go.mod h1:JwWn4nHO9Mp4F5qCie5sVIPQZ0X6cw8XAeMRvc/GXBE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= @@ -424,8 +429,8 @@ github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3 github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= -github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/gabriel-vasile/mimetype v1.4.0/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmxRtOJlERCzSmRvr8= github.com/gammazero/keymutex v0.0.2/go.mod h1:qtzWCCLMisQUmVa4dvqHVgwfh4BP2YB7JxNDGXnsKrs= github.com/gammazero/radixtree v0.2.5/go.mod h1:VPqqCDZ3YZZxAzUUsIF/ytFBigVWV7JIV1Stld8hri0= @@ -477,6 +482,8 @@ github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw= +github.com/gocql/gocql v1.3.1 h1:BTwM4rux+ah5G3oH6/MQa+tur/TDd/XAAOXDxBBs7rg= +github.com/gocql/gocql v1.3.1/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= github.com/godbus/dbus v0.0.0-20190402143921-271e53dc4968/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -617,6 +624,8 @@ github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go github.com/gxed/go-shellwords v1.0.3/go.mod h1:N7paucT91ByIjmVJHhvoarjoQnmsi3Jd3vH7VqgtMxQ= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026/go.mod h1:5Scbynm8dF1XAPwIwkGPqzkM/shndPm79Jd1003hTjE= github.com/hannahhoward/cbor-gen-for v0.0.0-20200817222906-ea96cece81f1/go.mod h1:jvfsLIxk0fY/2BKSQ1xf2406AKA5dwMmKKv0ADcOfN8= github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e/go.mod h1:I8h3MITA53gN9OnWGCgaMa0JWVRdXthWw4M3CPM54OY= @@ -911,8 +920,9 @@ github.com/ipld/go-car v0.4.0/go.mod h1:Uslcn4O9cBKK9wqHm/cLTFacg6RAPv6LZx2mxd2Y github.com/ipld/go-car v0.4.1-0.20220707083113-89de8134e58e/go.mod h1:Uslcn4O9cBKK9wqHm/cLTFacg6RAPv6LZx2mxd2Ypl4= github.com/ipld/go-car/v2 v2.1.1/go.mod h1:+2Yvf0Z3wzkv7NeI69i8tuZ+ft7jyjPYIWZzeVNeFcI= github.com/ipld/go-car/v2 v2.4.1/go.mod h1:zjpRf0Jew9gHqSvjsKVyoq9OY9SWoEKdYCQUKVaaPT0= -github.com/ipld/go-car/v2 v2.4.2-0.20220707083113-89de8134e58e h1:A4ttip3C2PLdE29/owgZAUgSX/qtIU6vphQU9CEPlN4= github.com/ipld/go-car/v2 v2.4.2-0.20220707083113-89de8134e58e/go.mod h1:sDHqspWMwG6cC0lrid3Lq2xtIR4R6iy6ymCNT0drhaI= +github.com/ipld/go-car/v2 v2.5.1 h1:U2ux9JS23upEgrJScW8VQuxmE94560kYxj9CQUpcfmk= +github.com/ipld/go-car/v2 v2.5.1/go.mod h1:jKjGOqoCj5zn6KjnabD6JbnCsMntqU2hLiU6baZVO3E= github.com/ipld/go-codec-dagpb v1.2.0/go.mod h1:6nBN7X7h8EOsEejZGqC7tej5drsdBAXbMHyBT+Fne5s= github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= github.com/ipld/go-codec-dagpb v1.3.1/go.mod h1:ErNNglIi5KMur/MfFE/svtgQthzVvf+43MrzLbpcIZY= @@ -1013,8 +1023,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= +github.com/koron/go-ssdp v0.0.2 h1:fL3wAoyT6hXHQlORyXUW4Q23kkQpJRgEAYcZB5BR71o= github.com/koron/go-ssdp v0.0.2/go.mod h1:XoLfkAiA2KeZsYh4DbHxD7h3nR2AZNqVQOa+LJuqPYs= -github.com/koron/go-ssdp v0.0.3 h1:JivLMY45N76b4p/vsWGOKewBQu6uf39y8l+AQ7sDKx8= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -1028,6 +1038,7 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E= @@ -1047,6 +1058,7 @@ github.com/libp2p/go-conn-security-multistream v0.3.0/go.mod h1:EEP47t4fw/bTelVm github.com/libp2p/go-doh-resolver v0.3.1/go.mod h1:y5go1ZppAq9N2eppbX0xON01CyPBeUg2yS6BTssssog= github.com/libp2p/go-eventbus v0.0.2/go.mod h1:Hr/yGlwxA/stuLnpMiu82lpNKpvRy3EaJxPu40XYOwk= github.com/libp2p/go-eventbus v0.1.0/go.mod h1:vROgu5cs5T7cv7POWlWxBaVLxfSegC5UGQf8A2eEmx4= +github.com/libp2p/go-eventbus v0.2.1 h1:VanAdErQnpTioN2TowqNcOijf6YwhuODe4pPKSDpxGc= github.com/libp2p/go-eventbus v0.2.1/go.mod h1:jc2S4SoEVPP48H9Wpzm5aiGwUCBMfGhVhhBjyhhCJs8= github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8= github.com/libp2p/go-flow-metrics v0.0.2/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= @@ -1071,9 +1083,8 @@ github.com/libp2p/go-libp2p v0.18.0/go.mod h1:+veaZ9z1SZQhmc5PW78jvnnxZ89Mgvmh4c github.com/libp2p/go-libp2p v0.19.4/go.mod h1:MIt8y481VDhUe4ErWi1a4bvt/CjjFfOq6kZTothWIXY= github.com/libp2p/go-libp2p v0.20.0/go.mod h1:g0C5Fu+aXXbCXkusCzLycuBowEih3ElmDqtbo61Em7k= github.com/libp2p/go-libp2p v0.20.1/go.mod h1:XgJHsOhEBVBXp/2Sj9bm/yEyD94uunAaP6oaegdcKks= +github.com/libp2p/go-libp2p v0.20.3 h1:tjjDNfp7FqdI/7v1rXtB/BtELaPlAThL2uzlj18kcrw= github.com/libp2p/go-libp2p v0.20.3/go.mod h1:I+vndVanE/p/SjFbnA+BEmmfAUEpWxrdXZeyQ1Dus5c= -github.com/libp2p/go-libp2p v0.23.2 h1:yqyTeKQJyofWXxEv/eEVUvOrGdt/9x+0PIQ4N1kaxmE= -github.com/libp2p/go-libp2p v0.23.2/go.mod h1:s9DEa5NLR4g+LZS+md5uGU4emjMWFiqkZr6hBTY8UxI= github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052/go.mod h1:nRMRTab+kZuk0LnKZpxhOVH/ndsdr2Nr//Zltc/vwgo= github.com/libp2p/go-libp2p-asn-util v0.1.0/go.mod h1:wu+AnM9Ii2KgO5jMmS1rz9dvzTdj8BXqsPR9HR0XB7I= github.com/libp2p/go-libp2p-asn-util v0.2.0 h1:rg3+Os8jbnO5DxkC7K/Utdi+DkY3q/d1/1q+8WeNAsw= @@ -1216,6 +1227,7 @@ github.com/libp2p/go-libp2p-peerstore v0.2.7/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuD github.com/libp2p/go-libp2p-peerstore v0.2.8/go.mod h1:gGiPlXdz7mIHd2vfAsHzBNAMqSDkt2UBFwgcITgw1lA= github.com/libp2p/go-libp2p-peerstore v0.4.0/go.mod h1:rDJUFyzEWPpXpEwywkcTYYzDHlwza8riYMaUzaN6hX0= github.com/libp2p/go-libp2p-peerstore v0.6.0/go.mod h1:DGEmKdXrcYpK9Jha3sS7MhqYdInxJy84bIPtSu65bKc= +github.com/libp2p/go-libp2p-peerstore v0.7.0 h1:2iIUwok3vtmnWJTZeTeLgnBO6GbkXcwSRwgZHEKrQZs= github.com/libp2p/go-libp2p-peerstore v0.7.0/go.mod h1:cdUWTHro83vpg6unCpGUr8qJoX3e93Vy8o97u5ppIM0= github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYcO0BW4wssv21LA= github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s= @@ -1282,9 +1294,8 @@ github.com/libp2p/go-libp2p-testing v0.6.0/go.mod h1:QBk8fqIL1XNcno/l3/hhaIEn4aL github.com/libp2p/go-libp2p-testing v0.7.0/go.mod h1:OLbdn9DbgdMwv00v+tlp1l3oe2Cl+FAjoWIA2pa0X6E= github.com/libp2p/go-libp2p-testing v0.8.0/go.mod h1:gRdsNxQSxAZowTgcLY7CC33xPmleZzoBpqSYbWenqPc= github.com/libp2p/go-libp2p-testing v0.9.0/go.mod h1:Td7kbdkWqYTJYQGTwzlgXwaqldraIanyjuRiAbK/XQU= +github.com/libp2p/go-libp2p-testing v0.9.2 h1:dCpODRtRaDZKF8HXT9qqqgON+OMEB423Knrgeod8j84= github.com/libp2p/go-libp2p-testing v0.9.2/go.mod h1:Td7kbdkWqYTJYQGTwzlgXwaqldraIanyjuRiAbK/XQU= -github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= -github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg= github.com/libp2p/go-libp2p-tls v0.1.3/go.mod h1:wZfuewxOndz5RTnCAxFliGjvYSDA40sKitV4c50uI1M= github.com/libp2p/go-libp2p-tls v0.3.0/go.mod h1:fwF5X6PWGxm6IDRwF3V8AVCCj/hOd5oFlg+wo2FxJDY= github.com/libp2p/go-libp2p-tls v0.3.1/go.mod h1:fwF5X6PWGxm6IDRwF3V8AVCCj/hOd5oFlg+wo2FxJDY= @@ -1502,8 +1513,8 @@ github.com/miekg/dns v1.1.28/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7 github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= +github.com/miekg/dns v1.1.48 h1:Ucfr7IIVyMBz4lRE8qmGUuZ4Wt3/ZGu9hmcMT3Uu4tQ= github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= -github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b/go.mod h1:lxPUiZwKoFL8DUUmalo2yJJUCxbPKtm8OKfqr2/FTNU= github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc/go.mod h1:cGKTAVKx4SxOuR/czcZ/E2RSJ3sfHs8FpHhQ5CWMf9s= @@ -2659,6 +2670,7 @@ gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=