Skip to content

Commit

Permalink
Expose additional migration APIs (#8153)
Browse files Browse the repository at this point in the history 
* Expose additional migration APIs

Expose migration APIs for reading migration config and creating migration fetchers.  This allows implementation of commands and external applications that want to retrieve migrations according to the Migration portion of the IPFS config.

This change also moves some functionality that is specific to fetching migrations via IPFS into the `ipfsfetcher` package.
  • Loading branch information
@gammazero
gammazero committed Jul 30, 2021
1 parent cab67f6 commit 3b6f577
Show file tree
Hide file tree
Showing 10 changed files with 526 additions and 511 deletions.
139 changes: 1 addition & 138 deletions cmd/ipfs/migration.go → cmd/ipfs/add_migrations.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,13 @@ package main

import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"net/url"
"os"
"path/filepath"
"strings"

config "github.com/ipfs/go-ipfs-config"
"github.com/ipfs/go-ipfs-files"
"github.com/ipfs/go-ipfs/core"
"github.com/ipfs/go-ipfs/core/coreapi"
Expand All @@ -24,140 +20,7 @@ import (
"github.com/libp2p/go-libp2p-core/peer"
)

// readMigrationConfig reads the migration config out of the config, avoiding
// reading anything other than the migration section. That way, we're free to
// make arbitrary changes to all _other_ sections in migrations.
func readMigrationConfig(repoRoot string) (*config.Migration, error) {
var cfg struct {
Migration config.Migration
}

cfgPath, err := config.Filename(repoRoot)
if err != nil {
return nil, err
}

cfgFile, err := os.Open(cfgPath)
if err != nil {
return nil, err
}
defer cfgFile.Close()

err = json.NewDecoder(cfgFile).Decode(&cfg)
if err != nil {
return nil, err
}

switch cfg.Migration.Keep {
case "":
cfg.Migration.Keep = config.DefaultMigrationKeep
case "discard", "cache", "keep":
default:
return nil, errors.New("unknown config value, Migrations.Keep must be 'cache', 'pin', or 'discard'")
}

if len(cfg.Migration.DownloadSources) == 0 {
cfg.Migration.DownloadSources = config.DefaultMigrationDownloadSources
}

return &cfg.Migration, nil
}

func readIpfsConfig(repoRoot *string) (bootstrap []string, peers []peer.AddrInfo) {
if repoRoot == nil {
return
}

cfgPath, err := config.Filename(*repoRoot)
if err != nil {
fmt.Fprintln(os.Stderr, err)
return
}

cfgFile, err := os.Open(cfgPath)
if err != nil {
fmt.Fprintln(os.Stderr, err)
return
}
defer cfgFile.Close()

// Attempt to read bootstrap addresses
var bootstrapCfg struct {
Bootstrap []string
}
err = json.NewDecoder(cfgFile).Decode(&bootstrapCfg)
if err != nil {
fmt.Fprintln(os.Stderr, "cannot read bootstrap peers from config")
} else {
bootstrap = bootstrapCfg.Bootstrap
}

if _, err = cfgFile.Seek(0, 0); err != nil {
fmt.Fprintln(os.Stderr, err)
}

// Attempt to read peers
var peeringCfg struct {
Peering config.Peering
}
err = json.NewDecoder(cfgFile).Decode(&peeringCfg)
if err != nil {
fmt.Fprintln(os.Stderr, "cannot read peering from config")
} else {
peers = peeringCfg.Peering.Peers
}

return
}

// getMigrationFetcher creates one or more fetchers according to
// config.Migration.DownloadSources. If an IpfsFetcher is required, then
// bootstrap and peer information in read from the config file in repoRoot,
// unless repoRoot is nil.
func getMigrationFetcher(cfg *config.Migration, repoRoot *string) (migrations.Fetcher, error) {
const httpUserAgent = "go-ipfs"

// Fetch migrations from current distribution, or location from environ
fetchDistPath := migrations.GetDistPathEnv(migrations.CurrentIpfsDist)

var fetchers []migrations.Fetcher
for _, src := range cfg.DownloadSources {
src := strings.TrimSpace(src)
switch src {
case "IPFS", "ipfs":
bootstrap, peers := readIpfsConfig(repoRoot)
fetchers = append(fetchers, ipfsfetcher.NewIpfsFetcher(fetchDistPath, 0, bootstrap, peers))
case "HTTPS", "https", "HTTP", "http":
fetchers = append(fetchers, migrations.NewHttpFetcher(fetchDistPath, "", httpUserAgent, 0))
default:
u, err := url.Parse(src)
if err != nil {
return nil, fmt.Errorf("bad gateway address: %s", err)
}
switch u.Scheme {
case "":
u.Scheme = "https"
case "https", "http":
default:
return nil, errors.New("bad gateway address: url scheme must be http or https")
}
fetchers = append(fetchers, migrations.NewHttpFetcher(fetchDistPath, u.String(), httpUserAgent, 0))
case "":
// Ignore empty string
}
}
if len(fetchers) == 0 {
return nil, errors.New("no sources specified")
}

if len(fetchers) == 1 {
return fetchers[0], nil
}

// Wrap fetchers in a MultiFetcher to try them in order
return migrations.NewMultiFetcher(fetchers...), nil
}

// addMigrations adds any migration downloaded by the fetcher to the IPFS node
func addMigrations(ctx context.Context, node *core.IpfsNode, fetcher migrations.Fetcher, pin bool) error {
var fetchers []migrations.Fetcher
if mf, ok := fetcher.(*migrations.MultiFetcher); ok {
Expand Down
19 changes: 17 additions & 2 deletions cmd/ipfs/daemon.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
nodeMount "github.com/ipfs/go-ipfs/fuse/node"
fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
"github.com/ipfs/go-ipfs/repo/fsrepo/migrations"
"github.com/ipfs/go-ipfs/repo/fsrepo/migrations/ipfsfetcher"
sockets "github.com/libp2p/go-socket-activation"

cmds "github.com/ipfs/go-ipfs-cmds"
Expand Down Expand Up @@ -294,12 +295,26 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
return fmt.Errorf("fs-repo requires migration")
}

migrationCfg, err := readMigrationConfig(cctx.ConfigRoot)
// Read Migration section of IPFS config
migrationCfg, err := migrations.ReadMigrationConfig(cctx.ConfigRoot)
if err != nil {
return err
}

fetcher, err = getMigrationFetcher(migrationCfg, &cctx.ConfigRoot)
// Define function to create IPFS fetcher. Do not supply an
// already-constructed IPFS fetcher, because this may be expensive and
// not needed according to migration config. Instead, supply a function
// to construct the particular IPFS fetcher implementation used here,
// which is called only if an IPFS fetcher is needed.
newIpfsFetcher := func(distPath string) migrations.Fetcher {
return ipfsfetcher.NewIpfsFetcher(distPath, 0, &cctx.ConfigRoot)
}

// Fetch migrations from current distribution, or location from environ
fetchDistPath := migrations.GetDistPathEnv(migrations.CurrentIpfsDist)

// Create fetchers according to migrationCfg.DownloadSources
fetcher, err = migrations.GetMigrationFetcher(migrationCfg.DownloadSources, fetchDistPath, newIpfsFetcher)
if err != nil {
return err
}
Expand Down
Loading

0 comments on commit 3b6f577

Please sign in to comment.