Skip to content

Commit

Permalink
deps: bump github.com/sylabs/squashfs to v1.0.0 (#265)
Browse files Browse the repository at this point in the history
* deps: bump github.com/sylabs/squashfs to v1.0.0

Signed-off-by: Adam Hughes <9903835+tri-adam@users.noreply.github.com>

* refactor: add error from Opener

This allows the squashfsVisitor to return errors when they are
encountered. Previously, there was not a way to return an error opening
the file, so an error could go unnoticed.

Signed-off-by: Adam Hughes <9903835+tri-adam@users.noreply.github.com>

---------

Signed-off-by: Adam Hughes <9903835+tri-adam@users.noreply.github.com>
  • Loading branch information
tri-adam committed Jul 11, 2024
1 parent 89215f4 commit 41c9674
Show file tree
Hide file tree
Showing 7 changed files with 93 additions and 71 deletions.
10 changes: 5 additions & 5 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/anchore/stereoscope

go 1.21.0
go 1.21.5

require (
github.com/GoogleCloudPlatform/docker-credential-gcr v2.0.5+incompatible
Expand Down Expand Up @@ -29,7 +29,7 @@ require (
github.com/spf13/afero v1.11.0
github.com/stretchr/testify v1.9.0
github.com/sylabs/sif/v2 v2.17.1
github.com/sylabs/squashfs v0.6.1
github.com/sylabs/squashfs v1.0.0
github.com/wagoodman/go-partybus v0.0.0-20200526224238-eb215533f07d
github.com/wagoodman/go-progress v0.0.0-20230925121702-07e42b3cdba0
golang.org/x/crypto v0.25.0
Expand Down Expand Up @@ -78,7 +78,7 @@ require (
github.com/google/uuid v1.6.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/klauspost/compress v1.16.5 // indirect
github.com/klauspost/compress v1.17.4 // indirect
github.com/kr/pretty v0.3.0 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
Expand All @@ -93,12 +93,12 @@ require (
github.com/opencontainers/runc v1.1.12 // indirect
github.com/opencontainers/runtime-spec v1.1.0-rc.1 // indirect
github.com/opencontainers/selinux v1.11.0 // indirect
github.com/pierrec/lz4/v4 v4.1.15 // indirect
github.com/pierrec/lz4/v4 v4.1.19 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/therootcompany/xz v1.0.1 // indirect
github.com/ulikunitz/xz v0.5.10 // indirect
github.com/ulikunitz/xz v0.5.11 // indirect
github.com/vbatts/tar-split v0.11.3 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect
Expand Down
16 changes: 8 additions & 8 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -168,8 +168,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
Expand Down Expand Up @@ -215,8 +215,8 @@ github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaL
github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec=
github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pierrec/lz4/v4 v4.1.19 h1:tYLzDnjDXh9qIxSTKHwXwOYmm9d887Y7Y1ZkyXYHAN4=
github.com/pierrec/lz4/v4 v4.1.19/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
Expand Down Expand Up @@ -259,12 +259,12 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/sylabs/sif/v2 v2.17.1 h1:p6Sl0LWyShXBj2SBsS1dMOMIMrZHe8pwBnBrYt6uo4M=
github.com/sylabs/sif/v2 v2.17.1/go.mod h1:XUGB6AQUXGkms3qPOPdevctT3lBLRLWZNWHVnt5HMKE=
github.com/sylabs/squashfs v0.6.1 h1:4hgvHnD9JGlYWwT0bPYNt9zaz23mAV3Js+VEgQoRGYQ=
github.com/sylabs/squashfs v0.6.1/go.mod h1:ZwpbPCj0ocIvMy2br6KZmix6Gzh6fsGQcCnydMF+Kx8=
github.com/sylabs/squashfs v1.0.0 h1:xAyMS21ogglkuR5HaY55PCfqY3H32ma9GkasTYo28Zg=
github.com/sylabs/squashfs v1.0.0/go.mod h1:rhWzvgefq1X+R+LZdts10hfMsTg3g74OfGunW8tvg/4=
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8=
github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck=
github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY=
Expand Down
2 changes: 1 addition & 1 deletion pkg/file/opener.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@ package file

import "io"

type Opener func() io.ReadCloser
type Opener func() (io.ReadCloser, error)
38 changes: 17 additions & 21 deletions pkg/file/squashfs_walk.go
Original file line number Diff line number Diff line change
@@ -1,53 +1,49 @@
package file

import (
"io"
"io/fs"
"os"

"github.com/sylabs/squashfs"
)

// SquashFSVisitor is the type of the function called by WalkSquashFS to visit each file or
// directory.
//
// The path argument contains the full path, and d is the corresponding fs.DirEntry.
// The sqfsPath argument contains the path to the SquashFS filesystem that was passed to
// WalkSquashFS. The filePath argument contains the full path of the file or directory within the
// SquashFS filesystem.
//
// The error result returned by the function controls how WalkSquashFS continues. If the function
// returns the special value fs.SkipDir, WalkSquashFS skips the current directory (path if
// d.IsDir() is true, otherwise path's parent directory). Otherwise, if the function returns a non-
// nil error, WalkSquashFS stops entirely and returns that error.
type SquashFSVisitor func(fsys fs.FS, path string, d fs.DirEntry) error
// returns the special value fs.SkipDir, WalkSquashFS skips the current directory (filePath if
// d.IsDir() is true, otherwise filePath's parent directory). Otherwise, if the function returns a
// non-nil error, WalkSquashFS stops entirely and returns that error.
type SquashFSVisitor func(fsys fs.FS, sqfsPath, filePath string) error

// WalkSquashFS walks the file tree within the SquashFS filesystem read from r, calling fn for each
// WalkSquashFS walks the file tree within the SquashFS filesystem at sqfsPath, calling fn for each
// file or directory in the tree, including root.
func WalkSquashFS(r io.ReaderAt, fn SquashFSVisitor) error {
fsys, err := squashfs.NewReader(r)
func WalkSquashFS(sqfsPath string, fn SquashFSVisitor) error {
f, err := os.Open(sqfsPath)
if err != nil {
return err
}
defer f.Close()

return fs.WalkDir(fsys, ".", walkDir(fsys, fn))
}

// WalkSquashFSFromReader walks the file tree within the SquashFS filesystem read from r, calling
// fn for each file or directory in the tree, including root. Callers should use WalkSquashFS
// where possible, as this function is considerably less efficient.
func WalkSquashFSFromReader(r io.Reader, fn SquashFSVisitor) error {
fsys, err := squashfs.NewReaderFromReader(r)
fsys, err := squashfs.NewReader(f)
if err != nil {
return err
}

return fs.WalkDir(fsys, ".", walkDir(fsys, fn))
return fs.WalkDir(fsys, ".", walkDir(fsys, sqfsPath, fn))
}

// walkDir returns a fs.WalkDirFunc bound to fn.
func walkDir(fsys fs.FS, fn SquashFSVisitor) fs.WalkDirFunc {
return func(path string, d fs.DirEntry, err error) error {
func walkDir(fsys fs.FS, sqfsPath string, fn SquashFSVisitor) fs.WalkDirFunc {
return func(path string, _ fs.DirEntry, err error) error {
if err != nil {
return err
}

return fn(fsys, path, d)
return fn(fsys, sqfsPath, path)
}
}
2 changes: 1 addition & 1 deletion pkg/image/file_catalog.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,5 +70,5 @@ func (c *FileCatalog) Open(f file.Reference) (io.ReadCloser, error) {
return nil, fmt.Errorf("no contents available for file: %+v", f.RealPath)
}

return opener(), nil
return opener()
}
4 changes: 2 additions & 2 deletions pkg/image/file_catalog_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -144,8 +144,8 @@ func TestFileCatalog_Open(t *testing.T) {

require.Len(t, entries, 1)

opener := func() io.ReadCloser {
return io.NopCloser(entries[0].Reader)
opener := func() (io.ReadCloser, error) {
return io.NopCloser(entries[0].Reader), nil
}

catalog := NewFileCatalog()
Expand Down
92 changes: 59 additions & 33 deletions pkg/image/layer.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package image

import (
"bytes"
"errors"
"fmt"
"io"
Expand Down Expand Up @@ -50,32 +49,32 @@ func NewLayer(layer v1.Layer) *Layer {
}
}

func (l *Layer) uncompressedTarCache(uncompressedLayersCacheDir string) (string, error) {
func (l *Layer) uncompressedCache(uncompressedLayersCacheDir string) (string, error) {
if uncompressedLayersCacheDir == "" {
return "", fmt.Errorf("no cache directory given")
}

tarPath := path.Join(uncompressedLayersCacheDir, l.Metadata.Digest+".tar")
path := path.Join(uncompressedLayersCacheDir, l.Metadata.Digest)

if _, err := os.Stat(tarPath); !os.IsNotExist(err) {
return tarPath, nil
if _, err := os.Stat(path); !os.IsNotExist(err) {
return path, nil
}

rawReader, err := l.layer.Uncompressed()
if err != nil {
return "", err
}

fh, err := os.Create(tarPath)
fh, err := os.Create(path)
if err != nil {
return "", fmt.Errorf("unable to create layer cache dir=%q : %w", tarPath, err)
return "", fmt.Errorf("unable to create layer cache dir=%q : %w", path, err)
}

if _, err := io.Copy(fh, rawReader); err != nil {
return "", fmt.Errorf("unable to populate layer cache dir=%q : %w", tarPath, err)
return "", fmt.Errorf("unable to populate layer cache dir=%q : %w", path, err)
}

return tarPath, nil
return path, nil
}

// Read parses information from the underlying layer tar into this struct. This includes layer metadata, the layer
Expand Down Expand Up @@ -104,7 +103,7 @@ func (l *Layer) Read(catalog *FileCatalog, idx int, uncompressedLayersCacheDir s
return err
}
case SingularitySquashFSLayer:
err := l.readSingularityImageLayer(idx, tree)
err := l.readSingularityImageLayer(idx, uncompressedLayersCacheDir, tree)
if err != nil {
return err
}
Expand All @@ -130,7 +129,7 @@ func (l *Layer) readStandardImageLayer(idx int, uncompressedLayersCacheDir strin
l.Metadata.Digest,
l.Metadata.MediaType)

tarFilePath, err := l.uncompressedTarCache(uncompressedLayersCacheDir)
tarFilePath, err := l.uncompressedCache(uncompressedLayersCacheDir)
if err != nil {
return err
}
Expand All @@ -147,7 +146,7 @@ func (l *Layer) readStandardImageLayer(idx int, uncompressedLayersCacheDir strin
return nil
}

func (l *Layer) readSingularityImageLayer(idx int, tree *filetree.FileTree) error {
func (l *Layer) readSingularityImageLayer(idx int, uncompressedLayersCacheDir string, tree *filetree.FileTree) error {
var err error
l.Metadata, err = newLayerMetadata(l.layer, idx)
if err != nil {
Expand All @@ -160,19 +159,12 @@ func (l *Layer) readSingularityImageLayer(idx int, tree *filetree.FileTree) erro
l.Metadata.MediaType)

monitor := trackReadProgress(l.Metadata)
r, err := l.layer.Uncompressed()
sqfsFilePath, err := l.uncompressedCache(uncompressedLayersCacheDir)
if err != nil {
return fmt.Errorf("failed to read layer=%q: %w", l.Metadata.Digest, err)
return err
}
// defer r.Close() // TODO: if we close this here, we can't read file contents after we return.

// Walk the more efficient walk if we're blessed with an io.ReaderAt.
if ra, ok := r.(io.ReaderAt); ok {
err = file.WalkSquashFS(ra, squashfsVisitor(tree, l.fileCatalog, &l.Metadata.Size, l, monitor))
} else {
err = file.WalkSquashFSFromReader(r, squashfsVisitor(tree, l.fileCatalog, &l.Metadata.Size, l, monitor))
}
if err != nil {
if err := file.WalkSquashFS(sqfsFilePath, squashfsVisitor(tree, l.fileCatalog, &l.Metadata.Size, l, monitor)); err != nil {
return fmt.Errorf("failed to walk layer=%q: %w", l.Metadata.Digest, err)
}

Expand Down Expand Up @@ -271,7 +263,9 @@ func layerTarIndexer(ft filetree.Writer, fileCatalog *FileCatalog, size *int64,
if size != nil {
*(size) += metadata.Size()
}
fileCatalog.addImageReferences(ref.ID(), layerRef, index.Open)
fileCatalog.addImageReferences(ref.ID(), layerRef, func() (io.ReadCloser, error) {
return index.Open(), nil
})

if monitor != nil {
monitor.Increment()
Expand All @@ -280,10 +274,49 @@ func layerTarIndexer(ft filetree.Writer, fileCatalog *FileCatalog, size *int64,
}
}

// squashfsReader implements an io.ReadCloser that reads a file from within a SquashFS filesystem.
type squashfsReader struct {
fs.File
backingFile *os.File
}

// newSquashfsFileReader returns a io.ReadCloser that reads the file at path within the SquashFS
// filesystem at sqfsPath.
func newSquashfsFileReader(sqfsPath, path string) (io.ReadCloser, error) {
f, err := os.Open(sqfsPath)
if err != nil {
return nil, err
}

fsys, err := squashfs.NewReader(f)
if err != nil {
return nil, err
}

r, err := fsys.Open(path)
if err != nil {
return nil, err
}

return &squashfsReader{
File: r,
backingFile: f,
}, nil
}

// Close closes the SquashFS file as well as the backing filesystem.
func (f *squashfsReader) Close() error {
if err := f.File.Close(); err != nil {
return err
}

return f.backingFile.Close()
}

func squashfsVisitor(ft filetree.Writer, fileCatalog *FileCatalog, size *int64, layerRef *Layer, monitor *progress.Manual) file.SquashFSVisitor {
builder := filetree.NewBuilder(ft, fileCatalog.Index)

return func(fsys fs.FS, path string, _ fs.DirEntry) error {
return func(fsys fs.FS, sqfsPath, path string) error {
ff, err := fsys.Open(path)
if err != nil {
return err
Expand All @@ -308,15 +341,8 @@ func squashfsVisitor(ft filetree.Writer, fileCatalog *FileCatalog, size *int64,
if size != nil {
*(size) += metadata.Size()
}
fileCatalog.addImageReferences(fileReference.ID(), layerRef, func() io.ReadCloser {
r, err := fsys.Open(path)
if err != nil {
// The file.Opener interface doesn't give us a way to return an error, and callers
// don't seem to handle a nil return. So, return a zero-byte reader.
log.Debug(err)
return io.NopCloser(bytes.NewReader(nil)) // TODO
}
return r
fileCatalog.addImageReferences(fileReference.ID(), layerRef, func() (io.ReadCloser, error) {
return newSquashfsFileReader(sqfsPath, path)
})

monitor.Increment()
Expand Down

0 comments on commit 41c9674

Please sign in to comment.